// Command yaegi-check replicates the Traefik Plugin Catalog analyzer // (piceus)'s yaegiMiddlewareCheck for this plugin. It loads the plugin // package under Yaegi exactly as the catalog and Traefik's own plugin // loader do: interpret the source tree, evaluate CreateConfig(), decode // the plugin's real .traefik.yml testData into the resulting Config // value, then call the plugin's New constructor by reflection from // outside the interpreter. It then exercises the returned handler with // two real HTTP requests, proving ServeHTTP, auth, the model registry, // statusTrackingWriter, and the gateway's logger all run under Yaegi — // not merely that the package's imports resolve. // // Any failure — a Yaegi-incompatible construct in the plugin's stdlib // usage, a constructor error, a nil handler, or a wrong response from the // live handler — exits non-zero: this is the gate the plugin must pass // before it can list in the Traefik Plugin Catalog. // // This module is intentionally separate from the plugin module // (llmgw-yaegi-check, not github.com/lukaszraczylo/traefik-llmgateway): // this harness's own dependencies (yaegi itself) must never become the // plugin's, and nothing here may be reachable from interpreted code. // // The plugin module carries exactly ONE non-stdlib runtime dependency, // github.com/lukaszraczylo/oss-telemetry, which is vendored and therefore // interpretable — Yaegi resolves it out of vendor/ in the GOPATH copy, // the same way Traefik resolves it from the cloned plugin tree. That is // why vendor/ is no longer excluded below. Every OTHER runtime import // must stay stdlib-only: an unvendored third-party import breaks under // interpretation even though `go build` never notices, and this harness // is what catches it. package main import ( "bufio" "bytes" "context" "encoding/json" "fmt" "io" "io/fs" "math" "net/http" "net/http/httptest" "net/url" "os" "path/filepath" "reflect" "regexp" "strconv" "strings" "sync/atomic" "time" dto "github.com/prometheus/client_model/go" "github.com/prometheus/common/expfmt" "github.com/prometheus/common/model" "github.com/traefik/yaegi/interp" "github.com/traefik/yaegi/stdlib" ) // testDataUserAPIKey and testDataWantModel are drawn directly from the // plugin's own .traefik.yml testData block (users.inline[0].apiKey and // providers.openai.models[0]) — see exerciseHandler. // // attemptAccountingAdminAPIKey names the second, admin-flagged user run() // layers on top of testData (see its own override comment) — SHOULD-4 // (v0.22 review round): this harness proves recordProviderAttempt's // SHOULD-1 deadline/cancel classification (limits.go's matchesSentinel/ // isDeadlineExceeded) actually runs correctly INTERPRETED, not merely // compiled: a bug there would not show up under `go test`, only here — // which is exactly what happened in round 2 (see slowProviderName's own // doc comment immediately below, and matchesSentinel's in limits.go, for // the full account of the interpreter-only bug this harness caught). // // slowProviderName/slowProviderModel/slowUpstreamSleep/ // slowRequestDeadline back exerciseAttemptAccounting's SECOND request // (SHOULD-A, v0.22 review round, round 2): the FIRST version of this // harness only ever drove an upstream that answers instantly, so // isDeadlineExceeded never actually saw a non-nil error, interpreted or // otherwise — a comment claiming to prove the interpreted deadline path // while never exercising it. Once fixed to actually drive a timeout, it // caught a real bug: round 2's original matchesSentinel (a hand-rolled // Unwrap walk against two locally declared interfaces, comma-ok matched) // passed every compiled `go test` row for a double-%w error shape but // silently misclassified it under the real interpreter — an interpreted // interface type failing to match a compiled concrete value's method // set. matchesSentinel now calls real errors.Is instead (limits.go's own // doc comment has the full account of why that is safe here // specifically). The second request below: a second, DELIBERATELY slow // provider, addressed by its own provider-prefixed model id // (`slowProviderName+"/"+slowProviderModel`, routableModelId's own // "provider/model" convention) so it can never collide with // testDataWantModel's bare "gpt-test" on the "openai" provider, driven // with a request context deadline shorter than the upstream's own // sleep — the exact shape the route-level Go test regression (routes_ // unified_test.go's TestHandleChat_ContextDeadlineExceeded_ // RecordsProviderFailure) already proves compiled; this proves it // interpreted. const ( testDataUserAPIKey = "test-user-key" testDataWantModel = "gpt-test" attemptAccountingAdminAPIKey = "sk-admin1" slowProviderName = "slow" slowProviderModel = "gpt-test" slowUpstreamSleep = 600 * time.Millisecond slowRequestDeadline = 60 * time.Millisecond // yaegiMetaContextTokens is the modelMeta config-override // contextTokens value the attempt-accounting harness sets for // testDataWantModel (feature v0.23) — a string, not an int, since it // is spliced directly into attemptAccountingOverride's hand-built // JSON text below. yaegiMetaContextTokens = "128000" ) // timeoutProviderName/timeoutProviderModel/timeoutProviderRequestTimeout/ // timeoutUpstreamSleep/timeoutProbeMaxWait back exerciseRequestTimeout // (feature: request timeout): a provider whose own ProviderConfig. // RequestTimeout ("80ms") is far shorter than timeoutUpstream's // deliberate 600ms silence before it ever writes a byte. Distinct from // slowProviderName/slowRequestDeadline above (SHOULD-A): that probe // proves a CLIENT-set context deadline is classified correctly; // timeoutProviderName proves the GATEWAY'S OWN adapter-level timeout // (newAdapterHTTPClient's Transport.ResponseHeaderTimeout, providers.go; // watchdogBody, timeout.go) aborts a hang under a request that sets NO // deadline of its own — the shape of a real, unbounded production // client — interpreted, not merely compiled. const ( timeoutProviderName = "hungtimeout" // timeoutProviderModel is deliberately NOT "gpt-test" // (testDataWantModel): a bare model id shared across providers // resolves to exactly one winner (modelRegistry's own bare-id rule), // and reusing testDataWantModel here made "hungtimeout" win it // instead of "openai" — silently hijacking exerciseAttemptAccounting's // own bare-"gpt-test" request onto this hung provider and failing // that unrelated probe. A unique id keeps this provider's own model // space from ever colliding with another probe's. timeoutProviderModel = "gpt-timeout-test" timeoutProviderRequestTimeout = "80ms" timeoutUpstreamSleep = 600 * time.Millisecond // timeoutProbeMaxWait bounds exerciseRequestTimeout's own assertion: // generous relative to timeoutProviderRequestTimeout (80ms), but far // under timeoutUpstreamSleep (600ms) — a pass proves the request was // aborted BY the timeout feature, not by timeoutUpstream eventually // answering on its own. timeoutProbeMaxWait = 3 * time.Second ) // hungBodyProviderName/hungBodyProviderModel/hungBodyProviderRequestTimeout/ // hungBodyUpstreamSleep back exerciseRequestTimeoutMidBodyStall (finding // F7, coordinator adversarial review, 2026-08-23): timeoutProviderName // above never sends headers at all, so it only exercises Transport. // ResponseHeaderTimeout — watchdogBody.fire (timeout.go) — the // interpreted method value handed to time.AfterFunc, plus sync/atomic's // function API and a context.CancelFunc — never runs under that probe. // hungBodyUpstream sends headers immediately, then goes silent, so the // body-read path (upstreamBytes' watchdogBody wrap, providers.go) is // what has to abort it, proving the OTHER half of this feature // interpreted. const ( hungBodyProviderName = "hungbody" // hungBodyProviderModel: same reasoning as timeoutProviderModel's own // doc comment above — must not collide with testDataWantModel or any // other probe's model id. hungBodyProviderModel = "gpt-bodystall-test" hungBodyProviderRequestTimeout = "80ms" hungBodyUpstreamSleep = 600 * time.Millisecond ) // provenanceEstProviderName/provenanceEstProviderModel back // exerciseUsageProvenance (feat: expose token-accounting provenance): a // provider whose upstream answers 200 with NO "usage" field at all — // decodes to chatUsagePayload's zero value (provider_openai.go's // forwardJSON), the exact shape runUnified's own estimation fallback // (routes_unified.go) classifies as "estimated" — proving the // substitution math (ceil(len(body)/4)) and the new // llmgateway_usage_provenance_requests_total/-_tokens_total families // (metrics.go) all run correctly under the REAL interpreter, not merely // compiled. "reported" is already exercised, under the interpreter, by // exerciseAttemptAccounting's own real, usage-bearing "openai" traffic // above — no separate fixture needed for that provenance here. const ( provenanceEstProviderName = "provenance-est" // provenanceEstProviderModel: same reasoning as timeoutProviderModel's // own doc comment above — must not collide with testDataWantModel or // any other probe's model id. provenanceEstProviderModel = "gpt-provenance-est-test" ) // excludedTopLevelDirs lists repo-root directories the GOPATH copy must // never include: build tooling, integration fixtures, planning docs and // VCS metadata have nothing to do with the plugin package Yaegi imports. // // vendor/ USED TO be excluded here, on the reasoning that it held only // the test-only testify dependency and _test.go files are never // interpreted. That stopped being true when the plugin took on // oss-telemetry as a runtime dependency: leaving vendor/ out would make // the interpreted copy unable to resolve an import the real plugin tree // resolves fine, so this harness would fail on code that works in // production — or, worse, quietly stop representing it. Copying the whole // vendor tree, nested "go.yaml.in"-style module dirs included, resolves // correctly under yaegi v0.16.1 (verified: the gate passes with the // telemetry import live and reached). // // webui/ (Vue admin-panel task) is still excluded: its own go.mod already // walls it off from the repo root's `go build`/`vet`/`test ./...`, and // its node_modules tree — hundreds of MB, // including a stray bundled .go file // (node_modules/flatted/golang/pkg/flatted.go) — would otherwise both // balloon copyRepoSource's wall time and risk Yaegi tripping over code // this repo neither wrote nor can build. The plugin only ever imports the // generated admin_assets_gen.go at the repo root, never anything under // webui/ itself. var excludedTopLevelDirs = map[string]bool{ "tools": true, "integration": true, ".superpowers": true, "docs": true, ".git": true, "webui": true, } func main() { if err := run(); err != nil { fmt.Fprintln(os.Stderr, "yaegi-check: FAIL:", err) os.Exit(1) } fmt.Println("yaegi-check: OK") } // run resolves the plugin's repo root (os.Args[1], defaulting to the // current directory), builds a temporary GOPATH containing a copy of its // source, interprets the plugin package under Yaegi, and exercises the // constructed handler. It returns the first error encountered, wrapped // with enough context to diagnose which stage failed. func run() error { repoRoot := "." if len(os.Args) > 1 { repoRoot = os.Args[1] } repoRoot, err := filepath.Abs(repoRoot) if err != nil { return fmt.Errorf("resolve repo root: %w", err) } modulePath, err := readModulePath(filepath.Join(repoRoot, "go.mod")) if err != nil { return fmt.Errorf("read module path: %w", err) } gopath, err := os.MkdirTemp("", "llmgw-yaegi-check-*") if err != nil { return fmt.Errorf("create temp GOPATH: %w", err) } defer os.RemoveAll(gopath) //nolint:errcheck // best-effort cleanup of a temp dir pkgDir := filepath.Join(gopath, "src", modulePath) if err = copyRepoSource(repoRoot, pkgDir); err != nil { return fmt.Errorf("copy repo source into GOPATH: %w", err) } // Suppress telemetry BEFORE anything is interpreted, and keep it // suppressed for the whole run. stampReleaseVersion below makes the // interpreted copy look like a stamped release build so that New's // telemetry call is actually REACHED under Yaegi; without these env // vars that would fire a real ping at the public ingest endpoint from // every CI run and every developer's `make yaegi-check`. Both names // are set because either one alone suppresses, so a rename upstream // cannot silently re-enable it. if err = os.Setenv("DO_NOT_TRACK", "1"); err != nil { return fmt.Errorf("set DO_NOT_TRACK: %w", err) } if err = os.Setenv("OSS_TELEMETRY_DISABLED", "1"); err != nil { return fmt.Errorf("set OSS_TELEMETRY_DISABLED: %w", err) } if err = stampReleaseVersion(pkgDir); err != nil { return fmt.Errorf("stamp a release version into the interpreted copy: %w", err) } pkgName, err := readPackageName(pkgDir) if err != nil { return fmt.Errorf("read plugin package name: %w", err) } i := interp.New(interp.Options{GoPath: gopath}) if err = i.Use(stdlib.Symbols); err != nil { return fmt.Errorf("interp.Use(stdlib.Symbols): %w", err) } if _, err = i.Eval(`import "` + modulePath + `"`); err != nil { return fmt.Errorf("import plugin package under yaegi: %w", err) } cfgVal, err := i.Eval(pkgName + ".CreateConfig()") if err != nil { return fmt.Errorf("call %s.CreateConfig() under yaegi: %w", pkgName, err) } if !cfgVal.CanInterface() { return fmt.Errorf("%s.CreateConfig() result cannot be used via reflect.Value.Interface", pkgName) } testData, err := extractTraefikYAMLTestData(filepath.Join(repoRoot, ".traefik.yml")) if err != nil { return fmt.Errorf("extract .traefik.yml testData: %w", err) } testDataJSON, err := json.Marshal(testData) if err != nil { return fmt.Errorf("marshal testData: %w", err) } if err = json.Unmarshal(testDataJSON, cfgVal.Interface()); err != nil { return fmt.Errorf("decode .traefik.yml testData into the interpreted Config: %w", err) } // Feature A (v0.22) attempt-accounting harness (SHOULD-4, review // round): a second json.Unmarshal into the same cfgVal layers this // override on top of testData's own decode, rather than replacing it // outright — encoding/json's own map/struct-merge semantics keep // every testData field this override does not mention (Groups.default // in particular, which .traefik.yml's own testData.groups block // already provides and this override never touches). It replaces // providers.openai wholesale (a real map key, not merged field-by- // field) with one pointed at a local httptest upstream so the harness // can drive a real chat completion without a live provider, and adds // a second, admin-flagged user (Users.Inline is a slice — fully // replaced, not merged, which is why the "tester" user from testData // is respecified here too, identical apiKey/group, rather than // silently dropped). upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") w.WriteHeader(http.StatusOK) _, _ = w.Write([]byte(`{"id":"c1","object":"chat.completion","model":"gpt-test","choices":[{"index":0,"message":{"role":"assistant","content":"hi"},"finish_reason":"stop"}],"usage":{"prompt_tokens":1,"completion_tokens":1}}`)) })) defer upstream.Close() // slowUpstream never answers within slowRequestDeadline — see // slowProviderName's own doc comment above (SHOULD-A) for why this // second provider exists at all. slowUpstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { time.Sleep(slowUpstreamSleep) w.Header().Set("Content-Type", "application/json") _, _ = w.Write([]byte(`{"id":"c2","object":"chat.completion","model":"gpt-test","choices":[{"index":0,"message":{"role":"assistant","content":"hi"},"finish_reason":"stop"}],"usage":{"prompt_tokens":1,"completion_tokens":1}}`)) })) defer slowUpstream.Close() // timeoutUpstream writes nothing at all for timeoutUpstreamSleep, // well past timeoutProviderName's own 80ms ProviderConfig. // RequestTimeout (attemptAccountingOverride below) — see the // timeoutProviderName const block's own doc comment for why this // probe exists alongside slowUpstream. timeoutUpstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { time.Sleep(timeoutUpstreamSleep) w.Header().Set("Content-Type", "application/json") _, _ = w.Write([]byte(`{"id":"c4","object":"chat.completion","model":"gpt-test","choices":[{"index":0,"message":{"role":"assistant","content":"hi"},"finish_reason":"stop"}],"usage":{"prompt_tokens":1,"completion_tokens":1}}`)) })) defer timeoutUpstream.Close() // hungBodyUpstream sends headers immediately (so ResponseHeaderTimeout // is satisfied and never fires), then goes silent for // hungBodyUpstreamSleep before ever writing a body byte — well past // hungBodyProviderName's own 80ms ProviderConfig.RequestTimeout. See // the hungBodyProviderName const block's own doc comment (finding F7) // for why this probe exists alongside timeoutUpstream, not instead of // it. hungBodyUpstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") w.WriteHeader(http.StatusOK) if fl, ok := w.(http.Flusher); ok { fl.Flush() } time.Sleep(hungBodyUpstreamSleep) _, _ = w.Write([]byte(`{"id":"c5","object":"chat.completion","model":"gpt-test","choices":[{"index":0,"message":{"role":"assistant","content":"hi"},"finish_reason":"stop"}],"usage":{"prompt_tokens":1,"completion_tokens":1}}`)) })) defer hungBodyUpstream.Close() // provenanceEstUpstream backs exerciseUsageProvenance (feat: expose // token-accounting provenance): answers 200 with NO "usage" field at // all — see provenanceEstProviderName's own const block doc comment // for why this is the exact shape runUnified's estimation fallback // classifies as "estimated". provenanceEstUpstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") w.WriteHeader(http.StatusOK) _, _ = w.Write([]byte(`{"id":"c-prov","object":"chat.completion","model":"` + provenanceEstProviderModel + `","choices":[{"index":0,"message":{"role":"assistant","content":"hi"},"finish_reason":"stop"}]}`)) })) defer provenanceEstUpstream.Close() // eventsUpstreamErrorUpstream always 500s — see // eventsUpstreamErrorProviderName's own doc comment for why this // dedicated, single-candidate provider exists rather than reusing // failoverAUpstream (which a correctly working failover loop never // surfaces to the client, so its own 500 never reaches // handleAdapterErrorEnvelope's client-facing write). eventsUpstreamErrorUpstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") w.WriteHeader(http.StatusInternalServerError) _, _ = w.Write([]byte(`{"error":{"message":"eventsprobe-upstream always fails","type":"server_error"}}`)) })) defer eventsUpstreamErrorUpstream.Close() // mcpProbeUpstream answers federation's outbound tools/call with a // body deliberately larger than mcpBackendCallResponseMaxBytes, so // exerciseHandler's POST /mcp probe below drives doBackendJSONRPC's // oversize branch — and therefore its errors.Is(err, // errMCPResponseTooLarge) match — under the REAL interpreter. // // This exists because a security review round objected that no gate // exercised federation's tools/call path interpreted, which left // errors.Is there justified by reasoning rather than by evidence. // Reasoning is exactly what the Yaegi traps in this codebase have // repeatedly defeated, so the path is now covered instead of argued. mcpProbeUpstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") _, _ = w.Write([]byte(`{"jsonrpc":"2.0","id":1,"result":{"content":[{"type":"text","text":"`)) chunk := bytes.Repeat([]byte("x"), 1<<20) for written := 0; written <= mcpProbeOversizeBytes; written += len(chunk) { if _, writeErr := w.Write(chunk); writeErr != nil { return } } _, _ = w.Write([]byte(`"}]}}`)) })) defer mcpProbeUpstream.Close() // targetHealthDownUpstream is opened only to capture a real, unique // "http://127.0.0.1:PORT" URL, then immediately closed — feat/ // target-health's own exerciseTargetHealth (below) drives real 502s // through this dead target's per-server proxy path // (/mcp/target-health-down/...) to prove the interpreted tracker // flips it to "unhealthy" after targetHealthFailureThreshold // consecutive failures, without disturbing mcpProbeUpstream's own // (successful, federation-routed) traffic above. targetHealthDownUpstream := httptest.NewServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) {})) targetHealthDownURL := targetHealthDownUpstream.URL targetHealthDownUpstream.Close() // targetHealthProbeOnlyUpstream answers any JSON-RPC call — in // practice only ever "initialize", from the active probe sweep // (target_health.go's probeMCPTarget) — with a small, valid result // well under mcpBackendResponseMaxBytes (4MiB), so the probe's // handshake genuinely SUCCEEDS. targetHealthProbeOnlyServerName's own // doc comment explains why this server backs a dedicated, // never-proxied target rather than reusing mcpProbeUpstream (whose // 11MiB body exceeds every response-size cap this plugin enforces, // probe included, so it can only ever record a failure). targetHealthProbeOnlyUpstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") _, _ = w.Write([]byte(`{"jsonrpc":"2.0","id":"federated","result":{"protocolVersion":"2025-11-25","capabilities":{}}}`)) })) defer targetHealthProbeOnlyUpstream.Close() // anthropicProbeUpstream answers an Anthropic Messages API call with a // real Anthropic-shaped body, so the /v1/messages passthrough branch // (callAnthropicMessagesPassthrough) runs interpreted. A body // containing "make-it-fail" gets a 429, driving // handleAdapterErrorEnvelope's *providerHTTPError branch interpreted // too — the exact construct class that has broken under Yaegi before // (see handleAdapterError's own doc comment, routes_unified.go). anthropicProbeUpstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { raw, _ := io.ReadAll(r.Body) if bytes.Contains(raw, []byte("make-it-fail")) { w.Header().Set("Content-Type", "application/json") w.WriteHeader(http.StatusTooManyRequests) _, _ = w.Write([]byte(`{"type":"error","error":{"type":"rate_limit_error","message":"slow down"}}`)) return } if r.Header.Get("anthropic-version") == "" || r.Header.Get("x-api-key") == "" { w.WriteHeader(http.StatusBadRequest) _, _ = w.Write([]byte(`{"type":"error","error":{"type":"invalid_request_error","message":"missing anthropic headers"}}`)) return } w.Header().Set("Content-Type", "application/json") w.WriteHeader(http.StatusOK) _, _ = w.Write([]byte(`{"id":"msg_probe","type":"message","role":"assistant","model":"claude-upstream-real","content":[{"type":"text","text":"pong"}],"stop_reason":"end_turn","usage":{"input_tokens":7,"output_tokens":3}}`)) })) defer anthropicProbeUpstream.Close() // builtinLookupContextTokens (review fix, SHOULD-6): read directly // out of the generated pricing_data_gen.go rather than hardcoding a // number, so this assertion self-updates across a `make // pricing-sync` regeneration instead of pinning a value that could // legitimately drift when LiteLLM's own upstream data changes. builtinLookupContextTokens, err := readBuiltinContextTokens(repoRoot, builtinLookupModelID) if err != nil { return fmt.Errorf("read builtin context tokens for %q from pricing_data_gen.go: %w", builtinLookupModelID, err) } // --- breaker probe (feat/provider-health): a provider whose // /v1/models 401s, exactly the xiaomi production case, driven under // the REAL interpreter. brkHealthy flips the upstream from always-401 // to serving real data once the probe wants to observe recovery; // brkModelsHits counts every /v1/models hit so exerciseBreaker can // prove the open window actually suppresses discovery traffic (a // count staying flat), not merely that healthState reads "open". brkModelsHits := new(int64) brkHealthy := new(int64) brkUpstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if strings.HasSuffix(r.URL.Path, "/v1/models") { atomic.AddInt64(brkModelsHits, 1) if atomic.LoadInt64(brkHealthy) == 0 { w.Header().Set("Content-Type", "application/json") w.WriteHeader(http.StatusUnauthorized) _, _ = w.Write([]byte(`{"error":{"message":"invalid api key","type":"invalid_request_error"}}`)) return } w.Header().Set("Content-Type", "application/json") _, _ = w.Write([]byte(`{"object":"list","data":[{"id":"brk-model"}]}`)) return } w.Header().Set("Content-Type", "application/json") _, _ = w.Write([]byte(`{"id":"c3","object":"chat.completion","model":"brk-model","choices":[],"usage":{}}`)) })) defer brkUpstream.Close() // warmRetryUpstream backs exerciseWarmFillRetry (production incident // fix, registry.go's warmFillRetryDelay): 401s on the FIRST /v1/models // hit only — the synchronous warmFill call inside New() itself — then // serves real data on every hit after, so the scheduled background // retry (and ONLY the retry) succeeds. Failing fast rather than // sleeping past warmFillTimeout keeps this probe's cost down to // roughly warmFillRetryDelay, not warmFillTimeout+warmFillRetryDelay. warmRetryHits := new(int64) warmRetryUpstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if strings.HasSuffix(r.URL.Path, "/v1/models") { if atomic.AddInt64(warmRetryHits, 1) == 1 { w.Header().Set("Content-Type", "application/json") w.WriteHeader(http.StatusUnauthorized) _, _ = w.Write([]byte(`{"error":{"message":"cold-start timeout","type":"invalid_request_error"}}`)) return } w.Header().Set("Content-Type", "application/json") _, _ = w.Write([]byte(`{"object":"list","data":[{"id":"warmretry-model"}]}`)) return } w.Header().Set("Content-Type", "application/json") _, _ = w.Write([]byte(`{"id":"c-warmretry","object":"chat.completion","model":"warmretry-model","choices":[],"usage":{}}`)) })) defer warmRetryUpstream.Close() // failoverAUpstream/failoverBUpstream back exerciseFailover (feat/ // failover, below): two providers configured with the IDENTICAL bare // model id (failoverModelID) — the exact shape registry.go's bareWinner // collision handles, and the shape production logs show for whisper-1/ // tts-1/tts-1-hd/the embeddings models across openai/openai-audio/ // copilot. A always answers 500; B always answers 200. This proves the // failover candidate loop (runMeteredCall, routes_unified.go) falls // through and serves correctly under the REAL interpreter, not merely // compiled — this package's own doc comment is exactly why that // distinction matters on this codebase. failoverAUpstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") w.WriteHeader(http.StatusInternalServerError) _, _ = w.Write([]byte(`{"error":{"message":"failover-a is down","type":"server_error"}}`)) })) defer failoverAUpstream.Close() failoverBHits := new(int64) failoverBUpstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { atomic.AddInt64(failoverBHits, 1) w.Header().Set("Content-Type", "application/json") w.WriteHeader(http.StatusOK) _, _ = w.Write([]byte(`{"id":"c-failover","object":"chat.completion","model":"` + failoverModelID + `","choices":[{"index":0,"message":{"role":"assistant","content":"served by failover-b"},"finish_reason":"stop"}],"usage":{"prompt_tokens":1,"completion_tokens":1}}`)) })) defer failoverBUpstream.Close() // bareWinnerAUpstream/bareWinnerBUpstream back exerciseBareWinnerGroupAware // (fix/group-aware-bare-winner, 2026-08-27): two providers configured // with the IDENTICAL bare model id (bareWinnerModelID) — the same // collision shape failoverAUpstream/failoverBUpstream above drive, but // exercising the bareWinner AUTHORIZATION fix, not failover. A must // NEVER be hit: bareWinnerGroupName can only use "bare-winner-b", so a // correct group-aware bareWinner never even considers "bare-winner-a" // as a candidate for this group's request, unlike a failover fall- // through (which WOULD hit A first). bareWinnerAHits catches a // regression back to the pre-fix, group-blind selection, which would // route here and get an error response instead of the expected 200. bareWinnerAHits := new(int64) bareWinnerAUpstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { atomic.AddInt64(bareWinnerAHits, 1) w.Header().Set("Content-Type", "application/json") w.WriteHeader(http.StatusInternalServerError) _, _ = w.Write([]byte(`{"error":{"message":"bare-winner-a must never be reached by bareWinnerGroupName","type":"server_error"}}`)) })) defer bareWinnerAUpstream.Close() bareWinnerBUpstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") w.WriteHeader(http.StatusOK) _, _ = w.Write([]byte(`{"id":"c-bare-winner","object":"chat.completion","model":"` + bareWinnerModelID + `","choices":[{"index":0,"message":{"role":"assistant","content":"served by bare-winner-b"},"finish_reason":"stop"}],"usage":{"prompt_tokens":1,"completion_tokens":1}}`)) })) defer bareWinnerBUpstream.Close() // multiGroupBUpstream backs exerciseMultiGroupPersonalGrant (multi- // group/personal-grant feature, UserConfig.Groups/Providers/Models, // llmgateway.go): the ONLY grant that reaches "multi-group-b" at all // is the harness user's own PERSONAL grant (run()'s "users" // override), restricted to multiGroupAllowedModelID alone — neither // of the user's two member groups (multiGroupEngGroupName/ // multiGroupOpsGroupName) names "multi-group-b" in their own // providers list. A CORRECT per-grant check must still deny // multiGroupDeniedModelID here; a naive "any grant allows the // provider (union) AND any grant allows the model (union)" check // would instead WRONGLY ALLOW it — neither member group restricts its // own models at all, so the models-side union collapses to "allow // anything," and pairing that with the personal grant's own // providers-side reach to "multi-group-b" authorizes a combination // neither grant alone ever granted — see this const block's own doc // comments below for the full shape. multiGroupBHits := new(int64) multiGroupBUpstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { atomic.AddInt64(multiGroupBHits, 1) w.Header().Set("Content-Type", "application/json") w.WriteHeader(http.StatusOK) _, _ = w.Write([]byte(`{"id":"c-multi-group","object":"chat.completion","model":"` + multiGroupAllowedModelID + `","choices":[{"index":0,"message":{"role":"assistant","content":"served by multi-group-b"},"finish_reason":"stop"}],"usage":{"prompt_tokens":1,"completion_tokens":1}}`)) })) defer multiGroupBUpstream.Close() // multiGroupCUpstream/multiGroupSharedXUpstream back // exerciseMultiGroupBareWinnerFailoverAndPassthroughPaths (LOW-8, // review round 2 — extends the multi-group/personal-grant probe // above to also cover HIGH-1's provider-coupled passthroughPaths fix // and HIGH-2's per-grant bareWinner/failover fix under the REAL // interpreter). Both serve multiGroupSharedModelID as a BARE id // alongside "multi-group-b" (all three configured below), the exact // three-owner collision shape HIGH-2's own compiled regression test // (TestModelRegistry_Resolve_MultiGrant_BareWinnerPrefersGrantThatActuallyAuthorizes) // drives, here reachable only through the harness user's THREE // distinct grants (multiGroupEngGroupName, multiGroupOpsGroupName, // and the personal grant) — sorted provider-name order is // "multi-group-b" < "multi-group-c" < "multi-group-shared-x", and // only the personal grant (restricted to multiGroupAllowedModelID // alone) reaches "multi-group-b", the sorted-first owner, so a // pre-fix bareWinner picks it and wrongly denies the whole request // even though multiGroupOpsGroupName's own unrestricted grant on // "multi-group-c" would happily serve it. // // multiGroupCUpstream also DOUBLES as the primary-failure trigger for // the failover half of that same assertion: it always 500s a chat- // completions call (path-detected below) specifically so // runMeteredCall's own failover loop falls through past the correctly // -chosen primary ("multi-group-c") to the next candidate // ("multi-group-shared-x", reachable via multiGroupEngGroupName's own // unrestricted grant) — while any OTHER path it receives (a native // passthrough request) succeeds normally, backing the HIGH-1 // passthroughPaths assertion multiGroupOpsGroupName's own // "passthroughPaths" override (below) restricts. multiGroupCChatHits := new(int64) multiGroupCPassthroughHits := new(int64) multiGroupCUpstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.URL.Path == "/v1/chat/completions" { atomic.AddInt64(multiGroupCChatHits, 1) w.Header().Set("Content-Type", "application/json") w.WriteHeader(http.StatusInternalServerError) _, _ = w.Write([]byte(`{"error":{"message":"multi-group-c chat completions always fails, forcing failover","type":"server_error"}}`)) return } atomic.AddInt64(multiGroupCPassthroughHits, 1) w.WriteHeader(http.StatusOK) })) defer multiGroupCUpstream.Close() multiGroupSharedXHits := new(int64) multiGroupSharedXUpstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.URL.Path == "/v1/chat/completions" { atomic.AddInt64(multiGroupSharedXHits, 1) w.Header().Set("Content-Type", "application/json") w.WriteHeader(http.StatusOK) _, _ = w.Write([]byte(`{"id":"c-multi-group-shared-x","object":"chat.completion","model":"` + multiGroupSharedModelID + `","choices":[{"index":0,"message":{"role":"assistant","content":"served by multi-group-shared-x"},"finish_reason":"stop"}],"usage":{"prompt_tokens":1,"completion_tokens":1}}`)) return } w.WriteHeader(http.StatusOK) })) defer multiGroupSharedXUpstream.Close() // round3SharedUpstream backs exerciseRound3ModelAndPathFromSameGrant // (HIGH fix, review round 3): a native passthrough request never // carries a translated "/v1/chat/completions" path the way the // chat-completions route does (handlePassthrough forwards rest // verbatim), so this upstream only ever needs to record that it was // reached at all — both P1 and P2 (this probe's own doc comment) must // deny before ever calling it. round3PassthroughHits := new(int64) round3PassthroughUpstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { atomic.AddInt64(round3PassthroughHits, 1) w.WriteHeader(http.StatusOK) })) defer round3PassthroughUpstream.Close() // failover.enabled must be set explicitly here: FailoverConfig.Enabled // defaults to false (coordinator ruling — a version upgrade with no // config change must preserve prior behavior), so without this the // exerciseFailover probe below would find failover-a's 500 returned // to the client unchanged, never reaching failover-b at all. // metricsProbeTrickyUserNameJSON: a user name containing a double // quote, a backslash, and a newline — auth.go's buildEntry validates // a name as non-empty only, never restricted to a URL-path-segment // character set (see its own doc comment), so this is a real, // legal config, not a contrived one. json.Marshal, not hand-built // string concatenation, produces the correctly JSON-escaped literal // to splice into attemptAccountingOverride below — this is JSON // escaping for the CONFIG document, a separate concern from the // Prometheus label escaping exerciseMetricsRoute actually verifies // in the RENDERED /metrics body. trickyNameJSON, err := json.Marshal(metricsProbeTrickyUserName) if err != nil { return fmt.Errorf("marshal metrics escaping probe user name: %w", err) } attemptAccountingOverride := `{"failover":{"enabled":true},"breaker":{"failureThreshold":2,"openDuration":"3s","maxOpenDuration":"6s"},` + `"providers":{"openai":{"type":"openai","baseUrl":"` + upstream.URL + `","apiKey":"sk-up","models":["` + testDataWantModel + `","` + builtinLookupModelID + `"]},` + `"brk":{"type":"openai","baseUrl":"` + brkUpstream.URL + `","apiKey":"sk-up","discovery":true,"discoveryInterval":"1ms"},` + // "warmretry" backs exerciseWarmFillRetry (production incident fix, // registry.go's warmFillRetryDelay) — deliberately NO // discoveryInterval override: the default (1h) keeps maybeRefresh's // own request-driven interval gate from ever firing again during // this probe, once warmFill's failed synchronous attempt (or the // retry that follows it) has set lastRefresh, so the retry is the // ONLY thing that can produce warmRetryUpstream's second hit. `"warmretry":{"type":"openai","baseUrl":"` + warmRetryUpstream.URL + `","apiKey":"sk-up","discovery":true},"` + slowProviderName + `":{"type":"openai","baseUrl":"` + slowUpstream.URL + `","apiKey":"sk-up","models":["` + slowProviderModel + `"]},` + // timeoutProviderName (feature: request timeout) — see its own // const block doc comment above for why this provider exists // alongside slowProviderName. `"` + timeoutProviderName + `":{"type":"openai","baseUrl":"` + timeoutUpstream.URL + `","apiKey":"sk-up","requestTimeout":"` + timeoutProviderRequestTimeout + `","models":["` + timeoutProviderModel + `"]},` + // hungBodyProviderName (finding F7) — see its own const block doc // comment above for why this provider exists alongside // timeoutProviderName. `"` + hungBodyProviderName + `":{"type":"openai","baseUrl":"` + hungBodyUpstream.URL + `","apiKey":"sk-up","requestTimeout":"` + hungBodyProviderRequestTimeout + `","models":["` + hungBodyProviderModel + `"]},` + // provenanceEstProviderName (feat: expose token-accounting // provenance) — see its own const block doc comment above. `"` + provenanceEstProviderName + `":{"type":"openai","baseUrl":"` + provenanceEstUpstream.URL + `","apiKey":"sk-up","models":["` + provenanceEstProviderModel + `"]},` + // eventsUpstreamErrorProviderName (F3 hook 2, coordinator request) // — see its own const block doc comment above for why this // provider carries no failover candidate. `"` + eventsUpstreamErrorProviderName + `":{"type":"openai","baseUrl":"` + eventsUpstreamErrorUpstream.URL + `","apiKey":"sk-up","models":["` + eventsUpstreamErrorProviderModel + `"]},` + // "anthropic" backs the /v1/messages passthrough probes // (exerciseMessagesRoute, below): a real anthropic-type provider, // interpreted end to end through // callAnthropicMessagesPassthrough. `"anthropic":{"type":"anthropic","baseUrl":"` + anthropicProbeUpstream.URL + `","apiKey":"sk-anth","models":["claude-test"]},` + // #nosec G101 -- test fixture literal, not a real credential `"failover-a":{"type":"openai","baseUrl":"` + failoverAUpstream.URL + `","apiKey":"sk-up","models":["` + failoverModelID + `"]},` + `"failover-b":{"type":"openai","baseUrl":"` + failoverBUpstream.URL + `","apiKey":"sk-up","models":["` + failoverModelID + `"]},` + // bareWinnerModelID's own const doc comment (above) has the full // account of why "bare-winner-a"/"bare-winner-b" exist alongside // failover-a/failover-b: same collision shape, different fix under // test. `"bare-winner-a":{"type":"openai","baseUrl":"` + bareWinnerAUpstream.URL + `","apiKey":"sk-up","models":["` + bareWinnerModelID + `"]},` + `"bare-winner-b":{"type":"openai","baseUrl":"` + bareWinnerBUpstream.URL + `","apiKey":"sk-up","models":["` + bareWinnerModelID + `"]},` + // "multi-group-b" backs exerciseMultiGroupPersonalGrant — see // multiGroupBUpstream's own doc comment above. Its models list // ALSO includes multiGroupSharedModelID (LOW-8 extension) so it // participates as the (wrongly-preferred, pre-fix) sorted-first // owner in the bareWinner/failover collision // exerciseMultiGroupBareWinnerFailoverAndPassthroughPaths drives // — see multiGroupCUpstream's own doc comment above for the full // three-provider shape. `"multi-group-b":{"type":"openai","baseUrl":"` + multiGroupBUpstream.URL + `","apiKey":"sk-up","models":["only","other","` + multiGroupSharedModelID + `"]},` + `"multi-group-c":{"type":"openai","baseUrl":"` + multiGroupCUpstream.URL + `","apiKey":"sk-up","models":["` + multiGroupSharedModelID + `"]},` + `"multi-group-shared-x":{"type":"openai","baseUrl":"` + multiGroupSharedXUpstream.URL + `","apiKey":"sk-up","models":["` + multiGroupSharedModelID + `"]},` + // "round3-shared" backs exerciseRound3ModelAndPathFromSameGrant // (HIGH fix, review round 3) — a single real provider both P1's // and P2's own groups reach, native passthrough only (no Models // list needed: passthrough authorization is purely group-driven, // never consults a provider's own configured Models). `"round3-shared":{"type":"openai","baseUrl":"` + round3PassthroughUpstream.URL + `","apiKey":"sk-up"}},` + // groups: ADDS bareWinnerGroupName and the two exerciseMultiGroupPersonalGrant // member groups to the map Unmarshal already populated from // .traefik.yml's own testData.groups.default — map keys merge // (run()'s own doc comment, above, on why "brk" and the other new // provider keys above are additive rather than replacing), so // "default" (providers: [], allow-all) is untouched. Neither // multiGroupEngGroupName nor multiGroupOpsGroupName names // "multi-group-b" — only the harness user's own PERSONAL grant // (below) does — so a correct implementation reaches "multi-group-b" // through that personal grant alone, not through either group. // multiGroupOpsGroupName's own "passthroughPaths" restricts ITS // OWN provider ("multi-group-c") to multiGroupAllowedPathID alone // (LOW-8/HIGH-1 extension) — multiGroupEngGroupName carries no // such restriction on its own provider ("multi-group-shared-x"), // and the fix under test (auth.go's // allowsPassthroughPathForProvider) must keep the two from // leaking into each other. // round3P1AGroupName/round3P1BGroupName (P1) and // round3P2HomeGroupName/round3P2FriendsGroupName (P2) — see // exerciseRound3ModelAndPathFromSameGrant's own doc comment for the // full shape each pair reproduces. `"groups":{"` + bareWinnerGroupName + `":{"providers":["bare-winner-b"]},` + `"` + multiGroupEngGroupName + `":{"providers":["multi-group-shared-x"]},` + `"` + multiGroupOpsGroupName + `":{"providers":["multi-group-c"],"passthroughPaths":["` + multiGroupAllowedPathID + `"]},` + `"` + round3P1AGroupName + `":{"providers":["round3-shared"],"models":["` + round3ModelID + `"]},` + `"` + round3P1BGroupName + `":{"providers":["round3-shared"],"passthroughPaths":["v1/chat/completions"]},` + `"` + round3P2HomeGroupName + `":{"providers":["round3-alpha"]},` + `"` + round3P2FriendsGroupName + `":{"providers":["round3-shared"],"passthroughPaths":["v1/chat/completions"]}},` + // modelMeta (feature v0.23): a config-override entry for // testDataWantModel, so exerciseHandler's GET /v1/models // assertion below proves resolveModelMeta's config-override // layer, modelObject's context_window/pricing extension fields, // and the whole registry.go/modelmeta.go resolution path all // run correctly INTERPRETED, not merely compiled — cheap to add // to this existing harness request, per this feature's own gate // requirement. builtinLookupModelID carries NO config override // at all (SHOULD-6, review round) — its context_window can only // come from an interpreted LOOKUP into the real, 200+-entry // builtinModelMetaTable map literal, proving that specific path // runs correctly under Yaegi too, not just the config-override // one testDataWantModel already exercises. // provenanceEstProviderModel additionally carries a modelMeta // free:true override (free-models feature) — it has no // configured price at all (unifiedCostMicros already bills it 0 // via the "unknown price, warn" path, exerciseUsageProvenance's // own assertions above never check its cost value, so this adds // no regression there), so free:true here is the ONLY reason // adminUsageModelFree (admin.go) marks it free: exactly the // modelMeta path exerciseFreeModelUsageDetail (below) exists to // exercise interpreted, as opposed to the OTHER free path (a // KNOWN price of exactly 0), which carries no fixture of its own // here — both are already covered compiled (admin_test.go). `"modelMeta":{"` + testDataWantModel + `":{"contextTokens":` + yaegiMetaContextTokens + `,"inputCostPerMTokMicroUsd":1250000,"outputCostPerMTokMicroUsd":10000000},"` + provenanceEstProviderModel + `":{"free":true}},` + `"mcpServers":{"` + mcpProbeServerName + `":{"url":"` + mcpProbeUpstream.URL + `"},"` + targetHealthDownServerName + `":{"url":"` + targetHealthDownURL + `"},"` + targetHealthProbeOnlyServerName + `":{"url":"` + targetHealthProbeOnlyUpstream.URL + `"}},` + // targetHealth (feat/target-health): enabled. probeInterval is // deliberately well ABOVE the floor (10s): exerciseTargetHealthProbeWarmup // (below) spends this gateway's ONE sweep for the whole // probeInterval window right after construction, before any other // exercise runs, specifically so no LATER exercise's own GET // /admin/api/targets, /metrics, /v1/mcp/servers, or /v1/agents // call can trigger a second sweep mid-run and race a passive- // traffic assertion (F5, feat/target-health review — see // exerciseTargetHealth's own doc comment for the exact race this // avoids). "60s" comfortably outlasts this whole harness's // real-world runtime. `"targetHealth":{"enabled":true,"probeInterval":"60s"},` + // metrics (Prometheus text-exposition endpoint): enabled with // modelLabel on, so exerciseMetricsRoute below exercises the // opt-in per-(provider,model) breakdown under the interpreter // too, not just the always-on families. allowedCIDRs names the // exact block httptest.NewRequest's own default RemoteAddr // (192.0.2.1:1234, verified empirically) falls inside, so the // CIDR-bypass path is reachable with a plain httptest.NewRequest // carrying no explicit RemoteAddr override. `"metrics":{"enabled":true,"allowedCIDRs":["` + metricsProbeAllowedCIDR + `"],"modelLabel":true},` + // The third inline user (metricsProbeTrickyUserName) exists // purely so exerciseMetricsRoute can prove the escaping path // under the interpreter — it makes no requests of its own, but // still renders a real (zero-traffic) llmgateway_requests_total // series, since writeUsageMetrics (metrics.go) lists every // active user regardless of traffic. // // The fourth inline user (bareWinnerFriendAPIKey) is // exerciseBareWinnerGroupAware's own — group bareWinnerGroupName, // authorized for "bare-winner-b" only (the "groups" override // above). // // The fifth inline user (multiGroupUserAPIKey) is // exerciseMultiGroupPersonalGrant's own — member of BOTH // multiGroupEngGroupName and multiGroupOpsGroupName (group + // groups, multi-group support), plus a personal grant // (providers/models) restricting "multi-group-b" to // multiGroupAllowedModelID alone. // // The sixth inline user (round3P1UserAPIKey) is // exerciseRound3ModelAndPathFromSameGrant's own P1 principal — // member of BOTH round3P1AGroupName (models-restricted) and // round3P1BGroupName (paths-restricted), no personal grant. // // The seventh inline user (round3P2UserAPIKey) is that same // probe's own P2 principal — member of round3P2HomeGroupName // (primary, unrestricted) and round3P2FriendsGroupName // (paths-restricted), plus a personal grant restricting // "round3-shared" to round3PersonalModelID alone. `"admin":{"enabled":true},"users":{"inline":[{"name":"tester","group":"default","apiKey":"` + testDataUserAPIKey + `"},{"name":"admin1","group":"default","apiKey":"` + attemptAccountingAdminAPIKey + `","admin":true},{"name":` + string(trickyNameJSON) + `,"group":"default","apiKey":"sk-tricky"},{"name":"bare-winner-friend","group":"` + bareWinnerGroupName + `","apiKey":"` + bareWinnerFriendAPIKey + `"},{"name":"multi-group-user","group":"` + multiGroupEngGroupName + `","groups":["` + multiGroupOpsGroupName + `"],"apiKey":"` + multiGroupUserAPIKey + `","providers":["multi-group-b"],"models":["` + multiGroupAllowedModelID + `"]},{"name":"round3-p1-user","group":"` + round3P1AGroupName + `","groups":["` + round3P1BGroupName + `"],"apiKey":"` + round3P1UserAPIKey + `"},{"name":"round3-p2-user","group":"` + round3P2HomeGroupName + `","groups":["` + round3P2FriendsGroupName + `"],"apiKey":"` + round3P2UserAPIKey + `","providers":["round3-shared"],"models":["` + round3PersonalModelID + `"]},` + // The eighth inline user (eventsProbeUserAPIKey) is // exerciseRateLimitEventAndRejections' own — group "default" // (allow-all), but with its own requestsPerMinute:1 limit so a // second call under this key alone 429s deterministically. `{"name":"events-probe","group":"default","apiKey":"` + eventsProbeUserAPIKey + `","limits":{"requestsPerMinute":1}}]}}` if err = json.Unmarshal([]byte(attemptAccountingOverride), cfgVal.Interface()); err != nil { return fmt.Errorf("decode attempt-accounting harness override into the interpreted Config: %w", err) } newVal, err := i.Eval(pkgName + ".New") if err != nil { return fmt.Errorf("resolve %s.New under yaegi: %w", pkgName, err) } if newVal.Kind() != reflect.Func { return fmt.Errorf("%s.New is not a function (kind %s)", pkgName, newVal.Kind()) } next := http.HandlerFunc(func(http.ResponseWriter, *http.Request) {}) results := newVal.Call([]reflect.Value{ reflect.ValueOf(context.Background()), reflect.ValueOf(next), cfgVal, reflect.ValueOf("llmgw-check"), }) if len(results) != 2 { return fmt.Errorf("%s.New returned %d values, want 2 (http.Handler, error)", pkgName, len(results)) } if errVal := results[1]; !errVal.IsNil() { return fmt.Errorf("%s.New returned an error: %v", pkgName, errVal.Interface()) } handlerVal := results[0] if handlerVal.IsNil() { return fmt.Errorf("%s.New returned a nil handler", pkgName) } handler, ok := handlerVal.Interface().(http.Handler) if !ok { return fmt.Errorf("%s.New's returned value does not implement http.Handler", pkgName) } // exerciseTargetHealthProbeWarmup (feat/target-health, F5 review) runs // FIRST, before exerciseHandler and everything after it: its own doc // comment explains why this ordering is what makes every later // target-health assertion in this harness deterministic rather than // racing the active probe sweep under Yaegi's interpreter overhead. if err := exerciseTargetHealthProbeWarmup(handler); err != nil { return err } if err := exerciseHandler(handler, builtinLookupContextTokens, failoverBHits); err != nil { return err } // exerciseTargetHealth (feat/target-health) runs right after // exerciseHandler: its own part (a) reads the passive-recording // side effect of exerciseFederatedTooLarge's federated tools/call, // already driven above as one of exerciseHandler's own sub-probes, // so it must not run before that attempt has happened. Its part (b) // drives its own, independent target (targetHealthDownServerName), // so it cannot perturb any later probe's own counters or /metrics // assertions. if err := exerciseTargetHealth(handler); err != nil { return err } // exerciseBareWinnerGroupAware (fix/group-aware-bare-winner, // 2026-08-27) runs right after exerciseHandler and before // exerciseBreaker: it drives its own, freshly named providers and // model id (bareWinnerModelID's own doc comment above), so it cannot // perturb exerciseBreaker's "brk"-specific hit counting or any later // probe's /metrics assertions, which are all keyed to OTHER provider/ // model names. if err := exerciseBareWinnerGroupAware(handler, bareWinnerAHits); err != nil { return err } // exerciseMultiGroupPersonalGrant (multi-group/personal-grant feature) // runs right after exerciseBareWinnerGroupAware, for the identical // reason: it drives its own, freshly named provider/groups/user, so it // cannot perturb exerciseBreaker's "brk"-specific hit counting or any // later probe's /metrics assertions, which are all keyed to OTHER // provider/model names. if err := exerciseMultiGroupPersonalGrant(handler, multiGroupBHits); err != nil { return err } // exerciseMultiGroupBareWinnerFailoverAndPassthroughPaths (LOW-8, // review round 2) runs right after exerciseMultiGroupPersonalGrant, // against the SAME multi-group-user principal but its own, freshly // named providers/models/paths, so it cannot perturb any earlier or // later probe's own hit counters or /metrics assertions. if err := exerciseMultiGroupBareWinnerFailoverAndPassthroughPaths(handler, multiGroupBHits, multiGroupCChatHits, multiGroupCPassthroughHits, multiGroupSharedXHits); err != nil { return err } // exerciseRound3ModelAndPathFromSameGrant (HIGH fix, review round 3) // runs right after, against its own, freshly named P1/P2 providers/ // groups/users, so it cannot perturb any earlier or later probe's own // hit counters or /metrics assertions. if err := exerciseRound3ModelAndPathFromSameGrant(handler, round3PassthroughHits); err != nil { return err } // exerciseBreaker (feat/provider-health, adversarial-review round 2): // runs after exerciseHandler, against the SAME interpreted handler — // proves the discovery circuit breaker's open/half-open/closed state // machine, its new interpreted breakerState (a named int32 with a // String() method), and recordHealthLocked's classification call all // run correctly under Yaegi, not merely compiled. limits.go's own // matchesSentinel doc comment (Trap-4/reverse-direction history) is // exactly why this class of divergence needs its own interpreted // probe: a compiled `go test` pass here would prove nothing about the // interpreter. if err := exerciseBreaker(handler, brkModelsHits, brkHealthy); err != nil { return err } // exerciseWarmFillRetry (production incident fix, registry.go's // warmFillRetryDelay) runs right after exerciseBreaker: it drives its // own, dedicated "warmretry" provider/upstream, so it cannot perturb // exerciseBreaker's own "brk"-specific hit counting, and nothing below // depends on "warmretry"'s own hit count either. if err := exerciseWarmFillRetry(handler, warmRetryHits); err != nil { return err } // exerciseMetricsRoute runs after exerciseBreaker has already settled: // GET /metrics runs registry.maybeRefresh like every other route // (ServeHTTP's own unconditional call at entry), and exerciseBreaker's // own hit-counting is timing-sensitive against the "brk" provider // specifically — probing metrics first could perturb the very // discovery-attempt counts that test samples. Ordering the metrics // probe after avoids any such interaction. if err := exerciseMetricsRoute(handler); err != nil { return err } // exerciseLatencyMetrics (feat: instrument upstream latency) runs // after exerciseMetricsRoute, for the same reason exerciseMetricsRoute // itself used to be last: it reads the exact same /metrics endpoint // and must not race or perturb anything exerciseBreaker/ // exerciseMetricsRoute still care about. It relies on exerciseHandler's // own exerciseAttemptAccounting sub-probe (already run, above) having // driven a real, non-streaming POST /v1/chat/completions against the // "openai" provider — see its own doc comment. if err := exerciseLatencyMetrics(handler); err != nil { return err } // exerciseUsageProvenance (feat: expose token-accounting provenance) // runs LAST, for the identical reason: it reads /metrics and GET // /admin/api/overview one more time and must not perturb any counter // an earlier probe already asserted on. if err := exerciseUsageProvenance(handler); err != nil { return err } // exerciseModelUsageRanking (feat: per-model usage statistics) runs // after everything above: it only READS admin endpoints plus one more // chat completion of its own, and asserts on a kindModel counter no // earlier probe touches, so it can neither perturb nor be perturbed by // the provider-level counters they assert on. if err := exerciseModelUsageRanking(handler); err != nil { return err } // exerciseFreeModelUsageDetail/exerciseMediaModelUsage (free-models // feature) run right after: the first only READS admin endpoints // (detail=1), the second sends one real POST /v1/images/generations // that adds exactly 1 to modelUsageCanonicalID's own req counter — a // perturbation exerciseModelUsageRanking's own assertions (above, // already run) never re-check, and nothing below depends on that // counter's exact value either (exerciseUsageModelsSpan only checks // the echoed span; exerciseRateLimitEventAndRejections uses its own // dedicated principal). if err := exerciseFreeModelUsageDetail(handler); err != nil { return err } if err := exerciseMediaModelUsage(handler); err != nil { return err } // exerciseAdminOverviewReplicaAndWarnings/exerciseUsageModelsSpan (F4/ // F9/F10/F2, v0.3 dashboard task) run next: both only READ admin // endpoints, so neither perturbs nor is perturbed by anything above. if err := exerciseAdminOverviewReplicaAndWarnings(handler); err != nil { return err } if err := exerciseUsageModelsSpan(handler); err != nil { return err } // exerciseRateLimitEventAndRejections (F3/F4, v0.3 dashboard task) // runs next, against its own dedicated requestsPerMinute:1 user // (eventsProbeUserAPIKey), so its two calls cannot perturb any earlier // probe's own per-user counters. if err := exerciseRateLimitEventAndRejections(handler); err != nil { return err } // exerciseUpstreamAndTimeoutEvents (F3 hook 2, coordinator request // 2026-09-23) runs LAST: each of its two assertions reads GET // /admin/api/events?limit=1 for the single newest entry right after // producing it, so nothing after it depends on event ordering staying // undisturbed. if err := exerciseUpstreamAndTimeoutEvents(handler); err != nil { return err } // exerciseAdminRedesignStats (admin_redesign.go, admin dashboard // redesign WP-G) runs LAST of all, against its own dedicated Gateway // instance (own config, own fake Redis, own upstream) rather than the // shared `handler` above — see that function's own doc comment for // why. return exerciseAdminRedesignStats(i, pkgName) } // builtinLookupModelID is a real, stable entry in the generated // builtinModelMetaTable (pricing_data_gen.go) — review fix, SHOULD-6 — // added to the harness's openai provider with NO modelMeta config // override of its own, so its GET /v1/models context_window can only // ever come from an interpreted lookup into the real map literal, not // the config-override path testDataWantModel already exercises. const builtinLookupModelID = "gpt-4o" // failoverModelID is the bare model id both "failover-a" and "failover-b" // (run(), above) are configured with — the run()-owned upstream servers // exerciseFailover (below) drives. const failoverModelID = "failover-shared" // bareWinnerModelID is the bare model id both "bare-winner-a" and // "bare-winner-b" (run(), above) are configured with — deliberately the // identical collision SHAPE as failoverModelID above, but exercising // registry.go's bareWinner AUTHORIZATION fix (2026-08-27) rather than // failover: bareWinnerGroupName (below) is authorized for "bare-winner-b" // only, yet "bare-winner-a" sorts first alphabetically. Before the fix, // bareWinner always returned the GLOBAL sorted-first owner regardless of // the caller, so this exact group got errModelDenied for a model it was // actually entitled to use — the production repro this whole fix exists // for (uni/macstudio serving text-embedding-multilingual-e5-base, the // "friends" group entitled to uni but denied because macstudio sorts // first). exerciseBareWinnerGroupAware (below) proves the fix resolves // and serves under the REAL interpreter, not merely compiled. const bareWinnerModelID = "bare-winner-shared" // bareWinnerGroupName is the group exerciseBareWinnerGroupAware // authenticates as: authorized for "bare-winner-b" only (run()'s "groups" // override), never "bare-winner-a" — see bareWinnerModelID's own doc // comment for why that specific asymmetry is the point. const bareWinnerGroupName = "bare-winner-friends" // bareWinnerFriendAPIKey authenticates bareWinnerGroupName's one inline // user (run()'s "users" override). const bareWinnerFriendAPIKey = "sk-bare-winner-friend" // #nosec G101 -- test fixture literal, not a real credential // multiGroupEngGroupName/multiGroupOpsGroupName are the two member groups // exerciseMultiGroupPersonalGrant's own inline user belongs to // (UserConfig.Group/Groups, llmgateway.go, run()'s "users" override) — // neither names "multi-group-b" in its own "groups" override providers // list, so the only grant reaching it at all is the user's own personal // one (multiGroupUserAPIKey's own doc comment, below). const ( multiGroupEngGroupName = "multi-group-eng" multiGroupOpsGroupName = "multi-group-ops" ) // multiGroupUserAPIKey authenticates exerciseMultiGroupPersonalGrant's // own inline user: member of BOTH multiGroupEngGroupName and // multiGroupOpsGroupName (run()'s "users" override), plus a personal // grant (Providers/Models) restricting "multi-group-b" to // multiGroupAllowedModelID alone. const multiGroupUserAPIKey = "sk-multi-group-user" // #nosec G101 -- test fixture literal, not a real credential // multiGroupAllowedModelID/multiGroupDeniedModelID are two ids served by // the SAME real provider ("multi-group-b", run()'s "providers" override): // the personal grant's own Models list (run()'s "users" override) names // only the "only" form, so the "other" form must resolve as // errModelDenied — proving the multi-group/personal-grant feature's // per-grant (not per-union) model check runs correctly under Yaegi, not // merely compiled (registry_test.go's own // TestModelRegistry_Resolve_MultiGroup_CrossGrantLeakDenied covers the // identical rule compiled). const ( multiGroupAllowedModelID = "multi-group-b/only" multiGroupDeniedModelID = "multi-group-b/other" ) // multiGroupSharedModelID is the BARE id (no provider prefix) all three // of "multi-group-b", "multi-group-c", and "multi-group-shared-x" serve // (run()'s "providers" override) — LOW-8's extension of this probe // (review round 2), exercising HIGH-2's bareWinner/listFor/failover fix // under the REAL interpreter: sorted provider-name order is // "multi-group-b" < "multi-group-c" < "multi-group-shared-x", and only // the harness user's PERSONAL grant reaches "multi-group-b" — restricted // to multiGroupAllowedModelID alone, which does NOT cover this bare id — // so a pre-fix, union-based bareWinner picks "multi-group-b" first and // wrongly denies the whole request, even though multiGroupOpsGroupName's // own unrestricted grant on "multi-group-c" (sorted next) would happily // serve it. multiGroupCUpstream's own doc comment (above) has the full // three-provider shape, including how it doubles as the failover // trigger. const multiGroupSharedModelID = "multi-group-shared" // multiGroupAllowedPathID is the one native-passthrough REST path // multiGroupOpsGroupName's own "passthroughPaths" override (run()'s // "groups" override) allows on ITS OWN provider ("multi-group-c") — // LOW-8's HIGH-1 extension (review round 2): multiGroupEngGroupName // carries no such restriction on its own, DIFFERENT provider // ("multi-group-shared-x"), and // exerciseMultiGroupBareWinnerFailoverAndPassthroughPaths (below) proves // the two never leak into each other under the REAL interpreter — the // identical rule routes_passthrough_test.go's own // TestHandlePassthrough_MultiGroup_PassthroughPathsNeverUnionAcrossProviders // covers compiled. const multiGroupAllowedPathID = "v1/allowed-path" // multiGroupDeniedPathID is any path NOT in multiGroupOpsGroupName's own // "passthroughPaths" list — must be denied on "multi-group-c" regardless // of multiGroupEngGroupName's own unrestricted paths on its DIFFERENT // provider. const multiGroupDeniedPathID = "v1/other-path" // round3P1AGroupName/round3P1BGroupName back // exerciseRound3ModelAndPathFromSameGrant's own P1 reproduction (HIGH // fix, review round 3): group A restricts models (round3ModelID) but not // paths; group B restricts paths ("v1/chat/completions") but not models // — both cover the SAME provider ("round3-shared"). round3P1UserAPIKey's // own principal belongs to BOTH, no personal grant. const ( round3P1AGroupName = "round3-p1-a" round3P1BGroupName = "round3-p1-b" round3P1UserAPIKey = "sk-round3-p1-user" // #nosec G101 -- test fixture literal, not a real credential round3ModelID = "round3-m1" ) // round3P2HomeGroupName/round3P2FriendsGroupName back // exerciseRound3ModelAndPathFromSameGrant's own P2 reproduction: "home" // is the PRIMARY group (no path restriction of its own); "friends" // restricts paths on "round3-shared" but not models. // round3P2UserAPIKey's own principal belongs to both, PLUS a personal // grant restricting "round3-shared" to round3PersonalModelID alone — // borrowing home's own unrestricted paths. const ( round3P2HomeGroupName = "round3-p2-home" round3P2FriendsGroupName = "round3-p2-friends" round3P2UserAPIKey = "sk-round3-p2-user" // #nosec G101 -- test fixture literal, not a real credential round3PersonalModelID = "round3-shared/m1" ) // eventsProbeUserAPIKey authenticates exerciseRateLimitEventAndRejections' // own inline user (run()'s "users" override): group "default" (allow-all, // like every other probe user), but with its own requestsPerMinute:1 // limit, so a SECOND call under this key alone 429s regardless of what // every other probe already did to testDataUserAPIKey's own counters — // no other probe may reuse this key. const eventsProbeUserAPIKey = "sk-events-probe" // #nosec G101 -- test fixture literal, not a real credential // eventsUpstreamErrorProviderName/-Model (F3 hook 2, coordinator request // 2026-09-23) back exerciseUpstreamAndTimeoutEvents: a SINGLE provider — // deliberately no failover candidate configured for this model id — whose // upstream always 500s, so the client-facing response IS the classified // *providerHTTPError handleAdapterErrorEnvelope hands to writeUpstream, // and recordUpstreamEvent's default branch runs on every call, not just // the first of a failover attempt sequence. const ( eventsUpstreamErrorProviderName = "eventsprobe-upstream" eventsUpstreamErrorProviderModel = "gpt-eventsprobe-upstream-test" ) // mcpProbeServerName is the federated MCP server run() configures against // mcpProbeUpstream, and mcpProbeToolName is a tool id carrying its // "_" prefix so resolveFederatedTool routes POST /mcp's // tools/call there. The tool needs no tools/list entry — resolution is // pure prefix matching on the configured, group-allowed server names. const ( mcpProbeServerName = "probe" mcpProbeToolName = mcpProbeServerName + "_big" ) // targetHealthDownServerName is the second MCP server run() configures, // against targetHealthDownUpstream — a URL nothing listens on, opened // then immediately closed (feat/target-health, exerciseTargetHealth, // below). targetHealthFailureThreshold mirrors the plugin's own // defaultTargetHealthFailureThreshold (target_health.go): this harness // module cannot import that unexported constant across the Yaegi // reflection boundary, so it is copied here, the same independent- // literal approach every other harness constant mirroring plugin // internals already takes in this file (e.g. testDataWantModel). const ( targetHealthDownServerName = "target-health-down" targetHealthFailureThreshold = 3 ) // targetHealthProbeOnlyServerName is a THIRD MCP server run() configures, // against targetHealthProbeOnlyUpstream — a real, always-answering // upstream this harness never proxies traffic through (no per-server // proxy call, no federated tools/list or tools/call ever names it): its // health entry can only ever be written by the ACTIVE PROBE sweep // (target_health.go's probeMCPTarget), never by passive traffic. F5, // feat/target-health review: exerciseTargetHealthProbeWarmup (below) // uses this determinism to assert source == "probe" without racing // anything, and to prove the interpreted initialize/close handshake path // itself, not just "some sweep ran". const targetHealthProbeOnlyServerName = "target-health-probe-only" // metricsProbeAllowedCIDR is the CIDR block attemptAccountingOverride's // own metrics.allowedCIDRs names — see that literal's own comment for why // this must contain httptest.NewRequest's default RemoteAddr. const metricsProbeAllowedCIDR = "192.0.2.0/24" // metricsProbeOutsideAddr is a source address deliberately OUTSIDE // metricsProbeAllowedCIDR (TEST-NET-3, RFC 5737) — exerciseMetricsRoute's // own proof that the allowlist is not simply matching everything. const metricsProbeOutsideAddr = "203.0.113.9:5555" // metricsProbeTrickyUserName is a configured user name containing a // double quote, a backslash, and a newline — the exact character set // escapeLabelValue (metrics.go) exists to handle, driven under the REAL // interpreter (review fix, adversarial verification 2026-08-23): the // compiled test suite already proves escaping correct // (TestMetrics_LabelValuesWithSpecialChars_Escaped, // TestEscapeLabelValue_InvalidUTF8_Injective), but this codebase's own // history is that a compiled pass has repeatedly said nothing about the // interpreted shape. const metricsProbeTrickyUserName = "weird\"user\\name\nwith-newline" // mcpProbeOversizeBytes is how much filler mcpProbeUpstream writes: over // mcpBackendCallResponseMaxBytes (maxRequestBytes, 10MiB) so the response // trips doBackendJSONRPC's cap. Kept as its own named value rather than // importing the plugin's const, because this harness deliberately builds // nothing from the plugin package at compile time — everything it asserts // about the plugin must come from the interpreter. const mcpProbeOversizeBytes = 11 << 20 // mcpTooLargeWantMessage is the exact JSON-RPC error message // mcpFederatedToolsCall writes when errors.Is matches // errMCPResponseTooLarge. "upstream error" here instead means errors.Is // returned false under the interpreter for a chain that is true when // compiled — which is precisely the class of Yaegi divergence this // harness exists to catch, and which no compiled test can see. const mcpTooLargeWantMessage = "response too large" // stampedCheckVersion is the fake release version stampReleaseVersion // writes into the interpreted copy. It only has to differ from // devPluginVersion; nothing asserts its value. const stampedCheckVersion = "9.9.9-yaegicheck" // stampReleaseVersion rewrites pluginVersion in the GOPATH COPY of // version.go (never the repo's own file) so the copy no longer carries // the dev sentinel. // // Without this, New's telemetry gate returns before ever calling into the // vendored oss-telemetry package, and this harness would report OK while // leaving the plugin's only non-stdlib runtime import unexercised under // the interpreter — precisely the blind spot yaegi-check exists to close. // Import resolution alone is not enough: a cross-package call into // interpreted third-party code is its own risk under Yaegi. // // Telemetry stays suppressed by the DO_NOT_TRACK/OSS_TELEMETRY_DISABLED // env vars the caller sets first, so Send is entered and returns without // touching the network. The HTTP path beyond that env check is plain // net/http, already exercised elsewhere in this harness. func stampReleaseVersion(pkgDir string) error { path := filepath.Join(pkgDir, "version.go") data, err := os.ReadFile(path) //nolint:gosec // a temp dir this harness just created if err != nil { return fmt.Errorf("read %s: %w", path, err) } pattern := regexp.MustCompile(`(pluginVersion\s*=\s*")[^"]*(")`) if !pattern.Match(data) { return fmt.Errorf("no pluginVersion constant found in %s — has version.go been renamed?", path) } stamped := pattern.ReplaceAll(data, []byte(`${1}`+stampedCheckVersion+`${2}`)) // pkgDir is a temp dir run() created itself, and the filename is a // constant — no external input reaches this path. if err := os.WriteFile(path, stamped, 0o600); err != nil { //nolint:gosec // G703: path is filepath.Join(, "version.go") return fmt.Errorf("write %s: %w", path, err) } return nil } // readBuiltinContextTokens reads repoRoot/pricing_data_gen.go and // extracts modelID's own ContextTokens value directly out of the // generated source (review fix, SHOULD-6) — rather than hardcoding a // number in this harness, which would need hand-updating (and could // silently drift out of sync) every time `make pricing-sync` // regenerates the table against updated upstream data. func readBuiltinContextTokens(repoRoot, modelID string) (int, error) { data, err := os.ReadFile(filepath.Join(repoRoot, "pricing_data_gen.go")) if err != nil { return 0, fmt.Errorf("read pricing_data_gen.go: %w", err) } pattern := regexp.MustCompile(regexp.QuoteMeta(`"`+modelID+`":`) + `\s*\{ContextTokens:\s*(\d+)`) m := pattern.FindSubmatch(data) if m == nil { return 0, fmt.Errorf("no builtinModelMetaTable entry found for %q — pick a different, currently-present stable id", modelID) } n, err := strconv.Atoi(string(m[1])) if err != nil { return 0, fmt.Errorf("parse ContextTokens for %q: %w", modelID, err) } if n <= 0 { return 0, fmt.Errorf("builtinModelMetaTable entry for %q has ContextTokens=%d, want > 0", modelID, n) } return n, nil } // readHealth returns provider name's healthState string from the admin // overview, decoded generically like readProviderAttemptCounters does // (feat/provider-health). func readHealth(handler http.Handler, name string) (string, error) { req := httptest.NewRequest(http.MethodGet, "/admin/api/overview", nil) req.Header.Set("Authorization", "Bearer "+attemptAccountingAdminAPIKey) rec := httptest.NewRecorder() handler.ServeHTTP(rec, req) if rec.Code != http.StatusOK { return "", fmt.Errorf("GET /admin/api/overview: status = %d, body=%s", rec.Code, rec.Body.String()) } var body map[string]any if err := json.Unmarshal(rec.Body.Bytes(), &body); err != nil { return "", err } providers, _ := body["providers"].([]any) for _, raw := range providers { p, ok := raw.(map[string]any) if !ok { continue } if p["name"] == name { s, _ := p["healthState"].(string) return s, nil } } return "", fmt.Errorf("provider %q not in overview: %s", name, rec.Body.String()) } // readProviderModels returns provider name's known model id set from the // admin overview (registry.go's providerSnapshot.models, // provider-model-accordion task) — decoded generically like readHealth // above, for exerciseWarmFillRetry's own poll loop below. func readProviderModels(handler http.Handler, name string) ([]string, error) { req := httptest.NewRequest(http.MethodGet, "/admin/api/overview", nil) req.Header.Set("Authorization", "Bearer "+attemptAccountingAdminAPIKey) rec := httptest.NewRecorder() handler.ServeHTTP(rec, req) if rec.Code != http.StatusOK { return nil, fmt.Errorf("GET /admin/api/overview: status = %d, body=%s", rec.Code, rec.Body.String()) } var body map[string]any if err := json.Unmarshal(rec.Body.Bytes(), &body); err != nil { return nil, err } providers, _ := body["providers"].([]any) for _, raw := range providers { p, ok := raw.(map[string]any) if !ok { continue } if p["name"] != name { continue } rawModels, _ := p["models"].([]any) models := make([]string, 0, len(rawModels)) for _, rm := range rawModels { if s, ok := rm.(string); ok { models = append(models, s) } } return models, nil } return nil, fmt.Errorf("provider %q not in overview: %s", name, rec.Body.String()) } // drive fires n authenticated GET /v1/models requests, each of which runs // maybeRefresh at ServeHTTP entry, with a small gap so the 1ms discovery // interval always permits a fresh attempt when the breaker is closed // (feat/provider-health). func drive(handler http.Handler, n int) { for i := 0; i < n; i++ { req := httptest.NewRequest(http.MethodGet, "/v1/models", nil) req.Header.Set("Authorization", "Bearer "+testDataUserAPIKey) handler.ServeHTTP(httptest.NewRecorder(), req) time.Sleep(20 * time.Millisecond) } } // exerciseBreaker drives the "brk" provider (attemptAccountingOverride, // run()) through the discovery circuit breaker's full state cycle — // closed -> open -> half-open -> closed again — against the REAL // interpreter, proving under Yaegi what registry_health_test.go already // proves compiled: tryBeginRefresh/finishRefresh/recordHealthLocked, the // interpreted breakerState named-int32 type and its String() method, and // the classification call recordHealthLocked makes all behave correctly // interpreted, not merely compiled (feat/provider-health, adversarial- // review round 2 — limits.go's matchesSentinel doc comment is exactly why // this class of divergence needs its own interpreted probe). func exerciseBreaker(handler http.Handler, hits, healthy *int64) error { // Phase 1: hammer with the upstream 401ing. The breaker must open. deadline := time.Now().Add(10 * time.Second) state := "" for time.Now().Before(deadline) { drive(handler, 5) s, err := readHealth(handler, "brk") if err != nil { return err } state = s if s == "open" { break } } if state != "open" { return fmt.Errorf("BREAKER-1: healthState = %q after sustained 401s, want %q (upstream /v1/models hits=%d)", state, "open", atomic.LoadInt64(hits)) } openedAtHits := atomic.LoadInt64(hits) fmt.Printf("yaegi-check: breaker opened after %d upstream /v1/models hits\n", openedAtHits) // Phase 2: while open, further traffic must NOT reach the upstream. // openDuration is 3s; hammer for ~1s and require zero new hits. drive(handler, 30) duringOpen := atomic.LoadInt64(hits) - openedAtHits if duringOpen != 0 { return fmt.Errorf("BREAKER-2: %d upstream /v1/models hits during the open window, want 0 — backoff did not suppress discovery", duringOpen) } fmt.Println("yaegi-check: breaker open window suppressed all discovery hits") // Phase 3: provider recovers. After the backoff window elapses the // half-open probe must run and CLOSE the breaker again. atomic.StoreInt64(healthy, 1) recovered := false recoverDeadline := time.Now().Add(20 * time.Second) for time.Now().Before(recoverDeadline) { drive(handler, 5) s, err := readHealth(handler, "brk") if err != nil { return err } if s == "closed" { recovered = true break } } if !recovered { return fmt.Errorf("BREAKER-3: breaker never returned to closed after the provider recovered (hits=%d)", atomic.LoadInt64(hits)) } fmt.Println("yaegi-check: recovered provider closed the breaker again") return nil } // exerciseWarmFillRetry proves registry.go's warm-fill retry (production // incident fix: a provider whose SYNCHRONOUS initial discovery fetch // fails, at pod start, is left with zero discovered models until a // scheduled background retry recovers it) actually schedules and runs // under the REAL interpreter — time.AfterFunc, the retry's own // context.WithTimeout budget, and the finishRefresh bookkeeping it shares // with warmFill/refreshProvider all proven interpreted, not merely // compiled, per this harness's own stated purpose. // // "warmretry" (attemptAccountingOverride, run()) 401s on its very first // /v1/models hit — warmFill's own synchronous call, spent inside New() // itself, well before this probe (or any other) runs — then serves real // data on every hit after. Only the scheduled retry, firing after // registry.go's warmFillRetryDelay (10s, not overridable from outside // the package), can ever produce a second hit and populate // "warmretry-model"; this probe does not assume anything about how much // wall-clock time the OTHER probes ahead of it in run()'s orchestration // already burned relative to that 10s window — by the time this runs the // retry may already have completed, or may still be pending — it only // polls until the recovered state is reached (or times out) and then // checks hits landed on exactly 2, not more. func exerciseWarmFillRetry(handler http.Handler, hits *int64) error { // warmFillRetryDelay (10s) + warmFillTimeout (5s) + slack: the same // budget shape exerciseBreaker's own phase-3 recovery wait above // uses, just against this feature's fixed, non-configurable delay // instead of a config-driven breaker backoff. var models []string var err error recovered := false deadline := time.Now().Add(20 * time.Second) for time.Now().Before(deadline) { models, err = readProviderModels(handler, "warmretry") if err != nil { return err } if len(models) == 1 && models[0] == "warmretry-model" { recovered = true break } time.Sleep(100 * time.Millisecond) } if !recovered { return fmt.Errorf("WARMFILLRETRY-1: provider %q never showed the retried model %q within 20s (hits=%d, models=%v)", "warmretry", "warmretry-model", atomic.LoadInt64(hits), models) } if got := atomic.LoadInt64(hits); got != 2 { return fmt.Errorf("WARMFILLRETRY-2: warmRetryUpstream saw %d /v1/models hits once recovered, want exactly 2 (the failed warm fill plus one retry — no maybeRefresh interference)", got) } fmt.Println("yaegi-check: warm-fill retry recovered the provider's discovered models after the scheduled delay") return nil } // exerciseMetricsRoute drives GET /metrics against the interpreted // handler and proves both its auth gate and its rendering run correctly // under Yaegi — the metrics feature's own gate requirement: compiled // tests have repeatedly passed on this codebase while the interpreted // shape failed (matchesSentinel's own doc comment, limits.go, is the // canonical example), so a metrics-specific probe is not optional here. // // Four requests, mirroring TestMetrics_GateMatrix (metrics_test.go) // exactly so this harness and the compiled test suite assert the // identical contract from two different angles: // 1. unauthenticated, from an address outside metrics.allowedCIDRs -> 401 // 2. authenticated as "tester" (not admin) -> 403 // 3. authenticated as the admin user -> 200, body inspected // 4. unauthenticated, from an address INSIDE metrics.allowedCIDRs -> 200 // // Request 3's body is inspected for real, live-data content — not just a // 200 status — because a Yaegi-specific bug in this codebase's history // (matchesSentinel, limits.go) was a case where the interpreted code ran // without panicking yet produced a WRONG answer; a bare status-code check // would not have caught that class of failure here either. Specifically: // the "openai" provider's name, and — since attemptAccountingOverride // enables metrics.modelLabel — testDataWantModel's own id inside the // opt-in provider-model breakdown, both of which can only appear via a // live registry.snapshot() walk and a live providerModelScopeID join // under the real interpreter, not a static string in this harness. func exerciseMetricsRoute(handler http.Handler) error { unauthReq := httptest.NewRequest(http.MethodGet, "/metrics", nil) unauthReq.RemoteAddr = metricsProbeOutsideAddr rec := httptest.NewRecorder() handler.ServeHTTP(rec, unauthReq) if rec.Code != http.StatusUnauthorized { return fmt.Errorf("GET /metrics (no key, outside allowedCIDRs): status = %d, want 401, body=%s", rec.Code, rec.Body.String()) } nonAdminReq := httptest.NewRequest(http.MethodGet, "/metrics", nil) nonAdminReq.RemoteAddr = metricsProbeOutsideAddr nonAdminReq.Header.Set("Authorization", "Bearer "+testDataUserAPIKey) rec = httptest.NewRecorder() handler.ServeHTTP(rec, nonAdminReq) if rec.Code != http.StatusForbidden { return fmt.Errorf("GET /metrics (non-admin key): status = %d, want 403, body=%s", rec.Code, rec.Body.String()) } adminReq := httptest.NewRequest(http.MethodGet, "/metrics", nil) adminReq.RemoteAddr = metricsProbeOutsideAddr adminReq.Header.Set("Authorization", "Bearer "+attemptAccountingAdminAPIKey) rec = httptest.NewRecorder() handler.ServeHTTP(rec, adminReq) if rec.Code != http.StatusOK { return fmt.Errorf("GET /metrics (admin key): status = %d, want 200, body=%s", rec.Code, rec.Body.String()) } if ct := rec.Header().Get("Content-Type"); !strings.HasPrefix(ct, "text/plain") { return fmt.Errorf("GET /metrics (admin key): Content-Type = %q, want a text/plain prefix", ct) } body := rec.Body.String() // metricsProbeEscapedTrickyName is escapeLabelValue's OWN expected // output for metricsProbeTrickyUserName — a raw Go string literal // (backticks: every backslash below is a literal backslash // character, not a Go escape), computed by hand from the format // rule (\\ -> \\\\, " -> \\", newline -> \\n) rather than by calling // the function itself, so this assertion cannot pass merely because // the interpreted and this harness's own understanding of the rule // happen to agree by construction. const metricsProbeEscapedTrickyName = `weird\"user\\name\nwith-newline` for _, want := range []string{ "# TYPE llmgateway_requests_total counter", "# TYPE llmgateway_provider_healthy gauge", `llmgateway_provider_attempts_total{provider="openai"}`, "# TYPE llmgateway_provider_model_attempts_total counter", `llmgateway_provider_model_attempts_total{provider="openai",model="` + testDataWantModel + `"}`, `llmgateway_requests_total{scope_kind="user",scope_id="` + metricsProbeEscapedTrickyName + `"}`, } { if !strings.Contains(body, want) { return fmt.Errorf("GET /metrics (admin key): body missing %q — interpreted rendering diverged from the compiled shape; body=%s", want, body) } } if strings.Contains(body, metricsProbeTrickyUserName) { return fmt.Errorf("GET /metrics (admin key): body contains the RAW, unescaped tricky user name — escapeLabelValue did not run interpreted; body=%s", body) } allowlistedReq := httptest.NewRequest(http.MethodGet, "/metrics", nil) // default RemoteAddr, inside metricsProbeAllowedCIDR rec = httptest.NewRecorder() handler.ServeHTTP(rec, allowlistedReq) if rec.Code != http.StatusOK { return fmt.Errorf("GET /metrics (no key, inside allowedCIDRs): status = %d, want 200, body=%s", rec.Code, rec.Body.String()) } fmt.Println("yaegi-check: GET /metrics auth gate and rendering both verified interpreted") return nil } // exerciseLatencyMetrics proves the upstream-latency histograms (feat: // instrument upstream latency, task brief) render correctly under the // REAL interpreter — not merely that a compiled `go test` accepts the // same bytes. This package's own doc comment explains why that // distinction matters here specifically: compiled tests have repeatedly // passed on this codebase while the interpreted shape failed. // // No new traffic is driven here: exerciseHandler's own // exerciseAttemptAccounting sub-probe (already run, above, as part of // the exerciseHandler call in run()) already drove a real, non-streaming // POST /v1/chat/completions against the "openai" provider, so // llmgateway_upstream_duration_seconds{provider="openai",stream="false"} // already carries at least one real, interpreted observation by the // time this runs. // // Unlike exerciseMetricsRoute's own strings.Contains checks, this parses // the rendered body with prometheus/common/expfmt's REAL TextParser — // the same library a real Prometheus server's scrape loop is built on — // instead of this project's own hand-rolled parser (parsePrometheusText, // metrics_test.go, used by the compiled suite). A malformed histogram // (wrong cumulative order, a missing +Inf bucket, a mismatched // sample_count) does not merely fail one assertion below, it fails to // PARSE at all — exactly the failure mode that takes a real scrape // target DOWN rather than dropping one metric (metricWriter.histogram's // own doc comment, metrics.go, has the full account of why that // distinction matters). func exerciseLatencyMetrics(handler http.Handler) error { req := httptest.NewRequest(http.MethodGet, "/metrics", nil) req.RemoteAddr = metricsProbeOutsideAddr req.Header.Set("Authorization", "Bearer "+attemptAccountingAdminAPIKey) rec := httptest.NewRecorder() handler.ServeHTTP(rec, req) if rec.Code != http.StatusOK { return fmt.Errorf("GET /metrics (latency histogram probe): status = %d, want 200, body=%s", rec.Code, rec.Body.String()) } parser := expfmt.NewTextParser(model.LegacyValidation) families, err := parser.TextToMetricFamilies(strings.NewReader(rec.Body.String())) if err != nil { return fmt.Errorf("GET /metrics: a real Prometheus TextParser rejected the interpreted rendering: %w (body=%s)", err, rec.Body.String()) } durationFamily, ok := families["llmgateway_upstream_duration_seconds"] if !ok { return fmt.Errorf("GET /metrics: no llmgateway_upstream_duration_seconds family in the parsed output (body=%s)", rec.Body.String()) } if durationFamily.GetType() != dto.MetricType_HISTOGRAM { return fmt.Errorf("llmgateway_upstream_duration_seconds type = %s, want HISTOGRAM", durationFamily.GetType()) } var openaiHist *dto.Histogram for _, m := range durationFamily.GetMetric() { var provider, stream string for _, lp := range m.GetLabel() { switch lp.GetName() { case "provider": provider = lp.GetValue() case "stream": stream = lp.GetValue() } } if provider == "openai" && stream == "false" { openaiHist = m.GetHistogram() break } } if openaiHist == nil { return fmt.Errorf("GET /metrics: no llmgateway_upstream_duration_seconds{provider=\"openai\",stream=\"false\"} series found (body=%s)", rec.Body.String()) } // Real-parser structural checks, matching the compiled suite's own // TestMetrics_LatencyHistogram_BucketsCumulativeOrdered_ // PlusInfPresent_SumCountConsistent (metrics_test.go) — but here, // against dto.Bucket values the REAL parser produced, not this // project's own hand-rolled promSample. buckets := openaiHist.GetBucket() if len(buckets) == 0 { return fmt.Errorf("llmgateway_upstream_duration_seconds{provider=\"openai\",stream=\"false\"}: no buckets parsed") } prevBound := math.Inf(-1) var prevCount uint64 for i, b := range buckets { if b.GetUpperBound() <= prevBound { return fmt.Errorf("bucket %d upper_bound=%v is not strictly greater than the previous bucket's %v — buckets must be strictly ascending", i, b.GetUpperBound(), prevBound) } if b.GetCumulativeCount() < prevCount { return fmt.Errorf("bucket %d cumulative_count=%d is less than the previous bucket's %d — buckets must be cumulative (non-decreasing)", i, b.GetCumulativeCount(), prevCount) } prevBound = b.GetUpperBound() prevCount = b.GetCumulativeCount() } if !math.IsInf(buckets[len(buckets)-1].GetUpperBound(), 1) { return fmt.Errorf("last bucket upper_bound = %v, want +Inf", buckets[len(buckets)-1].GetUpperBound()) } if openaiHist.GetSampleCount() != prevCount { return fmt.Errorf("sample_count = %d, want %d — must exactly equal the final +Inf cumulative bucket", openaiHist.GetSampleCount(), prevCount) } if openaiHist.GetSampleCount() == 0 { return fmt.Errorf("sample_count = 0, want at least 1 (exerciseAttemptAccounting already drove one real, non-streaming chat completion against \"openai\")") } ttfbFamily, ok := families["llmgateway_upstream_ttfb_seconds"] if !ok { return fmt.Errorf("GET /metrics: no llmgateway_upstream_ttfb_seconds family in the parsed output (body=%s)", rec.Body.String()) } if ttfbFamily.GetType() != dto.MetricType_HISTOGRAM { return fmt.Errorf("llmgateway_upstream_ttfb_seconds type = %s, want HISTOGRAM", ttfbFamily.GetType()) } fmt.Println("yaegi-check: upstream-latency histograms parsed by a real Prometheus TextParser; buckets cumulative, strictly ascending, +Inf-terminated, sample_count consistent") return nil } // exerciseUsageProvenance proves the usage-provenance metric surface // (feat: expose token-accounting provenance, metrics.go's // provenanceStore/writeProvenanceMetrics, admin.go's // buildAdminProvenanceViews) renders correctly under the REAL // interpreter — not merely that a compiled `go test` accepts the same // bytes, this package's own doc comment's exact concern. // // Drives one real, non-streaming POST /v1/chat/completions against // provenanceEstProviderName, whose upstream (provenanceEstUpstream, run()) // answers 200 with no "usage" field at all — the exact shape // runUnified's estimation fallback (routes_unified.go) classifies as // "estimated". "reported" is not driven fresh here: exerciseHandler's own // exerciseAttemptAccounting sub-probe (already run, above) already drove // a real, usage-bearing POST /v1/chat/completions against "openai", // which is reported provenance by construction — reusing that traffic // instead of a third fixture, mirroring exerciseLatencyMetrics' own // "no new traffic" reasoning for its own duration-histogram assertion. func exerciseUsageProvenance(handler http.Handler) error { chatReq := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader( `{"model":"`+provenanceEstProviderName+`/`+provenanceEstProviderModel+`","messages":[{"role":"user","content":"hi"}]}`, )) chatReq.Header.Set("Authorization", "Bearer "+testDataUserAPIKey) chatReq.Header.Set("Content-Type", "application/json") chatRec := httptest.NewRecorder() handler.ServeHTTP(chatRec, chatReq) if chatRec.Code != http.StatusOK { return fmt.Errorf("POST /v1/chat/completions (usage-provenance harness): status = %d, want 200, body=%s", chatRec.Code, chatRec.Body.String()) } req := httptest.NewRequest(http.MethodGet, "/metrics", nil) req.RemoteAddr = metricsProbeOutsideAddr req.Header.Set("Authorization", "Bearer "+attemptAccountingAdminAPIKey) rec := httptest.NewRecorder() handler.ServeHTTP(rec, req) if rec.Code != http.StatusOK { return fmt.Errorf("GET /metrics (usage-provenance probe): status = %d, want 200, body=%s", rec.Code, rec.Body.String()) } parser := expfmt.NewTextParser(model.LegacyValidation) families, err := parser.TextToMetricFamilies(strings.NewReader(rec.Body.String())) if err != nil { return fmt.Errorf("GET /metrics: a real Prometheus TextParser rejected the interpreted rendering: %w (body=%s)", err, rec.Body.String()) } requestsFamily, ok := families["llmgateway_usage_provenance_requests_total"] if !ok { return fmt.Errorf("GET /metrics: no llmgateway_usage_provenance_requests_total family in the parsed output (body=%s)", rec.Body.String()) } if requestsFamily.GetType() != dto.MetricType_COUNTER { return fmt.Errorf("llmgateway_usage_provenance_requests_total type = %s, want COUNTER", requestsFamily.GetType()) } tokensFamily, ok := families["llmgateway_usage_provenance_tokens_total"] if !ok { return fmt.Errorf("GET /metrics: no llmgateway_usage_provenance_tokens_total family in the parsed output (body=%s)", rec.Body.String()) } if tokensFamily.GetType() != dto.MetricType_COUNTER { return fmt.Errorf("llmgateway_usage_provenance_tokens_total type = %s, want COUNTER", tokensFamily.GetType()) } findSample := func(fam *dto.MetricFamily, provider, provenance string) *dto.Metric { for _, m := range fam.GetMetric() { var p, k string for _, lp := range m.GetLabel() { switch lp.GetName() { case "provider": p = lp.GetValue() case "provenance": k = lp.GetValue() } } if p == provider && k == provenance { return m } } return nil } estReq := findSample(requestsFamily, provenanceEstProviderName, "estimated") if estReq == nil { return fmt.Errorf(`GET /metrics: no llmgateway_usage_provenance_requests_total{provider=%q,provenance="estimated"} sample found (body=%s)`, provenanceEstProviderName, rec.Body.String()) } if estReq.GetCounter().GetValue() != 1 { return fmt.Errorf(`llmgateway_usage_provenance_requests_total{provider=%q,provenance="estimated"} = %v, want 1`, provenanceEstProviderName, estReq.GetCounter().GetValue()) } estTok := findSample(tokensFamily, provenanceEstProviderName, "estimated") if estTok == nil { return fmt.Errorf(`GET /metrics: no llmgateway_usage_provenance_tokens_total{provider=%q,provenance="estimated"} sample found (body=%s)`, provenanceEstProviderName, rec.Body.String()) } if estTok.GetCounter().GetValue() <= 0 { return fmt.Errorf(`llmgateway_usage_provenance_tokens_total{provider=%q,provenance="estimated"} = %v, want > 0 (estimated substitutes a nonzero prompt count)`, provenanceEstProviderName, estTok.GetCounter().GetValue()) } reportedReq := findSample(requestsFamily, "openai", "reported") if reportedReq == nil { return fmt.Errorf(`GET /metrics: no llmgateway_usage_provenance_requests_total{provider="openai",provenance="reported"} sample found (body=%s) — exerciseAttemptAccounting's own earlier real, usage-bearing traffic must have been classified "reported"`, rec.Body.String()) } if reportedReq.GetCounter().GetValue() < 1 { return fmt.Errorf(`llmgateway_usage_provenance_requests_total{provider="openai",provenance="reported"} = %v, want >= 1`, reportedReq.GetCounter().GetValue()) } // GET /admin/api/overview: proves admin.go's buildAdminProvenanceViews // wiring also runs correctly interpreted, decoded generically (this // harness module is compiled, not interpreted; adminOverviewResponse/ // adminProviderView are unexported — see readProviderAttemptCounters' // own doc comment for why a generic decode is simpler here than // exporting test-only types across that boundary). adminReq := httptest.NewRequest(http.MethodGet, "/admin/api/overview", nil) adminReq.Header.Set("Authorization", "Bearer "+attemptAccountingAdminAPIKey) adminRec := httptest.NewRecorder() handler.ServeHTTP(adminRec, adminReq) if adminRec.Code != http.StatusOK { return fmt.Errorf("GET /admin/api/overview (usage-provenance probe): status = %d, want 200, body=%s", adminRec.Code, adminRec.Body.String()) } var overview map[string]any if err := json.Unmarshal(adminRec.Body.Bytes(), &overview); err != nil { return fmt.Errorf("decode GET /admin/api/overview body: %w", err) } providers, _ := overview["providers"].([]any) var provenanceEstProvider map[string]any for _, p := range providers { pm, ok := p.(map[string]any) if !ok { continue } if pm["name"] == provenanceEstProviderName { provenanceEstProvider = pm break } } if provenanceEstProvider == nil { return fmt.Errorf("GET /admin/api/overview: no provider entry named %q (body=%s)", provenanceEstProviderName, adminRec.Body.String()) } provenanceField, _ := provenanceEstProvider["provenance"].(map[string]any) estimatedEntry, _ := provenanceField["estimated"].(map[string]any) if estimatedEntry == nil { return fmt.Errorf(`GET /admin/api/overview: provider %q has no provenance.estimated entry: %+v`, provenanceEstProviderName, provenanceField) } if reqs, _ := estimatedEntry["requests"].(float64); reqs != 1 { return fmt.Errorf("GET /admin/api/overview: provider %q provenance.estimated.requests = %v, want 1", provenanceEstProviderName, estimatedEntry["requests"]) } if _, hasReported := provenanceField["reported"]; hasReported { return fmt.Errorf(`GET /admin/api/overview: provider %q provenance has a "reported" key, want it excluded from this compact admin summary: %+v`, provenanceEstProviderName, provenanceField) } fmt.Println("yaegi-check: usage-provenance metrics (llmgateway_usage_provenance_requests_total/-_tokens_total) and admin overview parsed by a real Prometheus TextParser/JSON decoder; estimated and reported provenances both present and distinct") return nil } // exerciseHandler runs two real requests against the interpreted // handler, drawn from the plugin's own .traefik.yml testData: an // authenticated GET /v1/models, which must return the configured // testDataWantModel; and the same route without credentials, which must // be refused with 401 — proving ServeHTTP, auth.identify, the model // registry, statusTrackingWriter, and the gateway's logger all execute // for real under Yaegi, not merely that the package's imports resolve. // builtinContextTokens is builtinLookupModelID's own real, extracted // builtinModelMetaTable value (SHOULD-6) — see that constant's own doc // comment. failoverBHits backs exerciseFailover's own call, below. func exerciseHandler(handler http.Handler, builtinContextTokens int, failoverBHits *int64) error { authedReq := httptest.NewRequest(http.MethodGet, "/v1/models", nil) authedReq.Header.Set("Authorization", "Bearer "+testDataUserAPIKey) authedRec := httptest.NewRecorder() handler.ServeHTTP(authedRec, authedReq) if authedRec.Code != http.StatusOK { return fmt.Errorf("GET /v1/models with the testData user key: status = %d, want 200, body=%s", authedRec.Code, authedRec.Body.String()) } if !strings.Contains(authedRec.Body.String(), testDataWantModel) { return fmt.Errorf("GET /v1/models body does not contain %q: %s", testDataWantModel, authedRec.Body.String()) } // modelMeta (feature v0.23), interpreted, decoded structurally (not // a raw substring match — SHOULD-6, review round: a bare Contains // check cannot tell WHICH model entry a given context_window value // belongs to, which matters once two entries can legitimately share // the same number) so each assertion below is pinned to its own // named model id. var modelsBody struct { Data []map[string]any `json:"data"` } if err := json.Unmarshal(authedRec.Body.Bytes(), &modelsBody); err != nil { return fmt.Errorf("decode GET /v1/models body: %w (body=%s)", err, authedRec.Body.String()) } byID := make(map[string]map[string]any, len(modelsBody.Data)) for _, entry := range modelsBody.Data { if id, ok := entry["id"].(string); ok { byID[id] = entry } } // run() layers a modelMeta config override for testDataWantModel // onto cfgVal before New() is ever called (attemptAccountingOverride // above) — proving resolveModelMeta's config-override layer and // modelObject's context_window/pricing extension fields both run // correctly, interpreted, through registry.go's listFor/ // resolveMetaFor. overridden, ok := byID[testDataWantModel] if !ok { return fmt.Errorf("GET /v1/models body has no entry for %q: %s", testDataWantModel, authedRec.Body.String()) } if cw, _ := overridden["context_window"].(float64); cw != 128000 { return fmt.Errorf("GET /v1/models %q context_window = %v, want 128000 (feature v0.23 config-override metadata harness): %s", testDataWantModel, overridden["context_window"], authedRec.Body.String()) } pricing, _ := overridden["pricing"].(map[string]any) if pricing == nil || pricing["input_per_mtok_usd"] != 1.25 || pricing["output_per_mtok_usd"] != 10.0 { return fmt.Errorf("GET /v1/models %q pricing = %v, want {input_per_mtok_usd:1.25 output_per_mtok_usd:10} (feature v0.23 config-override metadata harness): %s", testDataWantModel, overridden["pricing"], authedRec.Body.String()) } // builtinLookupModelID (SHOULD-6, review round) carries NO modelMeta // config override — its context_window can only come from an // interpreted LOOKUP into the real, 200+-entry builtinModelMetaTable // map literal (modelmeta.go's resolveModelMeta, builtin branch), // proving that specific path — not just the config-override one // above — runs correctly under Yaegi. builtinLookedUp, ok := byID[builtinLookupModelID] if !ok { return fmt.Errorf("GET /v1/models body has no entry for %q: %s", builtinLookupModelID, authedRec.Body.String()) } if cw, _ := builtinLookedUp["context_window"].(float64); int(cw) != builtinContextTokens { return fmt.Errorf("GET /v1/models %q context_window = %v, want %d (feature v0.23 builtin-table-lookup metadata harness, SHOULD-6): %s", builtinLookupModelID, builtinLookedUp["context_window"], builtinContextTokens, authedRec.Body.String()) } if err := exerciseAttemptAccounting(handler); err != nil { return err } if err := exerciseFailover(handler, failoverBHits); err != nil { return err } if err := exerciseRequestTimeout(handler); err != nil { return err } if err := exerciseRequestTimeoutMidBodyStall(handler); err != nil { return err } if err := exerciseFederatedTooLarge(handler); err != nil { return err } if err := exerciseFederatedSSENegotiation(handler); err != nil { return err } if err := exerciseStampedVersion(handler); err != nil { return err } if err := exerciseMessagesRoute(handler); err != nil { return err } unauthedReq := httptest.NewRequest(http.MethodGet, "/v1/models", nil) unauthedRec := httptest.NewRecorder() handler.ServeHTTP(unauthedRec, unauthedReq) if unauthedRec.Code != http.StatusUnauthorized { return fmt.Errorf("GET /v1/models without a key: status = %d, want 401, body=%s", unauthedRec.Code, unauthedRec.Body.String()) } return nil } // exerciseAttemptAccounting drives two real requests through handler — // Feature A's (v0.22) per-provider success-rate accounting, exercised // end to end under Yaegi: runUnified's attemptRecorder (routes_unified. // go) -> retryPolicy.do's context lookup (retry.go) -> // limiter.recordProviderAttempt's isTransient/isDeadlineExceeded // classification (limits.go, including matchesSentinel's errors.Is call // through providers.go's own multi-%w wrapping) -> its fire-and-forget // spawn (SHOULD-5) -> buildAdminOverview's own batched providerUsage read // (admin.go). Any interpreter-only failure in that chain — a construct // `go build`/`go test` cannot catch, exactly the class of bug this // harness exists for (SHOULD-4, v0.22 review round) — surfaces here as a // non-1/non-0 count or an outright panic under Yaegi. // // The first request (against "openai", instant upstream) proves the // success path: one attempt, zero failures. The second (against // slowProviderName, an upstream that never answers within // slowRequestDeadline — SHOULD-A, round 2) proves the FAILURE path: // exactly what round 1 of this harness claimed to prove but never // actually drove — the interpreted plugin never saw a non-nil error at // all, so isDeadlineExceeded was never exercised under this harness even // while it ran (and passed) under `go test`. THIS is what SHOULD-A was // for: the first version of this second request, run against round 2's // hand-rolled comma-ok Unwrap-walk fix (limits.go's matchesSentinel, // since replaced), FAILED right here — under the interpreter only, // despite passing every compiled `go test` row for the identical error // shape — because an interpreted interface type does not correctly match // a compiled concrete value's method set in Yaegi. matchesSentinel now // uses real errors.Is instead (its own doc comment in limits.go has the // full account); this harness is what caught the difference. // exerciseStampedVersion asserts the interpreted plugin reports // stampedCheckVersion in GET /admin/api/overview. // // This is the proof that stampReleaseVersion actually landed, and so the // proof that New's telemetry gate was PASSED rather than short-circuited: // the gate compares this same pluginVersion constant against // devPluginVersion, so a version of stampedCheckVersion here means the // interpreted code went on to call into the vendored oss-telemetry // package. Without this assertion a silently failed stamp would leave the // dev sentinel in place and the harness would still print OK, claiming // coverage it does not have. func exerciseStampedVersion(handler http.Handler) error { req := httptest.NewRequest(http.MethodGet, "/admin/api/overview", nil) req.Header.Set("Authorization", "Bearer "+attemptAccountingAdminAPIKey) rec := httptest.NewRecorder() handler.ServeHTTP(rec, req) if rec.Code != http.StatusOK { return fmt.Errorf("GET /admin/api/overview: status = %d, want 200, body=%s", rec.Code, rec.Body.String()) } var overview struct { Version string `json:"version"` } if err := json.Unmarshal(rec.Body.Bytes(), &overview); err != nil { return fmt.Errorf("decode GET /admin/api/overview body: %w", err) } if overview.Version != stampedCheckVersion { return fmt.Errorf("interpreted plugin reports version %q, want %q — stampReleaseVersion did not take effect, so New's telemetry gate short-circuited on the dev sentinel and the vendored oss-telemetry call was never exercised under Yaegi", overview.Version, stampedCheckVersion) } return nil } // postMessages drives one POST /v1/messages through handler with the given // auth header name/value and JSON body, returning the recorder. func postMessages(handler http.Handler, hdrName, hdrValue, body string) *httptest.ResponseRecorder { req := httptest.NewRequest(http.MethodPost, "/v1/messages", strings.NewReader(body)) req.Header.Set("Content-Type", "application/json") req.Header.Set(hdrName, hdrValue) rec := httptest.NewRecorder() handler.ServeHTTP(rec, req) return rec } // exerciseMessagesRoute drives the inbound Anthropic Messages API route // under the REAL interpreter: both auth header forms, the streaming // rejection, a bad key's error envelope, an unknown model's error // envelope, the openai-type TRANSLATED path (openAIRequestFromAnthropic // -> chatCompletion -> anthropicResponseFromOpenAI), the anthropic-type // PASSTHROUGH path including its own upstream-error branch, and a // tool_use round trip through the translated path. This exists because // go test never runs under Yaegi — a construct that compiles and passes // `go test` can still panic or silently misbehave interpreted (this // package's own doc comment has the general case; handleAdapterError's // doc comment, routes_unified.go, has this route's own specific history // with plain type assertions over errors.As). func exerciseMessagesRoute(handler http.Handler) error { // 1. x-api-key auth + translated path (openai-type provider). rec := postMessages(handler, "x-api-key", testDataUserAPIKey, `{"model":"`+testDataWantModel+`","max_tokens":64,"system":"be terse","messages":[{"role":"user","content":[{"type":"text","text":"ping"}]}]}`) if rec.Code != http.StatusOK { return fmt.Errorf("POST /v1/messages (x-api-key, translated): status = %d, want 200, body=%s", rec.Code, rec.Body.String()) } var msg map[string]any if err := json.Unmarshal(rec.Body.Bytes(), &msg); err != nil { return fmt.Errorf("POST /v1/messages: decode body: %w (body=%s)", err, rec.Body.String()) } if msg["type"] != "message" || msg["role"] != "assistant" { return fmt.Errorf("POST /v1/messages translated: type/role = %v/%v, want message/assistant: %s", msg["type"], msg["role"], rec.Body.String()) } if msg["model"] != testDataWantModel { return fmt.Errorf("POST /v1/messages translated: model = %v, want %q (the client's requested alias echoed back, item 4 fix): %s", msg["model"], testDataWantModel, rec.Body.String()) } if msg["stop_reason"] != "end_turn" { return fmt.Errorf("POST /v1/messages translated: stop_reason = %v, want end_turn: %s", msg["stop_reason"], rec.Body.String()) } content, _ := msg["content"].([]any) if len(content) != 1 { return fmt.Errorf("POST /v1/messages translated: content len = %d, want 1: %s", len(content), rec.Body.String()) } block, _ := content[0].(map[string]any) if block["type"] != "text" || block["text"] != "hi" { return fmt.Errorf("POST /v1/messages translated: content[0] = %v, want text/hi: %s", content[0], rec.Body.String()) } usg, _ := msg["usage"].(map[string]any) if usg == nil || usg["input_tokens"] != float64(1) || usg["output_tokens"] != float64(1) { return fmt.Errorf("POST /v1/messages translated: usage = %v, want input=1 output=1: %s", msg["usage"], rec.Body.String()) } // 2. Authorization: Bearer must authenticate the same route. bearerRec := postMessages(handler, "Authorization", "Bearer "+testDataUserAPIKey, `{"model":"`+testDataWantModel+`","max_tokens":64,"messages":[{"role":"user","content":"ping"}]}`) if bearerRec.Code != http.StatusOK { return fmt.Errorf("POST /v1/messages (Authorization: Bearer): status = %d, want 200, body=%s", bearerRec.Code, bearerRec.Body.String()) } // 3. A bad key gets a 401 in the ANTHROPIC error shape. badRec := postMessages(handler, "x-api-key", "not-a-real-key", `{"model":"`+testDataWantModel+`","max_tokens":64,"messages":[{"role":"user","content":"ping"}]}`) if badRec.Code != http.StatusUnauthorized { return fmt.Errorf("POST /v1/messages with a bad key: status = %d, want 401, body=%s", badRec.Code, badRec.Body.String()) } if err := assertAnthropicErrorShape("bad key 401", badRec.Body.Bytes()); err != nil { return err } // 4. "stream": true is rejected explicitly, in the Anthropic shape, // as a 400 (item 2 fix, 2026-08-22 review) — NOT 501: a 501 makes // anthropic-sdk-python (which retries any status >= 500) turn every // streaming call into a three-request retry storm against this // route's own rate limit. 400 maps to the SDK's non-retryable // BadRequestError. streamRec := postMessages(handler, "x-api-key", testDataUserAPIKey, `{"model":"`+testDataWantModel+`","max_tokens":64,"stream":true,"messages":[{"role":"user","content":"ping"}]}`) if streamRec.Code != http.StatusBadRequest { return fmt.Errorf("POST /v1/messages with stream=true: status = %d, want 400, body=%s", streamRec.Code, streamRec.Body.String()) } if err := assertAnthropicErrorShape("stream rejection", streamRec.Body.Bytes()); err != nil { return err } // 5. An unknown model's 404 must also be Anthropic-shaped. unknownRec := postMessages(handler, "x-api-key", testDataUserAPIKey, `{"model":"no-such-model-anywhere","max_tokens":64,"messages":[{"role":"user","content":"ping"}]}`) if unknownRec.Code != http.StatusNotFound { return fmt.Errorf("POST /v1/messages with an unknown model: status = %d, want 404, body=%s", unknownRec.Code, unknownRec.Body.String()) } if err := assertAnthropicErrorShape("unknown model 404", unknownRec.Body.Bytes()); err != nil { return err } // 6. The anthropic-type PASSTHROUGH branch, interpreted: every field // survives except "model", rewritten to the client's own requested // alias (item 4 fix) — a routing rewrite, not message-body // translation. passRec := postMessages(handler, "x-api-key", testDataUserAPIKey, `{"model":"claude-test","max_tokens":64,"system":"be terse","messages":[{"role":"user","content":"ping"}]}`) if passRec.Code != http.StatusOK { return fmt.Errorf("POST /v1/messages (anthropic passthrough): status = %d, want 200, body=%s", passRec.Code, passRec.Body.String()) } var pass map[string]any if err := json.Unmarshal(passRec.Body.Bytes(), &pass); err != nil { return fmt.Errorf("POST /v1/messages passthrough: decode body: %w (body=%s)", err, passRec.Body.String()) } if pass["id"] != "msg_probe" { return fmt.Errorf("POST /v1/messages passthrough: id = %v, want msg_probe (every field but \"model\" must survive the passthrough unchanged): %s", pass["id"], passRec.Body.String()) } if pass["model"] != "claude-test" { return fmt.Errorf("POST /v1/messages passthrough: model = %v, want \"claude-test\" (the client's requested alias echoed back, item 4 fix — not anthropicProbeUpstream's own \"claude-upstream-real\"): %s", pass["model"], passRec.Body.String()) } // 6b. handleAdapterErrorEnvelope's *providerHTTPError type assertion, // interpreted — the exact construct class that has broken under Yaegi // before (see handleAdapterError's own doc comment, routes_unified.go). upErrRec := postMessages(handler, "x-api-key", testDataUserAPIKey, `{"model":"claude-test","max_tokens":64,"messages":[{"role":"user","content":"make-it-fail"}]}`) if upErrRec.Code != http.StatusTooManyRequests { return fmt.Errorf("POST /v1/messages (upstream 429): status = %d, want 429, body=%s", upErrRec.Code, upErrRec.Body.String()) } if err := assertAnthropicErrorShape("upstream 429", upErrRec.Body.Bytes()); err != nil { return err } // A stale X-Llmgw-Cache header on an error response was deliberately // NOT asserted here (F6 fix, 2026-08-23 review): this harness's own // fixture configures no cache/Redis, so g.cache is nil and the // header is never set on any response regardless of what this route // does — the assertion could not fail no matter what code ran, // confirmed by a mutation pass that deleted the header-clear line it // was meant to guard (handleAdapterErrorEnvelope, routes_unified.go) // with make yaegi-check still reporting OK. That behavior is already // covered where it can actually fail — a real cache fixture, go // test: TestHandleChat_CacheableRequest_UpstreamDown_502HasNoCacheHeader // (routes_unified_test.go). // 7. A tool_use round trip through the TRANSLATED path, to drive // openAIToolCallFromAnthropic/openAIToolMessageFromAnthropic and // anthropicToolInputFromOpenAI interpreted. toolRec := postMessages(handler, "x-api-key", testDataUserAPIKey, `{"model":"`+testDataWantModel+`","max_tokens":64,`+ `"tools":[{"name":"get_weather","description":"w","input_schema":{"type":"object","properties":{"city":{"type":"string"}}}}],`+ `"tool_choice":{"type":"auto"},`+ `"messages":[{"role":"user","content":"weather?"},`+ `{"role":"assistant","content":[{"type":"tool_use","id":"toolu_1","name":"get_weather","input":{"city":"London"}}]},`+ `{"role":"user","content":[{"type":"tool_result","tool_use_id":"toolu_1","content":"sunny"}]}]}`) if toolRec.Code != http.StatusOK { return fmt.Errorf("POST /v1/messages (tool_use round trip): status = %d, want 200, body=%s", toolRec.Code, toolRec.Body.String()) } return nil } // assertAnthropicErrorShape checks b is {"type":"error","error":{"type":..., // "message":...}} — what an Anthropic SDK client parses. func assertAnthropicErrorShape(label string, b []byte) error { var env map[string]any if err := json.Unmarshal(b, &env); err != nil { return fmt.Errorf("/v1/messages %s: body is not JSON: %w (body=%s)", label, err, string(b)) } if env["type"] != "error" { return fmt.Errorf("/v1/messages %s: top-level \"type\" = %v, want \"error\" (Anthropic envelope): %s", label, env["type"], string(b)) } inner, _ := env["error"].(map[string]any) if inner == nil { return fmt.Errorf("/v1/messages %s: no \"error\" object: %s", label, string(b)) } if _, ok := inner["type"].(string); !ok { return fmt.Errorf("/v1/messages %s: error.type missing: %s", label, string(b)) } if _, ok := inner["message"].(string); !ok { return fmt.Errorf("/v1/messages %s: error.message missing: %s", label, string(b)) } return nil } // exerciseFederatedTooLarge drives POST /mcp (federation's tools/call // path) against mcpProbeUpstream, whose response deliberately exceeds // mcpBackendCallResponseMaxBytes, and asserts the interpreted plugin // answers with mcpTooLargeWantMessage rather than the generic "upstream // error". // // What this actually pins is errors.Is under the interpreter. A security // review round replaced that errors.Is with a strings.Contains marker on // the grounds that no gate exercised this path interpreted — true at the // time, and the honest objection. The reasoning for errors.Is is sound // (errors.New and fmt.Errorf both return COMPILED values, so declaring // the sentinel variable in interpreted code leaves the whole chain // errors.Is walks compiled, which is why retry.go's isTransient has // worked in production since v0.2.0), and a standalone yaegi v0.16.1 // harness confirmed it. But this codebase's own history — matchesSentinel // (limits.go) — is a case where sound reasoning about the reflect // boundary was wrong and only an interpreted probe caught it. So the // path is covered here rather than argued in a comment. // // A failure surfaces as "upstream error": errors.Is returning false // interpreted for a chain that is true compiled. No `go test` row can // see that difference. func exerciseFederatedTooLarge(handler http.Handler) error { body := `{"jsonrpc":"2.0","id":7,"method":"tools/call","params":{"name":"` + mcpProbeToolName + `","arguments":{}}}` req := httptest.NewRequest(http.MethodPost, "/mcp", strings.NewReader(body)) req.Header.Set("Authorization", "Bearer "+testDataUserAPIKey) req.Header.Set("Content-Type", "application/json") rec := httptest.NewRecorder() handler.ServeHTTP(rec, req) if rec.Code != http.StatusOK { return fmt.Errorf("POST /mcp tools/call: status = %d, want 200 (JSON-RPC reports the failure in the body, not the status), body=%s", rec.Code, rec.Body.String()) } var resp struct { Error *struct { Message string `json:"message"` } `json:"error"` } if err := json.Unmarshal(rec.Body.Bytes(), &resp); err != nil { return fmt.Errorf("decode POST /mcp tools/call body: %w (body=%s)", err, rec.Body.String()) } if resp.Error == nil { return fmt.Errorf("POST /mcp tools/call returned no JSON-RPC error, want %q — the probe upstream's oversized response should have tripped the cap: %s", mcpTooLargeWantMessage, rec.Body.String()) } if resp.Error.Message != mcpTooLargeWantMessage { return fmt.Errorf("POST /mcp tools/call error message = %q, want %q — errors.Is(err, errMCPResponseTooLarge) did not match under the interpreter (mcp_federation.go)", resp.Error.Message, mcpTooLargeWantMessage) } return nil } // targetHealthProbeView is a shape-tolerant decode of one adminTargetView // entry's own "health" object (admin.go's adminTargetHealthView) — // this harness module cannot import the plugin's unexported type across // the Yaegi reflection boundary (New is resolved and called by // reflection, not a normal Go import — this whole file's own doc // comment explains why), so it decodes the JSON shape independently, the // same approach readHealth/readProviderAttemptCounters already take for // other admin responses. type targetHealthProbeView struct { State string `json:"state"` Source string `json:"source"` } // pollTargetHealth drives GET /admin/api/targets against handler // repeatedly until name's health satisfies want, or a 5s budget elapses, // then returns its final reading either way — mirrors // pollProviderAttemptCounters's identical poll-with-deadline shape // (below), applied here to feat/target-health's own async sweep // (maybeSweepTargetHealth spawns the actual probing goroutine and // returns immediately, target_health.go's own doc comment) instead of // limiter.spawn's async counter write. func pollTargetHealth(handler http.Handler, name string, want func(targetHealthProbeView) bool) (targetHealthProbeView, error) { deadline := time.Now().Add(5 * time.Second) for { health, err := readTargetHealth(handler, name) if err != nil { return targetHealthProbeView{}, err } if want(health) || time.Now().After(deadline) { return health, nil } time.Sleep(10 * time.Millisecond) } } // readTargetHealth returns name's own "health" object from GET // /admin/api/targets, searching both mcpServers and agents (mcpProbeServerName // and targetHealthDownServerName are both MCP servers, but this checks // both arrays so a future agent-shaped call site can reuse it unchanged). func readTargetHealth(handler http.Handler, name string) (targetHealthProbeView, error) { req := httptest.NewRequest(http.MethodGet, "/admin/api/targets", nil) req.Header.Set("Authorization", "Bearer "+attemptAccountingAdminAPIKey) rec := httptest.NewRecorder() handler.ServeHTTP(rec, req) if rec.Code != http.StatusOK { return targetHealthProbeView{}, fmt.Errorf("GET /admin/api/targets: status = %d, want 200, body=%s", rec.Code, rec.Body.String()) } var body struct { MCPServers []struct { Name string `json:"name"` Health targetHealthProbeView `json:"health"` } `json:"mcpServers"` Agents []struct { Name string `json:"name"` Health targetHealthProbeView `json:"health"` } `json:"agents"` } if err := json.Unmarshal(rec.Body.Bytes(), &body); err != nil { return targetHealthProbeView{}, fmt.Errorf("decode GET /admin/api/targets body: %w (body=%s)", err, rec.Body.String()) } for _, s := range body.MCPServers { if s.Name == name { return s.Health, nil } } for _, a := range body.Agents { if a.Name == name { return a.Health, nil } } return targetHealthProbeView{}, fmt.Errorf("target %q not in GET /admin/api/targets: %s", name, rec.Body.String()) } // exerciseTargetHealthProbeWarmup proves feat/target-health's ACTIVE // PROBE sweep end to end under the REAL interpreter, and — just as // important — makes every LATER target-health assertion in this harness // deterministic. Called from run() BEFORE exerciseHandler, i.e. before // any other route this harness hits: GET /admin/api/targets is the // first-ever sweep-capable route (maybeSweepTargetHealth's own doc // comment, target_health.go, lists the four: serveMetrics, // serveAdminTargets, handleMCPServers, handleAgents) this handler sees, // so it deterministically owns the ONE sweep this gateway's // probeInterval (configured to "60s", well above this harness's // real-world runtime) allows for the rest of the run. // // This matters because targetHealthProbeOnlyServerName's own record() is // not the only one a sweep touches — EVERY configured MCP server and // agent gets probed in the same sweep (sweepTargetHealth's own doc // comment), mcpProbeServerName included. Spending that one sweep here, // before exerciseFederatedTooLarge ever drives mcpProbeServerName's real // traffic observation, guarantees the traffic observation is the LAST // write to that entry for the remainder of the run — no later // admin/metrics/listing route can trigger a second sweep to race it // (F5, feat/target-health review: this is what closes the race that // used to make TARGETHEALTH-2, below, racy under Yaegi's interpreter // overhead). // // G5, feat/target-health review round 2: that guarantee only holds once // EVERY configured MCP server's own probe observation has actually // landed, not merely targetHealthProbeOnlyServerName's. sweepTargetHealth // probes every configured server CONCURRENTLY, on its own goroutines — // waiting for one target's record() to land says nothing about whether // mcpProbeServerName's own probe goroutine has finished yet. A version of // this function that returned as soon as targetHealthProbeOnlyServerName // alone changed could return while mcpProbeServerName's probe write was // still in flight; that write could then land AFTER // exerciseFederatedTooLarge's later traffic write, silently reverting // mcpProbeServerName's source back to "probe" and corrupting // exerciseTargetHealth's own TARGETHEALTH-2 assertion (source == // "traffic"). Polling every configured MCP server for its own source == // "probe" closes that gap: this function cannot return until the whole // sweep — not just the one target it happens to check last — has // finished writing. // // targetHealthProbeOnlyServerName is additionally asserted to the FULL // "healthy" state, not just source == "probe": it is NEVER proxied or // federated through by any other exercise in this harness, so its health // entry can only ever be written by the probe — recover's own swallow // (sweepTargetHealth's defer, target_health.go) means an interpreted // panic mid-probe would otherwise degrade to a silently-stuck "unknown" // rather than a loud failure, and asserting the full success shape here // is what makes a Yaegi incompatibility in the initialize/close handshake // path visible instead of passing silently. func exerciseTargetHealthProbeWarmup(handler http.Handler) error { warmupServerNames := []string{mcpProbeServerName, targetHealthDownServerName, targetHealthProbeOnlyServerName} var probeOnlyHealth targetHealthProbeView for _, name := range warmupServerNames { health, err := pollTargetHealth(handler, name, func(h targetHealthProbeView) bool { return h.Source == "probe" }) if err != nil { return err } if health.Source != "probe" { return fmt.Errorf("TARGETHEALTH-PROBE-1: GET /admin/api/targets: %q health.source = %q, want %q (the warmup sweep must probe every configured MCP server before any later exercise runs)", name, health.Source, "probe") } if name == targetHealthProbeOnlyServerName { probeOnlyHealth = health } } if probeOnlyHealth.State != "healthy" { return fmt.Errorf("TARGETHEALTH-PROBE-2: GET /admin/api/targets: %q health.state = %q, want healthy (targetHealthProbeOnlyUpstream always answers initialize successfully)", targetHealthProbeOnlyServerName, probeOnlyHealth.State) } fmt.Println("yaegi-check: target health active probe sweep ran and recorded correctly interpreted") return nil } // exerciseTargetHealth proves feat/target-health's PASSIVE recording end // to end under the REAL interpreter, now that // exerciseTargetHealthProbeWarmup (run() calls it first, above) has // already spent this gateway's one probeInterval-bounded sweep — no // further sweep can fire during this function, so neither part below // races the active probe path. // // Part (a): mcpProbeServerName receives one real federated tools/call // attempt via exerciseFederatedTooLarge (exerciseHandler, called before // this function) — its oversized response trips errMCPResponseTooLarge, // a transport-level failure from doBackendJSONRPC's own perspective // (mcp_federation.go), which mcpFederatedToolsCall records as a // target-health failure. This is the ONLY observation this target ever // receives (the warmup above never touches it — see // exerciseTargetHealthProbeWarmup's own doc comment for why spending the // sweep before this call matters), so it has never once succeeded: // F4 (feat/target-health review) reads that as "unknown", not "healthy" // — the failure-threshold gate still protects against this single // transient-looking failure, so it does not read "unhealthy" either. // // Part (b): drives targetHealthFailureThreshold consecutive failures // through the direct per-server proxy path (/mcp/{name}/..., // handleTargetProxy) against targetHealthDownServerName's dead upstream, // then asserts the SAME tracker flips that target to "unhealthy" on // both GET /admin/api/targets and llmgateway_target_healthy on // /metrics. func exerciseTargetHealth(handler http.Handler) error { probeHealth, err := readTargetHealth(handler, mcpProbeServerName) if err != nil { return err } if probeHealth.State != "unknown" { return fmt.Errorf("TARGETHEALTH-1: GET /admin/api/targets: %q health.state = %q, want unknown (one failed federated tools/call attempt, never having succeeded, below threshold — F4): %+v", mcpProbeServerName, probeHealth.State, probeHealth) } if probeHealth.Source != "traffic" { return fmt.Errorf("TARGETHEALTH-2: GET /admin/api/targets: %q health.source = %q, want %q", mcpProbeServerName, probeHealth.Source, "traffic") } for i := 0; i < targetHealthFailureThreshold; i++ { req := httptest.NewRequest(http.MethodGet, "/mcp/"+targetHealthDownServerName+"/tools/list", nil) req.Header.Set("Authorization", "Bearer "+testDataUserAPIKey) rec := httptest.NewRecorder() handler.ServeHTTP(rec, req) if rec.Code != http.StatusBadGateway { return fmt.Errorf("TARGETHEALTH-3: GET /mcp/%s/tools/list attempt %d: status = %d, want 502 (dead upstream), body=%s", targetHealthDownServerName, i, rec.Code, rec.Body.String()) } } downHealth, err := readTargetHealth(handler, targetHealthDownServerName) if err != nil { return err } if downHealth.State != "unhealthy" { return fmt.Errorf("TARGETHEALTH-4: GET /admin/api/targets: %q health.state = %q, want unhealthy after %d consecutive failures: %+v", targetHealthDownServerName, downHealth.State, targetHealthFailureThreshold, downHealth) } metricsReq := httptest.NewRequest(http.MethodGet, "/metrics", nil) metricsReq.Header.Set("Authorization", "Bearer "+attemptAccountingAdminAPIKey) metricsRec := httptest.NewRecorder() handler.ServeHTTP(metricsRec, metricsReq) if metricsRec.Code != http.StatusOK { return fmt.Errorf("GET /metrics: status = %d, want 200, body=%s", metricsRec.Code, metricsRec.Body.String()) } wantSample := `llmgateway_target_healthy{kind="mcp",target="` + targetHealthDownServerName + `"} 0` if !strings.Contains(metricsRec.Body.String(), wantSample) { return fmt.Errorf("TARGETHEALTH-5: GET /metrics missing %q — interpreted target-health metric diverged from the compiled shape; body=%s", wantSample, metricsRec.Body.String()) } fmt.Println("yaegi-check: target health tracked and exposed correctly interpreted (admin API + /metrics)") return nil } // exerciseFederatedSSENegotiation drives two real POST /mcp "ping" calls // against the interpreted handler — one carrying no Accept header // (today's plain JSON framing) and one carrying the reported client's // exact "Accept: application/json, text/event-stream" header — proving // content negotiation (mcpNegotiateFormat, mcpAcceptsSSE) and the SSE // branch's reuse of sse.go's newSSEWriter/writeData both run correctly // under the REAL interpreter. mcp_federation_sse_test.go already covers // this compiled; this harness exists because a compiled pass on this // codebase has repeatedly said nothing about the interpreted shape. // // "ping" is deliberately chosen over tools/list or tools/call: it is // answered entirely locally (handleMCPFederated's own doc comment), so // this probe needs no additional upstream server and cannot be confused // with mcpProbeUpstream's own, deliberately oversized, tools/call-only // response above. // // This is also the first time under this harness that sse.go's // newSSEWriter/writeData actually executes interpreted at all. The // streaming (chat completions) path is documented as losing its // incremental Flush under Yaegi (newSSEWriter's own doc comment, // confirmed by Task 15's integration suite) — harmless here, since this // is a single-shot envelope that writes once and returns rather than a // response that depends on Flush for time-to-first-byte, but it means no // existing gate has ever run this writer interpreted before now. func exerciseFederatedSSENegotiation(handler http.Handler) error { pingBody := `{"jsonrpc":"2.0","id":1,"method":"ping"}` jsonReq := httptest.NewRequest(http.MethodPost, "/mcp", strings.NewReader(pingBody)) jsonReq.Header.Set("Authorization", "Bearer "+testDataUserAPIKey) jsonReq.Header.Set("Content-Type", "application/json") jsonRec := httptest.NewRecorder() handler.ServeHTTP(jsonRec, jsonReq) if jsonRec.Code != http.StatusOK { return fmt.Errorf("POST /mcp ping (no Accept header): status = %d, want 200, body=%s", jsonRec.Code, jsonRec.Body.String()) } if ct := jsonRec.Header().Get("Content-Type"); ct != "application/json" { return fmt.Errorf("POST /mcp ping (no Accept header): Content-Type = %q, want %q — default framing must not change interpreted", ct, "application/json") } sseReq := httptest.NewRequest(http.MethodPost, "/mcp", strings.NewReader(pingBody)) sseReq.Header.Set("Authorization", "Bearer "+testDataUserAPIKey) sseReq.Header.Set("Content-Type", "application/json") sseReq.Header.Set("Accept", "application/json, text/event-stream") sseRec := httptest.NewRecorder() handler.ServeHTTP(sseRec, sseReq) if sseRec.Code != http.StatusOK { return fmt.Errorf("POST /mcp ping (Accept: text/event-stream): status = %d, want 200, body=%s", sseRec.Code, sseRec.Body.String()) } if ct := sseRec.Header().Get("Content-Type"); ct != "text/event-stream" { return fmt.Errorf("POST /mcp ping (Accept: text/event-stream): Content-Type = %q, want %q — SSE negotiation did not run interpreted", ct, "text/event-stream") } body := sseRec.Body.String() if !strings.HasPrefix(body, "data: ") { return fmt.Errorf("POST /mcp ping (Accept: text/event-stream): body does not start with \"data: \": %q", body) } if !strings.HasSuffix(body, "\n\n") { return fmt.Errorf("POST /mcp ping (Accept: text/event-stream): body does not end with the SSE blank-line terminator — the exact missing-terminator defect class this negotiation was added to fix: %q", body) } dataLine := strings.TrimSuffix(strings.TrimPrefix(body, "data: "), "\n\n") var resp struct { Result map[string]any `json:"result"` ID json.RawMessage `json:"id"` } if err := json.Unmarshal([]byte(dataLine), &resp); err != nil { return fmt.Errorf("POST /mcp ping (Accept: text/event-stream): data line is not valid JSON-RPC: %w (line=%q)", err, dataLine) } if string(resp.ID) != "1" { return fmt.Errorf("POST /mcp ping (Accept: text/event-stream): id = %s, want the client's own id 1 echoed back", resp.ID) } fmt.Println("yaegi-check: federated MCP content negotiation served both JSON and SSE framing correctly, interpreted") return nil } // exerciseFailover drives one real POST /v1/chat/completions against the // interpreted handler for failoverModelID — a bare model id both // "failover-a" (always 500s) and "failover-b" (always 200s) are // configured with — proving the failover candidate loop (runMeteredCall, // routes_unified.go/failover.go) falls through a genuine provider // failure and serves the second provider's response under the REAL // interpreter. Compiled tests already prove this (failover_test.go); this // harness exists because compiled tests have repeatedly passed on this // codebase while the interpreted shape failed (this package's own doc // comment). func exerciseFailover(handler http.Handler, bHits *int64) error { req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader( `{"model":"`+failoverModelID+`","messages":[{"role":"user","content":"hi"}]}`, )) req.Header.Set("Authorization", "Bearer "+testDataUserAPIKey) req.Header.Set("Content-Type", "application/json") rec := httptest.NewRecorder() handler.ServeHTTP(rec, req) if rec.Code != http.StatusOK { return fmt.Errorf("POST /v1/chat/completions (feat/failover harness): status = %d, want 200 (failover-a's 500 must fall through to failover-b), body=%s", rec.Code, rec.Body.String()) } if !strings.Contains(rec.Body.String(), "served by failover-b") { return fmt.Errorf("POST /v1/chat/completions (feat/failover harness): body does not contain failover-b's own content, want the SECOND provider's response: %s", rec.Body.String()) } if atomic.LoadInt64(bHits) != 1 { return fmt.Errorf("feat/failover harness: failover-b upstream hit count = %d, want exactly 1", atomic.LoadInt64(bHits)) } fmt.Println("yaegi-check: failover fell through from failover-a to failover-b and served its response") return nil } // exerciseBareWinnerGroupAware proves registry.go's group-aware bareWinner // fix (2026-08-27) under the REAL interpreter: "bare-winner-a" and // "bare-winner-b" both serve bareWinnerModelID; bareWinnerGroupName may // use "bare-winner-b" only, yet "bare-winner-a" sorts first // alphabetically. Before the fix, resolve's bareWinner call always // returned the GLOBAL sorted-first owner regardless of the caller's // group, so this exact request got errModelDenied for a model the group // was actually entitled to use — this is the production repro the fix // exists for (bug report: "friends" group entitled to "uni", denied a // bare id because "macstudio" sorts first and also serves it). // // Unlike exerciseFailover (immediately above, same colliding-bare-id // shape): a correct group-aware bareWinner never even CONSIDERS // "bare-winner-a" as a candidate for this group's request — it is not a // failover fall-through after a failed attempt. aHits staying at 0 proves // that: a pre-fix, group-blind bareWinner would route here first, and // "bare-winner-a"'s upstream (run(), above) answers 500 with a body // naming the exact wrong outcome, so a regression is unambiguous either // way (aHits != 0, or a non-200 status, or the wrong response body). // // MUTATION VERIFIED: temporarily reverting bareWinner (registry.go) to // its pre-fix, group-blind body (ignore grp entirely, always return the // first provider in m.providerNames whose known model set contains id) // made this fail with "status = 403" (errModelDenied) — bareWinnerGroupName // cannot use "bare-winner-a", the pre-fix global winner. Reverted before // committing. func exerciseBareWinnerGroupAware(handler http.Handler, aHits *int64) error { req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader( `{"model":"`+bareWinnerModelID+`","messages":[{"role":"user","content":"hi"}]}`, )) req.Header.Set("Authorization", "Bearer "+bareWinnerFriendAPIKey) req.Header.Set("Content-Type", "application/json") rec := httptest.NewRecorder() handler.ServeHTTP(rec, req) if rec.Code != http.StatusOK { return fmt.Errorf("POST /v1/chat/completions (fix/group-aware-bare-winner harness): status = %d, want 200 (%s is entitled to \"bare-winner-b\", which serves the bare id), body=%s", rec.Code, bareWinnerGroupName, rec.Body.String()) } if !strings.Contains(rec.Body.String(), "served by bare-winner-b") { return fmt.Errorf("POST /v1/chat/completions (fix/group-aware-bare-winner harness): body does not contain bare-winner-b's own content, want the group-visible provider's response: %s", rec.Body.String()) } if got := atomic.LoadInt64(aHits); got != 0 { return fmt.Errorf("fix/group-aware-bare-winner harness: bare-winner-a upstream hit count = %d, want exactly 0 (%s can never reach bare-winner-a)", got, bareWinnerGroupName) } // GET /v1/models, same group: listFor's own half of the fix // (registry.go) — the bare id must be attributed to "bare-winner-b" // (the group's own visible winner), and "bare-winner-a" must never // appear anywhere in this group's catalog, bare or prefixed. listReq := httptest.NewRequest(http.MethodGet, "/v1/models", nil) listReq.Header.Set("Authorization", "Bearer "+bareWinnerFriendAPIKey) listRec := httptest.NewRecorder() handler.ServeHTTP(listRec, listReq) if listRec.Code != http.StatusOK { return fmt.Errorf("GET /v1/models (fix/group-aware-bare-winner harness): status = %d, want 200, body=%s", listRec.Code, listRec.Body.String()) } if strings.Contains(listRec.Body.String(), "bare-winner-a") { return fmt.Errorf("GET /v1/models (fix/group-aware-bare-winner harness): body names bare-winner-a, want it invisible to %s entirely: %s", bareWinnerGroupName, listRec.Body.String()) } var listBody struct { Data []map[string]any `json:"data"` } if err := json.Unmarshal(listRec.Body.Bytes(), &listBody); err != nil { return fmt.Errorf("decode GET /v1/models body (fix/group-aware-bare-winner harness): %w (body=%s)", err, listRec.Body.String()) } found := false for _, entry := range listBody.Data { if entry["id"] != bareWinnerModelID { continue } found = true if entry["owned_by"] != "bare-winner-b" { return fmt.Errorf("GET /v1/models entry %v: owned_by = %v, want %q (this group's own visible winner)", entry, entry["owned_by"], "bare-winner-b") } } if !found { return fmt.Errorf("GET /v1/models body has no bare entry for %q: %s", bareWinnerModelID, listRec.Body.String()) } fmt.Println("yaegi-check: group-aware bareWinner resolved and listed bare-winner-shared via bare-winner-b, never touching bare-winner-a") return nil } // exerciseMultiGroupPersonalGrant proves the multi-group/personal-grant // feature (UserConfig.Groups/Providers/Models, llmgateway.go) resolves // correctly under the REAL interpreter: multiGroupUserAPIKey's own // principal belongs to TWO groups (multiGroupEngGroupName, // multiGroupOpsGroupName — neither reaching "multi-group-b" at all) // plus a personal grant restricting "multi-group-b" to // multiGroupAllowedModelID alone. A request for the personal grant's own // allowed model must succeed; a request for a DIFFERENT model on the // SAME provider must be denied — the no-cross-grant-leak rule // (group.allowsProviderModel, auth.go) a naive union-based check would // get wrong. func exerciseMultiGroupPersonalGrant(handler http.Handler, bHits *int64) error { allowedBody := `{"model":"` + multiGroupAllowedModelID + `","messages":[{"role":"user","content":"hi"}]}` allowedReq := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(allowedBody)) allowedReq.Header.Set("Authorization", "Bearer "+multiGroupUserAPIKey) allowedReq.Header.Set("Content-Type", "application/json") allowedRec := httptest.NewRecorder() handler.ServeHTTP(allowedRec, allowedReq) if allowedRec.Code != http.StatusOK { return fmt.Errorf("POST /v1/chat/completions (multi-group/personal-grant harness, allowed model): status = %d, want 200, body=%s", allowedRec.Code, allowedRec.Body.String()) } if !strings.Contains(allowedRec.Body.String(), "served by multi-group-b") { return fmt.Errorf("POST /v1/chat/completions (multi-group/personal-grant harness, allowed model): body does not contain multi-group-b's own content: %s", allowedRec.Body.String()) } if got := atomic.LoadInt64(bHits); got != 1 { return fmt.Errorf("multi-group/personal-grant harness: multi-group-b upstream hit count = %d, want exactly 1 after the allowed request", got) } deniedBody := `{"model":"` + multiGroupDeniedModelID + `","messages":[{"role":"user","content":"hi"}]}` deniedReq := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(deniedBody)) deniedReq.Header.Set("Authorization", "Bearer "+multiGroupUserAPIKey) deniedReq.Header.Set("Content-Type", "application/json") deniedRec := httptest.NewRecorder() handler.ServeHTTP(deniedRec, deniedReq) if deniedRec.Code != http.StatusForbidden { return fmt.Errorf("POST /v1/chat/completions (multi-group/personal-grant harness, cross-grant model): status = %d, want 403 (the personal grant's own Models list denies %q; neither member group's grant may leak into authorizing it), body=%s", deniedRec.Code, multiGroupDeniedModelID, deniedRec.Body.String()) } if got := atomic.LoadInt64(bHits); got != 1 { return fmt.Errorf("multi-group/personal-grant harness: multi-group-b upstream hit count = %d, want still exactly 1 (the denied request must never reach the upstream)", got) } fmt.Println("yaegi-check: multi-group user with a personal grant resolved the personal-grant provider's allowed model and was denied a cross-grant model") return nil } // exerciseMultiGroupBareWinnerFailoverAndPassthroughPaths extends the // multi-group/personal-grant probe above (LOW-8, review round 2) to // additionally assert, under the REAL interpreter: // // - HIGH-2: bareWinner/listFor prefer a grant that actually authorizes // a bare-id collision, not merely the sorted-first PROVIDER-visible // owner — resolving multiGroupSharedModelID (a bare, three-provider // collision: "multi-group-b" < "multi-group-c" < // "multi-group-shared-x") must land on "multi-group-c" // (multiGroupOpsGroupName's own unrestricted grant), never get // denied via "multi-group-b" (the personal grant's own, narrower // one, which sorts first but does not cover this id). // - The SAME multi-grant principal's failover pool correctly includes // "multi-group-shared-x" (multiGroupEngGroupName's own grant) and // correctly excludes "multi-group-b" — multiGroupCUpstream always // 500s its chat-completions call, forcing the fall-through past the // correctly-chosen primary. // - GET /v1/models shows exactly the union this principal is entitled // to for multiGroupSharedModelID: the bare id (owned by // "multi-group-c"), "multi-group-c/"+id, and // "multi-group-shared-x/"+id — never "multi-group-b/"+id. // - HIGH-1: a native passthrough request to "multi-group-c" is denied // on multiGroupDeniedPathID and allowed on multiGroupAllowedPathID // (multiGroupOpsGroupName's own "passthroughPaths"), while // "multi-group-shared-x" (multiGroupEngGroupName's own, unrestricted // provider) stays reachable on an arbitrary path either way — // proving passthroughPaths stays coupled to the SAME grant that // authorizes the provider, never unioned across this principal's // member groups. func exerciseMultiGroupBareWinnerFailoverAndPassthroughPaths(handler http.Handler, bHits, cChatHits, cPassthroughHits, sharedXHits *int64) error { bHitsBefore := atomic.LoadInt64(bHits) // exerciseMultiGroupPersonalGrant already drove one hit // --- HIGH-2 + failover --- body := `{"model":"` + multiGroupSharedModelID + `","messages":[{"role":"user","content":"hi"}]}` req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(body)) req.Header.Set("Authorization", "Bearer "+multiGroupUserAPIKey) req.Header.Set("Content-Type", "application/json") rec := httptest.NewRecorder() handler.ServeHTTP(rec, req) if rec.Code != http.StatusOK { return fmt.Errorf("POST /v1/chat/completions (multi-group bareWinner/failover harness): status = %d, want 200 (bareWinner must land on \"multi-group-c\" via multiGroupOpsGroupName's own grant, then fail over to \"multi-group-shared-x\" once multi-group-c's own upstream 500s), body=%s", rec.Code, rec.Body.String()) } if !strings.Contains(rec.Body.String(), "served by multi-group-shared-x") { return fmt.Errorf("POST /v1/chat/completions (multi-group bareWinner/failover harness): body does not contain multi-group-shared-x's own content, want the failover candidate's response: %s", rec.Body.String()) } if got := atomic.LoadInt64(cChatHits); got != 1 { return fmt.Errorf("multi-group bareWinner/failover harness: multi-group-c chat-completions hit count = %d, want exactly 1 (bareWinner must choose multi-group-c as primary — a pre-fix, union-based bareWinner would pick \"multi-group-b\" first and never reach multi-group-c at all)", got) } if got := atomic.LoadInt64(sharedXHits); got != 1 { return fmt.Errorf("multi-group bareWinner/failover harness: multi-group-shared-x hit count = %d, want exactly 1 (the correctly-included failover candidate)", got) } if got := atomic.LoadInt64(bHits) - bHitsBefore; got != 0 { return fmt.Errorf("multi-group bareWinner/failover harness: multi-group-b hit count delta = %d, want exactly 0 (the personal grant's own Models list does not cover %q; multi-group-b must never be offered as a failover candidate for it)", got, multiGroupSharedModelID) } // --- GET /v1/models: the union this principal is entitled to --- listReq := httptest.NewRequest(http.MethodGet, "/v1/models", nil) listReq.Header.Set("Authorization", "Bearer "+multiGroupUserAPIKey) listRec := httptest.NewRecorder() handler.ServeHTTP(listRec, listReq) if listRec.Code != http.StatusOK { return fmt.Errorf("GET /v1/models (multi-group bareWinner harness): status = %d, want 200, body=%s", listRec.Code, listRec.Body.String()) } var listBody struct { Data []map[string]any `json:"data"` } if err := json.Unmarshal(listRec.Body.Bytes(), &listBody); err != nil { return fmt.Errorf("decode GET /v1/models body (multi-group bareWinner harness): %w (body=%s)", err, listRec.Body.String()) } ownedBy := map[string]string{} for _, entry := range listBody.Data { id, idOK := entry["id"].(string) owner, ownerOK := entry["owned_by"].(string) if idOK && ownerOK { ownedBy[id] = owner } } if ownedBy[multiGroupSharedModelID] != "multi-group-c" { return fmt.Errorf("GET /v1/models (multi-group bareWinner harness): bare %q owned_by = %q, want \"multi-group-c\"", multiGroupSharedModelID, ownedBy[multiGroupSharedModelID]) } if _, ok := ownedBy["multi-group-c/"+multiGroupSharedModelID]; !ok { return fmt.Errorf("GET /v1/models (multi-group bareWinner harness): missing \"multi-group-c/%s\"", multiGroupSharedModelID) } if _, ok := ownedBy["multi-group-shared-x/"+multiGroupSharedModelID]; !ok { return fmt.Errorf("GET /v1/models (multi-group bareWinner harness): missing \"multi-group-shared-x/%s\"", multiGroupSharedModelID) } if _, ok := ownedBy["multi-group-b/"+multiGroupSharedModelID]; ok { return fmt.Errorf("GET /v1/models (multi-group bareWinner harness): \"multi-group-b/%s\" must be absent (the personal grant's own Models list does not cover it)", multiGroupSharedModelID) } // --- HIGH-1: passthroughPaths coupled to the provider --- deniedPathReq := httptest.NewRequest(http.MethodPost, "/multi-group-c/"+multiGroupDeniedPathID, strings.NewReader(`{}`)) deniedPathReq.Header.Set("Authorization", "Bearer "+multiGroupUserAPIKey) deniedPathReq.Header.Set("Content-Type", "application/json") deniedPathRec := httptest.NewRecorder() handler.ServeHTTP(deniedPathRec, deniedPathReq) if deniedPathRec.Code != http.StatusForbidden { return fmt.Errorf("POST /multi-group-c/%s (multi-group passthroughPaths harness): status = %d, want 403 (multiGroupOpsGroupName's own path restriction on its OWN provider must survive, never stripped by multiGroupEngGroupName's own unrestricted paths on a DIFFERENT provider), body=%s", multiGroupDeniedPathID, deniedPathRec.Code, deniedPathRec.Body.String()) } if got := atomic.LoadInt64(cPassthroughHits); got != 0 { return fmt.Errorf("multi-group passthroughPaths harness: multi-group-c passthrough hit count = %d, want exactly 0 (the denied-path request must never reach the upstream)", got) } allowedPathReq := httptest.NewRequest(http.MethodPost, "/multi-group-c/"+multiGroupAllowedPathID, strings.NewReader(`{}`)) allowedPathReq.Header.Set("Authorization", "Bearer "+multiGroupUserAPIKey) allowedPathReq.Header.Set("Content-Type", "application/json") allowedPathRec := httptest.NewRecorder() handler.ServeHTTP(allowedPathRec, allowedPathReq) if allowedPathRec.Code != http.StatusOK { return fmt.Errorf("POST /multi-group-c/%s (multi-group passthroughPaths harness): status = %d, want 200 (multiGroupOpsGroupName's own allowed path), body=%s", multiGroupAllowedPathID, allowedPathRec.Code, allowedPathRec.Body.String()) } if got := atomic.LoadInt64(cPassthroughHits); got != 1 { return fmt.Errorf("multi-group passthroughPaths harness: multi-group-c passthrough hit count = %d, want exactly 1 after the allowed-path request", got) } sharedXPathReq := httptest.NewRequest(http.MethodPost, "/multi-group-shared-x/"+multiGroupDeniedPathID, strings.NewReader(`{}`)) sharedXPathReq.Header.Set("Authorization", "Bearer "+multiGroupUserAPIKey) sharedXPathReq.Header.Set("Content-Type", "application/json") sharedXPathRec := httptest.NewRecorder() handler.ServeHTTP(sharedXPathRec, sharedXPathReq) if sharedXPathRec.Code != http.StatusOK { return fmt.Errorf("POST /multi-group-shared-x/%s (multi-group passthroughPaths harness): status = %d, want 200 (multiGroupEngGroupName's own provider carries no path restriction at all, regardless of multiGroupOpsGroupName's own restriction on a DIFFERENT provider), body=%s", multiGroupDeniedPathID, sharedXPathRec.Code, sharedXPathRec.Body.String()) } fmt.Println("yaegi-check: multi-group principal's bareWinner/listFor/failover chose the grant-authorized owner, GET /v1/models showed exactly the union, and passthroughPaths stayed coupled to its own provider") return nil } // exerciseRound3ModelAndPathFromSameGrant proves the HIGH fix (review // round 3) under the REAL interpreter: model and path authorization for // native passthrough must come from ONE grant, never two different ones. // // - P1: round3P1AGroupName restricts models (round3ModelID) but not // paths; round3P1BGroupName restricts paths ("v1/chat/completions") // but not models — both cover "round3-shared". round3-p1-user // (member of both, no personal grant) POSTs a model // round3P1AGroupName denies, on a path only round3P1BGroupName would // allow: round3P1AGroupName's own unrestricted path must never // combine with round3P1BGroupName's own unrestricted model to // authorize it. // - P2: round3P2HomeGroupName (primary) carries no path restriction; // round3P2FriendsGroupName restricts paths on "round3-shared" but not // models; round3-p2-user's own personal grant restricts // "round3-shared" to round3PersonalModelID alone. The personal grant // borrows home's own unrestricted paths, but its OWN model // restriction must still apply — friends' own unrestricted models // must never leak into it via a shared provider. // // Both must deny (403) and never reach round3PassthroughUpstream. func exerciseRound3ModelAndPathFromSameGrant(handler http.Handler, hits *int64) error { hitsBefore := atomic.LoadInt64(hits) p1Body := `{"model":"round3-m2"}` p1Req := httptest.NewRequest(http.MethodPost, "/round3-shared/v1/files", strings.NewReader(p1Body)) p1Req.Header.Set("Authorization", "Bearer "+round3P1UserAPIKey) p1Req.Header.Set("Content-Type", "application/json") p1Rec := httptest.NewRecorder() handler.ServeHTTP(p1Rec, p1Req) if p1Rec.Code != http.StatusForbidden { return fmt.Errorf("P1 POST /round3-shared/v1/files (round-3 model+path-from-same-grant harness): status = %d, want 403 — round3P1AGroupName's own unrestricted path must never combine with round3P1BGroupName's own unrestricted model to authorize %q, body=%s", p1Rec.Code, round3ModelID, p1Rec.Body.String()) } p2Body := `{"model":"round3-shared/other"}` p2Req := httptest.NewRequest(http.MethodPost, "/round3-shared/v1/files", strings.NewReader(p2Body)) p2Req.Header.Set("Authorization", "Bearer "+round3P2UserAPIKey) p2Req.Header.Set("Content-Type", "application/json") p2Rec := httptest.NewRecorder() handler.ServeHTTP(p2Rec, p2Req) if p2Rec.Code != http.StatusForbidden { return fmt.Errorf("P2 POST /round3-shared/v1/files (round-3 model+path-from-same-grant harness): status = %d, want 403 — home's own unrestricted paths and friends' own unrestricted models must not combine to authorize a model the personal grant itself denies, body=%s", p2Rec.Code, p2Rec.Body.String()) } if got := atomic.LoadInt64(hits) - hitsBefore; got != 0 { return fmt.Errorf("round-3 model+path-from-same-grant harness: round3-shared upstream hit count delta = %d, want exactly 0 (neither P1's nor P2's denied request may ever reach the upstream)", got) } // P2+ (positive control, review round 4): the personal grant's own // ALLOWED model (round3PersonalModelID), on a path the PRIMARY group // (round3P2HomeGroupName) allows — home carries no path restriction // at all, so ANY path is allowed via the "personal grant borrows the // primary group's own paths" rule (auth.go's personalGrantAllowsPath). // Proves the round-3 coupling fix denies the WRONG combination (P2 // above, a different model on the same provider+path) without also // wrongly denying the RIGHT one — the body IS peeked (the personal // grant restricts models) and the model IS matched. p2PositiveBody := `{"model":"` + round3PersonalModelID + `"}` p2PositiveReq := httptest.NewRequest(http.MethodPost, "/round3-shared/v1/files", strings.NewReader(p2PositiveBody)) p2PositiveReq.Header.Set("Authorization", "Bearer "+round3P2UserAPIKey) p2PositiveReq.Header.Set("Content-Type", "application/json") p2PositiveRec := httptest.NewRecorder() handler.ServeHTTP(p2PositiveRec, p2PositiveReq) if p2PositiveRec.Code != http.StatusOK { return fmt.Errorf("P2+ POST /round3-shared/v1/files (round-3 model+path-from-same-grant harness): status = %d, want 200 — the personal grant's own model (%q) on a path the PRIMARY group (home) allows must succeed, body=%s", p2PositiveRec.Code, round3PersonalModelID, p2PositiveRec.Body.String()) } if got := atomic.LoadInt64(hits) - hitsBefore; got != 1 { return fmt.Errorf("round-3 model+path-from-same-grant harness: round3-shared upstream hit count delta after the P2+ positive case = %d, want exactly 1", got) } fmt.Println("yaegi-check: native passthrough denied a model and a path that could only be authorized by combining two DIFFERENT grants (P1: two member groups; P2: primary group's paths + a personal grant's own model restriction), and allowed the personal grant's own model on the primary group's own path (P2+)") return nil } // exerciseRequestTimeout proves the request-timeout feature aborts a // hung provider under the REAL interpreter, not merely in compiled // tests: timeoutProviderName's own ProviderConfig.RequestTimeout ("80ms") // is far shorter than timeoutUpstream's deliberate 600ms silence, and the // request below carries NO client-side deadline of its own — unlike // exerciseAttemptAccounting's slowProviderName probe (SHOULD-A), which // proves a CLIENT-set context deadline classifies correctly, this proves // the GATEWAY'S OWN adapter-level timeout aborts a hang no caller ever // bounded, exactly the shape of a real, unbounded production client. // A pass here means newAdapterHTTPClient's Transport. // ResponseHeaderTimeout (providers.go) and watchdogBody's construction // (timeout.go — context.WithCancel, time.AfterFunc, sync/atomic's // function API) all compile and run correctly interpreted; a failure // (a status other than 502, or an elapsed time anywhere near // timeoutUpstreamSleep) would mean this feature works compiled but not // under Yaegi, the exact class of divergence this harness exists to // catch. // // MUTATION VERIFIED: temporarily removing // `tr.ResponseHeaderTimeout = timeout` from newAdapterHTTPClient // (providers.go) made `make yaegi-check` fail here with "status = 200, // want 502" — timeoutUpstream's 600ms sleep let the request succeed // instead of aborting at timeoutProviderRequestTimeout (80ms), proving // this probe genuinely exercises the interpreted mechanism rather than // passing regardless. Reverted before committing. func exerciseRequestTimeout(handler http.Handler) error { body := `{"model":"` + timeoutProviderName + `/` + timeoutProviderModel + `","messages":[{"role":"user","content":"hi"}]}` req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(body)) req.Header.Set("Authorization", "Bearer "+testDataUserAPIKey) req.Header.Set("Content-Type", "application/json") rec := httptest.NewRecorder() start := time.Now() handler.ServeHTTP(rec, req) elapsed := time.Since(start) if elapsed > timeoutProbeMaxWait { return fmt.Errorf("POST /v1/chat/completions (hung provider %q) took %s, want well under %s — a hung provider must be aborted under the interpreter, not merely in compiled tests: %s", timeoutProviderName, elapsed, timeoutProbeMaxWait, rec.Body.String()) } if rec.Code != http.StatusBadGateway { return fmt.Errorf("POST /v1/chat/completions (hung provider %q): status = %d, want 502 (upstream connection error, provider-attributed), body=%s", timeoutProviderName, rec.Code, rec.Body.String()) } return nil } // exerciseRequestTimeoutMidBodyStall proves the OTHER half of the // request-timeout feature aborts under the REAL interpreter (finding F7, // coordinator adversarial review, 2026-08-23): exerciseRequestTimeout // above drives timeoutProviderName, whose upstream never sends headers // at all, so it only exercises Transport.ResponseHeaderTimeout — // watchdogBody.fire (timeout.go) never runs under that probe, since the // request never gets far enough to construct a watchdogBody in the first // place. hungBodyProviderName's upstream sends headers immediately, then // stalls, so this drives the idle-progress body watchdog itself: // time.AfterFunc handed the interpreted method value wb.fire, // context.WithCancel/CancelFunc, and sync/atomic's function API // (atomic.StoreInt32/LoadInt32) all have to work correctly interpreted // for this to abort rather than hang. // // MUTATION VERIFIED: temporarily removing the `resp.Body = // newWatchdogBody(...)` line from upstreamBytes (providers.go) made // exerciseRequestTimeout (the header-timeout probe) still PASS // unchanged, while this probe failed with "status = 200, want 502" — // hungBodyUpstream's 600ms sleep let the request succeed once nothing // guarded the body read. This confirms the two probes exercise genuinely // different code paths, and that this one specifically needs // watchdogBody to pass. Reverted before committing. func exerciseRequestTimeoutMidBodyStall(handler http.Handler) error { body := `{"model":"` + hungBodyProviderName + `/` + hungBodyProviderModel + `","messages":[{"role":"user","content":"hi"}]}` req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(body)) req.Header.Set("Authorization", "Bearer "+testDataUserAPIKey) req.Header.Set("Content-Type", "application/json") rec := httptest.NewRecorder() start := time.Now() handler.ServeHTTP(rec, req) elapsed := time.Since(start) if elapsed > timeoutProbeMaxWait { return fmt.Errorf("POST /v1/chat/completions (mid-body-stall provider %q) took %s, want well under %s — a provider that sends headers then stalls must still be aborted under the interpreter, not merely in compiled tests: %s", hungBodyProviderName, elapsed, timeoutProbeMaxWait, rec.Body.String()) } if rec.Code != http.StatusBadGateway { return fmt.Errorf("POST /v1/chat/completions (mid-body-stall provider %q): status = %d, want 502 (upstream connection error, provider-attributed), body=%s", hungBodyProviderName, rec.Code, rec.Body.String()) } return nil } func exerciseAttemptAccounting(handler http.Handler) error { chatReq := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader( `{"model":"`+testDataWantModel+`","messages":[{"role":"user","content":"hi"}]}`, )) chatReq.Header.Set("Authorization", "Bearer "+testDataUserAPIKey) chatReq.Header.Set("Content-Type", "application/json") chatRec := httptest.NewRecorder() handler.ServeHTTP(chatRec, chatReq) if chatRec.Code != http.StatusOK { return fmt.Errorf("POST /v1/chat/completions (attempt-accounting harness): status = %d, want 200, body=%s", chatRec.Code, chatRec.Body.String()) } attemptsDay, failuresDay, err := pollProviderAttemptCounters(handler, "openai") if err != nil { return err } if attemptsDay != 1 { return fmt.Errorf(`admin overview provider "openai" attemptsDay = %v, want 1 (Feature A attempt-accounting harness)`, attemptsDay) } if failuresDay != 0 { return fmt.Errorf(`admin overview provider "openai" failuresDay = %v, want 0`, failuresDay) } // SHOULD-A: a real, interpreted context-deadline timeout against a // deliberately slow upstream must record a FAILURE — the whole // reason this harness exists after round 2's blocker. slowCtx, cancel := context.WithTimeout(context.Background(), slowRequestDeadline) defer cancel() slowReq := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader( `{"model":"`+slowProviderName+`/`+slowProviderModel+`","messages":[{"role":"user","content":"hi"}]}`, )).WithContext(slowCtx) slowReq.Header.Set("Authorization", "Bearer "+testDataUserAPIKey) slowReq.Header.Set("Content-Type", "application/json") slowRec := httptest.NewRecorder() handler.ServeHTTP(slowRec, slowReq) // No status assertion here on purpose: the point of interest is the // PROVIDER-HEALTH counter a timed-out upstream attempt leaves behind, // not what status code a canceled/timed-out client request itself // gets back (which the existing route-level Go tests already pin). slowAttemptsDay, slowFailuresDay, err := pollProviderAttemptCounters(handler, slowProviderName) if err != nil { return err } if slowAttemptsDay != 1 { return fmt.Errorf("admin overview provider %q attemptsDay = %v, want 1 (SHOULD-A timeout harness)", slowProviderName, slowAttemptsDay) } if slowFailuresDay != 1 { return fmt.Errorf("admin overview provider %q failuresDay = %v, want 1 — a real interpreted context-deadline timeout must count as a provider-health failure (SHOULD-A, closes the round-2 blocker)", slowProviderName, slowFailuresDay) } return nil } // pollProviderAttemptCounters drives GET /admin/api/overview against // handler repeatedly until providerName's attemptsDay is non-zero or a 2s // budget elapses, then returns its final reading either way — SHOULD-5 // (v0.22 review round): recordProviderAttempt's store write runs on its // own goroutine (limiter.spawn), off the request that triggered it, so a // single immediate read right after that request returns could race // ahead of the write landing. A real goroutine still schedules promptly // under Yaegi (only the INTERPRETED code driving it runs slower, not the // underlying Go runtime's scheduler), so this is a short poll, not a // long one. func pollProviderAttemptCounters(handler http.Handler, providerName string) (attemptsDay, failuresDay float64, err error) { deadline := time.Now().Add(2 * time.Second) for { attemptsDay, failuresDay, err = readProviderAttemptCounters(handler, providerName) if err != nil { return 0, 0, err } if attemptsDay != 0 || time.Now().After(deadline) { return attemptsDay, failuresDay, nil } time.Sleep(10 * time.Millisecond) } } // readProviderAttemptCounters reads GET /admin/api/overview, finds the // entry named providerName, and returns its attemptsDay/failuresDay // fields via a generic map[string]any decode — this harness module is // compiled, not interpreted, so it could import the plugin's own admin.go // types directly, but they are unexported (adminOverviewResponse, // adminProviderView); decoding generically here is simpler than exporting // test-only types across that boundary just for this one harness. Looked // up by name, not index 0 (round 1 of this harness only ever had one // configured provider): buildAdminOverview sorts providers by name // (registry.go), so adding slowProviderName ("slow") after "openai" // changed which index held which provider. func readProviderAttemptCounters(handler http.Handler, providerName string) (attemptsDay, failuresDay float64, err error) { req := httptest.NewRequest(http.MethodGet, "/admin/api/overview", nil) req.Header.Set("Authorization", "Bearer "+attemptAccountingAdminAPIKey) rec := httptest.NewRecorder() handler.ServeHTTP(rec, req) if rec.Code != http.StatusOK { return 0, 0, fmt.Errorf("GET /admin/api/overview: status = %d, want 200, body=%s", rec.Code, rec.Body.String()) } var body map[string]any if err := json.Unmarshal(rec.Body.Bytes(), &body); err != nil { return 0, 0, fmt.Errorf("decode GET /admin/api/overview body: %w", err) } providers, ok := body["providers"].([]any) if !ok { return 0, 0, fmt.Errorf("admin overview body has no providers array: %s", rec.Body.String()) } for _, raw := range providers { p, ok := raw.(map[string]any) if !ok { continue } if name, _ := p["name"].(string); name != providerName { continue } attemptsDay, _ = p["attemptsDay"].(float64) failuresDay, _ = p["failuresDay"].(float64) return attemptsDay, failuresDay, nil } return 0, 0, fmt.Errorf("admin overview body has no provider named %q: %s", providerName, rec.Body.String()) } // readModulePath returns the module path declared by goModPath's // "module" directive. func readModulePath(goModPath string) (string, error) { f, err := os.Open(goModPath) if err != nil { return "", err } defer f.Close() //nolint:errcheck // read-side close; nothing actionable on failure scanner := bufio.NewScanner(f) for scanner.Scan() { line := strings.TrimSpace(scanner.Text()) if after, ok := strings.CutPrefix(line, "module "); ok { return strings.TrimSpace(after), nil } } if err := scanner.Err(); err != nil { return "", err } return "", fmt.Errorf("no module directive found in %s", goModPath) } // readPackageName returns the package name declared by the first // non-test .go file it finds directly inside dir. The plugin is a single // root package, so any one of its files' declarations suffices. func readPackageName(dir string) (string, error) { entries, err := os.ReadDir(dir) if err != nil { return "", err } for _, e := range entries { name := e.Name() if e.IsDir() || !strings.HasSuffix(name, ".go") || strings.HasSuffix(name, "_test.go") { continue } pkgName, err := scanPackageDecl(filepath.Join(dir, name)) if err != nil { return "", err } if pkgName != "" { return pkgName, nil } } return "", fmt.Errorf("no package declaration found in %s", dir) } // scanPackageDecl reads path's leading "package X" line, if any. func scanPackageDecl(path string) (string, error) { f, err := os.Open(path) if err != nil { return "", err } defer f.Close() //nolint:errcheck // read-side close; nothing actionable on failure scanner := bufio.NewScanner(f) for scanner.Scan() { line := strings.TrimSpace(scanner.Text()) if after, ok := strings.CutPrefix(line, "package "); ok { return strings.TrimSpace(after), nil } } return "", scanner.Err() } // copyRepoSource copies repoRoot into destDir, skipping every directory // named in excludedTopLevelDirs. destDir plays the role of // GOPATH/src/{modulePath} for the interpreter: Yaegi's source loader // resolves the plugin's own import path by walking GOPATH exactly as the // standard toolchain would for a pre-modules GOPATH-mode build. func copyRepoSource(repoRoot, destDir string) error { entries, err := os.ReadDir(repoRoot) if err != nil { return err } if err := os.MkdirAll(destDir, 0o750); err != nil { return err } for _, e := range entries { if excludedTopLevelDirs[e.Name()] { continue } src := filepath.Join(repoRoot, e.Name()) dst := filepath.Join(destDir, e.Name()) if e.IsDir() { if err := copyDir(src, dst); err != nil { return err } continue } if err := copyFile(src, dst); err != nil { return err } } return nil } // copyDir recursively copies src to dst. func copyDir(src, dst string) error { return filepath.WalkDir(src, func(path string, d fs.DirEntry, err error) error { if err != nil { return err } rel, err := filepath.Rel(src, path) if err != nil { return err } target := filepath.Join(dst, rel) if d.IsDir() { return os.MkdirAll(target, 0o750) } return copyFile(path, target) }) } // copyFile copies src to dst, creating dst's parent directory if needed. func copyFile(src, dst string) error { in, err := os.Open(src) if err != nil { return err } defer in.Close() //nolint:errcheck // read-side close; nothing actionable on failure if err = os.MkdirAll(filepath.Dir(dst), 0o750); err != nil { return err } out, err := os.Create(dst) if err != nil { return err } defer out.Close() //nolint:errcheck // Close's error is checked explicitly below if _, err := io.Copy(out, in); err != nil { return err } return out.Close() } // yamlLine is one non-blank, non-comment line of a YAML file, with // leading whitespace stripped and recorded as indent. type yamlLine struct { text string indent int } // extractTraefikYAMLTestData reads path (the plugin's .traefik.yml) and // decodes its top-level "testData:" block into a generic // map[string]any/[]any/scalar tree, using a small hand-rolled, // indentation-based YAML subset parser: just enough for this one file's // shape (nested maps, lists of scalars, lists of flat maps, "{}"/"[]" // empty collections), not a general YAML implementation. It exists only // in this harness module — the plugin module never parses YAML itself, // Traefik does that before handing the plugin its already-decoded Config. func extractTraefikYAMLTestData(path string) (map[string]any, error) { raw, err := os.ReadFile(path) if err != nil { return nil, err } lines := tokenizeYAMLLines(string(raw)) block := extractTestDataBlock(lines) if block == nil { return nil, fmt.Errorf("%s has no top-level testData: block", path) } pos := 0 v := parseBlock(block, &pos, block[0].indent) m, ok := v.(map[string]any) if !ok { return nil, fmt.Errorf("%s testData: block is not a map", path) } return m, nil } // tokenizeYAMLLines splits raw into non-blank, non-comment yamlLines, // each carrying its own leading-space count as indent. func tokenizeYAMLLines(raw string) []yamlLine { var out []yamlLine for _, ln := range strings.Split(raw, "\n") { trimmedRight := strings.TrimRight(ln, " \t\r") content := strings.TrimLeft(trimmedRight, " ") if content == "" || strings.HasPrefix(content, "#") { continue } out = append(out, yamlLine{indent: len(trimmedRight) - len(content), text: content}) } return out } // extractTestDataBlock returns the lines nested under a top-level (indent // 0) "testData:" line, or nil if lines has none. func extractTestDataBlock(lines []yamlLine) []yamlLine { for i, l := range lines { if l.indent != 0 || l.text != "testData:" { continue } var block []yamlLine for j := i + 1; j < len(lines) && lines[j].indent > 0; j++ { block = append(block, lines[j]) } return block } return nil } // parseBlock parses the run of lines starting at *pos that share indent, // dispatching to parseList when the first such line is a "-" item and to // parseMap otherwise. func parseBlock(lines []yamlLine, pos *int, indent int) any { if *pos >= len(lines) || lines[*pos].indent != indent { return map[string]any{} } if strings.HasPrefix(lines[*pos].text, "-") { return parseList(lines, pos, indent) } return parseMap(lines, pos, indent) } // parseMap consumes every consecutive "key: value" line at indent, // recursing into parseBlock for a key whose value is empty (a nested // block follows on deeper-indented lines). func parseMap(lines []yamlLine, pos *int, indent int) map[string]any { m := map[string]any{} for *pos < len(lines) && lines[*pos].indent == indent { key, val, _ := strings.Cut(lines[*pos].text, ":") key = strings.TrimSpace(key) val = strings.TrimSpace(val) *pos++ if val != "" { m[key] = collectionOrScalar(val) continue } if *pos < len(lines) && lines[*pos].indent > indent { m[key] = parseBlock(lines, pos, lines[*pos].indent) } else { m[key] = map[string]any{} } } return m } // parseList consumes every consecutive "- ..." line at indent. A "-" // item containing a colon starts a flat map (its own "key: value" plus // every following line indented at least indent+2, until a line back at // indent or shallower); any other item is a bare scalar. func parseList(lines []yamlLine, pos *int, indent int) []any { var out []any for *pos < len(lines) && lines[*pos].indent == indent && strings.HasPrefix(lines[*pos].text, "-") { item := strings.TrimSpace(strings.TrimPrefix(lines[*pos].text, "-")) if item == "" { *pos++ continue } if !strings.Contains(item, ":") { out = append(out, parseScalar(item)) *pos++ continue } m := map[string]any{} key, val, _ := strings.Cut(item, ":") m[strings.TrimSpace(key)] = collectionOrScalar(strings.TrimSpace(val)) *pos++ childIndent := indent + 2 for *pos < len(lines) && lines[*pos].indent >= childIndent { k, v, _ := strings.Cut(lines[*pos].text, ":") m[strings.TrimSpace(k)] = collectionOrScalar(strings.TrimSpace(v)) *pos++ } out = append(out, m) } return out } // collectionOrScalar maps a "key: value" line's already-trimmed val to // an empty map/slice for "{}"/"[]", or to parseScalar(val) otherwise. It // does not recurse into a further nested block — no key in this file's // testData needs a map/list value inside a list item's own fields, and // this harness parses only what .traefik.yml actually contains. func collectionOrScalar(val string) any { switch val { case "{}": return map[string]any{} case "[]": return []any{} default: return parseScalar(val) } } // parseScalar converts a YAML scalar token to a bool, int64, float64, or // string, in that preference order, stripping a matching pair of quotes // first. func parseScalar(s string) any { if len(s) >= 2 { if (s[0] == '"' && s[len(s)-1] == '"') || (s[0] == '\'' && s[len(s)-1] == '\'') { return s[1 : len(s)-1] } } switch s { case "true": return true case "false": return false } if n, err := strconv.ParseInt(s, 10, 64); err == nil { return n } if f, err := strconv.ParseFloat(s, 64); err == nil { return f } return s } // modelUsageCanonicalID is the kindModel scope id exerciseModelUsageRanking // expects its own chat completion to be attributed to: the canonical // "provider/model" of the bare testDataWantModel's winning provider // ("openai" — the same attribution exerciseAttemptAccounting already // relies on). const modelUsageCanonicalID = "openai/" + testDataWantModel // exerciseModelUsageRanking drives the per-model usage feature end to end // under the INTERPRETER (feat: per-model usage statistics): one real chat // completion, then GET /admin/api/usage/models for the ranking and GET // /admin/api/usage/history for that same model's own series. // // Why this needs an interpreted probe at all, when compiled tests already // cover the same code: the feature adds a scope kind resolved POST-response // (limits.go's withModelScope/kindModel), reached through a sort.Slice // closure and a generic JSON encode of a newly declared struct type. Every // one of those is a construct this repo has previously seen diverge under // Yaegi — a compiled pass proves nothing about the interpreter, which is // the whole premise of this harness. func exerciseModelUsageRanking(handler http.Handler) error { chatReq := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader( `{"model":"`+testDataWantModel+`","messages":[{"role":"user","content":"hi"}]}`, )) chatReq.Header.Set("Authorization", "Bearer "+testDataUserAPIKey) chatReq.Header.Set("Content-Type", "application/json") chatRec := httptest.NewRecorder() handler.ServeHTTP(chatRec, chatReq) if chatRec.Code != http.StatusOK { return fmt.Errorf("POST /v1/chat/completions (model-usage harness): status = %d, want 200, body=%s", chatRec.Code, chatRec.Body.String()) } models, err := pollModelRanking(handler) if err != nil { return err } // The ranking must contain the model that just served, and EVERY entry // must be non-zero — the operator requirement the server-side filter // exists for. A zero here would mean the filter did not run under the // interpreter even though it does when compiled. var found bool for _, raw := range models { entry, ok := raw.(map[string]any) if !ok { return fmt.Errorf("GET /admin/api/usage/models: entry is not an object: %v", raw) } id, _ := entry["id"].(string) value, _ := entry["value"].(float64) if value == 0 { return fmt.Errorf("GET /admin/api/usage/models listed %q at value 0 — only non-zero models may be returned", id) } if id == modelUsageCanonicalID { found = true } } if !found { return fmt.Errorf("GET /admin/api/usage/models did not list %q after a served request: %v", modelUsageCanonicalID, models) } // The same model must also be reachable as an ordinary history scope — // the point of keying model usage as a scope kind rather than a // bespoke counter. histReq := httptest.NewRequest(http.MethodGet, "/admin/api/usage/history?scope=model:"+url.QueryEscape(modelUsageCanonicalID)+"&metric=req&window=day&span=1", nil) histReq.Header.Set("Authorization", "Bearer "+attemptAccountingAdminAPIKey) histRec := httptest.NewRecorder() handler.ServeHTTP(histRec, histReq) if histRec.Code != http.StatusOK { return fmt.Errorf("GET /admin/api/usage/history for scope model:%s: status = %d, want 200, body=%s", modelUsageCanonicalID, histRec.Code, histRec.Body.String()) } var hist map[string]any if err := json.Unmarshal(histRec.Body.Bytes(), &hist); err != nil { return fmt.Errorf("decode model history body: %w", err) } points, ok := hist["points"].([]any) if !ok || len(points) != 1 { return fmt.Errorf("model history points = %v, want exactly 1 bucket: %s", hist["points"], histRec.Body.String()) } point, ok := points[0].(map[string]any) if !ok { return fmt.Errorf("model history point is not an object: %v", points[0]) } if value, _ := point["value"].(float64); value < 1 { return fmt.Errorf("model history value = %v, want at least 1 request attributed to %q", point["value"], modelUsageCanonicalID) } // An id outside the catalog must 404 rather than answer an empty // series — the guard that keeps a mistyped or retired model from // looking like a model with no traffic. unknownReq := httptest.NewRequest(http.MethodGet, "/admin/api/usage/history?scope=model:openai/no-such-model&metric=req&window=day&span=1", nil) unknownReq.Header.Set("Authorization", "Bearer "+attemptAccountingAdminAPIKey) unknownRec := httptest.NewRecorder() handler.ServeHTTP(unknownRec, unknownReq) if unknownRec.Code != http.StatusNotFound { return fmt.Errorf("GET /admin/api/usage/history for an uncatalogued model: status = %d, want 404, body=%s", unknownRec.Code, unknownRec.Body.String()) } fmt.Println("yaegi-check: per-model usage ranking and model-scoped history served under the interpreter; only non-zero models listed, uncatalogued model 404s") return nil } // pollModelRanking reads GET /admin/api/usage/models until it reports at // least one model or a 2s budget elapses. The poll mirrors // pollProviderAttemptCounters' own rationale: interpreted code driving the // request runs slower than compiled code, so a single immediate read can // outrun the accounting write the preceding request triggered. func pollModelRanking(handler http.Handler) ([]any, error) { deadline := time.Now().Add(2 * time.Second) for { req := httptest.NewRequest(http.MethodGet, "/admin/api/usage/models?metric=req&window=day", nil) req.Header.Set("Authorization", "Bearer "+attemptAccountingAdminAPIKey) rec := httptest.NewRecorder() handler.ServeHTTP(rec, req) if rec.Code != http.StatusOK { return nil, fmt.Errorf("GET /admin/api/usage/models: status = %d, want 200, body=%s", rec.Code, rec.Body.String()) } var body map[string]any if err := json.Unmarshal(rec.Body.Bytes(), &body); err != nil { return nil, fmt.Errorf("decode GET /admin/api/usage/models body: %w", err) } models, ok := body["models"].([]any) if !ok { return nil, fmt.Errorf("admin usage/models body has no models array: %s", rec.Body.String()) } if len(models) > 0 || time.Now().After(deadline) { return models, nil } time.Sleep(10 * time.Millisecond) } } // freeModelDetailCanonicalID is the kindModel scope id // exerciseFreeModelUsageDetail expects to find, with Free=true, in GET // /admin/api/usage/models' detail=1 response: provenanceEstProviderModel // already carries one real request (exerciseUsageProvenance's own chat // completion, above) but no configured price at all — see its own // modelMeta:{"free":true} override's doc comment (attemptAccountingOverride) // for why free:true there is the ONLY reason this shows up free here. const freeModelDetailCanonicalID = provenanceEstProviderName + "/" + provenanceEstProviderModel // pollUsageModelsDetail is pollModelRanking's own detail=1 counterpart // (free-models feature): GET /admin/api/usage/models?metric=cost&window=day&detail=1, // polled the identical way — a single immediate read can outrun the // accounting write an immediately preceding request triggered, under the // slower interpreted path. func pollUsageModelsDetail(handler http.Handler) ([]any, error) { deadline := time.Now().Add(2 * time.Second) for { req := httptest.NewRequest(http.MethodGet, "/admin/api/usage/models?metric=cost&window=day&detail=1", nil) req.Header.Set("Authorization", "Bearer "+attemptAccountingAdminAPIKey) rec := httptest.NewRecorder() handler.ServeHTTP(rec, req) if rec.Code != http.StatusOK { return nil, fmt.Errorf("GET /admin/api/usage/models?detail=1: status = %d, want 200, body=%s", rec.Code, rec.Body.String()) } var body map[string]any if err := json.Unmarshal(rec.Body.Bytes(), &body); err != nil { return nil, fmt.Errorf("decode GET /admin/api/usage/models?detail=1 body: %w", err) } models, ok := body["models"].([]any) if !ok { return nil, fmt.Errorf("admin usage/models?detail=1 body has no models array: %s", rec.Body.String()) } if len(models) > 0 || time.Now().After(deadline) { return models, nil } time.Sleep(10 * time.Millisecond) } } // exerciseFreeModelUsageDetail drives GET /admin/api/usage/models' // detail=1 mode (free-models feature) under the INTERPRETER: a // modelMeta-declared free model with real request traffic but a // genuinely unpriced (hence zero-cost) model must still appear in a // metric=cost ranking once detail=1 is set — the non-detail path drops // any zero-value entry entirely, which is exactly the gap this feature // closes. Reached through serveAdminUsageModelsDetail's own sort.Slice // closure, its pointer-typed struct fields, and adminUsageModelFree's // two-source free() check — none of which any earlier probe in this // harness exercises. func exerciseFreeModelUsageDetail(handler http.Handler) error { models, err := pollUsageModelsDetail(handler) if err != nil { return err } for _, raw := range models { entry, ok := raw.(map[string]any) if !ok { return fmt.Errorf("GET /admin/api/usage/models?detail=1: entry is not an object: %v", raw) } if id, _ := entry["id"].(string); id != freeModelDetailCanonicalID { continue } value, _ := entry["value"].(float64) if value != 0 { return fmt.Errorf("detail=1 entry %q: value = %v, want 0 (metric=cost, and its price is genuinely unknown)", freeModelDetailCanonicalID, value) } free, _ := entry["free"].(bool) if !free { return fmt.Errorf("detail=1 entry %q: free = %v, want true (modelMeta free:true override)", freeModelDetailCanonicalID, entry["free"]) } requests, _ := entry["requests"].(float64) if requests <= 0 { return fmt.Errorf("detail=1 entry %q: requests = %v, want > 0", freeModelDetailCanonicalID, entry["requests"]) } fmt.Println("yaegi-check: GET /admin/api/usage/models detail=1 lists a modelMeta-free model at value 0 with its real request count, under the interpreter") return nil } return fmt.Errorf("GET /admin/api/usage/models?detail=1 did not list %q: %v", freeModelDetailCanonicalID, models) } // modelUsageDetailRequests reads GET /admin/api/usage/models' detail=1 // "requests" count for id right now. No poll: account()'s own store // write (limits.go) is synchronous — unlike recordProviderAttempt, it // never goes through limiter.spawn — so by the time ServeHTTP returns // for the request that triggered it, the counter this reads is already // current; pollUsageModelsDetail's own poll loop exists for a DIFFERENT // reason (an empty response, not a stale one) and still applies here // unchanged. An id with no traffic at all yet is legitimately absent // from the response (only a nonzero metric total is listed even under // detail=1), so a miss reports 0, not an error. func modelUsageDetailRequests(handler http.Handler, id string) (int64, error) { models, err := pollUsageModelsDetail(handler) if err != nil { return 0, err } for _, raw := range models { entry, ok := raw.(map[string]any) if !ok { continue } if entry["id"] != id { continue } reqs, _ := entry["requests"].(float64) return int64(reqs), nil } return 0, nil } // exerciseMediaModelUsage drives POST /v1/images/generations (free- // models feature: media-route model request counters) under the // INTERPRETER, reusing the existing "openai"/testDataWantModel provider/ // model pair — upstream answers 200 for any request path (its own doc // comment, above), so no new fixture provider is needed here. It reads // modelUsageCanonicalID's own kindModel req counter (via GET // /admin/api/usage/models detail=1) before and after the request: // recordMediaModelUsage (routes_media.go) must have added exactly 1, // reached through withModelScope called with an empty scope list — a // construct no earlier media-route probe in this harness exercises. func exerciseMediaModelUsage(handler http.Handler) error { before, err := modelUsageDetailRequests(handler, modelUsageCanonicalID) if err != nil { return err } imgReq := httptest.NewRequest(http.MethodPost, "/v1/images/generations", strings.NewReader( `{"model":"`+testDataWantModel+`","prompt":"a cat"}`, )) imgReq.Header.Set("Authorization", "Bearer "+testDataUserAPIKey) imgReq.Header.Set("Content-Type", "application/json") imgRec := httptest.NewRecorder() handler.ServeHTTP(imgRec, imgReq) if imgRec.Code != http.StatusOK { return fmt.Errorf("POST /v1/images/generations (media-model-usage harness): status = %d, want 200, body=%s", imgRec.Code, imgRec.Body.String()) } after, err := modelUsageDetailRequests(handler, modelUsageCanonicalID) if err != nil { return err } if after != before+1 { return fmt.Errorf("GET /admin/api/usage/models detail=1 requests for %q = %d, want exactly %d (before=%d, one served media request must add exactly one)", modelUsageCanonicalID, after, before+1, before) } fmt.Println("yaegi-check: a served media route (POST /v1/images/generations) incremented its model's own per-model request counter, under the interpreter") return nil } // exerciseAdminOverviewReplicaAndWarnings proves GET /admin/api/overview's // two new fields (F4/F9 replica id, F10 config-warnings collection, v0.3 // dashboard task) are populated correctly under the REAL interpreter: // replica must be non-empty (g.replica: os.Hostname() -> $HOSTNAME -> // "unknown", llmgateway.go) and warnings must decode as a genuine JSON // array, never null (configWarningsSnapshot's own "never nil" contract, // logger.go) — a null here would mean noteConfigWarning's slice append // under the interpreter produced a nil rather than an empty slice, a // construct class this codebase has seen diverge before. Content is // fixture-dependent (this harness's own config may or may not trip a // warnf during construction), so only presence/type is asserted, not any // particular message. func exerciseAdminOverviewReplicaAndWarnings(handler http.Handler) error { req := httptest.NewRequest(http.MethodGet, "/admin/api/overview", nil) req.Header.Set("Authorization", "Bearer "+attemptAccountingAdminAPIKey) rec := httptest.NewRecorder() handler.ServeHTTP(rec, req) if rec.Code != http.StatusOK { return fmt.Errorf("GET /admin/api/overview (replica/warnings probe): status = %d, want 200, body=%s", rec.Code, rec.Body.String()) } var body map[string]any if err := json.Unmarshal(rec.Body.Bytes(), &body); err != nil { return fmt.Errorf("decode GET /admin/api/overview body: %w", err) } replica, _ := body["replica"].(string) if replica == "" { return fmt.Errorf(`GET /admin/api/overview: "replica" = %q, want non-empty (F4/F9: os.Hostname() -> $HOSTNAME -> "unknown")`, replica) } if _, ok := body["warnings"].([]any); !ok { return fmt.Errorf(`GET /admin/api/overview: "warnings" did not decode as a JSON array (want never-nil, F10): %v`, body["warnings"]) } fmt.Println("yaegi-check: GET /admin/api/overview carries a non-empty replica id and a never-nil warnings array under the interpreter") return nil } // exerciseUsageModelsSpan proves GET /admin/api/usage/models' optional // ?span= parameter (F2, v0.3 dashboard task) is parsed, validated, and // echoed back correctly under the REAL interpreter: modelSpanTotals' // per-id historyBucketKeys loop and chunkedModelSpanTotals' chunking // arithmetic (admin.go/limits.go) are both new, interpreter-untested // code this harness's own compiled test suite cannot vouch for. window // "hour" caps span at 48 (historyMaxSpan), so 24 is comfortably valid. func exerciseUsageModelsSpan(handler http.Handler) error { req := httptest.NewRequest(http.MethodGet, "/admin/api/usage/models?metric=req&window=hour&span=24", nil) req.Header.Set("Authorization", "Bearer "+attemptAccountingAdminAPIKey) rec := httptest.NewRecorder() handler.ServeHTTP(rec, req) if rec.Code != http.StatusOK { return fmt.Errorf("GET /admin/api/usage/models?span=24: status = %d, want 200, body=%s", rec.Code, rec.Body.String()) } var body map[string]any if err := json.Unmarshal(rec.Body.Bytes(), &body); err != nil { return fmt.Errorf("decode GET /admin/api/usage/models body: %w", err) } if span, _ := body["span"].(float64); span != 24 { return fmt.Errorf(`GET /admin/api/usage/models?span=24: "span" = %v, want 24 (F2's own echoed, resolved span)`, body["span"]) } fmt.Println("yaegi-check: GET /admin/api/usage/models honors ?span=24 on window=hour and echoes it back under the interpreter") return nil } // exerciseRateLimitEventAndRejections proves a real checkAndCount // rejection is BOTH recorded to GET /admin/api/events as a rate_limit // event (F3 hook 1) AND folded into GET /admin/api/usage's fleet-wide // total.rejectionsPerDay counter (F4, Q6: "attribute rejections to // total/all too") under the REAL interpreter — settleRejection's own // single storeIncrMulti round trip (limits.go) and recordLimitEvent's // event-ring append (events.go) are both new code no compiled test here // can vouch for. eventsProbeUserAPIKey's own dedicated // requestsPerMinute:1 limit (run()'s "users" override) makes the second // call 429 deterministically, regardless of what every other probe // already did to testDataUserAPIKey's own counters. func exerciseRateLimitEventAndRejections(handler http.Handler) error { body := `{"model":"` + testDataWantModel + `","messages":[{"role":"user","content":"hi"}]}` req1 := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(body)) req1.Header.Set("Authorization", "Bearer "+eventsProbeUserAPIKey) req1.Header.Set("Content-Type", "application/json") rec1 := httptest.NewRecorder() handler.ServeHTTP(rec1, req1) if rec1.Code != http.StatusOK { return fmt.Errorf("POST /v1/chat/completions (rate-limit-event harness, 1st call): status = %d, want 200, body=%s", rec1.Code, rec1.Body.String()) } req2 := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(body)) req2.Header.Set("Authorization", "Bearer "+eventsProbeUserAPIKey) req2.Header.Set("Content-Type", "application/json") rec2 := httptest.NewRecorder() handler.ServeHTTP(rec2, req2) if rec2.Code != http.StatusTooManyRequests { return fmt.Errorf("POST /v1/chat/completions (rate-limit-event harness, 2nd call): status = %d, want 429 (over requestsPerMinute:1), body=%s", rec2.Code, rec2.Body.String()) } eventsReq := httptest.NewRequest(http.MethodGet, "/admin/api/events?limit=10", nil) eventsReq.Header.Set("Authorization", "Bearer "+attemptAccountingAdminAPIKey) eventsRec := httptest.NewRecorder() handler.ServeHTTP(eventsRec, eventsReq) if eventsRec.Code != http.StatusOK { return fmt.Errorf("GET /admin/api/events?limit=10: status = %d, want 200, body=%s", eventsRec.Code, eventsRec.Body.String()) } var eventsBody map[string]any if err := json.Unmarshal(eventsRec.Body.Bytes(), &eventsBody); err != nil { return fmt.Errorf("decode GET /admin/api/events body: %w", err) } events, _ := eventsBody["events"].([]any) if len(events) == 0 { return fmt.Errorf("GET /admin/api/events: want at least one event, body=%s", eventsRec.Body.String()) } first, ok := events[0].(map[string]any) if !ok { return fmt.Errorf("GET /admin/api/events: events[0] is not an object: %v", events[0]) } if kind, _ := first["kind"].(string); kind != "rate_limit" { return fmt.Errorf(`GET /admin/api/events: events[0] (newest first).kind = %q, want "rate_limit"`, kind) } if route, _ := first["route"].(string); route != "chat/completions" { return fmt.Errorf(`GET /admin/api/events: events[0].route = %q, want "chat/completions"`, route) } usageReq := httptest.NewRequest(http.MethodGet, "/admin/api/usage", nil) usageReq.Header.Set("Authorization", "Bearer "+attemptAccountingAdminAPIKey) usageRec := httptest.NewRecorder() handler.ServeHTTP(usageRec, usageReq) if usageRec.Code != http.StatusOK { return fmt.Errorf("GET /admin/api/usage: status = %d, want 200, body=%s", usageRec.Code, usageRec.Body.String()) } var usageBody map[string]any if err := json.Unmarshal(usageRec.Body.Bytes(), &usageBody); err != nil { return fmt.Errorf("decode GET /admin/api/usage body: %w", err) } total, ok := usageBody["total"].(map[string]any) if !ok { return fmt.Errorf("GET /admin/api/usage: no total object: %s", usageRec.Body.String()) } if rejPerDay, _ := total["rejectionsPerDay"].(float64); rejPerDay < 1 { return fmt.Errorf("GET /admin/api/usage: total.rejectionsPerDay = %v, want >= 1 after a real 429 (F4)", total["rejectionsPerDay"]) } fmt.Println("yaegi-check: a real 429 recorded a rate_limit event (GET /admin/api/events) and incremented total.rejectionsPerDay (GET /admin/api/usage) under the interpreter") return nil } // exerciseUpstreamAndTimeoutEvents proves recordUpstreamEvent's own // classification (events.go, F3 hook 2) runs correctly under the REAL // interpreter. Added per a backend verifier's finding (coordinator // request, 2026-09-23): Yaegi v0.16.1 was found to leave kind/status // empty for a multi-value assignment whose right-hand side contains a // method call — exactly recordUpstreamEvent's own // `kind, status, message = eventKindTimeout, http.StatusGatewayTimeout, // err.Error()` shape — a construct class this codebase has repeatedly // seen diverge under the interpreter (this package's own doc comment). // // eventsUpstreamErrorProviderName is a single, no-failover-candidate // provider that always 500s, so the client-facing response IS the // classified *providerHTTPError and the newest event must be kind // "upstream" with that same 500 status. timeoutProviderName // (exerciseRequestTimeout's own fixture, reused here rather than adding // a third upstream — cheap) then makes the newest event a "timeout" one: // its client-facing response is 502 (exerciseRequestTimeout's own // assertion), but the EVENT's own Status field is independently set to // http.StatusGatewayTimeout (504) by recordUpstreamEvent — the two are // deliberately different fields of different origin, both asserted here. func exerciseUpstreamAndTimeoutEvents(handler http.Handler) error { upstreamReq := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader( `{"model":"`+eventsUpstreamErrorProviderModel+`","messages":[{"role":"user","content":"hi"}]}`, )) upstreamReq.Header.Set("Authorization", "Bearer "+testDataUserAPIKey) upstreamReq.Header.Set("Content-Type", "application/json") upstreamRec := httptest.NewRecorder() handler.ServeHTTP(upstreamRec, upstreamReq) if upstreamRec.Code != http.StatusInternalServerError { return fmt.Errorf("POST /v1/chat/completions (upstream-event harness, provider %q): status = %d, want 500 (no failover candidate configured), body=%s", eventsUpstreamErrorProviderName, upstreamRec.Code, upstreamRec.Body.String()) } kind, status, err := latestEventKindAndStatus(handler) if err != nil { return err } if kind != "upstream" { return fmt.Errorf(`GET /admin/api/events: newest event kind = %q, want "upstream" after a real 500`, kind) } if status != http.StatusInternalServerError { return fmt.Errorf("GET /admin/api/events: newest (upstream) event status = %d, want 500", status) } timeoutReq := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader( `{"model":"`+timeoutProviderName+`/`+timeoutProviderModel+`","messages":[{"role":"user","content":"hi"}]}`, )) timeoutReq.Header.Set("Authorization", "Bearer "+testDataUserAPIKey) timeoutReq.Header.Set("Content-Type", "application/json") timeoutRec := httptest.NewRecorder() handler.ServeHTTP(timeoutRec, timeoutReq) if timeoutRec.Code != http.StatusBadGateway { return fmt.Errorf("POST /v1/chat/completions (timeout-event harness, provider %q): status = %d, want 502 (upstream connection error, provider-attributed — exerciseRequestTimeout's own contract), body=%s", timeoutProviderName, timeoutRec.Code, timeoutRec.Body.String()) } kind, status, err = latestEventKindAndStatus(handler) if err != nil { return err } if kind != "timeout" { return fmt.Errorf(`GET /admin/api/events: newest event kind = %q, want "timeout" after a real request-timeout abort`, kind) } if status != http.StatusGatewayTimeout { return fmt.Errorf("GET /admin/api/events: newest (timeout) event status = %d, want 504 (recordUpstreamEvent's own fixed value, distinct from the 502 the client saw)", status) } fmt.Println("yaegi-check: recordUpstreamEvent classified a real 500 as kind \"upstream\"/500 and a real request-timeout abort as kind \"timeout\"/504 under the interpreter") return nil } // latestEventKindAndStatus reads GET /admin/api/events?limit=1 as admin // and returns the single newest event's kind/status — the shared read // exerciseUpstreamAndTimeoutEvents' two assertions both drive, limit=1 // so each call cares only about the entry its own immediately preceding // request produced, never an earlier probe's. func latestEventKindAndStatus(handler http.Handler) (kind string, status int, err error) { req := httptest.NewRequest(http.MethodGet, "/admin/api/events?limit=1", nil) req.Header.Set("Authorization", "Bearer "+attemptAccountingAdminAPIKey) rec := httptest.NewRecorder() handler.ServeHTTP(rec, req) if rec.Code != http.StatusOK { return "", 0, fmt.Errorf("GET /admin/api/events?limit=1: status = %d, want 200, body=%s", rec.Code, rec.Body.String()) } var body map[string]any if jsonErr := json.Unmarshal(rec.Body.Bytes(), &body); jsonErr != nil { return "", 0, fmt.Errorf("decode GET /admin/api/events body: %w", jsonErr) } events, _ := body["events"].([]any) if len(events) == 0 { return "", 0, fmt.Errorf("GET /admin/api/events?limit=1: want at least one event, body=%s", rec.Body.String()) } first, ok := events[0].(map[string]any) if !ok { return "", 0, fmt.Errorf("GET /admin/api/events?limit=1: events[0] is not an object: %v", events[0]) } k, _ := first["kind"].(string) s, _ := first["status"].(float64) return k, int(s), nil }