Files
traefik-llmgateway/tools/yaegi-check/main.go
T
lukaszraczylo d421f49ff1 test(yaegi-check): verify the metrics endpoint under the interpreter
Compiled tests have repeatedly passed on this codebase while the
interpreted shape failed, so the metrics feature needs its own probe
in this harness, not just go test coverage.

Enable metrics in the harness config, with modelLabel on so the
opt-in provider-model breakdown runs interpreted too. Add
exerciseMetricsRoute, driving four real requests: unauthenticated
from outside the allowed CIDR block returns 401, a non-admin key
returns 403, the admin key returns 200 with the expected TYPE lines
and live provider and model data in the body, and a keyless request
from inside the allowed CIDR block returns 200.

Run the new probe last, after the breaker probe settles, since GET
/metrics triggers the same maybeRefresh call every route does and
the breaker probe's own hit counting is timing-sensitive.
2026-08-23 00:19:09 +01:00

1553 lines
70 KiB
Go

// Command yaegi-check replicates the Traefik Plugin Catalog analyzer
// (piceus)'s yaegiMiddlewareCheck for this plugin. It loads the plugin
// package under Yaegi exactly as the catalog and Traefik's own plugin
// loader do: interpret the source tree, evaluate CreateConfig(), decode
// the plugin's real .traefik.yml testData into the resulting Config
// value, then call the plugin's New constructor by reflection from
// outside the interpreter. It then exercises the returned handler with
// two real HTTP requests, proving ServeHTTP, auth, the model registry,
// statusTrackingWriter, and the gateway's logger all run under Yaegi —
// not merely that the package's imports resolve.
//
// Any failure — a Yaegi-incompatible construct in the plugin's stdlib
// usage, a constructor error, a nil handler, or a wrong response from the
// live handler — exits non-zero: this is the gate the plugin must pass
// before it can list in the Traefik Plugin Catalog.
//
// This module is intentionally separate from the plugin module
// (llmgw-yaegi-check, not github.com/lukaszraczylo/traefik-llmgateway):
// this harness's own dependencies (yaegi itself) must never become the
// plugin's, and nothing here may be reachable from interpreted code.
//
// The plugin module carries exactly ONE non-stdlib runtime dependency,
// github.com/lukaszraczylo/oss-telemetry, which is vendored and therefore
// interpretable — Yaegi resolves it out of vendor/ in the GOPATH copy,
// the same way Traefik resolves it from the cloned plugin tree. That is
// why vendor/ is no longer excluded below. Every OTHER runtime import
// must stay stdlib-only: an unvendored third-party import breaks under
// interpretation even though `go build` never notices, and this harness
// is what catches it.
package main
import (
"bufio"
"bytes"
"context"
"encoding/json"
"fmt"
"io"
"io/fs"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"reflect"
"regexp"
"strconv"
"strings"
"sync/atomic"
"time"
"github.com/traefik/yaegi/interp"
"github.com/traefik/yaegi/stdlib"
)
// testDataUserAPIKey and testDataWantModel are drawn directly from the
// plugin's own .traefik.yml testData block (users.inline[0].apiKey and
// providers.openai.models[0]) — see exerciseHandler.
//
// attemptAccountingAdminAPIKey names the second, admin-flagged user run()
// layers on top of testData (see its own override comment) — SHOULD-4
// (v0.22 review round): this harness proves recordProviderAttempt's
// SHOULD-1 deadline/cancel classification (limits.go's matchesSentinel/
// isDeadlineExceeded) actually runs correctly INTERPRETED, not merely
// compiled: a bug there would not show up under `go test`, only here —
// which is exactly what happened in round 2 (see slowProviderName's own
// doc comment immediately below, and matchesSentinel's in limits.go, for
// the full account of the interpreter-only bug this harness caught).
//
// slowProviderName/slowProviderModel/slowUpstreamSleep/
// slowRequestDeadline back exerciseAttemptAccounting's SECOND request
// (SHOULD-A, v0.22 review round, round 2): the FIRST version of this
// harness only ever drove an upstream that answers instantly, so
// isDeadlineExceeded never actually saw a non-nil error, interpreted or
// otherwise — a comment claiming to prove the interpreted deadline path
// while never exercising it. Once fixed to actually drive a timeout, it
// caught a real bug: round 2's original matchesSentinel (a hand-rolled
// Unwrap walk against two locally declared interfaces, comma-ok matched)
// passed every compiled `go test` row for a double-%w error shape but
// silently misclassified it under the real interpreter — an interpreted
// interface type failing to match a compiled concrete value's method
// set. matchesSentinel now calls real errors.Is instead (limits.go's own
// doc comment has the full account of why that is safe here
// specifically). The second request below: a second, DELIBERATELY slow
// provider, addressed by its own provider-prefixed model id
// (`slowProviderName+"/"+slowProviderModel`, routableModelId's own
// "provider/model" convention) so it can never collide with
// testDataWantModel's bare "gpt-test" on the "openai" provider, driven
// with a request context deadline shorter than the upstream's own
// sleep — the exact shape the route-level Go test regression (routes_
// unified_test.go's TestHandleChat_ContextDeadlineExceeded_
// RecordsProviderFailure) already proves compiled; this proves it
// interpreted.
const (
testDataUserAPIKey = "test-user-key"
testDataWantModel = "gpt-test"
attemptAccountingAdminAPIKey = "sk-admin1"
slowProviderName = "slow"
slowProviderModel = "gpt-test"
slowUpstreamSleep = 600 * time.Millisecond
slowRequestDeadline = 60 * time.Millisecond
// yaegiMetaContextTokens is the modelMeta config-override
// contextTokens value the attempt-accounting harness sets for
// testDataWantModel (feature v0.23) — a string, not an int, since it
// is spliced directly into attemptAccountingOverride's hand-built
// JSON text below.
yaegiMetaContextTokens = "128000"
)
// excludedTopLevelDirs lists repo-root directories the GOPATH copy must
// never include: build tooling, integration fixtures, planning docs and
// VCS metadata have nothing to do with the plugin package Yaegi imports.
//
// vendor/ USED TO be excluded here, on the reasoning that it held only
// the test-only testify dependency and _test.go files are never
// interpreted. That stopped being true when the plugin took on
// oss-telemetry as a runtime dependency: leaving vendor/ out would make
// the interpreted copy unable to resolve an import the real plugin tree
// resolves fine, so this harness would fail on code that works in
// production — or, worse, quietly stop representing it. Copying the whole
// vendor tree, nested "go.yaml.in"-style module dirs included, resolves
// correctly under yaegi v0.16.1 (verified: the gate passes with the
// telemetry import live and reached).
//
// webui/ (Vue admin-panel task) is still excluded: its own go.mod already
// walls it off from the repo root's `go build`/`vet`/`test ./...`, and
// its node_modules tree — hundreds of MB,
// including a stray bundled .go file
// (node_modules/flatted/golang/pkg/flatted.go) — would otherwise both
// balloon copyRepoSource's wall time and risk Yaegi tripping over code
// this repo neither wrote nor can build. The plugin only ever imports the
// generated admin_assets_gen.go at the repo root, never anything under
// webui/ itself.
var excludedTopLevelDirs = map[string]bool{
"tools": true,
"integration": true,
".superpowers": true,
"docs": true,
".git": true,
"webui": true,
}
func main() {
if err := run(); err != nil {
fmt.Fprintln(os.Stderr, "yaegi-check: FAIL:", err)
os.Exit(1)
}
fmt.Println("yaegi-check: OK")
}
// run resolves the plugin's repo root (os.Args[1], defaulting to the
// current directory), builds a temporary GOPATH containing a copy of its
// source, interprets the plugin package under Yaegi, and exercises the
// constructed handler. It returns the first error encountered, wrapped
// with enough context to diagnose which stage failed.
func run() error {
repoRoot := "."
if len(os.Args) > 1 {
repoRoot = os.Args[1]
}
repoRoot, err := filepath.Abs(repoRoot)
if err != nil {
return fmt.Errorf("resolve repo root: %w", err)
}
modulePath, err := readModulePath(filepath.Join(repoRoot, "go.mod"))
if err != nil {
return fmt.Errorf("read module path: %w", err)
}
gopath, err := os.MkdirTemp("", "llmgw-yaegi-check-*")
if err != nil {
return fmt.Errorf("create temp GOPATH: %w", err)
}
defer os.RemoveAll(gopath) //nolint:errcheck // best-effort cleanup of a temp dir
pkgDir := filepath.Join(gopath, "src", modulePath)
if err = copyRepoSource(repoRoot, pkgDir); err != nil {
return fmt.Errorf("copy repo source into GOPATH: %w", err)
}
// Suppress telemetry BEFORE anything is interpreted, and keep it
// suppressed for the whole run. stampReleaseVersion below makes the
// interpreted copy look like a stamped release build so that New's
// telemetry call is actually REACHED under Yaegi; without these env
// vars that would fire a real ping at the public ingest endpoint from
// every CI run and every developer's `make yaegi-check`. Both names
// are set because either one alone suppresses, so a rename upstream
// cannot silently re-enable it.
if err = os.Setenv("DO_NOT_TRACK", "1"); err != nil {
return fmt.Errorf("set DO_NOT_TRACK: %w", err)
}
if err = os.Setenv("OSS_TELEMETRY_DISABLED", "1"); err != nil {
return fmt.Errorf("set OSS_TELEMETRY_DISABLED: %w", err)
}
if err = stampReleaseVersion(pkgDir); err != nil {
return fmt.Errorf("stamp a release version into the interpreted copy: %w", err)
}
pkgName, err := readPackageName(pkgDir)
if err != nil {
return fmt.Errorf("read plugin package name: %w", err)
}
i := interp.New(interp.Options{GoPath: gopath})
if err = i.Use(stdlib.Symbols); err != nil {
return fmt.Errorf("interp.Use(stdlib.Symbols): %w", err)
}
if _, err = i.Eval(`import "` + modulePath + `"`); err != nil {
return fmt.Errorf("import plugin package under yaegi: %w", err)
}
cfgVal, err := i.Eval(pkgName + ".CreateConfig()")
if err != nil {
return fmt.Errorf("call %s.CreateConfig() under yaegi: %w", pkgName, err)
}
if !cfgVal.CanInterface() {
return fmt.Errorf("%s.CreateConfig() result cannot be used via reflect.Value.Interface", pkgName)
}
testData, err := extractTraefikYAMLTestData(filepath.Join(repoRoot, ".traefik.yml"))
if err != nil {
return fmt.Errorf("extract .traefik.yml testData: %w", err)
}
testDataJSON, err := json.Marshal(testData)
if err != nil {
return fmt.Errorf("marshal testData: %w", err)
}
if err = json.Unmarshal(testDataJSON, cfgVal.Interface()); err != nil {
return fmt.Errorf("decode .traefik.yml testData into the interpreted Config: %w", err)
}
// Feature A (v0.22) attempt-accounting harness (SHOULD-4, review
// round): a second json.Unmarshal into the same cfgVal layers this
// override on top of testData's own decode, rather than replacing it
// outright — encoding/json's own map/struct-merge semantics keep
// every testData field this override does not mention (Groups.default
// in particular, which .traefik.yml's own testData.groups block
// already provides and this override never touches). It replaces
// providers.openai wholesale (a real map key, not merged field-by-
// field) with one pointed at a local httptest upstream so the harness
// can drive a real chat completion without a live provider, and adds
// a second, admin-flagged user (Users.Inline is a slice — fully
// replaced, not merged, which is why the "tester" user from testData
// is respecified here too, identical apiKey/group, rather than
// silently dropped).
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusOK)
_, _ = w.Write([]byte(`{"id":"c1","object":"chat.completion","model":"gpt-test","choices":[{"index":0,"message":{"role":"assistant","content":"hi"},"finish_reason":"stop"}],"usage":{"prompt_tokens":1,"completion_tokens":1}}`))
}))
defer upstream.Close()
// slowUpstream never answers within slowRequestDeadline — see
// slowProviderName's own doc comment above (SHOULD-A) for why this
// second provider exists at all.
slowUpstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
time.Sleep(slowUpstreamSleep)
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`{"id":"c2","object":"chat.completion","model":"gpt-test","choices":[{"index":0,"message":{"role":"assistant","content":"hi"},"finish_reason":"stop"}],"usage":{"prompt_tokens":1,"completion_tokens":1}}`))
}))
defer slowUpstream.Close()
// mcpProbeUpstream answers federation's outbound tools/call with a
// body deliberately larger than mcpBackendCallResponseMaxBytes, so
// exerciseHandler's POST /mcp probe below drives doBackendJSONRPC's
// oversize branch — and therefore its errors.Is(err,
// errMCPResponseTooLarge) match — under the REAL interpreter.
//
// This exists because a security review round objected that no gate
// exercised federation's tools/call path interpreted, which left
// errors.Is there justified by reasoning rather than by evidence.
// Reasoning is exactly what the Yaegi traps in this codebase have
// repeatedly defeated, so the path is now covered instead of argued.
mcpProbeUpstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`{"jsonrpc":"2.0","id":1,"result":{"content":[{"type":"text","text":"`))
chunk := bytes.Repeat([]byte("x"), 1<<20)
for written := 0; written <= mcpProbeOversizeBytes; written += len(chunk) {
if _, writeErr := w.Write(chunk); writeErr != nil {
return
}
}
_, _ = w.Write([]byte(`"}]}}`))
}))
defer mcpProbeUpstream.Close()
// anthropicProbeUpstream answers an Anthropic Messages API call with a
// real Anthropic-shaped body, so the /v1/messages passthrough branch
// (callAnthropicMessagesPassthrough) runs interpreted. A body
// containing "make-it-fail" gets a 429, driving
// handleAdapterErrorEnvelope's *providerHTTPError branch interpreted
// too — the exact construct class that has broken under Yaegi before
// (see handleAdapterError's own doc comment, routes_unified.go).
anthropicProbeUpstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
raw, _ := io.ReadAll(r.Body)
if bytes.Contains(raw, []byte("make-it-fail")) {
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusTooManyRequests)
_, _ = w.Write([]byte(`{"type":"error","error":{"type":"rate_limit_error","message":"slow down"}}`))
return
}
if r.Header.Get("anthropic-version") == "" || r.Header.Get("x-api-key") == "" {
w.WriteHeader(http.StatusBadRequest)
_, _ = w.Write([]byte(`{"type":"error","error":{"type":"invalid_request_error","message":"missing anthropic headers"}}`))
return
}
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusOK)
_, _ = w.Write([]byte(`{"id":"msg_probe","type":"message","role":"assistant","model":"claude-upstream-real","content":[{"type":"text","text":"pong"}],"stop_reason":"end_turn","usage":{"input_tokens":7,"output_tokens":3}}`))
}))
defer anthropicProbeUpstream.Close()
// builtinLookupContextTokens (review fix, SHOULD-6): read directly
// out of the generated pricing_data_gen.go rather than hardcoding a
// number, so this assertion self-updates across a `make
// pricing-sync` regeneration instead of pinning a value that could
// legitimately drift when LiteLLM's own upstream data changes.
builtinLookupContextTokens, err := readBuiltinContextTokens(repoRoot, builtinLookupModelID)
if err != nil {
return fmt.Errorf("read builtin context tokens for %q from pricing_data_gen.go: %w", builtinLookupModelID, err)
}
// --- breaker probe (feat/provider-health): a provider whose
// /v1/models 401s, exactly the xiaomi production case, driven under
// the REAL interpreter. brkHealthy flips the upstream from always-401
// to serving real data once the probe wants to observe recovery;
// brkModelsHits counts every /v1/models hit so exerciseBreaker can
// prove the open window actually suppresses discovery traffic (a
// count staying flat), not merely that healthState reads "open".
brkModelsHits := new(int64)
brkHealthy := new(int64)
brkUpstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if strings.HasSuffix(r.URL.Path, "/v1/models") {
atomic.AddInt64(brkModelsHits, 1)
if atomic.LoadInt64(brkHealthy) == 0 {
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusUnauthorized)
_, _ = w.Write([]byte(`{"error":{"message":"invalid api key","type":"invalid_request_error"}}`))
return
}
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`{"object":"list","data":[{"id":"brk-model"}]}`))
return
}
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`{"id":"c3","object":"chat.completion","model":"brk-model","choices":[],"usage":{}}`))
}))
defer brkUpstream.Close()
attemptAccountingOverride := `{"breaker":{"failureThreshold":2,"openDuration":"3s","maxOpenDuration":"6s"},` +
`"providers":{"openai":{"type":"openai","baseUrl":"` + upstream.URL + `","apiKey":"sk-up","models":["` + testDataWantModel + `","` + builtinLookupModelID + `"]},` +
`"brk":{"type":"openai","baseUrl":"` + brkUpstream.URL + `","apiKey":"sk-up","discovery":true,"discoveryInterval":"1ms"},"` +
slowProviderName + `":{"type":"openai","baseUrl":"` + slowUpstream.URL + `","apiKey":"sk-up","models":["` + slowProviderModel + `"]},` +
// "anthropic" backs the /v1/messages passthrough probes
// (exerciseMessagesRoute, below): a real anthropic-type provider,
// interpreted end to end through
// callAnthropicMessagesPassthrough.
`"anthropic":{"type":"anthropic","baseUrl":"` + anthropicProbeUpstream.URL + `","apiKey":"sk-anth","models":["claude-test"]}},` + // #nosec G101 -- test fixture literal, not a real credential
// modelMeta (feature v0.23): a config-override entry for
// testDataWantModel, so exerciseHandler's GET /v1/models
// assertion below proves resolveModelMeta's config-override
// layer, modelObject's context_window/pricing extension fields,
// and the whole registry.go/modelmeta.go resolution path all
// run correctly INTERPRETED, not merely compiled — cheap to add
// to this existing harness request, per this feature's own gate
// requirement. builtinLookupModelID carries NO config override
// at all (SHOULD-6, review round) — its context_window can only
// come from an interpreted LOOKUP into the real, 200+-entry
// builtinModelMetaTable map literal, proving that specific path
// runs correctly under Yaegi too, not just the config-override
// one testDataWantModel already exercises.
`"modelMeta":{"` + testDataWantModel + `":{"contextTokens":` + yaegiMetaContextTokens + `,"inputCostPerMTokMicroUsd":1250000,"outputCostPerMTokMicroUsd":10000000}},` +
`"mcpServers":{"` + mcpProbeServerName + `":{"url":"` + mcpProbeUpstream.URL + `"}},` +
// metrics (Prometheus text-exposition endpoint): enabled with
// modelLabel on, so exerciseMetricsRoute below exercises the
// opt-in per-(provider,model) breakdown under the interpreter
// too, not just the always-on families. allowedCIDRs names the
// exact block httptest.NewRequest's own default RemoteAddr
// (192.0.2.1:1234, verified empirically) falls inside, so the
// CIDR-bypass path is reachable with a plain httptest.NewRequest
// carrying no explicit RemoteAddr override.
`"metrics":{"enabled":true,"allowedCIDRs":["` + metricsProbeAllowedCIDR + `"],"modelLabel":true},` +
`"admin":{"enabled":true},"users":{"inline":[{"name":"tester","group":"default","apiKey":"` + testDataUserAPIKey + `"},{"name":"admin1","group":"default","apiKey":"` + attemptAccountingAdminAPIKey + `","admin":true}]}}`
if err = json.Unmarshal([]byte(attemptAccountingOverride), cfgVal.Interface()); err != nil {
return fmt.Errorf("decode attempt-accounting harness override into the interpreted Config: %w", err)
}
newVal, err := i.Eval(pkgName + ".New")
if err != nil {
return fmt.Errorf("resolve %s.New under yaegi: %w", pkgName, err)
}
if newVal.Kind() != reflect.Func {
return fmt.Errorf("%s.New is not a function (kind %s)", pkgName, newVal.Kind())
}
next := http.HandlerFunc(func(http.ResponseWriter, *http.Request) {})
results := newVal.Call([]reflect.Value{
reflect.ValueOf(context.Background()),
reflect.ValueOf(next),
cfgVal,
reflect.ValueOf("llmgw-check"),
})
if len(results) != 2 {
return fmt.Errorf("%s.New returned %d values, want 2 (http.Handler, error)", pkgName, len(results))
}
if errVal := results[1]; !errVal.IsNil() {
return fmt.Errorf("%s.New returned an error: %v", pkgName, errVal.Interface())
}
handlerVal := results[0]
if handlerVal.IsNil() {
return fmt.Errorf("%s.New returned a nil handler", pkgName)
}
handler, ok := handlerVal.Interface().(http.Handler)
if !ok {
return fmt.Errorf("%s.New's returned value does not implement http.Handler", pkgName)
}
if err := exerciseHandler(handler, builtinLookupContextTokens); err != nil {
return err
}
// exerciseBreaker (feat/provider-health, adversarial-review round 2):
// runs after exerciseHandler, against the SAME interpreted handler —
// proves the discovery circuit breaker's open/half-open/closed state
// machine, its new interpreted breakerState (a named int32 with a
// String() method), and recordHealthLocked's classification call all
// run correctly under Yaegi, not merely compiled. limits.go's own
// matchesSentinel doc comment (Trap-4/reverse-direction history) is
// exactly why this class of divergence needs its own interpreted
// probe: a compiled `go test` pass here would prove nothing about the
// interpreter.
if err := exerciseBreaker(handler, brkModelsHits, brkHealthy); err != nil {
return err
}
// exerciseMetricsRoute runs LAST, after exerciseBreaker has already
// settled: GET /metrics runs registry.maybeRefresh like every other
// route (ServeHTTP's own unconditional call at entry), and
// exerciseBreaker's own hit-counting is timing-sensitive against the
// "brk" provider specifically — probing metrics first could perturb
// the very discovery-attempt counts that test samples. Ordering the
// metrics probe after avoids any such interaction.
return exerciseMetricsRoute(handler)
}
// builtinLookupModelID is a real, stable entry in the generated
// builtinModelMetaTable (pricing_data_gen.go) — review fix, SHOULD-6 —
// added to the harness's openai provider with NO modelMeta config
// override of its own, so its GET /v1/models context_window can only
// ever come from an interpreted lookup into the real map literal, not
// the config-override path testDataWantModel already exercises.
const builtinLookupModelID = "gpt-4o"
// mcpProbeServerName is the federated MCP server run() configures against
// mcpProbeUpstream, and mcpProbeToolName is a tool id carrying its
// "<server>_" prefix so resolveFederatedTool routes POST /mcp's
// tools/call there. The tool needs no tools/list entry — resolution is
// pure prefix matching on the configured, group-allowed server names.
const (
mcpProbeServerName = "probe"
mcpProbeToolName = mcpProbeServerName + "_big"
)
// metricsProbeAllowedCIDR is the CIDR block attemptAccountingOverride's
// own metrics.allowedCIDRs names — see that literal's own comment for why
// this must contain httptest.NewRequest's default RemoteAddr.
const metricsProbeAllowedCIDR = "192.0.2.0/24"
// metricsProbeOutsideAddr is a source address deliberately OUTSIDE
// metricsProbeAllowedCIDR (TEST-NET-3, RFC 5737) — exerciseMetricsRoute's
// own proof that the allowlist is not simply matching everything.
const metricsProbeOutsideAddr = "203.0.113.9:5555"
// mcpProbeOversizeBytes is how much filler mcpProbeUpstream writes: over
// mcpBackendCallResponseMaxBytes (maxRequestBytes, 10MiB) so the response
// trips doBackendJSONRPC's cap. Kept as its own named value rather than
// importing the plugin's const, because this harness deliberately builds
// nothing from the plugin package at compile time — everything it asserts
// about the plugin must come from the interpreter.
const mcpProbeOversizeBytes = 11 << 20
// mcpTooLargeWantMessage is the exact JSON-RPC error message
// mcpFederatedToolsCall writes when errors.Is matches
// errMCPResponseTooLarge. "upstream error" here instead means errors.Is
// returned false under the interpreter for a chain that is true when
// compiled — which is precisely the class of Yaegi divergence this
// harness exists to catch, and which no compiled test can see.
const mcpTooLargeWantMessage = "response too large"
// stampedCheckVersion is the fake release version stampReleaseVersion
// writes into the interpreted copy. It only has to differ from
// devPluginVersion; nothing asserts its value.
const stampedCheckVersion = "9.9.9-yaegicheck"
// stampReleaseVersion rewrites pluginVersion in the GOPATH COPY of
// version.go (never the repo's own file) so the copy no longer carries
// the dev sentinel.
//
// Without this, New's telemetry gate returns before ever calling into the
// vendored oss-telemetry package, and this harness would report OK while
// leaving the plugin's only non-stdlib runtime import unexercised under
// the interpreter — precisely the blind spot yaegi-check exists to close.
// Import resolution alone is not enough: a cross-package call into
// interpreted third-party code is its own risk under Yaegi.
//
// Telemetry stays suppressed by the DO_NOT_TRACK/OSS_TELEMETRY_DISABLED
// env vars the caller sets first, so Send is entered and returns without
// touching the network. The HTTP path beyond that env check is plain
// net/http, already exercised elsewhere in this harness.
func stampReleaseVersion(pkgDir string) error {
path := filepath.Join(pkgDir, "version.go")
data, err := os.ReadFile(path) //nolint:gosec // a temp dir this harness just created
if err != nil {
return fmt.Errorf("read %s: %w", path, err)
}
pattern := regexp.MustCompile(`(pluginVersion\s*=\s*")[^"]*(")`)
if !pattern.Match(data) {
return fmt.Errorf("no pluginVersion constant found in %s — has version.go been renamed?", path)
}
stamped := pattern.ReplaceAll(data, []byte(`${1}`+stampedCheckVersion+`${2}`))
// pkgDir is a temp dir run() created itself, and the filename is a
// constant — no external input reaches this path.
if err := os.WriteFile(path, stamped, 0o600); err != nil { //nolint:gosec // G703: path is filepath.Join(<temp dir we created>, "version.go")
return fmt.Errorf("write %s: %w", path, err)
}
return nil
}
// readBuiltinContextTokens reads repoRoot/pricing_data_gen.go and
// extracts modelID's own ContextTokens value directly out of the
// generated source (review fix, SHOULD-6) — rather than hardcoding a
// number in this harness, which would need hand-updating (and could
// silently drift out of sync) every time `make pricing-sync`
// regenerates the table against updated upstream data.
func readBuiltinContextTokens(repoRoot, modelID string) (int, error) {
data, err := os.ReadFile(filepath.Join(repoRoot, "pricing_data_gen.go"))
if err != nil {
return 0, fmt.Errorf("read pricing_data_gen.go: %w", err)
}
pattern := regexp.MustCompile(regexp.QuoteMeta(`"`+modelID+`":`) + `\s*\{ContextTokens:\s*(\d+)`)
m := pattern.FindSubmatch(data)
if m == nil {
return 0, fmt.Errorf("no builtinModelMetaTable entry found for %q — pick a different, currently-present stable id", modelID)
}
n, err := strconv.Atoi(string(m[1]))
if err != nil {
return 0, fmt.Errorf("parse ContextTokens for %q: %w", modelID, err)
}
if n <= 0 {
return 0, fmt.Errorf("builtinModelMetaTable entry for %q has ContextTokens=%d, want > 0", modelID, n)
}
return n, nil
}
// readHealth returns provider name's healthState string from the admin
// overview, decoded generically like readProviderAttemptCounters does
// (feat/provider-health).
func readHealth(handler http.Handler, name string) (string, error) {
req := httptest.NewRequest(http.MethodGet, "/admin/api/overview", nil)
req.Header.Set("Authorization", "Bearer "+attemptAccountingAdminAPIKey)
rec := httptest.NewRecorder()
handler.ServeHTTP(rec, req)
if rec.Code != http.StatusOK {
return "", fmt.Errorf("GET /admin/api/overview: status = %d, body=%s", rec.Code, rec.Body.String())
}
var body map[string]any
if err := json.Unmarshal(rec.Body.Bytes(), &body); err != nil {
return "", err
}
providers, _ := body["providers"].([]any)
for _, raw := range providers {
p, ok := raw.(map[string]any)
if !ok {
continue
}
if p["name"] == name {
s, _ := p["healthState"].(string)
return s, nil
}
}
return "", fmt.Errorf("provider %q not in overview: %s", name, rec.Body.String())
}
// drive fires n authenticated GET /v1/models requests, each of which runs
// maybeRefresh at ServeHTTP entry, with a small gap so the 1ms discovery
// interval always permits a fresh attempt when the breaker is closed
// (feat/provider-health).
func drive(handler http.Handler, n int) {
for i := 0; i < n; i++ {
req := httptest.NewRequest(http.MethodGet, "/v1/models", nil)
req.Header.Set("Authorization", "Bearer "+testDataUserAPIKey)
handler.ServeHTTP(httptest.NewRecorder(), req)
time.Sleep(20 * time.Millisecond)
}
}
// exerciseBreaker drives the "brk" provider (attemptAccountingOverride,
// run()) through the discovery circuit breaker's full state cycle —
// closed -> open -> half-open -> closed again — against the REAL
// interpreter, proving under Yaegi what registry_health_test.go already
// proves compiled: tryBeginRefresh/finishRefresh/recordHealthLocked, the
// interpreted breakerState named-int32 type and its String() method, and
// the classification call recordHealthLocked makes all behave correctly
// interpreted, not merely compiled (feat/provider-health, adversarial-
// review round 2 — limits.go's matchesSentinel doc comment is exactly why
// this class of divergence needs its own interpreted probe).
func exerciseBreaker(handler http.Handler, hits, healthy *int64) error {
// Phase 1: hammer with the upstream 401ing. The breaker must open.
deadline := time.Now().Add(10 * time.Second)
state := ""
for time.Now().Before(deadline) {
drive(handler, 5)
s, err := readHealth(handler, "brk")
if err != nil {
return err
}
state = s
if s == "open" {
break
}
}
if state != "open" {
return fmt.Errorf("BREAKER-1: healthState = %q after sustained 401s, want %q (upstream /v1/models hits=%d)", state, "open", atomic.LoadInt64(hits))
}
openedAtHits := atomic.LoadInt64(hits)
fmt.Printf("yaegi-check: breaker opened after %d upstream /v1/models hits\n", openedAtHits)
// Phase 2: while open, further traffic must NOT reach the upstream.
// openDuration is 3s; hammer for ~1s and require zero new hits.
drive(handler, 30)
duringOpen := atomic.LoadInt64(hits) - openedAtHits
if duringOpen != 0 {
return fmt.Errorf("BREAKER-2: %d upstream /v1/models hits during the open window, want 0 — backoff did not suppress discovery", duringOpen)
}
fmt.Println("yaegi-check: breaker open window suppressed all discovery hits")
// Phase 3: provider recovers. After the backoff window elapses the
// half-open probe must run and CLOSE the breaker again.
atomic.StoreInt64(healthy, 1)
recovered := false
recoverDeadline := time.Now().Add(20 * time.Second)
for time.Now().Before(recoverDeadline) {
drive(handler, 5)
s, err := readHealth(handler, "brk")
if err != nil {
return err
}
if s == "closed" {
recovered = true
break
}
}
if !recovered {
return fmt.Errorf("BREAKER-3: breaker never returned to closed after the provider recovered (hits=%d)", atomic.LoadInt64(hits))
}
fmt.Println("yaegi-check: recovered provider closed the breaker again")
return nil
}
// exerciseMetricsRoute drives GET /metrics against the interpreted
// handler and proves both its auth gate and its rendering run correctly
// under Yaegi — the metrics feature's own gate requirement: compiled
// tests have repeatedly passed on this codebase while the interpreted
// shape failed (matchesSentinel's own doc comment, limits.go, is the
// canonical example), so a metrics-specific probe is not optional here.
//
// Four requests, mirroring TestMetrics_GateMatrix (metrics_test.go)
// exactly so this harness and the compiled test suite assert the
// identical contract from two different angles:
// 1. unauthenticated, from an address outside metrics.allowedCIDRs -> 401
// 2. authenticated as "tester" (not admin) -> 403
// 3. authenticated as the admin user -> 200, body inspected
// 4. unauthenticated, from an address INSIDE metrics.allowedCIDRs -> 200
//
// Request 3's body is inspected for real, live-data content — not just a
// 200 status — because a Yaegi-specific bug in this codebase's history
// (matchesSentinel, limits.go) was a case where the interpreted code ran
// without panicking yet produced a WRONG answer; a bare status-code check
// would not have caught that class of failure here either. Specifically:
// the "openai" provider's name, and — since attemptAccountingOverride
// enables metrics.modelLabel — testDataWantModel's own id inside the
// opt-in provider-model breakdown, both of which can only appear via a
// live registry.snapshot() walk and a live providerModelScopeID join
// under the real interpreter, not a static string in this harness.
func exerciseMetricsRoute(handler http.Handler) error {
unauthReq := httptest.NewRequest(http.MethodGet, "/metrics", nil)
unauthReq.RemoteAddr = metricsProbeOutsideAddr
rec := httptest.NewRecorder()
handler.ServeHTTP(rec, unauthReq)
if rec.Code != http.StatusUnauthorized {
return fmt.Errorf("GET /metrics (no key, outside allowedCIDRs): status = %d, want 401, body=%s", rec.Code, rec.Body.String())
}
nonAdminReq := httptest.NewRequest(http.MethodGet, "/metrics", nil)
nonAdminReq.RemoteAddr = metricsProbeOutsideAddr
nonAdminReq.Header.Set("Authorization", "Bearer "+testDataUserAPIKey)
rec = httptest.NewRecorder()
handler.ServeHTTP(rec, nonAdminReq)
if rec.Code != http.StatusForbidden {
return fmt.Errorf("GET /metrics (non-admin key): status = %d, want 403, body=%s", rec.Code, rec.Body.String())
}
adminReq := httptest.NewRequest(http.MethodGet, "/metrics", nil)
adminReq.RemoteAddr = metricsProbeOutsideAddr
adminReq.Header.Set("Authorization", "Bearer "+attemptAccountingAdminAPIKey)
rec = httptest.NewRecorder()
handler.ServeHTTP(rec, adminReq)
if rec.Code != http.StatusOK {
return fmt.Errorf("GET /metrics (admin key): status = %d, want 200, body=%s", rec.Code, rec.Body.String())
}
if ct := rec.Header().Get("Content-Type"); !strings.HasPrefix(ct, "text/plain") {
return fmt.Errorf("GET /metrics (admin key): Content-Type = %q, want a text/plain prefix", ct)
}
body := rec.Body.String()
for _, want := range []string{
"# TYPE llmgateway_requests_total counter",
"# TYPE llmgateway_provider_healthy gauge",
`llmgateway_provider_attempts_total{provider="openai"}`,
"# TYPE llmgateway_provider_model_attempts_total counter",
`llmgateway_provider_model_attempts_total{provider="openai",model="` + testDataWantModel + `"}`,
} {
if !strings.Contains(body, want) {
return fmt.Errorf("GET /metrics (admin key): body missing %q — interpreted rendering diverged from the compiled shape; body=%s", want, body)
}
}
allowlistedReq := httptest.NewRequest(http.MethodGet, "/metrics", nil) // default RemoteAddr, inside metricsProbeAllowedCIDR
rec = httptest.NewRecorder()
handler.ServeHTTP(rec, allowlistedReq)
if rec.Code != http.StatusOK {
return fmt.Errorf("GET /metrics (no key, inside allowedCIDRs): status = %d, want 200, body=%s", rec.Code, rec.Body.String())
}
fmt.Println("yaegi-check: GET /metrics auth gate and rendering both verified interpreted")
return nil
}
// exerciseHandler runs two real requests against the interpreted
// handler, drawn from the plugin's own .traefik.yml testData: an
// authenticated GET /v1/models, which must return the configured
// testDataWantModel; and the same route without credentials, which must
// be refused with 401 — proving ServeHTTP, auth.identify, the model
// registry, statusTrackingWriter, and the gateway's logger all execute
// for real under Yaegi, not merely that the package's imports resolve.
// builtinContextTokens is builtinLookupModelID's own real, extracted
// builtinModelMetaTable value (SHOULD-6) — see that constant's own doc
// comment.
func exerciseHandler(handler http.Handler, builtinContextTokens int) error {
authedReq := httptest.NewRequest(http.MethodGet, "/v1/models", nil)
authedReq.Header.Set("Authorization", "Bearer "+testDataUserAPIKey)
authedRec := httptest.NewRecorder()
handler.ServeHTTP(authedRec, authedReq)
if authedRec.Code != http.StatusOK {
return fmt.Errorf("GET /v1/models with the testData user key: status = %d, want 200, body=%s", authedRec.Code, authedRec.Body.String())
}
if !strings.Contains(authedRec.Body.String(), testDataWantModel) {
return fmt.Errorf("GET /v1/models body does not contain %q: %s", testDataWantModel, authedRec.Body.String())
}
// modelMeta (feature v0.23), interpreted, decoded structurally (not
// a raw substring match — SHOULD-6, review round: a bare Contains
// check cannot tell WHICH model entry a given context_window value
// belongs to, which matters once two entries can legitimately share
// the same number) so each assertion below is pinned to its own
// named model id.
var modelsBody struct {
Data []map[string]any `json:"data"`
}
if err := json.Unmarshal(authedRec.Body.Bytes(), &modelsBody); err != nil {
return fmt.Errorf("decode GET /v1/models body: %w (body=%s)", err, authedRec.Body.String())
}
byID := make(map[string]map[string]any, len(modelsBody.Data))
for _, entry := range modelsBody.Data {
if id, ok := entry["id"].(string); ok {
byID[id] = entry
}
}
// run() layers a modelMeta config override for testDataWantModel
// onto cfgVal before New() is ever called (attemptAccountingOverride
// above) — proving resolveModelMeta's config-override layer and
// modelObject's context_window/pricing extension fields both run
// correctly, interpreted, through registry.go's listFor/
// resolveMetaFor.
overridden, ok := byID[testDataWantModel]
if !ok {
return fmt.Errorf("GET /v1/models body has no entry for %q: %s", testDataWantModel, authedRec.Body.String())
}
if cw, _ := overridden["context_window"].(float64); cw != 128000 {
return fmt.Errorf("GET /v1/models %q context_window = %v, want 128000 (feature v0.23 config-override metadata harness): %s", testDataWantModel, overridden["context_window"], authedRec.Body.String())
}
pricing, _ := overridden["pricing"].(map[string]any)
if pricing == nil || pricing["input_per_mtok_usd"] != 1.25 || pricing["output_per_mtok_usd"] != 10.0 {
return fmt.Errorf("GET /v1/models %q pricing = %v, want {input_per_mtok_usd:1.25 output_per_mtok_usd:10} (feature v0.23 config-override metadata harness): %s", testDataWantModel, overridden["pricing"], authedRec.Body.String())
}
// builtinLookupModelID (SHOULD-6, review round) carries NO modelMeta
// config override — its context_window can only come from an
// interpreted LOOKUP into the real, 200+-entry builtinModelMetaTable
// map literal (modelmeta.go's resolveModelMeta, builtin branch),
// proving that specific path — not just the config-override one
// above — runs correctly under Yaegi.
builtinLookedUp, ok := byID[builtinLookupModelID]
if !ok {
return fmt.Errorf("GET /v1/models body has no entry for %q: %s", builtinLookupModelID, authedRec.Body.String())
}
if cw, _ := builtinLookedUp["context_window"].(float64); int(cw) != builtinContextTokens {
return fmt.Errorf("GET /v1/models %q context_window = %v, want %d (feature v0.23 builtin-table-lookup metadata harness, SHOULD-6): %s", builtinLookupModelID, builtinLookedUp["context_window"], builtinContextTokens, authedRec.Body.String())
}
if err := exerciseAttemptAccounting(handler); err != nil {
return err
}
if err := exerciseFederatedTooLarge(handler); err != nil {
return err
}
if err := exerciseStampedVersion(handler); err != nil {
return err
}
if err := exerciseMessagesRoute(handler); err != nil {
return err
}
unauthedReq := httptest.NewRequest(http.MethodGet, "/v1/models", nil)
unauthedRec := httptest.NewRecorder()
handler.ServeHTTP(unauthedRec, unauthedReq)
if unauthedRec.Code != http.StatusUnauthorized {
return fmt.Errorf("GET /v1/models without a key: status = %d, want 401, body=%s", unauthedRec.Code, unauthedRec.Body.String())
}
return nil
}
// exerciseAttemptAccounting drives two real requests through handler —
// Feature A's (v0.22) per-provider success-rate accounting, exercised
// end to end under Yaegi: runUnified's attemptRecorder (routes_unified.
// go) -> retryPolicy.do's context lookup (retry.go) ->
// limiter.recordProviderAttempt's isTransient/isDeadlineExceeded
// classification (limits.go, including matchesSentinel's errors.Is call
// through providers.go's own multi-%w wrapping) -> its fire-and-forget
// spawn (SHOULD-5) -> buildAdminOverview's own batched providerUsage read
// (admin.go). Any interpreter-only failure in that chain — a construct
// `go build`/`go test` cannot catch, exactly the class of bug this
// harness exists for (SHOULD-4, v0.22 review round) — surfaces here as a
// non-1/non-0 count or an outright panic under Yaegi.
//
// The first request (against "openai", instant upstream) proves the
// success path: one attempt, zero failures. The second (against
// slowProviderName, an upstream that never answers within
// slowRequestDeadline — SHOULD-A, round 2) proves the FAILURE path:
// exactly what round 1 of this harness claimed to prove but never
// actually drove — the interpreted plugin never saw a non-nil error at
// all, so isDeadlineExceeded was never exercised under this harness even
// while it ran (and passed) under `go test`. THIS is what SHOULD-A was
// for: the first version of this second request, run against round 2's
// hand-rolled comma-ok Unwrap-walk fix (limits.go's matchesSentinel,
// since replaced), FAILED right here — under the interpreter only,
// despite passing every compiled `go test` row for the identical error
// shape — because an interpreted interface type does not correctly match
// a compiled concrete value's method set in Yaegi. matchesSentinel now
// uses real errors.Is instead (its own doc comment in limits.go has the
// full account); this harness is what caught the difference.
// exerciseStampedVersion asserts the interpreted plugin reports
// stampedCheckVersion in GET /admin/api/overview.
//
// This is the proof that stampReleaseVersion actually landed, and so the
// proof that New's telemetry gate was PASSED rather than short-circuited:
// the gate compares this same pluginVersion constant against
// devPluginVersion, so a version of stampedCheckVersion here means the
// interpreted code went on to call into the vendored oss-telemetry
// package. Without this assertion a silently failed stamp would leave the
// dev sentinel in place and the harness would still print OK, claiming
// coverage it does not have.
func exerciseStampedVersion(handler http.Handler) error {
req := httptest.NewRequest(http.MethodGet, "/admin/api/overview", nil)
req.Header.Set("Authorization", "Bearer "+attemptAccountingAdminAPIKey)
rec := httptest.NewRecorder()
handler.ServeHTTP(rec, req)
if rec.Code != http.StatusOK {
return fmt.Errorf("GET /admin/api/overview: status = %d, want 200, body=%s", rec.Code, rec.Body.String())
}
var overview struct {
Version string `json:"version"`
}
if err := json.Unmarshal(rec.Body.Bytes(), &overview); err != nil {
return fmt.Errorf("decode GET /admin/api/overview body: %w", err)
}
if overview.Version != stampedCheckVersion {
return fmt.Errorf("interpreted plugin reports version %q, want %q — stampReleaseVersion did not take effect, so New's telemetry gate short-circuited on the dev sentinel and the vendored oss-telemetry call was never exercised under Yaegi", overview.Version, stampedCheckVersion)
}
return nil
}
// postMessages drives one POST /v1/messages through handler with the given
// auth header name/value and JSON body, returning the recorder.
func postMessages(handler http.Handler, hdrName, hdrValue, body string) *httptest.ResponseRecorder {
req := httptest.NewRequest(http.MethodPost, "/v1/messages", strings.NewReader(body))
req.Header.Set("Content-Type", "application/json")
req.Header.Set(hdrName, hdrValue)
rec := httptest.NewRecorder()
handler.ServeHTTP(rec, req)
return rec
}
// exerciseMessagesRoute drives the inbound Anthropic Messages API route
// under the REAL interpreter: both auth header forms, the streaming
// rejection, a bad key's error envelope, an unknown model's error
// envelope, the openai-type TRANSLATED path (openAIRequestFromAnthropic
// -> chatCompletion -> anthropicResponseFromOpenAI), the anthropic-type
// PASSTHROUGH path including its own upstream-error branch, and a
// tool_use round trip through the translated path. This exists because
// go test never runs under Yaegi — a construct that compiles and passes
// `go test` can still panic or silently misbehave interpreted (this
// package's own doc comment has the general case; handleAdapterError's
// doc comment, routes_unified.go, has this route's own specific history
// with plain type assertions over errors.As).
func exerciseMessagesRoute(handler http.Handler) error {
// 1. x-api-key auth + translated path (openai-type provider).
rec := postMessages(handler, "x-api-key", testDataUserAPIKey,
`{"model":"`+testDataWantModel+`","max_tokens":64,"system":"be terse","messages":[{"role":"user","content":[{"type":"text","text":"ping"}]}]}`)
if rec.Code != http.StatusOK {
return fmt.Errorf("POST /v1/messages (x-api-key, translated): status = %d, want 200, body=%s", rec.Code, rec.Body.String())
}
var msg map[string]any
if err := json.Unmarshal(rec.Body.Bytes(), &msg); err != nil {
return fmt.Errorf("POST /v1/messages: decode body: %w (body=%s)", err, rec.Body.String())
}
if msg["type"] != "message" || msg["role"] != "assistant" {
return fmt.Errorf("POST /v1/messages translated: type/role = %v/%v, want message/assistant: %s", msg["type"], msg["role"], rec.Body.String())
}
if msg["model"] != testDataWantModel {
return fmt.Errorf("POST /v1/messages translated: model = %v, want %q (the client's requested alias echoed back, item 4 fix): %s", msg["model"], testDataWantModel, rec.Body.String())
}
if msg["stop_reason"] != "end_turn" {
return fmt.Errorf("POST /v1/messages translated: stop_reason = %v, want end_turn: %s", msg["stop_reason"], rec.Body.String())
}
content, _ := msg["content"].([]any)
if len(content) != 1 {
return fmt.Errorf("POST /v1/messages translated: content len = %d, want 1: %s", len(content), rec.Body.String())
}
block, _ := content[0].(map[string]any)
if block["type"] != "text" || block["text"] != "hi" {
return fmt.Errorf("POST /v1/messages translated: content[0] = %v, want text/hi: %s", content[0], rec.Body.String())
}
usg, _ := msg["usage"].(map[string]any)
if usg == nil || usg["input_tokens"] != float64(1) || usg["output_tokens"] != float64(1) {
return fmt.Errorf("POST /v1/messages translated: usage = %v, want input=1 output=1: %s", msg["usage"], rec.Body.String())
}
// 2. Authorization: Bearer must authenticate the same route.
bearerRec := postMessages(handler, "Authorization", "Bearer "+testDataUserAPIKey,
`{"model":"`+testDataWantModel+`","max_tokens":64,"messages":[{"role":"user","content":"ping"}]}`)
if bearerRec.Code != http.StatusOK {
return fmt.Errorf("POST /v1/messages (Authorization: Bearer): status = %d, want 200, body=%s", bearerRec.Code, bearerRec.Body.String())
}
// 3. A bad key gets a 401 in the ANTHROPIC error shape.
badRec := postMessages(handler, "x-api-key", "not-a-real-key",
`{"model":"`+testDataWantModel+`","max_tokens":64,"messages":[{"role":"user","content":"ping"}]}`)
if badRec.Code != http.StatusUnauthorized {
return fmt.Errorf("POST /v1/messages with a bad key: status = %d, want 401, body=%s", badRec.Code, badRec.Body.String())
}
if err := assertAnthropicErrorShape("bad key 401", badRec.Body.Bytes()); err != nil {
return err
}
// 4. "stream": true is rejected explicitly, in the Anthropic shape,
// as a 400 (item 2 fix, 2026-08-22 review) — NOT 501: a 501 makes
// anthropic-sdk-python (which retries any status >= 500) turn every
// streaming call into a three-request retry storm against this
// route's own rate limit. 400 maps to the SDK's non-retryable
// BadRequestError.
streamRec := postMessages(handler, "x-api-key", testDataUserAPIKey,
`{"model":"`+testDataWantModel+`","max_tokens":64,"stream":true,"messages":[{"role":"user","content":"ping"}]}`)
if streamRec.Code != http.StatusBadRequest {
return fmt.Errorf("POST /v1/messages with stream=true: status = %d, want 400, body=%s", streamRec.Code, streamRec.Body.String())
}
if err := assertAnthropicErrorShape("stream rejection", streamRec.Body.Bytes()); err != nil {
return err
}
// 5. An unknown model's 404 must also be Anthropic-shaped.
unknownRec := postMessages(handler, "x-api-key", testDataUserAPIKey,
`{"model":"no-such-model-anywhere","max_tokens":64,"messages":[{"role":"user","content":"ping"}]}`)
if unknownRec.Code != http.StatusNotFound {
return fmt.Errorf("POST /v1/messages with an unknown model: status = %d, want 404, body=%s", unknownRec.Code, unknownRec.Body.String())
}
if err := assertAnthropicErrorShape("unknown model 404", unknownRec.Body.Bytes()); err != nil {
return err
}
// 6. The anthropic-type PASSTHROUGH branch, interpreted: every field
// survives except "model", rewritten to the client's own requested
// alias (item 4 fix) — a routing rewrite, not message-body
// translation.
passRec := postMessages(handler, "x-api-key", testDataUserAPIKey,
`{"model":"claude-test","max_tokens":64,"system":"be terse","messages":[{"role":"user","content":"ping"}]}`)
if passRec.Code != http.StatusOK {
return fmt.Errorf("POST /v1/messages (anthropic passthrough): status = %d, want 200, body=%s", passRec.Code, passRec.Body.String())
}
var pass map[string]any
if err := json.Unmarshal(passRec.Body.Bytes(), &pass); err != nil {
return fmt.Errorf("POST /v1/messages passthrough: decode body: %w (body=%s)", err, passRec.Body.String())
}
if pass["id"] != "msg_probe" {
return fmt.Errorf("POST /v1/messages passthrough: id = %v, want msg_probe (every field but \"model\" must survive the passthrough unchanged): %s", pass["id"], passRec.Body.String())
}
if pass["model"] != "claude-test" {
return fmt.Errorf("POST /v1/messages passthrough: model = %v, want \"claude-test\" (the client's requested alias echoed back, item 4 fix — not anthropicProbeUpstream's own \"claude-upstream-real\"): %s", pass["model"], passRec.Body.String())
}
// 6b. handleAdapterErrorEnvelope's *providerHTTPError type assertion,
// interpreted — the exact construct class that has broken under Yaegi
// before (see handleAdapterError's own doc comment, routes_unified.go).
upErrRec := postMessages(handler, "x-api-key", testDataUserAPIKey,
`{"model":"claude-test","max_tokens":64,"messages":[{"role":"user","content":"make-it-fail"}]}`)
if upErrRec.Code != http.StatusTooManyRequests {
return fmt.Errorf("POST /v1/messages (upstream 429): status = %d, want 429, body=%s", upErrRec.Code, upErrRec.Body.String())
}
if err := assertAnthropicErrorShape("upstream 429", upErrRec.Body.Bytes()); err != nil {
return err
}
// A stale X-Llmgw-Cache header on an error response was deliberately
// NOT asserted here (F6 fix, 2026-08-23 review): this harness's own
// fixture configures no cache/Redis, so g.cache is nil and the
// header is never set on any response regardless of what this route
// does — the assertion could not fail no matter what code ran,
// confirmed by a mutation pass that deleted the header-clear line it
// was meant to guard (handleAdapterErrorEnvelope, routes_unified.go)
// with make yaegi-check still reporting OK. That behavior is already
// covered where it can actually fail — a real cache fixture, go
// test: TestHandleChat_CacheableRequest_UpstreamDown_502HasNoCacheHeader
// (routes_unified_test.go).
// 7. A tool_use round trip through the TRANSLATED path, to drive
// openAIToolCallFromAnthropic/openAIToolMessageFromAnthropic and
// anthropicToolInputFromOpenAI interpreted.
toolRec := postMessages(handler, "x-api-key", testDataUserAPIKey,
`{"model":"`+testDataWantModel+`","max_tokens":64,`+
`"tools":[{"name":"get_weather","description":"w","input_schema":{"type":"object","properties":{"city":{"type":"string"}}}}],`+
`"tool_choice":{"type":"auto"},`+
`"messages":[{"role":"user","content":"weather?"},`+
`{"role":"assistant","content":[{"type":"tool_use","id":"toolu_1","name":"get_weather","input":{"city":"London"}}]},`+
`{"role":"user","content":[{"type":"tool_result","tool_use_id":"toolu_1","content":"sunny"}]}]}`)
if toolRec.Code != http.StatusOK {
return fmt.Errorf("POST /v1/messages (tool_use round trip): status = %d, want 200, body=%s", toolRec.Code, toolRec.Body.String())
}
return nil
}
// assertAnthropicErrorShape checks b is {"type":"error","error":{"type":...,
// "message":...}} — what an Anthropic SDK client parses.
func assertAnthropicErrorShape(label string, b []byte) error {
var env map[string]any
if err := json.Unmarshal(b, &env); err != nil {
return fmt.Errorf("/v1/messages %s: body is not JSON: %w (body=%s)", label, err, string(b))
}
if env["type"] != "error" {
return fmt.Errorf("/v1/messages %s: top-level \"type\" = %v, want \"error\" (Anthropic envelope): %s", label, env["type"], string(b))
}
inner, _ := env["error"].(map[string]any)
if inner == nil {
return fmt.Errorf("/v1/messages %s: no \"error\" object: %s", label, string(b))
}
if _, ok := inner["type"].(string); !ok {
return fmt.Errorf("/v1/messages %s: error.type missing: %s", label, string(b))
}
if _, ok := inner["message"].(string); !ok {
return fmt.Errorf("/v1/messages %s: error.message missing: %s", label, string(b))
}
return nil
}
// exerciseFederatedTooLarge drives POST /mcp (federation's tools/call
// path) against mcpProbeUpstream, whose response deliberately exceeds
// mcpBackendCallResponseMaxBytes, and asserts the interpreted plugin
// answers with mcpTooLargeWantMessage rather than the generic "upstream
// error".
//
// What this actually pins is errors.Is under the interpreter. A security
// review round replaced that errors.Is with a strings.Contains marker on
// the grounds that no gate exercised this path interpreted — true at the
// time, and the honest objection. The reasoning for errors.Is is sound
// (errors.New and fmt.Errorf both return COMPILED values, so declaring
// the sentinel variable in interpreted code leaves the whole chain
// errors.Is walks compiled, which is why retry.go's isTransient has
// worked in production since v0.2.0), and a standalone yaegi v0.16.1
// harness confirmed it. But this codebase's own history — matchesSentinel
// (limits.go) — is a case where sound reasoning about the reflect
// boundary was wrong and only an interpreted probe caught it. So the
// path is covered here rather than argued in a comment.
//
// A failure surfaces as "upstream error": errors.Is returning false
// interpreted for a chain that is true compiled. No `go test` row can
// see that difference.
func exerciseFederatedTooLarge(handler http.Handler) error {
body := `{"jsonrpc":"2.0","id":7,"method":"tools/call","params":{"name":"` + mcpProbeToolName + `","arguments":{}}}`
req := httptest.NewRequest(http.MethodPost, "/mcp", strings.NewReader(body))
req.Header.Set("Authorization", "Bearer "+testDataUserAPIKey)
req.Header.Set("Content-Type", "application/json")
rec := httptest.NewRecorder()
handler.ServeHTTP(rec, req)
if rec.Code != http.StatusOK {
return fmt.Errorf("POST /mcp tools/call: status = %d, want 200 (JSON-RPC reports the failure in the body, not the status), body=%s", rec.Code, rec.Body.String())
}
var resp struct {
Error *struct {
Message string `json:"message"`
} `json:"error"`
}
if err := json.Unmarshal(rec.Body.Bytes(), &resp); err != nil {
return fmt.Errorf("decode POST /mcp tools/call body: %w (body=%s)", err, rec.Body.String())
}
if resp.Error == nil {
return fmt.Errorf("POST /mcp tools/call returned no JSON-RPC error, want %q — the probe upstream's oversized response should have tripped the cap: %s", mcpTooLargeWantMessage, rec.Body.String())
}
if resp.Error.Message != mcpTooLargeWantMessage {
return fmt.Errorf("POST /mcp tools/call error message = %q, want %q — errors.Is(err, errMCPResponseTooLarge) did not match under the interpreter (mcp_federation.go)", resp.Error.Message, mcpTooLargeWantMessage)
}
return nil
}
func exerciseAttemptAccounting(handler http.Handler) error {
chatReq := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(
`{"model":"`+testDataWantModel+`","messages":[{"role":"user","content":"hi"}]}`,
))
chatReq.Header.Set("Authorization", "Bearer "+testDataUserAPIKey)
chatReq.Header.Set("Content-Type", "application/json")
chatRec := httptest.NewRecorder()
handler.ServeHTTP(chatRec, chatReq)
if chatRec.Code != http.StatusOK {
return fmt.Errorf("POST /v1/chat/completions (attempt-accounting harness): status = %d, want 200, body=%s", chatRec.Code, chatRec.Body.String())
}
attemptsDay, failuresDay, err := pollProviderAttemptCounters(handler, "openai")
if err != nil {
return err
}
if attemptsDay != 1 {
return fmt.Errorf(`admin overview provider "openai" attemptsDay = %v, want 1 (Feature A attempt-accounting harness)`, attemptsDay)
}
if failuresDay != 0 {
return fmt.Errorf(`admin overview provider "openai" failuresDay = %v, want 0`, failuresDay)
}
// SHOULD-A: a real, interpreted context-deadline timeout against a
// deliberately slow upstream must record a FAILURE — the whole
// reason this harness exists after round 2's blocker.
slowCtx, cancel := context.WithTimeout(context.Background(), slowRequestDeadline)
defer cancel()
slowReq := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(
`{"model":"`+slowProviderName+`/`+slowProviderModel+`","messages":[{"role":"user","content":"hi"}]}`,
)).WithContext(slowCtx)
slowReq.Header.Set("Authorization", "Bearer "+testDataUserAPIKey)
slowReq.Header.Set("Content-Type", "application/json")
slowRec := httptest.NewRecorder()
handler.ServeHTTP(slowRec, slowReq)
// No status assertion here on purpose: the point of interest is the
// PROVIDER-HEALTH counter a timed-out upstream attempt leaves behind,
// not what status code a canceled/timed-out client request itself
// gets back (which the existing route-level Go tests already pin).
slowAttemptsDay, slowFailuresDay, err := pollProviderAttemptCounters(handler, slowProviderName)
if err != nil {
return err
}
if slowAttemptsDay != 1 {
return fmt.Errorf("admin overview provider %q attemptsDay = %v, want 1 (SHOULD-A timeout harness)", slowProviderName, slowAttemptsDay)
}
if slowFailuresDay != 1 {
return fmt.Errorf("admin overview provider %q failuresDay = %v, want 1 — a real interpreted context-deadline timeout must count as a provider-health failure (SHOULD-A, closes the round-2 blocker)", slowProviderName, slowFailuresDay)
}
return nil
}
// pollProviderAttemptCounters drives GET /admin/api/overview against
// handler repeatedly until providerName's attemptsDay is non-zero or a 2s
// budget elapses, then returns its final reading either way — SHOULD-5
// (v0.22 review round): recordProviderAttempt's store write runs on its
// own goroutine (limiter.spawn), off the request that triggered it, so a
// single immediate read right after that request returns could race
// ahead of the write landing. A real goroutine still schedules promptly
// under Yaegi (only the INTERPRETED code driving it runs slower, not the
// underlying Go runtime's scheduler), so this is a short poll, not a
// long one.
func pollProviderAttemptCounters(handler http.Handler, providerName string) (attemptsDay, failuresDay float64, err error) {
deadline := time.Now().Add(2 * time.Second)
for {
attemptsDay, failuresDay, err = readProviderAttemptCounters(handler, providerName)
if err != nil {
return 0, 0, err
}
if attemptsDay != 0 || time.Now().After(deadline) {
return attemptsDay, failuresDay, nil
}
time.Sleep(10 * time.Millisecond)
}
}
// readProviderAttemptCounters reads GET /admin/api/overview, finds the
// entry named providerName, and returns its attemptsDay/failuresDay
// fields via a generic map[string]any decode — this harness module is
// compiled, not interpreted, so it could import the plugin's own admin.go
// types directly, but they are unexported (adminOverviewResponse,
// adminProviderView); decoding generically here is simpler than exporting
// test-only types across that boundary just for this one harness. Looked
// up by name, not index 0 (round 1 of this harness only ever had one
// configured provider): buildAdminOverview sorts providers by name
// (registry.go), so adding slowProviderName ("slow") after "openai"
// changed which index held which provider.
func readProviderAttemptCounters(handler http.Handler, providerName string) (attemptsDay, failuresDay float64, err error) {
req := httptest.NewRequest(http.MethodGet, "/admin/api/overview", nil)
req.Header.Set("Authorization", "Bearer "+attemptAccountingAdminAPIKey)
rec := httptest.NewRecorder()
handler.ServeHTTP(rec, req)
if rec.Code != http.StatusOK {
return 0, 0, fmt.Errorf("GET /admin/api/overview: status = %d, want 200, body=%s", rec.Code, rec.Body.String())
}
var body map[string]any
if err := json.Unmarshal(rec.Body.Bytes(), &body); err != nil {
return 0, 0, fmt.Errorf("decode GET /admin/api/overview body: %w", err)
}
providers, ok := body["providers"].([]any)
if !ok {
return 0, 0, fmt.Errorf("admin overview body has no providers array: %s", rec.Body.String())
}
for _, raw := range providers {
p, ok := raw.(map[string]any)
if !ok {
continue
}
if name, _ := p["name"].(string); name != providerName {
continue
}
attemptsDay, _ = p["attemptsDay"].(float64)
failuresDay, _ = p["failuresDay"].(float64)
return attemptsDay, failuresDay, nil
}
return 0, 0, fmt.Errorf("admin overview body has no provider named %q: %s", providerName, rec.Body.String())
}
// readModulePath returns the module path declared by goModPath's
// "module" directive.
func readModulePath(goModPath string) (string, error) {
f, err := os.Open(goModPath)
if err != nil {
return "", err
}
defer f.Close() //nolint:errcheck // read-side close; nothing actionable on failure
scanner := bufio.NewScanner(f)
for scanner.Scan() {
line := strings.TrimSpace(scanner.Text())
if after, ok := strings.CutPrefix(line, "module "); ok {
return strings.TrimSpace(after), nil
}
}
if err := scanner.Err(); err != nil {
return "", err
}
return "", fmt.Errorf("no module directive found in %s", goModPath)
}
// readPackageName returns the package name declared by the first
// non-test .go file it finds directly inside dir. The plugin is a single
// root package, so any one of its files' declarations suffices.
func readPackageName(dir string) (string, error) {
entries, err := os.ReadDir(dir)
if err != nil {
return "", err
}
for _, e := range entries {
name := e.Name()
if e.IsDir() || !strings.HasSuffix(name, ".go") || strings.HasSuffix(name, "_test.go") {
continue
}
pkgName, err := scanPackageDecl(filepath.Join(dir, name))
if err != nil {
return "", err
}
if pkgName != "" {
return pkgName, nil
}
}
return "", fmt.Errorf("no package declaration found in %s", dir)
}
// scanPackageDecl reads path's leading "package X" line, if any.
func scanPackageDecl(path string) (string, error) {
f, err := os.Open(path)
if err != nil {
return "", err
}
defer f.Close() //nolint:errcheck // read-side close; nothing actionable on failure
scanner := bufio.NewScanner(f)
for scanner.Scan() {
line := strings.TrimSpace(scanner.Text())
if after, ok := strings.CutPrefix(line, "package "); ok {
return strings.TrimSpace(after), nil
}
}
return "", scanner.Err()
}
// copyRepoSource copies repoRoot into destDir, skipping every directory
// named in excludedTopLevelDirs. destDir plays the role of
// GOPATH/src/{modulePath} for the interpreter: Yaegi's source loader
// resolves the plugin's own import path by walking GOPATH exactly as the
// standard toolchain would for a pre-modules GOPATH-mode build.
func copyRepoSource(repoRoot, destDir string) error {
entries, err := os.ReadDir(repoRoot)
if err != nil {
return err
}
if err := os.MkdirAll(destDir, 0o750); err != nil {
return err
}
for _, e := range entries {
if excludedTopLevelDirs[e.Name()] {
continue
}
src := filepath.Join(repoRoot, e.Name())
dst := filepath.Join(destDir, e.Name())
if e.IsDir() {
if err := copyDir(src, dst); err != nil {
return err
}
continue
}
if err := copyFile(src, dst); err != nil {
return err
}
}
return nil
}
// copyDir recursively copies src to dst.
func copyDir(src, dst string) error {
return filepath.WalkDir(src, func(path string, d fs.DirEntry, err error) error {
if err != nil {
return err
}
rel, err := filepath.Rel(src, path)
if err != nil {
return err
}
target := filepath.Join(dst, rel)
if d.IsDir() {
return os.MkdirAll(target, 0o750)
}
return copyFile(path, target)
})
}
// copyFile copies src to dst, creating dst's parent directory if needed.
func copyFile(src, dst string) error {
in, err := os.Open(src)
if err != nil {
return err
}
defer in.Close() //nolint:errcheck // read-side close; nothing actionable on failure
if err = os.MkdirAll(filepath.Dir(dst), 0o750); err != nil {
return err
}
out, err := os.Create(dst)
if err != nil {
return err
}
defer out.Close() //nolint:errcheck // Close's error is checked explicitly below
if _, err := io.Copy(out, in); err != nil {
return err
}
return out.Close()
}
// yamlLine is one non-blank, non-comment line of a YAML file, with
// leading whitespace stripped and recorded as indent.
type yamlLine struct {
text string
indent int
}
// extractTraefikYAMLTestData reads path (the plugin's .traefik.yml) and
// decodes its top-level "testData:" block into a generic
// map[string]any/[]any/scalar tree, using a small hand-rolled,
// indentation-based YAML subset parser: just enough for this one file's
// shape (nested maps, lists of scalars, lists of flat maps, "{}"/"[]"
// empty collections), not a general YAML implementation. It exists only
// in this harness module — the plugin module never parses YAML itself,
// Traefik does that before handing the plugin its already-decoded Config.
func extractTraefikYAMLTestData(path string) (map[string]any, error) {
raw, err := os.ReadFile(path)
if err != nil {
return nil, err
}
lines := tokenizeYAMLLines(string(raw))
block := extractTestDataBlock(lines)
if block == nil {
return nil, fmt.Errorf("%s has no top-level testData: block", path)
}
pos := 0
v := parseBlock(block, &pos, block[0].indent)
m, ok := v.(map[string]any)
if !ok {
return nil, fmt.Errorf("%s testData: block is not a map", path)
}
return m, nil
}
// tokenizeYAMLLines splits raw into non-blank, non-comment yamlLines,
// each carrying its own leading-space count as indent.
func tokenizeYAMLLines(raw string) []yamlLine {
var out []yamlLine
for _, ln := range strings.Split(raw, "\n") {
trimmedRight := strings.TrimRight(ln, " \t\r")
content := strings.TrimLeft(trimmedRight, " ")
if content == "" || strings.HasPrefix(content, "#") {
continue
}
out = append(out, yamlLine{indent: len(trimmedRight) - len(content), text: content})
}
return out
}
// extractTestDataBlock returns the lines nested under a top-level (indent
// 0) "testData:" line, or nil if lines has none.
func extractTestDataBlock(lines []yamlLine) []yamlLine {
for i, l := range lines {
if l.indent != 0 || l.text != "testData:" {
continue
}
var block []yamlLine
for j := i + 1; j < len(lines) && lines[j].indent > 0; j++ {
block = append(block, lines[j])
}
return block
}
return nil
}
// parseBlock parses the run of lines starting at *pos that share indent,
// dispatching to parseList when the first such line is a "-" item and to
// parseMap otherwise.
func parseBlock(lines []yamlLine, pos *int, indent int) any {
if *pos >= len(lines) || lines[*pos].indent != indent {
return map[string]any{}
}
if strings.HasPrefix(lines[*pos].text, "-") {
return parseList(lines, pos, indent)
}
return parseMap(lines, pos, indent)
}
// parseMap consumes every consecutive "key: value" line at indent,
// recursing into parseBlock for a key whose value is empty (a nested
// block follows on deeper-indented lines).
func parseMap(lines []yamlLine, pos *int, indent int) map[string]any {
m := map[string]any{}
for *pos < len(lines) && lines[*pos].indent == indent {
key, val, _ := strings.Cut(lines[*pos].text, ":")
key = strings.TrimSpace(key)
val = strings.TrimSpace(val)
*pos++
if val != "" {
m[key] = collectionOrScalar(val)
continue
}
if *pos < len(lines) && lines[*pos].indent > indent {
m[key] = parseBlock(lines, pos, lines[*pos].indent)
} else {
m[key] = map[string]any{}
}
}
return m
}
// parseList consumes every consecutive "- ..." line at indent. A "-"
// item containing a colon starts a flat map (its own "key: value" plus
// every following line indented at least indent+2, until a line back at
// indent or shallower); any other item is a bare scalar.
func parseList(lines []yamlLine, pos *int, indent int) []any {
var out []any
for *pos < len(lines) && lines[*pos].indent == indent && strings.HasPrefix(lines[*pos].text, "-") {
item := strings.TrimSpace(strings.TrimPrefix(lines[*pos].text, "-"))
if item == "" {
*pos++
continue
}
if !strings.Contains(item, ":") {
out = append(out, parseScalar(item))
*pos++
continue
}
m := map[string]any{}
key, val, _ := strings.Cut(item, ":")
m[strings.TrimSpace(key)] = collectionOrScalar(strings.TrimSpace(val))
*pos++
childIndent := indent + 2
for *pos < len(lines) && lines[*pos].indent >= childIndent {
k, v, _ := strings.Cut(lines[*pos].text, ":")
m[strings.TrimSpace(k)] = collectionOrScalar(strings.TrimSpace(v))
*pos++
}
out = append(out, m)
}
return out
}
// collectionOrScalar maps a "key: value" line's already-trimmed val to
// an empty map/slice for "{}"/"[]", or to parseScalar(val) otherwise. It
// does not recurse into a further nested block — no key in this file's
// testData needs a map/list value inside a list item's own fields, and
// this harness parses only what .traefik.yml actually contains.
func collectionOrScalar(val string) any {
switch val {
case "{}":
return map[string]any{}
case "[]":
return []any{}
default:
return parseScalar(val)
}
}
// parseScalar converts a YAML scalar token to a bool, int64, float64, or
// string, in that preference order, stripping a matching pair of quotes
// first.
func parseScalar(s string) any {
if len(s) >= 2 {
if (s[0] == '"' && s[len(s)-1] == '"') || (s[0] == '\'' && s[len(s)-1] == '\'') {
return s[1 : len(s)-1]
}
}
switch s {
case "true":
return true
case "false":
return false
}
if n, err := strconv.ParseInt(s, 10, 64); err == nil {
return n
}
if f, err := strconv.ParseFloat(s, 64); err == nil {
return f
}
return s
}