mirror of
https://github.com/lukaszraczylo/traefik-llmgateway.git
synced 2026-10-12 04:54:45 +00:00
A provider whose discovery failed during the warm fill at pod start went without discovered models for a full discovery interval, which is 1 hour by default. It also stayed that way until a request arrived after the interval. Production logs showed this for openai-audio and anthropic when outbound calls were slow at startup. A provider that fails the warm fill now gets one background retry after 10s. The retry has its own timeout and uses the same refresh bookkeeping. It is skipped when a refresh is already in flight or has finished since the warm fill, and it is not scheduled when the parent context was canceled. A recovery logs one INFO line. If the retry also fails, the existing breaker behaviour applies unchanged. The retry delay and the warm-fill timeout can be set per registry for tests. yaegi-check probes the retry under the interpreter. Claude-Session: https://claude.ai/code/session_01V5jYJNxpDFdJkyyExpADrv
3997 lines
203 KiB
Go
3997 lines
203 KiB
Go
// Command yaegi-check replicates the Traefik Plugin Catalog analyzer
|
|
// (piceus)'s yaegiMiddlewareCheck for this plugin. It loads the plugin
|
|
// package under Yaegi exactly as the catalog and Traefik's own plugin
|
|
// loader do: interpret the source tree, evaluate CreateConfig(), decode
|
|
// the plugin's real .traefik.yml testData into the resulting Config
|
|
// value, then call the plugin's New constructor by reflection from
|
|
// outside the interpreter. It then exercises the returned handler with
|
|
// two real HTTP requests, proving ServeHTTP, auth, the model registry,
|
|
// statusTrackingWriter, and the gateway's logger all run under Yaegi —
|
|
// not merely that the package's imports resolve.
|
|
//
|
|
// Any failure — a Yaegi-incompatible construct in the plugin's stdlib
|
|
// usage, a constructor error, a nil handler, or a wrong response from the
|
|
// live handler — exits non-zero: this is the gate the plugin must pass
|
|
// before it can list in the Traefik Plugin Catalog.
|
|
//
|
|
// This module is intentionally separate from the plugin module
|
|
// (llmgw-yaegi-check, not github.com/lukaszraczylo/traefik-llmgateway):
|
|
// this harness's own dependencies (yaegi itself) must never become the
|
|
// plugin's, and nothing here may be reachable from interpreted code.
|
|
//
|
|
// The plugin module carries exactly ONE non-stdlib runtime dependency,
|
|
// github.com/lukaszraczylo/oss-telemetry, which is vendored and therefore
|
|
// interpretable — Yaegi resolves it out of vendor/ in the GOPATH copy,
|
|
// the same way Traefik resolves it from the cloned plugin tree. That is
|
|
// why vendor/ is no longer excluded below. Every OTHER runtime import
|
|
// must stay stdlib-only: an unvendored third-party import breaks under
|
|
// interpretation even though `go build` never notices, and this harness
|
|
// is what catches it.
|
|
package main
|
|
|
|
import (
|
|
"bufio"
|
|
"bytes"
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"io"
|
|
"io/fs"
|
|
"math"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"net/url"
|
|
"os"
|
|
"path/filepath"
|
|
"reflect"
|
|
"regexp"
|
|
"strconv"
|
|
"strings"
|
|
"sync/atomic"
|
|
"time"
|
|
|
|
dto "github.com/prometheus/client_model/go"
|
|
"github.com/prometheus/common/expfmt"
|
|
"github.com/prometheus/common/model"
|
|
"github.com/traefik/yaegi/interp"
|
|
"github.com/traefik/yaegi/stdlib"
|
|
)
|
|
|
|
// testDataUserAPIKey and testDataWantModel are drawn directly from the
|
|
// plugin's own .traefik.yml testData block (users.inline[0].apiKey and
|
|
// providers.openai.models[0]) — see exerciseHandler.
|
|
//
|
|
// attemptAccountingAdminAPIKey names the second, admin-flagged user run()
|
|
// layers on top of testData (see its own override comment) — SHOULD-4
|
|
// (v0.22 review round): this harness proves recordProviderAttempt's
|
|
// SHOULD-1 deadline/cancel classification (limits.go's matchesSentinel/
|
|
// isDeadlineExceeded) actually runs correctly INTERPRETED, not merely
|
|
// compiled: a bug there would not show up under `go test`, only here —
|
|
// which is exactly what happened in round 2 (see slowProviderName's own
|
|
// doc comment immediately below, and matchesSentinel's in limits.go, for
|
|
// the full account of the interpreter-only bug this harness caught).
|
|
//
|
|
// slowProviderName/slowProviderModel/slowUpstreamSleep/
|
|
// slowRequestDeadline back exerciseAttemptAccounting's SECOND request
|
|
// (SHOULD-A, v0.22 review round, round 2): the FIRST version of this
|
|
// harness only ever drove an upstream that answers instantly, so
|
|
// isDeadlineExceeded never actually saw a non-nil error, interpreted or
|
|
// otherwise — a comment claiming to prove the interpreted deadline path
|
|
// while never exercising it. Once fixed to actually drive a timeout, it
|
|
// caught a real bug: round 2's original matchesSentinel (a hand-rolled
|
|
// Unwrap walk against two locally declared interfaces, comma-ok matched)
|
|
// passed every compiled `go test` row for a double-%w error shape but
|
|
// silently misclassified it under the real interpreter — an interpreted
|
|
// interface type failing to match a compiled concrete value's method
|
|
// set. matchesSentinel now calls real errors.Is instead (limits.go's own
|
|
// doc comment has the full account of why that is safe here
|
|
// specifically). The second request below: a second, DELIBERATELY slow
|
|
// provider, addressed by its own provider-prefixed model id
|
|
// (`slowProviderName+"/"+slowProviderModel`, routableModelId's own
|
|
// "provider/model" convention) so it can never collide with
|
|
// testDataWantModel's bare "gpt-test" on the "openai" provider, driven
|
|
// with a request context deadline shorter than the upstream's own
|
|
// sleep — the exact shape the route-level Go test regression (routes_
|
|
// unified_test.go's TestHandleChat_ContextDeadlineExceeded_
|
|
// RecordsProviderFailure) already proves compiled; this proves it
|
|
// interpreted.
|
|
const (
|
|
testDataUserAPIKey = "test-user-key"
|
|
testDataWantModel = "gpt-test"
|
|
attemptAccountingAdminAPIKey = "sk-admin1"
|
|
slowProviderName = "slow"
|
|
slowProviderModel = "gpt-test"
|
|
slowUpstreamSleep = 600 * time.Millisecond
|
|
slowRequestDeadline = 60 * time.Millisecond
|
|
// yaegiMetaContextTokens is the modelMeta config-override
|
|
// contextTokens value the attempt-accounting harness sets for
|
|
// testDataWantModel (feature v0.23) — a string, not an int, since it
|
|
// is spliced directly into attemptAccountingOverride's hand-built
|
|
// JSON text below.
|
|
yaegiMetaContextTokens = "128000"
|
|
)
|
|
|
|
// timeoutProviderName/timeoutProviderModel/timeoutProviderRequestTimeout/
|
|
// timeoutUpstreamSleep/timeoutProbeMaxWait back exerciseRequestTimeout
|
|
// (feature: request timeout): a provider whose own ProviderConfig.
|
|
// RequestTimeout ("80ms") is far shorter than timeoutUpstream's
|
|
// deliberate 600ms silence before it ever writes a byte. Distinct from
|
|
// slowProviderName/slowRequestDeadline above (SHOULD-A): that probe
|
|
// proves a CLIENT-set context deadline is classified correctly;
|
|
// timeoutProviderName proves the GATEWAY'S OWN adapter-level timeout
|
|
// (newAdapterHTTPClient's Transport.ResponseHeaderTimeout, providers.go;
|
|
// watchdogBody, timeout.go) aborts a hang under a request that sets NO
|
|
// deadline of its own — the shape of a real, unbounded production
|
|
// client — interpreted, not merely compiled.
|
|
const (
|
|
timeoutProviderName = "hungtimeout"
|
|
// timeoutProviderModel is deliberately NOT "gpt-test"
|
|
// (testDataWantModel): a bare model id shared across providers
|
|
// resolves to exactly one winner (modelRegistry's own bare-id rule),
|
|
// and reusing testDataWantModel here made "hungtimeout" win it
|
|
// instead of "openai" — silently hijacking exerciseAttemptAccounting's
|
|
// own bare-"gpt-test" request onto this hung provider and failing
|
|
// that unrelated probe. A unique id keeps this provider's own model
|
|
// space from ever colliding with another probe's.
|
|
timeoutProviderModel = "gpt-timeout-test"
|
|
timeoutProviderRequestTimeout = "80ms"
|
|
timeoutUpstreamSleep = 600 * time.Millisecond
|
|
// timeoutProbeMaxWait bounds exerciseRequestTimeout's own assertion:
|
|
// generous relative to timeoutProviderRequestTimeout (80ms), but far
|
|
// under timeoutUpstreamSleep (600ms) — a pass proves the request was
|
|
// aborted BY the timeout feature, not by timeoutUpstream eventually
|
|
// answering on its own.
|
|
timeoutProbeMaxWait = 3 * time.Second
|
|
)
|
|
|
|
// hungBodyProviderName/hungBodyProviderModel/hungBodyProviderRequestTimeout/
|
|
// hungBodyUpstreamSleep back exerciseRequestTimeoutMidBodyStall (finding
|
|
// F7, coordinator adversarial review, 2026-08-23): timeoutProviderName
|
|
// above never sends headers at all, so it only exercises Transport.
|
|
// ResponseHeaderTimeout — watchdogBody.fire (timeout.go) — the
|
|
// interpreted method value handed to time.AfterFunc, plus sync/atomic's
|
|
// function API and a context.CancelFunc — never runs under that probe.
|
|
// hungBodyUpstream sends headers immediately, then goes silent, so the
|
|
// body-read path (upstreamBytes' watchdogBody wrap, providers.go) is
|
|
// what has to abort it, proving the OTHER half of this feature
|
|
// interpreted.
|
|
const (
|
|
hungBodyProviderName = "hungbody"
|
|
// hungBodyProviderModel: same reasoning as timeoutProviderModel's own
|
|
// doc comment above — must not collide with testDataWantModel or any
|
|
// other probe's model id.
|
|
hungBodyProviderModel = "gpt-bodystall-test"
|
|
hungBodyProviderRequestTimeout = "80ms"
|
|
hungBodyUpstreamSleep = 600 * time.Millisecond
|
|
)
|
|
|
|
// provenanceEstProviderName/provenanceEstProviderModel back
|
|
// exerciseUsageProvenance (feat: expose token-accounting provenance): a
|
|
// provider whose upstream answers 200 with NO "usage" field at all —
|
|
// decodes to chatUsagePayload's zero value (provider_openai.go's
|
|
// forwardJSON), the exact shape runUnified's own estimation fallback
|
|
// (routes_unified.go) classifies as "estimated" — proving the
|
|
// substitution math (ceil(len(body)/4)) and the new
|
|
// llmgateway_usage_provenance_requests_total/-_tokens_total families
|
|
// (metrics.go) all run correctly under the REAL interpreter, not merely
|
|
// compiled. "reported" is already exercised, under the interpreter, by
|
|
// exerciseAttemptAccounting's own real, usage-bearing "openai" traffic
|
|
// above — no separate fixture needed for that provenance here.
|
|
const (
|
|
provenanceEstProviderName = "provenance-est"
|
|
// provenanceEstProviderModel: same reasoning as timeoutProviderModel's
|
|
// own doc comment above — must not collide with testDataWantModel or
|
|
// any other probe's model id.
|
|
provenanceEstProviderModel = "gpt-provenance-est-test"
|
|
)
|
|
|
|
// excludedTopLevelDirs lists repo-root directories the GOPATH copy must
|
|
// never include: build tooling, integration fixtures, planning docs and
|
|
// VCS metadata have nothing to do with the plugin package Yaegi imports.
|
|
//
|
|
// vendor/ USED TO be excluded here, on the reasoning that it held only
|
|
// the test-only testify dependency and _test.go files are never
|
|
// interpreted. That stopped being true when the plugin took on
|
|
// oss-telemetry as a runtime dependency: leaving vendor/ out would make
|
|
// the interpreted copy unable to resolve an import the real plugin tree
|
|
// resolves fine, so this harness would fail on code that works in
|
|
// production — or, worse, quietly stop representing it. Copying the whole
|
|
// vendor tree, nested "go.yaml.in"-style module dirs included, resolves
|
|
// correctly under yaegi v0.16.1 (verified: the gate passes with the
|
|
// telemetry import live and reached).
|
|
//
|
|
// webui/ (Vue admin-panel task) is still excluded: its own go.mod already
|
|
// walls it off from the repo root's `go build`/`vet`/`test ./...`, and
|
|
// its node_modules tree — hundreds of MB,
|
|
// including a stray bundled .go file
|
|
// (node_modules/flatted/golang/pkg/flatted.go) — would otherwise both
|
|
// balloon copyRepoSource's wall time and risk Yaegi tripping over code
|
|
// this repo neither wrote nor can build. The plugin only ever imports the
|
|
// generated admin_assets_gen.go at the repo root, never anything under
|
|
// webui/ itself.
|
|
var excludedTopLevelDirs = map[string]bool{
|
|
"tools": true,
|
|
"integration": true,
|
|
".superpowers": true,
|
|
"docs": true,
|
|
".git": true,
|
|
"webui": true,
|
|
}
|
|
|
|
func main() {
|
|
if err := run(); err != nil {
|
|
fmt.Fprintln(os.Stderr, "yaegi-check: FAIL:", err)
|
|
os.Exit(1)
|
|
}
|
|
fmt.Println("yaegi-check: OK")
|
|
}
|
|
|
|
// run resolves the plugin's repo root (os.Args[1], defaulting to the
|
|
// current directory), builds a temporary GOPATH containing a copy of its
|
|
// source, interprets the plugin package under Yaegi, and exercises the
|
|
// constructed handler. It returns the first error encountered, wrapped
|
|
// with enough context to diagnose which stage failed.
|
|
func run() error {
|
|
repoRoot := "."
|
|
if len(os.Args) > 1 {
|
|
repoRoot = os.Args[1]
|
|
}
|
|
repoRoot, err := filepath.Abs(repoRoot)
|
|
if err != nil {
|
|
return fmt.Errorf("resolve repo root: %w", err)
|
|
}
|
|
|
|
modulePath, err := readModulePath(filepath.Join(repoRoot, "go.mod"))
|
|
if err != nil {
|
|
return fmt.Errorf("read module path: %w", err)
|
|
}
|
|
|
|
gopath, err := os.MkdirTemp("", "llmgw-yaegi-check-*")
|
|
if err != nil {
|
|
return fmt.Errorf("create temp GOPATH: %w", err)
|
|
}
|
|
defer os.RemoveAll(gopath) //nolint:errcheck // best-effort cleanup of a temp dir
|
|
|
|
pkgDir := filepath.Join(gopath, "src", modulePath)
|
|
if err = copyRepoSource(repoRoot, pkgDir); err != nil {
|
|
return fmt.Errorf("copy repo source into GOPATH: %w", err)
|
|
}
|
|
|
|
// Suppress telemetry BEFORE anything is interpreted, and keep it
|
|
// suppressed for the whole run. stampReleaseVersion below makes the
|
|
// interpreted copy look like a stamped release build so that New's
|
|
// telemetry call is actually REACHED under Yaegi; without these env
|
|
// vars that would fire a real ping at the public ingest endpoint from
|
|
// every CI run and every developer's `make yaegi-check`. Both names
|
|
// are set because either one alone suppresses, so a rename upstream
|
|
// cannot silently re-enable it.
|
|
if err = os.Setenv("DO_NOT_TRACK", "1"); err != nil {
|
|
return fmt.Errorf("set DO_NOT_TRACK: %w", err)
|
|
}
|
|
if err = os.Setenv("OSS_TELEMETRY_DISABLED", "1"); err != nil {
|
|
return fmt.Errorf("set OSS_TELEMETRY_DISABLED: %w", err)
|
|
}
|
|
if err = stampReleaseVersion(pkgDir); err != nil {
|
|
return fmt.Errorf("stamp a release version into the interpreted copy: %w", err)
|
|
}
|
|
|
|
pkgName, err := readPackageName(pkgDir)
|
|
if err != nil {
|
|
return fmt.Errorf("read plugin package name: %w", err)
|
|
}
|
|
|
|
i := interp.New(interp.Options{GoPath: gopath})
|
|
if err = i.Use(stdlib.Symbols); err != nil {
|
|
return fmt.Errorf("interp.Use(stdlib.Symbols): %w", err)
|
|
}
|
|
if _, err = i.Eval(`import "` + modulePath + `"`); err != nil {
|
|
return fmt.Errorf("import plugin package under yaegi: %w", err)
|
|
}
|
|
|
|
cfgVal, err := i.Eval(pkgName + ".CreateConfig()")
|
|
if err != nil {
|
|
return fmt.Errorf("call %s.CreateConfig() under yaegi: %w", pkgName, err)
|
|
}
|
|
if !cfgVal.CanInterface() {
|
|
return fmt.Errorf("%s.CreateConfig() result cannot be used via reflect.Value.Interface", pkgName)
|
|
}
|
|
|
|
testData, err := extractTraefikYAMLTestData(filepath.Join(repoRoot, ".traefik.yml"))
|
|
if err != nil {
|
|
return fmt.Errorf("extract .traefik.yml testData: %w", err)
|
|
}
|
|
testDataJSON, err := json.Marshal(testData)
|
|
if err != nil {
|
|
return fmt.Errorf("marshal testData: %w", err)
|
|
}
|
|
if err = json.Unmarshal(testDataJSON, cfgVal.Interface()); err != nil {
|
|
return fmt.Errorf("decode .traefik.yml testData into the interpreted Config: %w", err)
|
|
}
|
|
|
|
// Feature A (v0.22) attempt-accounting harness (SHOULD-4, review
|
|
// round): a second json.Unmarshal into the same cfgVal layers this
|
|
// override on top of testData's own decode, rather than replacing it
|
|
// outright — encoding/json's own map/struct-merge semantics keep
|
|
// every testData field this override does not mention (Groups.default
|
|
// in particular, which .traefik.yml's own testData.groups block
|
|
// already provides and this override never touches). It replaces
|
|
// providers.openai wholesale (a real map key, not merged field-by-
|
|
// field) with one pointed at a local httptest upstream so the harness
|
|
// can drive a real chat completion without a live provider, and adds
|
|
// a second, admin-flagged user (Users.Inline is a slice — fully
|
|
// replaced, not merged, which is why the "tester" user from testData
|
|
// is respecified here too, identical apiKey/group, rather than
|
|
// silently dropped).
|
|
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
w.Header().Set("Content-Type", "application/json")
|
|
w.WriteHeader(http.StatusOK)
|
|
_, _ = w.Write([]byte(`{"id":"c1","object":"chat.completion","model":"gpt-test","choices":[{"index":0,"message":{"role":"assistant","content":"hi"},"finish_reason":"stop"}],"usage":{"prompt_tokens":1,"completion_tokens":1}}`))
|
|
}))
|
|
defer upstream.Close()
|
|
|
|
// slowUpstream never answers within slowRequestDeadline — see
|
|
// slowProviderName's own doc comment above (SHOULD-A) for why this
|
|
// second provider exists at all.
|
|
slowUpstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
time.Sleep(slowUpstreamSleep)
|
|
w.Header().Set("Content-Type", "application/json")
|
|
_, _ = w.Write([]byte(`{"id":"c2","object":"chat.completion","model":"gpt-test","choices":[{"index":0,"message":{"role":"assistant","content":"hi"},"finish_reason":"stop"}],"usage":{"prompt_tokens":1,"completion_tokens":1}}`))
|
|
}))
|
|
defer slowUpstream.Close()
|
|
|
|
// timeoutUpstream writes nothing at all for timeoutUpstreamSleep,
|
|
// well past timeoutProviderName's own 80ms ProviderConfig.
|
|
// RequestTimeout (attemptAccountingOverride below) — see the
|
|
// timeoutProviderName const block's own doc comment for why this
|
|
// probe exists alongside slowUpstream.
|
|
timeoutUpstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
time.Sleep(timeoutUpstreamSleep)
|
|
w.Header().Set("Content-Type", "application/json")
|
|
_, _ = w.Write([]byte(`{"id":"c4","object":"chat.completion","model":"gpt-test","choices":[{"index":0,"message":{"role":"assistant","content":"hi"},"finish_reason":"stop"}],"usage":{"prompt_tokens":1,"completion_tokens":1}}`))
|
|
}))
|
|
defer timeoutUpstream.Close()
|
|
|
|
// hungBodyUpstream sends headers immediately (so ResponseHeaderTimeout
|
|
// is satisfied and never fires), then goes silent for
|
|
// hungBodyUpstreamSleep before ever writing a body byte — well past
|
|
// hungBodyProviderName's own 80ms ProviderConfig.RequestTimeout. See
|
|
// the hungBodyProviderName const block's own doc comment (finding F7)
|
|
// for why this probe exists alongside timeoutUpstream, not instead of
|
|
// it.
|
|
hungBodyUpstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
w.Header().Set("Content-Type", "application/json")
|
|
w.WriteHeader(http.StatusOK)
|
|
if fl, ok := w.(http.Flusher); ok {
|
|
fl.Flush()
|
|
}
|
|
time.Sleep(hungBodyUpstreamSleep)
|
|
_, _ = w.Write([]byte(`{"id":"c5","object":"chat.completion","model":"gpt-test","choices":[{"index":0,"message":{"role":"assistant","content":"hi"},"finish_reason":"stop"}],"usage":{"prompt_tokens":1,"completion_tokens":1}}`))
|
|
}))
|
|
defer hungBodyUpstream.Close()
|
|
|
|
// provenanceEstUpstream backs exerciseUsageProvenance (feat: expose
|
|
// token-accounting provenance): answers 200 with NO "usage" field at
|
|
// all — see provenanceEstProviderName's own const block doc comment
|
|
// for why this is the exact shape runUnified's estimation fallback
|
|
// classifies as "estimated".
|
|
provenanceEstUpstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
w.Header().Set("Content-Type", "application/json")
|
|
w.WriteHeader(http.StatusOK)
|
|
_, _ = w.Write([]byte(`{"id":"c-prov","object":"chat.completion","model":"` + provenanceEstProviderModel + `","choices":[{"index":0,"message":{"role":"assistant","content":"hi"},"finish_reason":"stop"}]}`))
|
|
}))
|
|
defer provenanceEstUpstream.Close()
|
|
|
|
// eventsUpstreamErrorUpstream always 500s — see
|
|
// eventsUpstreamErrorProviderName's own doc comment for why this
|
|
// dedicated, single-candidate provider exists rather than reusing
|
|
// failoverAUpstream (which a correctly working failover loop never
|
|
// surfaces to the client, so its own 500 never reaches
|
|
// handleAdapterErrorEnvelope's client-facing write).
|
|
eventsUpstreamErrorUpstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
w.Header().Set("Content-Type", "application/json")
|
|
w.WriteHeader(http.StatusInternalServerError)
|
|
_, _ = w.Write([]byte(`{"error":{"message":"eventsprobe-upstream always fails","type":"server_error"}}`))
|
|
}))
|
|
defer eventsUpstreamErrorUpstream.Close()
|
|
|
|
// mcpProbeUpstream answers federation's outbound tools/call with a
|
|
// body deliberately larger than mcpBackendCallResponseMaxBytes, so
|
|
// exerciseHandler's POST /mcp probe below drives doBackendJSONRPC's
|
|
// oversize branch — and therefore its errors.Is(err,
|
|
// errMCPResponseTooLarge) match — under the REAL interpreter.
|
|
//
|
|
// This exists because a security review round objected that no gate
|
|
// exercised federation's tools/call path interpreted, which left
|
|
// errors.Is there justified by reasoning rather than by evidence.
|
|
// Reasoning is exactly what the Yaegi traps in this codebase have
|
|
// repeatedly defeated, so the path is now covered instead of argued.
|
|
mcpProbeUpstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
w.Header().Set("Content-Type", "application/json")
|
|
_, _ = w.Write([]byte(`{"jsonrpc":"2.0","id":1,"result":{"content":[{"type":"text","text":"`))
|
|
chunk := bytes.Repeat([]byte("x"), 1<<20)
|
|
for written := 0; written <= mcpProbeOversizeBytes; written += len(chunk) {
|
|
if _, writeErr := w.Write(chunk); writeErr != nil {
|
|
return
|
|
}
|
|
}
|
|
_, _ = w.Write([]byte(`"}]}}`))
|
|
}))
|
|
defer mcpProbeUpstream.Close()
|
|
// targetHealthDownUpstream is opened only to capture a real, unique
|
|
// "http://127.0.0.1:PORT" URL, then immediately closed — feat/
|
|
// target-health's own exerciseTargetHealth (below) drives real 502s
|
|
// through this dead target's per-server proxy path
|
|
// (/mcp/target-health-down/...) to prove the interpreted tracker
|
|
// flips it to "unhealthy" after targetHealthFailureThreshold
|
|
// consecutive failures, without disturbing mcpProbeUpstream's own
|
|
// (successful, federation-routed) traffic above.
|
|
targetHealthDownUpstream := httptest.NewServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) {}))
|
|
targetHealthDownURL := targetHealthDownUpstream.URL
|
|
targetHealthDownUpstream.Close()
|
|
// targetHealthProbeOnlyUpstream answers any JSON-RPC call — in
|
|
// practice only ever "initialize", from the active probe sweep
|
|
// (target_health.go's probeMCPTarget) — with a small, valid result
|
|
// well under mcpBackendResponseMaxBytes (4MiB), so the probe's
|
|
// handshake genuinely SUCCEEDS. targetHealthProbeOnlyServerName's own
|
|
// doc comment explains why this server backs a dedicated,
|
|
// never-proxied target rather than reusing mcpProbeUpstream (whose
|
|
// 11MiB body exceeds every response-size cap this plugin enforces,
|
|
// probe included, so it can only ever record a failure).
|
|
targetHealthProbeOnlyUpstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
w.Header().Set("Content-Type", "application/json")
|
|
_, _ = w.Write([]byte(`{"jsonrpc":"2.0","id":"federated","result":{"protocolVersion":"2025-11-25","capabilities":{}}}`))
|
|
}))
|
|
defer targetHealthProbeOnlyUpstream.Close()
|
|
// anthropicProbeUpstream answers an Anthropic Messages API call with a
|
|
// real Anthropic-shaped body, so the /v1/messages passthrough branch
|
|
// (callAnthropicMessagesPassthrough) runs interpreted. A body
|
|
// containing "make-it-fail" gets a 429, driving
|
|
// handleAdapterErrorEnvelope's *providerHTTPError branch interpreted
|
|
// too — the exact construct class that has broken under Yaegi before
|
|
// (see handleAdapterError's own doc comment, routes_unified.go).
|
|
anthropicProbeUpstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
raw, _ := io.ReadAll(r.Body)
|
|
if bytes.Contains(raw, []byte("make-it-fail")) {
|
|
w.Header().Set("Content-Type", "application/json")
|
|
w.WriteHeader(http.StatusTooManyRequests)
|
|
_, _ = w.Write([]byte(`{"type":"error","error":{"type":"rate_limit_error","message":"slow down"}}`))
|
|
return
|
|
}
|
|
if r.Header.Get("anthropic-version") == "" || r.Header.Get("x-api-key") == "" {
|
|
w.WriteHeader(http.StatusBadRequest)
|
|
_, _ = w.Write([]byte(`{"type":"error","error":{"type":"invalid_request_error","message":"missing anthropic headers"}}`))
|
|
return
|
|
}
|
|
w.Header().Set("Content-Type", "application/json")
|
|
w.WriteHeader(http.StatusOK)
|
|
_, _ = w.Write([]byte(`{"id":"msg_probe","type":"message","role":"assistant","model":"claude-upstream-real","content":[{"type":"text","text":"pong"}],"stop_reason":"end_turn","usage":{"input_tokens":7,"output_tokens":3}}`))
|
|
}))
|
|
defer anthropicProbeUpstream.Close()
|
|
|
|
// builtinLookupContextTokens (review fix, SHOULD-6): read directly
|
|
// out of the generated pricing_data_gen.go rather than hardcoding a
|
|
// number, so this assertion self-updates across a `make
|
|
// pricing-sync` regeneration instead of pinning a value that could
|
|
// legitimately drift when LiteLLM's own upstream data changes.
|
|
builtinLookupContextTokens, err := readBuiltinContextTokens(repoRoot, builtinLookupModelID)
|
|
if err != nil {
|
|
return fmt.Errorf("read builtin context tokens for %q from pricing_data_gen.go: %w", builtinLookupModelID, err)
|
|
}
|
|
|
|
// --- breaker probe (feat/provider-health): a provider whose
|
|
// /v1/models 401s, exactly the xiaomi production case, driven under
|
|
// the REAL interpreter. brkHealthy flips the upstream from always-401
|
|
// to serving real data once the probe wants to observe recovery;
|
|
// brkModelsHits counts every /v1/models hit so exerciseBreaker can
|
|
// prove the open window actually suppresses discovery traffic (a
|
|
// count staying flat), not merely that healthState reads "open".
|
|
brkModelsHits := new(int64)
|
|
brkHealthy := new(int64)
|
|
brkUpstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
if strings.HasSuffix(r.URL.Path, "/v1/models") {
|
|
atomic.AddInt64(brkModelsHits, 1)
|
|
if atomic.LoadInt64(brkHealthy) == 0 {
|
|
w.Header().Set("Content-Type", "application/json")
|
|
w.WriteHeader(http.StatusUnauthorized)
|
|
_, _ = w.Write([]byte(`{"error":{"message":"invalid api key","type":"invalid_request_error"}}`))
|
|
return
|
|
}
|
|
w.Header().Set("Content-Type", "application/json")
|
|
_, _ = w.Write([]byte(`{"object":"list","data":[{"id":"brk-model"}]}`))
|
|
return
|
|
}
|
|
w.Header().Set("Content-Type", "application/json")
|
|
_, _ = w.Write([]byte(`{"id":"c3","object":"chat.completion","model":"brk-model","choices":[],"usage":{}}`))
|
|
}))
|
|
defer brkUpstream.Close()
|
|
|
|
// warmRetryUpstream backs exerciseWarmFillRetry (production incident
|
|
// fix, registry.go's warmFillRetryDelay): 401s on the FIRST /v1/models
|
|
// hit only — the synchronous warmFill call inside New() itself — then
|
|
// serves real data on every hit after, so the scheduled background
|
|
// retry (and ONLY the retry) succeeds. Failing fast rather than
|
|
// sleeping past warmFillTimeout keeps this probe's cost down to
|
|
// roughly warmFillRetryDelay, not warmFillTimeout+warmFillRetryDelay.
|
|
warmRetryHits := new(int64)
|
|
warmRetryUpstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
if strings.HasSuffix(r.URL.Path, "/v1/models") {
|
|
if atomic.AddInt64(warmRetryHits, 1) == 1 {
|
|
w.Header().Set("Content-Type", "application/json")
|
|
w.WriteHeader(http.StatusUnauthorized)
|
|
_, _ = w.Write([]byte(`{"error":{"message":"cold-start timeout","type":"invalid_request_error"}}`))
|
|
return
|
|
}
|
|
w.Header().Set("Content-Type", "application/json")
|
|
_, _ = w.Write([]byte(`{"object":"list","data":[{"id":"warmretry-model"}]}`))
|
|
return
|
|
}
|
|
w.Header().Set("Content-Type", "application/json")
|
|
_, _ = w.Write([]byte(`{"id":"c-warmretry","object":"chat.completion","model":"warmretry-model","choices":[],"usage":{}}`))
|
|
}))
|
|
defer warmRetryUpstream.Close()
|
|
|
|
// failoverAUpstream/failoverBUpstream back exerciseFailover (feat/
|
|
// failover, below): two providers configured with the IDENTICAL bare
|
|
// model id (failoverModelID) — the exact shape registry.go's bareWinner
|
|
// collision handles, and the shape production logs show for whisper-1/
|
|
// tts-1/tts-1-hd/the embeddings models across openai/openai-audio/
|
|
// copilot. A always answers 500; B always answers 200. This proves the
|
|
// failover candidate loop (runMeteredCall, routes_unified.go) falls
|
|
// through and serves correctly under the REAL interpreter, not merely
|
|
// compiled — this package's own doc comment is exactly why that
|
|
// distinction matters on this codebase.
|
|
failoverAUpstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
w.Header().Set("Content-Type", "application/json")
|
|
w.WriteHeader(http.StatusInternalServerError)
|
|
_, _ = w.Write([]byte(`{"error":{"message":"failover-a is down","type":"server_error"}}`))
|
|
}))
|
|
defer failoverAUpstream.Close()
|
|
failoverBHits := new(int64)
|
|
failoverBUpstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
atomic.AddInt64(failoverBHits, 1)
|
|
w.Header().Set("Content-Type", "application/json")
|
|
w.WriteHeader(http.StatusOK)
|
|
_, _ = w.Write([]byte(`{"id":"c-failover","object":"chat.completion","model":"` + failoverModelID + `","choices":[{"index":0,"message":{"role":"assistant","content":"served by failover-b"},"finish_reason":"stop"}],"usage":{"prompt_tokens":1,"completion_tokens":1}}`))
|
|
}))
|
|
defer failoverBUpstream.Close()
|
|
|
|
// bareWinnerAUpstream/bareWinnerBUpstream back exerciseBareWinnerGroupAware
|
|
// (fix/group-aware-bare-winner, 2026-08-27): two providers configured
|
|
// with the IDENTICAL bare model id (bareWinnerModelID) — the same
|
|
// collision shape failoverAUpstream/failoverBUpstream above drive, but
|
|
// exercising the bareWinner AUTHORIZATION fix, not failover. A must
|
|
// NEVER be hit: bareWinnerGroupName can only use "bare-winner-b", so a
|
|
// correct group-aware bareWinner never even considers "bare-winner-a"
|
|
// as a candidate for this group's request, unlike a failover fall-
|
|
// through (which WOULD hit A first). bareWinnerAHits catches a
|
|
// regression back to the pre-fix, group-blind selection, which would
|
|
// route here and get an error response instead of the expected 200.
|
|
bareWinnerAHits := new(int64)
|
|
bareWinnerAUpstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
atomic.AddInt64(bareWinnerAHits, 1)
|
|
w.Header().Set("Content-Type", "application/json")
|
|
w.WriteHeader(http.StatusInternalServerError)
|
|
_, _ = w.Write([]byte(`{"error":{"message":"bare-winner-a must never be reached by bareWinnerGroupName","type":"server_error"}}`))
|
|
}))
|
|
defer bareWinnerAUpstream.Close()
|
|
bareWinnerBUpstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
w.Header().Set("Content-Type", "application/json")
|
|
w.WriteHeader(http.StatusOK)
|
|
_, _ = w.Write([]byte(`{"id":"c-bare-winner","object":"chat.completion","model":"` + bareWinnerModelID + `","choices":[{"index":0,"message":{"role":"assistant","content":"served by bare-winner-b"},"finish_reason":"stop"}],"usage":{"prompt_tokens":1,"completion_tokens":1}}`))
|
|
}))
|
|
defer bareWinnerBUpstream.Close()
|
|
|
|
// multiGroupBUpstream backs exerciseMultiGroupPersonalGrant (multi-
|
|
// group/personal-grant feature, UserConfig.Groups/Providers/Models,
|
|
// llmgateway.go): the ONLY grant that reaches "multi-group-b" at all
|
|
// is the harness user's own PERSONAL grant (run()'s "users"
|
|
// override), restricted to multiGroupAllowedModelID alone — neither
|
|
// of the user's two member groups (multiGroupEngGroupName/
|
|
// multiGroupOpsGroupName) names "multi-group-b" in their own
|
|
// providers list. A CORRECT per-grant check must still deny
|
|
// multiGroupDeniedModelID here; a naive "any grant allows the
|
|
// provider (union) AND any grant allows the model (union)" check
|
|
// would instead WRONGLY ALLOW it — neither member group restricts its
|
|
// own models at all, so the models-side union collapses to "allow
|
|
// anything," and pairing that with the personal grant's own
|
|
// providers-side reach to "multi-group-b" authorizes a combination
|
|
// neither grant alone ever granted — see this const block's own doc
|
|
// comments below for the full shape.
|
|
multiGroupBHits := new(int64)
|
|
multiGroupBUpstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
atomic.AddInt64(multiGroupBHits, 1)
|
|
w.Header().Set("Content-Type", "application/json")
|
|
w.WriteHeader(http.StatusOK)
|
|
_, _ = w.Write([]byte(`{"id":"c-multi-group","object":"chat.completion","model":"` + multiGroupAllowedModelID + `","choices":[{"index":0,"message":{"role":"assistant","content":"served by multi-group-b"},"finish_reason":"stop"}],"usage":{"prompt_tokens":1,"completion_tokens":1}}`))
|
|
}))
|
|
defer multiGroupBUpstream.Close()
|
|
|
|
// multiGroupCUpstream/multiGroupSharedXUpstream back
|
|
// exerciseMultiGroupBareWinnerFailoverAndPassthroughPaths (LOW-8,
|
|
// review round 2 — extends the multi-group/personal-grant probe
|
|
// above to also cover HIGH-1's provider-coupled passthroughPaths fix
|
|
// and HIGH-2's per-grant bareWinner/failover fix under the REAL
|
|
// interpreter). Both serve multiGroupSharedModelID as a BARE id
|
|
// alongside "multi-group-b" (all three configured below), the exact
|
|
// three-owner collision shape HIGH-2's own compiled regression test
|
|
// (TestModelRegistry_Resolve_MultiGrant_BareWinnerPrefersGrantThatActuallyAuthorizes)
|
|
// drives, here reachable only through the harness user's THREE
|
|
// distinct grants (multiGroupEngGroupName, multiGroupOpsGroupName,
|
|
// and the personal grant) — sorted provider-name order is
|
|
// "multi-group-b" < "multi-group-c" < "multi-group-shared-x", and
|
|
// only the personal grant (restricted to multiGroupAllowedModelID
|
|
// alone) reaches "multi-group-b", the sorted-first owner, so a
|
|
// pre-fix bareWinner picks it and wrongly denies the whole request
|
|
// even though multiGroupOpsGroupName's own unrestricted grant on
|
|
// "multi-group-c" would happily serve it.
|
|
//
|
|
// multiGroupCUpstream also DOUBLES as the primary-failure trigger for
|
|
// the failover half of that same assertion: it always 500s a chat-
|
|
// completions call (path-detected below) specifically so
|
|
// runMeteredCall's own failover loop falls through past the correctly
|
|
// -chosen primary ("multi-group-c") to the next candidate
|
|
// ("multi-group-shared-x", reachable via multiGroupEngGroupName's own
|
|
// unrestricted grant) — while any OTHER path it receives (a native
|
|
// passthrough request) succeeds normally, backing the HIGH-1
|
|
// passthroughPaths assertion multiGroupOpsGroupName's own
|
|
// "passthroughPaths" override (below) restricts.
|
|
multiGroupCChatHits := new(int64)
|
|
multiGroupCPassthroughHits := new(int64)
|
|
multiGroupCUpstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
if r.URL.Path == "/v1/chat/completions" {
|
|
atomic.AddInt64(multiGroupCChatHits, 1)
|
|
w.Header().Set("Content-Type", "application/json")
|
|
w.WriteHeader(http.StatusInternalServerError)
|
|
_, _ = w.Write([]byte(`{"error":{"message":"multi-group-c chat completions always fails, forcing failover","type":"server_error"}}`))
|
|
return
|
|
}
|
|
atomic.AddInt64(multiGroupCPassthroughHits, 1)
|
|
w.WriteHeader(http.StatusOK)
|
|
}))
|
|
defer multiGroupCUpstream.Close()
|
|
multiGroupSharedXHits := new(int64)
|
|
multiGroupSharedXUpstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
if r.URL.Path == "/v1/chat/completions" {
|
|
atomic.AddInt64(multiGroupSharedXHits, 1)
|
|
w.Header().Set("Content-Type", "application/json")
|
|
w.WriteHeader(http.StatusOK)
|
|
_, _ = w.Write([]byte(`{"id":"c-multi-group-shared-x","object":"chat.completion","model":"` + multiGroupSharedModelID + `","choices":[{"index":0,"message":{"role":"assistant","content":"served by multi-group-shared-x"},"finish_reason":"stop"}],"usage":{"prompt_tokens":1,"completion_tokens":1}}`))
|
|
return
|
|
}
|
|
w.WriteHeader(http.StatusOK)
|
|
}))
|
|
defer multiGroupSharedXUpstream.Close()
|
|
|
|
// round3SharedUpstream backs exerciseRound3ModelAndPathFromSameGrant
|
|
// (HIGH fix, review round 3): a native passthrough request never
|
|
// carries a translated "/v1/chat/completions" path the way the
|
|
// chat-completions route does (handlePassthrough forwards rest
|
|
// verbatim), so this upstream only ever needs to record that it was
|
|
// reached at all — both P1 and P2 (this probe's own doc comment) must
|
|
// deny before ever calling it.
|
|
round3PassthroughHits := new(int64)
|
|
round3PassthroughUpstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
atomic.AddInt64(round3PassthroughHits, 1)
|
|
w.WriteHeader(http.StatusOK)
|
|
}))
|
|
defer round3PassthroughUpstream.Close()
|
|
|
|
// failover.enabled must be set explicitly here: FailoverConfig.Enabled
|
|
// defaults to false (coordinator ruling — a version upgrade with no
|
|
// config change must preserve prior behavior), so without this the
|
|
// exerciseFailover probe below would find failover-a's 500 returned
|
|
// to the client unchanged, never reaching failover-b at all.
|
|
|
|
// metricsProbeTrickyUserNameJSON: a user name containing a double
|
|
// quote, a backslash, and a newline — auth.go's buildEntry validates
|
|
// a name as non-empty only, never restricted to a URL-path-segment
|
|
// character set (see its own doc comment), so this is a real,
|
|
// legal config, not a contrived one. json.Marshal, not hand-built
|
|
// string concatenation, produces the correctly JSON-escaped literal
|
|
// to splice into attemptAccountingOverride below — this is JSON
|
|
// escaping for the CONFIG document, a separate concern from the
|
|
// Prometheus label escaping exerciseMetricsRoute actually verifies
|
|
// in the RENDERED /metrics body.
|
|
trickyNameJSON, err := json.Marshal(metricsProbeTrickyUserName)
|
|
if err != nil {
|
|
return fmt.Errorf("marshal metrics escaping probe user name: %w", err)
|
|
}
|
|
|
|
attemptAccountingOverride := `{"failover":{"enabled":true},"breaker":{"failureThreshold":2,"openDuration":"3s","maxOpenDuration":"6s"},` +
|
|
`"providers":{"openai":{"type":"openai","baseUrl":"` + upstream.URL + `","apiKey":"sk-up","models":["` + testDataWantModel + `","` + builtinLookupModelID + `"]},` +
|
|
`"brk":{"type":"openai","baseUrl":"` + brkUpstream.URL + `","apiKey":"sk-up","discovery":true,"discoveryInterval":"1ms"},` +
|
|
// "warmretry" backs exerciseWarmFillRetry (production incident fix,
|
|
// registry.go's warmFillRetryDelay) — deliberately NO
|
|
// discoveryInterval override: the default (1h) keeps maybeRefresh's
|
|
// own request-driven interval gate from ever firing again during
|
|
// this probe, once warmFill's failed synchronous attempt (or the
|
|
// retry that follows it) has set lastRefresh, so the retry is the
|
|
// ONLY thing that can produce warmRetryUpstream's second hit.
|
|
`"warmretry":{"type":"openai","baseUrl":"` + warmRetryUpstream.URL + `","apiKey":"sk-up","discovery":true},"` +
|
|
slowProviderName + `":{"type":"openai","baseUrl":"` + slowUpstream.URL + `","apiKey":"sk-up","models":["` + slowProviderModel + `"]},` +
|
|
// timeoutProviderName (feature: request timeout) — see its own
|
|
// const block doc comment above for why this provider exists
|
|
// alongside slowProviderName.
|
|
`"` + timeoutProviderName + `":{"type":"openai","baseUrl":"` + timeoutUpstream.URL + `","apiKey":"sk-up","requestTimeout":"` + timeoutProviderRequestTimeout + `","models":["` + timeoutProviderModel + `"]},` +
|
|
// hungBodyProviderName (finding F7) — see its own const block doc
|
|
// comment above for why this provider exists alongside
|
|
// timeoutProviderName.
|
|
`"` + hungBodyProviderName + `":{"type":"openai","baseUrl":"` + hungBodyUpstream.URL + `","apiKey":"sk-up","requestTimeout":"` + hungBodyProviderRequestTimeout + `","models":["` + hungBodyProviderModel + `"]},` +
|
|
// provenanceEstProviderName (feat: expose token-accounting
|
|
// provenance) — see its own const block doc comment above.
|
|
`"` + provenanceEstProviderName + `":{"type":"openai","baseUrl":"` + provenanceEstUpstream.URL + `","apiKey":"sk-up","models":["` + provenanceEstProviderModel + `"]},` +
|
|
// eventsUpstreamErrorProviderName (F3 hook 2, coordinator request)
|
|
// — see its own const block doc comment above for why this
|
|
// provider carries no failover candidate.
|
|
`"` + eventsUpstreamErrorProviderName + `":{"type":"openai","baseUrl":"` + eventsUpstreamErrorUpstream.URL + `","apiKey":"sk-up","models":["` + eventsUpstreamErrorProviderModel + `"]},` +
|
|
// "anthropic" backs the /v1/messages passthrough probes
|
|
// (exerciseMessagesRoute, below): a real anthropic-type provider,
|
|
// interpreted end to end through
|
|
// callAnthropicMessagesPassthrough.
|
|
`"anthropic":{"type":"anthropic","baseUrl":"` + anthropicProbeUpstream.URL + `","apiKey":"sk-anth","models":["claude-test"]},` + // #nosec G101 -- test fixture literal, not a real credential
|
|
`"failover-a":{"type":"openai","baseUrl":"` + failoverAUpstream.URL + `","apiKey":"sk-up","models":["` + failoverModelID + `"]},` +
|
|
`"failover-b":{"type":"openai","baseUrl":"` + failoverBUpstream.URL + `","apiKey":"sk-up","models":["` + failoverModelID + `"]},` +
|
|
// bareWinnerModelID's own const doc comment (above) has the full
|
|
// account of why "bare-winner-a"/"bare-winner-b" exist alongside
|
|
// failover-a/failover-b: same collision shape, different fix under
|
|
// test.
|
|
`"bare-winner-a":{"type":"openai","baseUrl":"` + bareWinnerAUpstream.URL + `","apiKey":"sk-up","models":["` + bareWinnerModelID + `"]},` +
|
|
`"bare-winner-b":{"type":"openai","baseUrl":"` + bareWinnerBUpstream.URL + `","apiKey":"sk-up","models":["` + bareWinnerModelID + `"]},` +
|
|
// "multi-group-b" backs exerciseMultiGroupPersonalGrant — see
|
|
// multiGroupBUpstream's own doc comment above. Its models list
|
|
// ALSO includes multiGroupSharedModelID (LOW-8 extension) so it
|
|
// participates as the (wrongly-preferred, pre-fix) sorted-first
|
|
// owner in the bareWinner/failover collision
|
|
// exerciseMultiGroupBareWinnerFailoverAndPassthroughPaths drives
|
|
// — see multiGroupCUpstream's own doc comment above for the full
|
|
// three-provider shape.
|
|
`"multi-group-b":{"type":"openai","baseUrl":"` + multiGroupBUpstream.URL + `","apiKey":"sk-up","models":["only","other","` + multiGroupSharedModelID + `"]},` +
|
|
`"multi-group-c":{"type":"openai","baseUrl":"` + multiGroupCUpstream.URL + `","apiKey":"sk-up","models":["` + multiGroupSharedModelID + `"]},` +
|
|
`"multi-group-shared-x":{"type":"openai","baseUrl":"` + multiGroupSharedXUpstream.URL + `","apiKey":"sk-up","models":["` + multiGroupSharedModelID + `"]},` +
|
|
// "round3-shared" backs exerciseRound3ModelAndPathFromSameGrant
|
|
// (HIGH fix, review round 3) — a single real provider both P1's
|
|
// and P2's own groups reach, native passthrough only (no Models
|
|
// list needed: passthrough authorization is purely group-driven,
|
|
// never consults a provider's own configured Models).
|
|
`"round3-shared":{"type":"openai","baseUrl":"` + round3PassthroughUpstream.URL + `","apiKey":"sk-up"}},` +
|
|
// groups: ADDS bareWinnerGroupName and the two exerciseMultiGroupPersonalGrant
|
|
// member groups to the map Unmarshal already populated from
|
|
// .traefik.yml's own testData.groups.default — map keys merge
|
|
// (run()'s own doc comment, above, on why "brk" and the other new
|
|
// provider keys above are additive rather than replacing), so
|
|
// "default" (providers: [], allow-all) is untouched. Neither
|
|
// multiGroupEngGroupName nor multiGroupOpsGroupName names
|
|
// "multi-group-b" — only the harness user's own PERSONAL grant
|
|
// (below) does — so a correct implementation reaches "multi-group-b"
|
|
// through that personal grant alone, not through either group.
|
|
// multiGroupOpsGroupName's own "passthroughPaths" restricts ITS
|
|
// OWN provider ("multi-group-c") to multiGroupAllowedPathID alone
|
|
// (LOW-8/HIGH-1 extension) — multiGroupEngGroupName carries no
|
|
// such restriction on its own provider ("multi-group-shared-x"),
|
|
// and the fix under test (auth.go's
|
|
// allowsPassthroughPathForProvider) must keep the two from
|
|
// leaking into each other.
|
|
// round3P1AGroupName/round3P1BGroupName (P1) and
|
|
// round3P2HomeGroupName/round3P2FriendsGroupName (P2) — see
|
|
// exerciseRound3ModelAndPathFromSameGrant's own doc comment for the
|
|
// full shape each pair reproduces.
|
|
`"groups":{"` + bareWinnerGroupName + `":{"providers":["bare-winner-b"]},` +
|
|
`"` + multiGroupEngGroupName + `":{"providers":["multi-group-shared-x"]},` +
|
|
`"` + multiGroupOpsGroupName + `":{"providers":["multi-group-c"],"passthroughPaths":["` + multiGroupAllowedPathID + `"]},` +
|
|
`"` + round3P1AGroupName + `":{"providers":["round3-shared"],"models":["` + round3ModelID + `"]},` +
|
|
`"` + round3P1BGroupName + `":{"providers":["round3-shared"],"passthroughPaths":["v1/chat/completions"]},` +
|
|
`"` + round3P2HomeGroupName + `":{"providers":["round3-alpha"]},` +
|
|
`"` + round3P2FriendsGroupName + `":{"providers":["round3-shared"],"passthroughPaths":["v1/chat/completions"]}},` +
|
|
// modelMeta (feature v0.23): a config-override entry for
|
|
// testDataWantModel, so exerciseHandler's GET /v1/models
|
|
// assertion below proves resolveModelMeta's config-override
|
|
// layer, modelObject's context_window/pricing extension fields,
|
|
// and the whole registry.go/modelmeta.go resolution path all
|
|
// run correctly INTERPRETED, not merely compiled — cheap to add
|
|
// to this existing harness request, per this feature's own gate
|
|
// requirement. builtinLookupModelID carries NO config override
|
|
// at all (SHOULD-6, review round) — its context_window can only
|
|
// come from an interpreted LOOKUP into the real, 200+-entry
|
|
// builtinModelMetaTable map literal, proving that specific path
|
|
// runs correctly under Yaegi too, not just the config-override
|
|
// one testDataWantModel already exercises.
|
|
// provenanceEstProviderModel additionally carries a modelMeta
|
|
// free:true override (free-models feature) — it has no
|
|
// configured price at all (unifiedCostMicros already bills it 0
|
|
// via the "unknown price, warn" path, exerciseUsageProvenance's
|
|
// own assertions above never check its cost value, so this adds
|
|
// no regression there), so free:true here is the ONLY reason
|
|
// adminUsageModelFree (admin.go) marks it free: exactly the
|
|
// modelMeta path exerciseFreeModelUsageDetail (below) exists to
|
|
// exercise interpreted, as opposed to the OTHER free path (a
|
|
// KNOWN price of exactly 0), which carries no fixture of its own
|
|
// here — both are already covered compiled (admin_test.go).
|
|
`"modelMeta":{"` + testDataWantModel + `":{"contextTokens":` + yaegiMetaContextTokens + `,"inputCostPerMTokMicroUsd":1250000,"outputCostPerMTokMicroUsd":10000000},"` + provenanceEstProviderModel + `":{"free":true}},` +
|
|
`"mcpServers":{"` + mcpProbeServerName + `":{"url":"` + mcpProbeUpstream.URL + `"},"` + targetHealthDownServerName + `":{"url":"` + targetHealthDownURL + `"},"` + targetHealthProbeOnlyServerName + `":{"url":"` + targetHealthProbeOnlyUpstream.URL + `"}},` +
|
|
// targetHealth (feat/target-health): enabled. probeInterval is
|
|
// deliberately well ABOVE the floor (10s): exerciseTargetHealthProbeWarmup
|
|
// (below) spends this gateway's ONE sweep for the whole
|
|
// probeInterval window right after construction, before any other
|
|
// exercise runs, specifically so no LATER exercise's own GET
|
|
// /admin/api/targets, /metrics, /v1/mcp/servers, or /v1/agents
|
|
// call can trigger a second sweep mid-run and race a passive-
|
|
// traffic assertion (F5, feat/target-health review — see
|
|
// exerciseTargetHealth's own doc comment for the exact race this
|
|
// avoids). "60s" comfortably outlasts this whole harness's
|
|
// real-world runtime.
|
|
`"targetHealth":{"enabled":true,"probeInterval":"60s"},` +
|
|
// metrics (Prometheus text-exposition endpoint): enabled with
|
|
// modelLabel on, so exerciseMetricsRoute below exercises the
|
|
// opt-in per-(provider,model) breakdown under the interpreter
|
|
// too, not just the always-on families. allowedCIDRs names the
|
|
// exact block httptest.NewRequest's own default RemoteAddr
|
|
// (192.0.2.1:1234, verified empirically) falls inside, so the
|
|
// CIDR-bypass path is reachable with a plain httptest.NewRequest
|
|
// carrying no explicit RemoteAddr override.
|
|
`"metrics":{"enabled":true,"allowedCIDRs":["` + metricsProbeAllowedCIDR + `"],"modelLabel":true},` +
|
|
// The third inline user (metricsProbeTrickyUserName) exists
|
|
// purely so exerciseMetricsRoute can prove the escaping path
|
|
// under the interpreter — it makes no requests of its own, but
|
|
// still renders a real (zero-traffic) llmgateway_requests_total
|
|
// series, since writeUsageMetrics (metrics.go) lists every
|
|
// active user regardless of traffic.
|
|
//
|
|
// The fourth inline user (bareWinnerFriendAPIKey) is
|
|
// exerciseBareWinnerGroupAware's own — group bareWinnerGroupName,
|
|
// authorized for "bare-winner-b" only (the "groups" override
|
|
// above).
|
|
//
|
|
// The fifth inline user (multiGroupUserAPIKey) is
|
|
// exerciseMultiGroupPersonalGrant's own — member of BOTH
|
|
// multiGroupEngGroupName and multiGroupOpsGroupName (group +
|
|
// groups, multi-group support), plus a personal grant
|
|
// (providers/models) restricting "multi-group-b" to
|
|
// multiGroupAllowedModelID alone.
|
|
//
|
|
// The sixth inline user (round3P1UserAPIKey) is
|
|
// exerciseRound3ModelAndPathFromSameGrant's own P1 principal —
|
|
// member of BOTH round3P1AGroupName (models-restricted) and
|
|
// round3P1BGroupName (paths-restricted), no personal grant.
|
|
//
|
|
// The seventh inline user (round3P2UserAPIKey) is that same
|
|
// probe's own P2 principal — member of round3P2HomeGroupName
|
|
// (primary, unrestricted) and round3P2FriendsGroupName
|
|
// (paths-restricted), plus a personal grant restricting
|
|
// "round3-shared" to round3PersonalModelID alone.
|
|
`"admin":{"enabled":true},"users":{"inline":[{"name":"tester","group":"default","apiKey":"` + testDataUserAPIKey + `"},{"name":"admin1","group":"default","apiKey":"` + attemptAccountingAdminAPIKey + `","admin":true},{"name":` + string(trickyNameJSON) + `,"group":"default","apiKey":"sk-tricky"},{"name":"bare-winner-friend","group":"` + bareWinnerGroupName + `","apiKey":"` + bareWinnerFriendAPIKey + `"},{"name":"multi-group-user","group":"` + multiGroupEngGroupName + `","groups":["` + multiGroupOpsGroupName + `"],"apiKey":"` + multiGroupUserAPIKey + `","providers":["multi-group-b"],"models":["` + multiGroupAllowedModelID + `"]},{"name":"round3-p1-user","group":"` + round3P1AGroupName + `","groups":["` + round3P1BGroupName + `"],"apiKey":"` + round3P1UserAPIKey + `"},{"name":"round3-p2-user","group":"` + round3P2HomeGroupName + `","groups":["` + round3P2FriendsGroupName + `"],"apiKey":"` + round3P2UserAPIKey + `","providers":["round3-shared"],"models":["` + round3PersonalModelID + `"]},` +
|
|
// The eighth inline user (eventsProbeUserAPIKey) is
|
|
// exerciseRateLimitEventAndRejections' own — group "default"
|
|
// (allow-all), but with its own requestsPerMinute:1 limit so a
|
|
// second call under this key alone 429s deterministically.
|
|
`{"name":"events-probe","group":"default","apiKey":"` + eventsProbeUserAPIKey + `","limits":{"requestsPerMinute":1}}]}}`
|
|
if err = json.Unmarshal([]byte(attemptAccountingOverride), cfgVal.Interface()); err != nil {
|
|
return fmt.Errorf("decode attempt-accounting harness override into the interpreted Config: %w", err)
|
|
}
|
|
|
|
newVal, err := i.Eval(pkgName + ".New")
|
|
if err != nil {
|
|
return fmt.Errorf("resolve %s.New under yaegi: %w", pkgName, err)
|
|
}
|
|
if newVal.Kind() != reflect.Func {
|
|
return fmt.Errorf("%s.New is not a function (kind %s)", pkgName, newVal.Kind())
|
|
}
|
|
|
|
next := http.HandlerFunc(func(http.ResponseWriter, *http.Request) {})
|
|
results := newVal.Call([]reflect.Value{
|
|
reflect.ValueOf(context.Background()),
|
|
reflect.ValueOf(next),
|
|
cfgVal,
|
|
reflect.ValueOf("llmgw-check"),
|
|
})
|
|
if len(results) != 2 {
|
|
return fmt.Errorf("%s.New returned %d values, want 2 (http.Handler, error)", pkgName, len(results))
|
|
}
|
|
if errVal := results[1]; !errVal.IsNil() {
|
|
return fmt.Errorf("%s.New returned an error: %v", pkgName, errVal.Interface())
|
|
}
|
|
handlerVal := results[0]
|
|
if handlerVal.IsNil() {
|
|
return fmt.Errorf("%s.New returned a nil handler", pkgName)
|
|
}
|
|
handler, ok := handlerVal.Interface().(http.Handler)
|
|
if !ok {
|
|
return fmt.Errorf("%s.New's returned value does not implement http.Handler", pkgName)
|
|
}
|
|
|
|
// exerciseTargetHealthProbeWarmup (feat/target-health, F5 review) runs
|
|
// FIRST, before exerciseHandler and everything after it: its own doc
|
|
// comment explains why this ordering is what makes every later
|
|
// target-health assertion in this harness deterministic rather than
|
|
// racing the active probe sweep under Yaegi's interpreter overhead.
|
|
if err := exerciseTargetHealthProbeWarmup(handler); err != nil {
|
|
return err
|
|
}
|
|
|
|
if err := exerciseHandler(handler, builtinLookupContextTokens, failoverBHits); err != nil {
|
|
return err
|
|
}
|
|
// exerciseTargetHealth (feat/target-health) runs right after
|
|
// exerciseHandler: its own part (a) reads the passive-recording
|
|
// side effect of exerciseFederatedTooLarge's federated tools/call,
|
|
// already driven above as one of exerciseHandler's own sub-probes,
|
|
// so it must not run before that attempt has happened. Its part (b)
|
|
// drives its own, independent target (targetHealthDownServerName),
|
|
// so it cannot perturb any later probe's own counters or /metrics
|
|
// assertions.
|
|
if err := exerciseTargetHealth(handler); err != nil {
|
|
return err
|
|
}
|
|
// exerciseBareWinnerGroupAware (fix/group-aware-bare-winner,
|
|
// 2026-08-27) runs right after exerciseHandler and before
|
|
// exerciseBreaker: it drives its own, freshly named providers and
|
|
// model id (bareWinnerModelID's own doc comment above), so it cannot
|
|
// perturb exerciseBreaker's "brk"-specific hit counting or any later
|
|
// probe's /metrics assertions, which are all keyed to OTHER provider/
|
|
// model names.
|
|
if err := exerciseBareWinnerGroupAware(handler, bareWinnerAHits); err != nil {
|
|
return err
|
|
}
|
|
// exerciseMultiGroupPersonalGrant (multi-group/personal-grant feature)
|
|
// runs right after exerciseBareWinnerGroupAware, for the identical
|
|
// reason: it drives its own, freshly named provider/groups/user, so it
|
|
// cannot perturb exerciseBreaker's "brk"-specific hit counting or any
|
|
// later probe's /metrics assertions, which are all keyed to OTHER
|
|
// provider/model names.
|
|
if err := exerciseMultiGroupPersonalGrant(handler, multiGroupBHits); err != nil {
|
|
return err
|
|
}
|
|
// exerciseMultiGroupBareWinnerFailoverAndPassthroughPaths (LOW-8,
|
|
// review round 2) runs right after exerciseMultiGroupPersonalGrant,
|
|
// against the SAME multi-group-user principal but its own, freshly
|
|
// named providers/models/paths, so it cannot perturb any earlier or
|
|
// later probe's own hit counters or /metrics assertions.
|
|
if err := exerciseMultiGroupBareWinnerFailoverAndPassthroughPaths(handler, multiGroupBHits, multiGroupCChatHits, multiGroupCPassthroughHits, multiGroupSharedXHits); err != nil {
|
|
return err
|
|
}
|
|
// exerciseRound3ModelAndPathFromSameGrant (HIGH fix, review round 3)
|
|
// runs right after, against its own, freshly named P1/P2 providers/
|
|
// groups/users, so it cannot perturb any earlier or later probe's own
|
|
// hit counters or /metrics assertions.
|
|
if err := exerciseRound3ModelAndPathFromSameGrant(handler, round3PassthroughHits); err != nil {
|
|
return err
|
|
}
|
|
// exerciseBreaker (feat/provider-health, adversarial-review round 2):
|
|
// runs after exerciseHandler, against the SAME interpreted handler —
|
|
// proves the discovery circuit breaker's open/half-open/closed state
|
|
// machine, its new interpreted breakerState (a named int32 with a
|
|
// String() method), and recordHealthLocked's classification call all
|
|
// run correctly under Yaegi, not merely compiled. limits.go's own
|
|
// matchesSentinel doc comment (Trap-4/reverse-direction history) is
|
|
// exactly why this class of divergence needs its own interpreted
|
|
// probe: a compiled `go test` pass here would prove nothing about the
|
|
// interpreter.
|
|
if err := exerciseBreaker(handler, brkModelsHits, brkHealthy); err != nil {
|
|
return err
|
|
}
|
|
// exerciseWarmFillRetry (production incident fix, registry.go's
|
|
// warmFillRetryDelay) runs right after exerciseBreaker: it drives its
|
|
// own, dedicated "warmretry" provider/upstream, so it cannot perturb
|
|
// exerciseBreaker's own "brk"-specific hit counting, and nothing below
|
|
// depends on "warmretry"'s own hit count either.
|
|
if err := exerciseWarmFillRetry(handler, warmRetryHits); err != nil {
|
|
return err
|
|
}
|
|
// exerciseMetricsRoute runs after exerciseBreaker has already settled:
|
|
// GET /metrics runs registry.maybeRefresh like every other route
|
|
// (ServeHTTP's own unconditional call at entry), and exerciseBreaker's
|
|
// own hit-counting is timing-sensitive against the "brk" provider
|
|
// specifically — probing metrics first could perturb the very
|
|
// discovery-attempt counts that test samples. Ordering the metrics
|
|
// probe after avoids any such interaction.
|
|
if err := exerciseMetricsRoute(handler); err != nil {
|
|
return err
|
|
}
|
|
// exerciseLatencyMetrics (feat: instrument upstream latency) runs
|
|
// after exerciseMetricsRoute, for the same reason exerciseMetricsRoute
|
|
// itself used to be last: it reads the exact same /metrics endpoint
|
|
// and must not race or perturb anything exerciseBreaker/
|
|
// exerciseMetricsRoute still care about. It relies on exerciseHandler's
|
|
// own exerciseAttemptAccounting sub-probe (already run, above) having
|
|
// driven a real, non-streaming POST /v1/chat/completions against the
|
|
// "openai" provider — see its own doc comment.
|
|
if err := exerciseLatencyMetrics(handler); err != nil {
|
|
return err
|
|
}
|
|
// exerciseUsageProvenance (feat: expose token-accounting provenance)
|
|
// runs LAST, for the identical reason: it reads /metrics and GET
|
|
// /admin/api/overview one more time and must not perturb any counter
|
|
// an earlier probe already asserted on.
|
|
if err := exerciseUsageProvenance(handler); err != nil {
|
|
return err
|
|
}
|
|
// exerciseModelUsageRanking (feat: per-model usage statistics) runs
|
|
// after everything above: it only READS admin endpoints plus one more
|
|
// chat completion of its own, and asserts on a kindModel counter no
|
|
// earlier probe touches, so it can neither perturb nor be perturbed by
|
|
// the provider-level counters they assert on.
|
|
if err := exerciseModelUsageRanking(handler); err != nil {
|
|
return err
|
|
}
|
|
// exerciseFreeModelUsageDetail/exerciseMediaModelUsage (free-models
|
|
// feature) run right after: the first only READS admin endpoints
|
|
// (detail=1), the second sends one real POST /v1/images/generations
|
|
// that adds exactly 1 to modelUsageCanonicalID's own req counter — a
|
|
// perturbation exerciseModelUsageRanking's own assertions (above,
|
|
// already run) never re-check, and nothing below depends on that
|
|
// counter's exact value either (exerciseUsageModelsSpan only checks
|
|
// the echoed span; exerciseRateLimitEventAndRejections uses its own
|
|
// dedicated principal).
|
|
if err := exerciseFreeModelUsageDetail(handler); err != nil {
|
|
return err
|
|
}
|
|
if err := exerciseMediaModelUsage(handler); err != nil {
|
|
return err
|
|
}
|
|
// exerciseAdminOverviewReplicaAndWarnings/exerciseUsageModelsSpan (F4/
|
|
// F9/F10/F2, v0.3 dashboard task) run next: both only READ admin
|
|
// endpoints, so neither perturbs nor is perturbed by anything above.
|
|
if err := exerciseAdminOverviewReplicaAndWarnings(handler); err != nil {
|
|
return err
|
|
}
|
|
if err := exerciseUsageModelsSpan(handler); err != nil {
|
|
return err
|
|
}
|
|
// exerciseRateLimitEventAndRejections (F3/F4, v0.3 dashboard task)
|
|
// runs next, against its own dedicated requestsPerMinute:1 user
|
|
// (eventsProbeUserAPIKey), so its two calls cannot perturb any earlier
|
|
// probe's own per-user counters.
|
|
if err := exerciseRateLimitEventAndRejections(handler); err != nil {
|
|
return err
|
|
}
|
|
// exerciseUpstreamAndTimeoutEvents (F3 hook 2, coordinator request
|
|
// 2026-09-23) runs LAST: each of its two assertions reads GET
|
|
// /admin/api/events?limit=1 for the single newest entry right after
|
|
// producing it, so nothing after it depends on event ordering staying
|
|
// undisturbed.
|
|
if err := exerciseUpstreamAndTimeoutEvents(handler); err != nil {
|
|
return err
|
|
}
|
|
// exerciseAdminRedesignStats (admin_redesign.go, admin dashboard
|
|
// redesign WP-G) runs LAST of all, against its own dedicated Gateway
|
|
// instance (own config, own fake Redis, own upstream) rather than the
|
|
// shared `handler` above — see that function's own doc comment for
|
|
// why.
|
|
return exerciseAdminRedesignStats(i, pkgName)
|
|
}
|
|
|
|
// builtinLookupModelID is a real, stable entry in the generated
|
|
// builtinModelMetaTable (pricing_data_gen.go) — review fix, SHOULD-6 —
|
|
// added to the harness's openai provider with NO modelMeta config
|
|
// override of its own, so its GET /v1/models context_window can only
|
|
// ever come from an interpreted lookup into the real map literal, not
|
|
// the config-override path testDataWantModel already exercises.
|
|
const builtinLookupModelID = "gpt-4o"
|
|
|
|
// failoverModelID is the bare model id both "failover-a" and "failover-b"
|
|
// (run(), above) are configured with — the run()-owned upstream servers
|
|
// exerciseFailover (below) drives.
|
|
const failoverModelID = "failover-shared"
|
|
|
|
// bareWinnerModelID is the bare model id both "bare-winner-a" and
|
|
// "bare-winner-b" (run(), above) are configured with — deliberately the
|
|
// identical collision SHAPE as failoverModelID above, but exercising
|
|
// registry.go's bareWinner AUTHORIZATION fix (2026-08-27) rather than
|
|
// failover: bareWinnerGroupName (below) is authorized for "bare-winner-b"
|
|
// only, yet "bare-winner-a" sorts first alphabetically. Before the fix,
|
|
// bareWinner always returned the GLOBAL sorted-first owner regardless of
|
|
// the caller, so this exact group got errModelDenied for a model it was
|
|
// actually entitled to use — the production repro this whole fix exists
|
|
// for (uni/macstudio serving text-embedding-multilingual-e5-base, the
|
|
// "friends" group entitled to uni but denied because macstudio sorts
|
|
// first). exerciseBareWinnerGroupAware (below) proves the fix resolves
|
|
// and serves under the REAL interpreter, not merely compiled.
|
|
const bareWinnerModelID = "bare-winner-shared"
|
|
|
|
// bareWinnerGroupName is the group exerciseBareWinnerGroupAware
|
|
// authenticates as: authorized for "bare-winner-b" only (run()'s "groups"
|
|
// override), never "bare-winner-a" — see bareWinnerModelID's own doc
|
|
// comment for why that specific asymmetry is the point.
|
|
const bareWinnerGroupName = "bare-winner-friends"
|
|
|
|
// bareWinnerFriendAPIKey authenticates bareWinnerGroupName's one inline
|
|
// user (run()'s "users" override).
|
|
const bareWinnerFriendAPIKey = "sk-bare-winner-friend" // #nosec G101 -- test fixture literal, not a real credential
|
|
|
|
// multiGroupEngGroupName/multiGroupOpsGroupName are the two member groups
|
|
// exerciseMultiGroupPersonalGrant's own inline user belongs to
|
|
// (UserConfig.Group/Groups, llmgateway.go, run()'s "users" override) —
|
|
// neither names "multi-group-b" in its own "groups" override providers
|
|
// list, so the only grant reaching it at all is the user's own personal
|
|
// one (multiGroupUserAPIKey's own doc comment, below).
|
|
const (
|
|
multiGroupEngGroupName = "multi-group-eng"
|
|
multiGroupOpsGroupName = "multi-group-ops"
|
|
)
|
|
|
|
// multiGroupUserAPIKey authenticates exerciseMultiGroupPersonalGrant's
|
|
// own inline user: member of BOTH multiGroupEngGroupName and
|
|
// multiGroupOpsGroupName (run()'s "users" override), plus a personal
|
|
// grant (Providers/Models) restricting "multi-group-b" to
|
|
// multiGroupAllowedModelID alone.
|
|
const multiGroupUserAPIKey = "sk-multi-group-user" // #nosec G101 -- test fixture literal, not a real credential
|
|
|
|
// multiGroupAllowedModelID/multiGroupDeniedModelID are two ids served by
|
|
// the SAME real provider ("multi-group-b", run()'s "providers" override):
|
|
// the personal grant's own Models list (run()'s "users" override) names
|
|
// only the "only" form, so the "other" form must resolve as
|
|
// errModelDenied — proving the multi-group/personal-grant feature's
|
|
// per-grant (not per-union) model check runs correctly under Yaegi, not
|
|
// merely compiled (registry_test.go's own
|
|
// TestModelRegistry_Resolve_MultiGroup_CrossGrantLeakDenied covers the
|
|
// identical rule compiled).
|
|
const (
|
|
multiGroupAllowedModelID = "multi-group-b/only"
|
|
multiGroupDeniedModelID = "multi-group-b/other"
|
|
)
|
|
|
|
// multiGroupSharedModelID is the BARE id (no provider prefix) all three
|
|
// of "multi-group-b", "multi-group-c", and "multi-group-shared-x" serve
|
|
// (run()'s "providers" override) — LOW-8's extension of this probe
|
|
// (review round 2), exercising HIGH-2's bareWinner/listFor/failover fix
|
|
// under the REAL interpreter: sorted provider-name order is
|
|
// "multi-group-b" < "multi-group-c" < "multi-group-shared-x", and only
|
|
// the harness user's PERSONAL grant reaches "multi-group-b" — restricted
|
|
// to multiGroupAllowedModelID alone, which does NOT cover this bare id —
|
|
// so a pre-fix, union-based bareWinner picks "multi-group-b" first and
|
|
// wrongly denies the whole request, even though multiGroupOpsGroupName's
|
|
// own unrestricted grant on "multi-group-c" (sorted next) would happily
|
|
// serve it. multiGroupCUpstream's own doc comment (above) has the full
|
|
// three-provider shape, including how it doubles as the failover
|
|
// trigger.
|
|
const multiGroupSharedModelID = "multi-group-shared"
|
|
|
|
// multiGroupAllowedPathID is the one native-passthrough REST path
|
|
// multiGroupOpsGroupName's own "passthroughPaths" override (run()'s
|
|
// "groups" override) allows on ITS OWN provider ("multi-group-c") —
|
|
// LOW-8's HIGH-1 extension (review round 2): multiGroupEngGroupName
|
|
// carries no such restriction on its own, DIFFERENT provider
|
|
// ("multi-group-shared-x"), and
|
|
// exerciseMultiGroupBareWinnerFailoverAndPassthroughPaths (below) proves
|
|
// the two never leak into each other under the REAL interpreter — the
|
|
// identical rule routes_passthrough_test.go's own
|
|
// TestHandlePassthrough_MultiGroup_PassthroughPathsNeverUnionAcrossProviders
|
|
// covers compiled.
|
|
const multiGroupAllowedPathID = "v1/allowed-path"
|
|
|
|
// multiGroupDeniedPathID is any path NOT in multiGroupOpsGroupName's own
|
|
// "passthroughPaths" list — must be denied on "multi-group-c" regardless
|
|
// of multiGroupEngGroupName's own unrestricted paths on its DIFFERENT
|
|
// provider.
|
|
const multiGroupDeniedPathID = "v1/other-path"
|
|
|
|
// round3P1AGroupName/round3P1BGroupName back
|
|
// exerciseRound3ModelAndPathFromSameGrant's own P1 reproduction (HIGH
|
|
// fix, review round 3): group A restricts models (round3ModelID) but not
|
|
// paths; group B restricts paths ("v1/chat/completions") but not models
|
|
// — both cover the SAME provider ("round3-shared"). round3P1UserAPIKey's
|
|
// own principal belongs to BOTH, no personal grant.
|
|
const (
|
|
round3P1AGroupName = "round3-p1-a"
|
|
round3P1BGroupName = "round3-p1-b"
|
|
round3P1UserAPIKey = "sk-round3-p1-user" // #nosec G101 -- test fixture literal, not a real credential
|
|
round3ModelID = "round3-m1"
|
|
)
|
|
|
|
// round3P2HomeGroupName/round3P2FriendsGroupName back
|
|
// exerciseRound3ModelAndPathFromSameGrant's own P2 reproduction: "home"
|
|
// is the PRIMARY group (no path restriction of its own); "friends"
|
|
// restricts paths on "round3-shared" but not models.
|
|
// round3P2UserAPIKey's own principal belongs to both, PLUS a personal
|
|
// grant restricting "round3-shared" to round3PersonalModelID alone —
|
|
// borrowing home's own unrestricted paths.
|
|
const (
|
|
round3P2HomeGroupName = "round3-p2-home"
|
|
round3P2FriendsGroupName = "round3-p2-friends"
|
|
round3P2UserAPIKey = "sk-round3-p2-user" // #nosec G101 -- test fixture literal, not a real credential
|
|
round3PersonalModelID = "round3-shared/m1"
|
|
)
|
|
|
|
// eventsProbeUserAPIKey authenticates exerciseRateLimitEventAndRejections'
|
|
// own inline user (run()'s "users" override): group "default" (allow-all,
|
|
// like every other probe user), but with its own requestsPerMinute:1
|
|
// limit, so a SECOND call under this key alone 429s regardless of what
|
|
// every other probe already did to testDataUserAPIKey's own counters —
|
|
// no other probe may reuse this key.
|
|
const eventsProbeUserAPIKey = "sk-events-probe" // #nosec G101 -- test fixture literal, not a real credential
|
|
|
|
// eventsUpstreamErrorProviderName/-Model (F3 hook 2, coordinator request
|
|
// 2026-09-23) back exerciseUpstreamAndTimeoutEvents: a SINGLE provider —
|
|
// deliberately no failover candidate configured for this model id — whose
|
|
// upstream always 500s, so the client-facing response IS the classified
|
|
// *providerHTTPError handleAdapterErrorEnvelope hands to writeUpstream,
|
|
// and recordUpstreamEvent's default branch runs on every call, not just
|
|
// the first of a failover attempt sequence.
|
|
const (
|
|
eventsUpstreamErrorProviderName = "eventsprobe-upstream"
|
|
eventsUpstreamErrorProviderModel = "gpt-eventsprobe-upstream-test"
|
|
)
|
|
|
|
// mcpProbeServerName is the federated MCP server run() configures against
|
|
// mcpProbeUpstream, and mcpProbeToolName is a tool id carrying its
|
|
// "<server>_" prefix so resolveFederatedTool routes POST /mcp's
|
|
// tools/call there. The tool needs no tools/list entry — resolution is
|
|
// pure prefix matching on the configured, group-allowed server names.
|
|
const (
|
|
mcpProbeServerName = "probe"
|
|
mcpProbeToolName = mcpProbeServerName + "_big"
|
|
)
|
|
|
|
// targetHealthDownServerName is the second MCP server run() configures,
|
|
// against targetHealthDownUpstream — a URL nothing listens on, opened
|
|
// then immediately closed (feat/target-health, exerciseTargetHealth,
|
|
// below). targetHealthFailureThreshold mirrors the plugin's own
|
|
// defaultTargetHealthFailureThreshold (target_health.go): this harness
|
|
// module cannot import that unexported constant across the Yaegi
|
|
// reflection boundary, so it is copied here, the same independent-
|
|
// literal approach every other harness constant mirroring plugin
|
|
// internals already takes in this file (e.g. testDataWantModel).
|
|
const (
|
|
targetHealthDownServerName = "target-health-down"
|
|
targetHealthFailureThreshold = 3
|
|
)
|
|
|
|
// targetHealthProbeOnlyServerName is a THIRD MCP server run() configures,
|
|
// against targetHealthProbeOnlyUpstream — a real, always-answering
|
|
// upstream this harness never proxies traffic through (no per-server
|
|
// proxy call, no federated tools/list or tools/call ever names it): its
|
|
// health entry can only ever be written by the ACTIVE PROBE sweep
|
|
// (target_health.go's probeMCPTarget), never by passive traffic. F5,
|
|
// feat/target-health review: exerciseTargetHealthProbeWarmup (below)
|
|
// uses this determinism to assert source == "probe" without racing
|
|
// anything, and to prove the interpreted initialize/close handshake path
|
|
// itself, not just "some sweep ran".
|
|
const targetHealthProbeOnlyServerName = "target-health-probe-only"
|
|
|
|
// metricsProbeAllowedCIDR is the CIDR block attemptAccountingOverride's
|
|
// own metrics.allowedCIDRs names — see that literal's own comment for why
|
|
// this must contain httptest.NewRequest's default RemoteAddr.
|
|
const metricsProbeAllowedCIDR = "192.0.2.0/24"
|
|
|
|
// metricsProbeOutsideAddr is a source address deliberately OUTSIDE
|
|
// metricsProbeAllowedCIDR (TEST-NET-3, RFC 5737) — exerciseMetricsRoute's
|
|
// own proof that the allowlist is not simply matching everything.
|
|
const metricsProbeOutsideAddr = "203.0.113.9:5555"
|
|
|
|
// metricsProbeTrickyUserName is a configured user name containing a
|
|
// double quote, a backslash, and a newline — the exact character set
|
|
// escapeLabelValue (metrics.go) exists to handle, driven under the REAL
|
|
// interpreter (review fix, adversarial verification 2026-08-23): the
|
|
// compiled test suite already proves escaping correct
|
|
// (TestMetrics_LabelValuesWithSpecialChars_Escaped,
|
|
// TestEscapeLabelValue_InvalidUTF8_Injective), but this codebase's own
|
|
// history is that a compiled pass has repeatedly said nothing about the
|
|
// interpreted shape.
|
|
const metricsProbeTrickyUserName = "weird\"user\\name\nwith-newline"
|
|
|
|
// mcpProbeOversizeBytes is how much filler mcpProbeUpstream writes: over
|
|
// mcpBackendCallResponseMaxBytes (maxRequestBytes, 10MiB) so the response
|
|
// trips doBackendJSONRPC's cap. Kept as its own named value rather than
|
|
// importing the plugin's const, because this harness deliberately builds
|
|
// nothing from the plugin package at compile time — everything it asserts
|
|
// about the plugin must come from the interpreter.
|
|
const mcpProbeOversizeBytes = 11 << 20
|
|
|
|
// mcpTooLargeWantMessage is the exact JSON-RPC error message
|
|
// mcpFederatedToolsCall writes when errors.Is matches
|
|
// errMCPResponseTooLarge. "upstream error" here instead means errors.Is
|
|
// returned false under the interpreter for a chain that is true when
|
|
// compiled — which is precisely the class of Yaegi divergence this
|
|
// harness exists to catch, and which no compiled test can see.
|
|
const mcpTooLargeWantMessage = "response too large"
|
|
|
|
// stampedCheckVersion is the fake release version stampReleaseVersion
|
|
// writes into the interpreted copy. It only has to differ from
|
|
// devPluginVersion; nothing asserts its value.
|
|
const stampedCheckVersion = "9.9.9-yaegicheck"
|
|
|
|
// stampReleaseVersion rewrites pluginVersion in the GOPATH COPY of
|
|
// version.go (never the repo's own file) so the copy no longer carries
|
|
// the dev sentinel.
|
|
//
|
|
// Without this, New's telemetry gate returns before ever calling into the
|
|
// vendored oss-telemetry package, and this harness would report OK while
|
|
// leaving the plugin's only non-stdlib runtime import unexercised under
|
|
// the interpreter — precisely the blind spot yaegi-check exists to close.
|
|
// Import resolution alone is not enough: a cross-package call into
|
|
// interpreted third-party code is its own risk under Yaegi.
|
|
//
|
|
// Telemetry stays suppressed by the DO_NOT_TRACK/OSS_TELEMETRY_DISABLED
|
|
// env vars the caller sets first, so Send is entered and returns without
|
|
// touching the network. The HTTP path beyond that env check is plain
|
|
// net/http, already exercised elsewhere in this harness.
|
|
func stampReleaseVersion(pkgDir string) error {
|
|
path := filepath.Join(pkgDir, "version.go")
|
|
data, err := os.ReadFile(path) //nolint:gosec // a temp dir this harness just created
|
|
if err != nil {
|
|
return fmt.Errorf("read %s: %w", path, err)
|
|
}
|
|
pattern := regexp.MustCompile(`(pluginVersion\s*=\s*")[^"]*(")`)
|
|
if !pattern.Match(data) {
|
|
return fmt.Errorf("no pluginVersion constant found in %s — has version.go been renamed?", path)
|
|
}
|
|
stamped := pattern.ReplaceAll(data, []byte(`${1}`+stampedCheckVersion+`${2}`))
|
|
// pkgDir is a temp dir run() created itself, and the filename is a
|
|
// constant — no external input reaches this path.
|
|
if err := os.WriteFile(path, stamped, 0o600); err != nil { //nolint:gosec // G703: path is filepath.Join(<temp dir we created>, "version.go")
|
|
return fmt.Errorf("write %s: %w", path, err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// readBuiltinContextTokens reads repoRoot/pricing_data_gen.go and
|
|
// extracts modelID's own ContextTokens value directly out of the
|
|
// generated source (review fix, SHOULD-6) — rather than hardcoding a
|
|
// number in this harness, which would need hand-updating (and could
|
|
// silently drift out of sync) every time `make pricing-sync`
|
|
// regenerates the table against updated upstream data.
|
|
func readBuiltinContextTokens(repoRoot, modelID string) (int, error) {
|
|
data, err := os.ReadFile(filepath.Join(repoRoot, "pricing_data_gen.go"))
|
|
if err != nil {
|
|
return 0, fmt.Errorf("read pricing_data_gen.go: %w", err)
|
|
}
|
|
pattern := regexp.MustCompile(regexp.QuoteMeta(`"`+modelID+`":`) + `\s*\{ContextTokens:\s*(\d+)`)
|
|
m := pattern.FindSubmatch(data)
|
|
if m == nil {
|
|
return 0, fmt.Errorf("no builtinModelMetaTable entry found for %q — pick a different, currently-present stable id", modelID)
|
|
}
|
|
n, err := strconv.Atoi(string(m[1]))
|
|
if err != nil {
|
|
return 0, fmt.Errorf("parse ContextTokens for %q: %w", modelID, err)
|
|
}
|
|
if n <= 0 {
|
|
return 0, fmt.Errorf("builtinModelMetaTable entry for %q has ContextTokens=%d, want > 0", modelID, n)
|
|
}
|
|
return n, nil
|
|
}
|
|
|
|
// readHealth returns provider name's healthState string from the admin
|
|
// overview, decoded generically like readProviderAttemptCounters does
|
|
// (feat/provider-health).
|
|
func readHealth(handler http.Handler, name string) (string, error) {
|
|
req := httptest.NewRequest(http.MethodGet, "/admin/api/overview", nil)
|
|
req.Header.Set("Authorization", "Bearer "+attemptAccountingAdminAPIKey)
|
|
rec := httptest.NewRecorder()
|
|
handler.ServeHTTP(rec, req)
|
|
if rec.Code != http.StatusOK {
|
|
return "", fmt.Errorf("GET /admin/api/overview: status = %d, body=%s", rec.Code, rec.Body.String())
|
|
}
|
|
var body map[string]any
|
|
if err := json.Unmarshal(rec.Body.Bytes(), &body); err != nil {
|
|
return "", err
|
|
}
|
|
providers, _ := body["providers"].([]any)
|
|
for _, raw := range providers {
|
|
p, ok := raw.(map[string]any)
|
|
if !ok {
|
|
continue
|
|
}
|
|
if p["name"] == name {
|
|
s, _ := p["healthState"].(string)
|
|
return s, nil
|
|
}
|
|
}
|
|
return "", fmt.Errorf("provider %q not in overview: %s", name, rec.Body.String())
|
|
}
|
|
|
|
// readProviderModels returns provider name's known model id set from the
|
|
// admin overview (registry.go's providerSnapshot.models,
|
|
// provider-model-accordion task) — decoded generically like readHealth
|
|
// above, for exerciseWarmFillRetry's own poll loop below.
|
|
func readProviderModels(handler http.Handler, name string) ([]string, error) {
|
|
req := httptest.NewRequest(http.MethodGet, "/admin/api/overview", nil)
|
|
req.Header.Set("Authorization", "Bearer "+attemptAccountingAdminAPIKey)
|
|
rec := httptest.NewRecorder()
|
|
handler.ServeHTTP(rec, req)
|
|
if rec.Code != http.StatusOK {
|
|
return nil, fmt.Errorf("GET /admin/api/overview: status = %d, body=%s", rec.Code, rec.Body.String())
|
|
}
|
|
var body map[string]any
|
|
if err := json.Unmarshal(rec.Body.Bytes(), &body); err != nil {
|
|
return nil, err
|
|
}
|
|
providers, _ := body["providers"].([]any)
|
|
for _, raw := range providers {
|
|
p, ok := raw.(map[string]any)
|
|
if !ok {
|
|
continue
|
|
}
|
|
if p["name"] != name {
|
|
continue
|
|
}
|
|
rawModels, _ := p["models"].([]any)
|
|
models := make([]string, 0, len(rawModels))
|
|
for _, rm := range rawModels {
|
|
if s, ok := rm.(string); ok {
|
|
models = append(models, s)
|
|
}
|
|
}
|
|
return models, nil
|
|
}
|
|
return nil, fmt.Errorf("provider %q not in overview: %s", name, rec.Body.String())
|
|
}
|
|
|
|
// drive fires n authenticated GET /v1/models requests, each of which runs
|
|
// maybeRefresh at ServeHTTP entry, with a small gap so the 1ms discovery
|
|
// interval always permits a fresh attempt when the breaker is closed
|
|
// (feat/provider-health).
|
|
func drive(handler http.Handler, n int) {
|
|
for i := 0; i < n; i++ {
|
|
req := httptest.NewRequest(http.MethodGet, "/v1/models", nil)
|
|
req.Header.Set("Authorization", "Bearer "+testDataUserAPIKey)
|
|
handler.ServeHTTP(httptest.NewRecorder(), req)
|
|
time.Sleep(20 * time.Millisecond)
|
|
}
|
|
}
|
|
|
|
// exerciseBreaker drives the "brk" provider (attemptAccountingOverride,
|
|
// run()) through the discovery circuit breaker's full state cycle —
|
|
// closed -> open -> half-open -> closed again — against the REAL
|
|
// interpreter, proving under Yaegi what registry_health_test.go already
|
|
// proves compiled: tryBeginRefresh/finishRefresh/recordHealthLocked, the
|
|
// interpreted breakerState named-int32 type and its String() method, and
|
|
// the classification call recordHealthLocked makes all behave correctly
|
|
// interpreted, not merely compiled (feat/provider-health, adversarial-
|
|
// review round 2 — limits.go's matchesSentinel doc comment is exactly why
|
|
// this class of divergence needs its own interpreted probe).
|
|
func exerciseBreaker(handler http.Handler, hits, healthy *int64) error {
|
|
// Phase 1: hammer with the upstream 401ing. The breaker must open.
|
|
deadline := time.Now().Add(10 * time.Second)
|
|
state := ""
|
|
for time.Now().Before(deadline) {
|
|
drive(handler, 5)
|
|
s, err := readHealth(handler, "brk")
|
|
if err != nil {
|
|
return err
|
|
}
|
|
state = s
|
|
if s == "open" {
|
|
break
|
|
}
|
|
}
|
|
if state != "open" {
|
|
return fmt.Errorf("BREAKER-1: healthState = %q after sustained 401s, want %q (upstream /v1/models hits=%d)", state, "open", atomic.LoadInt64(hits))
|
|
}
|
|
openedAtHits := atomic.LoadInt64(hits)
|
|
fmt.Printf("yaegi-check: breaker opened after %d upstream /v1/models hits\n", openedAtHits)
|
|
|
|
// Phase 2: while open, further traffic must NOT reach the upstream.
|
|
// openDuration is 3s; hammer for ~1s and require zero new hits.
|
|
drive(handler, 30)
|
|
duringOpen := atomic.LoadInt64(hits) - openedAtHits
|
|
if duringOpen != 0 {
|
|
return fmt.Errorf("BREAKER-2: %d upstream /v1/models hits during the open window, want 0 — backoff did not suppress discovery", duringOpen)
|
|
}
|
|
fmt.Println("yaegi-check: breaker open window suppressed all discovery hits")
|
|
|
|
// Phase 3: provider recovers. After the backoff window elapses the
|
|
// half-open probe must run and CLOSE the breaker again.
|
|
atomic.StoreInt64(healthy, 1)
|
|
recovered := false
|
|
recoverDeadline := time.Now().Add(20 * time.Second)
|
|
for time.Now().Before(recoverDeadline) {
|
|
drive(handler, 5)
|
|
s, err := readHealth(handler, "brk")
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if s == "closed" {
|
|
recovered = true
|
|
break
|
|
}
|
|
}
|
|
if !recovered {
|
|
return fmt.Errorf("BREAKER-3: breaker never returned to closed after the provider recovered (hits=%d)", atomic.LoadInt64(hits))
|
|
}
|
|
fmt.Println("yaegi-check: recovered provider closed the breaker again")
|
|
return nil
|
|
}
|
|
|
|
// exerciseWarmFillRetry proves registry.go's warm-fill retry (production
|
|
// incident fix: a provider whose SYNCHRONOUS initial discovery fetch
|
|
// fails, at pod start, is left with zero discovered models until a
|
|
// scheduled background retry recovers it) actually schedules and runs
|
|
// under the REAL interpreter — time.AfterFunc, the retry's own
|
|
// context.WithTimeout budget, and the finishRefresh bookkeeping it shares
|
|
// with warmFill/refreshProvider all proven interpreted, not merely
|
|
// compiled, per this harness's own stated purpose.
|
|
//
|
|
// "warmretry" (attemptAccountingOverride, run()) 401s on its very first
|
|
// /v1/models hit — warmFill's own synchronous call, spent inside New()
|
|
// itself, well before this probe (or any other) runs — then serves real
|
|
// data on every hit after. Only the scheduled retry, firing after
|
|
// registry.go's warmFillRetryDelay (10s, not overridable from outside
|
|
// the package), can ever produce a second hit and populate
|
|
// "warmretry-model"; this probe does not assume anything about how much
|
|
// wall-clock time the OTHER probes ahead of it in run()'s orchestration
|
|
// already burned relative to that 10s window — by the time this runs the
|
|
// retry may already have completed, or may still be pending — it only
|
|
// polls until the recovered state is reached (or times out) and then
|
|
// checks hits landed on exactly 2, not more.
|
|
func exerciseWarmFillRetry(handler http.Handler, hits *int64) error {
|
|
// warmFillRetryDelay (10s) + warmFillTimeout (5s) + slack: the same
|
|
// budget shape exerciseBreaker's own phase-3 recovery wait above
|
|
// uses, just against this feature's fixed, non-configurable delay
|
|
// instead of a config-driven breaker backoff.
|
|
var models []string
|
|
var err error
|
|
recovered := false
|
|
deadline := time.Now().Add(20 * time.Second)
|
|
for time.Now().Before(deadline) {
|
|
models, err = readProviderModels(handler, "warmretry")
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if len(models) == 1 && models[0] == "warmretry-model" {
|
|
recovered = true
|
|
break
|
|
}
|
|
time.Sleep(100 * time.Millisecond)
|
|
}
|
|
if !recovered {
|
|
return fmt.Errorf("WARMFILLRETRY-1: provider %q never showed the retried model %q within 20s (hits=%d, models=%v)", "warmretry", "warmretry-model", atomic.LoadInt64(hits), models)
|
|
}
|
|
if got := atomic.LoadInt64(hits); got != 2 {
|
|
return fmt.Errorf("WARMFILLRETRY-2: warmRetryUpstream saw %d /v1/models hits once recovered, want exactly 2 (the failed warm fill plus one retry — no maybeRefresh interference)", got)
|
|
}
|
|
fmt.Println("yaegi-check: warm-fill retry recovered the provider's discovered models after the scheduled delay")
|
|
return nil
|
|
}
|
|
|
|
// exerciseMetricsRoute drives GET /metrics against the interpreted
|
|
// handler and proves both its auth gate and its rendering run correctly
|
|
// under Yaegi — the metrics feature's own gate requirement: compiled
|
|
// tests have repeatedly passed on this codebase while the interpreted
|
|
// shape failed (matchesSentinel's own doc comment, limits.go, is the
|
|
// canonical example), so a metrics-specific probe is not optional here.
|
|
//
|
|
// Four requests, mirroring TestMetrics_GateMatrix (metrics_test.go)
|
|
// exactly so this harness and the compiled test suite assert the
|
|
// identical contract from two different angles:
|
|
// 1. unauthenticated, from an address outside metrics.allowedCIDRs -> 401
|
|
// 2. authenticated as "tester" (not admin) -> 403
|
|
// 3. authenticated as the admin user -> 200, body inspected
|
|
// 4. unauthenticated, from an address INSIDE metrics.allowedCIDRs -> 200
|
|
//
|
|
// Request 3's body is inspected for real, live-data content — not just a
|
|
// 200 status — because a Yaegi-specific bug in this codebase's history
|
|
// (matchesSentinel, limits.go) was a case where the interpreted code ran
|
|
// without panicking yet produced a WRONG answer; a bare status-code check
|
|
// would not have caught that class of failure here either. Specifically:
|
|
// the "openai" provider's name, and — since attemptAccountingOverride
|
|
// enables metrics.modelLabel — testDataWantModel's own id inside the
|
|
// opt-in provider-model breakdown, both of which can only appear via a
|
|
// live registry.snapshot() walk and a live providerModelScopeID join
|
|
// under the real interpreter, not a static string in this harness.
|
|
func exerciseMetricsRoute(handler http.Handler) error {
|
|
unauthReq := httptest.NewRequest(http.MethodGet, "/metrics", nil)
|
|
unauthReq.RemoteAddr = metricsProbeOutsideAddr
|
|
rec := httptest.NewRecorder()
|
|
handler.ServeHTTP(rec, unauthReq)
|
|
if rec.Code != http.StatusUnauthorized {
|
|
return fmt.Errorf("GET /metrics (no key, outside allowedCIDRs): status = %d, want 401, body=%s", rec.Code, rec.Body.String())
|
|
}
|
|
|
|
nonAdminReq := httptest.NewRequest(http.MethodGet, "/metrics", nil)
|
|
nonAdminReq.RemoteAddr = metricsProbeOutsideAddr
|
|
nonAdminReq.Header.Set("Authorization", "Bearer "+testDataUserAPIKey)
|
|
rec = httptest.NewRecorder()
|
|
handler.ServeHTTP(rec, nonAdminReq)
|
|
if rec.Code != http.StatusForbidden {
|
|
return fmt.Errorf("GET /metrics (non-admin key): status = %d, want 403, body=%s", rec.Code, rec.Body.String())
|
|
}
|
|
|
|
adminReq := httptest.NewRequest(http.MethodGet, "/metrics", nil)
|
|
adminReq.RemoteAddr = metricsProbeOutsideAddr
|
|
adminReq.Header.Set("Authorization", "Bearer "+attemptAccountingAdminAPIKey)
|
|
rec = httptest.NewRecorder()
|
|
handler.ServeHTTP(rec, adminReq)
|
|
if rec.Code != http.StatusOK {
|
|
return fmt.Errorf("GET /metrics (admin key): status = %d, want 200, body=%s", rec.Code, rec.Body.String())
|
|
}
|
|
if ct := rec.Header().Get("Content-Type"); !strings.HasPrefix(ct, "text/plain") {
|
|
return fmt.Errorf("GET /metrics (admin key): Content-Type = %q, want a text/plain prefix", ct)
|
|
}
|
|
body := rec.Body.String()
|
|
// metricsProbeEscapedTrickyName is escapeLabelValue's OWN expected
|
|
// output for metricsProbeTrickyUserName — a raw Go string literal
|
|
// (backticks: every backslash below is a literal backslash
|
|
// character, not a Go escape), computed by hand from the format
|
|
// rule (\\ -> \\\\, " -> \\", newline -> \\n) rather than by calling
|
|
// the function itself, so this assertion cannot pass merely because
|
|
// the interpreted and this harness's own understanding of the rule
|
|
// happen to agree by construction.
|
|
const metricsProbeEscapedTrickyName = `weird\"user\\name\nwith-newline`
|
|
for _, want := range []string{
|
|
"# TYPE llmgateway_requests_total counter",
|
|
"# TYPE llmgateway_provider_healthy gauge",
|
|
`llmgateway_provider_attempts_total{provider="openai"}`,
|
|
"# TYPE llmgateway_provider_model_attempts_total counter",
|
|
`llmgateway_provider_model_attempts_total{provider="openai",model="` + testDataWantModel + `"}`,
|
|
`llmgateway_requests_total{scope_kind="user",scope_id="` + metricsProbeEscapedTrickyName + `"}`,
|
|
} {
|
|
if !strings.Contains(body, want) {
|
|
return fmt.Errorf("GET /metrics (admin key): body missing %q — interpreted rendering diverged from the compiled shape; body=%s", want, body)
|
|
}
|
|
}
|
|
if strings.Contains(body, metricsProbeTrickyUserName) {
|
|
return fmt.Errorf("GET /metrics (admin key): body contains the RAW, unescaped tricky user name — escapeLabelValue did not run interpreted; body=%s", body)
|
|
}
|
|
|
|
allowlistedReq := httptest.NewRequest(http.MethodGet, "/metrics", nil) // default RemoteAddr, inside metricsProbeAllowedCIDR
|
|
rec = httptest.NewRecorder()
|
|
handler.ServeHTTP(rec, allowlistedReq)
|
|
if rec.Code != http.StatusOK {
|
|
return fmt.Errorf("GET /metrics (no key, inside allowedCIDRs): status = %d, want 200, body=%s", rec.Code, rec.Body.String())
|
|
}
|
|
|
|
fmt.Println("yaegi-check: GET /metrics auth gate and rendering both verified interpreted")
|
|
return nil
|
|
}
|
|
|
|
// exerciseLatencyMetrics proves the upstream-latency histograms (feat:
|
|
// instrument upstream latency, task brief) render correctly under the
|
|
// REAL interpreter — not merely that a compiled `go test` accepts the
|
|
// same bytes. This package's own doc comment explains why that
|
|
// distinction matters here specifically: compiled tests have repeatedly
|
|
// passed on this codebase while the interpreted shape failed.
|
|
//
|
|
// No new traffic is driven here: exerciseHandler's own
|
|
// exerciseAttemptAccounting sub-probe (already run, above, as part of
|
|
// the exerciseHandler call in run()) already drove a real, non-streaming
|
|
// POST /v1/chat/completions against the "openai" provider, so
|
|
// llmgateway_upstream_duration_seconds{provider="openai",stream="false"}
|
|
// already carries at least one real, interpreted observation by the
|
|
// time this runs.
|
|
//
|
|
// Unlike exerciseMetricsRoute's own strings.Contains checks, this parses
|
|
// the rendered body with prometheus/common/expfmt's REAL TextParser —
|
|
// the same library a real Prometheus server's scrape loop is built on —
|
|
// instead of this project's own hand-rolled parser (parsePrometheusText,
|
|
// metrics_test.go, used by the compiled suite). A malformed histogram
|
|
// (wrong cumulative order, a missing +Inf bucket, a mismatched
|
|
// sample_count) does not merely fail one assertion below, it fails to
|
|
// PARSE at all — exactly the failure mode that takes a real scrape
|
|
// target DOWN rather than dropping one metric (metricWriter.histogram's
|
|
// own doc comment, metrics.go, has the full account of why that
|
|
// distinction matters).
|
|
func exerciseLatencyMetrics(handler http.Handler) error {
|
|
req := httptest.NewRequest(http.MethodGet, "/metrics", nil)
|
|
req.RemoteAddr = metricsProbeOutsideAddr
|
|
req.Header.Set("Authorization", "Bearer "+attemptAccountingAdminAPIKey)
|
|
rec := httptest.NewRecorder()
|
|
handler.ServeHTTP(rec, req)
|
|
if rec.Code != http.StatusOK {
|
|
return fmt.Errorf("GET /metrics (latency histogram probe): status = %d, want 200, body=%s", rec.Code, rec.Body.String())
|
|
}
|
|
|
|
parser := expfmt.NewTextParser(model.LegacyValidation)
|
|
families, err := parser.TextToMetricFamilies(strings.NewReader(rec.Body.String()))
|
|
if err != nil {
|
|
return fmt.Errorf("GET /metrics: a real Prometheus TextParser rejected the interpreted rendering: %w (body=%s)", err, rec.Body.String())
|
|
}
|
|
|
|
durationFamily, ok := families["llmgateway_upstream_duration_seconds"]
|
|
if !ok {
|
|
return fmt.Errorf("GET /metrics: no llmgateway_upstream_duration_seconds family in the parsed output (body=%s)", rec.Body.String())
|
|
}
|
|
if durationFamily.GetType() != dto.MetricType_HISTOGRAM {
|
|
return fmt.Errorf("llmgateway_upstream_duration_seconds type = %s, want HISTOGRAM", durationFamily.GetType())
|
|
}
|
|
|
|
var openaiHist *dto.Histogram
|
|
for _, m := range durationFamily.GetMetric() {
|
|
var provider, stream string
|
|
for _, lp := range m.GetLabel() {
|
|
switch lp.GetName() {
|
|
case "provider":
|
|
provider = lp.GetValue()
|
|
case "stream":
|
|
stream = lp.GetValue()
|
|
}
|
|
}
|
|
if provider == "openai" && stream == "false" {
|
|
openaiHist = m.GetHistogram()
|
|
break
|
|
}
|
|
}
|
|
if openaiHist == nil {
|
|
return fmt.Errorf("GET /metrics: no llmgateway_upstream_duration_seconds{provider=\"openai\",stream=\"false\"} series found (body=%s)", rec.Body.String())
|
|
}
|
|
|
|
// Real-parser structural checks, matching the compiled suite's own
|
|
// TestMetrics_LatencyHistogram_BucketsCumulativeOrdered_
|
|
// PlusInfPresent_SumCountConsistent (metrics_test.go) — but here,
|
|
// against dto.Bucket values the REAL parser produced, not this
|
|
// project's own hand-rolled promSample.
|
|
buckets := openaiHist.GetBucket()
|
|
if len(buckets) == 0 {
|
|
return fmt.Errorf("llmgateway_upstream_duration_seconds{provider=\"openai\",stream=\"false\"}: no buckets parsed")
|
|
}
|
|
prevBound := math.Inf(-1)
|
|
var prevCount uint64
|
|
for i, b := range buckets {
|
|
if b.GetUpperBound() <= prevBound {
|
|
return fmt.Errorf("bucket %d upper_bound=%v is not strictly greater than the previous bucket's %v — buckets must be strictly ascending", i, b.GetUpperBound(), prevBound)
|
|
}
|
|
if b.GetCumulativeCount() < prevCount {
|
|
return fmt.Errorf("bucket %d cumulative_count=%d is less than the previous bucket's %d — buckets must be cumulative (non-decreasing)", i, b.GetCumulativeCount(), prevCount)
|
|
}
|
|
prevBound = b.GetUpperBound()
|
|
prevCount = b.GetCumulativeCount()
|
|
}
|
|
if !math.IsInf(buckets[len(buckets)-1].GetUpperBound(), 1) {
|
|
return fmt.Errorf("last bucket upper_bound = %v, want +Inf", buckets[len(buckets)-1].GetUpperBound())
|
|
}
|
|
if openaiHist.GetSampleCount() != prevCount {
|
|
return fmt.Errorf("sample_count = %d, want %d — must exactly equal the final +Inf cumulative bucket", openaiHist.GetSampleCount(), prevCount)
|
|
}
|
|
if openaiHist.GetSampleCount() == 0 {
|
|
return fmt.Errorf("sample_count = 0, want at least 1 (exerciseAttemptAccounting already drove one real, non-streaming chat completion against \"openai\")")
|
|
}
|
|
|
|
ttfbFamily, ok := families["llmgateway_upstream_ttfb_seconds"]
|
|
if !ok {
|
|
return fmt.Errorf("GET /metrics: no llmgateway_upstream_ttfb_seconds family in the parsed output (body=%s)", rec.Body.String())
|
|
}
|
|
if ttfbFamily.GetType() != dto.MetricType_HISTOGRAM {
|
|
return fmt.Errorf("llmgateway_upstream_ttfb_seconds type = %s, want HISTOGRAM", ttfbFamily.GetType())
|
|
}
|
|
|
|
fmt.Println("yaegi-check: upstream-latency histograms parsed by a real Prometheus TextParser; buckets cumulative, strictly ascending, +Inf-terminated, sample_count consistent")
|
|
return nil
|
|
}
|
|
|
|
// exerciseUsageProvenance proves the usage-provenance metric surface
|
|
// (feat: expose token-accounting provenance, metrics.go's
|
|
// provenanceStore/writeProvenanceMetrics, admin.go's
|
|
// buildAdminProvenanceViews) renders correctly under the REAL
|
|
// interpreter — not merely that a compiled `go test` accepts the same
|
|
// bytes, this package's own doc comment's exact concern.
|
|
//
|
|
// Drives one real, non-streaming POST /v1/chat/completions against
|
|
// provenanceEstProviderName, whose upstream (provenanceEstUpstream, run())
|
|
// answers 200 with no "usage" field at all — the exact shape
|
|
// runUnified's estimation fallback (routes_unified.go) classifies as
|
|
// "estimated". "reported" is not driven fresh here: exerciseHandler's own
|
|
// exerciseAttemptAccounting sub-probe (already run, above) already drove
|
|
// a real, usage-bearing POST /v1/chat/completions against "openai",
|
|
// which is reported provenance by construction — reusing that traffic
|
|
// instead of a third fixture, mirroring exerciseLatencyMetrics' own
|
|
// "no new traffic" reasoning for its own duration-histogram assertion.
|
|
func exerciseUsageProvenance(handler http.Handler) error {
|
|
chatReq := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(
|
|
`{"model":"`+provenanceEstProviderName+`/`+provenanceEstProviderModel+`","messages":[{"role":"user","content":"hi"}]}`,
|
|
))
|
|
chatReq.Header.Set("Authorization", "Bearer "+testDataUserAPIKey)
|
|
chatReq.Header.Set("Content-Type", "application/json")
|
|
chatRec := httptest.NewRecorder()
|
|
handler.ServeHTTP(chatRec, chatReq)
|
|
if chatRec.Code != http.StatusOK {
|
|
return fmt.Errorf("POST /v1/chat/completions (usage-provenance harness): status = %d, want 200, body=%s", chatRec.Code, chatRec.Body.String())
|
|
}
|
|
|
|
req := httptest.NewRequest(http.MethodGet, "/metrics", nil)
|
|
req.RemoteAddr = metricsProbeOutsideAddr
|
|
req.Header.Set("Authorization", "Bearer "+attemptAccountingAdminAPIKey)
|
|
rec := httptest.NewRecorder()
|
|
handler.ServeHTTP(rec, req)
|
|
if rec.Code != http.StatusOK {
|
|
return fmt.Errorf("GET /metrics (usage-provenance probe): status = %d, want 200, body=%s", rec.Code, rec.Body.String())
|
|
}
|
|
|
|
parser := expfmt.NewTextParser(model.LegacyValidation)
|
|
families, err := parser.TextToMetricFamilies(strings.NewReader(rec.Body.String()))
|
|
if err != nil {
|
|
return fmt.Errorf("GET /metrics: a real Prometheus TextParser rejected the interpreted rendering: %w (body=%s)", err, rec.Body.String())
|
|
}
|
|
|
|
requestsFamily, ok := families["llmgateway_usage_provenance_requests_total"]
|
|
if !ok {
|
|
return fmt.Errorf("GET /metrics: no llmgateway_usage_provenance_requests_total family in the parsed output (body=%s)", rec.Body.String())
|
|
}
|
|
if requestsFamily.GetType() != dto.MetricType_COUNTER {
|
|
return fmt.Errorf("llmgateway_usage_provenance_requests_total type = %s, want COUNTER", requestsFamily.GetType())
|
|
}
|
|
tokensFamily, ok := families["llmgateway_usage_provenance_tokens_total"]
|
|
if !ok {
|
|
return fmt.Errorf("GET /metrics: no llmgateway_usage_provenance_tokens_total family in the parsed output (body=%s)", rec.Body.String())
|
|
}
|
|
if tokensFamily.GetType() != dto.MetricType_COUNTER {
|
|
return fmt.Errorf("llmgateway_usage_provenance_tokens_total type = %s, want COUNTER", tokensFamily.GetType())
|
|
}
|
|
|
|
findSample := func(fam *dto.MetricFamily, provider, provenance string) *dto.Metric {
|
|
for _, m := range fam.GetMetric() {
|
|
var p, k string
|
|
for _, lp := range m.GetLabel() {
|
|
switch lp.GetName() {
|
|
case "provider":
|
|
p = lp.GetValue()
|
|
case "provenance":
|
|
k = lp.GetValue()
|
|
}
|
|
}
|
|
if p == provider && k == provenance {
|
|
return m
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
estReq := findSample(requestsFamily, provenanceEstProviderName, "estimated")
|
|
if estReq == nil {
|
|
return fmt.Errorf(`GET /metrics: no llmgateway_usage_provenance_requests_total{provider=%q,provenance="estimated"} sample found (body=%s)`, provenanceEstProviderName, rec.Body.String())
|
|
}
|
|
if estReq.GetCounter().GetValue() != 1 {
|
|
return fmt.Errorf(`llmgateway_usage_provenance_requests_total{provider=%q,provenance="estimated"} = %v, want 1`, provenanceEstProviderName, estReq.GetCounter().GetValue())
|
|
}
|
|
estTok := findSample(tokensFamily, provenanceEstProviderName, "estimated")
|
|
if estTok == nil {
|
|
return fmt.Errorf(`GET /metrics: no llmgateway_usage_provenance_tokens_total{provider=%q,provenance="estimated"} sample found (body=%s)`, provenanceEstProviderName, rec.Body.String())
|
|
}
|
|
if estTok.GetCounter().GetValue() <= 0 {
|
|
return fmt.Errorf(`llmgateway_usage_provenance_tokens_total{provider=%q,provenance="estimated"} = %v, want > 0 (estimated substitutes a nonzero prompt count)`, provenanceEstProviderName, estTok.GetCounter().GetValue())
|
|
}
|
|
|
|
reportedReq := findSample(requestsFamily, "openai", "reported")
|
|
if reportedReq == nil {
|
|
return fmt.Errorf(`GET /metrics: no llmgateway_usage_provenance_requests_total{provider="openai",provenance="reported"} sample found (body=%s) — exerciseAttemptAccounting's own earlier real, usage-bearing traffic must have been classified "reported"`, rec.Body.String())
|
|
}
|
|
if reportedReq.GetCounter().GetValue() < 1 {
|
|
return fmt.Errorf(`llmgateway_usage_provenance_requests_total{provider="openai",provenance="reported"} = %v, want >= 1`, reportedReq.GetCounter().GetValue())
|
|
}
|
|
|
|
// GET /admin/api/overview: proves admin.go's buildAdminProvenanceViews
|
|
// wiring also runs correctly interpreted, decoded generically (this
|
|
// harness module is compiled, not interpreted; adminOverviewResponse/
|
|
// adminProviderView are unexported — see readProviderAttemptCounters'
|
|
// own doc comment for why a generic decode is simpler here than
|
|
// exporting test-only types across that boundary).
|
|
adminReq := httptest.NewRequest(http.MethodGet, "/admin/api/overview", nil)
|
|
adminReq.Header.Set("Authorization", "Bearer "+attemptAccountingAdminAPIKey)
|
|
adminRec := httptest.NewRecorder()
|
|
handler.ServeHTTP(adminRec, adminReq)
|
|
if adminRec.Code != http.StatusOK {
|
|
return fmt.Errorf("GET /admin/api/overview (usage-provenance probe): status = %d, want 200, body=%s", adminRec.Code, adminRec.Body.String())
|
|
}
|
|
var overview map[string]any
|
|
if err := json.Unmarshal(adminRec.Body.Bytes(), &overview); err != nil {
|
|
return fmt.Errorf("decode GET /admin/api/overview body: %w", err)
|
|
}
|
|
providers, _ := overview["providers"].([]any)
|
|
var provenanceEstProvider map[string]any
|
|
for _, p := range providers {
|
|
pm, ok := p.(map[string]any)
|
|
if !ok {
|
|
continue
|
|
}
|
|
if pm["name"] == provenanceEstProviderName {
|
|
provenanceEstProvider = pm
|
|
break
|
|
}
|
|
}
|
|
if provenanceEstProvider == nil {
|
|
return fmt.Errorf("GET /admin/api/overview: no provider entry named %q (body=%s)", provenanceEstProviderName, adminRec.Body.String())
|
|
}
|
|
provenanceField, _ := provenanceEstProvider["provenance"].(map[string]any)
|
|
estimatedEntry, _ := provenanceField["estimated"].(map[string]any)
|
|
if estimatedEntry == nil {
|
|
return fmt.Errorf(`GET /admin/api/overview: provider %q has no provenance.estimated entry: %+v`, provenanceEstProviderName, provenanceField)
|
|
}
|
|
if reqs, _ := estimatedEntry["requests"].(float64); reqs != 1 {
|
|
return fmt.Errorf("GET /admin/api/overview: provider %q provenance.estimated.requests = %v, want 1", provenanceEstProviderName, estimatedEntry["requests"])
|
|
}
|
|
if _, hasReported := provenanceField["reported"]; hasReported {
|
|
return fmt.Errorf(`GET /admin/api/overview: provider %q provenance has a "reported" key, want it excluded from this compact admin summary: %+v`, provenanceEstProviderName, provenanceField)
|
|
}
|
|
|
|
fmt.Println("yaegi-check: usage-provenance metrics (llmgateway_usage_provenance_requests_total/-_tokens_total) and admin overview parsed by a real Prometheus TextParser/JSON decoder; estimated and reported provenances both present and distinct")
|
|
return nil
|
|
}
|
|
|
|
// exerciseHandler runs two real requests against the interpreted
|
|
// handler, drawn from the plugin's own .traefik.yml testData: an
|
|
// authenticated GET /v1/models, which must return the configured
|
|
// testDataWantModel; and the same route without credentials, which must
|
|
// be refused with 401 — proving ServeHTTP, auth.identify, the model
|
|
// registry, statusTrackingWriter, and the gateway's logger all execute
|
|
// for real under Yaegi, not merely that the package's imports resolve.
|
|
// builtinContextTokens is builtinLookupModelID's own real, extracted
|
|
// builtinModelMetaTable value (SHOULD-6) — see that constant's own doc
|
|
// comment. failoverBHits backs exerciseFailover's own call, below.
|
|
func exerciseHandler(handler http.Handler, builtinContextTokens int, failoverBHits *int64) error {
|
|
authedReq := httptest.NewRequest(http.MethodGet, "/v1/models", nil)
|
|
authedReq.Header.Set("Authorization", "Bearer "+testDataUserAPIKey)
|
|
authedRec := httptest.NewRecorder()
|
|
handler.ServeHTTP(authedRec, authedReq)
|
|
if authedRec.Code != http.StatusOK {
|
|
return fmt.Errorf("GET /v1/models with the testData user key: status = %d, want 200, body=%s", authedRec.Code, authedRec.Body.String())
|
|
}
|
|
if !strings.Contains(authedRec.Body.String(), testDataWantModel) {
|
|
return fmt.Errorf("GET /v1/models body does not contain %q: %s", testDataWantModel, authedRec.Body.String())
|
|
}
|
|
|
|
// modelMeta (feature v0.23), interpreted, decoded structurally (not
|
|
// a raw substring match — SHOULD-6, review round: a bare Contains
|
|
// check cannot tell WHICH model entry a given context_window value
|
|
// belongs to, which matters once two entries can legitimately share
|
|
// the same number) so each assertion below is pinned to its own
|
|
// named model id.
|
|
var modelsBody struct {
|
|
Data []map[string]any `json:"data"`
|
|
}
|
|
if err := json.Unmarshal(authedRec.Body.Bytes(), &modelsBody); err != nil {
|
|
return fmt.Errorf("decode GET /v1/models body: %w (body=%s)", err, authedRec.Body.String())
|
|
}
|
|
byID := make(map[string]map[string]any, len(modelsBody.Data))
|
|
for _, entry := range modelsBody.Data {
|
|
if id, ok := entry["id"].(string); ok {
|
|
byID[id] = entry
|
|
}
|
|
}
|
|
|
|
// run() layers a modelMeta config override for testDataWantModel
|
|
// onto cfgVal before New() is ever called (attemptAccountingOverride
|
|
// above) — proving resolveModelMeta's config-override layer and
|
|
// modelObject's context_window/pricing extension fields both run
|
|
// correctly, interpreted, through registry.go's listFor/
|
|
// resolveMetaFor.
|
|
overridden, ok := byID[testDataWantModel]
|
|
if !ok {
|
|
return fmt.Errorf("GET /v1/models body has no entry for %q: %s", testDataWantModel, authedRec.Body.String())
|
|
}
|
|
if cw, _ := overridden["context_window"].(float64); cw != 128000 {
|
|
return fmt.Errorf("GET /v1/models %q context_window = %v, want 128000 (feature v0.23 config-override metadata harness): %s", testDataWantModel, overridden["context_window"], authedRec.Body.String())
|
|
}
|
|
pricing, _ := overridden["pricing"].(map[string]any)
|
|
if pricing == nil || pricing["input_per_mtok_usd"] != 1.25 || pricing["output_per_mtok_usd"] != 10.0 {
|
|
return fmt.Errorf("GET /v1/models %q pricing = %v, want {input_per_mtok_usd:1.25 output_per_mtok_usd:10} (feature v0.23 config-override metadata harness): %s", testDataWantModel, overridden["pricing"], authedRec.Body.String())
|
|
}
|
|
|
|
// builtinLookupModelID (SHOULD-6, review round) carries NO modelMeta
|
|
// config override — its context_window can only come from an
|
|
// interpreted LOOKUP into the real, 200+-entry builtinModelMetaTable
|
|
// map literal (modelmeta.go's resolveModelMeta, builtin branch),
|
|
// proving that specific path — not just the config-override one
|
|
// above — runs correctly under Yaegi.
|
|
builtinLookedUp, ok := byID[builtinLookupModelID]
|
|
if !ok {
|
|
return fmt.Errorf("GET /v1/models body has no entry for %q: %s", builtinLookupModelID, authedRec.Body.String())
|
|
}
|
|
if cw, _ := builtinLookedUp["context_window"].(float64); int(cw) != builtinContextTokens {
|
|
return fmt.Errorf("GET /v1/models %q context_window = %v, want %d (feature v0.23 builtin-table-lookup metadata harness, SHOULD-6): %s", builtinLookupModelID, builtinLookedUp["context_window"], builtinContextTokens, authedRec.Body.String())
|
|
}
|
|
|
|
if err := exerciseAttemptAccounting(handler); err != nil {
|
|
return err
|
|
}
|
|
|
|
if err := exerciseFailover(handler, failoverBHits); err != nil {
|
|
return err
|
|
}
|
|
|
|
if err := exerciseRequestTimeout(handler); err != nil {
|
|
return err
|
|
}
|
|
|
|
if err := exerciseRequestTimeoutMidBodyStall(handler); err != nil {
|
|
return err
|
|
}
|
|
|
|
if err := exerciseFederatedTooLarge(handler); err != nil {
|
|
return err
|
|
}
|
|
|
|
if err := exerciseFederatedSSENegotiation(handler); err != nil {
|
|
return err
|
|
}
|
|
|
|
if err := exerciseStampedVersion(handler); err != nil {
|
|
return err
|
|
}
|
|
|
|
if err := exerciseMessagesRoute(handler); err != nil {
|
|
return err
|
|
}
|
|
|
|
unauthedReq := httptest.NewRequest(http.MethodGet, "/v1/models", nil)
|
|
unauthedRec := httptest.NewRecorder()
|
|
handler.ServeHTTP(unauthedRec, unauthedReq)
|
|
if unauthedRec.Code != http.StatusUnauthorized {
|
|
return fmt.Errorf("GET /v1/models without a key: status = %d, want 401, body=%s", unauthedRec.Code, unauthedRec.Body.String())
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// exerciseAttemptAccounting drives two real requests through handler —
|
|
// Feature A's (v0.22) per-provider success-rate accounting, exercised
|
|
// end to end under Yaegi: runUnified's attemptRecorder (routes_unified.
|
|
// go) -> retryPolicy.do's context lookup (retry.go) ->
|
|
// limiter.recordProviderAttempt's isTransient/isDeadlineExceeded
|
|
// classification (limits.go, including matchesSentinel's errors.Is call
|
|
// through providers.go's own multi-%w wrapping) -> its fire-and-forget
|
|
// spawn (SHOULD-5) -> buildAdminOverview's own batched providerUsage read
|
|
// (admin.go). Any interpreter-only failure in that chain — a construct
|
|
// `go build`/`go test` cannot catch, exactly the class of bug this
|
|
// harness exists for (SHOULD-4, v0.22 review round) — surfaces here as a
|
|
// non-1/non-0 count or an outright panic under Yaegi.
|
|
//
|
|
// The first request (against "openai", instant upstream) proves the
|
|
// success path: one attempt, zero failures. The second (against
|
|
// slowProviderName, an upstream that never answers within
|
|
// slowRequestDeadline — SHOULD-A, round 2) proves the FAILURE path:
|
|
// exactly what round 1 of this harness claimed to prove but never
|
|
// actually drove — the interpreted plugin never saw a non-nil error at
|
|
// all, so isDeadlineExceeded was never exercised under this harness even
|
|
// while it ran (and passed) under `go test`. THIS is what SHOULD-A was
|
|
// for: the first version of this second request, run against round 2's
|
|
// hand-rolled comma-ok Unwrap-walk fix (limits.go's matchesSentinel,
|
|
// since replaced), FAILED right here — under the interpreter only,
|
|
// despite passing every compiled `go test` row for the identical error
|
|
// shape — because an interpreted interface type does not correctly match
|
|
// a compiled concrete value's method set in Yaegi. matchesSentinel now
|
|
// uses real errors.Is instead (its own doc comment in limits.go has the
|
|
// full account); this harness is what caught the difference.
|
|
// exerciseStampedVersion asserts the interpreted plugin reports
|
|
// stampedCheckVersion in GET /admin/api/overview.
|
|
//
|
|
// This is the proof that stampReleaseVersion actually landed, and so the
|
|
// proof that New's telemetry gate was PASSED rather than short-circuited:
|
|
// the gate compares this same pluginVersion constant against
|
|
// devPluginVersion, so a version of stampedCheckVersion here means the
|
|
// interpreted code went on to call into the vendored oss-telemetry
|
|
// package. Without this assertion a silently failed stamp would leave the
|
|
// dev sentinel in place and the harness would still print OK, claiming
|
|
// coverage it does not have.
|
|
func exerciseStampedVersion(handler http.Handler) error {
|
|
req := httptest.NewRequest(http.MethodGet, "/admin/api/overview", nil)
|
|
req.Header.Set("Authorization", "Bearer "+attemptAccountingAdminAPIKey)
|
|
rec := httptest.NewRecorder()
|
|
handler.ServeHTTP(rec, req)
|
|
if rec.Code != http.StatusOK {
|
|
return fmt.Errorf("GET /admin/api/overview: status = %d, want 200, body=%s", rec.Code, rec.Body.String())
|
|
}
|
|
var overview struct {
|
|
Version string `json:"version"`
|
|
}
|
|
if err := json.Unmarshal(rec.Body.Bytes(), &overview); err != nil {
|
|
return fmt.Errorf("decode GET /admin/api/overview body: %w", err)
|
|
}
|
|
if overview.Version != stampedCheckVersion {
|
|
return fmt.Errorf("interpreted plugin reports version %q, want %q — stampReleaseVersion did not take effect, so New's telemetry gate short-circuited on the dev sentinel and the vendored oss-telemetry call was never exercised under Yaegi", overview.Version, stampedCheckVersion)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// postMessages drives one POST /v1/messages through handler with the given
|
|
// auth header name/value and JSON body, returning the recorder.
|
|
func postMessages(handler http.Handler, hdrName, hdrValue, body string) *httptest.ResponseRecorder {
|
|
req := httptest.NewRequest(http.MethodPost, "/v1/messages", strings.NewReader(body))
|
|
req.Header.Set("Content-Type", "application/json")
|
|
req.Header.Set(hdrName, hdrValue)
|
|
rec := httptest.NewRecorder()
|
|
handler.ServeHTTP(rec, req)
|
|
return rec
|
|
}
|
|
|
|
// exerciseMessagesRoute drives the inbound Anthropic Messages API route
|
|
// under the REAL interpreter: both auth header forms, the streaming
|
|
// rejection, a bad key's error envelope, an unknown model's error
|
|
// envelope, the openai-type TRANSLATED path (openAIRequestFromAnthropic
|
|
// -> chatCompletion -> anthropicResponseFromOpenAI), the anthropic-type
|
|
// PASSTHROUGH path including its own upstream-error branch, and a
|
|
// tool_use round trip through the translated path. This exists because
|
|
// go test never runs under Yaegi — a construct that compiles and passes
|
|
// `go test` can still panic or silently misbehave interpreted (this
|
|
// package's own doc comment has the general case; handleAdapterError's
|
|
// doc comment, routes_unified.go, has this route's own specific history
|
|
// with plain type assertions over errors.As).
|
|
func exerciseMessagesRoute(handler http.Handler) error {
|
|
// 1. x-api-key auth + translated path (openai-type provider).
|
|
rec := postMessages(handler, "x-api-key", testDataUserAPIKey,
|
|
`{"model":"`+testDataWantModel+`","max_tokens":64,"system":"be terse","messages":[{"role":"user","content":[{"type":"text","text":"ping"}]}]}`)
|
|
if rec.Code != http.StatusOK {
|
|
return fmt.Errorf("POST /v1/messages (x-api-key, translated): status = %d, want 200, body=%s", rec.Code, rec.Body.String())
|
|
}
|
|
var msg map[string]any
|
|
if err := json.Unmarshal(rec.Body.Bytes(), &msg); err != nil {
|
|
return fmt.Errorf("POST /v1/messages: decode body: %w (body=%s)", err, rec.Body.String())
|
|
}
|
|
if msg["type"] != "message" || msg["role"] != "assistant" {
|
|
return fmt.Errorf("POST /v1/messages translated: type/role = %v/%v, want message/assistant: %s", msg["type"], msg["role"], rec.Body.String())
|
|
}
|
|
if msg["model"] != testDataWantModel {
|
|
return fmt.Errorf("POST /v1/messages translated: model = %v, want %q (the client's requested alias echoed back, item 4 fix): %s", msg["model"], testDataWantModel, rec.Body.String())
|
|
}
|
|
if msg["stop_reason"] != "end_turn" {
|
|
return fmt.Errorf("POST /v1/messages translated: stop_reason = %v, want end_turn: %s", msg["stop_reason"], rec.Body.String())
|
|
}
|
|
content, _ := msg["content"].([]any)
|
|
if len(content) != 1 {
|
|
return fmt.Errorf("POST /v1/messages translated: content len = %d, want 1: %s", len(content), rec.Body.String())
|
|
}
|
|
block, _ := content[0].(map[string]any)
|
|
if block["type"] != "text" || block["text"] != "hi" {
|
|
return fmt.Errorf("POST /v1/messages translated: content[0] = %v, want text/hi: %s", content[0], rec.Body.String())
|
|
}
|
|
usg, _ := msg["usage"].(map[string]any)
|
|
if usg == nil || usg["input_tokens"] != float64(1) || usg["output_tokens"] != float64(1) {
|
|
return fmt.Errorf("POST /v1/messages translated: usage = %v, want input=1 output=1: %s", msg["usage"], rec.Body.String())
|
|
}
|
|
|
|
// 2. Authorization: Bearer must authenticate the same route.
|
|
bearerRec := postMessages(handler, "Authorization", "Bearer "+testDataUserAPIKey,
|
|
`{"model":"`+testDataWantModel+`","max_tokens":64,"messages":[{"role":"user","content":"ping"}]}`)
|
|
if bearerRec.Code != http.StatusOK {
|
|
return fmt.Errorf("POST /v1/messages (Authorization: Bearer): status = %d, want 200, body=%s", bearerRec.Code, bearerRec.Body.String())
|
|
}
|
|
|
|
// 3. A bad key gets a 401 in the ANTHROPIC error shape.
|
|
badRec := postMessages(handler, "x-api-key", "not-a-real-key",
|
|
`{"model":"`+testDataWantModel+`","max_tokens":64,"messages":[{"role":"user","content":"ping"}]}`)
|
|
if badRec.Code != http.StatusUnauthorized {
|
|
return fmt.Errorf("POST /v1/messages with a bad key: status = %d, want 401, body=%s", badRec.Code, badRec.Body.String())
|
|
}
|
|
if err := assertAnthropicErrorShape("bad key 401", badRec.Body.Bytes()); err != nil {
|
|
return err
|
|
}
|
|
|
|
// 4. "stream": true is rejected explicitly, in the Anthropic shape,
|
|
// as a 400 (item 2 fix, 2026-08-22 review) — NOT 501: a 501 makes
|
|
// anthropic-sdk-python (which retries any status >= 500) turn every
|
|
// streaming call into a three-request retry storm against this
|
|
// route's own rate limit. 400 maps to the SDK's non-retryable
|
|
// BadRequestError.
|
|
streamRec := postMessages(handler, "x-api-key", testDataUserAPIKey,
|
|
`{"model":"`+testDataWantModel+`","max_tokens":64,"stream":true,"messages":[{"role":"user","content":"ping"}]}`)
|
|
if streamRec.Code != http.StatusBadRequest {
|
|
return fmt.Errorf("POST /v1/messages with stream=true: status = %d, want 400, body=%s", streamRec.Code, streamRec.Body.String())
|
|
}
|
|
if err := assertAnthropicErrorShape("stream rejection", streamRec.Body.Bytes()); err != nil {
|
|
return err
|
|
}
|
|
|
|
// 5. An unknown model's 404 must also be Anthropic-shaped.
|
|
unknownRec := postMessages(handler, "x-api-key", testDataUserAPIKey,
|
|
`{"model":"no-such-model-anywhere","max_tokens":64,"messages":[{"role":"user","content":"ping"}]}`)
|
|
if unknownRec.Code != http.StatusNotFound {
|
|
return fmt.Errorf("POST /v1/messages with an unknown model: status = %d, want 404, body=%s", unknownRec.Code, unknownRec.Body.String())
|
|
}
|
|
if err := assertAnthropicErrorShape("unknown model 404", unknownRec.Body.Bytes()); err != nil {
|
|
return err
|
|
}
|
|
|
|
// 6. The anthropic-type PASSTHROUGH branch, interpreted: every field
|
|
// survives except "model", rewritten to the client's own requested
|
|
// alias (item 4 fix) — a routing rewrite, not message-body
|
|
// translation.
|
|
passRec := postMessages(handler, "x-api-key", testDataUserAPIKey,
|
|
`{"model":"claude-test","max_tokens":64,"system":"be terse","messages":[{"role":"user","content":"ping"}]}`)
|
|
if passRec.Code != http.StatusOK {
|
|
return fmt.Errorf("POST /v1/messages (anthropic passthrough): status = %d, want 200, body=%s", passRec.Code, passRec.Body.String())
|
|
}
|
|
var pass map[string]any
|
|
if err := json.Unmarshal(passRec.Body.Bytes(), &pass); err != nil {
|
|
return fmt.Errorf("POST /v1/messages passthrough: decode body: %w (body=%s)", err, passRec.Body.String())
|
|
}
|
|
if pass["id"] != "msg_probe" {
|
|
return fmt.Errorf("POST /v1/messages passthrough: id = %v, want msg_probe (every field but \"model\" must survive the passthrough unchanged): %s", pass["id"], passRec.Body.String())
|
|
}
|
|
if pass["model"] != "claude-test" {
|
|
return fmt.Errorf("POST /v1/messages passthrough: model = %v, want \"claude-test\" (the client's requested alias echoed back, item 4 fix — not anthropicProbeUpstream's own \"claude-upstream-real\"): %s", pass["model"], passRec.Body.String())
|
|
}
|
|
|
|
// 6b. handleAdapterErrorEnvelope's *providerHTTPError type assertion,
|
|
// interpreted — the exact construct class that has broken under Yaegi
|
|
// before (see handleAdapterError's own doc comment, routes_unified.go).
|
|
upErrRec := postMessages(handler, "x-api-key", testDataUserAPIKey,
|
|
`{"model":"claude-test","max_tokens":64,"messages":[{"role":"user","content":"make-it-fail"}]}`)
|
|
if upErrRec.Code != http.StatusTooManyRequests {
|
|
return fmt.Errorf("POST /v1/messages (upstream 429): status = %d, want 429, body=%s", upErrRec.Code, upErrRec.Body.String())
|
|
}
|
|
if err := assertAnthropicErrorShape("upstream 429", upErrRec.Body.Bytes()); err != nil {
|
|
return err
|
|
}
|
|
// A stale X-Llmgw-Cache header on an error response was deliberately
|
|
// NOT asserted here (F6 fix, 2026-08-23 review): this harness's own
|
|
// fixture configures no cache/Redis, so g.cache is nil and the
|
|
// header is never set on any response regardless of what this route
|
|
// does — the assertion could not fail no matter what code ran,
|
|
// confirmed by a mutation pass that deleted the header-clear line it
|
|
// was meant to guard (handleAdapterErrorEnvelope, routes_unified.go)
|
|
// with make yaegi-check still reporting OK. That behavior is already
|
|
// covered where it can actually fail — a real cache fixture, go
|
|
// test: TestHandleChat_CacheableRequest_UpstreamDown_502HasNoCacheHeader
|
|
// (routes_unified_test.go).
|
|
|
|
// 7. A tool_use round trip through the TRANSLATED path, to drive
|
|
// openAIToolCallFromAnthropic/openAIToolMessageFromAnthropic and
|
|
// anthropicToolInputFromOpenAI interpreted.
|
|
toolRec := postMessages(handler, "x-api-key", testDataUserAPIKey,
|
|
`{"model":"`+testDataWantModel+`","max_tokens":64,`+
|
|
`"tools":[{"name":"get_weather","description":"w","input_schema":{"type":"object","properties":{"city":{"type":"string"}}}}],`+
|
|
`"tool_choice":{"type":"auto"},`+
|
|
`"messages":[{"role":"user","content":"weather?"},`+
|
|
`{"role":"assistant","content":[{"type":"tool_use","id":"toolu_1","name":"get_weather","input":{"city":"London"}}]},`+
|
|
`{"role":"user","content":[{"type":"tool_result","tool_use_id":"toolu_1","content":"sunny"}]}]}`)
|
|
if toolRec.Code != http.StatusOK {
|
|
return fmt.Errorf("POST /v1/messages (tool_use round trip): status = %d, want 200, body=%s", toolRec.Code, toolRec.Body.String())
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// assertAnthropicErrorShape checks b is {"type":"error","error":{"type":...,
|
|
// "message":...}} — what an Anthropic SDK client parses.
|
|
func assertAnthropicErrorShape(label string, b []byte) error {
|
|
var env map[string]any
|
|
if err := json.Unmarshal(b, &env); err != nil {
|
|
return fmt.Errorf("/v1/messages %s: body is not JSON: %w (body=%s)", label, err, string(b))
|
|
}
|
|
if env["type"] != "error" {
|
|
return fmt.Errorf("/v1/messages %s: top-level \"type\" = %v, want \"error\" (Anthropic envelope): %s", label, env["type"], string(b))
|
|
}
|
|
inner, _ := env["error"].(map[string]any)
|
|
if inner == nil {
|
|
return fmt.Errorf("/v1/messages %s: no \"error\" object: %s", label, string(b))
|
|
}
|
|
if _, ok := inner["type"].(string); !ok {
|
|
return fmt.Errorf("/v1/messages %s: error.type missing: %s", label, string(b))
|
|
}
|
|
if _, ok := inner["message"].(string); !ok {
|
|
return fmt.Errorf("/v1/messages %s: error.message missing: %s", label, string(b))
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// exerciseFederatedTooLarge drives POST /mcp (federation's tools/call
|
|
// path) against mcpProbeUpstream, whose response deliberately exceeds
|
|
// mcpBackendCallResponseMaxBytes, and asserts the interpreted plugin
|
|
// answers with mcpTooLargeWantMessage rather than the generic "upstream
|
|
// error".
|
|
//
|
|
// What this actually pins is errors.Is under the interpreter. A security
|
|
// review round replaced that errors.Is with a strings.Contains marker on
|
|
// the grounds that no gate exercised this path interpreted — true at the
|
|
// time, and the honest objection. The reasoning for errors.Is is sound
|
|
// (errors.New and fmt.Errorf both return COMPILED values, so declaring
|
|
// the sentinel variable in interpreted code leaves the whole chain
|
|
// errors.Is walks compiled, which is why retry.go's isTransient has
|
|
// worked in production since v0.2.0), and a standalone yaegi v0.16.1
|
|
// harness confirmed it. But this codebase's own history — matchesSentinel
|
|
// (limits.go) — is a case where sound reasoning about the reflect
|
|
// boundary was wrong and only an interpreted probe caught it. So the
|
|
// path is covered here rather than argued in a comment.
|
|
//
|
|
// A failure surfaces as "upstream error": errors.Is returning false
|
|
// interpreted for a chain that is true compiled. No `go test` row can
|
|
// see that difference.
|
|
func exerciseFederatedTooLarge(handler http.Handler) error {
|
|
body := `{"jsonrpc":"2.0","id":7,"method":"tools/call","params":{"name":"` + mcpProbeToolName + `","arguments":{}}}`
|
|
req := httptest.NewRequest(http.MethodPost, "/mcp", strings.NewReader(body))
|
|
req.Header.Set("Authorization", "Bearer "+testDataUserAPIKey)
|
|
req.Header.Set("Content-Type", "application/json")
|
|
rec := httptest.NewRecorder()
|
|
handler.ServeHTTP(rec, req)
|
|
if rec.Code != http.StatusOK {
|
|
return fmt.Errorf("POST /mcp tools/call: status = %d, want 200 (JSON-RPC reports the failure in the body, not the status), body=%s", rec.Code, rec.Body.String())
|
|
}
|
|
|
|
var resp struct {
|
|
Error *struct {
|
|
Message string `json:"message"`
|
|
} `json:"error"`
|
|
}
|
|
if err := json.Unmarshal(rec.Body.Bytes(), &resp); err != nil {
|
|
return fmt.Errorf("decode POST /mcp tools/call body: %w (body=%s)", err, rec.Body.String())
|
|
}
|
|
if resp.Error == nil {
|
|
return fmt.Errorf("POST /mcp tools/call returned no JSON-RPC error, want %q — the probe upstream's oversized response should have tripped the cap: %s", mcpTooLargeWantMessage, rec.Body.String())
|
|
}
|
|
if resp.Error.Message != mcpTooLargeWantMessage {
|
|
return fmt.Errorf("POST /mcp tools/call error message = %q, want %q — errors.Is(err, errMCPResponseTooLarge) did not match under the interpreter (mcp_federation.go)", resp.Error.Message, mcpTooLargeWantMessage)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// targetHealthProbeView is a shape-tolerant decode of one adminTargetView
|
|
// entry's own "health" object (admin.go's adminTargetHealthView) —
|
|
// this harness module cannot import the plugin's unexported type across
|
|
// the Yaegi reflection boundary (New is resolved and called by
|
|
// reflection, not a normal Go import — this whole file's own doc
|
|
// comment explains why), so it decodes the JSON shape independently, the
|
|
// same approach readHealth/readProviderAttemptCounters already take for
|
|
// other admin responses.
|
|
type targetHealthProbeView struct {
|
|
State string `json:"state"`
|
|
Source string `json:"source"`
|
|
}
|
|
|
|
// pollTargetHealth drives GET /admin/api/targets against handler
|
|
// repeatedly until name's health satisfies want, or a 5s budget elapses,
|
|
// then returns its final reading either way — mirrors
|
|
// pollProviderAttemptCounters's identical poll-with-deadline shape
|
|
// (below), applied here to feat/target-health's own async sweep
|
|
// (maybeSweepTargetHealth spawns the actual probing goroutine and
|
|
// returns immediately, target_health.go's own doc comment) instead of
|
|
// limiter.spawn's async counter write.
|
|
func pollTargetHealth(handler http.Handler, name string, want func(targetHealthProbeView) bool) (targetHealthProbeView, error) {
|
|
deadline := time.Now().Add(5 * time.Second)
|
|
for {
|
|
health, err := readTargetHealth(handler, name)
|
|
if err != nil {
|
|
return targetHealthProbeView{}, err
|
|
}
|
|
if want(health) || time.Now().After(deadline) {
|
|
return health, nil
|
|
}
|
|
time.Sleep(10 * time.Millisecond)
|
|
}
|
|
}
|
|
|
|
// readTargetHealth returns name's own "health" object from GET
|
|
// /admin/api/targets, searching both mcpServers and agents (mcpProbeServerName
|
|
// and targetHealthDownServerName are both MCP servers, but this checks
|
|
// both arrays so a future agent-shaped call site can reuse it unchanged).
|
|
func readTargetHealth(handler http.Handler, name string) (targetHealthProbeView, error) {
|
|
req := httptest.NewRequest(http.MethodGet, "/admin/api/targets", nil)
|
|
req.Header.Set("Authorization", "Bearer "+attemptAccountingAdminAPIKey)
|
|
rec := httptest.NewRecorder()
|
|
handler.ServeHTTP(rec, req)
|
|
if rec.Code != http.StatusOK {
|
|
return targetHealthProbeView{}, fmt.Errorf("GET /admin/api/targets: status = %d, want 200, body=%s", rec.Code, rec.Body.String())
|
|
}
|
|
var body struct {
|
|
MCPServers []struct {
|
|
Name string `json:"name"`
|
|
Health targetHealthProbeView `json:"health"`
|
|
} `json:"mcpServers"`
|
|
Agents []struct {
|
|
Name string `json:"name"`
|
|
Health targetHealthProbeView `json:"health"`
|
|
} `json:"agents"`
|
|
}
|
|
if err := json.Unmarshal(rec.Body.Bytes(), &body); err != nil {
|
|
return targetHealthProbeView{}, fmt.Errorf("decode GET /admin/api/targets body: %w (body=%s)", err, rec.Body.String())
|
|
}
|
|
for _, s := range body.MCPServers {
|
|
if s.Name == name {
|
|
return s.Health, nil
|
|
}
|
|
}
|
|
for _, a := range body.Agents {
|
|
if a.Name == name {
|
|
return a.Health, nil
|
|
}
|
|
}
|
|
return targetHealthProbeView{}, fmt.Errorf("target %q not in GET /admin/api/targets: %s", name, rec.Body.String())
|
|
}
|
|
|
|
// exerciseTargetHealthProbeWarmup proves feat/target-health's ACTIVE
|
|
// PROBE sweep end to end under the REAL interpreter, and — just as
|
|
// important — makes every LATER target-health assertion in this harness
|
|
// deterministic. Called from run() BEFORE exerciseHandler, i.e. before
|
|
// any other route this harness hits: GET /admin/api/targets is the
|
|
// first-ever sweep-capable route (maybeSweepTargetHealth's own doc
|
|
// comment, target_health.go, lists the four: serveMetrics,
|
|
// serveAdminTargets, handleMCPServers, handleAgents) this handler sees,
|
|
// so it deterministically owns the ONE sweep this gateway's
|
|
// probeInterval (configured to "60s", well above this harness's
|
|
// real-world runtime) allows for the rest of the run.
|
|
//
|
|
// This matters because targetHealthProbeOnlyServerName's own record() is
|
|
// not the only one a sweep touches — EVERY configured MCP server and
|
|
// agent gets probed in the same sweep (sweepTargetHealth's own doc
|
|
// comment), mcpProbeServerName included. Spending that one sweep here,
|
|
// before exerciseFederatedTooLarge ever drives mcpProbeServerName's real
|
|
// traffic observation, guarantees the traffic observation is the LAST
|
|
// write to that entry for the remainder of the run — no later
|
|
// admin/metrics/listing route can trigger a second sweep to race it
|
|
// (F5, feat/target-health review: this is what closes the race that
|
|
// used to make TARGETHEALTH-2, below, racy under Yaegi's interpreter
|
|
// overhead).
|
|
//
|
|
// G5, feat/target-health review round 2: that guarantee only holds once
|
|
// EVERY configured MCP server's own probe observation has actually
|
|
// landed, not merely targetHealthProbeOnlyServerName's. sweepTargetHealth
|
|
// probes every configured server CONCURRENTLY, on its own goroutines —
|
|
// waiting for one target's record() to land says nothing about whether
|
|
// mcpProbeServerName's own probe goroutine has finished yet. A version of
|
|
// this function that returned as soon as targetHealthProbeOnlyServerName
|
|
// alone changed could return while mcpProbeServerName's probe write was
|
|
// still in flight; that write could then land AFTER
|
|
// exerciseFederatedTooLarge's later traffic write, silently reverting
|
|
// mcpProbeServerName's source back to "probe" and corrupting
|
|
// exerciseTargetHealth's own TARGETHEALTH-2 assertion (source ==
|
|
// "traffic"). Polling every configured MCP server for its own source ==
|
|
// "probe" closes that gap: this function cannot return until the whole
|
|
// sweep — not just the one target it happens to check last — has
|
|
// finished writing.
|
|
//
|
|
// targetHealthProbeOnlyServerName is additionally asserted to the FULL
|
|
// "healthy" state, not just source == "probe": it is NEVER proxied or
|
|
// federated through by any other exercise in this harness, so its health
|
|
// entry can only ever be written by the probe — recover's own swallow
|
|
// (sweepTargetHealth's defer, target_health.go) means an interpreted
|
|
// panic mid-probe would otherwise degrade to a silently-stuck "unknown"
|
|
// rather than a loud failure, and asserting the full success shape here
|
|
// is what makes a Yaegi incompatibility in the initialize/close handshake
|
|
// path visible instead of passing silently.
|
|
func exerciseTargetHealthProbeWarmup(handler http.Handler) error {
|
|
warmupServerNames := []string{mcpProbeServerName, targetHealthDownServerName, targetHealthProbeOnlyServerName}
|
|
var probeOnlyHealth targetHealthProbeView
|
|
for _, name := range warmupServerNames {
|
|
health, err := pollTargetHealth(handler, name, func(h targetHealthProbeView) bool {
|
|
return h.Source == "probe"
|
|
})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if health.Source != "probe" {
|
|
return fmt.Errorf("TARGETHEALTH-PROBE-1: GET /admin/api/targets: %q health.source = %q, want %q (the warmup sweep must probe every configured MCP server before any later exercise runs)", name, health.Source, "probe")
|
|
}
|
|
if name == targetHealthProbeOnlyServerName {
|
|
probeOnlyHealth = health
|
|
}
|
|
}
|
|
if probeOnlyHealth.State != "healthy" {
|
|
return fmt.Errorf("TARGETHEALTH-PROBE-2: GET /admin/api/targets: %q health.state = %q, want healthy (targetHealthProbeOnlyUpstream always answers initialize successfully)", targetHealthProbeOnlyServerName, probeOnlyHealth.State)
|
|
}
|
|
|
|
fmt.Println("yaegi-check: target health active probe sweep ran and recorded correctly interpreted")
|
|
return nil
|
|
}
|
|
|
|
// exerciseTargetHealth proves feat/target-health's PASSIVE recording end
|
|
// to end under the REAL interpreter, now that
|
|
// exerciseTargetHealthProbeWarmup (run() calls it first, above) has
|
|
// already spent this gateway's one probeInterval-bounded sweep — no
|
|
// further sweep can fire during this function, so neither part below
|
|
// races the active probe path.
|
|
//
|
|
// Part (a): mcpProbeServerName receives one real federated tools/call
|
|
// attempt via exerciseFederatedTooLarge (exerciseHandler, called before
|
|
// this function) — its oversized response trips errMCPResponseTooLarge,
|
|
// a transport-level failure from doBackendJSONRPC's own perspective
|
|
// (mcp_federation.go), which mcpFederatedToolsCall records as a
|
|
// target-health failure. This is the ONLY observation this target ever
|
|
// receives (the warmup above never touches it — see
|
|
// exerciseTargetHealthProbeWarmup's own doc comment for why spending the
|
|
// sweep before this call matters), so it has never once succeeded:
|
|
// F4 (feat/target-health review) reads that as "unknown", not "healthy"
|
|
// — the failure-threshold gate still protects against this single
|
|
// transient-looking failure, so it does not read "unhealthy" either.
|
|
//
|
|
// Part (b): drives targetHealthFailureThreshold consecutive failures
|
|
// through the direct per-server proxy path (/mcp/{name}/...,
|
|
// handleTargetProxy) against targetHealthDownServerName's dead upstream,
|
|
// then asserts the SAME tracker flips that target to "unhealthy" on
|
|
// both GET /admin/api/targets and llmgateway_target_healthy on
|
|
// /metrics.
|
|
func exerciseTargetHealth(handler http.Handler) error {
|
|
probeHealth, err := readTargetHealth(handler, mcpProbeServerName)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if probeHealth.State != "unknown" {
|
|
return fmt.Errorf("TARGETHEALTH-1: GET /admin/api/targets: %q health.state = %q, want unknown (one failed federated tools/call attempt, never having succeeded, below threshold — F4): %+v", mcpProbeServerName, probeHealth.State, probeHealth)
|
|
}
|
|
if probeHealth.Source != "traffic" {
|
|
return fmt.Errorf("TARGETHEALTH-2: GET /admin/api/targets: %q health.source = %q, want %q", mcpProbeServerName, probeHealth.Source, "traffic")
|
|
}
|
|
|
|
for i := 0; i < targetHealthFailureThreshold; i++ {
|
|
req := httptest.NewRequest(http.MethodGet, "/mcp/"+targetHealthDownServerName+"/tools/list", nil)
|
|
req.Header.Set("Authorization", "Bearer "+testDataUserAPIKey)
|
|
rec := httptest.NewRecorder()
|
|
handler.ServeHTTP(rec, req)
|
|
if rec.Code != http.StatusBadGateway {
|
|
return fmt.Errorf("TARGETHEALTH-3: GET /mcp/%s/tools/list attempt %d: status = %d, want 502 (dead upstream), body=%s", targetHealthDownServerName, i, rec.Code, rec.Body.String())
|
|
}
|
|
}
|
|
|
|
downHealth, err := readTargetHealth(handler, targetHealthDownServerName)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if downHealth.State != "unhealthy" {
|
|
return fmt.Errorf("TARGETHEALTH-4: GET /admin/api/targets: %q health.state = %q, want unhealthy after %d consecutive failures: %+v", targetHealthDownServerName, downHealth.State, targetHealthFailureThreshold, downHealth)
|
|
}
|
|
|
|
metricsReq := httptest.NewRequest(http.MethodGet, "/metrics", nil)
|
|
metricsReq.Header.Set("Authorization", "Bearer "+attemptAccountingAdminAPIKey)
|
|
metricsRec := httptest.NewRecorder()
|
|
handler.ServeHTTP(metricsRec, metricsReq)
|
|
if metricsRec.Code != http.StatusOK {
|
|
return fmt.Errorf("GET /metrics: status = %d, want 200, body=%s", metricsRec.Code, metricsRec.Body.String())
|
|
}
|
|
wantSample := `llmgateway_target_healthy{kind="mcp",target="` + targetHealthDownServerName + `"} 0`
|
|
if !strings.Contains(metricsRec.Body.String(), wantSample) {
|
|
return fmt.Errorf("TARGETHEALTH-5: GET /metrics missing %q — interpreted target-health metric diverged from the compiled shape; body=%s", wantSample, metricsRec.Body.String())
|
|
}
|
|
|
|
fmt.Println("yaegi-check: target health tracked and exposed correctly interpreted (admin API + /metrics)")
|
|
return nil
|
|
}
|
|
|
|
// exerciseFederatedSSENegotiation drives two real POST /mcp "ping" calls
|
|
// against the interpreted handler — one carrying no Accept header
|
|
// (today's plain JSON framing) and one carrying the reported client's
|
|
// exact "Accept: application/json, text/event-stream" header — proving
|
|
// content negotiation (mcpNegotiateFormat, mcpAcceptsSSE) and the SSE
|
|
// branch's reuse of sse.go's newSSEWriter/writeData both run correctly
|
|
// under the REAL interpreter. mcp_federation_sse_test.go already covers
|
|
// this compiled; this harness exists because a compiled pass on this
|
|
// codebase has repeatedly said nothing about the interpreted shape.
|
|
//
|
|
// "ping" is deliberately chosen over tools/list or tools/call: it is
|
|
// answered entirely locally (handleMCPFederated's own doc comment), so
|
|
// this probe needs no additional upstream server and cannot be confused
|
|
// with mcpProbeUpstream's own, deliberately oversized, tools/call-only
|
|
// response above.
|
|
//
|
|
// This is also the first time under this harness that sse.go's
|
|
// newSSEWriter/writeData actually executes interpreted at all. The
|
|
// streaming (chat completions) path is documented as losing its
|
|
// incremental Flush under Yaegi (newSSEWriter's own doc comment,
|
|
// confirmed by Task 15's integration suite) — harmless here, since this
|
|
// is a single-shot envelope that writes once and returns rather than a
|
|
// response that depends on Flush for time-to-first-byte, but it means no
|
|
// existing gate has ever run this writer interpreted before now.
|
|
func exerciseFederatedSSENegotiation(handler http.Handler) error {
|
|
pingBody := `{"jsonrpc":"2.0","id":1,"method":"ping"}`
|
|
|
|
jsonReq := httptest.NewRequest(http.MethodPost, "/mcp", strings.NewReader(pingBody))
|
|
jsonReq.Header.Set("Authorization", "Bearer "+testDataUserAPIKey)
|
|
jsonReq.Header.Set("Content-Type", "application/json")
|
|
jsonRec := httptest.NewRecorder()
|
|
handler.ServeHTTP(jsonRec, jsonReq)
|
|
if jsonRec.Code != http.StatusOK {
|
|
return fmt.Errorf("POST /mcp ping (no Accept header): status = %d, want 200, body=%s", jsonRec.Code, jsonRec.Body.String())
|
|
}
|
|
if ct := jsonRec.Header().Get("Content-Type"); ct != "application/json" {
|
|
return fmt.Errorf("POST /mcp ping (no Accept header): Content-Type = %q, want %q — default framing must not change interpreted", ct, "application/json")
|
|
}
|
|
|
|
sseReq := httptest.NewRequest(http.MethodPost, "/mcp", strings.NewReader(pingBody))
|
|
sseReq.Header.Set("Authorization", "Bearer "+testDataUserAPIKey)
|
|
sseReq.Header.Set("Content-Type", "application/json")
|
|
sseReq.Header.Set("Accept", "application/json, text/event-stream")
|
|
sseRec := httptest.NewRecorder()
|
|
handler.ServeHTTP(sseRec, sseReq)
|
|
if sseRec.Code != http.StatusOK {
|
|
return fmt.Errorf("POST /mcp ping (Accept: text/event-stream): status = %d, want 200, body=%s", sseRec.Code, sseRec.Body.String())
|
|
}
|
|
if ct := sseRec.Header().Get("Content-Type"); ct != "text/event-stream" {
|
|
return fmt.Errorf("POST /mcp ping (Accept: text/event-stream): Content-Type = %q, want %q — SSE negotiation did not run interpreted", ct, "text/event-stream")
|
|
}
|
|
|
|
body := sseRec.Body.String()
|
|
if !strings.HasPrefix(body, "data: ") {
|
|
return fmt.Errorf("POST /mcp ping (Accept: text/event-stream): body does not start with \"data: \": %q", body)
|
|
}
|
|
if !strings.HasSuffix(body, "\n\n") {
|
|
return fmt.Errorf("POST /mcp ping (Accept: text/event-stream): body does not end with the SSE blank-line terminator — the exact missing-terminator defect class this negotiation was added to fix: %q", body)
|
|
}
|
|
|
|
dataLine := strings.TrimSuffix(strings.TrimPrefix(body, "data: "), "\n\n")
|
|
var resp struct {
|
|
Result map[string]any `json:"result"`
|
|
ID json.RawMessage `json:"id"`
|
|
}
|
|
if err := json.Unmarshal([]byte(dataLine), &resp); err != nil {
|
|
return fmt.Errorf("POST /mcp ping (Accept: text/event-stream): data line is not valid JSON-RPC: %w (line=%q)", err, dataLine)
|
|
}
|
|
if string(resp.ID) != "1" {
|
|
return fmt.Errorf("POST /mcp ping (Accept: text/event-stream): id = %s, want the client's own id 1 echoed back", resp.ID)
|
|
}
|
|
|
|
fmt.Println("yaegi-check: federated MCP content negotiation served both JSON and SSE framing correctly, interpreted")
|
|
return nil
|
|
}
|
|
|
|
// exerciseFailover drives one real POST /v1/chat/completions against the
|
|
// interpreted handler for failoverModelID — a bare model id both
|
|
// "failover-a" (always 500s) and "failover-b" (always 200s) are
|
|
// configured with — proving the failover candidate loop (runMeteredCall,
|
|
// routes_unified.go/failover.go) falls through a genuine provider
|
|
// failure and serves the second provider's response under the REAL
|
|
// interpreter. Compiled tests already prove this (failover_test.go); this
|
|
// harness exists because compiled tests have repeatedly passed on this
|
|
// codebase while the interpreted shape failed (this package's own doc
|
|
// comment).
|
|
func exerciseFailover(handler http.Handler, bHits *int64) error {
|
|
req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(
|
|
`{"model":"`+failoverModelID+`","messages":[{"role":"user","content":"hi"}]}`,
|
|
))
|
|
req.Header.Set("Authorization", "Bearer "+testDataUserAPIKey)
|
|
req.Header.Set("Content-Type", "application/json")
|
|
rec := httptest.NewRecorder()
|
|
handler.ServeHTTP(rec, req)
|
|
|
|
if rec.Code != http.StatusOK {
|
|
return fmt.Errorf("POST /v1/chat/completions (feat/failover harness): status = %d, want 200 (failover-a's 500 must fall through to failover-b), body=%s", rec.Code, rec.Body.String())
|
|
}
|
|
if !strings.Contains(rec.Body.String(), "served by failover-b") {
|
|
return fmt.Errorf("POST /v1/chat/completions (feat/failover harness): body does not contain failover-b's own content, want the SECOND provider's response: %s", rec.Body.String())
|
|
}
|
|
if atomic.LoadInt64(bHits) != 1 {
|
|
return fmt.Errorf("feat/failover harness: failover-b upstream hit count = %d, want exactly 1", atomic.LoadInt64(bHits))
|
|
}
|
|
fmt.Println("yaegi-check: failover fell through from failover-a to failover-b and served its response")
|
|
return nil
|
|
}
|
|
|
|
// exerciseBareWinnerGroupAware proves registry.go's group-aware bareWinner
|
|
// fix (2026-08-27) under the REAL interpreter: "bare-winner-a" and
|
|
// "bare-winner-b" both serve bareWinnerModelID; bareWinnerGroupName may
|
|
// use "bare-winner-b" only, yet "bare-winner-a" sorts first
|
|
// alphabetically. Before the fix, resolve's bareWinner call always
|
|
// returned the GLOBAL sorted-first owner regardless of the caller's
|
|
// group, so this exact request got errModelDenied for a model the group
|
|
// was actually entitled to use — this is the production repro the fix
|
|
// exists for (bug report: "friends" group entitled to "uni", denied a
|
|
// bare id because "macstudio" sorts first and also serves it).
|
|
//
|
|
// Unlike exerciseFailover (immediately above, same colliding-bare-id
|
|
// shape): a correct group-aware bareWinner never even CONSIDERS
|
|
// "bare-winner-a" as a candidate for this group's request — it is not a
|
|
// failover fall-through after a failed attempt. aHits staying at 0 proves
|
|
// that: a pre-fix, group-blind bareWinner would route here first, and
|
|
// "bare-winner-a"'s upstream (run(), above) answers 500 with a body
|
|
// naming the exact wrong outcome, so a regression is unambiguous either
|
|
// way (aHits != 0, or a non-200 status, or the wrong response body).
|
|
//
|
|
// MUTATION VERIFIED: temporarily reverting bareWinner (registry.go) to
|
|
// its pre-fix, group-blind body (ignore grp entirely, always return the
|
|
// first provider in m.providerNames whose known model set contains id)
|
|
// made this fail with "status = 403" (errModelDenied) — bareWinnerGroupName
|
|
// cannot use "bare-winner-a", the pre-fix global winner. Reverted before
|
|
// committing.
|
|
func exerciseBareWinnerGroupAware(handler http.Handler, aHits *int64) error {
|
|
req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(
|
|
`{"model":"`+bareWinnerModelID+`","messages":[{"role":"user","content":"hi"}]}`,
|
|
))
|
|
req.Header.Set("Authorization", "Bearer "+bareWinnerFriendAPIKey)
|
|
req.Header.Set("Content-Type", "application/json")
|
|
rec := httptest.NewRecorder()
|
|
handler.ServeHTTP(rec, req)
|
|
|
|
if rec.Code != http.StatusOK {
|
|
return fmt.Errorf("POST /v1/chat/completions (fix/group-aware-bare-winner harness): status = %d, want 200 (%s is entitled to \"bare-winner-b\", which serves the bare id), body=%s", rec.Code, bareWinnerGroupName, rec.Body.String())
|
|
}
|
|
if !strings.Contains(rec.Body.String(), "served by bare-winner-b") {
|
|
return fmt.Errorf("POST /v1/chat/completions (fix/group-aware-bare-winner harness): body does not contain bare-winner-b's own content, want the group-visible provider's response: %s", rec.Body.String())
|
|
}
|
|
if got := atomic.LoadInt64(aHits); got != 0 {
|
|
return fmt.Errorf("fix/group-aware-bare-winner harness: bare-winner-a upstream hit count = %d, want exactly 0 (%s can never reach bare-winner-a)", got, bareWinnerGroupName)
|
|
}
|
|
|
|
// GET /v1/models, same group: listFor's own half of the fix
|
|
// (registry.go) — the bare id must be attributed to "bare-winner-b"
|
|
// (the group's own visible winner), and "bare-winner-a" must never
|
|
// appear anywhere in this group's catalog, bare or prefixed.
|
|
listReq := httptest.NewRequest(http.MethodGet, "/v1/models", nil)
|
|
listReq.Header.Set("Authorization", "Bearer "+bareWinnerFriendAPIKey)
|
|
listRec := httptest.NewRecorder()
|
|
handler.ServeHTTP(listRec, listReq)
|
|
if listRec.Code != http.StatusOK {
|
|
return fmt.Errorf("GET /v1/models (fix/group-aware-bare-winner harness): status = %d, want 200, body=%s", listRec.Code, listRec.Body.String())
|
|
}
|
|
if strings.Contains(listRec.Body.String(), "bare-winner-a") {
|
|
return fmt.Errorf("GET /v1/models (fix/group-aware-bare-winner harness): body names bare-winner-a, want it invisible to %s entirely: %s", bareWinnerGroupName, listRec.Body.String())
|
|
}
|
|
var listBody struct {
|
|
Data []map[string]any `json:"data"`
|
|
}
|
|
if err := json.Unmarshal(listRec.Body.Bytes(), &listBody); err != nil {
|
|
return fmt.Errorf("decode GET /v1/models body (fix/group-aware-bare-winner harness): %w (body=%s)", err, listRec.Body.String())
|
|
}
|
|
found := false
|
|
for _, entry := range listBody.Data {
|
|
if entry["id"] != bareWinnerModelID {
|
|
continue
|
|
}
|
|
found = true
|
|
if entry["owned_by"] != "bare-winner-b" {
|
|
return fmt.Errorf("GET /v1/models entry %v: owned_by = %v, want %q (this group's own visible winner)", entry, entry["owned_by"], "bare-winner-b")
|
|
}
|
|
}
|
|
if !found {
|
|
return fmt.Errorf("GET /v1/models body has no bare entry for %q: %s", bareWinnerModelID, listRec.Body.String())
|
|
}
|
|
|
|
fmt.Println("yaegi-check: group-aware bareWinner resolved and listed bare-winner-shared via bare-winner-b, never touching bare-winner-a")
|
|
return nil
|
|
}
|
|
|
|
// exerciseMultiGroupPersonalGrant proves the multi-group/personal-grant
|
|
// feature (UserConfig.Groups/Providers/Models, llmgateway.go) resolves
|
|
// correctly under the REAL interpreter: multiGroupUserAPIKey's own
|
|
// principal belongs to TWO groups (multiGroupEngGroupName,
|
|
// multiGroupOpsGroupName — neither reaching "multi-group-b" at all)
|
|
// plus a personal grant restricting "multi-group-b" to
|
|
// multiGroupAllowedModelID alone. A request for the personal grant's own
|
|
// allowed model must succeed; a request for a DIFFERENT model on the
|
|
// SAME provider must be denied — the no-cross-grant-leak rule
|
|
// (group.allowsProviderModel, auth.go) a naive union-based check would
|
|
// get wrong.
|
|
func exerciseMultiGroupPersonalGrant(handler http.Handler, bHits *int64) error {
|
|
allowedBody := `{"model":"` + multiGroupAllowedModelID + `","messages":[{"role":"user","content":"hi"}]}`
|
|
allowedReq := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(allowedBody))
|
|
allowedReq.Header.Set("Authorization", "Bearer "+multiGroupUserAPIKey)
|
|
allowedReq.Header.Set("Content-Type", "application/json")
|
|
allowedRec := httptest.NewRecorder()
|
|
handler.ServeHTTP(allowedRec, allowedReq)
|
|
if allowedRec.Code != http.StatusOK {
|
|
return fmt.Errorf("POST /v1/chat/completions (multi-group/personal-grant harness, allowed model): status = %d, want 200, body=%s", allowedRec.Code, allowedRec.Body.String())
|
|
}
|
|
if !strings.Contains(allowedRec.Body.String(), "served by multi-group-b") {
|
|
return fmt.Errorf("POST /v1/chat/completions (multi-group/personal-grant harness, allowed model): body does not contain multi-group-b's own content: %s", allowedRec.Body.String())
|
|
}
|
|
if got := atomic.LoadInt64(bHits); got != 1 {
|
|
return fmt.Errorf("multi-group/personal-grant harness: multi-group-b upstream hit count = %d, want exactly 1 after the allowed request", got)
|
|
}
|
|
|
|
deniedBody := `{"model":"` + multiGroupDeniedModelID + `","messages":[{"role":"user","content":"hi"}]}`
|
|
deniedReq := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(deniedBody))
|
|
deniedReq.Header.Set("Authorization", "Bearer "+multiGroupUserAPIKey)
|
|
deniedReq.Header.Set("Content-Type", "application/json")
|
|
deniedRec := httptest.NewRecorder()
|
|
handler.ServeHTTP(deniedRec, deniedReq)
|
|
if deniedRec.Code != http.StatusForbidden {
|
|
return fmt.Errorf("POST /v1/chat/completions (multi-group/personal-grant harness, cross-grant model): status = %d, want 403 (the personal grant's own Models list denies %q; neither member group's grant may leak into authorizing it), body=%s", deniedRec.Code, multiGroupDeniedModelID, deniedRec.Body.String())
|
|
}
|
|
if got := atomic.LoadInt64(bHits); got != 1 {
|
|
return fmt.Errorf("multi-group/personal-grant harness: multi-group-b upstream hit count = %d, want still exactly 1 (the denied request must never reach the upstream)", got)
|
|
}
|
|
|
|
fmt.Println("yaegi-check: multi-group user with a personal grant resolved the personal-grant provider's allowed model and was denied a cross-grant model")
|
|
return nil
|
|
}
|
|
|
|
// exerciseMultiGroupBareWinnerFailoverAndPassthroughPaths extends the
|
|
// multi-group/personal-grant probe above (LOW-8, review round 2) to
|
|
// additionally assert, under the REAL interpreter:
|
|
//
|
|
// - HIGH-2: bareWinner/listFor prefer a grant that actually authorizes
|
|
// a bare-id collision, not merely the sorted-first PROVIDER-visible
|
|
// owner — resolving multiGroupSharedModelID (a bare, three-provider
|
|
// collision: "multi-group-b" < "multi-group-c" <
|
|
// "multi-group-shared-x") must land on "multi-group-c"
|
|
// (multiGroupOpsGroupName's own unrestricted grant), never get
|
|
// denied via "multi-group-b" (the personal grant's own, narrower
|
|
// one, which sorts first but does not cover this id).
|
|
// - The SAME multi-grant principal's failover pool correctly includes
|
|
// "multi-group-shared-x" (multiGroupEngGroupName's own grant) and
|
|
// correctly excludes "multi-group-b" — multiGroupCUpstream always
|
|
// 500s its chat-completions call, forcing the fall-through past the
|
|
// correctly-chosen primary.
|
|
// - GET /v1/models shows exactly the union this principal is entitled
|
|
// to for multiGroupSharedModelID: the bare id (owned by
|
|
// "multi-group-c"), "multi-group-c/"+id, and
|
|
// "multi-group-shared-x/"+id — never "multi-group-b/"+id.
|
|
// - HIGH-1: a native passthrough request to "multi-group-c" is denied
|
|
// on multiGroupDeniedPathID and allowed on multiGroupAllowedPathID
|
|
// (multiGroupOpsGroupName's own "passthroughPaths"), while
|
|
// "multi-group-shared-x" (multiGroupEngGroupName's own, unrestricted
|
|
// provider) stays reachable on an arbitrary path either way —
|
|
// proving passthroughPaths stays coupled to the SAME grant that
|
|
// authorizes the provider, never unioned across this principal's
|
|
// member groups.
|
|
func exerciseMultiGroupBareWinnerFailoverAndPassthroughPaths(handler http.Handler, bHits, cChatHits, cPassthroughHits, sharedXHits *int64) error {
|
|
bHitsBefore := atomic.LoadInt64(bHits) // exerciseMultiGroupPersonalGrant already drove one hit
|
|
|
|
// --- HIGH-2 + failover ---
|
|
body := `{"model":"` + multiGroupSharedModelID + `","messages":[{"role":"user","content":"hi"}]}`
|
|
req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(body))
|
|
req.Header.Set("Authorization", "Bearer "+multiGroupUserAPIKey)
|
|
req.Header.Set("Content-Type", "application/json")
|
|
rec := httptest.NewRecorder()
|
|
handler.ServeHTTP(rec, req)
|
|
if rec.Code != http.StatusOK {
|
|
return fmt.Errorf("POST /v1/chat/completions (multi-group bareWinner/failover harness): status = %d, want 200 (bareWinner must land on \"multi-group-c\" via multiGroupOpsGroupName's own grant, then fail over to \"multi-group-shared-x\" once multi-group-c's own upstream 500s), body=%s", rec.Code, rec.Body.String())
|
|
}
|
|
if !strings.Contains(rec.Body.String(), "served by multi-group-shared-x") {
|
|
return fmt.Errorf("POST /v1/chat/completions (multi-group bareWinner/failover harness): body does not contain multi-group-shared-x's own content, want the failover candidate's response: %s", rec.Body.String())
|
|
}
|
|
if got := atomic.LoadInt64(cChatHits); got != 1 {
|
|
return fmt.Errorf("multi-group bareWinner/failover harness: multi-group-c chat-completions hit count = %d, want exactly 1 (bareWinner must choose multi-group-c as primary — a pre-fix, union-based bareWinner would pick \"multi-group-b\" first and never reach multi-group-c at all)", got)
|
|
}
|
|
if got := atomic.LoadInt64(sharedXHits); got != 1 {
|
|
return fmt.Errorf("multi-group bareWinner/failover harness: multi-group-shared-x hit count = %d, want exactly 1 (the correctly-included failover candidate)", got)
|
|
}
|
|
if got := atomic.LoadInt64(bHits) - bHitsBefore; got != 0 {
|
|
return fmt.Errorf("multi-group bareWinner/failover harness: multi-group-b hit count delta = %d, want exactly 0 (the personal grant's own Models list does not cover %q; multi-group-b must never be offered as a failover candidate for it)", got, multiGroupSharedModelID)
|
|
}
|
|
|
|
// --- GET /v1/models: the union this principal is entitled to ---
|
|
listReq := httptest.NewRequest(http.MethodGet, "/v1/models", nil)
|
|
listReq.Header.Set("Authorization", "Bearer "+multiGroupUserAPIKey)
|
|
listRec := httptest.NewRecorder()
|
|
handler.ServeHTTP(listRec, listReq)
|
|
if listRec.Code != http.StatusOK {
|
|
return fmt.Errorf("GET /v1/models (multi-group bareWinner harness): status = %d, want 200, body=%s", listRec.Code, listRec.Body.String())
|
|
}
|
|
var listBody struct {
|
|
Data []map[string]any `json:"data"`
|
|
}
|
|
if err := json.Unmarshal(listRec.Body.Bytes(), &listBody); err != nil {
|
|
return fmt.Errorf("decode GET /v1/models body (multi-group bareWinner harness): %w (body=%s)", err, listRec.Body.String())
|
|
}
|
|
ownedBy := map[string]string{}
|
|
for _, entry := range listBody.Data {
|
|
id, idOK := entry["id"].(string)
|
|
owner, ownerOK := entry["owned_by"].(string)
|
|
if idOK && ownerOK {
|
|
ownedBy[id] = owner
|
|
}
|
|
}
|
|
if ownedBy[multiGroupSharedModelID] != "multi-group-c" {
|
|
return fmt.Errorf("GET /v1/models (multi-group bareWinner harness): bare %q owned_by = %q, want \"multi-group-c\"", multiGroupSharedModelID, ownedBy[multiGroupSharedModelID])
|
|
}
|
|
if _, ok := ownedBy["multi-group-c/"+multiGroupSharedModelID]; !ok {
|
|
return fmt.Errorf("GET /v1/models (multi-group bareWinner harness): missing \"multi-group-c/%s\"", multiGroupSharedModelID)
|
|
}
|
|
if _, ok := ownedBy["multi-group-shared-x/"+multiGroupSharedModelID]; !ok {
|
|
return fmt.Errorf("GET /v1/models (multi-group bareWinner harness): missing \"multi-group-shared-x/%s\"", multiGroupSharedModelID)
|
|
}
|
|
if _, ok := ownedBy["multi-group-b/"+multiGroupSharedModelID]; ok {
|
|
return fmt.Errorf("GET /v1/models (multi-group bareWinner harness): \"multi-group-b/%s\" must be absent (the personal grant's own Models list does not cover it)", multiGroupSharedModelID)
|
|
}
|
|
|
|
// --- HIGH-1: passthroughPaths coupled to the provider ---
|
|
deniedPathReq := httptest.NewRequest(http.MethodPost, "/multi-group-c/"+multiGroupDeniedPathID, strings.NewReader(`{}`))
|
|
deniedPathReq.Header.Set("Authorization", "Bearer "+multiGroupUserAPIKey)
|
|
deniedPathReq.Header.Set("Content-Type", "application/json")
|
|
deniedPathRec := httptest.NewRecorder()
|
|
handler.ServeHTTP(deniedPathRec, deniedPathReq)
|
|
if deniedPathRec.Code != http.StatusForbidden {
|
|
return fmt.Errorf("POST /multi-group-c/%s (multi-group passthroughPaths harness): status = %d, want 403 (multiGroupOpsGroupName's own path restriction on its OWN provider must survive, never stripped by multiGroupEngGroupName's own unrestricted paths on a DIFFERENT provider), body=%s", multiGroupDeniedPathID, deniedPathRec.Code, deniedPathRec.Body.String())
|
|
}
|
|
if got := atomic.LoadInt64(cPassthroughHits); got != 0 {
|
|
return fmt.Errorf("multi-group passthroughPaths harness: multi-group-c passthrough hit count = %d, want exactly 0 (the denied-path request must never reach the upstream)", got)
|
|
}
|
|
|
|
allowedPathReq := httptest.NewRequest(http.MethodPost, "/multi-group-c/"+multiGroupAllowedPathID, strings.NewReader(`{}`))
|
|
allowedPathReq.Header.Set("Authorization", "Bearer "+multiGroupUserAPIKey)
|
|
allowedPathReq.Header.Set("Content-Type", "application/json")
|
|
allowedPathRec := httptest.NewRecorder()
|
|
handler.ServeHTTP(allowedPathRec, allowedPathReq)
|
|
if allowedPathRec.Code != http.StatusOK {
|
|
return fmt.Errorf("POST /multi-group-c/%s (multi-group passthroughPaths harness): status = %d, want 200 (multiGroupOpsGroupName's own allowed path), body=%s", multiGroupAllowedPathID, allowedPathRec.Code, allowedPathRec.Body.String())
|
|
}
|
|
if got := atomic.LoadInt64(cPassthroughHits); got != 1 {
|
|
return fmt.Errorf("multi-group passthroughPaths harness: multi-group-c passthrough hit count = %d, want exactly 1 after the allowed-path request", got)
|
|
}
|
|
|
|
sharedXPathReq := httptest.NewRequest(http.MethodPost, "/multi-group-shared-x/"+multiGroupDeniedPathID, strings.NewReader(`{}`))
|
|
sharedXPathReq.Header.Set("Authorization", "Bearer "+multiGroupUserAPIKey)
|
|
sharedXPathReq.Header.Set("Content-Type", "application/json")
|
|
sharedXPathRec := httptest.NewRecorder()
|
|
handler.ServeHTTP(sharedXPathRec, sharedXPathReq)
|
|
if sharedXPathRec.Code != http.StatusOK {
|
|
return fmt.Errorf("POST /multi-group-shared-x/%s (multi-group passthroughPaths harness): status = %d, want 200 (multiGroupEngGroupName's own provider carries no path restriction at all, regardless of multiGroupOpsGroupName's own restriction on a DIFFERENT provider), body=%s", multiGroupDeniedPathID, sharedXPathRec.Code, sharedXPathRec.Body.String())
|
|
}
|
|
|
|
fmt.Println("yaegi-check: multi-group principal's bareWinner/listFor/failover chose the grant-authorized owner, GET /v1/models showed exactly the union, and passthroughPaths stayed coupled to its own provider")
|
|
return nil
|
|
}
|
|
|
|
// exerciseRound3ModelAndPathFromSameGrant proves the HIGH fix (review
|
|
// round 3) under the REAL interpreter: model and path authorization for
|
|
// native passthrough must come from ONE grant, never two different ones.
|
|
//
|
|
// - P1: round3P1AGroupName restricts models (round3ModelID) but not
|
|
// paths; round3P1BGroupName restricts paths ("v1/chat/completions")
|
|
// but not models — both cover "round3-shared". round3-p1-user
|
|
// (member of both, no personal grant) POSTs a model
|
|
// round3P1AGroupName denies, on a path only round3P1BGroupName would
|
|
// allow: round3P1AGroupName's own unrestricted path must never
|
|
// combine with round3P1BGroupName's own unrestricted model to
|
|
// authorize it.
|
|
// - P2: round3P2HomeGroupName (primary) carries no path restriction;
|
|
// round3P2FriendsGroupName restricts paths on "round3-shared" but not
|
|
// models; round3-p2-user's own personal grant restricts
|
|
// "round3-shared" to round3PersonalModelID alone. The personal grant
|
|
// borrows home's own unrestricted paths, but its OWN model
|
|
// restriction must still apply — friends' own unrestricted models
|
|
// must never leak into it via a shared provider.
|
|
//
|
|
// Both must deny (403) and never reach round3PassthroughUpstream.
|
|
func exerciseRound3ModelAndPathFromSameGrant(handler http.Handler, hits *int64) error {
|
|
hitsBefore := atomic.LoadInt64(hits)
|
|
|
|
p1Body := `{"model":"round3-m2"}`
|
|
p1Req := httptest.NewRequest(http.MethodPost, "/round3-shared/v1/files", strings.NewReader(p1Body))
|
|
p1Req.Header.Set("Authorization", "Bearer "+round3P1UserAPIKey)
|
|
p1Req.Header.Set("Content-Type", "application/json")
|
|
p1Rec := httptest.NewRecorder()
|
|
handler.ServeHTTP(p1Rec, p1Req)
|
|
if p1Rec.Code != http.StatusForbidden {
|
|
return fmt.Errorf("P1 POST /round3-shared/v1/files (round-3 model+path-from-same-grant harness): status = %d, want 403 — round3P1AGroupName's own unrestricted path must never combine with round3P1BGroupName's own unrestricted model to authorize %q, body=%s", p1Rec.Code, round3ModelID, p1Rec.Body.String())
|
|
}
|
|
|
|
p2Body := `{"model":"round3-shared/other"}`
|
|
p2Req := httptest.NewRequest(http.MethodPost, "/round3-shared/v1/files", strings.NewReader(p2Body))
|
|
p2Req.Header.Set("Authorization", "Bearer "+round3P2UserAPIKey)
|
|
p2Req.Header.Set("Content-Type", "application/json")
|
|
p2Rec := httptest.NewRecorder()
|
|
handler.ServeHTTP(p2Rec, p2Req)
|
|
if p2Rec.Code != http.StatusForbidden {
|
|
return fmt.Errorf("P2 POST /round3-shared/v1/files (round-3 model+path-from-same-grant harness): status = %d, want 403 — home's own unrestricted paths and friends' own unrestricted models must not combine to authorize a model the personal grant itself denies, body=%s", p2Rec.Code, p2Rec.Body.String())
|
|
}
|
|
|
|
if got := atomic.LoadInt64(hits) - hitsBefore; got != 0 {
|
|
return fmt.Errorf("round-3 model+path-from-same-grant harness: round3-shared upstream hit count delta = %d, want exactly 0 (neither P1's nor P2's denied request may ever reach the upstream)", got)
|
|
}
|
|
|
|
// P2+ (positive control, review round 4): the personal grant's own
|
|
// ALLOWED model (round3PersonalModelID), on a path the PRIMARY group
|
|
// (round3P2HomeGroupName) allows — home carries no path restriction
|
|
// at all, so ANY path is allowed via the "personal grant borrows the
|
|
// primary group's own paths" rule (auth.go's personalGrantAllowsPath).
|
|
// Proves the round-3 coupling fix denies the WRONG combination (P2
|
|
// above, a different model on the same provider+path) without also
|
|
// wrongly denying the RIGHT one — the body IS peeked (the personal
|
|
// grant restricts models) and the model IS matched.
|
|
p2PositiveBody := `{"model":"` + round3PersonalModelID + `"}`
|
|
p2PositiveReq := httptest.NewRequest(http.MethodPost, "/round3-shared/v1/files", strings.NewReader(p2PositiveBody))
|
|
p2PositiveReq.Header.Set("Authorization", "Bearer "+round3P2UserAPIKey)
|
|
p2PositiveReq.Header.Set("Content-Type", "application/json")
|
|
p2PositiveRec := httptest.NewRecorder()
|
|
handler.ServeHTTP(p2PositiveRec, p2PositiveReq)
|
|
if p2PositiveRec.Code != http.StatusOK {
|
|
return fmt.Errorf("P2+ POST /round3-shared/v1/files (round-3 model+path-from-same-grant harness): status = %d, want 200 — the personal grant's own model (%q) on a path the PRIMARY group (home) allows must succeed, body=%s", p2PositiveRec.Code, round3PersonalModelID, p2PositiveRec.Body.String())
|
|
}
|
|
if got := atomic.LoadInt64(hits) - hitsBefore; got != 1 {
|
|
return fmt.Errorf("round-3 model+path-from-same-grant harness: round3-shared upstream hit count delta after the P2+ positive case = %d, want exactly 1", got)
|
|
}
|
|
|
|
fmt.Println("yaegi-check: native passthrough denied a model and a path that could only be authorized by combining two DIFFERENT grants (P1: two member groups; P2: primary group's paths + a personal grant's own model restriction), and allowed the personal grant's own model on the primary group's own path (P2+)")
|
|
return nil
|
|
}
|
|
|
|
// exerciseRequestTimeout proves the request-timeout feature aborts a
|
|
// hung provider under the REAL interpreter, not merely in compiled
|
|
// tests: timeoutProviderName's own ProviderConfig.RequestTimeout ("80ms")
|
|
// is far shorter than timeoutUpstream's deliberate 600ms silence, and the
|
|
// request below carries NO client-side deadline of its own — unlike
|
|
// exerciseAttemptAccounting's slowProviderName probe (SHOULD-A), which
|
|
// proves a CLIENT-set context deadline classifies correctly, this proves
|
|
// the GATEWAY'S OWN adapter-level timeout aborts a hang no caller ever
|
|
// bounded, exactly the shape of a real, unbounded production client.
|
|
// A pass here means newAdapterHTTPClient's Transport.
|
|
// ResponseHeaderTimeout (providers.go) and watchdogBody's construction
|
|
// (timeout.go — context.WithCancel, time.AfterFunc, sync/atomic's
|
|
// function API) all compile and run correctly interpreted; a failure
|
|
// (a status other than 502, or an elapsed time anywhere near
|
|
// timeoutUpstreamSleep) would mean this feature works compiled but not
|
|
// under Yaegi, the exact class of divergence this harness exists to
|
|
// catch.
|
|
//
|
|
// MUTATION VERIFIED: temporarily removing
|
|
// `tr.ResponseHeaderTimeout = timeout` from newAdapterHTTPClient
|
|
// (providers.go) made `make yaegi-check` fail here with "status = 200,
|
|
// want 502" — timeoutUpstream's 600ms sleep let the request succeed
|
|
// instead of aborting at timeoutProviderRequestTimeout (80ms), proving
|
|
// this probe genuinely exercises the interpreted mechanism rather than
|
|
// passing regardless. Reverted before committing.
|
|
func exerciseRequestTimeout(handler http.Handler) error {
|
|
body := `{"model":"` + timeoutProviderName + `/` + timeoutProviderModel + `","messages":[{"role":"user","content":"hi"}]}`
|
|
req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(body))
|
|
req.Header.Set("Authorization", "Bearer "+testDataUserAPIKey)
|
|
req.Header.Set("Content-Type", "application/json")
|
|
rec := httptest.NewRecorder()
|
|
|
|
start := time.Now()
|
|
handler.ServeHTTP(rec, req)
|
|
elapsed := time.Since(start)
|
|
|
|
if elapsed > timeoutProbeMaxWait {
|
|
return fmt.Errorf("POST /v1/chat/completions (hung provider %q) took %s, want well under %s — a hung provider must be aborted under the interpreter, not merely in compiled tests: %s", timeoutProviderName, elapsed, timeoutProbeMaxWait, rec.Body.String())
|
|
}
|
|
if rec.Code != http.StatusBadGateway {
|
|
return fmt.Errorf("POST /v1/chat/completions (hung provider %q): status = %d, want 502 (upstream connection error, provider-attributed), body=%s", timeoutProviderName, rec.Code, rec.Body.String())
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// exerciseRequestTimeoutMidBodyStall proves the OTHER half of the
|
|
// request-timeout feature aborts under the REAL interpreter (finding F7,
|
|
// coordinator adversarial review, 2026-08-23): exerciseRequestTimeout
|
|
// above drives timeoutProviderName, whose upstream never sends headers
|
|
// at all, so it only exercises Transport.ResponseHeaderTimeout —
|
|
// watchdogBody.fire (timeout.go) never runs under that probe, since the
|
|
// request never gets far enough to construct a watchdogBody in the first
|
|
// place. hungBodyProviderName's upstream sends headers immediately, then
|
|
// stalls, so this drives the idle-progress body watchdog itself:
|
|
// time.AfterFunc handed the interpreted method value wb.fire,
|
|
// context.WithCancel/CancelFunc, and sync/atomic's function API
|
|
// (atomic.StoreInt32/LoadInt32) all have to work correctly interpreted
|
|
// for this to abort rather than hang.
|
|
//
|
|
// MUTATION VERIFIED: temporarily removing the `resp.Body =
|
|
// newWatchdogBody(...)` line from upstreamBytes (providers.go) made
|
|
// exerciseRequestTimeout (the header-timeout probe) still PASS
|
|
// unchanged, while this probe failed with "status = 200, want 502" —
|
|
// hungBodyUpstream's 600ms sleep let the request succeed once nothing
|
|
// guarded the body read. This confirms the two probes exercise genuinely
|
|
// different code paths, and that this one specifically needs
|
|
// watchdogBody to pass. Reverted before committing.
|
|
func exerciseRequestTimeoutMidBodyStall(handler http.Handler) error {
|
|
body := `{"model":"` + hungBodyProviderName + `/` + hungBodyProviderModel + `","messages":[{"role":"user","content":"hi"}]}`
|
|
req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(body))
|
|
req.Header.Set("Authorization", "Bearer "+testDataUserAPIKey)
|
|
req.Header.Set("Content-Type", "application/json")
|
|
rec := httptest.NewRecorder()
|
|
|
|
start := time.Now()
|
|
handler.ServeHTTP(rec, req)
|
|
elapsed := time.Since(start)
|
|
|
|
if elapsed > timeoutProbeMaxWait {
|
|
return fmt.Errorf("POST /v1/chat/completions (mid-body-stall provider %q) took %s, want well under %s — a provider that sends headers then stalls must still be aborted under the interpreter, not merely in compiled tests: %s", hungBodyProviderName, elapsed, timeoutProbeMaxWait, rec.Body.String())
|
|
}
|
|
if rec.Code != http.StatusBadGateway {
|
|
return fmt.Errorf("POST /v1/chat/completions (mid-body-stall provider %q): status = %d, want 502 (upstream connection error, provider-attributed), body=%s", hungBodyProviderName, rec.Code, rec.Body.String())
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func exerciseAttemptAccounting(handler http.Handler) error {
|
|
chatReq := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(
|
|
`{"model":"`+testDataWantModel+`","messages":[{"role":"user","content":"hi"}]}`,
|
|
))
|
|
chatReq.Header.Set("Authorization", "Bearer "+testDataUserAPIKey)
|
|
chatReq.Header.Set("Content-Type", "application/json")
|
|
chatRec := httptest.NewRecorder()
|
|
handler.ServeHTTP(chatRec, chatReq)
|
|
if chatRec.Code != http.StatusOK {
|
|
return fmt.Errorf("POST /v1/chat/completions (attempt-accounting harness): status = %d, want 200, body=%s", chatRec.Code, chatRec.Body.String())
|
|
}
|
|
|
|
attemptsDay, failuresDay, err := pollProviderAttemptCounters(handler, "openai")
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if attemptsDay != 1 {
|
|
return fmt.Errorf(`admin overview provider "openai" attemptsDay = %v, want 1 (Feature A attempt-accounting harness)`, attemptsDay)
|
|
}
|
|
if failuresDay != 0 {
|
|
return fmt.Errorf(`admin overview provider "openai" failuresDay = %v, want 0`, failuresDay)
|
|
}
|
|
|
|
// SHOULD-A: a real, interpreted context-deadline timeout against a
|
|
// deliberately slow upstream must record a FAILURE — the whole
|
|
// reason this harness exists after round 2's blocker.
|
|
slowCtx, cancel := context.WithTimeout(context.Background(), slowRequestDeadline)
|
|
defer cancel()
|
|
slowReq := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(
|
|
`{"model":"`+slowProviderName+`/`+slowProviderModel+`","messages":[{"role":"user","content":"hi"}]}`,
|
|
)).WithContext(slowCtx)
|
|
slowReq.Header.Set("Authorization", "Bearer "+testDataUserAPIKey)
|
|
slowReq.Header.Set("Content-Type", "application/json")
|
|
slowRec := httptest.NewRecorder()
|
|
handler.ServeHTTP(slowRec, slowReq)
|
|
// No status assertion here on purpose: the point of interest is the
|
|
// PROVIDER-HEALTH counter a timed-out upstream attempt leaves behind,
|
|
// not what status code a canceled/timed-out client request itself
|
|
// gets back (which the existing route-level Go tests already pin).
|
|
|
|
slowAttemptsDay, slowFailuresDay, err := pollProviderAttemptCounters(handler, slowProviderName)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if slowAttemptsDay != 1 {
|
|
return fmt.Errorf("admin overview provider %q attemptsDay = %v, want 1 (SHOULD-A timeout harness)", slowProviderName, slowAttemptsDay)
|
|
}
|
|
if slowFailuresDay != 1 {
|
|
return fmt.Errorf("admin overview provider %q failuresDay = %v, want 1 — a real interpreted context-deadline timeout must count as a provider-health failure (SHOULD-A, closes the round-2 blocker)", slowProviderName, slowFailuresDay)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// pollProviderAttemptCounters drives GET /admin/api/overview against
|
|
// handler repeatedly until providerName's attemptsDay is non-zero or a 2s
|
|
// budget elapses, then returns its final reading either way — SHOULD-5
|
|
// (v0.22 review round): recordProviderAttempt's store write runs on its
|
|
// own goroutine (limiter.spawn), off the request that triggered it, so a
|
|
// single immediate read right after that request returns could race
|
|
// ahead of the write landing. A real goroutine still schedules promptly
|
|
// under Yaegi (only the INTERPRETED code driving it runs slower, not the
|
|
// underlying Go runtime's scheduler), so this is a short poll, not a
|
|
// long one.
|
|
func pollProviderAttemptCounters(handler http.Handler, providerName string) (attemptsDay, failuresDay float64, err error) {
|
|
deadline := time.Now().Add(2 * time.Second)
|
|
for {
|
|
attemptsDay, failuresDay, err = readProviderAttemptCounters(handler, providerName)
|
|
if err != nil {
|
|
return 0, 0, err
|
|
}
|
|
if attemptsDay != 0 || time.Now().After(deadline) {
|
|
return attemptsDay, failuresDay, nil
|
|
}
|
|
time.Sleep(10 * time.Millisecond)
|
|
}
|
|
}
|
|
|
|
// readProviderAttemptCounters reads GET /admin/api/overview, finds the
|
|
// entry named providerName, and returns its attemptsDay/failuresDay
|
|
// fields via a generic map[string]any decode — this harness module is
|
|
// compiled, not interpreted, so it could import the plugin's own admin.go
|
|
// types directly, but they are unexported (adminOverviewResponse,
|
|
// adminProviderView); decoding generically here is simpler than exporting
|
|
// test-only types across that boundary just for this one harness. Looked
|
|
// up by name, not index 0 (round 1 of this harness only ever had one
|
|
// configured provider): buildAdminOverview sorts providers by name
|
|
// (registry.go), so adding slowProviderName ("slow") after "openai"
|
|
// changed which index held which provider.
|
|
func readProviderAttemptCounters(handler http.Handler, providerName string) (attemptsDay, failuresDay float64, err error) {
|
|
req := httptest.NewRequest(http.MethodGet, "/admin/api/overview", nil)
|
|
req.Header.Set("Authorization", "Bearer "+attemptAccountingAdminAPIKey)
|
|
rec := httptest.NewRecorder()
|
|
handler.ServeHTTP(rec, req)
|
|
if rec.Code != http.StatusOK {
|
|
return 0, 0, fmt.Errorf("GET /admin/api/overview: status = %d, want 200, body=%s", rec.Code, rec.Body.String())
|
|
}
|
|
|
|
var body map[string]any
|
|
if err := json.Unmarshal(rec.Body.Bytes(), &body); err != nil {
|
|
return 0, 0, fmt.Errorf("decode GET /admin/api/overview body: %w", err)
|
|
}
|
|
providers, ok := body["providers"].([]any)
|
|
if !ok {
|
|
return 0, 0, fmt.Errorf("admin overview body has no providers array: %s", rec.Body.String())
|
|
}
|
|
for _, raw := range providers {
|
|
p, ok := raw.(map[string]any)
|
|
if !ok {
|
|
continue
|
|
}
|
|
if name, _ := p["name"].(string); name != providerName {
|
|
continue
|
|
}
|
|
attemptsDay, _ = p["attemptsDay"].(float64)
|
|
failuresDay, _ = p["failuresDay"].(float64)
|
|
return attemptsDay, failuresDay, nil
|
|
}
|
|
return 0, 0, fmt.Errorf("admin overview body has no provider named %q: %s", providerName, rec.Body.String())
|
|
}
|
|
|
|
// readModulePath returns the module path declared by goModPath's
|
|
// "module" directive.
|
|
func readModulePath(goModPath string) (string, error) {
|
|
f, err := os.Open(goModPath)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
defer f.Close() //nolint:errcheck // read-side close; nothing actionable on failure
|
|
|
|
scanner := bufio.NewScanner(f)
|
|
for scanner.Scan() {
|
|
line := strings.TrimSpace(scanner.Text())
|
|
if after, ok := strings.CutPrefix(line, "module "); ok {
|
|
return strings.TrimSpace(after), nil
|
|
}
|
|
}
|
|
if err := scanner.Err(); err != nil {
|
|
return "", err
|
|
}
|
|
return "", fmt.Errorf("no module directive found in %s", goModPath)
|
|
}
|
|
|
|
// readPackageName returns the package name declared by the first
|
|
// non-test .go file it finds directly inside dir. The plugin is a single
|
|
// root package, so any one of its files' declarations suffices.
|
|
func readPackageName(dir string) (string, error) {
|
|
entries, err := os.ReadDir(dir)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
for _, e := range entries {
|
|
name := e.Name()
|
|
if e.IsDir() || !strings.HasSuffix(name, ".go") || strings.HasSuffix(name, "_test.go") {
|
|
continue
|
|
}
|
|
pkgName, err := scanPackageDecl(filepath.Join(dir, name))
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
if pkgName != "" {
|
|
return pkgName, nil
|
|
}
|
|
}
|
|
return "", fmt.Errorf("no package declaration found in %s", dir)
|
|
}
|
|
|
|
// scanPackageDecl reads path's leading "package X" line, if any.
|
|
func scanPackageDecl(path string) (string, error) {
|
|
f, err := os.Open(path)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
defer f.Close() //nolint:errcheck // read-side close; nothing actionable on failure
|
|
|
|
scanner := bufio.NewScanner(f)
|
|
for scanner.Scan() {
|
|
line := strings.TrimSpace(scanner.Text())
|
|
if after, ok := strings.CutPrefix(line, "package "); ok {
|
|
return strings.TrimSpace(after), nil
|
|
}
|
|
}
|
|
return "", scanner.Err()
|
|
}
|
|
|
|
// copyRepoSource copies repoRoot into destDir, skipping every directory
|
|
// named in excludedTopLevelDirs. destDir plays the role of
|
|
// GOPATH/src/{modulePath} for the interpreter: Yaegi's source loader
|
|
// resolves the plugin's own import path by walking GOPATH exactly as the
|
|
// standard toolchain would for a pre-modules GOPATH-mode build.
|
|
func copyRepoSource(repoRoot, destDir string) error {
|
|
entries, err := os.ReadDir(repoRoot)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if err := os.MkdirAll(destDir, 0o750); err != nil {
|
|
return err
|
|
}
|
|
for _, e := range entries {
|
|
if excludedTopLevelDirs[e.Name()] {
|
|
continue
|
|
}
|
|
src := filepath.Join(repoRoot, e.Name())
|
|
dst := filepath.Join(destDir, e.Name())
|
|
if e.IsDir() {
|
|
if err := copyDir(src, dst); err != nil {
|
|
return err
|
|
}
|
|
continue
|
|
}
|
|
if err := copyFile(src, dst); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// copyDir recursively copies src to dst.
|
|
func copyDir(src, dst string) error {
|
|
return filepath.WalkDir(src, func(path string, d fs.DirEntry, err error) error {
|
|
if err != nil {
|
|
return err
|
|
}
|
|
rel, err := filepath.Rel(src, path)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
target := filepath.Join(dst, rel)
|
|
if d.IsDir() {
|
|
return os.MkdirAll(target, 0o750)
|
|
}
|
|
return copyFile(path, target)
|
|
})
|
|
}
|
|
|
|
// copyFile copies src to dst, creating dst's parent directory if needed.
|
|
func copyFile(src, dst string) error {
|
|
in, err := os.Open(src)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer in.Close() //nolint:errcheck // read-side close; nothing actionable on failure
|
|
|
|
if err = os.MkdirAll(filepath.Dir(dst), 0o750); err != nil {
|
|
return err
|
|
}
|
|
out, err := os.Create(dst)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer out.Close() //nolint:errcheck // Close's error is checked explicitly below
|
|
if _, err := io.Copy(out, in); err != nil {
|
|
return err
|
|
}
|
|
return out.Close()
|
|
}
|
|
|
|
// yamlLine is one non-blank, non-comment line of a YAML file, with
|
|
// leading whitespace stripped and recorded as indent.
|
|
type yamlLine struct {
|
|
text string
|
|
indent int
|
|
}
|
|
|
|
// extractTraefikYAMLTestData reads path (the plugin's .traefik.yml) and
|
|
// decodes its top-level "testData:" block into a generic
|
|
// map[string]any/[]any/scalar tree, using a small hand-rolled,
|
|
// indentation-based YAML subset parser: just enough for this one file's
|
|
// shape (nested maps, lists of scalars, lists of flat maps, "{}"/"[]"
|
|
// empty collections), not a general YAML implementation. It exists only
|
|
// in this harness module — the plugin module never parses YAML itself,
|
|
// Traefik does that before handing the plugin its already-decoded Config.
|
|
func extractTraefikYAMLTestData(path string) (map[string]any, error) {
|
|
raw, err := os.ReadFile(path)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
lines := tokenizeYAMLLines(string(raw))
|
|
block := extractTestDataBlock(lines)
|
|
if block == nil {
|
|
return nil, fmt.Errorf("%s has no top-level testData: block", path)
|
|
}
|
|
pos := 0
|
|
v := parseBlock(block, &pos, block[0].indent)
|
|
m, ok := v.(map[string]any)
|
|
if !ok {
|
|
return nil, fmt.Errorf("%s testData: block is not a map", path)
|
|
}
|
|
return m, nil
|
|
}
|
|
|
|
// tokenizeYAMLLines splits raw into non-blank, non-comment yamlLines,
|
|
// each carrying its own leading-space count as indent.
|
|
func tokenizeYAMLLines(raw string) []yamlLine {
|
|
var out []yamlLine
|
|
for _, ln := range strings.Split(raw, "\n") {
|
|
trimmedRight := strings.TrimRight(ln, " \t\r")
|
|
content := strings.TrimLeft(trimmedRight, " ")
|
|
if content == "" || strings.HasPrefix(content, "#") {
|
|
continue
|
|
}
|
|
out = append(out, yamlLine{indent: len(trimmedRight) - len(content), text: content})
|
|
}
|
|
return out
|
|
}
|
|
|
|
// extractTestDataBlock returns the lines nested under a top-level (indent
|
|
// 0) "testData:" line, or nil if lines has none.
|
|
func extractTestDataBlock(lines []yamlLine) []yamlLine {
|
|
for i, l := range lines {
|
|
if l.indent != 0 || l.text != "testData:" {
|
|
continue
|
|
}
|
|
var block []yamlLine
|
|
for j := i + 1; j < len(lines) && lines[j].indent > 0; j++ {
|
|
block = append(block, lines[j])
|
|
}
|
|
return block
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// parseBlock parses the run of lines starting at *pos that share indent,
|
|
// dispatching to parseList when the first such line is a "-" item and to
|
|
// parseMap otherwise.
|
|
func parseBlock(lines []yamlLine, pos *int, indent int) any {
|
|
if *pos >= len(lines) || lines[*pos].indent != indent {
|
|
return map[string]any{}
|
|
}
|
|
if strings.HasPrefix(lines[*pos].text, "-") {
|
|
return parseList(lines, pos, indent)
|
|
}
|
|
return parseMap(lines, pos, indent)
|
|
}
|
|
|
|
// parseMap consumes every consecutive "key: value" line at indent,
|
|
// recursing into parseBlock for a key whose value is empty (a nested
|
|
// block follows on deeper-indented lines).
|
|
func parseMap(lines []yamlLine, pos *int, indent int) map[string]any {
|
|
m := map[string]any{}
|
|
for *pos < len(lines) && lines[*pos].indent == indent {
|
|
key, val, _ := strings.Cut(lines[*pos].text, ":")
|
|
key = strings.TrimSpace(key)
|
|
val = strings.TrimSpace(val)
|
|
*pos++
|
|
if val != "" {
|
|
m[key] = collectionOrScalar(val)
|
|
continue
|
|
}
|
|
if *pos < len(lines) && lines[*pos].indent > indent {
|
|
m[key] = parseBlock(lines, pos, lines[*pos].indent)
|
|
} else {
|
|
m[key] = map[string]any{}
|
|
}
|
|
}
|
|
return m
|
|
}
|
|
|
|
// parseList consumes every consecutive "- ..." line at indent. A "-"
|
|
// item containing a colon starts a flat map (its own "key: value" plus
|
|
// every following line indented at least indent+2, until a line back at
|
|
// indent or shallower); any other item is a bare scalar.
|
|
func parseList(lines []yamlLine, pos *int, indent int) []any {
|
|
var out []any
|
|
for *pos < len(lines) && lines[*pos].indent == indent && strings.HasPrefix(lines[*pos].text, "-") {
|
|
item := strings.TrimSpace(strings.TrimPrefix(lines[*pos].text, "-"))
|
|
if item == "" {
|
|
*pos++
|
|
continue
|
|
}
|
|
if !strings.Contains(item, ":") {
|
|
out = append(out, parseScalar(item))
|
|
*pos++
|
|
continue
|
|
}
|
|
|
|
m := map[string]any{}
|
|
key, val, _ := strings.Cut(item, ":")
|
|
m[strings.TrimSpace(key)] = collectionOrScalar(strings.TrimSpace(val))
|
|
*pos++
|
|
|
|
childIndent := indent + 2
|
|
for *pos < len(lines) && lines[*pos].indent >= childIndent {
|
|
k, v, _ := strings.Cut(lines[*pos].text, ":")
|
|
m[strings.TrimSpace(k)] = collectionOrScalar(strings.TrimSpace(v))
|
|
*pos++
|
|
}
|
|
out = append(out, m)
|
|
}
|
|
return out
|
|
}
|
|
|
|
// collectionOrScalar maps a "key: value" line's already-trimmed val to
|
|
// an empty map/slice for "{}"/"[]", or to parseScalar(val) otherwise. It
|
|
// does not recurse into a further nested block — no key in this file's
|
|
// testData needs a map/list value inside a list item's own fields, and
|
|
// this harness parses only what .traefik.yml actually contains.
|
|
func collectionOrScalar(val string) any {
|
|
switch val {
|
|
case "{}":
|
|
return map[string]any{}
|
|
case "[]":
|
|
return []any{}
|
|
default:
|
|
return parseScalar(val)
|
|
}
|
|
}
|
|
|
|
// parseScalar converts a YAML scalar token to a bool, int64, float64, or
|
|
// string, in that preference order, stripping a matching pair of quotes
|
|
// first.
|
|
func parseScalar(s string) any {
|
|
if len(s) >= 2 {
|
|
if (s[0] == '"' && s[len(s)-1] == '"') || (s[0] == '\'' && s[len(s)-1] == '\'') {
|
|
return s[1 : len(s)-1]
|
|
}
|
|
}
|
|
switch s {
|
|
case "true":
|
|
return true
|
|
case "false":
|
|
return false
|
|
}
|
|
if n, err := strconv.ParseInt(s, 10, 64); err == nil {
|
|
return n
|
|
}
|
|
if f, err := strconv.ParseFloat(s, 64); err == nil {
|
|
return f
|
|
}
|
|
return s
|
|
}
|
|
|
|
// modelUsageCanonicalID is the kindModel scope id exerciseModelUsageRanking
|
|
// expects its own chat completion to be attributed to: the canonical
|
|
// "provider/model" of the bare testDataWantModel's winning provider
|
|
// ("openai" — the same attribution exerciseAttemptAccounting already
|
|
// relies on).
|
|
const modelUsageCanonicalID = "openai/" + testDataWantModel
|
|
|
|
// exerciseModelUsageRanking drives the per-model usage feature end to end
|
|
// under the INTERPRETER (feat: per-model usage statistics): one real chat
|
|
// completion, then GET /admin/api/usage/models for the ranking and GET
|
|
// /admin/api/usage/history for that same model's own series.
|
|
//
|
|
// Why this needs an interpreted probe at all, when compiled tests already
|
|
// cover the same code: the feature adds a scope kind resolved POST-response
|
|
// (limits.go's withModelScope/kindModel), reached through a sort.Slice
|
|
// closure and a generic JSON encode of a newly declared struct type. Every
|
|
// one of those is a construct this repo has previously seen diverge under
|
|
// Yaegi — a compiled pass proves nothing about the interpreter, which is
|
|
// the whole premise of this harness.
|
|
func exerciseModelUsageRanking(handler http.Handler) error {
|
|
chatReq := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(
|
|
`{"model":"`+testDataWantModel+`","messages":[{"role":"user","content":"hi"}]}`,
|
|
))
|
|
chatReq.Header.Set("Authorization", "Bearer "+testDataUserAPIKey)
|
|
chatReq.Header.Set("Content-Type", "application/json")
|
|
chatRec := httptest.NewRecorder()
|
|
handler.ServeHTTP(chatRec, chatReq)
|
|
if chatRec.Code != http.StatusOK {
|
|
return fmt.Errorf("POST /v1/chat/completions (model-usage harness): status = %d, want 200, body=%s", chatRec.Code, chatRec.Body.String())
|
|
}
|
|
|
|
models, err := pollModelRanking(handler)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
// The ranking must contain the model that just served, and EVERY entry
|
|
// must be non-zero — the operator requirement the server-side filter
|
|
// exists for. A zero here would mean the filter did not run under the
|
|
// interpreter even though it does when compiled.
|
|
var found bool
|
|
for _, raw := range models {
|
|
entry, ok := raw.(map[string]any)
|
|
if !ok {
|
|
return fmt.Errorf("GET /admin/api/usage/models: entry is not an object: %v", raw)
|
|
}
|
|
id, _ := entry["id"].(string)
|
|
value, _ := entry["value"].(float64)
|
|
if value == 0 {
|
|
return fmt.Errorf("GET /admin/api/usage/models listed %q at value 0 — only non-zero models may be returned", id)
|
|
}
|
|
if id == modelUsageCanonicalID {
|
|
found = true
|
|
}
|
|
}
|
|
if !found {
|
|
return fmt.Errorf("GET /admin/api/usage/models did not list %q after a served request: %v", modelUsageCanonicalID, models)
|
|
}
|
|
|
|
// The same model must also be reachable as an ordinary history scope —
|
|
// the point of keying model usage as a scope kind rather than a
|
|
// bespoke counter.
|
|
histReq := httptest.NewRequest(http.MethodGet,
|
|
"/admin/api/usage/history?scope=model:"+url.QueryEscape(modelUsageCanonicalID)+"&metric=req&window=day&span=1", nil)
|
|
histReq.Header.Set("Authorization", "Bearer "+attemptAccountingAdminAPIKey)
|
|
histRec := httptest.NewRecorder()
|
|
handler.ServeHTTP(histRec, histReq)
|
|
if histRec.Code != http.StatusOK {
|
|
return fmt.Errorf("GET /admin/api/usage/history for scope model:%s: status = %d, want 200, body=%s",
|
|
modelUsageCanonicalID, histRec.Code, histRec.Body.String())
|
|
}
|
|
var hist map[string]any
|
|
if err := json.Unmarshal(histRec.Body.Bytes(), &hist); err != nil {
|
|
return fmt.Errorf("decode model history body: %w", err)
|
|
}
|
|
points, ok := hist["points"].([]any)
|
|
if !ok || len(points) != 1 {
|
|
return fmt.Errorf("model history points = %v, want exactly 1 bucket: %s", hist["points"], histRec.Body.String())
|
|
}
|
|
point, ok := points[0].(map[string]any)
|
|
if !ok {
|
|
return fmt.Errorf("model history point is not an object: %v", points[0])
|
|
}
|
|
if value, _ := point["value"].(float64); value < 1 {
|
|
return fmt.Errorf("model history value = %v, want at least 1 request attributed to %q", point["value"], modelUsageCanonicalID)
|
|
}
|
|
|
|
// An id outside the catalog must 404 rather than answer an empty
|
|
// series — the guard that keeps a mistyped or retired model from
|
|
// looking like a model with no traffic.
|
|
unknownReq := httptest.NewRequest(http.MethodGet,
|
|
"/admin/api/usage/history?scope=model:openai/no-such-model&metric=req&window=day&span=1", nil)
|
|
unknownReq.Header.Set("Authorization", "Bearer "+attemptAccountingAdminAPIKey)
|
|
unknownRec := httptest.NewRecorder()
|
|
handler.ServeHTTP(unknownRec, unknownReq)
|
|
if unknownRec.Code != http.StatusNotFound {
|
|
return fmt.Errorf("GET /admin/api/usage/history for an uncatalogued model: status = %d, want 404, body=%s",
|
|
unknownRec.Code, unknownRec.Body.String())
|
|
}
|
|
|
|
fmt.Println("yaegi-check: per-model usage ranking and model-scoped history served under the interpreter; only non-zero models listed, uncatalogued model 404s")
|
|
return nil
|
|
}
|
|
|
|
// pollModelRanking reads GET /admin/api/usage/models until it reports at
|
|
// least one model or a 2s budget elapses. The poll mirrors
|
|
// pollProviderAttemptCounters' own rationale: interpreted code driving the
|
|
// request runs slower than compiled code, so a single immediate read can
|
|
// outrun the accounting write the preceding request triggered.
|
|
func pollModelRanking(handler http.Handler) ([]any, error) {
|
|
deadline := time.Now().Add(2 * time.Second)
|
|
for {
|
|
req := httptest.NewRequest(http.MethodGet, "/admin/api/usage/models?metric=req&window=day", nil)
|
|
req.Header.Set("Authorization", "Bearer "+attemptAccountingAdminAPIKey)
|
|
rec := httptest.NewRecorder()
|
|
handler.ServeHTTP(rec, req)
|
|
if rec.Code != http.StatusOK {
|
|
return nil, fmt.Errorf("GET /admin/api/usage/models: status = %d, want 200, body=%s", rec.Code, rec.Body.String())
|
|
}
|
|
var body map[string]any
|
|
if err := json.Unmarshal(rec.Body.Bytes(), &body); err != nil {
|
|
return nil, fmt.Errorf("decode GET /admin/api/usage/models body: %w", err)
|
|
}
|
|
models, ok := body["models"].([]any)
|
|
if !ok {
|
|
return nil, fmt.Errorf("admin usage/models body has no models array: %s", rec.Body.String())
|
|
}
|
|
if len(models) > 0 || time.Now().After(deadline) {
|
|
return models, nil
|
|
}
|
|
time.Sleep(10 * time.Millisecond)
|
|
}
|
|
}
|
|
|
|
// freeModelDetailCanonicalID is the kindModel scope id
|
|
// exerciseFreeModelUsageDetail expects to find, with Free=true, in GET
|
|
// /admin/api/usage/models' detail=1 response: provenanceEstProviderModel
|
|
// already carries one real request (exerciseUsageProvenance's own chat
|
|
// completion, above) but no configured price at all — see its own
|
|
// modelMeta:{"free":true} override's doc comment (attemptAccountingOverride)
|
|
// for why free:true there is the ONLY reason this shows up free here.
|
|
const freeModelDetailCanonicalID = provenanceEstProviderName + "/" + provenanceEstProviderModel
|
|
|
|
// pollUsageModelsDetail is pollModelRanking's own detail=1 counterpart
|
|
// (free-models feature): GET /admin/api/usage/models?metric=cost&window=day&detail=1,
|
|
// polled the identical way — a single immediate read can outrun the
|
|
// accounting write an immediately preceding request triggered, under the
|
|
// slower interpreted path.
|
|
func pollUsageModelsDetail(handler http.Handler) ([]any, error) {
|
|
deadline := time.Now().Add(2 * time.Second)
|
|
for {
|
|
req := httptest.NewRequest(http.MethodGet, "/admin/api/usage/models?metric=cost&window=day&detail=1", nil)
|
|
req.Header.Set("Authorization", "Bearer "+attemptAccountingAdminAPIKey)
|
|
rec := httptest.NewRecorder()
|
|
handler.ServeHTTP(rec, req)
|
|
if rec.Code != http.StatusOK {
|
|
return nil, fmt.Errorf("GET /admin/api/usage/models?detail=1: status = %d, want 200, body=%s", rec.Code, rec.Body.String())
|
|
}
|
|
var body map[string]any
|
|
if err := json.Unmarshal(rec.Body.Bytes(), &body); err != nil {
|
|
return nil, fmt.Errorf("decode GET /admin/api/usage/models?detail=1 body: %w", err)
|
|
}
|
|
models, ok := body["models"].([]any)
|
|
if !ok {
|
|
return nil, fmt.Errorf("admin usage/models?detail=1 body has no models array: %s", rec.Body.String())
|
|
}
|
|
if len(models) > 0 || time.Now().After(deadline) {
|
|
return models, nil
|
|
}
|
|
time.Sleep(10 * time.Millisecond)
|
|
}
|
|
}
|
|
|
|
// exerciseFreeModelUsageDetail drives GET /admin/api/usage/models'
|
|
// detail=1 mode (free-models feature) under the INTERPRETER: a
|
|
// modelMeta-declared free model with real request traffic but a
|
|
// genuinely unpriced (hence zero-cost) model must still appear in a
|
|
// metric=cost ranking once detail=1 is set — the non-detail path drops
|
|
// any zero-value entry entirely, which is exactly the gap this feature
|
|
// closes. Reached through serveAdminUsageModelsDetail's own sort.Slice
|
|
// closure, its pointer-typed struct fields, and adminUsageModelFree's
|
|
// two-source free() check — none of which any earlier probe in this
|
|
// harness exercises.
|
|
func exerciseFreeModelUsageDetail(handler http.Handler) error {
|
|
models, err := pollUsageModelsDetail(handler)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
for _, raw := range models {
|
|
entry, ok := raw.(map[string]any)
|
|
if !ok {
|
|
return fmt.Errorf("GET /admin/api/usage/models?detail=1: entry is not an object: %v", raw)
|
|
}
|
|
if id, _ := entry["id"].(string); id != freeModelDetailCanonicalID {
|
|
continue
|
|
}
|
|
value, _ := entry["value"].(float64)
|
|
if value != 0 {
|
|
return fmt.Errorf("detail=1 entry %q: value = %v, want 0 (metric=cost, and its price is genuinely unknown)", freeModelDetailCanonicalID, value)
|
|
}
|
|
free, _ := entry["free"].(bool)
|
|
if !free {
|
|
return fmt.Errorf("detail=1 entry %q: free = %v, want true (modelMeta free:true override)", freeModelDetailCanonicalID, entry["free"])
|
|
}
|
|
requests, _ := entry["requests"].(float64)
|
|
if requests <= 0 {
|
|
return fmt.Errorf("detail=1 entry %q: requests = %v, want > 0", freeModelDetailCanonicalID, entry["requests"])
|
|
}
|
|
fmt.Println("yaegi-check: GET /admin/api/usage/models detail=1 lists a modelMeta-free model at value 0 with its real request count, under the interpreter")
|
|
return nil
|
|
}
|
|
return fmt.Errorf("GET /admin/api/usage/models?detail=1 did not list %q: %v", freeModelDetailCanonicalID, models)
|
|
}
|
|
|
|
// modelUsageDetailRequests reads GET /admin/api/usage/models' detail=1
|
|
// "requests" count for id right now. No poll: account()'s own store
|
|
// write (limits.go) is synchronous — unlike recordProviderAttempt, it
|
|
// never goes through limiter.spawn — so by the time ServeHTTP returns
|
|
// for the request that triggered it, the counter this reads is already
|
|
// current; pollUsageModelsDetail's own poll loop exists for a DIFFERENT
|
|
// reason (an empty response, not a stale one) and still applies here
|
|
// unchanged. An id with no traffic at all yet is legitimately absent
|
|
// from the response (only a nonzero metric total is listed even under
|
|
// detail=1), so a miss reports 0, not an error.
|
|
func modelUsageDetailRequests(handler http.Handler, id string) (int64, error) {
|
|
models, err := pollUsageModelsDetail(handler)
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
for _, raw := range models {
|
|
entry, ok := raw.(map[string]any)
|
|
if !ok {
|
|
continue
|
|
}
|
|
if entry["id"] != id {
|
|
continue
|
|
}
|
|
reqs, _ := entry["requests"].(float64)
|
|
return int64(reqs), nil
|
|
}
|
|
return 0, nil
|
|
}
|
|
|
|
// exerciseMediaModelUsage drives POST /v1/images/generations (free-
|
|
// models feature: media-route model request counters) under the
|
|
// INTERPRETER, reusing the existing "openai"/testDataWantModel provider/
|
|
// model pair — upstream answers 200 for any request path (its own doc
|
|
// comment, above), so no new fixture provider is needed here. It reads
|
|
// modelUsageCanonicalID's own kindModel req counter (via GET
|
|
// /admin/api/usage/models detail=1) before and after the request:
|
|
// recordMediaModelUsage (routes_media.go) must have added exactly 1,
|
|
// reached through withModelScope called with an empty scope list — a
|
|
// construct no earlier media-route probe in this harness exercises.
|
|
func exerciseMediaModelUsage(handler http.Handler) error {
|
|
before, err := modelUsageDetailRequests(handler, modelUsageCanonicalID)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
imgReq := httptest.NewRequest(http.MethodPost, "/v1/images/generations", strings.NewReader(
|
|
`{"model":"`+testDataWantModel+`","prompt":"a cat"}`,
|
|
))
|
|
imgReq.Header.Set("Authorization", "Bearer "+testDataUserAPIKey)
|
|
imgReq.Header.Set("Content-Type", "application/json")
|
|
imgRec := httptest.NewRecorder()
|
|
handler.ServeHTTP(imgRec, imgReq)
|
|
if imgRec.Code != http.StatusOK {
|
|
return fmt.Errorf("POST /v1/images/generations (media-model-usage harness): status = %d, want 200, body=%s", imgRec.Code, imgRec.Body.String())
|
|
}
|
|
|
|
after, err := modelUsageDetailRequests(handler, modelUsageCanonicalID)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if after != before+1 {
|
|
return fmt.Errorf("GET /admin/api/usage/models detail=1 requests for %q = %d, want exactly %d (before=%d, one served media request must add exactly one)", modelUsageCanonicalID, after, before+1, before)
|
|
}
|
|
|
|
fmt.Println("yaegi-check: a served media route (POST /v1/images/generations) incremented its model's own per-model request counter, under the interpreter")
|
|
return nil
|
|
}
|
|
|
|
// exerciseAdminOverviewReplicaAndWarnings proves GET /admin/api/overview's
|
|
// two new fields (F4/F9 replica id, F10 config-warnings collection, v0.3
|
|
// dashboard task) are populated correctly under the REAL interpreter:
|
|
// replica must be non-empty (g.replica: os.Hostname() -> $HOSTNAME ->
|
|
// "unknown", llmgateway.go) and warnings must decode as a genuine JSON
|
|
// array, never null (configWarningsSnapshot's own "never nil" contract,
|
|
// logger.go) — a null here would mean noteConfigWarning's slice append
|
|
// under the interpreter produced a nil rather than an empty slice, a
|
|
// construct class this codebase has seen diverge before. Content is
|
|
// fixture-dependent (this harness's own config may or may not trip a
|
|
// warnf during construction), so only presence/type is asserted, not any
|
|
// particular message.
|
|
func exerciseAdminOverviewReplicaAndWarnings(handler http.Handler) error {
|
|
req := httptest.NewRequest(http.MethodGet, "/admin/api/overview", nil)
|
|
req.Header.Set("Authorization", "Bearer "+attemptAccountingAdminAPIKey)
|
|
rec := httptest.NewRecorder()
|
|
handler.ServeHTTP(rec, req)
|
|
if rec.Code != http.StatusOK {
|
|
return fmt.Errorf("GET /admin/api/overview (replica/warnings probe): status = %d, want 200, body=%s", rec.Code, rec.Body.String())
|
|
}
|
|
var body map[string]any
|
|
if err := json.Unmarshal(rec.Body.Bytes(), &body); err != nil {
|
|
return fmt.Errorf("decode GET /admin/api/overview body: %w", err)
|
|
}
|
|
replica, _ := body["replica"].(string)
|
|
if replica == "" {
|
|
return fmt.Errorf(`GET /admin/api/overview: "replica" = %q, want non-empty (F4/F9: os.Hostname() -> $HOSTNAME -> "unknown")`, replica)
|
|
}
|
|
if _, ok := body["warnings"].([]any); !ok {
|
|
return fmt.Errorf(`GET /admin/api/overview: "warnings" did not decode as a JSON array (want never-nil, F10): %v`, body["warnings"])
|
|
}
|
|
fmt.Println("yaegi-check: GET /admin/api/overview carries a non-empty replica id and a never-nil warnings array under the interpreter")
|
|
return nil
|
|
}
|
|
|
|
// exerciseUsageModelsSpan proves GET /admin/api/usage/models' optional
|
|
// ?span= parameter (F2, v0.3 dashboard task) is parsed, validated, and
|
|
// echoed back correctly under the REAL interpreter: modelSpanTotals'
|
|
// per-id historyBucketKeys loop and chunkedModelSpanTotals' chunking
|
|
// arithmetic (admin.go/limits.go) are both new, interpreter-untested
|
|
// code this harness's own compiled test suite cannot vouch for. window
|
|
// "hour" caps span at 48 (historyMaxSpan), so 24 is comfortably valid.
|
|
func exerciseUsageModelsSpan(handler http.Handler) error {
|
|
req := httptest.NewRequest(http.MethodGet, "/admin/api/usage/models?metric=req&window=hour&span=24", nil)
|
|
req.Header.Set("Authorization", "Bearer "+attemptAccountingAdminAPIKey)
|
|
rec := httptest.NewRecorder()
|
|
handler.ServeHTTP(rec, req)
|
|
if rec.Code != http.StatusOK {
|
|
return fmt.Errorf("GET /admin/api/usage/models?span=24: status = %d, want 200, body=%s", rec.Code, rec.Body.String())
|
|
}
|
|
var body map[string]any
|
|
if err := json.Unmarshal(rec.Body.Bytes(), &body); err != nil {
|
|
return fmt.Errorf("decode GET /admin/api/usage/models body: %w", err)
|
|
}
|
|
if span, _ := body["span"].(float64); span != 24 {
|
|
return fmt.Errorf(`GET /admin/api/usage/models?span=24: "span" = %v, want 24 (F2's own echoed, resolved span)`, body["span"])
|
|
}
|
|
fmt.Println("yaegi-check: GET /admin/api/usage/models honors ?span=24 on window=hour and echoes it back under the interpreter")
|
|
return nil
|
|
}
|
|
|
|
// exerciseRateLimitEventAndRejections proves a real checkAndCount
|
|
// rejection is BOTH recorded to GET /admin/api/events as a rate_limit
|
|
// event (F3 hook 1) AND folded into GET /admin/api/usage's fleet-wide
|
|
// total.rejectionsPerDay counter (F4, Q6: "attribute rejections to
|
|
// total/all too") under the REAL interpreter — settleRejection's own
|
|
// single storeIncrMulti round trip (limits.go) and recordLimitEvent's
|
|
// event-ring append (events.go) are both new code no compiled test here
|
|
// can vouch for. eventsProbeUserAPIKey's own dedicated
|
|
// requestsPerMinute:1 limit (run()'s "users" override) makes the second
|
|
// call 429 deterministically, regardless of what every other probe
|
|
// already did to testDataUserAPIKey's own counters.
|
|
func exerciseRateLimitEventAndRejections(handler http.Handler) error {
|
|
body := `{"model":"` + testDataWantModel + `","messages":[{"role":"user","content":"hi"}]}`
|
|
|
|
req1 := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(body))
|
|
req1.Header.Set("Authorization", "Bearer "+eventsProbeUserAPIKey)
|
|
req1.Header.Set("Content-Type", "application/json")
|
|
rec1 := httptest.NewRecorder()
|
|
handler.ServeHTTP(rec1, req1)
|
|
if rec1.Code != http.StatusOK {
|
|
return fmt.Errorf("POST /v1/chat/completions (rate-limit-event harness, 1st call): status = %d, want 200, body=%s", rec1.Code, rec1.Body.String())
|
|
}
|
|
|
|
req2 := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(body))
|
|
req2.Header.Set("Authorization", "Bearer "+eventsProbeUserAPIKey)
|
|
req2.Header.Set("Content-Type", "application/json")
|
|
rec2 := httptest.NewRecorder()
|
|
handler.ServeHTTP(rec2, req2)
|
|
if rec2.Code != http.StatusTooManyRequests {
|
|
return fmt.Errorf("POST /v1/chat/completions (rate-limit-event harness, 2nd call): status = %d, want 429 (over requestsPerMinute:1), body=%s", rec2.Code, rec2.Body.String())
|
|
}
|
|
|
|
eventsReq := httptest.NewRequest(http.MethodGet, "/admin/api/events?limit=10", nil)
|
|
eventsReq.Header.Set("Authorization", "Bearer "+attemptAccountingAdminAPIKey)
|
|
eventsRec := httptest.NewRecorder()
|
|
handler.ServeHTTP(eventsRec, eventsReq)
|
|
if eventsRec.Code != http.StatusOK {
|
|
return fmt.Errorf("GET /admin/api/events?limit=10: status = %d, want 200, body=%s", eventsRec.Code, eventsRec.Body.String())
|
|
}
|
|
var eventsBody map[string]any
|
|
if err := json.Unmarshal(eventsRec.Body.Bytes(), &eventsBody); err != nil {
|
|
return fmt.Errorf("decode GET /admin/api/events body: %w", err)
|
|
}
|
|
events, _ := eventsBody["events"].([]any)
|
|
if len(events) == 0 {
|
|
return fmt.Errorf("GET /admin/api/events: want at least one event, body=%s", eventsRec.Body.String())
|
|
}
|
|
first, ok := events[0].(map[string]any)
|
|
if !ok {
|
|
return fmt.Errorf("GET /admin/api/events: events[0] is not an object: %v", events[0])
|
|
}
|
|
if kind, _ := first["kind"].(string); kind != "rate_limit" {
|
|
return fmt.Errorf(`GET /admin/api/events: events[0] (newest first).kind = %q, want "rate_limit"`, kind)
|
|
}
|
|
if route, _ := first["route"].(string); route != "chat/completions" {
|
|
return fmt.Errorf(`GET /admin/api/events: events[0].route = %q, want "chat/completions"`, route)
|
|
}
|
|
|
|
usageReq := httptest.NewRequest(http.MethodGet, "/admin/api/usage", nil)
|
|
usageReq.Header.Set("Authorization", "Bearer "+attemptAccountingAdminAPIKey)
|
|
usageRec := httptest.NewRecorder()
|
|
handler.ServeHTTP(usageRec, usageReq)
|
|
if usageRec.Code != http.StatusOK {
|
|
return fmt.Errorf("GET /admin/api/usage: status = %d, want 200, body=%s", usageRec.Code, usageRec.Body.String())
|
|
}
|
|
var usageBody map[string]any
|
|
if err := json.Unmarshal(usageRec.Body.Bytes(), &usageBody); err != nil {
|
|
return fmt.Errorf("decode GET /admin/api/usage body: %w", err)
|
|
}
|
|
total, ok := usageBody["total"].(map[string]any)
|
|
if !ok {
|
|
return fmt.Errorf("GET /admin/api/usage: no total object: %s", usageRec.Body.String())
|
|
}
|
|
if rejPerDay, _ := total["rejectionsPerDay"].(float64); rejPerDay < 1 {
|
|
return fmt.Errorf("GET /admin/api/usage: total.rejectionsPerDay = %v, want >= 1 after a real 429 (F4)", total["rejectionsPerDay"])
|
|
}
|
|
|
|
fmt.Println("yaegi-check: a real 429 recorded a rate_limit event (GET /admin/api/events) and incremented total.rejectionsPerDay (GET /admin/api/usage) under the interpreter")
|
|
return nil
|
|
}
|
|
|
|
// exerciseUpstreamAndTimeoutEvents proves recordUpstreamEvent's own
|
|
// classification (events.go, F3 hook 2) runs correctly under the REAL
|
|
// interpreter. Added per a backend verifier's finding (coordinator
|
|
// request, 2026-09-23): Yaegi v0.16.1 was found to leave kind/status
|
|
// empty for a multi-value assignment whose right-hand side contains a
|
|
// method call — exactly recordUpstreamEvent's own
|
|
// `kind, status, message = eventKindTimeout, http.StatusGatewayTimeout,
|
|
// err.Error()` shape — a construct class this codebase has repeatedly
|
|
// seen diverge under the interpreter (this package's own doc comment).
|
|
//
|
|
// eventsUpstreamErrorProviderName is a single, no-failover-candidate
|
|
// provider that always 500s, so the client-facing response IS the
|
|
// classified *providerHTTPError and the newest event must be kind
|
|
// "upstream" with that same 500 status. timeoutProviderName
|
|
// (exerciseRequestTimeout's own fixture, reused here rather than adding
|
|
// a third upstream — cheap) then makes the newest event a "timeout" one:
|
|
// its client-facing response is 502 (exerciseRequestTimeout's own
|
|
// assertion), but the EVENT's own Status field is independently set to
|
|
// http.StatusGatewayTimeout (504) by recordUpstreamEvent — the two are
|
|
// deliberately different fields of different origin, both asserted here.
|
|
func exerciseUpstreamAndTimeoutEvents(handler http.Handler) error {
|
|
upstreamReq := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(
|
|
`{"model":"`+eventsUpstreamErrorProviderModel+`","messages":[{"role":"user","content":"hi"}]}`,
|
|
))
|
|
upstreamReq.Header.Set("Authorization", "Bearer "+testDataUserAPIKey)
|
|
upstreamReq.Header.Set("Content-Type", "application/json")
|
|
upstreamRec := httptest.NewRecorder()
|
|
handler.ServeHTTP(upstreamRec, upstreamReq)
|
|
if upstreamRec.Code != http.StatusInternalServerError {
|
|
return fmt.Errorf("POST /v1/chat/completions (upstream-event harness, provider %q): status = %d, want 500 (no failover candidate configured), body=%s", eventsUpstreamErrorProviderName, upstreamRec.Code, upstreamRec.Body.String())
|
|
}
|
|
kind, status, err := latestEventKindAndStatus(handler)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if kind != "upstream" {
|
|
return fmt.Errorf(`GET /admin/api/events: newest event kind = %q, want "upstream" after a real 500`, kind)
|
|
}
|
|
if status != http.StatusInternalServerError {
|
|
return fmt.Errorf("GET /admin/api/events: newest (upstream) event status = %d, want 500", status)
|
|
}
|
|
|
|
timeoutReq := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(
|
|
`{"model":"`+timeoutProviderName+`/`+timeoutProviderModel+`","messages":[{"role":"user","content":"hi"}]}`,
|
|
))
|
|
timeoutReq.Header.Set("Authorization", "Bearer "+testDataUserAPIKey)
|
|
timeoutReq.Header.Set("Content-Type", "application/json")
|
|
timeoutRec := httptest.NewRecorder()
|
|
handler.ServeHTTP(timeoutRec, timeoutReq)
|
|
if timeoutRec.Code != http.StatusBadGateway {
|
|
return fmt.Errorf("POST /v1/chat/completions (timeout-event harness, provider %q): status = %d, want 502 (upstream connection error, provider-attributed — exerciseRequestTimeout's own contract), body=%s", timeoutProviderName, timeoutRec.Code, timeoutRec.Body.String())
|
|
}
|
|
kind, status, err = latestEventKindAndStatus(handler)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if kind != "timeout" {
|
|
return fmt.Errorf(`GET /admin/api/events: newest event kind = %q, want "timeout" after a real request-timeout abort`, kind)
|
|
}
|
|
if status != http.StatusGatewayTimeout {
|
|
return fmt.Errorf("GET /admin/api/events: newest (timeout) event status = %d, want 504 (recordUpstreamEvent's own fixed value, distinct from the 502 the client saw)", status)
|
|
}
|
|
|
|
fmt.Println("yaegi-check: recordUpstreamEvent classified a real 500 as kind \"upstream\"/500 and a real request-timeout abort as kind \"timeout\"/504 under the interpreter")
|
|
return nil
|
|
}
|
|
|
|
// latestEventKindAndStatus reads GET /admin/api/events?limit=1 as admin
|
|
// and returns the single newest event's kind/status — the shared read
|
|
// exerciseUpstreamAndTimeoutEvents' two assertions both drive, limit=1
|
|
// so each call cares only about the entry its own immediately preceding
|
|
// request produced, never an earlier probe's.
|
|
func latestEventKindAndStatus(handler http.Handler) (kind string, status int, err error) {
|
|
req := httptest.NewRequest(http.MethodGet, "/admin/api/events?limit=1", nil)
|
|
req.Header.Set("Authorization", "Bearer "+attemptAccountingAdminAPIKey)
|
|
rec := httptest.NewRecorder()
|
|
handler.ServeHTTP(rec, req)
|
|
if rec.Code != http.StatusOK {
|
|
return "", 0, fmt.Errorf("GET /admin/api/events?limit=1: status = %d, want 200, body=%s", rec.Code, rec.Body.String())
|
|
}
|
|
var body map[string]any
|
|
if jsonErr := json.Unmarshal(rec.Body.Bytes(), &body); jsonErr != nil {
|
|
return "", 0, fmt.Errorf("decode GET /admin/api/events body: %w", jsonErr)
|
|
}
|
|
events, _ := body["events"].([]any)
|
|
if len(events) == 0 {
|
|
return "", 0, fmt.Errorf("GET /admin/api/events?limit=1: want at least one event, body=%s", rec.Body.String())
|
|
}
|
|
first, ok := events[0].(map[string]any)
|
|
if !ok {
|
|
return "", 0, fmt.Errorf("GET /admin/api/events?limit=1: events[0] is not an object: %v", events[0])
|
|
}
|
|
k, _ := first["kind"].(string)
|
|
s, _ := first["status"].(float64)
|
|
return k, int(s), nil
|
|
}
|