mirror of
https://github.com/lukaszraczylo/traefik-llmgateway.git
synced 2026-10-11 04:47:00 +00:00
test(admin): probe the statistics endpoints under Yaegi and across replicas
yaegi-check adds probes for these, run under the interpreter against a fake Redis: - the last-seen SET reply - usage/series and usage/totals - performance with latency on - catalog price source - config redaction - consumers key source - cache-hit counters The integration suite adds three checks against two replicas: last seen is visible across replicas, latency counts match on both, and target caller totals appear after an MCP call. Claude-Session: https://claude.ai/code/session_01V5jYJNxpDFdJkyyExpADrv
This commit is contained in:
@@ -57,6 +57,19 @@ const (
|
||||
// real Yaegi interpretation; every other group/user in this suite
|
||||
// leaves both fields unset.
|
||||
daveKey = "sk-int-dave"
|
||||
// erinKey belongs to "eng" like alice, but is used ONLY by
|
||||
// TestLastSeenVisibleAcrossReplicas: last-seen writes are throttled
|
||||
// to once per 60s per (scope, replica) IN-PROCESS (lastSeenGate,
|
||||
// limits.go) — state flushRedis's own Redis FLUSHALL never touches,
|
||||
// since it lives in the plugin process's own memory, not Redis. This
|
||||
// whole suite runs in well under 60s, and alice already makes many
|
||||
// requests through replica 1 in earlier tests, so a second alice
|
||||
// request there would find the gate already "due" from one of those
|
||||
// and skip the write entirely — a false negative that has nothing to
|
||||
// do with whether last-seen itself works. A dedicated identity no
|
||||
// other test ever touches guarantees this test's own request is
|
||||
// genuinely alice-for-erin's first ever on replica 1.
|
||||
erinKey = "sk-int-erin"
|
||||
)
|
||||
|
||||
// composeFile returns the path to docker-compose.yml relative to this
|
||||
@@ -1394,3 +1407,160 @@ func TestAdminDashboard(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestLastSeenVisibleAcrossReplicas covers plan section 5's own
|
||||
// "two-replica" bullet for the admin dashboard redesign (WP-G): a
|
||||
// request admitted through replica 1 writes an absolute last-seen SET
|
||||
// (checkAndCount's own admission batch, limits.go/redis_store.go, DECISIONS
|
||||
// Q6) to the shared Redis, so GET /admin/api/usage read THROUGH REPLICA 2
|
||||
// must report that same user's lastSeen as non-zero — proving the value
|
||||
// is fleet-wide via the shared Redis, not a per-process in-memory figure
|
||||
// each container would otherwise carry independently.
|
||||
func TestLastSeenVisibleAcrossReplicas(t *testing.T) {
|
||||
flushRedis(t)
|
||||
|
||||
reqBody := map[string]any{
|
||||
"model": "openai/gpt-mock",
|
||||
"messages": []map[string]any{{"role": "user", "content": "last-seen probe"}},
|
||||
}
|
||||
resp, body := doJSON(t, http.MethodPost, traefik1URL+"/v1/chat/completions", erinKey, reqBody)
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("POST %s/v1/chat/completions: status = %d, body=%#v", traefik1URL, resp.StatusCode, body)
|
||||
}
|
||||
|
||||
usageResp, usageBody := doJSON(t, http.MethodGet, traefik2URL+"/admin/api/usage", adminKey, nil)
|
||||
if usageResp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("GET %s/admin/api/usage: status = %d, body=%#v", traefik2URL, usageResp.StatusCode, usageBody)
|
||||
}
|
||||
users, ok := usageBody["users"].([]any)
|
||||
if !ok {
|
||||
t.Fatalf("GET %s/admin/api/usage: users is not an array, body=%#v", traefik2URL, usageBody)
|
||||
}
|
||||
var erin map[string]any
|
||||
for _, u := range users {
|
||||
if entry, ok := u.(map[string]any); ok && entry["id"] == "erin" {
|
||||
erin = entry
|
||||
}
|
||||
}
|
||||
if erin == nil {
|
||||
t.Fatalf("GET %s/admin/api/usage: no user entry for erin, body=%#v", traefik2URL, usageBody)
|
||||
}
|
||||
if lastSeen, _ := erin["lastSeen"].(float64); lastSeen <= 0 {
|
||||
t.Errorf("GET %s/admin/api/usage: erin.lastSeen = %v, want a non-zero unix-seconds value read through replica 2 after a request admitted on replica 1", traefik2URL, erin["lastSeen"])
|
||||
}
|
||||
}
|
||||
|
||||
// performanceRowCount finds id's own "count" field within body's "rows"
|
||||
// array (GET /admin/api/performance's response shape) — TestPerformanceLatencyCountsEqualAcrossReplicas'
|
||||
// own helper.
|
||||
func performanceRowCount(t *testing.T, body map[string]any, id string) float64 {
|
||||
t.Helper()
|
||||
rows, ok := body["rows"].([]any)
|
||||
if !ok {
|
||||
t.Fatalf("performance response: rows is not an array, body=%#v", body)
|
||||
}
|
||||
for _, r := range rows {
|
||||
row, ok := r.(map[string]any)
|
||||
if ok && row["id"] == id {
|
||||
count, _ := row["count"].(float64)
|
||||
return count
|
||||
}
|
||||
}
|
||||
t.Fatalf("performance response: no row for id %q, body=%#v", id, body)
|
||||
return 0
|
||||
}
|
||||
|
||||
// TestPerformanceLatencyCountsEqualAcrossReplicas covers plan section 5's
|
||||
// "two-replica" bullet for GET /admin/api/performance (admin dashboard
|
||||
// redesign): admin.stats.latency is on (dynamic.yml.tmpl), so a real chat
|
||||
// completion's accountWith call writes its duration-bucket counters
|
||||
// (ld00..ld13, limits.go) to the SAME shared Redis every other counter
|
||||
// family already uses — reading them back through either replica must
|
||||
// therefore report the IDENTICAL count, not a per-process figure.
|
||||
func TestPerformanceLatencyCountsEqualAcrossReplicas(t *testing.T) {
|
||||
flushRedis(t)
|
||||
|
||||
reqBody := map[string]any{
|
||||
"model": "openai/gpt-mock",
|
||||
"messages": []map[string]any{{"role": "user", "content": "performance probe"}},
|
||||
}
|
||||
resp, body := doJSON(t, http.MethodPost, traefik1URL+"/v1/chat/completions", aliceKey, reqBody)
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("POST %s/v1/chat/completions: status = %d, body=%#v", traefik1URL, resp.StatusCode, body)
|
||||
}
|
||||
|
||||
perfPath := "/admin/api/performance?kind=provider&window=day&id=openai"
|
||||
perf1Resp, perf1Body := doJSON(t, http.MethodGet, traefik1URL+perfPath, adminKey, nil)
|
||||
if perf1Resp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("GET %s%s: status = %d, body=%#v", traefik1URL, perfPath, perf1Resp.StatusCode, perf1Body)
|
||||
}
|
||||
perf2Resp, perf2Body := doJSON(t, http.MethodGet, traefik2URL+perfPath, adminKey, nil)
|
||||
if perf2Resp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("GET %s%s: status = %d, body=%#v", traefik2URL, perfPath, perf2Resp.StatusCode, perf2Body)
|
||||
}
|
||||
if latencyEnabled, _ := perf1Body["latencyEnabled"].(bool); !latencyEnabled {
|
||||
t.Fatalf("GET %s%s: latencyEnabled = %v, want true (admin.stats.latency is on in dynamic.yml.tmpl)", traefik1URL, perfPath, perf1Body["latencyEnabled"])
|
||||
}
|
||||
|
||||
count1 := performanceRowCount(t, perf1Body, "openai")
|
||||
count2 := performanceRowCount(t, perf2Body, "openai")
|
||||
if count1 <= 0 {
|
||||
t.Fatalf("GET %s%s: openai row count = %v, want > 0 after a real chat completion", traefik1URL, perfPath, count1)
|
||||
}
|
||||
if count1 != count2 {
|
||||
t.Errorf("GET %s%s: count = %v, GET %s%s: count = %v, want identical (same shared Redis counter)", traefik1URL, perfPath, count1, traefik2URL, perfPath, count2)
|
||||
}
|
||||
}
|
||||
|
||||
// TestTargetCallerTotalsAfterMCPCall covers plan section 5's "two-replica"
|
||||
// bullet for the target x caller counter family (always-on-with-admin,
|
||||
// limits.go's countTargetRequestBy): a real MCP target-proxy call made
|
||||
// through replica 1 must be visible via GET /admin/api/usage/totals?
|
||||
// kind=targetcaller read through replica 2 — the shared Redis counter
|
||||
// again, not a per-process figure.
|
||||
func TestTargetCallerTotalsAfterMCPCall(t *testing.T) {
|
||||
flushRedis(t)
|
||||
|
||||
req, err := http.NewRequest(http.MethodGet, traefik1URL+"/mcp/tool/sse", nil)
|
||||
if err != nil {
|
||||
t.Fatalf("new mcp request: %v", err)
|
||||
}
|
||||
req.Header.Set("Authorization", "Bearer "+aliceKey)
|
||||
mcpResp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
t.Fatalf("mcp target proxy request: %v", err)
|
||||
}
|
||||
defer func() { _ = mcpResp.Body.Close() }()
|
||||
if mcpResp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("GET %s/mcp/tool/sse: status = %d", traefik1URL, mcpResp.StatusCode)
|
||||
}
|
||||
if _, err := io.ReadAll(mcpResp.Body); err != nil {
|
||||
t.Fatalf("read mcp target proxy body: %v", err)
|
||||
}
|
||||
|
||||
totalsResp, totalsBody := doJSON(t, http.MethodGet, traefik2URL+"/admin/api/usage/totals?kind=targetcaller&window=day&target=mcp/tool", adminKey, nil)
|
||||
if totalsResp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("GET %s/admin/api/usage/totals?kind=targetcaller: status = %d, body=%#v", traefik2URL, totalsResp.StatusCode, totalsBody)
|
||||
}
|
||||
rows, ok := totalsBody["rows"].([]any)
|
||||
if !ok {
|
||||
t.Fatalf("GET /admin/api/usage/totals?kind=targetcaller: rows is not an array, body=%#v", totalsBody)
|
||||
}
|
||||
wantID := "mcp/tool/alice"
|
||||
var found map[string]any
|
||||
for _, r := range rows {
|
||||
if row, ok := r.(map[string]any); ok && row["id"] == wantID {
|
||||
found = row
|
||||
}
|
||||
}
|
||||
if found == nil {
|
||||
t.Fatalf("GET %s/admin/api/usage/totals?kind=targetcaller: no row for id %q, body=%#v", traefik2URL, wantID, totalsBody)
|
||||
}
|
||||
values, ok := found["values"].(map[string]any)
|
||||
if !ok {
|
||||
t.Fatalf("GET /admin/api/usage/totals?kind=targetcaller: row %q values is not an object, got %#v", wantID, found["values"])
|
||||
}
|
||||
if reqCount, _ := values["req"].(float64); reqCount < 1 {
|
||||
t.Errorf("GET %s/admin/api/usage/totals?kind=targetcaller: %q.values.req = %v, want >= 1 after one MCP call through replica 1", traefik2URL, wantID, values["req"])
|
||||
}
|
||||
}
|
||||
|
||||
@@ -136,6 +136,14 @@ http:
|
||||
- name: dave
|
||||
group: restricted
|
||||
apiKey: "sk-int-dave"
|
||||
# TestLastSeenVisibleAcrossReplicas' dedicated identity
|
||||
# (admin dashboard redesign, WP-G integration coverage) —
|
||||
# see erinKey's own doc comment (integration_test.go) for
|
||||
# why last-seen's 60s-per-replica in-process throttle needs
|
||||
# an identity no other test in this suite ever touches.
|
||||
- name: erin
|
||||
group: eng
|
||||
apiKey: "sk-int-erin"
|
||||
file: "/users/users.json"
|
||||
redis:
|
||||
address: "redis:6379"
|
||||
@@ -167,6 +175,15 @@ http:
|
||||
ttl: "2m"
|
||||
admin:
|
||||
enabled: true
|
||||
# admin.stats.latency (admin dashboard redesign, WP-G
|
||||
# integration coverage): opt-in, default off (DECISIONS Q4) —
|
||||
# turned on here so TestPerformanceLatencyCountsEqualAcrossReplicas
|
||||
# can prove GET /admin/api/performance's latency percentiles
|
||||
# read identically through either replica, off the SAME
|
||||
# shared Redis every other counter family already uses.
|
||||
# userModel stays off: nothing in this suite exercises it.
|
||||
stats:
|
||||
latency: true
|
||||
# Task 6 (spec §5): one alias per mock provider type, so the
|
||||
# integration suite proves the echo-behavior asymmetry under
|
||||
# real Traefik+Yaegi — openai-type forwards the upstream body
|
||||
|
||||
@@ -0,0 +1,540 @@
|
||||
// This file adds the admin dashboard redesign's own yaegi-check probes
|
||||
// (COMMON3.md plan §5, WP-G): last-seen's absolute SET reply decode,
|
||||
// GET /admin/api/usage/series, /usage/totals, /performance, /catalog,
|
||||
// /config, and /consumers, and a response-cache hit's chit counter —
|
||||
// every one of them interpreted code this repo's tests (go test) never
|
||||
// run under Yaegi.
|
||||
//
|
||||
// These build their OWN Gateway instance — own config, own upstream, own
|
||||
// fake Redis — rather than reusing run()'s shared `handler`: they need
|
||||
// admin.stats.userModel/latency on and a configured Redis (the cache
|
||||
// probe needs a real response cache, which is Redis-backed only —
|
||||
// cache.go's buildResponseCache returns nil when no Redis client is
|
||||
// configured, regardless of cache.enabled), neither of which the shared
|
||||
// harness config carries, and layering them onto the shared handler
|
||||
// would add stats traffic every earlier probe's own counter assertions
|
||||
// never accounted for. A real (fake) Redis also matters here beyond the
|
||||
// cache probe alone: the last-seen family writes an ABSOLUTE "SET key v
|
||||
// EX secs" command (limits.go/redis_store.go, admin-redesign WP-A) whose
|
||||
// "+OK" reply resp.go's decodeReply turns into a Go STRING, not the
|
||||
// INTEGER reply every other counter write returns — exactly the kind of
|
||||
// interpreted-vs-compiled type-handling divergence this whole harness
|
||||
// exists to catch (ground truth: "redis store INCRBY + optional EXPIRE;
|
||||
// applyIncrReplies expects integers; +OK decodes to Go string",
|
||||
// resp.go:677-679) — a probe running only against the in-process
|
||||
// memoryStore fallback would never exercise that wire-protocol path at
|
||||
// all.
|
||||
package main
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"reflect"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"github.com/traefik/yaegi/interp"
|
||||
)
|
||||
|
||||
const (
|
||||
adminRedesignUserAPIKey = "sk-admin-redesign-user" // #nosec G101 -- test fixture literal, not a real credential
|
||||
adminRedesignAdminAPIKey = "sk-admin-redesign-admin" // #nosec G101 -- test fixture literal, not a real credential
|
||||
adminRedesignProviderName = "arprov"
|
||||
adminRedesignProviderAPIKey = "sk-admin-redesign-provider-literal-should-be-redacted" // #nosec G101 -- test fixture literal, not a real credential; the /config probe below asserts this exact string never appears in a redacted response
|
||||
adminRedesignModel = "ar-model"
|
||||
adminRedesignFreeModel = "ar-free-model"
|
||||
adminRedesignGroupName = "arg"
|
||||
)
|
||||
|
||||
// adminRedesignRespValue is one key's stored value plus whether it was
|
||||
// ever set — probeRedisFake's own map value, mirroring
|
||||
// redis_store_test.go's behavioralRedisServer.data (a plain
|
||||
// map[string]string would do too, but a struct keeps the zero-value
|
||||
// "never set" case explicit rather than relying on Go's own zero string
|
||||
// being indistinguishable from a genuinely empty stored value).
|
||||
type adminRedesignRespValue struct {
|
||||
v string
|
||||
}
|
||||
|
||||
// probeRedisFake is a minimal RESP2 server supporting exactly the
|
||||
// commands redis_store.go ever sends (SELECT/AUTH/EXPIRE always succeed;
|
||||
// GET/SET/INCRBY behave for real) — the identical command set
|
||||
// redis_store_test.go's own behavioralRedisServer supports, reimplemented
|
||||
// here because that helper lives in a _test.go file and cannot be
|
||||
// imported into this separate module/binary.
|
||||
type probeRedisFake struct {
|
||||
data map[string]adminRedesignRespValue
|
||||
mu sync.Mutex
|
||||
}
|
||||
|
||||
func newProbeRedisFake() (net.Listener, error) {
|
||||
ln, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("listen for the admin-redesign probe's fake Redis: %w", err)
|
||||
}
|
||||
s := &probeRedisFake{data: make(map[string]adminRedesignRespValue)}
|
||||
go s.acceptLoop(ln)
|
||||
return ln, nil
|
||||
}
|
||||
|
||||
func (s *probeRedisFake) acceptLoop(ln net.Listener) {
|
||||
for {
|
||||
conn, err := ln.Accept()
|
||||
if err != nil {
|
||||
return // listener closed once the probe returns; not an error
|
||||
}
|
||||
go s.serve(conn)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *probeRedisFake) serve(conn net.Conn) {
|
||||
defer func() { _ = conn.Close() }()
|
||||
r := bufio.NewReader(conn)
|
||||
for {
|
||||
args, err := adminRedesignReadRESPCommand(r)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
if _, err := conn.Write(s.handle(args)); err != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (s *probeRedisFake) handle(args []string) []byte {
|
||||
if len(args) == 0 {
|
||||
return []byte("-ERR empty command\r\n")
|
||||
}
|
||||
switch strings.ToUpper(args[0]) {
|
||||
case "SELECT", "AUTH", "EXPIRE":
|
||||
return []byte("+OK\r\n")
|
||||
case "GET":
|
||||
s.mu.Lock()
|
||||
val, ok := s.data[args[1]]
|
||||
s.mu.Unlock()
|
||||
if !ok {
|
||||
return []byte("$-1\r\n")
|
||||
}
|
||||
return []byte(fmt.Sprintf("$%d\r\n%s\r\n", len(val.v), val.v))
|
||||
case "SET":
|
||||
// args[2] is the value; a trailing "EX seconds" (redis_store.go's
|
||||
// own absolute-entry command, last-seen's own write) is accepted
|
||||
// and ignored — this fake never expires a key, matching
|
||||
// behavioralRedisServer's own "no test here runs long enough for
|
||||
// a real TTL to matter" reasoning.
|
||||
s.mu.Lock()
|
||||
s.data[args[1]] = adminRedesignRespValue{v: args[2]}
|
||||
s.mu.Unlock()
|
||||
return []byte("+OK\r\n")
|
||||
case "INCRBY":
|
||||
n, _ := strconv.ParseInt(args[2], 10, 64)
|
||||
s.mu.Lock()
|
||||
cur, _ := strconv.ParseInt(s.data[args[1]].v, 10, 64)
|
||||
cur += n
|
||||
s.data[args[1]] = adminRedesignRespValue{v: strconv.FormatInt(cur, 10)}
|
||||
s.mu.Unlock()
|
||||
return []byte(fmt.Appendf(nil, ":%d\r\n", cur))
|
||||
default:
|
||||
return []byte("-ERR unknown command\r\n")
|
||||
}
|
||||
}
|
||||
|
||||
// adminRedesignReadLine and adminRedesignReadRESPCommand port resp.go's
|
||||
// readLine and resp_test.go's readRESPCommand verbatim (this harness
|
||||
// cannot import either — see this file's own package doc comment) — the
|
||||
// exact same RESP2 command-array framing ("*N\r\n$len\r\narg\r\n...")
|
||||
// every respClient command uses.
|
||||
func adminRedesignReadLine(r *bufio.Reader) (string, error) {
|
||||
var buf []byte
|
||||
for {
|
||||
b, err := r.ReadByte()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if b == '\n' {
|
||||
return strings.TrimSuffix(string(buf), "\r"), nil
|
||||
}
|
||||
buf = append(buf, b)
|
||||
}
|
||||
}
|
||||
|
||||
func adminRedesignReadRESPCommand(r *bufio.Reader) ([]string, error) {
|
||||
line, err := adminRedesignReadLine(r)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !strings.HasPrefix(line, "*") {
|
||||
return nil, fmt.Errorf("want array header, got %q", line)
|
||||
}
|
||||
n, err := strconv.Atoi(line[1:])
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
args := make([]string, n)
|
||||
for i := 0; i < n; i++ {
|
||||
lenLine, err := adminRedesignReadLine(r)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !strings.HasPrefix(lenLine, "$") {
|
||||
return nil, fmt.Errorf("want bulk header, got %q", lenLine)
|
||||
}
|
||||
l, err := strconv.Atoi(lenLine[1:])
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
buf := make([]byte, l+2)
|
||||
if _, err := readFull(r, buf); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
args[i] = string(buf[:l])
|
||||
}
|
||||
return args, nil
|
||||
}
|
||||
|
||||
// readFull mirrors io.ReadFull without importing "io" solely for this one
|
||||
// call in a file that otherwise has no other use for it.
|
||||
func readFull(r *bufio.Reader, buf []byte) (int, error) {
|
||||
n := 0
|
||||
for n < len(buf) {
|
||||
m, err := r.Read(buf[n:])
|
||||
n += m
|
||||
if err != nil {
|
||||
return n, err
|
||||
}
|
||||
}
|
||||
return n, nil
|
||||
}
|
||||
|
||||
// adminRedesignSeriesResp/TotalsResp/PerfResp/CatalogResp/ConsumersResp/
|
||||
// UsageResp/ConfigResp are minimal local decodes of the new endpoints'
|
||||
// JSON response shapes (admin.go/stats_read.go/admin_catalog.go/
|
||||
// admin_config.go) — only the fields these probes actually assert on,
|
||||
// not full mirrors of the plugin's own view types (webui/src/types/api.ts
|
||||
// is that contract's real, complete mirror).
|
||||
type adminRedesignUsageResp struct {
|
||||
Users []struct {
|
||||
ID string `json:"id"`
|
||||
LastSeen int64 `json:"lastSeen"`
|
||||
} `json:"users"`
|
||||
}
|
||||
|
||||
type adminRedesignSeriesResp struct {
|
||||
Buckets []string `json:"buckets"`
|
||||
Series []struct {
|
||||
Scope string `json:"scope"`
|
||||
Points []int64 `json:"points"`
|
||||
} `json:"series"`
|
||||
}
|
||||
|
||||
type adminRedesignTotalsResp struct {
|
||||
Rows []struct {
|
||||
Values map[string]int64 `json:"values"`
|
||||
ID string `json:"id"`
|
||||
} `json:"rows"`
|
||||
}
|
||||
|
||||
type adminRedesignPerfResp struct {
|
||||
Rows []struct {
|
||||
ID string `json:"id"`
|
||||
Count int64 `json:"count"`
|
||||
} `json:"rows"`
|
||||
LatencyEnabled bool `json:"latencyEnabled"`
|
||||
}
|
||||
|
||||
type adminRedesignCatalogResp struct {
|
||||
Providers []struct {
|
||||
Name string `json:"name"`
|
||||
Models []struct {
|
||||
ID string `json:"id"`
|
||||
PriceSource string `json:"priceSource"`
|
||||
} `json:"models"`
|
||||
} `json:"providers"`
|
||||
}
|
||||
|
||||
type adminRedesignConsumersResp struct {
|
||||
Users []struct {
|
||||
Name string `json:"name"`
|
||||
Source string `json:"source"`
|
||||
} `json:"users"`
|
||||
}
|
||||
|
||||
// adminRedesignGetJSON issues req against handler and decodes a 200
|
||||
// response's body into out — every probe below's shared "fetch and
|
||||
// decode" step.
|
||||
func adminRedesignGetJSON(handler http.Handler, path, apiKey string, out any) ([]byte, error) {
|
||||
req := httptest.NewRequest(http.MethodGet, path, nil)
|
||||
req.Header.Set("Authorization", "Bearer "+apiKey)
|
||||
rec := httptest.NewRecorder()
|
||||
handler.ServeHTTP(rec, req)
|
||||
body := rec.Body.Bytes()
|
||||
if rec.Code != http.StatusOK {
|
||||
return body, fmt.Errorf("GET %s: status = %d, want 200, body=%s", path, rec.Code, body)
|
||||
}
|
||||
if out != nil {
|
||||
if err := json.Unmarshal(body, out); err != nil {
|
||||
return body, fmt.Errorf("GET %s: decode response: %w (body=%s)", path, err, body)
|
||||
}
|
||||
}
|
||||
return body, nil
|
||||
}
|
||||
|
||||
// newAdminRedesignHandler builds one interpreted Gateway for
|
||||
// exerciseAdminRedesignStats' own config (admin.stats.userModel/latency
|
||||
// on, a real Redis at redisAddr, cache enabled) — split out from
|
||||
// exerciseAdminRedesignStats itself purely so its own `err` variable's
|
||||
// scope ends here, at this function's return, rather than staying live
|
||||
// (and shadow-prone in every later `if _, err := ...` assertion) for the
|
||||
// rest of that much longer function.
|
||||
func newAdminRedesignHandler(i *interp.Interpreter, pkgName, upstreamURL, redisAddr string) (http.Handler, error) {
|
||||
cfgVal, err := i.Eval(pkgName + ".CreateConfig()")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("admin-redesign probe: call %s.CreateConfig() under yaegi: %w", pkgName, err)
|
||||
}
|
||||
if !cfgVal.CanInterface() {
|
||||
return nil, fmt.Errorf("admin-redesign probe: %s.CreateConfig() result cannot be used via reflect.Value.Interface", pkgName)
|
||||
}
|
||||
|
||||
cfgJSON := `{` +
|
||||
`"providers":{"` + adminRedesignProviderName + `":{"type":"openai","baseUrl":"` + upstreamURL + `","apiKey":"` + adminRedesignProviderAPIKey + `","models":["` + adminRedesignModel + `","` + adminRedesignFreeModel + `"]}},` +
|
||||
`"groups":{"` + adminRedesignGroupName + `":{}},` +
|
||||
`"users":{"inline":[` +
|
||||
`{"name":"ar-alice","group":"` + adminRedesignGroupName + `","apiKey":"` + adminRedesignUserAPIKey + `"},` +
|
||||
`{"name":"ar-admin","group":"` + adminRedesignGroupName + `","apiKey":"` + adminRedesignAdminAPIKey + `","admin":true}` +
|
||||
`]},` +
|
||||
`"admin":{"enabled":true,"stats":{"userModel":true,"latency":true}},` +
|
||||
`"modelMeta":{"` + adminRedesignProviderName + `/` + adminRedesignFreeModel + `":{"free":true}},` +
|
||||
`"redis":{"address":"` + redisAddr + `"},` +
|
||||
`"cache":{"enabled":true,"ttl":"1m"}` +
|
||||
`}`
|
||||
if err = json.Unmarshal([]byte(cfgJSON), cfgVal.Interface()); err != nil {
|
||||
return nil, fmt.Errorf("admin-redesign probe: decode its own config into the interpreted Config: %w", err)
|
||||
}
|
||||
|
||||
newVal, err := i.Eval(pkgName + ".New")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("admin-redesign probe: resolve %s.New under yaegi: %w", pkgName, err)
|
||||
}
|
||||
if newVal.Kind() != reflect.Func {
|
||||
return nil, fmt.Errorf("admin-redesign probe: %s.New is not a function (kind %s)", pkgName, newVal.Kind())
|
||||
}
|
||||
next := http.HandlerFunc(func(http.ResponseWriter, *http.Request) {})
|
||||
results := newVal.Call([]reflect.Value{
|
||||
reflect.ValueOf(context.Background()),
|
||||
reflect.ValueOf(next),
|
||||
cfgVal,
|
||||
reflect.ValueOf("llmgw-check-admin-redesign"),
|
||||
})
|
||||
if len(results) != 2 {
|
||||
return nil, fmt.Errorf("admin-redesign probe: %s.New returned %d values, want 2 (http.Handler, error)", pkgName, len(results))
|
||||
}
|
||||
if errVal := results[1]; !errVal.IsNil() {
|
||||
return nil, fmt.Errorf("admin-redesign probe: %s.New returned an error: %v", pkgName, errVal.Interface())
|
||||
}
|
||||
handler, ok := results[0].Interface().(http.Handler)
|
||||
if !ok || handler == nil {
|
||||
return nil, fmt.Errorf("admin-redesign probe: %s.New's returned value is not a usable http.Handler", pkgName)
|
||||
}
|
||||
return handler, nil
|
||||
}
|
||||
|
||||
// exerciseAdminRedesignStats builds its own interpreted Gateway (admin
|
||||
// dashboard redesign, WP-G's own yaegi-check probes — plan §5) and runs
|
||||
// every new-endpoint/new-counter-family assertion the plan's yaegi-check
|
||||
// list names, against it.
|
||||
func exerciseAdminRedesignStats(i *interp.Interpreter, pkgName string) error {
|
||||
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
_, _ = w.Write([]byte(`{"id":"c-ar-1","object":"chat.completion","model":"` + adminRedesignModel + `","choices":[{"index":0,"message":{"role":"assistant","content":"hi"},"finish_reason":"stop"}],"usage":{"prompt_tokens":10,"completion_tokens":5}}`))
|
||||
}))
|
||||
defer upstream.Close()
|
||||
|
||||
redisLn, redisErr := newProbeRedisFake()
|
||||
if redisErr != nil {
|
||||
return redisErr
|
||||
}
|
||||
defer func() { _ = redisLn.Close() }()
|
||||
|
||||
handler, handlerErr := newAdminRedesignHandler(i, pkgName, upstream.URL, redisLn.Addr().String())
|
||||
if handlerErr != nil {
|
||||
return handlerErr
|
||||
}
|
||||
|
||||
chatBody := `{"model":"` + adminRedesignModel + `","messages":[{"role":"user","content":"hi"}]}`
|
||||
chat := func() (*httptest.ResponseRecorder, error) {
|
||||
req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(chatBody))
|
||||
req.Header.Set("Authorization", "Bearer "+adminRedesignUserAPIKey)
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
rec := httptest.NewRecorder()
|
||||
handler.ServeHTTP(rec, req)
|
||||
if rec.Code != http.StatusOK {
|
||||
return rec, fmt.Errorf("POST /v1/chat/completions (admin-redesign probe): status = %d, want 200, body=%s", rec.Code, rec.Body.String())
|
||||
}
|
||||
return rec, nil
|
||||
}
|
||||
|
||||
// First request: a MISS (checkAndCount writes last-seen + admission
|
||||
// counters; accountWith writes umodel/latency; recordProviderAttempt
|
||||
// writes the provider's hourly attempt/latency counters). Second
|
||||
// request: the SAME body again, a cache HIT (recordCacheHit's own
|
||||
// async chit/csave write) — exactly the shape
|
||||
// TestRoundTrips_SingleGroupChat_CacheHit_AddsExactlyOneAsyncRoundTrip
|
||||
// (stats_write_test.go) drives compiled, now proven under Yaegi too.
|
||||
missRec, missErr := chat()
|
||||
if missErr != nil {
|
||||
return missErr
|
||||
}
|
||||
if got := missRec.Header().Get("X-Llmgw-Cache"); got != "miss" {
|
||||
return fmt.Errorf("admin-redesign probe: first POST /v1/chat/completions X-Llmgw-Cache = %q, want miss (body=%s)", got, missRec.Body.String())
|
||||
}
|
||||
hitRec, hitErr := chat()
|
||||
if hitErr != nil {
|
||||
return hitErr
|
||||
}
|
||||
if got := hitRec.Header().Get("X-Llmgw-Cache"); got != "hit" {
|
||||
return fmt.Errorf("admin-redesign probe: second POST /v1/chat/completions X-Llmgw-Cache = %q, want hit (body=%s)", got, hitRec.Body.String())
|
||||
}
|
||||
|
||||
// (1) last-seen: GET /admin/api/usage's own lastSeen field, populated
|
||||
// by the SAME absolute-SET write the fake Redis above just decoded a
|
||||
// "+OK" reply for.
|
||||
var usage adminRedesignUsageResp
|
||||
if _, err := adminRedesignGetJSON(handler, "/admin/api/usage", adminRedesignAdminAPIKey, &usage); err != nil {
|
||||
return err
|
||||
}
|
||||
found := false
|
||||
for _, u := range usage.Users {
|
||||
if u.ID == "ar-alice" {
|
||||
found = true
|
||||
if u.LastSeen == 0 {
|
||||
return fmt.Errorf("admin-redesign probe: GET /admin/api/usage: ar-alice.lastSeen = 0, want non-zero after an admitted request")
|
||||
}
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
return fmt.Errorf("admin-redesign probe: GET /admin/api/usage: no user entry named ar-alice")
|
||||
}
|
||||
|
||||
// (2) usage/series: two scopes, 200, bucket/point arrays aligned.
|
||||
var series adminRedesignSeriesResp
|
||||
if _, err := adminRedesignGetJSON(handler,
|
||||
"/admin/api/usage/series?scope=user:ar-alice&scope=total&metric=req&window=day&span=3",
|
||||
adminRedesignAdminAPIKey, &series); err != nil {
|
||||
return err
|
||||
}
|
||||
if len(series.Series) != 2 {
|
||||
return fmt.Errorf("admin-redesign probe: GET /admin/api/usage/series: %d series, want 2 (user:ar-alice, total)", len(series.Series))
|
||||
}
|
||||
for _, s := range series.Series {
|
||||
if len(s.Points) != len(series.Buckets) {
|
||||
return fmt.Errorf("admin-redesign probe: GET /admin/api/usage/series: scope %q has %d points, want %d (aligned with buckets)", s.Scope, len(s.Points), len(series.Buckets))
|
||||
}
|
||||
}
|
||||
|
||||
// (3) usage/totals?kind=user.
|
||||
var totals adminRedesignTotalsResp
|
||||
if _, err := adminRedesignGetJSON(handler, "/admin/api/usage/totals?kind=user&window=day", adminRedesignAdminAPIKey, &totals); err != nil {
|
||||
return err
|
||||
}
|
||||
totalsFound := false
|
||||
for _, r := range totals.Rows {
|
||||
if r.ID == "ar-alice" {
|
||||
totalsFound = true
|
||||
}
|
||||
}
|
||||
if !totalsFound {
|
||||
return fmt.Errorf("admin-redesign probe: GET /admin/api/usage/totals?kind=user: no row for ar-alice (rows=%v)", totals.Rows)
|
||||
}
|
||||
|
||||
// (4) performance?kind=provider, stats.latency on: count>0 after one
|
||||
// request that actually completed (the miss above).
|
||||
var perf adminRedesignPerfResp
|
||||
if _, err := adminRedesignGetJSON(handler, "/admin/api/performance?kind=provider&window=day&id="+adminRedesignProviderName, adminRedesignAdminAPIKey, &perf); err != nil {
|
||||
return err
|
||||
}
|
||||
if !perf.LatencyEnabled {
|
||||
return fmt.Errorf("admin-redesign probe: GET /admin/api/performance: latencyEnabled = false, want true (admin.stats.latency is on)")
|
||||
}
|
||||
perfFound := false
|
||||
for _, r := range perf.Rows {
|
||||
if r.ID == adminRedesignProviderName {
|
||||
perfFound = true
|
||||
if r.Count <= 0 {
|
||||
return fmt.Errorf("admin-redesign probe: GET /admin/api/performance: provider %q count = %d, want > 0 after a completed request", adminRedesignProviderName, r.Count)
|
||||
}
|
||||
}
|
||||
}
|
||||
if !perfFound {
|
||||
return fmt.Errorf("admin-redesign probe: GET /admin/api/performance: no row for provider %q", adminRedesignProviderName)
|
||||
}
|
||||
|
||||
// (5) catalog: the modelMeta-free model resolves to priceSource
|
||||
// "free" (billingPriceSource's modelMetaFree branch, pricing.go).
|
||||
var catalog adminRedesignCatalogResp
|
||||
if _, err := adminRedesignGetJSON(handler, "/admin/api/catalog", adminRedesignAdminAPIKey, &catalog); err != nil {
|
||||
return err
|
||||
}
|
||||
wantFreeID := adminRedesignProviderName + "/" + adminRedesignFreeModel
|
||||
catalogFound := false
|
||||
for _, p := range catalog.Providers {
|
||||
for _, m := range p.Models {
|
||||
if m.ID == wantFreeID {
|
||||
catalogFound = true
|
||||
if m.PriceSource != "free" {
|
||||
return fmt.Errorf("admin-redesign probe: GET /admin/api/catalog: %q priceSource = %q, want %q", wantFreeID, m.PriceSource, "free")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if !catalogFound {
|
||||
return fmt.Errorf("admin-redesign probe: GET /admin/api/catalog: no model entry for %q", wantFreeID)
|
||||
}
|
||||
|
||||
// (6) config: the literal provider API key never appears in the
|
||||
// redacted response, under the interpreter.
|
||||
configBody, configErr := adminRedesignGetJSON(handler, "/admin/api/config", adminRedesignAdminAPIKey, nil)
|
||||
if configErr != nil {
|
||||
return configErr
|
||||
}
|
||||
if strings.Contains(string(configBody), adminRedesignProviderAPIKey) {
|
||||
return fmt.Errorf("admin-redesign probe: GET /admin/api/config: the literal provider apiKey leaked into the redacted response: %s", configBody)
|
||||
}
|
||||
|
||||
// (7) consumers: ar-alice's source is "inline" (Users.Inline, never
|
||||
// Users.File, in this probe's own config).
|
||||
var consumers adminRedesignConsumersResp
|
||||
if _, err := adminRedesignGetJSON(handler, "/admin/api/consumers", adminRedesignAdminAPIKey, &consumers); err != nil {
|
||||
return err
|
||||
}
|
||||
consumersFound := false
|
||||
for _, u := range consumers.Users {
|
||||
if u.Name == "ar-alice" {
|
||||
consumersFound = true
|
||||
if u.Source != "inline" {
|
||||
return fmt.Errorf("admin-redesign probe: GET /admin/api/consumers: ar-alice.source = %q, want %q", u.Source, "inline")
|
||||
}
|
||||
}
|
||||
}
|
||||
if !consumersFound {
|
||||
return fmt.Errorf("admin-redesign probe: GET /admin/api/consumers: no user entry named ar-alice")
|
||||
}
|
||||
|
||||
// (8) cache hit increments the total chit series — the second
|
||||
// (hit) chat request above.
|
||||
var chitSeries adminRedesignSeriesResp
|
||||
if _, err := adminRedesignGetJSON(handler, "/admin/api/usage/series?scope=total&metric=chit&window=day&span=1", adminRedesignAdminAPIKey, &chitSeries); err != nil {
|
||||
return err
|
||||
}
|
||||
if len(chitSeries.Series) != 1 || len(chitSeries.Series[0].Points) != 1 || chitSeries.Series[0].Points[0] < 1 {
|
||||
return fmt.Errorf("admin-redesign probe: GET /admin/api/usage/series?metric=chit: %+v, want exactly one series/point with a value >= 1 after a cache hit", chitSeries.Series)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -1007,7 +1007,15 @@ func run() error {
|
||||
// /admin/api/events?limit=1 for the single newest entry right after
|
||||
// producing it, so nothing after it depends on event ordering staying
|
||||
// undisturbed.
|
||||
return exerciseUpstreamAndTimeoutEvents(handler)
|
||||
if err := exerciseUpstreamAndTimeoutEvents(handler); err != nil {
|
||||
return err
|
||||
}
|
||||
// exerciseAdminRedesignStats (admin_redesign.go, admin dashboard
|
||||
// redesign WP-G) runs LAST of all, against its own dedicated Gateway
|
||||
// instance (own config, own fake Redis, own upstream) rather than the
|
||||
// shared `handler` above — see that function's own doc comment for
|
||||
// why.
|
||||
return exerciseAdminRedesignStats(i, pkgName)
|
||||
}
|
||||
|
||||
// builtinLookupModelID is a real, stable entry in the generated
|
||||
|
||||
Reference in New Issue
Block a user