mirror of
https://github.com/lukaszraczylo/traefik-llmgateway.git
synced 2026-10-12 04:54:45 +00:00
lukaszraczylo
e8abb91dc8
yaegi-check adds probes for these, run under the interpreter against a fake Redis: - the last-seen SET reply - usage/series and usage/totals - performance with latency on - catalog price source - config redaction - consumers key source - cache-hit counters The integration suite adds three checks against two replicas: last seen is visible across replicas, latency counts match on both, and target caller totals appear after an MCP call. Claude-Session: https://claude.ai/code/session_01V5jYJNxpDFdJkyyExpADrv
233 lines
11 KiB
Cheetah
233 lines
11 KiB
Cheetah
http:
|
|
routers:
|
|
llmgateway-router:
|
|
rule: "PathPrefix(`/`)"
|
|
entryPoints: ["web"]
|
|
service: dummy-service
|
|
middlewares: ["llmgateway"]
|
|
priority: 1
|
|
|
|
llmgateway-real-router:
|
|
rule: "Host(`real.localhost`)"
|
|
entryPoints: ["web"]
|
|
service: dummy-service
|
|
middlewares: ["llmgateway-real"]
|
|
priority: 100
|
|
|
|
services:
|
|
# Placeholder upstream. Every route the plugin recognizes is
|
|
# short-circuited by the middleware itself; this backend is never
|
|
# actually reached by the requests the integration suite sends.
|
|
dummy-service:
|
|
loadBalancer:
|
|
servers:
|
|
- url: "http://mockopenai:8080"
|
|
|
|
middlewares:
|
|
llmgateway:
|
|
plugin:
|
|
llmgateway:
|
|
providers:
|
|
openai:
|
|
type: openai
|
|
baseUrl: "http://mockopenai:8080"
|
|
apiKey: "mock-openai-key"
|
|
discovery: true
|
|
models: ["gpt-mock"]
|
|
anthropic:
|
|
type: anthropic
|
|
baseUrl: "http://mockanthropic:8080"
|
|
apiKey: "mock-anthropic-key"
|
|
models: ["claude-mock"]
|
|
gemini:
|
|
type: gemini
|
|
baseUrl: "http://mockgemini:8080"
|
|
apiKey: "mock-gemini-key"
|
|
models: ["gemini-mock"]
|
|
# Backs task 5's retry coverage only: baseUrl points at the
|
|
# SAME mockopenai container, under its dedicated /flaky prefix
|
|
# (see integration/mock/main.go's handleFlakyOpenAIChat), so
|
|
# a.baseURL+"/v1/chat/completions" (provider_openai.go)
|
|
# resolves to /flaky/v1/chat/completions, and this provider's
|
|
# own native passthrough route (/flaky/reset) reaches
|
|
# handleFlakyReset the same way. discovery is left off — GET
|
|
# /flaky/v1/models isn't served — so the explicit "flaky-mock"
|
|
# model id below is the only way this provider's model is
|
|
# known.
|
|
flaky:
|
|
type: openai
|
|
baseUrl: "http://mockopenai:8080/flaky"
|
|
apiKey: "mock-openai-key"
|
|
models: ["flaky-mock"]
|
|
groups:
|
|
eng:
|
|
limits:
|
|
requestsPerMinute: 1000
|
|
limited:
|
|
limits:
|
|
requestsPerMinute: 3
|
|
# Dedicated to TestConfigHotReload's sustained ~20rps probe loop
|
|
# (integration/config_hot_reload_test.go), isolated from "eng"'s
|
|
# shared 1000/min bucket: that loop alone can run several
|
|
# hundred requests in one test, and "eng" is also driven by
|
|
# every other alice-keyed test in this suite. Sharing "eng"
|
|
# measurably risked exhausting its per-minute budget, which
|
|
# would surface as a 429 in the probe loop — indistinguishable
|
|
# from a real reload-caused blip to that test's own "zero
|
|
# non-200s" assertion, and a false failure signal for every
|
|
# OTHER "eng" test running around the same time. 100000/min is
|
|
# deliberately far above anything this suite could ever send.
|
|
hotreload:
|
|
limits:
|
|
requestsPerMinute: 100000
|
|
# Dedicated to TestPassthroughModelAndPathRestriction
|
|
# (security review finding 2/3, round 3, 2026-08-22 report
|
|
# correction): every OTHER group in this file leaves models
|
|
# and passthroughPaths unset, so hasModelRestriction() is
|
|
# always false and allowsPassthroughPath's non-empty-list
|
|
# branch is never reached — meaning the native-passthrough
|
|
# model-peek (peekPassthroughModel/scanTopLevelModel) and
|
|
# path-allowlist (allowsPassthroughPath) code paths were
|
|
# NEVER exercised under real Yaegi interpretation by this
|
|
# suite, contrary to what an earlier report claimed. This
|
|
# group exists specifically to close that gap.
|
|
restricted:
|
|
limits:
|
|
requestsPerMinute: 1000
|
|
models: ["gpt-mock"]
|
|
passthroughPaths: ["v1/chat/completions"]
|
|
users:
|
|
inline:
|
|
# limits.requestsPerDay is set (generously — every test in
|
|
# this suite together never approaches it) to exercise real
|
|
# per-user ENFORCEMENT alongside accounting, not because it is
|
|
# required for alice's own counters to move at all: since the
|
|
# v0.21 accounting fix, buildLimitScopes (routes_unified.go/
|
|
# routes_media.go/admin.go) always builds a "user" counter
|
|
# scope, limits or not, so the admin usage test's "alice's own
|
|
# request counters are > 0" assertion would hold even with
|
|
# this field unset.
|
|
- name: alice
|
|
group: eng
|
|
apiKey: "sk-int-alice"
|
|
limits:
|
|
requestsPerDay: 1000000
|
|
# Admin-flagged user for the admin dashboard integration
|
|
# coverage (spec §4, v0.2): same "eng" group as alice, so
|
|
# polling /admin/api/* repeatedly never hits a rate limit
|
|
# meant for a different scenario. limits.requestsPerDay is
|
|
# set for the same reason as alice's above.
|
|
- name: admin
|
|
group: eng
|
|
apiKey: "sk-int-admin"
|
|
admin: true
|
|
limits:
|
|
requestsPerDay: 1000000
|
|
# TestConfigHotReload's dedicated identity — see the
|
|
# "hotreload" group's own comment above for why it is not
|
|
# alice/eng. No per-user limits set: the group's
|
|
# requestsPerMinute alone (100000) already covers this user.
|
|
- name: hotreload-probe
|
|
group: hotreload
|
|
apiKey: "sk-int-hotreload"
|
|
# TestPassthroughModelAndPathRestriction's dedicated
|
|
# identity — see the "restricted" group's own comment
|
|
# above.
|
|
- name: dave
|
|
group: restricted
|
|
apiKey: "sk-int-dave"
|
|
# TestLastSeenVisibleAcrossReplicas' dedicated identity
|
|
# (admin dashboard redesign, WP-G integration coverage) —
|
|
# see erinKey's own doc comment (integration_test.go) for
|
|
# why last-seen's 60s-per-replica in-process throttle needs
|
|
# an identity no other test in this suite ever touches.
|
|
- name: erin
|
|
group: eng
|
|
apiKey: "sk-int-erin"
|
|
file: "/users/users.json"
|
|
redis:
|
|
address: "redis:6379"
|
|
mcpServers:
|
|
tool:
|
|
url: "http://mocktool:8080"
|
|
# toolstrict backs the federated-/mcp handshake-fallback
|
|
# integration coverage (SF4, review round 2, v0.21): mocktool's
|
|
# own "/strict" route (integration/mock/main.go's
|
|
# handleToolJSONRPCStrict) rejects a bare, session-less
|
|
# tools/list or tools/call with a JSON-RPC error and only
|
|
# honors one carrying a session id its own "initialize"
|
|
# issued — the same shape readitall/fetch answer live, and the
|
|
# exact trigger the plugin's bare-first-with-fallback retry
|
|
# (mcp_federation.go's mcpBackendCall) exists for. Proves that
|
|
# retry works under real Traefik+Yaegi, not just go test's own
|
|
# mocks.
|
|
toolstrict:
|
|
url: "http://mocktool:8080/strict"
|
|
# v0.2 features (spec §1, §2, §4, §5), all opt-in and off by
|
|
# default in v0.1: enabled here so the integration suite
|
|
# exercises every one under real Traefik+Yaegi, not just go
|
|
# test.
|
|
retry:
|
|
enabled: true
|
|
attempts: 1 # one retry after the first try — matches handleFlakyOpenAIChat's single forced 503
|
|
cache:
|
|
enabled: true
|
|
ttl: "2m"
|
|
admin:
|
|
enabled: true
|
|
# admin.stats.latency (admin dashboard redesign, WP-G
|
|
# integration coverage): opt-in, default off (DECISIONS Q4) —
|
|
# turned on here so TestPerformanceLatencyCountsEqualAcrossReplicas
|
|
# can prove GET /admin/api/performance's latency percentiles
|
|
# read identically through either replica, off the SAME
|
|
# shared Redis every other counter family already uses.
|
|
# userModel stays off: nothing in this suite exercises it.
|
|
stats:
|
|
latency: true
|
|
# Task 6 (spec §5): one alias per mock provider type, so the
|
|
# integration suite proves the echo-behavior asymmetry under
|
|
# real Traefik+Yaegi — openai-type forwards the upstream body
|
|
# verbatim (its "model" field carries "gpt-mock", not the
|
|
# alias); anthropic-type builds its own envelope and echoes the
|
|
# client's exact alias string back.
|
|
modelAliases:
|
|
aliased/mock: "gpt-mock"
|
|
aliased/claude: "claude-mock"
|
|
|
|
# Second, separate middleware instance carrying only the optional
|
|
# real-upstream smoke test's "uni" provider (task 7, INTEGRATION_REAL=1
|
|
# gated). Kept isolated from the mock-backed "llmgateway" middleware
|
|
# above so the always-on tests never depend on network egress.
|
|
llmgateway-real:
|
|
plugin:
|
|
llmgateway:
|
|
providers:
|
|
real:
|
|
type: openai
|
|
baseUrl: "__INTEGRATION_REAL_BASEURL__"
|
|
apiKey: ""
|
|
# Explicit model AND discovery: the explicit entry keeps
|
|
# resolution working offline (discovery failures are non-fatal
|
|
# by design - log + retry next interval), while discovery
|
|
# populates /v1/models with the gateway's full live catalog
|
|
# when the upstream is reachable (INTEGRATION_REAL=1 runs).
|
|
#
|
|
# The provider is deliberately NOT named "uni": the upstream
|
|
# gateway's own model ids carry a "uni/" prefix, and a
|
|
# provider of that name would make the plugin strip it
|
|
# (provider-prefix resolution) before proxying. With a
|
|
# non-colliding name the whole slash-bearing id below is a
|
|
# bare model id and is forwarded verbatim.
|
|
models: ["uni/deepseek-v4-flash-0731"]
|
|
discovery: true
|
|
discoveryInterval: "5m"
|
|
groups:
|
|
eng:
|
|
limits:
|
|
requestsPerMinute: 1000
|
|
users:
|
|
inline:
|
|
- name: alice-real
|
|
group: eng
|
|
apiKey: "sk-int-real"
|