mirror of
https://github.com/lukaszraczylo/traefik-llmgateway.git
synced 2026-10-12 04:54:45 +00:00
lukaszraczylo
c1bb9fe87a
Review response (5 Important + 3 folded):
1. README's cacheTTL row/bullet implied per-group scoping; cacheTTL sets
the TTL written when a group's own request populates a shared cache
entry, it never scopes the entry itself. Reworded only, cacheKey
unchanged.
2. README's fallback-counter claim ("used only when redis is not
configured") was wrong: limits.go also falls back during a Redis
outage whenever failOpen is true (the default). Reworded against
limiter.storeIncrBy/storeGet.
3. Dropped the unevidenced "in-flight requests finish on the old
instance" claim; replaced with what TestConfigHotReload actually
observes (zero non-200s across the rewrite window, every run).
4. Real bug: cleanup's settle poll checked for the injected alias to
disappear, which trivially no-ops on the skip path (the alias was
never live, so "absent" was already true on the first check) —
exactly the unguarded window that caused the earlier 404 cascade.
Replaced with settleToBaseline: captures the baseline /v1/models id
set up front, then polls unconditionally (both paths) for 3
consecutive matching polls, bounded 20s. Verified against a live run
that actually hit the skip path.
5. The ~20rps probe loop shared aliceKey/"eng" (1000/min) with most of
this suite's other tests; a budget-exhaustion 429 would look identical
to a reload blip. Added a dedicated hotreload group
(requestsPerMinute: 100000) and hotreload-probe user to
dynamic.yml.tmpl; every request TestConfigHotReload issues now uses
that key.
6. examples/docker-compose.yml still shipped providers.file.filename,
contradicting the new guidance; switched to providers.file.directory.
Verified by actually running the demo stack.
7. waitForModelAliasBounded and the new settle poll now use
doJSONNonFatal (returns an error instead of calling t.Fatalf), so a
transient transport error resets that poll's retry state instead of
aborting the whole test.
8. Reworded the skip-path Skip message: zero failures there proves only
that the stack survived an unreloaded rewrite attempt, not that a
successful reload is blip-free. Re-added the old single-file path to
.gitignore alongside the new directory entry.
Gates: go build/vet/gofmt clean on integration/; root go test -race,
gofmt, vet, yaegi-check all clean; make integration run cold twice, both
exit 0, plus a verbose run that landed on the skip path (the exact
scenario item 4 fixes) with the full suite, including
TestUsersFileHotReload, passing immediately after.
119 lines
4.6 KiB
YAML
119 lines
4.6 KiB
YAML
# Minimal, runnable local demo of the LLM Gateway plugin: one Traefik
|
|
# instance, loading this repository as a localPlugin, in front of a single
|
|
# OpenAI-type provider and a `whoami` dummy backend (every route the plugin
|
|
# recognizes is short-circuited by the middleware itself; whoami only
|
|
# answers a request that falls through as passthroughUnknown).
|
|
#
|
|
# Run from the repository root:
|
|
#
|
|
# OPENAI_API_KEY=sk-... docker compose -f examples/docker-compose.yml up
|
|
#
|
|
# Try it once traefik1 is healthy:
|
|
#
|
|
# curl -s http://localhost:8080/v1/models \
|
|
# -H "Authorization: Bearer sk-demo-local"
|
|
#
|
|
# curl -s http://localhost:8080/v1/chat/completions \
|
|
# -H "Authorization: Bearer sk-demo-local" \
|
|
# -H "Content-Type: application/json" \
|
|
# -d '{"model":"gpt-5-mini","messages":[{"role":"user","content":"hi"}]}'
|
|
#
|
|
# Override the demo API key with LLMGW_DEMO_KEY=... if you don't want the
|
|
# default "sk-demo-local" in your shell history, and the published port
|
|
# with LLMGW_DEMO_PORT=... if 8080 is already taken on your machine. See
|
|
# ../README.md for the full configuration reference; see ../.traefik.yml
|
|
# for the catalog-facing plugin manifest.
|
|
|
|
configs:
|
|
dynamic:
|
|
content: |
|
|
http:
|
|
routers:
|
|
llmgateway-router:
|
|
rule: "PathPrefix(`/`)"
|
|
entryPoints: ["web"]
|
|
service: whoami
|
|
middlewares: ["llmgateway"]
|
|
services:
|
|
whoami:
|
|
loadBalancer:
|
|
servers:
|
|
- url: "http://whoami:80"
|
|
middlewares:
|
|
llmgateway:
|
|
plugin:
|
|
llmgateway:
|
|
providers:
|
|
openai:
|
|
type: openai
|
|
apiKey: "env:OPENAI_API_KEY"
|
|
# Explicit rather than discovery:true, so the demo
|
|
# starts cleanly with no outbound network call — flip
|
|
# this to discovery:true (and drop models) to pull the
|
|
# live catalog from api.openai.com instead.
|
|
models: ["gpt-5-mini", "gpt-5", "gpt-4o"]
|
|
groups:
|
|
default:
|
|
providers: []
|
|
users:
|
|
inline:
|
|
- name: demo
|
|
group: default
|
|
apiKey: "${LLMGW_DEMO_KEY:-sk-demo-local}"
|
|
# v0.2 features — see ../README.md#retry, #caching, #admin.
|
|
retry:
|
|
enabled: true
|
|
attempts: 1
|
|
backoff: 250ms
|
|
cache:
|
|
# Left disabled: this minimal demo runs no Redis, and
|
|
# cache.enabled: true with no `redis` block configured
|
|
# silently stays off anyway (one warning logged) — see
|
|
# ../README.md#caching. Add a `redis` service and block
|
|
# to this compose file to actually exercise it.
|
|
enabled: false
|
|
admin:
|
|
# Flip to true, then add `admin: true` to the "demo"
|
|
# user above, to try the dashboard at
|
|
# http://localhost:${LLMGW_DEMO_PORT:-8080}/admin.
|
|
enabled: false
|
|
|
|
services:
|
|
whoami:
|
|
image: traefik/whoami:v1.10
|
|
networks: ["llmgw-demo"]
|
|
|
|
traefik1:
|
|
image: traefik:v3.5
|
|
command:
|
|
- "--entrypoints.web.address=:80"
|
|
- "--experimental.localPlugins.llmgateway.moduleName=github.com/lukaszraczylo/traefik-llmgateway"
|
|
# directory:, not filename:: a single-file watch's fsnotify does not
|
|
# fire reliably over a Docker bind/config mount (see
|
|
# ../README.md#config-hot-reload and ../integration/traefik/traefik.yml,
|
|
# which hit this directly). This demo does not itself demonstrate hot
|
|
# reload, but stays consistent with that guidance rather than
|
|
# showing the trap.
|
|
- "--providers.file.directory=/etc/traefik/dynamic"
|
|
- "--log.level=INFO"
|
|
environment:
|
|
# ":?" fails `docker compose config`/`up` immediately, with a clear
|
|
# message, instead of starting a plugin whose provider has no key.
|
|
OPENAI_API_KEY: "${OPENAI_API_KEY:?set OPENAI_API_KEY before running this demo}"
|
|
ports:
|
|
- "${LLMGW_DEMO_PORT:-8080}:80"
|
|
volumes:
|
|
# The plugin's own module, mounted read-only exactly as Traefik's
|
|
# localPlugins loader expects: <plugins-local>/src/<moduleName>.
|
|
- ..:/plugins-local/src/github.com/lukaszraczylo/traefik-llmgateway:ro
|
|
configs:
|
|
- source: dynamic
|
|
target: /etc/traefik/dynamic/dynamic.yml
|
|
networks: ["llmgw-demo"]
|
|
depends_on:
|
|
- whoami
|
|
|
|
networks:
|
|
llmgw-demo:
|
|
driver: bridge
|