mirror of
https://github.com/lukaszraczylo/traefik-llmgateway.git
synced 2026-10-12 04:54:45 +00:00
lukaszraczylo
722e6f3d29
Critical: attempts now counts retries performed AFTER the first try
(default 1 -> two tries total), not the total try count. The prior
reading made retry:{enabled:true}'s own documented default perform zero
retries. (*retryPolicy).totalTries() is the single conversion point,
clamping any attempts<1 to 1 so do()'s loop bound can never compute <=0
and skip calling call() entirely (no (nil,nil) return path).
Backoff waits are now ctx-aware: sleepFn func(time.Duration) is replaced
by waitFn func(context.Context, time.Duration) bool, whose production
implementation (defaultWaitFn) selects between time.After(d) and
ctx.Done(). A context canceled mid-wait aborts the remaining retries
immediately instead of blocking out the full backoff.
Also: classify a request-build failure (invalid method/URL) as
non-transient via errRequestBuildFailed, since retrying an identically
malformed request only wastes attempts; drop the drain cap to a
dedicated 64KiB constant instead of the 32MiB response cap; fix a
t.Fatal call from a non-test goroutine in the mid-stream-death test;
drop the misleading omitempty on Config.Retry (a struct value is never
'empty' to encoding/json); document the Retry-After >2s-falls-back-to-
backoff behavior and cover it with a direct table test.
README: add the retry config table, a RetryConfig field reference, and
a Retry section documenting the semantics. examples/kubernetes.yaml
gains a retry block alongside its other optional sections.
Gate: go test -race, gofmt, vet, yaegi-check, golangci-lint (incl a
fieldalignment fix on two new test structs) all clean. 295 subtests
pass, 0 failed. integration/ untouched.
203 lines
6.5 KiB
YAML
203 lines
6.5 KiB
YAML
# Example Kubernetes deployment of the LLM Gateway Traefik plugin: a
|
|
# Secret holding the file-backed users list, a Middleware custom resource
|
|
# carrying the plugin's own configuration, and a router wiring both onto a
|
|
# host. See ../README.md for the full configuration reference and
|
|
# ../.traefik.yml for the catalog-facing plugin manifest.
|
|
#
|
|
# Provider API keys are NOT put in this file. They are read by the plugin
|
|
# at startup via "env:VAR_NAME" (see the Middleware below), which means
|
|
# they must be set as environment variables on the Traefik pod itself —
|
|
# see the Helm values snippet at the bottom of this file.
|
|
#
|
|
# The empty-map gotcha: write an empty group as `providers: []`, never a
|
|
# bare `{}`. Confirmed against Traefik's file provider, twice, with this
|
|
# exact plugin build (examples/docker-compose.yml, plus an isolated
|
|
# minimal repro outside that demo's compose-specific config mechanism): a
|
|
# bare `default: {}` fails plugin construction with
|
|
# "expected a map, got 'string'"; an explicit empty list decodes
|
|
# correctly and means the same thing (all values allowed). Whether the
|
|
# Kubernetes CRD provider used below hits the same decode path as the
|
|
# file provider is NOT verified — this file uses the empty-list form
|
|
# defensively, not because the CRD path is confirmed affected.
|
|
|
|
apiVersion: v1
|
|
kind: Secret
|
|
metadata:
|
|
name: llmgw-users
|
|
namespace: traefik
|
|
type: Opaque
|
|
stringData:
|
|
# Read by the plugin via users.file (mounted below), and hot-reloaded —
|
|
# no Traefik restart needed to add, remove, or rotate a user. Shape is
|
|
# fixed by users_file.go: {"users":[{"name","group","apiKey","limits"}]}.
|
|
# limits is optional per user; omit it to inherit the user's group limits.
|
|
users.json: |
|
|
{
|
|
"users": [
|
|
{"name": "alice", "group": "engineering", "apiKey": "sk-llmgw-alice-CHANGE-ME"},
|
|
{"name": "bob", "group": "general", "apiKey": "sk-llmgw-bob-CHANGE-ME"}
|
|
]
|
|
}
|
|
|
|
---
|
|
apiVersion: traefik.io/v1alpha1
|
|
kind: Middleware
|
|
metadata:
|
|
name: llmgateway
|
|
namespace: traefik
|
|
spec:
|
|
plugin:
|
|
llmgateway:
|
|
providers:
|
|
openai:
|
|
type: openai
|
|
apiKey: "env:OPENAI_API_KEY"
|
|
discovery: true
|
|
discoveryInterval: 1h
|
|
anthropic:
|
|
type: anthropic
|
|
apiKey: "env:ANTHROPIC_API_KEY"
|
|
models: ["claude-opus-4-5", "claude-sonnet-4-5", "claude-haiku-4-5"]
|
|
gemini:
|
|
type: gemini
|
|
apiKey: "env:GEMINI_API_KEY"
|
|
models: ["gemini-2.5-pro", "gemini-2.5-flash"]
|
|
groups:
|
|
engineering:
|
|
# Empty lists, not omitted fields, so this block is a real map
|
|
# entry (see the empty-map gotcha note above) — every provider
|
|
# and model stays allowed either way (empty = all).
|
|
providers: []
|
|
models: []
|
|
limits:
|
|
requestsPerMinute: 60
|
|
requestsPerDay: 5000
|
|
tokensPerDay: 1000000
|
|
tokensPerMonth: 20000000
|
|
costPerDayUSD: 25.0
|
|
costPerMonthUSD: 250.0
|
|
mcpServers: []
|
|
agents: []
|
|
general:
|
|
providers: ["openai"]
|
|
models: ["gpt-5-mini", "gpt-4o"]
|
|
limits:
|
|
requestsPerMinute: 10
|
|
requestsPerDay: 500
|
|
users:
|
|
file: /secrets/llmgw-users/users.json
|
|
redis:
|
|
address: redis.shared-resources.svc:6379
|
|
password: "env:REDIS_PASSWORD"
|
|
failOpen: true
|
|
retry:
|
|
enabled: true
|
|
attempts: 2 # retries after the first try: 3 tries total
|
|
backoff: 250ms
|
|
pricing:
|
|
gpt-5-mini: {inputPerM: 0.25, outputPerM: 2.0}
|
|
mcpServers:
|
|
github:
|
|
url: http://mcp-github.tools.svc:8080
|
|
agents:
|
|
summarizer:
|
|
url: http://summarizer.agents.svc:8080
|
|
passthroughUnknown: false
|
|
|
|
---
|
|
# Traefik requires a real backing Service for a router even though this
|
|
# middleware short-circuits every path it recognizes; whoami (the same
|
|
# image already exercised in ../examples/docker-compose.yml) is a small,
|
|
# always-up dummy, only ever reached via passthroughUnknown (default
|
|
# false, which 404s instead). Any always-up internal service works here.
|
|
apiVersion: apps/v1
|
|
kind: Deployment
|
|
metadata:
|
|
name: llmgateway-dummy-backend
|
|
namespace: traefik
|
|
spec:
|
|
replicas: 1
|
|
selector:
|
|
matchLabels:
|
|
app: llmgateway-dummy-backend
|
|
template:
|
|
metadata:
|
|
labels:
|
|
app: llmgateway-dummy-backend
|
|
spec:
|
|
containers:
|
|
- name: whoami
|
|
image: traefik/whoami:v1.10
|
|
ports:
|
|
- containerPort: 80
|
|
|
|
---
|
|
apiVersion: v1
|
|
kind: Service
|
|
metadata:
|
|
name: llmgateway-dummy-backend
|
|
namespace: traefik
|
|
spec:
|
|
selector:
|
|
app: llmgateway-dummy-backend
|
|
ports:
|
|
- port: 80
|
|
targetPort: 80
|
|
|
|
---
|
|
apiVersion: traefik.io/v1alpha1
|
|
kind: IngressRoute
|
|
metadata:
|
|
name: llmgateway
|
|
namespace: traefik
|
|
spec:
|
|
entryPoints:
|
|
- websecure
|
|
routes:
|
|
- match: Host(`llm.example.com`)
|
|
kind: Rule
|
|
middlewares:
|
|
- name: llmgateway
|
|
services:
|
|
- name: llmgateway-dummy-backend
|
|
port: 80
|
|
|
|
---
|
|
# Helm values snippet (traefik/traefik-helm-chart) — illustrative, verify
|
|
# field names against the chart version you deploy. Two things are
|
|
# required beyond a stock install:
|
|
#
|
|
# 1. Load the plugin. Either the catalog form (once this plugin is listed
|
|
# in the Traefik Plugin Catalog, see ../.traefik.yml) or localPlugins,
|
|
# mounting this repository's source into the Traefik pod the same way
|
|
# ../examples/docker-compose.yml does for its local demo:
|
|
#
|
|
# experimental:
|
|
# plugins:
|
|
# llmgateway:
|
|
# moduleName: "github.com/lukaszraczylo/traefik-llmgateway"
|
|
# version: "v0.1.0"
|
|
#
|
|
# 2. Provider keys as env vars, sourced from Secrets — never inlined into
|
|
# the Middleware CR above, which is why every apiKey field there reads
|
|
# "env:VAR_NAME" instead of a literal:
|
|
#
|
|
# env:
|
|
# - name: OPENAI_API_KEY
|
|
# valueFrom: {secretKeyRef: {name: llmgw-provider-keys, key: openai}}
|
|
# - name: ANTHROPIC_API_KEY
|
|
# valueFrom: {secretKeyRef: {name: llmgw-provider-keys, key: anthropic}}
|
|
# - name: GEMINI_API_KEY
|
|
# valueFrom: {secretKeyRef: {name: llmgw-provider-keys, key: gemini}}
|
|
# - name: REDIS_PASSWORD
|
|
# valueFrom: {secretKeyRef: {name: llmgw-redis, key: password}}
|
|
#
|
|
# additionalVolumes:
|
|
# - name: llmgw-users
|
|
# secret: {secretName: llmgw-users}
|
|
# deployment:
|
|
# additionalVolumeMounts:
|
|
# - name: llmgw-users
|
|
# mountPath: /secrets/llmgw-users
|
|
# readOnly: true
|