mirror of
https://github.com/lukaszraczylo/traefik-llmgateway.git
synced 2026-10-12 04:54:45 +00:00
lukaszraczylo
945f41b9b0
README Kubernetes section: what a Middleware CR edit actually does (Traefik builds a new instance, warm-fills discovery, swaps it in; requests in flight finish on the old one), what survives a rebuild (Redis-backed counters and cache — windowKey's key never carries instance identity) vs what resets (in-memory fallback counters, down-latches, the discovery cache), why new models via discovery need no config edit, why the users Secret must be a whole-volume mount (subPath never receives a kubelet update) and how the plugin's own 5s mtime poll picks up the resulting symlink swap, and why a file-provider dynamic config needs directory: rather than filename: in Kubernetes (ConfigMap symlink swap) and under Docker Desktop (VirtioFS fsnotify) alike. examples/kubernetes.yaml: annotated the existing (already-correct, no-subPath) users Secret volume mount explaining why subPath would silently break Secret updates, so a future edit doesn't 'simplify' it into one.
248 lines
9.0 KiB
YAML
248 lines
9.0 KiB
YAML
# Example Kubernetes deployment of the LLM Gateway Traefik plugin: a
|
|
# Secret holding the file-backed users list, a Middleware custom resource
|
|
# carrying the plugin's own configuration, and a router wiring both onto a
|
|
# host. See ../README.md for the full configuration reference and
|
|
# ../.traefik.yml for the catalog-facing plugin manifest.
|
|
#
|
|
# Provider API keys are NOT put in this file. They are read by the plugin
|
|
# at startup via "env:VAR_NAME" (see the Middleware below), which means
|
|
# they must be set as environment variables on the Traefik pod itself —
|
|
# see the Helm values snippet at the bottom of this file.
|
|
#
|
|
# The empty-map gotcha: write an empty group as `providers: []`, never a
|
|
# bare `{}`. Confirmed against Traefik's file provider, twice, with this
|
|
# exact plugin build (examples/docker-compose.yml, plus an isolated
|
|
# minimal repro outside that demo's compose-specific config mechanism): a
|
|
# bare `default: {}` fails plugin construction with
|
|
# "expected a map, got 'string'"; an explicit empty list decodes
|
|
# correctly and means the same thing (all values allowed). Whether the
|
|
# Kubernetes CRD provider used below hits the same decode path as the
|
|
# file provider is NOT verified — this file uses the empty-list form
|
|
# defensively, not because the CRD path is confirmed affected.
|
|
|
|
apiVersion: v1
|
|
kind: Secret
|
|
metadata:
|
|
name: llmgw-users
|
|
namespace: traefik
|
|
type: Opaque
|
|
stringData:
|
|
# Read by the plugin via users.file (mounted below), and hot-reloaded —
|
|
# no Traefik restart needed to add, remove, or rotate a user. Shape is
|
|
# fixed by users_file.go: {"users":[{"name","group","apiKey","limits"}]}.
|
|
# limits is optional per user; omit it to inherit the user's group limits.
|
|
users.json: |
|
|
{
|
|
"users": [
|
|
{"name": "alice", "group": "engineering", "apiKey": "sk-llmgw-alice-CHANGE-ME"},
|
|
{"name": "bob", "group": "general", "apiKey": "sk-llmgw-bob-CHANGE-ME"}
|
|
]
|
|
}
|
|
|
|
---
|
|
apiVersion: traefik.io/v1alpha1
|
|
kind: Middleware
|
|
metadata:
|
|
name: llmgateway
|
|
namespace: traefik
|
|
spec:
|
|
plugin:
|
|
llmgateway:
|
|
providers:
|
|
openai:
|
|
type: openai
|
|
apiKey: "env:OPENAI_API_KEY"
|
|
discovery: true
|
|
discoveryInterval: 1h
|
|
anthropic:
|
|
type: anthropic
|
|
apiKey: "env:ANTHROPIC_API_KEY"
|
|
models: ["claude-opus-4-5", "claude-sonnet-4-5", "claude-haiku-4-5"]
|
|
gemini:
|
|
type: gemini
|
|
apiKey: "env:GEMINI_API_KEY"
|
|
models: ["gemini-2.5-pro", "gemini-2.5-flash"]
|
|
groups:
|
|
engineering:
|
|
# Empty lists, not omitted fields, so this block is a real map
|
|
# entry (see the empty-map gotcha note above) — every provider
|
|
# and model stays allowed either way (empty = all).
|
|
providers: []
|
|
models: []
|
|
limits:
|
|
requestsPerMinute: 60
|
|
requestsPerDay: 5000
|
|
tokensPerDay: 1000000
|
|
tokensPerMonth: 20000000
|
|
costPerDayUSD: 25.0
|
|
costPerMonthUSD: 250.0
|
|
mcpServers: []
|
|
agents: []
|
|
general:
|
|
providers: ["openai"]
|
|
models: ["gpt-5-mini", "gpt-4o"]
|
|
limits:
|
|
requestsPerMinute: 10
|
|
requestsPerDay: 500
|
|
users:
|
|
file: /secrets/llmgw-users/users.json
|
|
inline:
|
|
# Grants the admin dashboard (see the "admin" block below),
|
|
# which stays unreachable until admin.enabled is true. Declared
|
|
# inline here — rather than added to the users.json Secret
|
|
# above — only to keep this admin-granting example visible in
|
|
# one file; the key itself is NOT a literal, it uses the same
|
|
# env: form as every other apiKey in this Middleware, and its
|
|
# env var (LLMGW_ADMIN_KEY) must be set on the Traefik pod
|
|
# regardless of whether admin.enabled is true or false below —
|
|
# every inline user's apiKey resolves at construction time, not
|
|
# lazily on first admin request. See the Helm env snippet at
|
|
# the bottom of this file.
|
|
- name: dashboard-admin
|
|
group: engineering
|
|
apiKey: "env:LLMGW_ADMIN_KEY"
|
|
admin: true
|
|
redis:
|
|
address: redis.shared-resources.svc:6379
|
|
password: "env:REDIS_PASSWORD"
|
|
failOpen: true
|
|
retry:
|
|
enabled: true
|
|
attempts: 2 # retries after the first try: 3 tries total
|
|
backoff: 250ms
|
|
# Opt-in response cache for non-streaming chat/embeddings — see
|
|
# ../README.md#caching. Requires the redis block above; without it,
|
|
# this would silently stay disabled (one warning logged) rather than
|
|
# fail construction.
|
|
cache:
|
|
enabled: true
|
|
ttl: 5m
|
|
# Disabled here deliberately: flip to `enabled: true` to try the
|
|
# read-only dashboard at https://llm.example.com/admin — the
|
|
# "dashboard-admin" user above (users.inline) already grants access
|
|
# once this is on, and its LLMGW_ADMIN_KEY env var is already
|
|
# required regardless (see that user's own comment) — see
|
|
# ../README.md#admin.
|
|
admin:
|
|
enabled: false
|
|
pricing:
|
|
gpt-5-mini: {inputPerM: 0.25, outputPerM: 2.0}
|
|
mcpServers:
|
|
github:
|
|
url: http://mcp-github.tools.svc:8080
|
|
agents:
|
|
summarizer:
|
|
url: http://summarizer.agents.svc:8080
|
|
passthroughUnknown: false
|
|
|
|
---
|
|
# Traefik requires a real backing Service for a router even though this
|
|
# middleware short-circuits every path it recognizes; whoami (the same
|
|
# image already exercised in ../examples/docker-compose.yml) is a small,
|
|
# always-up dummy, only ever reached via passthroughUnknown (default
|
|
# false, which 404s instead). Any always-up internal service works here.
|
|
apiVersion: apps/v1
|
|
kind: Deployment
|
|
metadata:
|
|
name: llmgateway-dummy-backend
|
|
namespace: traefik
|
|
spec:
|
|
replicas: 1
|
|
selector:
|
|
matchLabels:
|
|
app: llmgateway-dummy-backend
|
|
template:
|
|
metadata:
|
|
labels:
|
|
app: llmgateway-dummy-backend
|
|
spec:
|
|
containers:
|
|
- name: whoami
|
|
image: traefik/whoami:v1.10
|
|
ports:
|
|
- containerPort: 80
|
|
|
|
---
|
|
apiVersion: v1
|
|
kind: Service
|
|
metadata:
|
|
name: llmgateway-dummy-backend
|
|
namespace: traefik
|
|
spec:
|
|
selector:
|
|
app: llmgateway-dummy-backend
|
|
ports:
|
|
- port: 80
|
|
targetPort: 80
|
|
|
|
---
|
|
apiVersion: traefik.io/v1alpha1
|
|
kind: IngressRoute
|
|
metadata:
|
|
name: llmgateway
|
|
namespace: traefik
|
|
spec:
|
|
entryPoints:
|
|
- websecure
|
|
routes:
|
|
- match: Host(`llm.example.com`)
|
|
kind: Rule
|
|
middlewares:
|
|
- name: llmgateway
|
|
services:
|
|
- name: llmgateway-dummy-backend
|
|
port: 80
|
|
|
|
---
|
|
# Helm values snippet (traefik/traefik-helm-chart) — illustrative, verify
|
|
# field names against the chart version you deploy. Two things are
|
|
# required beyond a stock install:
|
|
#
|
|
# 1. Load the plugin. Either the catalog form (once this plugin is listed
|
|
# in the Traefik Plugin Catalog, see ../.traefik.yml) or localPlugins,
|
|
# mounting this repository's source into the Traefik pod the same way
|
|
# ../examples/docker-compose.yml does for its local demo:
|
|
#
|
|
# experimental:
|
|
# plugins:
|
|
# llmgateway:
|
|
# moduleName: "github.com/lukaszraczylo/traefik-llmgateway"
|
|
# version: "v0.1.0"
|
|
#
|
|
# 2. Every "env:VAR_NAME" secret this Middleware references as env vars,
|
|
# sourced from Secrets — never inlined into the Middleware CR above.
|
|
# This is NOT just the three provider keys and the Redis password:
|
|
# LLMGW_ADMIN_KEY (the "dashboard-admin" user above) resolves at
|
|
# construction time too, so it must be set here even while
|
|
# admin.enabled is false — an unset "env:" var is a hard construction
|
|
# error regardless of whether the feature it gates is on.
|
|
#
|
|
# env:
|
|
# - name: OPENAI_API_KEY
|
|
# valueFrom: {secretKeyRef: {name: llmgw-provider-keys, key: openai}}
|
|
# - name: ANTHROPIC_API_KEY
|
|
# valueFrom: {secretKeyRef: {name: llmgw-provider-keys, key: anthropic}}
|
|
# - name: GEMINI_API_KEY
|
|
# valueFrom: {secretKeyRef: {name: llmgw-provider-keys, key: gemini}}
|
|
# - name: REDIS_PASSWORD
|
|
# valueFrom: {secretKeyRef: {name: llmgw-redis, key: password}}
|
|
# - name: LLMGW_ADMIN_KEY
|
|
# valueFrom: {secretKeyRef: {name: llmgw-admin-key, key: apiKey}}
|
|
#
|
|
# additionalVolumes:
|
|
# - name: llmgw-users
|
|
# secret: {secretName: llmgw-users}
|
|
# deployment:
|
|
# additionalVolumeMounts:
|
|
# # Whole-volume mount — no subPath — is required, not a style
|
|
# # choice: a subPath mount binds once at pod start and the
|
|
# # kubelet never updates it again, so editing the llmgw-users
|
|
# # Secret above would never reach this pod. A whole-volume mount
|
|
# # updates via the kubelet swapping a symlink for the whole
|
|
# # directory, which users.file's own 5-second mtime poll
|
|
# # (reloadEvery, users_file.go) picks up with no pod restart —
|
|
# # see ../README.md#config-hot-reload.
|
|
# - name: llmgw-users
|
|
# mountPath: /secrets/llmgw-users
|
|
# readOnly: true
|