mirror of
https://github.com/lukaszraczylo/traefik-llmgateway.git
synced 2026-10-11 04:47:00 +00:00
lukaszraczylo
95039d96c9
The VMServiceScrape example scraped Traefik's pods directly, so the request carried Host: <pod-ip>:<port>. Traefik evaluates a router's match rule before any middleware runs, and the only router in this file requires Host(`llm.example.com`). The scrape request never matched it, the llmgateway middleware never ran, and /metrics 404ed before the plugin ever saw the request. Add a second, Host-free IngressRoute matching PathPrefix(`/metrics`) and carrying the same middleware, so an in-cluster scraper reaches the route regardless of what Host header it sends. Document the reachability trade-off this introduces (this route is not Host-restricted, though /metrics stays gated by its own auth inside the plugin) and name the alternative fix the review also offered: relabelConfigs rewriting __address__ to the ingress hostname before the request is sent, keeping one Host-scoped router instead.
345 lines
14 KiB
YAML
345 lines
14 KiB
YAML
# Example Kubernetes deployment of the LLM Gateway Traefik plugin: a
|
|
# Secret holding the file-backed users list, a Middleware custom resource
|
|
# carrying the plugin's own configuration, and a router wiring both onto a
|
|
# host. See ../README.md for the full configuration reference and
|
|
# ../.traefik.yml for the catalog-facing plugin manifest.
|
|
#
|
|
# Provider API keys are NOT put in this file. They are read by the plugin
|
|
# at startup via "env:VAR_NAME" (see the Middleware below), which means
|
|
# they must be set as environment variables on the Traefik pod itself —
|
|
# see the Helm values snippet at the bottom of this file.
|
|
#
|
|
# The empty-map gotcha: write an empty group as `providers: []`, never a
|
|
# bare `{}`. Confirmed against Traefik's file provider, twice, with this
|
|
# exact plugin build (examples/docker-compose.yml, plus an isolated
|
|
# minimal repro outside that demo's compose-specific config mechanism): a
|
|
# bare `default: {}` fails plugin construction with
|
|
# "expected a map, got 'string'"; an explicit empty list decodes
|
|
# correctly and means the same thing (all values allowed). Whether the
|
|
# Kubernetes CRD provider used below hits the same decode path as the
|
|
# file provider is NOT verified — this file uses the empty-list form
|
|
# defensively, not because the CRD path is confirmed affected.
|
|
|
|
apiVersion: v1
|
|
kind: Secret
|
|
metadata:
|
|
name: llmgw-users
|
|
namespace: traefik
|
|
type: Opaque
|
|
stringData:
|
|
# Read by the plugin via users.file (mounted below), and hot-reloaded —
|
|
# no Traefik restart needed to add, remove, or rotate a user. Shape is
|
|
# fixed by users_file.go: {"users":[{"name","group","apiKey","limits"}]}.
|
|
# limits is optional per user; omit it to inherit the user's group limits.
|
|
users.json: |
|
|
{
|
|
"users": [
|
|
{"name": "alice", "group": "engineering", "apiKey": "sk-llmgw-alice-CHANGE-ME"},
|
|
{"name": "bob", "group": "general", "apiKey": "sk-llmgw-bob-CHANGE-ME"}
|
|
]
|
|
}
|
|
|
|
---
|
|
apiVersion: traefik.io/v1alpha1
|
|
kind: Middleware
|
|
metadata:
|
|
name: llmgateway
|
|
namespace: traefik
|
|
spec:
|
|
plugin:
|
|
llmgateway:
|
|
providers:
|
|
openai:
|
|
type: openai
|
|
apiKey: "env:OPENAI_API_KEY"
|
|
discovery: true
|
|
discoveryInterval: 1h
|
|
anthropic:
|
|
type: anthropic
|
|
apiKey: "env:ANTHROPIC_API_KEY"
|
|
models: ["claude-opus-4-5", "claude-sonnet-4-5", "claude-haiku-4-5"]
|
|
gemini:
|
|
type: gemini
|
|
apiKey: "env:GEMINI_API_KEY"
|
|
models: ["gemini-2.5-pro", "gemini-2.5-flash"]
|
|
groups:
|
|
engineering:
|
|
# Empty lists, not omitted fields, so this block is a real map
|
|
# entry (see the empty-map gotcha note above) — every provider
|
|
# and model stays allowed either way (empty = all).
|
|
providers: []
|
|
models: []
|
|
limits:
|
|
requestsPerMinute: 60
|
|
requestsPerDay: 5000
|
|
tokensPerDay: 1000000
|
|
tokensPerMonth: 20000000
|
|
costPerDayUSD: 25.0
|
|
costPerMonthUSD: 250.0
|
|
mcpServers: []
|
|
agents: []
|
|
general:
|
|
providers: ["openai"]
|
|
models: ["gpt-5-mini", "gpt-4o"]
|
|
limits:
|
|
requestsPerMinute: 10
|
|
requestsPerDay: 500
|
|
users:
|
|
file: /secrets/llmgw-users/users.json
|
|
inline:
|
|
# Grants the admin dashboard (see the "admin" block below),
|
|
# which stays unreachable until admin.enabled is true. Declared
|
|
# inline here — rather than added to the users.json Secret
|
|
# above — only to keep this admin-granting example visible in
|
|
# one file; the key itself is NOT a literal, it uses the same
|
|
# env: form as every other apiKey in this Middleware, and its
|
|
# env var (LLMGW_ADMIN_KEY) must be set on the Traefik pod
|
|
# regardless of whether admin.enabled is true or false below —
|
|
# every inline user's apiKey resolves at construction time, not
|
|
# lazily on first admin request. See the Helm env snippet at
|
|
# the bottom of this file.
|
|
- name: dashboard-admin
|
|
group: engineering
|
|
apiKey: "env:LLMGW_ADMIN_KEY"
|
|
admin: true
|
|
redis:
|
|
address: redis.shared-resources.svc:6379
|
|
password: "env:REDIS_PASSWORD"
|
|
failOpen: true
|
|
retry:
|
|
enabled: true
|
|
attempts: 2 # retries after the first try: 3 tries total
|
|
backoff: 250ms
|
|
# Opt-in response cache for non-streaming chat/embeddings — see
|
|
# ../README.md#caching. Requires the redis block above; without it,
|
|
# this would silently stay disabled (one warning logged) rather than
|
|
# fail construction.
|
|
cache:
|
|
enabled: true
|
|
ttl: 5m
|
|
# Disabled here deliberately: flip to `enabled: true` to try the
|
|
# read-only dashboard at https://llm.example.com/admin — the
|
|
# "dashboard-admin" user above (users.inline) already grants access
|
|
# once this is on, and its LLMGW_ADMIN_KEY env var is already
|
|
# required regardless (see that user's own comment) — see
|
|
# ../README.md#admin.
|
|
admin:
|
|
enabled: false
|
|
# Prometheus text-exposition endpoint — see ../README.md#metrics.
|
|
# allowedCIDRs is the cluster's OWN pod network (a placeholder here;
|
|
# replace 10.42.0.0/16 with your cluster CIDR, e.g. `kubectl get
|
|
# nodes -o jsonpath='{.items[0].spec.podCIDR}'`), letting an
|
|
# in-cluster scraper (VMServiceScrape/ServiceMonitor below) hit
|
|
# /metrics with no credential. NEVER widen this to a whole RFC1918
|
|
# range like 10.0.0.0/8 — see ../README.md#metrics for why that
|
|
# specific mistake is live-reproducible, not theoretical: a
|
|
# SNAT-ing router or load balancer makes EXTERNAL traffic arrive
|
|
# from an address inside that same private range too. The
|
|
# dashboard-admin key above (users.inline) works here regardless,
|
|
# for a scraper that lives outside allowedCIDRs.
|
|
metrics:
|
|
enabled: true
|
|
allowedCIDRs:
|
|
- 10.42.0.0/16
|
|
modelLabel: false
|
|
pricing:
|
|
gpt-5-mini: {inputPerM: 0.25, outputPerM: 2.0}
|
|
mcpServers:
|
|
github:
|
|
url: http://mcp-github.tools.svc:8080
|
|
agents:
|
|
summarizer:
|
|
url: http://summarizer.agents.svc:8080
|
|
passthroughUnknown: false
|
|
|
|
---
|
|
# Traefik requires a real backing Service for a router even though this
|
|
# middleware short-circuits every path it recognizes; whoami (the same
|
|
# image already exercised in ../examples/docker-compose.yml) is a small,
|
|
# always-up dummy, only ever reached via passthroughUnknown (default
|
|
# false, which 404s instead). Any always-up internal service works here.
|
|
apiVersion: apps/v1
|
|
kind: Deployment
|
|
metadata:
|
|
name: llmgateway-dummy-backend
|
|
namespace: traefik
|
|
spec:
|
|
replicas: 1
|
|
selector:
|
|
matchLabels:
|
|
app: llmgateway-dummy-backend
|
|
template:
|
|
metadata:
|
|
labels:
|
|
app: llmgateway-dummy-backend
|
|
spec:
|
|
containers:
|
|
- name: whoami
|
|
image: traefik/whoami:v1.10
|
|
ports:
|
|
- containerPort: 80
|
|
|
|
---
|
|
apiVersion: v1
|
|
kind: Service
|
|
metadata:
|
|
name: llmgateway-dummy-backend
|
|
namespace: traefik
|
|
spec:
|
|
selector:
|
|
app: llmgateway-dummy-backend
|
|
ports:
|
|
- port: 80
|
|
targetPort: 80
|
|
|
|
---
|
|
apiVersion: traefik.io/v1alpha1
|
|
kind: IngressRoute
|
|
metadata:
|
|
name: llmgateway
|
|
namespace: traefik
|
|
spec:
|
|
entryPoints:
|
|
- websecure
|
|
routes:
|
|
- match: Host(`llm.example.com`)
|
|
kind: Rule
|
|
middlewares:
|
|
- name: llmgateway
|
|
services:
|
|
- name: llmgateway-dummy-backend
|
|
port: 80
|
|
|
|
---
|
|
# A SEPARATE, Host-free IngressRoute for the metrics scrape (review fix,
|
|
# adversarial verification 2026-08-23): Traefik evaluates a router's
|
|
# match rule BEFORE any middleware runs, and the ONLY router above
|
|
# requires Host(`llm.example.com`). A VMServiceScrape/ServiceMonitor (below)
|
|
# reaches this Service through its cluster-internal DNS name or the pod
|
|
# IP directly, so the scrape request's Host header is never
|
|
# llm.example.com — it never matches that rule, the llmgateway
|
|
# middleware (and therefore /metrics itself) never runs, and the scrape
|
|
# 404s. PathPrefix, not Host, is deliberate here: this router exists
|
|
# specifically so an in-cluster scraper, addressing this Service by
|
|
# whatever internal name/IP it resolves to, still reaches the metrics
|
|
# route regardless of Host. /metrics is already gated by its own auth
|
|
# inside the plugin (an admin key, or metrics.allowedCIDRs — see
|
|
# ../README.md#metrics), so this being reachable without a Host match is
|
|
# not a second, unguarded door — but if your own Traefik instance fronts
|
|
# other Host-restricted routers on the SAME entryPoint, this route is
|
|
# reachable via any of THEIR hostnames too, not only in-cluster; narrow
|
|
# entryPoints below to an internal-only one, or add your own Host()
|
|
# match for your cluster's internal DNS name, if that reachability is
|
|
# broader than you want. The alternative the review that flagged this
|
|
# also named — keeping one Host-scoped router and instead using
|
|
# VMServiceScrape's own relabelConfigs to rewrite __address__ to
|
|
# llm.example.com before the request is sent — works too; this file uses
|
|
# the simpler of the two.
|
|
apiVersion: traefik.io/v1alpha1
|
|
kind: IngressRoute
|
|
metadata:
|
|
name: llmgateway-metrics
|
|
namespace: traefik
|
|
spec:
|
|
entryPoints:
|
|
- websecure
|
|
routes:
|
|
- match: PathPrefix(`/metrics`)
|
|
kind: Rule
|
|
middlewares:
|
|
- name: llmgateway
|
|
services:
|
|
- name: llmgateway-dummy-backend
|
|
port: 80
|
|
|
|
---
|
|
# Scraping /metrics (../README.md#metrics): this cluster runs
|
|
# VictoriaMetrics, not Prometheus, so the CRD is VMServiceScrape
|
|
# (victoriametrics/operator), not a Prometheus-operator ServiceMonitor —
|
|
# the exposition FORMAT this plugin emits is identical either way (a
|
|
# ServiceMonitor with the same selector/port/path works unchanged if
|
|
# your cluster runs Prometheus instead); only the CRD kind differs, and
|
|
# that is entirely outside this plugin's own config.
|
|
#
|
|
# IMPORTANT: /metrics is served by TRAEFIK ITSELF, not by
|
|
# llmgateway-dummy-backend above — this middleware intercepts the path
|
|
# before any backend service is ever reached (same as every other route
|
|
# this plugin owns), which is exactly why the Host-free IngressRoute
|
|
# above exists: without it this scrape 404s before the middleware ever
|
|
# sees the request. Point this scrape at YOUR Traefik installation's own
|
|
# Service (commonly named "traefik" — verify against your own Helm
|
|
# release), never at a backend service this file defines. Adjust
|
|
# selector/namespaceSelector/port to match your Traefik Service's real
|
|
# labels and named port.
|
|
apiVersion: operator.victoriametrics.com/v1beta1
|
|
kind: VMServiceScrape
|
|
metadata:
|
|
name: llmgateway-metrics
|
|
namespace: traefik
|
|
spec:
|
|
namespaceSelector:
|
|
matchNames:
|
|
- traefik
|
|
selector:
|
|
matchLabels:
|
|
app.kubernetes.io/name: traefik
|
|
endpoints:
|
|
- port: websecure # the Traefik Service's own named entryPoint port llmgateway is exposed on
|
|
path: /metrics # the default; matches metrics.path left unset above
|
|
scheme: https
|
|
tlsConfig:
|
|
insecureSkipVerify: true # adjust for your cluster's own CA/cert setup
|
|
|
|
---
|
|
# Helm values snippet (traefik/traefik-helm-chart) — illustrative, verify
|
|
# field names against the chart version you deploy. Two things are
|
|
# required beyond a stock install:
|
|
#
|
|
# 1. Load the plugin. Either the catalog form, which pins a released tag
|
|
# and is the better default for production, or localPlugins, mounting
|
|
# this repository's source into the Traefik pod the same way
|
|
# ../examples/docker-compose.yml does for its local demo:
|
|
#
|
|
# experimental:
|
|
# plugins:
|
|
# llmgateway:
|
|
# moduleName: "github.com/lukaszraczylo/traefik-llmgateway"
|
|
# version: "v1.0.3"
|
|
#
|
|
# 2. Every "env:VAR_NAME" secret this Middleware references as env vars,
|
|
# sourced from Secrets — never inlined into the Middleware CR above.
|
|
# This is NOT just the three provider keys and the Redis password:
|
|
# LLMGW_ADMIN_KEY (the "dashboard-admin" user above) resolves at
|
|
# construction time too, so it must be set here even while
|
|
# admin.enabled is false — an unset "env:" var is a hard construction
|
|
# error regardless of whether the feature it gates is on.
|
|
#
|
|
# env:
|
|
# - name: OPENAI_API_KEY
|
|
# valueFrom: {secretKeyRef: {name: llmgw-provider-keys, key: openai}}
|
|
# - name: ANTHROPIC_API_KEY
|
|
# valueFrom: {secretKeyRef: {name: llmgw-provider-keys, key: anthropic}}
|
|
# - name: GEMINI_API_KEY
|
|
# valueFrom: {secretKeyRef: {name: llmgw-provider-keys, key: gemini}}
|
|
# - name: REDIS_PASSWORD
|
|
# valueFrom: {secretKeyRef: {name: llmgw-redis, key: password}}
|
|
# - name: LLMGW_ADMIN_KEY
|
|
# valueFrom: {secretKeyRef: {name: llmgw-admin-key, key: apiKey}}
|
|
#
|
|
# additionalVolumes:
|
|
# - name: llmgw-users
|
|
# secret: {secretName: llmgw-users}
|
|
# deployment:
|
|
# additionalVolumeMounts:
|
|
# # Whole-volume mount — no subPath — is required, not a style
|
|
# # choice: a subPath mount binds once at pod start and the
|
|
# # kubelet never updates it again, so editing the llmgw-users
|
|
# # Secret above would never reach this pod. A whole-volume mount
|
|
# # updates via the kubelet swapping a symlink for the whole
|
|
# # directory, which users.file's own 5-second mtime poll
|
|
# # (reloadEvery, users_file.go) picks up with no pod restart —
|
|
# # see ../README.md#config-hot-reload.
|
|
# - name: llmgw-users
|
|
# mountPath: /secrets/llmgw-users
|
|
# readOnly: true
|