traefik-llmgateway/examples/kubernetes.yaml

# Example Kubernetes deployment of the LLM Gateway Traefik plugin: a
# Secret holding the file-backed users list, a Middleware custom resource
# carrying the plugin's own configuration, and a router wiring both onto a
# host. See ../README.md for the full configuration reference and
# ../.traefik.yml for the catalog-facing plugin manifest.
#
# Provider API keys are NOT put in this file. They are read by the plugin
# at startup via "env:VAR_NAME" (see the Middleware below), which means
# they must be set as environment variables on the Traefik pod itself —
# see the Helm values snippet at the bottom of this file.
#
# The empty-map gotcha: write an empty group as `providers: []`, never a
# bare `{}`. Confirmed against Traefik's file provider, twice, with this
# exact plugin build (examples/docker-compose.yml, plus an isolated
# minimal repro outside that demo's compose-specific config mechanism): a
# bare `default: {}` fails plugin construction with
# "expected a map, got 'string'"; an explicit empty list decodes
# correctly and means the same thing (all values allowed). Whether the
# Kubernetes CRD provider used below hits the same decode path as the
# file provider is NOT verified — this file uses the empty-list form
# defensively, not because the CRD path is confirmed affected.

apiVersion: v1
kind: Secret
metadata:
  name: llmgw-users
  namespace: traefik
type: Opaque
stringData:
  # Read by the plugin via users.file (mounted below), and hot-reloaded —
  # no Traefik restart needed to add, remove, or rotate a user. Shape is
  # fixed by users_file.go: {"users":[{"name","group","apiKey","limits"}]}.
  # limits is optional per user; omit it to inherit the user's group limits.
  users.json: |
    {
      "users": [
        {"name": "alice", "group": "engineering", "apiKey": "sk-llmgw-alice-CHANGE-ME"},
        {"name": "bob", "group": "general", "apiKey": "sk-llmgw-bob-CHANGE-ME"}
      ]
    }

---
apiVersion: traefik.io/v1alpha1
kind: Middleware
metadata:
  name: llmgateway
  namespace: traefik
spec:
  plugin:
    llmgateway:
      providers:
        openai:
          type: openai
          apiKey: "env:OPENAI_API_KEY"
          discovery: true
          discoveryInterval: 1h
        anthropic:
          type: anthropic
          apiKey: "env:ANTHROPIC_API_KEY"
          models: ["claude-opus-4-5", "claude-sonnet-4-5", "claude-haiku-4-5"]
        gemini:
          type: gemini
          apiKey: "env:GEMINI_API_KEY"
          models: ["gemini-2.5-pro", "gemini-2.5-flash"]
      groups:
        engineering:
          # Empty lists, not omitted fields, so this block is a real map
          # entry (see the empty-map gotcha note above) — every provider
          # and model stays allowed either way (empty = all).
          providers: []
          models: []
          limits:
            requestsPerMinute: 60
            requestsPerDay: 5000
            tokensPerDay: 1000000
            tokensPerMonth: 20000000
            costPerDayUSD: 25.0
            costPerMonthUSD: 250.0
          mcpServers: []
          agents: []
        general:
          providers: ["openai"]
          models: ["gpt-5-mini", "gpt-4o"]
          limits:
            requestsPerMinute: 10
            requestsPerDay: 500
      users:
        file: /secrets/llmgw-users/users.json
        inline:
          # Grants the admin dashboard (see the "admin" block below),
          # which stays unreachable until admin.enabled is true. Declared
          # inline here — rather than added to the users.json Secret
          # above — only to keep this admin-granting example visible in
          # one file; the key itself is NOT a literal, it uses the same
          # env: form as every other apiKey in this Middleware, and its
          # env var (LLMGW_ADMIN_KEY) must be set on the Traefik pod
          # regardless of whether admin.enabled is true or false below —
          # every inline user's apiKey resolves at construction time, not
          # lazily on first admin request. See the Helm env snippet at
          # the bottom of this file.
          - name: dashboard-admin
            group: engineering
            apiKey: "env:LLMGW_ADMIN_KEY"
            admin: true
      redis:
        address: redis.shared-resources.svc:6379
        password: "env:REDIS_PASSWORD"
        failOpen: true
      retry:
        enabled: true
        attempts: 2 # retries after the first try: 3 tries total
        backoff: 250ms
      # Opt-in response cache for non-streaming chat/embeddings — see
      # ../README.md#caching. Requires the redis block above; without it,
      # this would silently stay disabled (one warning logged) rather than
      # fail construction.
      cache:
        enabled: true
        ttl: 5m
      # Disabled here deliberately: flip to `enabled: true` to try the
      # read-only dashboard at https://llm.example.com/admin — the
      # "dashboard-admin" user above (users.inline) already grants access
      # once this is on, and its LLMGW_ADMIN_KEY env var is already
      # required regardless (see that user's own comment) — see
      # ../README.md#admin.
      admin:
        enabled: false
      # Prometheus text-exposition endpoint — see ../README.md#metrics.
      # allowedCIDRs is the cluster's OWN pod network (a placeholder here;
      # replace 10.42.0.0/16 with your cluster CIDR, e.g. `kubectl get
      # nodes -o jsonpath='{.items[0].spec.podCIDR}'`), letting an
      # in-cluster scraper (VMServiceScrape/ServiceMonitor below) hit
      # /metrics with no credential. NEVER widen this to a whole RFC1918
      # range like 10.0.0.0/8 — see ../README.md#metrics for why that
      # specific mistake is live-reproducible, not theoretical: a
      # SNAT-ing router or load balancer makes EXTERNAL traffic arrive
      # from an address inside that same private range too. The
      # dashboard-admin key above (users.inline) works here regardless,
      # for a scraper that lives outside allowedCIDRs.
      metrics:
        enabled: true
        allowedCIDRs:
          - 10.42.0.0/16
        modelLabel: false
      pricing:
        gpt-5-mini: {inputPerM: 0.25, outputPerM: 2.0}
      mcpServers:
        github:
          url: http://mcp-github.tools.svc:8080
      agents:
        summarizer:
          url: http://summarizer.agents.svc:8080
      passthroughUnknown: false

---
# Traefik requires a real backing Service for a router even though this
# middleware short-circuits every path it recognizes; whoami (the same
# image already exercised in ../examples/docker-compose.yml) is a small,
# always-up dummy, only ever reached via passthroughUnknown (default
# false, which 404s instead). Any always-up internal service works here.
apiVersion: apps/v1
kind: Deployment
metadata:
  name: llmgateway-dummy-backend
  namespace: traefik
spec:
  replicas: 1
  selector:
    matchLabels:
      app: llmgateway-dummy-backend
  template:
    metadata:
      labels:
        app: llmgateway-dummy-backend
    spec:
      containers:
        - name: whoami
          image: traefik/whoami:v1.10
          ports:
            - containerPort: 80

---
apiVersion: v1
kind: Service
metadata:
  name: llmgateway-dummy-backend
  namespace: traefik
spec:
  selector:
    app: llmgateway-dummy-backend
  ports:
    - port: 80
      targetPort: 80

---
apiVersion: traefik.io/v1alpha1
kind: IngressRoute
metadata:
  name: llmgateway
  namespace: traefik
spec:
  entryPoints:
    - websecure
  routes:
    - match: Host(`llm.example.com`)
      kind: Rule
      middlewares:
        - name: llmgateway
      services:
        - name: llmgateway-dummy-backend
          port: 80

---
# A SEPARATE, Host-free IngressRoute for the metrics scrape (review fix,
# adversarial verification 2026-08-23): Traefik evaluates a router's
# match rule BEFORE any middleware runs, and the ONLY router above
# requires Host(`llm.example.com`). A VMServiceScrape/ServiceMonitor (below)
# reaches this Service through its cluster-internal DNS name or the pod
# IP directly, so the scrape request's Host header is never
# llm.example.com — it never matches that rule, the llmgateway
# middleware (and therefore /metrics itself) never runs, and the scrape
# 404s. PathPrefix, not Host, is deliberate here: this router exists
# specifically so an in-cluster scraper, addressing this Service by
# whatever internal name/IP it resolves to, still reaches the metrics
# route regardless of Host. /metrics is already gated by its own auth
# inside the plugin (an admin key, or metrics.allowedCIDRs — see
# ../README.md#metrics), so this being reachable without a Host match is
# not a second, unguarded door — but if your own Traefik instance fronts
# other Host-restricted routers on the SAME entryPoint, this route is
# reachable via any of THEIR hostnames too, not only in-cluster; narrow
# entryPoints below to an internal-only one, or add your own Host()
# match for your cluster's internal DNS name, if that reachability is
# broader than you want. The alternative the review that flagged this
# also named — keeping one Host-scoped router and instead using
# VMServiceScrape's own relabelConfigs to rewrite __address__ to
# llm.example.com before the request is sent — works too; this file uses
# the simpler of the two.
apiVersion: traefik.io/v1alpha1
kind: IngressRoute
metadata:
  name: llmgateway-metrics
  namespace: traefik
spec:
  entryPoints:
    - websecure
  routes:
    - match: PathPrefix(`/metrics`)
      kind: Rule
      middlewares:
        - name: llmgateway
      services:
        - name: llmgateway-dummy-backend
          port: 80

---
# Scraping /metrics (../README.md#metrics): this cluster runs
# VictoriaMetrics, not Prometheus, so the CRD is VMServiceScrape
# (victoriametrics/operator), not a Prometheus-operator ServiceMonitor —
# the exposition FORMAT this plugin emits is identical either way (a
# ServiceMonitor with the same selector/port/path works unchanged if
# your cluster runs Prometheus instead); only the CRD kind differs, and
# that is entirely outside this plugin's own config.
#
# IMPORTANT: /metrics is served by TRAEFIK ITSELF, not by
# llmgateway-dummy-backend above — this middleware intercepts the path
# before any backend service is ever reached (same as every other route
# this plugin owns), which is exactly why the Host-free IngressRoute
# above exists: without it this scrape 404s before the middleware ever
# sees the request. Point this scrape at YOUR Traefik installation's own
# Service (commonly named "traefik" — verify against your own Helm
# release), never at a backend service this file defines. Adjust
# selector/namespaceSelector/port to match your Traefik Service's real
# labels and named port.
apiVersion: operator.victoriametrics.com/v1beta1
kind: VMServiceScrape
metadata:
  name: llmgateway-metrics
  namespace: traefik
spec:
  namespaceSelector:
    matchNames:
      - traefik
  selector:
    matchLabels:
      app.kubernetes.io/name: traefik
  endpoints:
    - port: websecure # the Traefik Service's own named entryPoint port llmgateway is exposed on
      path: /metrics # the default; matches metrics.path left unset above
      scheme: https
      tlsConfig:
        insecureSkipVerify: true # adjust for your cluster's own CA/cert setup

---
# Helm values snippet (traefik/traefik-helm-chart) — illustrative, verify
# field names against the chart version you deploy. Two things are
# required beyond a stock install:
#
# 1. Load the plugin. Either the catalog form, which pins a released tag
#    and is the better default for production, or localPlugins, mounting
#    this repository's source into the Traefik pod the same way
#    ../examples/docker-compose.yml does for its local demo:
#
#      experimental:
#        plugins:
#          llmgateway:
#            moduleName: "github.com/lukaszraczylo/traefik-llmgateway"
#            version: "v1.0.3"
#
# 2. Every "env:VAR_NAME" secret this Middleware references as env vars,
#    sourced from Secrets — never inlined into the Middleware CR above.
#    This is NOT just the three provider keys and the Redis password:
#    LLMGW_ADMIN_KEY (the "dashboard-admin" user above) resolves at
#    construction time too, so it must be set here even while
#    admin.enabled is false — an unset "env:" var is a hard construction
#    error regardless of whether the feature it gates is on.
#
#      env:
#        - name: OPENAI_API_KEY
#          valueFrom: {secretKeyRef: {name: llmgw-provider-keys, key: openai}}
#        - name: ANTHROPIC_API_KEY
#          valueFrom: {secretKeyRef: {name: llmgw-provider-keys, key: anthropic}}
#        - name: GEMINI_API_KEY
#          valueFrom: {secretKeyRef: {name: llmgw-provider-keys, key: gemini}}
#        - name: REDIS_PASSWORD
#          valueFrom: {secretKeyRef: {name: llmgw-redis, key: password}}
#        - name: LLMGW_ADMIN_KEY
#          valueFrom: {secretKeyRef: {name: llmgw-admin-key, key: apiKey}}
#
#      additionalVolumes:
#        - name: llmgw-users
#          secret: {secretName: llmgw-users}
#      deployment:
#        additionalVolumeMounts:
#          # Whole-volume mount — no subPath — is required, not a style
#          # choice: a subPath mount binds once at pod start and the
#          # kubelet never updates it again, so editing the llmgw-users
#          # Secret above would never reach this pod. A whole-volume mount
#          # updates via the kubelet swapping a symlink for the whole
#          # directory, which users.file's own 5-second mtime poll
#          # (reloadEvery, users_file.go) picks up with no pod restart —
#          # see ../README.md#config-hot-reload.
#          - name: llmgw-users
#            mountPath: /secrets/llmgw-users
#            readOnly: true