traefik-llmgateway/examples/kubernetes.yaml

# Example Kubernetes deployment of the LLM Gateway Traefik plugin: a
# Secret holding the file-backed users list, a Middleware custom resource
# carrying the plugin's own configuration, and a router wiring both onto a
# host. See ../README.md for the full configuration reference and
# ../.traefik.yml for the catalog-facing plugin manifest.
#
# Provider API keys are NOT put in this file. They are read by the plugin
# at startup via "env:VAR_NAME" (see the Middleware below), which means
# they must be set as environment variables on the Traefik pod itself —
# see the Helm values snippet at the bottom of this file.
#
# The empty-map gotcha: write an empty group as `providers: []`, never a
# bare `{}`. Confirmed against Traefik's file provider, twice, with this
# exact plugin build (examples/docker-compose.yml, plus an isolated
# minimal repro outside that demo's compose-specific config mechanism): a
# bare `default: {}` fails plugin construction with
# "expected a map, got 'string'"; an explicit empty list decodes
# correctly and means the same thing (all values allowed). Whether the
# Kubernetes CRD provider used below hits the same decode path as the
# file provider is NOT verified — this file uses the empty-list form
# defensively, not because the CRD path is confirmed affected.

apiVersion: v1
kind: Secret
metadata:
  name: llmgw-users
  namespace: traefik
type: Opaque
stringData:
  # Read by the plugin via users.file (mounted below), and hot-reloaded —
  # no Traefik restart needed to add, remove, or rotate a user. Shape is
  # fixed by users_file.go: {"users":[{"name","group","apiKey","limits"}]}.
  # limits is optional per user; omit it to inherit the user's group limits.
  users.json: |
    {
      "users": [
        {"name": "alice", "group": "engineering", "apiKey": "sk-llmgw-alice-CHANGE-ME"},
        {"name": "bob", "group": "general", "apiKey": "sk-llmgw-bob-CHANGE-ME"}
      ]
    }

---
apiVersion: traefik.io/v1alpha1
kind: Middleware
metadata:
  name: llmgateway
  namespace: traefik
spec:
  plugin:
    llmgateway:
      providers:
        openai:
          type: openai
          apiKey: "env:OPENAI_API_KEY"
          discovery: true
          discoveryInterval: 1h
        anthropic:
          type: anthropic
          apiKey: "env:ANTHROPIC_API_KEY"
          models: ["claude-opus-4-5", "claude-sonnet-4-5", "claude-haiku-4-5"]
        gemini:
          type: gemini
          apiKey: "env:GEMINI_API_KEY"
          models: ["gemini-2.5-pro", "gemini-2.5-flash"]
      groups:
        engineering:
          # Empty lists, not omitted fields, so this block is a real map
          # entry (see the empty-map gotcha note above) — every provider
          # and model stays allowed either way (empty = all).
          providers: []
          models: []
          limits:
            requestsPerMinute: 60
            requestsPerDay: 5000
            tokensPerDay: 1000000
            tokensPerMonth: 20000000
            costPerDayUSD: 25.0
            costPerMonthUSD: 250.0
          mcpServers: []
          agents: []
        general:
          providers: ["openai"]
          models: ["gpt-5-mini", "gpt-4o"]
          limits:
            requestsPerMinute: 10
            requestsPerDay: 500
      users:
        file: /secrets/llmgw-users/users.json
        inline:
          # Grants the admin dashboard (see the "admin" block below),
          # which stays unreachable until admin.enabled is true. Declared
          # inline here — rather than added to the users.json Secret
          # above — only to keep this admin-granting example visible in
          # one file; the key itself is NOT a literal, it uses the same
          # env: form as every other apiKey in this Middleware, and its
          # env var (LLMGW_ADMIN_KEY) must be set on the Traefik pod
          # regardless of whether admin.enabled is true or false below —
          # every inline user's apiKey resolves at construction time, not
          # lazily on first admin request. See the Helm env snippet at
          # the bottom of this file.
          - name: dashboard-admin
            group: engineering
            apiKey: "env:LLMGW_ADMIN_KEY"
            admin: true
      redis:
        address: redis.shared-resources.svc:6379
        password: "env:REDIS_PASSWORD"
        failOpen: true
      retry:
        enabled: true
        attempts: 2 # retries after the first try: 3 tries total
        backoff: 250ms
      # Opt-in response cache for non-streaming chat/embeddings — see
      # ../README.md#caching. Requires the redis block above; without it,
      # this would silently stay disabled (one warning logged) rather than
      # fail construction.
      cache:
        enabled: true
        ttl: 5m
      # Disabled here deliberately: flip to `enabled: true` to try the
      # read-only dashboard at https://llm.example.com/admin — the
      # "dashboard-admin" user above (users.inline) already grants access
      # once this is on, and its LLMGW_ADMIN_KEY env var is already
      # required regardless (see that user's own comment) — see
      # ../README.md#admin.
      admin:
        enabled: false
      pricing:
        gpt-5-mini: {inputPerM: 0.25, outputPerM: 2.0}
      mcpServers:
        github:
          url: http://mcp-github.tools.svc:8080
      agents:
        summarizer:
          url: http://summarizer.agents.svc:8080
      passthroughUnknown: false

---
# Traefik requires a real backing Service for a router even though this
# middleware short-circuits every path it recognizes; whoami (the same
# image already exercised in ../examples/docker-compose.yml) is a small,
# always-up dummy, only ever reached via passthroughUnknown (default
# false, which 404s instead). Any always-up internal service works here.
apiVersion: apps/v1
kind: Deployment
metadata:
  name: llmgateway-dummy-backend
  namespace: traefik
spec:
  replicas: 1
  selector:
    matchLabels:
      app: llmgateway-dummy-backend
  template:
    metadata:
      labels:
        app: llmgateway-dummy-backend
    spec:
      containers:
        - name: whoami
          image: traefik/whoami:v1.10
          ports:
            - containerPort: 80

---
apiVersion: v1
kind: Service
metadata:
  name: llmgateway-dummy-backend
  namespace: traefik
spec:
  selector:
    app: llmgateway-dummy-backend
  ports:
    - port: 80
      targetPort: 80

---
apiVersion: traefik.io/v1alpha1
kind: IngressRoute
metadata:
  name: llmgateway
  namespace: traefik
spec:
  entryPoints:
    - websecure
  routes:
    - match: Host(`llm.example.com`)
      kind: Rule
      middlewares:
        - name: llmgateway
      services:
        - name: llmgateway-dummy-backend
          port: 80

---
# Helm values snippet (traefik/traefik-helm-chart) — illustrative, verify
# field names against the chart version you deploy. Two things are
# required beyond a stock install:
#
# 1. Load the plugin. Either the catalog form (once this plugin is listed
#    in the Traefik Plugin Catalog, see ../.traefik.yml) or localPlugins,
#    mounting this repository's source into the Traefik pod the same way
#    ../examples/docker-compose.yml does for its local demo:
#
#      experimental:
#        plugins:
#          llmgateway:
#            moduleName: "github.com/lukaszraczylo/traefik-llmgateway"
#            version: "v0.1.0"
#
# 2. Every "env:VAR_NAME" secret this Middleware references as env vars,
#    sourced from Secrets — never inlined into the Middleware CR above.
#    This is NOT just the three provider keys and the Redis password:
#    LLMGW_ADMIN_KEY (the "dashboard-admin" user above) resolves at
#    construction time too, so it must be set here even while
#    admin.enabled is false — an unset "env:" var is a hard construction
#    error regardless of whether the feature it gates is on.
#
#      env:
#        - name: OPENAI_API_KEY
#          valueFrom: {secretKeyRef: {name: llmgw-provider-keys, key: openai}}
#        - name: ANTHROPIC_API_KEY
#          valueFrom: {secretKeyRef: {name: llmgw-provider-keys, key: anthropic}}
#        - name: GEMINI_API_KEY
#          valueFrom: {secretKeyRef: {name: llmgw-provider-keys, key: gemini}}
#        - name: REDIS_PASSWORD
#          valueFrom: {secretKeyRef: {name: llmgw-redis, key: password}}
#        - name: LLMGW_ADMIN_KEY
#          valueFrom: {secretKeyRef: {name: llmgw-admin-key, key: apiKey}}
#
#      additionalVolumes:
#        - name: llmgw-users
#          secret: {secretName: llmgw-users}
#      deployment:
#        additionalVolumeMounts:
#          - name: llmgw-users
#            mountPath: /secrets/llmgw-users
#            readOnly: true