mirror of
https://github.com/lukaszraczylo/traefik-llmgateway.git
synced 2026-10-12 04:54:45 +00:00
docs: README, kubernetes and compose examples, MIT license
- README.md: what-it-is, quickstart (localPlugins today, catalog form
once published), full config reference table for every Config/
ProviderConfig/GroupConfig/LimitsConfig/UsersConfig/UserConfig/
RedisConfig/ModelPricing/TargetConfig/AgentConfig field, model routing,
limits and accounting semantics, unified vs passthrough differences,
MCP/A2A, security notes, known limitations (streaming buffering under
Yaegi, quota-on-reject nuances, identity-encoding, path-traversal
rejects), Kubernetes deployment notes, and development/catalog-
submission status. Every claim grounded against the actual source read
this session; two draft claims (user-limits "replace" group limits, an
unqualified "go-redis is documented as unsuitable in resp.go/
providers.go") were caught and corrected against the real
buildLimitScopes semantics and an actual grep before committing.
- examples/kubernetes.yaml: Secret (users.json) + Middleware CR +
IngressRoute + a Helm values snippet. The Middleware's plugin config
round-trips through the real Config struct and New() (verified via a
throwaway, since-deleted test), not just YAML-syntax-checked.
- examples/docker-compose.yml: minimal localPlugins-based demo (traefik +
whoami + inline file-provider config via compose's inline configs.content).
Brought up and exercised end-to-end against real OpenAI this session:
auth 401/200, /v1/models, /v1/chat/completions (upstream 401 envelope
with a fake key, proving passthrough+key-injection+wrapping all work),
and the unknown-route 404 envelope. Found and fixed a real bug in the
process: a bare "{}" empty group value fails Traefik's own dynamic-
config decode with "expected a map, got 'string'"; fixed to an
explicit empty list and documented the gotcha in both example files and
the README.
- LICENSE: MIT, copyright 2026 Lukasz Raczylo.
This commit is contained in:
@@ -0,0 +1,95 @@
|
||||
# Minimal, runnable local demo of the LLM Gateway plugin: one Traefik
|
||||
# instance, loading this repository as a localPlugin, in front of a single
|
||||
# OpenAI-type provider and a `whoami` dummy backend (every route the plugin
|
||||
# recognizes is short-circuited by the middleware itself; whoami only
|
||||
# answers a request that falls through as passthroughUnknown).
|
||||
#
|
||||
# Run from the repository root:
|
||||
#
|
||||
# OPENAI_API_KEY=sk-... docker compose -f examples/docker-compose.yml up
|
||||
#
|
||||
# Try it once traefik1 is healthy:
|
||||
#
|
||||
# curl -s http://localhost:8080/v1/models \
|
||||
# -H "Authorization: Bearer sk-demo-local"
|
||||
#
|
||||
# curl -s http://localhost:8080/v1/chat/completions \
|
||||
# -H "Authorization: Bearer sk-demo-local" \
|
||||
# -H "Content-Type: application/json" \
|
||||
# -d '{"model":"gpt-5-mini","messages":[{"role":"user","content":"hi"}]}'
|
||||
#
|
||||
# Override the demo API key with LLMGW_DEMO_KEY=... if you don't want the
|
||||
# default "sk-demo-local" in your shell history, and the published port
|
||||
# with LLMGW_DEMO_PORT=... if 8080 is already taken on your machine. See
|
||||
# ../README.md for the full configuration reference; see ../.traefik.yml
|
||||
# for the catalog-facing plugin manifest.
|
||||
|
||||
configs:
|
||||
dynamic:
|
||||
content: |
|
||||
http:
|
||||
routers:
|
||||
llmgateway-router:
|
||||
rule: "PathPrefix(`/`)"
|
||||
entryPoints: ["web"]
|
||||
service: whoami
|
||||
middlewares: ["llmgateway"]
|
||||
services:
|
||||
whoami:
|
||||
loadBalancer:
|
||||
servers:
|
||||
- url: "http://whoami:80"
|
||||
middlewares:
|
||||
llmgateway:
|
||||
plugin:
|
||||
llmgateway:
|
||||
providers:
|
||||
openai:
|
||||
type: openai
|
||||
apiKey: "env:OPENAI_API_KEY"
|
||||
# Explicit rather than discovery:true, so the demo
|
||||
# starts cleanly with no outbound network call — flip
|
||||
# this to discovery:true (and drop models) to pull the
|
||||
# live catalog from api.openai.com instead.
|
||||
models: ["gpt-5-mini", "gpt-5", "gpt-4o"]
|
||||
groups:
|
||||
default:
|
||||
providers: []
|
||||
users:
|
||||
inline:
|
||||
- name: demo
|
||||
group: default
|
||||
apiKey: "${LLMGW_DEMO_KEY:-sk-demo-local}"
|
||||
|
||||
services:
|
||||
whoami:
|
||||
image: traefik/whoami:v1.10
|
||||
networks: ["llmgw-demo"]
|
||||
|
||||
traefik1:
|
||||
image: traefik:v3.5
|
||||
command:
|
||||
- "--entrypoints.web.address=:80"
|
||||
- "--experimental.localPlugins.llmgateway.moduleName=github.com/lukaszraczylo/traefik-llmgateway"
|
||||
- "--providers.file.filename=/etc/traefik/dynamic.yml"
|
||||
- "--log.level=INFO"
|
||||
environment:
|
||||
# ":?" fails `docker compose config`/`up` immediately, with a clear
|
||||
# message, instead of starting a plugin whose provider has no key.
|
||||
OPENAI_API_KEY: "${OPENAI_API_KEY:?set OPENAI_API_KEY before running this demo}"
|
||||
ports:
|
||||
- "${LLMGW_DEMO_PORT:-8080}:80"
|
||||
volumes:
|
||||
# The plugin's own module, mounted read-only exactly as Traefik's
|
||||
# localPlugins loader expects: <plugins-local>/src/<moduleName>.
|
||||
- ..:/plugins-local/src/github.com/lukaszraczylo/traefik-llmgateway:ro
|
||||
configs:
|
||||
- source: dynamic
|
||||
target: /etc/traefik/dynamic.yml
|
||||
networks: ["llmgw-demo"]
|
||||
depends_on:
|
||||
- whoami
|
||||
|
||||
networks:
|
||||
llmgw-demo:
|
||||
driver: bridge
|
||||
@@ -0,0 +1,160 @@
|
||||
# Example Kubernetes deployment of the LLM Gateway Traefik plugin: a
|
||||
# Secret holding the file-backed users list, a Middleware custom resource
|
||||
# carrying the plugin's own configuration, and a router wiring both onto a
|
||||
# host. See ../README.md for the full configuration reference and
|
||||
# ../.traefik.yml for the catalog-facing plugin manifest.
|
||||
#
|
||||
# Provider API keys are NOT put in this file. They are read by the plugin
|
||||
# at startup via "env:VAR_NAME" (see the Middleware below), which means
|
||||
# they must be set as environment variables on the Traefik pod itself —
|
||||
# see the Helm values snippet at the bottom of this file.
|
||||
#
|
||||
# The empty-map gotcha: write an empty group as `providers: []`, never a
|
||||
# bare `{}`. Traefik's own dynamic-config decode pipeline (which the
|
||||
# Kubernetes CRD provider funnels plugin config through, same as the file
|
||||
# provider) can fail to round-trip a truly empty nested object with
|
||||
# "expected a map, got 'string'" — verified against this exact shape in
|
||||
# examples/docker-compose.yml. An explicit empty list decodes correctly
|
||||
# and means the same thing (all values allowed).
|
||||
|
||||
apiVersion: v1
|
||||
kind: Secret
|
||||
metadata:
|
||||
name: llmgw-users
|
||||
namespace: traefik
|
||||
type: Opaque
|
||||
stringData:
|
||||
# Read by the plugin via users.file (mounted below), and hot-reloaded —
|
||||
# no Traefik restart needed to add, remove, or rotate a user. Shape is
|
||||
# fixed by users_file.go: {"users":[{"name","group","apiKey","limits"}]}.
|
||||
# limits is optional per user; omit it to inherit the user's group limits.
|
||||
users.json: |
|
||||
{
|
||||
"users": [
|
||||
{"name": "alice", "group": "engineering", "apiKey": "sk-llmgw-alice-CHANGE-ME"},
|
||||
{"name": "bob", "group": "general", "apiKey": "sk-llmgw-bob-CHANGE-ME"}
|
||||
]
|
||||
}
|
||||
|
||||
---
|
||||
apiVersion: traefik.io/v1alpha1
|
||||
kind: Middleware
|
||||
metadata:
|
||||
name: llmgateway
|
||||
namespace: traefik
|
||||
spec:
|
||||
plugin:
|
||||
llmgateway:
|
||||
providers:
|
||||
openai:
|
||||
type: openai
|
||||
apiKey: "env:OPENAI_API_KEY"
|
||||
discovery: true
|
||||
discoveryInterval: 1h
|
||||
anthropic:
|
||||
type: anthropic
|
||||
apiKey: "env:ANTHROPIC_API_KEY"
|
||||
models: ["claude-opus-4-5", "claude-sonnet-4-5", "claude-haiku-4-5"]
|
||||
gemini:
|
||||
type: gemini
|
||||
apiKey: "env:GEMINI_API_KEY"
|
||||
models: ["gemini-2.5-pro", "gemini-2.5-flash"]
|
||||
groups:
|
||||
engineering:
|
||||
# Empty lists, not omitted fields, so this block is a real map
|
||||
# entry (see the empty-map gotcha note above) — every provider
|
||||
# and model stays allowed either way (empty = all).
|
||||
providers: []
|
||||
models: []
|
||||
limits:
|
||||
requestsPerMinute: 60
|
||||
requestsPerDay: 5000
|
||||
tokensPerDay: 1000000
|
||||
tokensPerMonth: 20000000
|
||||
costPerDayUSD: 25.0
|
||||
costPerMonthUSD: 250.0
|
||||
mcpServers: []
|
||||
agents: []
|
||||
general:
|
||||
providers: ["openai"]
|
||||
models: ["gpt-5-mini", "gpt-4o"]
|
||||
limits:
|
||||
requestsPerMinute: 10
|
||||
requestsPerDay: 500
|
||||
users:
|
||||
file: /secrets/llmgw-users/users.json
|
||||
redis:
|
||||
address: redis.shared-resources.svc:6379
|
||||
password: "env:REDIS_PASSWORD"
|
||||
failOpen: true
|
||||
pricing:
|
||||
gpt-5-mini: {inputPerM: 0.25, outputPerM: 2.0}
|
||||
mcpServers:
|
||||
github:
|
||||
url: http://mcp-github.tools.svc:8080
|
||||
agents:
|
||||
summarizer:
|
||||
url: http://summarizer.agents.svc:8080
|
||||
passthroughUnknown: false
|
||||
|
||||
---
|
||||
apiVersion: traefik.io/v1alpha1
|
||||
kind: IngressRoute
|
||||
metadata:
|
||||
name: llmgateway
|
||||
namespace: traefik
|
||||
spec:
|
||||
entryPoints:
|
||||
- websecure
|
||||
routes:
|
||||
- match: Host(`llm.example.com`)
|
||||
kind: Rule
|
||||
middlewares:
|
||||
- name: llmgateway
|
||||
# Traefik requires a real backing Service for a router even though
|
||||
# this middleware short-circuits every path it recognizes; a small
|
||||
# dummy such as traefik/whoami (see ../examples/docker-compose.yml)
|
||||
# or any always-up internal service works, since it is only ever
|
||||
# reached via passthroughUnknown (default false, which 404s instead).
|
||||
services:
|
||||
- name: llmgateway-dummy-backend
|
||||
port: 80
|
||||
|
||||
---
|
||||
# Helm values snippet (traefik/traefik-helm-chart) — illustrative, verify
|
||||
# field names against the chart version you deploy. Two things are
|
||||
# required beyond a stock install:
|
||||
#
|
||||
# 1. Load the plugin. Either the catalog form (once this plugin is listed
|
||||
# in the Traefik Plugin Catalog, see ../.traefik.yml) or localPlugins,
|
||||
# mounting this repository's source into the Traefik pod the same way
|
||||
# ../examples/docker-compose.yml does for its local demo:
|
||||
#
|
||||
# experimental:
|
||||
# plugins:
|
||||
# llmgateway:
|
||||
# moduleName: "github.com/lukaszraczylo/traefik-llmgateway"
|
||||
# version: "v0.1.0"
|
||||
#
|
||||
# 2. Provider keys as env vars, sourced from Secrets — never inlined into
|
||||
# the Middleware CR above, which is why every apiKey field there reads
|
||||
# "env:VAR_NAME" instead of a literal:
|
||||
#
|
||||
# env:
|
||||
# - name: OPENAI_API_KEY
|
||||
# valueFrom: {secretKeyRef: {name: llmgw-provider-keys, key: openai}}
|
||||
# - name: ANTHROPIC_API_KEY
|
||||
# valueFrom: {secretKeyRef: {name: llmgw-provider-keys, key: anthropic}}
|
||||
# - name: GEMINI_API_KEY
|
||||
# valueFrom: {secretKeyRef: {name: llmgw-provider-keys, key: gemini}}
|
||||
# - name: REDIS_PASSWORD
|
||||
# valueFrom: {secretKeyRef: {name: llmgw-redis, key: password}}
|
||||
#
|
||||
# additionalVolumes:
|
||||
# - name: llmgw-users
|
||||
# secret: {secretName: llmgw-users}
|
||||
# deployment:
|
||||
# additionalVolumeMounts:
|
||||
# - name: llmgw-users
|
||||
# mountPath: /secrets/llmgw-users
|
||||
# readOnly: true
|
||||
Reference in New Issue
Block a user