docs: README, kubernetes and compose examples, MIT license

- README.md: what-it-is, quickstart (localPlugins today, catalog form
  once published), full config reference table for every Config/
  ProviderConfig/GroupConfig/LimitsConfig/UsersConfig/UserConfig/
  RedisConfig/ModelPricing/TargetConfig/AgentConfig field, model routing,
  limits and accounting semantics, unified vs passthrough differences,
  MCP/A2A, security notes, known limitations (streaming buffering under
  Yaegi, quota-on-reject nuances, identity-encoding, path-traversal
  rejects), Kubernetes deployment notes, and development/catalog-
  submission status. Every claim grounded against the actual source read
  this session; two draft claims (user-limits "replace" group limits, an
  unqualified "go-redis is documented as unsuitable in resp.go/
  providers.go") were caught and corrected against the real
  buildLimitScopes semantics and an actual grep before committing.
- examples/kubernetes.yaml: Secret (users.json) + Middleware CR +
  IngressRoute + a Helm values snippet. The Middleware's plugin config
  round-trips through the real Config struct and New() (verified via a
  throwaway, since-deleted test), not just YAML-syntax-checked.
- examples/docker-compose.yml: minimal localPlugins-based demo (traefik +
  whoami + inline file-provider config via compose's inline configs.content).
  Brought up and exercised end-to-end against real OpenAI this session:
  auth 401/200, /v1/models, /v1/chat/completions (upstream 401 envelope
  with a fake key, proving passthrough+key-injection+wrapping all work),
  and the unknown-route 404 envelope. Found and fixed a real bug in the
  process: a bare "{}" empty group value fails Traefik's own dynamic-
  config decode with "expected a map, got 'string'"; fixed to an
  explicit empty list and documented the gotcha in both example files and
  the README.
- LICENSE: MIT, copyright 2026 Lukasz Raczylo.
This commit is contained in:
2026-08-20 14:12:05 +01:00
parent ae4d6bab96
commit 08dca6ad98
4 changed files with 801 additions and 0 deletions
+95
View File
@@ -0,0 +1,95 @@
# Minimal, runnable local demo of the LLM Gateway plugin: one Traefik
# instance, loading this repository as a localPlugin, in front of a single
# OpenAI-type provider and a `whoami` dummy backend (every route the plugin
# recognizes is short-circuited by the middleware itself; whoami only
# answers a request that falls through as passthroughUnknown).
#
# Run from the repository root:
#
# OPENAI_API_KEY=sk-... docker compose -f examples/docker-compose.yml up
#
# Try it once traefik1 is healthy:
#
# curl -s http://localhost:8080/v1/models \
# -H "Authorization: Bearer sk-demo-local"
#
# curl -s http://localhost:8080/v1/chat/completions \
# -H "Authorization: Bearer sk-demo-local" \
# -H "Content-Type: application/json" \
# -d '{"model":"gpt-5-mini","messages":[{"role":"user","content":"hi"}]}'
#
# Override the demo API key with LLMGW_DEMO_KEY=... if you don't want the
# default "sk-demo-local" in your shell history, and the published port
# with LLMGW_DEMO_PORT=... if 8080 is already taken on your machine. See
# ../README.md for the full configuration reference; see ../.traefik.yml
# for the catalog-facing plugin manifest.
configs:
dynamic:
content: |
http:
routers:
llmgateway-router:
rule: "PathPrefix(`/`)"
entryPoints: ["web"]
service: whoami
middlewares: ["llmgateway"]
services:
whoami:
loadBalancer:
servers:
- url: "http://whoami:80"
middlewares:
llmgateway:
plugin:
llmgateway:
providers:
openai:
type: openai
apiKey: "env:OPENAI_API_KEY"
# Explicit rather than discovery:true, so the demo
# starts cleanly with no outbound network call — flip
# this to discovery:true (and drop models) to pull the
# live catalog from api.openai.com instead.
models: ["gpt-5-mini", "gpt-5", "gpt-4o"]
groups:
default:
providers: []
users:
inline:
- name: demo
group: default
apiKey: "${LLMGW_DEMO_KEY:-sk-demo-local}"
services:
whoami:
image: traefik/whoami:v1.10
networks: ["llmgw-demo"]
traefik1:
image: traefik:v3.5
command:
- "--entrypoints.web.address=:80"
- "--experimental.localPlugins.llmgateway.moduleName=github.com/lukaszraczylo/traefik-llmgateway"
- "--providers.file.filename=/etc/traefik/dynamic.yml"
- "--log.level=INFO"
environment:
# ":?" fails `docker compose config`/`up` immediately, with a clear
# message, instead of starting a plugin whose provider has no key.
OPENAI_API_KEY: "${OPENAI_API_KEY:?set OPENAI_API_KEY before running this demo}"
ports:
- "${LLMGW_DEMO_PORT:-8080}:80"
volumes:
# The plugin's own module, mounted read-only exactly as Traefik's
# localPlugins loader expects: <plugins-local>/src/<moduleName>.
- ..:/plugins-local/src/github.com/lukaszraczylo/traefik-llmgateway:ro
configs:
- source: dynamic
target: /etc/traefik/dynamic.yml
networks: ["llmgw-demo"]
depends_on:
- whoami
networks:
llmgw-demo:
driver: bridge
+160
View File
@@ -0,0 +1,160 @@
# Example Kubernetes deployment of the LLM Gateway Traefik plugin: a
# Secret holding the file-backed users list, a Middleware custom resource
# carrying the plugin's own configuration, and a router wiring both onto a
# host. See ../README.md for the full configuration reference and
# ../.traefik.yml for the catalog-facing plugin manifest.
#
# Provider API keys are NOT put in this file. They are read by the plugin
# at startup via "env:VAR_NAME" (see the Middleware below), which means
# they must be set as environment variables on the Traefik pod itself —
# see the Helm values snippet at the bottom of this file.
#
# The empty-map gotcha: write an empty group as `providers: []`, never a
# bare `{}`. Traefik's own dynamic-config decode pipeline (which the
# Kubernetes CRD provider funnels plugin config through, same as the file
# provider) can fail to round-trip a truly empty nested object with
# "expected a map, got 'string'" — verified against this exact shape in
# examples/docker-compose.yml. An explicit empty list decodes correctly
# and means the same thing (all values allowed).
apiVersion: v1
kind: Secret
metadata:
name: llmgw-users
namespace: traefik
type: Opaque
stringData:
# Read by the plugin via users.file (mounted below), and hot-reloaded —
# no Traefik restart needed to add, remove, or rotate a user. Shape is
# fixed by users_file.go: {"users":[{"name","group","apiKey","limits"}]}.
# limits is optional per user; omit it to inherit the user's group limits.
users.json: |
{
"users": [
{"name": "alice", "group": "engineering", "apiKey": "sk-llmgw-alice-CHANGE-ME"},
{"name": "bob", "group": "general", "apiKey": "sk-llmgw-bob-CHANGE-ME"}
]
}
---
apiVersion: traefik.io/v1alpha1
kind: Middleware
metadata:
name: llmgateway
namespace: traefik
spec:
plugin:
llmgateway:
providers:
openai:
type: openai
apiKey: "env:OPENAI_API_KEY"
discovery: true
discoveryInterval: 1h
anthropic:
type: anthropic
apiKey: "env:ANTHROPIC_API_KEY"
models: ["claude-opus-4-5", "claude-sonnet-4-5", "claude-haiku-4-5"]
gemini:
type: gemini
apiKey: "env:GEMINI_API_KEY"
models: ["gemini-2.5-pro", "gemini-2.5-flash"]
groups:
engineering:
# Empty lists, not omitted fields, so this block is a real map
# entry (see the empty-map gotcha note above) — every provider
# and model stays allowed either way (empty = all).
providers: []
models: []
limits:
requestsPerMinute: 60
requestsPerDay: 5000
tokensPerDay: 1000000
tokensPerMonth: 20000000
costPerDayUSD: 25.0
costPerMonthUSD: 250.0
mcpServers: []
agents: []
general:
providers: ["openai"]
models: ["gpt-5-mini", "gpt-4o"]
limits:
requestsPerMinute: 10
requestsPerDay: 500
users:
file: /secrets/llmgw-users/users.json
redis:
address: redis.shared-resources.svc:6379
password: "env:REDIS_PASSWORD"
failOpen: true
pricing:
gpt-5-mini: {inputPerM: 0.25, outputPerM: 2.0}
mcpServers:
github:
url: http://mcp-github.tools.svc:8080
agents:
summarizer:
url: http://summarizer.agents.svc:8080
passthroughUnknown: false
---
apiVersion: traefik.io/v1alpha1
kind: IngressRoute
metadata:
name: llmgateway
namespace: traefik
spec:
entryPoints:
- websecure
routes:
- match: Host(`llm.example.com`)
kind: Rule
middlewares:
- name: llmgateway
# Traefik requires a real backing Service for a router even though
# this middleware short-circuits every path it recognizes; a small
# dummy such as traefik/whoami (see ../examples/docker-compose.yml)
# or any always-up internal service works, since it is only ever
# reached via passthroughUnknown (default false, which 404s instead).
services:
- name: llmgateway-dummy-backend
port: 80
---
# Helm values snippet (traefik/traefik-helm-chart) — illustrative, verify
# field names against the chart version you deploy. Two things are
# required beyond a stock install:
#
# 1. Load the plugin. Either the catalog form (once this plugin is listed
# in the Traefik Plugin Catalog, see ../.traefik.yml) or localPlugins,
# mounting this repository's source into the Traefik pod the same way
# ../examples/docker-compose.yml does for its local demo:
#
# experimental:
# plugins:
# llmgateway:
# moduleName: "github.com/lukaszraczylo/traefik-llmgateway"
# version: "v0.1.0"
#
# 2. Provider keys as env vars, sourced from Secrets — never inlined into
# the Middleware CR above, which is why every apiKey field there reads
# "env:VAR_NAME" instead of a literal:
#
# env:
# - name: OPENAI_API_KEY
# valueFrom: {secretKeyRef: {name: llmgw-provider-keys, key: openai}}
# - name: ANTHROPIC_API_KEY
# valueFrom: {secretKeyRef: {name: llmgw-provider-keys, key: anthropic}}
# - name: GEMINI_API_KEY
# valueFrom: {secretKeyRef: {name: llmgw-provider-keys, key: gemini}}
# - name: REDIS_PASSWORD
# valueFrom: {secretKeyRef: {name: llmgw-redis, key: password}}
#
# additionalVolumes:
# - name: llmgw-users
# secret: {secretName: llmgw-users}
# deployment:
# additionalVolumeMounts:
# - name: llmgw-users
# mountPath: /secrets/llmgw-users
# readOnly: true