From a593b6fc70fb533e7ab9912623453e2b8cc79cb4 Mon Sep 17 00:00:00 2001 From: Lukasz Raczylo Date: Wed, 23 Sep 2026 09:35:13 +0100 Subject: [PATCH] feat(cmd): add a standalone gateway binary and container image cmd/gateway runs the middleware as its own HTTP server behind Traefik. It reads a YAML config, converts it to JSON and decodes it with DisallowUnknownFields, so a misspelled key stops startup. It warns when metrics.allowedCIDRs is set, because behind a proxy the source address is the proxy's. The Makefile gains build-binary, docker-build and run-local targets. The Dockerfile builds from the committed vendor tree and stamps main.version from the VERSION build argument. go.mod lists go.yaml.in/yaml/v3 as a direct dependency. Claude-Session: https://claude.ai/code/session_01V5jYJNxpDFdJkyyExpADrv --- .dockerignore | 27 ++ .gitignore | 5 + Dockerfile | 69 +++ Makefile | 62 ++- cmd/gateway/main.go | 601 +++++++++++++++++++++++++ cmd/gateway/main_test.go | 533 ++++++++++++++++++++++ cmd/gateway/testdata/llmgw-config.yaml | 295 ++++++++++++ go.mod | 2 +- 8 files changed, 1592 insertions(+), 2 deletions(-) create mode 100644 .dockerignore create mode 100644 Dockerfile create mode 100644 cmd/gateway/main.go create mode 100644 cmd/gateway/main_test.go create mode 100644 cmd/gateway/testdata/llmgw-config.yaml diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..7f96f69 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,27 @@ +# The Docker build context is the repository root, because the binary +# build needs both go.mod and the committed vendor/ tree. Everything below +# is irrelevant to `go build ./cmd/gateway` and would otherwise be +# transferred on every build — webui/node_modules alone is a large tree of +# files the Go build never reads. +# +# vendor/ is deliberately NOT listed: the image build runs with +# -mod=vendor and cannot resolve a single import without it. +.git +.github +.superpowers +docs +integration +webui +dist +bin +examples +testdata + +# Tests and build tooling are not part of the runtime binary. tools/ has +# its own separate go modules (yaegi-check, pricing-sync) that must never +# become the gateway's dependencies. +tools +*_test.go + +Dockerfile +.dockerignore diff --git a/.gitignore b/.gitignore index 840642c..819f443 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,11 @@ tools/yaegi-check/llmgw-yaegi-check tools/pricing-sync/llmgw-pricing-sync dist/ +# `make build-binary`'s output — the standalone binary (deployment form B, +# see README's "Deployment modes"). Built locally on demand, never +# committed, exactly like the two tool binaries above. +bin/ + # webui/'s build output (dist/, matched above) and npm dependency tree — # neither is source: dist/ is regenerated by `vite build`, node_modules/ by # `npm ci`. admin_assets_gen.go (repo root, `make admin-ui`'s other output) diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..8511760 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,69 @@ +# Image for deployment form (B): the standalone compiled binary +# (cmd/gateway), which runs BEHIND Traefik rather than inside it. Form (A), +# the Yaegi-interpreted plugin, needs no image at all — Traefik loads this +# repository's source directly — so nothing here affects it. +# +# Build context is the repository root (both go.mod and vendor/ must be +# reachable); .dockerignore keeps webui/node_modules, .git and the +# integration fixtures out of the transfer. +# +# Built locally, never in CI: `make docker-build`. This repository +# deliberately does not add GitHub Actions for it, and .goreleaser.yaml +# stays at `builds: - skip: true` because its job is publishing the plugin +# SOURCE tarball for the Traefik catalog, not producing binaries. + +# --platform=$BUILDPLATFORM pins the toolchain to the BUILDER's native +# architecture and cross-compiles from there via GOOS/GOARCH below. Without +# it, a multi-arch build runs the compiler itself under QEMU emulation for +# every foreign arch, which is dramatically slower for no benefit — Go +# cross-compiles natively. +FROM --platform=$BUILDPLATFORM golang:1.25-alpine AS build + +# Supplied by BuildKit for the arch currently being produced. +ARG TARGETOS +ARG TARGETARCH + +# VERSION stamps main.version (cmd/gateway/main.go), the same var +# `make build-binary`'s own -ldflags -X main.version=$(BINARY_VERSION) +# stamps for the bin/llmgateway build (Makefile) — C3, review round 3, +# 2026-09. Left at its "dev" default when no --build-arg is passed +# (`make docker-build` passes BINARY_VERSION, below), so `llmgateway +# -version` inside the image matches whatever the caller stamped, instead +# of always printing "dev". +ARG VERSION=dev + +WORKDIR /src + +# -mod=vendor: vendor/ is committed (see .gitattributes' own comment for +# why it must stay that way), so the build never reaches the network and +# builds the exact dependency source this repository ships. +ENV CGO_ENABLED=0 GOFLAGS=-mod=vendor + +COPY . . + +# -trimpath strips local filesystem paths from the binary; -s -w drops the +# symbol and DWARF tables. CGO_ENABLED=0 above is what makes the result +# genuinely static and therefore runnable on distroless-static below. +# -X main.version=$VERSION matches build-binary's own stamping exactly +# (Makefile), so the two build paths can never disagree on what a +# "version" build-arg/var means. +RUN GOOS=$TARGETOS GOARCH=$TARGETARCH \ + go build -trimpath -ldflags "-s -w -X main.version=$VERSION" -o /out/llmgateway ./cmd/gateway + +# distroless-static carries CA certificates (needed: providers are reached +# over HTTPS) and nothing else — no shell, no package manager. :nonroot +# runs as uid 65532. +FROM gcr.io/distroless/static-debian12:nonroot + +COPY --from=build /out/llmgateway /usr/local/bin/llmgateway + +USER nonroot:nonroot +EXPOSE 8080 + +# The config path, listen address and every other setting are overridable +# by flag or by the matching LLMGW_* environment variable (see +# cmd/gateway/main.go). Mount the config at this path, and mount the same +# secret volumes the plugin form uses — a `file:/llmgw/providers/...` +# apiKey resolves identically in both forms and needs the file to exist. +ENTRYPOINT ["/usr/local/bin/llmgateway"] +CMD ["-config", "/etc/llmgateway/config.yaml"] diff --git a/Makefile b/Makefile index 616b439..156ee62 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,7 @@ # two of them concurrently would corrupt each other's containers. .NOTPARALLEL: -.PHONY: test lint yaegi-check admin-ui pricing-sync integration integration-keep integration-up integration-wait integration-down +.PHONY: test lint yaegi-check admin-ui pricing-sync integration integration-keep integration-up integration-wait integration-down build-binary docker-build run-local test: go test ./... -count=1 @@ -102,3 +102,63 @@ integration: # `make integration-down` when done. integration-keep: integration-up integration-wait go -C integration test -tags integration -count=1 ./... + +# --------------------------------------------------------------------------- +# Standalone binary (deployment form B) — see README's "Deployment modes". +# +# The plugin form (A) needs none of these: Traefik interprets this +# repository's source directly. Everything below builds the compiled +# binary that runs BEHIND Traefik, which is what makes SSE stream +# incrementally (the Yaegi ResponseWriter erases http.Flusher; a compiled +# one does not). +# +# Built locally on purpose. .goreleaser.yaml stays at `builds: - skip: +# true` because its job is publishing the plugin SOURCE tarball for the +# Traefik catalog, and turning binary builds on there would put this on +# the GitHub Actions release path rather than the operator's machine. +# --------------------------------------------------------------------------- + +# BINARY_VERSION stamps THIS binary's own -version output. It is NOT the +# plugin's pluginVersion, which is a const rewritten in source by +# workflow-prepare.sh at release time — `-ldflags -X` cannot write a const +# (verified: the build succeeds and the value never lands in the binary), +# so the two are stamped by different mechanisms and can disagree. +BINARY_VERSION ?= dev + +build-binary: + GOWORK=off CGO_ENABLED=0 go build -mod=vendor -trimpath \ + -ldflags "-s -w -X main.version=$(BINARY_VERSION)" \ + -o bin/llmgateway ./cmd/gateway + +# IMAGE/PLATFORMS are overridable the same way INTEGRATION_REAL_BASEURL is, +# so this never hardcodes one operator's registry. +IMAGE ?= llmgateway:dev +PLATFORMS ?= linux/arm64,linux/amd64 + +# docker-build produces a multi-arch image. --push is required rather than +# optional: a multi-arch result CANNOT be --load-ed into the local docker +# image store, and without either flag buildx leaves the result in the +# build cache only — it looks like it succeeded and produces nothing +# usable. For a single-arch image you can actually run locally, use: +# make docker-build PLATFORMS=linux/arm64 DOCKER_OUTPUT=--load +DOCKER_OUTPUT ?= --push + +# --build-arg VERSION=$(BINARY_VERSION) (C3, review round 3, 2026-09): +# without it the Dockerfile's own ARG VERSION=dev default applies, and +# `llmgateway -version` inside the image always prints "dev" regardless +# of what BINARY_VERSION was set to for this invocation — the same +# BINARY_VERSION build-binary above stamps into bin/llmgateway, so both +# build paths agree on one version for one invocation. +docker-build: + docker buildx build --platform $(PLATFORMS) --build-arg VERSION=$(BINARY_VERSION) -t $(IMAGE) $(DOCKER_OUTPUT) . + +# run-local runs the binary against a config on this machine — the fastest +# way to exercise form (B) without a cluster. Point LLMGW_CONFIG at any +# config file; the checked-in test fixture is a real production config and +# makes a reasonable smoke target, though its `file:` secrets and +# in-cluster hostnames will not resolve off-cluster. +LLMGW_CONFIG ?= cmd/gateway/testdata/llmgw-config.yaml +LLMGW_LISTEN ?= :8080 + +run-local: build-binary + ./bin/llmgateway -config $(LLMGW_CONFIG) -listen $(LLMGW_LISTEN) diff --git a/cmd/gateway/main.go b/cmd/gateway/main.go new file mode 100644 index 0000000..c60e1b5 --- /dev/null +++ b/cmd/gateway/main.go @@ -0,0 +1,601 @@ +// Command gateway runs the LLM gateway as a standalone compiled binary, +// sitting BEHIND Traefik rather than inside it. +// +// This is deployment form (B). Form (A) — the Yaegi-interpreted Traefik +// middleware plugin — is unchanged and remains the default; both forms are +// built from the same root package, which is why nothing in this directory +// may ever be imported by it. +// +// Why a binary exists at all: Yaegi hands the interpreted plugin a +// synthetic http.ResponseWriter that does not carry Traefik's own +// http.Flusher through the reflect boundary, so every per-chunk flush the +// plugin performs is a no-op and an SSE stream arrives in one burst once +// the handler returns (README's "Known limitations", and the upstream +// traefik/traefik#10269 it links). Compiled, the identical code's +// assertion in statusTrackingWriter.Flush (errors.go) succeeds against a +// real *http.response, so chunks reach the client as they are produced. +// +// THE yaml IMPORT BELOW MUST STAY IN THIS PACKAGE. The root package is +// interpreted by Yaegi in form (A) and by the Plugin Catalog's analyzer +// (piceus) at release time; keeping its non-stdlib surface to +// oss-telemetry alone is what makes both work. Nothing enforces this +// automatically — `make yaegi-check` resolves go.yaml.in/yaml/v3 out of +// vendor/ perfectly well and stays green even with the import at the root +// — so the rule lives here, in a comment, and in review. +package main + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "flag" + "fmt" + "net/http" + "os" + "os/signal" + "path/filepath" + "strings" + "sync/atomic" + "syscall" + "time" + + yaml "go.yaml.in/yaml/v3" + + traefikllmgateway "github.com/lukaszraczylo/traefik-llmgateway" +) + +// version is this BINARY's version, stamped at build time with +// `-ldflags "-X main.version=1.2.3"`. +// +// It is deliberately a var, not a const: the root package's own +// pluginVersion is a const (version.go), which `-X` silently refuses to +// write — verified, the build succeeds and the value never lands in the +// binary. The root package gets its version the only way a const can, by +// workflow-prepare.sh rewriting the source literal before the build, so +// the two can disagree unless the same stamping runs for both. -version +// reports this string; the admin API and telemetry report pluginVersion. +var version = "dev" + +// Default flag values. Each is overridable by the matching LLMGW_* env +// var, which in turn is overridden by an explicitly-passed flag — the +// house rule that an explicit override always wins, implemented by +// feeding the env value in as the flag's DEFAULT rather than reading it +// after parsing. +const ( + defaultConfigPath = "/etc/llmgateway/config.yaml" + defaultListenAddr = ":8080" + defaultInstanceName = "gateway" + defaultLogLevel = "info" + defaultShutdownWait = 30 * time.Second + defaultDrainDelay = 5 * time.Second + defaultHealthPath = "/healthz" + defaultReadyPath = "/readyz" + defaultReadHeaderWait = 10 * time.Second + defaultIdleTimeout = 120 * time.Second + defaultMaxHeaderBytes = 1 << 20 +) + +// telemetryOptOutVars are the environment variables oss-telemetry itself +// reads to suppress the anonymous startup ping (its telemetry.go). The +// binary never reimplements that decision — shouldSendTelemetry is +// unexported and the real gate also depends on whether the build was +// version-stamped — it only reports at startup which opt-out an operator +// actually set, so a deployment that believes it opted out can confirm it. +var telemetryOptOutVars = []string{ + "DO_NOT_TRACK", + "OSS_TELEMETRY_DISABLED", + "TRAEFIK_LLMGATEWAY_DISABLE_TELEMETRY", +} + +func main() { + if err := run(); err != nil { + fmt.Fprintf(os.Stderr, "llmgw-bin ERROR %v\n", err) + os.Exit(1) + } +} + +// options carries every resolved command-line/environment setting. +type options struct { + configPath string + listenAddr string + name string + logLevel string + healthPath string + readyPath string + shutdownWait time.Duration + drainDelay time.Duration + showVersion bool +} + +// parseOptions defines and parses the binary's flags. Every flag's +// default is the matching env var's value when set, so precedence reads +// flag > env > built-in default. +func parseOptions(args []string, stderr *os.File) (*options, error) { + fs := flag.NewFlagSet("gateway", flag.ContinueOnError) + fs.SetOutput(stderr) + o := &options{} + fs.StringVar(&o.configPath, "config", envOr("LLMGW_CONFIG", defaultConfigPath), + "path to the gateway config file (YAML, or JSON when the name ends .json) [LLMGW_CONFIG]") + fs.StringVar(&o.listenAddr, "listen", envOr("LLMGW_LISTEN", defaultListenAddr), + "address to serve HTTP on [LLMGW_LISTEN]") + fs.StringVar(&o.name, "name", envOr("LLMGW_NAME", defaultInstanceName), + "instance name, used as the llmgw[] log prefix [LLMGW_NAME]") + fs.StringVar(&o.logLevel, "log-level", envOr("LLMGW_LOG_LEVEL", defaultLogLevel), + "debug|info|warn|error. SCOPE: this filters only THIS BINARY's own lifecycle lines. "+ + "The gateway's own INFO/WARN/ERROR output is written unconditionally to stderr by the root "+ + "package (logger.go) through unexported helpers this command cannot reach, and is never filtered "+ + "[LLMGW_LOG_LEVEL]") + fs.StringVar(&o.healthPath, "health-path", envOr("LLMGW_HEALTH_PATH", defaultHealthPath), + "liveness path served by this binary, ahead of the gateway's own routes [LLMGW_HEALTH_PATH]") + fs.StringVar(&o.readyPath, "ready-path", envOr("LLMGW_READY_PATH", defaultReadyPath), + "readiness path served by this binary, ahead of the gateway's own routes [LLMGW_READY_PATH]") + fs.DurationVar(&o.shutdownWait, "shutdown-timeout", envDurationOr("LLMGW_SHUTDOWN_TIMEOUT", defaultShutdownWait), + "how long to wait for in-flight requests to finish before abandoning them [LLMGW_SHUTDOWN_TIMEOUT]") + fs.DurationVar(&o.drainDelay, "drain-delay", envDurationOr("LLMGW_DRAIN_DELAY", defaultDrainDelay), + "how long to keep serving after readiness starts failing, so the load balancer can drop this "+ + "endpoint before the listener closes [LLMGW_DRAIN_DELAY]") + fs.BoolVar(&o.showVersion, "version", false, "print this binary's version and exit") + if err := fs.Parse(args); err != nil { + return nil, err + } + if o.healthPath == o.readyPath { + return nil, fmt.Errorf("-health-path and -ready-path must differ (both are %q)", o.healthPath) + } + for _, p := range []string{o.healthPath, o.readyPath} { + if !strings.HasPrefix(p, "/") { + return nil, fmt.Errorf("health/ready path %q must start with /", p) + } + } + return o, nil +} + +// envOr returns the environment variable's value, or def when it is unset +// or empty. An empty value is treated as unset deliberately: a Kubernetes +// env entry with an empty `value:` is far more often an accident than a +// request for an empty listen address. +func envOr(key, def string) string { + if v := os.Getenv(key); v != "" { + return v + } + return def +} + +// envDurationOr is envOr for a Go duration string. An unparseable value is +// NOT silently ignored — it is reported and the default used, since +// swallowing it would leave an operator believing a timeout applies that +// never did. +func envDurationOr(key string, def time.Duration) time.Duration { + v := os.Getenv(key) + if v == "" { + return def + } + d, err := time.ParseDuration(v) + if err != nil { + fmt.Fprintf(os.Stderr, "llmgw-bin WARN %s=%q is not a duration (%v); using %s\n", key, v, err, def) + return def + } + return d +} + +// metricsCIDRWarning reports the startup warning run() logs, below, when +// cfg.Metrics.AllowedCIDRs is set (F5, review-auth.md finding 5) — split +// out as its own pure function, matching how passthroughUnknown's own +// warning stays inline in run(), so the DECISION (warn or not, and with +// what message) is unit-testable without capturing this binary's real +// os.Stderr. +// +// metrics.allowedCIDRs is an OPTIONAL bypass of the /metrics admin-key +// check for a caller whose SOURCE ADDRESS is trusted (MetricsConfig's +// own doc comment, metrics.go) — meaningful only when that source +// address is the real caller's. In THIS deployment form, it never is: +// Traefik sits in FRONT of this binary (this file's own package doc +// comment), so every request this process sees arrives with RemoteAddr +// set to Traefik's own address, not the original client's. An allowlist +// entry that matches Traefik's pod/service CIDR — an easy mistake, since +// that is exactly the CIDR an operator might expect to "trust" — +// bypasses the admin key for every client that can reach Traefik, not +// just a co-located scraper. +func metricsCIDRWarning(cfg *traefikllmgateway.Config) (msg string, warn bool) { + if cfg.Metrics == nil || len(cfg.Metrics.AllowedCIDRs) == 0 { + return "", false + } + return "metrics.allowedCIDRs is set, but this binary runs BEHIND Traefik: the source address it " + + "sees on every request is Traefik's own address, never the original caller's. An allowlist entry " + + "that matches Traefik's own pod/service CIDR bypasses the admin key for every client that reaches " + + "Traefik. Confirm the allowlist is scoped to this binary's actual peer address before relying on it.", true +} + +// run is main's body, split out so every exit path returns an error +// instead of calling os.Exit from inside nested logic. +func run() error { + opts, err := parseOptions(os.Args[1:], os.Stderr) + if err != nil { + return err + } + if opts.showVersion { + fmt.Println(version) + return nil + } + log := newLogger(opts.logLevel) + + cfg, err := loadConfig(opts.configPath) + if err != nil { + return err + } + log.info("starting version=%s name=%s config=%s listen=%s", version, opts.name, opts.configPath, opts.listenAddr) + logTelemetryOptOut(log) + + // passthroughUnknown routes an unrecognized path to the `next` + // handler. In form (A) that is the Traefik router's own backing + // service; in form (B) Traefik is IN FRONT of this process and there + // is no downstream at all, so the flag cannot mean what it means in + // the plugin. This is a warning rather than a hard error because the + // terminal handler below keeps the behaviour well-defined (a clean + // 404) — an operator carrying a working plugin config across should + // not be blocked, only told. + if cfg.PassthroughUnknown { + log.warn("passthroughUnknown is set, but a standalone binary has no downstream to pass through to: " + + "unknown routes return 404. Route non-gateway paths in Traefik instead.") + } + + if msg, warn := metricsCIDRWarning(cfg); warn { + log.warn(msg) + } + + // signal.NotifyContext, not context.Background(): New blocks on the + // registry's synchronous warm-fill, which with a large provider set + // can run for minutes before the listener ever opens. A SIGTERM + // arriving inside that window has to abort startup rather than be + // ignored until it finishes. + ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) + defer stop() + + handler, err := traefikllmgateway.New(ctx, terminalHandler(), cfg, opts.name) + if err != nil { + return fmt.Errorf("construct gateway: %w", err) + } + if ctx.Err() != nil { + log.info("signal received during startup; exiting before serving") + closeGateway(handler, log) + return nil + } + + var ready atomic.Bool + srv := &http.Server{ + Addr: opts.listenAddr, + Handler: withHealth(handler, &ready, opts.healthPath, opts.readyPath), + // Bounds a Slowloris header trickle (and satisfies gosec G112). + ReadHeaderTimeout: defaultReadHeaderWait, + // ReadTimeout and WriteTimeout are BOTH deliberately zero. + // + // net/http sets each as an ABSOLUTE deadline on the connection + // before the handler runs — neither is an idle timeout. A + // non-zero WriteTimeout therefore expires mid-generation on any + // stream that outlives it: writes start failing with + // os.ErrDeadlineExceeded and the client gets a truncated SSE + // stream with no terminal `data: [DONE]`, which is a worse + // defect than the burst-delivery bug this binary exists to fix. + // ReadTimeout has the same shape on the way in and would + // truncate a large multipart audio upload over a slow link. + // + // For the OUTBOUND leg this is covered one layer in: + // Config.RequestTimeout (timeout.go) is a PROGRESS-based + // watchdog, five minutes by default, with a per-provider + // override. Body size is capped inside the root package + // independently. + // + // CORRECTION (security audit run-1): that delegation does NOT + // cover the INBOUND leg, and this comment previously implied it + // did. newWatchdogBody wraps only an UPSTREAM RESPONSE body — its + // two production call sites both pass resp.Body (providers.go, + // routes_passthrough.go) — and is never applied to r.Body. So + // between a complete header block and the handler returning, + // nothing here bounds a client that trickles its request body, + // while that read holds one of the process-wide body-admission + // slots (acquireBodyAdmission, routes_unified.go). A deployment + // that fronts this binary with a proxy inherits that proxy's read + // timeout; one that exposes it directly has none. Fixing it + // properly needs a progress-based bound on r.Body for the + // duration of the admission-held read, in the root package — + // a finite ReadTimeout here is the wrong shape, for exactly the + // truncation reason stated above. + ReadTimeout: 0, + WriteTimeout: 0, + IdleTimeout: defaultIdleTimeout, + MaxHeaderBytes: defaultMaxHeaderBytes, + } + + ready.Store(true) + errCh := make(chan error, 1) + go func() { errCh <- srv.ListenAndServe() }() + log.info("listening on %s (health %s, ready %s)", opts.listenAddr, opts.healthPath, opts.readyPath) + + select { + case err := <-errCh: + // ErrServerClosed only ever arrives after a Shutdown this + // function did not start, so reaching it here is a real failure. + if !errors.Is(err, http.ErrServerClosed) { + closeGateway(handler, log) + return fmt.Errorf("listen on %s: %w", opts.listenAddr, err) + } + return nil + case <-ctx.Done(): + } + + // Ordering below is load-bearing and must not be reordered. + // + // stop() first restores the default signal disposition, so an + // impatient operator's SECOND SIGTERM kills the process immediately + // instead of being swallowed by the handler that is already draining. + stop() + log.info("signal received; draining") + + // Fail readiness BEFORE the listener stops accepting. Without the + // delay in between, the listener closes while the load balancer still + // routes to this pod, which turns every rollout into a burst of 503s. + ready.Store(false) + time.Sleep(opts.drainDelay) + + shutdownCtx, cancel := context.WithTimeout(context.Background(), opts.shutdownWait) + defer cancel() + // Shutdown WAITS for in-flight handlers rather than interrupting + // them, so shutdownWait is really "how long before we abandon a + // long generation". Raise it together with the pod's + // terminationGracePeriodSeconds where long streams matter. + if err := srv.Shutdown(shutdownCtx); err != nil { + log.error("shutdown: %v", err) + } + + // STRICTLY after Shutdown returns. Close releases the pooled Redis + // connections shared by the limiter's store and the response cache; + // closing them while requests are still draining would pull Redis out + // from under in-flight handlers, and under the common redis.failOpen + // default those requests would silently fall back to in-process + // counters rather than fail — a correctness regression invisible in + // the logs. + closeGateway(handler, log) + log.info("stopped") + return nil +} + +// closeGateway releases the Gateway's pooled resources when the handler +// New returned exposes Close. +// +// The assertion is comma-ok against a locally declared interface rather +// than a bare *traefikllmgateway.Gateway assertion: New's declared return +// type is http.Handler, so a bare assertion would panic the moment that +// return is ever wrapped, and matching on the method actually needed +// keeps this command uncoupled from the concrete type. +func closeGateway(h http.Handler, log *logger) { + c, ok := h.(interface{ Close() error }) + if !ok { + return + } + if err := c.Close(); err != nil { + log.error("gateway close: %v", err) + } +} + +// terminalHandler is the `next` handler passed to New. +// +// It is never nil: g.next is dereferenced unconditionally once +// passthroughUnknown is true (llmgateway.go), and a nil next there would +// be recovered by recoverPanic into a permanent, confusing 500 on every +// unknown path rather than an outright crash. It is never a reverse proxy +// either — in form (B) Traefik sits in front of this process, so there is +// no downstream by construction. +// +// The envelope is byte-shaped to match the root package's own unknown-route +// reply (writeOAIError, errors.go), which is unexported and so cannot be +// called from here: OpenAI-compatible SDKs parse this JSON shape, and +// http.NotFoundHandler's text/plain "404 page not found" breaks them. +// main_test.go pins the shape so the hand-copy cannot drift silently. +func terminalHandler() http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusNotFound) + _ = json.NewEncoder(w).Encode(map[string]any{ + "error": map[string]any{ + "message": "unknown route", + "type": "invalid_request_error", + "code": "404", + }, + }) + }) +} + +// withHealth serves the liveness and readiness paths ahead of the +// gateway, and delegates everything else VERBATIM. +// +// Deliberately hand-written rather than an http.ServeMux: ServeMux +// path-cleans and 301-redirects (collapsing "//" and resolving "..") +// before it dispatches, while the gateway routes on r.URL.EscapedPath() +// and routes_passthrough.go's traversal handling is built on that raw +// escaped form. Interposing a ServeMux would normalize paths before the +// gateway ever saw them, changing passthrough semantics and defeating +// those checks. +// +// Liveness is unconditional: making it depend on a provider or on Redis +// would turn any upstream outage into a restart loop. Readiness tracks +// only "constructed, and not yet shutting down" — warm-fill failures are +// non-fatal by design and Redis loss degrades to in-process counters +// under failOpen, so gating readiness on either would evict the pod for a +// condition the gateway is built to survive. +func withHealth(next http.Handler, ready *atomic.Bool, healthPath, readyPath string) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodGet || r.Method == http.MethodHead { + switch r.URL.Path { + case healthPath: + writePlain(w, http.StatusOK, "ok") + return + case readyPath: + if ready.Load() { + writePlain(w, http.StatusOK, "ready") + return + } + writePlain(w, http.StatusServiceUnavailable, "draining") + return + } + } + next.ServeHTTP(w, r) + }) +} + +// writePlain writes a health/readiness body. These endpoints are +// unauthenticated, so they carry a literal word and nothing else — no +// version, no provider list, no configuration. +func writePlain(w http.ResponseWriter, status int, body string) { + w.Header().Set("Content-Type", "text/plain; charset=utf-8") + w.Header().Set("Cache-Control", "no-store") + w.WriteHeader(status) + _, _ = w.Write([]byte(body + "\n")) +} + +// loadConfig reads path and decodes it into a *Config. +// +// THE ROUND-TRIP IS MANDATORY — see yamlToJSON. A .json file skips the +// YAML stage, which is safe because the JSON form is already the target +// shape (it is what `kubectl get middleware llmgateway -o json` emits) +// and decodes to an identical Config. +// +// Decoding starts from CreateConfig() for form parity with what Traefik +// itself does, so the two deployment forms cannot drift. It does NOT +// depend on CreateConfig pre-seeding anything: it returns a bare +// &Config{}, and every default in this codebase is applied at +// construction from a zero value rather than by pre-population. +// +// The decoder runs with DisallowUnknownFields (F12, review-auth.md +// finding 12): a plain json.Unmarshal silently drops a key that matches +// no Config field at any level, which is exactly the failure class this +// file already documents one incident of below (yamlToJSON's own doc +// comment) and cfg.PassthroughUnknown's own history repeats — a typo +// like "allowedCidrs" for "allowedCIDRs" would otherwise boot with that +// setting silently disabled rather than refusing to start. This is the +// STANDALONE BINARY's own decode only; the Traefik-plugin form (form A) +// stays on Traefik's own lenient decoder, which this package cannot +// change and does not attempt to here. +func loadConfig(path string) (*traefikllmgateway.Config, error) { + raw, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("read config: %w", err) + } + doc := raw + if !strings.EqualFold(filepath.Ext(path), ".json") { + if doc, err = yamlToJSON(raw); err != nil { + return nil, fmt.Errorf("config %s: %w", path, err) + } + } + cfg := traefikllmgateway.CreateConfig() + dec := json.NewDecoder(bytes.NewReader(doc)) + dec.DisallowUnknownFields() + if err := dec.Decode(cfg); err != nil { + return nil, fmt.Errorf("config %s: %w", path, err) + } + return cfg, nil +} + +// yamlToJSON converts a YAML document to JSON by way of a generic map. +// +// THIS INDIRECTION IS THE WHOLE POINT, AND IS NOT A STYLE CHOICE. The +// Config structs carry json tags only — there is not one yaml tag in the +// repository — so yaml.v3 falls back to lowercasing the entire Go field +// name when it decodes straight into a *Config: APIKey becomes "apikey", +// BaseURL "baseurl", MCPServers "mcpservers". Every all-lowercase key in a +// production config then matches by accident and every camelCase key +// misses, WITH A NIL ERROR. +// +// Measured against the live cluster's own config: the direct path returns +// no error and still produces a Config that boots, with all 18 providers +// present but every apiKey and baseUrl empty, mcpServers/modelAliases/ +// modelMeta dropped wholesale, and metrics.allowedCIDRs emptied. Empty +// BaseURLs then fall back to defaultBaseURLByType, so every openai-type +// provider silently retargets api.openai.com and the mounted file: +// secrets are never read. 84 leaves differ from the correct decode. +// main_test.go asserts both halves of that so no future "simplification" +// can reintroduce it quietly. +func yamlToJSON(raw []byte) ([]byte, error) { + var doc map[string]any + if err := yaml.Unmarshal(raw, &doc); err != nil { + return nil, fmt.Errorf("parse yaml: %w", err) + } + j, err := json.Marshal(doc) + if err != nil { + // The realistic trigger is a nested mapping key that YAML parses + // as a non-string — a model id like `4.1:` or `3.5:` written + // unquoted under modelMeta or pricing. The underlying error names + // an offset or an unsupported map type rather than the block, so + // it is wrapped with the fix rather than passed through bare. + return nil, fmt.Errorf("convert yaml to json: %w "+ + "(a mapping key that YAML reads as a number or boolean cannot become a JSON object key — "+ + "quote it, e.g. \"4.1\": under modelMeta or pricing)", err) + } + return j, nil +} + +// logTelemetryOptOut reports which anonymous-telemetry opt-out variable is +// set, if any, so a deployment that believes it opted out can confirm it +// from the startup log. It deliberately does not claim whether a ping +// will actually be sent: that also depends on the build being +// version-stamped, and the root package's decision function is +// unexported. +func logTelemetryOptOut(log *logger) { + for _, k := range telemetryOptOutVars { + if v := os.Getenv(k); v != "" { + log.info("anonymous usage reporting opted out via %s=%s", k, v) + return + } + } + log.info("anonymous usage reporting not opted out (set DO_NOT_TRACK=1 to disable); " + + "only version-stamped release builds ever send a ping") +} + +// logger is this binary's own leveled logger. It governs ONLY the +// lifecycle lines written here — the gateway's own output goes straight +// to stderr from the root package's unexported helpers and cannot be +// filtered from outside it without reassigning os.Stderr, which would +// reorder interleaved writes and is not worth a goroutine and a pipe. +// The prefix is llmgw-bin, distinct from the gateway's own llmgw[name], +// so the two are never confused in one stream. +type logger struct { + level int +} + +// Log levels, ordered so a message is written when its own level is at or +// above the configured threshold. +const ( + levelDebug = iota + levelInfo + levelWarn + levelError +) + +// newLogger resolves a level name. An unrecognized name falls back to +// info and says so, rather than silently picking a level the operator did +// not ask for. +func newLogger(name string) *logger { + switch strings.ToLower(strings.TrimSpace(name)) { + case "debug": + return &logger{level: levelDebug} + case "info": + return &logger{level: levelInfo} + case "warn", "warning": + return &logger{level: levelWarn} + case "error": + return &logger{level: levelError} + default: + fmt.Fprintf(os.Stderr, "llmgw-bin WARN unknown -log-level %q; using info\n", name) + return &logger{level: levelInfo} + } +} + +func (l *logger) write(at int, label, format string, args ...any) { + if at < l.level { + return + } + fmt.Fprintf(os.Stderr, "llmgw-bin %s %s\n", label, fmt.Sprintf(format, args...)) +} + +func (l *logger) info(format string, args ...any) { l.write(levelInfo, "INFO", format, args...) } +func (l *logger) warn(format string, args ...any) { l.write(levelWarn, "WARN", format, args...) } +func (l *logger) error(format string, args ...any) { l.write(levelError, "ERROR", format, args...) } diff --git a/cmd/gateway/main_test.go b/cmd/gateway/main_test.go new file mode 100644 index 0000000..68f247a --- /dev/null +++ b/cmd/gateway/main_test.go @@ -0,0 +1,533 @@ +package main + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "sync/atomic" + "testing" + + yaml "go.yaml.in/yaml/v3" + + traefikllmgateway "github.com/lukaszraczylo/traefik-llmgateway" +) + +// fixturePath is a real production config captured from the live cluster +// (`kubectl -n traefik get middleware llmgateway -o json`, converted to +// the ConfigMap-shaped YAML a standalone binary consumes). It carries no +// key material: every real provider key in it is a `file:` reference +// resolved at construction from a mounted volume, never a literal. +// +// It is checked in rather than read from a scratch directory so `make +// test` works from a clean clone. testdata/ is export-ignored +// (.gitattributes), so it never reaches the plugin's release tarball. +const fixturePath = "testdata/llmgw-config.yaml" + +// Counts and values below are taken from the fixture itself, not from the +// struct definitions, and are what the direct-yaml path destroys. +const ( + fixtureProviders = 18 + fixtureMCPServers = 11 + fixtureModelAliases = 3 + fixtureModelMeta = 21 + fixtureAgents = 34 + fixtureGroups = 3 + fixtureUsersFile = "/llmgw/users/users.json" + fixtureMetricsCIDR = "192.0.2.0/24" +) + +// bothFieldProviders are the providers in the fixture that genuinely +// carry BOTH apiKey and baseUrl, so both can be asserted non-empty. +// +// The set is deliberately NOT "every provider": anthropic has an apiKey +// but no baseUrl by design (it defaults upstream via +// defaultBaseURLByType), so a blanket assertion across all 18 would fail +// for a reason that has nothing to do with the decode being correct. +var bothFieldProviders = map[string]struct{ apiKey, baseURL string }{ + // G101 false positive: the "file:" prefix is resolveSecret's + // file-reference form (secrets.go) — a PATH the gateway reads the key + // out of at construction, not a key. This fixture carries no real + // credential. + //nolint:gosec + "alibaba": { + apiKey: "file:/llmgw/providers/alibaba", + baseURL: "https://ws-example.eu-central-1.maas.aliyuncs.com/compatible-mode", + }, + // G101 false positive, same as "alibaba" above: a resolveSecret file + // reference, not a credential. + //nolint:gosec + "cerebras": { + apiKey: "file:/llmgw/providers/cerebras", + baseURL: "https://api.cerebras.ai", + }, + "codex": { + apiKey: "unused", + baseURL: "http://codex-shim.mcp-servers.svc.example.invalid:8080", + }, +} + +// TestLoadConfigRoundTripPopulatesCamelCaseFields is the primary +// regression test for the loader's reason to exist. It asserts the real +// VALUES the round-trip recovers, not merely that loading returned no +// error — a nil error is exactly what the broken path also returns. +func TestLoadConfigRoundTripPopulatesCamelCaseFields(t *testing.T) { + cfg, err := loadConfig(fixturePath) + if err != nil { + t.Fatalf("loadConfig: %v", err) + } + + if got := len(cfg.Providers); got != fixtureProviders { + t.Fatalf("providers = %d, want %d", got, fixtureProviders) + } + for name, want := range bothFieldProviders { + pc, ok := cfg.Providers[name] + if !ok { + t.Fatalf("provider %q missing from decoded config", name) + } + if pc.APIKey != want.apiKey { + t.Errorf("provider %q apiKey = %q, want %q", name, pc.APIKey, want.apiKey) + } + if pc.BaseURL != want.baseURL { + t.Errorf("provider %q baseUrl = %q, want %q", name, pc.BaseURL, want.baseURL) + } + } + + // anthropic pins the other half of the contract: apiKey present, + // baseUrl legitimately absent. If a future change "fixes" this by + // populating a baseUrl, the defaulting behaviour changed. + anthropic, ok := cfg.Providers["anthropic"] + if !ok { + t.Fatal("provider \"anthropic\" missing from decoded config") + } + if anthropic.APIKey != "file:/llmgw/providers/anthropic" { + t.Errorf("anthropic apiKey = %q, want file:/llmgw/providers/anthropic", anthropic.APIKey) + } + if anthropic.BaseURL != "" { + t.Errorf("anthropic baseUrl = %q, want empty (it defaults upstream)", anthropic.BaseURL) + } + + // Per-provider camelCase keys beyond apiKey/baseUrl. + if got := cfg.Providers["gx10"].DiscoveryInterval; got != "5m" { + t.Errorf("gx10 discoveryInterval = %q, want 5m", got) + } + if got := cfg.Providers["gx10"].RequestTimeout; got != "15m" { + t.Errorf("gx10 requestTimeout = %q, want 15m", got) + } + if got := cfg.Providers["macstudio"].MetadataPath; got != "/api/v0/models" { + t.Errorf("macstudio metadataPath = %q, want /api/v0/models", got) + } + + // Blocks the direct-yaml path drops wholesale, because their own + // top-level keys are camelCase. + if got := len(cfg.MCPServers); got != fixtureMCPServers { + t.Errorf("mcpServers = %d, want %d", got, fixtureMCPServers) + } + if got := len(cfg.ModelAliases); got != fixtureModelAliases { + t.Errorf("modelAliases = %d, want %d", got, fixtureModelAliases) + } + if got := cfg.ModelAliases["deepseek"]; got != "uni/deepseek-v4-flash-0731" { + t.Errorf("modelAliases[deepseek] = %q, want uni/deepseek-v4-flash-0731", got) + } + if got := len(cfg.ModelMeta); got != fixtureModelMeta { + t.Errorf("modelMeta = %d, want %d", got, fixtureModelMeta) + } + + // metrics.allowedCIDRs decides whether an unauthenticated scrape is + // allowed at all, so silently emptying it is a security-relevant + // decode failure, not a cosmetic one. + if cfg.Metrics == nil { + t.Fatal("metrics block missing from decoded config") + } + if got := cfg.Metrics.AllowedCIDRs; len(got) != 1 || got[0] != fixtureMetricsCIDR { + t.Errorf("metrics.allowedCIDRs = %v, want [%s]", got, fixtureMetricsCIDR) + } + if cfg.Redis == nil || cfg.Redis.FailOpen == nil || !*cfg.Redis.FailOpen { + t.Errorf("redis.failOpen did not decode to *true (got %+v)", cfg.Redis) + } + + // Keys that are already all-lowercase survive BOTH paths. Asserting + // them here is what makes the direct-path test below meaningful: + // these are why the broken decode still boots and looks plausible. + if got := len(cfg.Agents); got != fixtureAgents { + t.Errorf("agents = %d, want %d", got, fixtureAgents) + } + if got := len(cfg.Groups); got != fixtureGroups { + t.Errorf("groups = %d, want %d", got, fixtureGroups) + } + if cfg.Users == nil || cfg.Users.File != fixtureUsersFile { + t.Errorf("users.file did not decode to %s (got %+v)", fixtureUsersFile, cfg.Users) + } +} + +// TestDirectYAMLUnmarshalSilentlyLosesCamelCaseFields documents WHY +// loadConfig round-trips through JSON, by proving the obvious-looking +// alternative fails the exact assertions above while returning a nil +// error. If yaml tags are ever added to the root package's Config, this +// test fails and the round-trip can be reconsidered deliberately — which +// is the point. +func TestDirectYAMLUnmarshalSilentlyLosesCamelCaseFields(t *testing.T) { + raw, err := os.ReadFile(fixturePath) + if err != nil { + t.Fatalf("ReadFile: %v", err) + } + + direct := traefikllmgateway.CreateConfig() + if err := yaml.Unmarshal(raw, direct); err != nil { + t.Fatalf("direct yaml.Unmarshal returned an error (%v); this test assumes it fails SILENTLY", err) + } + + // The failure is silent, not loud: the structure that makes the + // gateway boot is all still there. + if got := len(direct.Providers); got != fixtureProviders { + t.Fatalf("direct path providers = %d, want %d — the trap is that providers DO survive", got, fixtureProviders) + } + + // ...but every camelCase leaf is gone. + for name := range bothFieldProviders { + pc, ok := direct.Providers[name] + if !ok { + t.Fatalf("direct path: provider %q missing entirely", name) + } + if pc.APIKey != "" { + t.Errorf("direct path: provider %q apiKey = %q, want \"\" — expected the direct decode to lose it", name, pc.APIKey) + } + if pc.BaseURL != "" { + t.Errorf("direct path: provider %q baseUrl = %q, want \"\" — expected the direct decode to lose it", name, pc.BaseURL) + } + } + if len(direct.MCPServers) != 0 { + t.Errorf("direct path: mcpServers = %d, want 0 (key is camelCase)", len(direct.MCPServers)) + } + if len(direct.ModelAliases) != 0 { + t.Errorf("direct path: modelAliases = %d, want 0 (key is camelCase)", len(direct.ModelAliases)) + } + if len(direct.ModelMeta) != 0 { + t.Errorf("direct path: modelMeta = %d, want 0 (key is camelCase)", len(direct.ModelMeta)) + } + if direct.Metrics != nil && len(direct.Metrics.AllowedCIDRs) != 0 { + t.Errorf("direct path: metrics.allowedCIDRs = %v, want empty (key is camelCase)", + direct.Metrics.AllowedCIDRs) + } + + // The all-lowercase keys survive, which is precisely why the broken + // config boots rather than failing fast. + if direct.Users == nil || direct.Users.File != fixtureUsersFile { + t.Errorf("direct path: users.file = %+v, want %s to survive (key is all-lowercase)", direct.Users, fixtureUsersFile) + } + if len(direct.Agents) != fixtureAgents { + t.Errorf("direct path: agents = %d, want %d to survive (key is all-lowercase)", len(direct.Agents), fixtureAgents) + } +} + +// TestLoadConfigAcceptsJSON checks the .json shortcut decodes to the same +// Config as the YAML form, since the JSON shape is what kubectl emits. +func TestLoadConfigAcceptsJSON(t *testing.T) { + fromYAML, err := loadConfig(fixturePath) + if err != nil { + t.Fatalf("loadConfig(yaml): %v", err) + } + j, err := json.Marshal(fromYAML) + if err != nil { + t.Fatalf("Marshal: %v", err) + } + jsonPath := filepath.Join(t.TempDir(), "config.json") + if writeErr := os.WriteFile(jsonPath, j, 0o600); writeErr != nil { + t.Fatalf("WriteFile: %v", writeErr) + } + + fromJSON, err := loadConfig(jsonPath) + if err != nil { + t.Fatalf("loadConfig(json): %v", err) + } + again, err := json.Marshal(fromJSON) + if err != nil { + t.Fatalf("Marshal: %v", err) + } + if string(again) != string(j) { + t.Error("json config path decoded to a different Config than the yaml path") + } +} + +// TestLoadConfigRejectsNonStringMappingKey pins the one round-trip edge +// case that fails loudly: a mapping key YAML reads as a number cannot +// become a JSON object key. The wrapped message must name the fix, +// because the underlying error names only an offset or a map type. +func TestLoadConfigRejectsNonStringMappingKey(t *testing.T) { + const doc = ` +providers: + openai: + type: openai + apiKey: k +modelMeta: + 3.5: + free: true +` + p := filepath.Join(t.TempDir(), "config.yaml") + if err := os.WriteFile(p, []byte(doc), 0o600); err != nil { + t.Fatalf("WriteFile: %v", err) + } + _, err := loadConfig(p) + if err == nil { + t.Fatal("loadConfig accepted a numeric mapping key; want an error") + } + if !strings.Contains(err.Error(), "quote it") { + t.Errorf("error does not explain the fix: %v", err) + } +} + +// TestLoadConfigMissingFile checks a bad -config path fails loudly rather +// than yielding an empty Config that would fail much later with a +// confusing "at least one provider must be configured". +func TestLoadConfigMissingFile(t *testing.T) { + _, err := loadConfig(filepath.Join(t.TempDir(), "nope.yaml")) + if err == nil { + t.Fatal("loadConfig accepted a missing file; want an error") + } + if !strings.Contains(err.Error(), "read config") { + t.Errorf("error = %v, want it to name the read failure", err) + } +} + +// TestLoadConfigRejectsUnknownField is the F12 regression test +// (review-auth.md finding 12): loadConfig's decoder must run with +// DisallowUnknownFields so a misspelled or nonexistent key fails loudly +// at startup instead of silently doing nothing — the exact failure class +// yamlToJSON's own doc comment already documents one real incident of +// for the direct-yaml path (a whole different bug, same "wrong key, +// nil error" shape). This fails before the fix (plain json.Unmarshal +// drops the key silently, err == nil) and passes after. +func TestLoadConfigRejectsUnknownField(t *testing.T) { + const doc = ` +providers: + openai: + type: openai + apiKey: k +notARealConfigKey: true +` + p := filepath.Join(t.TempDir(), "config.yaml") + if err := os.WriteFile(p, []byte(doc), 0o600); err != nil { + t.Fatalf("WriteFile: %v", err) + } + _, err := loadConfig(p) + if err == nil { + t.Fatal("loadConfig accepted an unknown top-level key; want an error") + } + if !strings.Contains(err.Error(), "notARealConfigKey") { + t.Errorf("error = %v, want it to name the unknown field", err) + } +} + +// TestLoadConfigStillAcceptsRealFixture is the other half of F12's own +// regression coverage: the strict decoder must not reject a real, +// legitimate production config — the exact fixture +// TestLoadConfigRoundTripPopulatesCamelCaseFields already exercises, +// re-asserted here as its own named test so a future reviewer sees the +// negative (rejects unknown) and positive (accepts known) cases side by +// side. +func TestLoadConfigStillAcceptsRealFixture(t *testing.T) { + if _, err := loadConfig(fixturePath); err != nil { + t.Fatalf("loadConfig(fixturePath) = %v, want the real production fixture to still load under strict decoding", err) + } +} + +// TestMetricsCIDRWarning is the F5 regression test (review-auth.md +// finding 5): metricsCIDRWarning must warn exactly when +// Metrics.AllowedCIDRs is non-empty, and stay silent for every other +// shape (nil Metrics, Metrics with no CIDRs at all) — good/bad/edge, +// table-driven. +func TestMetricsCIDRWarning(t *testing.T) { + cases := []struct { + cfg *traefikllmgateway.Config + name string + want bool + }{ + {name: "nil metrics block", cfg: &traefikllmgateway.Config{}, want: false}, + { + name: "metrics enabled, no allowedCIDRs", + cfg: &traefikllmgateway.Config{Metrics: &traefikllmgateway.MetricsConfig{Enabled: true}}, + want: false, + }, + { + name: "allowedCIDRs set", + cfg: &traefikllmgateway.Config{Metrics: &traefikllmgateway.MetricsConfig{AllowedCIDRs: []string{"10.42.0.0/16"}}}, + want: true, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + msg, warn := metricsCIDRWarning(tc.cfg) + if warn != tc.want { + t.Fatalf("warn = %v, want %v", warn, tc.want) + } + if warn && !strings.Contains(msg, "BEHIND Traefik") { + t.Errorf("message = %q, want it to explain the source address is Traefik's own", msg) + } + if !warn && msg != "" { + t.Errorf("message = %q, want empty when warn is false", msg) + } + }) + } + + t.Run("real fixture triggers the warning", func(t *testing.T) { + cfg, err := loadConfig(fixturePath) + if err != nil { + t.Fatalf("loadConfig: %v", err) + } + if _, warn := metricsCIDRWarning(cfg); !warn { + t.Error("want the fixture's own metrics.allowedCIDRs to trigger the warning") + } + }) +} + +// TestTerminalHandlerEnvelope pins the hand-copied 404 envelope against +// the shape the root package's writeOAIError produces. writeOAIError is +// unexported, so this asserts the contract literally: if the root +// package's envelope ever changes, this is what catches the drift. +func TestTerminalHandlerEnvelope(t *testing.T) { + rec := httptest.NewRecorder() + terminalHandler().ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/nope", nil)) + + if rec.Code != http.StatusNotFound { + t.Errorf("status = %d, want 404", rec.Code) + } + if ct := rec.Header().Get("Content-Type"); ct != "application/json" { + t.Errorf("Content-Type = %q, want application/json", ct) + } + var got struct { + Error struct { + Message string `json:"message"` + Type string `json:"type"` + Code string `json:"code"` + } `json:"error"` + } + if err := json.Unmarshal(rec.Body.Bytes(), &got); err != nil { + t.Fatalf("body is not the OpenAI error envelope: %v (%s)", err, rec.Body.String()) + } + if got.Error.Message != "unknown route" || got.Error.Type != "invalid_request_error" || got.Error.Code != "404" { + t.Errorf("envelope = %+v, want {unknown route invalid_request_error 404}", got.Error) + } +} + +// TestWithHealth covers both probe paths and, more importantly, that +// everything else is delegated with the URL untouched. +func TestWithHealth(t *testing.T) { + var ready atomic.Bool + var seenPath, seenEscaped string + next := http.HandlerFunc(func(_ http.ResponseWriter, r *http.Request) { + seenPath = r.URL.Path + seenEscaped = r.URL.EscapedPath() + }) + h := withHealth(next, &ready, "/healthz", "/readyz") + + t.Run("liveness is unconditional", func(t *testing.T) { + rec := httptest.NewRecorder() + h.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/healthz", nil)) + if rec.Code != http.StatusOK { + t.Errorf("status = %d, want 200 even before ready", rec.Code) + } + }) + + t.Run("readiness reflects the flag", func(t *testing.T) { + rec := httptest.NewRecorder() + h.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/readyz", nil)) + if rec.Code != http.StatusServiceUnavailable { + t.Errorf("status = %d, want 503 while not ready", rec.Code) + } + ready.Store(true) + rec = httptest.NewRecorder() + h.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/readyz", nil)) + if rec.Code != http.StatusOK { + t.Errorf("status = %d, want 200 once ready", rec.Code) + } + }) + + t.Run("a POST to the probe path is delegated", func(t *testing.T) { + seenPath = "" + rec := httptest.NewRecorder() + h.ServeHTTP(rec, httptest.NewRequest(http.MethodPost, "/healthz", nil)) + if seenPath != "/healthz" { + t.Errorf("POST /healthz was answered by the probe; want it delegated (saw %q)", seenPath) + } + }) + + // The reason this wrapper is hand-written instead of an + // http.ServeMux: a mux would path-clean and 301-redirect these before + // dispatch, while the gateway routes on the raw EscapedPath and + // passthrough's traversal checks depend on it arriving unmodified. + t.Run("paths reach the gateway unnormalized", func(t *testing.T) { + for _, raw := range []string{"/v1//models", "/openai/a/../b", "/openai/a%2Fb"} { + seenPath, seenEscaped = "", "" + rec := httptest.NewRecorder() + h.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, raw, nil)) + if rec.Code == http.StatusMovedPermanently { + t.Fatalf("%s was redirected; the wrapper must never path-clean", raw) + } + if seenEscaped != raw { + t.Errorf("EscapedPath for %s = %q, want it delegated verbatim (Path was %q)", raw, seenEscaped, seenPath) + } + } + }) +} + +// TestParseOptionsPrecedence pins flag > env > default, the house rule +// that an explicit override always wins. +func TestParseOptionsPrecedence(t *testing.T) { + t.Run("default when neither is set", func(t *testing.T) { + o, err := parseOptions(nil, os.Stderr) + if err != nil { + t.Fatalf("parseOptions: %v", err) + } + if o.listenAddr != defaultListenAddr { + t.Errorf("listen = %q, want %q", o.listenAddr, defaultListenAddr) + } + if o.configPath != defaultConfigPath { + t.Errorf("config = %q, want %q", o.configPath, defaultConfigPath) + } + }) + + t.Run("env supplies the default", func(t *testing.T) { + t.Setenv("LLMGW_LISTEN", ":9999") + t.Setenv("LLMGW_SHUTDOWN_TIMEOUT", "45s") + o, err := parseOptions(nil, os.Stderr) + if err != nil { + t.Fatalf("parseOptions: %v", err) + } + if o.listenAddr != ":9999" { + t.Errorf("listen = %q, want :9999 from env", o.listenAddr) + } + if o.shutdownWait.String() != "45s" { + t.Errorf("shutdown-timeout = %s, want 45s from env", o.shutdownWait) + } + }) + + t.Run("an explicit flag beats env", func(t *testing.T) { + t.Setenv("LLMGW_LISTEN", ":9999") + o, err := parseOptions([]string{"-listen", ":7777"}, os.Stderr) + if err != nil { + t.Fatalf("parseOptions: %v", err) + } + if o.listenAddr != ":7777" { + t.Errorf("listen = %q, want :7777 from the flag", o.listenAddr) + } + }) + + t.Run("an unparseable env duration falls back", func(t *testing.T) { + t.Setenv("LLMGW_DRAIN_DELAY", "soon") + o, err := parseOptions(nil, os.Stderr) + if err != nil { + t.Fatalf("parseOptions: %v", err) + } + if o.drainDelay != defaultDrainDelay { + t.Errorf("drain-delay = %s, want the %s default", o.drainDelay, defaultDrainDelay) + } + }) + + t.Run("identical probe paths are rejected", func(t *testing.T) { + if _, err := parseOptions([]string{"-health-path", "/x", "-ready-path", "/x"}, os.Stderr); err == nil { + t.Error("parseOptions accepted identical health and ready paths") + } + }) +} diff --git a/cmd/gateway/testdata/llmgw-config.yaml b/cmd/gateway/testdata/llmgw-config.yaml new file mode 100644 index 0000000..87b1591 --- /dev/null +++ b/cmd/gateway/testdata/llmgw-config.yaml @@ -0,0 +1,295 @@ +admin: + enabled: true +agents: + agentkit: + url: http://agentkit.mcp-servers.svc.example.invalid:8080 + brainstorm: + url: http://agentkit.mcp-servers.svc.example.invalid:8080/brainstorm + budgeting-helper: + url: http://agentkit.mcp-servers.svc.example.invalid:8080/budgeting-helper + business-writer: + url: http://agentkit.mcp-servers.svc.example.invalid:8080/business-writer + car-advisor: + url: http://agentkit.mcp-servers.svc.example.invalid:8080/car-advisor + contract-review: + url: http://agentkit.mcp-servers.svc.example.invalid:8080/contract-review + creative: + url: http://agentkit.mcp-servers.svc.example.invalid:8080/creative + financial-research: + url: http://agentkit.mcp-servers.svc.example.invalid:8080/financial-research + fitness-coach: + url: http://agentkit.mcp-servers.svc.example.invalid:8080/fitness-coach + gift-finder: + url: http://agentkit.mcp-servers.svc.example.invalid:8080/gift-finder + health-researcher: + url: http://agentkit.mcp-servers.svc.example.invalid:8080/health-researcher + home-services-finder: + url: http://agentkit.mcp-servers.svc.example.invalid:8080/home-services-finder + job-search-assistant: + url: http://agentkit.mcp-servers.svc.example.invalid:8080/job-search-assistant + lead-research: + url: http://agentkit.mcp-servers.svc.example.invalid:8080/lead-research + local-events-finder: + url: http://agentkit.mcp-servers.svc.example.invalid:8080/local-events-finder + local-guide: + url: http://agentkit.mcp-servers.svc.example.invalid:8080/local-guide + market-research: + url: http://agentkit.mcp-servers.svc.example.invalid:8080/market-research + meal-prep-planner: + url: http://agentkit.mcp-servers.svc.example.invalid:8080/meal-prep-planner + meeting-prep: + url: http://agentkit.mcp-servers.svc.example.invalid:8080/meeting-prep + news-briefer: + url: http://agentkit.mcp-servers.svc.example.invalid:8080/news-briefer + productivity-planner: + url: http://agentkit.mcp-servers.svc.example.invalid:8080/productivity-planner + property-scout: + url: http://agentkit.mcp-servers.svc.example.invalid:8080/property-scout + recipe-finder: + url: http://agentkit.mcp-servers.svc.example.invalid:8080/recipe-finder + research: + url: http://agentkit.mcp-servers.svc.example.invalid:8080/research + researcher: + url: http://agentkit.mcp-servers.svc.example.invalid:8080/researcher + restaurant-scout: + url: http://agentkit.mcp-servers.svc.example.invalid:8080/restaurant-scout + science-tutor: + url: http://agentkit.mcp-servers.svc.example.invalid:8080/science-tutor + shopping-researcher: + url: http://agentkit.mcp-servers.svc.example.invalid:8080/shopping-researcher + study-helper: + url: http://agentkit.mcp-servers.svc.example.invalid:8080/study-helper + tech-support: + url: http://agentkit.mcp-servers.svc.example.invalid:8080/tech-support + translation: + url: http://agentkit.mcp-servers.svc.example.invalid:8080/translation + travel-planner: + url: http://agentkit.mcp-servers.svc.example.invalid:8080/travel-planner + weather-planner: + url: http://agentkit.mcp-servers.svc.example.invalid:8080/weather-planner + writer: + url: http://agentkit.mcp-servers.svc.example.invalid:8080/writer +cache: + enabled: true + ttl: 5m +groups: + friends: + agents: [] + mcpServers: [] + models: [] + providers: + - minimax + - uni + home: + agents: [] + mcpServers: [] + models: [] + providers: [] + projects: + agents: [] + mcpServers: [] + models: [] + providers: + - minimax + - uni + - gx10 +mcpServers: + brave-search: + url: http://mcp-brave-search.mcp-servers.svc.example.invalid:8080/mcp + captcha-solver: + url: http://mcp-captcha-solver.mcp-servers.svc.example.invalid:8085/mcp + fetch: + url: http://mcp-fetch.mcp-servers.svc.example.invalid:8088/mcp + google-maps: + url: http://mcp-google-maps.mcp-servers.svc.example.invalid:8084/mcp + lightpanda: + url: http://mcp-lightpanda.mcp-servers.svc.example.invalid:8081/mcp + playwright: + url: http://198.51.100.21:3000/sse + readitall: + url: http://mcp-readitall.mcp-servers.svc.example.invalid:8080/mcp + time: + url: http://mcp-time.mcp-servers.svc.example.invalid:8083/mcp + weather: + url: http://mcp-weather.mcp-servers.svc.example.invalid:8087/mcp + web-search: + url: http://mcp-web-search.mcp-servers.svc.example.invalid:8092/mcp + wikipedia: + url: http://mcp-wikipedia.mcp-servers.svc.example.invalid:8089/mcp +metrics: + allowedCIDRs: + - 192.0.2.0/24 + enabled: true + modelLabel: false +modelAliases: + deepseek: uni/deepseek-v4-flash-0731 + deepseek-v4-flash-vision-exp: gx10/current + local/current-model: gx10/current +modelMeta: + badtheorylabs_btl-4: + free: true + btl-4: + contextTokens: 262144 + free: true + btl-4-compact: + free: true + gx10-embed/text-embedding-qwen3-0.6b: + contextTokens: 6144 + free: true + gx10/GLM-5.3-Flash-EXL3: + free: true + gx10/Qwen3.8-Flash-Next-Uncensored-FP8: + free: true + gx10/current: + free: true + gx10/deepseek-v4-flash-vision-exp: + free: true + gx10/deepseek-v4-flash-vision-exp-ablit: + free: true + gx10/deepseek-v4-flash-vision-exp-uncensored: + free: true + huihui-deepseek-v4-flash-0731-abliterated: + free: true + huihui-ornith-1.0-35b-abliterated: + free: true + huihui-qwen3.6-27b-abliterated-mtp: + free: true + huihui-qwen3.6-35b-a3b-abliterated-mtp: + free: true + ornith-1.5-35b-a3b: + free: true + ornith-397b: + contextTokens: 262144 + free: true + qwen/qwen3.8-27b: + free: true + qwen2.5-0.5b-instruct-mlx: + free: true + qwen3.6-27b: + contextTokens: 262144 + free: true + tiny-aya-global: + free: true + uni/deepseek-v4-flash-0731: + contextTokens: 524288 + free: true +providers: + alibaba: + apiKey: file:/llmgw/providers/alibaba + baseUrl: https://ws-example.eu-central-1.maas.aliyuncs.com/compatible-mode + discovery: true + models: + - qwen3.8-max + - qwen3.8-flash + type: openai + anthropic: + apiKey: file:/llmgw/providers/anthropic + discovery: true + type: anthropic + cerebras: + apiKey: file:/llmgw/providers/cerebras + baseUrl: https://api.cerebras.ai + discovery: true + type: openai + codex: + apiKey: unused + baseUrl: http://codex-shim.mcp-servers.svc.example.invalid:8080 + discovery: true + type: openai + copilot: + apiKey: unused + baseUrl: http://copilot-shim.mcp-servers.svc.example.invalid:8080 + discovery: true + type: openai + deepseek: + apiKey: file:/llmgw/providers/deepseek + baseUrl: https://api.deepseek.com + discovery: true + type: openai + gx10: + apiKey: unused + baseUrl: http://llm-api.llm.svc.example.invalid:8000 + discovery: true + discoveryInterval: 5m + metadataPath: /v1/models + requestTimeout: 15m + type: openai + gx10-embed: + apiKey: unused + baseUrl: http://embed-qwen3.llm.svc.example.invalid:8000 + discovery: true + discoveryInterval: 5m + metadataPath: /v1/models + type: openai + macstudio: + apiKey: lm-studio + baseUrl: http://198.51.100.127:1234 + discovery: true + discoveryInterval: 5m + metadataPath: /api/v0/models + type: openai + macstudio-rerank: + apiKey: unused + baseUrl: http://rerank-bge.llm.svc.example.invalid:8000 + discovery: false + type: openai + minimax: + apiKey: file:/llmgw/providers/minimax + baseUrl: https://api.minimax.io + discovery: true + type: openai + openai: + apiKey: file:/llmgw/providers/openai + discovery: true + type: openai + openai-audio: + apiKey: file:/llmgw/providers/openai + discovery: true + type: openai + parakeet-mlx: + apiKey: unused + baseUrl: http://198.51.100.127:8001 + discovery: false + type: openai + piper-tts: + apiKey: unused + baseUrl: http://198.51.100.127:8002 + discovery: false + type: openai + uni: + apiKey: unused + baseUrl: https://agentgateway.example.invalid + discovery: true + metadataPath: /v1/models + models: + - qwen3.8-flash-next + - qwen3.8-27b + - qwen3.8-27b-uncensored + - ornith-1.5-35b + - ornith-397b + - qwen3.6-27b + - deepseek-v4-flash-0731 + requestTimeout: 15m + type: openai + xai: + apiKey: file:/llmgw/providers/xai + baseUrl: https://api.x.ai + discovery: true + type: openai + xiaomi: + apiKey: file:/llmgw/providers/xiaomi + baseUrl: https://token-plan.example.invalid + discovery: true + models: + - mimo-v2.5 + - mimo-v2.5-pro + type: openai +redis: + address: dragonfly-prod.shared-resources.svc.example.invalid:6379 + db: 7 + failOpen: true +retry: + attempts: 1 + enabled: true +users: + file: /llmgw/users/users.json diff --git a/go.mod b/go.mod index 96e17f5..4300814 100644 --- a/go.mod +++ b/go.mod @@ -7,4 +7,4 @@ require ( github.com/stretchr/testify v1.12.1 ) -require go.yaml.in/yaml/v3 v3.0.5 // indirect +require go.yaml.in/yaml/v3 v3.0.5