mirror of
https://github.com/lukaszraczylo/traefik-llmgateway.git
synced 2026-10-20 05:56:56 +00:00
feat: stdlib RESP client and redis-backed limit store with fail-open
Add resp.go: a minimal stdlib-only RESP2 client (respClient) for a single Redis-compatible server. It lazily dials one mutex-guarded connection, applies 2s dial/read/write deadlines, sends AUTH (if a password is set) then SELECT on every new connection, and reconnects once on any I/O error before giving up. do and pipeline encode commands as "*N\r\n$len\r\n..." and decode +/-/:/$/* replies. Add redis_store.go: redisStore implements counterStore over respClient. incrBy pipelines INCRBY and EXPIRE in one round trip and returns INCRBY's value; get treats a missing key (RESP null bulk) as 0. Wire the limiter's fail-open/fail-closed policy in limits.go, which Task 5 stored but never read: a configured store's error now routes through storeIncrBy/storeGet. failOpen=true falls back to the in-process memoryStore transparently; failOpen=false makes checkAndCount return a storeDown violation and makes account drop the sample. Both paths log through an injectable, 30s-rate-limited logf. Wire newGateway (llmgateway.go) to build a redisStore-backed limiter when Config.Redis is set (password through resolveSecret, failOpen defaulting to true unless RedisConfig.FailOpen says otherwise) and a memory-only limiter otherwise, and to route pricing.go's unknown-model warnings through the gateway's own logf. Validate LimitsConfig at construction: a new LimitsConfig.validate rejects a negative int64 count limit or a negative/NaN/Inf cost limit, called from newAuthStore for every group and every user (inline and file-sourced) so a malformed limit is a constructor error instead of silently behaving as unlimited. Add RequestsPerDay, TokensPerMonth, and CostPerMonthUSD enforcement tests alongside the existing per-minute/day/cost-per-day cases, plus an erroringStore-backed test pair proving the fail-open (fallback counted) and fail-closed (storeDown violation) branches directly. Folded the validation work into this single commit rather than a second one: it interleaves with the fail-open changes across limits.go, limits_test.go, and llmgateway_test.go closely enough that splitting by hunk would be artificial.
This commit is contained in:
+52
-7
@@ -101,12 +101,13 @@ func CreateConfig() *Config {
|
||||
|
||||
// Gateway is the Traefik middleware handler.
|
||||
type Gateway struct {
|
||||
next http.Handler
|
||||
cfg *Config
|
||||
auth *authStore
|
||||
name string
|
||||
// wired in later tasks: limiter *limiter, registry *modelRegistry,
|
||||
// adapters map[string]providerAdapter
|
||||
next http.Handler
|
||||
cfg *Config
|
||||
auth *authStore
|
||||
limiter *limiter
|
||||
name string
|
||||
// wired in later tasks: registry *modelRegistry, adapters
|
||||
// map[string]providerAdapter
|
||||
}
|
||||
|
||||
// New creates the middleware. NOTE: no tail call — Yaegi zeroes
|
||||
@@ -130,14 +131,58 @@ func newGateway(ctx context.Context, next http.Handler, config *Config, name str
|
||||
g := &Gateway{next: next, name: name, cfg: config, auth: auth}
|
||||
|
||||
if config.Users != nil && config.Users.File != "" {
|
||||
if err := attachUsersFile(auth, config.Users.File, g); err != nil {
|
||||
if err = attachUsersFile(auth, config.Users.File, g); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
lim, err := newConfiguredLimiter(config)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
lim.logf = g.errorf
|
||||
g.limiter = lim
|
||||
|
||||
setPricingWarnFn(func(msg string) {
|
||||
if msg == warnCapMessage {
|
||||
g.logf("%s", msg)
|
||||
return
|
||||
}
|
||||
g.logf("pricing: no price configured for model %q; cost will be recorded as 0", msg)
|
||||
})
|
||||
|
||||
return g, nil
|
||||
}
|
||||
|
||||
// newConfiguredLimiter builds the limiter for config, wiring config.Redis
|
||||
// when set. A configured Redis block wires a respClient-backed redisStore
|
||||
// behind AUTH/SELECT and resolveSecret'd credentials; otherwise the
|
||||
// limiter runs on its in-process fallback alone. failOpen defaults to
|
||||
// true (a Redis outage must not take the whole gateway down unless an
|
||||
// operator opts into strict enforcement) unless RedisConfig.FailOpen is
|
||||
// explicitly set.
|
||||
func newConfiguredLimiter(config *Config) (*limiter, error) {
|
||||
if config.Redis == nil {
|
||||
return newLimiter(nil, true), nil
|
||||
}
|
||||
if config.Redis.Address == "" {
|
||||
return nil, errors.New("llmgateway: redis: address must not be empty")
|
||||
}
|
||||
|
||||
password, err := resolveSecret(config.Redis.Password)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("llmgateway: redis: %w", err)
|
||||
}
|
||||
|
||||
failOpen := true
|
||||
if config.Redis.FailOpen != nil {
|
||||
failOpen = *config.Redis.FailOpen
|
||||
}
|
||||
|
||||
client := newRESPClient(config.Redis.Address, password, config.Redis.DB)
|
||||
return newLimiter(newRedisStore(client), failOpen), nil
|
||||
}
|
||||
|
||||
// attachUsersFile performs the synchronous initial load of a Users.File path
|
||||
// and wires auth up for throttled hot reload. A failing initial load is a
|
||||
// constructor error — fail fast on bad config rather than start with an
|
||||
|
||||
Reference in New Issue
Block a user