From a328a8687e5888cd20f22673684cbbcaaebbaa82 Mon Sep 17 00:00:00 2001 From: Lukasz Raczylo Date: Fri, 16 Oct 2026 00:34:14 +0100 Subject: [PATCH] feat(config): add maxRequestBytes for the unified body cap The unified chat and embeddings routes reject bodies above 4 MiB. Long agent sessions exceed this limit and receive a 413. Add Config.maxRequestBytes. Zero or a negative value keeps the 4 MiB default. A positive value raises the cap, clamped to 64 MiB. An adversarial body decodes to about 12x live heap. An operator who raises the cap must also set maxInFlightBodyRequests and the pod memory limit. --- README.md | 1 + llmgateway.go | 14 ++++++++ routes_unified.go | 6 +++- routes_unified_test.go | 76 ++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 96 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index d8f2ee1..2b09c46 100644 --- a/README.md +++ b/README.md @@ -384,6 +384,7 @@ config accepts them as YAML, which decodes to the same JSON shape. | `stripClientAnthropicBeta` | `bool` | `false` | When `true`, `POST /v1/messages` does not forward the client's `anthropic-beta` header to the provider, as native passthrough already does. `false` forwards it, so a client can enable beta features on the operator's own Anthropic account. | | `modelMeta` | `map[string]*ModelMetaConfig` | `{}` | Per-model context-window and cost overrides, keyed by exact `provider/model` or a bare model id — see [Model metadata](#model-metadata). Omitted or empty means no overrides; every model's metadata still resolves through discovery and the built-in table. | | `passthroughUnknown` | `bool` | `false` | `false`: a request matching none of the plugin's routes gets a 404 JSON envelope. `true`: it falls through to the router's own backing service. **Does not cover the three media routes** — see the upgrade note below. | +| `maxRequestBytes` | `int` | `0` (4 MiB) | Raises the unified chat/embeddings request-body cap. `0` or negative keeps 4 MiB; a positive value is clamped to 64 MiB. Pair with `maxInFlightBodyRequests` and the pod memory limit: an adversarial body decodes to ~12x live heap. | | `maxInFlightBodyRequests` | `int` | `0` (self-tuned) | Caps how many unified/media requests may concurrently be inside their own request-body read+decode step — see [Body-admission limit](#body-admission-limit). `0` self-tunes from the host's own `GOMAXPROCS`; a positive value pins an exact cap instead, clamped to a maximum of 10,000. | > **Upgrading from v0.1**: `POST /v1/images/generations`, `POST diff --git a/llmgateway.go b/llmgateway.go index 67f309e..cab4dec 100644 --- a/llmgateway.go +++ b/llmgateway.go @@ -127,6 +127,12 @@ type Config struct { // exposure (GET /admin/api/targets, llmgateway_target_healthy) stay // on unconditionally regardless of this block's presence. TargetHealth TargetHealthConfig `json:"targetHealth"` + // MaxRequestBytes raises the unified chat/embeddings body cap above the + // default maxUnifiedRequestBytes (4MiB), clamped to + // maxExplicitUnifiedRequestBytes. 0 or negative keeps the default. Size + // maxInFlightBodyRequests and the pod memory limit to match: an + // adversarial body decodes to ~12x live heap. + MaxRequestBytes int64 `json:"maxRequestBytes,omitempty"` // MaxInFlightBodyRequests caps how many unified and media JSON/ // multipart requests (routes_unified.go's runUnified via // readAndDecodeUnifiedBody; routes_media.go's decodeMediaJSONRequest/ @@ -771,6 +777,7 @@ type Gateway struct { next http.Handler events *eventLog bodyAdmission chan struct{} + unifiedRequestBytes int64 teeBudget *teeBudget fanoutGate chan struct{} failoverHealth *requestHealthTracker @@ -1061,6 +1068,13 @@ func newGateway(ctx context.Context, next http.Handler, config *Config, name str } } g.bodyAdmission = make(chan struct{}, bodyAdmissionCap) + g.unifiedRequestBytes = maxUnifiedRequestBytes + if config.MaxRequestBytes > 0 { + g.unifiedRequestBytes = config.MaxRequestBytes + if g.unifiedRequestBytes > maxExplicitUnifiedRequestBytes { + g.unifiedRequestBytes = maxExplicitUnifiedRequestBytes + } + } g.teeBudget = newTeeBudget(maxAccountingTeeTotalBytes) g.fanoutGate = make(chan struct{}, mcpFederatedFanoutGlobalConcurrency) diff --git a/routes_unified.go b/routes_unified.go index 86468bd..65591d1 100644 --- a/routes_unified.go +++ b/routes_unified.go @@ -74,6 +74,10 @@ const maxRequestBytes = 10 << 20 // package ships. const maxUnifiedRequestBytes = 4 << 20 +// maxExplicitUnifiedRequestBytes is the ceiling for Config.MaxRequestBytes, +// so an explicit value cannot disable the bound. +const maxExplicitUnifiedRequestBytes = 64 << 20 + // gatewayAliasKey is the request map's internal-convention key carrying // the client-requested model id through to an adapter (ruling a, ALIAS // ECHO), distinct from "model" itself, which runUnified rewrites to the @@ -1069,7 +1073,7 @@ func (g *Gateway) readAndDecodeUnifiedBody(sw *statusTrackingWriter, r *http.Req return nil, nil, false } - body, oversize, err := readCapped(r.Body, maxUnifiedRequestBytes) + body, oversize, err := readCapped(r.Body, g.unifiedRequestBytes) if err != nil { envelope(sw, http.StatusBadRequest, "invalid_request_error", "cannot read request body") return nil, nil, false diff --git a/routes_unified_test.go b/routes_unified_test.go index 1fe1679..3b5b6af 100644 --- a/routes_unified_test.go +++ b/routes_unified_test.go @@ -3128,3 +3128,79 @@ func TestCacheHit_BodyParseSkippedWhenStatsAdminOff(t *testing.T) { t.Errorf("ns/op with admin on (%d) is not meaningfully slower than admin off (%d) — the cached body's own JSON parse (cachedUsage) must be skipped entirely when statsAdmin is off, not merely cheap", nsOn, nsOff) } } + +func newMaxRequestBytesGateway(t *testing.T, upstream string, maxRequestBytes int64) *Gateway { + t.Helper() + cfg := CreateConfig() + cfg.MaxRequestBytes = maxRequestBytes + cfg.Providers = map[string]*ProviderConfig{ + "openai": {Type: "openai", BaseURL: upstream, APIKey: "k", Models: []string{"gpt-test"}}, + } + cfg.Groups = map[string]*GroupConfig{"default": {}} + cfg.Users = &UsersConfig{Inline: []*UserConfig{{Name: "alice", Group: "default", APIKey: "sk-alice"}}} + h, err := New(context.Background(), http.HandlerFunc(func(http.ResponseWriter, *http.Request) {}), cfg, "llmgw") + if err != nil { + t.Fatalf("New: %v", err) + } + return h.(*Gateway) +} + +func postPaddedChat(g *Gateway, padBytes int) *httptest.ResponseRecorder { + body := `{"model":"gpt-test","padding":"` + strings.Repeat("A", padBytes) + `","messages":[{"role":"user","content":"hi"}]}` + req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(body)) + req.Header.Set("Authorization", "Bearer sk-alice") + req.Header.Set("Content-Type", "application/json") + rec := httptest.NewRecorder() + g.ServeHTTP(rec, req) + return rec +} + +// A client that sends a long agent transcript gets 413 under the default cap +// and success once the operator raises maxRequestBytes. +func TestHandleChat_MaxRequestBytes_RaisesUnifiedCap(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = io.Copy(io.Discard, r.Body) + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"id":"c1","choices":[{"message":{"role":"assistant","content":"ok"}}],"usage":{"prompt_tokens":1,"completion_tokens":1}}`)) + })) + defer srv.Close() + + const pad = 5 << 20 + if rec := postPaddedChat(newMaxRequestBytesGateway(t, srv.URL, 0), pad); rec.Code != http.StatusRequestEntityTooLarge { + t.Fatalf("default cap: status = %d, want 413, body=%s", rec.Code, rec.Body.String()) + } + if rec := postPaddedChat(newMaxRequestBytesGateway(t, srv.URL, 16<<20), pad); rec.Code != http.StatusOK { + t.Fatalf("16MiB cap: status = %d, want 200, body=%s", rec.Code, rec.Body.String()) + } +} + +// The raised cap still rejects a body past it, so raising the limit never +// removes the bound. +func TestHandleChat_MaxRequestBytes_StillRejectsOversize(t *testing.T) { + rec := postPaddedChat(newMaxRequestBytesGateway(t, "http://127.0.0.1:1", 16<<20), 16<<20+1024) + if rec.Code != http.StatusRequestEntityTooLarge { + t.Fatalf("status = %d, want 413, body=%s", rec.Code, rec.Body.String()) + } +} + +// An explicit value cannot disable the bound: anything above the ceiling is +// clamped, and a negative value falls back to the default. +func TestNew_MaxRequestBytes_ClampedAndDefaulted(t *testing.T) { + cases := []struct { + name string + in int64 + want int64 + }{ + {"unset keeps default", 0, maxUnifiedRequestBytes}, + {"negative keeps default", -1, maxUnifiedRequestBytes}, + {"explicit value wins", 16 << 20, 16 << 20}, + {"above ceiling is clamped", 1 << 40, maxExplicitUnifiedRequestBytes}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if got := newMaxRequestBytesGateway(t, "http://127.0.0.1:1", tc.in).unifiedRequestBytes; got != tc.want { + t.Fatalf("unifiedRequestBytes = %d, want %d", got, tc.want) + } + }) + } +}