diff --git a/README.md b/README.md index d8f2ee1..2b09c46 100644 --- a/README.md +++ b/README.md @@ -384,6 +384,7 @@ config accepts them as YAML, which decodes to the same JSON shape. | `stripClientAnthropicBeta` | `bool` | `false` | When `true`, `POST /v1/messages` does not forward the client's `anthropic-beta` header to the provider, as native passthrough already does. `false` forwards it, so a client can enable beta features on the operator's own Anthropic account. | | `modelMeta` | `map[string]*ModelMetaConfig` | `{}` | Per-model context-window and cost overrides, keyed by exact `provider/model` or a bare model id — see [Model metadata](#model-metadata). Omitted or empty means no overrides; every model's metadata still resolves through discovery and the built-in table. | | `passthroughUnknown` | `bool` | `false` | `false`: a request matching none of the plugin's routes gets a 404 JSON envelope. `true`: it falls through to the router's own backing service. **Does not cover the three media routes** — see the upgrade note below. | +| `maxRequestBytes` | `int` | `0` (4 MiB) | Raises the unified chat/embeddings request-body cap. `0` or negative keeps 4 MiB; a positive value is clamped to 64 MiB. Pair with `maxInFlightBodyRequests` and the pod memory limit: an adversarial body decodes to ~12x live heap. | | `maxInFlightBodyRequests` | `int` | `0` (self-tuned) | Caps how many unified/media requests may concurrently be inside their own request-body read+decode step — see [Body-admission limit](#body-admission-limit). `0` self-tunes from the host's own `GOMAXPROCS`; a positive value pins an exact cap instead, clamped to a maximum of 10,000. | > **Upgrading from v0.1**: `POST /v1/images/generations`, `POST diff --git a/llmgateway.go b/llmgateway.go index 67f309e..cab4dec 100644 --- a/llmgateway.go +++ b/llmgateway.go @@ -127,6 +127,12 @@ type Config struct { // exposure (GET /admin/api/targets, llmgateway_target_healthy) stay // on unconditionally regardless of this block's presence. TargetHealth TargetHealthConfig `json:"targetHealth"` + // MaxRequestBytes raises the unified chat/embeddings body cap above the + // default maxUnifiedRequestBytes (4MiB), clamped to + // maxExplicitUnifiedRequestBytes. 0 or negative keeps the default. Size + // maxInFlightBodyRequests and the pod memory limit to match: an + // adversarial body decodes to ~12x live heap. + MaxRequestBytes int64 `json:"maxRequestBytes,omitempty"` // MaxInFlightBodyRequests caps how many unified and media JSON/ // multipart requests (routes_unified.go's runUnified via // readAndDecodeUnifiedBody; routes_media.go's decodeMediaJSONRequest/ @@ -771,6 +777,7 @@ type Gateway struct { next http.Handler events *eventLog bodyAdmission chan struct{} + unifiedRequestBytes int64 teeBudget *teeBudget fanoutGate chan struct{} failoverHealth *requestHealthTracker @@ -1061,6 +1068,13 @@ func newGateway(ctx context.Context, next http.Handler, config *Config, name str } } g.bodyAdmission = make(chan struct{}, bodyAdmissionCap) + g.unifiedRequestBytes = maxUnifiedRequestBytes + if config.MaxRequestBytes > 0 { + g.unifiedRequestBytes = config.MaxRequestBytes + if g.unifiedRequestBytes > maxExplicitUnifiedRequestBytes { + g.unifiedRequestBytes = maxExplicitUnifiedRequestBytes + } + } g.teeBudget = newTeeBudget(maxAccountingTeeTotalBytes) g.fanoutGate = make(chan struct{}, mcpFederatedFanoutGlobalConcurrency) diff --git a/routes_unified.go b/routes_unified.go index 86468bd..65591d1 100644 --- a/routes_unified.go +++ b/routes_unified.go @@ -74,6 +74,10 @@ const maxRequestBytes = 10 << 20 // package ships. const maxUnifiedRequestBytes = 4 << 20 +// maxExplicitUnifiedRequestBytes is the ceiling for Config.MaxRequestBytes, +// so an explicit value cannot disable the bound. +const maxExplicitUnifiedRequestBytes = 64 << 20 + // gatewayAliasKey is the request map's internal-convention key carrying // the client-requested model id through to an adapter (ruling a, ALIAS // ECHO), distinct from "model" itself, which runUnified rewrites to the @@ -1069,7 +1073,7 @@ func (g *Gateway) readAndDecodeUnifiedBody(sw *statusTrackingWriter, r *http.Req return nil, nil, false } - body, oversize, err := readCapped(r.Body, maxUnifiedRequestBytes) + body, oversize, err := readCapped(r.Body, g.unifiedRequestBytes) if err != nil { envelope(sw, http.StatusBadRequest, "invalid_request_error", "cannot read request body") return nil, nil, false diff --git a/routes_unified_test.go b/routes_unified_test.go index 1fe1679..3b5b6af 100644 --- a/routes_unified_test.go +++ b/routes_unified_test.go @@ -3128,3 +3128,79 @@ func TestCacheHit_BodyParseSkippedWhenStatsAdminOff(t *testing.T) { t.Errorf("ns/op with admin on (%d) is not meaningfully slower than admin off (%d) — the cached body's own JSON parse (cachedUsage) must be skipped entirely when statsAdmin is off, not merely cheap", nsOn, nsOff) } } + +func newMaxRequestBytesGateway(t *testing.T, upstream string, maxRequestBytes int64) *Gateway { + t.Helper() + cfg := CreateConfig() + cfg.MaxRequestBytes = maxRequestBytes + cfg.Providers = map[string]*ProviderConfig{ + "openai": {Type: "openai", BaseURL: upstream, APIKey: "k", Models: []string{"gpt-test"}}, + } + cfg.Groups = map[string]*GroupConfig{"default": {}} + cfg.Users = &UsersConfig{Inline: []*UserConfig{{Name: "alice", Group: "default", APIKey: "sk-alice"}}} + h, err := New(context.Background(), http.HandlerFunc(func(http.ResponseWriter, *http.Request) {}), cfg, "llmgw") + if err != nil { + t.Fatalf("New: %v", err) + } + return h.(*Gateway) +} + +func postPaddedChat(g *Gateway, padBytes int) *httptest.ResponseRecorder { + body := `{"model":"gpt-test","padding":"` + strings.Repeat("A", padBytes) + `","messages":[{"role":"user","content":"hi"}]}` + req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(body)) + req.Header.Set("Authorization", "Bearer sk-alice") + req.Header.Set("Content-Type", "application/json") + rec := httptest.NewRecorder() + g.ServeHTTP(rec, req) + return rec +} + +// A client that sends a long agent transcript gets 413 under the default cap +// and success once the operator raises maxRequestBytes. +func TestHandleChat_MaxRequestBytes_RaisesUnifiedCap(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = io.Copy(io.Discard, r.Body) + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"id":"c1","choices":[{"message":{"role":"assistant","content":"ok"}}],"usage":{"prompt_tokens":1,"completion_tokens":1}}`)) + })) + defer srv.Close() + + const pad = 5 << 20 + if rec := postPaddedChat(newMaxRequestBytesGateway(t, srv.URL, 0), pad); rec.Code != http.StatusRequestEntityTooLarge { + t.Fatalf("default cap: status = %d, want 413, body=%s", rec.Code, rec.Body.String()) + } + if rec := postPaddedChat(newMaxRequestBytesGateway(t, srv.URL, 16<<20), pad); rec.Code != http.StatusOK { + t.Fatalf("16MiB cap: status = %d, want 200, body=%s", rec.Code, rec.Body.String()) + } +} + +// The raised cap still rejects a body past it, so raising the limit never +// removes the bound. +func TestHandleChat_MaxRequestBytes_StillRejectsOversize(t *testing.T) { + rec := postPaddedChat(newMaxRequestBytesGateway(t, "http://127.0.0.1:1", 16<<20), 16<<20+1024) + if rec.Code != http.StatusRequestEntityTooLarge { + t.Fatalf("status = %d, want 413, body=%s", rec.Code, rec.Body.String()) + } +} + +// An explicit value cannot disable the bound: anything above the ceiling is +// clamped, and a negative value falls back to the default. +func TestNew_MaxRequestBytes_ClampedAndDefaulted(t *testing.T) { + cases := []struct { + name string + in int64 + want int64 + }{ + {"unset keeps default", 0, maxUnifiedRequestBytes}, + {"negative keeps default", -1, maxUnifiedRequestBytes}, + {"explicit value wins", 16 << 20, 16 << 20}, + {"above ceiling is clamped", 1 << 40, maxExplicitUnifiedRequestBytes}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if got := newMaxRequestBytesGateway(t, "http://127.0.0.1:1", tc.in).unifiedRequestBytes; got != tc.want { + t.Fatalf("unifiedRequestBytes = %d, want %d", got, tc.want) + } + }) + } +}