mirror of
https://github.com/lukaszraczylo/traefik-llmgateway.git
synced 2026-10-22 06:06:59 +00:00
feat(config): advertise concurrency via RateLimit-Policy with autodetect
The gateway now sends the concurrency limit as a RateLimit-Policy field from draft-ietf-httpapi-ratelimit-headers. The value is "concurrency";q=<n>;qu="concurrent-requests". The invented X-Concurrency-Limit header is gone. The new concurrencyAutodetect option is off by default. When on, the gateway learns a limit for each configured provider. A 429, 503, 504 or upstream timeout sets the limit to half the in-flight requests, with a minimum of 1. Four consecutive successes add 1. At 64 the limit counts as unknown again. Responses served via a provider advertise the smaller of maxConcurrency and the learned value. An explicit maxConcurrency always wins. Routing, retry and failover do not change.
This commit is contained in:
+2
-3
@@ -587,9 +587,7 @@ func (g *Gateway) runMeteredCall(sw *statusTrackingWriter, r *http.Request, scop
|
||||
return
|
||||
}
|
||||
|
||||
ctx := withAttemptRecorder(r.Context(), func(resp *http.Response, attemptErr error) {
|
||||
g.limiter.recordProviderAttempt(providerName, upstreamModel, resp, attemptErr)
|
||||
})
|
||||
ctx, endUpstream := g.trackUpstream(r.Context(), respWriter, providerName, upstreamModel)
|
||||
// feat: instrument upstream latency — gated on metricsEnabled OR
|
||||
// admin.stats.latency (admin-redesign WP-A), so a deployment with
|
||||
// BOTH off wires nothing here at all (not even the closure
|
||||
@@ -615,6 +613,7 @@ func (g *Gateway) runMeteredCall(sw *statusTrackingWriter, r *http.Request, scop
|
||||
})
|
||||
}
|
||||
result, callErr := call(cand.adapter, ctx, respWriter, attemptReq)
|
||||
endUpstream()
|
||||
|
||||
// Request-path health, recorded ONCE per candidate per logical
|
||||
// request (F3/F6 fix, adversarial-review) — see
|
||||
|
||||
Reference in New Issue
Block a user