lukaszraczylo/traefik-llmgateway
1
0
Fork 0
mirror of https://github.com/lukaszraczylo/traefik-llmgateway.git synced 2026-10-23 06:07:47 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(config): advertise concurrency via RateLimit-Policy with autodetect

Browse Source 
The gateway now sends the concurrency limit as a RateLimit-Policy
field from draft-ietf-httpapi-ratelimit-headers. The value is
"concurrency";q=<n>;qu="concurrent-requests". The invented
X-Concurrency-Limit header is gone.

The new concurrencyAutodetect option is off by default. When on,
the gateway learns a limit for each configured provider. A 429, 503,
504 or upstream timeout sets the limit to half the in-flight
requests, with a minimum of 1. Four consecutive successes add 1.
At 64 the limit counts as unknown again.

Responses served via a provider advertise the smaller of
maxConcurrency and the learned value. An explicit maxConcurrency
always wins. Routing, retry and failover do not change.
This commit is contained in:
lukaszraczylo
2026-10-18 13:44:08 +01:00
parent 3e3517eb62
commit 644b8115c8
7 changed files with 356 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files
routes_media.go
+6 -9
View File
@@ -124,9 +124,8 @@ func (g *Gateway) handleImagesGenerations(w http.ResponseWriter, r *http.Request
// Feature A (v0.22): see routes_unified.go's runUnified for the
// identical pattern this mirrors.
ctx := withAttemptRecorder(r.Context(), func(resp *http.Response, attemptErr error) {
g.limiter.recordProviderAttempt(adapter.name(), upstreamModel, resp, attemptErr)
})
ctx, endUpstream := g.trackUpstream(r.Context(), sw, adapter.name(), upstreamModel)
defer endUpstream()
// feat: instrument upstream latency — gated on metricsEnabled OR
// admin.stats.latency, same reasoning as runUnified's identical wiring
// (routes_unified.go).
@@ -200,9 +199,8 @@ func (g *Gateway) handleAudioSpeech(w http.ResponseWriter, r *http.Request, u *u
// Feature A (v0.22): see routes_unified.go's runUnified for the
// identical pattern this mirrors.
ctx := withAttemptRecorder(r.Context(), func(resp *http.Response, attemptErr error) {
g.limiter.recordProviderAttempt(adapter.name(), upstreamModel, resp, attemptErr)
})
ctx, endUpstream := g.trackUpstream(r.Context(), sw, adapter.name(), upstreamModel)
defer endUpstream()
// feat: instrument upstream latency — gated on metricsEnabled OR
// admin.stats.latency, same reasoning as runUnified's identical wiring
// (routes_unified.go).
@@ -315,9 +313,8 @@ func (g *Gateway) handleAudioTranscriptions(w http.ResponseWriter, r *http.Reque
// Feature A (v0.22): see routes_unified.go's runUnified for the
// identical pattern this mirrors.
ctx := withAttemptRecorder(r.Context(), func(resp *http.Response, attemptErr error) {
g.limiter.recordProviderAttempt(adapter.name(), upstreamModel, resp, attemptErr)
})
ctx, endUpstream := g.trackUpstream(r.Context(), sw, adapter.name(), upstreamModel)
defer endUpstream()
// feat: instrument upstream latency — gated on metricsEnabled OR
// admin.stats.latency, same reasoning as runUnified's identical wiring
// (routes_unified.go).