lukaszraczylo/traefik-llmgateway
1
0
Fork 0
mirror of https://github.com/lukaszraczylo/traefik-llmgateway.git synced 2026-10-21 05:57:53 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: same-provider retry for transient upstream failures

Browse Source 
Adds spec §1's opt-in retry engine: RetryConfig (top-level, zero value
disabled) and retryPolicy (retry.go), wired into upstreamJSON — the
single HTTP chokepoint every adapter's chat/embeddings/listModels call
already funneled through. A transient failure (net error, 429, 5xx)
retries up to Attempts times with exponential backoff capped at 2s,
honoring a qualifying Retry-After header on 429, draining and closing
each failed attempt's body before retrying. Never retries a non-429 4xx,
a translate error, or after context cancellation.

Because upstreamJSON returns before any adapter forwards a byte to the
client (forwardStream/forwardJSON run only after it returns), the
'zero bytes reached client' safety rule and the 'streaming retries only
the initial exchange' rule both hold by construction — no adapter
streaming logic needed to change.

buildAdapters(cfg) keeps its v0.1 signature: it derives the retryPolicy
from cfg.Retry internally rather than taking it as a parameter, so
every existing direct call (15+ across provider_openai_test.go) keeps
compiling unmodified. Every v0.1 test passes byte-identically, without
edits.
This commit is contained in:
lukaszraczylo
2026-08-20 17:11:40 +01:00
parent a280ee8aee
commit 34968e27b2
7 changed files with 847 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files
provider_openai.go
+10 -4
View File
@@ -13,7 +13,13 @@ import (
// itself, and any OpenAI-compatible upstream (the operator's own gateway
// included — a real deployment target for this adapter is keyless).
type openaiAdapter struct {
client *http.Client
client *http.Client
// retry is nil for every adapter built by a bare newOpenAIAdapter call
// (every v0.1 test, and any caller that never wires one) — nil-safe,
// per retryPolicy.do's own contract, so leaving it unset is exactly
// v0.1's one-attempt behavior. buildAdapters (providers.go) sets it
// from cfg.Retry for production use.
retry *retryPolicy
adapterName string
baseURL string
apiKey string
@@ -131,7 +137,7 @@ func (a *openaiAdapter) chatCompletion(ctx context.Context, w http.ResponseWrite
hdr.Set("Accept", "text/event-stream")
}
resp, err := upstreamJSON(ctx, a.client, http.MethodPost, a.baseURL+"/v1/chat/completions", hdr, req)
resp, err := upstreamJSON(ctx, a.client, http.MethodPost, a.baseURL+"/v1/chat/completions", hdr, req, a.retry)
if err != nil {
return usage{}, err
}
@@ -156,7 +162,7 @@ func (a *openaiAdapter) chatCompletion(ctx context.Context, w http.ResponseWrite
// non-streaming path.
func (a *openaiAdapter) embeddings(ctx context.Context, w http.ResponseWriter, req map[string]any) (usage, error) {
delete(req, gatewayAliasKey) // see chatCompletion's identical delete for why
resp, err := upstreamJSON(ctx, a.client, http.MethodPost, a.baseURL+"/v1/embeddings", a.requestHeaders(true), req)
resp, err := upstreamJSON(ctx, a.client, http.MethodPost, a.baseURL+"/v1/embeddings", a.requestHeaders(true), req, a.retry)
if err != nil {
return usage{}, err
}
@@ -258,7 +264,7 @@ type modelsPayload struct {
// listModels implements providerAdapter.
func (a *openaiAdapter) listModels(ctx context.Context) ([]string, error) {
resp, err := upstreamJSON(ctx, a.client, http.MethodGet, a.baseURL+"/v1/models", a.requestHeaders(false), nil)
resp, err := upstreamJSON(ctx, a.client, http.MethodGet, a.baseURL+"/v1/models", a.requestHeaders(false), nil, a.retry)
if err != nil {
return nil, err
}