mirror of
https://github.com/lukaszraczylo/traefik-llmgateway.git
synced 2026-10-21 05:57:53 +00:00
feat(providers): add progress-based upstream request timeout
Adapters previously issued upstream requests with no timeout at all. A provider that accepted a connection and then stalled hung the gateway request forever. newAdapterHTTPClient now sets Transport.ResponseHeaderTimeout, and every response body read (providers.go's upstreamBytes, routes_passthrough.go's proxyUpstream) runs through a watchdogBody that resets on every successful read and cancels the request once timeout passes with no progress. The idle-progress design lets a long stream that keeps emitting tokens run indefinitely. Only silence, never total duration, ends a request. Config.RequestTimeout sets the global default (five minutes when unset). ProviderConfig.RequestTimeout overrides it per provider. Both are Go duration strings, validated at construction; a zero or negative value is rejected rather than treated as unlimited, since that would reintroduce the hang this feature fixes. Native passthrough (routes_passthrough.go) and the MCP/A2A target proxy (mcp_a2a.go) share the identical watchdog mechanism through proxyUpstream's new timeout parameter. A watchdog-fired timeout wraps errProviderTimeout and names the provider. It never wraps context.Canceled, so it stays distinguishable from a genuine client disconnect in every existing log classification.
This commit is contained in:
+13
-3
@@ -29,6 +29,12 @@ type anthropicAdapter struct {
|
||||
adapterName string
|
||||
baseURL string
|
||||
apiKey string
|
||||
// timeout is this adapter's resolved per-request timeout —
|
||||
// resolveProviderTimeout, timeout.go; see openaiAdapter.timeout's own
|
||||
// doc comment (provider_openai.go) for why this is set to
|
||||
// defaultRequestTimeout at construction rather than left at its zero
|
||||
// value the way retry is.
|
||||
timeout time.Duration
|
||||
}
|
||||
|
||||
// newAnthropicAdapter returns an anthropicAdapter for provider name, with
|
||||
@@ -45,7 +51,8 @@ func newAnthropicAdapter(name, base, apiKey string) (*anthropicAdapter, error) {
|
||||
adapterName: name,
|
||||
baseURL: base,
|
||||
apiKey: apiKey,
|
||||
client: newAdapterHTTPClient(),
|
||||
client: newAdapterHTTPClient(defaultRequestTimeout),
|
||||
timeout: defaultRequestTimeout,
|
||||
}, nil
|
||||
}
|
||||
|
||||
@@ -61,6 +68,9 @@ func (a *anthropicAdapter) base() string { return a.baseURL }
|
||||
// httpClient implements providerAdapter.
|
||||
func (a *anthropicAdapter) httpClient() *http.Client { return a.client }
|
||||
|
||||
// requestTimeout implements providerAdapter.
|
||||
func (a *anthropicAdapter) requestTimeout() time.Duration { return a.timeout }
|
||||
|
||||
// injectAuth implements providerAdapter: Anthropic's x-api-key scheme,
|
||||
// plus the anthropic-version header every request must carry. Unlike
|
||||
// openai-type's injectAuth, this is never a no-op — the constructor
|
||||
@@ -112,7 +122,7 @@ func (a *anthropicAdapter) chatCompletion(ctx context.Context, w http.ResponseWr
|
||||
hdr.Set("Accept", "text/event-stream")
|
||||
}
|
||||
|
||||
resp, err := upstreamJSON(ctx, a.client, http.MethodPost, a.baseURL+anthropicMessagesPath, hdr, body, a.retry)
|
||||
resp, err := upstreamJSON(ctx, a.client, http.MethodPost, a.baseURL+anthropicMessagesPath, hdr, body, a.retry, a.timeout, a.adapterName)
|
||||
if err != nil {
|
||||
return usage{}, err
|
||||
}
|
||||
@@ -226,7 +236,7 @@ func (a *anthropicAdapter) audioTranscription(_ context.Context, _ http.Response
|
||||
// falls back to a provider's explicitly configured models on error rather
|
||||
// than this adapter guessing at a fallback itself.
|
||||
func (a *anthropicAdapter) listModels(ctx context.Context) ([]string, error) {
|
||||
resp, err := upstreamJSON(ctx, a.client, http.MethodGet, a.baseURL+anthropicModelsPath, a.requestHeaders(false), nil, a.retry)
|
||||
resp, err := upstreamJSON(ctx, a.client, http.MethodGet, a.baseURL+anthropicModelsPath, a.requestHeaders(false), nil, a.retry, a.timeout, a.adapterName)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user