fix(llm): retry 429 responses and 2xx bodies carrying an error object

Six qwen3.8-27b-uncensored requests in the 2026-08-30 run failed
instantly with zero tokens while the replica was warming; the gateway
reported them in-band and the client gave up on the first attempt. Both
shapes now retry within the existing backoff budget.
This commit is contained in:
2026-08-30 14:19:35 +01:00
parent d9b7ca9cc4
commit 7616f4bdee
2 changed files with 59 additions and 2 deletions
+8 -2
View File
@@ -135,8 +135,12 @@ func (c *OpenAIClient) Complete(ctx context.Context, req Request) (Response, err
finishReason = resp.Choices[0].FinishReason
}
if resp.Error != nil {
// A 2xx carrying a JSON error body is how some gateways report a
// transient upstream failure (observed as instant 0-token
// "errors" in the 2026-08-30 run while a replica was warming).
// Treat it like a 5xx: retry within the same attempt budget.
lastErr = fmt.Errorf("llm: api error: %s", resp.Error.Message)
return Response{}, lastErr
continue
}
return Response{
Text: text,
@@ -190,7 +194,9 @@ func (c *OpenAIClient) doRequest(ctx context.Context, payload []byte) (*chatComp
return nil, &retryableError{err: fmt.Errorf("read body: %w", err)}
}
if httpResp.StatusCode >= 500 {
// 429 is as transient as a 5xx for a self-hosted gateway (rate limit or
// a briefly saturated upstream); retry it with the same backoff budget.
if httpResp.StatusCode >= 500 || httpResp.StatusCode == http.StatusTooManyRequests {
return nil, &retryableError{status: httpResp.StatusCode}
}
if httpResp.StatusCode >= 400 {