From 7616f4bdee8835ec3bb2ebb4505d5c3a6909b0c7 Mon Sep 17 00:00:00 2001
From: Lukasz Raczylo <lukasz@raczylo.com>
Date: Sun, 30 Aug 2026 14:19:35 +0100
Subject: [PATCH] fix(llm): retry 429 responses and 2xx bodies carrying an
 error object

Six qwen3.8-27b-uncensored requests in the 2026-08-30 run failed
instantly with zero tokens while the replica was warming; the gateway
reported them in-band and the client gave up on the first attempt. Both
shapes now retry within the existing backoff budget.
---
 internal/llm/openai.go      | 10 ++++++--
 internal/llm/openai_test.go | 51 +++++++++++++++++++++++++++++++++++++
 2 files changed, 59 insertions(+), 2 deletions(-)

diff --git a/internal/llm/openai.go b/internal/llm/openai.go
index 35e65be..0778e05 100644
--- a/internal/llm/openai.go
+++ b/internal/llm/openai.go
@@ -135,8 +135,12 @@ func (c *OpenAIClient) Complete(ctx context.Context, req Request) (Response, err
 			finishReason = resp.Choices[0].FinishReason
 		}
 		if resp.Error != nil {
+			// A 2xx carrying a JSON error body is how some gateways report a
+			// transient upstream failure (observed as instant 0-token
+			// "errors" in the 2026-08-30 run while a replica was warming).
+			// Treat it like a 5xx: retry within the same attempt budget.
 			lastErr = fmt.Errorf("llm: api error: %s", resp.Error.Message)
-			return Response{}, lastErr
+			continue
 		}
 		return Response{
 			Text:             text,
@@ -190,7 +194,9 @@ func (c *OpenAIClient) doRequest(ctx context.Context, payload []byte) (*chatComp
 		return nil, &retryableError{err: fmt.Errorf("read body: %w", err)}
 	}
 
-	if httpResp.StatusCode >= 500 {
+	// 429 is as transient as a 5xx for a self-hosted gateway (rate limit or
+	// a briefly saturated upstream); retry it with the same backoff budget.
+	if httpResp.StatusCode >= 500 || httpResp.StatusCode == http.StatusTooManyRequests {
 		return nil, &retryableError{status: httpResp.StatusCode}
 	}
 	if httpResp.StatusCode >= 400 {
diff --git a/internal/llm/openai_test.go b/internal/llm/openai_test.go
index 47a11a3..4681ea7 100644
--- a/internal/llm/openai_test.go
+++ b/internal/llm/openai_test.go
@@ -236,7 +236,9 @@ func TestOpenAIClient_Complete_NoRetryOn4xx(t *testing.T) {
 }
 
 func TestOpenAIClient_Complete_APIErrorField(t *testing.T) {
+	var calls int32
 	srv := newTestServer(t, func(w http.ResponseWriter, r *http.Request) {
+		atomic.AddInt32(&calls, 1)
 		_, _ = w.Write([]byte(`{"error":{"message":"model overloaded"}}`))
 	})
 
@@ -248,6 +250,55 @@ func TestOpenAIClient_Complete_APIErrorField(t *testing.T) {
 	if !strings.Contains(err.Error(), "model overloaded") {
 		t.Errorf("error = %v, want mention of model overloaded", err)
 	}
+	if got := atomic.LoadInt32(&calls); got != maxRetries+1 {
+		t.Errorf("calls = %d, want %d (2xx-with-error-body is retried like a 5xx)", got, maxRetries+1)
+	}
+}
+
+func TestOpenAIClient_Complete_RetriesAPIErrorThenSucceeds(t *testing.T) {
+	var calls int32
+	srv := newTestServer(t, func(w http.ResponseWriter, r *http.Request) {
+		if atomic.AddInt32(&calls, 1) == 1 {
+			_, _ = w.Write([]byte(`{"error":{"message":"replica warming"}}`))
+			return
+		}
+		_, _ = w.Write([]byte(`{"choices":[{"message":{"content":"ok"},"finish_reason":"stop"}],"usage":{"prompt_tokens":1,"completion_tokens":1}}`))
+	})
+
+	client := NewOpenAIClient(srv.URL, "", time.Second)
+	resp, err := client.Complete(context.Background(), Request{Model: "m", Messages: []Message{{Role: "user", Content: "x"}}})
+	if err != nil {
+		t.Fatalf("Complete() error = %v, want nil", err)
+	}
+	if resp.Text != "ok" {
+		t.Errorf("Text = %q, want %q", resp.Text, "ok")
+	}
+	if got := atomic.LoadInt32(&calls); got != 2 {
+		t.Errorf("calls = %d, want 2 (one error body, one success)", got)
+	}
+}
+
+func TestOpenAIClient_Complete_RetriesOn429ThenSucceeds(t *testing.T) {
+	var calls int32
+	srv := newTestServer(t, func(w http.ResponseWriter, r *http.Request) {
+		if atomic.AddInt32(&calls, 1) == 1 {
+			w.WriteHeader(http.StatusTooManyRequests)
+			return
+		}
+		_, _ = w.Write([]byte(`{"choices":[{"message":{"content":"ok"},"finish_reason":"stop"}],"usage":{"prompt_tokens":1,"completion_tokens":1}}`))
+	})
+
+	client := NewOpenAIClient(srv.URL, "", time.Second)
+	resp, err := client.Complete(context.Background(), Request{Model: "m", Messages: []Message{{Role: "user", Content: "x"}}})
+	if err != nil {
+		t.Fatalf("Complete() error = %v, want nil", err)
+	}
+	if resp.Text != "ok" {
+		t.Errorf("Text = %q, want %q", resp.Text, "ok")
+	}
+	if got := atomic.LoadInt32(&calls); got != 2 {
+		t.Errorf("calls = %d, want 2 (one 429, one success)", got)
+	}
 }
 
 func TestOpenAIClient_Complete_ContextCanceled(t *testing.T) {