lukaszraczylo/llm-testbench
1
0
Fork 0
mirror of https://github.com/lukaszraczylo/llm-testbench.git synced 2026-10-14 00:57:45 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(llm): retry 429 responses and 2xx bodies carrying an error object

Browse Source 
Six qwen3.8-27b-uncensored requests in the 2026-08-30 run failed
instantly with zero tokens while the replica was warming; the gateway
reported them in-band and the client gave up on the first attempt. Both
shapes now retry within the existing backoff budget.
This commit is contained in:
lukaszraczylo
2026-08-30 14:19:35 +01:00
parent d9b7ca9cc4
commit 7616f4bdee
2 changed files with 59 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/llm/openai_test.go
+51
View File
@@ -236,7 +236,9 @@ func TestOpenAIClient_Complete_NoRetryOn4xx(t *testing.T) {
}
func TestOpenAIClient_Complete_APIErrorField(t *testing.T) {
var calls int32
srv := newTestServer(t, func(w http.ResponseWriter, r *http.Request) {
atomic.AddInt32(&calls, 1)
_, _ = w.Write([]byte(`{"error":{"message":"model overloaded"}}`))
})
@@ -248,6 +250,55 @@ func TestOpenAIClient_Complete_APIErrorField(t *testing.T) {
if !strings.Contains(err.Error(), "model overloaded") {
t.Errorf("error = %v, want mention of model overloaded", err)
}
if got := atomic.LoadInt32(&calls); got != maxRetries+1 {
t.Errorf("calls = %d, want %d (2xx-with-error-body is retried like a 5xx)", got, maxRetries+1)
}
}
func TestOpenAIClient_Complete_RetriesAPIErrorThenSucceeds(t *testing.T) {
var calls int32
srv := newTestServer(t, func(w http.ResponseWriter, r *http.Request) {
if atomic.AddInt32(&calls, 1) == 1 {
_, _ = w.Write([]byte(`{"error":{"message":"replica warming"}}`))
return
}
_, _ = w.Write([]byte(`{"choices":[{"message":{"content":"ok"},"finish_reason":"stop"}],"usage":{"prompt_tokens":1,"completion_tokens":1}}`))
})
client := NewOpenAIClient(srv.URL, "", time.Second)
resp, err := client.Complete(context.Background(), Request{Model: "m", Messages: []Message{{Role: "user", Content: "x"}}})
if err != nil {
t.Fatalf("Complete() error = %v, want nil", err)
}
if resp.Text != "ok" {
t.Errorf("Text = %q, want %q", resp.Text, "ok")
}
if got := atomic.LoadInt32(&calls); got != 2 {
t.Errorf("calls = %d, want 2 (one error body, one success)", got)
}
}
func TestOpenAIClient_Complete_RetriesOn429ThenSucceeds(t *testing.T) {
var calls int32
srv := newTestServer(t, func(w http.ResponseWriter, r *http.Request) {
if atomic.AddInt32(&calls, 1) == 1 {
w.WriteHeader(http.StatusTooManyRequests)
return
}
_, _ = w.Write([]byte(`{"choices":[{"message":{"content":"ok"},"finish_reason":"stop"}],"usage":{"prompt_tokens":1,"completion_tokens":1}}`))
})
client := NewOpenAIClient(srv.URL, "", time.Second)
resp, err := client.Complete(context.Background(), Request{Model: "m", Messages: []Message{{Role: "user", Content: "x"}}})
if err != nil {
t.Fatalf("Complete() error = %v, want nil", err)
}
if resp.Text != "ok" {
t.Errorf("Text = %q, want %q", resp.Text, "ok")
}
if got := atomic.LoadInt32(&calls); got != 2 {
t.Errorf("calls = %d, want 2 (one 429, one success)", got)
}
}
func TestOpenAIClient_Complete_ContextCanceled(t *testing.T) {