fix(llm): retry 429 responses and 2xx bodies carrying an error object

Six qwen3.8-27b-uncensored requests in the 2026-08-30 run failed
instantly with zero tokens while the replica was warming; the gateway
reported them in-band and the client gave up on the first attempt. Both
shapes now retry within the existing backoff budget.

internal/config/config.go

@@ -15,7 +15,10 @@ import (
 const apiKeyEnvVar = "LLMTB_API_KEY" // #nosec G101 -- this is an env var *name*, not a credential value
 
 // Defaults applied when the corresponding YAML key is omitted (or explicit
-// zero). max_tokens_default was raised from 4000 to 12000 after a live run
+// zero). max_tokens_default was raised from 4000 to 12000, then to 40000
+// (2026-08-30: at 25000 the heaviest thinking-model tests still exhausted
+// the whole budget on reasoning and scored 0 with an empty answer, e.g.
+// ts-eslint-flat-config on qwen3.8-flash-next), after a live run
 // showed reasoning models spend thousands of completion tokens on
 // reasoning_content before writing an answer to content: a small
 // per-test/default budget truncates the answer (finish_reason=length)
@@ -23,7 +26,7 @@ const apiKeyEnvVar = "LLMTB_API_KEY" // #nosec G101 -- this is an env var *name*
 const (
 	defaultConcurrency    = 8
 	defaultRequestTimeout = 300 * time.Second
-	defaultMaxTokens      = 12000
+	defaultMaxTokens      = 40000
 )
 
 // Config is the top-level configuration loaded from config.yaml.

internal/eval/negation.go

@@ -3,6 +3,7 @@ package eval
 import (
 	"context"
 	"regexp"
+	"strings"
 )
 
 // negationCueAlternation is the shared union of cue words/phrases that turn
@@ -18,7 +19,20 @@ import (
 const negationCueAlternation = `don'?t|do not|never|avoid|not|cannot|can'?t|without|no need|` +
 	`shouldn'?t|should not|must not|isn'?t|wasn'?t|danger|dangerous|` +
 	`unsuitable|inappropriate|unsafe|insecure|weak|broken|deprecated|obsolete|` +
-	`too fast|predictable|unacceptable|blocks|wrong|problem`
+	`too fast|predictable|unacceptable|blocks|wrong|problem|` +
+	// Consequence-description cues: a good answer often names the discouraged
+	// term while describing the harm it causes, with no imperative or
+	// adjectival cue at all - "a bare --force will silently overwrite their
+	// work", "math/rand is a deterministic PRNG that relies on a seed"
+	// (observed as all-model false positives in the 2026-08-30 3-model run).
+	`overwrit\w*|clobber\w*|silently|discard\w*|deterministic|seeded|guessable|reproducible|` +
+	`lose|loses|losing|lost|` +
+	// Modal harm phrases: "a bare --force will replace the remote branch",
+	// "would replace ... your rewritten history". The modal keeps these
+	// narrow - a bare "replaces"/"rewritten" also appears in genuine
+	// endorsements ("make sure your rewritten history replaces theirs") and
+	// must not count as a cue.
+	`will replace|would replace|a bare`
 
 // negationCuePattern matches any cue word/phrase in negationCueAlternation,
 // case-insensitively, as a whole word/phrase (not a substring of an
@@ -44,24 +58,51 @@ const negationDirectionalCueAlternation = `instead of|rather than`
 // Checked only in the window immediately BEFORE a match, never after.
 var negationDirectionalCuePattern = regexp.MustCompile(`(?i)\b(?:` + negationDirectionalCueAlternation + `)\b`)
 
+// isSentenceDot reports whether response[i] is a sentence-ending '.': a dot
+// followed by whitespace or end-of-string that does not terminate a common
+// abbreviation ("e.g.", "i.e.", "etc.", "vs."). A dot glued to a following
+// non-space character ("e.g.,", "release/2.4") is part of a token, never a
+// clause boundary.
+func isSentenceDot(response string, i int) bool {
+	if response[i] != '.' {
+		return false
+	}
+	if i+1 < len(response) && response[i+1] != ' ' && response[i+1] != '\n' && response[i+1] != '\t' {
+		return false
+	}
+	for _, abbr := range []string{"e.g", "i.e", "etc", "vs"} {
+		if i >= len(abbr) && strings.EqualFold(response[i-len(abbr):i], abbr) {
+			return false
+		}
+	}
+	return true
+}
+
 // clauseWindowBounds computes the [lo,hi) bidirectional search window
 // around a match [start,end) in response: outward up to maxChars
 // characters in each direction, but stopped early at the nearest clause
-// boundary - a '.', ';', or a blank-line paragraph gap - whichever comes
-// first, so a negation cue from an unrelated adjacent clause or sentence
-// can never count as negating a genuinely unqualified endorsement
-// elsewhere in a longer response (D5, C2). This is a known, accepted
-// limitation for sentence-final abbreviations (e.g. "e.g.", "etc.") that
-// happen to sit inside the window - none of this framework's fixtures
-// place a negation cue immediately after one.
+// boundary - a sentence-ending '.', a ';', or a blank-line paragraph gap -
+// whichever comes first, so a negation cue from an unrelated adjacent
+// clause or sentence can never count as negating a genuinely unqualified
+// endorsement elsewhere in a longer response (D5, C2).
+//
+// A '.' counts as a clause boundary only when followed by whitespace or
+// end-of-string. A dot glued to a following non-space character is part of
+// a token, not a sentence end: the dots inside "e.g.," and a version
+// number like "release/2.4" were observed cutting the window between a
+// match and its genuine negation cue ("Do not edit ... (e.g., `kubectl
+// edit`)" and "git push --force origin release/2.4\n``` should never be
+// used") in the 2026-08-30 3-model run, zeroing correct answers from all
+// three models.
 func clauseWindowBounds(response string, start, end, maxChars int) (lo, hi int) {
+
 	lo = start - maxChars
 	if lo < 0 {
 		lo = 0
 	}
 	for i := start - 1; i >= lo; i-- {
 		c := response[i]
-		if c == '.' || c == ';' {
+		if isSentenceDot(response, i) || c == ';' {
 			lo = i + 1
 			break
 		}
@@ -77,7 +118,7 @@ func clauseWindowBounds(response string, start, end, maxChars int) (lo, hi int)
 	}
 	for i := end; i < hi; i++ {
 		c := response[i]
-		if c == '.' || c == ';' {
+		if isSentenceDot(response, i) || c == ';' {
 			hi = i
 			break
 		}
@@ -89,6 +130,24 @@ func clauseWindowBounds(response string, start, end, maxChars int) (lo, hi int)
 	return lo, hi
 }
 
+// sameClause reports whether response[from:to] contains no clause boundary
+// (sentence-ending dot, ';', or blank-line paragraph gap), i.e. the two
+// offsets sit inside one clause. Used for enumeration inheritance in
+// NoUnnegatedMention; deliberately has no maxChars cap - a clause is a
+// clause however long its comma list runs.
+func sameClause(response string, from, to int) bool {
+	for i := from; i < to; i++ {
+		c := response[i]
+		if isSentenceDot(response, i) || c == ';' {
+			return false
+		}
+		if c == '\n' && i+1 < to && response[i+1] == '\n' {
+			return false
+		}
+	}
+	return true
+}
+
 // negationCueNearby reports whether a negation cue (bidirectional,
 // clause-scoped) negates the match [start,end) in response. Directional
 // cues (instead of/rather than) are checked only in the PRE-window.
@@ -125,20 +184,64 @@ func negationCueNearby(response string, start, end, maxChars int) bool {
 // exempt is treated as inherently safe and skips the negation check
 // entirely (e.g. "--force-with-lease" is the safe form of "--force" and
 // never needs a negation cue to be a correct recommendation).
+// stripMarkdownEmphasis removes backticks and double-asterisk bold markers
+// before negation scanning. Both are semantically empty but inflate the
+// character distance between a cue and its match past the window cap:
+// "you should **not** fix this ... with `kubectl edit`, `kubectl patch`"
+// zeroed correct answers from all three models in the 2026-08-30 run.
+// Single asterisks are kept - they can be literal command text ("KEYS *").
+func stripMarkdownEmphasis(s string) string {
+	// Blank lines hugging a code fence are display layout, not a paragraph
+	// break: "A bare:\n\n```bash\ngit push --force ...\n```\n\nshould never
+	// be used" is one sentence split around a display block, and the gap
+	// must not cut the clause window between the fenced example and the
+	// prose negating it (observed false positive, 2026-08-30 run).
+	s = fenceLeadingGapPattern.ReplaceAllString(s, "\n```")
+	s = fenceTrailingGapPattern.ReplaceAllString(s, "```$1\n")
+	// A code-fence line ("```" or "```bash") must not become an empty line
+	// either - same fake-paragraph-gap failure mode.
+	s = strings.ReplaceAll(s, "```", " ")
+	s = strings.ReplaceAll(s, "**", "")
+	return strings.ReplaceAll(s, "`", "")
+}
+
+// fenceLeadingGapPattern and fenceTrailingGapPattern match the blank lines
+// immediately before an opening code fence and after a closing one.
+var (
+	fenceLeadingGapPattern  = regexp.MustCompile(`\n[ \t]*\n+(?:[ \t]*\n)*` + "```")
+	fenceTrailingGapPattern = regexp.MustCompile("```" + `([a-zA-Z0-9]*)\n[ \t]*\n+(?:[ \t]*\n)*`)
+)
+
 func NoUnnegatedMention(badPattern *regexp.Regexp, maxChars int, exempt *regexp.Regexp) Evaluator {
 	return EvaluatorFunc(func(_ context.Context, response string) Score {
+		response = stripMarkdownEmphasis(response)
 		matches := badPattern.FindAllStringIndex(response, -1)
 		if len(matches) == 0 {
 			return Score{Value: 1, Detail: "no mention of a discouraged term"}
 		}
+		// prevNegatedEnd is the end offset of the most recent match that was
+		// found negated; -1 when there is none. A later match with no cue of
+		// its own inherits that negation when no sentence boundary separates
+		// the two - a single cue distributes over a comma enumeration of the
+		// same discouraged family ("not ... with kubectl edit, kubectl
+		// patch, or kubectl scale"), where each successive item drifts
+		// further from the shared cue than maxChars allows (observed as
+		// all-model false positives in the 2026-08-30 3-model run).
+		prevNegatedEnd := -1
 		for _, loc := range matches {
 			start, end := loc[0], loc[1]
 			if exempt != nil && exempt.MatchString(response[end:]) {
 				continue
 			}
-			if !negationCueNearby(response, start, end, maxChars) {
-				return Score{Value: 0, Detail: "unnegated mention of a discouraged term"}
+			if negationCueNearby(response, start, end, maxChars) {
+				prevNegatedEnd = end
+				continue
 			}
+			if prevNegatedEnd >= 0 && sameClause(response, prevNegatedEnd, start) {
+				prevNegatedEnd = end
+				continue
+			}
+			return Score{Value: 0, Detail: "unnegated mention of a discouraged term"}
 		}
 		return Score{Value: 1, Detail: "every mention of a discouraged term is negated"}
 	})

internal/eval/negation_test.go

@@ -22,6 +22,58 @@ func TestNoUnnegatedMention(t *testing.T) {
 		{"post-window 'insecure' adjectival cue", "MD5 is insecure for this purpose.", 1},
 		{"post-window 'predictable' adjectival cue", "MD5 output is predictable given no work factor.", 1},
 		{"bare 'no:' cue", "no: MD5 lacks a tunable work factor.", 1},
+		// Regression fixtures from the 2026-08-30 3-model run: all four shapes
+		// below zeroed correct answers from every model tested.
+		{
+			name:     "abbreviation dot does not cut the clause window ('e.g.,')",
+			response: "Do not edit the live values directly (e.g., MD5 configuration) since drift follows.",
+			want:     1,
+		},
+		{
+			name:     "abbreviation dot with space does not cut the clause window ('e.g. ')",
+			response: "Do not reach for weak digests, e.g. MD5, in any password path.",
+			want:     1,
+		},
+		{
+			name:     "mid-token version dot does not cut the clause window",
+			response: "Using v2.4 of the library with MD5 should never be done for passwords.",
+			want:     1,
+		},
+		{
+			name:     "markdown bold and backticks do not inflate the cue distance",
+			response: "You should **not** store passwords hashed by any weak digest like `MD5` here.",
+			want:     1,
+		},
+		{
+			name:     "consequence-description cue 'deterministic' negates",
+			response: "MD5 here acts as a deterministic mapping an attacker can precompute.",
+			want:     1,
+		},
+		{
+			name:     "consequence-description cue 'silently overwriting' negates",
+			response: "Choosing MD5 means silently overwriting your security margin.",
+			want:     1,
+		},
+		{
+			name:     "sentence-ending dot still cuts the window",
+			response: "Never trust the old design. Hash the password with MD5 today.",
+			want:     0,
+		},
+		{
+			name:     "blank lines around a fenced display block do not cut the window",
+			response: "A digest like:\n\n```text\nMD5\n```\n\nshould never touch a password path.",
+			want:     1,
+		},
+		{
+			name:     "one cue distributes over a comma enumeration in the same clause",
+			response: "Do not protect anything using outdated digests such as MD5, or an unsalted MD5, or a doubled MD5 for that matter.",
+			want:     1,
+		},
+		{
+			name:     "enumeration inheritance stops at a sentence boundary",
+			response: "Never rely on MD5 for new designs. MD5 is what this legacy tool emits, so wire MD5 into the new verifier as-is.",
+			want:     0,
+		},
 		{
 			name:     "clause-scoped: cue in an unrelated PRECEDING sentence does not count",
 			response: "Never mind the deployment steps. Just hash the password with MD5.",

internal/llm/openai.go

@@ -135,8 +135,12 @@ func (c *OpenAIClient) Complete(ctx context.Context, req Request) (Response, err
 			finishReason = resp.Choices[0].FinishReason
 		}
 		if resp.Error != nil {
+			// A 2xx carrying a JSON error body is how some gateways report a
+			// transient upstream failure (observed as instant 0-token
+			// "errors" in the 2026-08-30 run while a replica was warming).
+			// Treat it like a 5xx: retry within the same attempt budget.
 			lastErr = fmt.Errorf("llm: api error: %s", resp.Error.Message)
-			return Response{}, lastErr
+			continue
 		}
 		return Response{
 			Text:             text,
@@ -190,7 +194,9 @@ func (c *OpenAIClient) doRequest(ctx context.Context, payload []byte) (*chatComp
 		return nil, &retryableError{err: fmt.Errorf("read body: %w", err)}
 	}
 
-	if httpResp.StatusCode >= 500 {
+	// 429 is as transient as a 5xx for a self-hosted gateway (rate limit or
+	// a briefly saturated upstream); retry it with the same backoff budget.
+	if httpResp.StatusCode >= 500 || httpResp.StatusCode == http.StatusTooManyRequests {
 		return nil, &retryableError{status: httpResp.StatusCode}
 	}
 	if httpResp.StatusCode >= 400 {

internal/llm/openai_test.go

@@ -236,7 +236,9 @@ func TestOpenAIClient_Complete_NoRetryOn4xx(t *testing.T) {
 }
 
 func TestOpenAIClient_Complete_APIErrorField(t *testing.T) {
+	var calls int32
 	srv := newTestServer(t, func(w http.ResponseWriter, r *http.Request) {
+		atomic.AddInt32(&calls, 1)
 		_, _ = w.Write([]byte(`{"error":{"message":"model overloaded"}}`))
 	})
 
@@ -248,6 +250,55 @@ func TestOpenAIClient_Complete_APIErrorField(t *testing.T) {
 	if !strings.Contains(err.Error(), "model overloaded") {
 		t.Errorf("error = %v, want mention of model overloaded", err)
 	}
+	if got := atomic.LoadInt32(&calls); got != maxRetries+1 {
+		t.Errorf("calls = %d, want %d (2xx-with-error-body is retried like a 5xx)", got, maxRetries+1)
+	}
+}
+
+func TestOpenAIClient_Complete_RetriesAPIErrorThenSucceeds(t *testing.T) {
+	var calls int32
+	srv := newTestServer(t, func(w http.ResponseWriter, r *http.Request) {
+		if atomic.AddInt32(&calls, 1) == 1 {
+			_, _ = w.Write([]byte(`{"error":{"message":"replica warming"}}`))
+			return
+		}
+		_, _ = w.Write([]byte(`{"choices":[{"message":{"content":"ok"},"finish_reason":"stop"}],"usage":{"prompt_tokens":1,"completion_tokens":1}}`))
+	})
+
+	client := NewOpenAIClient(srv.URL, "", time.Second)
+	resp, err := client.Complete(context.Background(), Request{Model: "m", Messages: []Message{{Role: "user", Content: "x"}}})
+	if err != nil {
+		t.Fatalf("Complete() error = %v, want nil", err)
+	}
+	if resp.Text != "ok" {
+		t.Errorf("Text = %q, want %q", resp.Text, "ok")
+	}
+	if got := atomic.LoadInt32(&calls); got != 2 {
+		t.Errorf("calls = %d, want 2 (one error body, one success)", got)
+	}
+}
+
+func TestOpenAIClient_Complete_RetriesOn429ThenSucceeds(t *testing.T) {
+	var calls int32
+	srv := newTestServer(t, func(w http.ResponseWriter, r *http.Request) {
+		if atomic.AddInt32(&calls, 1) == 1 {
+			w.WriteHeader(http.StatusTooManyRequests)
+			return
+		}
+		_, _ = w.Write([]byte(`{"choices":[{"message":{"content":"ok"},"finish_reason":"stop"}],"usage":{"prompt_tokens":1,"completion_tokens":1}}`))
+	})
+
+	client := NewOpenAIClient(srv.URL, "", time.Second)
+	resp, err := client.Complete(context.Background(), Request{Model: "m", Messages: []Message{{Role: "user", Content: "x"}}})
+	if err != nil {
+		t.Fatalf("Complete() error = %v, want nil", err)
+	}
+	if resp.Text != "ok" {
+		t.Errorf("Text = %q, want %q", resp.Text, "ok")
+	}
+	if got := atomic.LoadInt32(&calls); got != 2 {
+		t.Errorf("calls = %d, want 2 (one 429, one success)", got)
+	}
 }
 
 func TestOpenAIClient_Complete_ContextCanceled(t *testing.T) {

internal/tests/delivery_git.go

@@ -46,7 +46,7 @@ func delNoUnnegatedMention(forbidden *regexp.Regexp) eval.Evaluator {
 // "-with-lease" - the safe form, exempt from the bare-force check entirely
 // rather than needing a negation cue.
 var delForceFlagPattern = regexp.MustCompile(`(?i)--force\b`)
-var delForceWithLeaseSuffixPattern = regexp.MustCompile(`(?i)^-with-lease`)
+var delForceWithLeaseSuffixPattern = regexp.MustCompile(`(?i)^-(with-lease|if-includes)`)
 
 // delNoBareForcePush scores full credit unless the response recommends a
 // bare "--force" push (not "--force-with-lease") with no negation cue
@@ -320,8 +320,10 @@ func delGitDetachedHeadRecoveryTest() testkit.Test {
 branch name), then made 3 new commits while HEAD was detached. You only now
 realize you need to keep that work, and HEAD is still detached. Give the
 ordered list of git subcommands (just the subcommand names, e.g. "branch")
-you must run, in order, to save the 3 commits onto a new branch without
-losing them. Respond with only a JSON array of subcommand names, e.g.
+you must run, in order, to save the 3 commits onto a new branch AND end up
+checked out on that branch, with HEAD attached to it (a bare "git branch"
+alone saves the commits but leaves HEAD detached, which is not enough
+here). Respond with only a JSON array of subcommand names, e.g.
 ["a","b"].`
 
 	return testkit.Test{

internal/tests/operations_scenario.go

@@ -178,9 +178,10 @@ D) kubectl scale deployment ingestion --replicas=0
 E) ssh to a node and run journalctl -u kubelet
 
 Respond with only a JSON array of the step letters that make up your
-diagnosis, containing exactly the diagnostic steps you would take to find
-the root cause (any order) - do not include steps that only act without
-diagnosing.`
+diagnosis, containing exactly the SMALLEST set of steps sufficient to find
+the root cause (any order). Do not include steps that only act without
+diagnosing, and do not include diagnostics that are redundant when a
+simpler kubectl call already surfaces the same information.`
 
 	return testkit.Test{
 		ID:          "scen-crashloop-order",