feat(tests): add 16 advanced distractor-rich scenarios

The 2026-08-30 suite-health review found 223/283 tests saturated. These
16 target the failure modes that still discriminated, one per existing
subcategory across all eight categories:

- multi-constraint reasoning: infeasible-plan detection, conflicting
  delegation policy, adversarial tool descriptions
- forensic root-causing from noisy evidence: RAG stage attribution,
  EXPLAIN ANALYZE with a red herring, interleaved multi-source incident
  logs, README-vs-config conflict resolution
- composed gotchas: a Go 1.22 capture + named-return-defer trace, a
  Python late-binding + shared-mutable-default trace
- constraint math: context-budget packing, capacity-fill projection,
  MVCC anomaly classification, Docker layer-rebuild prediction, dual-key
  rotation overlap, freeze-window scheduling, an open-redirect exploit
  chain

Every answer is a structured JSON value scored by an exact evaluator,
never a prose heuristic (the source of every false positive in the
review). A ground-truth self-consistency test asserts each documented
correct answer scores 1.0 through its real evaluator.

config.example.yaml

@@ -5,7 +5,7 @@ models:
   - uni/qwen3.6-27b
   - uni/btl-4
 concurrency: 8         # default when omitted; matches internal/config's default
-request_timeout: 300s  # default when omitted
+request_timeout: 900s  # default when omitted
 max_tokens_default: 12000  # default when omitted; a reasoning model can spend
                             # thousands of tokens on reasoning_content before
                             # writing an answer, so this must stay generous

internal/config/config.go

@@ -15,7 +15,10 @@ import (
 const apiKeyEnvVar = "LLMTB_API_KEY" // #nosec G101 -- this is an env var *name*, not a credential value
 
 // Defaults applied when the corresponding YAML key is omitted (or explicit
-// zero). max_tokens_default was raised from 4000 to 12000 after a live run
+// zero). max_tokens_default was raised from 4000 to 12000, then to 40000
+// (2026-08-30: at 25000 the heaviest thinking-model tests still exhausted
+// the whole budget on reasoning and scored 0 with an empty answer, e.g.
+// ts-eslint-flat-config on qwen3.8-flash-next), after a live run
 // showed reasoning models spend thousands of completion tokens on
 // reasoning_content before writing an answer to content: a small
 // per-test/default budget truncates the answer (finish_reason=length)
@@ -23,7 +26,7 @@ const apiKeyEnvVar = "LLMTB_API_KEY" // #nosec G101 -- this is an env var *name*
 const (
 	defaultConcurrency    = 8
 	defaultRequestTimeout = 300 * time.Second
-	defaultMaxTokens      = 12000
+	defaultMaxTokens      = 40000
 )
 
 // Config is the top-level configuration loaded from config.yaml.

internal/eval/negation.go

@@ -3,6 +3,7 @@ package eval
 import (
 	"context"
 	"regexp"
+	"strings"
 )
 
 // negationCueAlternation is the shared union of cue words/phrases that turn
@@ -18,7 +19,20 @@ import (
 const negationCueAlternation = `don'?t|do not|never|avoid|not|cannot|can'?t|without|no need|` +
 	`shouldn'?t|should not|must not|isn'?t|wasn'?t|danger|dangerous|` +
 	`unsuitable|inappropriate|unsafe|insecure|weak|broken|deprecated|obsolete|` +
-	`too fast|predictable|unacceptable|blocks|wrong|problem`
+	`too fast|predictable|unacceptable|blocks|wrong|problem|` +
+	// Consequence-description cues: a good answer often names the discouraged
+	// term while describing the harm it causes, with no imperative or
+	// adjectival cue at all - "a bare --force will silently overwrite their
+	// work", "math/rand is a deterministic PRNG that relies on a seed"
+	// (observed as all-model false positives in the 2026-08-30 3-model run).
+	`overwrit\w*|clobber\w*|silently|discard\w*|deterministic|seeded|guessable|reproducible|` +
+	`lose|loses|losing|lost|` +
+	// Modal harm phrases: "a bare --force will replace the remote branch",
+	// "would replace ... your rewritten history". The modal keeps these
+	// narrow - a bare "replaces"/"rewritten" also appears in genuine
+	// endorsements ("make sure your rewritten history replaces theirs") and
+	// must not count as a cue.
+	`will replace|would replace|a bare`
 
 // negationCuePattern matches any cue word/phrase in negationCueAlternation,
 // case-insensitively, as a whole word/phrase (not a substring of an
@@ -44,24 +58,51 @@ const negationDirectionalCueAlternation = `instead of|rather than`
 // Checked only in the window immediately BEFORE a match, never after.
 var negationDirectionalCuePattern = regexp.MustCompile(`(?i)\b(?:` + negationDirectionalCueAlternation + `)\b`)
 
+// isSentenceDot reports whether response[i] is a sentence-ending '.': a dot
+// followed by whitespace or end-of-string that does not terminate a common
+// abbreviation ("e.g.", "i.e.", "etc.", "vs."). A dot glued to a following
+// non-space character ("e.g.,", "release/2.4") is part of a token, never a
+// clause boundary.
+func isSentenceDot(response string, i int) bool {
+	if response[i] != '.' {
+		return false
+	}
+	if i+1 < len(response) && response[i+1] != ' ' && response[i+1] != '\n' && response[i+1] != '\t' {
+		return false
+	}
+	for _, abbr := range []string{"e.g", "i.e", "etc", "vs"} {
+		if i >= len(abbr) && strings.EqualFold(response[i-len(abbr):i], abbr) {
+			return false
+		}
+	}
+	return true
+}
+
 // clauseWindowBounds computes the [lo,hi) bidirectional search window
 // around a match [start,end) in response: outward up to maxChars
 // characters in each direction, but stopped early at the nearest clause
-// boundary - a '.', ';', or a blank-line paragraph gap - whichever comes
-// first, so a negation cue from an unrelated adjacent clause or sentence
-// can never count as negating a genuinely unqualified endorsement
-// elsewhere in a longer response (D5, C2). This is a known, accepted
-// limitation for sentence-final abbreviations (e.g. "e.g.", "etc.") that
-// happen to sit inside the window - none of this framework's fixtures
-// place a negation cue immediately after one.
+// boundary - a sentence-ending '.', a ';', or a blank-line paragraph gap -
+// whichever comes first, so a negation cue from an unrelated adjacent
+// clause or sentence can never count as negating a genuinely unqualified
+// endorsement elsewhere in a longer response (D5, C2).
+//
+// A '.' counts as a clause boundary only when followed by whitespace or
+// end-of-string. A dot glued to a following non-space character is part of
+// a token, not a sentence end: the dots inside "e.g.," and a version
+// number like "release/2.4" were observed cutting the window between a
+// match and its genuine negation cue ("Do not edit ... (e.g., `kubectl
+// edit`)" and "git push --force origin release/2.4\n``` should never be
+// used") in the 2026-08-30 3-model run, zeroing correct answers from all
+// three models.
 func clauseWindowBounds(response string, start, end, maxChars int) (lo, hi int) {
+
 	lo = start - maxChars
 	if lo < 0 {
 		lo = 0
 	}
 	for i := start - 1; i >= lo; i-- {
 		c := response[i]
-		if c == '.' || c == ';' {
+		if isSentenceDot(response, i) || c == ';' {
 			lo = i + 1
 			break
 		}
@@ -77,7 +118,7 @@ func clauseWindowBounds(response string, start, end, maxChars int) (lo, hi int)
 	}
 	for i := end; i < hi; i++ {
 		c := response[i]
-		if c == '.' || c == ';' {
+		if isSentenceDot(response, i) || c == ';' {
 			hi = i
 			break
 		}
@@ -89,6 +130,24 @@ func clauseWindowBounds(response string, start, end, maxChars int) (lo, hi int)
 	return lo, hi
 }
 
+// sameClause reports whether response[from:to] contains no clause boundary
+// (sentence-ending dot, ';', or blank-line paragraph gap), i.e. the two
+// offsets sit inside one clause. Used for enumeration inheritance in
+// NoUnnegatedMention; deliberately has no maxChars cap - a clause is a
+// clause however long its comma list runs.
+func sameClause(response string, from, to int) bool {
+	for i := from; i < to; i++ {
+		c := response[i]
+		if isSentenceDot(response, i) || c == ';' {
+			return false
+		}
+		if c == '\n' && i+1 < to && response[i+1] == '\n' {
+			return false
+		}
+	}
+	return true
+}
+
 // negationCueNearby reports whether a negation cue (bidirectional,
 // clause-scoped) negates the match [start,end) in response. Directional
 // cues (instead of/rather than) are checked only in the PRE-window.
@@ -125,20 +184,64 @@ func negationCueNearby(response string, start, end, maxChars int) bool {
 // exempt is treated as inherently safe and skips the negation check
 // entirely (e.g. "--force-with-lease" is the safe form of "--force" and
 // never needs a negation cue to be a correct recommendation).
+// stripMarkdownEmphasis removes backticks and double-asterisk bold markers
+// before negation scanning. Both are semantically empty but inflate the
+// character distance between a cue and its match past the window cap:
+// "you should **not** fix this ... with `kubectl edit`, `kubectl patch`"
+// zeroed correct answers from all three models in the 2026-08-30 run.
+// Single asterisks are kept - they can be literal command text ("KEYS *").
+func stripMarkdownEmphasis(s string) string {
+	// Blank lines hugging a code fence are display layout, not a paragraph
+	// break: "A bare:\n\n```bash\ngit push --force ...\n```\n\nshould never
+	// be used" is one sentence split around a display block, and the gap
+	// must not cut the clause window between the fenced example and the
+	// prose negating it (observed false positive, 2026-08-30 run).
+	s = fenceLeadingGapPattern.ReplaceAllString(s, "\n```")
+	s = fenceTrailingGapPattern.ReplaceAllString(s, "```$1\n")
+	// A code-fence line ("```" or "```bash") must not become an empty line
+	// either - same fake-paragraph-gap failure mode.
+	s = strings.ReplaceAll(s, "```", " ")
+	s = strings.ReplaceAll(s, "**", "")
+	return strings.ReplaceAll(s, "`", "")
+}
+
+// fenceLeadingGapPattern and fenceTrailingGapPattern match the blank lines
+// immediately before an opening code fence and after a closing one.
+var (
+	fenceLeadingGapPattern  = regexp.MustCompile(`\n[ \t]*\n+(?:[ \t]*\n)*` + "```")
+	fenceTrailingGapPattern = regexp.MustCompile("```" + `([a-zA-Z0-9]*)\n[ \t]*\n+(?:[ \t]*\n)*`)
+)
+
 func NoUnnegatedMention(badPattern *regexp.Regexp, maxChars int, exempt *regexp.Regexp) Evaluator {
 	return EvaluatorFunc(func(_ context.Context, response string) Score {
+		response = stripMarkdownEmphasis(response)
 		matches := badPattern.FindAllStringIndex(response, -1)
 		if len(matches) == 0 {
 			return Score{Value: 1, Detail: "no mention of a discouraged term"}
 		}
+		// prevNegatedEnd is the end offset of the most recent match that was
+		// found negated; -1 when there is none. A later match with no cue of
+		// its own inherits that negation when no sentence boundary separates
+		// the two - a single cue distributes over a comma enumeration of the
+		// same discouraged family ("not ... with kubectl edit, kubectl
+		// patch, or kubectl scale"), where each successive item drifts
+		// further from the shared cue than maxChars allows (observed as
+		// all-model false positives in the 2026-08-30 3-model run).
+		prevNegatedEnd := -1
 		for _, loc := range matches {
 			start, end := loc[0], loc[1]
 			if exempt != nil && exempt.MatchString(response[end:]) {
 				continue
 			}
-			if !negationCueNearby(response, start, end, maxChars) {
-				return Score{Value: 0, Detail: "unnegated mention of a discouraged term"}
+			if negationCueNearby(response, start, end, maxChars) {
+				prevNegatedEnd = end
+				continue
 			}
+			if prevNegatedEnd >= 0 && sameClause(response, prevNegatedEnd, start) {
+				prevNegatedEnd = end
+				continue
+			}
+			return Score{Value: 0, Detail: "unnegated mention of a discouraged term"}
 		}
 		return Score{Value: 1, Detail: "every mention of a discouraged term is negated"}
 	})

internal/eval/negation_test.go

@@ -22,6 +22,58 @@ func TestNoUnnegatedMention(t *testing.T) {
 		{"post-window 'insecure' adjectival cue", "MD5 is insecure for this purpose.", 1},
 		{"post-window 'predictable' adjectival cue", "MD5 output is predictable given no work factor.", 1},
 		{"bare 'no:' cue", "no: MD5 lacks a tunable work factor.", 1},
+		// Regression fixtures from the 2026-08-30 3-model run: all four shapes
+		// below zeroed correct answers from every model tested.
+		{
+			name:     "abbreviation dot does not cut the clause window ('e.g.,')",
+			response: "Do not edit the live values directly (e.g., MD5 configuration) since drift follows.",
+			want:     1,
+		},
+		{
+			name:     "abbreviation dot with space does not cut the clause window ('e.g. ')",
+			response: "Do not reach for weak digests, e.g. MD5, in any password path.",
+			want:     1,
+		},
+		{
+			name:     "mid-token version dot does not cut the clause window",
+			response: "Using v2.4 of the library with MD5 should never be done for passwords.",
+			want:     1,
+		},
+		{
+			name:     "markdown bold and backticks do not inflate the cue distance",
+			response: "You should **not** store passwords hashed by any weak digest like `MD5` here.",
+			want:     1,
+		},
+		{
+			name:     "consequence-description cue 'deterministic' negates",
+			response: "MD5 here acts as a deterministic mapping an attacker can precompute.",
+			want:     1,
+		},
+		{
+			name:     "consequence-description cue 'silently overwriting' negates",
+			response: "Choosing MD5 means silently overwriting your security margin.",
+			want:     1,
+		},
+		{
+			name:     "sentence-ending dot still cuts the window",
+			response: "Never trust the old design. Hash the password with MD5 today.",
+			want:     0,
+		},
+		{
+			name:     "blank lines around a fenced display block do not cut the window",
+			response: "A digest like:\n\n```text\nMD5\n```\n\nshould never touch a password path.",
+			want:     1,
+		},
+		{
+			name:     "one cue distributes over a comma enumeration in the same clause",
+			response: "Do not protect anything using outdated digests such as MD5, or an unsalted MD5, or a doubled MD5 for that matter.",
+			want:     1,
+		},
+		{
+			name:     "enumeration inheritance stops at a sentence boundary",
+			response: "Never rely on MD5 for new designs. MD5 is what this legacy tool emits, so wire MD5 into the new verifier as-is.",
+			want:     0,
+		},
 		{
 			name:     "clause-scoped: cue in an unrelated PRECEDING sentence does not count",
 			response: "Never mind the deployment steps. Just hash the password with MD5.",

internal/llm/openai.go

@@ -135,8 +135,12 @@ func (c *OpenAIClient) Complete(ctx context.Context, req Request) (Response, err
 			finishReason = resp.Choices[0].FinishReason
 		}
 		if resp.Error != nil {
+			// A 2xx carrying a JSON error body is how some gateways report a
+			// transient upstream failure (observed as instant 0-token
+			// "errors" in the 2026-08-30 run while a replica was warming).
+			// Treat it like a 5xx: retry within the same attempt budget.
 			lastErr = fmt.Errorf("llm: api error: %s", resp.Error.Message)
-			return Response{}, lastErr
+			continue
 		}
 		return Response{
 			Text:             text,
@@ -190,7 +194,9 @@ func (c *OpenAIClient) doRequest(ctx context.Context, payload []byte) (*chatComp
 		return nil, &retryableError{err: fmt.Errorf("read body: %w", err)}
 	}
 
-	if httpResp.StatusCode >= 500 {
+	// 429 is as transient as a 5xx for a self-hosted gateway (rate limit or
+	// a briefly saturated upstream); retry it with the same backoff budget.
+	if httpResp.StatusCode >= 500 || httpResp.StatusCode == http.StatusTooManyRequests {
 		return nil, &retryableError{status: httpResp.StatusCode}
 	}
 	if httpResp.StatusCode >= 400 {

internal/llm/openai_test.go

@@ -236,7 +236,9 @@ func TestOpenAIClient_Complete_NoRetryOn4xx(t *testing.T) {
 }
 
 func TestOpenAIClient_Complete_APIErrorField(t *testing.T) {
+	var calls int32
 	srv := newTestServer(t, func(w http.ResponseWriter, r *http.Request) {
+		atomic.AddInt32(&calls, 1)
 		_, _ = w.Write([]byte(`{"error":{"message":"model overloaded"}}`))
 	})
 
@@ -248,6 +250,55 @@ func TestOpenAIClient_Complete_APIErrorField(t *testing.T) {
 	if !strings.Contains(err.Error(), "model overloaded") {
 		t.Errorf("error = %v, want mention of model overloaded", err)
 	}
+	if got := atomic.LoadInt32(&calls); got != maxRetries+1 {
+		t.Errorf("calls = %d, want %d (2xx-with-error-body is retried like a 5xx)", got, maxRetries+1)
+	}
+}
+
+func TestOpenAIClient_Complete_RetriesAPIErrorThenSucceeds(t *testing.T) {
+	var calls int32
+	srv := newTestServer(t, func(w http.ResponseWriter, r *http.Request) {
+		if atomic.AddInt32(&calls, 1) == 1 {
+			_, _ = w.Write([]byte(`{"error":{"message":"replica warming"}}`))
+			return
+		}
+		_, _ = w.Write([]byte(`{"choices":[{"message":{"content":"ok"},"finish_reason":"stop"}],"usage":{"prompt_tokens":1,"completion_tokens":1}}`))
+	})
+
+	client := NewOpenAIClient(srv.URL, "", time.Second)
+	resp, err := client.Complete(context.Background(), Request{Model: "m", Messages: []Message{{Role: "user", Content: "x"}}})
+	if err != nil {
+		t.Fatalf("Complete() error = %v, want nil", err)
+	}
+	if resp.Text != "ok" {
+		t.Errorf("Text = %q, want %q", resp.Text, "ok")
+	}
+	if got := atomic.LoadInt32(&calls); got != 2 {
+		t.Errorf("calls = %d, want 2 (one error body, one success)", got)
+	}
+}
+
+func TestOpenAIClient_Complete_RetriesOn429ThenSucceeds(t *testing.T) {
+	var calls int32
+	srv := newTestServer(t, func(w http.ResponseWriter, r *http.Request) {
+		if atomic.AddInt32(&calls, 1) == 1 {
+			w.WriteHeader(http.StatusTooManyRequests)
+			return
+		}
+		_, _ = w.Write([]byte(`{"choices":[{"message":{"content":"ok"},"finish_reason":"stop"}],"usage":{"prompt_tokens":1,"completion_tokens":1}}`))
+	})
+
+	client := NewOpenAIClient(srv.URL, "", time.Second)
+	resp, err := client.Complete(context.Background(), Request{Model: "m", Messages: []Message{{Role: "user", Content: "x"}}})
+	if err != nil {
+		t.Fatalf("Complete() error = %v, want nil", err)
+	}
+	if resp.Text != "ok" {
+		t.Errorf("Text = %q, want %q", resp.Text, "ok")
+	}
+	if got := atomic.LoadInt32(&calls); got != 2 {
+		t.Errorf("calls = %d, want 2 (one 429, one success)", got)
+	}
 }
 
 func TestOpenAIClient_Complete_ContextCanceled(t *testing.T) {

internal/runner/runner.go

@@ -161,9 +161,14 @@ func (r *Runner) runOne(ctx context.Context, model string, test testkit.Test) Re
 		Temperature: r.cfg.Temperature,
 	}
 
+	// Wall-clock across all attempts, recorded even on error: an error row
+	// with zero latency reads as an instant rejection when it may be three
+	// exhausted timeouts (a 651-second generation looked "instant" in the
+	// 2026-08-30 run and misdirected the diagnosis).
+	callStart := time.Now()
 	resp, err := r.client.Complete(ctx, req)
 	if err != nil {
-		return Result{Model: model, TestID: test.ID, Err: err}
+		return Result{Model: model, TestID: test.ID, Err: err, Latency: time.Since(callStart)}
 	}
 
 	normalized := testkit.Normalize(resp.Text)

internal/tests/advanced.go

@@ -0,0 +1,534 @@
+package tests
+
+import (
+	"github.com/lukaszraczylo/llm-testbench/internal/eval"
+	"github.com/lukaszraczylo/llm-testbench/internal/testkit"
+)
+
+// registerAdvancedTests adds the harder, distractor-rich scenarios designed
+// after the 2026-08-30 suite-health review, which found 223/283 tests
+// saturated (every model passes). Each test here targets a failure mode
+// that still discriminated: multi-constraint reasoning, semantic tracing of
+// composed gotchas, forensic root-causing from noisy evidence, and
+// constraint math. Every answer is a structured JSON value scored by an
+// exact evaluator, never a prose-matching heuristic (the source of every
+// false positive in that review). Each lands in an EXISTING subcategory so
+// the catalog shape (one pair per category/subcategory) is unchanged.
+func registerAdvancedTests(r *testkit.Registry) {
+	r.Register(advAgentPlanInfeasibleTest())
+	r.Register(advAgentDelegConflictingPolicyTest())
+	r.Register(advAgentToolAdversarialDescTest())
+	r.Register(advRAGForensicsStageTest())
+	r.Register(advLLMContextBudgetTest())
+	r.Register(advPgExplainForensicsTest())
+	r.Register(advSQLMVCCAnomalyTest())
+	r.Register(advDockerLayerRebuildTest())
+	r.Register(advRelFreezeWindowTest())
+	r.Register(advScenMultiLogRootcauseTest())
+	r.Register(advLinuxCapacityFillTest())
+	r.Register(advHardComposedTraceTest())
+	r.Register(advPyComposedMutableTest())
+	r.Register(advCodeConflictingEvidenceTest())
+	r.Register(advSecExploitChainTest())
+	r.Register(advSecRotationBlastRadiusTest())
+}
+
+// advAgentPlanInfeasibleTest: a plan that cannot succeed because a later
+// step needs an artifact no earlier step produces. The discriminator is
+// admitting infeasibility instead of confabulating an order.
+//
+// ground truth: step "deploy" needs the built image; the only build step
+// ("build") needs the compiled binary from "compile"; "compile" needs
+// dependencies from "fetch-deps"; but "fetch-deps" is never listed as
+// runnable because it needs network egress the sandbox forbids (stated).
+// So the binding blocker is fetch-deps and the plan is infeasible.
+func advAgentPlanInfeasibleTest() testkit.Test {
+	prompt := `You must order these build steps into a runnable sequence:
+
+- compile: needs the fetched dependencies; produces the binary.
+- build-image: needs the binary; produces the container image.
+- deploy: needs the container image.
+- fetch-deps: needs outbound network egress to the package registry;
+  produces the fetched dependencies.
+
+Constraint: this runner has NO outbound network egress (air-gapped), and
+no dependency cache is present.
+
+Decide whether a runnable ordering exists. Respond with only a JSON object:
+{"feasible": true|false, "blocking_step": "<step id or empty string>"}`
+
+	return testkit.Test{
+		ID:          "adv-agent-plan-infeasible",
+		Category:    "agents",
+		Subcategory: "planning",
+		Description: "Detect that an air-gapped runner makes fetch-deps (and the whole chain) infeasible rather than confabulating an order.",
+		Prompt:      prompt,
+		Eval: eval.All(
+			eval.W(eval.JSONField("feasible", false), 2),
+			eval.W(eval.JSONField("blocking_step", "fetch-deps"), 1),
+		),
+	}
+}
+
+// advAgentDelegConflictingPolicyTest: pick the one assignment that satisfies
+// three simultaneous constraints; the distractors each violate exactly one.
+//
+// ground truth: the task needs GPU, must finish under a 2h deadline, and
+// must run in-region (data residency). agent-cpu-eu has no GPU (violates
+// capability); agent-gpu-us is out of region (violates residency);
+// agent-gpu-eu-slow takes 3h (violates deadline); agent-gpu-eu-fast meets
+// all three. Answer: agent-gpu-eu-fast.
+func advAgentDelegConflictingPolicyTest() testkit.Test {
+	prompt := `Assign this task to exactly one agent. The task REQUIRES a GPU,
+must complete within a 2-hour deadline, and must run in the EU region
+(data residency).
+
+- agent-cpu-eu: EU region, no GPU, would finish in 1h.
+- agent-gpu-us: US region, has GPU, would finish in 30m.
+- agent-gpu-eu-slow: EU region, has GPU, would finish in 3h.
+- agent-gpu-eu-fast: EU region, has GPU, would finish in 90m.
+
+Respond with only a JSON object: {"agent": "<agent id>"}`
+
+	return testkit.Test{
+		ID:          "adv-agent-deleg-conflicting-policy",
+		Category:    "agents",
+		Subcategory: "delegation",
+		Description: "Choose the single agent satisfying GPU + deadline + region when each distractor violates exactly one constraint.",
+		Prompt:      prompt,
+		Eval:        eval.JSONField("agent", "agent-gpu-eu-fast"),
+	}
+}
+
+// advAgentToolAdversarialDescTest: the tool whose NAME suggests the task is
+// wrong; the correct tool is identified only by reading descriptions.
+//
+// ground truth: task is to permanently delete a file. "file_manager" sounds
+// right but its description only LISTS files (read-only). "trash_bin"
+// sounds like recycling but its description permanently removes a path.
+// Answer: trash_bin.
+func advAgentToolAdversarialDescTest() testkit.Test {
+	prompt := `You must permanently delete the file at /data/tmp/old.log.
+Pick the correct tool by its description, not its name:
+
+- file_manager: "Returns a listing of files under a directory. Read-only."
+- disk_report: "Reports free and used disk space per mount."
+- trash_bin: "Permanently removes the file or directory at the given path
+  from disk. Not recoverable."
+- archive_tool: "Compresses a path into a .tar.gz beside it."
+
+Respond with only a JSON object: {"tool": "<tool name>"}`
+
+	return testkit.Test{
+		ID:          "adv-agent-tool-adversarial-desc",
+		Category:    "agents",
+		Subcategory: "tool-routing",
+		Description: "Select the deletion tool by description when its name (trash_bin) misleads and file_manager sounds right but is read-only.",
+		Prompt:      prompt,
+		Eval:        eval.JSONField("tool", "trash_bin"),
+	}
+}
+
+// advRAGForensicsStageTest: given a retrieval trace and a wrong answer,
+// name the failing pipeline stage.
+//
+// ground truth: the embedding query retrieved the right documents (they
+// appear in the trace with high scores and contain the fact), the reranker
+// kept them top-3, but the generation cited a number found in NONE of the
+// retrieved chunks - a generation-stage hallucination, not a retrieval or
+// rerank failure. Answer: generation.
+func advRAGForensicsStageTest() testkit.Test {
+	prompt := `A RAG system answered "the timeout is 45 seconds" but the
+correct value is 30. Here is the trace:
+
+- retrieval: top-3 chunks returned, all from the config reference; chunk 1
+  (score 0.91) contains the sentence "request_timeout defaults to 30s".
+- rerank: order unchanged; chunk 1 still ranked first.
+- generation: the model produced "45 seconds"; the string "45" appears in
+  none of the three retrieved chunks.
+
+Which single pipeline stage introduced the error? Respond with only a JSON
+object: {"stage": "retrieval"|"rerank"|"generation"}`
+
+	return testkit.Test{
+		ID:          "adv-rag-forensics-stage",
+		Category:    "ai",
+		Subcategory: "rag",
+		Description: "Attribute a wrong RAG answer to the generation stage when retrieval and rerank both surfaced the correct grounded chunk.",
+		Prompt:      prompt,
+		Eval:        eval.JSONField("stage", "generation"),
+	}
+}
+
+// advLLMContextBudgetTest: how many retrieved documents fit given a fixed
+// context window and reservations.
+//
+// ground truth: 200000 window - 8000 output reserve - 1200 system - 300
+// query = 190500 available; each doc is 3500 tokens; floor(190500/3500) =
+// 54.
+func advLLMContextBudgetTest() testkit.Test {
+	prompt := `A model has a 200000-token context window. You must reserve
+8000 tokens for the output, 1200 for the system prompt, and 300 for the
+user query. Each retrieved document is 3500 tokens. How many whole
+documents can you fit into the remaining context? Respond with only the
+integer.`
+
+	return testkit.Test{
+		ID:          "adv-llm-context-budget",
+		Category:    "ai",
+		Subcategory: "llm-integration",
+		Description: "Compute the document count that fits a 200K window after output/system/query reservations (floor(190500/3500)=54).",
+		Prompt:      prompt,
+		Eval:        eval.Numeric(eval.ExtractLastNumber[int], 54, 0),
+	}
+}
+
+// advPgExplainForensicsTest: read an EXPLAIN ANALYZE plan with a red herring
+// and name the real problem.
+//
+// ground truth: the plan shows a Seq Scan on a 2M-row table with a Filter
+// removing 1,999,900 rows (the actual cost driver), while a small Nested
+// Loop over 100 rows is a red herring. The fix class is an index on the
+// filtered column. Answer: the seq-scan / missing-index problem.
+func advPgExplainForensicsTest() testkit.Test {
+	prompt := `Here is a Postgres EXPLAIN (ANALYZE) fragment for a slow query:
+
+Nested Loop  (cost=0.29..8.34 rows=100 actual time=0.02..0.14 rows=100)
+  ->  Seq Scan on events  (rows=2000000 actual time=0.01..812.4 rows=2000000)
+        Filter: (tenant_id = 42)
+        Rows Removed by Filter: 1999900
+  ->  Index Scan using users_pkey on users  (actual time=0.001..0.001 rows=1)
+
+Planning Time: 0.2 ms
+Execution Time: 851.9 ms
+
+Identify the single dominant cause of the slow execution, and the fix
+class. Respond with only a JSON object:
+{"cause": "seq-scan"|"nested-loop"|"planning"|"index-scan",
+ "fix": "add-index"|"raise-work-mem"|"vacuum"|"rewrite-join"}`
+
+	return testkit.Test{
+		ID:          "adv-pg-explain-forensics",
+		Category:    "databases",
+		Subcategory: "postgres",
+		Description: "Attribute an 852ms plan to the Seq Scan filtering out ~2M rows (not the cheap nested loop) and pick add-index.",
+		Prompt:      prompt,
+		Eval: eval.All(
+			eval.W(eval.JSONField("cause", "seq-scan"), 2),
+			eval.W(eval.JSONField("fix", "add-index"), 1),
+		),
+	}
+}
+
+// advSQLMVCCAnomalyTest: name the concurrency anomaly from an interleaved
+// transaction timeline at a stated isolation level.
+//
+// ground truth: under READ COMMITTED, txn A reads balance=100, then txn B
+// commits balance=150, then txn A reads again and sees 150 - the same row
+// read twice in one transaction yielded different committed values: a
+// non-repeatable read. It is not a dirty read (B committed) nor a phantom
+// (same row, not a new row matching a predicate).
+func advSQLMVCCAnomalyTest() testkit.Test {
+	prompt := `Two transactions run under READ COMMITTED isolation:
+
+t1: A: BEGIN
+t2: A: SELECT balance FROM accounts WHERE id=1  -> 100
+t3: B: BEGIN
+t4: B: UPDATE accounts SET balance=150 WHERE id=1
+t5: B: COMMIT
+t6: A: SELECT balance FROM accounts WHERE id=1  -> 150
+t7: A: COMMIT
+
+Name the concurrency anomaly transaction A experienced. Respond with only a
+JSON object:
+{"anomaly": "dirty-read"|"non-repeatable-read"|"phantom-read"|"none"}`
+
+	return testkit.Test{
+		ID:          "adv-sql-mvcc-anomaly",
+		Category:    "databases",
+		Subcategory: "sql-tuning",
+		Description: "Classify a re-read yielding a different committed value under READ COMMITTED as a non-repeatable read (not dirty or phantom).",
+		Prompt:      prompt,
+		Eval:        eval.JSONField("anomaly", "non-repeatable-read"),
+	}
+}
+
+// advDockerLayerRebuildTest: predict exactly which layers rebuild after a
+// specific source change.
+//
+// ground truth: editing a file under src/ busts the "COPY src" layer (C);
+// every layer after it rebuilds (D: npm run build); the final-stage
+// "COPY --from=build /app/dist" (F) consumes D's output so it rebuilds too.
+// The package-file COPY (A) and npm ci (B) are before the change and stay
+// cached; the nginx base (E) is unaffected. Answer set: {C, D, F}.
+func advDockerLayerRebuildTest() testkit.Test {
+	prompt := `Given this Dockerfile, with BuildKit layer caching and a warm
+cache from a previous identical build:
+
+  FROM node:20 AS build
+  WORKDIR /app
+  COPY package.json package-lock.json ./   # layer A
+  RUN npm ci                               # layer B
+  COPY src ./src                           # layer C
+  RUN npm run build                        # layer D
+  FROM nginx:alpine                        # layer E
+  COPY --from=build /app/dist /usr/share/nginx/html   # layer F
+
+You edit one file inside src/ (no dependency changes). Which layers must
+rebuild? Respond with only a JSON array of the layer letters that rebuild,
+e.g. ["A","B"].`
+
+	return testkit.Test{
+		ID:          "adv-docker-layer-rebuild",
+		Category:    "delivery",
+		Subcategory: "containers",
+		Description: "Predict that a src/ edit rebuilds exactly the COPY src, npm build, and dist-copy layers (C, D, F), not the cached package layers.",
+		Prompt:      prompt,
+		Eval:        eval.JSONStringSet([]string{"C", "D", "F"}),
+	}
+}
+
+// advRelFreezeWindowTest: order release steps around a change-freeze
+// constraint.
+//
+// ground truth: the deploy-to-prod step must NOT fall inside the freeze;
+// every other step (build, test, stage-deploy, smoke-test) can run during
+// the freeze because they do not touch prod. The only step to defer past
+// the freeze is deploy-prod. Answer: deploy-prod.
+func advRelFreezeWindowTest() testkit.Test {
+	prompt := `A change freeze forbids any change to PRODUCTION between now and
+Monday 09:00. Your release pipeline has these steps:
+
+- build: compiles artifacts (no environment touched).
+- unit-test: runs tests (no environment touched).
+- stage-deploy: deploys to the staging environment.
+- smoke-test: runs checks against staging.
+- deploy-prod: deploys to production.
+
+Which single step must be deferred until after the freeze lifts? Respond
+with only a JSON object: {"deferred_step": "<step id>"}`
+
+	return testkit.Test{
+		ID:          "adv-rel-freeze-window",
+		Category:    "delivery",
+		Subcategory: "release-engineering",
+		Description: "Identify deploy-prod as the only step a production change freeze blocks, while build/test/stage steps may proceed.",
+		Prompt:      prompt,
+		Eval:        eval.JSONField("deferred_step", "deploy-prod"),
+	}
+}
+
+// advScenMultiLogRootcauseTest: interleaved logs from three sources with one
+// root cause and red herrings.
+//
+// ground truth: the load balancer logs 502s, the app logs OOMKilled restart
+// events, and the kubelet logs memory-pressure eviction - all downstream of
+// the app being OOM-killed. The 502s are a symptom (backend gone), the
+// eviction is a symptom (node pressure from the same leak). The root cause
+// is the application OOM. Answer: oom.
+func advScenMultiLogRootcauseTest() testkit.Test {
+	prompt := `Three log streams during an incident, timestamps interleaved:
+
+11:02:01 lb: upstream 10.42.3.9:8080 returned 502
+11:02:03 app: container killed, reason OOMKilled, restart #4
+11:02:03 kubelet: memory pressure on node gst02, evicting low-priority pods
+11:02:05 lb: upstream 10.42.3.9:8080 connection refused
+11:02:07 app: heap usage 1.98Gi of 2Gi limit before exit
+
+Identify the single ROOT cause (not a downstream symptom). Respond with
+only a JSON object:
+{"root_cause": "lb-misconfig"|"oom"|"node-eviction"|"network"}`
+
+	return testkit.Test{
+		ID:          "adv-scen-multilog-rootcause",
+		Category:    "operations",
+		Subcategory: "scenario",
+		Description: "Root-cause interleaved lb/app/kubelet logs to the application OOM, treating the 502s and eviction as downstream symptoms.",
+		Prompt:      prompt,
+		Eval:        eval.JSONField("root_cause", "oom"),
+	}
+}
+
+// advLinuxCapacityFillTest: linear capacity projection.
+//
+// ground truth: 500GB volume, 90% alert threshold = 450GB, currently 366GB
+// used, growing 12GB/day. (450-366)/12 = 84/12 = 7 whole days until it
+// reaches the threshold.
+func advLinuxCapacityFillTest() testkit.Test {
+	prompt := `A 500 GB data volume is currently 366 GB used and grows by a
+steady 12 GB per day. An alert fires when usage reaches 90% of capacity.
+In how many whole days from now will usage first reach the 90% alert
+threshold? Respond with only the integer number of days.`
+
+	return testkit.Test{
+		ID:          "adv-linux-capacity-fill",
+		Category:    "operations",
+		Subcategory: "linux",
+		Description: "Project days until a 500GB volume at 366GB growing 12GB/day reaches the 90% (450GB) threshold: 7.",
+		Prompt:      prompt,
+		Eval:        eval.Numeric(eval.ExtractLastNumber[int], 7, 0),
+	}
+}
+
+// advHardComposedTraceTest: a Go trace composing loop-variable capture, a
+// deferred named-return mutation, and closure order.
+//
+// ground truth (verified by running it): three closures each capture the
+// Go 1.22+ per-iteration loop variable i (1, 2, 3) and add it to the named
+// return result; result becomes 6; return result + 10 sets result to 16;
+// the deferred func doubles it to 32.
+func advHardComposedTraceTest() testkit.Test {
+	prompt := "In Go 1.22 or later, what does calc() print?\n\n" +
+		"```go\n" +
+		"func calc() (result int) {\n" +
+		"\tdefer func() { result *= 2 }()\n" +
+		"\tfns := []func(){}\n" +
+		"\tfor i := 1; i <= 3; i++ {\n" +
+		"\t\tfns = append(fns, func() { result += i })\n" +
+		"\t}\n" +
+		"\tfor _, fn := range fns {\n" +
+		"\t\tfn()\n" +
+		"\t}\n" +
+		"\treturn result + 10\n" +
+		"}\n" +
+		"```\n\n" +
+		"Respond with only the integer printed."
+
+	return testkit.Test{
+		ID:          "adv-hard-composed-trace",
+		Category:    "programming",
+		Subcategory: "hard",
+		Description: "Trace Go 1.22 per-iteration capture (1+2+3) plus return+10 plus a deferred double: 32.",
+		Prompt:      prompt,
+		Eval:        eval.Numeric(eval.ExtractLastNumber[int], 32, 0),
+	}
+}
+
+// advPyComposedMutableTest: a Python trace composing a mutable default
+// argument with a list comprehension of closures.
+//
+// ground truth (verified by running it): f uses a shared mutable default
+// list acc; fns[0]() calls f(0), appending 0 -> returns [0]... but the
+// closures capture i by late binding, and range(3) leaves i=2, so each
+// lambda calls f(2). fns[0]() -> f(2) appends 2 -> [2]. Then f(10) appends
+// to the SAME shared list -> [2, 10]. The second printed line is [2, 10].
+func advPyComposedMutableTest() testkit.Test {
+	prompt := "What does the SECOND print statement output?\n\n" +
+		"```python\n" +
+		"def f(x, acc=[]):\n" +
+		"    acc.append(x)\n" +
+		"    return acc\n" +
+		"\n" +
+		"fns = [lambda: f(i) for i in range(3)]\n" +
+		"print(fns[0]())\n" +
+		"print(f(10))\n" +
+		"```\n\n" +
+		"The second print outputs a list of two integers. Respond with only a " +
+		"JSON object naming its two elements in order: " +
+		"{\"first_element\": <int>, \"second_element\": <int>}"
+
+	return testkit.Test{
+		ID:          "adv-py-composed-mutable",
+		Category:    "programming",
+		Subcategory: "python",
+		Description: "Trace late-binding closure (i=2) plus a shared mutable default list across two calls: second print is [2, 10].",
+		Prompt:      prompt,
+		Eval: eval.All(
+			eval.W(eval.JSONField("first_element", 2), 1),
+			eval.W(eval.JSONField("second_element", 10), 1),
+		),
+	}
+}
+
+// advCodeConflictingEvidenceTest: two sources disagree; pick the claim that
+// survives with the correct reason.
+//
+// ground truth: the README says the default port is 8080; the actual
+// config file (authoritative, it is what the program reads) says 9090. When
+// documentation and the live config disagree, the config the program
+// actually loads wins. Answer: 9090, source config.
+func advCodeConflictingEvidenceTest() testkit.Test {
+	prompt := `Two sources describe the service's listen port:
+
+- README.md (last edited 14 months ago): "The service listens on port
+  8080 by default."
+- config/server.yaml (the file the running process loads on startup):
+  "listen_port: 9090"
+
+There is no environment-variable override. Which port does the running
+service actually listen on, and which source is authoritative? Respond with
+only a JSON object: {"port": <number>, "authoritative": "readme"|"config"}`
+
+	return testkit.Test{
+		ID:          "adv-code-conflicting-evidence",
+		Category:    "research",
+		Subcategory: "codebase",
+		Description: "Resolve a README-vs-config port conflict to the config the process actually loads (9090), not the stale doc.",
+		Prompt:      prompt,
+		Eval: eval.All(
+			eval.W(eval.JSONField("port", 9090), 2),
+			eval.W(eval.JSONField("authoritative", "config"), 1),
+		),
+	}
+}
+
+// advSecExploitChainTest: two individually-benign behaviors compose into an
+// exploit; name the composed vulnerability class.
+//
+// ground truth: an open redirect (benign-looking) plus the OAuth flow
+// echoing the redirect_uri without an allowlist lets an attacker steal the
+// authorization code by redirecting the victim's callback to an
+// attacker-controlled host - an account-takeover via open-redirect in the
+// OAuth redirect_uri. The class asked for is "open-redirect".
+func advSecExploitChainTest() testkit.Test {
+	prompt := `Two facts about an app, each individually looking minor:
+
+1. GET /go?next=<url> issues an HTTP 302 to whatever <url> is passed, with
+   no host allowlist.
+2. The OAuth login uses /go as the post-login landing page and forwards the
+   authorization code to the landing URL's query string.
+
+Combined, what vulnerability class does this enable? Respond with only a
+JSON object:
+{"vuln": "open-redirect"|"csrf"|"sql-injection"|"xss"|"ssrf"}`
+
+	return testkit.Test{
+		ID:          "adv-sec-exploit-chain",
+		Category:    "security",
+		Subcategory: "appsec",
+		Description: "Compose an unvalidated redirect with an OAuth code hand-off into an open-redirect account-takeover chain.",
+		Prompt:      prompt,
+		Eval:        eval.JSONField("vuln", "open-redirect"),
+	}
+}
+
+// advSecRotationBlastRadiusTest: dual-key rotation-overlap math across
+// dependent consumers with different cache TTLs.
+//
+// ground truth: consumers cache a validated key for up to their TTL, so the
+// old key must stay accepted for at least the longest consumer TTL after
+// the new key is published, or an in-flight cached validation breaks. TTLs
+// are 30, 15, and 5 minutes; the minimum safe overlap window is the max =
+// 30 minutes.
+func advSecRotationBlastRadiusTest() testkit.Test {
+	prompt := `You are rotating a signing key with a dual-key (old+new)
+overlap window. Three independent services validate signatures and cache
+the validated key material for up to their own TTL before re-fetching:
+
+- service-a: 30-minute cache TTL
+- service-b: 15-minute cache TTL
+- service-c: 5-minute cache TTL
+
+For how many minutes, at minimum, must the OLD key stay accepted after the
+new key is published, so that no service rejects a signature it should
+accept? Respond with only the integer number of minutes.`
+
+	return testkit.Test{
+		ID:          "adv-sec-rotation-blast-radius",
+		Category:    "security",
+		Subcategory: "secrets",
+		Description: "Size the dual-key overlap window to the longest consumer cache TTL (max(30,15,5)=30 minutes).",
+		Prompt:      prompt,
+		Eval:        eval.Numeric(eval.ExtractLastNumber[int], 30, 0),
+	}
+}

internal/tests/advanced_groundtruth_test.go

@@ -0,0 +1,49 @@
+package tests
+
+import (
+	"context"
+	"testing"
+)
+
+// TestAdvancedGroundTruthScoresFull asserts each advanced test's own
+// documented correct answer scores 1.0 through its real evaluator - a guard
+// that the eval and the intended ground truth agree (a JSON typo or a wrong
+// field path would silently make a test unwinnable).
+func TestAdvancedGroundTruthScoresFull(t *testing.T) {
+	golden := map[string]string{
+		"adv-agent-plan-infeasible":          `{"feasible": false, "blocking_step": "fetch-deps"}`,
+		"adv-agent-deleg-conflicting-policy": `{"agent": "agent-gpu-eu-fast"}`,
+		"adv-agent-tool-adversarial-desc":    `{"tool": "trash_bin"}`,
+		"adv-rag-forensics-stage":            `{"stage": "generation"}`,
+		"adv-llm-context-budget":             `54`,
+		"adv-pg-explain-forensics":           `{"cause": "seq-scan", "fix": "add-index"}`,
+		"adv-sql-mvcc-anomaly":               `{"anomaly": "non-repeatable-read"}`,
+		"adv-docker-layer-rebuild":           `["C","D","F"]`,
+		"adv-rel-freeze-window":              `{"deferred_step": "deploy-prod"}`,
+		"adv-scen-multilog-rootcause":        `{"root_cause": "oom"}`,
+		"adv-linux-capacity-fill":            `7`,
+		"adv-hard-composed-trace":            `32`,
+		"adv-py-composed-mutable":            `{"first_element": 2, "second_element": 10}`,
+		"adv-code-conflicting-evidence":      `{"port": 9090, "authoritative": "config"}`,
+		"adv-sec-exploit-chain":              `{"vuln": "open-redirect"}`,
+		"adv-sec-rotation-blast-radius":      `30`,
+	}
+
+	byID := map[string]int{}
+	all := All().All()
+	for i, tc := range all {
+		byID[tc.ID] = i
+	}
+
+	for id, answer := range golden {
+		idx, ok := byID[id]
+		if !ok {
+			t.Errorf("%s: not registered in catalog", id)
+			continue
+		}
+		s := all[idx].Eval.Evaluate(context.Background(), answer)
+		if s.Value != 1 {
+			t.Errorf("%s: golden answer %q scored %.2f, want 1.00 (%s)", id, answer, s.Value, s.Detail)
+		}
+	}
+}

internal/tests/catalog.go

@@ -26,5 +26,6 @@ func All() *testkit.Registry {
 	registerHardTests(r)
 	registerScenarioTests(r)
 	registerSaturationTests(r)
+	registerAdvancedTests(r)
 	return r
 }

internal/tests/delivery_git.go

@@ -46,7 +46,7 @@ func delNoUnnegatedMention(forbidden *regexp.Regexp) eval.Evaluator {
 // "-with-lease" - the safe form, exempt from the bare-force check entirely
 // rather than needing a negation cue.
 var delForceFlagPattern = regexp.MustCompile(`(?i)--force\b`)
-var delForceWithLeaseSuffixPattern = regexp.MustCompile(`(?i)^-with-lease`)
+var delForceWithLeaseSuffixPattern = regexp.MustCompile(`(?i)^-(with-lease|if-includes)`)
 
 // delNoBareForcePush scores full credit unless the response recommends a
 // bare "--force" push (not "--force-with-lease") with no negation cue
@@ -320,8 +320,10 @@ func delGitDetachedHeadRecoveryTest() testkit.Test {
 branch name), then made 3 new commits while HEAD was detached. You only now
 realize you need to keep that work, and HEAD is still detached. Give the
 ordered list of git subcommands (just the subcommand names, e.g. "branch")
-you must run, in order, to save the 3 commits onto a new branch without
-losing them. Respond with only a JSON array of subcommand names, e.g.
+you must run, in order, to save the 3 commits onto a new branch AND end up
+checked out on that branch, with HEAD attached to it (a bare "git branch"
+alone saves the commits but leaves HEAD detached, which is not enough
+here). Respond with only a JSON array of subcommand names, e.g.
 ["a","b"].`
 
 	return testkit.Test{

internal/tests/operations_scenario.go

@@ -178,9 +178,10 @@ D) kubectl scale deployment ingestion --replicas=0
 E) ssh to a node and run journalctl -u kubelet
 
 Respond with only a JSON array of the step letters that make up your
-diagnosis, containing exactly the diagnostic steps you would take to find
-the root cause (any order) - do not include steps that only act without
-diagnosing.`
+diagnosis, containing exactly the SMALLEST set of steps sufficient to find
+the root cause (any order). Do not include steps that only act without
+diagnosing, and do not include diagnostics that are redundant when a
+simpler kubectl call already surfaces the same information.`
 
 	return testkit.Test{
 		ID:          "scen-crashloop-order",