fix(catalog): harden evaluators against correct-answer phrasing variance

Both reviewers found evaluators that scored correct answers as zero:
hardcoded flag order, quote-style rigidity, missing multiline anchors,
comma-grouped numbers, ratio notation, and numeric strings in JSON.
A shared ExactToken evaluator now normalizes fences, quotes, and
trailing periods for exact-answer tests. The obsolete tmutil snapshot
reference now accepts the current localsnapshot verb.

internal/eval/exacttoken.go

@@ -0,0 +1,91 @@
+package eval
+
+import (
+	"context"
+	"fmt"
+	"strings"
+)
+
+// NormalizeExactToken strips formatting noise from s before an exact-token
+// comparison: it extracts a fenced code block if s has one (see
+// ExtractCodeBlock), trims surrounding whitespace, strips one trailing
+// sentence-ending period, strips one layer of surrounding quote/backtick
+// decoration, strips a *paired* leading/trailing run of asterisks
+// (**bolded**/*italic* markdown - never a single, unpaired asterisk, which
+// can be meaningful content rather than decoration: "*string" is a Go
+// pointer type, not "string" wrapped in emphasis), and trims again. The
+// quote/backtick/asterisk/period stripping repeats to a fixed point so
+// nested decoration ("**\"GET\"**.") fully resolves regardless of which
+// layer is outermost. It is exported so a test file that needs to compare
+// against more than one literal variant (e.g. "O(n^2)" vs "O(n²)" vs
+// "O(n**2)") can normalize once and compare the result to each variant,
+// rather than calling ExactToken repeatedly.
+func NormalizeExactToken(s string) string {
+	got := strings.TrimSpace(ExtractCodeBlock(s, ""))
+
+	periodStripped := false
+	for {
+		before := got
+		if !periodStripped && strings.HasSuffix(got, ".") {
+			got = strings.TrimSpace(strings.TrimSuffix(got, "."))
+			periodStripped = true
+		}
+		got = strings.Trim(got, "\"'`")
+		got = stripPairedAsterisks(got)
+		got = strings.TrimSpace(got)
+		if got == before {
+			break
+		}
+	}
+	return got
+}
+
+// stripPairedAsterisks removes a leading run of '*' and a trailing run of
+// '*' from s, but only when both runs have the same, non-zero length: a
+// balanced **bold** or *italic* wrapper. An unbalanced run (a single
+// leading '*' with no trailing one, as in the Go pointer type "*string")
+// is left untouched, since it is markdown noise for one and meaningful
+// content for the other, and this function cannot tell them apart from
+// the character alone.
+func stripPairedAsterisks(s string) string {
+	lead := 0
+	for lead < len(s) && s[lead] == '*' {
+		lead++
+	}
+	trail := 0
+	for trail < len(s)-lead && s[len(s)-1-trail] == '*' {
+		trail++
+	}
+	if lead > 0 && lead == trail {
+		return s[lead : len(s)-trail]
+	}
+	return s
+}
+
+// exactTokenEval scores full credit on a normalized, case-insensitive
+// whole-answer match.
+type exactTokenEval struct {
+	want string
+}
+
+// ExactToken returns an Evaluator awarding full credit when the response,
+// after NormalizeExactToken, equals want (also normalized) case-
+// insensitively. It exists for prompts that force a single, literal
+// answer token or short phrase - one the prompt itself may quote, bold, or
+// fence - where a substring-containment evaluator (ContainsAny etc.) would
+// wrongly score a response that merely restates the prompt's own option
+// list, or a negated answer that happens to contain the right substring
+// ("not O(n^2), it's O(n)"). ExactToken anchors to the whole normalized
+// answer, so neither false-positive scores.
+func ExactToken(want string) Evaluator {
+	return exactTokenEval{want: want}
+}
+
+func (e exactTokenEval) Evaluate(_ context.Context, response string) Score {
+	got := NormalizeExactToken(response)
+	want := NormalizeExactToken(e.want)
+	if strings.EqualFold(got, want) {
+		return Score{Value: 1, Detail: fmt.Sprintf("equals %q", want)}
+	}
+	return Score{Value: 0, Detail: fmt.Sprintf("got %q, want %q", got, want)}
+}

internal/eval/exacttoken_test.go

@@ -0,0 +1,71 @@
+package eval
+
+import (
+	"context"
+	"testing"
+)
+
+func TestNormalizeExactToken(t *testing.T) {
+	tests := []struct {
+		name string
+		in   string
+		want string
+	}{
+		{"bare", "GET", "GET"},
+		{"surrounding whitespace", "  GET  \n", "GET"},
+		{"single-quoted", "'GET'", "GET"},
+		{"double-quoted", "\"GET\"", "GET"},
+		{"backtick-quoted", "`GET`", "GET"},
+		{"bolded", "**GET**", "GET"},
+		{"trailing period", "GET.", "GET"},
+		{"quoted with trailing period", "\"GET\".", "GET"},
+		{"fenced, untagged", "```\nGET\n```", "GET"},
+		{"fenced, tagged", "```text\nGET\n```", "GET"},
+		{"fenced with surrounding prose", "The answer is:\n```\nGET\n```", "GET"},
+		{"multi-word phrase", "Access-Control-Allow-Headers", "Access-Control-Allow-Headers"},
+		{"multi-word phrase bolded with period", "**Access-Control-Allow-Headers**.", "Access-Control-Allow-Headers"},
+		// Regression: an unpaired leading asterisk is meaningful content
+		// (a Go pointer type), not markdown decoration, and must survive.
+		{"unpaired leading asterisk is content, not decoration", "*string", "*string"},
+		{"unpaired trailing asterisk is content, not decoration", "string*", "string*"},
+		{"nested quote-then-bold with trailing period", "**\"GET\"**.", "GET"},
+		{"italic single-asterisk wrap", "*GET*", "GET"},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := NormalizeExactToken(tt.in)
+			if got != tt.want {
+				t.Errorf("NormalizeExactToken(%q) = %q, want %q", tt.in, got, tt.want)
+			}
+		})
+	}
+}
+
+func TestExactToken(t *testing.T) {
+	tests := []struct {
+		name     string
+		want     string
+		response string
+		wantVal  float64
+	}{
+		{"bare exact match", "MX", "MX", 1},
+		{"case insensitive", "MX", "mx", 1},
+		{"quoted", "MX", "'MX'", 1},
+		{"bolded", "MX", "**MX**", 1},
+		{"trailing period", "MX", "MX.", 1},
+		{"fenced", "MX", "```\nMX\n```", 1},
+		{"restates a longer option list scores 0", "MX", "A, AAAA, CNAME, TXT, NS, MX", 0},
+		{"wrong answer", "MX", "TXT", 0},
+		{"empty response", "MX", "", 0},
+		{"phrase exact match", "Access-Control-Allow-Headers", "Access-Control-Allow-Headers", 1},
+		{"phrase restated with prose scores 0", "Access-Control-Allow-Headers", "The missing header is Access-Control-Allow-Headers.", 0},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := ExactToken(tt.want).Evaluate(context.Background(), tt.response)
+			if got.Value != tt.wantVal {
+				t.Errorf("ExactToken(%q).Evaluate(%q) = %v, want %v (detail: %s)", tt.want, tt.response, got.Value, tt.wantVal, got.Detail)
+			}
+		})
+	}
+}

internal/eval/jsonfield.go

@@ -111,12 +111,24 @@ func jsonValueAs[T comparable](v any) (T, error) {
 	}
 }
 
+// asFloat64 coerces a decoded JSON value to float64. A string is accepted
+// too (B9), coerced only when it parses cleanly as a number end to end
+// (strconv.ParseFloat on the trimmed string): a model that answers
+// {"line":"8"} instead of {"line":8} - both are the same value, and the
+// prompt asked for a JSON field, not a JSON type - must not score 0 for a
+// mechanical typing difference. A non-numeric string ("eight") still fails.
 func asFloat64(v any) (float64, error) {
 	switch n := v.(type) {
 	case float64:
 		return n, nil
 	case json.Number:
 		return n.Float64()
+	case string:
+		f, err := strconv.ParseFloat(strings.TrimSpace(n), 64)
+		if err != nil {
+			return 0, fmt.Errorf("expected number, got non-numeric string %q", n)
+		}
+		return f, nil
 	default:
 		return 0, fmt.Errorf("expected number, got %T (%v)", v, v)
 	}

internal/eval/jsonfield_test.go

@@ -90,6 +90,43 @@ func TestJSONField_Int(t *testing.T) {
 	}
 }
 
+// TestJSONField_Int_NumericStringCoercion is the B9 regression: a model
+// answering {"line":"8"} instead of {"line":8} must still score 1 - the
+// prompt asked for a JSON field with the right value, not a specific JSON
+// type.
+func TestJSONField_Int_NumericStringCoercion(t *testing.T) {
+	e := JSONField("line", 8)
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{"numeric string coerces", `{"line":"8"}`, 1},
+		{"numeric string with whitespace coerces", `{"line":" 8 "}`, 1},
+		{"wrong numeric string", `{"line":"9"}`, 0},
+		{"non-numeric string does not coerce", `{"line":"eight"}`, 0},
+		{"native number still works", `{"line":8}`, 1},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := e.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}
+
+// TestJSONField_Float_NumericStringCoercion mirrors the int case for a
+// float64-typed field.
+func TestJSONField_Float_NumericStringCoercion(t *testing.T) {
+	e := JSONField("port", float64(8080))
+	got := e.Evaluate(context.Background(), `{"port":"8080"}`)
+	if got.Value != 1 {
+		t.Errorf("Evaluate() = %v, want 1 (detail: %s)", got.Value, got.Detail)
+	}
+}
+
 func TestJSONField_NestedPath(t *testing.T) {
 	e := JSONField("task1", "search_web")
 	got := e.Evaluate(context.Background(), `{"task1":"search_web","task2":"read_file"}`)

internal/eval/numeric.go

@@ -18,12 +18,17 @@ type Number interface {
 }
 
 // digitPattern matches unsigned integers and decimals, including a
-// leading-dot form with no integer part (".9896"). Longer alternatives are
+// comma-grouped thousands form ("50,000", "1,234.5") and a leading-dot form
-// listed first so the match is not truncated to just the integer part of
+// with no integer part (".9896"). Longer/more specific alternatives are
-// "0.9896" or just the dot-less ".9896". It deliberately excludes the sign:
+// listed first so the match is not truncated to just "50" out of "50,000",
-// classifyMatch decides, per match, whether a leading '-' in the source
+// or just the integer part of "0.9896", or just the dot-less ".9896". The
-// text is a real minus sign or a compound-word hyphen ("x86-64").
+// comma-group alternative requires groups of exactly 3 digits after the
-var digitPattern = regexp.MustCompile(`\d+\.\d+|\.\d+|\d+`)
+// first comma (`\d{1,3}(?:,\d{3})+`), so it does not misfire on an
+// unrelated comma followed by a differently-sized number ("in 2024, 50
+// states"). It deliberately excludes the sign: classifyMatch decides, per
+// match, whether a leading '-' in the source text is a real minus sign or
+// a compound-word hyphen ("x86-64").
+var digitPattern = regexp.MustCompile(`\d{1,3}(?:,\d{3})+(?:\.\d+)?|\d+\.\d+|\.\d+|\d+`)
 
 // isWordByte reports whether b is a letter, digit, or underscore - the
 // standard regex \w character class - used to detect a digit run glued
@@ -47,6 +52,11 @@ type numberCandidate struct {
 	compoundHyphen bool
 }
 
+// isDigitByte reports whether b is an ASCII digit.
+func isDigitByte(b byte) bool {
+	return b >= '0' && b <= '9'
+}
+
 // classifyMatch inspects the characters immediately surrounding the
 // unsigned digit run text[start:end] and classifies it:
 //
@@ -56,9 +66,14 @@ type numberCandidate struct {
 //   - a '-' immediately before the run is a real minus sign only when the
 //     character before THAT is not alphanumeric, so "x86-64" does not read
 //     as "x86" followed by the negative number -64;
+//   - a digit run immediately preceded by ':' whose own left neighbour is
+//     a digit is the second term of a ratio ("64:1"'s "1") and is never a
+//     real candidate, at any tier, so extraction lands on the first term;
 //   - a number joined by a hyphen to an adjacent word on either side
 //     ("64-bit", "8-byte", the "64" in "x86-64" once its bogus sign is
-//     stripped) is flagged compoundHyphen rather than rejected outright.
+//     stripped) is flagged compoundHyphen rather than rejected outright;
+//   - a comma-grouped literal ("50,000", "1,234.5") has its commas
+//     stripped so the returned literal parses with strconv.ParseFloat.
 func classifyMatch(text string, start, end int) (numberCandidate, bool) {
 	literal := text[start:end]
 
@@ -73,6 +88,9 @@ func classifyMatch(text string, start, end int) (numberCandidate, bool) {
 	if start > 0 && isWordByte(text[start-1]) {
 		return numberCandidate{}, false // e.g. the "64" in "LP64"
 	}
+	if start > 0 && text[start-1] == ':' && start-2 >= 0 && isDigitByte(text[start-2]) {
+		return numberCandidate{}, false // e.g. the "1" in "64:1"
+	}
 	if end < len(text) && isWordByte(text[end]) {
 		// Permit a multiplier suffix ("64x" = 64 times): a single trailing
 		// x/X with nothing word-like after it. Hex ("0x2A") and identifiers
@@ -90,6 +108,7 @@ func classifyMatch(text string, start, end int) (numberCandidate, bool) {
 		compound = true // e.g. "64-bit", "24-byte"
 	}
 
+	literal = strings.ReplaceAll(literal, ",", "")
 	return numberCandidate{literal: literal, compoundHyphen: compound}, true
 }
 

internal/eval/numeric_test.go

@@ -35,6 +35,19 @@ func TestExtractLastNumber(t *testing.T) {
 		// the last standalone number, which here is not the answer a
 		// human would pick. Prompts should ask for the number alone.
 		{"ambiguous parenthetical breakdown returns the wrong number by design", "Total: 24 bytes (22 bytes of members plus 2 tail padding)", 2, false},
+		// B1: comma-grouped thousands separators.
+		{"comma-grouped thousands", "50,000", 50000, false},
+		{"comma-grouped thousands with unit", "50,000 URLs", 50000, false},
+		{"comma-grouped thousands in a sentence", "The limit is 50,000.", 50000, false},
+		{"comma-grouped with a decimal tail", "1,234.5", 1234.5, false},
+		{"comma-grouped, multiple groups", "12,345,678", 12345678, false},
+		{"unrelated comma is not a thousands group", "in 2024, 50 states voted", 50, false},
+		// B6: ratio form "N:M" must extract the first term, not the second.
+		{"ratio form extracts the first term", "64:1", 64, false},
+		// Documented limitation, not a regression (B6): the word-form ratio
+		// is not cheaply distinguishable from two unrelated numbers, so
+		// this pins the current (non-ideal) behaviour deliberately.
+		{"word-form ratio is a documented limitation, not fixed", "The ratio is 64 to 1.", 1, false},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {

internal/tests/agents_planning.go

@@ -477,21 +477,25 @@ Respond with only a JSON object: {"next_action":"..."}`
 //
 // ground truth: prep_env finishes on day 0+3=3, prep_data finishes on day
 // 0+5=5. Milestone M cannot start until BOTH finish, so M starts on
-// max(3,5)=5, and finishes 2 days later, on day 5+2=7.
+// max(3,5)=5, and finishes 2 days later, on day 5+2=7. B7: the prompt now
+// states the t+d finish-day rule explicitly rather than relying on a
+// "day 0" framing a reader could also read as 1-indexed (which would make
+// 6 a defensible, if unintended, answer); 7 remains the only correct one.
 var planMilestoneWant = max(3, 5) + 2
 
 func planMilestoneDateTest() testkit.Test {
-	prompt := `Two prerequisite work items both start on day 0 and run in
+	prompt := `Two prerequisite work items run in parallel, starting at day 0:
-parallel:
 - prep_env takes 3 days.
 - prep_data takes 5 days.
 
 Milestone M can only start once BOTH prep_env and prep_data have finished,
 and M itself takes 2 days once it starts.
 
-On what day number does milestone M finish (counting the day work starts
+Use this rule: a task starting at day t with duration d finishes at day
-as day 0, and each duration in whole days with no gaps)? Respond with only
+t+d. Work starts at day 0.
-the number.`
+
+On what day number does milestone M finish? Respond with only the
+number.`
 
 	return testkit.Test{
 		ID:          "agent-plan-milestone-date",

internal/tests/catalog.go

@@ -2,11 +2,10 @@ package tests
 
 import "github.com/lukaszraczylo/llm-testbench/internal/testkit"
 
-// All builds a Registry containing the full 17-test catalog: 3
+// All builds a Registry containing the full test catalog, spanning the
-// programming/golang, 2 programming/python, 1 programming/typescript, 1
+// programming (golang, python, typescript, c), operations (macos, linux,
-// programming/c, 2 operations/macos, 2 operations/linux, 1
+// kubernetes), research (web, whitepapers, codebase), and agents
-// operations/kubernetes, 1 research/web, 1 research/whitepapers, 1
+// (tool-routing, planning) categories.
-// research/codebase, and 2 agents (tool-routing, planning).
 func All() *testkit.Registry {
 	r := testkit.NewRegistry()
 	registerGolangTests(r)

internal/tests/codebase_analysis.go

@@ -1,6 +1,10 @@
 package tests
 
 import (
+	"context"
+	"fmt"
+	"regexp"
+
 	"github.com/lukaszraczylo/llm-testbench/internal/eval"
 	"github.com/lukaszraczylo/llm-testbench/internal/testkit"
 )
@@ -157,8 +161,26 @@ const codeDedupSnippet = `func Dedup(a []int) []int {
 
 // codeBigODedupPattern accepts every common notation for O(n^2): a caret,
 // a double-star, the unicode superscript two, or n*n, each optionally
-// wrapped as O(...) with flexible internal spacing.
+// wrapped as O(...) with flexible internal spacing. Anchored to the whole
-const codeBigODedupPattern = `(?i)o\(\s*n\s*(\^\s*2|\*\*\s*2|²|\*\s*n)\s*\)`
+// string (codeBigODedupEval matches it against the whole normalized
+// response, never a substring), so "not O(n^2), it's O(n)" - which
+// contains the literal substring "O(n^2)" - correctly fails (B5) instead
+// of matching the way an unanchored eval.Regex would.
+var codeBigODedupPattern = regexp.MustCompile(`(?i)^o\(\s*n\s*(\^\s*2|\*\*\s*2|²|\*\s*n)\s*\)$`)
+
+// codeBigODedupEval anchors eval.NormalizeExactToken's normalization (fence
+// extraction, quote/asterisk/period stripping) to codeBigODedupPattern,
+// combining "accept several equivalent notations" with "match the whole
+// answer, not a substring anywhere in it."
+func codeBigODedupEval() eval.Evaluator {
+	return eval.EvaluatorFunc(func(_ context.Context, response string) eval.Score {
+		normalized := eval.NormalizeExactToken(response)
+		if codeBigODedupPattern.MatchString(normalized) {
+			return eval.Score{Value: 1, Detail: fmt.Sprintf("matches O(n^2) notation: %q", normalized)}
+		}
+		return eval.Score{Value: 0, Detail: fmt.Sprintf("does not match O(n^2) notation: %q", normalized)}
+	})
+}
 
 // codeBigODedupTest: state the worst-case time complexity of an inline
 // nested-loop deduplication function.
@@ -181,7 +203,7 @@ terms of n = len(a)? Respond with only the Big-O expression, e.g. O(n).`
 		Subcategory: "codebase",
 		Description: "State the worst-case Big-O time complexity of an inline nested-loop deduplication function.",
 		Prompt:      prompt,
-		Eval:        eval.Regex(codeBigODedupPattern),
+		Eval:        codeBigODedupEval(),
 	}
 }
 

internal/tests/codebase_analysis_test.go

@@ -14,6 +14,10 @@ func TestCodeBugLineDiffTest_Eval(t *testing.T) {
 		want     float64
 	}{
 		{"correct: root-cause line", `{"line":8}`, 1},
+		// B9: a numeric-string JSON value must coerce the same as a
+		// native number - the prompt asked for the right line, not a
+		// specific JSON type.
+		{"correct as a numeric string", `{"line":"8"}`, 1},
 		{"wrong: points at the panic site instead of the root cause", `{"line":10}`, 0},
 		{"wrong line entirely", `{"line":6}`, 0},
 	}
@@ -87,8 +91,14 @@ func TestCodeBigODedupTest_Eval(t *testing.T) {
 		{"unicode superscript form", "O(n²)", 1},
 		{"n*n form", "O(n*n)", 1},
 		{"lowercase o", "o(n^2)", 1},
+		{"quoted", `"O(n^2)"`, 1},
+		{"trailing period", "O(n^2).", 1},
 		{"wrong: linear", "O(n)", 0},
 		{"wrong: linearithmic", "O(n log n)", 0},
+		// B5: the correct notation appears as a substring of a negated
+		// sentence; the whole-response-anchored evaluator must not match
+		// on that substring.
+		{"negated: contains the substring O(n^2) but names O(n) as the answer", "not O(n^2), it's O(n) with a hash set", 0},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {

internal/tests/codebase_trace.go

@@ -1,34 +1,25 @@
 package tests
 
 import (
-	"context"
-	"fmt"
 	"strconv"
-	"strings"
 
 	"github.com/lukaszraczylo/llm-testbench/internal/eval"
 	"github.com/lukaszraczylo/llm-testbench/internal/testkit"
 )
 
 // codeExactAnswer returns an Evaluator awarding full credit when the
-// response, trimmed of whitespace, at most one layer of surrounding quote
+// response equals want under eval.ExactToken's normalization: a fenced
-// characters (' , ", or `), and a single trailing sentence-ending period,
+// code block extracted if present, whitespace trimmed, one layer of
-// equals want case-insensitively. This accepts every materially-correct
+// surrounding quote/backtick/asterisk decoration stripped, one trailing
-// form of a forced single-token answer (bare, quoted, differently-cased,
+// sentence-ending period stripped, compared case-insensitively. This
-// or with trailing punctuation) without loosening the match to accept a
+// accepts every materially-correct form of a forced single-token answer
-// substring of a longer, wrong answer.
+// (bare, quoted, bolded, fenced, differently-cased, or with trailing
+// punctuation) without loosening the match to accept a substring of a
+// longer, wrong answer. Kept as a thin wrapper (rather than calling
+// eval.ExactToken directly at each call site) so this file's intent -
+// "the traced program's exact output token" - stays named and greppable.
 func codeExactAnswer(want string) eval.Evaluator {
-	return eval.EvaluatorFunc(func(_ context.Context, response string) eval.Score {
+	return eval.ExactToken(want)
-		got := strings.TrimSpace(response)
-		got = strings.Trim(got, "\"'`")
-		got = strings.TrimSuffix(strings.TrimSpace(got), ".")
-		got = strings.TrimSpace(got)
-		wantTrimmed := strings.TrimSpace(want)
-		if strings.EqualFold(got, wantTrimmed) {
-			return eval.Score{Value: 1, Detail: fmt.Sprintf("equals %q", wantTrimmed)}
-		}
-		return eval.Score{Value: 0, Detail: fmt.Sprintf("got %q, want %q", got, wantTrimmed)}
-	})
 }
 
 // codeTracePythonSource is the inline Python function traced by

internal/tests/codebase_trace_test.go

@@ -120,6 +120,39 @@ func TestCodeTraceTSWant_GroundTruth(t *testing.T) {
 	if got != codeTraceTSWant {
 		t.Errorf("codeTSPackTrace(%v) = %q, want %q", items, got, codeTraceTSWant)
 	}
+
+	// BN4: cross-check against a real JS engine too, not just the Go
+	// replica above. node cannot execute codeTraceTSSource's TypeScript
+	// type annotations directly, so this is the same logic with the two
+	// annotations (": number[]", ": string") stripped - everything else,
+	// including the reduce/ternary/toString(2) logic under test, is
+	// identical source.
+	if _, err := exec.LookPath("node"); err != nil {
+		t.Skip("node not found on PATH, skipping exec ground-truth cross-check")
+	}
+
+	const jsSource = `function pack(items) {
+  return items.reduce((acc, x, i) => {
+    if (i % 2 === 0) return acc + x;
+    return acc + x * 2;
+  }, 0).toString(2);
+}
+console.log(pack([3, 4, 5, 6, 7]));
+`
+	// #nosec G204 -- jsSource is a fixed, hardcoded string (the TS source
+	// embedded in the prompt with its two type annotations stripped), not
+	// external or response-controlled input.
+	cmd := exec.Command("node", "-e", jsSource) //nolint:gosec // see comment above
+	var stdout bytes.Buffer
+	cmd.Stdout = &stdout
+	cmd.Stderr = os.Stderr
+	if err := cmd.Run(); err != nil {
+		t.Fatalf("running node ground-truth script failed: %v", err)
+	}
+	nodeGot := strings.TrimSpace(stdout.String())
+	if nodeGot != codeTraceTSWant {
+		t.Errorf("node-computed pack([3,4,5,6,7]) = %q, want %q", nodeGot, codeTraceTSWant)
+	}
 }
 
 func TestCodeTraceTSTest_Eval(t *testing.T) {

internal/tests/golang.go

@@ -492,7 +492,9 @@ printed, nothing else.`
 		Subcategory: "golang",
 		Description: "Trace the exact output of a context cancellation propagating synchronously from a parent to a derived context.",
 		Prompt:      prompt,
-		Eval:        eval.Equals("context canceled"),
+		// A7: eval.ExactToken so a fenced or quoted rendering of the exact
+		// printed text still scores full credit.
+		Eval: eval.ExactToken("context canceled"),
 	}
 }
 

internal/tests/golang_test.go

@@ -479,6 +479,10 @@ func TestGoContextCancellationTraceTest_Eval(t *testing.T) {
 		{"exact correct", "context canceled", 1},
 		{"correct case-insensitive", "CONTEXT CANCELED", 1},
 		{"correct with surrounding whitespace", "  context canceled  ", 1},
+		// A7 regression: fenced/quoted decoration on the correct answer
+		// must still score full credit.
+		{"fenced", "```\ncontext canceled\n```", 1},
+		{"quoted with trailing period", `"context canceled".`, 1},
 		{"wrong: assumes select falls to default", "not done", 0},
 		{"wrong: wrong sentinel error", "context deadline exceeded", 0},
 	}

internal/tests/kubernetes.go

@@ -349,7 +349,10 @@ spec:
 // app=inventory-api-svc - a value that matches no pod's labels, so the
 // Service has zero Endpoints. The single wrong field is the Service's
 // spec.selector.app value; it must be changed to "inventory-api" to match
-// the pods.
+// the pods. The prompt's JSON template pre-fills "field":"selector.app"
+// as part of the requested response shape, so a model that never
+// diagnosed the field at all could still copy that value verbatim; only
+// "value" is actually evidence the model found the fix (AN2).
 func k8sServiceSelectorMismatchTest() testkit.Test {
 	prompt := `Requests to this Service time out with no available
 endpoints. Here is the Deployment's pod template and the Service:
@@ -360,10 +363,7 @@ Which single field in the Service YAML is wrong, and what value should it
 be changed to so the Service's Endpoints include these pods? Respond with
 only a JSON object: {"field":"selector.app","value":"<correct value>"}`
 
-	evaluator := eval.Mean(
+	evaluator := eval.JSONField("value", "inventory-api")
-		eval.JSONField("field", "selector.app"),
-		eval.JSONField("value", "inventory-api"),
-	)
 
 	return testkit.Test{
 		ID:          "k8s-service-selector-mismatch",
@@ -510,8 +510,11 @@ const k8sTraefikIngressRouteYAML = "apiVersion: traefik.io/v1alpha1\n" +
 	"          port: 80"
 
 // k8sTraefikIngressRoutePattern requires the corrected match expression,
-// with the backtick-quoted domain exactly matching app.raczylo.com.
+// with the domain app.raczylo.com quoted with either backticks (Traefik's
-const k8sTraefikIngressRoutePattern = "(?i)host\\(`app\\.raczylo\\.com`\\)"
+// own matcher syntax, as shown in the prompt's example) or double quotes
+// (a common, equally unambiguous way a model renders the same literal)
+// (AN4).
+const k8sTraefikIngressRoutePattern = "(?i)host\\([`\"]app\\.raczylo\\.com[`\"]\\)"
 
 // k8sTraefikIngressRouteHostTest: correct a Traefik IngressRoute's Host()
 // match rule to the actual domain the service must answer for.
@@ -581,7 +584,10 @@ BestEffort.`
 		Subcategory: "kubernetes",
 		Description: "Derive a pod's QoS class (Burstable) from requests that are set but not equal to limits.",
 		Prompt:      prompt,
-		Eval:        eval.Equals("Burstable"),
+		// A7: eval.ExactToken (not eval.Equals) so a fenced, quoted, or
+		// bolded "Burstable" - a common way a model formats a forced
+		// one-word answer - still scores full credit.
+		Eval: eval.ExactToken("Burstable"),
 	}
 }
 
@@ -600,10 +606,15 @@ with what minimum value, to enforce "at least 2 available"?`
 	// bounds voluntary disruptions (like a node drain evicting pods) for a
 	// selected set of pods. Setting its minAvailable field to 2 enforces
 	// that at least 2 of the 3 Postgres pods must remain available before
-	// the eviction API will permit another one to be drained.
+	// the eviction API will permit another one to be drained. The
+	// minAvailable check is a single regex tolerant of any short
+	// same-line phrasing between the key and the value (A10) - e.g.
+	// "minAvailable is set to 2" or "minAvailable: \"2\"" - rather than an
+	// enumerated list of exact phrasings that misses anything not on the
+	// list.
 	evaluator := eval.Mean(
 		eval.ContainsAll("PodDisruptionBudget"),
-		eval.ContainsAny("minAvailable: 2", "minAvailable=2", "minAvailable 2", "minAvailable of 2"),
+		eval.Regex(`(?i)minAvailable\b[^\n]{0,20}\b2\b`),
 	)
 
 	return testkit.Test{

internal/tests/kubernetes_test.go

@@ -238,7 +238,12 @@ func TestK8sServiceSelectorMismatchTest_Eval(t *testing.T) {
 		{"prose then JSON", "The selector doesn't match the pod labels.\n" + `{"field":"selector.app","value":"inventory-api"}`, 1},
 		{"fenced JSON", "```json\n" + `{"field":"selector.app","value":"inventory-api"}` + "\n```", 1},
 		{"points at the wrong field (targetPort instead of selector)", `{"field":"targetPort","value":"8080"}`, 0},
-		{"right field, wrong replacement value", `{"field":"selector.app","value":"inventory-api-svc"}`, 0.5},
+		// AN2: only "value" is graded, since the prompt's own JSON template
+		// pre-fills "field":"selector.app" - copying that back verbatim is
+		// not evidence of diagnosis, so it no longer earns partial credit
+		// on its own (this case used to score 0.5).
+		{"right field, wrong replacement value", `{"field":"selector.app","value":"inventory-api-svc"}`, 0},
+		{"field name wrong but value correct still scores full credit (AN2 bug probe)", `{"field":"wrongfield","value":"inventory-api"}`, 1},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
@@ -327,6 +332,7 @@ func TestK8sTraefikIngressRouteHostTest_Eval(t *testing.T) {
 		{"uppercase Host keyword variant", "MATCH: HOST(`app.raczylo.com`)", 1},
 		{"still the old domain", "match: Host(`app.example.com`)", 0},
 		{"right function, wrong domain typo", "match: Host(`app.raczlo.com`)", 0},
+		{"double-quoted domain (AN4 bug probe)", `match: Host("app.raczylo.com")`, 1},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
@@ -349,6 +355,11 @@ func TestK8sQoSClassTest_Eval(t *testing.T) {
 		{"exact word", "Burstable", 1},
 		{"lowercase", "burstable", 1},
 		{"trailing whitespace", "Burstable\n", 1},
+		// A7 regression: fenced/quoted/bolded/trailing-period decoration
+		// on the correct answer must still score full credit.
+		{"fenced", "```\nBurstable\n```", 1},
+		{"quoted", `"Burstable"`, 1},
+		{"bolded with period", "**Burstable**.", 1},
 		{"wrong class, mistakes it for Guaranteed", "Guaranteed", 0},
 		{"wrong class, mistakes it for BestEffort", "BestEffort", 0},
 	}
@@ -375,6 +386,8 @@ func TestK8sCNPGPDBTest_Eval(t *testing.T) {
 		{"english phrasing of the value", "Create a PodDisruptionBudget with a minAvailable of 2.", 1},
 		{"wrong resource entirely (a NetworkPolicy cannot do this)", "Use a NetworkPolicy with minAvailable: 2.", 0.5},
 		{"right resource, wrong/missing value", "Use a PodDisruptionBudget, but leave the fields at their defaults.", 0.5},
+		{"'set to' phrasing, not covered by the old enumerated substring list (A10 bug probe)", "Use a PodDisruptionBudget with minAvailable set to 2.", 1},
+		{"quoted numeric value in YAML (A10 bug probe)", `Use a PodDisruptionBudget with minAvailable: "2".`, 1},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {

internal/tests/linux.go

@@ -1,6 +1,9 @@
 package tests
 
 import (
+	"context"
+	"regexp"
+
 	"github.com/lukaszraczylo/llm-testbench/internal/eval"
 	"github.com/lukaszraczylo/llm-testbench/internal/testkit"
 )
@@ -74,11 +77,19 @@ contents only.`
 	}
 }
 
-// linuxJournalctlFilterPattern requires journalctl filtered to unit
+// linuxJournalctlUnitPattern matches the unit filter in either short (-u)
-// nginx.service and a since-window of 2 hours, in the canonical
+// or long (--unit) form, tolerant of "=" or whitespace between the option
-// "-u UNIT --since WINDOW" order (the overwhelmingly common idiom), on a
+// and its value (journalctl, via getopt_long, accepts both). It is checked
-// single line so a scattered mention across paragraphs does not count.
+// independently of linuxJournalctlSincePattern below (A3), so either
-const linuxJournalctlFilterPattern = `(?i)\bjournalctl\b[^\n]*?-u\s+nginx\.service\b[^\n]*?--since\s+["']?-?2\s*h(?:ours?)?(?:\s+ago)?["']?`
+// option can appear on the line before the other - the original single
+// regex hardcoded "-u ... --since ..." order and rejected the equally
+// valid "--unit"/"-S" long/short forms and "--since=" spelling.
+const linuxJournalctlUnitPattern = `(?i)\bjournalctl\b[^\n]*(?:-u|--unit)[=\s]+nginx\.service\b`
+
+// linuxJournalctlSincePattern matches the since-window filter in either
+// short (-S) or long (--since) form, independent of
+// linuxJournalctlUnitPattern (A3).
+const linuxJournalctlSincePattern = `(?i)\bjournalctl\b[^\n]*(?:-S|--since)[=\s]+["']?-?2\s*h(?:ours?)?(?:\s+ago)?["']?`
 
 // linuxJournalctlFilterTest: one-line journalctl command filtered to one
 // unit over a relative time window.
@@ -86,13 +97,18 @@ func linuxJournalctlFilterTest() testkit.Test {
 	prompt := `Give a single, one-line journalctl command that shows only
 log entries for the systemd unit "nginx.service" from the last 2 hours.`
 
+	evaluator := eval.All(
+		eval.W(eval.Regex(linuxJournalctlUnitPattern), 1),
+		eval.W(eval.Regex(linuxJournalctlSincePattern), 1),
+	)
+
 	return testkit.Test{
 		ID:          "linux-journalctl-filter",
 		Category:    "operations",
 		Subcategory: "linux",
 		Description: "One-line journalctl command filtered by unit and a relative 2-hour time window.",
 		Prompt:      prompt,
-		Eval:        eval.Regex(linuxJournalctlFilterPattern),
+		Eval:        evaluator,
 	}
 }
 
@@ -138,11 +154,16 @@ JSON object: {"oom_killed": true or false, "oom_kill_count": <number>}.`
 
 // linuxNftablesDNATPattern requires a DNAT rule (iptables or nft form) on
 // the nat table's prerouting chain, matching TCP port 8080 and forwarding
-// to 10.0.5.20:80, on a single line. iptables and nft flags/keywords follow
+// to 10.0.5.20:80, on a single line. The "nat" table and "prerouting"
-// a fixed canonical order (table/chain, then match, then target), so this
+// chain tokens are matched order-free relative to each other (A12): a
-// uses the same non-greedy same-line anchoring as the pct-exec lesson
+// syntactically valid iptables invocation can give -A PREROUTING before
-// rather than a permutation of independently-orderable flags.
+// -t nat (e.g. "iptables -A PREROUTING -t nat -p tcp --dport 8080 -j DNAT
-const linuxNftablesDNATPattern = `(?i)(?:iptables|nft)\b[^\n]*?nat[^\n]*?prerouting[^\n]*?(?:--dport|dport)\s*8080[^\n]*?dnat[^\n]*?10\.0\.5\.20:80`
+// --to-destination 10.0.5.20:80"), and the original pattern's fixed
+// "nat...prerouting" order rejected that ordering. The rest of the match
+// (dport, dnat, destination) keeps its original same-line, in-order
+// anchoring, since iptables and nft do put the match/target keywords after
+// the table/chain in both valid orderings.
+const linuxNftablesDNATPattern = `(?i)(?:iptables|nft)\b[^\n]*?(?:nat[^\n]*?prerouting|prerouting[^\n]*?nat)[^\n]*?(?:--dport|dport)\s*8080[^\n]*?dnat[^\n]*?10\.0\.5\.20:80`
 
 // linuxNftablesDNATTest: one-line DNAT rule forwarding an external port to
 // an internal host:port.
@@ -194,9 +215,17 @@ data. Respond with only a JSON array of the step ids, e.g.
 	}
 }
 
-// linuxSSHPortForwardPattern requires an ssh local port-forward (-L) from
+// linuxSSHPortForwardSpecPattern matches the -L local-forward flag and its
-// local port 15432 to the remote Postgres, connecting to the correct host.
+// argument, independent of where the SSH destination appears on the line -
-const linuxSSHPortForwardPattern = `(?i)\bssh\b[^\n]*?-L\s*15432:(?:127\.0\.0\.1|localhost):5432[^\n]*?ops@db1\.internal`
+// -L can come before or after the destination in a valid ssh invocation
+// (A6).
+const linuxSSHPortForwardSpecPattern = `(?i)\bssh\b[^\n]*-L\s*15432:(?:127\.0\.0\.1|localhost):5432\b`
+
+// linuxSSHPortForwardDestPattern matches the SSH destination in either its
+// "user@host" form or the equivalent "-l user ... host" form (A6): both
+// name the same destination, and the original pattern only accepted the
+// former.
+const linuxSSHPortForwardDestPattern = `(?i)\bssh\b[^\n]*(?:ops@db1\.internal|-l\s+ops\b[^\n]*\bdb1\.internal\b)`
 
 // linuxSSHPortForwardTest: ssh -L local port-forward command derived from a
 // scenario.
@@ -207,13 +236,18 @@ single, one-line ssh command, run from your laptop, that forwards your
 laptop's local port 15432 to that remote Postgres instance, so a local
 client can connect to localhost:15432.`
 
+	evaluator := eval.Mean(
+		eval.Regex(linuxSSHPortForwardSpecPattern),
+		eval.Regex(linuxSSHPortForwardDestPattern),
+	)
+
 	return testkit.Test{
 		ID:          "linux-ssh-port-forward",
 		Category:    "operations",
 		Subcategory: "linux",
 		Description: "One-line ssh -L local port-forward command reaching a Postgres instance that only listens on remote loopback.",
 		Prompt:      prompt,
-		Eval:        eval.Regex(linuxSSHPortForwardPattern),
+		Eval:        evaluator,
 	}
 }
 
@@ -221,7 +255,26 @@ client can connect to localhost:15432.`
 // "03:30 every Monday", tolerant only of whitespace width between fields -
 // unlike eval.Equals, this stays correct even if the model adds a leading
 // or trailing space, since the fields themselves have no valid rephrasing.
-const linuxCronExpressionPattern = `^\s*30\s+3\s+\*\s+\*\s+1\s*$`
+// (?m) makes ^/$ match per line rather than only at the very start/end of
+// the whole string, so the expression still matches as one line within a
+// longer, fenced, or multi-line response (A5).
+const linuxCronExpressionPattern = `(?m)^\s*30\s+3\s+\*\s+\*\s+1\s*$`
+
+// linuxCronExpressionEval strips a markdown code fence (if any) before
+// applying linuxCronExpressionPattern. Without this, a response that
+// wraps the correct expression in a ` + "```" + ` fence scored 0, because the fence
+// markers and surrounding whitespace kept the line-anchored regex from
+// finding a clean match (A5).
+func linuxCronExpressionEval() eval.Evaluator {
+	re := regexp.MustCompile(linuxCronExpressionPattern)
+	return eval.EvaluatorFunc(func(_ context.Context, response string) eval.Score {
+		stripped := eval.ExtractCodeBlock(response, "")
+		if re.MatchString(stripped) {
+			return eval.Score{Value: 1, Detail: "matches the 5-field cron expression"}
+		}
+		return eval.Score{Value: 0, Detail: "does not match the 5-field cron expression"}
+	})
+}
 
 // linuxCronExpressionTest: produce the 5-field cron expression matching a
 // stated schedule.
@@ -241,7 +294,7 @@ with only the expression, nothing else.`
 		Subcategory: "linux",
 		Description: "Produce the 5-field cron expression for a 03:30-every-Monday schedule.",
 		Prompt:      prompt,
-		Eval:        eval.Regex(linuxCronExpressionPattern),
+		Eval:        linuxCronExpressionEval(),
 	}
 }
 
@@ -258,11 +311,15 @@ unit's WantedBy needs.`
 	// paired with a same-named .service unit (the work). The .timer's
 	// OnCalendar= key sets the schedule (e.g. "*-*-* 02:00:00" for daily at
 	// 02:00), and its [Install] section needs WantedBy=timers.target for
-	// systemctl enable to activate it.
+	// systemctl enable to activate it. The WantedBy check accepts "=",
+	// ":", or "(is) set to" between the key and its value (A15), not just a
+	// bare "WantedBy=timers.target" substring - a response that phrases
+	// this in prose ("WantedBy is set to timers.target") states exactly the
+	// same fact.
 	evaluator := eval.Mean(
 		eval.ContainsAll(".service", ".timer"),
 		eval.ContainsAny("OnCalendar"),
-		eval.ContainsAll("WantedBy=timers.target"),
+		eval.Regex(`(?i)WantedBy\s*(?:=|:|\b(?:is\s+)?set to\b)\s*timers\.target`),
 	)
 
 	return testkit.Test{
@@ -284,7 +341,12 @@ Buffers:          512000 kB
 Cached:          3200000 kB`
 
 // linuxProcMeminfoAvailableWant is MemAvailable (9876000 kB) converted to
-// whole megabytes, floored: 9876000 / 1024 = 9644.53125 -> 9644.
+// whole megabytes, floored: 9876000 / 1024 = 9644.53125 -> 9644. The
+// prompt asks for the value rounded down, so 9644 is the single exact
+// correct answer - tolerance 0 below, not 1 (A17): a tolerance of 1 was
+// letting off-by-one wrong answers (9643 or 9645) score full credit for a
+// deterministic arithmetic conversion with no room for a legitimate
+// rephrasing.
 var linuxProcMeminfoAvailableWant = 9876000 / 1024
 
 // linuxProcMeminfoAvailableTest: interpret /proc/meminfo's MemAvailable
@@ -312,6 +374,6 @@ so convert by dividing by 1024, not 1000. Respond with only the number.`
 		Subcategory: "linux",
 		Description: "Convert /proc/meminfo's MemAvailable field (not MemFree) to whole megabytes.",
 		Prompt:      prompt,
-		Eval:        eval.Numeric(eval.ExtractLastNumber[int], linuxProcMeminfoAvailableWant, 1),
+		Eval:        eval.Numeric(eval.ExtractLastNumber[int], linuxProcMeminfoAvailableWant, 0),
 	}
 }

internal/tests/linux_test.go

@@ -138,8 +138,16 @@ func TestLinuxJournalctlFilterTest_Eval(t *testing.T) {
 		{"quoted negative window", `journalctl -u nginx.service --since "-2 hours"`, 1},
 		{"english phrasing window", `journalctl -u nginx.service --since "2 hours ago"`, 1},
 		{"compact -2h form, no quotes", `Run: journalctl -u nginx.service --since -2h`, 1},
-		{"wrong unit", `journalctl -u apache.service --since "-2 hours"`, 0},
+		// The unit and since checks are independent (A3): a response that
-		{"missing unit filter entirely", `journalctl --since "-2 hours"`, 0},
+		// nails one but not the other now earns that check's half-credit
+		// rather than scoring 0 outright, the same partial-credit
+		// convention eval.All already uses elsewhere in this file.
+		{"wrong unit still credits the correct since-window", `journalctl -u apache.service --since "-2 hours"`, 0.5},
+		{"missing unit filter still credits the correct since-window", `journalctl --since "-2 hours"`, 0.5},
+		{"missing both filters entirely scores 0", `journalctl -f`, 0},
+		{"long-form --unit= (A3 bug probe)", `journalctl --unit=nginx.service --since=-2h`, 1},
+		{"short-form -S (A3 bug probe)", `journalctl -u nginx.service -S -2h`, 1},
+		{"since option before unit option, order-free (A3 bug probe)", `journalctl --since "-2 hours" -u nginx.service`, 1},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
@@ -208,6 +216,11 @@ func TestLinuxNftablesDNATTest_Eval(t *testing.T) {
 			response: `iptables -t nat -A OUTPUT -p tcp --dport 8080 -j DNAT --to-destination 10.0.5.20:80`,
 			want:     0,
 		},
+		{
+			name:     "-A PREROUTING given before -t nat is equally valid iptables syntax (A12 bug probe)",
+			response: `iptables -A PREROUTING -t nat -p tcp --dport 8080 -j DNAT --to-destination 10.0.5.20:80`,
+			want:     1,
+		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
@@ -254,8 +267,13 @@ func TestLinuxSSHPortForwardTest_Eval(t *testing.T) {
 		{"loopback IP form", `ssh -L 15432:127.0.0.1:5432 [email protected]`, 1},
 		{"localhost form", `ssh -L 15432:localhost:5432 [email protected]`, 1},
 		{"extra flags before -L", `ssh -N -L 15432:127.0.0.1:5432 [email protected]`, 1},
-		{"wrong local port", `ssh -L 5432:127.0.0.1:5432 [email protected]`, 0},
+		// The forward-spec and destination checks are independent (A6):
-		{"wrong host", `ssh -L 15432:127.0.0.1:5432 [email protected]`, 0},
+		// getting one right and the other wrong now earns that check's
+		// half-credit instead of scoring 0 outright.
+		{"wrong local port still credits the correct destination", `ssh -L 5432:127.0.0.1:5432 [email protected]`, 0.5},
+		{"wrong host still credits the correct forward spec", `ssh -L 15432:127.0.0.1:5432 [email protected]`, 0.5},
+		{"destination given before -L, options-after-destination (A6 bug probe)", `ssh [email protected] -L 15432:127.0.0.1:5432`, 1},
+		{"-l user form for the destination (A6 bug probe)", `ssh -L 15432:127.0.0.1:5432 -l ops db1.internal`, 1},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
@@ -280,6 +298,8 @@ func TestLinuxCronExpressionTest_Eval(t *testing.T) {
 		{"tab-separated fields", "30\t3\t*\t*\t1", 1},
 		{"wrong day of week", "30 3 * * 2", 0},
 		{"wrong time", "45 3 * * 1", 0},
+		{"fenced code block, no language tag (A5 bug probe)", "```\n30 3 * * 1\n```", 1},
+		{"fenced code block with a language tag (A5 bug probe)", "```cron\n30 3 * * 1\n```", 1},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
@@ -326,6 +346,16 @@ WantedBy=timers.target.`,
 			response: `Just add a backup.service and backup.timer, and enable it with WantedBy=multi-user.target.`,
 			want:     1.0 / 3.0,
 		},
+		{
+			name:     "WantedBy with a colon separator (A15 bug probe)",
+			response: `backup.service + backup.timer, OnCalendar=*-*-* 02:00:00, WantedBy: timers.target`,
+			want:     1,
+		},
+		{
+			name:     "WantedBy phrased as 'set to' in prose (A15 bug probe)",
+			response: `Use backup.service and backup.timer. OnCalendar=02:00 daily. WantedBy is set to timers.target under [Install].`,
+			want:     1,
+		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
@@ -350,6 +380,8 @@ func TestLinuxProcMeminfoAvailableTest_Eval(t *testing.T) {
 		{"with unit suffix", "9644 MB", 1},
 		{"divided by 1000 instead of 1024", "9876", 0},
 		{"used MemFree instead of MemAvailable", "1000", 0},
+		{"off-by-one must not pass now that tolerance is 0, not 1 (A17 bug probe)", "9645", 0},
+		{"off-by-one the other direction must not pass (A17 bug probe)", "9643", 0},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {

internal/tests/macos.go

@@ -1,6 +1,10 @@
 package tests
 
 import (
+	"context"
+	"regexp"
+	"strings"
+
 	"github.com/lukaszraczylo/llm-testbench/internal/eval"
 	"github.com/lukaszraczylo/llm-testbench/internal/testkit"
 )
@@ -95,7 +99,12 @@ file belongs in for a per-machine (not per-user) job.`
 // single-quoted or double-quoted extension argument, invoking a foo->bar
 // substitution on file.txt, on a single line (no DOTALL) so a scattered,
 // multi-paragraph mention of the same tokens does not count as a real
-// one-line command.
+// one-line command. The substitution's delimiter is accepted as any of the
+// common sed delimiter characters (/, |, comma, #, colon, semicolon, @),
+// not just "/" - "s/foo/bar/" is the conventional example, but any
+// consistently-used delimiter is equally correct BSD sed syntax (AN3). Go's
+// RE2 engine has no backreferences, so each delimiter choice is spelled out
+// as its own alternative rather than captured-and-reused.
 //
 // ground truth: BSD sed's -i flag takes a mandatory in-place-suffix
 // argument (used for backup files); passing an empty single-quoted or
@@ -104,7 +113,7 @@ file belongs in for a per-machine (not per-user) job.`
 // "sed -i 's/foo/bar/' file.txt" (no argument to -i) works on Linux but
 // errors on macOS, where it is parsed as "-i 's/foo/bar/'" (using the sed
 // script itself as the backup suffix) followed by a missing script operand.
-const macosSedPattern = `(?i)\bsed\b[^\n]*-i\s*(''|"")[^\n]*s/foo/bar/g?[^\n]*file\.txt`
+const macosSedPattern = `(?i)\bsed\b[^\n]*-i\s*(''|"")[^\n]*(?:s/foo/bar/|s\|foo\|bar\||s,foo,bar,|s#foo#bar#|s:foo:bar:|s;foo;bar;|s@foo@bar@)g?[^\n]*file\.txt`
 
 // macosSedInplaceTest: give the portable BSD sed one-liner for in-place
 // substitution with no backup file, contrasted with the GNU-only form that
@@ -158,7 +167,7 @@ explicit domain targeting).`
 	evaluator := eval.Mean(
 		eval.ContainsAny("bootstrap"),
 		eval.ContainsAny("bootstrap system", "system domain", "domain system", "target system", "system target"),
-		eval.ContainsAny("deprecated", "avoid launchctl load", "no longer recommended", "outdated"),
+		eval.ContainsAny("deprecated", "avoid launchctl load", "no longer recommended", "outdated", "legacy", "superseded", "obsolete", "replaced"),
 	)
 
 	return testkit.Test{
@@ -171,9 +180,39 @@ explicit domain targeting).`
 	}
 }
 
-// macosPlutilPattern requires the built-in plutil -p pretty-printer
+// macosPlutilFlagPattern requires a properly bounded "-p" flag token
-// invoked directly on the plist path, on one line.
+// (immediately preceded by whitespace or the start of the response, and
-const macosPlutilPattern = `(?i)\bplutil\b[^\n]*-p[^\n]*/Library/Preferences/com\.example\.app\.plist`
+// immediately followed by whitespace or the end of the response). An
+// unbounded substring search for "-p" also matches inside the word
+// "pretty-print" (its internal hyphen followed by "p"), which would let a
+// response that only talks *about* pretty-printing - without actually
+// invoking "plutil -p" - score a false positive (A2).
+var macosPlutilFlagPattern = regexp.MustCompile(`(?i)(?:^|\s)-p(?:\s|$)`)
+
+// macosPlutilPlistReadEval accepts three materially correct ways to read a
+// plist file's contents in human-readable form using only macOS-bundled
+// tools: "plutil -p <path>" (with -p bounded as an actual flag token, not
+// matched as a substring of "pretty-print"), "plutil -convert xml1 -o -
+// <path>" (convert to XML and write to stdout instead of editing in
+// place), and "defaults read <domain-or-path>" (A2).
+func macosPlutilPlistReadEval() eval.Evaluator {
+	return eval.EvaluatorFunc(func(_ context.Context, response string) eval.Score {
+		if !strings.Contains(response, "com.example.app") {
+			return eval.Score{Value: 0, Detail: "does not name the target plist/domain"}
+		}
+		lower := strings.ToLower(response)
+		switch {
+		case strings.Contains(lower, "plutil") && macosPlutilFlagPattern.MatchString(response):
+			return eval.Score{Value: 1, Detail: "plutil -p (a properly bounded flag token)"}
+		case strings.Contains(lower, "plutil") && strings.Contains(lower, "-convert") && strings.Contains(lower, "xml1"):
+			return eval.Score{Value: 1, Detail: "plutil -convert xml1"}
+		case strings.Contains(lower, "defaults read"):
+			return eval.Score{Value: 1, Detail: "defaults read"}
+		default:
+			return eval.Score{Value: 0, Detail: "no recognized plist-reading command"}
+		}
+	})
+}
 
 // macosPlutilPlistReadTest: pretty-print a plist file's contents using a
 // tool that ships with macOS (no Homebrew installs).
@@ -190,17 +229,39 @@ that prints the plist's contents to your terminal in a readable
 	// ground truth: plutil ships with macOS and its -p flag pretty-prints a
 	// plist (binary or XML) to stdout in a human-readable form, taking the
 	// file path directly - no domain-name lookup or Homebrew install
-	// needed, unlike "defaults read <domain>".
+	// needed, unlike "defaults read <domain>". "plutil -convert xml1 -o -"
+	// (convert to XML, write to stdout) and "defaults read" are also
+	// accepted as materially correct alternatives (A2).
 	return testkit.Test{
 		ID:          "macos-plutil-plist-read",
 		Category:    "operations",
 		Subcategory: "macos",
 		Description: "Pretty-print a plist file's contents using the built-in plutil -p command.",
 		Prompt:      prompt,
-		Eval:        eval.Regex(macosPlutilPattern),
+		Eval:        macosPlutilPlistReadEval(),
 	}
 }
 
+// macosMdfindSpotlightEval gates the whole score on mdfind actually being
+// used: the prompt itself supplies both a plausible flag ("-name" is a real
+// "find" flag too) and the search term ("invoice"), so a bare filesystem
+// walk with "find" that happens to echo those prompt-supplied tokens must
+// not be able to reach a passing score. Only once mdfind is confirmed
+// present does the response get scored on the remaining, more permissive
+// criteria (A13).
+func macosMdfindSpotlightEval() eval.Evaluator {
+	sub := eval.Mean(
+		eval.ContainsAny("-onlyin", "-name"),
+		eval.ContainsAny("invoice"),
+	)
+	return eval.EvaluatorFunc(func(ctx context.Context, response string) eval.Score {
+		if !strings.Contains(strings.ToLower(response), "mdfind") {
+			return eval.Score{Value: 0, Detail: "mdfind not used; a filesystem walk is not what was asked for"}
+		}
+		return sub.Evaluate(ctx, response)
+	})
+}
+
 // macosMdfindSpotlightTest: use mdfind (Spotlight's metadata index) instead
 // of a full filesystem walk to find files by name.
 func macosMdfindSpotlightTest() testkit.Test {
@@ -211,20 +272,16 @@ whose name contains "invoice", scoped only to your home directory (~).`
 	// ground truth: mdfind queries the already-built Spotlight index rather
 	// than walking the filesystem; -onlyin scopes the search to a
 	// directory, and either -name or a plain query string matches on file
-	// name.
+	// name. mdfind's presence is a hard gate (A13): "-name" is also a real
-	evaluator := eval.Mean(
+	// "find" flag, and "invoice" is echoed straight from the prompt, so
-		eval.ContainsAll("mdfind"),
+	// neither can carry the score for a wrong, find-based answer.
-		eval.ContainsAny("-onlyin", "-name"),
-		eval.ContainsAny("invoice"),
-	)
-
 	return testkit.Test{
 		ID:          "macos-mdfind-spotlight",
 		Category:    "operations",
 		Subcategory: "macos",
 		Description: "Use mdfind against Spotlight's index, scoped to a directory, instead of a full filesystem walk.",
 		Prompt:      prompt,
-		Eval:        evaluator,
+		Eval:        macosMdfindSpotlightEval(),
 	}
 }
 
@@ -238,11 +295,14 @@ the command to create a local snapshot now, and the separate command to
 list the local snapshots that currently exist.`
 
 	// ground truth: tmutil manages local APFS snapshots independently of
-	// having a Time Machine backup destination attached. "tmutil snapshot"
+	// having a Time Machine backup destination attached. "tmutil
-	// creates one immediately; "tmutil listlocalsnapshots /" lists the
+	// localsnapshot" is the current, documented verb that creates one
-	// local snapshots that exist for the root volume.
+	// immediately ("tmutil snapshot" does not appear in tmutil's own usage
+	// listing on current macOS and is accepted only as a legacy/undocumented
+	// alias - verified on this machine, A1); "tmutil listlocalsnapshots /"
+	// lists the local snapshots that exist for the root volume.
 	evaluator := eval.Mean(
-		eval.ContainsAny("tmutil snapshot"),
+		eval.ContainsAny("tmutil localsnapshot", "tmutil snapshot"),
 		eval.ContainsAny("tmutil listlocalsnapshots", "listlocalsnapshots"),
 	)
 
@@ -300,11 +360,13 @@ tool built into macOS, that does this.`
 	// ground truth: caffeinate, bundled with macOS, takes a power-management
 	// assertion for as long as a given utility runs when you pass that
 	// utility as its argument, and releases the assertion automatically
-	// when the utility exits - no manual cleanup needed.
+	// when the utility exits - no manual cleanup needed. migrate.sh must be
-	evaluator := eval.Mean(
+	// caffeinate's own argument on the same command line (e.g.
-		eval.ContainsAll("caffeinate"),
+	// "caffeinate ./migrate.sh"), not merely mentioned somewhere in the
-		eval.ContainsAny("migrate.sh"),
+	// response: a backgrounded "caffeinate & ./migrate.sh" variant runs the
-	)
+	// two independently, so caffeinate's assertion does not track the
+	// script's lifetime and would not auto-release when it exits (A11).
+	evaluator := eval.Regex(`(?i)\bcaffeinate\b[^\n&;|]*\.?/?migrate\.sh`)
 
 	return testkit.Test{
 		ID:          "macos-caffeinate",
@@ -317,8 +379,11 @@ tool built into macOS, that does this.`
 }
 
 // macosStatPattern requires the BSD stat -f format-string form printing
-// only the byte size (%z) of file.txt.
+// only the byte size (%z) of file.txt. The %z format specifier is accepted
-const macosStatPattern = `(?i)\bstat\b[^\n]*-f\s*"?%z"?[^\n]*file\.txt`
+// bare or wrapped in either single or double quotes (both are valid shell
+// quoting for a format string with no special characters in it) - not just
+// double quotes (A4).
+const macosStatPattern = `(?i)\bstat\b[^\n]*-f\s*['"]?%z['"]?[^\n]*file\.txt`
 
 // macosStatBSDGNUTest: give the BSD stat command (macOS's built-in stat)
 // for a file's size in bytes, and name the differing GNU stat flag.

internal/tests/macos_test.go

@@ -92,6 +92,9 @@ func TestMacosSedInplaceTest_Eval(t *testing.T) {
 		{"global replace flag still matches", `sed -i '' 's/foo/bar/g' file.txt`, 1},
 		{"GNU-style with no extension argument is wrong on macOS", `sed -i 's/foo/bar/' file.txt`, 0},
 		{"backup extension creates a file, contradicting the requirement", `sed -i.bak 's/foo/bar/' file.txt`, 0},
+		{"pipe delimiter is equally valid sed syntax (AN3)", `sed -i '' 's|foo|bar|' file.txt`, 1},
+		{"comma delimiter (AN3)", `sed -i '' 's,foo,bar,g' file.txt`, 1},
+		{"hash delimiter (AN3)", `sed -i "" 's#foo#bar#' file.txt`, 1},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
@@ -128,6 +131,10 @@ Apple deprecated for lacking an explicit domain target.`
 		{"good answer, alternate phrasing", goodPhrasing, 1},
 		{"wrong subcommand (deprecated load only) scores 0", badWrongSubcommand, 0},
 		{"missing domain target and deprecation note loses two thirds", badMissingDomain, 1.0 / 3.0},
+		{"'legacy' deprecation cue (A14)", `Use launchctl bootstrap system /Library/LaunchDaemons/com.example.myjob.plist; launchctl load is the legacy subcommand.`, 1},
+		{"'superseded' deprecation cue (A14)", `launchctl bootstrap, system domain target, superseded launchctl load.`, 1},
+		{"'obsolete' deprecation cue (A14)", `bootstrap system domain target; launchctl load is obsolete.`, 1},
+		{"'replaced' deprecation cue (A14)", `launchctl bootstrap system domain target replaced launchctl load.`, 1},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
@@ -152,6 +159,9 @@ func TestMacosPlutilPlistReadTest_Eval(t *testing.T) {
 		{"quoted path", `plutil -p "/Library/Preferences/com.example.app.plist"`, 1},
 		{"wrong tool entirely (cat prints binary garbage)", `cat /Library/Preferences/com.example.app.plist`, 0},
 		{"plutil without -p just validates, does not print", `plutil /Library/Preferences/com.example.app.plist`, 0},
+		{"plutil -convert xml1 to stdout is a materially correct alternative (A2)", `plutil -convert xml1 -o - /Library/Preferences/com.example.app.plist`, 1},
+		{"defaults read is a materially correct alternative (A2)", `defaults read com.example.app`, 1},
+		{"mentions pretty-print in prose without invoking -p as a bounded flag (A2 bug probe)", `plutil can pretty-print the plist at /Library/Preferences/com.example.app.plist, though I don't recall the exact flag.`, 0},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
@@ -174,8 +184,9 @@ func TestMacosMdfindSpotlightTest_Eval(t *testing.T) {
 		{"onlyin with query", `mdfind -onlyin ~ "invoice"`, 1},
 		{"name flag", `mdfind -onlyin ~ -name invoice`, 1},
 		{"prose wrapped", `Run mdfind -onlyin ~ invoice to search the Spotlight index.`, 1},
-		{"uses find instead of mdfind, walks the filesystem", `find ~ -iname "*invoice*"`, 1.0 / 3.0},
+		{"uses find instead of mdfind, walks the filesystem, gated to 0 (A13)", `find ~ -iname "*invoice*"`, 0},
-		{"mdfind used but searching for the wrong term entirely", `mdfind -onlyin ~ "receipt"`, 2.0 / 3.0},
+		{"mdfind used but searching for the wrong term entirely", `mdfind -onlyin ~ "receipt"`, 0.5},
+		{"find command echoing both prompt-supplied tokens must not pass the gate (A13 bug probe)", `find ~ -name "*invoice*"`, 0},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
@@ -200,6 +211,8 @@ func TestMacosAPFSSnapshotTest_Eval(t *testing.T) {
 		{"listlocalsnapshots without explicit tmutil prefix restated", "First tmutil snapshot, then check with listlocalsnapshots /.", 1},
 		{"only creates, never lists", "Just run tmutil snapshot.", 0.5},
 		{"wrong tool (diskutil) mentioned instead of tmutil", "Use diskutil apfs snapshot / to create one.", 0},
+		{"current documented localsnapshot verb (A1)", "Create: tmutil localsnapshot\nList: tmutil listlocalsnapshots /", 1},
+		{"localsnapshot only, never lists (A1)", "Just run tmutil localsnapshot.", 0.5},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
@@ -266,8 +279,9 @@ func TestMacosCaffeinateTest_Eval(t *testing.T) {
 		{"bare invocation", `caffeinate ./migrate.sh`, 1},
 		{"with system-sleep flag", `caffeinate -s ./migrate.sh`, 1},
 		{"prose wrapped", "Run `caffeinate -i ./migrate.sh` and it stops holding the assertion once migrate.sh exits.", 1},
-		{"wrong script named, not the target", `caffeinate ./other-script.sh`, 0.5},
+		{"wrong script named, not the target, scores 0 now that migrate.sh must be on the same line (A11)", `caffeinate ./other-script.sh`, 0},
 		{"unrelated tool, does not auto-release on exit", `Just run pmset noidle before starting your script.`, 0},
+		{"backgrounded caffeinate is a broken answer and must not pass (A11 bug probe)", `caffeinate & ./migrate.sh`, 0},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
@@ -292,6 +306,7 @@ func TestMacosStatBSDGNUTest_Eval(t *testing.T) {
 		{"prose wrapped", "Run stat -f%z file.txt on macOS; the GNU equivalent flag is -c.", 1},
 		{"correct BSD command but never names the GNU flag", `stat -f%z file.txt`, 0.5},
 		{"GNU -c%s syntax used on macOS scores half: -c token matches but -f BSD form is missing", `stat -c%s file.txt`, 0.5},
+		{"single-quoted format string (A4)", `stat -f '%z' file.txt, and on Linux GNU stat uses -c.`, 1},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {

internal/tests/python.go

@@ -234,7 +234,9 @@ printed, nothing else.`
 		Subcategory: "python",
 		Description: "Trace the classic mutable-default-argument bug across three calls sharing one list.",
 		Prompt:      prompt,
-		Eval:        eval.Equals("[1, 2, 3]"),
+		// A7: eval.ExactToken so a fenced or quoted rendering of the exact
+		// printed text still scores full credit.
+		Eval: eval.ExactToken("[1, 2, 3]"),
 	}
 }
 
@@ -270,7 +272,9 @@ printed, nothing else.`
 		Subcategory: "python",
 		Description: "Trace the exact output of a length-filtered dict comprehension whose items are sorted and printed.",
 		Prompt:      prompt,
-		Eval:        eval.Equals("[('apple', 5), ('banana', 6), ('cherry', 6), ('date', 4)]"),
+		// A7: eval.ExactToken so a fenced or quoted rendering of the exact
+		// printed text still scores full credit.
+		Eval: eval.ExactToken("[('apple', 5), ('banana', 6), ('cherry', 6), ('date', 4)]"),
 	}
 }
 
@@ -313,7 +317,9 @@ printed, nothing else.`
 		Subcategory: "python",
 		Description: "Trace the exact output of iterating a generator a second time after it is already exhausted.",
 		Prompt:      prompt,
-		Eval:        eval.Equals("[0, 1, 2] []"),
+		// A7: eval.ExactToken so a fenced or quoted rendering of the exact
+		// printed text still scores full credit.
+		Eval: eval.ExactToken("[0, 1, 2] []"),
 	}
 }
 
@@ -345,9 +351,15 @@ keeping the same behavior: recursively find every ".yaml" file under
 base_dir and return their full paths. Respond with only the rewritten
 function.`
 
+	// A16: Path.rglob("*.yaml") and Path.glob("**/*.yaml") are behaviorally
+	// equivalent recursive searches ("**/" is glob's own recursive
+	// wildcard), so the recursive-method check accepts either instead of
+	// only the literal substring "rglob"; the "*.yaml" pattern is checked
+	// separately from which method carries it.
 	evaluator := eval.All(
 		eval.W(eval.ContainsAny("from pathlib import Path", "import pathlib"), 1),
-		eval.W(eval.ContainsAll("rglob", "*.yaml"), 2),
+		eval.W(eval.ContainsAny("rglob", `glob("**/`, "glob('**/"), 1),
+		eval.W(eval.ContainsAll("*.yaml"), 1),
 	)
 
 	return testkit.Test{

internal/tests/python_test.go

@@ -254,6 +254,10 @@ func TestPyMutableDefaultArgTest_Eval(t *testing.T) {
 		{"exact correct", "[1, 2, 3]", 1},
 		{"correct with surrounding whitespace", "  [1, 2, 3]  ", 1},
 		{"correct with trailing newline", "[1, 2, 3]\n", 1},
+		// A7 regression: fenced/quoted decoration on the correct answer
+		// must still score full credit.
+		{"fenced", "```\n[1, 2, 3]\n```", 1},
+		{"quoted", `"[1, 2, 3]"`, 1},
 		{"wrong: assumes a fresh list per call", "[3]", 0},
 		{"wrong: only counts explicit-arg calls", "[1, 2]", 0},
 	}
@@ -293,6 +297,9 @@ func TestPyDictComprehensionTraceTest_Eval(t *testing.T) {
 		{"exact correct", "[('apple', 5), ('banana', 6), ('cherry', 6), ('date', 4)]", 1},
 		{"correct with surrounding whitespace", "  [('apple', 5), ('banana', 6), ('cherry', 6), ('date', 4)]  ", 1},
 		{"correct uppercase-folded", "[('APPLE', 5), ('BANANA', 6), ('CHERRY', 6), ('DATE', 4)]", 1},
+		// A7 regression: fenced decoration on the correct answer must
+		// still score full credit.
+		{"fenced", "```\n[('apple', 5), ('banana', 6), ('cherry', 6), ('date', 4)]\n```", 1},
 		{"wrong: includes fig, forgets the length filter", "[('apple', 5), ('banana', 6), ('cherry', 6), ('date', 4), ('fig', 3)]", 0},
 		{"wrong: unsorted, dict insertion order", "[('apple', 5), ('banana', 6), ('cherry', 6), ('fig', 3), ('date', 4)]", 0},
 	}
@@ -331,6 +338,10 @@ func TestPyGeneratorExhaustionTraceTest_Eval(t *testing.T) {
 		{"exact correct", "[0, 1, 2] []", 1},
 		{"correct with surrounding whitespace", "  [0, 1, 2] []  ", 1},
 		{"correct with trailing newline", "[0, 1, 2] []\n", 1},
+		// A7 regression: fenced/quoted decoration on the correct answer
+		// must still score full credit.
+		{"fenced", "```\n[0, 1, 2] []\n```", 1},
+		{"quoted", `"[0, 1, 2] []"`, 1},
 		{"wrong: assumes the generator restarts", "[0, 1, 2] [0, 1, 2]", 0},
 		{"wrong: assumes an error is raised and printed", "StopIteration", 0},
 	}
@@ -370,6 +381,13 @@ def find_configs(base_dir):
     return [str(p) for p in Path(base_dir).glob("*.yaml")]
 ` + "```"
 
+	goodRewriteGlobRecursiveWildcard := "```python\n" + `from pathlib import Path
+
+
+def find_configs(base_dir):
+    return [str(p) for p in Path(base_dir).glob("**/*.yaml")]
+` + "```"
+
 	tests := []struct {
 		name     string
 		response string
@@ -378,7 +396,8 @@ def find_configs(base_dir):
 		{"good rewrite: from pathlib import Path", goodRewrite, 1},
 		{"good rewrite: import pathlib", goodRewriteAltImport, 1},
 		{"wrong: unchanged os.path/os.walk code", stillUsesOsPath, 0},
-		{"partial: uses pathlib but non-recursive glob, not rglob", missingRglob, 2.0 / 3.0}, // (1*1 + 2*0.5)/3: ContainsAny hits, ContainsAll("rglob","*.yaml") matches only "*.yaml"
+		{"partial: uses pathlib but non-recursive glob, not rglob", missingRglob, 2.0 / 3.0}, // (1*1 + 1*0 + 1*1)/3: pathlib import ok, neither rglob nor glob("**/ present, *.yaml ok
+		{"good rewrite: glob(\"**/*.yaml\") is behaviorally equivalent to rglob (A16 bug probe)", goodRewriteGlobRecursiveWildcard, 1},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {

internal/tests/typescript.go

@@ -142,9 +142,19 @@ given an interface with a nested object property, that nested object's own
 properties are also optional). Do not use the "any" type anywhere. Respond
 with only the type definition.`
 
+	// A9: the mapped-type variable name is the author's choice, not
+	// mandated syntax - "K" is one common convention but "P" (as used in
+	// TypeScript's own built-in Partial<T>) is equally correct, so the
+	// recursive-reference check accepts any identifier there rather than
+	// only the literal "DeepPartial<T[K]>" substring. The "keyof T" check
+	// is similarly relaxed to tolerate any whitespace width between the two
+	// words, not just a single space.
 	evaluator := eval.All(
-		eval.W(eval.ContainsAll("DeepPartial<T>", "keyof T"), 2),
+		eval.W(eval.Mean(
-		eval.W(eval.ContainsAny("DeepPartial<T[K]>"), 1),
+			eval.ContainsAll("DeepPartial<T>"),
+			eval.Regex(`(?i)keyof\s+T\b`),
+		), 2),
+		eval.W(eval.Regex(`DeepPartial<\s*T\[\s*\w+\s*\]\s*>`), 1),
 		eval.W(noAnyEscapeHatch(), 1),
 	)
 
@@ -316,8 +326,13 @@ configuration as an eslint.config.js flat config, preserving the same rules
 (semi: error, prefer-const: error) and the ecmaVersion/sourceType settings
 via languageOptions. Respond with only the eslint.config.js file content.`
 
+	// A flat eslint.config.js can equally validly be authored as an ES
+	// module (export default [...]) or as CommonJS (module.exports = [...]),
+	// depending on the project's own module system; requiring "export
+	// default" specifically rejected an otherwise-correct CommonJS answer
+	// (A8).
 	evaluator := eval.All(
-		eval.W(eval.ContainsAll("export default"), 2),
+		eval.W(eval.ContainsAny("export default", "module.exports"), 2),
 		eval.W(eval.ContainsAny("defineConfig(", "languageOptions"), 1),
 	)
 
@@ -447,6 +462,8 @@ printed, nothing else.`
 		Subcategory: "typescript",
 		Description: "Trace the exact output of optional chaining combined with nullish coalescing on a falsy-but-present value.",
 		Prompt:      prompt,
-		Eval:        eval.Equals("localhost:0"),
+		// A7: eval.ExactToken so a fenced or quoted rendering of the exact
+		// printed text still scores full credit.
+		Eval: eval.ExactToken("localhost:0"),
 	}
 }

internal/tests/typescript_test.go

@@ -163,6 +163,28 @@ func TestTsGenericUtilityTypeTest_Eval(t *testing.T) {
 		{"correct recursive DeepPartial", goodDeepPartial, 1},
 		{"wrong: shallow only, no recursion into nested objects", missingRecursion, 0.75}, // (2*1 + 1*0 + 1*1)/4: mapped type ok, no DeepPartial<T[K]> reference, no any
 		{"wrong: uses any, defeating the point", usesAny, 0.5},                            // (2*1 + 1*0 + 1*0)/4: mapped type ok, no recursion, uses any
+		{
+			// A9: "P" (TypeScript's own Partial<T> convention) is an
+			// equally valid mapped-type variable name to "K" - the
+			// recursive-reference check must not hardcode the literal
+			// identifier.
+			name: "correct recursive DeepPartial using P instead of K as the mapped-type variable (A9 bug probe)",
+			response: "```ts\n" + `type DeepPartial<T> = {
+  [P in keyof T]?: T[P] extends object ? DeepPartial<T[P]> : T[P];
+};
+` + "```",
+			want: 1,
+		},
+		{
+			// A9: "keyof T" with extra internal whitespace is the same
+			// requirement, just formatted differently.
+			name: "correct recursive DeepPartial with extra whitespace around keyof T (A9 bug probe)",
+			response: "```ts\n" + `type DeepPartial<T> = {
+  [K in keyof   T]?: T[K] extends object ? DeepPartial<T[K]> : T[K];
+};
+` + "```",
+			want: 1,
+		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
@@ -285,6 +307,14 @@ export default defineConfig([
     rules: { semi: "error", "prefer-const": "error" },
   },
 ]);
+` + "```"
+
+	goodFlatConfigCommonJS := "```js\n" + `module.exports = [
+  {
+    languageOptions: { ecmaVersion: "latest", sourceType: "module" },
+    rules: { semi: "error", "prefer-const": "error" },
+  },
+];
 ` + "```"
 
 	stillLegacyFormat := "```json\n" + tsOldEslintrcConfig + "\n```"
@@ -296,6 +326,7 @@ export default defineConfig([
 	}{
 		{"good: plain array export default", goodFlatConfig, 1},
 		{"good: defineConfig helper", goodFlatConfigWithHelper, 1},
+		{"good: CommonJS module.exports form is equally valid flat config (A8 bug probe)", goodFlatConfigCommonJS, 1},
 		{"wrong: left as legacy .eslintrc JSON", stillLegacyFormat, 0},
 	}
 	for _, tt := range tests {
@@ -400,6 +431,10 @@ func TestTsOptionalChainingNullishTraceTest_Eval(t *testing.T) {
 		{"exact correct", "localhost:0", 1},
 		{"correct with surrounding whitespace", "  localhost:0  ", 1},
 		{"correct case-insensitive", "LOCALHOST:0", 1},
+		// A7 regression: fenced/quoted decoration on the correct answer
+		// must still score full credit.
+		{"fenced", "```\nlocalhost:0\n```", 1},
+		{"quoted with trailing period", `"localhost:0".`, 1},
 		{"wrong: treats ?? like || and replaces the falsy 0", "localhost:8080", 0},
 		{"wrong: assumes host is present", "example.com:0", 0},
 	}

internal/tests/web_protocol.go

@@ -1,6 +1,9 @@
 package tests
 
 import (
+	"context"
+	"regexp"
+
 	"github.com/lukaszraczylo/llm-testbench/internal/eval"
 	"github.com/lukaszraczylo/llm-testbench/internal/testkit"
 )
@@ -34,8 +37,9 @@ scenario2: "The client did not send any authentication credentials, and
 the requested resource requires a logged-in user."
 scenario3: "The client is authenticated, but their account does not have
 permission to access the requested resource."
-scenario4: "The resource has permanently moved to a new URL; the server
+scenario4: "The resource has permanently moved to a new URL; the original
-wants clients (and search engines) to use the new URL from now on."`
+request was a GET, and the server wants clients (and search engines) to
+use the new URL from now on."`
 
 // webHTTPStatusScenariosTest: map 4 inline HTTP scenarios to their correct
 // status codes.
@@ -43,7 +47,11 @@ wants clients (and search engines) to use the new URL from now on."`
 // ground truth: malformed request body -> 400 Bad Request; missing
 // credentials on a resource that requires them -> 401 Unauthorized;
 // authenticated but lacking permission -> 403 Forbidden; permanent
-// relocation -> 301 Moved Permanently.
+// relocation of a GET request -> 301 Moved Permanently (B8: the scenario
+// states the original request is a GET, so 308 Permanent Redirect - which
+// exists specifically to guarantee an unchanged method and body on
+// replay, relevant for POST/PUT - is not equally defensible here; 301 is
+// the unique correct answer).
 func webHTTPStatusScenariosTest() testkit.Test {
 	prompt := `For each of these 4 request-handling scenarios, give the single most
 correct HTTP status code:
@@ -70,24 +78,39 @@ code as a number: {"scenario1":<number>,"scenario2":<number>,"scenario3":<number
 	}
 }
 
-// webDNSMXRecordPattern requires the literal token "MX" (word-bounded,
+// webDNSMXRecordTrailingWordPattern strips an optional trailing
-// case-insensitive) anywhere in the response, materially correct whether
+// "record"/"records" word (with or without a final period) that the
-// or not the response also says "record" or names the other record types
+// prompt's own "the record type" phrasing invites a model to echo back,
-// it is not.
+// e.g. "MX record" or "MX records.", before the exact-token comparison.
-const webDNSMXRecordPattern = `(?i)\bMX\b`
+var webDNSMXRecordTrailingWordPattern = regexp.MustCompile(`(?i)\s+records?\.?\s*$`)
+
+// webDNSMXRecordEval wraps eval.ExactToken(want) to also accept a response
+// carrying that optional trailing "record"/"records" word (B4).
+func webDNSMXRecordEval(want string) eval.Evaluator {
+	exact := eval.ExactToken(want)
+	return eval.EvaluatorFunc(func(ctx context.Context, response string) eval.Score {
+		stripped := webDNSMXRecordTrailingWordPattern.ReplaceAllString(response, "")
+		return exact.Evaluate(ctx, stripped)
+	})
+}
 
 // webDNSMXRecordTest: name the DNS record type used to publish a domain's
 // prioritized mail servers.
 //
 // ground truth: mail-routing priority is published exclusively via MX
 // (Mail Exchange) records; no other standard record type carries a
-// priority value for mail delivery.
+// priority value for mail delivery. B4: the prompt's own example list used
+// to include "MX" itself, which the prior substring evaluator would then
+// score correct even for a response that only restated the option list
+// without ever answering the question; MX is no longer in the example
+// list, and the evaluator now anchors to the whole answer instead of
+// matching a substring anywhere in the response.
 func webDNSMXRecordTest() testkit.Test {
 	prompt := `example.com wants mail delivered to mailA.example.com as its primary
 mail server (priority 10), falling back to mailB.example.com (priority
 20) only if the primary is unreachable. Which single DNS record type
 publishes this prioritized mail-routing configuration? Respond with only
-the record type (e.g. A, AAAA, CNAME, MX, TXT, NS).`
+the record type (e.g. A, AAAA, CNAME, TXT, NS).`
 
 	return testkit.Test{
 		ID:          "web-dns-mx-record",
@@ -95,7 +118,7 @@ the record type (e.g. A, AAAA, CNAME, MX, TXT, NS).`
 		Subcategory: "web",
 		Description: "Name the DNS record type used to publish a domain's prioritized mail servers.",
 		Prompt:      prompt,
-		Eval:        eval.Regex(webDNSMXRecordPattern),
+		Eval:        webDNSMXRecordEval("MX"),
 	}
 }
 
@@ -106,7 +129,10 @@ the record type (e.g. A, AAAA, CNAME, MX, TXT, NS).`
 // ground truth: a 301 redirect would break the tracking links (they would
 // stop resolving to their own URL), so the correct fix is a canonical
 // tag: it consolidates ranking signals onto the preferred URL while
-// leaving every tracked variant independently reachable.
+// leaving every tracked variant independently reachable. B3: the prompt
+// itself quotes both options ("canonical" or "redirect"), so scored with
+// eval.ExactToken - the prior anchored regex required the bare word and
+// scored a quoted or bolded correct answer 0.
 func webCanonicalVsRedirectTest() testkit.Test {
 	prompt := `An e-commerce site serves identical product content at these three
 URLs, which must all keep working since they are used as tracking links
@@ -129,6 +155,6 @@ you use a canonical tag or a 301 redirect? Respond with only one word:
 		Subcategory: "web",
 		Description: "Choose canonical tag over 301 redirect for a duplicate-content scenario requiring all URL variants to stay reachable.",
 		Prompt:      prompt,
-		Eval:        eval.Regex(`(?i)^\s*canonical\.?\s*$`),
+		Eval:        eval.ExactToken("canonical"),
 	}
 }

internal/tests/web_protocol_test.go

@@ -15,6 +15,10 @@ func TestWebSitemapMaxURLsTest_Eval(t *testing.T) {
 	}{
 		{"correct", "50000", 1},
 		{"prose wrapped", "The limit is 50000 URLs per sitemap file.", 1},
+		// B1: the prompt itself primes comma grouping with "52,000", so a
+		// comma-grouped correct answer must not score 0.
+		{"comma-grouped", "50,000", 1},
+		{"comma-grouped in a sentence", "The limit is 50,000 URLs.", 1},
 		{"wrong: confused with the uncompressed size limit in MB", "50", 0},
 		{"wrong entirely", "10000", 0},
 	}
@@ -61,8 +65,14 @@ func TestWebDNSMXRecordTest_Eval(t *testing.T) {
 		{"bare", "MX", 1},
 		{"lowercase", "mx", 1},
 		{"with the word record", "MX record", 1},
+		{"with the word records and a period", "MX records.", 1},
+		{"quoted", `"MX"`, 1},
 		{"wrong: CNAME", "CNAME", 0},
 		{"wrong: A record", "A", 0},
+		// B4: restating the prompt's example list (which no longer
+		// includes MX at all) must not score 1 the way the old
+		// substring-anywhere-in-the-response evaluator would have.
+		{"restating the full example list scores 0", "A, AAAA, CNAME, TXT, NS", 0},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
@@ -86,6 +96,9 @@ func TestWebCanonicalVsRedirectTest_Eval(t *testing.T) {
 		{"correct capitalized with period", "Canonical.", 1},
 		{"wrong: redirect", "redirect", 0},
 		{"wrong: extra words break the forced one-word format", "Use a canonical tag", 0},
+		// B3: the prompt's own phrasing quotes both options.
+		{"correct, quoted (prompt's own phrasing)", `"canonical"`, 1},
+		{"correct, bolded", "**canonical**", 1},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {

internal/tests/web_security.go

@@ -19,20 +19,24 @@ HTTP/1.1 204 No Content
 Access-Control-Allow-Origin: https://app.example.com
 Access-Control-Allow-Methods: GET, POST, PUT`
 
-// webCORSMissingHeaderPattern requires the literal token
+// webCORSMissingHeaderWant is the single header name this test's prompt
-// "Access-Control-Allow-Headers" (case-insensitive) anywhere in the
+// forces as the answer.
-// response, accepting it with or without a trailing colon or surrounding
-// prose.
-const webCORSMissingHeaderPattern = `(?i)access-control-allow-headers`
-
-// webCORSMissingHeaderTest: identify the single missing CORS response
-// header that will cause the browser to block the request.
 //
 // ground truth: the browser sent Access-Control-Request-Headers:
 // X-Custom-Auth in the preflight, but the server's response never echoes
 // that header name back in an Access-Control-Allow-Headers response
 // header, so the browser blocks the actual PUT request even though the
-// origin and method are both explicitly allowed.
+// origin and method are both explicitly allowed. B5: scored with
+// eval.ExactToken, anchored to the whole answer, rather than a substring
+// match anywhere in the response - a substring match would also score a
+// response that talks about the header without ever naming it as the
+// answer (e.g. explaining Access-Control-Allow-Methods at length while
+// mentioning "Access-Control-Allow-Headers" only as one of several names
+// it considered and rejected).
+const webCORSMissingHeaderWant = "Access-Control-Allow-Headers"
+
+// webCORSMissingHeaderTest: identify the single missing CORS response
+// header that will cause the browser to block the request.
 func webCORSMissingHeaderTest() testkit.Test {
 	prompt := `Here is a CORS preflight exchange:
 
@@ -49,7 +53,7 @@ nothing else.`
 		Subcategory: "web",
 		Description: "Identify the single missing CORS response header that causes a browser to block a preflighted request.",
 		Prompt:      prompt,
-		Eval:        eval.Regex(webCORSMissingHeaderPattern),
+		Eval:        eval.ExactToken(webCORSMissingHeaderWant),
 	}
 }
 

internal/tests/web_security_test.go

@@ -14,8 +14,17 @@ func TestWebCORSMissingHeaderTest_Eval(t *testing.T) {
 		want     float64
 	}{
 		{"bare header name", "Access-Control-Allow-Headers", 1},
-		{"with a colon", "Access-Control-Allow-Headers:", 1},
+		{"quoted", `"Access-Control-Allow-Headers"`, 1},
-		{"in a sentence", "The missing header is Access-Control-Allow-Headers.", 1},
+		{"bolded", "**Access-Control-Allow-Headers**", 1},
+		{"trailing period", "Access-Control-Allow-Headers.", 1},
+		// B5: ExactToken anchors to the whole normalized answer, per the
+		// prompt's own "Respond with only the header name, nothing else" -
+		// a colon is extra content beyond the header name, and a
+		// sentence wrapping the name is not the forced bare-answer format
+		// either; both now correctly score 0 rather than matching as a
+		// substring anywhere in the response.
+		{"with a colon: extra content, not the bare name", "Access-Control-Allow-Headers:", 0},
+		{"in a sentence: extra words break the forced format", "The missing header is Access-Control-Allow-Headers.", 0},
 		{"wrong: names an already-present header", "Access-Control-Allow-Origin", 0},
 		{"wrong: names an already-present header", "Access-Control-Allow-Methods", 0},
 	}

internal/tests/whitepapers_systems.go

@@ -117,7 +117,8 @@ Respond with only a JSON object: {"quorum_size":<number>,"max_tolerable_failures
 // description of a worst-case (minimum-fanout) B-tree capacity model
 // written for this test, pinning order m and a target key count.
 const paperBTreeExcerpt = `A B-tree of order m allows every internal node between ceil(m/2) and m
-children, keeping the tree shallow and well balanced even as it grows,
+children, except the root, which may have as few as two children, keeping
+the tree shallow and well balanced even as it grows,
 which bounds the number of disk reads a lookup needs to at most one per
 level. For capacity planning this report models the worst case: a tree
 where every internal node has the minimum allowed fanout, f equals
@@ -177,12 +178,19 @@ partition between two data centers, both sides keep accepting writes
 independently rather than refusing requests, accepting that the two sides
 may briefly disagree and need reconciliation once the partition heals.`
 
-// paperCAPAvailabilityPattern anchors on the trimmed response being
+// paperCAPAvailabilityWant is the single word this test's prompt forces as
-// exactly the single word "availability" (case-insensitive, with an
+// the answer.
-// optional trailing period), accepting every materially-correct form of
+//
-// the forced one-word answer without also matching a response that names
+// ground truth: the excerpt states both data centers "keep accepting
-// the wrong guarantee.
+// writes independently rather than refusing requests" during the
-const paperCAPAvailabilityPattern = `(?i)^\s*availability\.?\s*$`
+// partition. Per the CAP theorem as the excerpt defines it, continuing to
+// answer every request rather than refusing (erroring) is the definition
+// of availability, so the system prioritizes availability over
+// consistency during the partition. B2: scored with eval.ExactToken so a
+// quoted or bolded "Availability" - both forms the prompt's own phrasing
+// invites - still matches; the prior anchored regex required the bare
+// word and scored those correct answers 0.
+const paperCAPAvailabilityWant = "Availability"
 
 // paperCAPAvailabilityChoiceTest: identify, from an inline CAP-theorem
 // excerpt and a described design decision, which guarantee (consistency or
@@ -204,7 +212,7 @@ the CAP theorem? Respond with only one word: "Consistency" or
 		Subcategory: "whitepapers",
 		Description: "Identify which CAP-theorem guarantee a described partition-handling design decision prioritizes.",
 		Prompt:      prompt,
-		Eval:        eval.Regex(paperCAPAvailabilityPattern),
+		Eval:        eval.ExactToken(paperCAPAvailabilityWant),
 	}
 }
 

internal/tests/whitepapers_systems_test.go

@@ -164,6 +164,11 @@ func TestPaperCAPAvailabilityChoiceTest_Eval(t *testing.T) {
 		{"correct with trailing period", "Availability.", 1},
 		{"wrong: consistency", "Consistency", 0},
 		{"wrong: extra words break the forced one-word format", "The answer is availability", 0},
+		// B2: the prompt's own phrasing quotes the two options, so a
+		// quoted or bolded answer must still score full credit.
+		{"correct, quoted (prompt's own phrasing)", `"Availability"`, 1},
+		{"correct, bolded", "**Availability**", 1},
+		{"correct, quoted with trailing period", `"Availability".`, 1},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {

internal/tests/whitepapers_vector.go

@@ -114,8 +114,8 @@ total number of documents in the corpus and df(t) is the number of
 documents that contain t at least once. The TF-IDF weight of a term in a
 document is the product tf(t,d) times idf(t): a term that appears often in
 one document but rarely elsewhere scores highest, while a term appearing in
-every document scores near zero regardless of how often it repeats, since
+every document scores zero regardless of how often it repeats, since its
-its idf approaches the logarithm of one.`
+idf is ln(1) = 0 exactly.`
 
 // paperTFIDFDoc1, paperTFIDFDoc2, and paperTFIDFDoc3 are the pre-tokenized
 // (whitespace-split, lowercase) mini-corpus for paperTFIDFTermScoreTest.
@@ -185,6 +185,12 @@ surfaced, never recover ones it dropped.`
 // paperRecallRelevant and paperRecallRetrievedTop6 are the inline
 // relevant-set and top-6 retrieved-ranking fixtures for
 // paperRecallAtKTest.
+//
+// ground truth: relevant = {D2,D5,D7,D9} (size 4); of those, D2, D5, and
+// D7 appear somewhere in the top-6 retrieved list, but D9 does not, so the
+// intersection has size 3. recall@6 = |intersection| / |relevant| = 3/4 =
+// 0.75. whitepapers_vector_test.go independently recomputes this by
+// counting the intersection directly, not via wpRecallAtK.
 var (
 	paperRecallRelevant      = []string{"D2", "D5", "D7", "D9"}
 	paperRecallRetrievedTop6 = []string{"D1", "D5", "D3", "D7", "D6", "D2"}

internal/tests/whitepapers_vector_test.go

@@ -32,6 +32,9 @@ func TestPaperPQCompressionRatioTest_Eval(t *testing.T) {
 	}{
 		{"correct", "64", 1},
 		{"prose wrapped", "The compression ratio is 64x.", 1},
+		// B6: ratio notation must extract the first term (64), not the
+		// second (the "1" of "64:1").
+		{"ratio notation", "64:1", 1},
 		{"wrong", "8", 0},
 	}
 	for _, tt := range tests {