fix(catalog): harden evaluators against correct-answer phrasing variance

Both reviewers found evaluators that scored correct answers as zero:
hardcoded flag order, quote-style rigidity, missing multiline anchors,
comma-grouped numbers, ratio notation, and numeric strings in JSON.
A shared ExactToken evaluator now normalizes fences, quotes, and
trailing periods for exact-answer tests. The obsolete tmutil snapshot
reference now accepts the current localsnapshot verb.

internal/eval/exacttoken.go

@@ -0,0 +1,91 @@
+package eval
+
+import (
+	"context"
+	"fmt"
+	"strings"
+)
+
+// NormalizeExactToken strips formatting noise from s before an exact-token
+// comparison: it extracts a fenced code block if s has one (see
+// ExtractCodeBlock), trims surrounding whitespace, strips one trailing
+// sentence-ending period, strips one layer of surrounding quote/backtick
+// decoration, strips a *paired* leading/trailing run of asterisks
+// (**bolded**/*italic* markdown - never a single, unpaired asterisk, which
+// can be meaningful content rather than decoration: "*string" is a Go
+// pointer type, not "string" wrapped in emphasis), and trims again. The
+// quote/backtick/asterisk/period stripping repeats to a fixed point so
+// nested decoration ("**\"GET\"**.") fully resolves regardless of which
+// layer is outermost. It is exported so a test file that needs to compare
+// against more than one literal variant (e.g. "O(n^2)" vs "O(n²)" vs
+// "O(n**2)") can normalize once and compare the result to each variant,
+// rather than calling ExactToken repeatedly.
+func NormalizeExactToken(s string) string {
+	got := strings.TrimSpace(ExtractCodeBlock(s, ""))
+
+	periodStripped := false
+	for {
+		before := got
+		if !periodStripped && strings.HasSuffix(got, ".") {
+			got = strings.TrimSpace(strings.TrimSuffix(got, "."))
+			periodStripped = true
+		}
+		got = strings.Trim(got, "\"'`")
+		got = stripPairedAsterisks(got)
+		got = strings.TrimSpace(got)
+		if got == before {
+			break
+		}
+	}
+	return got
+}
+
+// stripPairedAsterisks removes a leading run of '*' and a trailing run of
+// '*' from s, but only when both runs have the same, non-zero length: a
+// balanced **bold** or *italic* wrapper. An unbalanced run (a single
+// leading '*' with no trailing one, as in the Go pointer type "*string")
+// is left untouched, since it is markdown noise for one and meaningful
+// content for the other, and this function cannot tell them apart from
+// the character alone.
+func stripPairedAsterisks(s string) string {
+	lead := 0
+	for lead < len(s) && s[lead] == '*' {
+		lead++
+	}
+	trail := 0
+	for trail < len(s)-lead && s[len(s)-1-trail] == '*' {
+		trail++
+	}
+	if lead > 0 && lead == trail {
+		return s[lead : len(s)-trail]
+	}
+	return s
+}
+
+// exactTokenEval scores full credit on a normalized, case-insensitive
+// whole-answer match.
+type exactTokenEval struct {
+	want string
+}
+
+// ExactToken returns an Evaluator awarding full credit when the response,
+// after NormalizeExactToken, equals want (also normalized) case-
+// insensitively. It exists for prompts that force a single, literal
+// answer token or short phrase - one the prompt itself may quote, bold, or
+// fence - where a substring-containment evaluator (ContainsAny etc.) would
+// wrongly score a response that merely restates the prompt's own option
+// list, or a negated answer that happens to contain the right substring
+// ("not O(n^2), it's O(n)"). ExactToken anchors to the whole normalized
+// answer, so neither false-positive scores.
+func ExactToken(want string) Evaluator {
+	return exactTokenEval{want: want}
+}
+
+func (e exactTokenEval) Evaluate(_ context.Context, response string) Score {
+	got := NormalizeExactToken(response)
+	want := NormalizeExactToken(e.want)
+	if strings.EqualFold(got, want) {
+		return Score{Value: 1, Detail: fmt.Sprintf("equals %q", want)}
+	}
+	return Score{Value: 0, Detail: fmt.Sprintf("got %q, want %q", got, want)}
+}

internal/eval/exacttoken_test.go

@@ -0,0 +1,71 @@
+package eval
+
+import (
+	"context"
+	"testing"
+)
+
+func TestNormalizeExactToken(t *testing.T) {
+	tests := []struct {
+		name string
+		in   string
+		want string
+	}{
+		{"bare", "GET", "GET"},
+		{"surrounding whitespace", "  GET  \n", "GET"},
+		{"single-quoted", "'GET'", "GET"},
+		{"double-quoted", "\"GET\"", "GET"},
+		{"backtick-quoted", "`GET`", "GET"},
+		{"bolded", "**GET**", "GET"},
+		{"trailing period", "GET.", "GET"},
+		{"quoted with trailing period", "\"GET\".", "GET"},
+		{"fenced, untagged", "```\nGET\n```", "GET"},
+		{"fenced, tagged", "```text\nGET\n```", "GET"},
+		{"fenced with surrounding prose", "The answer is:\n```\nGET\n```", "GET"},
+		{"multi-word phrase", "Access-Control-Allow-Headers", "Access-Control-Allow-Headers"},
+		{"multi-word phrase bolded with period", "**Access-Control-Allow-Headers**.", "Access-Control-Allow-Headers"},
+		// Regression: an unpaired leading asterisk is meaningful content
+		// (a Go pointer type), not markdown decoration, and must survive.
+		{"unpaired leading asterisk is content, not decoration", "*string", "*string"},
+		{"unpaired trailing asterisk is content, not decoration", "string*", "string*"},
+		{"nested quote-then-bold with trailing period", "**\"GET\"**.", "GET"},
+		{"italic single-asterisk wrap", "*GET*", "GET"},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := NormalizeExactToken(tt.in)
+			if got != tt.want {
+				t.Errorf("NormalizeExactToken(%q) = %q, want %q", tt.in, got, tt.want)
+			}
+		})
+	}
+}
+
+func TestExactToken(t *testing.T) {
+	tests := []struct {
+		name     string
+		want     string
+		response string
+		wantVal  float64
+	}{
+		{"bare exact match", "MX", "MX", 1},
+		{"case insensitive", "MX", "mx", 1},
+		{"quoted", "MX", "'MX'", 1},
+		{"bolded", "MX", "**MX**", 1},
+		{"trailing period", "MX", "MX.", 1},
+		{"fenced", "MX", "```\nMX\n```", 1},
+		{"restates a longer option list scores 0", "MX", "A, AAAA, CNAME, TXT, NS, MX", 0},
+		{"wrong answer", "MX", "TXT", 0},
+		{"empty response", "MX", "", 0},
+		{"phrase exact match", "Access-Control-Allow-Headers", "Access-Control-Allow-Headers", 1},
+		{"phrase restated with prose scores 0", "Access-Control-Allow-Headers", "The missing header is Access-Control-Allow-Headers.", 0},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := ExactToken(tt.want).Evaluate(context.Background(), tt.response)
+			if got.Value != tt.wantVal {
+				t.Errorf("ExactToken(%q).Evaluate(%q) = %v, want %v (detail: %s)", tt.want, tt.response, got.Value, tt.wantVal, got.Detail)
+			}
+		})
+	}
+}

internal/eval/jsonfield.go

@@ -111,12 +111,24 @@ func jsonValueAs[T comparable](v any) (T, error) {
 	}
 }
 
+// asFloat64 coerces a decoded JSON value to float64. A string is accepted
+// too (B9), coerced only when it parses cleanly as a number end to end
+// (strconv.ParseFloat on the trimmed string): a model that answers
+// {"line":"8"} instead of {"line":8} - both are the same value, and the
+// prompt asked for a JSON field, not a JSON type - must not score 0 for a
+// mechanical typing difference. A non-numeric string ("eight") still fails.
 func asFloat64(v any) (float64, error) {
 	switch n := v.(type) {
 	case float64:
 		return n, nil
 	case json.Number:
 		return n.Float64()
+	case string:
+		f, err := strconv.ParseFloat(strings.TrimSpace(n), 64)
+		if err != nil {
+			return 0, fmt.Errorf("expected number, got non-numeric string %q", n)
+		}
+		return f, nil
 	default:
 		return 0, fmt.Errorf("expected number, got %T (%v)", v, v)
 	}

internal/eval/jsonfield_test.go

@@ -90,6 +90,43 @@ func TestJSONField_Int(t *testing.T) {
 	}
 }
 
+// TestJSONField_Int_NumericStringCoercion is the B9 regression: a model
+// answering {"line":"8"} instead of {"line":8} must still score 1 - the
+// prompt asked for a JSON field with the right value, not a specific JSON
+// type.
+func TestJSONField_Int_NumericStringCoercion(t *testing.T) {
+	e := JSONField("line", 8)
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{"numeric string coerces", `{"line":"8"}`, 1},
+		{"numeric string with whitespace coerces", `{"line":" 8 "}`, 1},
+		{"wrong numeric string", `{"line":"9"}`, 0},
+		{"non-numeric string does not coerce", `{"line":"eight"}`, 0},
+		{"native number still works", `{"line":8}`, 1},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := e.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}
+
+// TestJSONField_Float_NumericStringCoercion mirrors the int case for a
+// float64-typed field.
+func TestJSONField_Float_NumericStringCoercion(t *testing.T) {
+	e := JSONField("port", float64(8080))
+	got := e.Evaluate(context.Background(), `{"port":"8080"}`)
+	if got.Value != 1 {
+		t.Errorf("Evaluate() = %v, want 1 (detail: %s)", got.Value, got.Detail)
+	}
+}
+
 func TestJSONField_NestedPath(t *testing.T) {
 	e := JSONField("task1", "search_web")
 	got := e.Evaluate(context.Background(), `{"task1":"search_web","task2":"read_file"}`)

internal/eval/numeric.go

@@ -18,12 +18,17 @@ type Number interface {
 }
 
 // digitPattern matches unsigned integers and decimals, including a
-// leading-dot form with no integer part (".9896"). Longer alternatives are
-// listed first so the match is not truncated to just the integer part of
-// "0.9896" or just the dot-less ".9896". It deliberately excludes the sign:
-// classifyMatch decides, per match, whether a leading '-' in the source
-// text is a real minus sign or a compound-word hyphen ("x86-64").
-var digitPattern = regexp.MustCompile(`\d+\.\d+|\.\d+|\d+`)
+// comma-grouped thousands form ("50,000", "1,234.5") and a leading-dot form
+// with no integer part (".9896"). Longer/more specific alternatives are
+// listed first so the match is not truncated to just "50" out of "50,000",
+// or just the integer part of "0.9896", or just the dot-less ".9896". The
+// comma-group alternative requires groups of exactly 3 digits after the
+// first comma (`\d{1,3}(?:,\d{3})+`), so it does not misfire on an
+// unrelated comma followed by a differently-sized number ("in 2024, 50
+// states"). It deliberately excludes the sign: classifyMatch decides, per
+// match, whether a leading '-' in the source text is a real minus sign or
+// a compound-word hyphen ("x86-64").
+var digitPattern = regexp.MustCompile(`\d{1,3}(?:,\d{3})+(?:\.\d+)?|\d+\.\d+|\.\d+|\d+`)
 
 // isWordByte reports whether b is a letter, digit, or underscore - the
 // standard regex \w character class - used to detect a digit run glued
@@ -47,6 +52,11 @@ type numberCandidate struct {
 	compoundHyphen bool
 }
 
+// isDigitByte reports whether b is an ASCII digit.
+func isDigitByte(b byte) bool {
+	return b >= '0' && b <= '9'
+}
+
 // classifyMatch inspects the characters immediately surrounding the
 // unsigned digit run text[start:end] and classifies it:
 //
@@ -56,9 +66,14 @@ type numberCandidate struct {
 //   - a '-' immediately before the run is a real minus sign only when the
 //     character before THAT is not alphanumeric, so "x86-64" does not read
 //     as "x86" followed by the negative number -64;
+//   - a digit run immediately preceded by ':' whose own left neighbour is
+//     a digit is the second term of a ratio ("64:1"'s "1") and is never a
+//     real candidate, at any tier, so extraction lands on the first term;
 //   - a number joined by a hyphen to an adjacent word on either side
 //     ("64-bit", "8-byte", the "64" in "x86-64" once its bogus sign is
-//     stripped) is flagged compoundHyphen rather than rejected outright.
+//     stripped) is flagged compoundHyphen rather than rejected outright;
+//   - a comma-grouped literal ("50,000", "1,234.5") has its commas
+//     stripped so the returned literal parses with strconv.ParseFloat.
 func classifyMatch(text string, start, end int) (numberCandidate, bool) {
 	literal := text[start:end]
 
@@ -73,6 +88,9 @@ func classifyMatch(text string, start, end int) (numberCandidate, bool) {
 	if start > 0 && isWordByte(text[start-1]) {
 		return numberCandidate{}, false // e.g. the "64" in "LP64"
 	}
+	if start > 0 && text[start-1] == ':' && start-2 >= 0 && isDigitByte(text[start-2]) {
+		return numberCandidate{}, false // e.g. the "1" in "64:1"
+	}
 	if end < len(text) && isWordByte(text[end]) {
 		// Permit a multiplier suffix ("64x" = 64 times): a single trailing
 		// x/X with nothing word-like after it. Hex ("0x2A") and identifiers
@@ -90,6 +108,7 @@ func classifyMatch(text string, start, end int) (numberCandidate, bool) {
 		compound = true // e.g. "64-bit", "24-byte"
 	}
 
+	literal = strings.ReplaceAll(literal, ",", "")
 	return numberCandidate{literal: literal, compoundHyphen: compound}, true
 }
 

internal/eval/numeric_test.go

@@ -35,6 +35,19 @@ func TestExtractLastNumber(t *testing.T) {
 		// the last standalone number, which here is not the answer a
 		// human would pick. Prompts should ask for the number alone.
 		{"ambiguous parenthetical breakdown returns the wrong number by design", "Total: 24 bytes (22 bytes of members plus 2 tail padding)", 2, false},
+		// B1: comma-grouped thousands separators.
+		{"comma-grouped thousands", "50,000", 50000, false},
+		{"comma-grouped thousands with unit", "50,000 URLs", 50000, false},
+		{"comma-grouped thousands in a sentence", "The limit is 50,000.", 50000, false},
+		{"comma-grouped with a decimal tail", "1,234.5", 1234.5, false},
+		{"comma-grouped, multiple groups", "12,345,678", 12345678, false},
+		{"unrelated comma is not a thousands group", "in 2024, 50 states voted", 50, false},
+		// B6: ratio form "N:M" must extract the first term, not the second.
+		{"ratio form extracts the first term", "64:1", 64, false},
+		// Documented limitation, not a regression (B6): the word-form ratio
+		// is not cheaply distinguishable from two unrelated numbers, so
+		// this pins the current (non-ideal) behaviour deliberately.
+		{"word-form ratio is a documented limitation, not fixed", "The ratio is 64 to 1.", 1, false},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {