fix(catalog): harden evaluators against correct-answer phrasing variance

Both reviewers found evaluators that scored correct answers as zero: hardcoded flag order, quote-style rigidity, missing multiline anchors, comma-grouped numbers, ratio notation, and numeric strings in JSON. A shared ExactToken evaluator now normalizes fences, quotes, and trailing periods for exact-answer tests. The obsolete tmutil snapshot reference now accepts the current localsnapshot verb.
2026-10-14 00:57:45 +00:00 · 2026-08-17 02:48:05 +01:00
parent 8ab83a57ca
commit 7264fe81b7
32 changed files with 815 additions and 152 deletions
@@ -1,6 +1,9 @@
 package tests

 import (
+	"context"
+	"regexp"
+
 	"github.com/lukaszraczylo/llm-testbench/internal/eval"
 	"github.com/lukaszraczylo/llm-testbench/internal/testkit"
 )
@@ -74,11 +77,19 @@ contents only.`
 	}
 }

-// linuxJournalctlFilterPattern requires journalctl filtered to unit
-// nginx.service and a since-window of 2 hours, in the canonical
-// "-u UNIT --since WINDOW" order (the overwhelmingly common idiom), on a
-// single line so a scattered mention across paragraphs does not count.
-const linuxJournalctlFilterPattern = `(?i)\bjournalctl\b[^\n]*?-u\s+nginx\.service\b[^\n]*?--since\s+["']?-?2\s*h(?:ours?)?(?:\s+ago)?["']?`
+// linuxJournalctlUnitPattern matches the unit filter in either short (-u)
+// or long (--unit) form, tolerant of "=" or whitespace between the option
+// and its value (journalctl, via getopt_long, accepts both). It is checked
+// independently of linuxJournalctlSincePattern below (A3), so either
+// option can appear on the line before the other - the original single
+// regex hardcoded "-u ... --since ..." order and rejected the equally
+// valid "--unit"/"-S" long/short forms and "--since=" spelling.
+const linuxJournalctlUnitPattern = `(?i)\bjournalctl\b[^\n]*(?:-u|--unit)[=\s]+nginx\.service\b`
+
+// linuxJournalctlSincePattern matches the since-window filter in either
+// short (-S) or long (--since) form, independent of
+// linuxJournalctlUnitPattern (A3).
+const linuxJournalctlSincePattern = `(?i)\bjournalctl\b[^\n]*(?:-S|--since)[=\s]+["']?-?2\s*h(?:ours?)?(?:\s+ago)?["']?`

 // linuxJournalctlFilterTest: one-line journalctl command filtered to one
 // unit over a relative time window.
@@ -86,13 +97,18 @@ func linuxJournalctlFilterTest() testkit.Test {
 	prompt := `Give a single, one-line journalctl command that shows only
 log entries for the systemd unit "nginx.service" from the last 2 hours.`

+	evaluator := eval.All(
+		eval.W(eval.Regex(linuxJournalctlUnitPattern), 1),
+		eval.W(eval.Regex(linuxJournalctlSincePattern), 1),
+	)
+
 	return testkit.Test{
 		ID:          "linux-journalctl-filter",
 		Category:    "operations",
 		Subcategory: "linux",
 		Description: "One-line journalctl command filtered by unit and a relative 2-hour time window.",
 		Prompt:      prompt,
-		Eval:        eval.Regex(linuxJournalctlFilterPattern),
+		Eval:        evaluator,
 	}
 }

@@ -138,11 +154,16 @@ JSON object: {"oom_killed": true or false, "oom_kill_count": <number>}.`

 // linuxNftablesDNATPattern requires a DNAT rule (iptables or nft form) on
 // the nat table's prerouting chain, matching TCP port 8080 and forwarding
-// to 10.0.5.20:80, on a single line. iptables and nft flags/keywords follow
-// a fixed canonical order (table/chain, then match, then target), so this
-// uses the same non-greedy same-line anchoring as the pct-exec lesson
-// rather than a permutation of independently-orderable flags.
-const linuxNftablesDNATPattern = `(?i)(?:iptables|nft)\b[^\n]*?nat[^\n]*?prerouting[^\n]*?(?:--dport|dport)\s*8080[^\n]*?dnat[^\n]*?10\.0\.5\.20:80`
+// to 10.0.5.20:80, on a single line. The "nat" table and "prerouting"
+// chain tokens are matched order-free relative to each other (A12): a
+// syntactically valid iptables invocation can give -A PREROUTING before
+// -t nat (e.g. "iptables -A PREROUTING -t nat -p tcp --dport 8080 -j DNAT
+// --to-destination 10.0.5.20:80"), and the original pattern's fixed
+// "nat...prerouting" order rejected that ordering. The rest of the match
+// (dport, dnat, destination) keeps its original same-line, in-order
+// anchoring, since iptables and nft do put the match/target keywords after
+// the table/chain in both valid orderings.
+const linuxNftablesDNATPattern = `(?i)(?:iptables|nft)\b[^\n]*?(?:nat[^\n]*?prerouting|prerouting[^\n]*?nat)[^\n]*?(?:--dport|dport)\s*8080[^\n]*?dnat[^\n]*?10\.0\.5\.20:80`

 // linuxNftablesDNATTest: one-line DNAT rule forwarding an external port to
 // an internal host:port.
@@ -194,9 +215,17 @@ data. Respond with only a JSON array of the step ids, e.g.
 	}
 }

-// linuxSSHPortForwardPattern requires an ssh local port-forward (-L) from
-// local port 15432 to the remote Postgres, connecting to the correct host.
-const linuxSSHPortForwardPattern = `(?i)\bssh\b[^\n]*?-L\s*15432:(?:127\.0\.0\.1|localhost):5432[^\n]*?ops@db1\.internal`
+// linuxSSHPortForwardSpecPattern matches the -L local-forward flag and its
+// argument, independent of where the SSH destination appears on the line -
+// -L can come before or after the destination in a valid ssh invocation
+// (A6).
+const linuxSSHPortForwardSpecPattern = `(?i)\bssh\b[^\n]*-L\s*15432:(?:127\.0\.0\.1|localhost):5432\b`
+
+// linuxSSHPortForwardDestPattern matches the SSH destination in either its
+// "user@host" form or the equivalent "-l user ... host" form (A6): both
+// name the same destination, and the original pattern only accepted the
+// former.
+const linuxSSHPortForwardDestPattern = `(?i)\bssh\b[^\n]*(?:ops@db1\.internal|-l\s+ops\b[^\n]*\bdb1\.internal\b)`

 // linuxSSHPortForwardTest: ssh -L local port-forward command derived from a
 // scenario.
@@ -207,13 +236,18 @@ single, one-line ssh command, run from your laptop, that forwards your
 laptop's local port 15432 to that remote Postgres instance, so a local
 client can connect to localhost:15432.`

+	evaluator := eval.Mean(
+		eval.Regex(linuxSSHPortForwardSpecPattern),
+		eval.Regex(linuxSSHPortForwardDestPattern),
+	)
+
 	return testkit.Test{
 		ID:          "linux-ssh-port-forward",
 		Category:    "operations",
 		Subcategory: "linux",
 		Description: "One-line ssh -L local port-forward command reaching a Postgres instance that only listens on remote loopback.",
 		Prompt:      prompt,
-		Eval:        eval.Regex(linuxSSHPortForwardPattern),
+		Eval:        evaluator,
 	}
 }

@@ -221,7 +255,26 @@ client can connect to localhost:15432.`
 // "03:30 every Monday", tolerant only of whitespace width between fields -
 // unlike eval.Equals, this stays correct even if the model adds a leading
 // or trailing space, since the fields themselves have no valid rephrasing.
-const linuxCronExpressionPattern = `^\s*30\s+3\s+\*\s+\*\s+1\s*$`
+// (?m) makes ^/$ match per line rather than only at the very start/end of
+// the whole string, so the expression still matches as one line within a
+// longer, fenced, or multi-line response (A5).
+const linuxCronExpressionPattern = `(?m)^\s*30\s+3\s+\*\s+\*\s+1\s*$`
+
+// linuxCronExpressionEval strips a markdown code fence (if any) before
+// applying linuxCronExpressionPattern. Without this, a response that
+// wraps the correct expression in a ` + "```" + ` fence scored 0, because the fence
+// markers and surrounding whitespace kept the line-anchored regex from
+// finding a clean match (A5).
+func linuxCronExpressionEval() eval.Evaluator {
+	re := regexp.MustCompile(linuxCronExpressionPattern)
+	return eval.EvaluatorFunc(func(_ context.Context, response string) eval.Score {
+		stripped := eval.ExtractCodeBlock(response, "")
+		if re.MatchString(stripped) {
+			return eval.Score{Value: 1, Detail: "matches the 5-field cron expression"}
+		}
+		return eval.Score{Value: 0, Detail: "does not match the 5-field cron expression"}
+	})
+}

 // linuxCronExpressionTest: produce the 5-field cron expression matching a
 // stated schedule.
@@ -241,7 +294,7 @@ with only the expression, nothing else.`
 		Subcategory: "linux",
 		Description: "Produce the 5-field cron expression for a 03:30-every-Monday schedule.",
 		Prompt:      prompt,
-		Eval:        eval.Regex(linuxCronExpressionPattern),
+		Eval:        linuxCronExpressionEval(),
 	}
 }

@@ -258,11 +311,15 @@ unit's WantedBy needs.`
 	// paired with a same-named .service unit (the work). The .timer's
 	// OnCalendar= key sets the schedule (e.g. "*-*-* 02:00:00" for daily at
 	// 02:00), and its [Install] section needs WantedBy=timers.target for
-	// systemctl enable to activate it.
+	// systemctl enable to activate it. The WantedBy check accepts "=",
+	// ":", or "(is) set to" between the key and its value (A15), not just a
+	// bare "WantedBy=timers.target" substring - a response that phrases
+	// this in prose ("WantedBy is set to timers.target") states exactly the
+	// same fact.
 	evaluator := eval.Mean(
 		eval.ContainsAll(".service", ".timer"),
 		eval.ContainsAny("OnCalendar"),
-		eval.ContainsAll("WantedBy=timers.target"),
+		eval.Regex(`(?i)WantedBy\s*(?:=|:|\b(?:is\s+)?set to\b)\s*timers\.target`),
 	)

 	return testkit.Test{
@@ -284,7 +341,12 @@ Buffers:          512000 kB
 Cached:          3200000 kB`

 // linuxProcMeminfoAvailableWant is MemAvailable (9876000 kB) converted to
-// whole megabytes, floored: 9876000 / 1024 = 9644.53125 -> 9644.
+// whole megabytes, floored: 9876000 / 1024 = 9644.53125 -> 9644. The
+// prompt asks for the value rounded down, so 9644 is the single exact
+// correct answer - tolerance 0 below, not 1 (A17): a tolerance of 1 was
+// letting off-by-one wrong answers (9643 or 9645) score full credit for a
+// deterministic arithmetic conversion with no room for a legitimate
+// rephrasing.
 var linuxProcMeminfoAvailableWant = 9876000 / 1024

 // linuxProcMeminfoAvailableTest: interpret /proc/meminfo's MemAvailable
@@ -312,6 +374,6 @@ so convert by dividing by 1024, not 1000. Respond with only the number.`
 		Subcategory: "linux",
 		Description: "Convert /proc/meminfo's MemAvailable field (not MemFree) to whole megabytes.",
 		Prompt:      prompt,
-		Eval:        eval.Numeric(eval.ExtractLastNumber[int], linuxProcMeminfoAvailableWant, 1),
+		Eval:        eval.Numeric(eval.ExtractLastNumber[int], linuxProcMeminfoAvailableWant, 0),
 	}
 }