fix(catalog): harden evaluators against correct-answer phrasing variance

Both reviewers found evaluators that scored correct answers as zero:
hardcoded flag order, quote-style rigidity, missing multiline anchors,
comma-grouped numbers, ratio notation, and numeric strings in JSON.
A shared ExactToken evaluator now normalizes fences, quotes, and
trailing periods for exact-answer tests. The obsolete tmutil snapshot
reference now accepts the current localsnapshot verb.
This commit is contained in:
2026-08-17 02:48:05 +01:00
parent 8ab83a57ca
commit 7264fe81b7
32 changed files with 815 additions and 152 deletions
+39 -13
View File
@@ -1,6 +1,9 @@
package tests
import (
"context"
"regexp"
"github.com/lukaszraczylo/llm-testbench/internal/eval"
"github.com/lukaszraczylo/llm-testbench/internal/testkit"
)
@@ -34,8 +37,9 @@ scenario2: "The client did not send any authentication credentials, and
the requested resource requires a logged-in user."
scenario3: "The client is authenticated, but their account does not have
permission to access the requested resource."
scenario4: "The resource has permanently moved to a new URL; the server
wants clients (and search engines) to use the new URL from now on."`
scenario4: "The resource has permanently moved to a new URL; the original
request was a GET, and the server wants clients (and search engines) to
use the new URL from now on."`
// webHTTPStatusScenariosTest: map 4 inline HTTP scenarios to their correct
// status codes.
@@ -43,7 +47,11 @@ wants clients (and search engines) to use the new URL from now on."`
// ground truth: malformed request body -> 400 Bad Request; missing
// credentials on a resource that requires them -> 401 Unauthorized;
// authenticated but lacking permission -> 403 Forbidden; permanent
// relocation -> 301 Moved Permanently.
// relocation of a GET request -> 301 Moved Permanently (B8: the scenario
// states the original request is a GET, so 308 Permanent Redirect - which
// exists specifically to guarantee an unchanged method and body on
// replay, relevant for POST/PUT - is not equally defensible here; 301 is
// the unique correct answer).
func webHTTPStatusScenariosTest() testkit.Test {
prompt := `For each of these 4 request-handling scenarios, give the single most
correct HTTP status code:
@@ -70,24 +78,39 @@ code as a number: {"scenario1":<number>,"scenario2":<number>,"scenario3":<number
}
}
// webDNSMXRecordPattern requires the literal token "MX" (word-bounded,
// case-insensitive) anywhere in the response, materially correct whether
// or not the response also says "record" or names the other record types
// it is not.
const webDNSMXRecordPattern = `(?i)\bMX\b`
// webDNSMXRecordTrailingWordPattern strips an optional trailing
// "record"/"records" word (with or without a final period) that the
// prompt's own "the record type" phrasing invites a model to echo back,
// e.g. "MX record" or "MX records.", before the exact-token comparison.
var webDNSMXRecordTrailingWordPattern = regexp.MustCompile(`(?i)\s+records?\.?\s*$`)
// webDNSMXRecordEval wraps eval.ExactToken(want) to also accept a response
// carrying that optional trailing "record"/"records" word (B4).
func webDNSMXRecordEval(want string) eval.Evaluator {
exact := eval.ExactToken(want)
return eval.EvaluatorFunc(func(ctx context.Context, response string) eval.Score {
stripped := webDNSMXRecordTrailingWordPattern.ReplaceAllString(response, "")
return exact.Evaluate(ctx, stripped)
})
}
// webDNSMXRecordTest: name the DNS record type used to publish a domain's
// prioritized mail servers.
//
// ground truth: mail-routing priority is published exclusively via MX
// (Mail Exchange) records; no other standard record type carries a
// priority value for mail delivery.
// priority value for mail delivery. B4: the prompt's own example list used
// to include "MX" itself, which the prior substring evaluator would then
// score correct even for a response that only restated the option list
// without ever answering the question; MX is no longer in the example
// list, and the evaluator now anchors to the whole answer instead of
// matching a substring anywhere in the response.
func webDNSMXRecordTest() testkit.Test {
prompt := `example.com wants mail delivered to mailA.example.com as its primary
mail server (priority 10), falling back to mailB.example.com (priority
20) only if the primary is unreachable. Which single DNS record type
publishes this prioritized mail-routing configuration? Respond with only
the record type (e.g. A, AAAA, CNAME, MX, TXT, NS).`
the record type (e.g. A, AAAA, CNAME, TXT, NS).`
return testkit.Test{
ID: "web-dns-mx-record",
@@ -95,7 +118,7 @@ the record type (e.g. A, AAAA, CNAME, MX, TXT, NS).`
Subcategory: "web",
Description: "Name the DNS record type used to publish a domain's prioritized mail servers.",
Prompt: prompt,
Eval: eval.Regex(webDNSMXRecordPattern),
Eval: webDNSMXRecordEval("MX"),
}
}
@@ -106,7 +129,10 @@ the record type (e.g. A, AAAA, CNAME, MX, TXT, NS).`
// ground truth: a 301 redirect would break the tracking links (they would
// stop resolving to their own URL), so the correct fix is a canonical
// tag: it consolidates ranking signals onto the preferred URL while
// leaving every tracked variant independently reachable.
// leaving every tracked variant independently reachable. B3: the prompt
// itself quotes both options ("canonical" or "redirect"), so scored with
// eval.ExactToken - the prior anchored regex required the bare word and
// scored a quoted or bolded correct answer 0.
func webCanonicalVsRedirectTest() testkit.Test {
prompt := `An e-commerce site serves identical product content at these three
URLs, which must all keep working since they are used as tracking links
@@ -129,6 +155,6 @@ you use a canonical tag or a 301 redirect? Respond with only one word:
Subcategory: "web",
Description: "Choose canonical tag over 301 redirect for a duplicate-content scenario requiring all URL variants to stay reachable.",
Prompt: prompt,
Eval: eval.Regex(`(?i)^\s*canonical\.?\s*$`),
Eval: eval.ExactToken("canonical"),
}
}