mirror of
https://github.com/lukaszraczylo/llm-testbench.git
synced 2026-10-13 00:48:29 +00:00
fix(catalog): harden evaluators against correct-answer phrasing variance
Both reviewers found evaluators that scored correct answers as zero: hardcoded flag order, quote-style rigidity, missing multiline anchors, comma-grouped numbers, ratio notation, and numeric strings in JSON. A shared ExactToken evaluator now normalizes fences, quotes, and trailing periods for exact-answer tests. The obsolete tmutil snapshot reference now accepts the current localsnapshot verb.
This commit is contained in:
@@ -1,6 +1,9 @@
|
||||
package tests
|
||||
|
||||
import (
|
||||
"context"
|
||||
"regexp"
|
||||
|
||||
"github.com/lukaszraczylo/llm-testbench/internal/eval"
|
||||
"github.com/lukaszraczylo/llm-testbench/internal/testkit"
|
||||
)
|
||||
@@ -34,8 +37,9 @@ scenario2: "The client did not send any authentication credentials, and
|
||||
the requested resource requires a logged-in user."
|
||||
scenario3: "The client is authenticated, but their account does not have
|
||||
permission to access the requested resource."
|
||||
scenario4: "The resource has permanently moved to a new URL; the server
|
||||
wants clients (and search engines) to use the new URL from now on."`
|
||||
scenario4: "The resource has permanently moved to a new URL; the original
|
||||
request was a GET, and the server wants clients (and search engines) to
|
||||
use the new URL from now on."`
|
||||
|
||||
// webHTTPStatusScenariosTest: map 4 inline HTTP scenarios to their correct
|
||||
// status codes.
|
||||
@@ -43,7 +47,11 @@ wants clients (and search engines) to use the new URL from now on."`
|
||||
// ground truth: malformed request body -> 400 Bad Request; missing
|
||||
// credentials on a resource that requires them -> 401 Unauthorized;
|
||||
// authenticated but lacking permission -> 403 Forbidden; permanent
|
||||
// relocation -> 301 Moved Permanently.
|
||||
// relocation of a GET request -> 301 Moved Permanently (B8: the scenario
|
||||
// states the original request is a GET, so 308 Permanent Redirect - which
|
||||
// exists specifically to guarantee an unchanged method and body on
|
||||
// replay, relevant for POST/PUT - is not equally defensible here; 301 is
|
||||
// the unique correct answer).
|
||||
func webHTTPStatusScenariosTest() testkit.Test {
|
||||
prompt := `For each of these 4 request-handling scenarios, give the single most
|
||||
correct HTTP status code:
|
||||
@@ -70,24 +78,39 @@ code as a number: {"scenario1":<number>,"scenario2":<number>,"scenario3":<number
|
||||
}
|
||||
}
|
||||
|
||||
// webDNSMXRecordPattern requires the literal token "MX" (word-bounded,
|
||||
// case-insensitive) anywhere in the response, materially correct whether
|
||||
// or not the response also says "record" or names the other record types
|
||||
// it is not.
|
||||
const webDNSMXRecordPattern = `(?i)\bMX\b`
|
||||
// webDNSMXRecordTrailingWordPattern strips an optional trailing
|
||||
// "record"/"records" word (with or without a final period) that the
|
||||
// prompt's own "the record type" phrasing invites a model to echo back,
|
||||
// e.g. "MX record" or "MX records.", before the exact-token comparison.
|
||||
var webDNSMXRecordTrailingWordPattern = regexp.MustCompile(`(?i)\s+records?\.?\s*$`)
|
||||
|
||||
// webDNSMXRecordEval wraps eval.ExactToken(want) to also accept a response
|
||||
// carrying that optional trailing "record"/"records" word (B4).
|
||||
func webDNSMXRecordEval(want string) eval.Evaluator {
|
||||
exact := eval.ExactToken(want)
|
||||
return eval.EvaluatorFunc(func(ctx context.Context, response string) eval.Score {
|
||||
stripped := webDNSMXRecordTrailingWordPattern.ReplaceAllString(response, "")
|
||||
return exact.Evaluate(ctx, stripped)
|
||||
})
|
||||
}
|
||||
|
||||
// webDNSMXRecordTest: name the DNS record type used to publish a domain's
|
||||
// prioritized mail servers.
|
||||
//
|
||||
// ground truth: mail-routing priority is published exclusively via MX
|
||||
// (Mail Exchange) records; no other standard record type carries a
|
||||
// priority value for mail delivery.
|
||||
// priority value for mail delivery. B4: the prompt's own example list used
|
||||
// to include "MX" itself, which the prior substring evaluator would then
|
||||
// score correct even for a response that only restated the option list
|
||||
// without ever answering the question; MX is no longer in the example
|
||||
// list, and the evaluator now anchors to the whole answer instead of
|
||||
// matching a substring anywhere in the response.
|
||||
func webDNSMXRecordTest() testkit.Test {
|
||||
prompt := `example.com wants mail delivered to mailA.example.com as its primary
|
||||
mail server (priority 10), falling back to mailB.example.com (priority
|
||||
20) only if the primary is unreachable. Which single DNS record type
|
||||
publishes this prioritized mail-routing configuration? Respond with only
|
||||
the record type (e.g. A, AAAA, CNAME, MX, TXT, NS).`
|
||||
the record type (e.g. A, AAAA, CNAME, TXT, NS).`
|
||||
|
||||
return testkit.Test{
|
||||
ID: "web-dns-mx-record",
|
||||
@@ -95,7 +118,7 @@ the record type (e.g. A, AAAA, CNAME, MX, TXT, NS).`
|
||||
Subcategory: "web",
|
||||
Description: "Name the DNS record type used to publish a domain's prioritized mail servers.",
|
||||
Prompt: prompt,
|
||||
Eval: eval.Regex(webDNSMXRecordPattern),
|
||||
Eval: webDNSMXRecordEval("MX"),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -106,7 +129,10 @@ the record type (e.g. A, AAAA, CNAME, MX, TXT, NS).`
|
||||
// ground truth: a 301 redirect would break the tracking links (they would
|
||||
// stop resolving to their own URL), so the correct fix is a canonical
|
||||
// tag: it consolidates ranking signals onto the preferred URL while
|
||||
// leaving every tracked variant independently reachable.
|
||||
// leaving every tracked variant independently reachable. B3: the prompt
|
||||
// itself quotes both options ("canonical" or "redirect"), so scored with
|
||||
// eval.ExactToken - the prior anchored regex required the bare word and
|
||||
// scored a quoted or bolded correct answer 0.
|
||||
func webCanonicalVsRedirectTest() testkit.Test {
|
||||
prompt := `An e-commerce site serves identical product content at these three
|
||||
URLs, which must all keep working since they are used as tracking links
|
||||
@@ -129,6 +155,6 @@ you use a canonical tag or a 301 redirect? Respond with only one word:
|
||||
Subcategory: "web",
|
||||
Description: "Choose canonical tag over 301 redirect for a duplicate-content scenario requiring all URL variants to stay reachable.",
|
||||
Prompt: prompt,
|
||||
Eval: eval.Regex(`(?i)^\s*canonical\.?\s*$`),
|
||||
Eval: eval.ExactToken("canonical"),
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user