fix(catalog): harden evaluators against correct-answer phrasing variance

Both reviewers found evaluators that scored correct answers as zero:
hardcoded flag order, quote-style rigidity, missing multiline anchors,
comma-grouped numbers, ratio notation, and numeric strings in JSON.
A shared ExactToken evaluator now normalizes fences, quotes, and
trailing periods for exact-answer tests. The obsolete tmutil snapshot
reference now accepts the current localsnapshot verb.
This commit is contained in:
2026-08-17 02:48:05 +01:00
parent 8ab83a57ca
commit 7264fe81b7
32 changed files with 815 additions and 152 deletions
+12
View File
@@ -111,12 +111,24 @@ func jsonValueAs[T comparable](v any) (T, error) {
}
}
// asFloat64 coerces a decoded JSON value to float64. A string is accepted
// too (B9), coerced only when it parses cleanly as a number end to end
// (strconv.ParseFloat on the trimmed string): a model that answers
// {"line":"8"} instead of {"line":8} - both are the same value, and the
// prompt asked for a JSON field, not a JSON type - must not score 0 for a
// mechanical typing difference. A non-numeric string ("eight") still fails.
func asFloat64(v any) (float64, error) {
switch n := v.(type) {
case float64:
return n, nil
case json.Number:
return n.Float64()
case string:
f, err := strconv.ParseFloat(strings.TrimSpace(n), 64)
if err != nil {
return 0, fmt.Errorf("expected number, got non-numeric string %q", n)
}
return f, nil
default:
return 0, fmt.Errorf("expected number, got %T (%v)", v, v)
}