mirror of
https://github.com/lukaszraczylo/llm-testbench.git
synced 2026-10-14 00:57:45 +00:00
fix(catalog): harden evaluators against correct-answer phrasing variance
Both reviewers found evaluators that scored correct answers as zero: hardcoded flag order, quote-style rigidity, missing multiline anchors, comma-grouped numbers, ratio notation, and numeric strings in JSON. A shared ExactToken evaluator now normalizes fences, quotes, and trailing periods for exact-answer tests. The obsolete tmutil snapshot reference now accepts the current localsnapshot verb.
This commit is contained in:
@@ -111,12 +111,24 @@ func jsonValueAs[T comparable](v any) (T, error) {
|
||||
}
|
||||
}
|
||||
|
||||
// asFloat64 coerces a decoded JSON value to float64. A string is accepted
|
||||
// too (B9), coerced only when it parses cleanly as a number end to end
|
||||
// (strconv.ParseFloat on the trimmed string): a model that answers
|
||||
// {"line":"8"} instead of {"line":8} - both are the same value, and the
|
||||
// prompt asked for a JSON field, not a JSON type - must not score 0 for a
|
||||
// mechanical typing difference. A non-numeric string ("eight") still fails.
|
||||
func asFloat64(v any) (float64, error) {
|
||||
switch n := v.(type) {
|
||||
case float64:
|
||||
return n, nil
|
||||
case json.Number:
|
||||
return n.Float64()
|
||||
case string:
|
||||
f, err := strconv.ParseFloat(strings.TrimSpace(n), 64)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("expected number, got non-numeric string %q", n)
|
||||
}
|
||||
return f, nil
|
||||
default:
|
||||
return 0, fmt.Errorf("expected number, got %T (%v)", v, v)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user