mirror of
https://github.com/lukaszraczylo/llm-testbench.git
synced 2026-10-16 01:07:47 +00:00
fix(catalog): unify negation guard and harden round-2 evaluators
A shared NoUnnegatedMention primitive replaces four per-category implementations, with a bidirectional clause-scoped window and an adjectival cue set. The docker layer fixture drops WORKDIR, which creates a real layer and made the pinned answer wrong. The replica failover test now forces a JSON verdict so a negated answer cannot score. Remaining exact-answer tests migrate to ExactToken. Ordering prompts shuffle their candidate lists so echoing scores zero.
This commit is contained in:
@@ -89,9 +89,9 @@ func jsonValueAs[T comparable](v any) (T, error) {
|
||||
}
|
||||
return any(s).(T), nil
|
||||
case bool:
|
||||
b, ok := v.(bool)
|
||||
if !ok {
|
||||
return zero, fmt.Errorf("expected bool, got %T (%v)", v, v)
|
||||
b, err := asBool(v)
|
||||
if err != nil {
|
||||
return zero, err
|
||||
}
|
||||
return any(b).(T), nil
|
||||
case float64:
|
||||
@@ -134,6 +134,30 @@ func asFloat64(v any) (float64, error) {
|
||||
}
|
||||
}
|
||||
|
||||
// asBool coerces a decoded JSON value to bool. A string is accepted too
|
||||
// (DC2, mirroring asFloat64's B9 numeric-string coercion), case-insensitive
|
||||
// and trimmed: a model that answers {"guaranteed_visible":"false"} instead
|
||||
// of {"guaranteed_visible":false} - both are the same value, and the
|
||||
// prompt asked for a JSON field, not a JSON type - must not score 0 for a
|
||||
// mechanical typing difference. Any other string fails.
|
||||
func asBool(v any) (bool, error) {
|
||||
switch b := v.(type) {
|
||||
case bool:
|
||||
return b, nil
|
||||
case string:
|
||||
switch strings.ToLower(strings.TrimSpace(b)) {
|
||||
case "true":
|
||||
return true, nil
|
||||
case "false":
|
||||
return false, nil
|
||||
default:
|
||||
return false, fmt.Errorf("expected bool, got non-boolean string %q", b)
|
||||
}
|
||||
default:
|
||||
return false, fmt.Errorf("expected bool, got %T (%v)", v, v)
|
||||
}
|
||||
}
|
||||
|
||||
// jsonFieldEval extracts one field by path from the response's JSON value
|
||||
// (see ExtractJSON: the first ```json fenced block if present, otherwise
|
||||
// the last balanced JSON value in the response) and compares it to want.
|
||||
@@ -193,7 +217,17 @@ func (j jsonFieldEval[T]) Evaluate(_ context.Context, response string) Score {
|
||||
func equalJSONValue[T comparable](got, want T) bool {
|
||||
if gs, ok := any(got).(string); ok {
|
||||
ws := any(want).(string)
|
||||
return strings.EqualFold(strings.TrimSpace(gs), strings.TrimSpace(ws))
|
||||
return strings.EqualFold(enumNormalize(gs), enumNormalize(ws))
|
||||
}
|
||||
return got == want
|
||||
}
|
||||
|
||||
// enumNormalize folds hyphens to spaces and collapses repeated whitespace
|
||||
// (DC4), so a closed-vocabulary enum answer like "non-repeatable-read" and
|
||||
// "non repeatable read" compare equal: a model choosing its own spelling
|
||||
// of a multi-word enum token should not be marked wrong for a formatting
|
||||
// difference the prompt's closed vocabulary did not unambiguously specify.
|
||||
func enumNormalize(s string) string {
|
||||
s = strings.ReplaceAll(s, "-", " ")
|
||||
return strings.Join(strings.Fields(s), " ")
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user