fix(catalog): unify negation guard and harden round-2 evaluators

A shared NoUnnegatedMention primitive replaces four per-category
implementations, with a bidirectional clause-scoped window and an
adjectival cue set. The docker layer fixture drops WORKDIR, which
creates a real layer and made the pinned answer wrong. The replica
failover test now forces a JSON verdict so a negated answer cannot
score. Remaining exact-answer tests migrate to ExactToken. Ordering
prompts shuffle their candidate lists so echoing scores zero.
This commit is contained in:
2026-08-17 03:50:57 +01:00
parent 9e5deda646
commit 9d1d4d89e9
35 changed files with 1417 additions and 453 deletions
+38 -4
View File
@@ -89,9 +89,9 @@ func jsonValueAs[T comparable](v any) (T, error) {
}
return any(s).(T), nil
case bool:
b, ok := v.(bool)
if !ok {
return zero, fmt.Errorf("expected bool, got %T (%v)", v, v)
b, err := asBool(v)
if err != nil {
return zero, err
}
return any(b).(T), nil
case float64:
@@ -134,6 +134,30 @@ func asFloat64(v any) (float64, error) {
}
}
// asBool coerces a decoded JSON value to bool. A string is accepted too
// (DC2, mirroring asFloat64's B9 numeric-string coercion), case-insensitive
// and trimmed: a model that answers {"guaranteed_visible":"false"} instead
// of {"guaranteed_visible":false} - both are the same value, and the
// prompt asked for a JSON field, not a JSON type - must not score 0 for a
// mechanical typing difference. Any other string fails.
func asBool(v any) (bool, error) {
switch b := v.(type) {
case bool:
return b, nil
case string:
switch strings.ToLower(strings.TrimSpace(b)) {
case "true":
return true, nil
case "false":
return false, nil
default:
return false, fmt.Errorf("expected bool, got non-boolean string %q", b)
}
default:
return false, fmt.Errorf("expected bool, got %T (%v)", v, v)
}
}
// jsonFieldEval extracts one field by path from the response's JSON value
// (see ExtractJSON: the first ```json fenced block if present, otherwise
// the last balanced JSON value in the response) and compares it to want.
@@ -193,7 +217,17 @@ func (j jsonFieldEval[T]) Evaluate(_ context.Context, response string) Score {
func equalJSONValue[T comparable](got, want T) bool {
if gs, ok := any(got).(string); ok {
ws := any(want).(string)
return strings.EqualFold(strings.TrimSpace(gs), strings.TrimSpace(ws))
return strings.EqualFold(enumNormalize(gs), enumNormalize(ws))
}
return got == want
}
// enumNormalize folds hyphens to spaces and collapses repeated whitespace
// (DC4), so a closed-vocabulary enum answer like "non-repeatable-read" and
// "non repeatable read" compare equal: a model choosing its own spelling
// of a multi-word enum token should not be marked wrong for a formatting
// difference the prompt's closed vocabulary did not unambiguously specify.
func enumNormalize(s string) string {
s = strings.ReplaceAll(s, "-", " ")
return strings.Join(strings.Fields(s), " ")
}