lukaszraczylo/llm-testbench
1
0
Fork 0
mirror of https://github.com/lukaszraczylo/llm-testbench.git synced 2026-10-14 00:57:45 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(catalog): unify negation guard and harden round-2 evaluators

Browse Source 
A shared NoUnnegatedMention primitive replaces four per-category
implementations, with a bidirectional clause-scoped window and an
adjectival cue set. The docker layer fixture drops WORKDIR, which
creates a real layer and made the pinned answer wrong. The replica
failover test now forces a JSON verdict so a negated answer cannot
score. Remaining exact-answer tests migrate to ExactToken. Ordering
prompts shuffle their candidate lists so echoing scores zero.
This commit is contained in:
lukaszraczylo
2026-08-17 03:50:57 +01:00
parent 9e5deda646
commit 9d1d4d89e9
35 changed files with 1417 additions and 453 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/tests/databases.go
-23
View File
@@ -4,7 +4,6 @@ import (
"context"
"encoding/json"
"fmt"
"strings"
"github.com/lukaszraczylo/llm-testbench/internal/eval"
"github.com/lukaszraczylo/llm-testbench/internal/testkit"
@@ -23,28 +22,6 @@ func registerDatabasesTests(r *testkit.Registry) {
registerDBSQLTuningTests(r)
}
// dbExactAnswer returns an Evaluator awarding full credit when the
// response, trimmed of whitespace, at most one layer of surrounding quote
// characters (' , ", or `), and a single trailing sentence-ending period,
// equals want case-insensitively. Used across the databases category for
// prompts that force a single short forced-vocabulary answer (e.g. "yes"
// or "no", "keyset" or "offset"): it accepts every materially-correct form
// of that answer (bare, quoted, differently-cased, or with trailing
// punctuation) without loosening the match to accept a wrong answer.
func dbExactAnswer(want string) eval.Evaluator {
return eval.EvaluatorFunc(func(_ context.Context, response string) eval.Score {
got := strings.TrimSpace(response)
got = strings.Trim(got, "\"'`")
got = strings.TrimSuffix(strings.TrimSpace(got), ".")
got = strings.TrimSpace(got)
wantTrimmed := strings.TrimSpace(want)
if strings.EqualFold(got, wantTrimmed) {
return eval.Score{Value: 1, Detail: fmt.Sprintf("equals %q", wantTrimmed)}
}
return eval.Score{Value: 0, Detail: fmt.Sprintf("got %q, want %q", got, wantTrimmed)}
})
}
// dbJSONArrayLength returns an Evaluator awarding full credit when the
// response's extracted JSON array (see eval.ExtractJSON) has exactly want
// elements, zero otherwise. Paired with per-index eval.JSONField checks so