mirror of
https://github.com/lukaszraczylo/llm-testbench.git
synced 2026-10-12 00:45:19 +00:00
fix(catalog): harden evaluators against correct-answer phrasing variance
Both reviewers found evaluators that scored correct answers as zero: hardcoded flag order, quote-style rigidity, missing multiline anchors, comma-grouped numbers, ratio notation, and numeric strings in JSON. A shared ExactToken evaluator now normalizes fences, quotes, and trailing periods for exact-answer tests. The obsolete tmutil snapshot reference now accepts the current localsnapshot verb.
This commit is contained in:
@@ -32,6 +32,9 @@ func TestPaperPQCompressionRatioTest_Eval(t *testing.T) {
|
||||
}{
|
||||
{"correct", "64", 1},
|
||||
{"prose wrapped", "The compression ratio is 64x.", 1},
|
||||
// B6: ratio notation must extract the first term (64), not the
|
||||
// second (the "1" of "64:1").
|
||||
{"ratio notation", "64:1", 1},
|
||||
{"wrong", "8", 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
|
||||
Reference in New Issue
Block a user