mirror of
https://github.com/lukaszraczylo/llm-testbench.git
synced 2026-10-15 01:07:26 +00:00
fix(catalog): harden evaluators against correct-answer phrasing variance
Both reviewers found evaluators that scored correct answers as zero: hardcoded flag order, quote-style rigidity, missing multiline anchors, comma-grouped numbers, ratio notation, and numeric strings in JSON. A shared ExactToken evaluator now normalizes fences, quotes, and trailing periods for exact-answer tests. The obsolete tmutil snapshot reference now accepts the current localsnapshot verb.
This commit is contained in:
@@ -479,6 +479,10 @@ func TestGoContextCancellationTraceTest_Eval(t *testing.T) {
|
||||
{"exact correct", "context canceled", 1},
|
||||
{"correct case-insensitive", "CONTEXT CANCELED", 1},
|
||||
{"correct with surrounding whitespace", " context canceled ", 1},
|
||||
// A7 regression: fenced/quoted decoration on the correct answer
|
||||
// must still score full credit.
|
||||
{"fenced", "```\ncontext canceled\n```", 1},
|
||||
{"quoted with trailing period", `"context canceled".`, 1},
|
||||
{"wrong: assumes select falls to default", "not done", 0},
|
||||
{"wrong: wrong sentinel error", "context deadline exceeded", 0},
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user