lukaszraczylo/llm-testbench
1
0
Fork 0
mirror of https://github.com/lukaszraczylo/llm-testbench.git synced 2026-10-17 01:17:33 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(catalog): unify negation guard and harden round-2 evaluators

Browse Source 
A shared NoUnnegatedMention primitive replaces four per-category
implementations, with a bidirectional clause-scoped window and an
adjectival cue set. The docker layer fixture drops WORKDIR, which
creates a real layer and made the pinned answer wrong. The replica
failover test now forces a JSON verdict so a negated answer cannot
score. Remaining exact-answer tests migrate to ExactToken. Ordering
prompts shuffle their candidate lists so echoing scores zero.
This commit is contained in:
lukaszraczylo
2026-08-17 03:50:57 +01:00
parent 9e5deda646
commit 9d1d4d89e9
35 changed files with 1417 additions and 453 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/tests/ai_vectorsearch_test.go
+33 -1
View File
@@ -23,7 +23,16 @@ func TestVecCosineVsDotTest_Eval(t *testing.T) {
want float64
}{
{"correct: yes", "yes", 1},
{"correct: yes, prose", "Yes, since both magnitudes are 1.", 0},
{"correct: different case", "Yes", 1},
{"correct: quoted", `"yes"`, 1},
// C3: this row's own label was self-contradictory - "correct"
// paired with want:0. The prompt asks for "only one word: yes or
// no"; a full sentence is not a bare/quoted/fenced token even
// though its content is substantively correct, so want:0 was
// already the right assertion - only the misleading label needed
// fixing, matching how eval.ExactToken treats sentence-wrapping
// everywhere else in this codebase (it is not a summarizer).
{"wrong: substantively correct but sentence-wrapped, not a bare token", "Yes, since both magnitudes are 1.", 0},
{"wrong: no", "no", 0},
}
for _, tt := range tests {
@@ -66,6 +75,7 @@ func TestVecRecallAtKTest_Eval(t *testing.T) {
}{
{"exact", "0.4", 1},
{"prose wrapped", "recall@5 is 0.4.", 1},
{"trailing period", "0.4.", 1},
{"wrong: counted an extra hit", "0.6", 0},
{"wrong: missed a hit", "0.2", 0},
}
@@ -87,7 +97,10 @@ func TestVecHNSWEfSearchTradeoffTest_Eval(t *testing.T) {
want float64
}{
{"correct: up", "up", 1},
{"correct: uppercase", "UP", 1},
{"correct: trailing period", "up.", 1},
{"wrong: down", "down", 0},
{"wrong: unchanged", "unchanged", 0},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
@@ -123,7 +136,9 @@ func TestVecPQMemoryMathTest_Eval(t *testing.T) {
}{
{"exact", "8000000", 1},
{"prose wrapped", "The total is 8000000 bytes.", 1},
{"fenced", "```\n8000000\n```", 1},
{"wrong", "512", 0},
{"wrong: bit/byte confusion (8x too large)", "64000000", 0},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
@@ -143,7 +158,10 @@ func TestVecPreVsPostFilteringTest_Eval(t *testing.T) {
want float64
}{
{"correct: pre", "pre", 1},
{"correct: uppercase", "PRE", 1},
{"correct: trailing period", "pre.", 1},
{"wrong: post", "post", 0},
{"wrong: both", "both", 0},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
@@ -180,6 +198,7 @@ func TestVecRRFFusionTest_Eval(t *testing.T) {
{"within tolerance", "0.0307", 1},
{"prose wrapped", "The RRF score for D5 is 0.0308.", 1},
{"wrong: only counted one list", "0.0159", 0},
{"wrong: outside tolerance", "0.0328", 0},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
@@ -214,7 +233,9 @@ func TestVecDistanceToSimilarityTest_Eval(t *testing.T) {
}{
{"exact", "0.82", 1},
{"prose wrapped", "cos_sim(a,b) = 0.82", 1},
{"fenced", "```\n0.82\n```", 1},
{"wrong: used d not d^2", "0.7", 0},
{"wrong: forgot to subtract from 1", "0.18", 0},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
@@ -252,6 +273,7 @@ func TestVecNearDuplicateThresholdTest_Eval(t *testing.T) {
}{
{"exact", "-0.07", 1},
{"prose wrapped", "similarity minus threshold is -0.07.", 1},
{"fenced", "```\n-0.07\n```", 1},
{"wrong: forgot the sign", "0.07", 0},
{"wrong: reported similarity, not the difference", "0.9", 0},
}
@@ -274,6 +296,7 @@ func TestVecIndexBuildQueryTradeoffTest_Eval(t *testing.T) {
}{
{"all correct", `{"scenario_a":"ivf-flat","scenario_b":"hnsw"}`, 1},
{"all correct fenced", "```json\n{\"scenario_a\":\"ivf-flat\",\"scenario_b\":\"hnsw\"}\n```", 1},
{"all correct, different case", `{"scenario_a":"IVF-FLAT","scenario_b":"HNSW"}`, 1},
{"scenario_a wrong", `{"scenario_a":"hnsw","scenario_b":"hnsw"}`, 0.5},
{"scenario_b wrong", `{"scenario_a":"ivf-flat","scenario_b":"ivf-flat"}`, 0.5},
{"both swapped", `{"scenario_a":"hnsw","scenario_b":"ivf-flat"}`, 0},
@@ -309,6 +332,15 @@ func TestVecEmbeddingDimensionTradeoffTest_Eval(t *testing.T) {
{"correct: 128", `{"choice":"128"}`, 1},
{"correct, fenced with prose", "The 128-dim option fits:\n```json\n{\"choice\":\"128\"}\n```", 1},
{"wrong: 768", `{"choice":"768"}`, 0},
{"wrong: neither valid option", `{"choice":"512"}`, 0},
{
// C7 bug probe: the prompt's own template shows choice
// unquoted ({"choice":128|768}), so a genuinely numeric
// (unquoted) response must also score full credit.
name: "correct: unquoted numeric form matching the prompt's own template (C7 bug probe)",
response: `{"choice":128}`,
want: 1,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {