lukaszraczylo/llm-testbench
1
0
Fork 0
mirror of https://github.com/lukaszraczylo/llm-testbench.git synced 2026-10-18 01:17:48 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(catalog): unify negation guard and harden round-2 evaluators

Browse Source 
A shared NoUnnegatedMention primitive replaces four per-category
implementations, with a bidirectional clause-scoped window and an
adjectival cue set. The docker layer fixture drops WORKDIR, which
creates a real layer and made the pinned answer wrong. The replica
failover test now forces a JSON verdict so a negated answer cannot
score. Remaining exact-answer tests migrate to ExactToken. Ordering
prompts shuffle their candidate lists so echoing scores zero.
This commit is contained in:
lukaszraczylo
2026-08-17 03:50:57 +01:00
parent 9e5deda646
commit 9d1d4d89e9
35 changed files with 1417 additions and 453 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/tests/ai_rag_test.go
+71 -1
View File
@@ -14,6 +14,7 @@ func TestRagChunkSizeTradeoffTest_Eval(t *testing.T) {
}{
{"correct: 512", `{"chunk_size_tokens":512}`, 1},
{"correct, fenced with prose", "The smallest viable option is:\n```json\n{\"chunk_size_tokens\":512}\n```", 1},
{"correct, different spacing", `{ "chunk_size_tokens": 512 }`, 1},
{"wrong: too small, would split a fact", `{"chunk_size_tokens":128}`, 0},
{"wrong: larger than needed, more dilution", `{"chunk_size_tokens":2048}`, 0},
}
@@ -39,6 +40,7 @@ func TestRagRerankerPlacementTest_Eval(t *testing.T) {
{"correct order different case", `["Retrieve","Rerank","Generate"]`, 1},
{"reranker before retrieval", `["rerank","retrieve","generate"]`, 0},
{"generate before rerank", `["retrieve","generate","rerank"]`, 0},
{"wrong: echoes the prompt's shuffled presentation order verbatim (C4 bug probe)", `["generate","retrieve","rerank"]`, 0},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
@@ -59,7 +61,9 @@ func TestRagRetrievalFailureModeTest_Eval(t *testing.T) {
}{
{"correct: keyword", "keyword", 1},
{"correct, different case", "Keyword", 1},
{"correct, quoted", `"keyword"`, 1},
{"wrong: semantic", "semantic", 0},
{"wrong: both", "both", 0},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
@@ -83,14 +87,28 @@ func TestRagCitationGroundingTest_Eval(t *testing.T) {
response: "Every claim must be traceable to a cited source in the retrieved context; if the context does not contain the answer, the system must say it cannot answer rather than guess.",
want: 1,
},
{
name: "both requirements present, alternate phrasing",
response: "Answers must be grounded in and supported by the provided context, with every claim traceable back to it; when the context lacks the needed information, the system must say it is unable to find it instead of guessing.",
want: 1,
},
{
name: "both requirements present, terse phrasing",
response: "Cite the source for every claim; if no relevant context exists, decline to answer.",
want: 1,
},
{
name: "only the citation requirement",
response: "Every claim in the answer must include a citation back to the retrieved context.",
want: 0.5,
},
{
// C10 note: rephrased to avoid "retrieved context" (now one of
// the widened traceability-group cues), so this case still
// cleanly isolates the decline requirement from the citation
// requirement.
name: "only the decline requirement",
response: "If the retrieved context does not contain the answer, the system must say it is unable to find it.",
response: "If the answer isn't there, the system must say it cannot answer.",
want: 0.5,
},
{
@@ -98,6 +116,11 @@ func TestRagCitationGroundingTest_Eval(t *testing.T) {
response: "The answer should just sound confident and helpful.",
want: 0,
},
{
name: "neither requirement, wrong advice entirely",
response: "Just make the answer as detailed as possible.",
want: 0,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
@@ -118,7 +141,9 @@ func TestRagContextSelectionVsStuffingTest_Eval(t *testing.T) {
}{
{"correct: selective", "selective", 1},
{"correct, different case", "Selective", 1},
{"correct, quoted", `"selective"`, 1},
{"wrong: stuffed", "stuffed", 0},
{"wrong: uses all the budget", "all of it", 0},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
@@ -147,6 +172,11 @@ func TestRagIndexStalenessTest_Eval(t *testing.T) {
response: "```json\n{\"problem\":\"stale-chunks-coexist\",\"fix\":\"delete-old-chunks-before-reinsert\"}\n```",
want: 1,
},
{
name: "all correct, different spacing",
response: `{ "problem": "stale-chunks-coexist", "fix": "delete-old-chunks-before-reinsert" }`,
want: 1,
},
{
name: "wrong problem",
response: `{"problem":"embedding-model-drift","fix":"delete-old-chunks-before-reinsert"}`,
@@ -190,6 +220,11 @@ func TestRagEvalMetricChoiceTest_Eval(t *testing.T) {
response: "```json\n{\"scenario_a\":\"context_precision\",\"scenario_b\":\"faithfulness\"}\n```",
want: 1,
},
{
name: "all correct, different spacing",
response: `{ "scenario_a": "context_precision", "scenario_b": "faithfulness" }`,
want: 1,
},
{
name: "scenario_a wrong",
response: `{"scenario_a":"context_recall","scenario_b":"faithfulness"}`,
@@ -233,6 +268,11 @@ func TestRagHallucinationMitigationOrderingTest_Eval(t *testing.T) {
response: "```json\n[\"step_retrieve\",\"step_instruct\",\"step_generate\",\"step_verify\",\"step_flag\"]\n```",
want: 1,
},
{
name: "correct order, different case",
response: `["Step_Retrieve","Step_Instruct","Step_Generate","Step_Verify","Step_Flag"]`,
want: 1,
},
{
name: "verify before generate",
response: `["step_retrieve","step_instruct","step_verify","step_generate","step_flag"]`,
@@ -243,6 +283,11 @@ func TestRagHallucinationMitigationOrderingTest_Eval(t *testing.T) {
response: `["step_retrieve","step_instruct","step_generate","step_flag","step_verify"]`,
want: 0,
},
{
name: "wrong: echoes the prompt's shuffled presentation order verbatim (C4 bug probe)",
response: `["step_flag","step_generate","step_retrieve","step_verify","step_instruct"]`,
want: 0,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
@@ -271,6 +316,11 @@ func TestRagMultihopDecompositionTest_Eval(t *testing.T) {
response: "```json\n[\"frag_q\",\"frag_p\",\"frag_s\",\"frag_r\"]\n```",
want: 1,
},
{
name: "correct order, different case",
response: `["Frag_Q","Frag_P","Frag_S","Frag_R"]`,
want: 1,
},
{
name: "distractor incorrectly included",
response: `["frag_q","frag_p","frag_s","frag_r","frag_x"]`,
@@ -281,6 +331,11 @@ func TestRagMultihopDecompositionTest_Eval(t *testing.T) {
response: `["frag_p","frag_q","frag_s","frag_r"]`,
want: 0,
},
{
name: "wrong: echoes the prompt's shuffled roster order verbatim, minus the distractor (C4 bug probe)",
response: `["frag_s","frag_r","frag_q","frag_p"]`,
want: 0,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
@@ -304,6 +359,16 @@ func TestRagPreAssemblyDedupTest_Eval(t *testing.T) {
response: "Passing all 3 copies wastes context window budget, and the redundant repetition risks biasing the answer toward that one duplicated point.",
want: 1,
},
{
name: "both reasons present, alternate phrasing",
response: "Sending all 3 duplicates dominates the token budget with the same content, and the repetition biases the model toward over-weighting that one point.",
want: 1,
},
{
name: "both reasons present, terse phrasing",
response: "It wastes tokens and skews the answer toward the duplicated content.",
want: 1,
},
{
name: "only the waste reason",
response: "It wastes token budget that could hold other useful chunks.",
@@ -319,6 +384,11 @@ func TestRagPreAssemblyDedupTest_Eval(t *testing.T) {
response: "It just looks messy to a human reviewer.",
want: 0,
},
{
name: "neither reason, wrong justification entirely",
response: "It makes the prompt harder for a human to read.",
want: 0,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {