fix(catalog): unify negation guard and harden round-2 evaluators

A shared NoUnnegatedMention primitive replaces four per-category
implementations, with a bidirectional clause-scoped window and an
adjectival cue set. The docker layer fixture drops WORKDIR, which
creates a real layer and made the pinned answer wrong. The replica
failover test now forces a JSON verdict so a negated answer cannot
score. Remaining exact-answer tests migrate to ExactToken. Ordering
prompts shuffle their candidate lists so echoing scores zero.
This commit is contained in:
2026-08-17 03:50:57 +01:00
parent 9e5deda646
commit 9d1d4d89e9
35 changed files with 1417 additions and 453 deletions
+28
View File
@@ -22,6 +22,11 @@ func TestLlmFieldSemanticsTest_Eval(t *testing.T) {
response: "Here is my answer:\n```json\n{\"temperature\":\"sampling_randomness\",\"max_tokens\":\"completion_token_cap\",\"stop\":\"generation_end_strings\"}\n```",
want: 1,
},
{
name: "all correct, different spacing",
response: `{ "temperature": "sampling_randomness", "max_tokens": "completion_token_cap", "stop": "generation_end_strings" }`,
want: 1,
},
{
name: "one wrong",
response: `{"temperature":"output_format","max_tokens":"completion_token_cap","stop":"generation_end_strings"}`,
@@ -65,6 +70,7 @@ func TestLlmTokenBudgetReasoningTest_Eval(t *testing.T) {
}{
{"exact", "63", 1},
{"prose wrapped", "63 completion tokens remain.", 1},
{"fenced", "```\n63\n```", 1},
{"wrong: forgot to subtract", "500", 0},
{"wrong: reported the reasoning tokens used, not remaining", "437", 0},
}
@@ -87,6 +93,7 @@ func TestLlm429RetryBackoffTest_Eval(t *testing.T) {
}{
{"all correct", `{"should_retry":"yes","wait_seconds":2}`, 1},
{"all correct fenced", "```json\n{\"should_retry\":\"yes\",\"wait_seconds\":2}\n```", 1},
{"all correct, different spacing", `{ "should_retry": "yes", "wait_seconds": 2 }`, 1},
{"wrong: should not retry", `{"should_retry":"no","wait_seconds":2}`, 0.5},
{"wrong: ignored Retry-After value", `{"should_retry":"yes","wait_seconds":30}`, 0.5},
{"both wrong", `{"should_retry":"no","wait_seconds":30}`, 0},
@@ -110,6 +117,7 @@ func TestLlmContextOverflowStrategyTest_Eval(t *testing.T) {
}{
{"correct: drop-oldest-turns", `{"strategy":"drop-oldest-turns"}`, 1},
{"correct, fenced with prose", "Given the no-altering constraint:\n```json\n{\"strategy\":\"drop-oldest-turns\"}\n```", 1},
{"correct, different spacing", `{ "strategy": "drop-oldest-turns" }`, 1},
{"wrong: summarize-oldest-turns", `{"strategy":"summarize-oldest-turns"}`, 0},
{"wrong: truncate-system-prompt", `{"strategy":"truncate-system-prompt"}`, 0},
}
@@ -132,6 +140,7 @@ func TestLlmToolCallHandlingTest_Eval(t *testing.T) {
}{
{"all correct", `{"next_role":"tool","id_field":"tool_call_id"}`, 1},
{"all correct fenced", "```json\n{\"next_role\":\"tool\",\"id_field\":\"tool_call_id\"}\n```", 1},
{"all correct, different spacing", `{ "next_role": "tool", "id_field": "tool_call_id" }`, 1},
{"wrong role", `{"next_role":"assistant","id_field":"tool_call_id"}`, 0.5},
{"wrong field", `{"next_role":"tool","id_field":"id"}`, 0.5},
{"both wrong", `{"next_role":"user","id_field":"call_id"}`, 0},
@@ -155,7 +164,9 @@ func TestLlmRolePlacementTest_Eval(t *testing.T) {
}{
{"correct: system", "system", 1},
{"correct, different case", "System", 1},
{"correct, quoted", `"system"`, 1},
{"wrong: user", "user", 0},
{"wrong: assistant", "assistant", 0},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
@@ -179,6 +190,16 @@ func TestLlmTemperatureZeroCaveatTest_Eval(t *testing.T) {
response: "Floating-point arithmetic is not strictly associative, and provider-side batching or GPU kernel routing changes summation order, which can flip the top token.",
want: 1,
},
{
name: "both reasons present, alternate phrasing",
response: "Numerical precision in floating-point math is not associative, and the hardware kernel path a batched request is routed through can differ between calls.",
want: 1,
},
{
name: "both reasons present, terse phrasing",
response: "Floating-point precision issues plus parallel execution batching on the GPU.",
want: 1,
},
{
name: "only the numerical reason",
response: "It is because of floating point rounding differences.",
@@ -194,6 +215,11 @@ func TestLlmTemperatureZeroCaveatTest_Eval(t *testing.T) {
response: "The model just has some inherent randomness left even at temperature zero.",
want: 0,
},
{
name: "neither reason, wrong explanation entirely",
response: "It happens because the random seed is not fixed across requests.",
want: 0,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
@@ -255,6 +281,7 @@ func TestLlmEmbeddingBatchMathTest_Eval(t *testing.T) {
}{
{"exact", "105", 1},
{"prose wrapped", "You need 105 requests.", 1},
{"fenced", "```\n105\n```", 1},
{"wrong: dropped the partial batch", "104", 0},
{"wrong: off by a lot", "96", 0},
}
@@ -277,6 +304,7 @@ func TestLlmStopSequenceTraceTest_Eval(t *testing.T) {
}{
{"correct", "Answer: 42", 1},
{"correct with whitespace", " Answer: 42 ", 1},
{"correct, quoted", `"Answer: 42"`, 1},
{"wrong: included text past the stop sequence", "Answer: 42\n\nExplanation: 6 times 7 is 42.", 0},
{"wrong: paraphrased instead of quoting exactly", "The answer is 42", 0},
}