lukaszraczylo/llm-testbench
1
0
Fork 0
mirror of https://github.com/lukaszraczylo/llm-testbench.git synced 2026-10-17 01:17:33 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(catalog): unify negation guard and harden round-2 evaluators

Browse Source 
A shared NoUnnegatedMention primitive replaces four per-category
implementations, with a bidirectional clause-scoped window and an
adjectival cue set. The docker layer fixture drops WORKDIR, which
creates a real layer and made the pinned answer wrong. The replica
failover test now forces a JSON verdict so a negated answer cannot
score. Remaining exact-answer tests migrate to ExactToken. Ordering
prompts shuffle their candidate lists so echoing scores zero.
This commit is contained in:
lukaszraczylo
2026-08-17 03:50:57 +01:00
parent 9e5deda646
commit 9d1d4d89e9
35 changed files with 1417 additions and 453 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/tests/delivery_release_test.go
+21
View File
@@ -18,6 +18,19 @@ func TestDelRelSemverBumpChangelogTest_Eval(t *testing.T) {
{name: "extra whitespace inside the object", response: `{ "version" : "2.4.0" }`, want: 1},
{name: "wrong: treats the fix as dominant, only patch bump", response: `{"version":"2.3.2"}`, want: 0},
{name: "wrong: major bump for no breaking change", response: `{"version":"3.0.0"}`, want: 0},
{
// DC3 bug probe: the prompt's own last-release version is
// v-prefixed ("v2.3.1"), so a model mirroring that convention
// must not be marked wrong for the same leading "v".
name: "correct: leading v prefix, mirroring the prompt's own convention (DC3 bug probe)",
response: `{"version":"v2.4.0"}`,
want: 1,
},
{
name: "correct: uppercase V prefix (DC3 bug probe)",
response: `{"version":"V2.4.0"}`,
want: 1,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
@@ -239,6 +252,14 @@ func TestDelRelChecksumsSigningPurposeTest_Eval(t *testing.T) {
response: "Both files just make the release page look more professional.",
want: 0,
},
{
// D8 bug probe: "publisher" alone (no "identity") is a
// sufficient authenticity term now that "identity" was
// replaced with more specific alternatives.
name: "correct: bare 'publisher' term (D8 bug probe)",
response: "The checksum catches corruption in transit; the signature ties the file back to the publisher.",
want: 1,
},
{
name: "wrong: only explains the checksum, not the signature",
response: "The checksum lets you confirm the binary was not corrupted in transit.",