lukaszraczylo/llm-testbench
1
0
Fork 0
mirror of https://github.com/lukaszraczylo/llm-testbench.git synced 2026-10-12 00:45:19 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(catalog): unify negation guard and harden round-2 evaluators

Browse Source 
A shared NoUnnegatedMention primitive replaces four per-category
implementations, with a bidirectional clause-scoped window and an
adjectival cue set. The docker layer fixture drops WORKDIR, which
creates a real layer and made the pinned answer wrong. The replica
failover test now forces a JSON verdict so a negated answer cannot
score. Remaining exact-answer tests migrate to ExactToken. Ordering
prompts shuffle their candidate lists so echoing scores zero.
This commit is contained in:
lukaszraczylo
2026-08-17 03:50:57 +01:00
parent 9e5deda646
commit 9d1d4d89e9
35 changed files with 1417 additions and 453 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/tests/security_secrets_test.go
+16
View File
@@ -62,6 +62,14 @@ func TestSecRemediationOrderTest_Eval(t *testing.T) {
response: `["rewrite-git-history-to-purge-it","rotate-the-leaked-credential","remove-secret-from-current-code","force-push-and-notify-collaborators"]`,
want: 0,
},
{
// C4 bug probe: the prompt's presented list is hand-shuffled to
// a different order than the correct answer, so echoing the
// prompt's own list order must score 0.
name: "wrong: echoes the prompt's shuffled presentation order verbatim (C4 bug probe)",
response: `["force-push-and-notify-collaborators","remove-secret-from-current-code","rotate-the-leaked-credential","rewrite-git-history-to-purge-it"]`,
want: 0,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
@@ -154,6 +162,14 @@ func TestSecRotationNoDowntimeTest_Eval(t *testing.T) {
response: `["create-new-credential-alongside-old","revoke-old-credential","deploy-config-with-new-credential-to-all-replicas","verify-all-replicas-using-new-credential"]`,
want: 0,
},
{
// C4 bug probe: the prompt's presented list is hand-shuffled to
// a different order than the correct answer, so echoing the
// prompt's own list order must score 0.
name: "wrong: echoes the prompt's shuffled presentation order verbatim (C4 bug probe)",
response: `["verify-all-replicas-using-new-credential","revoke-old-credential","create-new-credential-alongside-old","deploy-config-with-new-credential-to-all-replicas"]`,
want: 0,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {