mirror of
https://github.com/lukaszraczylo/llm-testbench.git
synced 2026-10-18 01:17:48 +00:00
feat(tests): add 16 advanced distractor-rich scenarios
The 2026-08-30 suite-health review found 223/283 tests saturated. These 16 target the failure modes that still discriminated, one per existing subcategory across all eight categories: - multi-constraint reasoning: infeasible-plan detection, conflicting delegation policy, adversarial tool descriptions - forensic root-causing from noisy evidence: RAG stage attribution, EXPLAIN ANALYZE with a red herring, interleaved multi-source incident logs, README-vs-config conflict resolution - composed gotchas: a Go 1.22 capture + named-return-defer trace, a Python late-binding + shared-mutable-default trace - constraint math: context-budget packing, capacity-fill projection, MVCC anomaly classification, Docker layer-rebuild prediction, dual-key rotation overlap, freeze-window scheduling, an open-redirect exploit chain Every answer is a structured JSON value scored by an exact evaluator, never a prose heuristic (the source of every false positive in the review). A ground-truth self-consistency test asserts each documented correct answer scores 1.0 through its real evaluator.
This commit is contained in:
@@ -0,0 +1,49 @@
|
||||
package tests
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// TestAdvancedGroundTruthScoresFull asserts each advanced test's own
|
||||
// documented correct answer scores 1.0 through its real evaluator - a guard
|
||||
// that the eval and the intended ground truth agree (a JSON typo or a wrong
|
||||
// field path would silently make a test unwinnable).
|
||||
func TestAdvancedGroundTruthScoresFull(t *testing.T) {
|
||||
golden := map[string]string{
|
||||
"adv-agent-plan-infeasible": `{"feasible": false, "blocking_step": "fetch-deps"}`,
|
||||
"adv-agent-deleg-conflicting-policy": `{"agent": "agent-gpu-eu-fast"}`,
|
||||
"adv-agent-tool-adversarial-desc": `{"tool": "trash_bin"}`,
|
||||
"adv-rag-forensics-stage": `{"stage": "generation"}`,
|
||||
"adv-llm-context-budget": `54`,
|
||||
"adv-pg-explain-forensics": `{"cause": "seq-scan", "fix": "add-index"}`,
|
||||
"adv-sql-mvcc-anomaly": `{"anomaly": "non-repeatable-read"}`,
|
||||
"adv-docker-layer-rebuild": `["C","D","F"]`,
|
||||
"adv-rel-freeze-window": `{"deferred_step": "deploy-prod"}`,
|
||||
"adv-scen-multilog-rootcause": `{"root_cause": "oom"}`,
|
||||
"adv-linux-capacity-fill": `7`,
|
||||
"adv-hard-composed-trace": `32`,
|
||||
"adv-py-composed-mutable": `{"first_element": 2, "second_element": 10}`,
|
||||
"adv-code-conflicting-evidence": `{"port": 9090, "authoritative": "config"}`,
|
||||
"adv-sec-exploit-chain": `{"vuln": "open-redirect"}`,
|
||||
"adv-sec-rotation-blast-radius": `30`,
|
||||
}
|
||||
|
||||
byID := map[string]int{}
|
||||
all := All().All()
|
||||
for i, tc := range all {
|
||||
byID[tc.ID] = i
|
||||
}
|
||||
|
||||
for id, answer := range golden {
|
||||
idx, ok := byID[id]
|
||||
if !ok {
|
||||
t.Errorf("%s: not registered in catalog", id)
|
||||
continue
|
||||
}
|
||||
s := all[idx].Eval.Evaluate(context.Background(), answer)
|
||||
if s.Value != 1 {
|
||||
t.Errorf("%s: golden answer %q scored %.2f, want 1.00 (%s)", id, answer, s.Value, s.Detail)
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user