test(catalog): expand agents to 10 tests per subcategory

Bring agents/tool-routing and agents/planning up to 10 tests each.
Add a new agents/delegation subcategory with 10 tests. Split
agents.go into agents_toolrouting.go, agents_planning.go, and
agents_delegation.go, each with a sibling _test.go file, and reduce
agents.go to a thin registration aggregator.

Every new prompt is self-contained and forces a JSON or number-only
answer. Set-valued answers use JSONStringSet or a custom stage-set
evaluator so no arbitrary order is required. Dependency-graph fixtures
that force a single valid order are verified unique in the unit tests
by brute-force enumeration over every permutation. Numeric ground
truth is derived by running the same algorithm the fixture describes,
then cross-checked against an independently hand-derived value.

MaxTokens is omitted on all 28 new tests per the expansion rules; the
2 existing tests keep their prior value unchanged.
This commit is contained in:
2026-08-17 01:47:36 +01:00
parent 764fe990fd
commit 053eeab1bb
8 changed files with 2689 additions and 173 deletions
+32 -72
View File
@@ -1,84 +1,44 @@
package tests
import (
"context"
"testing"
"github.com/lukaszraczylo/llm-testbench/internal/testkit"
)
func TestAgentToolRoutingTest_Eval(t *testing.T) {
tc := agentToolRoutingTest()
// TestRegisterAgentsTests_Wiring checks registerAgentsTests wires up all
// three agents subcategories - tool-routing, planning, delegation - with
// 10 tests each, all categorized "agents", and no duplicate IDs across the
// three sibling files (testkit.Registry.Register panics on a duplicate ID,
// which would fail this test outright).
func TestRegisterAgentsTests_Wiring(t *testing.T) {
r := testkit.NewRegistry()
registerAgentsTests(r)
tests := []struct {
name string
response string
want float64
}{
{
name: "all correct",
response: `{"task1":"none","task2":"query_db","task3":"search_web","task4":"send_email"}`,
want: 1,
},
{
name: "one wrong",
response: `{"task1":"run_shell","task2":"query_db","task3":"search_web","task4":"send_email"}`,
want: 0.75,
},
{
name: "all wrong",
response: `{"task1":"run_shell","task2":"search_web","task3":"query_db","task4":"none"}`,
want: 0,
},
const wantTotal = 30
if r.Len() != wantTotal {
t.Fatalf("registerAgentsTests: registry has %d tests, want %d", r.Len(), wantTotal)
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := tc.Eval.Evaluate(context.Background(), tt.response)
if got.Value != tt.want {
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
}
})
}
}
func TestAgentPlanOrderingTest_Eval(t *testing.T) {
tc := agentPlanOrderingTest()
tests := []struct {
name string
response string
want float64
}{
{
name: "correct order",
response: `["build","test","backup","deploy","verify","rollback"]`,
want: 1,
},
{
name: "correct order fenced",
response: "```json\n[\"build\",\"test\",\"backup\",\"deploy\",\"verify\",\"rollback\"]\n```",
want: 1,
},
{
name: "backup before test violates dependency",
response: `["build","backup","test","deploy","verify","rollback"]`,
want: 0,
},
{
name: "rollback before verify",
response: `["build","test","backup","deploy","rollback","verify"]`,
want: 0,
},
{
name: "missing a step",
response: `["build","test","backup","deploy","verify"]`,
want: 0,
},
counts := map[string]int{}
for _, tc := range r.All() {
if tc.Category != "agents" {
t.Errorf("test %q: Category = %q, want %q", tc.ID, tc.Category, "agents")
}
if tc.Prompt == "" {
t.Errorf("test %q: Prompt is empty", tc.ID)
}
if tc.Eval == nil {
t.Errorf("test %q: Eval is nil", tc.ID)
}
if tc.Description == "" {
t.Errorf("test %q: Description is empty", tc.ID)
}
counts[tc.Subcategory]++
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := tc.Eval.Evaluate(context.Background(), tt.response)
if got.Value != tt.want {
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
}
})
for _, sub := range []string{"tool-routing", "planning", "delegation"} {
if counts[sub] != 10 {
t.Errorf("subcategory %q: got %d tests, want 10", sub, counts[sub])
}
}
}