mirror of
https://github.com/lukaszraczylo/llm-testbench.git
synced 2026-10-12 00:45:19 +00:00
test(catalog): expand agents to 10 tests per subcategory
Bring agents/tool-routing and agents/planning up to 10 tests each. Add a new agents/delegation subcategory with 10 tests. Split agents.go into agents_toolrouting.go, agents_planning.go, and agents_delegation.go, each with a sibling _test.go file, and reduce agents.go to a thin registration aggregator. Every new prompt is self-contained and forces a JSON or number-only answer. Set-valued answers use JSONStringSet or a custom stage-set evaluator so no arbitrary order is required. Dependency-graph fixtures that force a single valid order are verified unique in the unit tests by brute-force enumeration over every permutation. Numeric ground truth is derived by running the same algorithm the fixture describes, then cross-checked against an independently hand-derived value. MaxTokens is omitted on all 28 new tests per the expansion rules; the 2 existing tests keep their prior value unchanged.
This commit is contained in:
@@ -0,0 +1,286 @@
|
||||
package tests
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestDelegTaskToSpecialistTest_Eval(t *testing.T) {
|
||||
tc := delegTaskToSpecialistTest()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{
|
||||
name: "all correct",
|
||||
response: `{"task1":"devops-debugger","task2":"test-writer","task3":"security-reviewer","task4":"release-manager"}`,
|
||||
want: 1,
|
||||
},
|
||||
{
|
||||
name: "all correct fenced with prose",
|
||||
response: "Here is my mapping:\n```json\n{\"task1\":\"devops-debugger\",\"task2\":\"test-writer\",\"task3\":\"security-reviewer\",\"task4\":\"release-manager\"}\n```",
|
||||
want: 1,
|
||||
},
|
||||
{
|
||||
name: "one wrong: code-writer for infra incident",
|
||||
response: `{"task1":"code-writer","task2":"test-writer","task3":"security-reviewer","task4":"release-manager"}`,
|
||||
want: 0.75,
|
||||
},
|
||||
{
|
||||
name: "one wrong: code-reviewer for security audit",
|
||||
response: `{"task1":"devops-debugger","task2":"test-writer","task3":"code-reviewer","task4":"release-manager"}`,
|
||||
want: 0.75,
|
||||
},
|
||||
{
|
||||
name: "all wrong",
|
||||
response: `{"task1":"orchestrator","task2":"code-writer","task3":"code-reviewer","task4":"code-writer"}`,
|
||||
want: 0,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDelegBuildThenVerifyTest_Eval(t *testing.T) {
|
||||
tc := delegBuildThenVerifyTest()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{name: "correct order", response: `["code-writer","code-reviewer"]`, want: 1},
|
||||
{name: "correct order fenced", response: "```json\n[\"code-writer\",\"code-reviewer\"]\n```", want: 1},
|
||||
{name: "correct order different case", response: `["Code-Writer","Code-Reviewer"]`, want: 1},
|
||||
{name: "reversed order", response: `["code-reviewer","code-writer"]`, want: 0},
|
||||
{name: "wrong second agent", response: `["code-writer","test-writer"]`, want: 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDelegHandoffContextTest_Eval(t *testing.T) {
|
||||
tc := delegHandoffContextTest()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{
|
||||
name: "exact correct set",
|
||||
response: `["file_paths_touched","failing_test_output","reproduction_steps","acceptance_criteria"]`,
|
||||
want: 1,
|
||||
},
|
||||
{
|
||||
name: "exact correct set, different order",
|
||||
response: `["acceptance_criteria","reproduction_steps","file_paths_touched","failing_test_output"]`,
|
||||
want: 1,
|
||||
},
|
||||
{
|
||||
name: "exact correct set, fenced",
|
||||
response: "```json\n[\"file_paths_touched\",\"failing_test_output\",\"reproduction_steps\",\"acceptance_criteria\"]\n```",
|
||||
want: 1,
|
||||
},
|
||||
{
|
||||
name: "includes noise item",
|
||||
response: `["file_paths_touched","failing_test_output","reproduction_steps","acceptance_criteria","unrelated_agent_chat_history"]`,
|
||||
want: 0.8,
|
||||
},
|
||||
{
|
||||
name: "missing an essential item",
|
||||
response: `["file_paths_touched","failing_test_output","reproduction_steps"]`,
|
||||
want: 0.75,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDelegVerifyVsTrustTest_Eval(t *testing.T) {
|
||||
tc := delegVerifyVsTrustTest()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{name: "all correct", response: `{"scenario_a":"trust","scenario_b":"verify"}`, want: 1},
|
||||
{name: "all correct fenced", response: "```json\n{\"scenario_a\":\"trust\",\"scenario_b\":\"verify\"}\n```", want: 1},
|
||||
{name: "scenario_a wrong", response: `{"scenario_a":"verify","scenario_b":"verify"}`, want: 0.5},
|
||||
{name: "scenario_b wrong", response: `{"scenario_a":"trust","scenario_b":"trust"}`, want: 0.5},
|
||||
{name: "both swapped", response: `{"scenario_a":"verify","scenario_b":"trust"}`, want: 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDelegBatchVsSeparateTest_Eval(t *testing.T) {
|
||||
tc := delegBatchVsSeparateTest()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{name: "all correct", response: `{"scenario_a":"batch","scenario_b":"separate"}`, want: 1},
|
||||
{name: "all correct fenced", response: "```json\n{\"scenario_a\":\"batch\",\"scenario_b\":\"separate\"}\n```", want: 1},
|
||||
{name: "scenario_a wrong", response: `{"scenario_a":"separate","scenario_b":"separate"}`, want: 0.5},
|
||||
{name: "scenario_b wrong", response: `{"scenario_a":"batch","scenario_b":"batch"}`, want: 0.5},
|
||||
{name: "both swapped", response: `{"scenario_a":"separate","scenario_b":"batch"}`, want: 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDelegMainThreadVsDelegateTest_Eval(t *testing.T) {
|
||||
tc := delegMainThreadVsDelegateTest()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{name: "all correct", response: `{"scenario_a":"main-thread","scenario_b":"delegate"}`, want: 1},
|
||||
{name: "all correct fenced", response: "```json\n{\"scenario_a\":\"main-thread\",\"scenario_b\":\"delegate\"}\n```", want: 1},
|
||||
{name: "scenario_a wrong", response: `{"scenario_a":"delegate","scenario_b":"delegate"}`, want: 0.5},
|
||||
{name: "scenario_b wrong", response: `{"scenario_a":"main-thread","scenario_b":"main-thread"}`, want: 0.5},
|
||||
{name: "both swapped", response: `{"scenario_a":"delegate","scenario_b":"main-thread"}`, want: 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDelegReviewerIndependenceTest_Eval(t *testing.T) {
|
||||
tc := delegReviewerIndependenceTest()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{name: "all correct", response: `{"case1":"code-reviewer","case2":"code-reviewer"}`, want: 1},
|
||||
{name: "all correct fenced", response: "```json\n{\"case1\":\"code-reviewer\",\"case2\":\"code-reviewer\"}\n```", want: 1},
|
||||
{name: "case1 wrong: self-review", response: `{"case1":"code-writer","case2":"code-reviewer"}`, want: 0.5},
|
||||
{name: "case2 wrong: self-sign-off", response: `{"case1":"code-reviewer","case2":"security-reviewer"}`, want: 0.5},
|
||||
{name: "both wrong", response: `{"case1":"code-writer","case2":"security-reviewer"}`, want: 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDelegEscalationToHumanTest_Eval(t *testing.T) {
|
||||
tc := delegEscalationToHumanTest()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{name: "all correct", response: `{"scenario_a":"escalate","scenario_b":"proceed"}`, want: 1},
|
||||
{name: "all correct fenced", response: "```json\n{\"scenario_a\":\"escalate\",\"scenario_b\":\"proceed\"}\n```", want: 1},
|
||||
{name: "scenario_a wrong", response: `{"scenario_a":"proceed","scenario_b":"proceed"}`, want: 0.5},
|
||||
{name: "scenario_b wrong", response: `{"scenario_a":"escalate","scenario_b":"escalate"}`, want: 0.5},
|
||||
{name: "both swapped", response: `{"scenario_a":"proceed","scenario_b":"escalate"}`, want: 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDelegParallelDispatchSafetyTest_Eval(t *testing.T) {
|
||||
tc := delegParallelDispatchSafetyTest()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{name: "all correct", response: `{"pair_1":"conflict","pair_2":"safe"}`, want: 1},
|
||||
{name: "all correct fenced", response: "```json\n{\"pair_1\":\"conflict\",\"pair_2\":\"safe\"}\n```", want: 1},
|
||||
{name: "pair_1 wrong", response: `{"pair_1":"safe","pair_2":"safe"}`, want: 0.5},
|
||||
{name: "pair_2 wrong", response: `{"pair_1":"conflict","pair_2":"conflict"}`, want: 0.5},
|
||||
{name: "both swapped", response: `{"pair_1":"safe","pair_2":"conflict"}`, want: 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDelegMinimalPrivilegeTest_Eval(t *testing.T) {
|
||||
tc := delegMinimalPrivilegeTest()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{name: "all correct", response: `{"scenario_a":"web-researcher","scenario_b":"release-manager"}`, want: 1},
|
||||
{name: "all correct fenced", response: "```json\n{\"scenario_a\":\"web-researcher\",\"scenario_b\":\"release-manager\"}\n```", want: 1},
|
||||
{name: "scenario_a over-privileged", response: `{"scenario_a":"orchestrator","scenario_b":"release-manager"}`, want: 0.5},
|
||||
{name: "scenario_b over-privileged", response: `{"scenario_a":"web-researcher","scenario_b":"code-writer"}`, want: 0.5},
|
||||
{name: "both over-privileged", response: `{"scenario_a":"orchestrator","scenario_b":"code-writer"}`, want: 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user