mirror of
https://github.com/lukaszraczylo/llm-testbench.git
synced 2026-10-12 00:45:19 +00:00
lukaszraczylo
053eeab1bb
Bring agents/tool-routing and agents/planning up to 10 tests each. Add a new agents/delegation subcategory with 10 tests. Split agents.go into agents_toolrouting.go, agents_planning.go, and agents_delegation.go, each with a sibling _test.go file, and reduce agents.go to a thin registration aggregator. Every new prompt is self-contained and forces a JSON or number-only answer. Set-valued answers use JSONStringSet or a custom stage-set evaluator so no arbitrary order is required. Dependency-graph fixtures that force a single valid order are verified unique in the unit tests by brute-force enumeration over every permutation. Numeric ground truth is derived by running the same algorithm the fixture describes, then cross-checked against an independently hand-derived value. MaxTokens is omitted on all 28 new tests per the expansion rules; the 2 existing tests keep their prior value unchanged.
322 lines
9.6 KiB
Go
322 lines
9.6 KiB
Go
package tests
|
|
|
|
import (
|
|
"context"
|
|
"testing"
|
|
)
|
|
|
|
func TestAgentToolRoutingTest_Eval(t *testing.T) {
|
|
tc := agentToolRoutingTest()
|
|
|
|
tests := []struct {
|
|
name string
|
|
response string
|
|
want float64
|
|
}{
|
|
{
|
|
name: "all correct",
|
|
response: `{"task1":"none","task2":"query_db","task3":"search_web","task4":"send_email"}`,
|
|
want: 1,
|
|
},
|
|
{
|
|
name: "all correct fenced with prose",
|
|
response: "Here is my answer:\n```json\n{\"task1\":\"none\",\"task2\":\"query_db\",\"task3\":\"search_web\",\"task4\":\"send_email\"}\n```",
|
|
want: 1,
|
|
},
|
|
{
|
|
name: "one wrong",
|
|
response: `{"task1":"run_shell","task2":"query_db","task3":"search_web","task4":"send_email"}`,
|
|
want: 0.75,
|
|
},
|
|
{
|
|
name: "all wrong",
|
|
response: `{"task1":"run_shell","task2":"search_web","task3":"query_db","task4":"none"}`,
|
|
want: 0,
|
|
},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
|
if got.Value != tt.want {
|
|
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestRouteDistractorsTest_Eval(t *testing.T) {
|
|
tc := routeDistractorsTest()
|
|
|
|
tests := []struct {
|
|
name string
|
|
response string
|
|
want float64
|
|
}{
|
|
{
|
|
name: "all correct",
|
|
response: `{"task_a":"fetch_url","task_b":"list_directory","task_c":"none","task_d":"search_web"}`,
|
|
want: 1,
|
|
},
|
|
{
|
|
name: "all correct with different casing and spacing",
|
|
response: " {\"task_a\": \"Fetch_Url\", \"task_b\": \"List_Directory\", \"task_c\": \"None\", \"task_d\": \"Search_Web\"} ",
|
|
want: 1,
|
|
},
|
|
{
|
|
name: "distractor picked for task_a",
|
|
response: `{"task_a":"search_web","task_b":"list_directory","task_c":"none","task_d":"search_web"}`,
|
|
want: 0.75,
|
|
},
|
|
{
|
|
name: "distractor picked for task_b",
|
|
response: `{"task_a":"fetch_url","task_b":"read_file","task_c":"none","task_d":"search_web"}`,
|
|
want: 0.75,
|
|
},
|
|
{
|
|
name: "all wrong",
|
|
response: `{"task_a":"none","task_b":"fetch_url","task_c":"read_file","task_d":"list_directory"}`,
|
|
want: 0,
|
|
},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
|
if got.Value != tt.want {
|
|
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestRouteNoToolNeededTest_Eval(t *testing.T) {
|
|
tc := routeNoToolNeededTest()
|
|
|
|
tests := []struct {
|
|
name string
|
|
response string
|
|
want float64
|
|
}{
|
|
{name: "correct: none", response: `{"tool":"none"}`, want: 1},
|
|
{name: "correct: none, fenced", response: "```json\n{\"tool\":\"none\"}\n```", want: 1},
|
|
{name: "correct: none, uppercase", response: `{"tool":"NONE"}`, want: 1},
|
|
{name: "wrong: read_file", response: `{"tool":"read_file"}`, want: 0},
|
|
{name: "wrong: search_web", response: `{"tool":"search_web"}`, want: 0},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
|
if got.Value != tt.want {
|
|
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestRouteMultistepTest_Eval(t *testing.T) {
|
|
tc := routeMultistepTest()
|
|
|
|
tests := []struct {
|
|
name string
|
|
response string
|
|
want float64
|
|
}{
|
|
{name: "correct order", response: `["search_web","send_email"]`, want: 1},
|
|
{name: "correct order fenced", response: "```json\n[\"search_web\",\"send_email\"]\n```", want: 1},
|
|
{name: "correct order different case", response: `["Search_Web","Send_Email"]`, want: 1},
|
|
{name: "reversed order", response: `["send_email","search_web"]`, want: 0},
|
|
{name: "missing a tool", response: `["search_web"]`, want: 0},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
|
if got.Value != tt.want {
|
|
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestRouteCheapestToolTest_Eval(t *testing.T) {
|
|
tc := routeCheapestToolTest()
|
|
|
|
tests := []struct {
|
|
name string
|
|
response string
|
|
want float64
|
|
}{
|
|
{name: "correct: read_file", response: `{"tool":"read_file"}`, want: 1},
|
|
{name: "correct: read_file, fenced with prose", response: "The cheapest option is:\n```json\n{\"tool\":\"read_file\"}\n```", want: 1},
|
|
{name: "correct: read_file, uppercase", response: `{"tool":"READ_FILE"}`, want: 1},
|
|
{name: "wrong: query_db", response: `{"tool":"query_db"}`, want: 0},
|
|
{name: "wrong: search_web", response: `{"tool":"search_web"}`, want: 0},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
|
if got.Value != tt.want {
|
|
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestRouteParallelDispatchTest_Eval(t *testing.T) {
|
|
tc := routeParallelDispatchTest()
|
|
|
|
tests := []struct {
|
|
name string
|
|
response string
|
|
want float64
|
|
}{
|
|
{
|
|
name: "all correct",
|
|
response: `{"task_x":"parallel","task_y":"sequential"}`,
|
|
want: 1,
|
|
},
|
|
{
|
|
name: "all correct fenced",
|
|
response: "```json\n{\"task_x\":\"parallel\",\"task_y\":\"sequential\"}\n```",
|
|
want: 1,
|
|
},
|
|
{
|
|
name: "task_x wrong",
|
|
response: `{"task_x":"sequential","task_y":"sequential"}`,
|
|
want: 0.5,
|
|
},
|
|
{
|
|
name: "task_y wrong",
|
|
response: `{"task_x":"parallel","task_y":"parallel"}`,
|
|
want: 0.5,
|
|
},
|
|
{
|
|
name: "both swapped",
|
|
response: `{"task_x":"sequential","task_y":"parallel"}`,
|
|
want: 0,
|
|
},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
|
if got.Value != tt.want {
|
|
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestRouteMissingParamTest_Eval(t *testing.T) {
|
|
tc := routeMissingParamTest()
|
|
|
|
tests := []struct {
|
|
name string
|
|
response string
|
|
want float64
|
|
}{
|
|
{
|
|
name: "correct: cannot, recipient",
|
|
response: `{"decision":"cannot","missing_param":"recipient"}`,
|
|
want: 1,
|
|
},
|
|
{
|
|
name: "correct, fenced with prose",
|
|
response: "I cannot call send_email as specified.\n```json\n{\"decision\":\"cannot\",\"missing_param\":\"recipient\"}\n```",
|
|
want: 1,
|
|
},
|
|
{
|
|
name: "correct, different case",
|
|
response: `{"decision":"CANNOT","missing_param":"Recipient"}`,
|
|
want: 1,
|
|
},
|
|
{
|
|
name: "wrong decision",
|
|
response: `{"decision":"can","missing_param":"recipient"}`,
|
|
want: 0.5,
|
|
},
|
|
{
|
|
name: "wrong missing param",
|
|
response: `{"decision":"cannot","missing_param":"subject"}`,
|
|
want: 0.5,
|
|
},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
|
if got.Value != tt.want {
|
|
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestRouteAmbiguousTest_Eval(t *testing.T) {
|
|
tc := routeAmbiguousTest()
|
|
|
|
tests := []struct {
|
|
name string
|
|
response string
|
|
want float64
|
|
}{
|
|
{name: "all correct", response: `{"task_p":"clarify","task_q":"act"}`, want: 1},
|
|
{name: "all correct fenced", response: "```json\n{\"task_p\":\"clarify\",\"task_q\":\"act\"}\n```", want: 1},
|
|
{name: "task_p wrong", response: `{"task_p":"act","task_q":"act"}`, want: 0.5},
|
|
{name: "task_q wrong", response: `{"task_p":"clarify","task_q":"clarify"}`, want: 0.5},
|
|
{name: "both swapped", response: `{"task_p":"act","task_q":"clarify"}`, want: 0},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
|
if got.Value != tt.want {
|
|
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestRouteChainingTest_Eval(t *testing.T) {
|
|
tc := routeChainingTest()
|
|
|
|
tests := []struct {
|
|
name string
|
|
response string
|
|
want float64
|
|
}{
|
|
{name: "correct: send_email", response: `{"next_tool":"send_email"}`, want: 1},
|
|
{name: "correct, fenced with prose", response: "Next I should:\n```json\n{\"next_tool\":\"send_email\"}\n```", want: 1},
|
|
{name: "correct, uppercase", response: `{"next_tool":"SEND_EMAIL"}`, want: 1},
|
|
{name: "wrong: run_shell", response: `{"next_tool":"run_shell"}`, want: 0},
|
|
{name: "wrong: none", response: `{"next_tool":"none"}`, want: 0},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
|
if got.Value != tt.want {
|
|
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestRouteSafetyTest_Eval(t *testing.T) {
|
|
tc := routeSafetyTest()
|
|
|
|
tests := []struct {
|
|
name string
|
|
response string
|
|
want float64
|
|
}{
|
|
{name: "correct: query_db", response: `{"tool":"query_db"}`, want: 1},
|
|
{name: "correct, fenced with prose", response: "The safest choice:\n```json\n{\"tool\":\"query_db\"}\n```", want: 1},
|
|
{name: "correct, uppercase", response: `{"tool":"QUERY_DB"}`, want: 1},
|
|
{name: "wrong: modify_db", response: `{"tool":"modify_db"}`, want: 0},
|
|
{name: "wrong: none", response: `{"tool":"none"}`, want: 0},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
|
if got.Value != tt.want {
|
|
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
|
}
|
|
})
|
|
}
|
|
}
|