mirror of
https://github.com/lukaszraczylo/llm-testbench.git
synced 2026-10-15 01:07:26 +00:00
feat: function-calling (MCP-style) tests plus ultra-hard tasks
Adds real OpenAI function-calling support end to end and two test tranches. Client (internal/llm): Request.Tools advertises tools with tool_choice auto; Response.ToolCalls carries the model's calls, decoding each function's JSON-encoded argument string into an object (malformed arguments preserved raw with Decoded=false). Verified live: the gateway round-trips tools and the qwen/deepseek tool-call parsers emit the expected format. Runner: when a test declares Tools, it serializes the model's tool calls into a canonical JSON envelope and scores that, not the free text. Eval (toolcall.go): ToolCalled, NoToolCalled, ToolCallWithArgs (subset match, number/string coercion), ToolSequence, FirstToolIs, ToolNotCalled, ToolArgValuesCover - all parse the envelope, no llm dependency. Tests: 12 function-calling tests (tools/function-calling) covering tool selection, argument extraction, enum constraints, chained and parallel calls, no-tool-needed and missing-param abstention, destructive-tool safety, and name-vs-description traps; 8 ultra-hard single-shot tasks in existing subcategories. Ground-truth self-consistency tests assert every documented answer and ideal tool envelope scores 1.0.
This commit is contained in:
@@ -0,0 +1,124 @@
|
||||
package eval
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestToolCalled(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{"called", `{"tool_calls":[{"name":"get_weather","arguments":{"city":"Paris"}}]}`, 1},
|
||||
{"case-insensitive", `{"tool_calls":[{"name":"Get_Weather","arguments":{}}]}`, 1},
|
||||
{"among several", `{"tool_calls":[{"name":"a","arguments":{}},{"name":"get_weather","arguments":{}}]}`, 1},
|
||||
{"not called", `{"tool_calls":[{"name":"other","arguments":{}}]}`, 0},
|
||||
{"none called", `{"tool_calls":[]}`, 0},
|
||||
{"malformed envelope", `not json`, 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := ToolCalled("get_weather").Evaluate(context.Background(), tt.response).Value
|
||||
if got != tt.want {
|
||||
t.Errorf("= %v, want %v", got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestNoToolCalled(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{"none called is correct", `{"tool_calls":[],"content":"the answer is 4"}`, 1},
|
||||
{"missing tool_calls key is empty", `{"content":"4"}`, 1},
|
||||
{"called a tool is wrong", `{"tool_calls":[{"name":"calc","arguments":{}}]}`, 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := NoToolCalled().Evaluate(context.Background(), tt.response).Value
|
||||
if got != tt.want {
|
||||
t.Errorf("= %v, want %v", got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestToolCallWithArgs(t *testing.T) {
|
||||
tests := []struct {
|
||||
wantArgs map[string]any
|
||||
name string
|
||||
response string
|
||||
wantName string
|
||||
want float64
|
||||
}{
|
||||
{
|
||||
name: "exact match",
|
||||
response: `{"tool_calls":[{"name":"book","arguments":{"city":"NYC","nights":3}}]}`,
|
||||
wantName: "book", wantArgs: map[string]any{"city": "NYC", "nights": 3}, want: 1,
|
||||
},
|
||||
{
|
||||
name: "subset match ignores extra args",
|
||||
response: `{"tool_calls":[{"name":"book","arguments":{"city":"NYC","nights":3,"pets":true}}]}`,
|
||||
wantName: "book", wantArgs: map[string]any{"city": "NYC"}, want: 1,
|
||||
},
|
||||
{
|
||||
name: "number-as-string coerced",
|
||||
response: `{"tool_calls":[{"name":"book","arguments":{"nights":"3"}}]}`,
|
||||
wantName: "book", wantArgs: map[string]any{"nights": 3}, want: 1,
|
||||
},
|
||||
{
|
||||
name: "wrong value",
|
||||
response: `{"tool_calls":[{"name":"book","arguments":{"city":"LA"}}]}`,
|
||||
wantName: "book", wantArgs: map[string]any{"city": "NYC"}, want: 0,
|
||||
},
|
||||
{
|
||||
name: "missing arg",
|
||||
response: `{"tool_calls":[{"name":"book","arguments":{"nights":3}}]}`,
|
||||
wantName: "book", wantArgs: map[string]any{"city": "NYC"}, want: 0,
|
||||
},
|
||||
{
|
||||
name: "right name malformed args",
|
||||
response: `{"tool_calls":[{"name":"book","arguments":null,"arguments_malformed":true}]}`,
|
||||
wantName: "book", wantArgs: map[string]any{"city": "NYC"}, want: 0,
|
||||
},
|
||||
{
|
||||
name: "wrong tool",
|
||||
response: `{"tool_calls":[{"name":"other","arguments":{"city":"NYC"}}]}`,
|
||||
wantName: "book", wantArgs: map[string]any{"city": "NYC"}, want: 0,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := ToolCallWithArgs(tt.wantName, tt.wantArgs).Evaluate(context.Background(), tt.response).Value
|
||||
if got != tt.want {
|
||||
t.Errorf("= %v, want %v", got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestToolSequence(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want []string
|
||||
wantVal float64
|
||||
}{
|
||||
{"exact order", `{"tool_calls":[{"name":"a","arguments":{}},{"name":"b","arguments":{}}]}`, []string{"a", "b"}, 1},
|
||||
{"wrong order", `{"tool_calls":[{"name":"b","arguments":{}},{"name":"a","arguments":{}}]}`, []string{"a", "b"}, 0},
|
||||
{"wrong count", `{"tool_calls":[{"name":"a","arguments":{}}]}`, []string{"a", "b"}, 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := ToolSequence(tt.want...).Evaluate(context.Background(), tt.response).Value
|
||||
if got != tt.wantVal {
|
||||
t.Errorf("= %v, want %v", got, tt.wantVal)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user