llm-testbench/internal/eval/toolcall_test.go

package eval

import (
	"context"
	"testing"
)

func TestToolCalled(t *testing.T) {
	tests := []struct {
		name     string
		response string
		want     float64
	}{
		{"called", `{"tool_calls":[{"name":"get_weather","arguments":{"city":"Paris"}}]}`, 1},
		{"case-insensitive", `{"tool_calls":[{"name":"Get_Weather","arguments":{}}]}`, 1},
		{"among several", `{"tool_calls":[{"name":"a","arguments":{}},{"name":"get_weather","arguments":{}}]}`, 1},
		{"not called", `{"tool_calls":[{"name":"other","arguments":{}}]}`, 0},
		{"none called", `{"tool_calls":[]}`, 0},
		{"malformed envelope", `not json`, 0},
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			got := ToolCalled("get_weather").Evaluate(context.Background(), tt.response).Value
			if got != tt.want {
				t.Errorf("= %v, want %v", got, tt.want)
			}
		})
	}
}

func TestNoToolCalled(t *testing.T) {
	tests := []struct {
		name     string
		response string
		want     float64
	}{
		{"none called is correct", `{"tool_calls":[],"content":"the answer is 4"}`, 1},
		{"missing tool_calls key is empty", `{"content":"4"}`, 1},
		{"called a tool is wrong", `{"tool_calls":[{"name":"calc","arguments":{}}]}`, 0},
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			got := NoToolCalled().Evaluate(context.Background(), tt.response).Value
			if got != tt.want {
				t.Errorf("= %v, want %v", got, tt.want)
			}
		})
	}
}

func TestToolCallWithArgs(t *testing.T) {
	tests := []struct {
		wantArgs map[string]any
		name     string
		response string
		wantName string
		want     float64
	}{
		{
			name:     "exact match",
			response: `{"tool_calls":[{"name":"book","arguments":{"city":"NYC","nights":3}}]}`,
			wantName: "book", wantArgs: map[string]any{"city": "NYC", "nights": 3}, want: 1,
		},
		{
			name:     "subset match ignores extra args",
			response: `{"tool_calls":[{"name":"book","arguments":{"city":"NYC","nights":3,"pets":true}}]}`,
			wantName: "book", wantArgs: map[string]any{"city": "NYC"}, want: 1,
		},
		{
			name:     "number-as-string coerced",
			response: `{"tool_calls":[{"name":"book","arguments":{"nights":"3"}}]}`,
			wantName: "book", wantArgs: map[string]any{"nights": 3}, want: 1,
		},
		{
			name:     "wrong value",
			response: `{"tool_calls":[{"name":"book","arguments":{"city":"LA"}}]}`,
			wantName: "book", wantArgs: map[string]any{"city": "NYC"}, want: 0,
		},
		{
			name:     "missing arg",
			response: `{"tool_calls":[{"name":"book","arguments":{"nights":3}}]}`,
			wantName: "book", wantArgs: map[string]any{"city": "NYC"}, want: 0,
		},
		{
			name:     "right name malformed args",
			response: `{"tool_calls":[{"name":"book","arguments":null,"arguments_malformed":true}]}`,
			wantName: "book", wantArgs: map[string]any{"city": "NYC"}, want: 0,
		},
		{
			name:     "wrong tool",
			response: `{"tool_calls":[{"name":"other","arguments":{"city":"NYC"}}]}`,
			wantName: "book", wantArgs: map[string]any{"city": "NYC"}, want: 0,
		},
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			got := ToolCallWithArgs(tt.wantName, tt.wantArgs).Evaluate(context.Background(), tt.response).Value
			if got != tt.want {
				t.Errorf("= %v, want %v", got, tt.want)
			}
		})
	}
}

func TestToolSequence(t *testing.T) {
	tests := []struct {
		name     string
		response string
		want     []string
		wantVal  float64
	}{
		{"exact order", `{"tool_calls":[{"name":"a","arguments":{}},{"name":"b","arguments":{}}]}`, []string{"a", "b"}, 1},
		{"wrong order", `{"tool_calls":[{"name":"b","arguments":{}},{"name":"a","arguments":{}}]}`, []string{"a", "b"}, 0},
		{"wrong count", `{"tool_calls":[{"name":"a","arguments":{}}]}`, []string{"a", "b"}, 0},
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			got := ToolSequence(tt.want...).Evaluate(context.Background(), tt.response).Value
			if got != tt.wantVal {
				t.Errorf("= %v, want %v", got, tt.wantVal)
			}
		})
	}
}