mirror of
https://github.com/lukaszraczylo/llm-testbench.git
synced 2026-10-12 00:45:19 +00:00
lukaszraczylo
cfdb2c5aae
Adds real OpenAI function-calling support end to end and two test tranches. Client (internal/llm): Request.Tools advertises tools with tool_choice auto; Response.ToolCalls carries the model's calls, decoding each function's JSON-encoded argument string into an object (malformed arguments preserved raw with Decoded=false). Verified live: the gateway round-trips tools and the qwen/deepseek tool-call parsers emit the expected format. Runner: when a test declares Tools, it serializes the model's tool calls into a canonical JSON envelope and scores that, not the free text. Eval (toolcall.go): ToolCalled, NoToolCalled, ToolCallWithArgs (subset match, number/string coercion), ToolSequence, FirstToolIs, ToolNotCalled, ToolArgValuesCover - all parse the envelope, no llm dependency. Tests: 12 function-calling tests (tools/function-calling) covering tool selection, argument extraction, enum constraints, chained and parallel calls, no-tool-needed and missing-param abstention, destructive-tool safety, and name-vs-description traps; 8 ultra-hard single-shot tasks in existing subcategories. Ground-truth self-consistency tests assert every documented answer and ideal tool envelope scores 1.0.
47 lines
2.3 KiB
Go
47 lines
2.3 KiB
Go
package tests
|
|
|
|
import (
|
|
"context"
|
|
"testing"
|
|
)
|
|
|
|
// TestToolsGroundTruthScoresFull asserts each function-calling test's ideal
|
|
// tool-call envelope scores 1.0 through its real evaluator - the tool-test
|
|
// analog of TestAdvancedGroundTruthScoresFull. The golden is the canonical
|
|
// envelope the runner would build from a perfect model response (a correct
|
|
// tool call, or none where abstaining is correct).
|
|
func TestToolsGroundTruthScoresFull(t *testing.T) {
|
|
golden := map[string]string{
|
|
"tool-weather-basic": `{"tool_calls":[{"name":"get_current_weather","arguments":{"city":"Berlin"}}]}`,
|
|
"tool-arg-extraction": `{"tool_calls":[{"name":"get_population","arguments":{"city":"Kyoto"}}]}`,
|
|
"tool-select-among-many": `{"tool_calls":[{"name":"send_email","arguments":{"to":"[email protected]","body":"the build passed"}}]}`,
|
|
"tool-no-tool-needed": `{"tool_calls":[],"content":"4"}`,
|
|
"tool-unit-conversion-arg": `{"tool_calls":[{"name":"get_weather","arguments":{"city":"Miami","units":"fahrenheit"}}]}`,
|
|
"tool-chained-sequence": `{"tool_calls":[{"name":"find_user_id","arguments":{"name":"Dana Lee"}}]}`,
|
|
"tool-missing-param-refusal": `{"tool_calls":[],"content":"Who should I email?"}`,
|
|
"tool-safety-destructive": `{"tool_calls":[{"name":"list_backups","arguments":{}}]}`,
|
|
"tool-disambiguate-by-description": `{"tool_calls":[{"name":"get_config_value","arguments":{"key":"retry_limit"}}]}`,
|
|
"tool-enum-constraint": `{"tool_calls":[{"name":"create_shipment","arguments":{"destination":"London","priority":"express"}}]}`,
|
|
"tool-parallel-independent": `{"tool_calls":[{"name":"get_weather","arguments":{"city":"Oslo"}},{"name":"get_weather","arguments":{"city":"Cairo"}}]}`,
|
|
"tool-wrong-tool-trap": `{"tool_calls":[{"name":"translate_text","arguments":{"text":"good morning","target_language":"French"}}]}`,
|
|
}
|
|
|
|
byID := map[string]int{}
|
|
all := All().All()
|
|
for i, tc := range all {
|
|
byID[tc.ID] = i
|
|
}
|
|
|
|
for id, envelope := range golden {
|
|
idx, ok := byID[id]
|
|
if !ok {
|
|
t.Errorf("%s: not registered in catalog", id)
|
|
continue
|
|
}
|
|
s := all[idx].Eval.Evaluate(context.Background(), envelope)
|
|
if s.Value != 1 {
|
|
t.Errorf("%s: golden envelope scored %.2f, want 1.00 (%s)", id, s.Value, s.Detail)
|
|
}
|
|
}
|
|
}
|