feat: function-calling (MCP-style) tests plus ultra-hard tasks

Adds real OpenAI function-calling support end to end and two test tranches.

Client (internal/llm): Request.Tools advertises tools with tool_choice
auto; Response.ToolCalls carries the model's calls, decoding each
function's JSON-encoded argument string into an object (malformed
arguments preserved raw with Decoded=false). Verified live: the gateway
round-trips tools and the qwen/deepseek tool-call parsers emit the
expected format.

Runner: when a test declares Tools, it serializes the model's tool calls
into a canonical JSON envelope and scores that, not the free text.

Eval (toolcall.go): ToolCalled, NoToolCalled, ToolCallWithArgs (subset
match, number/string coercion), ToolSequence, FirstToolIs, ToolNotCalled,
ToolArgValuesCover - all parse the envelope, no llm dependency.

Tests: 12 function-calling tests (tools/function-calling) covering tool
selection, argument extraction, enum constraints, chained and parallel
calls, no-tool-needed and missing-param abstention, destructive-tool
safety, and name-vs-description traps; 8 ultra-hard single-shot tasks in
existing subcategories. Ground-truth self-consistency tests assert every
documented answer and ideal tool envelope scores 1.0.
This commit is contained in:
2026-08-30 20:22:20 +01:00
parent 708622e2a8
commit cfdb2c5aae
13 changed files with 1076 additions and 9 deletions
+67 -2
View File
@@ -26,12 +26,12 @@ func newTestServer(t *testing.T, handler http.HandlerFunc) *httptest.Server {
// element type, so callers can append() it without repeating the anonymous
// struct type at each call site.
func testChoice(content, finishReason string) struct {
Message responseMsg `json:"message"`
FinishReason string `json:"finish_reason"`
Message responseMsg `json:"message"`
} {
return struct {
Message responseMsg `json:"message"`
FinishReason string `json:"finish_reason"`
Message responseMsg `json:"message"`
}{
Message: responseMsg{Role: "assistant", Content: &content},
FinishReason: finishReason,
@@ -329,3 +329,68 @@ func TestOpenAIClient_Complete_SendsAPIKey(t *testing.T) {
t.Fatalf("Complete() error = %v", err)
}
}
func TestOpenAIClient_Complete_ParsesToolCalls(t *testing.T) {
srv := newTestServer(t, func(w http.ResponseWriter, r *http.Request) {
var body chatCompletionRequest
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
t.Fatalf("decode request: %v", err)
}
// The client must advertise tools with tool_choice auto.
if len(body.Tools) != 1 || body.Tools[0].Function.Name != "get_weather" {
t.Errorf("tools not sent correctly: %+v", body.Tools)
}
if body.ToolChoice != "auto" {
t.Errorf("tool_choice = %q, want auto", body.ToolChoice)
}
_, _ = w.Write([]byte(`{"choices":[{"message":{"role":"assistant","content":null,"tool_calls":[` +
`{"type":"function","function":{"name":"get_weather","arguments":"{\"city\":\"Paris\",\"units\":\"c\"}"}}` +
`]},"finish_reason":"tool_calls"}],"usage":{"prompt_tokens":5,"completion_tokens":9}}`))
})
client := NewOpenAIClient(srv.URL, "", time.Second)
resp, err := client.Complete(context.Background(), Request{
Model: "m",
Messages: []Message{{Role: "user", Content: "weather in Paris?"}},
Tools: []Tool{{
Name: "get_weather",
Description: "Get weather for a city",
Parameters: map[string]any{"type": "object"},
}},
})
if err != nil {
t.Fatalf("Complete() error = %v", err)
}
if len(resp.ToolCalls) != 1 {
t.Fatalf("got %d tool calls, want 1", len(resp.ToolCalls))
}
tc := resp.ToolCalls[0]
if tc.Name != "get_weather" || !tc.Decoded {
t.Errorf("tool call = %+v", tc)
}
if tc.Arguments["city"] != "Paris" || tc.Arguments["units"] != "c" {
t.Errorf("arguments = %v", tc.Arguments)
}
}
func TestOpenAIClient_Complete_MalformedToolArgs(t *testing.T) {
srv := newTestServer(t, func(w http.ResponseWriter, r *http.Request) {
_, _ = w.Write([]byte(`{"choices":[{"message":{"role":"assistant","tool_calls":[` +
`{"type":"function","function":{"name":"f","arguments":"{not valid json"}}` +
`]},"finish_reason":"tool_calls"}],"usage":{}}`))
})
client := NewOpenAIClient(srv.URL, "", time.Second)
resp, err := client.Complete(context.Background(), Request{
Model: "m", Messages: []Message{{Role: "user", Content: "x"}},
Tools: []Tool{{Name: "f"}},
})
if err != nil {
t.Fatalf("Complete() error = %v", err)
}
if len(resp.ToolCalls) != 1 || resp.ToolCalls[0].Decoded {
t.Fatalf("expected 1 undecoded tool call, got %+v", resp.ToolCalls)
}
if resp.ToolCalls[0].RawArguments != "{not valid json" {
t.Errorf("raw args = %q", resp.ToolCalls[0].RawArguments)
}
}