feat: function-calling (MCP-style) tests plus ultra-hard tasks

Adds real OpenAI function-calling support end to end and two test tranches. Client (internal/llm): Request.Tools advertises tools with tool_choice auto; Response.ToolCalls carries the model's calls, decoding each function's JSON-encoded argument string into an object (malformed arguments preserved raw with Decoded=false). Verified live: the gateway round-trips tools and the qwen/deepseek tool-call parsers emit the expected format. Runner: when a test declares Tools, it serializes the model's tool calls into a canonical JSON envelope and scores that, not the free text. Eval (toolcall.go): ToolCalled, NoToolCalled, ToolCallWithArgs (subset match, number/string coercion), ToolSequence, FirstToolIs, ToolNotCalled, ToolArgValuesCover - all parse the envelope, no llm dependency. Tests: 12 function-calling tests (tools/function-calling) covering tool selection, argument extraction, enum constraints, chained and parallel calls, no-tool-needed and missing-param abstention, destructive-tool safety, and name-vs-description traps; 8 ultra-hard single-shot tasks in existing subcategories. Ground-truth self-consistency tests assert every documented answer and ideal tool envelope scores 1.0.
2026-10-13 00:48:29 +00:00 · 2026-08-30 20:22:20 +01:00
parent 708622e2a8
commit cfdb2c5aae
13 changed files with 1076 additions and 9 deletions
@@ -13,10 +13,33 @@ type Message struct {
 	Content string
 }

-// Request is one chat-completion request sent to a model.
+// Tool is one function the model may call, in OpenAI function-calling
+// shape. Parameters is a JSON Schema object describing the arguments.
+type Tool struct {
+	Parameters  map[string]any
+	Name        string
+	Description string
+}
+
+// ToolCall is one function invocation the model emitted. Arguments is the
+// decoded argument object (OpenAI sends it as a JSON-encoded string on the
+// wire; the client decodes it before returning). Decoded is false when the
+// model produced malformed argument JSON, in which case RawArguments holds
+// the undecodable string for scoring.
+type ToolCall struct {
+	Arguments    map[string]any
+	Name         string
+	RawArguments string
+	Decoded      bool
+}
+
+// Request is one chat-completion request sent to a model. When Tools is
+// non-empty the client advertises them with tool_choice "auto", so the
+// model decides whether and which to call.
 type Request struct {
 	Model       string
 	Messages    []Message
+	Tools       []Tool
 	MaxTokens   int
 	Temperature float64
 }
@@ -25,6 +48,7 @@ type Request struct {
 type Response struct {
 	Text             string
 	FinishReason     string
+	ToolCalls        []ToolCall
 	PromptTokens     int
 	CompletionTokens int
 	Latency          time.Duration
@@ -40,7 +40,9 @@ func NewOpenAIClient(endpoint, apiKey string, timeout time.Duration) *OpenAIClie

 type chatCompletionRequest struct {
 	Model       string    `json:"model"`
+	ToolChoice  string    `json:"tool_choice,omitempty"`
 	Messages    []chatMsg `json:"messages"`
+	Tools       []apiTool `json:"tools,omitempty"`
 	MaxTokens   int       `json:"max_tokens,omitempty"`
 	Temperature float64   `json:"temperature"`
 }
@@ -50,6 +52,28 @@ type chatMsg struct {
 	Content string `json:"content"`
 }

+// apiTool is the OpenAI tools[] wire shape: {"type":"function","function":{...}}.
+type apiTool struct {
+	Function apiToolFunction `json:"function"`
+	Type     string          `json:"type"`
+}
+
+type apiToolFunction struct {
+	Parameters  map[string]any `json:"parameters,omitempty"`
+	Name        string         `json:"name"`
+	Description string         `json:"description,omitempty"`
+}
+
+// apiToolCall is the OpenAI tool_calls[] wire shape. Function.Arguments is a
+// JSON-encoded STRING, not a nested object, per the OpenAI spec.
+type apiToolCall struct {
+	Function struct {
+		Name      string `json:"name"`
+		Arguments string `json:"arguments"`
+	} `json:"function"`
+	Type string `json:"type"`
+}
+
 // responseMsg is the assistant message inside one response choice. Content
 // is a pointer, not a plain string: some OpenAI-compatible backends return
 // "content": null when generation was cut off before any answer token was
@@ -61,8 +85,9 @@ type chatMsg struct {
 // Deliberately not evaluated: reasoning_content. The catalog's evaluators
 // score the answer a user would actually see, which is content only.
 type responseMsg struct {
-	Content *string `json:"content"`
-	Role    string  `json:"role"`
+	Content   *string       `json:"content"`
+	Role      string        `json:"role"`
+	ToolCalls []apiToolCall `json:"tool_calls"`
 }

 // Text returns m.Content, or "" if the backend sent "content": null.
@@ -78,8 +103,8 @@ type chatCompletionResponse struct {
 		Message string `json:"message"`
 	} `json:"error"`
 	Choices []struct {
-		Message      responseMsg `json:"message"`
 		FinishReason string      `json:"finish_reason"`
+		Message      responseMsg `json:"message"`
 	} `json:"choices"`
 	Usage struct {
 		PromptTokens     int `json:"prompt_tokens"`
@@ -87,6 +112,32 @@ type chatCompletionResponse struct {
 	} `json:"usage"`
 }

+// decodeToolCalls converts the OpenAI wire tool_calls into normalized
+// ToolCall values, decoding each function's JSON-encoded argument string
+// into an object. A call whose argument string is not valid JSON is kept
+// with Decoded=false and its raw string preserved, so a test can still see
+// that the tool was named even when the model emitted malformed arguments.
+func decodeToolCalls(raw []apiToolCall) []ToolCall {
+	if len(raw) == 0 {
+		return nil
+	}
+	out := make([]ToolCall, 0, len(raw))
+	for _, tc := range raw {
+		call := ToolCall{Name: tc.Function.Name, RawArguments: tc.Function.Arguments}
+		var args map[string]any
+		if tc.Function.Arguments == "" {
+			// No-argument tool call: an empty argument object, decoded.
+			call.Arguments = map[string]any{}
+			call.Decoded = true
+		} else if err := json.Unmarshal([]byte(tc.Function.Arguments), &args); err == nil {
+			call.Arguments = args
+			call.Decoded = true
+		}
+		out = append(out, call)
+	}
+	return out
+}
+
 // Complete implements Client. It retries transport failures and 5xx
 // responses up to maxRetries times with linear backoff.
 func (c *OpenAIClient) Complete(ctx context.Context, req Request) (Response, error) {
@@ -98,6 +149,17 @@ func (c *OpenAIClient) Complete(ctx context.Context, req Request) (Response, err
 	for _, m := range req.Messages {
 		body.Messages = append(body.Messages, chatMsg(m))
 	}
+	if len(req.Tools) > 0 {
+		// tool_choice "auto" lets the model decide whether and which tool to
+		// call: the honest test of both "must call X" and "needs no tool".
+		body.ToolChoice = "auto"
+		for _, t := range req.Tools {
+			body.Tools = append(body.Tools, apiTool{
+				Type:     "function",
+				Function: apiToolFunction(t),
+			})
+		}
+	}

 	payload, err := json.Marshal(body)
 	if err != nil {
@@ -130,9 +192,11 @@ func (c *OpenAIClient) Complete(ctx context.Context, req Request) (Response, err

 		latency := time.Since(attemptStart)
 		var text, finishReason string
+		var toolCalls []ToolCall
 		if len(resp.Choices) > 0 {
 			text = resp.Choices[0].Message.Text()
 			finishReason = resp.Choices[0].FinishReason
+			toolCalls = decodeToolCalls(resp.Choices[0].Message.ToolCalls)
 		}
 		if resp.Error != nil {
 			// A 2xx carrying a JSON error body is how some gateways report a
@@ -145,6 +209,7 @@ func (c *OpenAIClient) Complete(ctx context.Context, req Request) (Response, err
 		return Response{
 			Text:             text,
 			FinishReason:     finishReason,
+			ToolCalls:        toolCalls,
 			PromptTokens:     resp.Usage.PromptTokens,
 			CompletionTokens: resp.Usage.CompletionTokens,
 			Latency:          latency,
@@ -26,12 +26,12 @@ func newTestServer(t *testing.T, handler http.HandlerFunc) *httptest.Server {
 // element type, so callers can append() it without repeating the anonymous
 // struct type at each call site.
 func testChoice(content, finishReason string) struct {
-	Message      responseMsg `json:"message"`
 	FinishReason string      `json:"finish_reason"`
+	Message      responseMsg `json:"message"`
 } {
 	return struct {
-		Message      responseMsg `json:"message"`
 		FinishReason string      `json:"finish_reason"`
+		Message      responseMsg `json:"message"`
 	}{
 		Message:      responseMsg{Role: "assistant", Content: &content},
 		FinishReason: finishReason,
@@ -329,3 +329,68 @@ func TestOpenAIClient_Complete_SendsAPIKey(t *testing.T) {
 		t.Fatalf("Complete() error = %v", err)
 	}
 }
+
+func TestOpenAIClient_Complete_ParsesToolCalls(t *testing.T) {
+	srv := newTestServer(t, func(w http.ResponseWriter, r *http.Request) {
+		var body chatCompletionRequest
+		if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
+			t.Fatalf("decode request: %v", err)
+		}
+		// The client must advertise tools with tool_choice auto.
+		if len(body.Tools) != 1 || body.Tools[0].Function.Name != "get_weather" {
+			t.Errorf("tools not sent correctly: %+v", body.Tools)
+		}
+		if body.ToolChoice != "auto" {
+			t.Errorf("tool_choice = %q, want auto", body.ToolChoice)
+		}
+		_, _ = w.Write([]byte(`{"choices":[{"message":{"role":"assistant","content":null,"tool_calls":[` +
+			`{"type":"function","function":{"name":"get_weather","arguments":"{\"city\":\"Paris\",\"units\":\"c\"}"}}` +
+			`]},"finish_reason":"tool_calls"}],"usage":{"prompt_tokens":5,"completion_tokens":9}}`))
+	})
+
+	client := NewOpenAIClient(srv.URL, "", time.Second)
+	resp, err := client.Complete(context.Background(), Request{
+		Model:    "m",
+		Messages: []Message{{Role: "user", Content: "weather in Paris?"}},
+		Tools: []Tool{{
+			Name:        "get_weather",
+			Description: "Get weather for a city",
+			Parameters:  map[string]any{"type": "object"},
+		}},
+	})
+	if err != nil {
+		t.Fatalf("Complete() error = %v", err)
+	}
+	if len(resp.ToolCalls) != 1 {
+		t.Fatalf("got %d tool calls, want 1", len(resp.ToolCalls))
+	}
+	tc := resp.ToolCalls[0]
+	if tc.Name != "get_weather" || !tc.Decoded {
+		t.Errorf("tool call = %+v", tc)
+	}
+	if tc.Arguments["city"] != "Paris" || tc.Arguments["units"] != "c" {
+		t.Errorf("arguments = %v", tc.Arguments)
+	}
+}
+
+func TestOpenAIClient_Complete_MalformedToolArgs(t *testing.T) {
+	srv := newTestServer(t, func(w http.ResponseWriter, r *http.Request) {
+		_, _ = w.Write([]byte(`{"choices":[{"message":{"role":"assistant","tool_calls":[` +
+			`{"type":"function","function":{"name":"f","arguments":"{not valid json"}}` +
+			`]},"finish_reason":"tool_calls"}],"usage":{}}`))
+	})
+	client := NewOpenAIClient(srv.URL, "", time.Second)
+	resp, err := client.Complete(context.Background(), Request{
+		Model: "m", Messages: []Message{{Role: "user", Content: "x"}},
+		Tools: []Tool{{Name: "f"}},
+	})
+	if err != nil {
+		t.Fatalf("Complete() error = %v", err)
+	}
+	if len(resp.ToolCalls) != 1 || resp.ToolCalls[0].Decoded {
+		t.Fatalf("expected 1 undecoded tool call, got %+v", resp.ToolCalls)
+	}
+	if resp.ToolCalls[0].RawArguments != "{not valid json" {
+		t.Errorf("raw args = %q", resp.ToolCalls[0].RawArguments)
+	}
+}