feat: function-calling (MCP-style) tests plus ultra-hard tasks

Adds real OpenAI function-calling support end to end and two test tranches.

Client (internal/llm): Request.Tools advertises tools with tool_choice
auto; Response.ToolCalls carries the model's calls, decoding each
function's JSON-encoded argument string into an object (malformed
arguments preserved raw with Decoded=false). Verified live: the gateway
round-trips tools and the qwen/deepseek tool-call parsers emit the
expected format.

Runner: when a test declares Tools, it serializes the model's tool calls
into a canonical JSON envelope and scores that, not the free text.

Eval (toolcall.go): ToolCalled, NoToolCalled, ToolCallWithArgs (subset
match, number/string coercion), ToolSequence, FirstToolIs, ToolNotCalled,
ToolArgValuesCover - all parse the envelope, no llm dependency.

Tests: 12 function-calling tests (tools/function-calling) covering tool
selection, argument extraction, enum constraints, chained and parallel
calls, no-tool-needed and missing-param abstention, destructive-tool
safety, and name-vs-description traps; 8 ultra-hard single-shot tasks in
existing subcategories. Ground-truth self-consistency tests assert every
documented answer and ideal tool envelope scores 1.0.
This commit is contained in:
2026-08-30 20:22:20 +01:00
parent 708622e2a8
commit cfdb2c5aae
13 changed files with 1076 additions and 9 deletions
+43 -3
View File
@@ -4,6 +4,7 @@ package runner
import (
"context"
"encoding/json"
"sync/atomic"
"time"
@@ -14,6 +15,35 @@ import (
"github.com/lukaszraczylo/llm-testbench/internal/testkit"
)
// serializeToolEnvelope renders a response's tool calls into the canonical
// JSON envelope the eval.ToolCalled family parses: {"tool_calls":[{"name":
// ...,"arguments":{...}}],"content":...}. A tool call whose arguments the
// model emitted as malformed JSON serializes with a null arguments object
// and arguments_malformed=true so a test can still see the call happened.
func serializeToolEnvelope(resp llm.Response) string {
type envItem struct {
Arguments map[string]any `json:"arguments"`
Name string `json:"name"`
Malformed bool `json:"arguments_malformed,omitempty"`
}
env := struct {
Content string `json:"content"`
ToolCalls []envItem `json:"tool_calls"`
}{Content: resp.Text, ToolCalls: []envItem{}}
for _, c := range resp.ToolCalls {
env.ToolCalls = append(env.ToolCalls, envItem{
Name: c.Name,
Arguments: c.Arguments,
Malformed: !c.Decoded,
})
}
b, err := json.Marshal(env)
if err != nil {
return `{"tool_calls":[],"content":""}`
}
return string(b)
}
// truncatedDetailPrefix is prepended to Score.Detail when the model's
// response was cut off by the token budget (llm.FinishReasonLength), so
// the reason a test scored the way it did is visible without cross-
@@ -157,6 +187,7 @@ func (r *Runner) runOne(ctx context.Context, model string, test testkit.Test) Re
req := llm.Request{
Model: model,
Messages: messages,
Tools: test.Tools,
MaxTokens: maxTokens,
Temperature: r.cfg.Temperature,
}
@@ -171,8 +202,17 @@ func (r *Runner) runOne(ctx context.Context, model string, test testkit.Test) Re
return Result{Model: model, TestID: test.ID, Err: err, Latency: time.Since(callStart)}
}
normalized := testkit.Normalize(resp.Text)
score := test.Eval.Evaluate(ctx, normalized)
// A function-calling test is scored on the tool-call envelope, not the
// free-text answer: serialize the model's tool calls into the canonical
// JSON the eval.ToolCalled family parses. The envelope is also what gets
// stored as ResponseText, so the artifact shows exactly what was scored.
var scored string
if len(test.Tools) > 0 {
scored = serializeToolEnvelope(resp)
} else {
scored = testkit.Normalize(resp.Text)
}
score := test.Eval.Evaluate(ctx, scored)
if resp.FinishReason == llm.FinishReasonLength {
score.Detail = truncatedDetailPrefix + score.Detail
}
@@ -181,7 +221,7 @@ func (r *Runner) runOne(ctx context.Context, model string, test testkit.Test) Re
Model: model,
TestID: test.ID,
Score: score,
ResponseText: normalized,
ResponseText: scored,
FinishReason: resp.FinishReason,
Latency: resp.Latency,
PromptTokens: resp.PromptTokens,