mirror of
https://github.com/lukaszraczylo/llm-testbench.git
synced 2026-10-12 00:45:19 +00:00
lukaszraczylo
cfdb2c5aae
Adds real OpenAI function-calling support end to end and two test tranches. Client (internal/llm): Request.Tools advertises tools with tool_choice auto; Response.ToolCalls carries the model's calls, decoding each function's JSON-encoded argument string into an object (malformed arguments preserved raw with Decoded=false). Verified live: the gateway round-trips tools and the qwen/deepseek tool-call parsers emit the expected format. Runner: when a test declares Tools, it serializes the model's tool calls into a canonical JSON envelope and scores that, not the free text. Eval (toolcall.go): ToolCalled, NoToolCalled, ToolCallWithArgs (subset match, number/string coercion), ToolSequence, FirstToolIs, ToolNotCalled, ToolArgValuesCover - all parse the envelope, no llm dependency. Tests: 12 function-calling tests (tools/function-calling) covering tool selection, argument extraction, enum constraints, chained and parallel calls, no-tool-needed and missing-param abstention, destructive-tool safety, and name-vs-description traps; 8 ultra-hard single-shot tasks in existing subcategories. Ground-truth self-consistency tests assert every documented answer and ideal tool envelope scores 1.0.
58 lines
2.3 KiB
Go
58 lines
2.3 KiB
Go
package tests
|
|
|
|
import (
|
|
"context"
|
|
"testing"
|
|
)
|
|
|
|
// TestAdvancedGroundTruthScoresFull asserts each advanced test's own
|
|
// documented correct answer scores 1.0 through its real evaluator - a guard
|
|
// that the eval and the intended ground truth agree (a JSON typo or a wrong
|
|
// field path would silently make a test unwinnable).
|
|
func TestAdvancedGroundTruthScoresFull(t *testing.T) {
|
|
golden := map[string]string{
|
|
"adv-agent-plan-infeasible": `{"feasible": false, "blocking_step": "fetch-deps"}`,
|
|
"adv-agent-deleg-conflicting-policy": `{"agent": "agent-gpu-eu-fast"}`,
|
|
"adv-agent-tool-adversarial-desc": `{"tool": "trash_bin"}`,
|
|
"adv-rag-forensics-stage": `{"stage": "generation"}`,
|
|
"adv-llm-context-budget": `54`,
|
|
"adv-pg-explain-forensics": `{"cause": "seq-scan", "fix": "add-index"}`,
|
|
"adv-sql-mvcc-anomaly": `{"anomaly": "non-repeatable-read"}`,
|
|
"adv-docker-layer-rebuild": `["C","D","F"]`,
|
|
"adv-rel-freeze-window": `{"deferred_step": "deploy-prod"}`,
|
|
"adv-scen-multilog-rootcause": `{"root_cause": "oom"}`,
|
|
"adv-linux-capacity-fill": `7`,
|
|
"adv-hard-composed-trace": `32`,
|
|
"adv-py-composed-mutable": `{"first_element": 2, "second_element": 10}`,
|
|
"adv-code-conflicting-evidence": `{"port": 9090, "authoritative": "config"}`,
|
|
"adv-sec-exploit-chain": `{"vuln": "open-redirect"}`,
|
|
"adv-sec-rotation-blast-radius": `30`,
|
|
"ultra-c-int8-signed": `-76`,
|
|
"ultra-hard-modexp": `5`,
|
|
"ultra-hard-fp-equality": `{"equal": false}`,
|
|
"ultra-py-bankers-round": `6`,
|
|
"ultra-sec-redos": `{"vulnerable": true}`,
|
|
"ultra-scen-cap-partition": `{"sacrificed": "availability"}`,
|
|
"ultra-ai-pq-bytes": `16`,
|
|
"ultra-sql-deadlock-cycle": `{"fix": "consistent-lock-order"}`,
|
|
}
|
|
|
|
byID := map[string]int{}
|
|
all := All().All()
|
|
for i, tc := range all {
|
|
byID[tc.ID] = i
|
|
}
|
|
|
|
for id, answer := range golden {
|
|
idx, ok := byID[id]
|
|
if !ok {
|
|
t.Errorf("%s: not registered in catalog", id)
|
|
continue
|
|
}
|
|
s := all[idx].Eval.Evaluate(context.Background(), answer)
|
|
if s.Value != 1 {
|
|
t.Errorf("%s: golden answer %q scored %.2f, want 1.00 (%s)", id, answer, s.Value, s.Detail)
|
|
}
|
|
}
|
|
}
|