mirror of
https://github.com/lukaszraczylo/llm-testbench.git
synced 2026-10-14 00:57:45 +00:00
feat: function-calling (MCP-style) tests plus ultra-hard tasks
Adds real OpenAI function-calling support end to end and two test tranches. Client (internal/llm): Request.Tools advertises tools with tool_choice auto; Response.ToolCalls carries the model's calls, decoding each function's JSON-encoded argument string into an object (malformed arguments preserved raw with Decoded=false). Verified live: the gateway round-trips tools and the qwen/deepseek tool-call parsers emit the expected format. Runner: when a test declares Tools, it serializes the model's tool calls into a canonical JSON envelope and scores that, not the free text. Eval (toolcall.go): ToolCalled, NoToolCalled, ToolCallWithArgs (subset match, number/string coercion), ToolSequence, FirstToolIs, ToolNotCalled, ToolArgValuesCover - all parse the envelope, no llm dependency. Tests: 12 function-calling tests (tools/function-calling) covering tool selection, argument extraction, enum constraints, chained and parallel calls, no-tool-needed and missing-param abstention, destructive-tool safety, and name-vs-description traps; 8 ultra-hard single-shot tasks in existing subcategories. Ground-truth self-consistency tests assert every documented answer and ideal tool envelope scores 1.0.
This commit is contained in:
@@ -0,0 +1,146 @@
|
||||
package tests
|
||||
|
||||
import (
|
||||
"github.com/lukaszraczylo/llm-testbench/internal/eval"
|
||||
"github.com/lukaszraczylo/llm-testbench/internal/testkit"
|
||||
)
|
||||
|
||||
// registerUltraTests adds the hardest single-shot reasoning tasks - bit-level
|
||||
// semantics, number theory, IEEE-754 edge cases, language rounding rules,
|
||||
// algorithmic worst-case bounds, and distributed-systems trade-offs. Each
|
||||
// answer is a value a machine can verify exactly. They land in existing
|
||||
// subcategories, so the catalog shape is unchanged.
|
||||
func registerUltraTests(r *testkit.Registry) {
|
||||
r.Register(ultraCInt8SignedTest())
|
||||
r.Register(ultraHardModexpTest())
|
||||
r.Register(ultraHardFPEqualityTest())
|
||||
r.Register(ultraPyBankersRoundTest())
|
||||
r.Register(ultraSecReDoSTest())
|
||||
r.Register(ultraScenCAPPartitionTest())
|
||||
r.Register(ultraAIPQBytesTest())
|
||||
r.Register(ultraSQLDeadlockCycleTest())
|
||||
}
|
||||
|
||||
// ultraCInt8SignedTest: interpret a raw byte as a signed 8-bit two's
|
||||
// complement value. 0b10110100 = 180 unsigned; MSB set, so 180 - 256 = -76.
|
||||
func ultraCInt8SignedTest() testkit.Test {
|
||||
return testkit.Test{
|
||||
ID: "ultra-c-int8-signed",
|
||||
Category: "programming",
|
||||
Subcategory: "c",
|
||||
Description: "Interpret the byte 0b10110100 as a signed 8-bit two's complement integer: -76.",
|
||||
Prompt: "A single byte holds the bit pattern 10110100. Interpreted as a " +
|
||||
"signed 8-bit integer in two's complement, what decimal value is it? " +
|
||||
"Respond with only the integer.",
|
||||
Eval: eval.Numeric(eval.ExtractLastNumber[int], -76, 0),
|
||||
}
|
||||
}
|
||||
|
||||
// ultraHardModexpTest: modular exponentiation 7^123 mod 13 = 5.
|
||||
func ultraHardModexpTest() testkit.Test {
|
||||
return testkit.Test{
|
||||
ID: "ultra-hard-modexp",
|
||||
Category: "programming",
|
||||
Subcategory: "hard",
|
||||
Description: "Compute 7^123 mod 13 = 5 (modular exponentiation, e.g. via Fermat's little theorem).",
|
||||
Prompt: "Compute 7 raised to the power 123, modulo 13. Respond with only the integer result.",
|
||||
Eval: eval.Numeric(eval.ExtractLastNumber[int], 5, 0),
|
||||
}
|
||||
}
|
||||
|
||||
// ultraHardFPEqualityTest: in IEEE-754 double precision, 0.1 + 0.2 != 0.3.
|
||||
func ultraHardFPEqualityTest() testkit.Test {
|
||||
return testkit.Test{
|
||||
ID: "ultra-hard-fp-equality",
|
||||
Category: "programming",
|
||||
Subcategory: "hard",
|
||||
Description: "Recognize that 0.1 + 0.2 == 0.3 is false in IEEE-754 double precision.",
|
||||
Prompt: "In standard IEEE-754 double-precision floating point, does the " +
|
||||
"expression (0.1 + 0.2) == 0.3 evaluate to true? Respond with only a " +
|
||||
"JSON object: {\"equal\": true|false}",
|
||||
Eval: eval.JSONField("equal", false),
|
||||
}
|
||||
}
|
||||
|
||||
// ultraPyBankersRoundTest: Python 3 round() uses banker's rounding (round
|
||||
// half to even): round(2.5)=2, round(3.5)=4, sum = 6.
|
||||
func ultraPyBankersRoundTest() testkit.Test {
|
||||
return testkit.Test{
|
||||
ID: "ultra-py-bankers-round",
|
||||
Category: "programming",
|
||||
Subcategory: "python",
|
||||
Description: "Apply Python 3 banker's rounding: round(2.5)+round(3.5) = 2+4 = 6.",
|
||||
Prompt: "In Python 3, what is the value of round(2.5) + round(3.5)? " +
|
||||
"Account for Python's rounding mode. Respond with only the integer.",
|
||||
Eval: eval.Numeric(eval.ExtractLastNumber[int], 6, 0),
|
||||
}
|
||||
}
|
||||
|
||||
// ultraSecReDoSTest: the regex (a+)+$ has catastrophic backtracking on an
|
||||
// input of many 'a's followed by a non-matching character.
|
||||
func ultraSecReDoSTest() testkit.Test {
|
||||
return testkit.Test{
|
||||
ID: "ultra-sec-redos",
|
||||
Category: "security",
|
||||
Subcategory: "appsec",
|
||||
Description: "Identify (a+)+$ as vulnerable to catastrophic-backtracking ReDoS on a long non-matching input.",
|
||||
Prompt: "A backtracking regex engine evaluates the pattern (a+)+$ against " +
|
||||
"an input of 40 'a' characters followed by a single '!'. Is this " +
|
||||
"pattern vulnerable to catastrophic backtracking (ReDoS) on such input? " +
|
||||
"Respond with only a JSON object: {\"vulnerable\": true|false}",
|
||||
Eval: eval.JSONField("vulnerable", true),
|
||||
}
|
||||
}
|
||||
|
||||
// ultraScenCAPPartitionTest: a system that stays strongly consistent during
|
||||
// a network partition must sacrifice availability (a CP system).
|
||||
func ultraScenCAPPartitionTest() testkit.Test {
|
||||
return testkit.Test{
|
||||
ID: "ultra-scen-cap-partition",
|
||||
Category: "operations",
|
||||
Subcategory: "scenario",
|
||||
Description: "Under a network partition, a system choosing strong consistency sacrifices availability (CP).",
|
||||
Prompt: "During a network partition, a distributed database is configured " +
|
||||
"to refuse any read or write it cannot confirm against a quorum, so " +
|
||||
"clients on the minority side get errors rather than stale data. In CAP " +
|
||||
"terms, which property is this system sacrificing during the partition? " +
|
||||
"Respond with only a JSON object: " +
|
||||
"{\"sacrificed\": \"consistency\"|\"availability\"|\"partition-tolerance\"}",
|
||||
Eval: eval.JSONField("sacrificed", "availability"),
|
||||
}
|
||||
}
|
||||
|
||||
// ultraAIPQBytesTest: product quantization memory. With 16 subquantizers,
|
||||
// each code being 8 bits (1 byte), a vector's PQ code is 16 bytes,
|
||||
// independent of the original dimensionality.
|
||||
func ultraAIPQBytesTest() testkit.Test {
|
||||
return testkit.Test{
|
||||
ID: "ultra-ai-pq-bytes",
|
||||
Category: "ai",
|
||||
Subcategory: "vector-search",
|
||||
Description: "Compute product-quantization code size: 16 subquantizers at 8 bits each = 16 bytes per vector.",
|
||||
Prompt: "A vector index uses product quantization with 16 subquantizers, " +
|
||||
"each encoding its sub-vector into an 8-bit code, on originally " +
|
||||
"768-dimensional float32 vectors. How many bytes does the PQ code for " +
|
||||
"one vector occupy? Respond with only the integer number of bytes.",
|
||||
Eval: eval.Numeric(eval.ExtractLastNumber[int], 16, 0),
|
||||
}
|
||||
}
|
||||
|
||||
// ultraSQLDeadlockCycleTest: two transactions acquiring locks in opposite
|
||||
// order deadlock; the classic fix is a consistent global lock ordering.
|
||||
func ultraSQLDeadlockCycleTest() testkit.Test {
|
||||
return testkit.Test{
|
||||
ID: "ultra-sql-deadlock-cycle",
|
||||
Category: "databases",
|
||||
Subcategory: "sql-tuning",
|
||||
Description: "Diagnose an opposite-lock-order deadlock and prescribe a consistent global lock ordering.",
|
||||
Prompt: "Transaction T1 updates row A then row B; concurrently T2 updates " +
|
||||
"row B then row A. Both block and the database aborts one with a " +
|
||||
"deadlock error. What is the standard design fix that prevents this " +
|
||||
"class of deadlock without reducing concurrency to one transaction at a " +
|
||||
"time? Respond with only a JSON object: " +
|
||||
"{\"fix\": \"consistent-lock-order\"|\"longer-timeout\"|\"serializable-isolation\"|\"more-retries\"}",
|
||||
Eval: eval.JSONField("fix", "consistent-lock-order"),
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user