package tests import ( "github.com/lukaszraczylo/llm-testbench/internal/eval" "github.com/lukaszraczylo/llm-testbench/internal/testkit" ) // registerUltraTests adds the hardest single-shot reasoning tasks - bit-level // semantics, number theory, IEEE-754 edge cases, language rounding rules, // algorithmic worst-case bounds, and distributed-systems trade-offs. Each // answer is a value a machine can verify exactly. They land in existing // subcategories, so the catalog shape is unchanged. func registerUltraTests(r *testkit.Registry) { r.Register(ultraCInt8SignedTest()) r.Register(ultraHardModexpTest()) r.Register(ultraHardFPEqualityTest()) r.Register(ultraPyBankersRoundTest()) r.Register(ultraSecReDoSTest()) r.Register(ultraScenCAPPartitionTest()) r.Register(ultraAIPQBytesTest()) r.Register(ultraSQLDeadlockCycleTest()) } // ultraCInt8SignedTest: interpret a raw byte as a signed 8-bit two's // complement value. 0b10110100 = 180 unsigned; MSB set, so 180 - 256 = -76. func ultraCInt8SignedTest() testkit.Test { return testkit.Test{ ID: "ultra-c-int8-signed", Category: "programming", Subcategory: "c", Description: "Interpret the byte 0b10110100 as a signed 8-bit two's complement integer: -76.", Prompt: "A single byte holds the bit pattern 10110100. Interpreted as a " + "signed 8-bit integer in two's complement, what decimal value is it? " + "Respond with only the integer.", Eval: eval.Numeric(eval.ExtractLastNumber[int], -76, 0), } } // ultraHardModexpTest: modular exponentiation 7^123 mod 13 = 5. func ultraHardModexpTest() testkit.Test { return testkit.Test{ ID: "ultra-hard-modexp", Category: "programming", Subcategory: "hard", Description: "Compute 7^123 mod 13 = 5 (modular exponentiation, e.g. via Fermat's little theorem).", Prompt: "Compute 7 raised to the power 123, modulo 13. Respond with only the integer result.", Eval: eval.Numeric(eval.ExtractLastNumber[int], 5, 0), } } // ultraHardFPEqualityTest: in IEEE-754 double precision, 0.1 + 0.2 != 0.3. func ultraHardFPEqualityTest() testkit.Test { return testkit.Test{ ID: "ultra-hard-fp-equality", Category: "programming", Subcategory: "hard", Description: "Recognize that 0.1 + 0.2 == 0.3 is false in IEEE-754 double precision.", Prompt: "In standard IEEE-754 double-precision floating point, does the " + "expression (0.1 + 0.2) == 0.3 evaluate to true? Respond with only a " + "JSON object: {\"equal\": true|false}", Eval: eval.JSONField("equal", false), } } // ultraPyBankersRoundTest: Python 3 round() uses banker's rounding (round // half to even): round(2.5)=2, round(3.5)=4, sum = 6. func ultraPyBankersRoundTest() testkit.Test { return testkit.Test{ ID: "ultra-py-bankers-round", Category: "programming", Subcategory: "python", Description: "Apply Python 3 banker's rounding: round(2.5)+round(3.5) = 2+4 = 6.", Prompt: "In Python 3, what is the value of round(2.5) + round(3.5)? " + "Account for Python's rounding mode. Respond with only the integer.", Eval: eval.Numeric(eval.ExtractLastNumber[int], 6, 0), } } // ultraSecReDoSTest: the regex (a+)+$ has catastrophic backtracking on an // input of many 'a's followed by a non-matching character. func ultraSecReDoSTest() testkit.Test { return testkit.Test{ ID: "ultra-sec-redos", Category: "security", Subcategory: "appsec", Description: "Identify (a+)+$ as vulnerable to catastrophic-backtracking ReDoS on a long non-matching input.", Prompt: "A backtracking regex engine evaluates the pattern (a+)+$ against " + "an input of 40 'a' characters followed by a single '!'. Is this " + "pattern vulnerable to catastrophic backtracking (ReDoS) on such input? " + "Respond with only a JSON object: {\"vulnerable\": true|false}", Eval: eval.JSONField("vulnerable", true), } } // ultraScenCAPPartitionTest: a system that stays strongly consistent during // a network partition must sacrifice availability (a CP system). func ultraScenCAPPartitionTest() testkit.Test { return testkit.Test{ ID: "ultra-scen-cap-partition", Category: "operations", Subcategory: "scenario", Description: "Under a network partition, a system choosing strong consistency sacrifices availability (CP).", Prompt: "During a network partition, a distributed database is configured " + "to refuse any read or write it cannot confirm against a quorum, so " + "clients on the minority side get errors rather than stale data. In CAP " + "terms, which property is this system sacrificing during the partition? " + "Respond with only a JSON object: " + "{\"sacrificed\": \"consistency\"|\"availability\"|\"partition-tolerance\"}", Eval: eval.JSONField("sacrificed", "availability"), } } // ultraAIPQBytesTest: product quantization memory. With 16 subquantizers, // each code being 8 bits (1 byte), a vector's PQ code is 16 bytes, // independent of the original dimensionality. func ultraAIPQBytesTest() testkit.Test { return testkit.Test{ ID: "ultra-ai-pq-bytes", Category: "ai", Subcategory: "vector-search", Description: "Compute product-quantization code size: 16 subquantizers at 8 bits each = 16 bytes per vector.", Prompt: "A vector index uses product quantization with 16 subquantizers, " + "each encoding its sub-vector into an 8-bit code, on originally " + "768-dimensional float32 vectors. How many bytes does the PQ code for " + "one vector occupy? Respond with only the integer number of bytes.", Eval: eval.Numeric(eval.ExtractLastNumber[int], 16, 0), } } // ultraSQLDeadlockCycleTest: two transactions acquiring locks in opposite // order deadlock; the classic fix is a consistent global lock ordering. func ultraSQLDeadlockCycleTest() testkit.Test { return testkit.Test{ ID: "ultra-sql-deadlock-cycle", Category: "databases", Subcategory: "sql-tuning", Description: "Diagnose an opposite-lock-order deadlock and prescribe a consistent global lock ordering.", Prompt: "Transaction T1 updates row A then row B; concurrently T2 updates " + "row B then row A. Both block and the database aborts one with a " + "deadlock error. What is the standard design fix that prevents this " + "class of deadlock without reducing concurrency to one transaction at a " + "time? Respond with only a JSON object: " + "{\"fix\": \"consistent-lock-order\"|\"longer-timeout\"|\"serializable-isolation\"|\"more-retries\"}", Eval: eval.JSONField("fix", "consistent-lock-order"), } }