mirror of
https://github.com/lukaszraczylo/llm-testbench.git
synced 2026-10-16 01:07:47 +00:00
test(catalog): add databases category with 30 tests
Add a new databases category with three subcategories: postgres, redis, and sql-tuning. Each subcategory holds 10 self-contained tests with deterministic evaluators. Postgres tests cover EXPLAIN interpretation, index choice, N+1 detection, isolation anomalies, deadlocks, replica failover, bloat diagnosis, pool sizing, partial indexes, and LISTEN/NOTIFY. Redis tests cover structure choice, TTL eviction, INCR atomicity, MULTI versus pipelining, and SCAN versus KEYS. They also cover pub/sub delivery, Lua atomicity, memory estimation, cache stampede mitigation, and keyspace anti-patterns. The SCAN versus KEYS test uses a negation-aware evaluator, so a response that warns against KEYS does not lose credit for mentioning it. SQL-tuning tests trace query results against an inline orders table: aggregates, joins, NULL handling, GROUP BY plus HAVING, and window functions. The unit tests recompute every numeric or JSON answer independently, with a hand-written loop separate from the production query-logic helper. registerDatabasesTests registers the category. A wiring test verifies it directly, and the code does not touch catalog.go, so the orchestrator can wire the category in separately during integration.
This commit is contained in:
@@ -0,0 +1,410 @@
|
||||
package tests
|
||||
|
||||
import (
|
||||
"context"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/lukaszraczylo/llm-testbench/internal/eval"
|
||||
"github.com/lukaszraczylo/llm-testbench/internal/testkit"
|
||||
)
|
||||
|
||||
// registerDBRedisTests registers every databases/redis test.
|
||||
func registerDBRedisTests(r *testkit.Registry) {
|
||||
r.Register(dbRedisStructureChoiceTest())
|
||||
r.Register(dbRedisTTLEvictionTest())
|
||||
r.Register(dbRedisIncrAtomicityTest())
|
||||
r.Register(dbRedisMultiVsPipelineTest())
|
||||
r.Register(dbRedisScanVsKeysTest())
|
||||
r.Register(dbRedisPubSubDeliveryTest())
|
||||
r.Register(dbRedisLuaAtomicityTest())
|
||||
r.Register(dbRedisMemoryEstimateTest())
|
||||
r.Register(dbRedisCacheStampedeTest())
|
||||
r.Register(dbRedisKeyspaceAntiPatternTest())
|
||||
}
|
||||
|
||||
// dbRedisStructureScenarios lists the 4 inline scenarios for
|
||||
// dbRedisStructureChoiceTest.
|
||||
const dbRedisStructureScenarios = `a: "A leaderboard of user scores that must support efficient range and
|
||||
rank queries (e.g. 'give me the top 10' or 'what rank is this user')."
|
||||
b: "Counting approximately how many unique visitors hit the site today,
|
||||
at a scale of tens of millions of visitors, using minimal memory."
|
||||
c: "A user's session data as a group of named fields (name, plan,
|
||||
last_seen) that must each be readable and updatable individually
|
||||
without rewriting the whole session."
|
||||
d: "A FIFO job queue where worker processes block until a new job
|
||||
arrives, then pop and process jobs in the order they were pushed."`
|
||||
|
||||
// dbRedisStructureChoiceTest: pick the best-fit Redis data structure for
|
||||
// each of 4 inline usage scenarios, from a closed vocabulary.
|
||||
//
|
||||
// ground truth: (a) a leaderboard needing ranked range queries is exactly
|
||||
// what a sorted set (ZSET, score-ordered) provides. (b) approximate unique
|
||||
// counting at large scale with minimal memory is HyperLogLog's purpose
|
||||
// (a small fixed-size probabilistic structure), not an exact set which
|
||||
// would use memory proportional to visitor count. (c) named, individually
|
||||
// updatable fields is a hash, not a single string blob that would need a
|
||||
// full read-modify-write. (d) a FIFO queue with blocking pop is a list
|
||||
// (LPUSH/BRPOP), not a set (unordered, no blocking pop) or a stream
|
||||
// (heavier, consumer-group semantics not needed here).
|
||||
func dbRedisStructureChoiceTest() testkit.Test {
|
||||
prompt := `For each of these 4 scenarios, pick the single best-fit Redis
|
||||
data structure from exactly this vocabulary: string, list, hash, set,
|
||||
sorted-set, hyperloglog, stream, bitmap.
|
||||
|
||||
` + dbRedisStructureScenarios + `
|
||||
|
||||
Respond with only a JSON object mapping each letter to one structure name:
|
||||
{"a":"...","b":"...","c":"...","d":"..."}`
|
||||
|
||||
evaluator := eval.Mean(
|
||||
eval.JSONField("a", "sorted-set"),
|
||||
eval.JSONField("b", "hyperloglog"),
|
||||
eval.JSONField("c", "hash"),
|
||||
eval.JSONField("d", "list"),
|
||||
)
|
||||
|
||||
return testkit.Test{
|
||||
ID: "redis-structure-choice",
|
||||
Category: "databases",
|
||||
Subcategory: "redis",
|
||||
Description: "Pick the best-fit Redis data structure (sorted-set, hyperloglog, hash, list) for 4 inline usage scenarios from a closed vocabulary.",
|
||||
Prompt: prompt,
|
||||
Eval: evaluator,
|
||||
}
|
||||
}
|
||||
|
||||
// dbRedisTTLEvictionTest: determine what happens to a write under
|
||||
// maxmemory-policy=noeviction when Redis is already at maxmemory.
|
||||
//
|
||||
// ground truth: noeviction means Redis will never evict a key to make
|
||||
// room. A write that would grow memory usage while already at maxmemory
|
||||
// is instead rejected outright with an OOM error - the write does not
|
||||
// silently succeed and no key is evicted to make space.
|
||||
func dbRedisTTLEvictionTest() testkit.Test {
|
||||
prompt := `A Redis instance has maxmemory-policy set to "noeviction" and
|
||||
is currently at maxmemory. A client sends a write command that would
|
||||
increase memory usage further.
|
||||
|
||||
What happens to that write? Respond with only one word: reject or evict.`
|
||||
|
||||
return testkit.Test{
|
||||
ID: "redis-ttl-eviction",
|
||||
Category: "databases",
|
||||
Subcategory: "redis",
|
||||
Description: "Determine that a write is rejected (not silently allowed, and nothing evicted) under maxmemory-policy=noeviction at maxmemory.",
|
||||
Prompt: prompt,
|
||||
Eval: dbExactAnswer("reject"),
|
||||
}
|
||||
}
|
||||
|
||||
// dbRedisIncrAtomicityTest: contrast INCR's atomic guarantee against the
|
||||
// race in a manual GET-then-SET increment pattern.
|
||||
//
|
||||
// ground truth: INCR is a single atomic server-side operation, so 100
|
||||
// concurrent INCR calls on a counter starting at 0 are guaranteed to leave
|
||||
// it at exactly 100, with no lost updates. The GET-then-SET pattern is two
|
||||
// separate round trips per client; two clients can both GET the same
|
||||
// value before either SETs, so one client's increment can be silently
|
||||
// overwritten - the final value after 100 concurrent GET/SET increments is
|
||||
// not guaranteed to reach 100.
|
||||
func dbRedisIncrAtomicityTest() testkit.Test {
|
||||
prompt := `A Redis counter starts at 0. 100 clients each run exactly one
|
||||
of the two patterns below, concurrently:
|
||||
|
||||
Pattern A: each client calls INCR counter once.
|
||||
Pattern B: each client calls GET counter, adds 1 in its own application
|
||||
code, then calls SET counter <new value> once.
|
||||
|
||||
What is the guaranteed final value of the counter after Pattern A's 100
|
||||
calls, and is Pattern B's final value guaranteed to also reach that same
|
||||
total? Respond with only a JSON object:
|
||||
{"incr_final":<number>,"getset_race_safe":true|false}`
|
||||
|
||||
evaluator := eval.Mean(
|
||||
eval.JSONField("incr_final", 100),
|
||||
eval.JSONField("getset_race_safe", false),
|
||||
)
|
||||
|
||||
return testkit.Test{
|
||||
ID: "redis-incr-atomicity",
|
||||
Category: "databases",
|
||||
Subcategory: "redis",
|
||||
Description: "Contrast INCR's guaranteed atomic result (100) against the race-prone GET-then-SET increment pattern (not guaranteed to reach 100).",
|
||||
Prompt: prompt,
|
||||
Eval: evaluator,
|
||||
}
|
||||
}
|
||||
|
||||
// dbRedisMultiVsPipelineTest: confirm that plain pipelining, unlike
|
||||
// MULTI/EXEC, gives no atomicity/isolation guarantee against interleaving
|
||||
// from other clients.
|
||||
//
|
||||
// ground truth: pipelining is purely a network optimization - batching
|
||||
// several commands into one round trip - with no guarantee that the
|
||||
// server executes them back-to-back relative to other connected clients.
|
||||
// MULTI/EXEC (a transaction) is what guarantees no other client's command
|
||||
// can execute in between the queued commands.
|
||||
func dbRedisMultiVsPipelineTest() testkit.Test {
|
||||
prompt := `Does plain command pipelining (batching several commands into
|
||||
one network round trip, without wrapping them in MULTI/EXEC) guarantee
|
||||
that no other client's command can execute on the server in between the
|
||||
pipelined commands? Respond with only "yes" or "no".`
|
||||
|
||||
return testkit.Test{
|
||||
ID: "redis-multi-vs-pipeline",
|
||||
Category: "databases",
|
||||
Subcategory: "redis",
|
||||
Description: "Confirm plain pipelining (unlike MULTI/EXEC) gives no guarantee against another client's commands interleaving.",
|
||||
Prompt: prompt,
|
||||
Eval: dbExactAnswer("no"),
|
||||
}
|
||||
}
|
||||
|
||||
// dbKeysCommandPattern matches an explicit mention of the Redis KEYS
|
||||
// command actually being invoked - "KEYS *" (its most common prod-danger
|
||||
// form) or a backtick-quoted `KEYS` - not a generic use of the English
|
||||
// word "keys" elsewhere in prose ("the keys to fixing this...").
|
||||
var dbKeysCommandPattern = regexp.MustCompile("(?:`KEYS`|\\bKEYS\\s*\\*)")
|
||||
|
||||
// dbNegationCuePattern matches a word that turns a mention of the KEYS
|
||||
// command into a warning against running it in production, rather than an
|
||||
// instruction to run it.
|
||||
var dbNegationCuePattern = regexp.MustCompile(`(?i)\b(don'?t|do not|never|avoid|instead of|not|shouldn'?t|should not|rather than|without|no need|danger|dangerous)\b`)
|
||||
|
||||
// dbNegationWindow is how many characters before a KEYS-command mention
|
||||
// are searched for a negation cue.
|
||||
const dbNegationWindow = 60
|
||||
|
||||
// dbNegationWindowStart returns the earliest byte offset in response to
|
||||
// search for a negation cue before a KEYS-command mention starting at
|
||||
// start: the current line, extended back at most dbNegationWindow
|
||||
// characters.
|
||||
func dbNegationWindowStart(response string, start int) int {
|
||||
lineStart := strings.LastIndexByte(response[:start], '\n') + 1
|
||||
return max(start-dbNegationWindow, lineStart)
|
||||
}
|
||||
|
||||
// dbNoBareKeysInProd scores full credit unless the response instructs
|
||||
// running "KEYS *" against a production instance without any negating
|
||||
// context (a warning not to, or an instruction to use SCAN instead is
|
||||
// fine). This is deliberately not eval.NotContains, which would also zero
|
||||
// out the best possible answer: one that correctly explains why NOT to run
|
||||
// KEYS in production.
|
||||
func dbNoBareKeysInProd() eval.Evaluator {
|
||||
return eval.EvaluatorFunc(func(_ context.Context, response string) eval.Score {
|
||||
matches := dbKeysCommandPattern.FindAllStringIndex(response, -1)
|
||||
if len(matches) == 0 {
|
||||
return eval.Score{Value: 1, Detail: "no mention of running the KEYS command"}
|
||||
}
|
||||
for _, loc := range matches {
|
||||
start := loc[0]
|
||||
windowStart := dbNegationWindowStart(response, start)
|
||||
if !dbNegationCuePattern.MatchString(response[windowStart:start]) {
|
||||
return eval.Score{Value: 0, Detail: "unnegated mention of running the KEYS command"}
|
||||
}
|
||||
}
|
||||
return eval.Score{Value: 1, Detail: "every mention of KEYS is negated"}
|
||||
})
|
||||
}
|
||||
|
||||
// dbRedisScanVsKeysTest: require SCAN over KEYS for a hot-path production
|
||||
// lookup, using a negation-aware guard so an answer that correctly warns
|
||||
// against KEYS is not penalized for mentioning it.
|
||||
func dbRedisScanVsKeysTest() testkit.Test {
|
||||
prompt := `A request handler that runs thousands of times per second
|
||||
against a production Redis instance holding millions of keys currently
|
||||
calls "KEYS *" to find every key matching a pattern.
|
||||
|
||||
What is wrong with this in production, and what command should be used
|
||||
instead to iterate the keyspace safely?`
|
||||
|
||||
evaluator := eval.All(
|
||||
eval.W(dbNoBareKeysInProd(), 2),
|
||||
eval.W(eval.ContainsAny("SCAN"), 2),
|
||||
)
|
||||
|
||||
return testkit.Test{
|
||||
ID: "redis-scan-vs-keys",
|
||||
Category: "databases",
|
||||
Subcategory: "redis",
|
||||
Description: "Require the SCAN cursor over KEYS * for a hot-path production lookup, without penalizing a negated warning against KEYS.",
|
||||
Prompt: prompt,
|
||||
Eval: evaluator,
|
||||
}
|
||||
}
|
||||
|
||||
// dbRedisPubSubDeliveryTest: confirm Redis pub/sub gives no delivery
|
||||
// guarantee to a disconnected subscriber.
|
||||
//
|
||||
// ground truth: Redis pub/sub is fire-and-forget with no backlog or
|
||||
// replay - a message published while a subscriber is disconnected is
|
||||
// simply not delivered to it; reconnecting does not retrieve messages
|
||||
// missed while offline.
|
||||
func dbRedisPubSubDeliveryTest() testkit.Test {
|
||||
prompt := `A subscriber is disconnected (e.g. a brief network blip) at
|
||||
the exact moment a message is published on a channel it normally
|
||||
subscribes to. After the subscriber reconnects, will it receive that
|
||||
specific message it missed while disconnected? Respond with only "yes" or
|
||||
"no".`
|
||||
|
||||
return testkit.Test{
|
||||
ID: "redis-pubsub-delivery",
|
||||
Category: "databases",
|
||||
Subcategory: "redis",
|
||||
Description: "Confirm Redis pub/sub gives no delivery guarantee (no backlog/replay) to a subscriber that was disconnected when a message was published.",
|
||||
Prompt: prompt,
|
||||
Eval: dbExactAnswer("no"),
|
||||
}
|
||||
}
|
||||
|
||||
// dbRedisLuaAtomicityTest: confirm a Lua script run via EVAL executes
|
||||
// atomically with no other client command interleaved.
|
||||
//
|
||||
// ground truth: Redis executes commands (and an entire EVAL script) on a
|
||||
// single command-processing thread; while one client's script is running,
|
||||
// the server does not process any other client's command, so a script is
|
||||
// guaranteed to run start-to-finish with nothing interleaved.
|
||||
func dbRedisLuaAtomicityTest() testkit.Test {
|
||||
prompt := `While one client's Lua script is executing via EVAL, can any
|
||||
other client's command run on the Redis server before that script
|
||||
finishes? Respond with only "yes" or "no".`
|
||||
|
||||
return testkit.Test{
|
||||
ID: "redis-lua-atomicity",
|
||||
Category: "databases",
|
||||
Subcategory: "redis",
|
||||
Description: "Confirm a Lua script run via EVAL executes atomically, with no other client command able to interleave.",
|
||||
Prompt: prompt,
|
||||
Eval: dbExactAnswer("no"),
|
||||
}
|
||||
}
|
||||
|
||||
// dbRedisMemoryEstimateWant is the total estimated bytes computed by the
|
||||
// formula given inline in dbRedisMemoryEstimateTest's prompt.
|
||||
//
|
||||
// ground truth: total_bytes = count * (key_bytes + value_bytes +
|
||||
// overhead_bytes) = 200000 * (20 + 100 + 56) = 200000 * 176 = 35,200,000.
|
||||
// databases_redis_test.go recomputes this independently with the same
|
||||
// arithmetic written out plainly.
|
||||
const dbRedisMemoryEstimateWant = 200000 * (20 + 100 + 56)
|
||||
|
||||
// dbRedisMemoryEstimateTest: compute total memory bytes for an inline
|
||||
// dataset from a formula given entirely in the prompt.
|
||||
func dbRedisMemoryEstimateTest() testkit.Test {
|
||||
prompt := `Estimate total memory, in bytes, to store a Redis dataset of
|
||||
200,000 string key-value pairs, where each key name is 20 bytes, each
|
||||
value is 100 bytes, and Redis's per-key-value overhead (hash table entry
|
||||
plus object headers) is estimated at 56 bytes. Use this formula:
|
||||
|
||||
total_bytes = count * (key_bytes + value_bytes + overhead_bytes)
|
||||
|
||||
Respond with only the number.`
|
||||
|
||||
return testkit.Test{
|
||||
ID: "redis-memory-estimate",
|
||||
Category: "databases",
|
||||
Subcategory: "redis",
|
||||
Description: "Compute total memory bytes for a 200,000-key dataset from an inline per-key-value formula and byte sizes.",
|
||||
Prompt: prompt,
|
||||
Eval: eval.Numeric(eval.ExtractLastNumber[int], dbRedisMemoryEstimateWant, 0),
|
||||
}
|
||||
}
|
||||
|
||||
// dbRedisStampedeCandidates lists the 4 candidate mitigation techniques
|
||||
// for dbRedisCacheStampedeTest, mixing genuinely valid mitigations with
|
||||
// one plausible-sounding non-mitigation.
|
||||
const dbRedisStampedeCandidates = `- mutex_lock_on_regenerate: only one client regenerates a missing/expired
|
||||
key at a time; others wait for or read the in-flight result.
|
||||
- probabilistic_early_expiration: clients probabilistically recompute a
|
||||
soon-to-expire key slightly before it actually expires, spreading
|
||||
regeneration out instead of all racing at the exact expiry moment.
|
||||
- increase_replica_count: add more read replicas to the Redis deployment.
|
||||
- stale_while_revalidate: serve the stale cached value immediately while
|
||||
one background request regenerates it.`
|
||||
|
||||
// dbRedisStampedeWant is the subset of dbRedisStampedeCandidates that
|
||||
// actually mitigates cache stampede.
|
||||
//
|
||||
// ground truth: a cache stampede is a burst of concurrent regenerations
|
||||
// racing to rebuild the same expired key against the backing store.
|
||||
// mutex_lock_on_regenerate, probabilistic_early_expiration, and
|
||||
// stale_while_revalidate each directly prevent that race (by
|
||||
// serializing regeneration, spreading it out over time, or deferring it
|
||||
// entirely). Adding more replicas does not: replicas serve reads of
|
||||
// already-cached data, but every replica independently misses the same
|
||||
// expired key at the same moment, so it does not reduce or serialize the
|
||||
// regeneration race against the backing store at all.
|
||||
var dbRedisStampedeWant = []string{
|
||||
"mutex_lock_on_regenerate",
|
||||
"probabilistic_early_expiration",
|
||||
"stale_while_revalidate",
|
||||
}
|
||||
|
||||
func dbRedisCacheStampedeTest() testkit.Test {
|
||||
prompt := `A hot cache key expires and many concurrent requests all miss
|
||||
it at once, each racing to regenerate it against the backing store (a
|
||||
"cache stampede" / "thundering herd"). Here are 4 candidate mitigation
|
||||
techniques:
|
||||
|
||||
` + dbRedisStampedeCandidates + `
|
||||
|
||||
Which of these candidates actually mitigate a cache stampede? Respond with
|
||||
only a JSON array of the candidate names that do.`
|
||||
|
||||
return testkit.Test{
|
||||
ID: "redis-cache-stampede",
|
||||
Category: "databases",
|
||||
Subcategory: "redis",
|
||||
Description: "Select the cache-stampede mitigations (mutex, probabilistic early expiration, stale-while-revalidate) from 4 candidates that also include a non-mitigation.",
|
||||
Prompt: prompt,
|
||||
Eval: eval.JSONStringSet(dbRedisStampedeWant),
|
||||
}
|
||||
}
|
||||
|
||||
// dbRedisKeyspaceAntiPatternCode is the inline monolithic-key snippet for
|
||||
// dbRedisKeyspaceAntiPatternTest.
|
||||
const dbRedisKeyspaceAntiPatternCode = `def save_all_users(users):
|
||||
r.set("all_users", json.dumps(users)) # users: a list of 500,000 user dicts
|
||||
|
||||
def update_user(user_id, fields):
|
||||
users = json.loads(r.get("all_users"))
|
||||
for u in users:
|
||||
if u["id"] == user_id:
|
||||
u.update(fields)
|
||||
r.set("all_users", json.dumps(users)) # rewrites the entire 500,000-user blob`
|
||||
|
||||
// dbRedisKeyspaceAntiPatternTest: spot a monolithic single-key blob
|
||||
// keyspace anti-pattern in inline code and require the per-entity hash
|
||||
// fix.
|
||||
//
|
||||
// ground truth: every single-field update to one user rewrites a
|
||||
// serialized blob of all 500,000 users under one key, an O(n) operation
|
||||
// per update where n is the whole user count - a monolithic-key
|
||||
// anti-pattern, not a "too many small keys" problem (there is only one
|
||||
// key here) or a missing-TTL problem (nothing about expiry is shown). The
|
||||
// fix is to give each user its own Hash key (e.g. user:<id>), so a field
|
||||
// update touches O(1) data instead of the entire dataset.
|
||||
func dbRedisKeyspaceAntiPatternTest() testkit.Test {
|
||||
prompt := `Here is code managing user data in Redis:
|
||||
|
||||
` + "```python\n" + dbRedisKeyspaceAntiPatternCode + "\n```" + `
|
||||
|
||||
What keyspace design problem does this have, and what is the correct fix?
|
||||
Respond with only a JSON object:
|
||||
{"problem":"<one of: monolithic-key, too-many-small-keys, missing-ttl>","fix":"<one of: hash-per-user, single-key-ok, add-ttl>"}`
|
||||
|
||||
evaluator := eval.Mean(
|
||||
eval.JSONField("problem", "monolithic-key"),
|
||||
eval.JSONField("fix", "hash-per-user"),
|
||||
)
|
||||
|
||||
return testkit.Test{
|
||||
ID: "redis-keyspace-anti-pattern",
|
||||
Category: "databases",
|
||||
Subcategory: "redis",
|
||||
Description: "Spot a monolithic single-key user blob rewritten on every update and require the per-user-hash fix.",
|
||||
Prompt: prompt,
|
||||
Eval: evaluator,
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user