llm-testbench/internal/tests/databases_redis.go

package tests

import (
	"regexp"

	"github.com/lukaszraczylo/llm-testbench/internal/eval"
	"github.com/lukaszraczylo/llm-testbench/internal/testkit"
)

// registerDBRedisTests registers every databases/redis test.
func registerDBRedisTests(r *testkit.Registry) {
	r.Register(dbRedisStructureChoiceTest())
	r.Register(dbRedisNoevictionWriteTest())
	r.Register(dbRedisIncrAtomicityTest())
	r.Register(dbRedisMultiVsPipelineTest())
	r.Register(dbRedisScanVsKeysTest())
	r.Register(dbRedisPubSubDeliveryTest())
	r.Register(dbRedisLuaAtomicityTest())
	r.Register(dbRedisMemoryEstimateTest())
	r.Register(dbRedisCacheStampedeTest())
	r.Register(dbRedisKeyspaceAntiPatternTest())
}

// dbRedisStructureScenarios lists the 4 inline scenarios for
// dbRedisStructureChoiceTest.
const dbRedisStructureScenarios = `a: "A leaderboard of user scores that must support efficient range and
   rank queries (e.g. 'give me the top 10' or 'what rank is this user')."
b: "Counting approximately how many unique visitors hit the site today,
   at a scale of tens of millions of visitors, using minimal memory."
c: "A user's session data as a group of named fields (name, plan,
   last_seen) that must each be readable and updatable individually
   without rewriting the whole session."
d: "A FIFO job queue where worker processes block until a new job
   arrives, then pop and process jobs in the order they were pushed."`

// dbRedisStructureChoiceTest: pick the best-fit Redis data structure for
// each of 4 inline usage scenarios, from a closed vocabulary.
//
// ground truth: (a) a leaderboard needing ranked range queries is exactly
// what a sorted set (ZSET, score-ordered) provides. (b) approximate unique
// counting at large scale with minimal memory is HyperLogLog's purpose
// (a small fixed-size probabilistic structure), not an exact set which
// would use memory proportional to visitor count. (c) named, individually
// updatable fields is a hash, not a single string blob that would need a
// full read-modify-write. (d) a FIFO queue with blocking pop is a list
// (LPUSH/BRPOP), not a set (unordered, no blocking pop) or a stream
// (heavier, consumer-group semantics not needed here).
func dbRedisStructureChoiceTest() testkit.Test {
	prompt := `For each of these 4 scenarios, pick the single best-fit Redis
data structure from exactly this vocabulary: string, list, hash, set,
sorted-set, hyperloglog, stream, bitmap.

` + dbRedisStructureScenarios + `

Respond with only a JSON object mapping each letter to one structure name:
{"a":"...","b":"...","c":"...","d":"..."}`

	evaluator := eval.Mean(
		eval.JSONField("a", "sorted-set"),
		eval.JSONField("b", "hyperloglog"),
		eval.JSONField("c", "hash"),
		eval.JSONField("d", "list"),
	)

	return testkit.Test{
		ID:          "redis-structure-choice",
		Category:    "databases",
		Subcategory: "redis",
		Description: "Pick the best-fit Redis data structure (sorted-set, hyperloglog, hash, list) for 4 inline usage scenarios from a closed vocabulary.",
		Prompt:      prompt,
		Eval:        evaluator,
	}
}

// dbRedisNoevictionWriteTest: determine what happens to a write under
// maxmemory-policy=noeviction when Redis is already at maxmemory.
//
// ground truth: noeviction means Redis will never evict a key to make
// room. A write that would grow memory usage while already at maxmemory
// is instead rejected outright with an OOM error - the write does not
// silently succeed and no key is evicted to make space.
func dbRedisNoevictionWriteTest() testkit.Test {
	prompt := `A Redis instance has maxmemory-policy set to "noeviction" and
is currently at maxmemory. A client sends a write command that would
increase memory usage further.

What happens to that write? Respond with only one word: reject or evict.`

	return testkit.Test{
		ID:          "redis-noeviction-write",
		Category:    "databases",
		Subcategory: "redis",
		Description: "Determine that a write is rejected (not silently allowed, and nothing evicted) under maxmemory-policy=noeviction at maxmemory.",
		Prompt:      prompt,
		Eval:        eval.ExactToken("reject"),
	}
}

// dbRedisIncrAtomicityTest: contrast INCR's atomic guarantee against the
// race in a manual GET-then-SET increment pattern.
//
// ground truth: INCR is a single atomic server-side operation, so 100
// concurrent INCR calls on a counter starting at 0 are guaranteed to leave
// it at exactly 100, with no lost updates. The GET-then-SET pattern is two
// separate round trips per client; two clients can both GET the same
// value before either SETs, so one client's increment can be silently
// overwritten - the final value after 100 concurrent GET/SET increments is
// not guaranteed to reach 100.
func dbRedisIncrAtomicityTest() testkit.Test {
	prompt := `A Redis counter starts at 0. 100 clients each run exactly one
of the two patterns below, concurrently:

Pattern A: each client calls INCR counter once.
Pattern B: each client calls GET counter, adds 1 in its own application
code, then calls SET counter <new value> once.

What is the guaranteed final value of the counter after Pattern A's 100
calls, and is Pattern B's final value guaranteed to also reach that same
total? Respond with only a JSON object:
{"incr_final":<number>,"getset_race_safe":true|false}`

	evaluator := eval.Mean(
		eval.JSONField("incr_final", 100),
		eval.JSONField("getset_race_safe", false),
	)

	return testkit.Test{
		ID:          "redis-incr-atomicity",
		Category:    "databases",
		Subcategory: "redis",
		Description: "Contrast INCR's guaranteed atomic result (100) against the race-prone GET-then-SET increment pattern (not guaranteed to reach 100).",
		Prompt:      prompt,
		Eval:        evaluator,
	}
}

// dbRedisMultiVsPipelineTest: confirm that plain pipelining, unlike
// MULTI/EXEC, gives no atomicity/isolation guarantee against interleaving
// from other clients.
//
// ground truth: pipelining is purely a network optimization - batching
// several commands into one round trip - with no guarantee that the
// server executes them back-to-back relative to other connected clients.
// MULTI/EXEC (a transaction) is what guarantees no other client's command
// can execute in between the queued commands.
func dbRedisMultiVsPipelineTest() testkit.Test {
	prompt := `Does plain command pipelining (batching several commands into
one network round trip, without wrapping them in MULTI/EXEC) guarantee
that no other client's command can execute on the server in between the
pipelined commands? Respond with only "yes" or "no".`

	return testkit.Test{
		ID:          "redis-multi-vs-pipeline",
		Category:    "databases",
		Subcategory: "redis",
		Description: "Confirm plain pipelining (unlike MULTI/EXEC) gives no guarantee against another client's commands interleaving.",
		Prompt:      prompt,
		Eval:        eval.ExactToken("no"),
	}
}

// dbKeysEndorsementPattern matches a KEYS mention in an endorsement-shaped
// position — a verb recommending it ("use `KEYS`", "just run KEYS *"), a
// positive predicate ("KEYS * is fine/simplest"), or a bare imperative line
// ("run this:\nKEYS *") — rather than every mention. The guard then still
// requires a negation cue near such a mention (see dbNoBareKeysInProd).
//
// Triggering on any mention is too strict: a fully correct live answer
// (probe, 2026-08-29) opens with "`KEYS *` is dangerous..." and later says
// "so `KEYS *` can block the server while it walks through millions of
// keys" — harm descriptions whose words ("can block") sit outside the
// negation cue list, yet they are exactly the explanation the prompt asks
// for, not an endorsement.
//
// The predicate branch's leading (?:^|[^...]) guard stops a hyphen-joined
// identifier from matching mid-word and keeps the match start at the term
// area rather than arbitrary prose.
var dbKeysEndorsementPattern = regexp.MustCompile("(?i)(?:" +

	// a verb recommending it: "use `KEYS`", "just run KEYS *", "stick with KEYS *"

	"(?:use|using|used|run|running|runs|execute|executing|call|calling|should(?:\\s+(?:also|just))?|better\\s+to|go\\s+with|opt\\s+for|stick\\s+with|switch(?:ing)?\\s+(?:back\\s+)?to)\\s+(?:the\\s+)?(?:redis\\s+)?(?:command\\s+)?(?:`KEYS`|\\bKEYS\\s*\\*)" +

	// a positive predicate about it: "KEYS * is fine", "`KEYS` is simplest"

	"|(?:^|[^A-Za-z0-9_-])(?:`KEYS`|\\bKEYS\\s*\\*)[^.;\\n]{0,30}?\\b(?:is|are|was|were|seems?|looks?|works?|stays?|remains?|would\\s+be|should\\s+be)\\s+(?:a\\s+|just\\s+|totally\\s+|still\\s+|actually\\s+)?(?:fine|good|great|safe|ok|okay|acceptable|simplest|easiest|better|fast\\s+enough)" +

	// a bare command line after a colon or full stop: "run this:\\nKEYS *" -

	// requiring the break keeps a line-initial mention that merely

	// continues a wrapped sentence from triggering.

	"|(?:[.:]|^)[ \\t]*(?:\\r?\\n)[ \\t*`>-]*(?:`KEYS`|\\bKEYS\\s*\\*)" +

	")")

// dbNegationWindow is how many characters around a KEYS-command mention
// are searched for a negation cue.
const dbNegationWindow = 60

// dbNoBareKeysInProd scores full credit unless the response endorses
// running "KEYS *" against a production instance (see
// dbKeysEndorsementPattern) with no negating context nearby — a warning
// not to, or an instruction to use SCAN instead, is fine. This is
// deliberately not eval.NotContains, which would also zero out the best
// possible answer: one that correctly explains why NOT to run KEYS in
// production.
//
// Delegates to eval.NoUnnegatedMention (D5), the primitive shared with
// kubernetes.go, security.go, and delivery_git.go's equivalent guards.
func dbNoBareKeysInProd() eval.Evaluator {
	return eval.NoUnnegatedMention(dbKeysEndorsementPattern, dbNegationWindow, nil)
}

// dbRedisScanVsKeysTest: require SCAN over KEYS for a hot-path production
// lookup, using a negation-aware guard so an answer that correctly warns
// against KEYS is not penalized for mentioning it.
//
// ground truth (DC9): KEYS * is an O(n) full-keyspace scan that blocks
// Redis's single command-processing thread for its entire duration - on a
// production instance holding millions of keys, called thousands of times
// per second, this stalls every other client for the scan's whole
// duration. SCAN instead walks the keyspace incrementally via a cursor,
// returning a small batch per call and never blocking the server for more
// than that batch, making it the safe hot-path replacement.
func dbRedisScanVsKeysTest() testkit.Test {
	prompt := `A request handler that runs thousands of times per second
against a production Redis instance holding millions of keys currently
calls "KEYS *" to find every key matching a pattern.

What is wrong with this in production, and what command should be used
instead to iterate the keyspace safely?`

	evaluator := eval.All(
		eval.W(dbNoBareKeysInProd(), 2),
		eval.W(eval.ContainsAny("SCAN"), 2),
	)

	return testkit.Test{
		ID:          "redis-scan-vs-keys",
		Category:    "databases",
		Subcategory: "redis",
		Description: "Require the SCAN cursor over KEYS * for a hot-path production lookup, without penalizing a negated warning against KEYS.",
		Prompt:      prompt,
		Eval:        evaluator,
	}
}

// dbRedisPubSubDeliveryTest: confirm Redis pub/sub gives no delivery
// guarantee to a disconnected subscriber.
//
// ground truth: Redis pub/sub is fire-and-forget with no backlog or
// replay - a message published while a subscriber is disconnected is
// simply not delivered to it; reconnecting does not retrieve messages
// missed while offline.
func dbRedisPubSubDeliveryTest() testkit.Test {
	prompt := `A subscriber is disconnected (e.g. a brief network blip) at
the exact moment a message is published on a channel it normally
subscribes to. After the subscriber reconnects, will it receive that
specific message it missed while disconnected? Respond with only "yes" or
"no".`

	return testkit.Test{
		ID:          "redis-pubsub-delivery",
		Category:    "databases",
		Subcategory: "redis",
		Description: "Confirm Redis pub/sub gives no delivery guarantee (no backlog/replay) to a subscriber that was disconnected when a message was published.",
		Prompt:      prompt,
		Eval:        eval.ExactToken("no"),
	}
}

// dbRedisLuaAtomicityTest: confirm a Lua script run via EVAL executes
// atomically with no other client command interleaved.
//
// ground truth: Redis executes commands (and an entire EVAL script) on a
// single command-processing thread; while one client's script is running,
// the server does not process any other client's command, so a script is
// guaranteed to run start-to-finish with nothing interleaved.
func dbRedisLuaAtomicityTest() testkit.Test {
	// DC6: narrowed to a data command (SET) specifically - "any other
	// command whatsoever" is a broader claim than Redis's single-threaded
	// script guarantee actually covers, since some non-data/administrative
	// commands can be processed in certain configurations while a script
	// runs. A data command like SET (one that reads/writes the keyspace)
	// is unambiguously blocked, matching the ground truth exactly.
	prompt := `While one client's Lua script is executing via EVAL, can
another client's SET command (a data command touching the keyspace) run
on the Redis server before that script finishes? Respond with only "yes"
or "no".`

	return testkit.Test{
		ID:          "redis-lua-atomicity",
		Category:    "databases",
		Subcategory: "redis",
		Description: "Confirm a Lua script run via EVAL executes atomically, with no other client command able to interleave.",
		Prompt:      prompt,
		Eval:        eval.ExactToken("no"),
	}
}

// dbRedisMemoryEstimateWant is the total estimated bytes computed by the
// formula given inline in dbRedisMemoryEstimateTest's prompt.
//
// ground truth: total_bytes = count * (key_bytes + value_bytes +
// overhead_bytes) = 200000 * (20 + 100 + 56) = 200000 * 176 = 35,200,000.
// databases_redis_test.go recomputes this independently with the same
// arithmetic written out plainly.
const dbRedisMemoryEstimateWant = 200000 * (20 + 100 + 56)

// dbRedisMemoryEstimateTest: compute total memory bytes for an inline
// dataset from a formula given entirely in the prompt.
func dbRedisMemoryEstimateTest() testkit.Test {
	prompt := `Estimate total memory, in bytes, to store a Redis dataset of
200,000 string key-value pairs, where each key name is 20 bytes, each
value is 100 bytes, and Redis's per-key-value overhead (hash table entry
plus object headers) is estimated at 56 bytes. Use this formula:

total_bytes = count * (key_bytes + value_bytes + overhead_bytes)

Respond with only the number.`

	return testkit.Test{
		ID:          "redis-memory-estimate",
		Category:    "databases",
		Subcategory: "redis",
		Description: "Compute total memory bytes for a 200,000-key dataset from an inline per-key-value formula and byte sizes.",
		Prompt:      prompt,
		Eval:        eval.Numeric(eval.ExtractLastNumber[int], dbRedisMemoryEstimateWant, 0),
	}
}

// dbRedisStampedeCandidates lists the 4 candidate mitigation techniques
// for dbRedisCacheStampedeTest, mixing genuinely valid mitigations with
// one plausible-sounding non-mitigation.
const dbRedisStampedeCandidates = `- mutex_lock_on_regenerate: only one client regenerates a missing/expired
  key at a time; others wait for or read the in-flight result.
- probabilistic_early_expiration: clients probabilistically recompute a
  soon-to-expire key slightly before it actually expires, spreading
  regeneration out instead of all racing at the exact expiry moment.
- increase_replica_count: add more read replicas to the Redis deployment.
- stale_while_revalidate: serve the stale cached value immediately while
  one background request regenerates it.`

// dbRedisStampedeWant is the subset of dbRedisStampedeCandidates that
// actually mitigates cache stampede.
//
// ground truth: a cache stampede is a burst of concurrent regenerations
// racing to rebuild the same expired key against the backing store.
// mutex_lock_on_regenerate, probabilistic_early_expiration, and
// stale_while_revalidate each directly prevent that race (by
// serializing regeneration, spreading it out over time, or deferring it
// entirely). Adding more replicas does not: replicas serve reads of
// already-cached data, but every replica independently misses the same
// expired key at the same moment, so it does not reduce or serialize the
// regeneration race against the backing store at all.
var dbRedisStampedeWant = []string{
	"mutex_lock_on_regenerate",
	"probabilistic_early_expiration",
	"stale_while_revalidate",
}

func dbRedisCacheStampedeTest() testkit.Test {
	prompt := `A hot cache key expires and many concurrent requests all miss
it at once, each racing to regenerate it against the backing store (a
"cache stampede" / "thundering herd"). Here are 4 candidate mitigation
techniques:

` + dbRedisStampedeCandidates + `

Which of these candidates actually mitigate a cache stampede? Respond with
only a JSON array of the candidate names that do.`

	return testkit.Test{
		ID:          "redis-cache-stampede",
		Category:    "databases",
		Subcategory: "redis",
		Description: "Select the cache-stampede mitigations (mutex, probabilistic early expiration, stale-while-revalidate) from 4 candidates that also include a non-mitigation.",
		Prompt:      prompt,
		Eval:        eval.JSONStringSet(dbRedisStampedeWant),
	}
}

// dbRedisKeyspaceAntiPatternCode is the inline monolithic-key snippet for
// dbRedisKeyspaceAntiPatternTest.
const dbRedisKeyspaceAntiPatternCode = `def save_all_users(users):
    r.set("all_users", json.dumps(users))  # users: a list of 500,000 user dicts

def update_user(user_id, fields):
    users = json.loads(r.get("all_users"))
    for u in users:
        if u["id"] == user_id:
            u.update(fields)
    r.set("all_users", json.dumps(users))  # rewrites the entire 500,000-user blob`

// dbRedisKeyspaceAntiPatternTest: spot a monolithic single-key blob
// keyspace anti-pattern in inline code and require the per-entity hash
// fix.
//
// ground truth: every single-field update to one user rewrites a
// serialized blob of all 500,000 users under one key, an O(n) operation
// per update where n is the whole user count - a monolithic-key
// anti-pattern, not a "too many small keys" problem (there is only one
// key here) or a missing-TTL problem (nothing about expiry is shown). The
// fix is to give each user its own Hash key (e.g. user:<id>), so a field
// update touches O(1) data instead of the entire dataset.
func dbRedisKeyspaceAntiPatternTest() testkit.Test {
	prompt := `Here is code managing user data in Redis:

` + "```python\n" + dbRedisKeyspaceAntiPatternCode + "\n```" + `

What keyspace design problem does this have, and what is the correct fix?
Respond with only a JSON object:
{"problem":"<one of: monolithic-key, too-many-small-keys, missing-ttl>","fix":"<one of: hash-per-user, single-key-ok, add-ttl>"}`

	evaluator := eval.Mean(
		eval.JSONField("problem", "monolithic-key"),
		eval.JSONField("fix", "hash-per-user"),
	)

	return testkit.Test{
		ID:          "redis-keyspace-anti-pattern",
		Category:    "databases",
		Subcategory: "redis",
		Description: "Spot a monolithic single-key user blob rewritten on every update and require the per-user-hash fix.",
		Prompt:      prompt,
		Eval:        evaluator,
	}
}