From 0665d706f0d133af7fee86b563952245f01b821f Mon Sep 17 00:00:00 2001 From: Lukasz Raczylo Date: Mon, 17 Aug 2026 02:20:34 +0100 Subject: [PATCH] test(catalog): add databases category with 30 tests Add a new databases category with three subcategories: postgres, redis, and sql-tuning. Each subcategory holds 10 self-contained tests with deterministic evaluators. Postgres tests cover EXPLAIN interpretation, index choice, N+1 detection, isolation anomalies, deadlocks, replica failover, bloat diagnosis, pool sizing, partial indexes, and LISTEN/NOTIFY. Redis tests cover structure choice, TTL eviction, INCR atomicity, MULTI versus pipelining, and SCAN versus KEYS. They also cover pub/sub delivery, Lua atomicity, memory estimation, cache stampede mitigation, and keyspace anti-patterns. The SCAN versus KEYS test uses a negation-aware evaluator, so a response that warns against KEYS does not lose credit for mentioning it. SQL-tuning tests trace query results against an inline orders table: aggregates, joins, NULL handling, GROUP BY plus HAVING, and window functions. The unit tests recompute every numeric or JSON answer independently, with a hand-written loop separate from the production query-logic helper. registerDatabasesTests registers the category. A wiring test verifies it directly, and the code does not touch catalog.go, so the orchestrator can wire the category in separately during integration. --- internal/tests/databases.go | 69 +++ internal/tests/databases_postgres.go | 412 +++++++++++++ internal/tests/databases_postgres_test.go | 276 +++++++++ internal/tests/databases_redis.go | 410 +++++++++++++ internal/tests/databases_redis_test.go | 342 +++++++++++ internal/tests/databases_sqltuning.go | 634 +++++++++++++++++++++ internal/tests/databases_sqltuning_test.go | 393 +++++++++++++ internal/tests/databases_test.go | 58 ++ 8 files changed, 2594 insertions(+) create mode 100644 internal/tests/databases.go create mode 100644 internal/tests/databases_postgres.go create mode 100644 internal/tests/databases_postgres_test.go create mode 100644 internal/tests/databases_redis.go create mode 100644 internal/tests/databases_redis_test.go create mode 100644 internal/tests/databases_sqltuning.go create mode 100644 internal/tests/databases_sqltuning_test.go create mode 100644 internal/tests/databases_test.go diff --git a/internal/tests/databases.go b/internal/tests/databases.go new file mode 100644 index 0000000..9956e44 --- /dev/null +++ b/internal/tests/databases.go @@ -0,0 +1,69 @@ +package tests + +import ( + "context" + "encoding/json" + "fmt" + "strings" + + "github.com/lukaszraczylo/llm-testbench/internal/eval" + "github.com/lukaszraczylo/llm-testbench/internal/testkit" +) + +// registerDatabasesTests registers every databases-category test. Each +// subcategory - postgres, redis, sql-tuning - has its own register function +// and source file (databases_postgres.go, databases_redis.go, +// databases_sqltuning.go) to keep any one file from growing past a few +// hundred lines. The orchestrator wires this into the catalog's All() +// separately; this package intentionally does not register itself in +// catalog.go so parallel category worktrees stay independently buildable. +func registerDatabasesTests(r *testkit.Registry) { + registerDBPostgresTests(r) + registerDBRedisTests(r) + registerDBSQLTuningTests(r) +} + +// dbExactAnswer returns an Evaluator awarding full credit when the +// response, trimmed of whitespace, at most one layer of surrounding quote +// characters (' , ", or `), and a single trailing sentence-ending period, +// equals want case-insensitively. Used across the databases category for +// prompts that force a single short forced-vocabulary answer (e.g. "yes" +// or "no", "keyset" or "offset"): it accepts every materially-correct form +// of that answer (bare, quoted, differently-cased, or with trailing +// punctuation) without loosening the match to accept a wrong answer. +func dbExactAnswer(want string) eval.Evaluator { + return eval.EvaluatorFunc(func(_ context.Context, response string) eval.Score { + got := strings.TrimSpace(response) + got = strings.Trim(got, "\"'`") + got = strings.TrimSuffix(strings.TrimSpace(got), ".") + got = strings.TrimSpace(got) + wantTrimmed := strings.TrimSpace(want) + if strings.EqualFold(got, wantTrimmed) { + return eval.Score{Value: 1, Detail: fmt.Sprintf("equals %q", wantTrimmed)} + } + return eval.Score{Value: 0, Detail: fmt.Sprintf("got %q, want %q", got, wantTrimmed)} + }) +} + +// dbJSONArrayLength returns an Evaluator awarding full credit when the +// response's extracted JSON array (see eval.ExtractJSON) has exactly want +// elements, zero otherwise. Paired with per-index eval.JSONField checks so +// an answer with extra or missing elements (e.g. forgetting to apply a +// HAVING filter, so an extra group leaks through) cannot score full credit +// merely by getting the checked indices right. +func dbJSONArrayLength(want int) eval.Evaluator { + return eval.EvaluatorFunc(func(_ context.Context, response string) eval.Score { + raw, err := eval.ExtractJSON(response) + if err != nil { + return eval.Score{Value: 0, Detail: err.Error()} + } + var arr []json.RawMessage + if err := json.Unmarshal([]byte(raw), &arr); err != nil { + return eval.Score{Value: 0, Detail: fmt.Sprintf("invalid JSON array: %v", err)} + } + if len(arr) == want { + return eval.Score{Value: 1, Detail: fmt.Sprintf("array length %d", len(arr))} + } + return eval.Score{Value: 0, Detail: fmt.Sprintf("array length %d, want %d", len(arr), want)} + }) +} diff --git a/internal/tests/databases_postgres.go b/internal/tests/databases_postgres.go new file mode 100644 index 0000000..bf485f7 --- /dev/null +++ b/internal/tests/databases_postgres.go @@ -0,0 +1,412 @@ +package tests + +import ( + "github.com/lukaszraczylo/llm-testbench/internal/eval" + "github.com/lukaszraczylo/llm-testbench/internal/testkit" +) + +// registerDBPostgresTests registers every databases/postgres test. +func registerDBPostgresTests(r *testkit.Registry) { + r.Register(dbPGExplainSeqVsIndexTest()) + r.Register(dbPGIndexChoiceTest()) + r.Register(dbPGNPlusOneTest()) + r.Register(dbPGIsolationAnomalyTest()) + r.Register(dbPGDeadlockLockOrderTest()) + r.Register(dbPGReplicaFailoverTest()) + r.Register(dbPGBloatVacuumTest()) + r.Register(dbPGPoolSizingTest()) + r.Register(dbPGPartialIndexTest()) + r.Register(dbPGListenNotifyTest()) +} + +// dbPGExplainSeqVsIndexPlan is the inline EXPLAIN excerpt for +// dbPGExplainSeqVsIndexTest. +const dbPGExplainSeqVsIndexPlan = `Seq Scan on events (cost=0.00..215000.00 rows=4200000 width=8) + Filter: (status = 'active'::text)` + +// dbPGExplainSeqVsIndexTest: interpret why the planner chose a sequential +// scan over an available index, from an inline EXPLAIN plan and the table's +// row counts. +// +// ground truth: the events table holds 10,000,000 rows and the plan +// estimates 4,200,000 of them match status='active' - 42% of the table. +// Above roughly 5-10% selectivity, a sequential scan reading every page +// once beats an index scan that would revisit the heap almost as many +// times as a seq scan touches pages, so a low-selectivity filter (not a +// missing index - one exists - and not stale statistics) is why the +// planner picked Seq Scan. At a selectivity of 8,000/10,000,000 (0.08%), +// the same index would clearly be preferred instead. +func dbPGExplainSeqVsIndexTest() testkit.Test { + prompt := `The "events" table holds 10,000,000 rows and has a btree index +on its "status" column. Here is the EXPLAIN output for +"SELECT id FROM events WHERE status = 'active';": + +` + "```\n" + dbPGExplainSeqVsIndexPlan + "\n```" + ` + +The planner used a sequential scan even though an index on status exists. +Given the estimated row count in the plan versus the table's total row +count, what row-count property of the filter caused this choice? Also: if +instead only 8,000 of the 10,000,000 rows had status='rare' (a highly +selective filter), would the planner likely prefer the index scan instead? +Respond with only a JSON object: +{"reason":"","would_index_help_at_rare_status":true|false}` + + evaluator := eval.Mean( + eval.JSONField("reason", "low-selectivity"), + eval.JSONField("would_index_help_at_rare_status", true), + ) + + return testkit.Test{ + ID: "pg-explain-seq-vs-index", + Category: "databases", + Subcategory: "postgres", + Description: "Interpret an EXPLAIN plan's Seq Scan choice as a low-selectivity filter (42% of the table), not a missing index.", + Prompt: prompt, + Eval: evaluator, + } +} + +// dbPGIndexChoiceSchema is the inline schema + query for +// dbPGIndexChoiceTest. +const dbPGIndexChoiceSchema = `Table: orders(id bigserial, region text, order_date date, amount numeric) + +Query: +SELECT id, amount FROM orders +WHERE region = 'EU' AND order_date >= '2026-01-01' +ORDER BY order_date;` + +// dbPGIndexChoiceTest: pick the composite index column order (equality +// column first, then the range/sort column) that best serves an inline +// query. +// +// ground truth: region is filtered by exact equality, order_date is +// filtered by a range AND drives the ORDER BY. A composite index leads +// with the equality column so the index narrows to matching rows first, +// then with the range/sort column so those rows are already in the index's +// order needed by ORDER BY: (region, order_date). Reversing the order +// (order_date, region) forces a full scan of every order_date >= the bound +// across all regions before filtering by region. +func dbPGIndexChoiceTest() testkit.Test { + prompt := `Here is a table schema and a query that runs frequently: + +` + "```\n" + dbPGIndexChoiceSchema + "\n```" + ` + +Which two columns, in which order, should a single composite index be +defined on to best serve this exact query (equality filters should lead a +composite index, ahead of range/sort columns)? Respond with only a JSON +array of the two column names in index-definition order, e.g. ["a","b"].` + + return testkit.Test{ + ID: "pg-index-choice", + Category: "databases", + Subcategory: "postgres", + Description: "Pick composite index column order (region, order_date) for an inline equality+range+sort query.", + Prompt: prompt, + Eval: eval.JSONStringArrayEquals([]string{"region", "order_date"}), + } +} + +// dbPGNPlusOneCode is the inline query-in-a-loop snippet for +// dbPGNPlusOneTest. +const dbPGNPlusOneCode = `orders = db.query("SELECT id, customer_id FROM orders WHERE status = 'pending'") +for o in orders: + customer = db.query("SELECT name FROM customers WHERE id = %s", [o.customer_id]) + print(o.id, customer.name)` + +// dbPGNPlusOneTest: spot the N+1 query pattern in an inline loop and name +// its single-query fix. +// +// ground truth: the code runs one query to fetch pending orders, then one +// additional query per order to fetch that order's customer name inside +// the loop - the classic N+1 pattern, not a Cartesian product (which would +// return duplicated rows from a single query) or a missing index (which +// would not change the query count at all). The fix is to replace the +// per-row queries with a single JOIN (or a single WHERE id = ANY(...) +// batch query) so exactly one query fetches every needed customer name. +func dbPGNPlusOneTest() testkit.Test { + prompt := `Here is application code that fetches pending orders and their +customer names: + +` + "```python\n" + dbPGNPlusOneCode + "\n```" + ` + +What query pattern is this, and what single-query fix addresses it? +Respond with only a JSON object: +{"pattern":"","fix":""}` + + evaluator := eval.Mean( + eval.JSONField("pattern", "n-plus-one"), + eval.JSONField("fix", "single-join-query"), + ) + + return testkit.Test{ + ID: "pg-n-plus-one", + Category: "databases", + Subcategory: "postgres", + Description: "Spot the N+1 query pattern in an inline per-row query loop and require a single-JOIN fix.", + Prompt: prompt, + Eval: evaluator, + } +} + +// dbPGIsolationAnomalySchedule is the inline two-transaction interleaved +// schedule for dbPGIsolationAnomalyTest. +const dbPGIsolationAnomalySchedule = `T1: BEGIN; +T1: SELECT balance FROM accounts WHERE id = 1; -- returns 100 +T2: BEGIN; +T2: UPDATE accounts SET balance = 50 WHERE id = 1; +T2: COMMIT; +T1: SELECT balance FROM accounts WHERE id = 1; -- returns 50 +T1: COMMIT;` + +// dbPGIsolationAnomalyTest: name the isolation anomaly a READ COMMITTED +// transaction observes when its own two reads of the same row disagree +// because another transaction committed in between. +// +// ground truth: T1 reads balance=100, then, after T2 commits a change to +// the same row, T1's second read of the exact same row within the SAME +// transaction returns 50 - two reads of one row disagreeing inside one +// transaction is the definition of a non-repeatable read, not a dirty read +// (which would require reading T2's UPDATE before T2 committed) or a +// phantom read (which is about a changed set of rows matching a +// predicate, not one row's value). +func dbPGIsolationAnomalyTest() testkit.Test { + prompt := `T1 runs under READ COMMITTED isolation. Here is an interleaved +schedule of T1 and a concurrent T2: + +` + "```\n" + dbPGIsolationAnomalySchedule + "\n```" + ` + +Which isolation anomaly does T1 observe between its two SELECTs? Respond +with only a JSON object: +{"anomaly":""}` + + return testkit.Test{ + ID: "pg-isolation-anomaly", + Category: "databases", + Subcategory: "postgres", + Description: "Name the non-repeatable-read anomaly from an inline READ COMMITTED schedule where a row's value changes between two reads in one transaction.", + Prompt: prompt, + Eval: eval.JSONField("anomaly", "non-repeatable-read"), + } +} + +// dbPGDeadlockSchedule is the inline two-transaction opposite-lock-order +// schedule for dbPGDeadlockLockOrderTest. +const dbPGDeadlockSchedule = `T1: BEGIN; +T1: UPDATE accounts SET balance = balance - 100 WHERE id = 1; -- T1 locks id=1 +T1: UPDATE accounts SET balance = balance + 100 WHERE id = 2; -- T1 waits: T2 holds id=2 + +T2: BEGIN; +T2: UPDATE accounts SET balance = balance - 50 WHERE id = 2; -- T2 locks id=2 +T2: UPDATE accounts SET balance = balance + 50 WHERE id = 1; -- T2 waits: T1 holds id=1` + +// dbPGDeadlockLockOrderTest: recognize a deadlock caused by two +// transactions locking the same two rows in opposite order, and give the +// consistent lock-acquisition order that prevents it. +// +// ground truth: T1 locks id=1 then waits on id=2 (held by T2); T2 locks +// id=2 then waits on id=1 (held by T1) - a circular wait, so Postgres' +// deadlock detector must abort one transaction. Both transactions always +// touching rows in the same order - ascending id, so id=1 before id=2 - +// eliminates the circular wait, since neither transaction would ever hold +// a higher id's lock while waiting on a lower one. +func dbPGDeadlockLockOrderTest() testkit.Test { + prompt := `Here are two concurrent transactions on the same accounts table: + +` + "```\n" + dbPGDeadlockSchedule + "\n```" + ` + +Do these two transactions deadlock? If so, what consistent row lock +acquisition order (by id, ascending) would both transactions need to follow +to prevent this deadlock? Respond with only a JSON object: +{"deadlock":true|false,"safe_order_ids":[,]}` + + evaluator := eval.Mean( + eval.JSONField("deadlock", true), + eval.JSONField("safe_order_ids[0]", 1), + eval.JSONField("safe_order_ids[1]", 2), + ) + + return testkit.Test{ + ID: "pg-deadlock-lock-order", + Category: "databases", + Subcategory: "postgres", + Description: "Recognize a deadlock from two transactions locking rows id=1/id=2 in opposite order and require the ascending-id safe order.", + Prompt: prompt, + Eval: evaluator, + } +} + +// dbPGReplicaFailoverTest: explain that an asynchronously-replicated commit +// can be lost when the primary crashes before the replica catches up and +// gets promoted. +// +// ground truth: with asynchronous streaming replication (no +// synchronous_commit), a transaction's COMMIT returns to the client as +// soon as it is durable on the primary, before its WAL is guaranteed to +// have reached the replica. If the primary crashes in that window and the +// replica is promoted, the promoted replica becomes the new primary +// without ever having applied that transaction's WAL - so the committed +// transaction is not visible on it and is effectively lost, unless it can +// be recovered from the old primary's WAL after the fact. +func dbPGReplicaFailoverTest() testkit.Test { + prompt := `A Postgres primary has one streaming replica, using +asynchronous replication (synchronous_commit is not enabled). A client's +transaction COMMITs successfully on the primary. Before the WAL for that +commit reaches the replica, the primary host crashes, and the replica is +promoted to become the new primary. + +Is that committed transaction guaranteed to be visible on the newly +promoted primary? Explain what happens to it and why, in one or two +sentences.` + + evaluator := eval.All( + eval.W(eval.ContainsAny("lost", "not visible", "not guaranteed", "may be lost", "can be lost", "data loss"), 2), + eval.W(eval.ContainsAny("async", "asynchronous"), 1), + ) + + return testkit.Test{ + ID: "pg-replica-failover", + Category: "databases", + Subcategory: "postgres", + Description: "Explain that an asynchronously-replicated commit can be lost on failover if the primary crashes before the replica catches up.", + Prompt: prompt, + Eval: evaluator, + } +} + +// dbPGBloatVacuumStats is the inline pg_stat_user_tables excerpt for +// dbPGBloatVacuumTest. +const dbPGBloatVacuumStats = `relname | n_live_tup | n_dead_tup | last_autovacuum +orders | 500000 | 4500000 | (null)` + +// dbPGBloatVacuumTest: diagnose table bloat from an inline pg_stat_ +// user_tables excerpt showing dead tuples vastly outnumbering live ones and +// autovacuum never having run. +// +// ground truth: n_dead_tup (4,500,000) is 9x n_live_tup (500,000) - about +// 90% of the table's physical rows are dead - and last_autovacuum is null, +// meaning autovacuum has never completed on this table. That combination +// is table bloat from accumulated dead tuples, not a missing index (which +// this fixture says nothing about) or replication lag (unrelated to +// tuple counts). The fix is to VACUUM the table (reclaiming/reusing the +// dead tuple space), not to reindex or add an index. +func dbPGBloatVacuumTest() testkit.Test { + prompt := `Here is an excerpt of pg_stat_user_tables for one table: + +` + "```\n" + dbPGBloatVacuumStats + "\n```" + ` + +What is the likely cause of this table's condition, and what single +maintenance action addresses it? Respond with only a JSON object: +{"cause":"","action":""}` + + evaluator := eval.Mean( + eval.JSONField("cause", "bloat"), + eval.JSONField("action", "vacuum"), + ) + + return testkit.Test{ + ID: "pg-bloat-vacuum", + Category: "databases", + Subcategory: "postgres", + Description: "Diagnose table bloat (dead tuples 9x live, autovacuum never ran) from an inline pg_stat_user_tables excerpt and require the vacuum fix.", + Prompt: prompt, + Eval: evaluator, + } +} + +// dbPGPoolSizingWant is the pool_size computed by the formula given inline +// in dbPGPoolSizingTest's prompt. +// +// ground truth: pool_size = (core_count * 2) + effective_spindle_count = +// (8 * 2) + 1 = 17. databases_postgres_test.go recomputes this +// independently with the same arithmetic written out plainly. +const dbPGPoolSizingWant = 8*2 + 1 + +// dbPGPoolSizingTest: compute a connection pool size from a formula and +// parameters both given inline in the prompt. +func dbPGPoolSizingTest() testkit.Test { + prompt := `Use this connection pool sizing formula: + +pool_size = (core_count * 2) + effective_spindle_count + +A database server has core_count = 8. Its storage is a single NVMe SSD, +which this formula treats as effective_spindle_count = 1. Compute +pool_size. Respond with only the number.` + + return testkit.Test{ + ID: "pg-pool-sizing", + Category: "databases", + Subcategory: "postgres", + Description: "Compute a connection pool size from an inline formula (core_count*2 + effective_spindle_count) and given parameters.", + Prompt: prompt, + Eval: eval.Numeric(eval.ExtractLastNumber[int], dbPGPoolSizingWant, 0), + } +} + +// dbPGPartialIndexTest: judge whether a partial index is a good fit for an +// inline query pattern where the filtered value is a small, hot subset of +// a huge table. +// +// ground truth: only 120,000 of 10,000,000 rows (1.2%) have +// status='pending', and the application's hottest query filters on exactly +// that value almost every time it queries this table. A partial index +// `ON orders(...) WHERE status = 'pending'` indexes only that small, +// frequently-queried subset - far smaller and cheaper to maintain than a +// full index across all 10,000,000 rows, and it exactly matches the query +// pattern that needs it. +func dbPGPartialIndexTest() testkit.Test { + prompt := `The "orders" table holds 10,000,000 rows. Exactly 120,000 of +them have status='pending'. The application's single hottest query against +this table is "SELECT * FROM orders WHERE status = 'pending';", run almost +every time this table is queried at all. + +Is a partial index (CREATE INDEX ... ON orders(...) WHERE status = +'pending') a good fit here, and why? Respond with only a JSON object: +{"partial_index_applicable":true|false,"reason":""}` + + evaluator := eval.Mean( + eval.JSONField("partial_index_applicable", true), + eval.JSONField("reason", "small-frequently-queried-subset"), + ) + + return testkit.Test{ + ID: "pg-partial-index", + Category: "databases", + Subcategory: "postgres", + Description: "Judge a partial index as a good fit for a 1.2%-selectivity, frequently-queried status value on a 10M-row table.", + Prompt: prompt, + Eval: evaluator, + } +} + +// dbPGListenNotifyTest: pick LISTEN/NOTIFY over polling for near-real-time +// notification of new rows without a tight polling loop. +// +// ground truth: LISTEN/NOTIFY lets a connected client block until the +// database itself pushes a notification on a channel (via NOTIFY, typically +// from a trigger on INSERT), giving near-real-time delivery with zero +// polling queries in between - unlike a polling loop, which either wastes +// queries when idle or adds latency when its interval is too coarse. +func dbPGListenNotifyTest() testkit.Test { + prompt := `Background workers need near-real-time notification whenever a +new row is inserted into a "job_queue" table. The team wants to avoid a +tight polling loop that repeatedly queries the table (e.g. every second) +whether or not there is new work. + +Which built-in Postgres mechanism lets workers avoid polling entirely, +being pushed a notification only when a new row actually arrives? Name the +mechanism.` + + evaluator := eval.Mean( + eval.ContainsAny("LISTEN"), + eval.ContainsAny("NOTIFY"), + ) + + return testkit.Test{ + ID: "pg-listen-notify", + Category: "databases", + Subcategory: "postgres", + Description: "Pick LISTEN/NOTIFY over polling for near-real-time notification of new job_queue rows.", + Prompt: prompt, + Eval: evaluator, + } +} diff --git a/internal/tests/databases_postgres_test.go b/internal/tests/databases_postgres_test.go new file mode 100644 index 0000000..351bfc6 --- /dev/null +++ b/internal/tests/databases_postgres_test.go @@ -0,0 +1,276 @@ +package tests + +import ( + "context" + "testing" +) + +func TestDBPGExplainSeqVsIndexTest_Eval(t *testing.T) { + tc := dbPGExplainSeqVsIndexTest() + + tests := []struct { + name string + response string + want float64 + }{ + {"correct", `{"reason":"low-selectivity","would_index_help_at_rare_status":true}`, 1}, + {"correct fenced with prose", "Here you go:\n```json\n{\"reason\":\"low-selectivity\",\"would_index_help_at_rare_status\":true}\n```", 1}, + {"wrong reason: blames missing index", `{"reason":"missing-index","would_index_help_at_rare_status":true}`, 0.5}, + {"wrong: says rare-status index would not help", `{"reason":"low-selectivity","would_index_help_at_rare_status":false}`, 0.5}, + {"both wrong", `{"reason":"stale-statistics","would_index_help_at_rare_status":false}`, 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Value != tt.want { + t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail) + } + }) + } +} + +func TestDBPGIndexChoiceTest_Eval(t *testing.T) { + tc := dbPGIndexChoiceTest() + + tests := []struct { + name string + response string + want float64 + }{ + {"correct", `["region", "order_date"]`, 1}, + {"correct fenced", "```json\n[\"region\", \"order_date\"]\n```", 1}, + {"reversed order", `["order_date", "region"]`, 0}, + {"missing a column", `["region"]`, 0}, + {"wrong column entirely", `["region", "amount"]`, 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Value != tt.want { + t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail) + } + }) + } +} + +func TestDBPGNPlusOneTest_Eval(t *testing.T) { + tc := dbPGNPlusOneTest() + + tests := []struct { + name string + response string + want float64 + }{ + {"correct", `{"pattern":"n-plus-one","fix":"single-join-query"}`, 1}, + {"correct, different key order", `{"fix":"single-join-query","pattern":"n-plus-one"}`, 1}, + {"wrong pattern", `{"pattern":"cartesian-product","fix":"single-join-query"}`, 0.5}, + {"wrong fix", `{"pattern":"n-plus-one","fix":"add-cache"}`, 0.5}, + {"both wrong", `{"pattern":"missing-index","fix":"add-index"}`, 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Value != tt.want { + t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail) + } + }) + } +} + +func TestDBPGIsolationAnomalyTest_Eval(t *testing.T) { + tc := dbPGIsolationAnomalyTest() + + tests := []struct { + name string + response string + want float64 + }{ + {"correct", `{"anomaly":"non-repeatable-read"}`, 1}, + {"correct, uppercase value", `{"anomaly":"Non-Repeatable-Read"}`, 1}, + {"wrong: dirty read", `{"anomaly":"dirty-read"}`, 0}, + {"wrong: phantom read", `{"anomaly":"phantom-read"}`, 0}, + {"wrong: lost update", `{"anomaly":"lost-update"}`, 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Value != tt.want { + t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail) + } + }) + } +} + +func TestDBPGDeadlockLockOrderTest_Eval(t *testing.T) { + tc := dbPGDeadlockLockOrderTest() + + tests := []struct { + name string + response string + want float64 + }{ + {"correct", `{"deadlock":true,"safe_order_ids":[1,2]}`, 1}, + {"correct fenced with prose", "Yes, they deadlock.\n```json\n{\"deadlock\":true,\"safe_order_ids\":[1,2]}\n```", 1}, + {"says no deadlock", `{"deadlock":false,"safe_order_ids":[1,2]}`, 2.0 / 3.0}, + {"reversed safe order", `{"deadlock":true,"safe_order_ids":[2,1]}`, 1.0 / 3.0}, + {"everything wrong", `{"deadlock":false,"safe_order_ids":[2,1]}`, 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Value != tt.want { + t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail) + } + }) + } +} + +func TestDBPGReplicaFailoverTest_Eval(t *testing.T) { + tc := dbPGReplicaFailoverTest() + + tests := []struct { + name string + response string + want float64 + }{ + { + name: "correct: names data loss and async cause", + response: "Because replication is asynchronous, that commit may be lost - the replica never received its WAL before promotion.", + want: 1, + }, + { + name: "correct: alternate phrasing", + response: "No, it is not guaranteed to be visible; async replication means the promoted replica can be missing it.", + want: 1, + }, + { + name: "wrong: claims it is guaranteed durable", + response: "Yes, Postgres guarantees every committed transaction is always visible after failover.", + want: 0, + }, + { + name: "partial: mentions loss but not the async cause", + response: "That transaction can be lost after the failover.", + want: 2.0 / 3.0, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Value != tt.want { + t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail) + } + }) + } +} + +func TestDBPGBloatVacuumTest_Eval(t *testing.T) { + tc := dbPGBloatVacuumTest() + + tests := []struct { + name string + response string + want float64 + }{ + {"correct", `{"cause":"bloat","action":"vacuum"}`, 1}, + {"correct fenced", "```json\n{\"cause\":\"bloat\",\"action\":\"vacuum\"}\n```", 1}, + {"wrong cause", `{"cause":"missing-index","action":"vacuum"}`, 0.5}, + {"wrong action", `{"cause":"bloat","action":"reindex"}`, 0.5}, + {"both wrong", `{"cause":"replication-lag","action":"add-index"}`, 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Value != tt.want { + t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail) + } + }) + } +} + +// TestDBPGPoolSizingWant_GroundTruth independently re-derives pool_size +// with plain arithmetic, not via the dbPGPoolSizingWant constant's own +// expression. +func TestDBPGPoolSizingWant_GroundTruth(t *testing.T) { + const coreCount = 8 + const effectiveSpindleCount = 1 + want := coreCount*2 + effectiveSpindleCount + + if want != 17 { + t.Fatalf("independently recomputed pool_size = %d, want 17", want) + } + if dbPGPoolSizingWant != want { + t.Errorf("dbPGPoolSizingWant = %d, independently recomputed = %d", dbPGPoolSizingWant, want) + } +} + +func TestDBPGPoolSizingTest_Eval(t *testing.T) { + tc := dbPGPoolSizingTest() + + tests := []struct { + name string + response string + want float64 + }{ + {"correct", "17", 1}, + {"prose wrapped", "pool_size = 17", 1}, + {"wrong: forgot spindle term", "16", 0}, + {"wrong: doubled instead of adding", "18", 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Value != tt.want { + t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail) + } + }) + } +} + +func TestDBPGPartialIndexTest_Eval(t *testing.T) { + tc := dbPGPartialIndexTest() + + tests := []struct { + name string + response string + want float64 + }{ + {"correct", `{"partial_index_applicable":true,"reason":"small-frequently-queried-subset"}`, 1}, + {"correct fenced with prose", "Yes.\n```json\n{\"partial_index_applicable\":true,\"reason\":\"small-frequently-queried-subset\"}\n```", 1}, + {"wrong: says not applicable", `{"partial_index_applicable":false,"reason":"small-frequently-queried-subset"}`, 0.5}, + {"wrong reason", `{"partial_index_applicable":true,"reason":"subset-too-large"}`, 0.5}, + {"both wrong", `{"partial_index_applicable":false,"reason":"query-does-not-filter-on-column"}`, 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Value != tt.want { + t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail) + } + }) + } +} + +func TestDBPGListenNotifyTest_Eval(t *testing.T) { + tc := dbPGListenNotifyTest() + + tests := []struct { + name string + response string + want float64 + }{ + {"correct", "Use LISTEN/NOTIFY.", 1}, + {"correct, spelled out", "Postgres's LISTEN and NOTIFY commands let a worker block until notified.", 1}, + {"only names one half", "Use LISTEN.", 0.5}, + {"wrong: recommends polling harder", "Just poll more frequently, e.g. every 100ms.", 0}, + {"wrong: recommends a trigger with no LISTEN/NOTIFY", "Add a trigger that writes to an audit table.", 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Value != tt.want { + t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail) + } + }) + } +} diff --git a/internal/tests/databases_redis.go b/internal/tests/databases_redis.go new file mode 100644 index 0000000..fb0a427 --- /dev/null +++ b/internal/tests/databases_redis.go @@ -0,0 +1,410 @@ +package tests + +import ( + "context" + "regexp" + "strings" + + "github.com/lukaszraczylo/llm-testbench/internal/eval" + "github.com/lukaszraczylo/llm-testbench/internal/testkit" +) + +// registerDBRedisTests registers every databases/redis test. +func registerDBRedisTests(r *testkit.Registry) { + r.Register(dbRedisStructureChoiceTest()) + r.Register(dbRedisTTLEvictionTest()) + r.Register(dbRedisIncrAtomicityTest()) + r.Register(dbRedisMultiVsPipelineTest()) + r.Register(dbRedisScanVsKeysTest()) + r.Register(dbRedisPubSubDeliveryTest()) + r.Register(dbRedisLuaAtomicityTest()) + r.Register(dbRedisMemoryEstimateTest()) + r.Register(dbRedisCacheStampedeTest()) + r.Register(dbRedisKeyspaceAntiPatternTest()) +} + +// dbRedisStructureScenarios lists the 4 inline scenarios for +// dbRedisStructureChoiceTest. +const dbRedisStructureScenarios = `a: "A leaderboard of user scores that must support efficient range and + rank queries (e.g. 'give me the top 10' or 'what rank is this user')." +b: "Counting approximately how many unique visitors hit the site today, + at a scale of tens of millions of visitors, using minimal memory." +c: "A user's session data as a group of named fields (name, plan, + last_seen) that must each be readable and updatable individually + without rewriting the whole session." +d: "A FIFO job queue where worker processes block until a new job + arrives, then pop and process jobs in the order they were pushed."` + +// dbRedisStructureChoiceTest: pick the best-fit Redis data structure for +// each of 4 inline usage scenarios, from a closed vocabulary. +// +// ground truth: (a) a leaderboard needing ranked range queries is exactly +// what a sorted set (ZSET, score-ordered) provides. (b) approximate unique +// counting at large scale with minimal memory is HyperLogLog's purpose +// (a small fixed-size probabilistic structure), not an exact set which +// would use memory proportional to visitor count. (c) named, individually +// updatable fields is a hash, not a single string blob that would need a +// full read-modify-write. (d) a FIFO queue with blocking pop is a list +// (LPUSH/BRPOP), not a set (unordered, no blocking pop) or a stream +// (heavier, consumer-group semantics not needed here). +func dbRedisStructureChoiceTest() testkit.Test { + prompt := `For each of these 4 scenarios, pick the single best-fit Redis +data structure from exactly this vocabulary: string, list, hash, set, +sorted-set, hyperloglog, stream, bitmap. + +` + dbRedisStructureScenarios + ` + +Respond with only a JSON object mapping each letter to one structure name: +{"a":"...","b":"...","c":"...","d":"..."}` + + evaluator := eval.Mean( + eval.JSONField("a", "sorted-set"), + eval.JSONField("b", "hyperloglog"), + eval.JSONField("c", "hash"), + eval.JSONField("d", "list"), + ) + + return testkit.Test{ + ID: "redis-structure-choice", + Category: "databases", + Subcategory: "redis", + Description: "Pick the best-fit Redis data structure (sorted-set, hyperloglog, hash, list) for 4 inline usage scenarios from a closed vocabulary.", + Prompt: prompt, + Eval: evaluator, + } +} + +// dbRedisTTLEvictionTest: determine what happens to a write under +// maxmemory-policy=noeviction when Redis is already at maxmemory. +// +// ground truth: noeviction means Redis will never evict a key to make +// room. A write that would grow memory usage while already at maxmemory +// is instead rejected outright with an OOM error - the write does not +// silently succeed and no key is evicted to make space. +func dbRedisTTLEvictionTest() testkit.Test { + prompt := `A Redis instance has maxmemory-policy set to "noeviction" and +is currently at maxmemory. A client sends a write command that would +increase memory usage further. + +What happens to that write? Respond with only one word: reject or evict.` + + return testkit.Test{ + ID: "redis-ttl-eviction", + Category: "databases", + Subcategory: "redis", + Description: "Determine that a write is rejected (not silently allowed, and nothing evicted) under maxmemory-policy=noeviction at maxmemory.", + Prompt: prompt, + Eval: dbExactAnswer("reject"), + } +} + +// dbRedisIncrAtomicityTest: contrast INCR's atomic guarantee against the +// race in a manual GET-then-SET increment pattern. +// +// ground truth: INCR is a single atomic server-side operation, so 100 +// concurrent INCR calls on a counter starting at 0 are guaranteed to leave +// it at exactly 100, with no lost updates. The GET-then-SET pattern is two +// separate round trips per client; two clients can both GET the same +// value before either SETs, so one client's increment can be silently +// overwritten - the final value after 100 concurrent GET/SET increments is +// not guaranteed to reach 100. +func dbRedisIncrAtomicityTest() testkit.Test { + prompt := `A Redis counter starts at 0. 100 clients each run exactly one +of the two patterns below, concurrently: + +Pattern A: each client calls INCR counter once. +Pattern B: each client calls GET counter, adds 1 in its own application +code, then calls SET counter once. + +What is the guaranteed final value of the counter after Pattern A's 100 +calls, and is Pattern B's final value guaranteed to also reach that same +total? Respond with only a JSON object: +{"incr_final":,"getset_race_safe":true|false}` + + evaluator := eval.Mean( + eval.JSONField("incr_final", 100), + eval.JSONField("getset_race_safe", false), + ) + + return testkit.Test{ + ID: "redis-incr-atomicity", + Category: "databases", + Subcategory: "redis", + Description: "Contrast INCR's guaranteed atomic result (100) against the race-prone GET-then-SET increment pattern (not guaranteed to reach 100).", + Prompt: prompt, + Eval: evaluator, + } +} + +// dbRedisMultiVsPipelineTest: confirm that plain pipelining, unlike +// MULTI/EXEC, gives no atomicity/isolation guarantee against interleaving +// from other clients. +// +// ground truth: pipelining is purely a network optimization - batching +// several commands into one round trip - with no guarantee that the +// server executes them back-to-back relative to other connected clients. +// MULTI/EXEC (a transaction) is what guarantees no other client's command +// can execute in between the queued commands. +func dbRedisMultiVsPipelineTest() testkit.Test { + prompt := `Does plain command pipelining (batching several commands into +one network round trip, without wrapping them in MULTI/EXEC) guarantee +that no other client's command can execute on the server in between the +pipelined commands? Respond with only "yes" or "no".` + + return testkit.Test{ + ID: "redis-multi-vs-pipeline", + Category: "databases", + Subcategory: "redis", + Description: "Confirm plain pipelining (unlike MULTI/EXEC) gives no guarantee against another client's commands interleaving.", + Prompt: prompt, + Eval: dbExactAnswer("no"), + } +} + +// dbKeysCommandPattern matches an explicit mention of the Redis KEYS +// command actually being invoked - "KEYS *" (its most common prod-danger +// form) or a backtick-quoted `KEYS` - not a generic use of the English +// word "keys" elsewhere in prose ("the keys to fixing this..."). +var dbKeysCommandPattern = regexp.MustCompile("(?:`KEYS`|\\bKEYS\\s*\\*)") + +// dbNegationCuePattern matches a word that turns a mention of the KEYS +// command into a warning against running it in production, rather than an +// instruction to run it. +var dbNegationCuePattern = regexp.MustCompile(`(?i)\b(don'?t|do not|never|avoid|instead of|not|shouldn'?t|should not|rather than|without|no need|danger|dangerous)\b`) + +// dbNegationWindow is how many characters before a KEYS-command mention +// are searched for a negation cue. +const dbNegationWindow = 60 + +// dbNegationWindowStart returns the earliest byte offset in response to +// search for a negation cue before a KEYS-command mention starting at +// start: the current line, extended back at most dbNegationWindow +// characters. +func dbNegationWindowStart(response string, start int) int { + lineStart := strings.LastIndexByte(response[:start], '\n') + 1 + return max(start-dbNegationWindow, lineStart) +} + +// dbNoBareKeysInProd scores full credit unless the response instructs +// running "KEYS *" against a production instance without any negating +// context (a warning not to, or an instruction to use SCAN instead is +// fine). This is deliberately not eval.NotContains, which would also zero +// out the best possible answer: one that correctly explains why NOT to run +// KEYS in production. +func dbNoBareKeysInProd() eval.Evaluator { + return eval.EvaluatorFunc(func(_ context.Context, response string) eval.Score { + matches := dbKeysCommandPattern.FindAllStringIndex(response, -1) + if len(matches) == 0 { + return eval.Score{Value: 1, Detail: "no mention of running the KEYS command"} + } + for _, loc := range matches { + start := loc[0] + windowStart := dbNegationWindowStart(response, start) + if !dbNegationCuePattern.MatchString(response[windowStart:start]) { + return eval.Score{Value: 0, Detail: "unnegated mention of running the KEYS command"} + } + } + return eval.Score{Value: 1, Detail: "every mention of KEYS is negated"} + }) +} + +// dbRedisScanVsKeysTest: require SCAN over KEYS for a hot-path production +// lookup, using a negation-aware guard so an answer that correctly warns +// against KEYS is not penalized for mentioning it. +func dbRedisScanVsKeysTest() testkit.Test { + prompt := `A request handler that runs thousands of times per second +against a production Redis instance holding millions of keys currently +calls "KEYS *" to find every key matching a pattern. + +What is wrong with this in production, and what command should be used +instead to iterate the keyspace safely?` + + evaluator := eval.All( + eval.W(dbNoBareKeysInProd(), 2), + eval.W(eval.ContainsAny("SCAN"), 2), + ) + + return testkit.Test{ + ID: "redis-scan-vs-keys", + Category: "databases", + Subcategory: "redis", + Description: "Require the SCAN cursor over KEYS * for a hot-path production lookup, without penalizing a negated warning against KEYS.", + Prompt: prompt, + Eval: evaluator, + } +} + +// dbRedisPubSubDeliveryTest: confirm Redis pub/sub gives no delivery +// guarantee to a disconnected subscriber. +// +// ground truth: Redis pub/sub is fire-and-forget with no backlog or +// replay - a message published while a subscriber is disconnected is +// simply not delivered to it; reconnecting does not retrieve messages +// missed while offline. +func dbRedisPubSubDeliveryTest() testkit.Test { + prompt := `A subscriber is disconnected (e.g. a brief network blip) at +the exact moment a message is published on a channel it normally +subscribes to. After the subscriber reconnects, will it receive that +specific message it missed while disconnected? Respond with only "yes" or +"no".` + + return testkit.Test{ + ID: "redis-pubsub-delivery", + Category: "databases", + Subcategory: "redis", + Description: "Confirm Redis pub/sub gives no delivery guarantee (no backlog/replay) to a subscriber that was disconnected when a message was published.", + Prompt: prompt, + Eval: dbExactAnswer("no"), + } +} + +// dbRedisLuaAtomicityTest: confirm a Lua script run via EVAL executes +// atomically with no other client command interleaved. +// +// ground truth: Redis executes commands (and an entire EVAL script) on a +// single command-processing thread; while one client's script is running, +// the server does not process any other client's command, so a script is +// guaranteed to run start-to-finish with nothing interleaved. +func dbRedisLuaAtomicityTest() testkit.Test { + prompt := `While one client's Lua script is executing via EVAL, can any +other client's command run on the Redis server before that script +finishes? Respond with only "yes" or "no".` + + return testkit.Test{ + ID: "redis-lua-atomicity", + Category: "databases", + Subcategory: "redis", + Description: "Confirm a Lua script run via EVAL executes atomically, with no other client command able to interleave.", + Prompt: prompt, + Eval: dbExactAnswer("no"), + } +} + +// dbRedisMemoryEstimateWant is the total estimated bytes computed by the +// formula given inline in dbRedisMemoryEstimateTest's prompt. +// +// ground truth: total_bytes = count * (key_bytes + value_bytes + +// overhead_bytes) = 200000 * (20 + 100 + 56) = 200000 * 176 = 35,200,000. +// databases_redis_test.go recomputes this independently with the same +// arithmetic written out plainly. +const dbRedisMemoryEstimateWant = 200000 * (20 + 100 + 56) + +// dbRedisMemoryEstimateTest: compute total memory bytes for an inline +// dataset from a formula given entirely in the prompt. +func dbRedisMemoryEstimateTest() testkit.Test { + prompt := `Estimate total memory, in bytes, to store a Redis dataset of +200,000 string key-value pairs, where each key name is 20 bytes, each +value is 100 bytes, and Redis's per-key-value overhead (hash table entry +plus object headers) is estimated at 56 bytes. Use this formula: + +total_bytes = count * (key_bytes + value_bytes + overhead_bytes) + +Respond with only the number.` + + return testkit.Test{ + ID: "redis-memory-estimate", + Category: "databases", + Subcategory: "redis", + Description: "Compute total memory bytes for a 200,000-key dataset from an inline per-key-value formula and byte sizes.", + Prompt: prompt, + Eval: eval.Numeric(eval.ExtractLastNumber[int], dbRedisMemoryEstimateWant, 0), + } +} + +// dbRedisStampedeCandidates lists the 4 candidate mitigation techniques +// for dbRedisCacheStampedeTest, mixing genuinely valid mitigations with +// one plausible-sounding non-mitigation. +const dbRedisStampedeCandidates = `- mutex_lock_on_regenerate: only one client regenerates a missing/expired + key at a time; others wait for or read the in-flight result. +- probabilistic_early_expiration: clients probabilistically recompute a + soon-to-expire key slightly before it actually expires, spreading + regeneration out instead of all racing at the exact expiry moment. +- increase_replica_count: add more read replicas to the Redis deployment. +- stale_while_revalidate: serve the stale cached value immediately while + one background request regenerates it.` + +// dbRedisStampedeWant is the subset of dbRedisStampedeCandidates that +// actually mitigates cache stampede. +// +// ground truth: a cache stampede is a burst of concurrent regenerations +// racing to rebuild the same expired key against the backing store. +// mutex_lock_on_regenerate, probabilistic_early_expiration, and +// stale_while_revalidate each directly prevent that race (by +// serializing regeneration, spreading it out over time, or deferring it +// entirely). Adding more replicas does not: replicas serve reads of +// already-cached data, but every replica independently misses the same +// expired key at the same moment, so it does not reduce or serialize the +// regeneration race against the backing store at all. +var dbRedisStampedeWant = []string{ + "mutex_lock_on_regenerate", + "probabilistic_early_expiration", + "stale_while_revalidate", +} + +func dbRedisCacheStampedeTest() testkit.Test { + prompt := `A hot cache key expires and many concurrent requests all miss +it at once, each racing to regenerate it against the backing store (a +"cache stampede" / "thundering herd"). Here are 4 candidate mitigation +techniques: + +` + dbRedisStampedeCandidates + ` + +Which of these candidates actually mitigate a cache stampede? Respond with +only a JSON array of the candidate names that do.` + + return testkit.Test{ + ID: "redis-cache-stampede", + Category: "databases", + Subcategory: "redis", + Description: "Select the cache-stampede mitigations (mutex, probabilistic early expiration, stale-while-revalidate) from 4 candidates that also include a non-mitigation.", + Prompt: prompt, + Eval: eval.JSONStringSet(dbRedisStampedeWant), + } +} + +// dbRedisKeyspaceAntiPatternCode is the inline monolithic-key snippet for +// dbRedisKeyspaceAntiPatternTest. +const dbRedisKeyspaceAntiPatternCode = `def save_all_users(users): + r.set("all_users", json.dumps(users)) # users: a list of 500,000 user dicts + +def update_user(user_id, fields): + users = json.loads(r.get("all_users")) + for u in users: + if u["id"] == user_id: + u.update(fields) + r.set("all_users", json.dumps(users)) # rewrites the entire 500,000-user blob` + +// dbRedisKeyspaceAntiPatternTest: spot a monolithic single-key blob +// keyspace anti-pattern in inline code and require the per-entity hash +// fix. +// +// ground truth: every single-field update to one user rewrites a +// serialized blob of all 500,000 users under one key, an O(n) operation +// per update where n is the whole user count - a monolithic-key +// anti-pattern, not a "too many small keys" problem (there is only one +// key here) or a missing-TTL problem (nothing about expiry is shown). The +// fix is to give each user its own Hash key (e.g. user:), so a field +// update touches O(1) data instead of the entire dataset. +func dbRedisKeyspaceAntiPatternTest() testkit.Test { + prompt := `Here is code managing user data in Redis: + +` + "```python\n" + dbRedisKeyspaceAntiPatternCode + "\n```" + ` + +What keyspace design problem does this have, and what is the correct fix? +Respond with only a JSON object: +{"problem":"","fix":""}` + + evaluator := eval.Mean( + eval.JSONField("problem", "monolithic-key"), + eval.JSONField("fix", "hash-per-user"), + ) + + return testkit.Test{ + ID: "redis-keyspace-anti-pattern", + Category: "databases", + Subcategory: "redis", + Description: "Spot a monolithic single-key user blob rewritten on every update and require the per-user-hash fix.", + Prompt: prompt, + Eval: evaluator, + } +} diff --git a/internal/tests/databases_redis_test.go b/internal/tests/databases_redis_test.go new file mode 100644 index 0000000..ccebec2 --- /dev/null +++ b/internal/tests/databases_redis_test.go @@ -0,0 +1,342 @@ +package tests + +import ( + "context" + "testing" +) + +func TestDBRedisStructureChoiceTest_Eval(t *testing.T) { + tc := dbRedisStructureChoiceTest() + + tests := []struct { + name string + response string + want float64 + }{ + { + name: "all correct", + response: `{"a":"sorted-set","b":"hyperloglog","c":"hash","d":"list"}`, + want: 1, + }, + { + name: "all correct fenced with prose", + response: "Here is my answer:\n```json\n{\"a\":\"sorted-set\",\"b\":\"hyperloglog\",\"c\":\"hash\",\"d\":\"list\"}\n```", + want: 1, + }, + { + name: "one wrong", + response: `{"a":"set","b":"hyperloglog","c":"hash","d":"list"}`, + want: 0.75, + }, + { + name: "all wrong", + response: `{"a":"list","b":"set","c":"string","d":"stream"}`, + want: 0, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Value != tt.want { + t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail) + } + }) + } +} + +func TestDBRedisTTLEvictionTest_Eval(t *testing.T) { + tc := dbRedisTTLEvictionTest() + + tests := []struct { + name string + response string + want float64 + }{ + {"bare correct", "reject", 1}, + {"quoted correct", `"reject"`, 1}, + {"different case", "REJECT", 1}, + {"trailing period", "reject.", 1}, + {"wrong: evict", "evict", 0}, + {"wrong: claims silent success", "succeed", 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Value != tt.want { + t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail) + } + }) + } +} + +func TestDBRedisIncrAtomicityTest_Eval(t *testing.T) { + tc := dbRedisIncrAtomicityTest() + + tests := []struct { + name string + response string + want float64 + }{ + {"correct", `{"incr_final":100,"getset_race_safe":false}`, 1}, + {"correct fenced", "```json\n{\"incr_final\":100,\"getset_race_safe\":false}\n```", 1}, + {"wrong incr_final", `{"incr_final":99,"getset_race_safe":false}`, 0.5}, + {"wrongly claims getset is safe", `{"incr_final":100,"getset_race_safe":true}`, 0.5}, + {"both wrong", `{"incr_final":50,"getset_race_safe":true}`, 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Value != tt.want { + t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail) + } + }) + } +} + +func TestDBRedisMultiVsPipelineTest_Eval(t *testing.T) { + tc := dbRedisMultiVsPipelineTest() + + tests := []struct { + name string + response string + want float64 + }{ + {"bare correct", "no", 1}, + {"quoted correct", `"no"`, 1}, + {"different case", "No.", 1}, + {"wrong", "yes", 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Value != tt.want { + t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail) + } + }) + } +} + +// TestDBNoBareKeysInProd exercises the negation-aware guard directly, +// mirroring TestNoLiveKubectlMutation's coverage: an unnegated instruction +// to run KEYS scores 0, but a response that mentions KEYS only to warn +// against it (however it is phrased or wrapped) scores 1. +func TestDBNoBareKeysInProd(t *testing.T) { + tests := []struct { + name string + response string + want float64 + }{ + { + name: "no mention at all", + response: "Use SCAN with a cursor and MATCH pattern instead, iterating in small batches.", + want: 1, + }, + { + name: "negated mid-sentence mention", + response: "This runs against production, so do not use `KEYS` here - it blocks the whole server.", + want: 1, + }, + { + name: "imperative command at line start", + response: "Just run this:\nKEYS *", + want: 0, + }, + { + name: "unnegated mid-sentence mention", + response: "One quick option is to just run KEYS * to list them all.", + want: 0, + }, + { + name: "negation cue too far away to count", + response: "Never mind the migration for now, but you could casually run KEYS * to check this quickly.", + want: 0, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := dbNoBareKeysInProd().Evaluate(context.Background(), tt.response) + if got.Value != tt.want { + t.Errorf("dbNoBareKeysInProd().Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail) + } + }) + } +} + +func TestDBRedisScanVsKeysTest_Eval(t *testing.T) { + tc := dbRedisScanVsKeysTest() + + tests := []struct { + name string + response string + want float64 + }{ + { + name: "correct: warns against KEYS, recommends SCAN", + response: "Avoid running `KEYS *` in production - it blocks the entire server while it scans the whole keyspace, causing latency spikes for every other client. Use SCAN with a cursor and small COUNT instead, iterating incrementally.", + want: 1, + }, + { + name: "wrong: recommends running KEYS unnegated", + response: "Just run KEYS * - it will return every matching key.", + want: 0, + }, + { + name: "partial: correctly avoids KEYS but never names SCAN", + response: "Do not run that command against production; iterate the keyspace incrementally instead.", + want: 0.5, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Value != tt.want { + t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail) + } + }) + } +} + +func TestDBRedisPubSubDeliveryTest_Eval(t *testing.T) { + tc := dbRedisPubSubDeliveryTest() + + tests := []struct { + name string + response string + want float64 + }{ + {"bare correct", "no", 1}, + {"quoted correct", `"no"`, 1}, + {"wrong", "yes", 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Value != tt.want { + t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail) + } + }) + } +} + +func TestDBRedisLuaAtomicityTest_Eval(t *testing.T) { + tc := dbRedisLuaAtomicityTest() + + tests := []struct { + name string + response string + want float64 + }{ + {"bare correct", "no", 1}, + {"trailing period", "no.", 1}, + {"wrong", "yes", 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Value != tt.want { + t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail) + } + }) + } +} + +// TestDBRedisMemoryEstimateWant_GroundTruth independently re-derives the +// total-bytes estimate with plain arithmetic, not via the +// dbRedisMemoryEstimateWant constant's own expression. +func TestDBRedisMemoryEstimateWant_GroundTruth(t *testing.T) { + const count, keyBytes, valueBytes, overheadBytes = 200000, 20, 100, 56 + want := count * (keyBytes + valueBytes + overheadBytes) + + if want != 35200000 { + t.Fatalf("independently recomputed total bytes = %d, want 35200000", want) + } + if dbRedisMemoryEstimateWant != want { + t.Errorf("dbRedisMemoryEstimateWant = %d, independently recomputed = %d", dbRedisMemoryEstimateWant, want) + } +} + +func TestDBRedisMemoryEstimateTest_Eval(t *testing.T) { + tc := dbRedisMemoryEstimateTest() + + tests := []struct { + name string + response string + want float64 + }{ + {"correct", "35200000", 1}, + {"prose wrapped", "The estimate is about 35200000 bytes.", 1}, + {"wrong: forgot overhead term", "24000000", 0}, + {"wrong: used wrong overhead", "32000000", 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Value != tt.want { + t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail) + } + }) + } +} + +func TestDBRedisCacheStampedeTest_Eval(t *testing.T) { + tc := dbRedisCacheStampedeTest() + + tests := []struct { + name string + response string + want float64 + }{ + { + name: "exact correct set", + response: `["mutex_lock_on_regenerate", "probabilistic_early_expiration", "stale_while_revalidate"]`, + want: 1, + }, + { + name: "correct set, different order, fenced", + response: "```json\n[\"stale_while_revalidate\", \"mutex_lock_on_regenerate\", \"probabilistic_early_expiration\"]\n```", + want: 1, + }, + { + name: "includes the non-mitigation", + response: `["mutex_lock_on_regenerate", "probabilistic_early_expiration", "stale_while_revalidate", "increase_replica_count"]`, + want: 0.75, + }, + { + name: "only the non-mitigation", + response: `["increase_replica_count"]`, + want: 0, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Value != tt.want { + t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail) + } + }) + } +} + +func TestDBRedisKeyspaceAntiPatternTest_Eval(t *testing.T) { + tc := dbRedisKeyspaceAntiPatternTest() + + tests := []struct { + name string + response string + want float64 + }{ + {"correct", `{"problem":"monolithic-key","fix":"hash-per-user"}`, 1}, + {"correct fenced with prose", "Here's the issue:\n```json\n{\"problem\":\"monolithic-key\",\"fix\":\"hash-per-user\"}\n```", 1}, + {"wrong problem", `{"problem":"missing-ttl","fix":"hash-per-user"}`, 0.5}, + {"wrong fix", `{"problem":"monolithic-key","fix":"add-ttl"}`, 0.5}, + {"both wrong", `{"problem":"too-many-small-keys","fix":"single-key-ok"}`, 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Value != tt.want { + t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail) + } + }) + } +} diff --git a/internal/tests/databases_sqltuning.go b/internal/tests/databases_sqltuning.go new file mode 100644 index 0000000..e50731b --- /dev/null +++ b/internal/tests/databases_sqltuning.go @@ -0,0 +1,634 @@ +package tests + +import ( + "sort" + + "github.com/lukaszraczylo/llm-testbench/internal/eval" + "github.com/lukaszraczylo/llm-testbench/internal/testkit" +) + +// registerDBSQLTuningTests registers every databases/sql-tuning test. +func registerDBSQLTuningTests(r *testkit.Registry) { + r.Register(dbSQLAggregateSumTest()) + r.Register(dbSQLJoinCountsTest()) + r.Register(dbSQLNullInWhereTest()) + r.Register(dbSQLGroupByHavingTest()) + r.Register(dbSQLWindowRankTest()) + r.Register(dbSQLCompositeIndexOrderTest()) + r.Register(dbSQLKeysetPaginationTest()) + r.Register(dbSQLCoveringIndexTest()) + r.Register(dbSQLEquivalentRewriteTest()) + r.Register(dbSQLOrderByIndexServingTest()) +} + +// dbSQLOrder is one row of the shared inline "orders" table fixture used by +// several sql-tuning tests. Region is "" to represent SQL NULL (order id 4 +// has no region on file). +type dbSQLOrder struct { + Region string + Status string + ID int + CustomerID int + Amount int +} + +// dbSQLOrders is the shared inline "orders" table fixture. It is printed +// verbatim (see dbSQLOrdersTableText) into several sql-tuning prompts, and +// every one of those tests' expected answers is derived by calling a query- +// logic helper on this exact slice - never hardcoded independently of it. +var dbSQLOrders = []dbSQLOrder{ + {ID: 1, CustomerID: 101, Region: "EU", Amount: 250, Status: "paid"}, + {ID: 2, CustomerID: 102, Region: "US", Amount: 100, Status: "paid"}, + {ID: 3, CustomerID: 101, Region: "EU", Amount: 75, Status: "pending"}, + {ID: 4, CustomerID: 103, Region: "", Amount: 300, Status: "paid"}, + {ID: 5, CustomerID: 102, Region: "US", Amount: 50, Status: "refunded"}, + {ID: 6, CustomerID: 104, Region: "APAC", Amount: 400, Status: "paid"}, + {ID: 7, CustomerID: 101, Region: "EU", Amount: 125, Status: "paid"}, + {ID: 8, CustomerID: 103, Region: "US", Amount: 60, Status: "pending"}, +} + +// dbSQLOrdersTableText is dbSQLOrders rendered as a plain-text table for +// the prompt. Kept in sync with dbSQLOrders by hand; databases_sqltuning_ +// test.go's ground-truth tests operate on dbSQLOrders directly, so a +// mismatch here would only affect prompt readability, never the pinned +// expected answers. +const dbSQLOrdersTableText = `id | customer_id | region | amount | status +1 | 101 | EU | 250 | paid +2 | 102 | US | 100 | paid +3 | 101 | EU | 75 | pending +4 | 103 | NULL | 300 | paid +5 | 102 | US | 50 | refunded +6 | 104 | APAC | 400 | paid +7 | 101 | EU | 125 | paid +8 | 103 | US | 60 | pending` + +// dbSQLSumPaidAmount sums Amount over every row whose Status is "paid" - +// the query-logic equivalent of "SELECT SUM(amount) FROM orders WHERE +// status = 'paid'". +func dbSQLSumPaidAmount(rows []dbSQLOrder) int { + total := 0 + for _, r := range rows { + if r.Status == "paid" { + total += r.Amount + } + } + return total +} + +// dbSQLAggregateSumWant is derived by calling dbSQLSumPaidAmount, not +// hardcoded. +// +// ground truth: paid rows are id1(250), id2(100), id4(300), id6(400), +// id7(125): 250+100+300+400+125 = 1175. databases_sqltuning_test.go +// recomputes this independently with a hand-written loop over the same +// dbSQLOrders fixture. +var dbSQLAggregateSumWant = dbSQLSumPaidAmount(dbSQLOrders) + +// dbSQLAggregateSumTest: compute SUM(amount) WHERE status = 'paid' over an +// inline 8-row orders table. +func dbSQLAggregateSumTest() testkit.Test { + prompt := `Here is the "orders" table: + +` + "```\n" + dbSQLOrdersTableText + "\n```" + ` + +What does "SELECT SUM(amount) FROM orders WHERE status = 'paid';" return? +Respond with only the number.` + + return testkit.Test{ + ID: "sql-aggregate-sum", + Category: "databases", + Subcategory: "sql-tuning", + Description: "Compute SUM(amount) WHERE status = 'paid' over an inline 8-row orders table.", + Prompt: prompt, + Eval: eval.Numeric(eval.ExtractLastNumber[int], dbSQLAggregateSumWant, 0), + } +} + +// dbSQLCustomer is one row of the shared inline "customers" table fixture +// used by dbSQLJoinCountsTest. It intentionally does not cover every +// customer_id present in dbSQLOrders (103 and 104 are absent), and +// includes one customer (105) with no matching orders at all, so INNER +// JOIN and LEFT JOIN produce different row counts. +type dbSQLCustomer struct { + Name string + ID int +} + +// dbSQLCustomers is the shared inline "customers" table fixture, printed +// verbatim into dbSQLJoinCountsTest's prompt. +var dbSQLCustomers = []dbSQLCustomer{ + {ID: 101, Name: "Alice"}, + {ID: 102, Name: "Bob"}, + {ID: 105, Name: "Eve"}, +} + +// dbSQLCustomersTableText is dbSQLCustomers rendered as a plain-text table +// for the prompt. +const dbSQLCustomersTableText = `id | name +101 | Alice +102 | Bob +105 | Eve` + +// dbSQLInnerJoinCount counts the result rows of "orders o INNER JOIN +// customers c ON o.customer_id = c.id": one row per order whose +// customer_id matches some customer in customers, zero rows contributed by +// a customer or order with no match on the other side. +func dbSQLInnerJoinCount(orders []dbSQLOrder, customers []dbSQLCustomer) int { + knownCustomer := make(map[int]bool, len(customers)) + for _, c := range customers { + knownCustomer[c.ID] = true + } + count := 0 + for _, o := range orders { + if knownCustomer[o.CustomerID] { + count++ + } + } + return count +} + +// dbSQLLeftJoinCount counts the result rows of "customers c LEFT JOIN +// orders o ON o.customer_id = c.id": one row per matching order, or +// exactly one row (with NULL order columns) for a customer with zero +// matching orders. +func dbSQLLeftJoinCount(customers []dbSQLCustomer, orders []dbSQLOrder) int { + count := 0 + for _, c := range customers { + matches := 0 + for _, o := range orders { + if o.CustomerID == c.ID { + matches++ + } + } + if matches == 0 { + count++ + } else { + count += matches + } + } + return count +} + +// dbSQLInnerJoinCountWant and dbSQLLeftJoinCountWant are derived by calling +// dbSQLInnerJoinCount/dbSQLLeftJoinCount, not hardcoded. +// +// ground truth: orders' customer_id values are 101(x3: id1,3,7), 102(x2: +// id2,5), 103(x2: id4,8), 104(x1: id6). customers only holds ids +// {101,102,105}. INNER JOIN keeps only rows whose customer_id is in that +// set: 3+2 = 5 (customer 105 contributes nothing; customer_id 103/104 +// orders are dropped). LEFT JOIN, driven from customers, keeps every +// customer: 101 contributes 3, 102 contributes 2, 105 contributes exactly +// 1 row (NULL order columns, since it matches zero orders): 3+2+1 = 6. +// databases_sqltuning_test.go recomputes both independently with a +// hand-written loop. +var ( + dbSQLInnerJoinCountWant = dbSQLInnerJoinCount(dbSQLOrders, dbSQLCustomers) + dbSQLLeftJoinCountWant = dbSQLLeftJoinCount(dbSQLCustomers, dbSQLOrders) +) + +// dbSQLJoinCountsTest: compute the row counts of an INNER JOIN versus a +// LEFT JOIN between the same inline orders and customers tables. +func dbSQLJoinCountsTest() testkit.Test { + prompt := `Here are two tables: + +orders: +` + "```\n" + dbSQLOrdersTableText + "\n```" + ` + +customers: +` + "```\n" + dbSQLCustomersTableText + "\n```" + ` + +How many rows does each of these two queries return? + +query_inner: SELECT o.id FROM orders o INNER JOIN customers c ON o.customer_id = c.id; +query_left: SELECT c.name, o.id FROM customers c LEFT JOIN orders o ON o.customer_id = c.id; + +Respond with only a JSON object: +{"inner_join_count":,"left_join_count":}` + + evaluator := eval.Mean( + eval.JSONField("inner_join_count", dbSQLInnerJoinCountWant), + eval.JSONField("left_join_count", dbSQLLeftJoinCountWant), + ) + + return testkit.Test{ + ID: "sql-join-counts", + Category: "databases", + Subcategory: "sql-tuning", + Description: "Compute INNER JOIN (5) versus LEFT JOIN (6) row counts between an inline orders and customers table.", + Prompt: prompt, + Eval: evaluator, + } +} + +// dbSQLCountRegionNotEU counts rows where "region <> 'EU'" evaluates to +// SQL's three-valued TRUE. A row whose Region is SQL NULL (Region == "") +// makes that comparison evaluate to UNKNOWN, not TRUE, so it is excluded - +// not counted as matching EU and not counted as matching the negation +// either. +func dbSQLCountRegionNotEU(rows []dbSQLOrder) int { + count := 0 + for _, r := range rows { + if r.Region == "" { + continue // NULL: comparison is UNKNOWN, row excluded either way + } + if r.Region != "EU" { + count++ + } + } + return count +} + +// dbSQLNullInWhereWant is derived by calling dbSQLCountRegionNotEU, not +// hardcoded. +// +// ground truth: region values are EU(id1,3,7), US(id2,5,8), APAC(id6), +// NULL(id4). "region <> 'EU'" is TRUE for the 4 US/APAC rows, FALSE for +// the 3 EU rows, and UNKNOWN (excluded) for the 1 NULL row - the trap +// being that a naive 8-3=5 count forgets NULL is excluded from BOTH sides +// of the comparison. databases_sqltuning_test.go recomputes this +// independently with a hand-written loop. +var dbSQLNullInWhereWant = dbSQLCountRegionNotEU(dbSQLOrders) + +// dbSQLNullInWhereTest: compute the row count of a WHERE clause negating a +// column that contains a NULL, testing whether the model accounts for +// SQL's three-valued logic (NULL never satisfies <>, so it is excluded +// from both the match and its negation). +func dbSQLNullInWhereTest() testkit.Test { + prompt := `Here is the "orders" table (region is NULL for order id 4): + +` + "```\n" + dbSQLOrdersTableText + "\n```" + ` + +What does "SELECT COUNT(*) FROM orders WHERE region <> 'EU';" return? +Remember SQL's three-valued logic: a comparison against NULL evaluates to +UNKNOWN, not TRUE or FALSE, so a NULL row satisfies neither +"region = 'EU'" nor "region <> 'EU'". Respond with only the number.` + + return testkit.Test{ + ID: "sql-null-in-where", + Category: "databases", + Subcategory: "sql-tuning", + Description: "Compute COUNT(*) WHERE region <> 'EU' over an inline table with one NULL region, testing the NULL-exclusion trap.", + Prompt: prompt, + Eval: eval.Numeric(eval.ExtractLastNumber[int], dbSQLNullInWhereWant, 0), + } +} + +// dbSQLRegionTotal is one row of a GROUP BY region result: the region name +// and its summed amount. +type dbSQLRegionTotal struct { + Region string + Total int +} + +// dbSQLGroupByRegionHaving computes SUM(amount) per non-NULL region, keeps +// only groups whose total exceeds minTotal, and returns them sorted by +// region ascending - the query-logic equivalent of "... WHERE region IS +// NOT NULL GROUP BY region HAVING SUM(amount) > minTotal ORDER BY region". +func dbSQLGroupByRegionHaving(rows []dbSQLOrder, minTotal int) []dbSQLRegionTotal { + totals := make(map[string]int) + var regions []string + for _, r := range rows { + if r.Region == "" { + continue + } + if _, seen := totals[r.Region]; !seen { + regions = append(regions, r.Region) + } + totals[r.Region] += r.Amount + } + sort.Strings(regions) + + var out []dbSQLRegionTotal + for _, region := range regions { + if totals[region] > minTotal { + out = append(out, dbSQLRegionTotal{Region: region, Total: totals[region]}) + } + } + return out +} + +// dbSQLGroupByHavingWant is derived by calling dbSQLGroupByRegionHaving, +// not hardcoded. +// +// ground truth: non-NULL region totals are EU = 250+75+125 = 450, US = +// 100+50+60 = 210, APAC = 400. HAVING SUM(amount) > 300 keeps EU (450) and +// APAC (400), drops US (210). Sorted by region ascending: APAC before EU. +// databases_sqltuning_test.go recomputes this independently with a +// hand-written map + sort. +var dbSQLGroupByHavingWant = dbSQLGroupByRegionHaving(dbSQLOrders, 300) + +// dbSQLGroupByHavingTest: compute a GROUP BY + HAVING result (region +// totals over 300) over the inline orders table, as an ordered JSON array. +func dbSQLGroupByHavingTest() testkit.Test { + prompt := `Here is the "orders" table: + +` + "```\n" + dbSQLOrdersTableText + "\n```" + ` + +What does this query return? + +SELECT region, SUM(amount) AS total +FROM orders +WHERE region IS NOT NULL +GROUP BY region +HAVING SUM(amount) > 300 +ORDER BY region; + +Respond with only a JSON array of objects, ordered exactly as the query's +ORDER BY would return them: [{"region":"...","total":}, ...]` + + evaluator := eval.Mean( + dbJSONArrayLength(len(dbSQLGroupByHavingWant)), + eval.JSONField("[0].region", dbSQLGroupByHavingWant[0].Region), + eval.JSONField("[0].total", dbSQLGroupByHavingWant[0].Total), + eval.JSONField("[1].region", dbSQLGroupByHavingWant[1].Region), + eval.JSONField("[1].total", dbSQLGroupByHavingWant[1].Total), + ) + + return testkit.Test{ + ID: "sql-groupby-having", + Category: "databases", + Subcategory: "sql-tuning", + Description: "Compute a GROUP BY region + HAVING SUM(amount) > 300 result (APAC, EU) over the inline orders table, as an ordered JSON array.", + Prompt: prompt, + Eval: evaluator, + } +} + +// dbSQLEmployee is one row of the shared inline "employees" table fixture +// used by dbSQLWindowRankTest. +type dbSQLEmployee struct { + Name string + Department string + Salary int +} + +// dbSQLEmployees is the shared inline "employees" table fixture, printed +// verbatim into dbSQLWindowRankTest's prompt. No department has a salary +// tie, so RANK() ... = 1 identifies exactly one employee per department. +var dbSQLEmployees = []dbSQLEmployee{ + {Name: "Alice", Department: "Eng", Salary: 140}, + {Name: "Bob", Department: "Eng", Salary: 120}, + {Name: "Carol", Department: "Sales", Salary: 110}, + {Name: "Dave", Department: "Sales", Salary: 130}, + {Name: "Erin", Department: "Eng", Salary: 150}, + {Name: "Frank", Department: "Sales", Salary: 90}, +} + +// dbSQLEmployeesTableText is dbSQLEmployees rendered as a plain-text table +// for the prompt. +const dbSQLEmployeesTableText = `name | department | salary +Alice | Eng | 140 +Bob | Eng | 120 +Carol | Sales | 110 +Dave | Sales | 130 +Erin | Eng | 150 +Frank | Sales | 90` + +// dbSQLTopEarnerByDepartment returns, for each department present in rows, +// the name of the employee with the highest salary in that department - +// the query-logic equivalent of "RANK() OVER (PARTITION BY department +// ORDER BY salary DESC) = 1", assuming no in-department salary tie. +func dbSQLTopEarnerByDepartment(rows []dbSQLEmployee) map[string]string { + best := make(map[string]dbSQLEmployee, len(rows)) + for _, e := range rows { + cur, ok := best[e.Department] + if !ok || e.Salary > cur.Salary { + best[e.Department] = e + } + } + out := make(map[string]string, len(best)) + for dept, e := range best { + out[dept] = e.Name + } + return out +} + +// dbSQLWindowRankWant is derived by calling dbSQLTopEarnerByDepartment, not +// hardcoded. +// +// ground truth: Eng salaries are Alice=140, Bob=120, Erin=150 - Erin is +// highest. Sales salaries are Carol=110, Dave=130, Frank=90 - Dave is +// highest. databases_sqltuning_test.go recomputes this independently with +// a hand-written loop. +var dbSQLWindowRankWant = dbSQLTopEarnerByDepartment(dbSQLEmployees) + +// dbSQLWindowRankTest: identify the rank-1 (highest-salary) employee per +// department from a RANK() OVER (PARTITION BY ...) window function applied +// to an inline employees table. +func dbSQLWindowRankTest() testkit.Test { + prompt := `Here is the "employees" table: + +` + "```\n" + dbSQLEmployeesTableText + "\n```" + ` + +For "SELECT department, name, RANK() OVER (PARTITION BY department ORDER +BY salary DESC) AS rnk FROM employees;", which employee's name has rnk = 1 +in each department? Respond with only a JSON object: +{"Eng":"","Sales":""}` + + evaluator := eval.Mean( + eval.JSONField("Eng", dbSQLWindowRankWant["Eng"]), + eval.JSONField("Sales", dbSQLWindowRankWant["Sales"]), + ) + + return testkit.Test{ + ID: "sql-window-rank", + Category: "databases", + Subcategory: "sql-tuning", + Description: "Identify the RANK()=1 (highest-salary) employee per department from an inline 6-row employees table.", + Prompt: prompt, + Eval: evaluator, + } +} + +// dbSQLCompositeIndexSchema is the inline schema + query for +// dbSQLCompositeIndexOrderTest. +const dbSQLCompositeIndexSchema = `Table: sessions(id bigserial, user_id bigint, created_at timestamptz, ip inet) + +Query: +SELECT * FROM sessions +WHERE user_id = $1 AND created_at > $2 +ORDER BY created_at DESC +LIMIT 50;` + +// dbSQLCompositeIndexOrderTest: pick the composite index column order +// (equality column first, then the range/sort column) that best serves an +// inline query. +// +// ground truth: user_id is filtered by exact equality; created_at is +// filtered by a range AND drives ORDER BY. Leading a composite index with +// the equality column narrows to one user's rows first; following with +// created_at then serves both the range filter and the sort from the +// index directly. Reversing the order (created_at, user_id) would force +// scanning every session in the time range across all users before +// filtering by user_id. +func dbSQLCompositeIndexOrderTest() testkit.Test { + prompt := `Here is a table schema and a query that runs frequently: + +` + "```\n" + dbSQLCompositeIndexSchema + "\n```" + ` + +Which two columns, in which order, should a single composite index be +defined on to best serve this exact query (equality filters should lead a +composite index, ahead of range/sort columns)? Respond with only a JSON +array of the two column names in index-definition order, e.g. ["a","b"].` + + return testkit.Test{ + ID: "sql-composite-index-order", + Category: "databases", + Subcategory: "sql-tuning", + Description: "Pick composite index column order (user_id, created_at) for an inline equality+range+sort query.", + Prompt: prompt, + Eval: eval.JSONStringArrayEquals([]string{"user_id", "created_at"}), + } +} + +// dbSQLKeysetPaginationTest: pick keyset (seek) pagination over OFFSET +// pagination for a deep-paging scenario over a huge table. +// +// ground truth: "LIMIT 100 OFFSET 20000000" forces the database to walk +// and discard 20,000,000 rows before returning the next 100, so its cost +// grows with how deep the user pages. Keyset/seek pagination +// ("WHERE id > :last_seen_id ORDER BY id LIMIT 100") instead jumps +// straight to the right position via an index seek, so its latency stays +// roughly constant no matter how deep the user has paged. +func dbSQLKeysetPaginationTest() testkit.Test { + prompt := `A table has 50,000,000 rows. An API paginates results ordered +by primary key id ascending, 100 rows per page. A user navigates deep into +the results - page 200,000, equivalent to OFFSET 20,000,000. + +Which pagination technique keeps query latency roughly constant regardless +of how deep the user pages: OFFSET-based pagination +("LIMIT 100 OFFSET 20000000"), or keyset/seek pagination +("WHERE id > :last_seen_id ORDER BY id LIMIT 100")? Respond with only one +word: offset or keyset.` + + return testkit.Test{ + ID: "sql-keyset-pagination", + Category: "databases", + Subcategory: "sql-tuning", + Description: "Pick keyset/seek pagination over OFFSET pagination for constant latency 20M rows deep into a 50M-row table.", + Prompt: prompt, + Eval: dbExactAnswer("keyset"), + } +} + +// dbSQLCoveringIndexTest: confirm that a composite index containing every +// column a query needs allows an index-only scan, with no heap visit. +// +// ground truth: the query only reads customer_id and status, and both are +// present in the index (customer_id, status) itself. Since every column +// the query needs is already inside the index entry, Postgres can answer +// the query by reading the index alone - an index-only scan - without +// visiting the underlying table (the heap) for any row, provided the +// visibility map allows it (as the prompt states). +func dbSQLCoveringIndexTest() testkit.Test { + prompt := `Query: SELECT customer_id, status FROM orders WHERE customer_id = 42; + +An index exists on orders(customer_id, status) - both columns this query +reads are present in the index itself. Assuming the visibility map allows +an index-only scan, can Postgres answer this query using only the index, +without visiting the underlying table (the heap) for any row? Respond with +only "yes" or "no".` + + return testkit.Test{ + ID: "sql-covering-index", + Category: "databases", + Subcategory: "sql-tuning", + Description: "Confirm an index containing every column a query needs (customer_id, status) enables an index-only scan with no heap visit.", + Prompt: prompt, + Eval: dbExactAnswer("yes"), + } +} + +// dbSQLEquivalentRewriteQueries is the inline original query and 3 +// rewrite candidates for dbSQLEquivalentRewriteTest. +const dbSQLEquivalentRewriteQueries = `Original: SELECT * FROM orders WHERE status = 'paid' OR status = 'pending'; + +Candidate A: SELECT * FROM orders WHERE status IN ('paid', 'pending'); +Candidate B: SELECT * FROM orders WHERE status = 'paid' AND status = 'pending'; +Candidate C: SELECT * FROM orders WHERE status != 'refunded';` + +// dbSQLEquivalentRewriteTest: judge which of 3 rewrite candidates are +// semantically equivalent to an inline original query, given a stated +// closed set of possible column values. +// +// ground truth: status only ever takes one of exactly 3 values in this +// table: 'paid', 'pending', 'refunded' (stated in the prompt so C's +// equivalence is well-defined). Candidate A: "status IN (x, y)" is +// standard SQL sugar for "status = x OR status = y" - always equivalent +// to the original. Candidate B: a single column cannot equal two +// different literal values in the same row, so "status = 'paid' AND +// status = 'pending'" is always FALSE - never equivalent to a query that +// is often TRUE. Candidate C: since status only ever takes one of exactly +// 3 values, "status != 'refunded'" is TRUE exactly when status is 'paid' +// or 'pending' - equivalent to the original given that closed value set +// (it would NOT be equivalent if a 4th status value existed). +func dbSQLEquivalentRewriteTest() testkit.Test { + prompt := `The "status" column in this table only ever holds one of +exactly 3 values: 'paid', 'pending', or 'refunded'. + +` + dbSQLEquivalentRewriteQueries + ` + +Which of Candidate A, B, C are semantically equivalent to the Original +query (return exactly the same rows for any data satisfying that 3-value +constraint)? Respond with only a JSON object: +{"A":true|false,"B":true|false,"C":true|false}` + + evaluator := eval.Mean( + eval.JSONField("A", true), + eval.JSONField("B", false), + eval.JSONField("C", true), + ) + + return testkit.Test{ + ID: "sql-equivalent-rewrite", + Category: "databases", + Subcategory: "sql-tuning", + Description: "Judge 3 rewrite candidates (IN-sugar equivalent, impossible-AND non-equivalent, closed-set-negation equivalent) against an inline original OR query.", + Prompt: prompt, + Eval: evaluator, + } +} + +// dbSQLOrderByIndexServingSchema is the inline index definition + query +// for dbSQLOrderByIndexServingTest. +const dbSQLOrderByIndexServingSchema = `Index: CREATE INDEX idx_orders_status_created_at ON orders(status, created_at DESC); + +Query: +SELECT * FROM orders +WHERE status = 'paid' +ORDER BY created_at DESC +LIMIT 20;` + +// dbSQLOrderByIndexServingTest: judge whether an inline composite index +// can serve both a query's WHERE filter and its ORDER BY without a +// separate sort step. +// +// ground truth: the index leads with status (an exact equality match in +// the query), and its second column, created_at DESC, exactly matches +// both the query's remaining filter role and its ORDER BY created_at DESC +// direction. Postgres can therefore walk the matching status='paid' +// portion of the index in exactly the order the query needs, serving the +// ORDER BY directly from the index with no separate sort step. +func dbSQLOrderByIndexServingTest() testkit.Test { + prompt := `Here is an index and a query: + +` + "```\n" + dbSQLOrderByIndexServingSchema + "\n```" + ` + +Can Postgres use this index to satisfy both the WHERE filter and the +ORDER BY without a separate sort step? Respond with only a JSON object: +{"avoids_sort_step":true|false,"reason":""}` + + evaluator := eval.Mean( + eval.JSONField("avoids_sort_step", true), + eval.JSONField("reason", "index-matches-filter-and-order"), + ) + + return testkit.Test{ + ID: "sql-orderby-index-serving", + Category: "databases", + Subcategory: "sql-tuning", + Description: "Judge that an index on (status, created_at DESC) serves both an equality WHERE and an ORDER BY created_at DESC with no extra sort step.", + Prompt: prompt, + Eval: evaluator, + } +} diff --git a/internal/tests/databases_sqltuning_test.go b/internal/tests/databases_sqltuning_test.go new file mode 100644 index 0000000..3e03ade --- /dev/null +++ b/internal/tests/databases_sqltuning_test.go @@ -0,0 +1,393 @@ +package tests + +import ( + "context" + "sort" + "testing" +) + +// TestDBSQLAggregateSumWant_GroundTruth independently re-derives the paid- +// orders total by listing the paid amounts by hand, not by calling +// dbSQLSumPaidAmount. +func TestDBSQLAggregateSumWant_GroundTruth(t *testing.T) { + paidAmounts := []int{250, 100, 300, 400, 125} // ids 1, 2, 4, 6, 7 + want := 0 + for _, a := range paidAmounts { + want += a + } + + if want != 1175 { + t.Fatalf("independently recomputed sum = %d, want 1175", want) + } + if dbSQLAggregateSumWant != want { + t.Errorf("dbSQLAggregateSumWant = %d, independently recomputed = %d", dbSQLAggregateSumWant, want) + } +} + +func TestDBSQLAggregateSumTest_Eval(t *testing.T) { + tc := dbSQLAggregateSumTest() + + tests := []struct { + name string + response string + want float64 + }{ + {"correct", "1175", 1}, + {"prose wrapped", "The sum is 1175.", 1}, + {"wrong: included a pending row", "1250", 0}, + {"wrong: excluded a paid row", "875", 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Value != tt.want { + t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail) + } + }) + } +} + +// TestDBSQLJoinCountsWant_GroundTruth independently re-derives both join +// counts from a hand-written per-customer_id tally, not by calling +// dbSQLInnerJoinCount/dbSQLLeftJoinCount. +func TestDBSQLJoinCountsWant_GroundTruth(t *testing.T) { + orderCustomerIDs := []int{101, 102, 101, 103, 102, 104, 101, 103} // ids 1..8 + knownCustomers := map[int]bool{101: true, 102: true, 105: true} + + innerWant := 0 + perCustomerOrders := map[int]int{} + for _, cid := range orderCustomerIDs { + perCustomerOrders[cid]++ + if knownCustomers[cid] { + innerWant++ + } + } + if innerWant != 5 { + t.Fatalf("independently recomputed inner join count = %d, want 5", innerWant) + } + + leftWant := 0 + for cid := range knownCustomers { + if perCustomerOrders[cid] == 0 { + leftWant++ // one NULL-order row + } else { + leftWant += perCustomerOrders[cid] + } + } + if leftWant != 6 { + t.Fatalf("independently recomputed left join count = %d, want 6", leftWant) + } + + if dbSQLInnerJoinCountWant != innerWant { + t.Errorf("dbSQLInnerJoinCountWant = %d, independently recomputed = %d", dbSQLInnerJoinCountWant, innerWant) + } + if dbSQLLeftJoinCountWant != leftWant { + t.Errorf("dbSQLLeftJoinCountWant = %d, independently recomputed = %d", dbSQLLeftJoinCountWant, leftWant) + } +} + +func TestDBSQLJoinCountsTest_Eval(t *testing.T) { + tc := dbSQLJoinCountsTest() + + tests := []struct { + name string + response string + want float64 + }{ + {"correct", `{"inner_join_count":5,"left_join_count":6}`, 1}, + {"correct fenced", "```json\n{\"inner_join_count\":5,\"left_join_count\":6}\n```", 1}, + {"wrong inner count", `{"inner_join_count":4,"left_join_count":6}`, 0.5}, + {"wrong left count: forgot the zero-match customer's NULL row", `{"inner_join_count":5,"left_join_count":5}`, 0.5}, + {"both wrong: treated inner and left as equal", `{"inner_join_count":8,"left_join_count":8}`, 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Value != tt.want { + t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail) + } + }) + } +} + +// TestDBSQLNullInWhereWant_GroundTruth independently re-derives the +// region <> 'EU' count by listing every row's region by hand, not by +// calling dbSQLCountRegionNotEU. +func TestDBSQLNullInWhereWant_GroundTruth(t *testing.T) { + regions := []string{"EU", "US", "EU", "", "US", "APAC", "EU", "US"} // ids 1..8, "" = NULL + want := 0 + for _, region := range regions { + if region == "" { + continue // NULL: <> 'EU' is UNKNOWN, excluded + } + if region != "EU" { + want++ + } + } + + if want != 4 { + t.Fatalf("independently recomputed count = %d, want 4", want) + } + if dbSQLNullInWhereWant != want { + t.Errorf("dbSQLNullInWhereWant = %d, independently recomputed = %d", dbSQLNullInWhereWant, want) + } +} + +func TestDBSQLNullInWhereTest_Eval(t *testing.T) { + tc := dbSQLNullInWhereTest() + + tests := []struct { + name string + response string + want float64 + }{ + {"correct", "4", 1}, + {"prose wrapped", "The count is 4.", 1}, + {"wrong: forgot NULL exclusion (8 - 3 EU rows)", "5", 0}, + {"wrong: counted the NULL row as non-EU", "5", 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Value != tt.want { + t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail) + } + }) + } +} + +// TestDBSQLGroupByHavingWant_GroundTruth independently re-derives the +// GROUP BY + HAVING result from hand-written region totals, not by calling +// dbSQLGroupByRegionHaving. +func TestDBSQLGroupByHavingWant_GroundTruth(t *testing.T) { + totals := map[string]int{ + "EU": 250 + 75 + 125, + "US": 100 + 50 + 60, + "APAC": 400, + } + if totals["EU"] != 450 || totals["US"] != 210 || totals["APAC"] != 400 { + t.Fatalf("independently recomputed totals = %v, want EU=450 US=210 APAC=400", totals) + } + + var kept []string + for region, total := range totals { + if total > 300 { + kept = append(kept, region) + } + } + sort.Strings(kept) + + if len(kept) != 2 || kept[0] != "APAC" || kept[1] != "EU" { + t.Fatalf("independently recomputed kept regions = %v, want [APAC EU]", kept) + } + + if len(dbSQLGroupByHavingWant) != 2 { + t.Fatalf("dbSQLGroupByHavingWant has %d entries, want 2", len(dbSQLGroupByHavingWant)) + } + if dbSQLGroupByHavingWant[0].Region != "APAC" || dbSQLGroupByHavingWant[0].Total != 400 { + t.Errorf("dbSQLGroupByHavingWant[0] = %+v, want {APAC 400}", dbSQLGroupByHavingWant[0]) + } + if dbSQLGroupByHavingWant[1].Region != "EU" || dbSQLGroupByHavingWant[1].Total != 450 { + t.Errorf("dbSQLGroupByHavingWant[1] = %+v, want {EU 450}", dbSQLGroupByHavingWant[1]) + } +} + +func TestDBSQLGroupByHavingTest_Eval(t *testing.T) { + tc := dbSQLGroupByHavingTest() + + tests := []struct { + name string + response string + want float64 + }{ + {"correct", `[{"region":"APAC","total":400},{"region":"EU","total":450}]`, 1}, + {"correct fenced with prose", "Result:\n```json\n[{\"region\":\"APAC\",\"total\":400},{\"region\":\"EU\",\"total\":450}]\n```", 1}, + {"wrong order", `[{"region":"EU","total":450},{"region":"APAC","total":400}]`, 0.2}, + {"missing a group", `[{"region":"APAC","total":400}]`, 0.4}, + {"wrong total for one group", `[{"region":"APAC","total":401},{"region":"EU","total":450}]`, 0.8}, + {"forgot HAVING filter, includes US", `[{"region":"APAC","total":400},{"region":"EU","total":450},{"region":"US","total":210}]`, 0.8}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Value != tt.want { + t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail) + } + }) + } +} + +// TestDBSQLWindowRankWant_GroundTruth independently re-derives the top +// earner per department from hand-written salary lists, not by calling +// dbSQLTopEarnerByDepartment. +func TestDBSQLWindowRankWant_GroundTruth(t *testing.T) { + eng := map[string]int{"Alice": 140, "Bob": 120, "Erin": 150} + sales := map[string]int{"Carol": 110, "Dave": 130, "Frank": 90} + + topOf := func(salaries map[string]int) string { + best, bestSalary := "", -1 + for name, salary := range salaries { + if salary > bestSalary { + best, bestSalary = name, salary + } + } + return best + } + + if got := topOf(eng); got != "Erin" { + t.Fatalf("independently recomputed Eng top earner = %q, want Erin", got) + } + if got := topOf(sales); got != "Dave" { + t.Fatalf("independently recomputed Sales top earner = %q, want Dave", got) + } + + if dbSQLWindowRankWant["Eng"] != "Erin" { + t.Errorf(`dbSQLWindowRankWant["Eng"] = %q, want "Erin"`, dbSQLWindowRankWant["Eng"]) + } + if dbSQLWindowRankWant["Sales"] != "Dave" { + t.Errorf(`dbSQLWindowRankWant["Sales"] = %q, want "Dave"`, dbSQLWindowRankWant["Sales"]) + } +} + +func TestDBSQLWindowRankTest_Eval(t *testing.T) { + tc := dbSQLWindowRankTest() + + tests := []struct { + name string + response string + want float64 + }{ + {"correct", `{"Eng":"Erin","Sales":"Dave"}`, 1}, + {"correct fenced", "```json\n{\"Eng\":\"Erin\",\"Sales\":\"Dave\"}\n```", 1}, + {"wrong Eng", `{"Eng":"Alice","Sales":"Dave"}`, 0.5}, + {"wrong Sales", `{"Eng":"Erin","Sales":"Carol"}`, 0.5}, + {"both wrong", `{"Eng":"Bob","Sales":"Frank"}`, 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Value != tt.want { + t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail) + } + }) + } +} + +func TestDBSQLCompositeIndexOrderTest_Eval(t *testing.T) { + tc := dbSQLCompositeIndexOrderTest() + + tests := []struct { + name string + response string + want float64 + }{ + {"correct", `["user_id", "created_at"]`, 1}, + {"correct fenced", "```json\n[\"user_id\", \"created_at\"]\n```", 1}, + {"reversed order", `["created_at", "user_id"]`, 0}, + {"missing a column", `["user_id"]`, 0}, + {"wrong column entirely", `["user_id", "ip"]`, 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Value != tt.want { + t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail) + } + }) + } +} + +func TestDBSQLKeysetPaginationTest_Eval(t *testing.T) { + tc := dbSQLKeysetPaginationTest() + + tests := []struct { + name string + response string + want float64 + }{ + {"bare correct", "keyset", 1}, + {"quoted correct", `"keyset"`, 1}, + {"different case", "Keyset", 1}, + {"trailing period", "keyset.", 1}, + {"wrong", "offset", 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Value != tt.want { + t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail) + } + }) + } +} + +func TestDBSQLCoveringIndexTest_Eval(t *testing.T) { + tc := dbSQLCoveringIndexTest() + + tests := []struct { + name string + response string + want float64 + }{ + {"bare correct", "yes", 1}, + {"quoted correct", `"yes"`, 1}, + {"trailing period", "Yes.", 1}, + {"wrong", "no", 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Value != tt.want { + t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail) + } + }) + } +} + +func TestDBSQLEquivalentRewriteTest_Eval(t *testing.T) { + tc := dbSQLEquivalentRewriteTest() + + tests := []struct { + name string + response string + want float64 + }{ + {"correct", `{"A":true,"B":false,"C":true}`, 1}, + {"correct fenced with prose", "Here's my analysis:\n```json\n{\"A\":true,\"B\":false,\"C\":true}\n```", 1}, + {"wrongly accepts B (impossible AND)", `{"A":true,"B":true,"C":true}`, 2.0 / 3.0}, + {"wrongly rejects C (forgets closed value set)", `{"A":true,"B":false,"C":false}`, 2.0 / 3.0}, + {"everything wrong", `{"A":false,"B":true,"C":false}`, 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Value != tt.want { + t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail) + } + }) + } +} + +func TestDBSQLOrderByIndexServingTest_Eval(t *testing.T) { + tc := dbSQLOrderByIndexServingTest() + + tests := []struct { + name string + response string + want float64 + }{ + {"correct", `{"avoids_sort_step":true,"reason":"index-matches-filter-and-order"}`, 1}, + {"correct fenced", "```json\n{\"avoids_sort_step\":true,\"reason\":\"index-matches-filter-and-order\"}\n```", 1}, + {"wrongly says a sort step is needed", `{"avoids_sort_step":false,"reason":"index-matches-filter-and-order"}`, 0.5}, + {"wrong reason", `{"avoids_sort_step":true,"reason":"order-by-column-not-leading"}`, 0.5}, + {"both wrong", `{"avoids_sort_step":false,"reason":"sort-direction-mismatch"}`, 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Value != tt.want { + t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail) + } + }) + } +} diff --git a/internal/tests/databases_test.go b/internal/tests/databases_test.go new file mode 100644 index 0000000..f5c22a5 --- /dev/null +++ b/internal/tests/databases_test.go @@ -0,0 +1,58 @@ +package tests + +import ( + "strings" + "testing" + + "github.com/lukaszraczylo/llm-testbench/internal/testkit" +) + +// TestRegisterDatabasesTests_Wiring checks registerDatabasesTests wires up +// all three databases subcategories - postgres, redis, sql-tuning - with 10 +// tests each, all categorized "databases", all with the expected ID prefix +// per subcategory, MaxTokens omitted (0, per the round-2 authoring rule), +// and no duplicate IDs across the three sibling files (testkit.Registry. +// Register panics on a duplicate ID, which would fail this test outright). +func TestRegisterDatabasesTests_Wiring(t *testing.T) { + r := testkit.NewRegistry() + registerDatabasesTests(r) + + const wantTotal = 30 + if r.Len() != wantTotal { + t.Fatalf("registerDatabasesTests: registry has %d tests, want %d", r.Len(), wantTotal) + } + + wantPrefix := map[string]string{ + "postgres": "pg-", + "redis": "redis-", + "sql-tuning": "sql-", + } + + counts := map[string]int{} + for _, tc := range r.All() { + if tc.Category != "databases" { + t.Errorf("test %q: Category = %q, want %q", tc.ID, tc.Category, "databases") + } + if tc.Prompt == "" { + t.Errorf("test %q: Prompt is empty", tc.ID) + } + if tc.Eval == nil { + t.Errorf("test %q: Eval is nil", tc.ID) + } + if tc.Description == "" { + t.Errorf("test %q: Description is empty", tc.ID) + } + if tc.MaxTokens != 0 { + t.Errorf("test %q: MaxTokens = %d, want 0 (omitted)", tc.ID, tc.MaxTokens) + } + if prefix, ok := wantPrefix[tc.Subcategory]; ok && !strings.HasPrefix(tc.ID, prefix) { + t.Errorf("test %q: subcategory %q wants ID prefix %q", tc.ID, tc.Subcategory, prefix) + } + counts[tc.Subcategory]++ + } + for _, sub := range []string{"postgres", "redis", "sql-tuning"} { + if counts[sub] != 10 { + t.Errorf("subcategory %q: got %d tests, want 10", sub, counts[sub]) + } + } +}