mirror of
https://github.com/lukaszraczylo/llm-testbench.git
synced 2026-10-12 00:45:19 +00:00
Merge branch 'worktree-agent-a0b0f5406dcb45aa0'
This commit is contained in:
@@ -0,0 +1,69 @@
|
||||
package tests
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/lukaszraczylo/llm-testbench/internal/eval"
|
||||
"github.com/lukaszraczylo/llm-testbench/internal/testkit"
|
||||
)
|
||||
|
||||
// registerDatabasesTests registers every databases-category test. Each
|
||||
// subcategory - postgres, redis, sql-tuning - has its own register function
|
||||
// and source file (databases_postgres.go, databases_redis.go,
|
||||
// databases_sqltuning.go) to keep any one file from growing past a few
|
||||
// hundred lines. The orchestrator wires this into the catalog's All()
|
||||
// separately; this package intentionally does not register itself in
|
||||
// catalog.go so parallel category worktrees stay independently buildable.
|
||||
func registerDatabasesTests(r *testkit.Registry) {
|
||||
registerDBPostgresTests(r)
|
||||
registerDBRedisTests(r)
|
||||
registerDBSQLTuningTests(r)
|
||||
}
|
||||
|
||||
// dbExactAnswer returns an Evaluator awarding full credit when the
|
||||
// response, trimmed of whitespace, at most one layer of surrounding quote
|
||||
// characters (' , ", or `), and a single trailing sentence-ending period,
|
||||
// equals want case-insensitively. Used across the databases category for
|
||||
// prompts that force a single short forced-vocabulary answer (e.g. "yes"
|
||||
// or "no", "keyset" or "offset"): it accepts every materially-correct form
|
||||
// of that answer (bare, quoted, differently-cased, or with trailing
|
||||
// punctuation) without loosening the match to accept a wrong answer.
|
||||
func dbExactAnswer(want string) eval.Evaluator {
|
||||
return eval.EvaluatorFunc(func(_ context.Context, response string) eval.Score {
|
||||
got := strings.TrimSpace(response)
|
||||
got = strings.Trim(got, "\"'`")
|
||||
got = strings.TrimSuffix(strings.TrimSpace(got), ".")
|
||||
got = strings.TrimSpace(got)
|
||||
wantTrimmed := strings.TrimSpace(want)
|
||||
if strings.EqualFold(got, wantTrimmed) {
|
||||
return eval.Score{Value: 1, Detail: fmt.Sprintf("equals %q", wantTrimmed)}
|
||||
}
|
||||
return eval.Score{Value: 0, Detail: fmt.Sprintf("got %q, want %q", got, wantTrimmed)}
|
||||
})
|
||||
}
|
||||
|
||||
// dbJSONArrayLength returns an Evaluator awarding full credit when the
|
||||
// response's extracted JSON array (see eval.ExtractJSON) has exactly want
|
||||
// elements, zero otherwise. Paired with per-index eval.JSONField checks so
|
||||
// an answer with extra or missing elements (e.g. forgetting to apply a
|
||||
// HAVING filter, so an extra group leaks through) cannot score full credit
|
||||
// merely by getting the checked indices right.
|
||||
func dbJSONArrayLength(want int) eval.Evaluator {
|
||||
return eval.EvaluatorFunc(func(_ context.Context, response string) eval.Score {
|
||||
raw, err := eval.ExtractJSON(response)
|
||||
if err != nil {
|
||||
return eval.Score{Value: 0, Detail: err.Error()}
|
||||
}
|
||||
var arr []json.RawMessage
|
||||
if err := json.Unmarshal([]byte(raw), &arr); err != nil {
|
||||
return eval.Score{Value: 0, Detail: fmt.Sprintf("invalid JSON array: %v", err)}
|
||||
}
|
||||
if len(arr) == want {
|
||||
return eval.Score{Value: 1, Detail: fmt.Sprintf("array length %d", len(arr))}
|
||||
}
|
||||
return eval.Score{Value: 0, Detail: fmt.Sprintf("array length %d, want %d", len(arr), want)}
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,412 @@
|
||||
package tests
|
||||
|
||||
import (
|
||||
"github.com/lukaszraczylo/llm-testbench/internal/eval"
|
||||
"github.com/lukaszraczylo/llm-testbench/internal/testkit"
|
||||
)
|
||||
|
||||
// registerDBPostgresTests registers every databases/postgres test.
|
||||
func registerDBPostgresTests(r *testkit.Registry) {
|
||||
r.Register(dbPGExplainSeqVsIndexTest())
|
||||
r.Register(dbPGIndexChoiceTest())
|
||||
r.Register(dbPGNPlusOneTest())
|
||||
r.Register(dbPGIsolationAnomalyTest())
|
||||
r.Register(dbPGDeadlockLockOrderTest())
|
||||
r.Register(dbPGReplicaFailoverTest())
|
||||
r.Register(dbPGBloatVacuumTest())
|
||||
r.Register(dbPGPoolSizingTest())
|
||||
r.Register(dbPGPartialIndexTest())
|
||||
r.Register(dbPGListenNotifyTest())
|
||||
}
|
||||
|
||||
// dbPGExplainSeqVsIndexPlan is the inline EXPLAIN excerpt for
|
||||
// dbPGExplainSeqVsIndexTest.
|
||||
const dbPGExplainSeqVsIndexPlan = `Seq Scan on events (cost=0.00..215000.00 rows=4200000 width=8)
|
||||
Filter: (status = 'active'::text)`
|
||||
|
||||
// dbPGExplainSeqVsIndexTest: interpret why the planner chose a sequential
|
||||
// scan over an available index, from an inline EXPLAIN plan and the table's
|
||||
// row counts.
|
||||
//
|
||||
// ground truth: the events table holds 10,000,000 rows and the plan
|
||||
// estimates 4,200,000 of them match status='active' - 42% of the table.
|
||||
// Above roughly 5-10% selectivity, a sequential scan reading every page
|
||||
// once beats an index scan that would revisit the heap almost as many
|
||||
// times as a seq scan touches pages, so a low-selectivity filter (not a
|
||||
// missing index - one exists - and not stale statistics) is why the
|
||||
// planner picked Seq Scan. At a selectivity of 8,000/10,000,000 (0.08%),
|
||||
// the same index would clearly be preferred instead.
|
||||
func dbPGExplainSeqVsIndexTest() testkit.Test {
|
||||
prompt := `The "events" table holds 10,000,000 rows and has a btree index
|
||||
on its "status" column. Here is the EXPLAIN output for
|
||||
"SELECT id FROM events WHERE status = 'active';":
|
||||
|
||||
` + "```\n" + dbPGExplainSeqVsIndexPlan + "\n```" + `
|
||||
|
||||
The planner used a sequential scan even though an index on status exists.
|
||||
Given the estimated row count in the plan versus the table's total row
|
||||
count, what row-count property of the filter caused this choice? Also: if
|
||||
instead only 8,000 of the 10,000,000 rows had status='rare' (a highly
|
||||
selective filter), would the planner likely prefer the index scan instead?
|
||||
Respond with only a JSON object:
|
||||
{"reason":"<one of: low-selectivity, missing-index, stale-statistics>","would_index_help_at_rare_status":true|false}`
|
||||
|
||||
evaluator := eval.Mean(
|
||||
eval.JSONField("reason", "low-selectivity"),
|
||||
eval.JSONField("would_index_help_at_rare_status", true),
|
||||
)
|
||||
|
||||
return testkit.Test{
|
||||
ID: "pg-explain-seq-vs-index",
|
||||
Category: "databases",
|
||||
Subcategory: "postgres",
|
||||
Description: "Interpret an EXPLAIN plan's Seq Scan choice as a low-selectivity filter (42% of the table), not a missing index.",
|
||||
Prompt: prompt,
|
||||
Eval: evaluator,
|
||||
}
|
||||
}
|
||||
|
||||
// dbPGIndexChoiceSchema is the inline schema + query for
|
||||
// dbPGIndexChoiceTest.
|
||||
const dbPGIndexChoiceSchema = `Table: orders(id bigserial, region text, order_date date, amount numeric)
|
||||
|
||||
Query:
|
||||
SELECT id, amount FROM orders
|
||||
WHERE region = 'EU' AND order_date >= '2026-01-01'
|
||||
ORDER BY order_date;`
|
||||
|
||||
// dbPGIndexChoiceTest: pick the composite index column order (equality
|
||||
// column first, then the range/sort column) that best serves an inline
|
||||
// query.
|
||||
//
|
||||
// ground truth: region is filtered by exact equality, order_date is
|
||||
// filtered by a range AND drives the ORDER BY. A composite index leads
|
||||
// with the equality column so the index narrows to matching rows first,
|
||||
// then with the range/sort column so those rows are already in the index's
|
||||
// order needed by ORDER BY: (region, order_date). Reversing the order
|
||||
// (order_date, region) forces a full scan of every order_date >= the bound
|
||||
// across all regions before filtering by region.
|
||||
func dbPGIndexChoiceTest() testkit.Test {
|
||||
prompt := `Here is a table schema and a query that runs frequently:
|
||||
|
||||
` + "```\n" + dbPGIndexChoiceSchema + "\n```" + `
|
||||
|
||||
Which two columns, in which order, should a single composite index be
|
||||
defined on to best serve this exact query (equality filters should lead a
|
||||
composite index, ahead of range/sort columns)? Respond with only a JSON
|
||||
array of the two column names in index-definition order, e.g. ["a","b"].`
|
||||
|
||||
return testkit.Test{
|
||||
ID: "pg-index-choice",
|
||||
Category: "databases",
|
||||
Subcategory: "postgres",
|
||||
Description: "Pick composite index column order (region, order_date) for an inline equality+range+sort query.",
|
||||
Prompt: prompt,
|
||||
Eval: eval.JSONStringArrayEquals([]string{"region", "order_date"}),
|
||||
}
|
||||
}
|
||||
|
||||
// dbPGNPlusOneCode is the inline query-in-a-loop snippet for
|
||||
// dbPGNPlusOneTest.
|
||||
const dbPGNPlusOneCode = `orders = db.query("SELECT id, customer_id FROM orders WHERE status = 'pending'")
|
||||
for o in orders:
|
||||
customer = db.query("SELECT name FROM customers WHERE id = %s", [o.customer_id])
|
||||
print(o.id, customer.name)`
|
||||
|
||||
// dbPGNPlusOneTest: spot the N+1 query pattern in an inline loop and name
|
||||
// its single-query fix.
|
||||
//
|
||||
// ground truth: the code runs one query to fetch pending orders, then one
|
||||
// additional query per order to fetch that order's customer name inside
|
||||
// the loop - the classic N+1 pattern, not a Cartesian product (which would
|
||||
// return duplicated rows from a single query) or a missing index (which
|
||||
// would not change the query count at all). The fix is to replace the
|
||||
// per-row queries with a single JOIN (or a single WHERE id = ANY(...)
|
||||
// batch query) so exactly one query fetches every needed customer name.
|
||||
func dbPGNPlusOneTest() testkit.Test {
|
||||
prompt := `Here is application code that fetches pending orders and their
|
||||
customer names:
|
||||
|
||||
` + "```python\n" + dbPGNPlusOneCode + "\n```" + `
|
||||
|
||||
What query pattern is this, and what single-query fix addresses it?
|
||||
Respond with only a JSON object:
|
||||
{"pattern":"<one of: n-plus-one, missing-index, cartesian-product>","fix":"<one of: single-join-query, add-cache, add-index>"}`
|
||||
|
||||
evaluator := eval.Mean(
|
||||
eval.JSONField("pattern", "n-plus-one"),
|
||||
eval.JSONField("fix", "single-join-query"),
|
||||
)
|
||||
|
||||
return testkit.Test{
|
||||
ID: "pg-n-plus-one",
|
||||
Category: "databases",
|
||||
Subcategory: "postgres",
|
||||
Description: "Spot the N+1 query pattern in an inline per-row query loop and require a single-JOIN fix.",
|
||||
Prompt: prompt,
|
||||
Eval: evaluator,
|
||||
}
|
||||
}
|
||||
|
||||
// dbPGIsolationAnomalySchedule is the inline two-transaction interleaved
|
||||
// schedule for dbPGIsolationAnomalyTest.
|
||||
const dbPGIsolationAnomalySchedule = `T1: BEGIN;
|
||||
T1: SELECT balance FROM accounts WHERE id = 1; -- returns 100
|
||||
T2: BEGIN;
|
||||
T2: UPDATE accounts SET balance = 50 WHERE id = 1;
|
||||
T2: COMMIT;
|
||||
T1: SELECT balance FROM accounts WHERE id = 1; -- returns 50
|
||||
T1: COMMIT;`
|
||||
|
||||
// dbPGIsolationAnomalyTest: name the isolation anomaly a READ COMMITTED
|
||||
// transaction observes when its own two reads of the same row disagree
|
||||
// because another transaction committed in between.
|
||||
//
|
||||
// ground truth: T1 reads balance=100, then, after T2 commits a change to
|
||||
// the same row, T1's second read of the exact same row within the SAME
|
||||
// transaction returns 50 - two reads of one row disagreeing inside one
|
||||
// transaction is the definition of a non-repeatable read, not a dirty read
|
||||
// (which would require reading T2's UPDATE before T2 committed) or a
|
||||
// phantom read (which is about a changed set of rows matching a
|
||||
// predicate, not one row's value).
|
||||
func dbPGIsolationAnomalyTest() testkit.Test {
|
||||
prompt := `T1 runs under READ COMMITTED isolation. Here is an interleaved
|
||||
schedule of T1 and a concurrent T2:
|
||||
|
||||
` + "```\n" + dbPGIsolationAnomalySchedule + "\n```" + `
|
||||
|
||||
Which isolation anomaly does T1 observe between its two SELECTs? Respond
|
||||
with only a JSON object:
|
||||
{"anomaly":"<one of: dirty-read, non-repeatable-read, phantom-read, lost-update>"}`
|
||||
|
||||
return testkit.Test{
|
||||
ID: "pg-isolation-anomaly",
|
||||
Category: "databases",
|
||||
Subcategory: "postgres",
|
||||
Description: "Name the non-repeatable-read anomaly from an inline READ COMMITTED schedule where a row's value changes between two reads in one transaction.",
|
||||
Prompt: prompt,
|
||||
Eval: eval.JSONField("anomaly", "non-repeatable-read"),
|
||||
}
|
||||
}
|
||||
|
||||
// dbPGDeadlockSchedule is the inline two-transaction opposite-lock-order
|
||||
// schedule for dbPGDeadlockLockOrderTest.
|
||||
const dbPGDeadlockSchedule = `T1: BEGIN;
|
||||
T1: UPDATE accounts SET balance = balance - 100 WHERE id = 1; -- T1 locks id=1
|
||||
T1: UPDATE accounts SET balance = balance + 100 WHERE id = 2; -- T1 waits: T2 holds id=2
|
||||
|
||||
T2: BEGIN;
|
||||
T2: UPDATE accounts SET balance = balance - 50 WHERE id = 2; -- T2 locks id=2
|
||||
T2: UPDATE accounts SET balance = balance + 50 WHERE id = 1; -- T2 waits: T1 holds id=1`
|
||||
|
||||
// dbPGDeadlockLockOrderTest: recognize a deadlock caused by two
|
||||
// transactions locking the same two rows in opposite order, and give the
|
||||
// consistent lock-acquisition order that prevents it.
|
||||
//
|
||||
// ground truth: T1 locks id=1 then waits on id=2 (held by T2); T2 locks
|
||||
// id=2 then waits on id=1 (held by T1) - a circular wait, so Postgres'
|
||||
// deadlock detector must abort one transaction. Both transactions always
|
||||
// touching rows in the same order - ascending id, so id=1 before id=2 -
|
||||
// eliminates the circular wait, since neither transaction would ever hold
|
||||
// a higher id's lock while waiting on a lower one.
|
||||
func dbPGDeadlockLockOrderTest() testkit.Test {
|
||||
prompt := `Here are two concurrent transactions on the same accounts table:
|
||||
|
||||
` + "```\n" + dbPGDeadlockSchedule + "\n```" + `
|
||||
|
||||
Do these two transactions deadlock? If so, what consistent row lock
|
||||
acquisition order (by id, ascending) would both transactions need to follow
|
||||
to prevent this deadlock? Respond with only a JSON object:
|
||||
{"deadlock":true|false,"safe_order_ids":[<first id to lock>,<second id to lock>]}`
|
||||
|
||||
evaluator := eval.Mean(
|
||||
eval.JSONField("deadlock", true),
|
||||
eval.JSONField("safe_order_ids[0]", 1),
|
||||
eval.JSONField("safe_order_ids[1]", 2),
|
||||
)
|
||||
|
||||
return testkit.Test{
|
||||
ID: "pg-deadlock-lock-order",
|
||||
Category: "databases",
|
||||
Subcategory: "postgres",
|
||||
Description: "Recognize a deadlock from two transactions locking rows id=1/id=2 in opposite order and require the ascending-id safe order.",
|
||||
Prompt: prompt,
|
||||
Eval: evaluator,
|
||||
}
|
||||
}
|
||||
|
||||
// dbPGReplicaFailoverTest: explain that an asynchronously-replicated commit
|
||||
// can be lost when the primary crashes before the replica catches up and
|
||||
// gets promoted.
|
||||
//
|
||||
// ground truth: with asynchronous streaming replication (no
|
||||
// synchronous_commit), a transaction's COMMIT returns to the client as
|
||||
// soon as it is durable on the primary, before its WAL is guaranteed to
|
||||
// have reached the replica. If the primary crashes in that window and the
|
||||
// replica is promoted, the promoted replica becomes the new primary
|
||||
// without ever having applied that transaction's WAL - so the committed
|
||||
// transaction is not visible on it and is effectively lost, unless it can
|
||||
// be recovered from the old primary's WAL after the fact.
|
||||
func dbPGReplicaFailoverTest() testkit.Test {
|
||||
prompt := `A Postgres primary has one streaming replica, using
|
||||
asynchronous replication (synchronous_commit is not enabled). A client's
|
||||
transaction COMMITs successfully on the primary. Before the WAL for that
|
||||
commit reaches the replica, the primary host crashes, and the replica is
|
||||
promoted to become the new primary.
|
||||
|
||||
Is that committed transaction guaranteed to be visible on the newly
|
||||
promoted primary? Explain what happens to it and why, in one or two
|
||||
sentences.`
|
||||
|
||||
evaluator := eval.All(
|
||||
eval.W(eval.ContainsAny("lost", "not visible", "not guaranteed", "may be lost", "can be lost", "data loss"), 2),
|
||||
eval.W(eval.ContainsAny("async", "asynchronous"), 1),
|
||||
)
|
||||
|
||||
return testkit.Test{
|
||||
ID: "pg-replica-failover",
|
||||
Category: "databases",
|
||||
Subcategory: "postgres",
|
||||
Description: "Explain that an asynchronously-replicated commit can be lost on failover if the primary crashes before the replica catches up.",
|
||||
Prompt: prompt,
|
||||
Eval: evaluator,
|
||||
}
|
||||
}
|
||||
|
||||
// dbPGBloatVacuumStats is the inline pg_stat_user_tables excerpt for
|
||||
// dbPGBloatVacuumTest.
|
||||
const dbPGBloatVacuumStats = `relname | n_live_tup | n_dead_tup | last_autovacuum
|
||||
orders | 500000 | 4500000 | (null)`
|
||||
|
||||
// dbPGBloatVacuumTest: diagnose table bloat from an inline pg_stat_
|
||||
// user_tables excerpt showing dead tuples vastly outnumbering live ones and
|
||||
// autovacuum never having run.
|
||||
//
|
||||
// ground truth: n_dead_tup (4,500,000) is 9x n_live_tup (500,000) - about
|
||||
// 90% of the table's physical rows are dead - and last_autovacuum is null,
|
||||
// meaning autovacuum has never completed on this table. That combination
|
||||
// is table bloat from accumulated dead tuples, not a missing index (which
|
||||
// this fixture says nothing about) or replication lag (unrelated to
|
||||
// tuple counts). The fix is to VACUUM the table (reclaiming/reusing the
|
||||
// dead tuple space), not to reindex or add an index.
|
||||
func dbPGBloatVacuumTest() testkit.Test {
|
||||
prompt := `Here is an excerpt of pg_stat_user_tables for one table:
|
||||
|
||||
` + "```\n" + dbPGBloatVacuumStats + "\n```" + `
|
||||
|
||||
What is the likely cause of this table's condition, and what single
|
||||
maintenance action addresses it? Respond with only a JSON object:
|
||||
{"cause":"<one of: bloat, missing-index, replication-lag>","action":"<one of: vacuum, reindex, add-index>"}`
|
||||
|
||||
evaluator := eval.Mean(
|
||||
eval.JSONField("cause", "bloat"),
|
||||
eval.JSONField("action", "vacuum"),
|
||||
)
|
||||
|
||||
return testkit.Test{
|
||||
ID: "pg-bloat-vacuum",
|
||||
Category: "databases",
|
||||
Subcategory: "postgres",
|
||||
Description: "Diagnose table bloat (dead tuples 9x live, autovacuum never ran) from an inline pg_stat_user_tables excerpt and require the vacuum fix.",
|
||||
Prompt: prompt,
|
||||
Eval: evaluator,
|
||||
}
|
||||
}
|
||||
|
||||
// dbPGPoolSizingWant is the pool_size computed by the formula given inline
|
||||
// in dbPGPoolSizingTest's prompt.
|
||||
//
|
||||
// ground truth: pool_size = (core_count * 2) + effective_spindle_count =
|
||||
// (8 * 2) + 1 = 17. databases_postgres_test.go recomputes this
|
||||
// independently with the same arithmetic written out plainly.
|
||||
const dbPGPoolSizingWant = 8*2 + 1
|
||||
|
||||
// dbPGPoolSizingTest: compute a connection pool size from a formula and
|
||||
// parameters both given inline in the prompt.
|
||||
func dbPGPoolSizingTest() testkit.Test {
|
||||
prompt := `Use this connection pool sizing formula:
|
||||
|
||||
pool_size = (core_count * 2) + effective_spindle_count
|
||||
|
||||
A database server has core_count = 8. Its storage is a single NVMe SSD,
|
||||
which this formula treats as effective_spindle_count = 1. Compute
|
||||
pool_size. Respond with only the number.`
|
||||
|
||||
return testkit.Test{
|
||||
ID: "pg-pool-sizing",
|
||||
Category: "databases",
|
||||
Subcategory: "postgres",
|
||||
Description: "Compute a connection pool size from an inline formula (core_count*2 + effective_spindle_count) and given parameters.",
|
||||
Prompt: prompt,
|
||||
Eval: eval.Numeric(eval.ExtractLastNumber[int], dbPGPoolSizingWant, 0),
|
||||
}
|
||||
}
|
||||
|
||||
// dbPGPartialIndexTest: judge whether a partial index is a good fit for an
|
||||
// inline query pattern where the filtered value is a small, hot subset of
|
||||
// a huge table.
|
||||
//
|
||||
// ground truth: only 120,000 of 10,000,000 rows (1.2%) have
|
||||
// status='pending', and the application's hottest query filters on exactly
|
||||
// that value almost every time it queries this table. A partial index
|
||||
// `ON orders(...) WHERE status = 'pending'` indexes only that small,
|
||||
// frequently-queried subset - far smaller and cheaper to maintain than a
|
||||
// full index across all 10,000,000 rows, and it exactly matches the query
|
||||
// pattern that needs it.
|
||||
func dbPGPartialIndexTest() testkit.Test {
|
||||
prompt := `The "orders" table holds 10,000,000 rows. Exactly 120,000 of
|
||||
them have status='pending'. The application's single hottest query against
|
||||
this table is "SELECT * FROM orders WHERE status = 'pending';", run almost
|
||||
every time this table is queried at all.
|
||||
|
||||
Is a partial index (CREATE INDEX ... ON orders(...) WHERE status =
|
||||
'pending') a good fit here, and why? Respond with only a JSON object:
|
||||
{"partial_index_applicable":true|false,"reason":"<one of: small-frequently-queried-subset, subset-too-large, query-does-not-filter-on-column>"}`
|
||||
|
||||
evaluator := eval.Mean(
|
||||
eval.JSONField("partial_index_applicable", true),
|
||||
eval.JSONField("reason", "small-frequently-queried-subset"),
|
||||
)
|
||||
|
||||
return testkit.Test{
|
||||
ID: "pg-partial-index",
|
||||
Category: "databases",
|
||||
Subcategory: "postgres",
|
||||
Description: "Judge a partial index as a good fit for a 1.2%-selectivity, frequently-queried status value on a 10M-row table.",
|
||||
Prompt: prompt,
|
||||
Eval: evaluator,
|
||||
}
|
||||
}
|
||||
|
||||
// dbPGListenNotifyTest: pick LISTEN/NOTIFY over polling for near-real-time
|
||||
// notification of new rows without a tight polling loop.
|
||||
//
|
||||
// ground truth: LISTEN/NOTIFY lets a connected client block until the
|
||||
// database itself pushes a notification on a channel (via NOTIFY, typically
|
||||
// from a trigger on INSERT), giving near-real-time delivery with zero
|
||||
// polling queries in between - unlike a polling loop, which either wastes
|
||||
// queries when idle or adds latency when its interval is too coarse.
|
||||
func dbPGListenNotifyTest() testkit.Test {
|
||||
prompt := `Background workers need near-real-time notification whenever a
|
||||
new row is inserted into a "job_queue" table. The team wants to avoid a
|
||||
tight polling loop that repeatedly queries the table (e.g. every second)
|
||||
whether or not there is new work.
|
||||
|
||||
Which built-in Postgres mechanism lets workers avoid polling entirely,
|
||||
being pushed a notification only when a new row actually arrives? Name the
|
||||
mechanism.`
|
||||
|
||||
evaluator := eval.Mean(
|
||||
eval.ContainsAny("LISTEN"),
|
||||
eval.ContainsAny("NOTIFY"),
|
||||
)
|
||||
|
||||
return testkit.Test{
|
||||
ID: "pg-listen-notify",
|
||||
Category: "databases",
|
||||
Subcategory: "postgres",
|
||||
Description: "Pick LISTEN/NOTIFY over polling for near-real-time notification of new job_queue rows.",
|
||||
Prompt: prompt,
|
||||
Eval: evaluator,
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,276 @@
|
||||
package tests
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestDBPGExplainSeqVsIndexTest_Eval(t *testing.T) {
|
||||
tc := dbPGExplainSeqVsIndexTest()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{"correct", `{"reason":"low-selectivity","would_index_help_at_rare_status":true}`, 1},
|
||||
{"correct fenced with prose", "Here you go:\n```json\n{\"reason\":\"low-selectivity\",\"would_index_help_at_rare_status\":true}\n```", 1},
|
||||
{"wrong reason: blames missing index", `{"reason":"missing-index","would_index_help_at_rare_status":true}`, 0.5},
|
||||
{"wrong: says rare-status index would not help", `{"reason":"low-selectivity","would_index_help_at_rare_status":false}`, 0.5},
|
||||
{"both wrong", `{"reason":"stale-statistics","would_index_help_at_rare_status":false}`, 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDBPGIndexChoiceTest_Eval(t *testing.T) {
|
||||
tc := dbPGIndexChoiceTest()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{"correct", `["region", "order_date"]`, 1},
|
||||
{"correct fenced", "```json\n[\"region\", \"order_date\"]\n```", 1},
|
||||
{"reversed order", `["order_date", "region"]`, 0},
|
||||
{"missing a column", `["region"]`, 0},
|
||||
{"wrong column entirely", `["region", "amount"]`, 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDBPGNPlusOneTest_Eval(t *testing.T) {
|
||||
tc := dbPGNPlusOneTest()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{"correct", `{"pattern":"n-plus-one","fix":"single-join-query"}`, 1},
|
||||
{"correct, different key order", `{"fix":"single-join-query","pattern":"n-plus-one"}`, 1},
|
||||
{"wrong pattern", `{"pattern":"cartesian-product","fix":"single-join-query"}`, 0.5},
|
||||
{"wrong fix", `{"pattern":"n-plus-one","fix":"add-cache"}`, 0.5},
|
||||
{"both wrong", `{"pattern":"missing-index","fix":"add-index"}`, 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDBPGIsolationAnomalyTest_Eval(t *testing.T) {
|
||||
tc := dbPGIsolationAnomalyTest()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{"correct", `{"anomaly":"non-repeatable-read"}`, 1},
|
||||
{"correct, uppercase value", `{"anomaly":"Non-Repeatable-Read"}`, 1},
|
||||
{"wrong: dirty read", `{"anomaly":"dirty-read"}`, 0},
|
||||
{"wrong: phantom read", `{"anomaly":"phantom-read"}`, 0},
|
||||
{"wrong: lost update", `{"anomaly":"lost-update"}`, 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDBPGDeadlockLockOrderTest_Eval(t *testing.T) {
|
||||
tc := dbPGDeadlockLockOrderTest()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{"correct", `{"deadlock":true,"safe_order_ids":[1,2]}`, 1},
|
||||
{"correct fenced with prose", "Yes, they deadlock.\n```json\n{\"deadlock\":true,\"safe_order_ids\":[1,2]}\n```", 1},
|
||||
{"says no deadlock", `{"deadlock":false,"safe_order_ids":[1,2]}`, 2.0 / 3.0},
|
||||
{"reversed safe order", `{"deadlock":true,"safe_order_ids":[2,1]}`, 1.0 / 3.0},
|
||||
{"everything wrong", `{"deadlock":false,"safe_order_ids":[2,1]}`, 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDBPGReplicaFailoverTest_Eval(t *testing.T) {
|
||||
tc := dbPGReplicaFailoverTest()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{
|
||||
name: "correct: names data loss and async cause",
|
||||
response: "Because replication is asynchronous, that commit may be lost - the replica never received its WAL before promotion.",
|
||||
want: 1,
|
||||
},
|
||||
{
|
||||
name: "correct: alternate phrasing",
|
||||
response: "No, it is not guaranteed to be visible; async replication means the promoted replica can be missing it.",
|
||||
want: 1,
|
||||
},
|
||||
{
|
||||
name: "wrong: claims it is guaranteed durable",
|
||||
response: "Yes, Postgres guarantees every committed transaction is always visible after failover.",
|
||||
want: 0,
|
||||
},
|
||||
{
|
||||
name: "partial: mentions loss but not the async cause",
|
||||
response: "That transaction can be lost after the failover.",
|
||||
want: 2.0 / 3.0,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDBPGBloatVacuumTest_Eval(t *testing.T) {
|
||||
tc := dbPGBloatVacuumTest()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{"correct", `{"cause":"bloat","action":"vacuum"}`, 1},
|
||||
{"correct fenced", "```json\n{\"cause\":\"bloat\",\"action\":\"vacuum\"}\n```", 1},
|
||||
{"wrong cause", `{"cause":"missing-index","action":"vacuum"}`, 0.5},
|
||||
{"wrong action", `{"cause":"bloat","action":"reindex"}`, 0.5},
|
||||
{"both wrong", `{"cause":"replication-lag","action":"add-index"}`, 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestDBPGPoolSizingWant_GroundTruth independently re-derives pool_size
|
||||
// with plain arithmetic, not via the dbPGPoolSizingWant constant's own
|
||||
// expression.
|
||||
func TestDBPGPoolSizingWant_GroundTruth(t *testing.T) {
|
||||
const coreCount = 8
|
||||
const effectiveSpindleCount = 1
|
||||
want := coreCount*2 + effectiveSpindleCount
|
||||
|
||||
if want != 17 {
|
||||
t.Fatalf("independently recomputed pool_size = %d, want 17", want)
|
||||
}
|
||||
if dbPGPoolSizingWant != want {
|
||||
t.Errorf("dbPGPoolSizingWant = %d, independently recomputed = %d", dbPGPoolSizingWant, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDBPGPoolSizingTest_Eval(t *testing.T) {
|
||||
tc := dbPGPoolSizingTest()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{"correct", "17", 1},
|
||||
{"prose wrapped", "pool_size = 17", 1},
|
||||
{"wrong: forgot spindle term", "16", 0},
|
||||
{"wrong: doubled instead of adding", "18", 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDBPGPartialIndexTest_Eval(t *testing.T) {
|
||||
tc := dbPGPartialIndexTest()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{"correct", `{"partial_index_applicable":true,"reason":"small-frequently-queried-subset"}`, 1},
|
||||
{"correct fenced with prose", "Yes.\n```json\n{\"partial_index_applicable\":true,\"reason\":\"small-frequently-queried-subset\"}\n```", 1},
|
||||
{"wrong: says not applicable", `{"partial_index_applicable":false,"reason":"small-frequently-queried-subset"}`, 0.5},
|
||||
{"wrong reason", `{"partial_index_applicable":true,"reason":"subset-too-large"}`, 0.5},
|
||||
{"both wrong", `{"partial_index_applicable":false,"reason":"query-does-not-filter-on-column"}`, 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDBPGListenNotifyTest_Eval(t *testing.T) {
|
||||
tc := dbPGListenNotifyTest()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{"correct", "Use LISTEN/NOTIFY.", 1},
|
||||
{"correct, spelled out", "Postgres's LISTEN and NOTIFY commands let a worker block until notified.", 1},
|
||||
{"only names one half", "Use LISTEN.", 0.5},
|
||||
{"wrong: recommends polling harder", "Just poll more frequently, e.g. every 100ms.", 0},
|
||||
{"wrong: recommends a trigger with no LISTEN/NOTIFY", "Add a trigger that writes to an audit table.", 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,410 @@
|
||||
package tests
|
||||
|
||||
import (
|
||||
"context"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/lukaszraczylo/llm-testbench/internal/eval"
|
||||
"github.com/lukaszraczylo/llm-testbench/internal/testkit"
|
||||
)
|
||||
|
||||
// registerDBRedisTests registers every databases/redis test.
|
||||
func registerDBRedisTests(r *testkit.Registry) {
|
||||
r.Register(dbRedisStructureChoiceTest())
|
||||
r.Register(dbRedisTTLEvictionTest())
|
||||
r.Register(dbRedisIncrAtomicityTest())
|
||||
r.Register(dbRedisMultiVsPipelineTest())
|
||||
r.Register(dbRedisScanVsKeysTest())
|
||||
r.Register(dbRedisPubSubDeliveryTest())
|
||||
r.Register(dbRedisLuaAtomicityTest())
|
||||
r.Register(dbRedisMemoryEstimateTest())
|
||||
r.Register(dbRedisCacheStampedeTest())
|
||||
r.Register(dbRedisKeyspaceAntiPatternTest())
|
||||
}
|
||||
|
||||
// dbRedisStructureScenarios lists the 4 inline scenarios for
|
||||
// dbRedisStructureChoiceTest.
|
||||
const dbRedisStructureScenarios = `a: "A leaderboard of user scores that must support efficient range and
|
||||
rank queries (e.g. 'give me the top 10' or 'what rank is this user')."
|
||||
b: "Counting approximately how many unique visitors hit the site today,
|
||||
at a scale of tens of millions of visitors, using minimal memory."
|
||||
c: "A user's session data as a group of named fields (name, plan,
|
||||
last_seen) that must each be readable and updatable individually
|
||||
without rewriting the whole session."
|
||||
d: "A FIFO job queue where worker processes block until a new job
|
||||
arrives, then pop and process jobs in the order they were pushed."`
|
||||
|
||||
// dbRedisStructureChoiceTest: pick the best-fit Redis data structure for
|
||||
// each of 4 inline usage scenarios, from a closed vocabulary.
|
||||
//
|
||||
// ground truth: (a) a leaderboard needing ranked range queries is exactly
|
||||
// what a sorted set (ZSET, score-ordered) provides. (b) approximate unique
|
||||
// counting at large scale with minimal memory is HyperLogLog's purpose
|
||||
// (a small fixed-size probabilistic structure), not an exact set which
|
||||
// would use memory proportional to visitor count. (c) named, individually
|
||||
// updatable fields is a hash, not a single string blob that would need a
|
||||
// full read-modify-write. (d) a FIFO queue with blocking pop is a list
|
||||
// (LPUSH/BRPOP), not a set (unordered, no blocking pop) or a stream
|
||||
// (heavier, consumer-group semantics not needed here).
|
||||
func dbRedisStructureChoiceTest() testkit.Test {
|
||||
prompt := `For each of these 4 scenarios, pick the single best-fit Redis
|
||||
data structure from exactly this vocabulary: string, list, hash, set,
|
||||
sorted-set, hyperloglog, stream, bitmap.
|
||||
|
||||
` + dbRedisStructureScenarios + `
|
||||
|
||||
Respond with only a JSON object mapping each letter to one structure name:
|
||||
{"a":"...","b":"...","c":"...","d":"..."}`
|
||||
|
||||
evaluator := eval.Mean(
|
||||
eval.JSONField("a", "sorted-set"),
|
||||
eval.JSONField("b", "hyperloglog"),
|
||||
eval.JSONField("c", "hash"),
|
||||
eval.JSONField("d", "list"),
|
||||
)
|
||||
|
||||
return testkit.Test{
|
||||
ID: "redis-structure-choice",
|
||||
Category: "databases",
|
||||
Subcategory: "redis",
|
||||
Description: "Pick the best-fit Redis data structure (sorted-set, hyperloglog, hash, list) for 4 inline usage scenarios from a closed vocabulary.",
|
||||
Prompt: prompt,
|
||||
Eval: evaluator,
|
||||
}
|
||||
}
|
||||
|
||||
// dbRedisTTLEvictionTest: determine what happens to a write under
|
||||
// maxmemory-policy=noeviction when Redis is already at maxmemory.
|
||||
//
|
||||
// ground truth: noeviction means Redis will never evict a key to make
|
||||
// room. A write that would grow memory usage while already at maxmemory
|
||||
// is instead rejected outright with an OOM error - the write does not
|
||||
// silently succeed and no key is evicted to make space.
|
||||
func dbRedisTTLEvictionTest() testkit.Test {
|
||||
prompt := `A Redis instance has maxmemory-policy set to "noeviction" and
|
||||
is currently at maxmemory. A client sends a write command that would
|
||||
increase memory usage further.
|
||||
|
||||
What happens to that write? Respond with only one word: reject or evict.`
|
||||
|
||||
return testkit.Test{
|
||||
ID: "redis-ttl-eviction",
|
||||
Category: "databases",
|
||||
Subcategory: "redis",
|
||||
Description: "Determine that a write is rejected (not silently allowed, and nothing evicted) under maxmemory-policy=noeviction at maxmemory.",
|
||||
Prompt: prompt,
|
||||
Eval: dbExactAnswer("reject"),
|
||||
}
|
||||
}
|
||||
|
||||
// dbRedisIncrAtomicityTest: contrast INCR's atomic guarantee against the
|
||||
// race in a manual GET-then-SET increment pattern.
|
||||
//
|
||||
// ground truth: INCR is a single atomic server-side operation, so 100
|
||||
// concurrent INCR calls on a counter starting at 0 are guaranteed to leave
|
||||
// it at exactly 100, with no lost updates. The GET-then-SET pattern is two
|
||||
// separate round trips per client; two clients can both GET the same
|
||||
// value before either SETs, so one client's increment can be silently
|
||||
// overwritten - the final value after 100 concurrent GET/SET increments is
|
||||
// not guaranteed to reach 100.
|
||||
func dbRedisIncrAtomicityTest() testkit.Test {
|
||||
prompt := `A Redis counter starts at 0. 100 clients each run exactly one
|
||||
of the two patterns below, concurrently:
|
||||
|
||||
Pattern A: each client calls INCR counter once.
|
||||
Pattern B: each client calls GET counter, adds 1 in its own application
|
||||
code, then calls SET counter <new value> once.
|
||||
|
||||
What is the guaranteed final value of the counter after Pattern A's 100
|
||||
calls, and is Pattern B's final value guaranteed to also reach that same
|
||||
total? Respond with only a JSON object:
|
||||
{"incr_final":<number>,"getset_race_safe":true|false}`
|
||||
|
||||
evaluator := eval.Mean(
|
||||
eval.JSONField("incr_final", 100),
|
||||
eval.JSONField("getset_race_safe", false),
|
||||
)
|
||||
|
||||
return testkit.Test{
|
||||
ID: "redis-incr-atomicity",
|
||||
Category: "databases",
|
||||
Subcategory: "redis",
|
||||
Description: "Contrast INCR's guaranteed atomic result (100) against the race-prone GET-then-SET increment pattern (not guaranteed to reach 100).",
|
||||
Prompt: prompt,
|
||||
Eval: evaluator,
|
||||
}
|
||||
}
|
||||
|
||||
// dbRedisMultiVsPipelineTest: confirm that plain pipelining, unlike
|
||||
// MULTI/EXEC, gives no atomicity/isolation guarantee against interleaving
|
||||
// from other clients.
|
||||
//
|
||||
// ground truth: pipelining is purely a network optimization - batching
|
||||
// several commands into one round trip - with no guarantee that the
|
||||
// server executes them back-to-back relative to other connected clients.
|
||||
// MULTI/EXEC (a transaction) is what guarantees no other client's command
|
||||
// can execute in between the queued commands.
|
||||
func dbRedisMultiVsPipelineTest() testkit.Test {
|
||||
prompt := `Does plain command pipelining (batching several commands into
|
||||
one network round trip, without wrapping them in MULTI/EXEC) guarantee
|
||||
that no other client's command can execute on the server in between the
|
||||
pipelined commands? Respond with only "yes" or "no".`
|
||||
|
||||
return testkit.Test{
|
||||
ID: "redis-multi-vs-pipeline",
|
||||
Category: "databases",
|
||||
Subcategory: "redis",
|
||||
Description: "Confirm plain pipelining (unlike MULTI/EXEC) gives no guarantee against another client's commands interleaving.",
|
||||
Prompt: prompt,
|
||||
Eval: dbExactAnswer("no"),
|
||||
}
|
||||
}
|
||||
|
||||
// dbKeysCommandPattern matches an explicit mention of the Redis KEYS
|
||||
// command actually being invoked - "KEYS *" (its most common prod-danger
|
||||
// form) or a backtick-quoted `KEYS` - not a generic use of the English
|
||||
// word "keys" elsewhere in prose ("the keys to fixing this...").
|
||||
var dbKeysCommandPattern = regexp.MustCompile("(?:`KEYS`|\\bKEYS\\s*\\*)")
|
||||
|
||||
// dbNegationCuePattern matches a word that turns a mention of the KEYS
|
||||
// command into a warning against running it in production, rather than an
|
||||
// instruction to run it.
|
||||
var dbNegationCuePattern = regexp.MustCompile(`(?i)\b(don'?t|do not|never|avoid|instead of|not|shouldn'?t|should not|rather than|without|no need|danger|dangerous)\b`)
|
||||
|
||||
// dbNegationWindow is how many characters before a KEYS-command mention
|
||||
// are searched for a negation cue.
|
||||
const dbNegationWindow = 60
|
||||
|
||||
// dbNegationWindowStart returns the earliest byte offset in response to
|
||||
// search for a negation cue before a KEYS-command mention starting at
|
||||
// start: the current line, extended back at most dbNegationWindow
|
||||
// characters.
|
||||
func dbNegationWindowStart(response string, start int) int {
|
||||
lineStart := strings.LastIndexByte(response[:start], '\n') + 1
|
||||
return max(start-dbNegationWindow, lineStart)
|
||||
}
|
||||
|
||||
// dbNoBareKeysInProd scores full credit unless the response instructs
|
||||
// running "KEYS *" against a production instance without any negating
|
||||
// context (a warning not to, or an instruction to use SCAN instead is
|
||||
// fine). This is deliberately not eval.NotContains, which would also zero
|
||||
// out the best possible answer: one that correctly explains why NOT to run
|
||||
// KEYS in production.
|
||||
func dbNoBareKeysInProd() eval.Evaluator {
|
||||
return eval.EvaluatorFunc(func(_ context.Context, response string) eval.Score {
|
||||
matches := dbKeysCommandPattern.FindAllStringIndex(response, -1)
|
||||
if len(matches) == 0 {
|
||||
return eval.Score{Value: 1, Detail: "no mention of running the KEYS command"}
|
||||
}
|
||||
for _, loc := range matches {
|
||||
start := loc[0]
|
||||
windowStart := dbNegationWindowStart(response, start)
|
||||
if !dbNegationCuePattern.MatchString(response[windowStart:start]) {
|
||||
return eval.Score{Value: 0, Detail: "unnegated mention of running the KEYS command"}
|
||||
}
|
||||
}
|
||||
return eval.Score{Value: 1, Detail: "every mention of KEYS is negated"}
|
||||
})
|
||||
}
|
||||
|
||||
// dbRedisScanVsKeysTest: require SCAN over KEYS for a hot-path production
|
||||
// lookup, using a negation-aware guard so an answer that correctly warns
|
||||
// against KEYS is not penalized for mentioning it.
|
||||
func dbRedisScanVsKeysTest() testkit.Test {
|
||||
prompt := `A request handler that runs thousands of times per second
|
||||
against a production Redis instance holding millions of keys currently
|
||||
calls "KEYS *" to find every key matching a pattern.
|
||||
|
||||
What is wrong with this in production, and what command should be used
|
||||
instead to iterate the keyspace safely?`
|
||||
|
||||
evaluator := eval.All(
|
||||
eval.W(dbNoBareKeysInProd(), 2),
|
||||
eval.W(eval.ContainsAny("SCAN"), 2),
|
||||
)
|
||||
|
||||
return testkit.Test{
|
||||
ID: "redis-scan-vs-keys",
|
||||
Category: "databases",
|
||||
Subcategory: "redis",
|
||||
Description: "Require the SCAN cursor over KEYS * for a hot-path production lookup, without penalizing a negated warning against KEYS.",
|
||||
Prompt: prompt,
|
||||
Eval: evaluator,
|
||||
}
|
||||
}
|
||||
|
||||
// dbRedisPubSubDeliveryTest: confirm Redis pub/sub gives no delivery
|
||||
// guarantee to a disconnected subscriber.
|
||||
//
|
||||
// ground truth: Redis pub/sub is fire-and-forget with no backlog or
|
||||
// replay - a message published while a subscriber is disconnected is
|
||||
// simply not delivered to it; reconnecting does not retrieve messages
|
||||
// missed while offline.
|
||||
func dbRedisPubSubDeliveryTest() testkit.Test {
|
||||
prompt := `A subscriber is disconnected (e.g. a brief network blip) at
|
||||
the exact moment a message is published on a channel it normally
|
||||
subscribes to. After the subscriber reconnects, will it receive that
|
||||
specific message it missed while disconnected? Respond with only "yes" or
|
||||
"no".`
|
||||
|
||||
return testkit.Test{
|
||||
ID: "redis-pubsub-delivery",
|
||||
Category: "databases",
|
||||
Subcategory: "redis",
|
||||
Description: "Confirm Redis pub/sub gives no delivery guarantee (no backlog/replay) to a subscriber that was disconnected when a message was published.",
|
||||
Prompt: prompt,
|
||||
Eval: dbExactAnswer("no"),
|
||||
}
|
||||
}
|
||||
|
||||
// dbRedisLuaAtomicityTest: confirm a Lua script run via EVAL executes
|
||||
// atomically with no other client command interleaved.
|
||||
//
|
||||
// ground truth: Redis executes commands (and an entire EVAL script) on a
|
||||
// single command-processing thread; while one client's script is running,
|
||||
// the server does not process any other client's command, so a script is
|
||||
// guaranteed to run start-to-finish with nothing interleaved.
|
||||
func dbRedisLuaAtomicityTest() testkit.Test {
|
||||
prompt := `While one client's Lua script is executing via EVAL, can any
|
||||
other client's command run on the Redis server before that script
|
||||
finishes? Respond with only "yes" or "no".`
|
||||
|
||||
return testkit.Test{
|
||||
ID: "redis-lua-atomicity",
|
||||
Category: "databases",
|
||||
Subcategory: "redis",
|
||||
Description: "Confirm a Lua script run via EVAL executes atomically, with no other client command able to interleave.",
|
||||
Prompt: prompt,
|
||||
Eval: dbExactAnswer("no"),
|
||||
}
|
||||
}
|
||||
|
||||
// dbRedisMemoryEstimateWant is the total estimated bytes computed by the
|
||||
// formula given inline in dbRedisMemoryEstimateTest's prompt.
|
||||
//
|
||||
// ground truth: total_bytes = count * (key_bytes + value_bytes +
|
||||
// overhead_bytes) = 200000 * (20 + 100 + 56) = 200000 * 176 = 35,200,000.
|
||||
// databases_redis_test.go recomputes this independently with the same
|
||||
// arithmetic written out plainly.
|
||||
const dbRedisMemoryEstimateWant = 200000 * (20 + 100 + 56)
|
||||
|
||||
// dbRedisMemoryEstimateTest: compute total memory bytes for an inline
|
||||
// dataset from a formula given entirely in the prompt.
|
||||
func dbRedisMemoryEstimateTest() testkit.Test {
|
||||
prompt := `Estimate total memory, in bytes, to store a Redis dataset of
|
||||
200,000 string key-value pairs, where each key name is 20 bytes, each
|
||||
value is 100 bytes, and Redis's per-key-value overhead (hash table entry
|
||||
plus object headers) is estimated at 56 bytes. Use this formula:
|
||||
|
||||
total_bytes = count * (key_bytes + value_bytes + overhead_bytes)
|
||||
|
||||
Respond with only the number.`
|
||||
|
||||
return testkit.Test{
|
||||
ID: "redis-memory-estimate",
|
||||
Category: "databases",
|
||||
Subcategory: "redis",
|
||||
Description: "Compute total memory bytes for a 200,000-key dataset from an inline per-key-value formula and byte sizes.",
|
||||
Prompt: prompt,
|
||||
Eval: eval.Numeric(eval.ExtractLastNumber[int], dbRedisMemoryEstimateWant, 0),
|
||||
}
|
||||
}
|
||||
|
||||
// dbRedisStampedeCandidates lists the 4 candidate mitigation techniques
|
||||
// for dbRedisCacheStampedeTest, mixing genuinely valid mitigations with
|
||||
// one plausible-sounding non-mitigation.
|
||||
const dbRedisStampedeCandidates = `- mutex_lock_on_regenerate: only one client regenerates a missing/expired
|
||||
key at a time; others wait for or read the in-flight result.
|
||||
- probabilistic_early_expiration: clients probabilistically recompute a
|
||||
soon-to-expire key slightly before it actually expires, spreading
|
||||
regeneration out instead of all racing at the exact expiry moment.
|
||||
- increase_replica_count: add more read replicas to the Redis deployment.
|
||||
- stale_while_revalidate: serve the stale cached value immediately while
|
||||
one background request regenerates it.`
|
||||
|
||||
// dbRedisStampedeWant is the subset of dbRedisStampedeCandidates that
|
||||
// actually mitigates cache stampede.
|
||||
//
|
||||
// ground truth: a cache stampede is a burst of concurrent regenerations
|
||||
// racing to rebuild the same expired key against the backing store.
|
||||
// mutex_lock_on_regenerate, probabilistic_early_expiration, and
|
||||
// stale_while_revalidate each directly prevent that race (by
|
||||
// serializing regeneration, spreading it out over time, or deferring it
|
||||
// entirely). Adding more replicas does not: replicas serve reads of
|
||||
// already-cached data, but every replica independently misses the same
|
||||
// expired key at the same moment, so it does not reduce or serialize the
|
||||
// regeneration race against the backing store at all.
|
||||
var dbRedisStampedeWant = []string{
|
||||
"mutex_lock_on_regenerate",
|
||||
"probabilistic_early_expiration",
|
||||
"stale_while_revalidate",
|
||||
}
|
||||
|
||||
func dbRedisCacheStampedeTest() testkit.Test {
|
||||
prompt := `A hot cache key expires and many concurrent requests all miss
|
||||
it at once, each racing to regenerate it against the backing store (a
|
||||
"cache stampede" / "thundering herd"). Here are 4 candidate mitigation
|
||||
techniques:
|
||||
|
||||
` + dbRedisStampedeCandidates + `
|
||||
|
||||
Which of these candidates actually mitigate a cache stampede? Respond with
|
||||
only a JSON array of the candidate names that do.`
|
||||
|
||||
return testkit.Test{
|
||||
ID: "redis-cache-stampede",
|
||||
Category: "databases",
|
||||
Subcategory: "redis",
|
||||
Description: "Select the cache-stampede mitigations (mutex, probabilistic early expiration, stale-while-revalidate) from 4 candidates that also include a non-mitigation.",
|
||||
Prompt: prompt,
|
||||
Eval: eval.JSONStringSet(dbRedisStampedeWant),
|
||||
}
|
||||
}
|
||||
|
||||
// dbRedisKeyspaceAntiPatternCode is the inline monolithic-key snippet for
|
||||
// dbRedisKeyspaceAntiPatternTest.
|
||||
const dbRedisKeyspaceAntiPatternCode = `def save_all_users(users):
|
||||
r.set("all_users", json.dumps(users)) # users: a list of 500,000 user dicts
|
||||
|
||||
def update_user(user_id, fields):
|
||||
users = json.loads(r.get("all_users"))
|
||||
for u in users:
|
||||
if u["id"] == user_id:
|
||||
u.update(fields)
|
||||
r.set("all_users", json.dumps(users)) # rewrites the entire 500,000-user blob`
|
||||
|
||||
// dbRedisKeyspaceAntiPatternTest: spot a monolithic single-key blob
|
||||
// keyspace anti-pattern in inline code and require the per-entity hash
|
||||
// fix.
|
||||
//
|
||||
// ground truth: every single-field update to one user rewrites a
|
||||
// serialized blob of all 500,000 users under one key, an O(n) operation
|
||||
// per update where n is the whole user count - a monolithic-key
|
||||
// anti-pattern, not a "too many small keys" problem (there is only one
|
||||
// key here) or a missing-TTL problem (nothing about expiry is shown). The
|
||||
// fix is to give each user its own Hash key (e.g. user:<id>), so a field
|
||||
// update touches O(1) data instead of the entire dataset.
|
||||
func dbRedisKeyspaceAntiPatternTest() testkit.Test {
|
||||
prompt := `Here is code managing user data in Redis:
|
||||
|
||||
` + "```python\n" + dbRedisKeyspaceAntiPatternCode + "\n```" + `
|
||||
|
||||
What keyspace design problem does this have, and what is the correct fix?
|
||||
Respond with only a JSON object:
|
||||
{"problem":"<one of: monolithic-key, too-many-small-keys, missing-ttl>","fix":"<one of: hash-per-user, single-key-ok, add-ttl>"}`
|
||||
|
||||
evaluator := eval.Mean(
|
||||
eval.JSONField("problem", "monolithic-key"),
|
||||
eval.JSONField("fix", "hash-per-user"),
|
||||
)
|
||||
|
||||
return testkit.Test{
|
||||
ID: "redis-keyspace-anti-pattern",
|
||||
Category: "databases",
|
||||
Subcategory: "redis",
|
||||
Description: "Spot a monolithic single-key user blob rewritten on every update and require the per-user-hash fix.",
|
||||
Prompt: prompt,
|
||||
Eval: evaluator,
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,342 @@
|
||||
package tests
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestDBRedisStructureChoiceTest_Eval(t *testing.T) {
|
||||
tc := dbRedisStructureChoiceTest()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{
|
||||
name: "all correct",
|
||||
response: `{"a":"sorted-set","b":"hyperloglog","c":"hash","d":"list"}`,
|
||||
want: 1,
|
||||
},
|
||||
{
|
||||
name: "all correct fenced with prose",
|
||||
response: "Here is my answer:\n```json\n{\"a\":\"sorted-set\",\"b\":\"hyperloglog\",\"c\":\"hash\",\"d\":\"list\"}\n```",
|
||||
want: 1,
|
||||
},
|
||||
{
|
||||
name: "one wrong",
|
||||
response: `{"a":"set","b":"hyperloglog","c":"hash","d":"list"}`,
|
||||
want: 0.75,
|
||||
},
|
||||
{
|
||||
name: "all wrong",
|
||||
response: `{"a":"list","b":"set","c":"string","d":"stream"}`,
|
||||
want: 0,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDBRedisTTLEvictionTest_Eval(t *testing.T) {
|
||||
tc := dbRedisTTLEvictionTest()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{"bare correct", "reject", 1},
|
||||
{"quoted correct", `"reject"`, 1},
|
||||
{"different case", "REJECT", 1},
|
||||
{"trailing period", "reject.", 1},
|
||||
{"wrong: evict", "evict", 0},
|
||||
{"wrong: claims silent success", "succeed", 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDBRedisIncrAtomicityTest_Eval(t *testing.T) {
|
||||
tc := dbRedisIncrAtomicityTest()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{"correct", `{"incr_final":100,"getset_race_safe":false}`, 1},
|
||||
{"correct fenced", "```json\n{\"incr_final\":100,\"getset_race_safe\":false}\n```", 1},
|
||||
{"wrong incr_final", `{"incr_final":99,"getset_race_safe":false}`, 0.5},
|
||||
{"wrongly claims getset is safe", `{"incr_final":100,"getset_race_safe":true}`, 0.5},
|
||||
{"both wrong", `{"incr_final":50,"getset_race_safe":true}`, 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDBRedisMultiVsPipelineTest_Eval(t *testing.T) {
|
||||
tc := dbRedisMultiVsPipelineTest()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{"bare correct", "no", 1},
|
||||
{"quoted correct", `"no"`, 1},
|
||||
{"different case", "No.", 1},
|
||||
{"wrong", "yes", 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestDBNoBareKeysInProd exercises the negation-aware guard directly,
|
||||
// mirroring TestNoLiveKubectlMutation's coverage: an unnegated instruction
|
||||
// to run KEYS scores 0, but a response that mentions KEYS only to warn
|
||||
// against it (however it is phrased or wrapped) scores 1.
|
||||
func TestDBNoBareKeysInProd(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{
|
||||
name: "no mention at all",
|
||||
response: "Use SCAN with a cursor and MATCH pattern instead, iterating in small batches.",
|
||||
want: 1,
|
||||
},
|
||||
{
|
||||
name: "negated mid-sentence mention",
|
||||
response: "This runs against production, so do not use `KEYS` here - it blocks the whole server.",
|
||||
want: 1,
|
||||
},
|
||||
{
|
||||
name: "imperative command at line start",
|
||||
response: "Just run this:\nKEYS *",
|
||||
want: 0,
|
||||
},
|
||||
{
|
||||
name: "unnegated mid-sentence mention",
|
||||
response: "One quick option is to just run KEYS * to list them all.",
|
||||
want: 0,
|
||||
},
|
||||
{
|
||||
name: "negation cue too far away to count",
|
||||
response: "Never mind the migration for now, but you could casually run KEYS * to check this quickly.",
|
||||
want: 0,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := dbNoBareKeysInProd().Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("dbNoBareKeysInProd().Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDBRedisScanVsKeysTest_Eval(t *testing.T) {
|
||||
tc := dbRedisScanVsKeysTest()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{
|
||||
name: "correct: warns against KEYS, recommends SCAN",
|
||||
response: "Avoid running `KEYS *` in production - it blocks the entire server while it scans the whole keyspace, causing latency spikes for every other client. Use SCAN with a cursor and small COUNT instead, iterating incrementally.",
|
||||
want: 1,
|
||||
},
|
||||
{
|
||||
name: "wrong: recommends running KEYS unnegated",
|
||||
response: "Just run KEYS * - it will return every matching key.",
|
||||
want: 0,
|
||||
},
|
||||
{
|
||||
name: "partial: correctly avoids KEYS but never names SCAN",
|
||||
response: "Do not run that command against production; iterate the keyspace incrementally instead.",
|
||||
want: 0.5,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDBRedisPubSubDeliveryTest_Eval(t *testing.T) {
|
||||
tc := dbRedisPubSubDeliveryTest()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{"bare correct", "no", 1},
|
||||
{"quoted correct", `"no"`, 1},
|
||||
{"wrong", "yes", 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDBRedisLuaAtomicityTest_Eval(t *testing.T) {
|
||||
tc := dbRedisLuaAtomicityTest()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{"bare correct", "no", 1},
|
||||
{"trailing period", "no.", 1},
|
||||
{"wrong", "yes", 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestDBRedisMemoryEstimateWant_GroundTruth independently re-derives the
|
||||
// total-bytes estimate with plain arithmetic, not via the
|
||||
// dbRedisMemoryEstimateWant constant's own expression.
|
||||
func TestDBRedisMemoryEstimateWant_GroundTruth(t *testing.T) {
|
||||
const count, keyBytes, valueBytes, overheadBytes = 200000, 20, 100, 56
|
||||
want := count * (keyBytes + valueBytes + overheadBytes)
|
||||
|
||||
if want != 35200000 {
|
||||
t.Fatalf("independently recomputed total bytes = %d, want 35200000", want)
|
||||
}
|
||||
if dbRedisMemoryEstimateWant != want {
|
||||
t.Errorf("dbRedisMemoryEstimateWant = %d, independently recomputed = %d", dbRedisMemoryEstimateWant, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDBRedisMemoryEstimateTest_Eval(t *testing.T) {
|
||||
tc := dbRedisMemoryEstimateTest()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{"correct", "35200000", 1},
|
||||
{"prose wrapped", "The estimate is about 35200000 bytes.", 1},
|
||||
{"wrong: forgot overhead term", "24000000", 0},
|
||||
{"wrong: used wrong overhead", "32000000", 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDBRedisCacheStampedeTest_Eval(t *testing.T) {
|
||||
tc := dbRedisCacheStampedeTest()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{
|
||||
name: "exact correct set",
|
||||
response: `["mutex_lock_on_regenerate", "probabilistic_early_expiration", "stale_while_revalidate"]`,
|
||||
want: 1,
|
||||
},
|
||||
{
|
||||
name: "correct set, different order, fenced",
|
||||
response: "```json\n[\"stale_while_revalidate\", \"mutex_lock_on_regenerate\", \"probabilistic_early_expiration\"]\n```",
|
||||
want: 1,
|
||||
},
|
||||
{
|
||||
name: "includes the non-mitigation",
|
||||
response: `["mutex_lock_on_regenerate", "probabilistic_early_expiration", "stale_while_revalidate", "increase_replica_count"]`,
|
||||
want: 0.75,
|
||||
},
|
||||
{
|
||||
name: "only the non-mitigation",
|
||||
response: `["increase_replica_count"]`,
|
||||
want: 0,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDBRedisKeyspaceAntiPatternTest_Eval(t *testing.T) {
|
||||
tc := dbRedisKeyspaceAntiPatternTest()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{"correct", `{"problem":"monolithic-key","fix":"hash-per-user"}`, 1},
|
||||
{"correct fenced with prose", "Here's the issue:\n```json\n{\"problem\":\"monolithic-key\",\"fix\":\"hash-per-user\"}\n```", 1},
|
||||
{"wrong problem", `{"problem":"missing-ttl","fix":"hash-per-user"}`, 0.5},
|
||||
{"wrong fix", `{"problem":"monolithic-key","fix":"add-ttl"}`, 0.5},
|
||||
{"both wrong", `{"problem":"too-many-small-keys","fix":"single-key-ok"}`, 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,634 @@
|
||||
package tests
|
||||
|
||||
import (
|
||||
"sort"
|
||||
|
||||
"github.com/lukaszraczylo/llm-testbench/internal/eval"
|
||||
"github.com/lukaszraczylo/llm-testbench/internal/testkit"
|
||||
)
|
||||
|
||||
// registerDBSQLTuningTests registers every databases/sql-tuning test.
|
||||
func registerDBSQLTuningTests(r *testkit.Registry) {
|
||||
r.Register(dbSQLAggregateSumTest())
|
||||
r.Register(dbSQLJoinCountsTest())
|
||||
r.Register(dbSQLNullInWhereTest())
|
||||
r.Register(dbSQLGroupByHavingTest())
|
||||
r.Register(dbSQLWindowRankTest())
|
||||
r.Register(dbSQLCompositeIndexOrderTest())
|
||||
r.Register(dbSQLKeysetPaginationTest())
|
||||
r.Register(dbSQLCoveringIndexTest())
|
||||
r.Register(dbSQLEquivalentRewriteTest())
|
||||
r.Register(dbSQLOrderByIndexServingTest())
|
||||
}
|
||||
|
||||
// dbSQLOrder is one row of the shared inline "orders" table fixture used by
|
||||
// several sql-tuning tests. Region is "" to represent SQL NULL (order id 4
|
||||
// has no region on file).
|
||||
type dbSQLOrder struct {
|
||||
Region string
|
||||
Status string
|
||||
ID int
|
||||
CustomerID int
|
||||
Amount int
|
||||
}
|
||||
|
||||
// dbSQLOrders is the shared inline "orders" table fixture. It is printed
|
||||
// verbatim (see dbSQLOrdersTableText) into several sql-tuning prompts, and
|
||||
// every one of those tests' expected answers is derived by calling a query-
|
||||
// logic helper on this exact slice - never hardcoded independently of it.
|
||||
var dbSQLOrders = []dbSQLOrder{
|
||||
{ID: 1, CustomerID: 101, Region: "EU", Amount: 250, Status: "paid"},
|
||||
{ID: 2, CustomerID: 102, Region: "US", Amount: 100, Status: "paid"},
|
||||
{ID: 3, CustomerID: 101, Region: "EU", Amount: 75, Status: "pending"},
|
||||
{ID: 4, CustomerID: 103, Region: "", Amount: 300, Status: "paid"},
|
||||
{ID: 5, CustomerID: 102, Region: "US", Amount: 50, Status: "refunded"},
|
||||
{ID: 6, CustomerID: 104, Region: "APAC", Amount: 400, Status: "paid"},
|
||||
{ID: 7, CustomerID: 101, Region: "EU", Amount: 125, Status: "paid"},
|
||||
{ID: 8, CustomerID: 103, Region: "US", Amount: 60, Status: "pending"},
|
||||
}
|
||||
|
||||
// dbSQLOrdersTableText is dbSQLOrders rendered as a plain-text table for
|
||||
// the prompt. Kept in sync with dbSQLOrders by hand; databases_sqltuning_
|
||||
// test.go's ground-truth tests operate on dbSQLOrders directly, so a
|
||||
// mismatch here would only affect prompt readability, never the pinned
|
||||
// expected answers.
|
||||
const dbSQLOrdersTableText = `id | customer_id | region | amount | status
|
||||
1 | 101 | EU | 250 | paid
|
||||
2 | 102 | US | 100 | paid
|
||||
3 | 101 | EU | 75 | pending
|
||||
4 | 103 | NULL | 300 | paid
|
||||
5 | 102 | US | 50 | refunded
|
||||
6 | 104 | APAC | 400 | paid
|
||||
7 | 101 | EU | 125 | paid
|
||||
8 | 103 | US | 60 | pending`
|
||||
|
||||
// dbSQLSumPaidAmount sums Amount over every row whose Status is "paid" -
|
||||
// the query-logic equivalent of "SELECT SUM(amount) FROM orders WHERE
|
||||
// status = 'paid'".
|
||||
func dbSQLSumPaidAmount(rows []dbSQLOrder) int {
|
||||
total := 0
|
||||
for _, r := range rows {
|
||||
if r.Status == "paid" {
|
||||
total += r.Amount
|
||||
}
|
||||
}
|
||||
return total
|
||||
}
|
||||
|
||||
// dbSQLAggregateSumWant is derived by calling dbSQLSumPaidAmount, not
|
||||
// hardcoded.
|
||||
//
|
||||
// ground truth: paid rows are id1(250), id2(100), id4(300), id6(400),
|
||||
// id7(125): 250+100+300+400+125 = 1175. databases_sqltuning_test.go
|
||||
// recomputes this independently with a hand-written loop over the same
|
||||
// dbSQLOrders fixture.
|
||||
var dbSQLAggregateSumWant = dbSQLSumPaidAmount(dbSQLOrders)
|
||||
|
||||
// dbSQLAggregateSumTest: compute SUM(amount) WHERE status = 'paid' over an
|
||||
// inline 8-row orders table.
|
||||
func dbSQLAggregateSumTest() testkit.Test {
|
||||
prompt := `Here is the "orders" table:
|
||||
|
||||
` + "```\n" + dbSQLOrdersTableText + "\n```" + `
|
||||
|
||||
What does "SELECT SUM(amount) FROM orders WHERE status = 'paid';" return?
|
||||
Respond with only the number.`
|
||||
|
||||
return testkit.Test{
|
||||
ID: "sql-aggregate-sum",
|
||||
Category: "databases",
|
||||
Subcategory: "sql-tuning",
|
||||
Description: "Compute SUM(amount) WHERE status = 'paid' over an inline 8-row orders table.",
|
||||
Prompt: prompt,
|
||||
Eval: eval.Numeric(eval.ExtractLastNumber[int], dbSQLAggregateSumWant, 0),
|
||||
}
|
||||
}
|
||||
|
||||
// dbSQLCustomer is one row of the shared inline "customers" table fixture
|
||||
// used by dbSQLJoinCountsTest. It intentionally does not cover every
|
||||
// customer_id present in dbSQLOrders (103 and 104 are absent), and
|
||||
// includes one customer (105) with no matching orders at all, so INNER
|
||||
// JOIN and LEFT JOIN produce different row counts.
|
||||
type dbSQLCustomer struct {
|
||||
Name string
|
||||
ID int
|
||||
}
|
||||
|
||||
// dbSQLCustomers is the shared inline "customers" table fixture, printed
|
||||
// verbatim into dbSQLJoinCountsTest's prompt.
|
||||
var dbSQLCustomers = []dbSQLCustomer{
|
||||
{ID: 101, Name: "Alice"},
|
||||
{ID: 102, Name: "Bob"},
|
||||
{ID: 105, Name: "Eve"},
|
||||
}
|
||||
|
||||
// dbSQLCustomersTableText is dbSQLCustomers rendered as a plain-text table
|
||||
// for the prompt.
|
||||
const dbSQLCustomersTableText = `id | name
|
||||
101 | Alice
|
||||
102 | Bob
|
||||
105 | Eve`
|
||||
|
||||
// dbSQLInnerJoinCount counts the result rows of "orders o INNER JOIN
|
||||
// customers c ON o.customer_id = c.id": one row per order whose
|
||||
// customer_id matches some customer in customers, zero rows contributed by
|
||||
// a customer or order with no match on the other side.
|
||||
func dbSQLInnerJoinCount(orders []dbSQLOrder, customers []dbSQLCustomer) int {
|
||||
knownCustomer := make(map[int]bool, len(customers))
|
||||
for _, c := range customers {
|
||||
knownCustomer[c.ID] = true
|
||||
}
|
||||
count := 0
|
||||
for _, o := range orders {
|
||||
if knownCustomer[o.CustomerID] {
|
||||
count++
|
||||
}
|
||||
}
|
||||
return count
|
||||
}
|
||||
|
||||
// dbSQLLeftJoinCount counts the result rows of "customers c LEFT JOIN
|
||||
// orders o ON o.customer_id = c.id": one row per matching order, or
|
||||
// exactly one row (with NULL order columns) for a customer with zero
|
||||
// matching orders.
|
||||
func dbSQLLeftJoinCount(customers []dbSQLCustomer, orders []dbSQLOrder) int {
|
||||
count := 0
|
||||
for _, c := range customers {
|
||||
matches := 0
|
||||
for _, o := range orders {
|
||||
if o.CustomerID == c.ID {
|
||||
matches++
|
||||
}
|
||||
}
|
||||
if matches == 0 {
|
||||
count++
|
||||
} else {
|
||||
count += matches
|
||||
}
|
||||
}
|
||||
return count
|
||||
}
|
||||
|
||||
// dbSQLInnerJoinCountWant and dbSQLLeftJoinCountWant are derived by calling
|
||||
// dbSQLInnerJoinCount/dbSQLLeftJoinCount, not hardcoded.
|
||||
//
|
||||
// ground truth: orders' customer_id values are 101(x3: id1,3,7), 102(x2:
|
||||
// id2,5), 103(x2: id4,8), 104(x1: id6). customers only holds ids
|
||||
// {101,102,105}. INNER JOIN keeps only rows whose customer_id is in that
|
||||
// set: 3+2 = 5 (customer 105 contributes nothing; customer_id 103/104
|
||||
// orders are dropped). LEFT JOIN, driven from customers, keeps every
|
||||
// customer: 101 contributes 3, 102 contributes 2, 105 contributes exactly
|
||||
// 1 row (NULL order columns, since it matches zero orders): 3+2+1 = 6.
|
||||
// databases_sqltuning_test.go recomputes both independently with a
|
||||
// hand-written loop.
|
||||
var (
|
||||
dbSQLInnerJoinCountWant = dbSQLInnerJoinCount(dbSQLOrders, dbSQLCustomers)
|
||||
dbSQLLeftJoinCountWant = dbSQLLeftJoinCount(dbSQLCustomers, dbSQLOrders)
|
||||
)
|
||||
|
||||
// dbSQLJoinCountsTest: compute the row counts of an INNER JOIN versus a
|
||||
// LEFT JOIN between the same inline orders and customers tables.
|
||||
func dbSQLJoinCountsTest() testkit.Test {
|
||||
prompt := `Here are two tables:
|
||||
|
||||
orders:
|
||||
` + "```\n" + dbSQLOrdersTableText + "\n```" + `
|
||||
|
||||
customers:
|
||||
` + "```\n" + dbSQLCustomersTableText + "\n```" + `
|
||||
|
||||
How many rows does each of these two queries return?
|
||||
|
||||
query_inner: SELECT o.id FROM orders o INNER JOIN customers c ON o.customer_id = c.id;
|
||||
query_left: SELECT c.name, o.id FROM customers c LEFT JOIN orders o ON o.customer_id = c.id;
|
||||
|
||||
Respond with only a JSON object:
|
||||
{"inner_join_count":<number>,"left_join_count":<number>}`
|
||||
|
||||
evaluator := eval.Mean(
|
||||
eval.JSONField("inner_join_count", dbSQLInnerJoinCountWant),
|
||||
eval.JSONField("left_join_count", dbSQLLeftJoinCountWant),
|
||||
)
|
||||
|
||||
return testkit.Test{
|
||||
ID: "sql-join-counts",
|
||||
Category: "databases",
|
||||
Subcategory: "sql-tuning",
|
||||
Description: "Compute INNER JOIN (5) versus LEFT JOIN (6) row counts between an inline orders and customers table.",
|
||||
Prompt: prompt,
|
||||
Eval: evaluator,
|
||||
}
|
||||
}
|
||||
|
||||
// dbSQLCountRegionNotEU counts rows where "region <> 'EU'" evaluates to
|
||||
// SQL's three-valued TRUE. A row whose Region is SQL NULL (Region == "")
|
||||
// makes that comparison evaluate to UNKNOWN, not TRUE, so it is excluded -
|
||||
// not counted as matching EU and not counted as matching the negation
|
||||
// either.
|
||||
func dbSQLCountRegionNotEU(rows []dbSQLOrder) int {
|
||||
count := 0
|
||||
for _, r := range rows {
|
||||
if r.Region == "" {
|
||||
continue // NULL: comparison is UNKNOWN, row excluded either way
|
||||
}
|
||||
if r.Region != "EU" {
|
||||
count++
|
||||
}
|
||||
}
|
||||
return count
|
||||
}
|
||||
|
||||
// dbSQLNullInWhereWant is derived by calling dbSQLCountRegionNotEU, not
|
||||
// hardcoded.
|
||||
//
|
||||
// ground truth: region values are EU(id1,3,7), US(id2,5,8), APAC(id6),
|
||||
// NULL(id4). "region <> 'EU'" is TRUE for the 4 US/APAC rows, FALSE for
|
||||
// the 3 EU rows, and UNKNOWN (excluded) for the 1 NULL row - the trap
|
||||
// being that a naive 8-3=5 count forgets NULL is excluded from BOTH sides
|
||||
// of the comparison. databases_sqltuning_test.go recomputes this
|
||||
// independently with a hand-written loop.
|
||||
var dbSQLNullInWhereWant = dbSQLCountRegionNotEU(dbSQLOrders)
|
||||
|
||||
// dbSQLNullInWhereTest: compute the row count of a WHERE clause negating a
|
||||
// column that contains a NULL, testing whether the model accounts for
|
||||
// SQL's three-valued logic (NULL never satisfies <>, so it is excluded
|
||||
// from both the match and its negation).
|
||||
func dbSQLNullInWhereTest() testkit.Test {
|
||||
prompt := `Here is the "orders" table (region is NULL for order id 4):
|
||||
|
||||
` + "```\n" + dbSQLOrdersTableText + "\n```" + `
|
||||
|
||||
What does "SELECT COUNT(*) FROM orders WHERE region <> 'EU';" return?
|
||||
Remember SQL's three-valued logic: a comparison against NULL evaluates to
|
||||
UNKNOWN, not TRUE or FALSE, so a NULL row satisfies neither
|
||||
"region = 'EU'" nor "region <> 'EU'". Respond with only the number.`
|
||||
|
||||
return testkit.Test{
|
||||
ID: "sql-null-in-where",
|
||||
Category: "databases",
|
||||
Subcategory: "sql-tuning",
|
||||
Description: "Compute COUNT(*) WHERE region <> 'EU' over an inline table with one NULL region, testing the NULL-exclusion trap.",
|
||||
Prompt: prompt,
|
||||
Eval: eval.Numeric(eval.ExtractLastNumber[int], dbSQLNullInWhereWant, 0),
|
||||
}
|
||||
}
|
||||
|
||||
// dbSQLRegionTotal is one row of a GROUP BY region result: the region name
|
||||
// and its summed amount.
|
||||
type dbSQLRegionTotal struct {
|
||||
Region string
|
||||
Total int
|
||||
}
|
||||
|
||||
// dbSQLGroupByRegionHaving computes SUM(amount) per non-NULL region, keeps
|
||||
// only groups whose total exceeds minTotal, and returns them sorted by
|
||||
// region ascending - the query-logic equivalent of "... WHERE region IS
|
||||
// NOT NULL GROUP BY region HAVING SUM(amount) > minTotal ORDER BY region".
|
||||
func dbSQLGroupByRegionHaving(rows []dbSQLOrder, minTotal int) []dbSQLRegionTotal {
|
||||
totals := make(map[string]int)
|
||||
var regions []string
|
||||
for _, r := range rows {
|
||||
if r.Region == "" {
|
||||
continue
|
||||
}
|
||||
if _, seen := totals[r.Region]; !seen {
|
||||
regions = append(regions, r.Region)
|
||||
}
|
||||
totals[r.Region] += r.Amount
|
||||
}
|
||||
sort.Strings(regions)
|
||||
|
||||
var out []dbSQLRegionTotal
|
||||
for _, region := range regions {
|
||||
if totals[region] > minTotal {
|
||||
out = append(out, dbSQLRegionTotal{Region: region, Total: totals[region]})
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// dbSQLGroupByHavingWant is derived by calling dbSQLGroupByRegionHaving,
|
||||
// not hardcoded.
|
||||
//
|
||||
// ground truth: non-NULL region totals are EU = 250+75+125 = 450, US =
|
||||
// 100+50+60 = 210, APAC = 400. HAVING SUM(amount) > 300 keeps EU (450) and
|
||||
// APAC (400), drops US (210). Sorted by region ascending: APAC before EU.
|
||||
// databases_sqltuning_test.go recomputes this independently with a
|
||||
// hand-written map + sort.
|
||||
var dbSQLGroupByHavingWant = dbSQLGroupByRegionHaving(dbSQLOrders, 300)
|
||||
|
||||
// dbSQLGroupByHavingTest: compute a GROUP BY + HAVING result (region
|
||||
// totals over 300) over the inline orders table, as an ordered JSON array.
|
||||
func dbSQLGroupByHavingTest() testkit.Test {
|
||||
prompt := `Here is the "orders" table:
|
||||
|
||||
` + "```\n" + dbSQLOrdersTableText + "\n```" + `
|
||||
|
||||
What does this query return?
|
||||
|
||||
SELECT region, SUM(amount) AS total
|
||||
FROM orders
|
||||
WHERE region IS NOT NULL
|
||||
GROUP BY region
|
||||
HAVING SUM(amount) > 300
|
||||
ORDER BY region;
|
||||
|
||||
Respond with only a JSON array of objects, ordered exactly as the query's
|
||||
ORDER BY would return them: [{"region":"...","total":<number>}, ...]`
|
||||
|
||||
evaluator := eval.Mean(
|
||||
dbJSONArrayLength(len(dbSQLGroupByHavingWant)),
|
||||
eval.JSONField("[0].region", dbSQLGroupByHavingWant[0].Region),
|
||||
eval.JSONField("[0].total", dbSQLGroupByHavingWant[0].Total),
|
||||
eval.JSONField("[1].region", dbSQLGroupByHavingWant[1].Region),
|
||||
eval.JSONField("[1].total", dbSQLGroupByHavingWant[1].Total),
|
||||
)
|
||||
|
||||
return testkit.Test{
|
||||
ID: "sql-groupby-having",
|
||||
Category: "databases",
|
||||
Subcategory: "sql-tuning",
|
||||
Description: "Compute a GROUP BY region + HAVING SUM(amount) > 300 result (APAC, EU) over the inline orders table, as an ordered JSON array.",
|
||||
Prompt: prompt,
|
||||
Eval: evaluator,
|
||||
}
|
||||
}
|
||||
|
||||
// dbSQLEmployee is one row of the shared inline "employees" table fixture
|
||||
// used by dbSQLWindowRankTest.
|
||||
type dbSQLEmployee struct {
|
||||
Name string
|
||||
Department string
|
||||
Salary int
|
||||
}
|
||||
|
||||
// dbSQLEmployees is the shared inline "employees" table fixture, printed
|
||||
// verbatim into dbSQLWindowRankTest's prompt. No department has a salary
|
||||
// tie, so RANK() ... = 1 identifies exactly one employee per department.
|
||||
var dbSQLEmployees = []dbSQLEmployee{
|
||||
{Name: "Alice", Department: "Eng", Salary: 140},
|
||||
{Name: "Bob", Department: "Eng", Salary: 120},
|
||||
{Name: "Carol", Department: "Sales", Salary: 110},
|
||||
{Name: "Dave", Department: "Sales", Salary: 130},
|
||||
{Name: "Erin", Department: "Eng", Salary: 150},
|
||||
{Name: "Frank", Department: "Sales", Salary: 90},
|
||||
}
|
||||
|
||||
// dbSQLEmployeesTableText is dbSQLEmployees rendered as a plain-text table
|
||||
// for the prompt.
|
||||
const dbSQLEmployeesTableText = `name | department | salary
|
||||
Alice | Eng | 140
|
||||
Bob | Eng | 120
|
||||
Carol | Sales | 110
|
||||
Dave | Sales | 130
|
||||
Erin | Eng | 150
|
||||
Frank | Sales | 90`
|
||||
|
||||
// dbSQLTopEarnerByDepartment returns, for each department present in rows,
|
||||
// the name of the employee with the highest salary in that department -
|
||||
// the query-logic equivalent of "RANK() OVER (PARTITION BY department
|
||||
// ORDER BY salary DESC) = 1", assuming no in-department salary tie.
|
||||
func dbSQLTopEarnerByDepartment(rows []dbSQLEmployee) map[string]string {
|
||||
best := make(map[string]dbSQLEmployee, len(rows))
|
||||
for _, e := range rows {
|
||||
cur, ok := best[e.Department]
|
||||
if !ok || e.Salary > cur.Salary {
|
||||
best[e.Department] = e
|
||||
}
|
||||
}
|
||||
out := make(map[string]string, len(best))
|
||||
for dept, e := range best {
|
||||
out[dept] = e.Name
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// dbSQLWindowRankWant is derived by calling dbSQLTopEarnerByDepartment, not
|
||||
// hardcoded.
|
||||
//
|
||||
// ground truth: Eng salaries are Alice=140, Bob=120, Erin=150 - Erin is
|
||||
// highest. Sales salaries are Carol=110, Dave=130, Frank=90 - Dave is
|
||||
// highest. databases_sqltuning_test.go recomputes this independently with
|
||||
// a hand-written loop.
|
||||
var dbSQLWindowRankWant = dbSQLTopEarnerByDepartment(dbSQLEmployees)
|
||||
|
||||
// dbSQLWindowRankTest: identify the rank-1 (highest-salary) employee per
|
||||
// department from a RANK() OVER (PARTITION BY ...) window function applied
|
||||
// to an inline employees table.
|
||||
func dbSQLWindowRankTest() testkit.Test {
|
||||
prompt := `Here is the "employees" table:
|
||||
|
||||
` + "```\n" + dbSQLEmployeesTableText + "\n```" + `
|
||||
|
||||
For "SELECT department, name, RANK() OVER (PARTITION BY department ORDER
|
||||
BY salary DESC) AS rnk FROM employees;", which employee's name has rnk = 1
|
||||
in each department? Respond with only a JSON object:
|
||||
{"Eng":"<name>","Sales":"<name>"}`
|
||||
|
||||
evaluator := eval.Mean(
|
||||
eval.JSONField("Eng", dbSQLWindowRankWant["Eng"]),
|
||||
eval.JSONField("Sales", dbSQLWindowRankWant["Sales"]),
|
||||
)
|
||||
|
||||
return testkit.Test{
|
||||
ID: "sql-window-rank",
|
||||
Category: "databases",
|
||||
Subcategory: "sql-tuning",
|
||||
Description: "Identify the RANK()=1 (highest-salary) employee per department from an inline 6-row employees table.",
|
||||
Prompt: prompt,
|
||||
Eval: evaluator,
|
||||
}
|
||||
}
|
||||
|
||||
// dbSQLCompositeIndexSchema is the inline schema + query for
|
||||
// dbSQLCompositeIndexOrderTest.
|
||||
const dbSQLCompositeIndexSchema = `Table: sessions(id bigserial, user_id bigint, created_at timestamptz, ip inet)
|
||||
|
||||
Query:
|
||||
SELECT * FROM sessions
|
||||
WHERE user_id = $1 AND created_at > $2
|
||||
ORDER BY created_at DESC
|
||||
LIMIT 50;`
|
||||
|
||||
// dbSQLCompositeIndexOrderTest: pick the composite index column order
|
||||
// (equality column first, then the range/sort column) that best serves an
|
||||
// inline query.
|
||||
//
|
||||
// ground truth: user_id is filtered by exact equality; created_at is
|
||||
// filtered by a range AND drives ORDER BY. Leading a composite index with
|
||||
// the equality column narrows to one user's rows first; following with
|
||||
// created_at then serves both the range filter and the sort from the
|
||||
// index directly. Reversing the order (created_at, user_id) would force
|
||||
// scanning every session in the time range across all users before
|
||||
// filtering by user_id.
|
||||
func dbSQLCompositeIndexOrderTest() testkit.Test {
|
||||
prompt := `Here is a table schema and a query that runs frequently:
|
||||
|
||||
` + "```\n" + dbSQLCompositeIndexSchema + "\n```" + `
|
||||
|
||||
Which two columns, in which order, should a single composite index be
|
||||
defined on to best serve this exact query (equality filters should lead a
|
||||
composite index, ahead of range/sort columns)? Respond with only a JSON
|
||||
array of the two column names in index-definition order, e.g. ["a","b"].`
|
||||
|
||||
return testkit.Test{
|
||||
ID: "sql-composite-index-order",
|
||||
Category: "databases",
|
||||
Subcategory: "sql-tuning",
|
||||
Description: "Pick composite index column order (user_id, created_at) for an inline equality+range+sort query.",
|
||||
Prompt: prompt,
|
||||
Eval: eval.JSONStringArrayEquals([]string{"user_id", "created_at"}),
|
||||
}
|
||||
}
|
||||
|
||||
// dbSQLKeysetPaginationTest: pick keyset (seek) pagination over OFFSET
|
||||
// pagination for a deep-paging scenario over a huge table.
|
||||
//
|
||||
// ground truth: "LIMIT 100 OFFSET 20000000" forces the database to walk
|
||||
// and discard 20,000,000 rows before returning the next 100, so its cost
|
||||
// grows with how deep the user pages. Keyset/seek pagination
|
||||
// ("WHERE id > :last_seen_id ORDER BY id LIMIT 100") instead jumps
|
||||
// straight to the right position via an index seek, so its latency stays
|
||||
// roughly constant no matter how deep the user has paged.
|
||||
func dbSQLKeysetPaginationTest() testkit.Test {
|
||||
prompt := `A table has 50,000,000 rows. An API paginates results ordered
|
||||
by primary key id ascending, 100 rows per page. A user navigates deep into
|
||||
the results - page 200,000, equivalent to OFFSET 20,000,000.
|
||||
|
||||
Which pagination technique keeps query latency roughly constant regardless
|
||||
of how deep the user pages: OFFSET-based pagination
|
||||
("LIMIT 100 OFFSET 20000000"), or keyset/seek pagination
|
||||
("WHERE id > :last_seen_id ORDER BY id LIMIT 100")? Respond with only one
|
||||
word: offset or keyset.`
|
||||
|
||||
return testkit.Test{
|
||||
ID: "sql-keyset-pagination",
|
||||
Category: "databases",
|
||||
Subcategory: "sql-tuning",
|
||||
Description: "Pick keyset/seek pagination over OFFSET pagination for constant latency 20M rows deep into a 50M-row table.",
|
||||
Prompt: prompt,
|
||||
Eval: dbExactAnswer("keyset"),
|
||||
}
|
||||
}
|
||||
|
||||
// dbSQLCoveringIndexTest: confirm that a composite index containing every
|
||||
// column a query needs allows an index-only scan, with no heap visit.
|
||||
//
|
||||
// ground truth: the query only reads customer_id and status, and both are
|
||||
// present in the index (customer_id, status) itself. Since every column
|
||||
// the query needs is already inside the index entry, Postgres can answer
|
||||
// the query by reading the index alone - an index-only scan - without
|
||||
// visiting the underlying table (the heap) for any row, provided the
|
||||
// visibility map allows it (as the prompt states).
|
||||
func dbSQLCoveringIndexTest() testkit.Test {
|
||||
prompt := `Query: SELECT customer_id, status FROM orders WHERE customer_id = 42;
|
||||
|
||||
An index exists on orders(customer_id, status) - both columns this query
|
||||
reads are present in the index itself. Assuming the visibility map allows
|
||||
an index-only scan, can Postgres answer this query using only the index,
|
||||
without visiting the underlying table (the heap) for any row? Respond with
|
||||
only "yes" or "no".`
|
||||
|
||||
return testkit.Test{
|
||||
ID: "sql-covering-index",
|
||||
Category: "databases",
|
||||
Subcategory: "sql-tuning",
|
||||
Description: "Confirm an index containing every column a query needs (customer_id, status) enables an index-only scan with no heap visit.",
|
||||
Prompt: prompt,
|
||||
Eval: dbExactAnswer("yes"),
|
||||
}
|
||||
}
|
||||
|
||||
// dbSQLEquivalentRewriteQueries is the inline original query and 3
|
||||
// rewrite candidates for dbSQLEquivalentRewriteTest.
|
||||
const dbSQLEquivalentRewriteQueries = `Original: SELECT * FROM orders WHERE status = 'paid' OR status = 'pending';
|
||||
|
||||
Candidate A: SELECT * FROM orders WHERE status IN ('paid', 'pending');
|
||||
Candidate B: SELECT * FROM orders WHERE status = 'paid' AND status = 'pending';
|
||||
Candidate C: SELECT * FROM orders WHERE status != 'refunded';`
|
||||
|
||||
// dbSQLEquivalentRewriteTest: judge which of 3 rewrite candidates are
|
||||
// semantically equivalent to an inline original query, given a stated
|
||||
// closed set of possible column values.
|
||||
//
|
||||
// ground truth: status only ever takes one of exactly 3 values in this
|
||||
// table: 'paid', 'pending', 'refunded' (stated in the prompt so C's
|
||||
// equivalence is well-defined). Candidate A: "status IN (x, y)" is
|
||||
// standard SQL sugar for "status = x OR status = y" - always equivalent
|
||||
// to the original. Candidate B: a single column cannot equal two
|
||||
// different literal values in the same row, so "status = 'paid' AND
|
||||
// status = 'pending'" is always FALSE - never equivalent to a query that
|
||||
// is often TRUE. Candidate C: since status only ever takes one of exactly
|
||||
// 3 values, "status != 'refunded'" is TRUE exactly when status is 'paid'
|
||||
// or 'pending' - equivalent to the original given that closed value set
|
||||
// (it would NOT be equivalent if a 4th status value existed).
|
||||
func dbSQLEquivalentRewriteTest() testkit.Test {
|
||||
prompt := `The "status" column in this table only ever holds one of
|
||||
exactly 3 values: 'paid', 'pending', or 'refunded'.
|
||||
|
||||
` + dbSQLEquivalentRewriteQueries + `
|
||||
|
||||
Which of Candidate A, B, C are semantically equivalent to the Original
|
||||
query (return exactly the same rows for any data satisfying that 3-value
|
||||
constraint)? Respond with only a JSON object:
|
||||
{"A":true|false,"B":true|false,"C":true|false}`
|
||||
|
||||
evaluator := eval.Mean(
|
||||
eval.JSONField("A", true),
|
||||
eval.JSONField("B", false),
|
||||
eval.JSONField("C", true),
|
||||
)
|
||||
|
||||
return testkit.Test{
|
||||
ID: "sql-equivalent-rewrite",
|
||||
Category: "databases",
|
||||
Subcategory: "sql-tuning",
|
||||
Description: "Judge 3 rewrite candidates (IN-sugar equivalent, impossible-AND non-equivalent, closed-set-negation equivalent) against an inline original OR query.",
|
||||
Prompt: prompt,
|
||||
Eval: evaluator,
|
||||
}
|
||||
}
|
||||
|
||||
// dbSQLOrderByIndexServingSchema is the inline index definition + query
|
||||
// for dbSQLOrderByIndexServingTest.
|
||||
const dbSQLOrderByIndexServingSchema = `Index: CREATE INDEX idx_orders_status_created_at ON orders(status, created_at DESC);
|
||||
|
||||
Query:
|
||||
SELECT * FROM orders
|
||||
WHERE status = 'paid'
|
||||
ORDER BY created_at DESC
|
||||
LIMIT 20;`
|
||||
|
||||
// dbSQLOrderByIndexServingTest: judge whether an inline composite index
|
||||
// can serve both a query's WHERE filter and its ORDER BY without a
|
||||
// separate sort step.
|
||||
//
|
||||
// ground truth: the index leads with status (an exact equality match in
|
||||
// the query), and its second column, created_at DESC, exactly matches
|
||||
// both the query's remaining filter role and its ORDER BY created_at DESC
|
||||
// direction. Postgres can therefore walk the matching status='paid'
|
||||
// portion of the index in exactly the order the query needs, serving the
|
||||
// ORDER BY directly from the index with no separate sort step.
|
||||
func dbSQLOrderByIndexServingTest() testkit.Test {
|
||||
prompt := `Here is an index and a query:
|
||||
|
||||
` + "```\n" + dbSQLOrderByIndexServingSchema + "\n```" + `
|
||||
|
||||
Can Postgres use this index to satisfy both the WHERE filter and the
|
||||
ORDER BY without a separate sort step? Respond with only a JSON object:
|
||||
{"avoids_sort_step":true|false,"reason":"<one of: index-matches-filter-and-order, order-by-column-not-leading, sort-direction-mismatch>"}`
|
||||
|
||||
evaluator := eval.Mean(
|
||||
eval.JSONField("avoids_sort_step", true),
|
||||
eval.JSONField("reason", "index-matches-filter-and-order"),
|
||||
)
|
||||
|
||||
return testkit.Test{
|
||||
ID: "sql-orderby-index-serving",
|
||||
Category: "databases",
|
||||
Subcategory: "sql-tuning",
|
||||
Description: "Judge that an index on (status, created_at DESC) serves both an equality WHERE and an ORDER BY created_at DESC with no extra sort step.",
|
||||
Prompt: prompt,
|
||||
Eval: evaluator,
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,393 @@
|
||||
package tests
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sort"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// TestDBSQLAggregateSumWant_GroundTruth independently re-derives the paid-
|
||||
// orders total by listing the paid amounts by hand, not by calling
|
||||
// dbSQLSumPaidAmount.
|
||||
func TestDBSQLAggregateSumWant_GroundTruth(t *testing.T) {
|
||||
paidAmounts := []int{250, 100, 300, 400, 125} // ids 1, 2, 4, 6, 7
|
||||
want := 0
|
||||
for _, a := range paidAmounts {
|
||||
want += a
|
||||
}
|
||||
|
||||
if want != 1175 {
|
||||
t.Fatalf("independently recomputed sum = %d, want 1175", want)
|
||||
}
|
||||
if dbSQLAggregateSumWant != want {
|
||||
t.Errorf("dbSQLAggregateSumWant = %d, independently recomputed = %d", dbSQLAggregateSumWant, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDBSQLAggregateSumTest_Eval(t *testing.T) {
|
||||
tc := dbSQLAggregateSumTest()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{"correct", "1175", 1},
|
||||
{"prose wrapped", "The sum is 1175.", 1},
|
||||
{"wrong: included a pending row", "1250", 0},
|
||||
{"wrong: excluded a paid row", "875", 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestDBSQLJoinCountsWant_GroundTruth independently re-derives both join
|
||||
// counts from a hand-written per-customer_id tally, not by calling
|
||||
// dbSQLInnerJoinCount/dbSQLLeftJoinCount.
|
||||
func TestDBSQLJoinCountsWant_GroundTruth(t *testing.T) {
|
||||
orderCustomerIDs := []int{101, 102, 101, 103, 102, 104, 101, 103} // ids 1..8
|
||||
knownCustomers := map[int]bool{101: true, 102: true, 105: true}
|
||||
|
||||
innerWant := 0
|
||||
perCustomerOrders := map[int]int{}
|
||||
for _, cid := range orderCustomerIDs {
|
||||
perCustomerOrders[cid]++
|
||||
if knownCustomers[cid] {
|
||||
innerWant++
|
||||
}
|
||||
}
|
||||
if innerWant != 5 {
|
||||
t.Fatalf("independently recomputed inner join count = %d, want 5", innerWant)
|
||||
}
|
||||
|
||||
leftWant := 0
|
||||
for cid := range knownCustomers {
|
||||
if perCustomerOrders[cid] == 0 {
|
||||
leftWant++ // one NULL-order row
|
||||
} else {
|
||||
leftWant += perCustomerOrders[cid]
|
||||
}
|
||||
}
|
||||
if leftWant != 6 {
|
||||
t.Fatalf("independently recomputed left join count = %d, want 6", leftWant)
|
||||
}
|
||||
|
||||
if dbSQLInnerJoinCountWant != innerWant {
|
||||
t.Errorf("dbSQLInnerJoinCountWant = %d, independently recomputed = %d", dbSQLInnerJoinCountWant, innerWant)
|
||||
}
|
||||
if dbSQLLeftJoinCountWant != leftWant {
|
||||
t.Errorf("dbSQLLeftJoinCountWant = %d, independently recomputed = %d", dbSQLLeftJoinCountWant, leftWant)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDBSQLJoinCountsTest_Eval(t *testing.T) {
|
||||
tc := dbSQLJoinCountsTest()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{"correct", `{"inner_join_count":5,"left_join_count":6}`, 1},
|
||||
{"correct fenced", "```json\n{\"inner_join_count\":5,\"left_join_count\":6}\n```", 1},
|
||||
{"wrong inner count", `{"inner_join_count":4,"left_join_count":6}`, 0.5},
|
||||
{"wrong left count: forgot the zero-match customer's NULL row", `{"inner_join_count":5,"left_join_count":5}`, 0.5},
|
||||
{"both wrong: treated inner and left as equal", `{"inner_join_count":8,"left_join_count":8}`, 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestDBSQLNullInWhereWant_GroundTruth independently re-derives the
|
||||
// region <> 'EU' count by listing every row's region by hand, not by
|
||||
// calling dbSQLCountRegionNotEU.
|
||||
func TestDBSQLNullInWhereWant_GroundTruth(t *testing.T) {
|
||||
regions := []string{"EU", "US", "EU", "", "US", "APAC", "EU", "US"} // ids 1..8, "" = NULL
|
||||
want := 0
|
||||
for _, region := range regions {
|
||||
if region == "" {
|
||||
continue // NULL: <> 'EU' is UNKNOWN, excluded
|
||||
}
|
||||
if region != "EU" {
|
||||
want++
|
||||
}
|
||||
}
|
||||
|
||||
if want != 4 {
|
||||
t.Fatalf("independently recomputed count = %d, want 4", want)
|
||||
}
|
||||
if dbSQLNullInWhereWant != want {
|
||||
t.Errorf("dbSQLNullInWhereWant = %d, independently recomputed = %d", dbSQLNullInWhereWant, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDBSQLNullInWhereTest_Eval(t *testing.T) {
|
||||
tc := dbSQLNullInWhereTest()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{"correct", "4", 1},
|
||||
{"prose wrapped", "The count is 4.", 1},
|
||||
{"wrong: forgot NULL exclusion (8 - 3 EU rows)", "5", 0},
|
||||
{"wrong: counted the NULL row as non-EU", "5", 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestDBSQLGroupByHavingWant_GroundTruth independently re-derives the
|
||||
// GROUP BY + HAVING result from hand-written region totals, not by calling
|
||||
// dbSQLGroupByRegionHaving.
|
||||
func TestDBSQLGroupByHavingWant_GroundTruth(t *testing.T) {
|
||||
totals := map[string]int{
|
||||
"EU": 250 + 75 + 125,
|
||||
"US": 100 + 50 + 60,
|
||||
"APAC": 400,
|
||||
}
|
||||
if totals["EU"] != 450 || totals["US"] != 210 || totals["APAC"] != 400 {
|
||||
t.Fatalf("independently recomputed totals = %v, want EU=450 US=210 APAC=400", totals)
|
||||
}
|
||||
|
||||
var kept []string
|
||||
for region, total := range totals {
|
||||
if total > 300 {
|
||||
kept = append(kept, region)
|
||||
}
|
||||
}
|
||||
sort.Strings(kept)
|
||||
|
||||
if len(kept) != 2 || kept[0] != "APAC" || kept[1] != "EU" {
|
||||
t.Fatalf("independently recomputed kept regions = %v, want [APAC EU]", kept)
|
||||
}
|
||||
|
||||
if len(dbSQLGroupByHavingWant) != 2 {
|
||||
t.Fatalf("dbSQLGroupByHavingWant has %d entries, want 2", len(dbSQLGroupByHavingWant))
|
||||
}
|
||||
if dbSQLGroupByHavingWant[0].Region != "APAC" || dbSQLGroupByHavingWant[0].Total != 400 {
|
||||
t.Errorf("dbSQLGroupByHavingWant[0] = %+v, want {APAC 400}", dbSQLGroupByHavingWant[0])
|
||||
}
|
||||
if dbSQLGroupByHavingWant[1].Region != "EU" || dbSQLGroupByHavingWant[1].Total != 450 {
|
||||
t.Errorf("dbSQLGroupByHavingWant[1] = %+v, want {EU 450}", dbSQLGroupByHavingWant[1])
|
||||
}
|
||||
}
|
||||
|
||||
func TestDBSQLGroupByHavingTest_Eval(t *testing.T) {
|
||||
tc := dbSQLGroupByHavingTest()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{"correct", `[{"region":"APAC","total":400},{"region":"EU","total":450}]`, 1},
|
||||
{"correct fenced with prose", "Result:\n```json\n[{\"region\":\"APAC\",\"total\":400},{\"region\":\"EU\",\"total\":450}]\n```", 1},
|
||||
{"wrong order", `[{"region":"EU","total":450},{"region":"APAC","total":400}]`, 0.2},
|
||||
{"missing a group", `[{"region":"APAC","total":400}]`, 0.4},
|
||||
{"wrong total for one group", `[{"region":"APAC","total":401},{"region":"EU","total":450}]`, 0.8},
|
||||
{"forgot HAVING filter, includes US", `[{"region":"APAC","total":400},{"region":"EU","total":450},{"region":"US","total":210}]`, 0.8},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestDBSQLWindowRankWant_GroundTruth independently re-derives the top
|
||||
// earner per department from hand-written salary lists, not by calling
|
||||
// dbSQLTopEarnerByDepartment.
|
||||
func TestDBSQLWindowRankWant_GroundTruth(t *testing.T) {
|
||||
eng := map[string]int{"Alice": 140, "Bob": 120, "Erin": 150}
|
||||
sales := map[string]int{"Carol": 110, "Dave": 130, "Frank": 90}
|
||||
|
||||
topOf := func(salaries map[string]int) string {
|
||||
best, bestSalary := "", -1
|
||||
for name, salary := range salaries {
|
||||
if salary > bestSalary {
|
||||
best, bestSalary = name, salary
|
||||
}
|
||||
}
|
||||
return best
|
||||
}
|
||||
|
||||
if got := topOf(eng); got != "Erin" {
|
||||
t.Fatalf("independently recomputed Eng top earner = %q, want Erin", got)
|
||||
}
|
||||
if got := topOf(sales); got != "Dave" {
|
||||
t.Fatalf("independently recomputed Sales top earner = %q, want Dave", got)
|
||||
}
|
||||
|
||||
if dbSQLWindowRankWant["Eng"] != "Erin" {
|
||||
t.Errorf(`dbSQLWindowRankWant["Eng"] = %q, want "Erin"`, dbSQLWindowRankWant["Eng"])
|
||||
}
|
||||
if dbSQLWindowRankWant["Sales"] != "Dave" {
|
||||
t.Errorf(`dbSQLWindowRankWant["Sales"] = %q, want "Dave"`, dbSQLWindowRankWant["Sales"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestDBSQLWindowRankTest_Eval(t *testing.T) {
|
||||
tc := dbSQLWindowRankTest()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{"correct", `{"Eng":"Erin","Sales":"Dave"}`, 1},
|
||||
{"correct fenced", "```json\n{\"Eng\":\"Erin\",\"Sales\":\"Dave\"}\n```", 1},
|
||||
{"wrong Eng", `{"Eng":"Alice","Sales":"Dave"}`, 0.5},
|
||||
{"wrong Sales", `{"Eng":"Erin","Sales":"Carol"}`, 0.5},
|
||||
{"both wrong", `{"Eng":"Bob","Sales":"Frank"}`, 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDBSQLCompositeIndexOrderTest_Eval(t *testing.T) {
|
||||
tc := dbSQLCompositeIndexOrderTest()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{"correct", `["user_id", "created_at"]`, 1},
|
||||
{"correct fenced", "```json\n[\"user_id\", \"created_at\"]\n```", 1},
|
||||
{"reversed order", `["created_at", "user_id"]`, 0},
|
||||
{"missing a column", `["user_id"]`, 0},
|
||||
{"wrong column entirely", `["user_id", "ip"]`, 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDBSQLKeysetPaginationTest_Eval(t *testing.T) {
|
||||
tc := dbSQLKeysetPaginationTest()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{"bare correct", "keyset", 1},
|
||||
{"quoted correct", `"keyset"`, 1},
|
||||
{"different case", "Keyset", 1},
|
||||
{"trailing period", "keyset.", 1},
|
||||
{"wrong", "offset", 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDBSQLCoveringIndexTest_Eval(t *testing.T) {
|
||||
tc := dbSQLCoveringIndexTest()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{"bare correct", "yes", 1},
|
||||
{"quoted correct", `"yes"`, 1},
|
||||
{"trailing period", "Yes.", 1},
|
||||
{"wrong", "no", 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDBSQLEquivalentRewriteTest_Eval(t *testing.T) {
|
||||
tc := dbSQLEquivalentRewriteTest()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{"correct", `{"A":true,"B":false,"C":true}`, 1},
|
||||
{"correct fenced with prose", "Here's my analysis:\n```json\n{\"A\":true,\"B\":false,\"C\":true}\n```", 1},
|
||||
{"wrongly accepts B (impossible AND)", `{"A":true,"B":true,"C":true}`, 2.0 / 3.0},
|
||||
{"wrongly rejects C (forgets closed value set)", `{"A":true,"B":false,"C":false}`, 2.0 / 3.0},
|
||||
{"everything wrong", `{"A":false,"B":true,"C":false}`, 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDBSQLOrderByIndexServingTest_Eval(t *testing.T) {
|
||||
tc := dbSQLOrderByIndexServingTest()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{"correct", `{"avoids_sort_step":true,"reason":"index-matches-filter-and-order"}`, 1},
|
||||
{"correct fenced", "```json\n{\"avoids_sort_step\":true,\"reason\":\"index-matches-filter-and-order\"}\n```", 1},
|
||||
{"wrongly says a sort step is needed", `{"avoids_sort_step":false,"reason":"index-matches-filter-and-order"}`, 0.5},
|
||||
{"wrong reason", `{"avoids_sort_step":true,"reason":"order-by-column-not-leading"}`, 0.5},
|
||||
{"both wrong", `{"avoids_sort_step":false,"reason":"sort-direction-mismatch"}`, 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,58 @@
|
||||
package tests
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/lukaszraczylo/llm-testbench/internal/testkit"
|
||||
)
|
||||
|
||||
// TestRegisterDatabasesTests_Wiring checks registerDatabasesTests wires up
|
||||
// all three databases subcategories - postgres, redis, sql-tuning - with 10
|
||||
// tests each, all categorized "databases", all with the expected ID prefix
|
||||
// per subcategory, MaxTokens omitted (0, per the round-2 authoring rule),
|
||||
// and no duplicate IDs across the three sibling files (testkit.Registry.
|
||||
// Register panics on a duplicate ID, which would fail this test outright).
|
||||
func TestRegisterDatabasesTests_Wiring(t *testing.T) {
|
||||
r := testkit.NewRegistry()
|
||||
registerDatabasesTests(r)
|
||||
|
||||
const wantTotal = 30
|
||||
if r.Len() != wantTotal {
|
||||
t.Fatalf("registerDatabasesTests: registry has %d tests, want %d", r.Len(), wantTotal)
|
||||
}
|
||||
|
||||
wantPrefix := map[string]string{
|
||||
"postgres": "pg-",
|
||||
"redis": "redis-",
|
||||
"sql-tuning": "sql-",
|
||||
}
|
||||
|
||||
counts := map[string]int{}
|
||||
for _, tc := range r.All() {
|
||||
if tc.Category != "databases" {
|
||||
t.Errorf("test %q: Category = %q, want %q", tc.ID, tc.Category, "databases")
|
||||
}
|
||||
if tc.Prompt == "" {
|
||||
t.Errorf("test %q: Prompt is empty", tc.ID)
|
||||
}
|
||||
if tc.Eval == nil {
|
||||
t.Errorf("test %q: Eval is nil", tc.ID)
|
||||
}
|
||||
if tc.Description == "" {
|
||||
t.Errorf("test %q: Description is empty", tc.ID)
|
||||
}
|
||||
if tc.MaxTokens != 0 {
|
||||
t.Errorf("test %q: MaxTokens = %d, want 0 (omitted)", tc.ID, tc.MaxTokens)
|
||||
}
|
||||
if prefix, ok := wantPrefix[tc.Subcategory]; ok && !strings.HasPrefix(tc.ID, prefix) {
|
||||
t.Errorf("test %q: subcategory %q wants ID prefix %q", tc.ID, tc.Subcategory, prefix)
|
||||
}
|
||||
counts[tc.Subcategory]++
|
||||
}
|
||||
for _, sub := range []string{"postgres", "redis", "sql-tuning"} {
|
||||
if counts[sub] != 10 {
|
||||
t.Errorf("subcategory %q: got %d tests, want 10", sub, counts[sub])
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user