mirror of
https://github.com/lukaszraczylo/llm-testbench.git
synced 2026-10-14 00:57:45 +00:00
test(catalog): add databases category with 30 tests
Add a new databases category with three subcategories: postgres, redis, and sql-tuning. Each subcategory holds 10 self-contained tests with deterministic evaluators. Postgres tests cover EXPLAIN interpretation, index choice, N+1 detection, isolation anomalies, deadlocks, replica failover, bloat diagnosis, pool sizing, partial indexes, and LISTEN/NOTIFY. Redis tests cover structure choice, TTL eviction, INCR atomicity, MULTI versus pipelining, and SCAN versus KEYS. They also cover pub/sub delivery, Lua atomicity, memory estimation, cache stampede mitigation, and keyspace anti-patterns. The SCAN versus KEYS test uses a negation-aware evaluator, so a response that warns against KEYS does not lose credit for mentioning it. SQL-tuning tests trace query results against an inline orders table: aggregates, joins, NULL handling, GROUP BY plus HAVING, and window functions. The unit tests recompute every numeric or JSON answer independently, with a hand-written loop separate from the production query-logic helper. registerDatabasesTests registers the category. A wiring test verifies it directly, and the code does not touch catalog.go, so the orchestrator can wire the category in separately during integration.
This commit is contained in:
@@ -0,0 +1,58 @@
|
||||
package tests
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/lukaszraczylo/llm-testbench/internal/testkit"
|
||||
)
|
||||
|
||||
// TestRegisterDatabasesTests_Wiring checks registerDatabasesTests wires up
|
||||
// all three databases subcategories - postgres, redis, sql-tuning - with 10
|
||||
// tests each, all categorized "databases", all with the expected ID prefix
|
||||
// per subcategory, MaxTokens omitted (0, per the round-2 authoring rule),
|
||||
// and no duplicate IDs across the three sibling files (testkit.Registry.
|
||||
// Register panics on a duplicate ID, which would fail this test outright).
|
||||
func TestRegisterDatabasesTests_Wiring(t *testing.T) {
|
||||
r := testkit.NewRegistry()
|
||||
registerDatabasesTests(r)
|
||||
|
||||
const wantTotal = 30
|
||||
if r.Len() != wantTotal {
|
||||
t.Fatalf("registerDatabasesTests: registry has %d tests, want %d", r.Len(), wantTotal)
|
||||
}
|
||||
|
||||
wantPrefix := map[string]string{
|
||||
"postgres": "pg-",
|
||||
"redis": "redis-",
|
||||
"sql-tuning": "sql-",
|
||||
}
|
||||
|
||||
counts := map[string]int{}
|
||||
for _, tc := range r.All() {
|
||||
if tc.Category != "databases" {
|
||||
t.Errorf("test %q: Category = %q, want %q", tc.ID, tc.Category, "databases")
|
||||
}
|
||||
if tc.Prompt == "" {
|
||||
t.Errorf("test %q: Prompt is empty", tc.ID)
|
||||
}
|
||||
if tc.Eval == nil {
|
||||
t.Errorf("test %q: Eval is nil", tc.ID)
|
||||
}
|
||||
if tc.Description == "" {
|
||||
t.Errorf("test %q: Description is empty", tc.ID)
|
||||
}
|
||||
if tc.MaxTokens != 0 {
|
||||
t.Errorf("test %q: MaxTokens = %d, want 0 (omitted)", tc.ID, tc.MaxTokens)
|
||||
}
|
||||
if prefix, ok := wantPrefix[tc.Subcategory]; ok && !strings.HasPrefix(tc.ID, prefix) {
|
||||
t.Errorf("test %q: subcategory %q wants ID prefix %q", tc.ID, tc.Subcategory, prefix)
|
||||
}
|
||||
counts[tc.Subcategory]++
|
||||
}
|
||||
for _, sub := range []string{"postgres", "redis", "sql-tuning"} {
|
||||
if counts[sub] != 10 {
|
||||
t.Errorf("subcategory %q: got %d tests, want 10", sub, counts[sub])
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user