mirror of
https://github.com/lukaszraczylo/llm-testbench.git
synced 2026-10-12 00:45:19 +00:00
lukaszraczylo
0665d706f0
Add a new databases category with three subcategories: postgres, redis, and sql-tuning. Each subcategory holds 10 self-contained tests with deterministic evaluators. Postgres tests cover EXPLAIN interpretation, index choice, N+1 detection, isolation anomalies, deadlocks, replica failover, bloat diagnosis, pool sizing, partial indexes, and LISTEN/NOTIFY. Redis tests cover structure choice, TTL eviction, INCR atomicity, MULTI versus pipelining, and SCAN versus KEYS. They also cover pub/sub delivery, Lua atomicity, memory estimation, cache stampede mitigation, and keyspace anti-patterns. The SCAN versus KEYS test uses a negation-aware evaluator, so a response that warns against KEYS does not lose credit for mentioning it. SQL-tuning tests trace query results against an inline orders table: aggregates, joins, NULL handling, GROUP BY plus HAVING, and window functions. The unit tests recompute every numeric or JSON answer independently, with a hand-written loop separate from the production query-logic helper. registerDatabasesTests registers the category. A wiring test verifies it directly, and the code does not touch catalog.go, so the orchestrator can wire the category in separately during integration.
59 lines
1.8 KiB
Go
59 lines
1.8 KiB
Go
package tests
|
|
|
|
import (
|
|
"strings"
|
|
"testing"
|
|
|
|
"github.com/lukaszraczylo/llm-testbench/internal/testkit"
|
|
)
|
|
|
|
// TestRegisterDatabasesTests_Wiring checks registerDatabasesTests wires up
|
|
// all three databases subcategories - postgres, redis, sql-tuning - with 10
|
|
// tests each, all categorized "databases", all with the expected ID prefix
|
|
// per subcategory, MaxTokens omitted (0, per the round-2 authoring rule),
|
|
// and no duplicate IDs across the three sibling files (testkit.Registry.
|
|
// Register panics on a duplicate ID, which would fail this test outright).
|
|
func TestRegisterDatabasesTests_Wiring(t *testing.T) {
|
|
r := testkit.NewRegistry()
|
|
registerDatabasesTests(r)
|
|
|
|
const wantTotal = 30
|
|
if r.Len() != wantTotal {
|
|
t.Fatalf("registerDatabasesTests: registry has %d tests, want %d", r.Len(), wantTotal)
|
|
}
|
|
|
|
wantPrefix := map[string]string{
|
|
"postgres": "pg-",
|
|
"redis": "redis-",
|
|
"sql-tuning": "sql-",
|
|
}
|
|
|
|
counts := map[string]int{}
|
|
for _, tc := range r.All() {
|
|
if tc.Category != "databases" {
|
|
t.Errorf("test %q: Category = %q, want %q", tc.ID, tc.Category, "databases")
|
|
}
|
|
if tc.Prompt == "" {
|
|
t.Errorf("test %q: Prompt is empty", tc.ID)
|
|
}
|
|
if tc.Eval == nil {
|
|
t.Errorf("test %q: Eval is nil", tc.ID)
|
|
}
|
|
if tc.Description == "" {
|
|
t.Errorf("test %q: Description is empty", tc.ID)
|
|
}
|
|
if tc.MaxTokens != 0 {
|
|
t.Errorf("test %q: MaxTokens = %d, want 0 (omitted)", tc.ID, tc.MaxTokens)
|
|
}
|
|
if prefix, ok := wantPrefix[tc.Subcategory]; ok && !strings.HasPrefix(tc.ID, prefix) {
|
|
t.Errorf("test %q: subcategory %q wants ID prefix %q", tc.ID, tc.Subcategory, prefix)
|
|
}
|
|
counts[tc.Subcategory]++
|
|
}
|
|
for _, sub := range []string{"postgres", "redis", "sql-tuning"} {
|
|
if counts[sub] != 10 {
|
|
t.Errorf("subcategory %q: got %d tests, want 10", sub, counts[sub])
|
|
}
|
|
}
|
|
}
|