mirror of
https://github.com/lukaszraczylo/llm-testbench.git
synced 2026-10-14 00:57:45 +00:00
test(catalog): add databases category with 30 tests
Add a new databases category with three subcategories: postgres, redis, and sql-tuning. Each subcategory holds 10 self-contained tests with deterministic evaluators. Postgres tests cover EXPLAIN interpretation, index choice, N+1 detection, isolation anomalies, deadlocks, replica failover, bloat diagnosis, pool sizing, partial indexes, and LISTEN/NOTIFY. Redis tests cover structure choice, TTL eviction, INCR atomicity, MULTI versus pipelining, and SCAN versus KEYS. They also cover pub/sub delivery, Lua atomicity, memory estimation, cache stampede mitigation, and keyspace anti-patterns. The SCAN versus KEYS test uses a negation-aware evaluator, so a response that warns against KEYS does not lose credit for mentioning it. SQL-tuning tests trace query results against an inline orders table: aggregates, joins, NULL handling, GROUP BY plus HAVING, and window functions. The unit tests recompute every numeric or JSON answer independently, with a hand-written loop separate from the production query-logic helper. registerDatabasesTests registers the category. A wiring test verifies it directly, and the code does not touch catalog.go, so the orchestrator can wire the category in separately during integration.
This commit is contained in:
@@ -0,0 +1,69 @@
|
||||
package tests
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/lukaszraczylo/llm-testbench/internal/eval"
|
||||
"github.com/lukaszraczylo/llm-testbench/internal/testkit"
|
||||
)
|
||||
|
||||
// registerDatabasesTests registers every databases-category test. Each
|
||||
// subcategory - postgres, redis, sql-tuning - has its own register function
|
||||
// and source file (databases_postgres.go, databases_redis.go,
|
||||
// databases_sqltuning.go) to keep any one file from growing past a few
|
||||
// hundred lines. The orchestrator wires this into the catalog's All()
|
||||
// separately; this package intentionally does not register itself in
|
||||
// catalog.go so parallel category worktrees stay independently buildable.
|
||||
func registerDatabasesTests(r *testkit.Registry) {
|
||||
registerDBPostgresTests(r)
|
||||
registerDBRedisTests(r)
|
||||
registerDBSQLTuningTests(r)
|
||||
}
|
||||
|
||||
// dbExactAnswer returns an Evaluator awarding full credit when the
|
||||
// response, trimmed of whitespace, at most one layer of surrounding quote
|
||||
// characters (' , ", or `), and a single trailing sentence-ending period,
|
||||
// equals want case-insensitively. Used across the databases category for
|
||||
// prompts that force a single short forced-vocabulary answer (e.g. "yes"
|
||||
// or "no", "keyset" or "offset"): it accepts every materially-correct form
|
||||
// of that answer (bare, quoted, differently-cased, or with trailing
|
||||
// punctuation) without loosening the match to accept a wrong answer.
|
||||
func dbExactAnswer(want string) eval.Evaluator {
|
||||
return eval.EvaluatorFunc(func(_ context.Context, response string) eval.Score {
|
||||
got := strings.TrimSpace(response)
|
||||
got = strings.Trim(got, "\"'`")
|
||||
got = strings.TrimSuffix(strings.TrimSpace(got), ".")
|
||||
got = strings.TrimSpace(got)
|
||||
wantTrimmed := strings.TrimSpace(want)
|
||||
if strings.EqualFold(got, wantTrimmed) {
|
||||
return eval.Score{Value: 1, Detail: fmt.Sprintf("equals %q", wantTrimmed)}
|
||||
}
|
||||
return eval.Score{Value: 0, Detail: fmt.Sprintf("got %q, want %q", got, wantTrimmed)}
|
||||
})
|
||||
}
|
||||
|
||||
// dbJSONArrayLength returns an Evaluator awarding full credit when the
|
||||
// response's extracted JSON array (see eval.ExtractJSON) has exactly want
|
||||
// elements, zero otherwise. Paired with per-index eval.JSONField checks so
|
||||
// an answer with extra or missing elements (e.g. forgetting to apply a
|
||||
// HAVING filter, so an extra group leaks through) cannot score full credit
|
||||
// merely by getting the checked indices right.
|
||||
func dbJSONArrayLength(want int) eval.Evaluator {
|
||||
return eval.EvaluatorFunc(func(_ context.Context, response string) eval.Score {
|
||||
raw, err := eval.ExtractJSON(response)
|
||||
if err != nil {
|
||||
return eval.Score{Value: 0, Detail: err.Error()}
|
||||
}
|
||||
var arr []json.RawMessage
|
||||
if err := json.Unmarshal([]byte(raw), &arr); err != nil {
|
||||
return eval.Score{Value: 0, Detail: fmt.Sprintf("invalid JSON array: %v", err)}
|
||||
}
|
||||
if len(arr) == want {
|
||||
return eval.Score{Value: 1, Detail: fmt.Sprintf("array length %d", len(arr))}
|
||||
}
|
||||
return eval.Score{Value: 0, Detail: fmt.Sprintf("array length %d, want %d", len(arr), want)}
|
||||
})
|
||||
}
|
||||
Reference in New Issue
Block a user