mirror of
https://github.com/lukaszraczylo/llm-testbench.git
synced 2026-10-14 00:57:45 +00:00
lukaszraczylo
0665d706f0
Add a new databases category with three subcategories: postgres, redis, and sql-tuning. Each subcategory holds 10 self-contained tests with deterministic evaluators. Postgres tests cover EXPLAIN interpretation, index choice, N+1 detection, isolation anomalies, deadlocks, replica failover, bloat diagnosis, pool sizing, partial indexes, and LISTEN/NOTIFY. Redis tests cover structure choice, TTL eviction, INCR atomicity, MULTI versus pipelining, and SCAN versus KEYS. They also cover pub/sub delivery, Lua atomicity, memory estimation, cache stampede mitigation, and keyspace anti-patterns. The SCAN versus KEYS test uses a negation-aware evaluator, so a response that warns against KEYS does not lose credit for mentioning it. SQL-tuning tests trace query results against an inline orders table: aggregates, joins, NULL handling, GROUP BY plus HAVING, and window functions. The unit tests recompute every numeric or JSON answer independently, with a hand-written loop separate from the production query-logic helper. registerDatabasesTests registers the category. A wiring test verifies it directly, and the code does not touch catalog.go, so the orchestrator can wire the category in separately during integration.
70 lines
2.9 KiB
Go
70 lines
2.9 KiB
Go
package tests
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"strings"
|
|
|
|
"github.com/lukaszraczylo/llm-testbench/internal/eval"
|
|
"github.com/lukaszraczylo/llm-testbench/internal/testkit"
|
|
)
|
|
|
|
// registerDatabasesTests registers every databases-category test. Each
|
|
// subcategory - postgres, redis, sql-tuning - has its own register function
|
|
// and source file (databases_postgres.go, databases_redis.go,
|
|
// databases_sqltuning.go) to keep any one file from growing past a few
|
|
// hundred lines. The orchestrator wires this into the catalog's All()
|
|
// separately; this package intentionally does not register itself in
|
|
// catalog.go so parallel category worktrees stay independently buildable.
|
|
func registerDatabasesTests(r *testkit.Registry) {
|
|
registerDBPostgresTests(r)
|
|
registerDBRedisTests(r)
|
|
registerDBSQLTuningTests(r)
|
|
}
|
|
|
|
// dbExactAnswer returns an Evaluator awarding full credit when the
|
|
// response, trimmed of whitespace, at most one layer of surrounding quote
|
|
// characters (' , ", or `), and a single trailing sentence-ending period,
|
|
// equals want case-insensitively. Used across the databases category for
|
|
// prompts that force a single short forced-vocabulary answer (e.g. "yes"
|
|
// or "no", "keyset" or "offset"): it accepts every materially-correct form
|
|
// of that answer (bare, quoted, differently-cased, or with trailing
|
|
// punctuation) without loosening the match to accept a wrong answer.
|
|
func dbExactAnswer(want string) eval.Evaluator {
|
|
return eval.EvaluatorFunc(func(_ context.Context, response string) eval.Score {
|
|
got := strings.TrimSpace(response)
|
|
got = strings.Trim(got, "\"'`")
|
|
got = strings.TrimSuffix(strings.TrimSpace(got), ".")
|
|
got = strings.TrimSpace(got)
|
|
wantTrimmed := strings.TrimSpace(want)
|
|
if strings.EqualFold(got, wantTrimmed) {
|
|
return eval.Score{Value: 1, Detail: fmt.Sprintf("equals %q", wantTrimmed)}
|
|
}
|
|
return eval.Score{Value: 0, Detail: fmt.Sprintf("got %q, want %q", got, wantTrimmed)}
|
|
})
|
|
}
|
|
|
|
// dbJSONArrayLength returns an Evaluator awarding full credit when the
|
|
// response's extracted JSON array (see eval.ExtractJSON) has exactly want
|
|
// elements, zero otherwise. Paired with per-index eval.JSONField checks so
|
|
// an answer with extra or missing elements (e.g. forgetting to apply a
|
|
// HAVING filter, so an extra group leaks through) cannot score full credit
|
|
// merely by getting the checked indices right.
|
|
func dbJSONArrayLength(want int) eval.Evaluator {
|
|
return eval.EvaluatorFunc(func(_ context.Context, response string) eval.Score {
|
|
raw, err := eval.ExtractJSON(response)
|
|
if err != nil {
|
|
return eval.Score{Value: 0, Detail: err.Error()}
|
|
}
|
|
var arr []json.RawMessage
|
|
if err := json.Unmarshal([]byte(raw), &arr); err != nil {
|
|
return eval.Score{Value: 0, Detail: fmt.Sprintf("invalid JSON array: %v", err)}
|
|
}
|
|
if len(arr) == want {
|
|
return eval.Score{Value: 1, Detail: fmt.Sprintf("array length %d", len(arr))}
|
|
}
|
|
return eval.Score{Value: 0, Detail: fmt.Sprintf("array length %d, want %d", len(arr), want)}
|
|
})
|
|
}
|