mirror of
https://github.com/lukaszraczylo/llm-testbench.git
synced 2026-10-12 00:45:19 +00:00
feat: LLM accuracy testing framework with 17-test catalog
Framework runs a category/subcategory test catalog against multiple LLMs through an OpenAI-compatible endpoint. Deterministic evaluators score each response. Reports render as table, markdown, or JSON.
This commit is contained in:
@@ -0,0 +1,200 @@
|
||||
// Command llmtest runs the llm-testbench catalog against one or more
|
||||
// OpenAI-compatible models and reports deterministic scores.
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"flag"
|
||||
"fmt"
|
||||
"os"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/lukaszraczylo/llm-testbench/internal/config"
|
||||
"github.com/lukaszraczylo/llm-testbench/internal/llm"
|
||||
"github.com/lukaszraczylo/llm-testbench/internal/report"
|
||||
"github.com/lukaszraczylo/llm-testbench/internal/runner"
|
||||
"github.com/lukaszraczylo/llm-testbench/internal/testkit"
|
||||
"github.com/lukaszraczylo/llm-testbench/internal/tests"
|
||||
)
|
||||
|
||||
// requestTemperature is pinned to 0 for deterministic scoring, per PLAN.md;
|
||||
// it is not exposed as a flag.
|
||||
const requestTemperature = 0.0
|
||||
|
||||
func main() {
|
||||
if len(os.Args) < 2 {
|
||||
usage()
|
||||
os.Exit(2)
|
||||
}
|
||||
|
||||
var err error
|
||||
switch os.Args[1] {
|
||||
case "run":
|
||||
err = runCommand(os.Args[2:])
|
||||
case "list":
|
||||
err = listCommand(os.Args[2:])
|
||||
case "-h", "--help", "help":
|
||||
usage()
|
||||
return
|
||||
default:
|
||||
fmt.Fprintf(os.Stderr, "llmtest: unknown command %q\n\n", os.Args[1])
|
||||
usage()
|
||||
os.Exit(2)
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "llmtest: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
func usage() {
|
||||
fmt.Fprint(os.Stderr, `llmtest - LLM accuracy testing framework
|
||||
|
||||
Usage:
|
||||
llmtest run [flags] Run the test catalog against configured models.
|
||||
llmtest list [flags] List the test catalog.
|
||||
|
||||
Run "llmtest run -h" or "llmtest list -h" for flag details.
|
||||
`)
|
||||
}
|
||||
|
||||
// sharedFlags are accepted by both run and list to filter the catalog.
|
||||
type sharedFlags struct {
|
||||
configPath string
|
||||
category string
|
||||
subcategory string
|
||||
}
|
||||
|
||||
func bindSharedFlags(fs *flag.FlagSet) *sharedFlags {
|
||||
f := &sharedFlags{}
|
||||
fs.StringVar(&f.configPath, "config", "config.yaml", "path to config.yaml")
|
||||
fs.StringVar(&f.category, "category", "", "filter tests by category")
|
||||
fs.StringVar(&f.subcategory, "subcategory", "", "filter tests by subcategory")
|
||||
return f
|
||||
}
|
||||
|
||||
func listCommand(args []string) error {
|
||||
fs := flag.NewFlagSet("list", flag.ExitOnError)
|
||||
shared := bindSharedFlags(fs)
|
||||
if err := fs.Parse(args); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
registry := tests.All()
|
||||
filtered := registry.Filter(shared.category, shared.subcategory)
|
||||
sort.Slice(filtered, func(i, j int) bool { return filtered[i].ID < filtered[j].ID })
|
||||
|
||||
for _, t := range filtered {
|
||||
fmt.Printf("%-28s %-14s %-14s %s\n", t.ID, t.Category, t.Subcategory, t.Description)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func runCommand(args []string) error {
|
||||
fs := flag.NewFlagSet("run", flag.ExitOnError)
|
||||
shared := bindSharedFlags(fs)
|
||||
modelsCSV := fs.String("models", "", "comma-separated model override (defaults to config's models)")
|
||||
format := fs.String("format", "table", "output format: table|markdown|json")
|
||||
concurrency := fs.Int("concurrency", 0, "override config's concurrency (0 = use config)")
|
||||
timeout := fs.Duration("timeout", 0, "override config's request_timeout (0 = use config)")
|
||||
verbose := fs.Bool("verbose", false, "print progress to stderr as tests complete")
|
||||
if err := fs.Parse(args); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
cfg, err := config.Load(shared.configPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
models := cfg.Models
|
||||
if *modelsCSV != "" {
|
||||
models = splitCSV(*modelsCSV)
|
||||
}
|
||||
|
||||
reqTimeout := cfg.RequestTimeout
|
||||
if *timeout > 0 {
|
||||
reqTimeout = *timeout
|
||||
}
|
||||
|
||||
conc := cfg.Concurrency
|
||||
if *concurrency > 0 {
|
||||
conc = *concurrency
|
||||
}
|
||||
|
||||
client := llm.NewOpenAIClient(cfg.Endpoint, cfg.APIKey, reqTimeout)
|
||||
|
||||
registry := tests.All()
|
||||
selected := registry.Filter(shared.category, shared.subcategory)
|
||||
if len(selected) == 0 {
|
||||
return fmt.Errorf("no tests matched category=%q subcategory=%q", shared.category, shared.subcategory)
|
||||
}
|
||||
|
||||
r := runner.New(client, runner.Config{
|
||||
Concurrency: conc,
|
||||
Temperature: requestTemperature,
|
||||
MaxTokensDefault: cfg.MaxTokensDefault,
|
||||
})
|
||||
|
||||
ctx := context.Background()
|
||||
results := runWithProgress(ctx, r, models, selected, *verbose)
|
||||
|
||||
f, err := validateFormat(*format)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return report.Render(os.Stdout, f, selected, models, results)
|
||||
}
|
||||
|
||||
// validateFormat parses and validates the --format flag value, pulled out
|
||||
// of runCommand as its own function so it is unit-testable without going
|
||||
// through flag parsing / a live run (N8).
|
||||
func validateFormat(format string) (report.Format, error) {
|
||||
f := report.Format(format)
|
||||
switch f {
|
||||
case report.FormatTable, report.FormatMarkdown, report.FormatJSON:
|
||||
return f, nil
|
||||
default:
|
||||
return "", fmt.Errorf("unknown format %q (want table|markdown|json)", format)
|
||||
}
|
||||
}
|
||||
|
||||
// runWithProgress runs the given tests/models and, when verbose, prints one
|
||||
// progress line per completed (model, test) pair to stderr.
|
||||
func runWithProgress(ctx context.Context, r *runner.Runner, models []string, selected []testkit.Test, verbose bool) []runner.Result {
|
||||
if !verbose {
|
||||
return r.Run(ctx, models, selected)
|
||||
}
|
||||
|
||||
fmt.Fprintf(os.Stderr, "running %d tests x %d models = %d calls...\n", len(selected), len(models), len(selected)*len(models))
|
||||
start := time.Now()
|
||||
results := r.Run(ctx, models, selected)
|
||||
for _, res := range results {
|
||||
status := fmt.Sprintf("%.2f", res.Score.Value)
|
||||
switch {
|
||||
case res.Err != nil:
|
||||
status = "ERR: " + res.Err.Error()
|
||||
case res.Score.Skipped:
|
||||
status = "skip: " + res.Score.Detail
|
||||
}
|
||||
fmt.Fprintf(os.Stderr, " %-14s %-28s %s (%s)\n", res.Model, res.TestID, status, res.Latency)
|
||||
}
|
||||
fmt.Fprintf(os.Stderr, "done in %s\n", time.Since(start))
|
||||
return results
|
||||
}
|
||||
|
||||
func splitCSV(s string) []string {
|
||||
parts := strings.Split(s, ",")
|
||||
out := make([]string, 0, len(parts))
|
||||
for _, p := range parts {
|
||||
p = strings.TrimSpace(p)
|
||||
if p != "" {
|
||||
out = append(out, p)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,63 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/lukaszraczylo/llm-testbench/internal/report"
|
||||
)
|
||||
|
||||
func TestSplitCSV(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
in string
|
||||
want []string
|
||||
}{
|
||||
{"single model", "uni/deepseek-v4-flash-0731", []string{"uni/deepseek-v4-flash-0731"}},
|
||||
{"multiple models", "a,b,c", []string{"a", "b", "c"}},
|
||||
{"trims whitespace around entries", " a , b ,c ", []string{"a", "b", "c"}},
|
||||
{"drops empty entries from trailing comma", "a,b,", []string{"a", "b"}},
|
||||
{"drops empty entries from doubled comma", "a,,b", []string{"a", "b"}},
|
||||
{"empty string yields no entries", "", nil},
|
||||
{"whitespace only yields no entries", " ", nil},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := splitCSV(tt.in)
|
||||
if len(got) != len(tt.want) {
|
||||
t.Fatalf("splitCSV(%q) = %v, want %v", tt.in, got, tt.want)
|
||||
}
|
||||
for i := range got {
|
||||
if got[i] != tt.want[i] {
|
||||
t.Errorf("splitCSV(%q)[%d] = %q, want %q", tt.in, i, got[i], tt.want[i])
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateFormat(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
in string
|
||||
want report.Format
|
||||
wantErr bool
|
||||
}{
|
||||
{"table", "table", report.FormatTable, false},
|
||||
{"markdown", "markdown", report.FormatMarkdown, false},
|
||||
{"json", "json", report.FormatJSON, false},
|
||||
{"unknown format", "yaml", "", true},
|
||||
{"empty format", "", "", true},
|
||||
{"case sensitive: uppercase rejected", "TABLE", "", true},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got, err := validateFormat(tt.in)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Fatalf("validateFormat(%q) error = %v, wantErr %v", tt.in, err, tt.wantErr)
|
||||
}
|
||||
if err == nil && got != tt.want {
|
||||
t.Errorf("validateFormat(%q) = %q, want %q", tt.in, got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user