feat: LLM accuracy testing framework with 17-test catalog

Framework runs a category/subcategory test catalog against multiple
LLMs through an OpenAI-compatible endpoint. Deterministic evaluators
score each response. Reports render as table, markdown, or JSON.

cmd/llmtest/main.go

@@ -0,0 +1,200 @@
+// Command llmtest runs the llm-testbench catalog against one or more
+// OpenAI-compatible models and reports deterministic scores.
+package main
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"os"
+	"sort"
+	"strings"
+	"time"
+
+	"github.com/lukaszraczylo/llm-testbench/internal/config"
+	"github.com/lukaszraczylo/llm-testbench/internal/llm"
+	"github.com/lukaszraczylo/llm-testbench/internal/report"
+	"github.com/lukaszraczylo/llm-testbench/internal/runner"
+	"github.com/lukaszraczylo/llm-testbench/internal/testkit"
+	"github.com/lukaszraczylo/llm-testbench/internal/tests"
+)
+
+// requestTemperature is pinned to 0 for deterministic scoring, per PLAN.md;
+// it is not exposed as a flag.
+const requestTemperature = 0.0
+
+func main() {
+	if len(os.Args) < 2 {
+		usage()
+		os.Exit(2)
+	}
+
+	var err error
+	switch os.Args[1] {
+	case "run":
+		err = runCommand(os.Args[2:])
+	case "list":
+		err = listCommand(os.Args[2:])
+	case "-h", "--help", "help":
+		usage()
+		return
+	default:
+		fmt.Fprintf(os.Stderr, "llmtest: unknown command %q\n\n", os.Args[1])
+		usage()
+		os.Exit(2)
+	}
+
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "llmtest: %v\n", err)
+		os.Exit(1)
+	}
+}
+
+func usage() {
+	fmt.Fprint(os.Stderr, `llmtest - LLM accuracy testing framework
+
+Usage:
+  llmtest run  [flags]   Run the test catalog against configured models.
+  llmtest list [flags]   List the test catalog.
+
+Run "llmtest run -h" or "llmtest list -h" for flag details.
+`)
+}
+
+// sharedFlags are accepted by both run and list to filter the catalog.
+type sharedFlags struct {
+	configPath  string
+	category    string
+	subcategory string
+}
+
+func bindSharedFlags(fs *flag.FlagSet) *sharedFlags {
+	f := &sharedFlags{}
+	fs.StringVar(&f.configPath, "config", "config.yaml", "path to config.yaml")
+	fs.StringVar(&f.category, "category", "", "filter tests by category")
+	fs.StringVar(&f.subcategory, "subcategory", "", "filter tests by subcategory")
+	return f
+}
+
+func listCommand(args []string) error {
+	fs := flag.NewFlagSet("list", flag.ExitOnError)
+	shared := bindSharedFlags(fs)
+	if err := fs.Parse(args); err != nil {
+		return err
+	}
+
+	registry := tests.All()
+	filtered := registry.Filter(shared.category, shared.subcategory)
+	sort.Slice(filtered, func(i, j int) bool { return filtered[i].ID < filtered[j].ID })
+
+	for _, t := range filtered {
+		fmt.Printf("%-28s %-14s %-14s %s\n", t.ID, t.Category, t.Subcategory, t.Description)
+	}
+	return nil
+}
+
+func runCommand(args []string) error {
+	fs := flag.NewFlagSet("run", flag.ExitOnError)
+	shared := bindSharedFlags(fs)
+	modelsCSV := fs.String("models", "", "comma-separated model override (defaults to config's models)")
+	format := fs.String("format", "table", "output format: table|markdown|json")
+	concurrency := fs.Int("concurrency", 0, "override config's concurrency (0 = use config)")
+	timeout := fs.Duration("timeout", 0, "override config's request_timeout (0 = use config)")
+	verbose := fs.Bool("verbose", false, "print progress to stderr as tests complete")
+	if err := fs.Parse(args); err != nil {
+		return err
+	}
+
+	cfg, err := config.Load(shared.configPath)
+	if err != nil {
+		return err
+	}
+
+	models := cfg.Models
+	if *modelsCSV != "" {
+		models = splitCSV(*modelsCSV)
+	}
+
+	reqTimeout := cfg.RequestTimeout
+	if *timeout > 0 {
+		reqTimeout = *timeout
+	}
+
+	conc := cfg.Concurrency
+	if *concurrency > 0 {
+		conc = *concurrency
+	}
+
+	client := llm.NewOpenAIClient(cfg.Endpoint, cfg.APIKey, reqTimeout)
+
+	registry := tests.All()
+	selected := registry.Filter(shared.category, shared.subcategory)
+	if len(selected) == 0 {
+		return fmt.Errorf("no tests matched category=%q subcategory=%q", shared.category, shared.subcategory)
+	}
+
+	r := runner.New(client, runner.Config{
+		Concurrency:      conc,
+		Temperature:      requestTemperature,
+		MaxTokensDefault: cfg.MaxTokensDefault,
+	})
+
+	ctx := context.Background()
+	results := runWithProgress(ctx, r, models, selected, *verbose)
+
+	f, err := validateFormat(*format)
+	if err != nil {
+		return err
+	}
+
+	return report.Render(os.Stdout, f, selected, models, results)
+}
+
+// validateFormat parses and validates the --format flag value, pulled out
+// of runCommand as its own function so it is unit-testable without going
+// through flag parsing / a live run (N8).
+func validateFormat(format string) (report.Format, error) {
+	f := report.Format(format)
+	switch f {
+	case report.FormatTable, report.FormatMarkdown, report.FormatJSON:
+		return f, nil
+	default:
+		return "", fmt.Errorf("unknown format %q (want table|markdown|json)", format)
+	}
+}
+
+// runWithProgress runs the given tests/models and, when verbose, prints one
+// progress line per completed (model, test) pair to stderr.
+func runWithProgress(ctx context.Context, r *runner.Runner, models []string, selected []testkit.Test, verbose bool) []runner.Result {
+	if !verbose {
+		return r.Run(ctx, models, selected)
+	}
+
+	fmt.Fprintf(os.Stderr, "running %d tests x %d models = %d calls...\n", len(selected), len(models), len(selected)*len(models))
+	start := time.Now()
+	results := r.Run(ctx, models, selected)
+	for _, res := range results {
+		status := fmt.Sprintf("%.2f", res.Score.Value)
+		switch {
+		case res.Err != nil:
+			status = "ERR: " + res.Err.Error()
+		case res.Score.Skipped:
+			status = "skip: " + res.Score.Detail
+		}
+		fmt.Fprintf(os.Stderr, "  %-14s %-28s %s (%s)\n", res.Model, res.TestID, status, res.Latency)
+	}
+	fmt.Fprintf(os.Stderr, "done in %s\n", time.Since(start))
+	return results
+}
+
+func splitCSV(s string) []string {
+	parts := strings.Split(s, ",")
+	out := make([]string, 0, len(parts))
+	for _, p := range parts {
+		p = strings.TrimSpace(p)
+		if p != "" {
+			out = append(out, p)
+		}
+	}
+	return out
+}