feat: LLM accuracy testing framework with 17-test catalog

Framework runs a category/subcategory test catalog against multiple LLMs through an OpenAI-compatible endpoint. Deterministic evaluators score each response. Reports render as table, markdown, or JSON.
2026-10-12 00:45:19 +00:00 · 2026-08-17 01:29:45 +01:00
commit 1d51d4cbbd
68 changed files with 7089 additions and 0 deletions
@@ -0,0 +1,132 @@
+// Package runner fans a test catalog out across models with bounded
+// concurrency and collects per-(model,test) results.
+package runner
+
+import (
+	"context"
+	"time"
+
+	"golang.org/x/sync/errgroup"
+
+	"github.com/lukaszraczylo/llm-testbench/internal/eval"
+	"github.com/lukaszraczylo/llm-testbench/internal/llm"
+	"github.com/lukaszraczylo/llm-testbench/internal/testkit"
+)
+
+// Result is the outcome of running one test against one model.
+type Result struct {
+	Err              error
+	Model            string
+	TestID           string
+	Score            eval.Score
+	Latency          time.Duration
+	PromptTokens     int
+	CompletionTokens int
+}
+
+// TotalTokens returns PromptTokens + CompletionTokens, the total token
+// usage billed for this (model, test) call.
+func (r Result) TotalTokens() int {
+	return r.PromptTokens + r.CompletionTokens
+}
+
+// Config controls how the Runner fans work out.
+type Config struct {
+	// Concurrency bounds the number of in-flight (model,test) calls.
+	Concurrency int
+	// Temperature is sent on every request; PLAN.md pins this to 0 for
+	// determinism across runs.
+	Temperature float64
+	// MaxTokensDefault is used when a Test does not set its own MaxTokens.
+	MaxTokensDefault int
+}
+
+// Runner executes a set of tests against a set of models.
+type Runner struct {
+	client llm.Client
+	cfg    Config
+}
+
+// New builds a Runner that issues requests through client.
+func New(client llm.Client, cfg Config) *Runner {
+	return &Runner{client: client, cfg: cfg}
+}
+
+// Run executes every combination of models x tests, bounded by
+// cfg.Concurrency in-flight calls, and returns one Result per combination.
+// A per-call failure is captured in Result.Err rather than aborting the run.
+func (r *Runner) Run(ctx context.Context, models []string, tests []testkit.Test) []Result {
+	type job struct {
+		model string
+		test  testkit.Test
+	}
+
+	jobs := make([]job, 0, len(models)*len(tests))
+	for _, m := range models {
+		for _, t := range tests {
+			jobs = append(jobs, job{model: m, test: t})
+		}
+	}
+
+	results := make([]Result, len(jobs))
+
+	g, gctx := errgroup.WithContext(ctx)
+	concurrency := r.cfg.Concurrency
+	if concurrency <= 0 {
+		concurrency = 1
+	}
+	g.SetLimit(concurrency)
+
+	// go.mod's go directive is 1.22+, so loop variables are already
+	// per-iteration scoped; no manual i, j := i, j shadowing needed (N4).
+	for i, j := range jobs {
+		g.Go(func() error {
+			results[i] = r.runOne(gctx, j.model, j.test)
+			return nil
+		})
+	}
+	// Errors are captured per-result, not surfaced from Run; g.Wait() only
+	// ever returns non-nil if a goroutine itself returned an error, which
+	// runOne never does.
+	_ = g.Wait()
+
+	return results
+}
+
+// runOne executes a single (model, test) combination.
+func (r *Runner) runOne(ctx context.Context, model string, test testkit.Test) Result {
+	maxTokens := test.MaxTokens
+	if maxTokens <= 0 {
+		maxTokens = r.cfg.MaxTokensDefault
+	}
+
+	messages := make([]llm.Message, 0, 2)
+	if test.System != "" {
+		messages = append(messages, llm.Message{Role: "system", Content: test.System})
+	}
+	messages = append(messages, llm.Message{Role: "user", Content: test.Prompt})
+
+	req := llm.Request{
+		Model:       model,
+		Messages:    messages,
+		MaxTokens:   maxTokens,
+		Temperature: r.cfg.Temperature,
+	}
+
+	resp, err := r.client.Complete(ctx, req)
+	if err != nil {
+		return Result{Model: model, TestID: test.ID, Err: err}
+	}
+
+	normalized := testkit.Normalize(resp.Text)
+	score := test.Eval.Evaluate(ctx, normalized)
+
+	return Result{
+		Model:            model,
+		TestID:           test.ID,
+		Score:            score,
+		Latency:          resp.Latency,
+		PromptTokens:     resp.PromptTokens,
+		CompletionTokens: resp.CompletionTokens,
+	}
+}
@@ -0,0 +1,226 @@
+package runner
+
+import (
+	"context"
+	"errors"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/lukaszraczylo/llm-testbench/internal/eval"
+	"github.com/lukaszraczylo/llm-testbench/internal/llm"
+	"github.com/lukaszraczylo/llm-testbench/internal/testkit"
+)
+
+// mockClient is an offline llm.Client for unit tests; it never touches the
+// network.
+type mockClient struct {
+	respond func(req llm.Request) (llm.Response, error)
+
+	inFlight    int32
+	maxInFlight int32
+}
+
+func (m *mockClient) Complete(_ context.Context, req llm.Request) (llm.Response, error) {
+	cur := atomic.AddInt32(&m.inFlight, 1)
+	for {
+		max := atomic.LoadInt32(&m.maxInFlight)
+		if cur <= max || atomic.CompareAndSwapInt32(&m.maxInFlight, max, cur) {
+			break
+		}
+	}
+	defer atomic.AddInt32(&m.inFlight, -1)
+
+	if m.respond != nil {
+		return m.respond(req)
+	}
+	return llm.Response{Text: "OK"}, nil
+}
+
+func echoEval() eval.Evaluator {
+	return eval.EvaluatorFunc(func(_ context.Context, response string) eval.Score {
+		if response == "OK" {
+			return eval.Score{Value: 1}
+		}
+		return eval.Score{Value: 0, Detail: "got " + response}
+	})
+}
+
+func TestRunner_Run_FansOutAllCombinations(t *testing.T) {
+	client := &mockClient{}
+	r := New(client, Config{Concurrency: 4, MaxTokensDefault: 100})
+
+	tests := []testkit.Test{
+		{ID: "t1", Category: "c", Prompt: "p1", Eval: echoEval()},
+		{ID: "t2", Category: "c", Prompt: "p2", Eval: echoEval()},
+	}
+	models := []string{"model-a", "model-b", "model-c"}
+
+	results := r.Run(context.Background(), models, tests)
+
+	if len(results) != len(models)*len(tests) {
+		t.Fatalf("Run() returned %d results, want %d", len(results), len(models)*len(tests))
+	}
+
+	seen := make(map[string]bool)
+	for _, res := range results {
+		seen[res.Model+"/"+res.TestID] = true
+		if res.Err != nil {
+			t.Errorf("Result.Err = %v, want nil", res.Err)
+		}
+		if res.Score.Value != 1 {
+			t.Errorf("Result.Score.Value = %v, want 1", res.Score.Value)
+		}
+	}
+	for _, m := range models {
+		for _, tc := range tests {
+			key := m + "/" + tc.ID
+			if !seen[key] {
+				t.Errorf("Run() missing combination %s", key)
+			}
+		}
+	}
+}
+
+func TestRunner_Run_RespectsConcurrencyLimit(t *testing.T) {
+	client := &mockClient{
+		respond: func(_ llm.Request) (llm.Response, error) {
+			time.Sleep(20 * time.Millisecond)
+			return llm.Response{Text: "OK"}, nil
+		},
+	}
+	const limit = 2
+	r := New(client, Config{Concurrency: limit, MaxTokensDefault: 100})
+
+	var tests []testkit.Test
+	for i := range 8 {
+		tests = append(tests, testkit.Test{ID: string(rune('a' + i)), Category: "c", Prompt: "p", Eval: echoEval()})
+	}
+
+	r.Run(context.Background(), []string{"model-a"}, tests)
+
+	if got := atomic.LoadInt32(&client.maxInFlight); got > limit {
+		t.Errorf("max in-flight calls = %d, want <= %d", got, limit)
+	}
+}
+
+func TestRunner_Run_CapturesPerCallError(t *testing.T) {
+	wantErr := errors.New("transport failure")
+	client := &mockClient{
+		respond: func(_ llm.Request) (llm.Response, error) {
+			return llm.Response{}, wantErr
+		},
+	}
+	r := New(client, Config{Concurrency: 2, MaxTokensDefault: 100})
+
+	results := r.Run(context.Background(), []string{"model-a"}, []testkit.Test{
+		{ID: "t1", Category: "c", Prompt: "p", Eval: echoEval()},
+	})
+
+	if len(results) != 1 {
+		t.Fatalf("Run() returned %d results, want 1", len(results))
+	}
+	if !errors.Is(results[0].Err, wantErr) {
+		t.Errorf("Result.Err = %v, want %v", results[0].Err, wantErr)
+	}
+}
+
+func TestRunner_Run_NormalizesResponseBeforeEval(t *testing.T) {
+	client := &mockClient{
+		respond: func(_ llm.Request) (llm.Response, error) {
+			return llm.Response{Text: "<think>internal reasoning</think>\n\nOK"}, nil
+		},
+	}
+	r := New(client, Config{Concurrency: 1, MaxTokensDefault: 100})
+
+	results := r.Run(context.Background(), []string{"model-a"}, []testkit.Test{
+		{ID: "t1", Category: "c", Prompt: "p", Eval: echoEval()},
+	})
+
+	if results[0].Score.Value != 1 {
+		t.Errorf("Score.Value = %v, want 1 (detail: %s)", results[0].Score.Value, results[0].Score.Detail)
+	}
+}
+
+func TestRunner_Run_UsesTestMaxTokensOverDefault(t *testing.T) {
+	var gotMaxTokens int
+	client := &mockClient{
+		respond: func(req llm.Request) (llm.Response, error) {
+			gotMaxTokens = req.MaxTokens
+			return llm.Response{Text: "OK"}, nil
+		},
+	}
+	r := New(client, Config{Concurrency: 1, MaxTokensDefault: 100})
+
+	r.Run(context.Background(), []string{"model-a"}, []testkit.Test{
+		{ID: "t1", Category: "c", Prompt: "p", MaxTokens: 500, Eval: echoEval()},
+	})
+
+	if gotMaxTokens != 500 {
+		t.Errorf("MaxTokens = %d, want 500 (test override)", gotMaxTokens)
+	}
+}
+
+func TestRunner_Run_FallsBackToDefaultMaxTokens(t *testing.T) {
+	var gotMaxTokens int
+	client := &mockClient{
+		respond: func(req llm.Request) (llm.Response, error) {
+			gotMaxTokens = req.MaxTokens
+			return llm.Response{Text: "OK"}, nil
+		},
+	}
+	r := New(client, Config{Concurrency: 1, MaxTokensDefault: 100})
+
+	r.Run(context.Background(), []string{"model-a"}, []testkit.Test{
+		{ID: "t1", Category: "c", Prompt: "p", Eval: echoEval()},
+	})
+
+	if gotMaxTokens != 100 {
+		t.Errorf("MaxTokens = %d, want 100 (config default)", gotMaxTokens)
+	}
+}
+
+func TestRunner_Run_IncludesSystemMessageWhenSet(t *testing.T) {
+	var gotMessages []llm.Message
+	client := &mockClient{
+		respond: func(req llm.Request) (llm.Response, error) {
+			gotMessages = req.Messages
+			return llm.Response{Text: "OK"}, nil
+		},
+	}
+	r := New(client, Config{Concurrency: 1, MaxTokensDefault: 100})
+
+	r.Run(context.Background(), []string{"model-a"}, []testkit.Test{
+		{ID: "t1", Category: "c", System: "you are terse", Prompt: "p", Eval: echoEval()},
+	})
+
+	if len(gotMessages) != 2 || gotMessages[0].Role != "system" || gotMessages[1].Role != "user" {
+		t.Errorf("Messages = %+v, want [system, user]", gotMessages)
+	}
+}
+
+func TestRunner_Run_TracksLatencyAndTokens(t *testing.T) {
+	client := &mockClient{
+		respond: func(_ llm.Request) (llm.Response, error) {
+			return llm.Response{Text: "OK", PromptTokens: 10, CompletionTokens: 5, Latency: 42 * time.Millisecond}, nil
+		},
+	}
+	r := New(client, Config{Concurrency: 1, MaxTokensDefault: 100})
+
+	results := r.Run(context.Background(), []string{"model-a"}, []testkit.Test{
+		{ID: "t1", Category: "c", Prompt: "p", Eval: echoEval()},
+	})
+
+	if results[0].PromptTokens != 10 {
+		t.Errorf("PromptTokens = %d, want 10", results[0].PromptTokens)
+	}
+	if results[0].CompletionTokens != 5 {
+		t.Errorf("CompletionTokens = %d, want 5", results[0].CompletionTokens)
+	}
+	if results[0].TotalTokens() != 15 {
+		t.Errorf("TotalTokens() = %d, want 15", results[0].TotalTokens())
+	}
+	if results[0].Latency != 42*time.Millisecond {
+		t.Errorf("Latency = %v, want 42ms", results[0].Latency)
+	}
+}