mirror of
https://github.com/lukaszraczylo/llm-testbench.git
synced 2026-10-14 00:57:45 +00:00
feat: LLM accuracy testing framework with 17-test catalog
Framework runs a category/subcategory test catalog against multiple LLMs through an OpenAI-compatible endpoint. Deterministic evaluators score each response. Reports render as table, markdown, or JSON.
This commit is contained in:
@@ -0,0 +1,133 @@
|
||||
// Package testkit defines the Test type shared by the catalog in
|
||||
// internal/tests, a Registry to collect them, and response normalization
|
||||
// applied before evaluation.
|
||||
package testkit
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"github.com/lukaszraczylo/llm-testbench/internal/eval"
|
||||
)
|
||||
|
||||
// Test is one catalog entry: a fully self-contained prompt scored by a
|
||||
// deterministic Evaluator.
|
||||
type Test struct {
|
||||
Eval eval.Evaluator
|
||||
ID string
|
||||
Category string
|
||||
Subcategory string
|
||||
Description string
|
||||
System string
|
||||
Prompt string
|
||||
MaxTokens int
|
||||
}
|
||||
|
||||
// thinkBlockPattern strips a leading <think>...</think> or
|
||||
// <reasoning>...</reasoning> block, case-insensitive, before the real
|
||||
// answer. Models observed on this endpoint prepend one.
|
||||
var thinkBlockPattern = regexp.MustCompile(`(?is)^\s*<(think|reasoning)>.*?</(think|reasoning)>\s*`)
|
||||
|
||||
// thinkCloseTagPattern and thinkOpenTagPattern locate a closing/opening
|
||||
// <think>/<reasoning> tag anywhere in the text, used to detect an orphan
|
||||
// closing tag (S7): some models on this endpoint truncate the opening tag
|
||||
// (e.g. it falls outside a context/streaming window) but still emit the
|
||||
// closing tag, leaving stray reasoning text with no opener in front of it.
|
||||
var thinkCloseTagPattern = regexp.MustCompile(`(?i)</(?:think|reasoning)>`)
|
||||
var thinkOpenTagPattern = regexp.MustCompile(`(?i)<(?:think|reasoning)>`)
|
||||
|
||||
// stripOrphanThinkClose removes everything up to and including a closing
|
||||
// </think>/</reasoning> tag that has no matching opening tag earlier in s.
|
||||
// A tag pair with a real opener is left alone (thinkBlockPattern already
|
||||
// handles the well-formed, anchored-at-start case).
|
||||
func stripOrphanThinkClose(s string) string {
|
||||
closeLoc := thinkCloseTagPattern.FindStringIndex(s)
|
||||
if closeLoc == nil {
|
||||
return s
|
||||
}
|
||||
if thinkOpenTagPattern.MatchString(s[:closeLoc[0]]) {
|
||||
return s
|
||||
}
|
||||
return s[closeLoc[1]:]
|
||||
}
|
||||
|
||||
// Normalize prepares a raw model response for evaluation: it strips a
|
||||
// leading <think>/<reasoning> block (or, failing that, an orphan closing
|
||||
// tag with no opener), collapses Windows line endings, and trims
|
||||
// surrounding whitespace.
|
||||
func Normalize(response string) string {
|
||||
s := thinkBlockPattern.ReplaceAllString(response, "")
|
||||
s = stripOrphanThinkClose(s)
|
||||
s = strings.ReplaceAll(s, "\r\n", "\n")
|
||||
return strings.TrimSpace(s)
|
||||
}
|
||||
|
||||
// Registry collects Tests and provides lookup/filtering for the runner and
|
||||
// CLI.
|
||||
type Registry struct {
|
||||
byID map[string]Test
|
||||
tests []Test
|
||||
}
|
||||
|
||||
// NewRegistry builds an empty Registry.
|
||||
func NewRegistry() *Registry {
|
||||
return &Registry{byID: make(map[string]Test)}
|
||||
}
|
||||
|
||||
// Register adds t to the registry. It panics on a duplicate ID since the
|
||||
// catalog is assembled once at program startup via init()-time registration,
|
||||
// making a duplicate a programming error, not a runtime condition.
|
||||
func (r *Registry) Register(t Test) {
|
||||
if t.ID == "" {
|
||||
panic("testkit: Test.ID must not be empty")
|
||||
}
|
||||
if _, exists := r.byID[t.ID]; exists {
|
||||
panic(fmt.Sprintf("testkit: duplicate test ID %q", t.ID))
|
||||
}
|
||||
if t.Eval == nil {
|
||||
panic(fmt.Sprintf("testkit: test %q has a nil Evaluator", t.ID))
|
||||
}
|
||||
r.tests = append(r.tests, t)
|
||||
r.byID[t.ID] = t
|
||||
}
|
||||
|
||||
// All returns every registered test, sorted by ID for deterministic output.
|
||||
func (r *Registry) All() []Test {
|
||||
out := make([]Test, len(r.tests))
|
||||
copy(out, r.tests)
|
||||
sort.Slice(out, func(i, j int) bool { return out[i].ID < out[j].ID })
|
||||
return out
|
||||
}
|
||||
|
||||
// Filter returns registered tests whose Category/Subcategory match the
|
||||
// given filters. Empty filters match everything.
|
||||
func (r *Registry) Filter(category, subcategory string) []Test {
|
||||
all := r.All()
|
||||
if category == "" && subcategory == "" {
|
||||
return all
|
||||
}
|
||||
var out []Test
|
||||
for _, t := range all {
|
||||
if category != "" && !strings.EqualFold(t.Category, category) {
|
||||
continue
|
||||
}
|
||||
if subcategory != "" && !strings.EqualFold(t.Subcategory, subcategory) {
|
||||
continue
|
||||
}
|
||||
out = append(out, t)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// Get returns the test with the given ID.
|
||||
func (r *Registry) Get(id string) (Test, bool) {
|
||||
t, ok := r.byID[id]
|
||||
return t, ok
|
||||
}
|
||||
|
||||
// Len returns the number of registered tests.
|
||||
func (r *Registry) Len() int {
|
||||
return len(r.tests)
|
||||
}
|
||||
@@ -0,0 +1,153 @@
|
||||
package testkit
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"github.com/lukaszraczylo/llm-testbench/internal/eval"
|
||||
)
|
||||
|
||||
func TestNormalize(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
in string
|
||||
want string
|
||||
}{
|
||||
{"trims whitespace", " hello ", "hello"},
|
||||
{"leading whitespace before answer", "\n\nOK", "OK"},
|
||||
{"strips think block", "<think>reasoning here</think>OK", "OK"},
|
||||
{"strips think block with whitespace after", "<think>reasoning here</think>\n\nOK", "OK"},
|
||||
{"strips think block case insensitive", "<THINK>x</THINK>OK", "OK"},
|
||||
{"strips reasoning block", "<reasoning>plan</reasoning>final answer", "final answer"},
|
||||
{"collapses windows line endings", "line1\r\nline2", "line1\nline2"},
|
||||
{"multiline think block", "<think>\nstep 1\nstep 2\n</think>\nanswer", "answer"},
|
||||
{"no think block passthrough", "just a plain answer", "just a plain answer"},
|
||||
{"think tag not at start is left alone", "prefix <think>x</think> suffix", "prefix <think>x</think> suffix"},
|
||||
// S7: an orphan closing tag (opening tag truncated by the gateway)
|
||||
// must still be stripped.
|
||||
{"orphan think close tag with no opener", "leftover reasoning residue</think>OK", "OK"},
|
||||
{"orphan reasoning close tag with no opener", "leftover plan</reasoning>final answer", "final answer"},
|
||||
{"orphan close tag case insensitive", "residue</THINK>OK", "OK"},
|
||||
{"orphan close tag with whitespace after", "residue</think>\n\nOK", "OK"},
|
||||
{"no close tag at all passthrough", "just a plain answer with no tags", "just a plain answer with no tags"},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := Normalize(tt.in)
|
||||
if got != tt.want {
|
||||
t.Errorf("Normalize(%q) = %q, want %q", tt.in, got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func newDummyEval(value float64) eval.Evaluator {
|
||||
return eval.EvaluatorFunc(func(_ context.Context, _ string) eval.Score {
|
||||
return eval.Score{Value: value}
|
||||
})
|
||||
}
|
||||
|
||||
func TestRegistry_RegisterAndAll(t *testing.T) {
|
||||
r := NewRegistry()
|
||||
r.Register(Test{ID: "b-test", Category: "cat", Subcategory: "sub", Prompt: "p", Eval: newDummyEval(1)})
|
||||
r.Register(Test{ID: "a-test", Category: "cat", Subcategory: "sub", Prompt: "p", Eval: newDummyEval(1)})
|
||||
|
||||
all := r.All()
|
||||
if len(all) != 2 {
|
||||
t.Fatalf("All() len = %d, want 2", len(all))
|
||||
}
|
||||
if all[0].ID != "a-test" || all[1].ID != "b-test" {
|
||||
t.Errorf("All() not sorted by ID: got %q, %q", all[0].ID, all[1].ID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRegistry_RegisterDuplicateIDPanics(t *testing.T) {
|
||||
r := NewRegistry()
|
||||
r.Register(Test{ID: "dup", Category: "c", Prompt: "p", Eval: newDummyEval(1)})
|
||||
|
||||
defer func() {
|
||||
if recover() == nil {
|
||||
t.Fatal("Register() with duplicate ID did not panic")
|
||||
}
|
||||
}()
|
||||
r.Register(Test{ID: "dup", Category: "c", Prompt: "p", Eval: newDummyEval(1)})
|
||||
}
|
||||
|
||||
func TestRegistry_RegisterEmptyIDPanics(t *testing.T) {
|
||||
r := NewRegistry()
|
||||
defer func() {
|
||||
if recover() == nil {
|
||||
t.Fatal("Register() with empty ID did not panic")
|
||||
}
|
||||
}()
|
||||
r.Register(Test{Category: "c", Prompt: "p", Eval: newDummyEval(1)})
|
||||
}
|
||||
|
||||
func TestRegistry_RegisterNilEvaluatorPanics(t *testing.T) {
|
||||
r := NewRegistry()
|
||||
defer func() {
|
||||
if recover() == nil {
|
||||
t.Fatal("Register() with nil Evaluator did not panic")
|
||||
}
|
||||
}()
|
||||
r.Register(Test{ID: "x", Category: "c", Prompt: "p"})
|
||||
}
|
||||
|
||||
func TestRegistry_Filter(t *testing.T) {
|
||||
r := NewRegistry()
|
||||
r.Register(Test{ID: "go-1", Category: "programming", Subcategory: "golang", Prompt: "p", Eval: newDummyEval(1)})
|
||||
r.Register(Test{ID: "py-1", Category: "programming", Subcategory: "python", Prompt: "p", Eval: newDummyEval(1)})
|
||||
r.Register(Test{ID: "k8s-1", Category: "operations", Subcategory: "kubernetes", Prompt: "p", Eval: newDummyEval(1)})
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
category string
|
||||
subcategory string
|
||||
wantIDs []string
|
||||
}{
|
||||
{"no filter", "", "", []string{"go-1", "k8s-1", "py-1"}},
|
||||
{"by category", "programming", "", []string{"go-1", "py-1"}},
|
||||
{"by category case insensitive", "PROGRAMMING", "", []string{"go-1", "py-1"}},
|
||||
{"by subcategory", "", "golang", []string{"go-1"}},
|
||||
{"by category and subcategory", "programming", "python", []string{"py-1"}},
|
||||
{"no match", "nonexistent", "", nil},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := r.Filter(tt.category, tt.subcategory)
|
||||
if len(got) != len(tt.wantIDs) {
|
||||
t.Fatalf("Filter(%q, %q) len = %d, want %d", tt.category, tt.subcategory, len(got), len(tt.wantIDs))
|
||||
}
|
||||
for i, want := range tt.wantIDs {
|
||||
if got[i].ID != want {
|
||||
t.Errorf("Filter(%q, %q)[%d].ID = %q, want %q", tt.category, tt.subcategory, i, got[i].ID, want)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRegistry_Get(t *testing.T) {
|
||||
r := NewRegistry()
|
||||
r.Register(Test{ID: "x", Category: "c", Prompt: "p", Eval: newDummyEval(1)})
|
||||
|
||||
got, ok := r.Get("x")
|
||||
if !ok || got.ID != "x" {
|
||||
t.Errorf("Get(%q) = %+v, %v, want ID x, true", "x", got, ok)
|
||||
}
|
||||
_, ok = r.Get("missing")
|
||||
if ok {
|
||||
t.Error("Get() for missing ID returned ok=true")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRegistry_Len(t *testing.T) {
|
||||
r := NewRegistry()
|
||||
if r.Len() != 0 {
|
||||
t.Errorf("Len() = %d, want 0", r.Len())
|
||||
}
|
||||
r.Register(Test{ID: "x", Category: "c", Prompt: "p", Eval: newDummyEval(1)})
|
||||
if r.Len() != 1 {
|
||||
t.Errorf("Len() = %d, want 1", r.Len())
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user