package testkit import ( "context" "testing" "github.com/lukaszraczylo/llm-testbench/internal/eval" ) func TestNormalize(t *testing.T) { tests := []struct { name string in string want string }{ {"trims whitespace", " hello ", "hello"}, {"leading whitespace before answer", "\n\nOK", "OK"}, {"strips think block", "reasoning hereOK", "OK"}, {"strips think block with whitespace after", "reasoning here\n\nOK", "OK"}, {"strips think block case insensitive", "xOK", "OK"}, {"strips reasoning block", "planfinal answer", "final answer"}, {"collapses windows line endings", "line1\r\nline2", "line1\nline2"}, {"multiline think block", "\nstep 1\nstep 2\n\nanswer", "answer"}, {"no think block passthrough", "just a plain answer", "just a plain answer"}, {"think tag not at start is left alone", "prefix x suffix", "prefix x suffix"}, // S7: an orphan closing tag (opening tag truncated by the gateway) // must still be stripped. {"orphan think close tag with no opener", "leftover reasoning residueOK", "OK"}, {"orphan reasoning close tag with no opener", "leftover planfinal answer", "final answer"}, {"orphan close tag case insensitive", "residueOK", "OK"}, {"orphan close tag with whitespace after", "residue\n\nOK", "OK"}, {"no close tag at all passthrough", "just a plain answer with no tags", "just a plain answer with no tags"}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { got := Normalize(tt.in) if got != tt.want { t.Errorf("Normalize(%q) = %q, want %q", tt.in, got, tt.want) } }) } } func newDummyEval(value float64) eval.Evaluator { return eval.EvaluatorFunc(func(_ context.Context, _ string) eval.Score { return eval.Score{Value: value} }) } func TestRegistry_RegisterAndAll(t *testing.T) { r := NewRegistry() r.Register(Test{ID: "b-test", Category: "cat", Subcategory: "sub", Prompt: "p", Eval: newDummyEval(1)}) r.Register(Test{ID: "a-test", Category: "cat", Subcategory: "sub", Prompt: "p", Eval: newDummyEval(1)}) all := r.All() if len(all) != 2 { t.Fatalf("All() len = %d, want 2", len(all)) } if all[0].ID != "a-test" || all[1].ID != "b-test" { t.Errorf("All() not sorted by ID: got %q, %q", all[0].ID, all[1].ID) } } func TestRegistry_RegisterDuplicateIDPanics(t *testing.T) { r := NewRegistry() r.Register(Test{ID: "dup", Category: "c", Prompt: "p", Eval: newDummyEval(1)}) defer func() { if recover() == nil { t.Fatal("Register() with duplicate ID did not panic") } }() r.Register(Test{ID: "dup", Category: "c", Prompt: "p", Eval: newDummyEval(1)}) } func TestRegistry_RegisterEmptyIDPanics(t *testing.T) { r := NewRegistry() defer func() { if recover() == nil { t.Fatal("Register() with empty ID did not panic") } }() r.Register(Test{Category: "c", Prompt: "p", Eval: newDummyEval(1)}) } func TestRegistry_RegisterNilEvaluatorPanics(t *testing.T) { r := NewRegistry() defer func() { if recover() == nil { t.Fatal("Register() with nil Evaluator did not panic") } }() r.Register(Test{ID: "x", Category: "c", Prompt: "p"}) } func TestRegistry_Filter(t *testing.T) { r := NewRegistry() r.Register(Test{ID: "go-1", Category: "programming", Subcategory: "golang", Prompt: "p", Eval: newDummyEval(1)}) r.Register(Test{ID: "py-1", Category: "programming", Subcategory: "python", Prompt: "p", Eval: newDummyEval(1)}) r.Register(Test{ID: "k8s-1", Category: "operations", Subcategory: "kubernetes", Prompt: "p", Eval: newDummyEval(1)}) tests := []struct { name string category string subcategory string wantIDs []string }{ {"no filter", "", "", []string{"go-1", "k8s-1", "py-1"}}, {"by category", "programming", "", []string{"go-1", "py-1"}}, {"by category case insensitive", "PROGRAMMING", "", []string{"go-1", "py-1"}}, {"by subcategory", "", "golang", []string{"go-1"}}, {"by category and subcategory", "programming", "python", []string{"py-1"}}, {"no match", "nonexistent", "", nil}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { got := r.Filter(tt.category, tt.subcategory) if len(got) != len(tt.wantIDs) { t.Fatalf("Filter(%q, %q) len = %d, want %d", tt.category, tt.subcategory, len(got), len(tt.wantIDs)) } for i, want := range tt.wantIDs { if got[i].ID != want { t.Errorf("Filter(%q, %q)[%d].ID = %q, want %q", tt.category, tt.subcategory, i, got[i].ID, want) } } }) } } func TestRegistry_Get(t *testing.T) { r := NewRegistry() r.Register(Test{ID: "x", Category: "c", Prompt: "p", Eval: newDummyEval(1)}) got, ok := r.Get("x") if !ok || got.ID != "x" { t.Errorf("Get(%q) = %+v, %v, want ID x, true", "x", got, ok) } _, ok = r.Get("missing") if ok { t.Error("Get() for missing ID returned ok=true") } } func TestRegistry_Len(t *testing.T) { r := NewRegistry() if r.Len() != 0 { t.Errorf("Len() = %d, want 0", r.Len()) } r.Register(Test{ID: "x", Category: "c", Prompt: "p", Eval: newDummyEval(1)}) if r.Len() != 1 { t.Errorf("Len() = %d, want 1", r.Len()) } }