mirror of
https://github.com/lukaszraczylo/llm-testbench.git
synced 2026-10-12 00:45:19 +00:00
lukaszraczylo
d4b3d4601f
Add a new ai test category with three subcategories: vector-search, llm-integration, and rag. Each subcategory registers 10 tests, for 30 tests total. Vector-search tests cover cosine-vs-dot equivalence, recall@k, HNSW efSearch tradeoffs, product-quantization memory math, ANN pre/post filtering, Reciprocal Rank Fusion, distance-to-similarity conversion, near-duplicate thresholds, and index/embedding-dimension tradeoffs. LLM-integration tests cover OpenAI-compatible field semantics, token budget math for reasoning models, 429 retry/backoff, context-overflow strategy, tool-call handling, role placement, the temperature=0 determinism caveat, SSE stream parsing, embedding batch math, and stop-sequence behavior. RAG tests cover chunk-size tradeoffs, reranker placement, retrieval failure modes, citation grounding, context selection versus stuffing, index staleness, evaluation metric choice, hallucination mitigation ordering, multi-hop decomposition, and pre-assembly deduplication. Every numeric test derives its expected value from an ai-prefixed helper or an existing mathutil/whitepapers helper, and each unit test independently recomputes the same value with plain arithmetic. New files only: ai.go, ai_vectorsearch.go, ai_llmintegration.go, ai_rag.go, and their _test.go siblings, plus a wiring test that calls registerAITests directly. catalog.go and catalog_test.go are untouched; the orchestrator wires registration in at integration.
50 lines
1.5 KiB
Go
50 lines
1.5 KiB
Go
package tests
|
|
|
|
import (
|
|
"testing"
|
|
|
|
"github.com/lukaszraczylo/llm-testbench/internal/testkit"
|
|
)
|
|
|
|
// TestRegisterAITests_Wiring checks registerAITests wires up all three ai
|
|
// subcategories - vector-search, llm-integration, rag - with 10 tests
|
|
// each, all categorized "ai", and no duplicate IDs across the three
|
|
// sibling files (testkit.Registry.Register panics on a duplicate ID,
|
|
// which would fail this test outright). This is the integration point the
|
|
// orchestrator's catalog wiring depends on; catalog.go itself is not
|
|
// touched by this worktree.
|
|
func TestRegisterAITests_Wiring(t *testing.T) {
|
|
r := testkit.NewRegistry()
|
|
registerAITests(r)
|
|
|
|
const wantTotal = 30
|
|
if r.Len() != wantTotal {
|
|
t.Fatalf("registerAITests: registry has %d tests, want %d", r.Len(), wantTotal)
|
|
}
|
|
|
|
counts := map[string]int{}
|
|
for _, tc := range r.All() {
|
|
if tc.Category != "ai" {
|
|
t.Errorf("test %q: Category = %q, want %q", tc.ID, tc.Category, "ai")
|
|
}
|
|
if tc.Prompt == "" {
|
|
t.Errorf("test %q: Prompt is empty", tc.ID)
|
|
}
|
|
if tc.Eval == nil {
|
|
t.Errorf("test %q: Eval is nil", tc.ID)
|
|
}
|
|
if tc.Description == "" {
|
|
t.Errorf("test %q: Description is empty", tc.ID)
|
|
}
|
|
if tc.MaxTokens != 0 {
|
|
t.Errorf("test %q: MaxTokens = %d, want 0 (omitted, per round-2 rule 7)", tc.ID, tc.MaxTokens)
|
|
}
|
|
counts[tc.Subcategory]++
|
|
}
|
|
for _, sub := range []string{"vector-search", "llm-integration", "rag"} {
|
|
if counts[sub] != 10 {
|
|
t.Errorf("subcategory %q: got %d tests, want 10", sub, counts[sub])
|
|
}
|
|
}
|
|
}
|