mirror of
https://github.com/lukaszraczylo/llm-testbench.git
synced 2026-10-12 00:45:19 +00:00
lukaszraczylo
d4b3d4601f
Add a new ai test category with three subcategories: vector-search, llm-integration, and rag. Each subcategory registers 10 tests, for 30 tests total. Vector-search tests cover cosine-vs-dot equivalence, recall@k, HNSW efSearch tradeoffs, product-quantization memory math, ANN pre/post filtering, Reciprocal Rank Fusion, distance-to-similarity conversion, near-duplicate thresholds, and index/embedding-dimension tradeoffs. LLM-integration tests cover OpenAI-compatible field semantics, token budget math for reasoning models, 429 retry/backoff, context-overflow strategy, tool-call handling, role placement, the temperature=0 determinism caveat, SSE stream parsing, embedding batch math, and stop-sequence behavior. RAG tests cover chunk-size tradeoffs, reranker placement, retrieval failure modes, citation grounding, context selection versus stuffing, index staleness, evaluation metric choice, hallucination mitigation ordering, multi-hop decomposition, and pre-assembly deduplication. Every numeric test derives its expected value from an ai-prefixed helper or an existing mathutil/whitepapers helper, and each unit test independently recomputes the same value with plain arithmetic. New files only: ai.go, ai_vectorsearch.go, ai_llmintegration.go, ai_rag.go, and their _test.go siblings, plus a wiring test that calls registerAITests directly. catalog.go and catalog_test.go are untouched; the orchestrator wires registration in at integration.
322 lines
9.5 KiB
Go
322 lines
9.5 KiB
Go
package tests
|
|
|
|
import (
|
|
"context"
|
|
"math"
|
|
"testing"
|
|
)
|
|
|
|
func TestVecCosineVsDotTest_Eval(t *testing.T) {
|
|
// Independent re-derivation: |a| = sqrt(0.6^2+0.8^2), |b| =
|
|
// sqrt(0.8^2+0.6^2), both exactly 1, so cosine similarity's
|
|
// denominator is 1 and the dot product equals the cosine similarity.
|
|
magA := math.Sqrt(0.6*0.6 + 0.8*0.8)
|
|
magB := math.Sqrt(0.8*0.8 + 0.6*0.6)
|
|
if math.Abs(magA-1) > 1e-9 || math.Abs(magB-1) > 1e-9 {
|
|
t.Fatalf("independently recomputed magnitudes = %v, %v, want both 1", magA, magB)
|
|
}
|
|
|
|
tc := vecCosineVsDotTest()
|
|
tests := []struct {
|
|
name string
|
|
response string
|
|
want float64
|
|
}{
|
|
{"correct: yes", "yes", 1},
|
|
{"correct: yes, prose", "Yes, since both magnitudes are 1.", 0},
|
|
{"wrong: no", "no", 0},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
|
if got.Value != tt.want {
|
|
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestVecRecallAtKWant_GroundTruth(t *testing.T) {
|
|
// Independent re-derivation with a plain set intersection, not via
|
|
// wpRecallAtK.
|
|
relevant := map[string]bool{"V3": true, "V8": true, "V11": true, "V14": true, "V20": true}
|
|
retrieved := []string{"V8", "V1", "V14", "V6", "V9"}
|
|
hit := 0
|
|
for _, id := range retrieved {
|
|
if relevant[id] {
|
|
hit++
|
|
}
|
|
}
|
|
want := float64(hit) / float64(len(relevant))
|
|
|
|
if want != 0.4 {
|
|
t.Fatalf("independently recomputed recall@5 = %v, want 0.4", want)
|
|
}
|
|
if vecRecallAtKWant != want {
|
|
t.Errorf("vecRecallAtKWant = %v, independently recomputed = %v", vecRecallAtKWant, want)
|
|
}
|
|
}
|
|
|
|
func TestVecRecallAtKTest_Eval(t *testing.T) {
|
|
tc := vecRecallAtKTest()
|
|
tests := []struct {
|
|
name string
|
|
response string
|
|
want float64
|
|
}{
|
|
{"exact", "0.4", 1},
|
|
{"prose wrapped", "recall@5 is 0.4.", 1},
|
|
{"wrong: counted an extra hit", "0.6", 0},
|
|
{"wrong: missed a hit", "0.2", 0},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
|
if got.Value != tt.want {
|
|
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestVecHNSWEfSearchTradeoffTest_Eval(t *testing.T) {
|
|
tc := vecHNSWEfSearchTradeoffTest()
|
|
tests := []struct {
|
|
name string
|
|
response string
|
|
want float64
|
|
}{
|
|
{"correct: up", "up", 1},
|
|
{"wrong: down", "down", 0},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
|
if got.Value != tt.want {
|
|
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestVecPQMemoryMathWant_GroundTruth(t *testing.T) {
|
|
// Independent re-derivation with plain arithmetic, not via
|
|
// wpQuantizedCodeBytes.
|
|
const m, nbits, n = 16, 4, 1000000
|
|
codeBytes := m * nbits / 8
|
|
want := codeBytes * n
|
|
|
|
if want != 8000000 {
|
|
t.Fatalf("independently recomputed total bytes = %d, want 8000000", want)
|
|
}
|
|
if vecPQMemoryMathWant != want {
|
|
t.Errorf("vecPQMemoryMathWant = %d, independently recomputed = %d", vecPQMemoryMathWant, want)
|
|
}
|
|
}
|
|
|
|
func TestVecPQMemoryMathTest_Eval(t *testing.T) {
|
|
tc := vecPQMemoryMathTest()
|
|
tests := []struct {
|
|
name string
|
|
response string
|
|
want float64
|
|
}{
|
|
{"exact", "8000000", 1},
|
|
{"prose wrapped", "The total is 8000000 bytes.", 1},
|
|
{"wrong", "512", 0},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
|
if got.Value != tt.want {
|
|
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestVecPreVsPostFilteringTest_Eval(t *testing.T) {
|
|
tc := vecPreVsPostFilteringTest()
|
|
tests := []struct {
|
|
name string
|
|
response string
|
|
want float64
|
|
}{
|
|
{"correct: pre", "pre", 1},
|
|
{"wrong: post", "post", 0},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
|
if got.Value != tt.want {
|
|
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestVecRRFFusionWant_GroundTruth(t *testing.T) {
|
|
// Independent re-derivation with plain arithmetic, not via aiRRFScore.
|
|
const k = 60
|
|
const keywordRank, vectorRank = 3, 7
|
|
want := math.Round((1.0/float64(k+keywordRank)+1.0/float64(k+vectorRank))*10000) / 10000
|
|
|
|
if math.Abs(want-0.0308) > 1e-9 {
|
|
t.Fatalf("independently recomputed RRF score = %v, want ~0.0308", want)
|
|
}
|
|
if vecRRFFusionWant != want {
|
|
t.Errorf("vecRRFFusionWant = %v, independently recomputed = %v", vecRRFFusionWant, want)
|
|
}
|
|
}
|
|
|
|
func TestVecRRFFusionTest_Eval(t *testing.T) {
|
|
tc := vecRRFFusionTest()
|
|
tests := []struct {
|
|
name string
|
|
response string
|
|
want float64
|
|
}{
|
|
{"exact", "0.0308", 1},
|
|
{"within tolerance", "0.0307", 1},
|
|
{"prose wrapped", "The RRF score for D5 is 0.0308.", 1},
|
|
{"wrong: only counted one list", "0.0159", 0},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
|
if got.Value != tt.want {
|
|
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestVecDistanceToSimilarityWant_GroundTruth(t *testing.T) {
|
|
// Independent re-derivation with plain arithmetic, not via
|
|
// aiUnitDistanceToCosineSimilarity.
|
|
const d = 0.6
|
|
want := math.Round((1-d*d/2)*10000) / 10000
|
|
|
|
if math.Abs(want-0.82) > 1e-9 {
|
|
t.Fatalf("independently recomputed cosine similarity = %v, want 0.82", want)
|
|
}
|
|
if vecDistanceToSimilarityWant != want {
|
|
t.Errorf("vecDistanceToSimilarityWant = %v, independently recomputed = %v", vecDistanceToSimilarityWant, want)
|
|
}
|
|
}
|
|
|
|
func TestVecDistanceToSimilarityTest_Eval(t *testing.T) {
|
|
tc := vecDistanceToSimilarityTest()
|
|
tests := []struct {
|
|
name string
|
|
response string
|
|
want float64
|
|
}{
|
|
{"exact", "0.82", 1},
|
|
{"prose wrapped", "cos_sim(a,b) = 0.82", 1},
|
|
{"wrong: used d not d^2", "0.7", 0},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
|
if got.Value != tt.want {
|
|
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestVecNearDuplicateThresholdWant_GroundTruth(t *testing.T) {
|
|
// Independent re-derivation with plain arithmetic, not via
|
|
// cosineSimilarity.
|
|
dot := 1.0*2 + 2.0*2 + 2.0*1 + 1.0*1
|
|
normA := math.Sqrt(1*1 + 2*2 + 2*2 + 1*1)
|
|
normB := math.Sqrt(2*2 + 2*2 + 1*1 + 1*1)
|
|
cos := dot / (normA * normB)
|
|
want := math.Round((cos-0.97)*10000) / 10000
|
|
|
|
if math.Abs(want-(-0.07)) > 1e-9 {
|
|
t.Fatalf("independently recomputed difference = %v, want -0.07", want)
|
|
}
|
|
if vecNearDuplicateThresholdWant != want {
|
|
t.Errorf("vecNearDuplicateThresholdWant = %v, independently recomputed = %v", vecNearDuplicateThresholdWant, want)
|
|
}
|
|
}
|
|
|
|
func TestVecNearDuplicateThresholdTest_Eval(t *testing.T) {
|
|
tc := vecNearDuplicateThresholdTest()
|
|
tests := []struct {
|
|
name string
|
|
response string
|
|
want float64
|
|
}{
|
|
{"exact", "-0.07", 1},
|
|
{"prose wrapped", "similarity minus threshold is -0.07.", 1},
|
|
{"wrong: forgot the sign", "0.07", 0},
|
|
{"wrong: reported similarity, not the difference", "0.9", 0},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
|
if got.Value != tt.want {
|
|
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestVecIndexBuildQueryTradeoffTest_Eval(t *testing.T) {
|
|
tc := vecIndexBuildQueryTradeoffTest()
|
|
tests := []struct {
|
|
name string
|
|
response string
|
|
want float64
|
|
}{
|
|
{"all correct", `{"scenario_a":"ivf-flat","scenario_b":"hnsw"}`, 1},
|
|
{"all correct fenced", "```json\n{\"scenario_a\":\"ivf-flat\",\"scenario_b\":\"hnsw\"}\n```", 1},
|
|
{"scenario_a wrong", `{"scenario_a":"hnsw","scenario_b":"hnsw"}`, 0.5},
|
|
{"scenario_b wrong", `{"scenario_a":"ivf-flat","scenario_b":"ivf-flat"}`, 0.5},
|
|
{"both swapped", `{"scenario_a":"hnsw","scenario_b":"ivf-flat"}`, 0},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
|
if got.Value != tt.want {
|
|
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestVecEmbeddingDimensionTradeoffTest_Eval(t *testing.T) {
|
|
// Independent re-derivation confirming only 128-dim fits the budget.
|
|
const budgetBytes = 60_000_000_000
|
|
bytes128 := 128 * 4 * 100_000_000
|
|
bytes768 := 768 * 4 * 100_000_000
|
|
if bytes128 > budgetBytes {
|
|
t.Fatalf("128-dim total %d exceeds budget %d, expected it to fit", bytes128, budgetBytes)
|
|
}
|
|
if bytes768 <= budgetBytes {
|
|
t.Fatalf("768-dim total %d fits budget %d, expected it not to fit", bytes768, budgetBytes)
|
|
}
|
|
|
|
tc := vecEmbeddingDimensionTradeoffTest()
|
|
tests := []struct {
|
|
name string
|
|
response string
|
|
want float64
|
|
}{
|
|
{"correct: 128", `{"choice":"128"}`, 1},
|
|
{"correct, fenced with prose", "The 128-dim option fits:\n```json\n{\"choice\":\"128\"}\n```", 1},
|
|
{"wrong: 768", `{"choice":"768"}`, 0},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
|
if got.Value != tt.want {
|
|
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
|
}
|
|
})
|
|
}
|
|
}
|