mirror of
https://github.com/lukaszraczylo/llm-testbench.git
synced 2026-10-12 00:45:19 +00:00
lukaszraczylo
9d1d4d89e9
A shared NoUnnegatedMention primitive replaces four per-category implementations, with a bidirectional clause-scoped window and an adjectival cue set. The docker layer fixture drops WORKDIR, which creates a real layer and made the pinned answer wrong. The replica failover test now forces a JSON verdict so a negated answer cannot score. Remaining exact-answer tests migrate to ExactToken. Ordering prompts shuffle their candidate lists so echoing scores zero.
354 lines
11 KiB
Go
354 lines
11 KiB
Go
package tests
|
|
|
|
import (
|
|
"context"
|
|
"math"
|
|
"testing"
|
|
)
|
|
|
|
func TestVecCosineVsDotTest_Eval(t *testing.T) {
|
|
// Independent re-derivation: |a| = sqrt(0.6^2+0.8^2), |b| =
|
|
// sqrt(0.8^2+0.6^2), both exactly 1, so cosine similarity's
|
|
// denominator is 1 and the dot product equals the cosine similarity.
|
|
magA := math.Sqrt(0.6*0.6 + 0.8*0.8)
|
|
magB := math.Sqrt(0.8*0.8 + 0.6*0.6)
|
|
if math.Abs(magA-1) > 1e-9 || math.Abs(magB-1) > 1e-9 {
|
|
t.Fatalf("independently recomputed magnitudes = %v, %v, want both 1", magA, magB)
|
|
}
|
|
|
|
tc := vecCosineVsDotTest()
|
|
tests := []struct {
|
|
name string
|
|
response string
|
|
want float64
|
|
}{
|
|
{"correct: yes", "yes", 1},
|
|
{"correct: different case", "Yes", 1},
|
|
{"correct: quoted", `"yes"`, 1},
|
|
// C3: this row's own label was self-contradictory - "correct"
|
|
// paired with want:0. The prompt asks for "only one word: yes or
|
|
// no"; a full sentence is not a bare/quoted/fenced token even
|
|
// though its content is substantively correct, so want:0 was
|
|
// already the right assertion - only the misleading label needed
|
|
// fixing, matching how eval.ExactToken treats sentence-wrapping
|
|
// everywhere else in this codebase (it is not a summarizer).
|
|
{"wrong: substantively correct but sentence-wrapped, not a bare token", "Yes, since both magnitudes are 1.", 0},
|
|
{"wrong: no", "no", 0},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
|
if got.Value != tt.want {
|
|
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestVecRecallAtKWant_GroundTruth(t *testing.T) {
|
|
// Independent re-derivation with a plain set intersection, not via
|
|
// wpRecallAtK.
|
|
relevant := map[string]bool{"V3": true, "V8": true, "V11": true, "V14": true, "V20": true}
|
|
retrieved := []string{"V8", "V1", "V14", "V6", "V9"}
|
|
hit := 0
|
|
for _, id := range retrieved {
|
|
if relevant[id] {
|
|
hit++
|
|
}
|
|
}
|
|
want := float64(hit) / float64(len(relevant))
|
|
|
|
if want != 0.4 {
|
|
t.Fatalf("independently recomputed recall@5 = %v, want 0.4", want)
|
|
}
|
|
if vecRecallAtKWant != want {
|
|
t.Errorf("vecRecallAtKWant = %v, independently recomputed = %v", vecRecallAtKWant, want)
|
|
}
|
|
}
|
|
|
|
func TestVecRecallAtKTest_Eval(t *testing.T) {
|
|
tc := vecRecallAtKTest()
|
|
tests := []struct {
|
|
name string
|
|
response string
|
|
want float64
|
|
}{
|
|
{"exact", "0.4", 1},
|
|
{"prose wrapped", "recall@5 is 0.4.", 1},
|
|
{"trailing period", "0.4.", 1},
|
|
{"wrong: counted an extra hit", "0.6", 0},
|
|
{"wrong: missed a hit", "0.2", 0},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
|
if got.Value != tt.want {
|
|
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestVecHNSWEfSearchTradeoffTest_Eval(t *testing.T) {
|
|
tc := vecHNSWEfSearchTradeoffTest()
|
|
tests := []struct {
|
|
name string
|
|
response string
|
|
want float64
|
|
}{
|
|
{"correct: up", "up", 1},
|
|
{"correct: uppercase", "UP", 1},
|
|
{"correct: trailing period", "up.", 1},
|
|
{"wrong: down", "down", 0},
|
|
{"wrong: unchanged", "unchanged", 0},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
|
if got.Value != tt.want {
|
|
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestVecPQMemoryMathWant_GroundTruth(t *testing.T) {
|
|
// Independent re-derivation with plain arithmetic, not via
|
|
// wpQuantizedCodeBytes.
|
|
const m, nbits, n = 16, 4, 1000000
|
|
codeBytes := m * nbits / 8
|
|
want := codeBytes * n
|
|
|
|
if want != 8000000 {
|
|
t.Fatalf("independently recomputed total bytes = %d, want 8000000", want)
|
|
}
|
|
if vecPQMemoryMathWant != want {
|
|
t.Errorf("vecPQMemoryMathWant = %d, independently recomputed = %d", vecPQMemoryMathWant, want)
|
|
}
|
|
}
|
|
|
|
func TestVecPQMemoryMathTest_Eval(t *testing.T) {
|
|
tc := vecPQMemoryMathTest()
|
|
tests := []struct {
|
|
name string
|
|
response string
|
|
want float64
|
|
}{
|
|
{"exact", "8000000", 1},
|
|
{"prose wrapped", "The total is 8000000 bytes.", 1},
|
|
{"fenced", "```\n8000000\n```", 1},
|
|
{"wrong", "512", 0},
|
|
{"wrong: bit/byte confusion (8x too large)", "64000000", 0},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
|
if got.Value != tt.want {
|
|
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestVecPreVsPostFilteringTest_Eval(t *testing.T) {
|
|
tc := vecPreVsPostFilteringTest()
|
|
tests := []struct {
|
|
name string
|
|
response string
|
|
want float64
|
|
}{
|
|
{"correct: pre", "pre", 1},
|
|
{"correct: uppercase", "PRE", 1},
|
|
{"correct: trailing period", "pre.", 1},
|
|
{"wrong: post", "post", 0},
|
|
{"wrong: both", "both", 0},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
|
if got.Value != tt.want {
|
|
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestVecRRFFusionWant_GroundTruth(t *testing.T) {
|
|
// Independent re-derivation with plain arithmetic, not via aiRRFScore.
|
|
const k = 60
|
|
const keywordRank, vectorRank = 3, 7
|
|
want := math.Round((1.0/float64(k+keywordRank)+1.0/float64(k+vectorRank))*10000) / 10000
|
|
|
|
if math.Abs(want-0.0308) > 1e-9 {
|
|
t.Fatalf("independently recomputed RRF score = %v, want ~0.0308", want)
|
|
}
|
|
if vecRRFFusionWant != want {
|
|
t.Errorf("vecRRFFusionWant = %v, independently recomputed = %v", vecRRFFusionWant, want)
|
|
}
|
|
}
|
|
|
|
func TestVecRRFFusionTest_Eval(t *testing.T) {
|
|
tc := vecRRFFusionTest()
|
|
tests := []struct {
|
|
name string
|
|
response string
|
|
want float64
|
|
}{
|
|
{"exact", "0.0308", 1},
|
|
{"within tolerance", "0.0307", 1},
|
|
{"prose wrapped", "The RRF score for D5 is 0.0308.", 1},
|
|
{"wrong: only counted one list", "0.0159", 0},
|
|
{"wrong: outside tolerance", "0.0328", 0},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
|
if got.Value != tt.want {
|
|
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestVecDistanceToSimilarityWant_GroundTruth(t *testing.T) {
|
|
// Independent re-derivation with plain arithmetic, not via
|
|
// aiUnitDistanceToCosineSimilarity.
|
|
const d = 0.6
|
|
want := math.Round((1-d*d/2)*10000) / 10000
|
|
|
|
if math.Abs(want-0.82) > 1e-9 {
|
|
t.Fatalf("independently recomputed cosine similarity = %v, want 0.82", want)
|
|
}
|
|
if vecDistanceToSimilarityWant != want {
|
|
t.Errorf("vecDistanceToSimilarityWant = %v, independently recomputed = %v", vecDistanceToSimilarityWant, want)
|
|
}
|
|
}
|
|
|
|
func TestVecDistanceToSimilarityTest_Eval(t *testing.T) {
|
|
tc := vecDistanceToSimilarityTest()
|
|
tests := []struct {
|
|
name string
|
|
response string
|
|
want float64
|
|
}{
|
|
{"exact", "0.82", 1},
|
|
{"prose wrapped", "cos_sim(a,b) = 0.82", 1},
|
|
{"fenced", "```\n0.82\n```", 1},
|
|
{"wrong: used d not d^2", "0.7", 0},
|
|
{"wrong: forgot to subtract from 1", "0.18", 0},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
|
if got.Value != tt.want {
|
|
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestVecNearDuplicateThresholdWant_GroundTruth(t *testing.T) {
|
|
// Independent re-derivation with plain arithmetic, not via
|
|
// cosineSimilarity.
|
|
dot := 1.0*2 + 2.0*2 + 2.0*1 + 1.0*1
|
|
normA := math.Sqrt(1*1 + 2*2 + 2*2 + 1*1)
|
|
normB := math.Sqrt(2*2 + 2*2 + 1*1 + 1*1)
|
|
cos := dot / (normA * normB)
|
|
want := math.Round((cos-0.97)*10000) / 10000
|
|
|
|
if math.Abs(want-(-0.07)) > 1e-9 {
|
|
t.Fatalf("independently recomputed difference = %v, want -0.07", want)
|
|
}
|
|
if vecNearDuplicateThresholdWant != want {
|
|
t.Errorf("vecNearDuplicateThresholdWant = %v, independently recomputed = %v", vecNearDuplicateThresholdWant, want)
|
|
}
|
|
}
|
|
|
|
func TestVecNearDuplicateThresholdTest_Eval(t *testing.T) {
|
|
tc := vecNearDuplicateThresholdTest()
|
|
tests := []struct {
|
|
name string
|
|
response string
|
|
want float64
|
|
}{
|
|
{"exact", "-0.07", 1},
|
|
{"prose wrapped", "similarity minus threshold is -0.07.", 1},
|
|
{"fenced", "```\n-0.07\n```", 1},
|
|
{"wrong: forgot the sign", "0.07", 0},
|
|
{"wrong: reported similarity, not the difference", "0.9", 0},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
|
if got.Value != tt.want {
|
|
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestVecIndexBuildQueryTradeoffTest_Eval(t *testing.T) {
|
|
tc := vecIndexBuildQueryTradeoffTest()
|
|
tests := []struct {
|
|
name string
|
|
response string
|
|
want float64
|
|
}{
|
|
{"all correct", `{"scenario_a":"ivf-flat","scenario_b":"hnsw"}`, 1},
|
|
{"all correct fenced", "```json\n{\"scenario_a\":\"ivf-flat\",\"scenario_b\":\"hnsw\"}\n```", 1},
|
|
{"all correct, different case", `{"scenario_a":"IVF-FLAT","scenario_b":"HNSW"}`, 1},
|
|
{"scenario_a wrong", `{"scenario_a":"hnsw","scenario_b":"hnsw"}`, 0.5},
|
|
{"scenario_b wrong", `{"scenario_a":"ivf-flat","scenario_b":"ivf-flat"}`, 0.5},
|
|
{"both swapped", `{"scenario_a":"hnsw","scenario_b":"ivf-flat"}`, 0},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
|
if got.Value != tt.want {
|
|
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestVecEmbeddingDimensionTradeoffTest_Eval(t *testing.T) {
|
|
// Independent re-derivation confirming only 128-dim fits the budget.
|
|
const budgetBytes = 60_000_000_000
|
|
bytes128 := 128 * 4 * 100_000_000
|
|
bytes768 := 768 * 4 * 100_000_000
|
|
if bytes128 > budgetBytes {
|
|
t.Fatalf("128-dim total %d exceeds budget %d, expected it to fit", bytes128, budgetBytes)
|
|
}
|
|
if bytes768 <= budgetBytes {
|
|
t.Fatalf("768-dim total %d fits budget %d, expected it not to fit", bytes768, budgetBytes)
|
|
}
|
|
|
|
tc := vecEmbeddingDimensionTradeoffTest()
|
|
tests := []struct {
|
|
name string
|
|
response string
|
|
want float64
|
|
}{
|
|
{"correct: 128", `{"choice":"128"}`, 1},
|
|
{"correct, fenced with prose", "The 128-dim option fits:\n```json\n{\"choice\":\"128\"}\n```", 1},
|
|
{"wrong: 768", `{"choice":"768"}`, 0},
|
|
{"wrong: neither valid option", `{"choice":"512"}`, 0},
|
|
{
|
|
// C7 bug probe: the prompt's own template shows choice
|
|
// unquoted ({"choice":128|768}), so a genuinely numeric
|
|
// (unquoted) response must also score full credit.
|
|
name: "correct: unquoted numeric form matching the prompt's own template (C7 bug probe)",
|
|
response: `{"choice":128}`,
|
|
want: 1,
|
|
},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
|
if got.Value != tt.want {
|
|
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
|
}
|
|
})
|
|
}
|
|
}
|