diff --git a/internal/tests/codebase.go b/internal/tests/codebase.go index 980fbde..71da040 100644 --- a/internal/tests/codebase.go +++ b/internal/tests/codebase.go @@ -7,6 +7,15 @@ import ( func registerCodebaseTests(r *testkit.Registry) { r.Register(codeTraceGoTest()) + r.Register(codeTracePythonTest()) + r.Register(codeTraceTSTest()) + r.Register(codeBugLineDiffTest()) + r.Register(codeRaceVariableTest()) + r.Register(codeDeadFunctionsTest()) + r.Register(codeBigODedupTest()) + r.Register(codeImportCycleTest()) + r.Register(codeGenericReturnTypeTest()) + r.Register(codeCommitBisectTest()) } // mix is the traced function for codeTraceGoTest: deterministic bit diff --git a/internal/tests/codebase_analysis.go b/internal/tests/codebase_analysis.go new file mode 100644 index 0000000..4fb3e5d --- /dev/null +++ b/internal/tests/codebase_analysis.go @@ -0,0 +1,304 @@ +package tests + +import ( + "github.com/lukaszraczylo/llm-testbench/internal/eval" + "github.com/lukaszraczylo/llm-testbench/internal/testkit" +) + +// codeBugDiffSnippet is a unified diff for codeBugLineDiffTest, with each +// displayed line prefixed by its 1-based position in this listing (not the +// diff's own hunk line numbers), so "which line" has one unambiguous +// meaning. +const codeBugDiffSnippet = `1: @@ -1,9 +1,9 @@ +2: func Average(nums []float64) float64 { +3: if len(nums) == 0 { +4: return 0 +5: } +6: sum := 0.0 +7: - for _, n := range nums { +8: + for i := 0; i <= len(nums); i++ { +9: - sum += n +10: + sum += nums[i] +11: } +12: return sum / float64(len(nums)) +13: }` + +// codeBugLineDiffTest: identify the root-cause line of an off-by-one +// out-of-bounds bug in an inline, line-numbered diff. +// +// ground truth: line 8's new loop condition `i <= len(nums)` allows i to +// reach len(nums), and line 10 then indexes nums[i] at that out-of-range +// value, panicking. Line 8 is the root cause (the wrong bound); line 10 is +// only where the panic actually fires - the correct bound is `i < +// len(nums)`. +func codeBugLineDiffTest() testkit.Test { + prompt := `Here is a diff modifying a Go function that computes an average. Each +displayed line below is prefixed with its position in this listing (not +the diff's own internal hunk numbering): + +` + codeBugDiffSnippet + ` + +This change introduces a bug that causes a runtime panic: index out of +range. Which line number contains the root cause of the panic - the +incorrect loop bound - as opposed to the line where the panic actually +fires? Respond with only a JSON object: {"line":}` + + return testkit.Test{ + ID: "code-bug-line-diff", + Category: "research", + Subcategory: "codebase", + Description: "Identify the root-cause line of an off-by-one out-of-bounds bug in an inline, line-numbered diff.", + Prompt: prompt, + Eval: eval.JSONField("line", 8), + } +} + +// codeRaceSnippet is an inline Go function for codeRaceVariableTest with a +// classic unsynchronized shared-counter data race. +const codeRaceSnippet = `func Count(n int) int { + total := 0 + var wg sync.WaitGroup + for i := 0; i < n; i++ { + wg.Add(1) + go func() { + defer wg.Done() + total++ + }() + } + wg.Wait() + return total +}` + +// codeRaceVariableTest: name the variable with a data race in an inline Go +// snippet. +// +// ground truth: every launched goroutine executes `total++` (a +// read-modify-write) on the same shared `total` variable with no mutex, +// atomic operation, or channel serializing access, so concurrent +// increments race. `n` and `i` are each goroutine-local or loop-local and +// are not raced on by this snippet. +func codeRaceVariableTest() testkit.Test { + prompt := `Here is a Go function: + +` + "```go\n" + codeRaceSnippet + "\n```" + ` + +Which single variable has a data race in this snippet - concurrent, +unsynchronized reads and writes from multiple goroutines? Respond with +only a JSON object: {"variable":""}` + + return testkit.Test{ + ID: "code-race-variable", + Category: "research", + Subcategory: "codebase", + Description: "Name the racy shared variable in an inline Go snippet with an unsynchronized concurrent counter.", + Prompt: prompt, + Eval: eval.JSONField("variable", "total"), + } +} + +// codeDeadFunctionsSnippet is an inline Go file for codeDeadFunctionsTest +// with two functions reachable from main and two that are never called. +const codeDeadFunctionsSnippet = `package main + +import "fmt" + +func helperA() int { return 1 } +func helperB() int { return helperA() + 1 } +func helperC() int { return 3 } +func helperD() int { return helperC() * 2 } + +func main() { + fmt.Println(helperB()) +}` + +// codeDeadFunctionsTest: identify functions unreachable from main in an +// inline Go file. +// +// ground truth: main calls only helperB, which calls helperA - both are +// reachable. Nothing calls helperD, and the only caller of helperC is +// helperD itself, so neither helperC nor helperD is reachable from main; +// both are dead code. +func codeDeadFunctionsTest() testkit.Test { + prompt := `Here is a small Go file: + +` + "```go\n" + codeDeadFunctionsSnippet + "\n```" + ` + +Which functions are dead code - never reachable, directly or transitively, +from main? Respond with only a JSON array of function names.` + + return testkit.Test{ + ID: "code-dead-functions", + Category: "research", + Subcategory: "codebase", + Description: "Identify functions unreachable from main in an inline Go file with both live and dead call chains.", + Prompt: prompt, + Eval: eval.JSONStringSet([]string{"helperC", "helperD"}), + } +} + +// codeDedupSnippet is an inline Go function for codeBigODedupTest with a +// nested-loop worst-case-quadratic deduplication implementation. +const codeDedupSnippet = `func Dedup(a []int) []int { + var out []int + for _, x := range a { + found := false + for _, y := range out { + if x == y { + found = true + break + } + } + if !found { + out = append(out, x) + } + } + return out +}` + +// codeBigODedupPattern accepts every common notation for O(n^2): a caret, +// a double-star, the unicode superscript two, or n*n, each optionally +// wrapped as O(...) with flexible internal spacing. +const codeBigODedupPattern = `(?i)o\(\s*n\s*(\^\s*2|\*\*\s*2|²|\*\s*n)\s*\)` + +// codeBigODedupTest: state the worst-case time complexity of an inline +// nested-loop deduplication function. +// +// ground truth: when a has no duplicates, the inner loop scans the whole +// of out, which grows to length n, for every one of the n outer +// iterations, giving n + (n-1) + ... = O(n^2) comparisons in the worst +// case. +func codeBigODedupTest() testkit.Test { + prompt := `Here is a Go function: + +` + "```go\n" + codeDedupSnippet + "\n```" + ` + +What is the worst-case time complexity of Dedup, in Big-O notation, in +terms of n = len(a)? Respond with only the Big-O expression, e.g. O(n).` + + return testkit.Test{ + ID: "code-bigo-dedup", + Category: "research", + Subcategory: "codebase", + Description: "State the worst-case Big-O time complexity of an inline nested-loop deduplication function.", + Prompt: prompt, + Eval: eval.Regex(codeBigODedupPattern), + } +} + +// codeImportListSnippet lists each inline package's direct imports for +// codeImportCycleTest, with one cycle among four packages. +const codeImportListSnippet = `- package a imports: [b] +- package b imports: [c] +- package c imports: [a, d] +- package d imports: []` + +// codeImportCycleTest: detect which packages form an import cycle from an +// inline list of per-package direct imports. +// +// ground truth: following the edges a->b->c->a closes a cycle among a, b, +// and c. Package d is imported by c but does not import anything itself, +// so it is not part of any cycle. +func codeImportCycleTest() testkit.Test { + prompt := `Here are the direct imports of four packages: + +` + codeImportListSnippet + ` + +Which packages form an import cycle? Respond with only a JSON array of +package names involved in the cycle (any order).` + + return testkit.Test{ + ID: "code-import-cycle", + Category: "research", + Subcategory: "codebase", + Description: "Detect which packages form an import cycle from an inline list of per-package direct imports.", + Prompt: prompt, + Eval: eval.JSONStringSet([]string{"a", "b", "c"}), + } +} + +// codeGenericFirstSnippet is an inline Go generic function plus a call +// site for codeGenericReturnTypeTest. +const codeGenericFirstSnippet = `func First[T any](s []T) (T, bool) { + if len(s) == 0 { + var zero T + return zero, false + } + return s[0], true +} + +result, ok := First([]string{"a", "b"})` + +// codeGenericReturnTypeTest: infer the concrete instantiated type of a +// generic function's result at a specific call site. +// +// ground truth: First is instantiated with T = string because it is +// called with []string{"a", "b"}, so result's concrete type is string. +func codeGenericReturnTypeTest() testkit.Test { + prompt := `Here is a generic Go function and a call site: + +` + "```go\n" + codeGenericFirstSnippet + "\n```" + ` + +What is the concrete type of the variable "result" in the code above? +Respond with only the type name, nothing else.` + + return testkit.Test{ + ID: "code-generic-return-type", + Category: "research", + Subcategory: "codebase", + Description: "Infer the concrete instantiated type of a generic Go function's result at a specific call site.", + Prompt: prompt, + Eval: codeExactAnswer("string"), + } +} + +// codeBisectCommitLog is an inline commit history for +// codeCommitBisectTest, where exactly one commit changes safeDivide's +// result cache to be keyed only by "a" and checked before validation. +const codeBisectCommitLog = `c1a1a1a "init: add safeDivide(a, b) that returns an error if b == 0, +otherwise returns a / b" + +c2b2b2b "refactor: rewrite safeDivide's internal branch as a plain +if/else, no behavior change" + +c3c3c3c "perf: add a result cache to safeDivide, keyed only by a (not by +the pair a and b), checked BEFORE the b == 0 validation runs" + +c4d4d4d "chore: add debug logging around cache hits in safeDivide" + +c5e5e5e "docs: fix a typo in the cache-hit log message"` + +// codeCommitBisectTest: use git-bisect-style reasoning over an inline +// commit log and a described symptom to name the exact commit that +// introduced the bug. +// +// ground truth: c3c3c3c's cache is keyed only by the first argument a and +// is checked before the b == 0 validation runs. A prior call +// safeDivide(10, 2) caches a successful result under key a=10; a later +// call safeDivide(10, 0) then hits that stale cache entry (same a, cache +// ignores b) and returns it instead of reaching the validation that would +// raise the b == 0 error. c1 predates the cache entirely; c2 is a +// no-behavior-change refactor; c4 and c5 only touch logging text and do +// not change the cache's key or check order, so the bug already existed +// before either of them. +func codeCommitBisectTest() testkit.Test { + prompt := `Here is a commit history for a function safeDivide(a, b), oldest first: + +` + codeBisectCommitLog + ` + +Symptom observed in production after deploying through the latest commit: +calling safeDivide(10, 2) returns 5 (correct). Immediately afterward, +calling safeDivide(10, 0) returns 5 instead of raising the expected +"b == 0" error. + +Using git-bisect-style reasoning, which commit introduced this bug? +Respond with only the commit id (the 7-character id shown), nothing else.` + + return testkit.Test{ + ID: "code-commit-bisect", + Category: "research", + Subcategory: "codebase", + Description: "Use git-bisect-style reasoning over an inline commit log and a symptom to name the commit that introduced the bug.", + Prompt: prompt, + Eval: codeExactAnswer("c3c3c3c"), + } +} diff --git a/internal/tests/codebase_analysis_test.go b/internal/tests/codebase_analysis_test.go new file mode 100644 index 0000000..bb4640d --- /dev/null +++ b/internal/tests/codebase_analysis_test.go @@ -0,0 +1,173 @@ +package tests + +import ( + "context" + "testing" +) + +func TestCodeBugLineDiffTest_Eval(t *testing.T) { + tc := codeBugLineDiffTest() + + tests := []struct { + name string + response string + want float64 + }{ + {"correct: root-cause line", `{"line":8}`, 1}, + {"wrong: points at the panic site instead of the root cause", `{"line":10}`, 0}, + {"wrong line entirely", `{"line":6}`, 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Value != tt.want { + t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail) + } + }) + } +} + +func TestCodeRaceVariableTest_Eval(t *testing.T) { + tc := codeRaceVariableTest() + + tests := []struct { + name string + response string + want float64 + }{ + {"correct", `{"variable":"total"}`, 1}, + {"correct different case", `{"variable":"Total"}`, 1}, + {"wrong: loop variable, not the shared counter", `{"variable":"i"}`, 0}, + {"wrong: function parameter, not the shared counter", `{"variable":"n"}`, 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Value != tt.want { + t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail) + } + }) + } +} + +func TestCodeDeadFunctionsTest_Eval(t *testing.T) { + tc := codeDeadFunctionsTest() + + tests := []struct { + name string + response string + want float64 + }{ + {"correct set", `["helperC","helperD"]`, 1}, + {"correct set, different order", `["helperD","helperC"]`, 1}, + {"missing one dead function", `["helperD"]`, 0.5}, + {"wrongly includes a live function", `["helperC","helperD","helperB"]`, 2.0 / 3.0}, + {"wrong: says nothing is dead", `[]`, 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Value != tt.want { + t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail) + } + }) + } +} + +func TestCodeBigODedupTest_Eval(t *testing.T) { + tc := codeBigODedupTest() + + tests := []struct { + name string + response string + want float64 + }{ + {"caret form", "O(n^2)", 1}, + {"double-star form", "O(n**2)", 1}, + {"unicode superscript form", "O(n²)", 1}, + {"n*n form", "O(n*n)", 1}, + {"lowercase o", "o(n^2)", 1}, + {"wrong: linear", "O(n)", 0}, + {"wrong: linearithmic", "O(n log n)", 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Value != tt.want { + t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail) + } + }) + } +} + +func TestCodeImportCycleTest_Eval(t *testing.T) { + tc := codeImportCycleTest() + + tests := []struct { + name string + response string + want float64 + }{ + {"correct set", `["a","b","c"]`, 1}, + {"correct set, different order", `["c","a","b"]`, 1}, + {"wrongly includes non-cyclic d", `["a","b","c","d"]`, 0.75}, + {"missing one cycle member", `["a","b"]`, 2.0 / 3.0}, + {"wrong: only d, which has no imports at all", `["d"]`, 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Value != tt.want { + t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail) + } + }) + } +} + +func TestCodeGenericReturnTypeTest_Eval(t *testing.T) { + tc := codeGenericReturnTypeTest() + + tests := []struct { + name string + response string + want float64 + }{ + {"correct", "string", 1}, + {"correct capitalized", "String", 1}, + {"correct quoted", "`string`", 1}, + {"wrong: the slice element pointer type", "*string", 0}, + {"wrong: the generic parameter name itself, not its instantiation", "T", 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Value != tt.want { + t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail) + } + }) + } +} + +func TestCodeCommitBisectTest_Eval(t *testing.T) { + tc := codeCommitBisectTest() + + tests := []struct { + name string + response string + want float64 + }{ + {"correct", "c3c3c3c", 1}, + {"correct with trailing period", "c3c3c3c.", 1}, + {"wrong: the initial commit, predates the cache entirely", "c1a1a1a", 0}, + {"wrong: only a logging change, no behavior change", "c4d4d4d", 0}, + {"wrong: the docs-only commit", "c5e5e5e", 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Value != tt.want { + t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail) + } + }) + } +} diff --git a/internal/tests/codebase_trace.go b/internal/tests/codebase_trace.go new file mode 100644 index 0000000..48c4411 --- /dev/null +++ b/internal/tests/codebase_trace.go @@ -0,0 +1,127 @@ +package tests + +import ( + "context" + "fmt" + "strconv" + "strings" + + "github.com/lukaszraczylo/llm-testbench/internal/eval" + "github.com/lukaszraczylo/llm-testbench/internal/testkit" +) + +// codeExactAnswer returns an Evaluator awarding full credit when the +// response, trimmed of whitespace, at most one layer of surrounding quote +// characters (' , ", or `), and a single trailing sentence-ending period, +// equals want case-insensitively. This accepts every materially-correct +// form of a forced single-token answer (bare, quoted, differently-cased, +// or with trailing punctuation) without loosening the match to accept a +// substring of a longer, wrong answer. +func codeExactAnswer(want string) eval.Evaluator { + return eval.EvaluatorFunc(func(_ context.Context, response string) eval.Score { + got := strings.TrimSpace(response) + got = strings.Trim(got, "\"'`") + got = strings.TrimSuffix(strings.TrimSpace(got), ".") + got = strings.TrimSpace(got) + wantTrimmed := strings.TrimSpace(want) + if strings.EqualFold(got, wantTrimmed) { + return eval.Score{Value: 1, Detail: fmt.Sprintf("equals %q", wantTrimmed)} + } + return eval.Score{Value: 0, Detail: fmt.Sprintf("got %q, want %q", got, wantTrimmed)} + }) +} + +// codeTracePythonSource is the inline Python function traced by +// codeTracePythonTest. +const codeTracePythonSource = `def transform(s, n): + if n == 0: + return s + if len(s) % 2 == 0: + s = s[1:] + s[0] + else: + s = s[::-1] + return transform(s, n - 1)` + +// codeTracePythonWant is the exact result of transform("abcdef", 4). +// +// ground truth: len("abcdef") is 6 (even) at every recursive call, since +// the rotate branch never changes length, so every call takes the rotate +// branch: "abcdef"->"bcdefa"->"cdefab"->"defabc"->"efabcd" over 4 calls +// (n=4,3,2,1), then n=0 returns "efabcd". codebase_trace_test.go both +// hand-derives this independently and, when python3 is available, runs +// the source above directly and compares stdout. +const codeTracePythonWant = "efabcd" + +// codeTracePythonTest: trace a recursive Python function combining a +// parity check, string rotation, and full reversal to its exact output. +func codeTracePythonTest() testkit.Test { + prompt := `Here is a Python function: + +` + "```python\n" + codeTracePythonSource + "\n```" + ` + +What is the exact return value of transform("abcdef", 4)? Respond with +only the resulting string, with no quotes and no other text.` + + return testkit.Test{ + ID: "code-trace-python", + Category: "research", + Subcategory: "codebase", + Description: "Trace a recursive Python function combining a parity check, string rotation, and full reversal to its exact output.", + Prompt: prompt, + Eval: codeExactAnswer(codeTracePythonWant), + } +} + +// codeTraceTSSource is the inline TypeScript function traced by +// codeTraceTSTest. +const codeTraceTSSource = `function pack(items: number[]): string { + return items.reduce((acc, x, i) => { + if (i % 2 === 0) return acc + x; + return acc + x * 2; + }, 0).toString(2); +}` + +// codeTSPackTrace replicates pack's reduce logic in Go so +// codebase_trace_test.go can cross-check the hand-derived trace by +// running equivalent logic, since the TypeScript itself is not executed. +func codeTSPackTrace(items []int) string { + acc := 0 + for i, x := range items { + if i%2 == 0 { + acc += x + } else { + acc += x * 2 + } + } + return strconv.FormatInt(int64(acc), 2) +} + +// codeTraceTSWant is the exact result of pack([3,4,5,6,7]). +// +// ground truth: acc starts at 0; i=0,x=3 (even index) -> acc=3; i=1,x=4 +// (odd index) -> acc=3+4*2=11; i=2,x=5 -> acc=11+5=16; i=3,x=6 -> +// acc=16+6*2=28; i=4,x=7 -> acc=28+7=35. 35 in binary is 100011 +// (32+2+1=35). codebase_trace_test.go recomputes this both by hand and by +// calling codeTSPackTrace. +const codeTraceTSWant = "100011" + +// codeTraceTSTest: trace a TypeScript reduce-based function mixing +// index-parity branching and binary string conversion to its exact +// output. +func codeTraceTSTest() testkit.Test { + prompt := `Here is a TypeScript function: + +` + "```ts\n" + codeTraceTSSource + "\n```" + ` + +What is the exact return value of pack([3, 4, 5, 6, 7])? Respond with only +the resulting string, with no quotes and no other text.` + + return testkit.Test{ + ID: "code-trace-ts", + Category: "research", + Subcategory: "codebase", + Description: "Trace a TypeScript reduce-based function mixing index-parity branching and binary string conversion to its exact output.", + Prompt: prompt, + Eval: codeExactAnswer(codeTraceTSWant), + } +} diff --git a/internal/tests/codebase_trace_test.go b/internal/tests/codebase_trace_test.go new file mode 100644 index 0000000..26c8b0b --- /dev/null +++ b/internal/tests/codebase_trace_test.go @@ -0,0 +1,146 @@ +package tests + +import ( + "bytes" + "context" + "os" + "os/exec" + "strings" + "testing" +) + +func TestCodeExactAnswer(t *testing.T) { + ev := codeExactAnswer("efabcd") + + tests := []struct { + name string + response string + want float64 + }{ + {"bare correct", "efabcd", 1}, + {"double-quoted correct", `"efabcd"`, 1}, + {"different case", "EFABCD", 1}, + {"trailing period", "efabcd.", 1}, + {"wrong string", "abcdef", 0}, + {"correct but truncated", "efabc", 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := ev.Evaluate(context.Background(), tt.response) + if got.Value != tt.want { + t.Errorf("Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail) + } + }) + } +} + +func TestCodeTracePythonWant_GroundTruth(t *testing.T) { + // Hand-derivation, double-checked: len("abcdef")=6 is even at every + // step, so every call takes the rotate branch (s[1:]+s[0]) rather than + // the reverse branch. + s := "abcdef" + for i := 0; i < 4; i++ { + if len(s)%2 == 0 { + s = s[1:] + string(s[0]) + } else { + for l, r := 0, len(s)-1; l < r; l, r = l+1, r-1 { + b := []byte(s) + b[l], b[r] = b[r], b[l] + s = string(b) + } + } + } + if s != codeTracePythonWant { + t.Errorf("hand-derived trace = %q, want %q", s, codeTracePythonWant) + } + + if _, err := exec.LookPath("python3"); err != nil { + t.Skip("python3 not found on PATH, skipping exec ground-truth cross-check") + } + + script := codeTracePythonSource + "\nprint(transform(\"abcdef\", 4))\n" + // #nosec G204 -- script is a fixed, hardcoded string built from the + // same codeTracePythonSource constant embedded in the prompt, not + // external or response-controlled input. + cmd := exec.Command("python3", "-c", script) //nolint:gosec // see comment above + var stdout bytes.Buffer + cmd.Stdout = &stdout + cmd.Stderr = os.Stderr + if err := cmd.Run(); err != nil { + t.Fatalf("running python3 ground-truth script failed: %v", err) + } + got := strings.TrimSpace(stdout.String()) + if got != codeTracePythonWant { + t.Errorf("python3-computed transform(\"abcdef\", 4) = %q, want %q", got, codeTracePythonWant) + } +} + +func TestCodeTracePythonTest_Eval(t *testing.T) { + tc := codeTracePythonTest() + + tests := []struct { + name string + response string + want float64 + }{ + {"correct", "efabcd", 1}, + {"correct quoted", `"efabcd"`, 1}, + {"wrong: forgot final rotation", "defabc", 0}, + {"wrong: reversed instead of rotated", "fedcba", 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Value != tt.want { + t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail) + } + }) + } +} + +func TestCodeTraceTSWant_GroundTruth(t *testing.T) { + // Hand-derivation, double-checked against codeTSPackTrace: acc=0; + // i=0,x=3 (even index) -> 3; i=1,x=4 (odd) -> 3+8=11; i=2,x=5 (even) -> + // 16; i=3,x=6 (odd) -> 16+12=28; i=4,x=7 (even) -> 35. 35 = 32+2+1 in + // binary is 100011. + acc := 0 + items := []int{3, 4, 5, 6, 7} + for i, x := range items { + if i%2 == 0 { + acc += x + } else { + acc += x * 2 + } + } + if acc != 35 { + t.Fatalf("hand-derived acc = %d, want 35", acc) + } + + got := codeTSPackTrace(items) + if got != codeTraceTSWant { + t.Errorf("codeTSPackTrace(%v) = %q, want %q", items, got, codeTraceTSWant) + } +} + +func TestCodeTraceTSTest_Eval(t *testing.T) { + tc := codeTraceTSTest() + + tests := []struct { + name string + response string + want float64 + }{ + {"correct", "100011", 1}, + {"correct quoted", `"100011"`, 1}, + {"wrong: doubled every element instead of alternating", "1000110", 0}, + {"wrong: decimal instead of binary", "35", 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Value != tt.want { + t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail) + } + }) + } +} diff --git a/internal/tests/web.go b/internal/tests/web.go index eae5368..926b418 100644 --- a/internal/tests/web.go +++ b/internal/tests/web.go @@ -7,6 +7,15 @@ import ( func registerWebTests(r *testkit.Registry) { r.Register(webRobotsAICrawlersTest()) + r.Register(webHTMLProductExtractTest()) + r.Register(webHreflangReciprocalTest()) + r.Register(webURLParseComponentsTest()) + r.Register(webSitemapMaxURLsTest()) + r.Register(webHTTPStatusScenariosTest()) + r.Register(webDNSMXRecordTest()) + r.Register(webCanonicalVsRedirectTest()) + r.Register(webCORSMissingHeaderTest()) + r.Register(webSecurityHeadersAuditTest()) } // webRobotsAICrawlersRobotsTxt is the inline robots.txt for diff --git a/internal/tests/web_extract.go b/internal/tests/web_extract.go new file mode 100644 index 0000000..06d8b40 --- /dev/null +++ b/internal/tests/web_extract.go @@ -0,0 +1,137 @@ +package tests + +import ( + "github.com/lukaszraczylo/llm-testbench/internal/eval" + "github.com/lukaszraczylo/llm-testbench/internal/testkit" +) + +// webProductCardHTML is the inline HTML product-card snippet for +// webHTMLProductExtractTest. +const webProductCardHTML = `
+

Wireless Mechanical Keyboard

+ 129.99 + In Stock +
` + +// webHTMLProductExtractTest: extract a product's name, price, and SKU +// from inline HTML into JSON. +// +// ground truth: name is the h2.product-name text; price is the +// span.price text as a number; sku is the data-sku attribute on the +// outer div - all read directly from the markup, no computation. +func webHTMLProductExtractTest() testkit.Test { + prompt := `Here is an HTML product card: + +` + "```html\n" + webProductCardHTML + "\n```" + ` + +Extract the product's name, price (as a number), and SKU into JSON. +Respond with only a JSON object: {"name":"...","price":,"sku":"..."}` + + evaluator := eval.Mean( + eval.JSONField("name", "Wireless Mechanical Keyboard"), + eval.JSONField("price", 129.99), + eval.JSONField("sku", "WBH-2201"), + ) + + return testkit.Test{ + ID: "web-html-product-extract", + Category: "research", + Subcategory: "web", + Description: "Extract a product's name, price, and SKU from an inline HTML product card into JSON.", + Prompt: prompt, + Eval: evaluator, + } +} + +// webHreflangPagesSnippet lists the hreflang tags of three pages +// for webHreflangReciprocalTest. /en/ links to both /es/ and /fr/, and +// /es/ links back to both, but /fr/ omits the reciprocal link back to +// /en/. +const webHreflangPagesSnippet = `Page https://example.com/en/ contains: + + + + +Page https://example.com/es/ contains: + + + + +Page https://example.com/fr/ contains: + +` + +// webHreflangReciprocalTest: find the one non-reciprocal hreflang link +// among three inline pages. +// +// ground truth: /en/ declares an hreflang link to /fr/, but /fr/'s own +// link set has no entry pointing back to https://example.com/en/, so the +// pair (/en/, /fr/) is not reciprocal; /fr/ is the page missing the link, +// and https://example.com/en/ is the target it is missing. Every other +// pair in the three pages is fully reciprocal. +func webHreflangReciprocalTest() testkit.Test { + prompt := `hreflang annotations must be reciprocal: if page A links to page B +via hreflang, page B must also link back to page A. Here are the hreflang +link tags for three pages: + +` + webHreflangPagesSnippet + ` + +Exactly one pair among these three pages is broken (not reciprocal). +Respond with only a JSON object: +{"page_missing_link":"","missing_target":""}` + + evaluator := eval.Mean( + eval.JSONField("page_missing_link", "https://example.com/fr/"), + eval.JSONField("missing_target", "https://example.com/en/"), + ) + + return testkit.Test{ + ID: "web-hreflang-reciprocal", + Category: "research", + Subcategory: "web", + Description: "Find the one non-reciprocal hreflang link among three inline pages' link tag sets.", + Prompt: prompt, + Eval: evaluator, + } +} + +// webParseURL is the inline URL for webURLParseComponentsTest. +const webParseURL = "https://api.example.com:8443/v2/search?q=hello+world&sort=desc#results" + +// webURLParseComponentsTest: parse an inline URL into its component parts +// as JSON. +// +// ground truth: computed directly from net/url.Parse in +// web_extract_test.go, not hand-derived - scheme "https", host (hostname, +// no port) "api.example.com", port "8443", path "/v2/search", raw query +// "q=hello+world&sort=desc", fragment "results". +func webURLParseComponentsTest() testkit.Test { + prompt := `Parse this URL into its components: + +` + webParseURL + ` + +Respond with only a JSON object: +{"scheme":"...","host":"...","port":,"path":"...","query":"...","fragment":"..."} + +"host" must be the hostname only (no port). "port" must be a number. +"query" must be the raw query string exactly as it appears after the "?", +not split into individual parameters.` + + evaluator := eval.Mean( + eval.JSONField("scheme", "https"), + eval.JSONField("host", "api.example.com"), + eval.JSONField("port", 8443), + eval.JSONField("path", "/v2/search"), + eval.JSONField("query", "q=hello+world&sort=desc"), + eval.JSONField("fragment", "results"), + ) + + return testkit.Test{ + ID: "web-url-parse-components", + Category: "research", + Subcategory: "web", + Description: "Parse an inline URL into its scheme, host, port, path, query, and fragment components as JSON.", + Prompt: prompt, + Eval: evaluator, + } +} diff --git a/internal/tests/web_extract_test.go b/internal/tests/web_extract_test.go new file mode 100644 index 0000000..72da322 --- /dev/null +++ b/internal/tests/web_extract_test.go @@ -0,0 +1,113 @@ +package tests + +import ( + "context" + "net/url" + "testing" +) + +func TestWebHTMLProductExtractTest_Eval(t *testing.T) { + tc := webHTMLProductExtractTest() + + tests := []struct { + name string + response string + want float64 + }{ + {"all correct", `{"name":"Wireless Mechanical Keyboard","price":129.99,"sku":"WBH-2201"}`, 1}, + {"one field wrong", `{"name":"Wireless Mechanical Keyboard","price":99.99,"sku":"WBH-2201"}`, 2.0 / 3.0}, + {"all wrong", `{"name":"Wrong Product","price":1,"sku":"XXX"}`, 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Value != tt.want { + t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail) + } + }) + } +} + +func TestWebHreflangReciprocalTest_Eval(t *testing.T) { + tc := webHreflangReciprocalTest() + + tests := []struct { + name string + response string + want float64 + }{ + {"correct", `{"page_missing_link":"https://example.com/fr/","missing_target":"https://example.com/en/"}`, 1}, + {"correct page, wrong target", `{"page_missing_link":"https://example.com/fr/","missing_target":"https://example.com/es/"}`, 0.5}, + {"wrong page entirely", `{"page_missing_link":"https://example.com/es/","missing_target":"https://example.com/en/"}`, 0.5}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Value != tt.want { + t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail) + } + }) + } +} + +func TestWebParseURL_GroundTruth(t *testing.T) { + // Independent re-derivation via net/url.Parse, not hand-derived, per + // the ground-truth discipline. + u, err := url.Parse(webParseURL) + if err != nil { + t.Fatalf("url.Parse(%q): %v", webParseURL, err) + } + + if got, want := u.Scheme, "https"; got != want { + t.Errorf("Scheme = %q, want %q", got, want) + } + if got, want := u.Hostname(), "api.example.com"; got != want { + t.Errorf("Hostname() = %q, want %q", got, want) + } + if got, want := u.Port(), "8443"; got != want { + t.Errorf("Port() = %q, want %q", got, want) + } + if got, want := u.Path, "/v2/search"; got != want { + t.Errorf("Path = %q, want %q", got, want) + } + if got, want := u.RawQuery, "q=hello+world&sort=desc"; got != want { + t.Errorf("RawQuery = %q, want %q", got, want) + } + if got, want := u.Fragment, "results"; got != want { + t.Errorf("Fragment = %q, want %q", got, want) + } +} + +func TestWebURLParseComponentsTest_Eval(t *testing.T) { + tc := webURLParseComponentsTest() + + tests := []struct { + name string + response string + want float64 + }{ + { + "all correct", + `{"scheme":"https","host":"api.example.com","port":8443,"path":"/v2/search","query":"q=hello+world&sort=desc","fragment":"results"}`, + 1, + }, + { + "port included in host field is still wrong for host", + `{"scheme":"https","host":"api.example.com:8443","port":8443,"path":"/v2/search","query":"q=hello+world&sort=desc","fragment":"results"}`, + 5.0 / 6.0, + }, + { + "all wrong", + `{"scheme":"http","host":"wrong.example.com","port":80,"path":"/","query":"","fragment":""}`, + 0, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Value != tt.want { + t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail) + } + }) + } +} diff --git a/internal/tests/web_protocol.go b/internal/tests/web_protocol.go new file mode 100644 index 0000000..b00f02a --- /dev/null +++ b/internal/tests/web_protocol.go @@ -0,0 +1,134 @@ +package tests + +import ( + "github.com/lukaszraczylo/llm-testbench/internal/eval" + "github.com/lukaszraczylo/llm-testbench/internal/testkit" +) + +// webSitemapMaxURLsTest: state the sitemaps.org protocol's maximum URL +// count per sitemap file. +// +// ground truth: the sitemaps.org protocol caps a single sitemap file at +// 50,000 URLs (and 50MB uncompressed); a file exceeding that must be +// split across multiple sitemap files listed in a sitemap index. +func webSitemapMaxURLsTest() testkit.Test { + prompt := `A site's sitemap.xml file lists 52,000 entries inside a single +. Per the sitemaps.org protocol, what is the maximum number of +URLs permitted in a single sitemap file (the limit this file violates)? +Respond with only the number.` + + return testkit.Test{ + ID: "web-sitemap-max-urls", + Category: "research", + Subcategory: "web", + Description: "State the sitemaps.org protocol's maximum URL-per-file limit, applied to a scenario that exceeds it.", + Prompt: prompt, + Eval: eval.Numeric(eval.ExtractLastNumber[int], 50000, 0), + } +} + +// webHTTPStatusScenarios lists four inline request-handling scenarios for +// webHTTPStatusScenariosTest. +const webHTTPStatusScenarios = `scenario1: "The client sent a request body that is not valid JSON." +scenario2: "The client did not send any authentication credentials, and +the requested resource requires a logged-in user." +scenario3: "The client is authenticated, but their account does not have +permission to access the requested resource." +scenario4: "The resource has permanently moved to a new URL; the server +wants clients (and search engines) to use the new URL from now on."` + +// webHTTPStatusScenariosTest: map 4 inline HTTP scenarios to their correct +// status codes. +// +// ground truth: malformed request body -> 400 Bad Request; missing +// credentials on a resource that requires them -> 401 Unauthorized; +// authenticated but lacking permission -> 403 Forbidden; permanent +// relocation -> 301 Moved Permanently. +func webHTTPStatusScenariosTest() testkit.Test { + prompt := `For each of these 4 request-handling scenarios, give the single most +correct HTTP status code: + +` + webHTTPStatusScenarios + ` + +Respond with only a JSON object mapping each scenario id to its status +code as a number: {"scenario1":,"scenario2":,"scenario3":,"scenario4":}` + + evaluator := eval.Mean( + eval.JSONField("scenario1", 400), + eval.JSONField("scenario2", 401), + eval.JSONField("scenario3", 403), + eval.JSONField("scenario4", 301), + ) + + return testkit.Test{ + ID: "web-http-status-scenarios", + Category: "research", + Subcategory: "web", + Description: "Map 4 inline request-handling scenarios to their correct HTTP status codes.", + Prompt: prompt, + Eval: evaluator, + } +} + +// webDNSMXRecordPattern requires the literal token "MX" (word-bounded, +// case-insensitive) anywhere in the response, materially correct whether +// or not the response also says "record" or names the other record types +// it is not. +const webDNSMXRecordPattern = `(?i)\bMX\b` + +// webDNSMXRecordTest: name the DNS record type used to publish a domain's +// prioritized mail servers. +// +// ground truth: mail-routing priority is published exclusively via MX +// (Mail Exchange) records; no other standard record type carries a +// priority value for mail delivery. +func webDNSMXRecordTest() testkit.Test { + prompt := `example.com wants mail delivered to mailA.example.com as its primary +mail server (priority 10), falling back to mailB.example.com (priority +20) only if the primary is unreachable. Which single DNS record type +publishes this prioritized mail-routing configuration? Respond with only +the record type (e.g. A, AAAA, CNAME, MX, TXT, NS).` + + return testkit.Test{ + ID: "web-dns-mx-record", + Category: "research", + Subcategory: "web", + Description: "Name the DNS record type used to publish a domain's prioritized mail servers.", + Prompt: prompt, + Eval: eval.Regex(webDNSMXRecordPattern), + } +} + +// webCanonicalVsRedirectTest: choose between a canonical tag and a 301 +// redirect for a duplicate-content scenario where the duplicate URLs must +// stay independently reachable. +// +// ground truth: a 301 redirect would break the tracking links (they would +// stop resolving to their own URL), so the correct fix is a canonical +// tag: it consolidates ranking signals onto the preferred URL while +// leaving every tracked variant independently reachable. +func webCanonicalVsRedirectTest() testkit.Test { + prompt := `An e-commerce site serves identical product content at these three +URLs, which must all keep working since they are used as tracking links +in email and social campaigns: + +https://example.com/product?id=123 +https://example.com/product?id=123&ref=email +https://example.com/product?id=123&ref=social + +Search engines are treating these as duplicate content and splitting +ranking signal between them. You need to consolidate that ranking signal +onto https://example.com/product?id=123, while keeping all three URLs +independently reachable for the tracking links to keep working. Should +you use a canonical tag or a 301 redirect? Respond with only one word: +"canonical" or "redirect".` + + return testkit.Test{ + ID: "web-canonical-vs-redirect", + Category: "research", + Subcategory: "web", + Description: "Choose canonical tag over 301 redirect for a duplicate-content scenario requiring all URL variants to stay reachable.", + Prompt: prompt, + Eval: eval.Regex(`(?i)^\s*canonical\.?\s*$`), + } +} diff --git a/internal/tests/web_protocol_test.go b/internal/tests/web_protocol_test.go new file mode 100644 index 0000000..960d354 --- /dev/null +++ b/internal/tests/web_protocol_test.go @@ -0,0 +1,98 @@ +package tests + +import ( + "context" + "testing" +) + +func TestWebSitemapMaxURLsTest_Eval(t *testing.T) { + tc := webSitemapMaxURLsTest() + + tests := []struct { + name string + response string + want float64 + }{ + {"correct", "50000", 1}, + {"prose wrapped", "The limit is 50000 URLs per sitemap file.", 1}, + {"wrong: confused with the uncompressed size limit in MB", "50", 0}, + {"wrong entirely", "10000", 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Value != tt.want { + t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail) + } + }) + } +} + +func TestWebHTTPStatusScenariosTest_Eval(t *testing.T) { + tc := webHTTPStatusScenariosTest() + + tests := []struct { + name string + response string + want float64 + }{ + {"all correct", `{"scenario1":400,"scenario2":401,"scenario3":403,"scenario4":301}`, 1}, + {"swapped 401 and 403", `{"scenario1":400,"scenario2":403,"scenario3":401,"scenario4":301}`, 0.5}, + {"used 500 for a client error", `{"scenario1":500,"scenario2":401,"scenario3":403,"scenario4":301}`, 0.75}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Value != tt.want { + t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail) + } + }) + } +} + +func TestWebDNSMXRecordTest_Eval(t *testing.T) { + tc := webDNSMXRecordTest() + + tests := []struct { + name string + response string + want float64 + }{ + {"bare", "MX", 1}, + {"lowercase", "mx", 1}, + {"with the word record", "MX record", 1}, + {"wrong: CNAME", "CNAME", 0}, + {"wrong: A record", "A", 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Value != tt.want { + t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail) + } + }) + } +} + +func TestWebCanonicalVsRedirectTest_Eval(t *testing.T) { + tc := webCanonicalVsRedirectTest() + + tests := []struct { + name string + response string + want float64 + }{ + {"correct lowercase", "canonical", 1}, + {"correct capitalized with period", "Canonical.", 1}, + {"wrong: redirect", "redirect", 0}, + {"wrong: extra words break the forced one-word format", "Use a canonical tag", 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Value != tt.want { + t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail) + } + }) + } +} diff --git a/internal/tests/web_security.go b/internal/tests/web_security.go new file mode 100644 index 0000000..f6ab6ba --- /dev/null +++ b/internal/tests/web_security.go @@ -0,0 +1,97 @@ +package tests + +import ( + "github.com/lukaszraczylo/llm-testbench/internal/eval" + "github.com/lukaszraczylo/llm-testbench/internal/testkit" +) + +// webCORSPreflightExchange is the inline preflight request/response +// exchange for webCORSMissingHeaderTest. +const webCORSPreflightExchange = `Browser preflight request (from https://app.example.com): +OPTIONS /data HTTP/1.1 +Host: api.example.com +Origin: https://app.example.com +Access-Control-Request-Method: PUT +Access-Control-Request-Headers: X-Custom-Auth + +Server's preflight response: +HTTP/1.1 204 No Content +Access-Control-Allow-Origin: https://app.example.com +Access-Control-Allow-Methods: GET, POST, PUT` + +// webCORSMissingHeaderPattern requires the literal token +// "Access-Control-Allow-Headers" (case-insensitive) anywhere in the +// response, accepting it with or without a trailing colon or surrounding +// prose. +const webCORSMissingHeaderPattern = `(?i)access-control-allow-headers` + +// webCORSMissingHeaderTest: identify the single missing CORS response +// header that will cause the browser to block the request. +// +// ground truth: the browser sent Access-Control-Request-Headers: +// X-Custom-Auth in the preflight, but the server's response never echoes +// that header name back in an Access-Control-Allow-Headers response +// header, so the browser blocks the actual PUT request even though the +// origin and method are both explicitly allowed. +func webCORSMissingHeaderTest() testkit.Test { + prompt := `Here is a CORS preflight exchange: + +` + webCORSPreflightExchange + ` + +The browser will block the actual PUT request that follows this +preflight. Which single response header is missing from the server's +preflight response, causing the block? Respond with only the header name, +nothing else.` + + return testkit.Test{ + ID: "web-cors-missing-header", + Category: "research", + Subcategory: "web", + Description: "Identify the single missing CORS response header that causes a browser to block a preflighted request.", + Prompt: prompt, + Eval: eval.Regex(webCORSMissingHeaderPattern), + } +} + +// webSecurityHeadersResponse is the inline HTTP response header block for +// webSecurityHeadersAuditTest. +const webSecurityHeadersResponse = `HTTP/1.1 200 OK +Content-Type: text/html; charset=utf-8 +Cache-Control: no-cache +Set-Cookie: session=abc123; HttpOnly +Server: nginx +X-Content-Type-Options: nosniff +Referrer-Policy: strict-origin-when-cross-origin` + +// webSecurityHeadersBaseline is the recommended baseline of security +// headers checked against webSecurityHeadersResponse. +const webSecurityHeadersBaseline = `Content-Security-Policy, Strict-Transport-Security, +X-Content-Type-Options, X-Frame-Options, Referrer-Policy` + +// webSecurityHeadersAuditTest: identify which baseline security headers +// are missing from an inline HTTP response. +// +// ground truth: of the 5 baseline headers, X-Content-Type-Options and +// Referrer-Policy are present in the response; Content-Security-Policy, +// Strict-Transport-Security, and X-Frame-Options are absent. +func webSecurityHeadersAuditTest() testkit.Test { + prompt := `Here is an HTTP response's headers: + +` + webSecurityHeadersResponse + ` + +Here is the recommended baseline of security headers: + +` + webSecurityHeadersBaseline + ` + +Which of the baseline headers are missing from the response above? +Respond with only a JSON array of the missing header names.` + + return testkit.Test{ + ID: "web-security-headers-audit", + Category: "research", + Subcategory: "web", + Description: "Identify which baseline security headers are missing from an inline HTTP response, given a partial-compliance fixture.", + Prompt: prompt, + Eval: eval.JSONStringSet([]string{"Content-Security-Policy", "Strict-Transport-Security", "X-Frame-Options"}), + } +} diff --git a/internal/tests/web_security_test.go b/internal/tests/web_security_test.go new file mode 100644 index 0000000..9c65485 --- /dev/null +++ b/internal/tests/web_security_test.go @@ -0,0 +1,54 @@ +package tests + +import ( + "context" + "testing" +) + +func TestWebCORSMissingHeaderTest_Eval(t *testing.T) { + tc := webCORSMissingHeaderTest() + + tests := []struct { + name string + response string + want float64 + }{ + {"bare header name", "Access-Control-Allow-Headers", 1}, + {"with a colon", "Access-Control-Allow-Headers:", 1}, + {"in a sentence", "The missing header is Access-Control-Allow-Headers.", 1}, + {"wrong: names an already-present header", "Access-Control-Allow-Origin", 0}, + {"wrong: names an already-present header", "Access-Control-Allow-Methods", 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Value != tt.want { + t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail) + } + }) + } +} + +func TestWebSecurityHeadersAuditTest_Eval(t *testing.T) { + tc := webSecurityHeadersAuditTest() + + tests := []struct { + name string + response string + want float64 + }{ + {"correct set", `["Content-Security-Policy","Strict-Transport-Security","X-Frame-Options"]`, 1}, + {"correct set, different order", `["X-Frame-Options","Content-Security-Policy","Strict-Transport-Security"]`, 1}, + {"missing one", `["Content-Security-Policy","Strict-Transport-Security"]`, 2.0 / 3.0}, + {"wrongly includes a present header", `["Content-Security-Policy","Strict-Transport-Security","X-Frame-Options","Referrer-Policy"]`, 0.75}, + {"wrong: says nothing is missing", `[]`, 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Value != tt.want { + t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail) + } + }) + } +} diff --git a/internal/tests/whitepapers.go b/internal/tests/whitepapers.go index 060408e..d679328 100644 --- a/internal/tests/whitepapers.go +++ b/internal/tests/whitepapers.go @@ -7,6 +7,15 @@ import ( func registerWhitepapersTests(r *testkit.Registry) { r.Register(paperHnswParamsTest()) + r.Register(paperPQCompressionRatioTest()) + r.Register(paperBloomFPRateTest()) + r.Register(paperTFIDFTermScoreTest()) + r.Register(paperRecallAtKTest()) + r.Register(paperLSMWriteAmplificationTest()) + r.Register(paperRaftQuorumFailuresTest()) + r.Register(paperBTreeHeightLevelsTest()) + r.Register(paperCAPAvailabilityChoiceTest()) + r.Register(paperAttentionScaleFactorTest()) } // paperHnswParamsExcerpt is a ~150-word, technically accurate original diff --git a/internal/tests/whitepapers_compute.go b/internal/tests/whitepapers_compute.go new file mode 100644 index 0000000..bcda996 --- /dev/null +++ b/internal/tests/whitepapers_compute.go @@ -0,0 +1,105 @@ +package tests + +import ( + "math" + "strings" +) + +// wpQuantizedCodeBytes returns the number of bytes needed to store one +// product-quantization code: m subvectors, each an nbits-bit centroid +// index, packed with no wasted bits. It assumes m*nbits is a multiple of +// 8, true for every configuration used in this catalog. +func wpQuantizedCodeBytes(m, nbits int) int { + return m * nbits / 8 +} + +// wpBloomFalsePositiveRate computes the standard Bloom filter +// false-positive probability p = (1 - e^(-k*n/m))^k for an m-bit filter +// holding n inserted elements and k hash functions. +func wpBloomFalsePositiveRate(m, n, k int) float64 { + exponent := -float64(k) * float64(n) / float64(m) + return math.Pow(1-math.Exp(exponent), float64(k)) +} + +// wpTermFrequency counts how many times term appears in doc, splitting on +// whitespace (doc is pre-tokenized) and comparing case-insensitively. +func wpTermFrequency(term string, doc []string) int { + count := 0 + for _, w := range doc { + if strings.EqualFold(w, term) { + count++ + } + } + return count +} + +// wpDocumentFrequency counts how many of docs contain term at least once. +func wpDocumentFrequency(term string, docs [][]string) int { + count := 0 + for _, doc := range docs { + if wpTermFrequency(term, doc) > 0 { + count++ + } + } + return count +} + +// wpTFIDF computes the plain tf * ln(N/df) weight: tf is the raw term +// count in one document, df is the number of documents containing the +// term, and n is the total document count. No smoothing is applied, +// matching the formula stated in the wp-tfidf-term-score excerpt. +func wpTFIDF(tf, df, n int) float64 { + return float64(tf) * math.Log(float64(n)/float64(df)) +} + +// wpRecallAtK computes the standard recall@k retrieval metric: the +// fraction of relevant that appears anywhere in retrievedTopK, regardless +// of rank. +func wpRecallAtK(relevant, retrievedTopK []string) float64 { + retrievedSet := make(map[string]struct{}, len(retrievedTopK)) + for _, id := range retrievedTopK { + retrievedSet[id] = struct{}{} + } + hit := 0 + for _, id := range relevant { + if _, ok := retrievedSet[id]; ok { + hit++ + } + } + return float64(hit) / float64(len(relevant)) +} + +// wpLSMWriteAmplification returns the write amplification factor for a +// byte that is rewritten rewriteCount times by compaction after its +// original flush write: the original write plus every rewrite, divided by +// the single original write. +func wpLSMWriteAmplification(rewriteCount int) int { + return 1 + rewriteCount +} + +// wpRaftQuorumSize returns the minimum number of votes/acknowledgments +// needed for a strict majority of an n-node Raft cluster: floor(n/2)+1. +func wpRaftQuorumSize(n int) int { + return n/2 + 1 +} + +// wpRaftMaxTolerableFailures returns how many simultaneous node failures +// an n-node Raft cluster can sustain while a majority quorum, sized +// relative to the full n-node membership, remains reachable. +func wpRaftMaxTolerableFailures(n int) int { + return n - wpRaftQuorumSize(n) +} + +// wpBTreeMinBranchingLevels returns the smallest h (root counted as h=0) +// such that a tree with minimum fanout f can index at least n leaf key +// slots, per the f^(h+1) >= n bound stated in the wp-btree-height-levels +// excerpt. +func wpBTreeMinBranchingLevels(f, n int) int { + h := 0 + capacity := f + for capacity < n { + capacity *= f + h++ + } + return h +} diff --git a/internal/tests/whitepapers_systems.go b/internal/tests/whitepapers_systems.go new file mode 100644 index 0000000..87fff98 --- /dev/null +++ b/internal/tests/whitepapers_systems.go @@ -0,0 +1,267 @@ +package tests + +import ( + "github.com/lukaszraczylo/llm-testbench/internal/eval" + "github.com/lukaszraczylo/llm-testbench/internal/testkit" +) + +// paperLSMExcerpt is a ~152-word, technically accurate original +// description of LSM-tree leveled compaction and write amplification +// written for this test, pinning a concrete 4-level rewrite path. +const paperLSMExcerpt = `An LSM-tree buffers writes in memory and periodically flushes them as +immutable sorted files, then merges those files together in the background +through a process called compaction, so reads do not have to check an +unbounded number of files. In leveled compaction, each level on disk is +merged into the next level down, and every byte that a compaction step +touches gets rewritten to a new file even though its value did not change. +A storage engine described in this report uses four levels, L0 through L3. +A byte first flushed into L0 is compacted into L1, then later compacted +from L1 into L2, and finally from L2 into L3 before it reaches its +long-term resting level, meaning it is rewritten three separate times after +its original flush write. Write amplification here is defined as total +bytes written to disk, including every rewrite, divided by the bytes of +user data originally written.` + +// paperLSMWriteAmpWant is derived by calling wpLSMWriteAmplification, not +// hardcoded. +// +// ground truth: the excerpt states a byte is rewritten 3 times (L0->L1, +// L1->L2, L2->L3) after its original flush write, so total disk writes per +// user byte = 1 (original) + 3 (rewrites) = 4. +// whitepapers_systems_test.go independently recomputes this by counting +// the rewrite path described in the excerpt text. +var paperLSMWriteAmpWant = wpLSMWriteAmplification(3) + +// paperLSMWriteAmplificationTest: derive an LSM-tree's write amplification +// factor from an inline excerpt describing a byte's full compaction path. +func paperLSMWriteAmplificationTest() testkit.Test { + prompt := `Read this excerpt about LSM-tree compaction: + +` + paperLSMExcerpt + ` + +Using the byte's full path through this system (an L0 flush, then +compaction into L1, then L2, then L3) and the write-amplification +definition given above, what is the write amplification factor for that +byte? Respond with only the number.` + + return testkit.Test{ + ID: "paper-lsm-write-amplification", + Category: "research", + Subcategory: "whitepapers", + Description: "Derive an LSM-tree's write amplification factor from an inline excerpt describing a byte's full leveled-compaction path.", + Prompt: prompt, + Eval: eval.Numeric(eval.ExtractLastNumber[int], paperLSMWriteAmpWant, 0), + } +} + +// paperRaftExcerpt is a ~155-word, technically accurate original +// description of Raft's majority-quorum requirement written for this +// test, pinning a 5-node cluster. +const paperRaftExcerpt = `Raft is a consensus protocol that keeps a replicated log consistent across +a cluster of nodes by electing a single leader for each term and having +that leader replicate every log entry to the other nodes. A node starts an +election after its randomized election timeout, typically drawn from the +150 to 300 millisecond range in the original paper, expires without +hearing from a leader. A candidate becomes leader only once it collects +votes from a strict majority of all nodes in the cluster, not merely a +majority of the nodes currently reachable, and the same majority rule +governs when a log entry counts as committed. This report deployed Raft +across a cluster of N equal to 5 nodes, so both leader election and log +commitment require votes or acknowledgments from a majority quorum sized +relative to that total membership of 5, not to however many nodes happen to +be healthy at a given moment.` + +// paperRaftQuorumWant and paperRaftFailuresWant are derived by calling +// wpRaftQuorumSize and wpRaftMaxTolerableFailures, not hardcoded. +// +// ground truth: quorum = floor(5/2)+1 = 3; a 5-node cluster can lose at +// most 5-3 = 2 nodes and still reach that quorum. +// whitepapers_systems_test.go independently recomputes both for several N. +var ( + paperRaftQuorumWant = wpRaftQuorumSize(5) + paperRaftFailuresWant = wpRaftMaxTolerableFailures(5) +) + +// paperRaftQuorumFailuresTest: derive a Raft cluster's majority quorum +// size and maximum tolerable simultaneous failures from an inline excerpt +// and a stated cluster size. +func paperRaftQuorumFailuresTest() testkit.Test { + prompt := `Read this excerpt about the Raft consensus protocol: + +` + paperRaftExcerpt + ` + +For this N=5 node cluster: +1. What is the minimum number of votes/acknowledgments needed for a + majority quorum? +2. How many simultaneous node failures can the cluster sustain and still + reach that quorum? + +Respond with only a JSON object: {"quorum_size":,"max_tolerable_failures":}` + + evaluator := eval.Mean( + eval.JSONField("quorum_size", paperRaftQuorumWant), + eval.JSONField("max_tolerable_failures", paperRaftFailuresWant), + ) + + return testkit.Test{ + ID: "paper-raft-quorum-failures", + Category: "research", + Subcategory: "whitepapers", + Description: "Derive a 5-node Raft cluster's majority quorum size and maximum tolerable simultaneous failures from an inline excerpt.", + Prompt: prompt, + Eval: evaluator, + } +} + +// paperBTreeExcerpt is a ~146-word, technically accurate original +// description of a worst-case (minimum-fanout) B-tree capacity model +// written for this test, pinning order m and a target key count. +const paperBTreeExcerpt = `A B-tree of order m allows every internal node between ceil(m/2) and m +children, keeping the tree shallow and well balanced even as it grows, +which bounds the number of disk reads a lookup needs to at most one per +level. For capacity planning this report models the worst case: a tree +where every internal node has the minimum allowed fanout, f equals +ceil(m/2), rather than the maximum. With that minimum fanout, and counting +the root itself as branching level h equal to zero, a tree with h +branching levels can index at most f to the power of (h+1) leaf key slots, +since each additional level below the root multiplies the reachable leaf +count by f. This benchmark used order m equal to 200, giving a minimum +fanout f equal to 100, and needed to index at least 5,000,000 keys without +exceeding that capacity bound.` + +// paperBTreeHeightWant is derived by calling wpBTreeMinBranchingLevels, +// not hardcoded. +// +// ground truth: smallest h with 100^(h+1) >= 5,000,000: h=0 -> 100; +// h=1 -> 10,000; h=2 -> 1,000,000 (still short); h=3 -> 100,000,000 +// (first to clear 5,000,000). whitepapers_systems_test.go independently +// recomputes this with a plain loop over math.Pow. +var paperBTreeHeightWant = wpBTreeMinBranchingLevels(100, 5000000) + +// paperBTreeHeightLevelsTest: derive the minimum number of branching +// levels a worst-case B-tree needs to index a target key count, from an +// inline excerpt's stated capacity formula. +func paperBTreeHeightLevelsTest() testkit.Test { + prompt := `Read this excerpt about B-tree capacity planning: + +` + paperBTreeExcerpt + ` + +Using the formula given above (f^(h+1) >= n keys, root counted as h=0), +what is the minimum number of branching levels h needed so this tree can +index at least 5,000,000 keys with f=100? Respond with only the number.` + + return testkit.Test{ + ID: "paper-btree-height-levels", + Category: "research", + Subcategory: "whitepapers", + Description: "Derive the minimum branching-level count a worst-case-fanout B-tree needs to index a target key count.", + Prompt: prompt, + Eval: eval.Numeric(eval.ExtractLastNumber[int], paperBTreeHeightWant, 0), + } +} + +// paperCAPExcerpt is a ~135-word, technically accurate original +// description of the CAP theorem written for this test, describing a +// scenario that unambiguously chooses availability over consistency. +const paperCAPExcerpt = `The CAP theorem states that a distributed data store can provide at most +two of three guarantees during a network partition: consistency, meaning +every read sees the most recent write or an error; availability, meaning +every request receives a non-error response; and partition tolerance, +meaning the system keeps operating despite dropped or delayed messages +between nodes. Because network partitions are unavoidable in any real +distributed deployment, partition tolerance is treated as a fixed +requirement rather than a design choice, leaving consistency and +availability as the actual trade-off a system architect makes. A key-value +store described in this report was explicitly designed so that, during a +partition between two data centers, both sides keep accepting writes +independently rather than refusing requests, accepting that the two sides +may briefly disagree and need reconciliation once the partition heals.` + +// paperCAPAvailabilityPattern anchors on the trimmed response being +// exactly the single word "availability" (case-insensitive, with an +// optional trailing period), accepting every materially-correct form of +// the forced one-word answer without also matching a response that names +// the wrong guarantee. +const paperCAPAvailabilityPattern = `(?i)^\s*availability\.?\s*$` + +// paperCAPAvailabilityChoiceTest: identify, from an inline CAP-theorem +// excerpt and a described design decision, which guarantee (consistency or +// availability) the system prioritizes during a partition. +func paperCAPAvailabilityChoiceTest() testkit.Test { + prompt := `Read this excerpt about the CAP theorem: + +` + paperCAPExcerpt + ` + +Given this design decision - both data centers keep accepting writes +during the partition rather than refusing requests - which of the two +trade-off guarantees does this system prioritize during a partition, per +the CAP theorem? Respond with only one word: "Consistency" or +"Availability".` + + return testkit.Test{ + ID: "paper-cap-availability-choice", + Category: "research", + Subcategory: "whitepapers", + Description: "Identify which CAP-theorem guarantee a described partition-handling design decision prioritizes.", + Prompt: prompt, + Eval: eval.Regex(paperCAPAvailabilityPattern), + } +} + +// paperAttentionExcerpt is a ~158-word, technically accurate original +// description of scaled dot-product attention's sqrt(d_k) scaling written +// for this test, pinning a concrete per-head dimension. +const paperAttentionExcerpt = `Scaled dot-product attention computes a weighted average of value vectors, +where the weight given to each value comes from a softmax over the dot +products between one query vector and every key vector. When query and key +entries are drawn independently with roughly unit variance, the variance of +their dot product grows in proportion to the dimensionality d_k of those +vectors, so without correction, larger d_k pushes the raw dot products to +extreme magnitudes. Feeding those extreme values into softmax drives most +of the output probability mass onto one entry and squashes the gradient +almost to zero everywhere else, stalling training. To counteract this, the +dot products are divided by the square root of d_k before the softmax, +which rescales their variance back down to roughly one regardless of +dimensionality. A benchmark model in this report used d_k equal to 64 per +attention head, so this scaling divided every raw dot product by the square +root of 64.` + +// paperAttentionDK and paperAttentionScaleFactorWant pin the excerpt's +// stated d_k and its square root. +// +// ground truth: d_k = 64 (stated verbatim in the excerpt); scale factor = +// sqrt(64) = 8. whitepapers_systems_test.go independently recomputes the +// scale factor with math.Sqrt. +const ( + paperAttentionDK = 64 + paperAttentionScaleFactorWant = 8 +) + +// paperAttentionScaleFactorTest: extract d_k and derive the sqrt(d_k) +// attention scale factor from an inline excerpt. +func paperAttentionScaleFactorTest() testkit.Test { + prompt := `Read this excerpt about scaled dot-product attention: + +` + paperAttentionExcerpt + ` + +For the benchmark model described: +1. What is d_k? +2. By what number are the raw dot products divided before the softmax + (the scale factor)? + +Respond with only a JSON object: {"d_k":,"scale_factor":}` + + evaluator := eval.Mean( + eval.JSONField("d_k", paperAttentionDK), + eval.JSONField("scale_factor", paperAttentionScaleFactorWant), + ) + + return testkit.Test{ + ID: "paper-attention-scale-factor", + Category: "research", + Subcategory: "whitepapers", + Description: "Extract d_k and derive the sqrt(d_k) attention scale factor from an inline excerpt.", + Prompt: prompt, + Eval: evaluator, + } +} diff --git a/internal/tests/whitepapers_systems_test.go b/internal/tests/whitepapers_systems_test.go new file mode 100644 index 0000000..eac2c40 --- /dev/null +++ b/internal/tests/whitepapers_systems_test.go @@ -0,0 +1,203 @@ +package tests + +import ( + "context" + "math" + "strings" + "testing" +) + +func TestPaperLSMWriteAmpWant_GroundTruth(t *testing.T) { + // Independent re-derivation by counting the excerpt's stated rewrite + // path (L0->L1, L1->L2, L2->L3 = 3 rewrites) plus the original write, + // not via wpLSMWriteAmplification. + // The excerpt names exactly three compaction moves; check each phrase is + // present rather than counting "into L" substrings, since the initial + // "flushed into L0" also contains that substring without being a + // compaction rewrite. + moves := []string{"compacted into L1", "L1 into L2", "L2 into L3"} + rewrites := 0 + for _, phrase := range moves { + if strings.Contains(paperLSMExcerpt, phrase) { + rewrites++ + } + } + if rewrites != 3 { + t.Fatalf("excerpt describes %d of the 3 expected compaction moves", rewrites) + } + want := 1 + rewrites + + if want != 4 { + t.Fatalf("independently recomputed write amplification = %d, want 4", want) + } + if paperLSMWriteAmpWant != want { + t.Errorf("paperLSMWriteAmpWant = %d, independently recomputed = %d", paperLSMWriteAmpWant, want) + } +} + +func TestPaperLSMWriteAmplificationTest_Eval(t *testing.T) { + tc := paperLSMWriteAmplificationTest() + + tests := []struct { + name string + response string + want float64 + }{ + {"correct", "4", 1}, + {"prose wrapped", "The write amplification factor is 4.", 1}, + {"forgot the original write", "3", 0}, + {"off by one", "5", 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Value != tt.want { + t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail) + } + }) + } +} + +func TestPaperRaftQuorumWant_GroundTruth(t *testing.T) { + // Independent re-derivation of floor(n/2)+1 and n-quorum for several N, + // not via wpRaftQuorumSize/wpRaftMaxTolerableFailures. + tests := []struct { + n int + wantQuorum int + wantFailures int + }{ + {3, 2, 1}, + {5, 3, 2}, + {7, 4, 3}, + } + for _, tt := range tests { + gotQuorum := tt.n/2 + 1 + gotFailures := tt.n - gotQuorum + if gotQuorum != tt.wantQuorum { + t.Errorf("quorum(n=%d) = %d, want %d", tt.n, gotQuorum, tt.wantQuorum) + } + if gotFailures != tt.wantFailures { + t.Errorf("maxFailures(n=%d) = %d, want %d", tt.n, gotFailures, tt.wantFailures) + } + } + + if paperRaftQuorumWant != 3 { + t.Errorf("paperRaftQuorumWant = %d, want 3", paperRaftQuorumWant) + } + if paperRaftFailuresWant != 2 { + t.Errorf("paperRaftFailuresWant = %d, want 2", paperRaftFailuresWant) + } +} + +func TestPaperRaftQuorumFailuresTest_Eval(t *testing.T) { + tc := paperRaftQuorumFailuresTest() + + tests := []struct { + name string + response string + want float64 + }{ + {"all correct", `{"quorum_size":3,"max_tolerable_failures":2}`, 1}, + {"one field wrong", `{"quorum_size":3,"max_tolerable_failures":3}`, 0.5}, + {"both wrong: used simple majority of reachable nodes", `{"quorum_size":2,"max_tolerable_failures":1}`, 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Value != tt.want { + t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail) + } + }) + } +} + +func TestPaperBTreeHeightWant_GroundTruth(t *testing.T) { + // Independent re-derivation with math.Pow, not via + // wpBTreeMinBranchingLevels's iterative loop. + const f, n = 100.0, 5000000.0 + h := 0 + for math.Pow(f, float64(h+1)) < n { + h++ + } + + if h != 3 { + t.Fatalf("independently recomputed h = %d, want 3", h) + } + if paperBTreeHeightWant != h { + t.Errorf("paperBTreeHeightWant = %d, independently recomputed = %d", paperBTreeHeightWant, h) + } +} + +func TestPaperBTreeHeightLevelsTest_Eval(t *testing.T) { + tc := paperBTreeHeightLevelsTest() + + tests := []struct { + name string + response string + want float64 + }{ + {"correct", "3", 1}, + {"prose wrapped", "The minimum height is h=3.", 1}, + {"one level short", "2", 0}, + {"one level too many", "4", 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Value != tt.want { + t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail) + } + }) + } +} + +func TestPaperCAPAvailabilityChoiceTest_Eval(t *testing.T) { + tc := paperCAPAvailabilityChoiceTest() + + tests := []struct { + name string + response string + want float64 + }{ + {"correct lowercase", "availability", 1}, + {"correct capitalized", "Availability", 1}, + {"correct with trailing period", "Availability.", 1}, + {"wrong: consistency", "Consistency", 0}, + {"wrong: extra words break the forced one-word format", "The answer is availability", 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Value != tt.want { + t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail) + } + }) + } +} + +func TestPaperAttentionScaleFactorTest_Eval(t *testing.T) { + tc := paperAttentionScaleFactorTest() + + // Ground truth: sqrt(64) = 8, independently recomputed with math.Sqrt. + if got := math.Sqrt(64); got != 8 { + t.Fatalf("math.Sqrt(64) = %v, want 8", got) + } + + tests := []struct { + name string + response string + want float64 + }{ + {"all correct", `{"d_k":64,"scale_factor":8}`, 1}, + {"one field wrong", `{"d_k":64,"scale_factor":64}`, 0.5}, + {"both wrong", `{"d_k":512,"scale_factor":22.6}`, 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Value != tt.want { + t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail) + } + }) + } +} diff --git a/internal/tests/whitepapers_vector.go b/internal/tests/whitepapers_vector.go new file mode 100644 index 0000000..c5e58e3 --- /dev/null +++ b/internal/tests/whitepapers_vector.go @@ -0,0 +1,218 @@ +package tests + +import ( + "github.com/lukaszraczylo/llm-testbench/internal/eval" + "github.com/lukaszraczylo/llm-testbench/internal/testkit" +) + +// paperPQCompressionExcerpt is a ~143-word, technically accurate original +// description of product quantization written for this test, pinning the +// dimension, m, and nbits parameters needed to derive a compression ratio. +const paperPQCompressionExcerpt = `Product quantization compresses high-dimensional vectors for approximate +nearest-neighbor search by splitting each vector into m equal-length +subvectors, then quantizing each subvector independently against its own +codebook. Each codebook is trained offline with k-means and holds 2^nbits +centroids. Rather than storing the original subvector, the index stores only +the index of its nearest centroid: an nbits-bit code. Concatenating the m +per-subvector codes gives the full compressed representation of the vector. +A benchmark index in this report indexed 128-dimensional vectors stored as +32-bit floats, so one uncompressed vector occupies 128 times 4, or 512, +bytes. The index split each vector into m equal to 8 subvectors and used +nbits equal to 8 bits per subvector, giving each subquantizer's codebook 256 +centroids. Search computes approximate distances directly against the +stored codes using precomputed per-centroid distance tables, never +decompressing a vector back to its original 512-byte form.` + +// paperPQCompressionRatioWant is derived by calling wpQuantizedCodeBytes, +// not hardcoded. +// +// ground truth: code bytes = m*nbits/8 = 8*8/8 = 8 bytes per vector; +// compression ratio = raw bytes / code bytes = 512/8 = 64. +// whitepapers_vector_test.go independently recomputes this with plain +// arithmetic. +var paperPQCompressionRatioWant = 512 / wpQuantizedCodeBytes(8, 8) + +// paperPQCompressionRatioTest: derive a product-quantization compression +// ratio from an inline excerpt's m/nbits/dimension parameters. +func paperPQCompressionRatioTest() testkit.Test { + prompt := `Read this excerpt about product quantization (PQ) for vector search: + +` + paperPQCompressionExcerpt + ` + +The benchmark index above stores each compressed vector as its m +per-subvector codes packed with no wasted bits. What is the compression +ratio of the compressed representation versus the original 512-byte +float32 vector (raw bytes divided by compressed bytes)? Respond with only +the number.` + + return testkit.Test{ + ID: "paper-pq-compression-ratio", + Category: "research", + Subcategory: "whitepapers", + Description: "Derive a product-quantization compression ratio from an inline excerpt's m, nbits, and raw-dimension parameters.", + Prompt: prompt, + Eval: eval.Numeric(eval.ExtractLastNumber[int], paperPQCompressionRatioWant, 0), + } +} + +// paperBloomFPRateExcerpt is a ~157-word, technically accurate original +// description of the Bloom filter false-positive formula written for this +// test, pinning m, n, and k. +const paperBloomFPRateExcerpt = `A Bloom filter is a space-efficient structure for testing set membership +that trades a small, tunable false-positive rate for large memory savings, +and never produces a false negative. It holds an m-bit array, initially all +zero, and uses k independent hash functions. Inserting an element sets the +k bit positions its hashes point to; querying an element checks whether all +k of those positions are set, and if so reports "possibly present." The +false-positive probability after inserting n elements is well approximated +by p equals the quantity one minus e to the power of negative k times n +divided by m, that whole quantity raised to the power k. A caching layer +described in this report sized its filter at m equal to 10000 bits for n +equal to 1000 expected keys, and chose k equal to 7 hash functions, +following the standard guidance to pick k near m over n times the natural +log of 2.` + +// paperBloomFPRateWant is derived by calling wpBloomFalsePositiveRate, not +// hardcoded. +// +// ground truth: p = (1 - e^(-k*n/m))^k with m=10000, n=1000, k=7. +// whitepapers_vector_test.go recomputes this independently with math.Pow +// and math.Exp inline. +var paperBloomFPRateWant = round4dp(wpBloomFalsePositiveRate(10000, 1000, 7)) + +// paperBloomFPRateTest: compute a Bloom filter's false-positive rate from +// an inline excerpt's formula and m/n/k parameters. +func paperBloomFPRateTest() testkit.Test { + prompt := `Read this excerpt about Bloom filters: + +` + paperBloomFPRateExcerpt + ` + +Using the exact formula given above and the caching layer's parameters +(m=10000, n=1000, k=7), compute the false-positive probability p. Respond +with only the number, rounded to 4 decimal places.` + + return testkit.Test{ + ID: "paper-bloom-fp-rate", + Category: "research", + Subcategory: "whitepapers", + Description: "Compute a Bloom filter's false-positive rate from an inline excerpt's formula and m/n/k parameters.", + Prompt: prompt, + Eval: eval.Numeric(eval.ExtractLastNumber[float64], paperBloomFPRateWant, 0.0005), + } +} + +// paperTFIDFExcerpt is a ~148-word, technically accurate original +// description of the plain (unsmoothed) TF-IDF formula written for this +// test. +const paperTFIDFExcerpt = `TF-IDF scores how important a term is to one document within a corpus by +combining two signals: how often the term appears in that document, and how +rare the term is across the whole corpus. This report uses the plainest +textbook variant. Term frequency, tf(t,d), is simply the raw count of term +t in document d, with no normalization. Inverse document frequency, +idf(t), is the natural logarithm of N divided by df(t), where N is the +total number of documents in the corpus and df(t) is the number of +documents that contain t at least once. The TF-IDF weight of a term in a +document is the product tf(t,d) times idf(t): a term that appears often in +one document but rarely elsewhere scores highest, while a term appearing in +every document scores near zero regardless of how often it repeats, since +its idf approaches the logarithm of one.` + +// paperTFIDFDoc1, paperTFIDFDoc2, and paperTFIDFDoc3 are the pre-tokenized +// (whitespace-split, lowercase) mini-corpus for paperTFIDFTermScoreTest. +var ( + paperTFIDFDoc1 = []string{"the", "cat", "sat", "on", "the", "mat"} + paperTFIDFDoc2 = []string{"the", "dog", "sat", "on", "the", "log"} + paperTFIDFDoc3 = []string{"cats", "and", "dogs", "are", "great", "pets"} + paperTFIDFCorpus = [][]string{paperTFIDFDoc1, paperTFIDFDoc2, paperTFIDFDoc3} +) + +// paperTFIDFTermScoreWant is derived by calling wpTermFrequency, +// wpDocumentFrequency, and wpTFIDF, not hardcoded. +// +// ground truth: tf("sat", doc1) = 1; df("sat") = 2 (doc1 and doc2 contain +// it, doc3 does not); N = 3; idf = ln(3/2); tfidf = 1 * ln(3/2). +// whitepapers_vector_test.go recomputes this independently with math.Log. +var paperTFIDFTermScoreWant = round4dp(wpTFIDF( + wpTermFrequency("sat", paperTFIDFDoc1), + wpDocumentFrequency("sat", paperTFIDFCorpus), + len(paperTFIDFCorpus), +)) + +// paperTFIDFTermScoreTest: compute the TF-IDF weight of one term in one +// document of an inline 3-document mini-corpus, using the exact formula +// given in the excerpt. +func paperTFIDFTermScoreTest() testkit.Test { + prompt := `Read this excerpt about TF-IDF: + +` + paperTFIDFExcerpt + ` + +Here is a 3-document corpus: + +Document 1: "the cat sat on the mat" +Document 2: "the dog sat on the log" +Document 3: "cats and dogs are great pets" + +Using the exact formulas given above (tf = raw count of the term in the +document, idf = natural log of N divided by df), compute the TF-IDF weight +of the term "sat" in Document 1. Respond with only the number, rounded to +4 decimal places.` + + return testkit.Test{ + ID: "paper-tfidf-term-score", + Category: "research", + Subcategory: "whitepapers", + Description: "Compute a term's TF-IDF weight in one document of an inline 3-document mini-corpus using a formula given in the excerpt.", + Prompt: prompt, + Eval: eval.Numeric(eval.ExtractLastNumber[float64], paperTFIDFTermScoreWant, 0.0005), + } +} + +// paperRecallExcerpt is a ~137-word, technically accurate original +// description of the recall@k retrieval metric written for this test. +const paperRecallExcerpt = `Recall@k is a standard retrieval-quality metric for evaluating a nearest- +neighbor or search system: for one query, it is the fraction of that +query's truly relevant items that appear anywhere within the system's top +k returned results, regardless of their exact rank inside that top-k list. +Formally, recall@k equals the size of the intersection between the +relevant set and the top-k retrieved set, divided by the size of the +relevant set. A perfect recall@k of 1.0 means every relevant item was +retrieved somewhere in the top k, even if none of them landed in rank one; +a lower score means some relevant items were pushed outside the top k +entirely and are effectively lost to any downstream ranking or re-ranking +stage, since a re-ranker can only reorder candidates that retrieval already +surfaced, never recover ones it dropped.` + +// paperRecallRelevant and paperRecallRetrievedTop6 are the inline +// relevant-set and top-6 retrieved-ranking fixtures for +// paperRecallAtKTest. +var ( + paperRecallRelevant = []string{"D2", "D5", "D7", "D9"} + paperRecallRetrievedTop6 = []string{"D1", "D5", "D3", "D7", "D6", "D2"} + paperRecallAtKWant = round4dp(wpRecallAtK(paperRecallRelevant, paperRecallRetrievedTop6)) +) + +// paperRecallAtKTest: compute recall@k from an inline relevant-item set and +// a ranked top-k retrieval list. +func paperRecallAtKTest() testkit.Test { + prompt := `Read this excerpt about the recall@k retrieval metric: + +` + paperRecallExcerpt + ` + +A query's truly relevant documents are: D2, D5, D7, D9. + +A retrieval system's ranked top-6 results for that query are, in rank +order: D1, D5, D3, D7, D6, D2. + +Using the definition given above, compute recall@6 for this query. Respond +with only the number, as a decimal (e.g. 0.5), rounded to 4 decimal +places.` + + return testkit.Test{ + ID: "paper-recall-at-k", + Category: "research", + Subcategory: "whitepapers", + Description: "Compute recall@k from an inline relevant-item set and a ranked top-k retrieval list.", + Prompt: prompt, + Eval: eval.Numeric(eval.ExtractLastNumber[float64], paperRecallAtKWant, 0.0005), + } +} diff --git a/internal/tests/whitepapers_vector_test.go b/internal/tests/whitepapers_vector_test.go new file mode 100644 index 0000000..a31384c --- /dev/null +++ b/internal/tests/whitepapers_vector_test.go @@ -0,0 +1,165 @@ +package tests + +import ( + "context" + "math" + "testing" +) + +func TestPaperPQCompressionRatioWant_GroundTruth(t *testing.T) { + // Independent re-derivation with plain arithmetic, not via + // wpQuantizedCodeBytes. + const dim, m, nbits = 128, 8, 8 + rawBytes := dim * 4 + codeBytes := m * nbits / 8 + want := rawBytes / codeBytes + + if want != 64 { + t.Fatalf("independently recomputed ratio = %d, want 64", want) + } + if paperPQCompressionRatioWant != want { + t.Errorf("paperPQCompressionRatioWant = %d, independently recomputed = %d", paperPQCompressionRatioWant, want) + } +} + +func TestPaperPQCompressionRatioTest_Eval(t *testing.T) { + tc := paperPQCompressionRatioTest() + + tests := []struct { + name string + response string + want float64 + }{ + {"correct", "64", 1}, + {"prose wrapped", "The compression ratio is 64x.", 1}, + {"wrong", "8", 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Value != tt.want { + t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail) + } + }) + } +} + +func TestPaperBloomFPRateWant_GroundTruth(t *testing.T) { + // Independent re-derivation with math.Pow/math.Exp inline, not via + // wpBloomFalsePositiveRate. + const m, n, k = 10000.0, 1000.0, 7.0 + want := math.Round(math.Pow(1-math.Exp(-k*n/m), k)*10000) / 10000 + + if math.Abs(want-0.0082) > 1e-9 { + t.Fatalf("independently recomputed FP rate = %v, want ~0.0082", want) + } + if paperBloomFPRateWant != want { + t.Errorf("paperBloomFPRateWant = %v, independently recomputed = %v", paperBloomFPRateWant, want) + } +} + +func TestPaperBloomFPRateTest_Eval(t *testing.T) { + tc := paperBloomFPRateTest() + + tests := []struct { + name string + response string + want float64 + }{ + {"exact", "0.0082", 1}, + {"within tolerance", "0.0081", 1}, + {"prose wrapped", "p is approximately 0.0082.", 1}, + {"outside tolerance", "0.05", 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Value != tt.want { + t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail) + } + }) + } +} + +func TestPaperTFIDFTermScoreWant_GroundTruth(t *testing.T) { + // Independent re-derivation by counting the raw corpus text directly, + // not via wpTermFrequency/wpDocumentFrequency/wpTFIDF. + tf := 1 // "sat" appears once in doc1 + df := 2 // doc1 and doc2 contain "sat"; doc3 does not + n := 3 // 3 documents total + want := math.Round(float64(tf)*math.Log(float64(n)/float64(df))*10000) / 10000 + + if math.Abs(want-0.4055) > 1e-9 { + t.Fatalf("independently recomputed tfidf = %v, want ~0.4055", want) + } + if paperTFIDFTermScoreWant != want { + t.Errorf("paperTFIDFTermScoreWant = %v, independently recomputed = %v", paperTFIDFTermScoreWant, want) + } +} + +func TestPaperTFIDFTermScoreTest_Eval(t *testing.T) { + tc := paperTFIDFTermScoreTest() + + tests := []struct { + name string + response string + want float64 + }{ + {"exact", "0.4055", 1}, + {"within tolerance", "0.4053", 1}, + {"prose wrapped", "The TF-IDF weight is 0.4055.", 1}, + {"outside tolerance", "1.0986", 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Value != tt.want { + t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail) + } + }) + } +} + +func TestPaperRecallAtKWant_GroundTruth(t *testing.T) { + // Independent re-derivation with a plain set intersection, not via + // wpRecallAtK. + relevant := map[string]bool{"D2": true, "D5": true, "D7": true, "D9": true} + retrieved := []string{"D1", "D5", "D3", "D7", "D6", "D2"} + hit := 0 + for _, id := range retrieved { + if relevant[id] { + hit++ + } + } + want := float64(hit) / float64(len(relevant)) + + if want != 0.75 { + t.Fatalf("independently recomputed recall@6 = %v, want 0.75", want) + } + if paperRecallAtKWant != want { + t.Errorf("paperRecallAtKWant = %v, independently recomputed = %v", paperRecallAtKWant, want) + } +} + +func TestPaperRecallAtKTest_Eval(t *testing.T) { + tc := paperRecallAtKTest() + + tests := []struct { + name string + response string + want float64 + }{ + {"exact", "0.75", 1}, + {"prose wrapped", "recall@6 is 0.75.", 1}, + {"wrong: counted D9 as retrieved", "1.0", 0}, + {"wrong: missed one hit", "0.5", 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Value != tt.want { + t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail) + } + }) + } +}