Merge branch 'worktree-agent-a975228ccb1ce06d4'

internal/tests/codebase.go

@@ -7,6 +7,15 @@ import (
 
 func registerCodebaseTests(r *testkit.Registry) {
 	r.Register(codeTraceGoTest())
+	r.Register(codeTracePythonTest())
+	r.Register(codeTraceTSTest())
+	r.Register(codeBugLineDiffTest())
+	r.Register(codeRaceVariableTest())
+	r.Register(codeDeadFunctionsTest())
+	r.Register(codeBigODedupTest())
+	r.Register(codeImportCycleTest())
+	r.Register(codeGenericReturnTypeTest())
+	r.Register(codeCommitBisectTest())
 }
 
 // mix is the traced function for codeTraceGoTest: deterministic bit

internal/tests/codebase_analysis.go

@@ -0,0 +1,304 @@
+package tests
+
+import (
+	"github.com/lukaszraczylo/llm-testbench/internal/eval"
+	"github.com/lukaszraczylo/llm-testbench/internal/testkit"
+)
+
+// codeBugDiffSnippet is a unified diff for codeBugLineDiffTest, with each
+// displayed line prefixed by its 1-based position in this listing (not the
+// diff's own hunk line numbers), so "which line" has one unambiguous
+// meaning.
+const codeBugDiffSnippet = `1: @@ -1,9 +1,9 @@
+2:  func Average(nums []float64) float64 {
+3:      if len(nums) == 0 {
+4:          return 0
+5:      }
+6:      sum := 0.0
+7: -    for _, n := range nums {
+8: +    for i := 0; i <= len(nums); i++ {
+9: -        sum += n
+10: +        sum += nums[i]
+11:      }
+12:      return sum / float64(len(nums))
+13:  }`
+
+// codeBugLineDiffTest: identify the root-cause line of an off-by-one
+// out-of-bounds bug in an inline, line-numbered diff.
+//
+// ground truth: line 8's new loop condition `i <= len(nums)` allows i to
+// reach len(nums), and line 10 then indexes nums[i] at that out-of-range
+// value, panicking. Line 8 is the root cause (the wrong bound); line 10 is
+// only where the panic actually fires - the correct bound is `i <
+// len(nums)`.
+func codeBugLineDiffTest() testkit.Test {
+	prompt := `Here is a diff modifying a Go function that computes an average. Each
+displayed line below is prefixed with its position in this listing (not
+the diff's own internal hunk numbering):
+
+` + codeBugDiffSnippet + `
+
+This change introduces a bug that causes a runtime panic: index out of
+range. Which line number contains the root cause of the panic - the
+incorrect loop bound - as opposed to the line where the panic actually
+fires? Respond with only a JSON object: {"line":<number>}`
+
+	return testkit.Test{
+		ID:          "code-bug-line-diff",
+		Category:    "research",
+		Subcategory: "codebase",
+		Description: "Identify the root-cause line of an off-by-one out-of-bounds bug in an inline, line-numbered diff.",
+		Prompt:      prompt,
+		Eval:        eval.JSONField("line", 8),
+	}
+}
+
+// codeRaceSnippet is an inline Go function for codeRaceVariableTest with a
+// classic unsynchronized shared-counter data race.
+const codeRaceSnippet = `func Count(n int) int {
+	total := 0
+	var wg sync.WaitGroup
+	for i := 0; i < n; i++ {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			total++
+		}()
+	}
+	wg.Wait()
+	return total
+}`
+
+// codeRaceVariableTest: name the variable with a data race in an inline Go
+// snippet.
+//
+// ground truth: every launched goroutine executes `total++` (a
+// read-modify-write) on the same shared `total` variable with no mutex,
+// atomic operation, or channel serializing access, so concurrent
+// increments race. `n` and `i` are each goroutine-local or loop-local and
+// are not raced on by this snippet.
+func codeRaceVariableTest() testkit.Test {
+	prompt := `Here is a Go function:
+
+` + "```go\n" + codeRaceSnippet + "\n```" + `
+
+Which single variable has a data race in this snippet - concurrent,
+unsynchronized reads and writes from multiple goroutines? Respond with
+only a JSON object: {"variable":"<name>"}`
+
+	return testkit.Test{
+		ID:          "code-race-variable",
+		Category:    "research",
+		Subcategory: "codebase",
+		Description: "Name the racy shared variable in an inline Go snippet with an unsynchronized concurrent counter.",
+		Prompt:      prompt,
+		Eval:        eval.JSONField("variable", "total"),
+	}
+}
+
+// codeDeadFunctionsSnippet is an inline Go file for codeDeadFunctionsTest
+// with two functions reachable from main and two that are never called.
+const codeDeadFunctionsSnippet = `package main
+
+import "fmt"
+
+func helperA() int { return 1 }
+func helperB() int { return helperA() + 1 }
+func helperC() int { return 3 }
+func helperD() int { return helperC() * 2 }
+
+func main() {
+	fmt.Println(helperB())
+}`
+
+// codeDeadFunctionsTest: identify functions unreachable from main in an
+// inline Go file.
+//
+// ground truth: main calls only helperB, which calls helperA - both are
+// reachable. Nothing calls helperD, and the only caller of helperC is
+// helperD itself, so neither helperC nor helperD is reachable from main;
+// both are dead code.
+func codeDeadFunctionsTest() testkit.Test {
+	prompt := `Here is a small Go file:
+
+` + "```go\n" + codeDeadFunctionsSnippet + "\n```" + `
+
+Which functions are dead code - never reachable, directly or transitively,
+from main? Respond with only a JSON array of function names.`
+
+	return testkit.Test{
+		ID:          "code-dead-functions",
+		Category:    "research",
+		Subcategory: "codebase",
+		Description: "Identify functions unreachable from main in an inline Go file with both live and dead call chains.",
+		Prompt:      prompt,
+		Eval:        eval.JSONStringSet([]string{"helperC", "helperD"}),
+	}
+}
+
+// codeDedupSnippet is an inline Go function for codeBigODedupTest with a
+// nested-loop worst-case-quadratic deduplication implementation.
+const codeDedupSnippet = `func Dedup(a []int) []int {
+	var out []int
+	for _, x := range a {
+		found := false
+		for _, y := range out {
+			if x == y {
+				found = true
+				break
+			}
+		}
+		if !found {
+			out = append(out, x)
+		}
+	}
+	return out
+}`
+
+// codeBigODedupPattern accepts every common notation for O(n^2): a caret,
+// a double-star, the unicode superscript two, or n*n, each optionally
+// wrapped as O(...) with flexible internal spacing.
+const codeBigODedupPattern = `(?i)o\(\s*n\s*(\^\s*2|\*\*\s*2|²|\*\s*n)\s*\)`
+
+// codeBigODedupTest: state the worst-case time complexity of an inline
+// nested-loop deduplication function.
+//
+// ground truth: when a has no duplicates, the inner loop scans the whole
+// of out, which grows to length n, for every one of the n outer
+// iterations, giving n + (n-1) + ... = O(n^2) comparisons in the worst
+// case.
+func codeBigODedupTest() testkit.Test {
+	prompt := `Here is a Go function:
+
+` + "```go\n" + codeDedupSnippet + "\n```" + `
+
+What is the worst-case time complexity of Dedup, in Big-O notation, in
+terms of n = len(a)? Respond with only the Big-O expression, e.g. O(n).`
+
+	return testkit.Test{
+		ID:          "code-bigo-dedup",
+		Category:    "research",
+		Subcategory: "codebase",
+		Description: "State the worst-case Big-O time complexity of an inline nested-loop deduplication function.",
+		Prompt:      prompt,
+		Eval:        eval.Regex(codeBigODedupPattern),
+	}
+}
+
+// codeImportListSnippet lists each inline package's direct imports for
+// codeImportCycleTest, with one cycle among four packages.
+const codeImportListSnippet = `- package a imports: [b]
+- package b imports: [c]
+- package c imports: [a, d]
+- package d imports: []`
+
+// codeImportCycleTest: detect which packages form an import cycle from an
+// inline list of per-package direct imports.
+//
+// ground truth: following the edges a->b->c->a closes a cycle among a, b,
+// and c. Package d is imported by c but does not import anything itself,
+// so it is not part of any cycle.
+func codeImportCycleTest() testkit.Test {
+	prompt := `Here are the direct imports of four packages:
+
+` + codeImportListSnippet + `
+
+Which packages form an import cycle? Respond with only a JSON array of
+package names involved in the cycle (any order).`
+
+	return testkit.Test{
+		ID:          "code-import-cycle",
+		Category:    "research",
+		Subcategory: "codebase",
+		Description: "Detect which packages form an import cycle from an inline list of per-package direct imports.",
+		Prompt:      prompt,
+		Eval:        eval.JSONStringSet([]string{"a", "b", "c"}),
+	}
+}
+
+// codeGenericFirstSnippet is an inline Go generic function plus a call
+// site for codeGenericReturnTypeTest.
+const codeGenericFirstSnippet = `func First[T any](s []T) (T, bool) {
+	if len(s) == 0 {
+		var zero T
+		return zero, false
+	}
+	return s[0], true
+}
+
+result, ok := First([]string{"a", "b"})`
+
+// codeGenericReturnTypeTest: infer the concrete instantiated type of a
+// generic function's result at a specific call site.
+//
+// ground truth: First is instantiated with T = string because it is
+// called with []string{"a", "b"}, so result's concrete type is string.
+func codeGenericReturnTypeTest() testkit.Test {
+	prompt := `Here is a generic Go function and a call site:
+
+` + "```go\n" + codeGenericFirstSnippet + "\n```" + `
+
+What is the concrete type of the variable "result" in the code above?
+Respond with only the type name, nothing else.`
+
+	return testkit.Test{
+		ID:          "code-generic-return-type",
+		Category:    "research",
+		Subcategory: "codebase",
+		Description: "Infer the concrete instantiated type of a generic Go function's result at a specific call site.",
+		Prompt:      prompt,
+		Eval:        codeExactAnswer("string"),
+	}
+}
+
+// codeBisectCommitLog is an inline commit history for
+// codeCommitBisectTest, where exactly one commit changes safeDivide's
+// result cache to be keyed only by "a" and checked before validation.
+const codeBisectCommitLog = `c1a1a1a "init: add safeDivide(a, b) that returns an error if b == 0,
+otherwise returns a / b"
+
+c2b2b2b "refactor: rewrite safeDivide's internal branch as a plain
+if/else, no behavior change"
+
+c3c3c3c "perf: add a result cache to safeDivide, keyed only by a (not by
+the pair a and b), checked BEFORE the b == 0 validation runs"
+
+c4d4d4d "chore: add debug logging around cache hits in safeDivide"
+
+c5e5e5e "docs: fix a typo in the cache-hit log message"`
+
+// codeCommitBisectTest: use git-bisect-style reasoning over an inline
+// commit log and a described symptom to name the exact commit that
+// introduced the bug.
+//
+// ground truth: c3c3c3c's cache is keyed only by the first argument a and
+// is checked before the b == 0 validation runs. A prior call
+// safeDivide(10, 2) caches a successful result under key a=10; a later
+// call safeDivide(10, 0) then hits that stale cache entry (same a, cache
+// ignores b) and returns it instead of reaching the validation that would
+// raise the b == 0 error. c1 predates the cache entirely; c2 is a
+// no-behavior-change refactor; c4 and c5 only touch logging text and do
+// not change the cache's key or check order, so the bug already existed
+// before either of them.
+func codeCommitBisectTest() testkit.Test {
+	prompt := `Here is a commit history for a function safeDivide(a, b), oldest first:
+
+` + codeBisectCommitLog + `
+
+Symptom observed in production after deploying through the latest commit:
+calling safeDivide(10, 2) returns 5 (correct). Immediately afterward,
+calling safeDivide(10, 0) returns 5 instead of raising the expected
+"b == 0" error.
+
+Using git-bisect-style reasoning, which commit introduced this bug?
+Respond with only the commit id (the 7-character id shown), nothing else.`
+
+	return testkit.Test{
+		ID:          "code-commit-bisect",
+		Category:    "research",
+		Subcategory: "codebase",
+		Description: "Use git-bisect-style reasoning over an inline commit log and a symptom to name the commit that introduced the bug.",
+		Prompt:      prompt,
+		Eval:        codeExactAnswer("c3c3c3c"),
+	}
+}

internal/tests/codebase_analysis_test.go

@@ -0,0 +1,173 @@
+package tests
+
+import (
+	"context"
+	"testing"
+)
+
+func TestCodeBugLineDiffTest_Eval(t *testing.T) {
+	tc := codeBugLineDiffTest()
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{"correct: root-cause line", `{"line":8}`, 1},
+		{"wrong: points at the panic site instead of the root cause", `{"line":10}`, 0},
+		{"wrong line entirely", `{"line":6}`, 0},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}
+
+func TestCodeRaceVariableTest_Eval(t *testing.T) {
+	tc := codeRaceVariableTest()
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{"correct", `{"variable":"total"}`, 1},
+		{"correct different case", `{"variable":"Total"}`, 1},
+		{"wrong: loop variable, not the shared counter", `{"variable":"i"}`, 0},
+		{"wrong: function parameter, not the shared counter", `{"variable":"n"}`, 0},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}
+
+func TestCodeDeadFunctionsTest_Eval(t *testing.T) {
+	tc := codeDeadFunctionsTest()
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{"correct set", `["helperC","helperD"]`, 1},
+		{"correct set, different order", `["helperD","helperC"]`, 1},
+		{"missing one dead function", `["helperD"]`, 0.5},
+		{"wrongly includes a live function", `["helperC","helperD","helperB"]`, 2.0 / 3.0},
+		{"wrong: says nothing is dead", `[]`, 0},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}
+
+func TestCodeBigODedupTest_Eval(t *testing.T) {
+	tc := codeBigODedupTest()
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{"caret form", "O(n^2)", 1},
+		{"double-star form", "O(n**2)", 1},
+		{"unicode superscript form", "O(n²)", 1},
+		{"n*n form", "O(n*n)", 1},
+		{"lowercase o", "o(n^2)", 1},
+		{"wrong: linear", "O(n)", 0},
+		{"wrong: linearithmic", "O(n log n)", 0},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}
+
+func TestCodeImportCycleTest_Eval(t *testing.T) {
+	tc := codeImportCycleTest()
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{"correct set", `["a","b","c"]`, 1},
+		{"correct set, different order", `["c","a","b"]`, 1},
+		{"wrongly includes non-cyclic d", `["a","b","c","d"]`, 0.75},
+		{"missing one cycle member", `["a","b"]`, 2.0 / 3.0},
+		{"wrong: only d, which has no imports at all", `["d"]`, 0},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}
+
+func TestCodeGenericReturnTypeTest_Eval(t *testing.T) {
+	tc := codeGenericReturnTypeTest()
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{"correct", "string", 1},
+		{"correct capitalized", "String", 1},
+		{"correct quoted", "`string`", 1},
+		{"wrong: the slice element pointer type", "*string", 0},
+		{"wrong: the generic parameter name itself, not its instantiation", "T", 0},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}
+
+func TestCodeCommitBisectTest_Eval(t *testing.T) {
+	tc := codeCommitBisectTest()
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{"correct", "c3c3c3c", 1},
+		{"correct with trailing period", "c3c3c3c.", 1},
+		{"wrong: the initial commit, predates the cache entirely", "c1a1a1a", 0},
+		{"wrong: only a logging change, no behavior change", "c4d4d4d", 0},
+		{"wrong: the docs-only commit", "c5e5e5e", 0},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}

internal/tests/codebase_trace.go

@@ -0,0 +1,127 @@
+package tests
+
+import (
+	"context"
+	"fmt"
+	"strconv"
+	"strings"
+
+	"github.com/lukaszraczylo/llm-testbench/internal/eval"
+	"github.com/lukaszraczylo/llm-testbench/internal/testkit"
+)
+
+// codeExactAnswer returns an Evaluator awarding full credit when the
+// response, trimmed of whitespace, at most one layer of surrounding quote
+// characters (' , ", or `), and a single trailing sentence-ending period,
+// equals want case-insensitively. This accepts every materially-correct
+// form of a forced single-token answer (bare, quoted, differently-cased,
+// or with trailing punctuation) without loosening the match to accept a
+// substring of a longer, wrong answer.
+func codeExactAnswer(want string) eval.Evaluator {
+	return eval.EvaluatorFunc(func(_ context.Context, response string) eval.Score {
+		got := strings.TrimSpace(response)
+		got = strings.Trim(got, "\"'`")
+		got = strings.TrimSuffix(strings.TrimSpace(got), ".")
+		got = strings.TrimSpace(got)
+		wantTrimmed := strings.TrimSpace(want)
+		if strings.EqualFold(got, wantTrimmed) {
+			return eval.Score{Value: 1, Detail: fmt.Sprintf("equals %q", wantTrimmed)}
+		}
+		return eval.Score{Value: 0, Detail: fmt.Sprintf("got %q, want %q", got, wantTrimmed)}
+	})
+}
+
+// codeTracePythonSource is the inline Python function traced by
+// codeTracePythonTest.
+const codeTracePythonSource = `def transform(s, n):
+    if n == 0:
+        return s
+    if len(s) % 2 == 0:
+        s = s[1:] + s[0]
+    else:
+        s = s[::-1]
+    return transform(s, n - 1)`
+
+// codeTracePythonWant is the exact result of transform("abcdef", 4).
+//
+// ground truth: len("abcdef") is 6 (even) at every recursive call, since
+// the rotate branch never changes length, so every call takes the rotate
+// branch: "abcdef"->"bcdefa"->"cdefab"->"defabc"->"efabcd" over 4 calls
+// (n=4,3,2,1), then n=0 returns "efabcd". codebase_trace_test.go both
+// hand-derives this independently and, when python3 is available, runs
+// the source above directly and compares stdout.
+const codeTracePythonWant = "efabcd"
+
+// codeTracePythonTest: trace a recursive Python function combining a
+// parity check, string rotation, and full reversal to its exact output.
+func codeTracePythonTest() testkit.Test {
+	prompt := `Here is a Python function:
+
+` + "```python\n" + codeTracePythonSource + "\n```" + `
+
+What is the exact return value of transform("abcdef", 4)? Respond with
+only the resulting string, with no quotes and no other text.`
+
+	return testkit.Test{
+		ID:          "code-trace-python",
+		Category:    "research",
+		Subcategory: "codebase",
+		Description: "Trace a recursive Python function combining a parity check, string rotation, and full reversal to its exact output.",
+		Prompt:      prompt,
+		Eval:        codeExactAnswer(codeTracePythonWant),
+	}
+}
+
+// codeTraceTSSource is the inline TypeScript function traced by
+// codeTraceTSTest.
+const codeTraceTSSource = `function pack(items: number[]): string {
+  return items.reduce((acc, x, i) => {
+    if (i % 2 === 0) return acc + x;
+    return acc + x * 2;
+  }, 0).toString(2);
+}`
+
+// codeTSPackTrace replicates pack's reduce logic in Go so
+// codebase_trace_test.go can cross-check the hand-derived trace by
+// running equivalent logic, since the TypeScript itself is not executed.
+func codeTSPackTrace(items []int) string {
+	acc := 0
+	for i, x := range items {
+		if i%2 == 0 {
+			acc += x
+		} else {
+			acc += x * 2
+		}
+	}
+	return strconv.FormatInt(int64(acc), 2)
+}
+
+// codeTraceTSWant is the exact result of pack([3,4,5,6,7]).
+//
+// ground truth: acc starts at 0; i=0,x=3 (even index) -> acc=3; i=1,x=4
+// (odd index) -> acc=3+4*2=11; i=2,x=5 -> acc=11+5=16; i=3,x=6 ->
+// acc=16+6*2=28; i=4,x=7 -> acc=28+7=35. 35 in binary is 100011
+// (32+2+1=35). codebase_trace_test.go recomputes this both by hand and by
+// calling codeTSPackTrace.
+const codeTraceTSWant = "100011"
+
+// codeTraceTSTest: trace a TypeScript reduce-based function mixing
+// index-parity branching and binary string conversion to its exact
+// output.
+func codeTraceTSTest() testkit.Test {
+	prompt := `Here is a TypeScript function:
+
+` + "```ts\n" + codeTraceTSSource + "\n```" + `
+
+What is the exact return value of pack([3, 4, 5, 6, 7])? Respond with only
+the resulting string, with no quotes and no other text.`
+
+	return testkit.Test{
+		ID:          "code-trace-ts",
+		Category:    "research",
+		Subcategory: "codebase",
+		Description: "Trace a TypeScript reduce-based function mixing index-parity branching and binary string conversion to its exact output.",
+		Prompt:      prompt,
+		Eval:        codeExactAnswer(codeTraceTSWant),
+	}
+}

internal/tests/codebase_trace_test.go

@@ -0,0 +1,146 @@
+package tests
+
+import (
+	"bytes"
+	"context"
+	"os"
+	"os/exec"
+	"strings"
+	"testing"
+)
+
+func TestCodeExactAnswer(t *testing.T) {
+	ev := codeExactAnswer("efabcd")
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{"bare correct", "efabcd", 1},
+		{"double-quoted correct", `"efabcd"`, 1},
+		{"different case", "EFABCD", 1},
+		{"trailing period", "efabcd.", 1},
+		{"wrong string", "abcdef", 0},
+		{"correct but truncated", "efabc", 0},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := ev.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}
+
+func TestCodeTracePythonWant_GroundTruth(t *testing.T) {
+	// Hand-derivation, double-checked: len("abcdef")=6 is even at every
+	// step, so every call takes the rotate branch (s[1:]+s[0]) rather than
+	// the reverse branch.
+	s := "abcdef"
+	for i := 0; i < 4; i++ {
+		if len(s)%2 == 0 {
+			s = s[1:] + string(s[0])
+		} else {
+			for l, r := 0, len(s)-1; l < r; l, r = l+1, r-1 {
+				b := []byte(s)
+				b[l], b[r] = b[r], b[l]
+				s = string(b)
+			}
+		}
+	}
+	if s != codeTracePythonWant {
+		t.Errorf("hand-derived trace = %q, want %q", s, codeTracePythonWant)
+	}
+
+	if _, err := exec.LookPath("python3"); err != nil {
+		t.Skip("python3 not found on PATH, skipping exec ground-truth cross-check")
+	}
+
+	script := codeTracePythonSource + "\nprint(transform(\"abcdef\", 4))\n"
+	// #nosec G204 -- script is a fixed, hardcoded string built from the
+	// same codeTracePythonSource constant embedded in the prompt, not
+	// external or response-controlled input.
+	cmd := exec.Command("python3", "-c", script) //nolint:gosec // see comment above
+	var stdout bytes.Buffer
+	cmd.Stdout = &stdout
+	cmd.Stderr = os.Stderr
+	if err := cmd.Run(); err != nil {
+		t.Fatalf("running python3 ground-truth script failed: %v", err)
+	}
+	got := strings.TrimSpace(stdout.String())
+	if got != codeTracePythonWant {
+		t.Errorf("python3-computed transform(\"abcdef\", 4) = %q, want %q", got, codeTracePythonWant)
+	}
+}
+
+func TestCodeTracePythonTest_Eval(t *testing.T) {
+	tc := codeTracePythonTest()
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{"correct", "efabcd", 1},
+		{"correct quoted", `"efabcd"`, 1},
+		{"wrong: forgot final rotation", "defabc", 0},
+		{"wrong: reversed instead of rotated", "fedcba", 0},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}
+
+func TestCodeTraceTSWant_GroundTruth(t *testing.T) {
+	// Hand-derivation, double-checked against codeTSPackTrace: acc=0;
+	// i=0,x=3 (even index) -> 3; i=1,x=4 (odd) -> 3+8=11; i=2,x=5 (even) ->
+	// 16; i=3,x=6 (odd) -> 16+12=28; i=4,x=7 (even) -> 35. 35 = 32+2+1 in
+	// binary is 100011.
+	acc := 0
+	items := []int{3, 4, 5, 6, 7}
+	for i, x := range items {
+		if i%2 == 0 {
+			acc += x
+		} else {
+			acc += x * 2
+		}
+	}
+	if acc != 35 {
+		t.Fatalf("hand-derived acc = %d, want 35", acc)
+	}
+
+	got := codeTSPackTrace(items)
+	if got != codeTraceTSWant {
+		t.Errorf("codeTSPackTrace(%v) = %q, want %q", items, got, codeTraceTSWant)
+	}
+}
+
+func TestCodeTraceTSTest_Eval(t *testing.T) {
+	tc := codeTraceTSTest()
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{"correct", "100011", 1},
+		{"correct quoted", `"100011"`, 1},
+		{"wrong: doubled every element instead of alternating", "1000110", 0},
+		{"wrong: decimal instead of binary", "35", 0},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}

internal/tests/web.go

@@ -7,6 +7,15 @@ import (
 
 func registerWebTests(r *testkit.Registry) {
 	r.Register(webRobotsAICrawlersTest())
+	r.Register(webHTMLProductExtractTest())
+	r.Register(webHreflangReciprocalTest())
+	r.Register(webURLParseComponentsTest())
+	r.Register(webSitemapMaxURLsTest())
+	r.Register(webHTTPStatusScenariosTest())
+	r.Register(webDNSMXRecordTest())
+	r.Register(webCanonicalVsRedirectTest())
+	r.Register(webCORSMissingHeaderTest())
+	r.Register(webSecurityHeadersAuditTest())
 }
 
 // webRobotsAICrawlersRobotsTxt is the inline robots.txt for

internal/tests/web_extract.go

@@ -0,0 +1,137 @@
+package tests
+
+import (
+	"github.com/lukaszraczylo/llm-testbench/internal/eval"
+	"github.com/lukaszraczylo/llm-testbench/internal/testkit"
+)
+
+// webProductCardHTML is the inline HTML product-card snippet for
+// webHTMLProductExtractTest.
+const webProductCardHTML = `<div class="product-card" data-sku="WBH-2201">
+  <h2 class="product-name">Wireless Mechanical Keyboard</h2>
+  <span class="price" data-currency="USD">129.99</span>
+  <span class="stock" data-available="true">In Stock</span>
+</div>`
+
+// webHTMLProductExtractTest: extract a product's name, price, and SKU
+// from inline HTML into JSON.
+//
+// ground truth: name is the h2.product-name text; price is the
+// span.price text as a number; sku is the data-sku attribute on the
+// outer div - all read directly from the markup, no computation.
+func webHTMLProductExtractTest() testkit.Test {
+	prompt := `Here is an HTML product card:
+
+` + "```html\n" + webProductCardHTML + "\n```" + `
+
+Extract the product's name, price (as a number), and SKU into JSON.
+Respond with only a JSON object: {"name":"...","price":<number>,"sku":"..."}`
+
+	evaluator := eval.Mean(
+		eval.JSONField("name", "Wireless Mechanical Keyboard"),
+		eval.JSONField("price", 129.99),
+		eval.JSONField("sku", "WBH-2201"),
+	)
+
+	return testkit.Test{
+		ID:          "web-html-product-extract",
+		Category:    "research",
+		Subcategory: "web",
+		Description: "Extract a product's name, price, and SKU from an inline HTML product card into JSON.",
+		Prompt:      prompt,
+		Eval:        evaluator,
+	}
+}
+
+// webHreflangPagesSnippet lists the hreflang <link> tags of three pages
+// for webHreflangReciprocalTest. /en/ links to both /es/ and /fr/, and
+// /es/ links back to both, but /fr/ omits the reciprocal link back to
+// /en/.
+const webHreflangPagesSnippet = `Page https://example.com/en/ <head> contains:
+<link rel="alternate" hreflang="en-us" href="https://example.com/en/" />
+<link rel="alternate" hreflang="es" href="https://example.com/es/" />
+<link rel="alternate" hreflang="fr" href="https://example.com/fr/" />
+
+Page https://example.com/es/ <head> contains:
+<link rel="alternate" hreflang="en-us" href="https://example.com/en/" />
+<link rel="alternate" hreflang="es" href="https://example.com/es/" />
+<link rel="alternate" hreflang="fr" href="https://example.com/fr/" />
+
+Page https://example.com/fr/ <head> contains:
+<link rel="alternate" hreflang="es" href="https://example.com/es/" />
+<link rel="alternate" hreflang="fr" href="https://example.com/fr/" />`
+
+// webHreflangReciprocalTest: find the one non-reciprocal hreflang link
+// among three inline pages.
+//
+// ground truth: /en/ declares an hreflang link to /fr/, but /fr/'s own
+// link set has no entry pointing back to https://example.com/en/, so the
+// pair (/en/, /fr/) is not reciprocal; /fr/ is the page missing the link,
+// and https://example.com/en/ is the target it is missing. Every other
+// pair in the three pages is fully reciprocal.
+func webHreflangReciprocalTest() testkit.Test {
+	prompt := `hreflang annotations must be reciprocal: if page A links to page B
+via hreflang, page B must also link back to page A. Here are the hreflang
+link tags for three pages:
+
+` + webHreflangPagesSnippet + `
+
+Exactly one pair among these three pages is broken (not reciprocal).
+Respond with only a JSON object:
+{"page_missing_link":"<url of the page that is missing a link>","missing_target":"<url that page should link to but does not>"}`
+
+	evaluator := eval.Mean(
+		eval.JSONField("page_missing_link", "https://example.com/fr/"),
+		eval.JSONField("missing_target", "https://example.com/en/"),
+	)
+
+	return testkit.Test{
+		ID:          "web-hreflang-reciprocal",
+		Category:    "research",
+		Subcategory: "web",
+		Description: "Find the one non-reciprocal hreflang link among three inline pages' link tag sets.",
+		Prompt:      prompt,
+		Eval:        evaluator,
+	}
+}
+
+// webParseURL is the inline URL for webURLParseComponentsTest.
+const webParseURL = "https://api.example.com:8443/v2/search?q=hello+world&sort=desc#results"
+
+// webURLParseComponentsTest: parse an inline URL into its component parts
+// as JSON.
+//
+// ground truth: computed directly from net/url.Parse in
+// web_extract_test.go, not hand-derived - scheme "https", host (hostname,
+// no port) "api.example.com", port "8443", path "/v2/search", raw query
+// "q=hello+world&sort=desc", fragment "results".
+func webURLParseComponentsTest() testkit.Test {
+	prompt := `Parse this URL into its components:
+
+` + webParseURL + `
+
+Respond with only a JSON object:
+{"scheme":"...","host":"...","port":<number>,"path":"...","query":"...","fragment":"..."}
+
+"host" must be the hostname only (no port). "port" must be a number.
+"query" must be the raw query string exactly as it appears after the "?",
+not split into individual parameters.`
+
+	evaluator := eval.Mean(
+		eval.JSONField("scheme", "https"),
+		eval.JSONField("host", "api.example.com"),
+		eval.JSONField("port", 8443),
+		eval.JSONField("path", "/v2/search"),
+		eval.JSONField("query", "q=hello+world&sort=desc"),
+		eval.JSONField("fragment", "results"),
+	)
+
+	return testkit.Test{
+		ID:          "web-url-parse-components",
+		Category:    "research",
+		Subcategory: "web",
+		Description: "Parse an inline URL into its scheme, host, port, path, query, and fragment components as JSON.",
+		Prompt:      prompt,
+		Eval:        evaluator,
+	}
+}

internal/tests/web_extract_test.go

@@ -0,0 +1,113 @@
+package tests
+
+import (
+	"context"
+	"net/url"
+	"testing"
+)
+
+func TestWebHTMLProductExtractTest_Eval(t *testing.T) {
+	tc := webHTMLProductExtractTest()
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{"all correct", `{"name":"Wireless Mechanical Keyboard","price":129.99,"sku":"WBH-2201"}`, 1},
+		{"one field wrong", `{"name":"Wireless Mechanical Keyboard","price":99.99,"sku":"WBH-2201"}`, 2.0 / 3.0},
+		{"all wrong", `{"name":"Wrong Product","price":1,"sku":"XXX"}`, 0},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}
+
+func TestWebHreflangReciprocalTest_Eval(t *testing.T) {
+	tc := webHreflangReciprocalTest()
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{"correct", `{"page_missing_link":"https://example.com/fr/","missing_target":"https://example.com/en/"}`, 1},
+		{"correct page, wrong target", `{"page_missing_link":"https://example.com/fr/","missing_target":"https://example.com/es/"}`, 0.5},
+		{"wrong page entirely", `{"page_missing_link":"https://example.com/es/","missing_target":"https://example.com/en/"}`, 0.5},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}
+
+func TestWebParseURL_GroundTruth(t *testing.T) {
+	// Independent re-derivation via net/url.Parse, not hand-derived, per
+	// the ground-truth discipline.
+	u, err := url.Parse(webParseURL)
+	if err != nil {
+		t.Fatalf("url.Parse(%q): %v", webParseURL, err)
+	}
+
+	if got, want := u.Scheme, "https"; got != want {
+		t.Errorf("Scheme = %q, want %q", got, want)
+	}
+	if got, want := u.Hostname(), "api.example.com"; got != want {
+		t.Errorf("Hostname() = %q, want %q", got, want)
+	}
+	if got, want := u.Port(), "8443"; got != want {
+		t.Errorf("Port() = %q, want %q", got, want)
+	}
+	if got, want := u.Path, "/v2/search"; got != want {
+		t.Errorf("Path = %q, want %q", got, want)
+	}
+	if got, want := u.RawQuery, "q=hello+world&sort=desc"; got != want {
+		t.Errorf("RawQuery = %q, want %q", got, want)
+	}
+	if got, want := u.Fragment, "results"; got != want {
+		t.Errorf("Fragment = %q, want %q", got, want)
+	}
+}
+
+func TestWebURLParseComponentsTest_Eval(t *testing.T) {
+	tc := webURLParseComponentsTest()
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{
+			"all correct",
+			`{"scheme":"https","host":"api.example.com","port":8443,"path":"/v2/search","query":"q=hello+world&sort=desc","fragment":"results"}`,
+			1,
+		},
+		{
+			"port included in host field is still wrong for host",
+			`{"scheme":"https","host":"api.example.com:8443","port":8443,"path":"/v2/search","query":"q=hello+world&sort=desc","fragment":"results"}`,
+			5.0 / 6.0,
+		},
+		{
+			"all wrong",
+			`{"scheme":"http","host":"wrong.example.com","port":80,"path":"/","query":"","fragment":""}`,
+			0,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}

internal/tests/web_protocol.go

@@ -0,0 +1,134 @@
+package tests
+
+import (
+	"github.com/lukaszraczylo/llm-testbench/internal/eval"
+	"github.com/lukaszraczylo/llm-testbench/internal/testkit"
+)
+
+// webSitemapMaxURLsTest: state the sitemaps.org protocol's maximum URL
+// count per sitemap file.
+//
+// ground truth: the sitemaps.org protocol caps a single sitemap file at
+// 50,000 URLs (and 50MB uncompressed); a file exceeding that must be
+// split across multiple sitemap files listed in a sitemap index.
+func webSitemapMaxURLsTest() testkit.Test {
+	prompt := `A site's sitemap.xml file lists 52,000 <url> entries inside a single
+<urlset>. Per the sitemaps.org protocol, what is the maximum number of
+URLs permitted in a single sitemap file (the limit this file violates)?
+Respond with only the number.`
+
+	return testkit.Test{
+		ID:          "web-sitemap-max-urls",
+		Category:    "research",
+		Subcategory: "web",
+		Description: "State the sitemaps.org protocol's maximum URL-per-file limit, applied to a scenario that exceeds it.",
+		Prompt:      prompt,
+		Eval:        eval.Numeric(eval.ExtractLastNumber[int], 50000, 0),
+	}
+}
+
+// webHTTPStatusScenarios lists four inline request-handling scenarios for
+// webHTTPStatusScenariosTest.
+const webHTTPStatusScenarios = `scenario1: "The client sent a request body that is not valid JSON."
+scenario2: "The client did not send any authentication credentials, and
+the requested resource requires a logged-in user."
+scenario3: "The client is authenticated, but their account does not have
+permission to access the requested resource."
+scenario4: "The resource has permanently moved to a new URL; the server
+wants clients (and search engines) to use the new URL from now on."`
+
+// webHTTPStatusScenariosTest: map 4 inline HTTP scenarios to their correct
+// status codes.
+//
+// ground truth: malformed request body -> 400 Bad Request; missing
+// credentials on a resource that requires them -> 401 Unauthorized;
+// authenticated but lacking permission -> 403 Forbidden; permanent
+// relocation -> 301 Moved Permanently.
+func webHTTPStatusScenariosTest() testkit.Test {
+	prompt := `For each of these 4 request-handling scenarios, give the single most
+correct HTTP status code:
+
+` + webHTTPStatusScenarios + `
+
+Respond with only a JSON object mapping each scenario id to its status
+code as a number: {"scenario1":<number>,"scenario2":<number>,"scenario3":<number>,"scenario4":<number>}`
+
+	evaluator := eval.Mean(
+		eval.JSONField("scenario1", 400),
+		eval.JSONField("scenario2", 401),
+		eval.JSONField("scenario3", 403),
+		eval.JSONField("scenario4", 301),
+	)
+
+	return testkit.Test{
+		ID:          "web-http-status-scenarios",
+		Category:    "research",
+		Subcategory: "web",
+		Description: "Map 4 inline request-handling scenarios to their correct HTTP status codes.",
+		Prompt:      prompt,
+		Eval:        evaluator,
+	}
+}
+
+// webDNSMXRecordPattern requires the literal token "MX" (word-bounded,
+// case-insensitive) anywhere in the response, materially correct whether
+// or not the response also says "record" or names the other record types
+// it is not.
+const webDNSMXRecordPattern = `(?i)\bMX\b`
+
+// webDNSMXRecordTest: name the DNS record type used to publish a domain's
+// prioritized mail servers.
+//
+// ground truth: mail-routing priority is published exclusively via MX
+// (Mail Exchange) records; no other standard record type carries a
+// priority value for mail delivery.
+func webDNSMXRecordTest() testkit.Test {
+	prompt := `example.com wants mail delivered to mailA.example.com as its primary
+mail server (priority 10), falling back to mailB.example.com (priority
+20) only if the primary is unreachable. Which single DNS record type
+publishes this prioritized mail-routing configuration? Respond with only
+the record type (e.g. A, AAAA, CNAME, MX, TXT, NS).`
+
+	return testkit.Test{
+		ID:          "web-dns-mx-record",
+		Category:    "research",
+		Subcategory: "web",
+		Description: "Name the DNS record type used to publish a domain's prioritized mail servers.",
+		Prompt:      prompt,
+		Eval:        eval.Regex(webDNSMXRecordPattern),
+	}
+}
+
+// webCanonicalVsRedirectTest: choose between a canonical tag and a 301
+// redirect for a duplicate-content scenario where the duplicate URLs must
+// stay independently reachable.
+//
+// ground truth: a 301 redirect would break the tracking links (they would
+// stop resolving to their own URL), so the correct fix is a canonical
+// tag: it consolidates ranking signals onto the preferred URL while
+// leaving every tracked variant independently reachable.
+func webCanonicalVsRedirectTest() testkit.Test {
+	prompt := `An e-commerce site serves identical product content at these three
+URLs, which must all keep working since they are used as tracking links
+in email and social campaigns:
+
+https://example.com/product?id=123
+https://example.com/product?id=123&ref=email
+https://example.com/product?id=123&ref=social
+
+Search engines are treating these as duplicate content and splitting
+ranking signal between them. You need to consolidate that ranking signal
+onto https://example.com/product?id=123, while keeping all three URLs
+independently reachable for the tracking links to keep working. Should
+you use a canonical tag or a 301 redirect? Respond with only one word:
+"canonical" or "redirect".`
+
+	return testkit.Test{
+		ID:          "web-canonical-vs-redirect",
+		Category:    "research",
+		Subcategory: "web",
+		Description: "Choose canonical tag over 301 redirect for a duplicate-content scenario requiring all URL variants to stay reachable.",
+		Prompt:      prompt,
+		Eval:        eval.Regex(`(?i)^\s*canonical\.?\s*$`),
+	}
+}

internal/tests/web_protocol_test.go

@@ -0,0 +1,98 @@
+package tests
+
+import (
+	"context"
+	"testing"
+)
+
+func TestWebSitemapMaxURLsTest_Eval(t *testing.T) {
+	tc := webSitemapMaxURLsTest()
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{"correct", "50000", 1},
+		{"prose wrapped", "The limit is 50000 URLs per sitemap file.", 1},
+		{"wrong: confused with the uncompressed size limit in MB", "50", 0},
+		{"wrong entirely", "10000", 0},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}
+
+func TestWebHTTPStatusScenariosTest_Eval(t *testing.T) {
+	tc := webHTTPStatusScenariosTest()
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{"all correct", `{"scenario1":400,"scenario2":401,"scenario3":403,"scenario4":301}`, 1},
+		{"swapped 401 and 403", `{"scenario1":400,"scenario2":403,"scenario3":401,"scenario4":301}`, 0.5},
+		{"used 500 for a client error", `{"scenario1":500,"scenario2":401,"scenario3":403,"scenario4":301}`, 0.75},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}
+
+func TestWebDNSMXRecordTest_Eval(t *testing.T) {
+	tc := webDNSMXRecordTest()
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{"bare", "MX", 1},
+		{"lowercase", "mx", 1},
+		{"with the word record", "MX record", 1},
+		{"wrong: CNAME", "CNAME", 0},
+		{"wrong: A record", "A", 0},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}
+
+func TestWebCanonicalVsRedirectTest_Eval(t *testing.T) {
+	tc := webCanonicalVsRedirectTest()
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{"correct lowercase", "canonical", 1},
+		{"correct capitalized with period", "Canonical.", 1},
+		{"wrong: redirect", "redirect", 0},
+		{"wrong: extra words break the forced one-word format", "Use a canonical tag", 0},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}

internal/tests/web_security.go

@@ -0,0 +1,97 @@
+package tests
+
+import (
+	"github.com/lukaszraczylo/llm-testbench/internal/eval"
+	"github.com/lukaszraczylo/llm-testbench/internal/testkit"
+)
+
+// webCORSPreflightExchange is the inline preflight request/response
+// exchange for webCORSMissingHeaderTest.
+const webCORSPreflightExchange = `Browser preflight request (from https://app.example.com):
+OPTIONS /data HTTP/1.1
+Host: api.example.com
+Origin: https://app.example.com
+Access-Control-Request-Method: PUT
+Access-Control-Request-Headers: X-Custom-Auth
+
+Server's preflight response:
+HTTP/1.1 204 No Content
+Access-Control-Allow-Origin: https://app.example.com
+Access-Control-Allow-Methods: GET, POST, PUT`
+
+// webCORSMissingHeaderPattern requires the literal token
+// "Access-Control-Allow-Headers" (case-insensitive) anywhere in the
+// response, accepting it with or without a trailing colon or surrounding
+// prose.
+const webCORSMissingHeaderPattern = `(?i)access-control-allow-headers`
+
+// webCORSMissingHeaderTest: identify the single missing CORS response
+// header that will cause the browser to block the request.
+//
+// ground truth: the browser sent Access-Control-Request-Headers:
+// X-Custom-Auth in the preflight, but the server's response never echoes
+// that header name back in an Access-Control-Allow-Headers response
+// header, so the browser blocks the actual PUT request even though the
+// origin and method are both explicitly allowed.
+func webCORSMissingHeaderTest() testkit.Test {
+	prompt := `Here is a CORS preflight exchange:
+
+` + webCORSPreflightExchange + `
+
+The browser will block the actual PUT request that follows this
+preflight. Which single response header is missing from the server's
+preflight response, causing the block? Respond with only the header name,
+nothing else.`
+
+	return testkit.Test{
+		ID:          "web-cors-missing-header",
+		Category:    "research",
+		Subcategory: "web",
+		Description: "Identify the single missing CORS response header that causes a browser to block a preflighted request.",
+		Prompt:      prompt,
+		Eval:        eval.Regex(webCORSMissingHeaderPattern),
+	}
+}
+
+// webSecurityHeadersResponse is the inline HTTP response header block for
+// webSecurityHeadersAuditTest.
+const webSecurityHeadersResponse = `HTTP/1.1 200 OK
+Content-Type: text/html; charset=utf-8
+Cache-Control: no-cache
+Set-Cookie: session=abc123; HttpOnly
+Server: nginx
+X-Content-Type-Options: nosniff
+Referrer-Policy: strict-origin-when-cross-origin`
+
+// webSecurityHeadersBaseline is the recommended baseline of security
+// headers checked against webSecurityHeadersResponse.
+const webSecurityHeadersBaseline = `Content-Security-Policy, Strict-Transport-Security,
+X-Content-Type-Options, X-Frame-Options, Referrer-Policy`
+
+// webSecurityHeadersAuditTest: identify which baseline security headers
+// are missing from an inline HTTP response.
+//
+// ground truth: of the 5 baseline headers, X-Content-Type-Options and
+// Referrer-Policy are present in the response; Content-Security-Policy,
+// Strict-Transport-Security, and X-Frame-Options are absent.
+func webSecurityHeadersAuditTest() testkit.Test {
+	prompt := `Here is an HTTP response's headers:
+
+` + webSecurityHeadersResponse + `
+
+Here is the recommended baseline of security headers:
+
+` + webSecurityHeadersBaseline + `
+
+Which of the baseline headers are missing from the response above?
+Respond with only a JSON array of the missing header names.`
+
+	return testkit.Test{
+		ID:          "web-security-headers-audit",
+		Category:    "research",
+		Subcategory: "web",
+		Description: "Identify which baseline security headers are missing from an inline HTTP response, given a partial-compliance fixture.",
+		Prompt:      prompt,
+		Eval:        eval.JSONStringSet([]string{"Content-Security-Policy", "Strict-Transport-Security", "X-Frame-Options"}),
+	}
+}

internal/tests/web_security_test.go

@@ -0,0 +1,54 @@
+package tests
+
+import (
+	"context"
+	"testing"
+)
+
+func TestWebCORSMissingHeaderTest_Eval(t *testing.T) {
+	tc := webCORSMissingHeaderTest()
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{"bare header name", "Access-Control-Allow-Headers", 1},
+		{"with a colon", "Access-Control-Allow-Headers:", 1},
+		{"in a sentence", "The missing header is Access-Control-Allow-Headers.", 1},
+		{"wrong: names an already-present header", "Access-Control-Allow-Origin", 0},
+		{"wrong: names an already-present header", "Access-Control-Allow-Methods", 0},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}
+
+func TestWebSecurityHeadersAuditTest_Eval(t *testing.T) {
+	tc := webSecurityHeadersAuditTest()
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{"correct set", `["Content-Security-Policy","Strict-Transport-Security","X-Frame-Options"]`, 1},
+		{"correct set, different order", `["X-Frame-Options","Content-Security-Policy","Strict-Transport-Security"]`, 1},
+		{"missing one", `["Content-Security-Policy","Strict-Transport-Security"]`, 2.0 / 3.0},
+		{"wrongly includes a present header", `["Content-Security-Policy","Strict-Transport-Security","X-Frame-Options","Referrer-Policy"]`, 0.75},
+		{"wrong: says nothing is missing", `[]`, 0},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}

internal/tests/whitepapers.go

@@ -7,6 +7,15 @@ import (
 
 func registerWhitepapersTests(r *testkit.Registry) {
 	r.Register(paperHnswParamsTest())
+	r.Register(paperPQCompressionRatioTest())
+	r.Register(paperBloomFPRateTest())
+	r.Register(paperTFIDFTermScoreTest())
+	r.Register(paperRecallAtKTest())
+	r.Register(paperLSMWriteAmplificationTest())
+	r.Register(paperRaftQuorumFailuresTest())
+	r.Register(paperBTreeHeightLevelsTest())
+	r.Register(paperCAPAvailabilityChoiceTest())
+	r.Register(paperAttentionScaleFactorTest())
 }
 
 // paperHnswParamsExcerpt is a ~150-word, technically accurate original

internal/tests/whitepapers_compute.go

@@ -0,0 +1,105 @@
+package tests
+
+import (
+	"math"
+	"strings"
+)
+
+// wpQuantizedCodeBytes returns the number of bytes needed to store one
+// product-quantization code: m subvectors, each an nbits-bit centroid
+// index, packed with no wasted bits. It assumes m*nbits is a multiple of
+// 8, true for every configuration used in this catalog.
+func wpQuantizedCodeBytes(m, nbits int) int {
+	return m * nbits / 8
+}
+
+// wpBloomFalsePositiveRate computes the standard Bloom filter
+// false-positive probability p = (1 - e^(-k*n/m))^k for an m-bit filter
+// holding n inserted elements and k hash functions.
+func wpBloomFalsePositiveRate(m, n, k int) float64 {
+	exponent := -float64(k) * float64(n) / float64(m)
+	return math.Pow(1-math.Exp(exponent), float64(k))
+}
+
+// wpTermFrequency counts how many times term appears in doc, splitting on
+// whitespace (doc is pre-tokenized) and comparing case-insensitively.
+func wpTermFrequency(term string, doc []string) int {
+	count := 0
+	for _, w := range doc {
+		if strings.EqualFold(w, term) {
+			count++
+		}
+	}
+	return count
+}
+
+// wpDocumentFrequency counts how many of docs contain term at least once.
+func wpDocumentFrequency(term string, docs [][]string) int {
+	count := 0
+	for _, doc := range docs {
+		if wpTermFrequency(term, doc) > 0 {
+			count++
+		}
+	}
+	return count
+}
+
+// wpTFIDF computes the plain tf * ln(N/df) weight: tf is the raw term
+// count in one document, df is the number of documents containing the
+// term, and n is the total document count. No smoothing is applied,
+// matching the formula stated in the wp-tfidf-term-score excerpt.
+func wpTFIDF(tf, df, n int) float64 {
+	return float64(tf) * math.Log(float64(n)/float64(df))
+}
+
+// wpRecallAtK computes the standard recall@k retrieval metric: the
+// fraction of relevant that appears anywhere in retrievedTopK, regardless
+// of rank.
+func wpRecallAtK(relevant, retrievedTopK []string) float64 {
+	retrievedSet := make(map[string]struct{}, len(retrievedTopK))
+	for _, id := range retrievedTopK {
+		retrievedSet[id] = struct{}{}
+	}
+	hit := 0
+	for _, id := range relevant {
+		if _, ok := retrievedSet[id]; ok {
+			hit++
+		}
+	}
+	return float64(hit) / float64(len(relevant))
+}
+
+// wpLSMWriteAmplification returns the write amplification factor for a
+// byte that is rewritten rewriteCount times by compaction after its
+// original flush write: the original write plus every rewrite, divided by
+// the single original write.
+func wpLSMWriteAmplification(rewriteCount int) int {
+	return 1 + rewriteCount
+}
+
+// wpRaftQuorumSize returns the minimum number of votes/acknowledgments
+// needed for a strict majority of an n-node Raft cluster: floor(n/2)+1.
+func wpRaftQuorumSize(n int) int {
+	return n/2 + 1
+}
+
+// wpRaftMaxTolerableFailures returns how many simultaneous node failures
+// an n-node Raft cluster can sustain while a majority quorum, sized
+// relative to the full n-node membership, remains reachable.
+func wpRaftMaxTolerableFailures(n int) int {
+	return n - wpRaftQuorumSize(n)
+}
+
+// wpBTreeMinBranchingLevels returns the smallest h (root counted as h=0)
+// such that a tree with minimum fanout f can index at least n leaf key
+// slots, per the f^(h+1) >= n bound stated in the wp-btree-height-levels
+// excerpt.
+func wpBTreeMinBranchingLevels(f, n int) int {
+	h := 0
+	capacity := f
+	for capacity < n {
+		capacity *= f
+		h++
+	}
+	return h
+}

internal/tests/whitepapers_systems.go

@@ -0,0 +1,267 @@
+package tests
+
+import (
+	"github.com/lukaszraczylo/llm-testbench/internal/eval"
+	"github.com/lukaszraczylo/llm-testbench/internal/testkit"
+)
+
+// paperLSMExcerpt is a ~152-word, technically accurate original
+// description of LSM-tree leveled compaction and write amplification
+// written for this test, pinning a concrete 4-level rewrite path.
+const paperLSMExcerpt = `An LSM-tree buffers writes in memory and periodically flushes them as
+immutable sorted files, then merges those files together in the background
+through a process called compaction, so reads do not have to check an
+unbounded number of files. In leveled compaction, each level on disk is
+merged into the next level down, and every byte that a compaction step
+touches gets rewritten to a new file even though its value did not change.
+A storage engine described in this report uses four levels, L0 through L3.
+A byte first flushed into L0 is compacted into L1, then later compacted
+from L1 into L2, and finally from L2 into L3 before it reaches its
+long-term resting level, meaning it is rewritten three separate times after
+its original flush write. Write amplification here is defined as total
+bytes written to disk, including every rewrite, divided by the bytes of
+user data originally written.`
+
+// paperLSMWriteAmpWant is derived by calling wpLSMWriteAmplification, not
+// hardcoded.
+//
+// ground truth: the excerpt states a byte is rewritten 3 times (L0->L1,
+// L1->L2, L2->L3) after its original flush write, so total disk writes per
+// user byte = 1 (original) + 3 (rewrites) = 4.
+// whitepapers_systems_test.go independently recomputes this by counting
+// the rewrite path described in the excerpt text.
+var paperLSMWriteAmpWant = wpLSMWriteAmplification(3)
+
+// paperLSMWriteAmplificationTest: derive an LSM-tree's write amplification
+// factor from an inline excerpt describing a byte's full compaction path.
+func paperLSMWriteAmplificationTest() testkit.Test {
+	prompt := `Read this excerpt about LSM-tree compaction:
+
+` + paperLSMExcerpt + `
+
+Using the byte's full path through this system (an L0 flush, then
+compaction into L1, then L2, then L3) and the write-amplification
+definition given above, what is the write amplification factor for that
+byte? Respond with only the number.`
+
+	return testkit.Test{
+		ID:          "paper-lsm-write-amplification",
+		Category:    "research",
+		Subcategory: "whitepapers",
+		Description: "Derive an LSM-tree's write amplification factor from an inline excerpt describing a byte's full leveled-compaction path.",
+		Prompt:      prompt,
+		Eval:        eval.Numeric(eval.ExtractLastNumber[int], paperLSMWriteAmpWant, 0),
+	}
+}
+
+// paperRaftExcerpt is a ~155-word, technically accurate original
+// description of Raft's majority-quorum requirement written for this
+// test, pinning a 5-node cluster.
+const paperRaftExcerpt = `Raft is a consensus protocol that keeps a replicated log consistent across
+a cluster of nodes by electing a single leader for each term and having
+that leader replicate every log entry to the other nodes. A node starts an
+election after its randomized election timeout, typically drawn from the
+150 to 300 millisecond range in the original paper, expires without
+hearing from a leader. A candidate becomes leader only once it collects
+votes from a strict majority of all nodes in the cluster, not merely a
+majority of the nodes currently reachable, and the same majority rule
+governs when a log entry counts as committed. This report deployed Raft
+across a cluster of N equal to 5 nodes, so both leader election and log
+commitment require votes or acknowledgments from a majority quorum sized
+relative to that total membership of 5, not to however many nodes happen to
+be healthy at a given moment.`
+
+// paperRaftQuorumWant and paperRaftFailuresWant are derived by calling
+// wpRaftQuorumSize and wpRaftMaxTolerableFailures, not hardcoded.
+//
+// ground truth: quorum = floor(5/2)+1 = 3; a 5-node cluster can lose at
+// most 5-3 = 2 nodes and still reach that quorum.
+// whitepapers_systems_test.go independently recomputes both for several N.
+var (
+	paperRaftQuorumWant   = wpRaftQuorumSize(5)
+	paperRaftFailuresWant = wpRaftMaxTolerableFailures(5)
+)
+
+// paperRaftQuorumFailuresTest: derive a Raft cluster's majority quorum
+// size and maximum tolerable simultaneous failures from an inline excerpt
+// and a stated cluster size.
+func paperRaftQuorumFailuresTest() testkit.Test {
+	prompt := `Read this excerpt about the Raft consensus protocol:
+
+` + paperRaftExcerpt + `
+
+For this N=5 node cluster:
+1. What is the minimum number of votes/acknowledgments needed for a
+   majority quorum?
+2. How many simultaneous node failures can the cluster sustain and still
+   reach that quorum?
+
+Respond with only a JSON object: {"quorum_size":<number>,"max_tolerable_failures":<number>}`
+
+	evaluator := eval.Mean(
+		eval.JSONField("quorum_size", paperRaftQuorumWant),
+		eval.JSONField("max_tolerable_failures", paperRaftFailuresWant),
+	)
+
+	return testkit.Test{
+		ID:          "paper-raft-quorum-failures",
+		Category:    "research",
+		Subcategory: "whitepapers",
+		Description: "Derive a 5-node Raft cluster's majority quorum size and maximum tolerable simultaneous failures from an inline excerpt.",
+		Prompt:      prompt,
+		Eval:        evaluator,
+	}
+}
+
+// paperBTreeExcerpt is a ~146-word, technically accurate original
+// description of a worst-case (minimum-fanout) B-tree capacity model
+// written for this test, pinning order m and a target key count.
+const paperBTreeExcerpt = `A B-tree of order m allows every internal node between ceil(m/2) and m
+children, keeping the tree shallow and well balanced even as it grows,
+which bounds the number of disk reads a lookup needs to at most one per
+level. For capacity planning this report models the worst case: a tree
+where every internal node has the minimum allowed fanout, f equals
+ceil(m/2), rather than the maximum. With that minimum fanout, and counting
+the root itself as branching level h equal to zero, a tree with h
+branching levels can index at most f to the power of (h+1) leaf key slots,
+since each additional level below the root multiplies the reachable leaf
+count by f. This benchmark used order m equal to 200, giving a minimum
+fanout f equal to 100, and needed to index at least 5,000,000 keys without
+exceeding that capacity bound.`
+
+// paperBTreeHeightWant is derived by calling wpBTreeMinBranchingLevels,
+// not hardcoded.
+//
+// ground truth: smallest h with 100^(h+1) >= 5,000,000: h=0 -> 100;
+// h=1 -> 10,000; h=2 -> 1,000,000 (still short); h=3 -> 100,000,000
+// (first to clear 5,000,000). whitepapers_systems_test.go independently
+// recomputes this with a plain loop over math.Pow.
+var paperBTreeHeightWant = wpBTreeMinBranchingLevels(100, 5000000)
+
+// paperBTreeHeightLevelsTest: derive the minimum number of branching
+// levels a worst-case B-tree needs to index a target key count, from an
+// inline excerpt's stated capacity formula.
+func paperBTreeHeightLevelsTest() testkit.Test {
+	prompt := `Read this excerpt about B-tree capacity planning:
+
+` + paperBTreeExcerpt + `
+
+Using the formula given above (f^(h+1) >= n keys, root counted as h=0),
+what is the minimum number of branching levels h needed so this tree can
+index at least 5,000,000 keys with f=100? Respond with only the number.`
+
+	return testkit.Test{
+		ID:          "paper-btree-height-levels",
+		Category:    "research",
+		Subcategory: "whitepapers",
+		Description: "Derive the minimum branching-level count a worst-case-fanout B-tree needs to index a target key count.",
+		Prompt:      prompt,
+		Eval:        eval.Numeric(eval.ExtractLastNumber[int], paperBTreeHeightWant, 0),
+	}
+}
+
+// paperCAPExcerpt is a ~135-word, technically accurate original
+// description of the CAP theorem written for this test, describing a
+// scenario that unambiguously chooses availability over consistency.
+const paperCAPExcerpt = `The CAP theorem states that a distributed data store can provide at most
+two of three guarantees during a network partition: consistency, meaning
+every read sees the most recent write or an error; availability, meaning
+every request receives a non-error response; and partition tolerance,
+meaning the system keeps operating despite dropped or delayed messages
+between nodes. Because network partitions are unavoidable in any real
+distributed deployment, partition tolerance is treated as a fixed
+requirement rather than a design choice, leaving consistency and
+availability as the actual trade-off a system architect makes. A key-value
+store described in this report was explicitly designed so that, during a
+partition between two data centers, both sides keep accepting writes
+independently rather than refusing requests, accepting that the two sides
+may briefly disagree and need reconciliation once the partition heals.`
+
+// paperCAPAvailabilityPattern anchors on the trimmed response being
+// exactly the single word "availability" (case-insensitive, with an
+// optional trailing period), accepting every materially-correct form of
+// the forced one-word answer without also matching a response that names
+// the wrong guarantee.
+const paperCAPAvailabilityPattern = `(?i)^\s*availability\.?\s*$`
+
+// paperCAPAvailabilityChoiceTest: identify, from an inline CAP-theorem
+// excerpt and a described design decision, which guarantee (consistency or
+// availability) the system prioritizes during a partition.
+func paperCAPAvailabilityChoiceTest() testkit.Test {
+	prompt := `Read this excerpt about the CAP theorem:
+
+` + paperCAPExcerpt + `
+
+Given this design decision - both data centers keep accepting writes
+during the partition rather than refusing requests - which of the two
+trade-off guarantees does this system prioritize during a partition, per
+the CAP theorem? Respond with only one word: "Consistency" or
+"Availability".`
+
+	return testkit.Test{
+		ID:          "paper-cap-availability-choice",
+		Category:    "research",
+		Subcategory: "whitepapers",
+		Description: "Identify which CAP-theorem guarantee a described partition-handling design decision prioritizes.",
+		Prompt:      prompt,
+		Eval:        eval.Regex(paperCAPAvailabilityPattern),
+	}
+}
+
+// paperAttentionExcerpt is a ~158-word, technically accurate original
+// description of scaled dot-product attention's sqrt(d_k) scaling written
+// for this test, pinning a concrete per-head dimension.
+const paperAttentionExcerpt = `Scaled dot-product attention computes a weighted average of value vectors,
+where the weight given to each value comes from a softmax over the dot
+products between one query vector and every key vector. When query and key
+entries are drawn independently with roughly unit variance, the variance of
+their dot product grows in proportion to the dimensionality d_k of those
+vectors, so without correction, larger d_k pushes the raw dot products to
+extreme magnitudes. Feeding those extreme values into softmax drives most
+of the output probability mass onto one entry and squashes the gradient
+almost to zero everywhere else, stalling training. To counteract this, the
+dot products are divided by the square root of d_k before the softmax,
+which rescales their variance back down to roughly one regardless of
+dimensionality. A benchmark model in this report used d_k equal to 64 per
+attention head, so this scaling divided every raw dot product by the square
+root of 64.`
+
+// paperAttentionDK and paperAttentionScaleFactorWant pin the excerpt's
+// stated d_k and its square root.
+//
+// ground truth: d_k = 64 (stated verbatim in the excerpt); scale factor =
+// sqrt(64) = 8. whitepapers_systems_test.go independently recomputes the
+// scale factor with math.Sqrt.
+const (
+	paperAttentionDK              = 64
+	paperAttentionScaleFactorWant = 8
+)
+
+// paperAttentionScaleFactorTest: extract d_k and derive the sqrt(d_k)
+// attention scale factor from an inline excerpt.
+func paperAttentionScaleFactorTest() testkit.Test {
+	prompt := `Read this excerpt about scaled dot-product attention:
+
+` + paperAttentionExcerpt + `
+
+For the benchmark model described:
+1. What is d_k?
+2. By what number are the raw dot products divided before the softmax
+   (the scale factor)?
+
+Respond with only a JSON object: {"d_k":<number>,"scale_factor":<number>}`
+
+	evaluator := eval.Mean(
+		eval.JSONField("d_k", paperAttentionDK),
+		eval.JSONField("scale_factor", paperAttentionScaleFactorWant),
+	)
+
+	return testkit.Test{
+		ID:          "paper-attention-scale-factor",
+		Category:    "research",
+		Subcategory: "whitepapers",
+		Description: "Extract d_k and derive the sqrt(d_k) attention scale factor from an inline excerpt.",
+		Prompt:      prompt,
+		Eval:        evaluator,
+	}
+}

internal/tests/whitepapers_systems_test.go

@@ -0,0 +1,203 @@
+package tests
+
+import (
+	"context"
+	"math"
+	"strings"
+	"testing"
+)
+
+func TestPaperLSMWriteAmpWant_GroundTruth(t *testing.T) {
+	// Independent re-derivation by counting the excerpt's stated rewrite
+	// path (L0->L1, L1->L2, L2->L3 = 3 rewrites) plus the original write,
+	// not via wpLSMWriteAmplification.
+	// The excerpt names exactly three compaction moves; check each phrase is
+	// present rather than counting "into L" substrings, since the initial
+	// "flushed into L0" also contains that substring without being a
+	// compaction rewrite.
+	moves := []string{"compacted into L1", "L1 into L2", "L2 into L3"}
+	rewrites := 0
+	for _, phrase := range moves {
+		if strings.Contains(paperLSMExcerpt, phrase) {
+			rewrites++
+		}
+	}
+	if rewrites != 3 {
+		t.Fatalf("excerpt describes %d of the 3 expected compaction moves", rewrites)
+	}
+	want := 1 + rewrites
+
+	if want != 4 {
+		t.Fatalf("independently recomputed write amplification = %d, want 4", want)
+	}
+	if paperLSMWriteAmpWant != want {
+		t.Errorf("paperLSMWriteAmpWant = %d, independently recomputed = %d", paperLSMWriteAmpWant, want)
+	}
+}
+
+func TestPaperLSMWriteAmplificationTest_Eval(t *testing.T) {
+	tc := paperLSMWriteAmplificationTest()
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{"correct", "4", 1},
+		{"prose wrapped", "The write amplification factor is 4.", 1},
+		{"forgot the original write", "3", 0},
+		{"off by one", "5", 0},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}
+
+func TestPaperRaftQuorumWant_GroundTruth(t *testing.T) {
+	// Independent re-derivation of floor(n/2)+1 and n-quorum for several N,
+	// not via wpRaftQuorumSize/wpRaftMaxTolerableFailures.
+	tests := []struct {
+		n            int
+		wantQuorum   int
+		wantFailures int
+	}{
+		{3, 2, 1},
+		{5, 3, 2},
+		{7, 4, 3},
+	}
+	for _, tt := range tests {
+		gotQuorum := tt.n/2 + 1
+		gotFailures := tt.n - gotQuorum
+		if gotQuorum != tt.wantQuorum {
+			t.Errorf("quorum(n=%d) = %d, want %d", tt.n, gotQuorum, tt.wantQuorum)
+		}
+		if gotFailures != tt.wantFailures {
+			t.Errorf("maxFailures(n=%d) = %d, want %d", tt.n, gotFailures, tt.wantFailures)
+		}
+	}
+
+	if paperRaftQuorumWant != 3 {
+		t.Errorf("paperRaftQuorumWant = %d, want 3", paperRaftQuorumWant)
+	}
+	if paperRaftFailuresWant != 2 {
+		t.Errorf("paperRaftFailuresWant = %d, want 2", paperRaftFailuresWant)
+	}
+}
+
+func TestPaperRaftQuorumFailuresTest_Eval(t *testing.T) {
+	tc := paperRaftQuorumFailuresTest()
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{"all correct", `{"quorum_size":3,"max_tolerable_failures":2}`, 1},
+		{"one field wrong", `{"quorum_size":3,"max_tolerable_failures":3}`, 0.5},
+		{"both wrong: used simple majority of reachable nodes", `{"quorum_size":2,"max_tolerable_failures":1}`, 0},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}
+
+func TestPaperBTreeHeightWant_GroundTruth(t *testing.T) {
+	// Independent re-derivation with math.Pow, not via
+	// wpBTreeMinBranchingLevels's iterative loop.
+	const f, n = 100.0, 5000000.0
+	h := 0
+	for math.Pow(f, float64(h+1)) < n {
+		h++
+	}
+
+	if h != 3 {
+		t.Fatalf("independently recomputed h = %d, want 3", h)
+	}
+	if paperBTreeHeightWant != h {
+		t.Errorf("paperBTreeHeightWant = %d, independently recomputed = %d", paperBTreeHeightWant, h)
+	}
+}
+
+func TestPaperBTreeHeightLevelsTest_Eval(t *testing.T) {
+	tc := paperBTreeHeightLevelsTest()
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{"correct", "3", 1},
+		{"prose wrapped", "The minimum height is h=3.", 1},
+		{"one level short", "2", 0},
+		{"one level too many", "4", 0},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}
+
+func TestPaperCAPAvailabilityChoiceTest_Eval(t *testing.T) {
+	tc := paperCAPAvailabilityChoiceTest()
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{"correct lowercase", "availability", 1},
+		{"correct capitalized", "Availability", 1},
+		{"correct with trailing period", "Availability.", 1},
+		{"wrong: consistency", "Consistency", 0},
+		{"wrong: extra words break the forced one-word format", "The answer is availability", 0},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}
+
+func TestPaperAttentionScaleFactorTest_Eval(t *testing.T) {
+	tc := paperAttentionScaleFactorTest()
+
+	// Ground truth: sqrt(64) = 8, independently recomputed with math.Sqrt.
+	if got := math.Sqrt(64); got != 8 {
+		t.Fatalf("math.Sqrt(64) = %v, want 8", got)
+	}
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{"all correct", `{"d_k":64,"scale_factor":8}`, 1},
+		{"one field wrong", `{"d_k":64,"scale_factor":64}`, 0.5},
+		{"both wrong", `{"d_k":512,"scale_factor":22.6}`, 0},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}

internal/tests/whitepapers_vector.go

@@ -0,0 +1,218 @@
+package tests
+
+import (
+	"github.com/lukaszraczylo/llm-testbench/internal/eval"
+	"github.com/lukaszraczylo/llm-testbench/internal/testkit"
+)
+
+// paperPQCompressionExcerpt is a ~143-word, technically accurate original
+// description of product quantization written for this test, pinning the
+// dimension, m, and nbits parameters needed to derive a compression ratio.
+const paperPQCompressionExcerpt = `Product quantization compresses high-dimensional vectors for approximate
+nearest-neighbor search by splitting each vector into m equal-length
+subvectors, then quantizing each subvector independently against its own
+codebook. Each codebook is trained offline with k-means and holds 2^nbits
+centroids. Rather than storing the original subvector, the index stores only
+the index of its nearest centroid: an nbits-bit code. Concatenating the m
+per-subvector codes gives the full compressed representation of the vector.
+A benchmark index in this report indexed 128-dimensional vectors stored as
+32-bit floats, so one uncompressed vector occupies 128 times 4, or 512,
+bytes. The index split each vector into m equal to 8 subvectors and used
+nbits equal to 8 bits per subvector, giving each subquantizer's codebook 256
+centroids. Search computes approximate distances directly against the
+stored codes using precomputed per-centroid distance tables, never
+decompressing a vector back to its original 512-byte form.`
+
+// paperPQCompressionRatioWant is derived by calling wpQuantizedCodeBytes,
+// not hardcoded.
+//
+// ground truth: code bytes = m*nbits/8 = 8*8/8 = 8 bytes per vector;
+// compression ratio = raw bytes / code bytes = 512/8 = 64.
+// whitepapers_vector_test.go independently recomputes this with plain
+// arithmetic.
+var paperPQCompressionRatioWant = 512 / wpQuantizedCodeBytes(8, 8)
+
+// paperPQCompressionRatioTest: derive a product-quantization compression
+// ratio from an inline excerpt's m/nbits/dimension parameters.
+func paperPQCompressionRatioTest() testkit.Test {
+	prompt := `Read this excerpt about product quantization (PQ) for vector search:
+
+` + paperPQCompressionExcerpt + `
+
+The benchmark index above stores each compressed vector as its m
+per-subvector codes packed with no wasted bits. What is the compression
+ratio of the compressed representation versus the original 512-byte
+float32 vector (raw bytes divided by compressed bytes)? Respond with only
+the number.`
+
+	return testkit.Test{
+		ID:          "paper-pq-compression-ratio",
+		Category:    "research",
+		Subcategory: "whitepapers",
+		Description: "Derive a product-quantization compression ratio from an inline excerpt's m, nbits, and raw-dimension parameters.",
+		Prompt:      prompt,
+		Eval:        eval.Numeric(eval.ExtractLastNumber[int], paperPQCompressionRatioWant, 0),
+	}
+}
+
+// paperBloomFPRateExcerpt is a ~157-word, technically accurate original
+// description of the Bloom filter false-positive formula written for this
+// test, pinning m, n, and k.
+const paperBloomFPRateExcerpt = `A Bloom filter is a space-efficient structure for testing set membership
+that trades a small, tunable false-positive rate for large memory savings,
+and never produces a false negative. It holds an m-bit array, initially all
+zero, and uses k independent hash functions. Inserting an element sets the
+k bit positions its hashes point to; querying an element checks whether all
+k of those positions are set, and if so reports "possibly present." The
+false-positive probability after inserting n elements is well approximated
+by p equals the quantity one minus e to the power of negative k times n
+divided by m, that whole quantity raised to the power k. A caching layer
+described in this report sized its filter at m equal to 10000 bits for n
+equal to 1000 expected keys, and chose k equal to 7 hash functions,
+following the standard guidance to pick k near m over n times the natural
+log of 2.`
+
+// paperBloomFPRateWant is derived by calling wpBloomFalsePositiveRate, not
+// hardcoded.
+//
+// ground truth: p = (1 - e^(-k*n/m))^k with m=10000, n=1000, k=7.
+// whitepapers_vector_test.go recomputes this independently with math.Pow
+// and math.Exp inline.
+var paperBloomFPRateWant = round4dp(wpBloomFalsePositiveRate(10000, 1000, 7))
+
+// paperBloomFPRateTest: compute a Bloom filter's false-positive rate from
+// an inline excerpt's formula and m/n/k parameters.
+func paperBloomFPRateTest() testkit.Test {
+	prompt := `Read this excerpt about Bloom filters:
+
+` + paperBloomFPRateExcerpt + `
+
+Using the exact formula given above and the caching layer's parameters
+(m=10000, n=1000, k=7), compute the false-positive probability p. Respond
+with only the number, rounded to 4 decimal places.`
+
+	return testkit.Test{
+		ID:          "paper-bloom-fp-rate",
+		Category:    "research",
+		Subcategory: "whitepapers",
+		Description: "Compute a Bloom filter's false-positive rate from an inline excerpt's formula and m/n/k parameters.",
+		Prompt:      prompt,
+		Eval:        eval.Numeric(eval.ExtractLastNumber[float64], paperBloomFPRateWant, 0.0005),
+	}
+}
+
+// paperTFIDFExcerpt is a ~148-word, technically accurate original
+// description of the plain (unsmoothed) TF-IDF formula written for this
+// test.
+const paperTFIDFExcerpt = `TF-IDF scores how important a term is to one document within a corpus by
+combining two signals: how often the term appears in that document, and how
+rare the term is across the whole corpus. This report uses the plainest
+textbook variant. Term frequency, tf(t,d), is simply the raw count of term
+t in document d, with no normalization. Inverse document frequency,
+idf(t), is the natural logarithm of N divided by df(t), where N is the
+total number of documents in the corpus and df(t) is the number of
+documents that contain t at least once. The TF-IDF weight of a term in a
+document is the product tf(t,d) times idf(t): a term that appears often in
+one document but rarely elsewhere scores highest, while a term appearing in
+every document scores near zero regardless of how often it repeats, since
+its idf approaches the logarithm of one.`
+
+// paperTFIDFDoc1, paperTFIDFDoc2, and paperTFIDFDoc3 are the pre-tokenized
+// (whitespace-split, lowercase) mini-corpus for paperTFIDFTermScoreTest.
+var (
+	paperTFIDFDoc1   = []string{"the", "cat", "sat", "on", "the", "mat"}
+	paperTFIDFDoc2   = []string{"the", "dog", "sat", "on", "the", "log"}
+	paperTFIDFDoc3   = []string{"cats", "and", "dogs", "are", "great", "pets"}
+	paperTFIDFCorpus = [][]string{paperTFIDFDoc1, paperTFIDFDoc2, paperTFIDFDoc3}
+)
+
+// paperTFIDFTermScoreWant is derived by calling wpTermFrequency,
+// wpDocumentFrequency, and wpTFIDF, not hardcoded.
+//
+// ground truth: tf("sat", doc1) = 1; df("sat") = 2 (doc1 and doc2 contain
+// it, doc3 does not); N = 3; idf = ln(3/2); tfidf = 1 * ln(3/2).
+// whitepapers_vector_test.go recomputes this independently with math.Log.
+var paperTFIDFTermScoreWant = round4dp(wpTFIDF(
+	wpTermFrequency("sat", paperTFIDFDoc1),
+	wpDocumentFrequency("sat", paperTFIDFCorpus),
+	len(paperTFIDFCorpus),
+))
+
+// paperTFIDFTermScoreTest: compute the TF-IDF weight of one term in one
+// document of an inline 3-document mini-corpus, using the exact formula
+// given in the excerpt.
+func paperTFIDFTermScoreTest() testkit.Test {
+	prompt := `Read this excerpt about TF-IDF:
+
+` + paperTFIDFExcerpt + `
+
+Here is a 3-document corpus:
+
+Document 1: "the cat sat on the mat"
+Document 2: "the dog sat on the log"
+Document 3: "cats and dogs are great pets"
+
+Using the exact formulas given above (tf = raw count of the term in the
+document, idf = natural log of N divided by df), compute the TF-IDF weight
+of the term "sat" in Document 1. Respond with only the number, rounded to
+4 decimal places.`
+
+	return testkit.Test{
+		ID:          "paper-tfidf-term-score",
+		Category:    "research",
+		Subcategory: "whitepapers",
+		Description: "Compute a term's TF-IDF weight in one document of an inline 3-document mini-corpus using a formula given in the excerpt.",
+		Prompt:      prompt,
+		Eval:        eval.Numeric(eval.ExtractLastNumber[float64], paperTFIDFTermScoreWant, 0.0005),
+	}
+}
+
+// paperRecallExcerpt is a ~137-word, technically accurate original
+// description of the recall@k retrieval metric written for this test.
+const paperRecallExcerpt = `Recall@k is a standard retrieval-quality metric for evaluating a nearest-
+neighbor or search system: for one query, it is the fraction of that
+query's truly relevant items that appear anywhere within the system's top
+k returned results, regardless of their exact rank inside that top-k list.
+Formally, recall@k equals the size of the intersection between the
+relevant set and the top-k retrieved set, divided by the size of the
+relevant set. A perfect recall@k of 1.0 means every relevant item was
+retrieved somewhere in the top k, even if none of them landed in rank one;
+a lower score means some relevant items were pushed outside the top k
+entirely and are effectively lost to any downstream ranking or re-ranking
+stage, since a re-ranker can only reorder candidates that retrieval already
+surfaced, never recover ones it dropped.`
+
+// paperRecallRelevant and paperRecallRetrievedTop6 are the inline
+// relevant-set and top-6 retrieved-ranking fixtures for
+// paperRecallAtKTest.
+var (
+	paperRecallRelevant      = []string{"D2", "D5", "D7", "D9"}
+	paperRecallRetrievedTop6 = []string{"D1", "D5", "D3", "D7", "D6", "D2"}
+	paperRecallAtKWant       = round4dp(wpRecallAtK(paperRecallRelevant, paperRecallRetrievedTop6))
+)
+
+// paperRecallAtKTest: compute recall@k from an inline relevant-item set and
+// a ranked top-k retrieval list.
+func paperRecallAtKTest() testkit.Test {
+	prompt := `Read this excerpt about the recall@k retrieval metric:
+
+` + paperRecallExcerpt + `
+
+A query's truly relevant documents are: D2, D5, D7, D9.
+
+A retrieval system's ranked top-6 results for that query are, in rank
+order: D1, D5, D3, D7, D6, D2.
+
+Using the definition given above, compute recall@6 for this query. Respond
+with only the number, as a decimal (e.g. 0.5), rounded to 4 decimal
+places.`
+
+	return testkit.Test{
+		ID:          "paper-recall-at-k",
+		Category:    "research",
+		Subcategory: "whitepapers",
+		Description: "Compute recall@k from an inline relevant-item set and a ranked top-k retrieval list.",
+		Prompt:      prompt,
+		Eval:        eval.Numeric(eval.ExtractLastNumber[float64], paperRecallAtKWant, 0.0005),
+	}
+}

internal/tests/whitepapers_vector_test.go

@@ -0,0 +1,165 @@
+package tests
+
+import (
+	"context"
+	"math"
+	"testing"
+)
+
+func TestPaperPQCompressionRatioWant_GroundTruth(t *testing.T) {
+	// Independent re-derivation with plain arithmetic, not via
+	// wpQuantizedCodeBytes.
+	const dim, m, nbits = 128, 8, 8
+	rawBytes := dim * 4
+	codeBytes := m * nbits / 8
+	want := rawBytes / codeBytes
+
+	if want != 64 {
+		t.Fatalf("independently recomputed ratio = %d, want 64", want)
+	}
+	if paperPQCompressionRatioWant != want {
+		t.Errorf("paperPQCompressionRatioWant = %d, independently recomputed = %d", paperPQCompressionRatioWant, want)
+	}
+}
+
+func TestPaperPQCompressionRatioTest_Eval(t *testing.T) {
+	tc := paperPQCompressionRatioTest()
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{"correct", "64", 1},
+		{"prose wrapped", "The compression ratio is 64x.", 1},
+		{"wrong", "8", 0},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}
+
+func TestPaperBloomFPRateWant_GroundTruth(t *testing.T) {
+	// Independent re-derivation with math.Pow/math.Exp inline, not via
+	// wpBloomFalsePositiveRate.
+	const m, n, k = 10000.0, 1000.0, 7.0
+	want := math.Round(math.Pow(1-math.Exp(-k*n/m), k)*10000) / 10000
+
+	if math.Abs(want-0.0082) > 1e-9 {
+		t.Fatalf("independently recomputed FP rate = %v, want ~0.0082", want)
+	}
+	if paperBloomFPRateWant != want {
+		t.Errorf("paperBloomFPRateWant = %v, independently recomputed = %v", paperBloomFPRateWant, want)
+	}
+}
+
+func TestPaperBloomFPRateTest_Eval(t *testing.T) {
+	tc := paperBloomFPRateTest()
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{"exact", "0.0082", 1},
+		{"within tolerance", "0.0081", 1},
+		{"prose wrapped", "p is approximately 0.0082.", 1},
+		{"outside tolerance", "0.05", 0},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}
+
+func TestPaperTFIDFTermScoreWant_GroundTruth(t *testing.T) {
+	// Independent re-derivation by counting the raw corpus text directly,
+	// not via wpTermFrequency/wpDocumentFrequency/wpTFIDF.
+	tf := 1 // "sat" appears once in doc1
+	df := 2 // doc1 and doc2 contain "sat"; doc3 does not
+	n := 3  // 3 documents total
+	want := math.Round(float64(tf)*math.Log(float64(n)/float64(df))*10000) / 10000
+
+	if math.Abs(want-0.4055) > 1e-9 {
+		t.Fatalf("independently recomputed tfidf = %v, want ~0.4055", want)
+	}
+	if paperTFIDFTermScoreWant != want {
+		t.Errorf("paperTFIDFTermScoreWant = %v, independently recomputed = %v", paperTFIDFTermScoreWant, want)
+	}
+}
+
+func TestPaperTFIDFTermScoreTest_Eval(t *testing.T) {
+	tc := paperTFIDFTermScoreTest()
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{"exact", "0.4055", 1},
+		{"within tolerance", "0.4053", 1},
+		{"prose wrapped", "The TF-IDF weight is 0.4055.", 1},
+		{"outside tolerance", "1.0986", 0},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}
+
+func TestPaperRecallAtKWant_GroundTruth(t *testing.T) {
+	// Independent re-derivation with a plain set intersection, not via
+	// wpRecallAtK.
+	relevant := map[string]bool{"D2": true, "D5": true, "D7": true, "D9": true}
+	retrieved := []string{"D1", "D5", "D3", "D7", "D6", "D2"}
+	hit := 0
+	for _, id := range retrieved {
+		if relevant[id] {
+			hit++
+		}
+	}
+	want := float64(hit) / float64(len(relevant))
+
+	if want != 0.75 {
+		t.Fatalf("independently recomputed recall@6 = %v, want 0.75", want)
+	}
+	if paperRecallAtKWant != want {
+		t.Errorf("paperRecallAtKWant = %v, independently recomputed = %v", paperRecallAtKWant, want)
+	}
+}
+
+func TestPaperRecallAtKTest_Eval(t *testing.T) {
+	tc := paperRecallAtKTest()
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{"exact", "0.75", 1},
+		{"prose wrapped", "recall@6 is 0.75.", 1},
+		{"wrong: counted D9 as retrieved", "1.0", 0},
+		{"wrong: missed one hit", "0.5", 0},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}