test(catalog): expand research to 10 tests per subcategory

Add 9 tests to research/web, research/whitepapers, and
research/codebase, bringing each subcategory from 1 test to 10.

Web tests extract data from inline HTML, headers, and URLs, and
answer protocol scenarios for HTTP status codes, CORS, DNS, and
canonicalization. Every fixture stays inline; no test makes a live
web call.

Whitepaper tests each pair a 120 to 180 word, technically accurate
excerpt with numeric or JSON extraction questions. Topics cover
product quantization, Bloom filters, TF-IDF, recall@k, LSM-tree
write amplification, Raft quorum sizing, B-tree capacity, the CAP
theorem, and attention scaling. Every derived constant carries a
ground-truth comment and a unit test that recomputes the value with
an independent formula.

Codebase tests trace a Python function and a TypeScript function to
exact output, then reason over an inline diff, a data race, dead
code, Big-O complexity, an import cycle, a generic return type, and
a git-bisect scenario. The Python trace test runs the same source
through python3 when available and skips otherwise; the TypeScript
trace test cross-checks its hand-derivation against an equivalent Go
function.

New package-level helpers use a wp or code prefix to avoid name
collisions with sibling category authors sharing this package. Test
IDs keep the paper, web, and code prefixes already established by
the existing catalog entries. New tests omit MaxTokens so the runner
falls back to its default, per the catalog's authoring rules.

catalog_test.go's exact-map assertion now fails as expected, since
it still lists only the original 17-test catalog; that assertion is
owned centrally and is not touched here.

internal/tests/codebase.go

@@ -7,6 +7,15 @@ import (
 
 func registerCodebaseTests(r *testkit.Registry) {
 	r.Register(codeTraceGoTest())
+	r.Register(codeTracePythonTest())
+	r.Register(codeTraceTSTest())
+	r.Register(codeBugLineDiffTest())
+	r.Register(codeRaceVariableTest())
+	r.Register(codeDeadFunctionsTest())
+	r.Register(codeBigODedupTest())
+	r.Register(codeImportCycleTest())
+	r.Register(codeGenericReturnTypeTest())
+	r.Register(codeCommitBisectTest())
 }
 
 // mix is the traced function for codeTraceGoTest: deterministic bit

internal/tests/codebase_analysis.go

@@ -0,0 +1,304 @@
+package tests
+
+import (
+	"github.com/lukaszraczylo/llm-testbench/internal/eval"
+	"github.com/lukaszraczylo/llm-testbench/internal/testkit"
+)
+
+// codeBugDiffSnippet is a unified diff for codeBugLineDiffTest, with each
+// displayed line prefixed by its 1-based position in this listing (not the
+// diff's own hunk line numbers), so "which line" has one unambiguous
+// meaning.
+const codeBugDiffSnippet = `1: @@ -1,9 +1,9 @@
+2:  func Average(nums []float64) float64 {
+3:      if len(nums) == 0 {
+4:          return 0
+5:      }
+6:      sum := 0.0
+7: -    for _, n := range nums {
+8: +    for i := 0; i <= len(nums); i++ {
+9: -        sum += n
+10: +        sum += nums[i]
+11:      }
+12:      return sum / float64(len(nums))
+13:  }`
+
+// codeBugLineDiffTest: identify the root-cause line of an off-by-one
+// out-of-bounds bug in an inline, line-numbered diff.
+//
+// ground truth: line 8's new loop condition `i <= len(nums)` allows i to
+// reach len(nums), and line 10 then indexes nums[i] at that out-of-range
+// value, panicking. Line 8 is the root cause (the wrong bound); line 10 is
+// only where the panic actually fires - the correct bound is `i <
+// len(nums)`.
+func codeBugLineDiffTest() testkit.Test {
+	prompt := `Here is a diff modifying a Go function that computes an average. Each
+displayed line below is prefixed with its position in this listing (not
+the diff's own internal hunk numbering):
+
+` + codeBugDiffSnippet + `
+
+This change introduces a bug that causes a runtime panic: index out of
+range. Which line number contains the root cause of the panic - the
+incorrect loop bound - as opposed to the line where the panic actually
+fires? Respond with only a JSON object: {"line":<number>}`
+
+	return testkit.Test{
+		ID:          "code-bug-line-diff",
+		Category:    "research",
+		Subcategory: "codebase",
+		Description: "Identify the root-cause line of an off-by-one out-of-bounds bug in an inline, line-numbered diff.",
+		Prompt:      prompt,
+		Eval:        eval.JSONField("line", 8),
+	}
+}
+
+// codeRaceSnippet is an inline Go function for codeRaceVariableTest with a
+// classic unsynchronized shared-counter data race.
+const codeRaceSnippet = `func Count(n int) int {
+	total := 0
+	var wg sync.WaitGroup
+	for i := 0; i < n; i++ {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			total++
+		}()
+	}
+	wg.Wait()
+	return total
+}`
+
+// codeRaceVariableTest: name the variable with a data race in an inline Go
+// snippet.
+//
+// ground truth: every launched goroutine executes `total++` (a
+// read-modify-write) on the same shared `total` variable with no mutex,
+// atomic operation, or channel serializing access, so concurrent
+// increments race. `n` and `i` are each goroutine-local or loop-local and
+// are not raced on by this snippet.
+func codeRaceVariableTest() testkit.Test {
+	prompt := `Here is a Go function:
+
+` + "```go\n" + codeRaceSnippet + "\n```" + `
+
+Which single variable has a data race in this snippet - concurrent,
+unsynchronized reads and writes from multiple goroutines? Respond with
+only a JSON object: {"variable":"<name>"}`
+
+	return testkit.Test{
+		ID:          "code-race-variable",
+		Category:    "research",
+		Subcategory: "codebase",
+		Description: "Name the racy shared variable in an inline Go snippet with an unsynchronized concurrent counter.",
+		Prompt:      prompt,
+		Eval:        eval.JSONField("variable", "total"),
+	}
+}
+
+// codeDeadFunctionsSnippet is an inline Go file for codeDeadFunctionsTest
+// with two functions reachable from main and two that are never called.
+const codeDeadFunctionsSnippet = `package main
+
+import "fmt"
+
+func helperA() int { return 1 }
+func helperB() int { return helperA() + 1 }
+func helperC() int { return 3 }
+func helperD() int { return helperC() * 2 }
+
+func main() {
+	fmt.Println(helperB())
+}`
+
+// codeDeadFunctionsTest: identify functions unreachable from main in an
+// inline Go file.
+//
+// ground truth: main calls only helperB, which calls helperA - both are
+// reachable. Nothing calls helperD, and the only caller of helperC is
+// helperD itself, so neither helperC nor helperD is reachable from main;
+// both are dead code.
+func codeDeadFunctionsTest() testkit.Test {
+	prompt := `Here is a small Go file:
+
+` + "```go\n" + codeDeadFunctionsSnippet + "\n```" + `
+
+Which functions are dead code - never reachable, directly or transitively,
+from main? Respond with only a JSON array of function names.`
+
+	return testkit.Test{
+		ID:          "code-dead-functions",
+		Category:    "research",
+		Subcategory: "codebase",
+		Description: "Identify functions unreachable from main in an inline Go file with both live and dead call chains.",
+		Prompt:      prompt,
+		Eval:        eval.JSONStringSet([]string{"helperC", "helperD"}),
+	}
+}
+
+// codeDedupSnippet is an inline Go function for codeBigODedupTest with a
+// nested-loop worst-case-quadratic deduplication implementation.
+const codeDedupSnippet = `func Dedup(a []int) []int {
+	var out []int
+	for _, x := range a {
+		found := false
+		for _, y := range out {
+			if x == y {
+				found = true
+				break
+			}
+		}
+		if !found {
+			out = append(out, x)
+		}
+	}
+	return out
+}`
+
+// codeBigODedupPattern accepts every common notation for O(n^2): a caret,
+// a double-star, the unicode superscript two, or n*n, each optionally
+// wrapped as O(...) with flexible internal spacing.
+const codeBigODedupPattern = `(?i)o\(\s*n\s*(\^\s*2|\*\*\s*2|²|\*\s*n)\s*\)`
+
+// codeBigODedupTest: state the worst-case time complexity of an inline
+// nested-loop deduplication function.
+//
+// ground truth: when a has no duplicates, the inner loop scans the whole
+// of out, which grows to length n, for every one of the n outer
+// iterations, giving n + (n-1) + ... = O(n^2) comparisons in the worst
+// case.
+func codeBigODedupTest() testkit.Test {
+	prompt := `Here is a Go function:
+
+` + "```go\n" + codeDedupSnippet + "\n```" + `
+
+What is the worst-case time complexity of Dedup, in Big-O notation, in
+terms of n = len(a)? Respond with only the Big-O expression, e.g. O(n).`
+
+	return testkit.Test{
+		ID:          "code-bigo-dedup",
+		Category:    "research",
+		Subcategory: "codebase",
+		Description: "State the worst-case Big-O time complexity of an inline nested-loop deduplication function.",
+		Prompt:      prompt,
+		Eval:        eval.Regex(codeBigODedupPattern),
+	}
+}
+
+// codeImportListSnippet lists each inline package's direct imports for
+// codeImportCycleTest, with one cycle among four packages.
+const codeImportListSnippet = `- package a imports: [b]
+- package b imports: [c]
+- package c imports: [a, d]
+- package d imports: []`
+
+// codeImportCycleTest: detect which packages form an import cycle from an
+// inline list of per-package direct imports.
+//
+// ground truth: following the edges a->b->c->a closes a cycle among a, b,
+// and c. Package d is imported by c but does not import anything itself,
+// so it is not part of any cycle.
+func codeImportCycleTest() testkit.Test {
+	prompt := `Here are the direct imports of four packages:
+
+` + codeImportListSnippet + `
+
+Which packages form an import cycle? Respond with only a JSON array of
+package names involved in the cycle (any order).`
+
+	return testkit.Test{
+		ID:          "code-import-cycle",
+		Category:    "research",
+		Subcategory: "codebase",
+		Description: "Detect which packages form an import cycle from an inline list of per-package direct imports.",
+		Prompt:      prompt,
+		Eval:        eval.JSONStringSet([]string{"a", "b", "c"}),
+	}
+}
+
+// codeGenericFirstSnippet is an inline Go generic function plus a call
+// site for codeGenericReturnTypeTest.
+const codeGenericFirstSnippet = `func First[T any](s []T) (T, bool) {
+	if len(s) == 0 {
+		var zero T
+		return zero, false
+	}
+	return s[0], true
+}
+
+result, ok := First([]string{"a", "b"})`
+
+// codeGenericReturnTypeTest: infer the concrete instantiated type of a
+// generic function's result at a specific call site.
+//
+// ground truth: First is instantiated with T = string because it is
+// called with []string{"a", "b"}, so result's concrete type is string.
+func codeGenericReturnTypeTest() testkit.Test {
+	prompt := `Here is a generic Go function and a call site:
+
+` + "```go\n" + codeGenericFirstSnippet + "\n```" + `
+
+What is the concrete type of the variable "result" in the code above?
+Respond with only the type name, nothing else.`
+
+	return testkit.Test{
+		ID:          "code-generic-return-type",
+		Category:    "research",
+		Subcategory: "codebase",
+		Description: "Infer the concrete instantiated type of a generic Go function's result at a specific call site.",
+		Prompt:      prompt,
+		Eval:        codeExactAnswer("string"),
+	}
+}
+
+// codeBisectCommitLog is an inline commit history for
+// codeCommitBisectTest, where exactly one commit changes safeDivide's
+// result cache to be keyed only by "a" and checked before validation.
+const codeBisectCommitLog = `c1a1a1a "init: add safeDivide(a, b) that returns an error if b == 0,
+otherwise returns a / b"
+
+c2b2b2b "refactor: rewrite safeDivide's internal branch as a plain
+if/else, no behavior change"
+
+c3c3c3c "perf: add a result cache to safeDivide, keyed only by a (not by
+the pair a and b), checked BEFORE the b == 0 validation runs"
+
+c4d4d4d "chore: add debug logging around cache hits in safeDivide"
+
+c5e5e5e "docs: fix a typo in the cache-hit log message"`
+
+// codeCommitBisectTest: use git-bisect-style reasoning over an inline
+// commit log and a described symptom to name the exact commit that
+// introduced the bug.
+//
+// ground truth: c3c3c3c's cache is keyed only by the first argument a and
+// is checked before the b == 0 validation runs. A prior call
+// safeDivide(10, 2) caches a successful result under key a=10; a later
+// call safeDivide(10, 0) then hits that stale cache entry (same a, cache
+// ignores b) and returns it instead of reaching the validation that would
+// raise the b == 0 error. c1 predates the cache entirely; c2 is a
+// no-behavior-change refactor; c4 and c5 only touch logging text and do
+// not change the cache's key or check order, so the bug already existed
+// before either of them.
+func codeCommitBisectTest() testkit.Test {
+	prompt := `Here is a commit history for a function safeDivide(a, b), oldest first:
+
+` + codeBisectCommitLog + `
+
+Symptom observed in production after deploying through the latest commit:
+calling safeDivide(10, 2) returns 5 (correct). Immediately afterward,
+calling safeDivide(10, 0) returns 5 instead of raising the expected
+"b == 0" error.
+
+Using git-bisect-style reasoning, which commit introduced this bug?
+Respond with only the commit id (the 7-character id shown), nothing else.`
+
+	return testkit.Test{
+		ID:          "code-commit-bisect",
+		Category:    "research",
+		Subcategory: "codebase",
+		Description: "Use git-bisect-style reasoning over an inline commit log and a symptom to name the commit that introduced the bug.",
+		Prompt:      prompt,
+		Eval:        codeExactAnswer("c3c3c3c"),
+	}
+}

internal/tests/codebase_analysis_test.go

@@ -0,0 +1,173 @@
+package tests
+
+import (
+	"context"
+	"testing"
+)
+
+func TestCodeBugLineDiffTest_Eval(t *testing.T) {
+	tc := codeBugLineDiffTest()
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{"correct: root-cause line", `{"line":8}`, 1},
+		{"wrong: points at the panic site instead of the root cause", `{"line":10}`, 0},
+		{"wrong line entirely", `{"line":6}`, 0},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}
+
+func TestCodeRaceVariableTest_Eval(t *testing.T) {
+	tc := codeRaceVariableTest()
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{"correct", `{"variable":"total"}`, 1},
+		{"correct different case", `{"variable":"Total"}`, 1},
+		{"wrong: loop variable, not the shared counter", `{"variable":"i"}`, 0},
+		{"wrong: function parameter, not the shared counter", `{"variable":"n"}`, 0},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}
+
+func TestCodeDeadFunctionsTest_Eval(t *testing.T) {
+	tc := codeDeadFunctionsTest()
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{"correct set", `["helperC","helperD"]`, 1},
+		{"correct set, different order", `["helperD","helperC"]`, 1},
+		{"missing one dead function", `["helperD"]`, 0.5},
+		{"wrongly includes a live function", `["helperC","helperD","helperB"]`, 2.0 / 3.0},
+		{"wrong: says nothing is dead", `[]`, 0},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}
+
+func TestCodeBigODedupTest_Eval(t *testing.T) {
+	tc := codeBigODedupTest()
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{"caret form", "O(n^2)", 1},
+		{"double-star form", "O(n**2)", 1},
+		{"unicode superscript form", "O(n²)", 1},
+		{"n*n form", "O(n*n)", 1},
+		{"lowercase o", "o(n^2)", 1},
+		{"wrong: linear", "O(n)", 0},
+		{"wrong: linearithmic", "O(n log n)", 0},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}
+
+func TestCodeImportCycleTest_Eval(t *testing.T) {
+	tc := codeImportCycleTest()
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{"correct set", `["a","b","c"]`, 1},
+		{"correct set, different order", `["c","a","b"]`, 1},
+		{"wrongly includes non-cyclic d", `["a","b","c","d"]`, 0.75},
+		{"missing one cycle member", `["a","b"]`, 2.0 / 3.0},
+		{"wrong: only d, which has no imports at all", `["d"]`, 0},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}
+
+func TestCodeGenericReturnTypeTest_Eval(t *testing.T) {
+	tc := codeGenericReturnTypeTest()
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{"correct", "string", 1},
+		{"correct capitalized", "String", 1},
+		{"correct quoted", "`string`", 1},
+		{"wrong: the slice element pointer type", "*string", 0},
+		{"wrong: the generic parameter name itself, not its instantiation", "T", 0},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}
+
+func TestCodeCommitBisectTest_Eval(t *testing.T) {
+	tc := codeCommitBisectTest()
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{"correct", "c3c3c3c", 1},
+		{"correct with trailing period", "c3c3c3c.", 1},
+		{"wrong: the initial commit, predates the cache entirely", "c1a1a1a", 0},
+		{"wrong: only a logging change, no behavior change", "c4d4d4d", 0},
+		{"wrong: the docs-only commit", "c5e5e5e", 0},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}

internal/tests/codebase_trace.go

@@ -0,0 +1,127 @@
+package tests
+
+import (
+	"context"
+	"fmt"
+	"strconv"
+	"strings"
+
+	"github.com/lukaszraczylo/llm-testbench/internal/eval"
+	"github.com/lukaszraczylo/llm-testbench/internal/testkit"
+)
+
+// codeExactAnswer returns an Evaluator awarding full credit when the
+// response, trimmed of whitespace, at most one layer of surrounding quote
+// characters (' , ", or `), and a single trailing sentence-ending period,
+// equals want case-insensitively. This accepts every materially-correct
+// form of a forced single-token answer (bare, quoted, differently-cased,
+// or with trailing punctuation) without loosening the match to accept a
+// substring of a longer, wrong answer.
+func codeExactAnswer(want string) eval.Evaluator {
+	return eval.EvaluatorFunc(func(_ context.Context, response string) eval.Score {
+		got := strings.TrimSpace(response)
+		got = strings.Trim(got, "\"'`")
+		got = strings.TrimSuffix(strings.TrimSpace(got), ".")
+		got = strings.TrimSpace(got)
+		wantTrimmed := strings.TrimSpace(want)
+		if strings.EqualFold(got, wantTrimmed) {
+			return eval.Score{Value: 1, Detail: fmt.Sprintf("equals %q", wantTrimmed)}
+		}
+		return eval.Score{Value: 0, Detail: fmt.Sprintf("got %q, want %q", got, wantTrimmed)}
+	})
+}
+
+// codeTracePythonSource is the inline Python function traced by
+// codeTracePythonTest.
+const codeTracePythonSource = `def transform(s, n):
+    if n == 0:
+        return s
+    if len(s) % 2 == 0:
+        s = s[1:] + s[0]
+    else:
+        s = s[::-1]
+    return transform(s, n - 1)`
+
+// codeTracePythonWant is the exact result of transform("abcdef", 4).
+//
+// ground truth: len("abcdef") is 6 (even) at every recursive call, since
+// the rotate branch never changes length, so every call takes the rotate
+// branch: "abcdef"->"bcdefa"->"cdefab"->"defabc"->"efabcd" over 4 calls
+// (n=4,3,2,1), then n=0 returns "efabcd". codebase_trace_test.go both
+// hand-derives this independently and, when python3 is available, runs
+// the source above directly and compares stdout.
+const codeTracePythonWant = "efabcd"
+
+// codeTracePythonTest: trace a recursive Python function combining a
+// parity check, string rotation, and full reversal to its exact output.
+func codeTracePythonTest() testkit.Test {
+	prompt := `Here is a Python function:
+
+` + "```python\n" + codeTracePythonSource + "\n```" + `
+
+What is the exact return value of transform("abcdef", 4)? Respond with
+only the resulting string, with no quotes and no other text.`
+
+	return testkit.Test{
+		ID:          "code-trace-python",
+		Category:    "research",
+		Subcategory: "codebase",
+		Description: "Trace a recursive Python function combining a parity check, string rotation, and full reversal to its exact output.",
+		Prompt:      prompt,
+		Eval:        codeExactAnswer(codeTracePythonWant),
+	}
+}
+
+// codeTraceTSSource is the inline TypeScript function traced by
+// codeTraceTSTest.
+const codeTraceTSSource = `function pack(items: number[]): string {
+  return items.reduce((acc, x, i) => {
+    if (i % 2 === 0) return acc + x;
+    return acc + x * 2;
+  }, 0).toString(2);
+}`
+
+// codeTSPackTrace replicates pack's reduce logic in Go so
+// codebase_trace_test.go can cross-check the hand-derived trace by
+// running equivalent logic, since the TypeScript itself is not executed.
+func codeTSPackTrace(items []int) string {
+	acc := 0
+	for i, x := range items {
+		if i%2 == 0 {
+			acc += x
+		} else {
+			acc += x * 2
+		}
+	}
+	return strconv.FormatInt(int64(acc), 2)
+}
+
+// codeTraceTSWant is the exact result of pack([3,4,5,6,7]).
+//
+// ground truth: acc starts at 0; i=0,x=3 (even index) -> acc=3; i=1,x=4
+// (odd index) -> acc=3+4*2=11; i=2,x=5 -> acc=11+5=16; i=3,x=6 ->
+// acc=16+6*2=28; i=4,x=7 -> acc=28+7=35. 35 in binary is 100011
+// (32+2+1=35). codebase_trace_test.go recomputes this both by hand and by
+// calling codeTSPackTrace.
+const codeTraceTSWant = "100011"
+
+// codeTraceTSTest: trace a TypeScript reduce-based function mixing
+// index-parity branching and binary string conversion to its exact
+// output.
+func codeTraceTSTest() testkit.Test {
+	prompt := `Here is a TypeScript function:
+
+` + "```ts\n" + codeTraceTSSource + "\n```" + `
+
+What is the exact return value of pack([3, 4, 5, 6, 7])? Respond with only
+the resulting string, with no quotes and no other text.`
+
+	return testkit.Test{
+		ID:          "code-trace-ts",
+		Category:    "research",
+		Subcategory: "codebase",
+		Description: "Trace a TypeScript reduce-based function mixing index-parity branching and binary string conversion to its exact output.",
+		Prompt:      prompt,
+		Eval:        codeExactAnswer(codeTraceTSWant),
+	}
+}

internal/tests/codebase_trace_test.go

@@ -0,0 +1,146 @@
+package tests
+
+import (
+	"bytes"
+	"context"
+	"os"
+	"os/exec"
+	"strings"
+	"testing"
+)
+
+func TestCodeExactAnswer(t *testing.T) {
+	ev := codeExactAnswer("efabcd")
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{"bare correct", "efabcd", 1},
+		{"double-quoted correct", `"efabcd"`, 1},
+		{"different case", "EFABCD", 1},
+		{"trailing period", "efabcd.", 1},
+		{"wrong string", "abcdef", 0},
+		{"correct but truncated", "efabc", 0},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := ev.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}
+
+func TestCodeTracePythonWant_GroundTruth(t *testing.T) {
+	// Hand-derivation, double-checked: len("abcdef")=6 is even at every
+	// step, so every call takes the rotate branch (s[1:]+s[0]) rather than
+	// the reverse branch.
+	s := "abcdef"
+	for i := 0; i < 4; i++ {
+		if len(s)%2 == 0 {
+			s = s[1:] + string(s[0])
+		} else {
+			for l, r := 0, len(s)-1; l < r; l, r = l+1, r-1 {
+				b := []byte(s)
+				b[l], b[r] = b[r], b[l]
+				s = string(b)
+			}
+		}
+	}
+	if s != codeTracePythonWant {
+		t.Errorf("hand-derived trace = %q, want %q", s, codeTracePythonWant)
+	}
+
+	if _, err := exec.LookPath("python3"); err != nil {
+		t.Skip("python3 not found on PATH, skipping exec ground-truth cross-check")
+	}
+
+	script := codeTracePythonSource + "\nprint(transform(\"abcdef\", 4))\n"
+	// #nosec G204 -- script is a fixed, hardcoded string built from the
+	// same codeTracePythonSource constant embedded in the prompt, not
+	// external or response-controlled input.
+	cmd := exec.Command("python3", "-c", script) //nolint:gosec // see comment above
+	var stdout bytes.Buffer
+	cmd.Stdout = &stdout
+	cmd.Stderr = os.Stderr
+	if err := cmd.Run(); err != nil {
+		t.Fatalf("running python3 ground-truth script failed: %v", err)
+	}
+	got := strings.TrimSpace(stdout.String())
+	if got != codeTracePythonWant {
+		t.Errorf("python3-computed transform(\"abcdef\", 4) = %q, want %q", got, codeTracePythonWant)
+	}
+}
+
+func TestCodeTracePythonTest_Eval(t *testing.T) {
+	tc := codeTracePythonTest()
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{"correct", "efabcd", 1},
+		{"correct quoted", `"efabcd"`, 1},
+		{"wrong: forgot final rotation", "defabc", 0},
+		{"wrong: reversed instead of rotated", "fedcba", 0},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}
+
+func TestCodeTraceTSWant_GroundTruth(t *testing.T) {
+	// Hand-derivation, double-checked against codeTSPackTrace: acc=0;
+	// i=0,x=3 (even index) -> 3; i=1,x=4 (odd) -> 3+8=11; i=2,x=5 (even) ->
+	// 16; i=3,x=6 (odd) -> 16+12=28; i=4,x=7 (even) -> 35. 35 = 32+2+1 in
+	// binary is 100011.
+	acc := 0
+	items := []int{3, 4, 5, 6, 7}
+	for i, x := range items {
+		if i%2 == 0 {
+			acc += x
+		} else {
+			acc += x * 2
+		}
+	}
+	if acc != 35 {
+		t.Fatalf("hand-derived acc = %d, want 35", acc)
+	}
+
+	got := codeTSPackTrace(items)
+	if got != codeTraceTSWant {
+		t.Errorf("codeTSPackTrace(%v) = %q, want %q", items, got, codeTraceTSWant)
+	}
+}
+
+func TestCodeTraceTSTest_Eval(t *testing.T) {
+	tc := codeTraceTSTest()
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{"correct", "100011", 1},
+		{"correct quoted", `"100011"`, 1},
+		{"wrong: doubled every element instead of alternating", "1000110", 0},
+		{"wrong: decimal instead of binary", "35", 0},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}

internal/tests/web.go

@@ -7,6 +7,15 @@ import (
 
 func registerWebTests(r *testkit.Registry) {
 	r.Register(webRobotsAICrawlersTest())
+	r.Register(webHTMLProductExtractTest())
+	r.Register(webHreflangReciprocalTest())
+	r.Register(webURLParseComponentsTest())
+	r.Register(webSitemapMaxURLsTest())
+	r.Register(webHTTPStatusScenariosTest())
+	r.Register(webDNSMXRecordTest())
+	r.Register(webCanonicalVsRedirectTest())
+	r.Register(webCORSMissingHeaderTest())
+	r.Register(webSecurityHeadersAuditTest())
 }
 
 // webRobotsAICrawlersRobotsTxt is the inline robots.txt for

internal/tests/web_extract.go

@@ -0,0 +1,137 @@
+package tests
+
+import (
+	"github.com/lukaszraczylo/llm-testbench/internal/eval"
+	"github.com/lukaszraczylo/llm-testbench/internal/testkit"
+)
+
+// webProductCardHTML is the inline HTML product-card snippet for
+// webHTMLProductExtractTest.
+const webProductCardHTML = `<div class="product-card" data-sku="WBH-2201">
+  <h2 class="product-name">Wireless Mechanical Keyboard</h2>
+  <span class="price" data-currency="USD">129.99</span>
+  <span class="stock" data-available="true">In Stock</span>
+</div>`
+
+// webHTMLProductExtractTest: extract a product's name, price, and SKU
+// from inline HTML into JSON.
+//
+// ground truth: name is the h2.product-name text; price is the
+// span.price text as a number; sku is the data-sku attribute on the
+// outer div - all read directly from the markup, no computation.
+func webHTMLProductExtractTest() testkit.Test {
+	prompt := `Here is an HTML product card:
+
+` + "```html\n" + webProductCardHTML + "\n```" + `
+
+Extract the product's name, price (as a number), and SKU into JSON.
+Respond with only a JSON object: {"name":"...","price":<number>,"sku":"..."}`
+
+	evaluator := eval.Mean(
+		eval.JSONField("name", "Wireless Mechanical Keyboard"),
+		eval.JSONField("price", 129.99),
+		eval.JSONField("sku", "WBH-2201"),
+	)
+
+	return testkit.Test{
+		ID:          "web-html-product-extract",
+		Category:    "research",
+		Subcategory: "web",
+		Description: "Extract a product's name, price, and SKU from an inline HTML product card into JSON.",
+		Prompt:      prompt,
+		Eval:        evaluator,
+	}
+}
+
+// webHreflangPagesSnippet lists the hreflang <link> tags of three pages
+// for webHreflangReciprocalTest. /en/ links to both /es/ and /fr/, and
+// /es/ links back to both, but /fr/ omits the reciprocal link back to
+// /en/.
+const webHreflangPagesSnippet = `Page https://example.com/en/ <head> contains:
+<link rel="alternate" hreflang="en-us" href="https://example.com/en/" />
+<link rel="alternate" hreflang="es" href="https://example.com/es/" />
+<link rel="alternate" hreflang="fr" href="https://example.com/fr/" />
+
+Page https://example.com/es/ <head> contains:
+<link rel="alternate" hreflang="en-us" href="https://example.com/en/" />
+<link rel="alternate" hreflang="es" href="https://example.com/es/" />
+<link rel="alternate" hreflang="fr" href="https://example.com/fr/" />
+
+Page https://example.com/fr/ <head> contains:
+<link rel="alternate" hreflang="es" href="https://example.com/es/" />
+<link rel="alternate" hreflang="fr" href="https://example.com/fr/" />`
+
+// webHreflangReciprocalTest: find the one non-reciprocal hreflang link
+// among three inline pages.
+//
+// ground truth: /en/ declares an hreflang link to /fr/, but /fr/'s own
+// link set has no entry pointing back to https://example.com/en/, so the
+// pair (/en/, /fr/) is not reciprocal; /fr/ is the page missing the link,
+// and https://example.com/en/ is the target it is missing. Every other
+// pair in the three pages is fully reciprocal.
+func webHreflangReciprocalTest() testkit.Test {
+	prompt := `hreflang annotations must be reciprocal: if page A links to page B
+via hreflang, page B must also link back to page A. Here are the hreflang
+link tags for three pages:
+
+` + webHreflangPagesSnippet + `
+
+Exactly one pair among these three pages is broken (not reciprocal).
+Respond with only a JSON object:
+{"page_missing_link":"<url of the page that is missing a link>","missing_target":"<url that page should link to but does not>"}`
+
+	evaluator := eval.Mean(
+		eval.JSONField("page_missing_link", "https://example.com/fr/"),
+		eval.JSONField("missing_target", "https://example.com/en/"),
+	)
+
+	return testkit.Test{
+		ID:          "web-hreflang-reciprocal",
+		Category:    "research",
+		Subcategory: "web",
+		Description: "Find the one non-reciprocal hreflang link among three inline pages' link tag sets.",
+		Prompt:      prompt,
+		Eval:        evaluator,
+	}
+}
+
+// webParseURL is the inline URL for webURLParseComponentsTest.
+const webParseURL = "https://api.example.com:8443/v2/search?q=hello+world&sort=desc#results"
+
+// webURLParseComponentsTest: parse an inline URL into its component parts
+// as JSON.
+//
+// ground truth: computed directly from net/url.Parse in
+// web_extract_test.go, not hand-derived - scheme "https", host (hostname,
+// no port) "api.example.com", port "8443", path "/v2/search", raw query
+// "q=hello+world&sort=desc", fragment "results".
+func webURLParseComponentsTest() testkit.Test {
+	prompt := `Parse this URL into its components:
+
+` + webParseURL + `
+
+Respond with only a JSON object:
+{"scheme":"...","host":"...","port":<number>,"path":"...","query":"...","fragment":"..."}
+
+"host" must be the hostname only (no port). "port" must be a number.
+"query" must be the raw query string exactly as it appears after the "?",
+not split into individual parameters.`
+
+	evaluator := eval.Mean(
+		eval.JSONField("scheme", "https"),
+		eval.JSONField("host", "api.example.com"),
+		eval.JSONField("port", 8443),
+		eval.JSONField("path", "/v2/search"),
+		eval.JSONField("query", "q=hello+world&sort=desc"),
+		eval.JSONField("fragment", "results"),
+	)
+
+	return testkit.Test{
+		ID:          "web-url-parse-components",
+		Category:    "research",
+		Subcategory: "web",
+		Description: "Parse an inline URL into its scheme, host, port, path, query, and fragment components as JSON.",
+		Prompt:      prompt,
+		Eval:        evaluator,
+	}
+}

internal/tests/web_extract_test.go

@@ -0,0 +1,113 @@
+package tests
+
+import (
+	"context"
+	"net/url"
+	"testing"
+)
+
+func TestWebHTMLProductExtractTest_Eval(t *testing.T) {
+	tc := webHTMLProductExtractTest()
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{"all correct", `{"name":"Wireless Mechanical Keyboard","price":129.99,"sku":"WBH-2201"}`, 1},
+		{"one field wrong", `{"name":"Wireless Mechanical Keyboard","price":99.99,"sku":"WBH-2201"}`, 2.0 / 3.0},
+		{"all wrong", `{"name":"Wrong Product","price":1,"sku":"XXX"}`, 0},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}
+
+func TestWebHreflangReciprocalTest_Eval(t *testing.T) {
+	tc := webHreflangReciprocalTest()
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{"correct", `{"page_missing_link":"https://example.com/fr/","missing_target":"https://example.com/en/"}`, 1},
+		{"correct page, wrong target", `{"page_missing_link":"https://example.com/fr/","missing_target":"https://example.com/es/"}`, 0.5},
+		{"wrong page entirely", `{"page_missing_link":"https://example.com/es/","missing_target":"https://example.com/en/"}`, 0.5},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}
+
+func TestWebParseURL_GroundTruth(t *testing.T) {
+	// Independent re-derivation via net/url.Parse, not hand-derived, per
+	// the ground-truth discipline.
+	u, err := url.Parse(webParseURL)
+	if err != nil {
+		t.Fatalf("url.Parse(%q): %v", webParseURL, err)
+	}
+
+	if got, want := u.Scheme, "https"; got != want {
+		t.Errorf("Scheme = %q, want %q", got, want)
+	}
+	if got, want := u.Hostname(), "api.example.com"; got != want {
+		t.Errorf("Hostname() = %q, want %q", got, want)
+	}
+	if got, want := u.Port(), "8443"; got != want {
+		t.Errorf("Port() = %q, want %q", got, want)
+	}
+	if got, want := u.Path, "/v2/search"; got != want {
+		t.Errorf("Path = %q, want %q", got, want)
+	}
+	if got, want := u.RawQuery, "q=hello+world&sort=desc"; got != want {
+		t.Errorf("RawQuery = %q, want %q", got, want)
+	}
+	if got, want := u.Fragment, "results"; got != want {
+		t.Errorf("Fragment = %q, want %q", got, want)
+	}
+}
+
+func TestWebURLParseComponentsTest_Eval(t *testing.T) {
+	tc := webURLParseComponentsTest()
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{
+			"all correct",
+			`{"scheme":"https","host":"api.example.com","port":8443,"path":"/v2/search","query":"q=hello+world&sort=desc","fragment":"results"}`,
+			1,
+		},
+		{
+			"port included in host field is still wrong for host",
+			`{"scheme":"https","host":"api.example.com:8443","port":8443,"path":"/v2/search","query":"q=hello+world&sort=desc","fragment":"results"}`,
+			5.0 / 6.0,
+		},
+		{
+			"all wrong",
+			`{"scheme":"http","host":"wrong.example.com","port":80,"path":"/","query":"","fragment":""}`,
+			0,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}

internal/tests/web_protocol.go

@@ -0,0 +1,134 @@
+package tests
+
+import (
+	"github.com/lukaszraczylo/llm-testbench/internal/eval"
+	"github.com/lukaszraczylo/llm-testbench/internal/testkit"
+)
+
+// webSitemapMaxURLsTest: state the sitemaps.org protocol's maximum URL
+// count per sitemap file.
+//
+// ground truth: the sitemaps.org protocol caps a single sitemap file at
+// 50,000 URLs (and 50MB uncompressed); a file exceeding that must be
+// split across multiple sitemap files listed in a sitemap index.
+func webSitemapMaxURLsTest() testkit.Test {
+	prompt := `A site's sitemap.xml file lists 52,000 <url> entries inside a single
+<urlset>. Per the sitemaps.org protocol, what is the maximum number of
+URLs permitted in a single sitemap file (the limit this file violates)?
+Respond with only the number.`
+
+	return testkit.Test{
+		ID:          "web-sitemap-max-urls",
+		Category:    "research",
+		Subcategory: "web",
+		Description: "State the sitemaps.org protocol's maximum URL-per-file limit, applied to a scenario that exceeds it.",
+		Prompt:      prompt,
+		Eval:        eval.Numeric(eval.ExtractLastNumber[int], 50000, 0),
+	}
+}
+
+// webHTTPStatusScenarios lists four inline request-handling scenarios for
+// webHTTPStatusScenariosTest.
+const webHTTPStatusScenarios = `scenario1: "The client sent a request body that is not valid JSON."
+scenario2: "The client did not send any authentication credentials, and
+the requested resource requires a logged-in user."
+scenario3: "The client is authenticated, but their account does not have
+permission to access the requested resource."
+scenario4: "The resource has permanently moved to a new URL; the server
+wants clients (and search engines) to use the new URL from now on."`
+
+// webHTTPStatusScenariosTest: map 4 inline HTTP scenarios to their correct
+// status codes.
+//
+// ground truth: malformed request body -> 400 Bad Request; missing
+// credentials on a resource that requires them -> 401 Unauthorized;
+// authenticated but lacking permission -> 403 Forbidden; permanent
+// relocation -> 301 Moved Permanently.
+func webHTTPStatusScenariosTest() testkit.Test {
+	prompt := `For each of these 4 request-handling scenarios, give the single most
+correct HTTP status code:
+
+` + webHTTPStatusScenarios + `
+
+Respond with only a JSON object mapping each scenario id to its status
+code as a number: {"scenario1":<number>,"scenario2":<number>,"scenario3":<number>,"scenario4":<number>}`
+
+	evaluator := eval.Mean(
+		eval.JSONField("scenario1", 400),
+		eval.JSONField("scenario2", 401),
+		eval.JSONField("scenario3", 403),
+		eval.JSONField("scenario4", 301),
+	)
+
+	return testkit.Test{
+		ID:          "web-http-status-scenarios",
+		Category:    "research",
+		Subcategory: "web",
+		Description: "Map 4 inline request-handling scenarios to their correct HTTP status codes.",
+		Prompt:      prompt,
+		Eval:        evaluator,
+	}
+}
+
+// webDNSMXRecordPattern requires the literal token "MX" (word-bounded,
+// case-insensitive) anywhere in the response, materially correct whether
+// or not the response also says "record" or names the other record types
+// it is not.
+const webDNSMXRecordPattern = `(?i)\bMX\b`
+
+// webDNSMXRecordTest: name the DNS record type used to publish a domain's
+// prioritized mail servers.
+//
+// ground truth: mail-routing priority is published exclusively via MX
+// (Mail Exchange) records; no other standard record type carries a
+// priority value for mail delivery.
+func webDNSMXRecordTest() testkit.Test {
+	prompt := `example.com wants mail delivered to mailA.example.com as its primary
+mail server (priority 10), falling back to mailB.example.com (priority
+20) only if the primary is unreachable. Which single DNS record type
+publishes this prioritized mail-routing configuration? Respond with only
+the record type (e.g. A, AAAA, CNAME, MX, TXT, NS).`
+
+	return testkit.Test{
+		ID:          "web-dns-mx-record",
+		Category:    "research",
+		Subcategory: "web",
+		Description: "Name the DNS record type used to publish a domain's prioritized mail servers.",
+		Prompt:      prompt,
+		Eval:        eval.Regex(webDNSMXRecordPattern),
+	}
+}
+
+// webCanonicalVsRedirectTest: choose between a canonical tag and a 301
+// redirect for a duplicate-content scenario where the duplicate URLs must
+// stay independently reachable.
+//
+// ground truth: a 301 redirect would break the tracking links (they would
+// stop resolving to their own URL), so the correct fix is a canonical
+// tag: it consolidates ranking signals onto the preferred URL while
+// leaving every tracked variant independently reachable.
+func webCanonicalVsRedirectTest() testkit.Test {
+	prompt := `An e-commerce site serves identical product content at these three
+URLs, which must all keep working since they are used as tracking links
+in email and social campaigns:
+
+https://example.com/product?id=123
+https://example.com/product?id=123&ref=email
+https://example.com/product?id=123&ref=social
+
+Search engines are treating these as duplicate content and splitting
+ranking signal between them. You need to consolidate that ranking signal
+onto https://example.com/product?id=123, while keeping all three URLs
+independently reachable for the tracking links to keep working. Should
+you use a canonical tag or a 301 redirect? Respond with only one word:
+"canonical" or "redirect".`
+
+	return testkit.Test{
+		ID:          "web-canonical-vs-redirect",
+		Category:    "research",
+		Subcategory: "web",
+		Description: "Choose canonical tag over 301 redirect for a duplicate-content scenario requiring all URL variants to stay reachable.",
+		Prompt:      prompt,
+		Eval:        eval.Regex(`(?i)^\s*canonical\.?\s*$`),
+	}
+}

internal/tests/web_protocol_test.go

@@ -0,0 +1,98 @@
+package tests
+
+import (
+	"context"
+	"testing"
+)
+
+func TestWebSitemapMaxURLsTest_Eval(t *testing.T) {
+	tc := webSitemapMaxURLsTest()
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{"correct", "50000", 1},
+		{"prose wrapped", "The limit is 50000 URLs per sitemap file.", 1},
+		{"wrong: confused with the uncompressed size limit in MB", "50", 0},
+		{"wrong entirely", "10000", 0},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}
+
+func TestWebHTTPStatusScenariosTest_Eval(t *testing.T) {
+	tc := webHTTPStatusScenariosTest()
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{"all correct", `{"scenario1":400,"scenario2":401,"scenario3":403,"scenario4":301}`, 1},
+		{"swapped 401 and 403", `{"scenario1":400,"scenario2":403,"scenario3":401,"scenario4":301}`, 0.5},
+		{"used 500 for a client error", `{"scenario1":500,"scenario2":401,"scenario3":403,"scenario4":301}`, 0.75},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}
+
+func TestWebDNSMXRecordTest_Eval(t *testing.T) {
+	tc := webDNSMXRecordTest()
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{"bare", "MX", 1},
+		{"lowercase", "mx", 1},
+		{"with the word record", "MX record", 1},
+		{"wrong: CNAME", "CNAME", 0},
+		{"wrong: A record", "A", 0},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}
+
+func TestWebCanonicalVsRedirectTest_Eval(t *testing.T) {
+	tc := webCanonicalVsRedirectTest()
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{"correct lowercase", "canonical", 1},
+		{"correct capitalized with period", "Canonical.", 1},
+		{"wrong: redirect", "redirect", 0},
+		{"wrong: extra words break the forced one-word format", "Use a canonical tag", 0},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}

internal/tests/web_security.go

@@ -0,0 +1,97 @@
+package tests
+
+import (
+	"github.com/lukaszraczylo/llm-testbench/internal/eval"
+	"github.com/lukaszraczylo/llm-testbench/internal/testkit"
+)
+
+// webCORSPreflightExchange is the inline preflight request/response
+// exchange for webCORSMissingHeaderTest.
+const webCORSPreflightExchange = `Browser preflight request (from https://app.example.com):
+OPTIONS /data HTTP/1.1
+Host: api.example.com
+Origin: https://app.example.com
+Access-Control-Request-Method: PUT
+Access-Control-Request-Headers: X-Custom-Auth
+
+Server's preflight response:
+HTTP/1.1 204 No Content
+Access-Control-Allow-Origin: https://app.example.com
+Access-Control-Allow-Methods: GET, POST, PUT`
+
+// webCORSMissingHeaderPattern requires the literal token
+// "Access-Control-Allow-Headers" (case-insensitive) anywhere in the
+// response, accepting it with or without a trailing colon or surrounding
+// prose.
+const webCORSMissingHeaderPattern = `(?i)access-control-allow-headers`
+
+// webCORSMissingHeaderTest: identify the single missing CORS response
+// header that will cause the browser to block the request.
+//
+// ground truth: the browser sent Access-Control-Request-Headers:
+// X-Custom-Auth in the preflight, but the server's response never echoes
+// that header name back in an Access-Control-Allow-Headers response
+// header, so the browser blocks the actual PUT request even though the
+// origin and method are both explicitly allowed.
+func webCORSMissingHeaderTest() testkit.Test {
+	prompt := `Here is a CORS preflight exchange:
+
+` + webCORSPreflightExchange + `
+
+The browser will block the actual PUT request that follows this
+preflight. Which single response header is missing from the server's
+preflight response, causing the block? Respond with only the header name,
+nothing else.`
+
+	return testkit.Test{
+		ID:          "web-cors-missing-header",
+		Category:    "research",
+		Subcategory: "web",
+		Description: "Identify the single missing CORS response header that causes a browser to block a preflighted request.",
+		Prompt:      prompt,
+		Eval:        eval.Regex(webCORSMissingHeaderPattern),
+	}
+}
+
+// webSecurityHeadersResponse is the inline HTTP response header block for
+// webSecurityHeadersAuditTest.
+const webSecurityHeadersResponse = `HTTP/1.1 200 OK
+Content-Type: text/html; charset=utf-8
+Cache-Control: no-cache
+Set-Cookie: session=abc123; HttpOnly
+Server: nginx
+X-Content-Type-Options: nosniff
+Referrer-Policy: strict-origin-when-cross-origin`
+
+// webSecurityHeadersBaseline is the recommended baseline of security
+// headers checked against webSecurityHeadersResponse.
+const webSecurityHeadersBaseline = `Content-Security-Policy, Strict-Transport-Security,
+X-Content-Type-Options, X-Frame-Options, Referrer-Policy`
+
+// webSecurityHeadersAuditTest: identify which baseline security headers
+// are missing from an inline HTTP response.
+//
+// ground truth: of the 5 baseline headers, X-Content-Type-Options and
+// Referrer-Policy are present in the response; Content-Security-Policy,
+// Strict-Transport-Security, and X-Frame-Options are absent.
+func webSecurityHeadersAuditTest() testkit.Test {
+	prompt := `Here is an HTTP response's headers:
+
+` + webSecurityHeadersResponse + `
+
+Here is the recommended baseline of security headers:
+
+` + webSecurityHeadersBaseline + `
+
+Which of the baseline headers are missing from the response above?
+Respond with only a JSON array of the missing header names.`
+
+	return testkit.Test{
+		ID:          "web-security-headers-audit",
+		Category:    "research",
+		Subcategory: "web",
+		Description: "Identify which baseline security headers are missing from an inline HTTP response, given a partial-compliance fixture.",
+		Prompt:      prompt,
+		Eval:        eval.JSONStringSet([]string{"Content-Security-Policy", "Strict-Transport-Security", "X-Frame-Options"}),
+	}
+}

internal/tests/web_security_test.go

@@ -0,0 +1,54 @@
+package tests
+
+import (
+	"context"
+	"testing"
+)
+
+func TestWebCORSMissingHeaderTest_Eval(t *testing.T) {
+	tc := webCORSMissingHeaderTest()
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{"bare header name", "Access-Control-Allow-Headers", 1},
+		{"with a colon", "Access-Control-Allow-Headers:", 1},
+		{"in a sentence", "The missing header is Access-Control-Allow-Headers.", 1},
+		{"wrong: names an already-present header", "Access-Control-Allow-Origin", 0},
+		{"wrong: names an already-present header", "Access-Control-Allow-Methods", 0},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}
+
+func TestWebSecurityHeadersAuditTest_Eval(t *testing.T) {
+	tc := webSecurityHeadersAuditTest()
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{"correct set", `["Content-Security-Policy","Strict-Transport-Security","X-Frame-Options"]`, 1},
+		{"correct set, different order", `["X-Frame-Options","Content-Security-Policy","Strict-Transport-Security"]`, 1},
+		{"missing one", `["Content-Security-Policy","Strict-Transport-Security"]`, 2.0 / 3.0},
+		{"wrongly includes a present header", `["Content-Security-Policy","Strict-Transport-Security","X-Frame-Options","Referrer-Policy"]`, 0.75},
+		{"wrong: says nothing is missing", `[]`, 0},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}

internal/tests/whitepapers.go

@@ -7,6 +7,15 @@ import (
 
 func registerWhitepapersTests(r *testkit.Registry) {
 	r.Register(paperHnswParamsTest())
+	r.Register(paperPQCompressionRatioTest())
+	r.Register(paperBloomFPRateTest())
+	r.Register(paperTFIDFTermScoreTest())
+	r.Register(paperRecallAtKTest())
+	r.Register(paperLSMWriteAmplificationTest())
+	r.Register(paperRaftQuorumFailuresTest())
+	r.Register(paperBTreeHeightLevelsTest())
+	r.Register(paperCAPAvailabilityChoiceTest())
+	r.Register(paperAttentionScaleFactorTest())
 }
 
 // paperHnswParamsExcerpt is a ~150-word, technically accurate original

internal/tests/whitepapers_compute.go

@@ -0,0 +1,105 @@
+package tests
+
+import (
+	"math"
+	"strings"
+)
+
+// wpQuantizedCodeBytes returns the number of bytes needed to store one
+// product-quantization code: m subvectors, each an nbits-bit centroid
+// index, packed with no wasted bits. It assumes m*nbits is a multiple of
+// 8, true for every configuration used in this catalog.
+func wpQuantizedCodeBytes(m, nbits int) int {
+	return m * nbits / 8
+}
+
+// wpBloomFalsePositiveRate computes the standard Bloom filter
+// false-positive probability p = (1 - e^(-k*n/m))^k for an m-bit filter
+// holding n inserted elements and k hash functions.
+func wpBloomFalsePositiveRate(m, n, k int) float64 {
+	exponent := -float64(k) * float64(n) / float64(m)
+	return math.Pow(1-math.Exp(exponent), float64(k))
+}
+
+// wpTermFrequency counts how many times term appears in doc, splitting on
+// whitespace (doc is pre-tokenized) and comparing case-insensitively.
+func wpTermFrequency(term string, doc []string) int {
+	count := 0
+	for _, w := range doc {
+		if strings.EqualFold(w, term) {
+			count++
+		}
+	}
+	return count
+}
+
+// wpDocumentFrequency counts how many of docs contain term at least once.
+func wpDocumentFrequency(term string, docs [][]string) int {
+	count := 0
+	for _, doc := range docs {
+		if wpTermFrequency(term, doc) > 0 {
+			count++
+		}
+	}
+	return count
+}
+
+// wpTFIDF computes the plain tf * ln(N/df) weight: tf is the raw term
+// count in one document, df is the number of documents containing the
+// term, and n is the total document count. No smoothing is applied,
+// matching the formula stated in the wp-tfidf-term-score excerpt.
+func wpTFIDF(tf, df, n int) float64 {
+	return float64(tf) * math.Log(float64(n)/float64(df))
+}
+
+// wpRecallAtK computes the standard recall@k retrieval metric: the
+// fraction of relevant that appears anywhere in retrievedTopK, regardless
+// of rank.
+func wpRecallAtK(relevant, retrievedTopK []string) float64 {
+	retrievedSet := make(map[string]struct{}, len(retrievedTopK))
+	for _, id := range retrievedTopK {
+		retrievedSet[id] = struct{}{}
+	}
+	hit := 0
+	for _, id := range relevant {
+		if _, ok := retrievedSet[id]; ok {
+			hit++
+		}
+	}
+	return float64(hit) / float64(len(relevant))
+}
+
+// wpLSMWriteAmplification returns the write amplification factor for a
+// byte that is rewritten rewriteCount times by compaction after its
+// original flush write: the original write plus every rewrite, divided by
+// the single original write.
+func wpLSMWriteAmplification(rewriteCount int) int {
+	return 1 + rewriteCount
+}
+
+// wpRaftQuorumSize returns the minimum number of votes/acknowledgments
+// needed for a strict majority of an n-node Raft cluster: floor(n/2)+1.
+func wpRaftQuorumSize(n int) int {
+	return n/2 + 1
+}
+
+// wpRaftMaxTolerableFailures returns how many simultaneous node failures
+// an n-node Raft cluster can sustain while a majority quorum, sized
+// relative to the full n-node membership, remains reachable.
+func wpRaftMaxTolerableFailures(n int) int {
+	return n - wpRaftQuorumSize(n)
+}
+
+// wpBTreeMinBranchingLevels returns the smallest h (root counted as h=0)
+// such that a tree with minimum fanout f can index at least n leaf key
+// slots, per the f^(h+1) >= n bound stated in the wp-btree-height-levels
+// excerpt.
+func wpBTreeMinBranchingLevels(f, n int) int {
+	h := 0
+	capacity := f
+	for capacity < n {
+		capacity *= f
+		h++
+	}
+	return h
+}

internal/tests/whitepapers_systems.go

@@ -0,0 +1,267 @@
+package tests
+
+import (
+	"github.com/lukaszraczylo/llm-testbench/internal/eval"
+	"github.com/lukaszraczylo/llm-testbench/internal/testkit"
+)
+
+// paperLSMExcerpt is a ~152-word, technically accurate original
+// description of LSM-tree leveled compaction and write amplification
+// written for this test, pinning a concrete 4-level rewrite path.
+const paperLSMExcerpt = `An LSM-tree buffers writes in memory and periodically flushes them as
+immutable sorted files, then merges those files together in the background
+through a process called compaction, so reads do not have to check an
+unbounded number of files. In leveled compaction, each level on disk is
+merged into the next level down, and every byte that a compaction step
+touches gets rewritten to a new file even though its value did not change.
+A storage engine described in this report uses four levels, L0 through L3.
+A byte first flushed into L0 is compacted into L1, then later compacted
+from L1 into L2, and finally from L2 into L3 before it reaches its
+long-term resting level, meaning it is rewritten three separate times after
+its original flush write. Write amplification here is defined as total
+bytes written to disk, including every rewrite, divided by the bytes of
+user data originally written.`
+
+// paperLSMWriteAmpWant is derived by calling wpLSMWriteAmplification, not
+// hardcoded.
+//
+// ground truth: the excerpt states a byte is rewritten 3 times (L0->L1,
+// L1->L2, L2->L3) after its original flush write, so total disk writes per
+// user byte = 1 (original) + 3 (rewrites) = 4.
+// whitepapers_systems_test.go independently recomputes this by counting
+// the rewrite path described in the excerpt text.
+var paperLSMWriteAmpWant = wpLSMWriteAmplification(3)
+
+// paperLSMWriteAmplificationTest: derive an LSM-tree's write amplification
+// factor from an inline excerpt describing a byte's full compaction path.
+func paperLSMWriteAmplificationTest() testkit.Test {
+	prompt := `Read this excerpt about LSM-tree compaction:
+
+` + paperLSMExcerpt + `
+
+Using the byte's full path through this system (an L0 flush, then
+compaction into L1, then L2, then L3) and the write-amplification
+definition given above, what is the write amplification factor for that
+byte? Respond with only the number.`
+
+	return testkit.Test{
+		ID:          "paper-lsm-write-amplification",
+		Category:    "research",
+		Subcategory: "whitepapers",
+		Description: "Derive an LSM-tree's write amplification factor from an inline excerpt describing a byte's full leveled-compaction path.",
+		Prompt:      prompt,
+		Eval:        eval.Numeric(eval.ExtractLastNumber[int], paperLSMWriteAmpWant, 0),
+	}
+}
+
+// paperRaftExcerpt is a ~155-word, technically accurate original
+// description of Raft's majority-quorum requirement written for this
+// test, pinning a 5-node cluster.
+const paperRaftExcerpt = `Raft is a consensus protocol that keeps a replicated log consistent across
+a cluster of nodes by electing a single leader for each term and having
+that leader replicate every log entry to the other nodes. A node starts an
+election after its randomized election timeout, typically drawn from the
+150 to 300 millisecond range in the original paper, expires without
+hearing from a leader. A candidate becomes leader only once it collects
+votes from a strict majority of all nodes in the cluster, not merely a
+majority of the nodes currently reachable, and the same majority rule
+governs when a log entry counts as committed. This report deployed Raft
+across a cluster of N equal to 5 nodes, so both leader election and log
+commitment require votes or acknowledgments from a majority quorum sized
+relative to that total membership of 5, not to however many nodes happen to
+be healthy at a given moment.`
+
+// paperRaftQuorumWant and paperRaftFailuresWant are derived by calling
+// wpRaftQuorumSize and wpRaftMaxTolerableFailures, not hardcoded.
+//
+// ground truth: quorum = floor(5/2)+1 = 3; a 5-node cluster can lose at
+// most 5-3 = 2 nodes and still reach that quorum.
+// whitepapers_systems_test.go independently recomputes both for several N.
+var (
+	paperRaftQuorumWant   = wpRaftQuorumSize(5)
+	paperRaftFailuresWant = wpRaftMaxTolerableFailures(5)
+)
+
+// paperRaftQuorumFailuresTest: derive a Raft cluster's majority quorum
+// size and maximum tolerable simultaneous failures from an inline excerpt
+// and a stated cluster size.
+func paperRaftQuorumFailuresTest() testkit.Test {
+	prompt := `Read this excerpt about the Raft consensus protocol:
+
+` + paperRaftExcerpt + `
+
+For this N=5 node cluster:
+1. What is the minimum number of votes/acknowledgments needed for a
+   majority quorum?
+2. How many simultaneous node failures can the cluster sustain and still
+   reach that quorum?
+
+Respond with only a JSON object: {"quorum_size":<number>,"max_tolerable_failures":<number>}`
+
+	evaluator := eval.Mean(
+		eval.JSONField("quorum_size", paperRaftQuorumWant),
+		eval.JSONField("max_tolerable_failures", paperRaftFailuresWant),
+	)
+
+	return testkit.Test{
+		ID:          "paper-raft-quorum-failures",
+		Category:    "research",
+		Subcategory: "whitepapers",
+		Description: "Derive a 5-node Raft cluster's majority quorum size and maximum tolerable simultaneous failures from an inline excerpt.",
+		Prompt:      prompt,
+		Eval:        evaluator,
+	}
+}
+
+// paperBTreeExcerpt is a ~146-word, technically accurate original
+// description of a worst-case (minimum-fanout) B-tree capacity model
+// written for this test, pinning order m and a target key count.
+const paperBTreeExcerpt = `A B-tree of order m allows every internal node between ceil(m/2) and m
+children, keeping the tree shallow and well balanced even as it grows,
+which bounds the number of disk reads a lookup needs to at most one per
+level. For capacity planning this report models the worst case: a tree
+where every internal node has the minimum allowed fanout, f equals
+ceil(m/2), rather than the maximum. With that minimum fanout, and counting
+the root itself as branching level h equal to zero, a tree with h
+branching levels can index at most f to the power of (h+1) leaf key slots,
+since each additional level below the root multiplies the reachable leaf
+count by f. This benchmark used order m equal to 200, giving a minimum
+fanout f equal to 100, and needed to index at least 5,000,000 keys without
+exceeding that capacity bound.`
+
+// paperBTreeHeightWant is derived by calling wpBTreeMinBranchingLevels,
+// not hardcoded.
+//
+// ground truth: smallest h with 100^(h+1) >= 5,000,000: h=0 -> 100;
+// h=1 -> 10,000; h=2 -> 1,000,000 (still short); h=3 -> 100,000,000
+// (first to clear 5,000,000). whitepapers_systems_test.go independently
+// recomputes this with a plain loop over math.Pow.
+var paperBTreeHeightWant = wpBTreeMinBranchingLevels(100, 5000000)
+
+// paperBTreeHeightLevelsTest: derive the minimum number of branching
+// levels a worst-case B-tree needs to index a target key count, from an
+// inline excerpt's stated capacity formula.
+func paperBTreeHeightLevelsTest() testkit.Test {
+	prompt := `Read this excerpt about B-tree capacity planning:
+
+` + paperBTreeExcerpt + `
+
+Using the formula given above (f^(h+1) >= n keys, root counted as h=0),
+what is the minimum number of branching levels h needed so this tree can
+index at least 5,000,000 keys with f=100? Respond with only the number.`
+
+	return testkit.Test{
+		ID:          "paper-btree-height-levels",
+		Category:    "research",
+		Subcategory: "whitepapers",
+		Description: "Derive the minimum branching-level count a worst-case-fanout B-tree needs to index a target key count.",
+		Prompt:      prompt,
+		Eval:        eval.Numeric(eval.ExtractLastNumber[int], paperBTreeHeightWant, 0),
+	}
+}
+
+// paperCAPExcerpt is a ~135-word, technically accurate original
+// description of the CAP theorem written for this test, describing a
+// scenario that unambiguously chooses availability over consistency.
+const paperCAPExcerpt = `The CAP theorem states that a distributed data store can provide at most
+two of three guarantees during a network partition: consistency, meaning
+every read sees the most recent write or an error; availability, meaning
+every request receives a non-error response; and partition tolerance,
+meaning the system keeps operating despite dropped or delayed messages
+between nodes. Because network partitions are unavoidable in any real
+distributed deployment, partition tolerance is treated as a fixed
+requirement rather than a design choice, leaving consistency and
+availability as the actual trade-off a system architect makes. A key-value
+store described in this report was explicitly designed so that, during a
+partition between two data centers, both sides keep accepting writes
+independently rather than refusing requests, accepting that the two sides
+may briefly disagree and need reconciliation once the partition heals.`
+
+// paperCAPAvailabilityPattern anchors on the trimmed response being
+// exactly the single word "availability" (case-insensitive, with an
+// optional trailing period), accepting every materially-correct form of
+// the forced one-word answer without also matching a response that names
+// the wrong guarantee.
+const paperCAPAvailabilityPattern = `(?i)^\s*availability\.?\s*$`
+
+// paperCAPAvailabilityChoiceTest: identify, from an inline CAP-theorem
+// excerpt and a described design decision, which guarantee (consistency or
+// availability) the system prioritizes during a partition.
+func paperCAPAvailabilityChoiceTest() testkit.Test {
+	prompt := `Read this excerpt about the CAP theorem:
+
+` + paperCAPExcerpt + `
+
+Given this design decision - both data centers keep accepting writes
+during the partition rather than refusing requests - which of the two
+trade-off guarantees does this system prioritize during a partition, per
+the CAP theorem? Respond with only one word: "Consistency" or
+"Availability".`
+
+	return testkit.Test{
+		ID:          "paper-cap-availability-choice",
+		Category:    "research",
+		Subcategory: "whitepapers",
+		Description: "Identify which CAP-theorem guarantee a described partition-handling design decision prioritizes.",
+		Prompt:      prompt,
+		Eval:        eval.Regex(paperCAPAvailabilityPattern),
+	}
+}
+
+// paperAttentionExcerpt is a ~158-word, technically accurate original
+// description of scaled dot-product attention's sqrt(d_k) scaling written
+// for this test, pinning a concrete per-head dimension.
+const paperAttentionExcerpt = `Scaled dot-product attention computes a weighted average of value vectors,
+where the weight given to each value comes from a softmax over the dot
+products between one query vector and every key vector. When query and key
+entries are drawn independently with roughly unit variance, the variance of
+their dot product grows in proportion to the dimensionality d_k of those
+vectors, so without correction, larger d_k pushes the raw dot products to
+extreme magnitudes. Feeding those extreme values into softmax drives most
+of the output probability mass onto one entry and squashes the gradient
+almost to zero everywhere else, stalling training. To counteract this, the
+dot products are divided by the square root of d_k before the softmax,
+which rescales their variance back down to roughly one regardless of
+dimensionality. A benchmark model in this report used d_k equal to 64 per
+attention head, so this scaling divided every raw dot product by the square
+root of 64.`
+
+// paperAttentionDK and paperAttentionScaleFactorWant pin the excerpt's
+// stated d_k and its square root.
+//
+// ground truth: d_k = 64 (stated verbatim in the excerpt); scale factor =
+// sqrt(64) = 8. whitepapers_systems_test.go independently recomputes the
+// scale factor with math.Sqrt.
+const (
+	paperAttentionDK              = 64
+	paperAttentionScaleFactorWant = 8
+)
+
+// paperAttentionScaleFactorTest: extract d_k and derive the sqrt(d_k)
+// attention scale factor from an inline excerpt.
+func paperAttentionScaleFactorTest() testkit.Test {
+	prompt := `Read this excerpt about scaled dot-product attention:
+
+` + paperAttentionExcerpt + `
+
+For the benchmark model described:
+1. What is d_k?
+2. By what number are the raw dot products divided before the softmax
+   (the scale factor)?
+
+Respond with only a JSON object: {"d_k":<number>,"scale_factor":<number>}`
+
+	evaluator := eval.Mean(
+		eval.JSONField("d_k", paperAttentionDK),
+		eval.JSONField("scale_factor", paperAttentionScaleFactorWant),
+	)
+
+	return testkit.Test{
+		ID:          "paper-attention-scale-factor",
+		Category:    "research",
+		Subcategory: "whitepapers",
+		Description: "Extract d_k and derive the sqrt(d_k) attention scale factor from an inline excerpt.",
+		Prompt:      prompt,
+		Eval:        evaluator,
+	}
+}

internal/tests/whitepapers_systems_test.go

@@ -0,0 +1,203 @@
+package tests
+
+import (
+	"context"
+	"math"
+	"strings"
+	"testing"
+)
+
+func TestPaperLSMWriteAmpWant_GroundTruth(t *testing.T) {
+	// Independent re-derivation by counting the excerpt's stated rewrite
+	// path (L0->L1, L1->L2, L2->L3 = 3 rewrites) plus the original write,
+	// not via wpLSMWriteAmplification.
+	// The excerpt names exactly three compaction moves; check each phrase is
+	// present rather than counting "into L" substrings, since the initial
+	// "flushed into L0" also contains that substring without being a
+	// compaction rewrite.
+	moves := []string{"compacted into L1", "L1 into L2", "L2 into L3"}
+	rewrites := 0
+	for _, phrase := range moves {
+		if strings.Contains(paperLSMExcerpt, phrase) {
+			rewrites++
+		}
+	}
+	if rewrites != 3 {
+		t.Fatalf("excerpt describes %d of the 3 expected compaction moves", rewrites)
+	}
+	want := 1 + rewrites
+
+	if want != 4 {
+		t.Fatalf("independently recomputed write amplification = %d, want 4", want)
+	}
+	if paperLSMWriteAmpWant != want {
+		t.Errorf("paperLSMWriteAmpWant = %d, independently recomputed = %d", paperLSMWriteAmpWant, want)
+	}
+}
+
+func TestPaperLSMWriteAmplificationTest_Eval(t *testing.T) {
+	tc := paperLSMWriteAmplificationTest()
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{"correct", "4", 1},
+		{"prose wrapped", "The write amplification factor is 4.", 1},
+		{"forgot the original write", "3", 0},
+		{"off by one", "5", 0},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}
+
+func TestPaperRaftQuorumWant_GroundTruth(t *testing.T) {
+	// Independent re-derivation of floor(n/2)+1 and n-quorum for several N,
+	// not via wpRaftQuorumSize/wpRaftMaxTolerableFailures.
+	tests := []struct {
+		n            int
+		wantQuorum   int
+		wantFailures int
+	}{
+		{3, 2, 1},
+		{5, 3, 2},
+		{7, 4, 3},
+	}
+	for _, tt := range tests {
+		gotQuorum := tt.n/2 + 1
+		gotFailures := tt.n - gotQuorum
+		if gotQuorum != tt.wantQuorum {
+			t.Errorf("quorum(n=%d) = %d, want %d", tt.n, gotQuorum, tt.wantQuorum)
+		}
+		if gotFailures != tt.wantFailures {
+			t.Errorf("maxFailures(n=%d) = %d, want %d", tt.n, gotFailures, tt.wantFailures)
+		}
+	}
+
+	if paperRaftQuorumWant != 3 {
+		t.Errorf("paperRaftQuorumWant = %d, want 3", paperRaftQuorumWant)
+	}
+	if paperRaftFailuresWant != 2 {
+		t.Errorf("paperRaftFailuresWant = %d, want 2", paperRaftFailuresWant)
+	}
+}
+
+func TestPaperRaftQuorumFailuresTest_Eval(t *testing.T) {
+	tc := paperRaftQuorumFailuresTest()
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{"all correct", `{"quorum_size":3,"max_tolerable_failures":2}`, 1},
+		{"one field wrong", `{"quorum_size":3,"max_tolerable_failures":3}`, 0.5},
+		{"both wrong: used simple majority of reachable nodes", `{"quorum_size":2,"max_tolerable_failures":1}`, 0},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}
+
+func TestPaperBTreeHeightWant_GroundTruth(t *testing.T) {
+	// Independent re-derivation with math.Pow, not via
+	// wpBTreeMinBranchingLevels's iterative loop.
+	const f, n = 100.0, 5000000.0
+	h := 0
+	for math.Pow(f, float64(h+1)) < n {
+		h++
+	}
+
+	if h != 3 {
+		t.Fatalf("independently recomputed h = %d, want 3", h)
+	}
+	if paperBTreeHeightWant != h {
+		t.Errorf("paperBTreeHeightWant = %d, independently recomputed = %d", paperBTreeHeightWant, h)
+	}
+}
+
+func TestPaperBTreeHeightLevelsTest_Eval(t *testing.T) {
+	tc := paperBTreeHeightLevelsTest()
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{"correct", "3", 1},
+		{"prose wrapped", "The minimum height is h=3.", 1},
+		{"one level short", "2", 0},
+		{"one level too many", "4", 0},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}
+
+func TestPaperCAPAvailabilityChoiceTest_Eval(t *testing.T) {
+	tc := paperCAPAvailabilityChoiceTest()
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{"correct lowercase", "availability", 1},
+		{"correct capitalized", "Availability", 1},
+		{"correct with trailing period", "Availability.", 1},
+		{"wrong: consistency", "Consistency", 0},
+		{"wrong: extra words break the forced one-word format", "The answer is availability", 0},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}
+
+func TestPaperAttentionScaleFactorTest_Eval(t *testing.T) {
+	tc := paperAttentionScaleFactorTest()
+
+	// Ground truth: sqrt(64) = 8, independently recomputed with math.Sqrt.
+	if got := math.Sqrt(64); got != 8 {
+		t.Fatalf("math.Sqrt(64) = %v, want 8", got)
+	}
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{"all correct", `{"d_k":64,"scale_factor":8}`, 1},
+		{"one field wrong", `{"d_k":64,"scale_factor":64}`, 0.5},
+		{"both wrong", `{"d_k":512,"scale_factor":22.6}`, 0},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}

internal/tests/whitepapers_vector.go

@@ -0,0 +1,218 @@
+package tests
+
+import (
+	"github.com/lukaszraczylo/llm-testbench/internal/eval"
+	"github.com/lukaszraczylo/llm-testbench/internal/testkit"
+)
+
+// paperPQCompressionExcerpt is a ~143-word, technically accurate original
+// description of product quantization written for this test, pinning the
+// dimension, m, and nbits parameters needed to derive a compression ratio.
+const paperPQCompressionExcerpt = `Product quantization compresses high-dimensional vectors for approximate
+nearest-neighbor search by splitting each vector into m equal-length
+subvectors, then quantizing each subvector independently against its own
+codebook. Each codebook is trained offline with k-means and holds 2^nbits
+centroids. Rather than storing the original subvector, the index stores only
+the index of its nearest centroid: an nbits-bit code. Concatenating the m
+per-subvector codes gives the full compressed representation of the vector.
+A benchmark index in this report indexed 128-dimensional vectors stored as
+32-bit floats, so one uncompressed vector occupies 128 times 4, or 512,
+bytes. The index split each vector into m equal to 8 subvectors and used
+nbits equal to 8 bits per subvector, giving each subquantizer's codebook 256
+centroids. Search computes approximate distances directly against the
+stored codes using precomputed per-centroid distance tables, never
+decompressing a vector back to its original 512-byte form.`
+
+// paperPQCompressionRatioWant is derived by calling wpQuantizedCodeBytes,
+// not hardcoded.
+//
+// ground truth: code bytes = m*nbits/8 = 8*8/8 = 8 bytes per vector;
+// compression ratio = raw bytes / code bytes = 512/8 = 64.
+// whitepapers_vector_test.go independently recomputes this with plain
+// arithmetic.
+var paperPQCompressionRatioWant = 512 / wpQuantizedCodeBytes(8, 8)
+
+// paperPQCompressionRatioTest: derive a product-quantization compression
+// ratio from an inline excerpt's m/nbits/dimension parameters.
+func paperPQCompressionRatioTest() testkit.Test {
+	prompt := `Read this excerpt about product quantization (PQ) for vector search:
+
+` + paperPQCompressionExcerpt + `
+
+The benchmark index above stores each compressed vector as its m
+per-subvector codes packed with no wasted bits. What is the compression
+ratio of the compressed representation versus the original 512-byte
+float32 vector (raw bytes divided by compressed bytes)? Respond with only
+the number.`
+
+	return testkit.Test{
+		ID:          "paper-pq-compression-ratio",
+		Category:    "research",
+		Subcategory: "whitepapers",
+		Description: "Derive a product-quantization compression ratio from an inline excerpt's m, nbits, and raw-dimension parameters.",
+		Prompt:      prompt,
+		Eval:        eval.Numeric(eval.ExtractLastNumber[int], paperPQCompressionRatioWant, 0),
+	}
+}
+
+// paperBloomFPRateExcerpt is a ~157-word, technically accurate original
+// description of the Bloom filter false-positive formula written for this
+// test, pinning m, n, and k.
+const paperBloomFPRateExcerpt = `A Bloom filter is a space-efficient structure for testing set membership
+that trades a small, tunable false-positive rate for large memory savings,
+and never produces a false negative. It holds an m-bit array, initially all
+zero, and uses k independent hash functions. Inserting an element sets the
+k bit positions its hashes point to; querying an element checks whether all
+k of those positions are set, and if so reports "possibly present." The
+false-positive probability after inserting n elements is well approximated
+by p equals the quantity one minus e to the power of negative k times n
+divided by m, that whole quantity raised to the power k. A caching layer
+described in this report sized its filter at m equal to 10000 bits for n
+equal to 1000 expected keys, and chose k equal to 7 hash functions,
+following the standard guidance to pick k near m over n times the natural
+log of 2.`
+
+// paperBloomFPRateWant is derived by calling wpBloomFalsePositiveRate, not
+// hardcoded.
+//
+// ground truth: p = (1 - e^(-k*n/m))^k with m=10000, n=1000, k=7.
+// whitepapers_vector_test.go recomputes this independently with math.Pow
+// and math.Exp inline.
+var paperBloomFPRateWant = round4dp(wpBloomFalsePositiveRate(10000, 1000, 7))
+
+// paperBloomFPRateTest: compute a Bloom filter's false-positive rate from
+// an inline excerpt's formula and m/n/k parameters.
+func paperBloomFPRateTest() testkit.Test {
+	prompt := `Read this excerpt about Bloom filters:
+
+` + paperBloomFPRateExcerpt + `
+
+Using the exact formula given above and the caching layer's parameters
+(m=10000, n=1000, k=7), compute the false-positive probability p. Respond
+with only the number, rounded to 4 decimal places.`
+
+	return testkit.Test{
+		ID:          "paper-bloom-fp-rate",
+		Category:    "research",
+		Subcategory: "whitepapers",
+		Description: "Compute a Bloom filter's false-positive rate from an inline excerpt's formula and m/n/k parameters.",
+		Prompt:      prompt,
+		Eval:        eval.Numeric(eval.ExtractLastNumber[float64], paperBloomFPRateWant, 0.0005),
+	}
+}
+
+// paperTFIDFExcerpt is a ~148-word, technically accurate original
+// description of the plain (unsmoothed) TF-IDF formula written for this
+// test.
+const paperTFIDFExcerpt = `TF-IDF scores how important a term is to one document within a corpus by
+combining two signals: how often the term appears in that document, and how
+rare the term is across the whole corpus. This report uses the plainest
+textbook variant. Term frequency, tf(t,d), is simply the raw count of term
+t in document d, with no normalization. Inverse document frequency,
+idf(t), is the natural logarithm of N divided by df(t), where N is the
+total number of documents in the corpus and df(t) is the number of
+documents that contain t at least once. The TF-IDF weight of a term in a
+document is the product tf(t,d) times idf(t): a term that appears often in
+one document but rarely elsewhere scores highest, while a term appearing in
+every document scores near zero regardless of how often it repeats, since
+its idf approaches the logarithm of one.`
+
+// paperTFIDFDoc1, paperTFIDFDoc2, and paperTFIDFDoc3 are the pre-tokenized
+// (whitespace-split, lowercase) mini-corpus for paperTFIDFTermScoreTest.
+var (
+	paperTFIDFDoc1   = []string{"the", "cat", "sat", "on", "the", "mat"}
+	paperTFIDFDoc2   = []string{"the", "dog", "sat", "on", "the", "log"}
+	paperTFIDFDoc3   = []string{"cats", "and", "dogs", "are", "great", "pets"}
+	paperTFIDFCorpus = [][]string{paperTFIDFDoc1, paperTFIDFDoc2, paperTFIDFDoc3}
+)
+
+// paperTFIDFTermScoreWant is derived by calling wpTermFrequency,
+// wpDocumentFrequency, and wpTFIDF, not hardcoded.
+//
+// ground truth: tf("sat", doc1) = 1; df("sat") = 2 (doc1 and doc2 contain
+// it, doc3 does not); N = 3; idf = ln(3/2); tfidf = 1 * ln(3/2).
+// whitepapers_vector_test.go recomputes this independently with math.Log.
+var paperTFIDFTermScoreWant = round4dp(wpTFIDF(
+	wpTermFrequency("sat", paperTFIDFDoc1),
+	wpDocumentFrequency("sat", paperTFIDFCorpus),
+	len(paperTFIDFCorpus),
+))
+
+// paperTFIDFTermScoreTest: compute the TF-IDF weight of one term in one
+// document of an inline 3-document mini-corpus, using the exact formula
+// given in the excerpt.
+func paperTFIDFTermScoreTest() testkit.Test {
+	prompt := `Read this excerpt about TF-IDF:
+
+` + paperTFIDFExcerpt + `
+
+Here is a 3-document corpus:
+
+Document 1: "the cat sat on the mat"
+Document 2: "the dog sat on the log"
+Document 3: "cats and dogs are great pets"
+
+Using the exact formulas given above (tf = raw count of the term in the
+document, idf = natural log of N divided by df), compute the TF-IDF weight
+of the term "sat" in Document 1. Respond with only the number, rounded to
+4 decimal places.`
+
+	return testkit.Test{
+		ID:          "paper-tfidf-term-score",
+		Category:    "research",
+		Subcategory: "whitepapers",
+		Description: "Compute a term's TF-IDF weight in one document of an inline 3-document mini-corpus using a formula given in the excerpt.",
+		Prompt:      prompt,
+		Eval:        eval.Numeric(eval.ExtractLastNumber[float64], paperTFIDFTermScoreWant, 0.0005),
+	}
+}
+
+// paperRecallExcerpt is a ~137-word, technically accurate original
+// description of the recall@k retrieval metric written for this test.
+const paperRecallExcerpt = `Recall@k is a standard retrieval-quality metric for evaluating a nearest-
+neighbor or search system: for one query, it is the fraction of that
+query's truly relevant items that appear anywhere within the system's top
+k returned results, regardless of their exact rank inside that top-k list.
+Formally, recall@k equals the size of the intersection between the
+relevant set and the top-k retrieved set, divided by the size of the
+relevant set. A perfect recall@k of 1.0 means every relevant item was
+retrieved somewhere in the top k, even if none of them landed in rank one;
+a lower score means some relevant items were pushed outside the top k
+entirely and are effectively lost to any downstream ranking or re-ranking
+stage, since a re-ranker can only reorder candidates that retrieval already
+surfaced, never recover ones it dropped.`
+
+// paperRecallRelevant and paperRecallRetrievedTop6 are the inline
+// relevant-set and top-6 retrieved-ranking fixtures for
+// paperRecallAtKTest.
+var (
+	paperRecallRelevant      = []string{"D2", "D5", "D7", "D9"}
+	paperRecallRetrievedTop6 = []string{"D1", "D5", "D3", "D7", "D6", "D2"}
+	paperRecallAtKWant       = round4dp(wpRecallAtK(paperRecallRelevant, paperRecallRetrievedTop6))
+)
+
+// paperRecallAtKTest: compute recall@k from an inline relevant-item set and
+// a ranked top-k retrieval list.
+func paperRecallAtKTest() testkit.Test {
+	prompt := `Read this excerpt about the recall@k retrieval metric:
+
+` + paperRecallExcerpt + `
+
+A query's truly relevant documents are: D2, D5, D7, D9.
+
+A retrieval system's ranked top-6 results for that query are, in rank
+order: D1, D5, D3, D7, D6, D2.
+
+Using the definition given above, compute recall@6 for this query. Respond
+with only the number, as a decimal (e.g. 0.5), rounded to 4 decimal
+places.`
+
+	return testkit.Test{
+		ID:          "paper-recall-at-k",
+		Category:    "research",
+		Subcategory: "whitepapers",
+		Description: "Compute recall@k from an inline relevant-item set and a ranked top-k retrieval list.",
+		Prompt:      prompt,
+		Eval:        eval.Numeric(eval.ExtractLastNumber[float64], paperRecallAtKWant, 0.0005),
+	}
+}

internal/tests/whitepapers_vector_test.go

@@ -0,0 +1,165 @@
+package tests
+
+import (
+	"context"
+	"math"
+	"testing"
+)
+
+func TestPaperPQCompressionRatioWant_GroundTruth(t *testing.T) {
+	// Independent re-derivation with plain arithmetic, not via
+	// wpQuantizedCodeBytes.
+	const dim, m, nbits = 128, 8, 8
+	rawBytes := dim * 4
+	codeBytes := m * nbits / 8
+	want := rawBytes / codeBytes
+
+	if want != 64 {
+		t.Fatalf("independently recomputed ratio = %d, want 64", want)
+	}
+	if paperPQCompressionRatioWant != want {
+		t.Errorf("paperPQCompressionRatioWant = %d, independently recomputed = %d", paperPQCompressionRatioWant, want)
+	}
+}
+
+func TestPaperPQCompressionRatioTest_Eval(t *testing.T) {
+	tc := paperPQCompressionRatioTest()
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{"correct", "64", 1},
+		{"prose wrapped", "The compression ratio is 64x.", 1},
+		{"wrong", "8", 0},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}
+
+func TestPaperBloomFPRateWant_GroundTruth(t *testing.T) {
+	// Independent re-derivation with math.Pow/math.Exp inline, not via
+	// wpBloomFalsePositiveRate.
+	const m, n, k = 10000.0, 1000.0, 7.0
+	want := math.Round(math.Pow(1-math.Exp(-k*n/m), k)*10000) / 10000
+
+	if math.Abs(want-0.0082) > 1e-9 {
+		t.Fatalf("independently recomputed FP rate = %v, want ~0.0082", want)
+	}
+	if paperBloomFPRateWant != want {
+		t.Errorf("paperBloomFPRateWant = %v, independently recomputed = %v", paperBloomFPRateWant, want)
+	}
+}
+
+func TestPaperBloomFPRateTest_Eval(t *testing.T) {
+	tc := paperBloomFPRateTest()
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{"exact", "0.0082", 1},
+		{"within tolerance", "0.0081", 1},
+		{"prose wrapped", "p is approximately 0.0082.", 1},
+		{"outside tolerance", "0.05", 0},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}
+
+func TestPaperTFIDFTermScoreWant_GroundTruth(t *testing.T) {
+	// Independent re-derivation by counting the raw corpus text directly,
+	// not via wpTermFrequency/wpDocumentFrequency/wpTFIDF.
+	tf := 1 // "sat" appears once in doc1
+	df := 2 // doc1 and doc2 contain "sat"; doc3 does not
+	n := 3  // 3 documents total
+	want := math.Round(float64(tf)*math.Log(float64(n)/float64(df))*10000) / 10000
+
+	if math.Abs(want-0.4055) > 1e-9 {
+		t.Fatalf("independently recomputed tfidf = %v, want ~0.4055", want)
+	}
+	if paperTFIDFTermScoreWant != want {
+		t.Errorf("paperTFIDFTermScoreWant = %v, independently recomputed = %v", paperTFIDFTermScoreWant, want)
+	}
+}
+
+func TestPaperTFIDFTermScoreTest_Eval(t *testing.T) {
+	tc := paperTFIDFTermScoreTest()
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{"exact", "0.4055", 1},
+		{"within tolerance", "0.4053", 1},
+		{"prose wrapped", "The TF-IDF weight is 0.4055.", 1},
+		{"outside tolerance", "1.0986", 0},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}
+
+func TestPaperRecallAtKWant_GroundTruth(t *testing.T) {
+	// Independent re-derivation with a plain set intersection, not via
+	// wpRecallAtK.
+	relevant := map[string]bool{"D2": true, "D5": true, "D7": true, "D9": true}
+	retrieved := []string{"D1", "D5", "D3", "D7", "D6", "D2"}
+	hit := 0
+	for _, id := range retrieved {
+		if relevant[id] {
+			hit++
+		}
+	}
+	want := float64(hit) / float64(len(relevant))
+
+	if want != 0.75 {
+		t.Fatalf("independently recomputed recall@6 = %v, want 0.75", want)
+	}
+	if paperRecallAtKWant != want {
+		t.Errorf("paperRecallAtKWant = %v, independently recomputed = %v", paperRecallAtKWant, want)
+	}
+}
+
+func TestPaperRecallAtKTest_Eval(t *testing.T) {
+	tc := paperRecallAtKTest()
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{"exact", "0.75", 1},
+		{"prose wrapped", "recall@6 is 0.75.", 1},
+		{"wrong: counted D9 as retrieved", "1.0", 0},
+		{"wrong: missed one hit", "0.5", 0},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}