test(catalog): expand research to 10 tests per subcategory

Add 9 tests to research/web, research/whitepapers, and
research/codebase, bringing each subcategory from 1 test to 10.

Web tests extract data from inline HTML, headers, and URLs, and
answer protocol scenarios for HTTP status codes, CORS, DNS, and
canonicalization. Every fixture stays inline; no test makes a live
web call.

Whitepaper tests each pair a 120 to 180 word, technically accurate
excerpt with numeric or JSON extraction questions. Topics cover
product quantization, Bloom filters, TF-IDF, recall@k, LSM-tree
write amplification, Raft quorum sizing, B-tree capacity, the CAP
theorem, and attention scaling. Every derived constant carries a
ground-truth comment and a unit test that recomputes the value with
an independent formula.

Codebase tests trace a Python function and a TypeScript function to
exact output, then reason over an inline diff, a data race, dead
code, Big-O complexity, an import cycle, a generic return type, and
a git-bisect scenario. The Python trace test runs the same source
through python3 when available and skips otherwise; the TypeScript
trace test cross-checks its hand-derivation against an equivalent Go
function.

New package-level helpers use a wp or code prefix to avoid name
collisions with sibling category authors sharing this package. Test
IDs keep the paper, web, and code prefixes already established by
the existing catalog entries. New tests omit MaxTokens so the runner
falls back to its default, per the catalog's authoring rules.

catalog_test.go's exact-map assertion now fails as expected, since
it still lists only the original 17-test catalog; that assertion is
owned centrally and is not touched here.
This commit is contained in:
2026-08-17 01:51:11 +01:00
parent 764fe990fd
commit e7479e9169
18 changed files with 2368 additions and 0 deletions
+137
View File
@@ -0,0 +1,137 @@
package tests
import (
"github.com/lukaszraczylo/llm-testbench/internal/eval"
"github.com/lukaszraczylo/llm-testbench/internal/testkit"
)
// webProductCardHTML is the inline HTML product-card snippet for
// webHTMLProductExtractTest.
const webProductCardHTML = `<div class="product-card" data-sku="WBH-2201">
<h2 class="product-name">Wireless Mechanical Keyboard</h2>
<span class="price" data-currency="USD">129.99</span>
<span class="stock" data-available="true">In Stock</span>
</div>`
// webHTMLProductExtractTest: extract a product's name, price, and SKU
// from inline HTML into JSON.
//
// ground truth: name is the h2.product-name text; price is the
// span.price text as a number; sku is the data-sku attribute on the
// outer div - all read directly from the markup, no computation.
func webHTMLProductExtractTest() testkit.Test {
prompt := `Here is an HTML product card:
` + "```html\n" + webProductCardHTML + "\n```" + `
Extract the product's name, price (as a number), and SKU into JSON.
Respond with only a JSON object: {"name":"...","price":<number>,"sku":"..."}`
evaluator := eval.Mean(
eval.JSONField("name", "Wireless Mechanical Keyboard"),
eval.JSONField("price", 129.99),
eval.JSONField("sku", "WBH-2201"),
)
return testkit.Test{
ID: "web-html-product-extract",
Category: "research",
Subcategory: "web",
Description: "Extract a product's name, price, and SKU from an inline HTML product card into JSON.",
Prompt: prompt,
Eval: evaluator,
}
}
// webHreflangPagesSnippet lists the hreflang <link> tags of three pages
// for webHreflangReciprocalTest. /en/ links to both /es/ and /fr/, and
// /es/ links back to both, but /fr/ omits the reciprocal link back to
// /en/.
const webHreflangPagesSnippet = `Page https://example.com/en/ <head> contains:
<link rel="alternate" hreflang="en-us" href="https://example.com/en/" />
<link rel="alternate" hreflang="es" href="https://example.com/es/" />
<link rel="alternate" hreflang="fr" href="https://example.com/fr/" />
Page https://example.com/es/ <head> contains:
<link rel="alternate" hreflang="en-us" href="https://example.com/en/" />
<link rel="alternate" hreflang="es" href="https://example.com/es/" />
<link rel="alternate" hreflang="fr" href="https://example.com/fr/" />
Page https://example.com/fr/ <head> contains:
<link rel="alternate" hreflang="es" href="https://example.com/es/" />
<link rel="alternate" hreflang="fr" href="https://example.com/fr/" />`
// webHreflangReciprocalTest: find the one non-reciprocal hreflang link
// among three inline pages.
//
// ground truth: /en/ declares an hreflang link to /fr/, but /fr/'s own
// link set has no entry pointing back to https://example.com/en/, so the
// pair (/en/, /fr/) is not reciprocal; /fr/ is the page missing the link,
// and https://example.com/en/ is the target it is missing. Every other
// pair in the three pages is fully reciprocal.
func webHreflangReciprocalTest() testkit.Test {
prompt := `hreflang annotations must be reciprocal: if page A links to page B
via hreflang, page B must also link back to page A. Here are the hreflang
link tags for three pages:
` + webHreflangPagesSnippet + `
Exactly one pair among these three pages is broken (not reciprocal).
Respond with only a JSON object:
{"page_missing_link":"<url of the page that is missing a link>","missing_target":"<url that page should link to but does not>"}`
evaluator := eval.Mean(
eval.JSONField("page_missing_link", "https://example.com/fr/"),
eval.JSONField("missing_target", "https://example.com/en/"),
)
return testkit.Test{
ID: "web-hreflang-reciprocal",
Category: "research",
Subcategory: "web",
Description: "Find the one non-reciprocal hreflang link among three inline pages' link tag sets.",
Prompt: prompt,
Eval: evaluator,
}
}
// webParseURL is the inline URL for webURLParseComponentsTest.
const webParseURL = "https://api.example.com:8443/v2/search?q=hello+world&sort=desc#results"
// webURLParseComponentsTest: parse an inline URL into its component parts
// as JSON.
//
// ground truth: computed directly from net/url.Parse in
// web_extract_test.go, not hand-derived - scheme "https", host (hostname,
// no port) "api.example.com", port "8443", path "/v2/search", raw query
// "q=hello+world&sort=desc", fragment "results".
func webURLParseComponentsTest() testkit.Test {
prompt := `Parse this URL into its components:
` + webParseURL + `
Respond with only a JSON object:
{"scheme":"...","host":"...","port":<number>,"path":"...","query":"...","fragment":"..."}
"host" must be the hostname only (no port). "port" must be a number.
"query" must be the raw query string exactly as it appears after the "?",
not split into individual parameters.`
evaluator := eval.Mean(
eval.JSONField("scheme", "https"),
eval.JSONField("host", "api.example.com"),
eval.JSONField("port", 8443),
eval.JSONField("path", "/v2/search"),
eval.JSONField("query", "q=hello+world&sort=desc"),
eval.JSONField("fragment", "results"),
)
return testkit.Test{
ID: "web-url-parse-components",
Category: "research",
Subcategory: "web",
Description: "Parse an inline URL into its scheme, host, port, path, query, and fragment components as JSON.",
Prompt: prompt,
Eval: evaluator,
}
}