feat: LLM accuracy testing framework with 17-test catalog

Framework runs a category/subcategory test catalog against multiple
LLMs through an OpenAI-compatible endpoint. Deterministic evaluators
score each response. Reports render as table, markdown, or JSON.

internal/tests/web.go

@@ -0,0 +1,66 @@
+package tests
+
+import (
+	"github.com/lukaszraczylo/llm-testbench/internal/eval"
+	"github.com/lukaszraczylo/llm-testbench/internal/testkit"
+)
+
+func registerWebTests(r *testkit.Registry) {
+	r.Register(webRobotsAICrawlersTest())
+}
+
+// webRobotsAICrawlersRobotsTxt is the inline robots.txt for
+// webRobotsAICrawlersTest. Each listed user-agent has an explicit block, so
+// the most specific matching block applies per bot (standard robots.txt
+// precedence); the wildcard block only applies to bots with no dedicated
+// block of their own.
+const webRobotsAICrawlersRobotsTxt = `User-agent: GPTBot
+Disallow: /
+
+User-agent: ClaudeBot
+Allow: /
+
+User-agent: PerplexityBot
+Disallow: /docs/
+
+User-agent: Googlebot
+Allow: /
+
+User-agent: *
+Disallow: /private/`
+
+// webRobotsAICrawlersTest: determine which AI crawlers (not just any
+// allowed bot) can fetch /docs/.
+//
+// ground truth: for path /docs/, GPTBot's dedicated block disallows /
+// entirely (blocked), ClaudeBot's dedicated block allows / (can fetch),
+// PerplexityBot's dedicated block disallows /docs/ specifically (blocked),
+// and Googlebot's dedicated block allows / (can fetch) - but Googlebot is
+// Google's general-purpose search-indexing crawler, not an AI crawler, so
+// it does not belong in an "AI crawlers" answer even though it is
+// technically allowed. The only AI crawler that can fetch /docs/ is
+// ClaudeBot.
+func webRobotsAICrawlersTest() testkit.Test {
+	prompt := `Here is a robots.txt file:
+
+` + webRobotsAICrawlersRobotsTxt + `
+
+GPTBot (OpenAI), ClaudeBot (Anthropic), and PerplexityBot (Perplexity) are
+AI crawlers used to fetch content for AI systems. Googlebot is Google's
+general-purpose search-indexing crawler, not an AI crawler.
+
+Which of the AI crawlers listed above are permitted, by this robots.txt, to
+fetch a page at path /docs/? Respond with only a JSON array of crawler
+names, e.g. ["ClaudeBot"]. Do not include Googlebot even if it is allowed,
+since it is not an AI crawler.`
+
+	return testkit.Test{
+		ID:          "web-robots-ai-crawlers",
+		Category:    "research",
+		Subcategory: "web",
+		Description: "Determine which AI crawlers (excluding a general search-engine decoy) may fetch a path per robots.txt rules.",
+		Prompt:      prompt,
+		MaxTokens:   300,
+		Eval:        eval.JSONStringSet([]string{"ClaudeBot"}),
+	}
+}