mirror of
https://github.com/lukaszraczylo/llm-testbench.git
synced 2026-10-12 00:45:19 +00:00
lukaszraczylo
1d51d4cbbd
Framework runs a category/subcategory test catalog against multiple LLMs through an OpenAI-compatible endpoint. Deterministic evaluators score each response. Reports render as table, markdown, or JSON.
67 lines
2.3 KiB
Go
67 lines
2.3 KiB
Go
package tests
|
|
|
|
import (
|
|
"github.com/lukaszraczylo/llm-testbench/internal/eval"
|
|
"github.com/lukaszraczylo/llm-testbench/internal/testkit"
|
|
)
|
|
|
|
func registerWebTests(r *testkit.Registry) {
|
|
r.Register(webRobotsAICrawlersTest())
|
|
}
|
|
|
|
// webRobotsAICrawlersRobotsTxt is the inline robots.txt for
|
|
// webRobotsAICrawlersTest. Each listed user-agent has an explicit block, so
|
|
// the most specific matching block applies per bot (standard robots.txt
|
|
// precedence); the wildcard block only applies to bots with no dedicated
|
|
// block of their own.
|
|
const webRobotsAICrawlersRobotsTxt = `User-agent: GPTBot
|
|
Disallow: /
|
|
|
|
User-agent: ClaudeBot
|
|
Allow: /
|
|
|
|
User-agent: PerplexityBot
|
|
Disallow: /docs/
|
|
|
|
User-agent: Googlebot
|
|
Allow: /
|
|
|
|
User-agent: *
|
|
Disallow: /private/`
|
|
|
|
// webRobotsAICrawlersTest: determine which AI crawlers (not just any
|
|
// allowed bot) can fetch /docs/.
|
|
//
|
|
// ground truth: for path /docs/, GPTBot's dedicated block disallows /
|
|
// entirely (blocked), ClaudeBot's dedicated block allows / (can fetch),
|
|
// PerplexityBot's dedicated block disallows /docs/ specifically (blocked),
|
|
// and Googlebot's dedicated block allows / (can fetch) - but Googlebot is
|
|
// Google's general-purpose search-indexing crawler, not an AI crawler, so
|
|
// it does not belong in an "AI crawlers" answer even though it is
|
|
// technically allowed. The only AI crawler that can fetch /docs/ is
|
|
// ClaudeBot.
|
|
func webRobotsAICrawlersTest() testkit.Test {
|
|
prompt := `Here is a robots.txt file:
|
|
|
|
` + webRobotsAICrawlersRobotsTxt + `
|
|
|
|
GPTBot (OpenAI), ClaudeBot (Anthropic), and PerplexityBot (Perplexity) are
|
|
AI crawlers used to fetch content for AI systems. Googlebot is Google's
|
|
general-purpose search-indexing crawler, not an AI crawler.
|
|
|
|
Which of the AI crawlers listed above are permitted, by this robots.txt, to
|
|
fetch a page at path /docs/? Respond with only a JSON array of crawler
|
|
names, e.g. ["ClaudeBot"]. Do not include Googlebot even if it is allowed,
|
|
since it is not an AI crawler.`
|
|
|
|
return testkit.Test{
|
|
ID: "web-robots-ai-crawlers",
|
|
Category: "research",
|
|
Subcategory: "web",
|
|
Description: "Determine which AI crawlers (excluding a general search-engine decoy) may fetch a path per robots.txt rules.",
|
|
Prompt: prompt,
|
|
MaxTokens: 300,
|
|
Eval: eval.JSONStringSet([]string{"ClaudeBot"}),
|
|
}
|
|
}
|