llm-testbench/internal/tests/web_protocol_test.go

package tests

import (
	"context"
	"testing"
)

func TestWebSitemapMaxURLsTest_Eval(t *testing.T) {
	tc := webSitemapMaxURLsTest()

	tests := []struct {
		name     string
		response string
		want     float64
	}{
		{"correct", "50000", 1},
		{"prose wrapped", "The limit is 50000 URLs per sitemap file.", 1},
		// B1: the prompt itself primes comma grouping with "52,000", so a
		// comma-grouped correct answer must not score 0.
		{"comma-grouped", "50,000", 1},
		{"comma-grouped in a sentence", "The limit is 50,000 URLs.", 1},
		{"wrong: confused with the uncompressed size limit in MB", "50", 0},
		{"wrong entirely", "10000", 0},
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			got := tc.Eval.Evaluate(context.Background(), tt.response)
			if got.Value != tt.want {
				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
			}
		})
	}
}

func TestWebHTTPStatusScenariosTest_Eval(t *testing.T) {
	tc := webHTTPStatusScenariosTest()

	tests := []struct {
		name     string
		response string
		want     float64
	}{
		{"all correct", `{"scenario1":400,"scenario2":401,"scenario3":403,"scenario4":301}`, 1},
		{"swapped 401 and 403", `{"scenario1":400,"scenario2":403,"scenario3":401,"scenario4":301}`, 0.5},
		{"used 500 for a client error", `{"scenario1":500,"scenario2":401,"scenario3":403,"scenario4":301}`, 0.75},
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			got := tc.Eval.Evaluate(context.Background(), tt.response)
			if got.Value != tt.want {
				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
			}
		})
	}
}

func TestWebDNSMXRecordTest_Eval(t *testing.T) {
	tc := webDNSMXRecordTest()

	tests := []struct {
		name     string
		response string
		want     float64
	}{
		{"bare", "MX", 1},
		{"lowercase", "mx", 1},
		{"with the word record", "MX record", 1},
		{"with the word records and a period", "MX records.", 1},
		{"quoted", `"MX"`, 1},
		{"wrong: CNAME", "CNAME", 0},
		{"wrong: A record", "A", 0},
		// B4: restating the prompt's example list (which no longer
		// includes MX at all) must not score 1 the way the old
		// substring-anywhere-in-the-response evaluator would have.
		{"restating the full example list scores 0", "A, AAAA, CNAME, TXT, NS", 0},
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			got := tc.Eval.Evaluate(context.Background(), tt.response)
			if got.Value != tt.want {
				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
			}
		})
	}
}

func TestWebCanonicalVsRedirectTest_Eval(t *testing.T) {
	tc := webCanonicalVsRedirectTest()

	tests := []struct {
		name     string
		response string
		want     float64
	}{
		{"correct lowercase", "canonical", 1},
		{"correct capitalized with period", "Canonical.", 1},
		{"wrong: redirect", "redirect", 0},
		{"wrong: extra words break the forced one-word format", "Use a canonical tag", 0},
		// B3: the prompt's own phrasing quotes both options.
		{"correct, quoted (prompt's own phrasing)", `"canonical"`, 1},
		{"correct, bolded", "**canonical**", 1},
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			got := tc.Eval.Evaluate(context.Background(), tt.response)
			if got.Value != tt.want {
				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
			}
		})
	}
}