feat: LLM accuracy testing framework with 17-test catalog

Framework runs a category/subcategory test catalog against multiple
LLMs through an OpenAI-compatible endpoint. Deterministic evaluators
score each response. Reports render as table, markdown, or JSON.

internal/config/config.go

@@ -0,0 +1,109 @@
+// Package config loads and validates the llm-testbench YAML configuration.
+package config
+
+import (
+	"fmt"
+	"os"
+	"time"
+
+	"gopkg.in/yaml.v3"
+)
+
+// apiKeyEnvVar names the environment variable that overrides Config.APIKey.
+//
+//nolint:gosec // this is an env var *name*, not a credential value
+const apiKeyEnvVar = "LLMTB_API_KEY" // #nosec G101 -- this is an env var *name*, not a credential value
+
+// Config is the top-level configuration loaded from config.yaml.
+type Config struct {
+	Endpoint         string        `yaml:"endpoint"`
+	APIKey           string        `yaml:"api_key"`
+	Models           []string      `yaml:"models"`
+	Concurrency      int           `yaml:"concurrency"`
+	RequestTimeout   time.Duration `yaml:"request_timeout"`
+	MaxTokensDefault int           `yaml:"max_tokens_default"`
+}
+
+// rawConfig mirrors Config but with RequestTimeout as a string, since
+// yaml.v3 does not natively decode Go duration strings.
+type rawConfig struct {
+	Endpoint         string   `yaml:"endpoint"`
+	APIKey           string   `yaml:"api_key"`
+	RequestTimeout   string   `yaml:"request_timeout"`
+	Models           []string `yaml:"models"`
+	Concurrency      int      `yaml:"concurrency"`
+	MaxTokensDefault int      `yaml:"max_tokens_default"`
+}
+
+// Load reads and parses the YAML config at path, applies the LLMTB_API_KEY
+// environment override (which always wins over the file value), and
+// validates the result.
+func Load(path string) (Config, error) {
+	// #nosec G304 -- path is the operator's own --config flag value (a
+	// local CLI tool reading its own config file), not externally supplied
+	// or network input.
+	data, err := os.ReadFile(path) //nolint:gosec // path is the operator's own --config flag value, not external input
+	if err != nil {
+		return Config{}, fmt.Errorf("config: read %s: %w", path, err)
+	}
+	return parse(data)
+}
+
+// parse decodes YAML bytes into a validated Config. Split out from Load for
+// direct, file-free unit testing.
+func parse(data []byte) (Config, error) {
+	var raw rawConfig
+	if err := yaml.Unmarshal(data, &raw); err != nil {
+		return Config{}, fmt.Errorf("config: parse yaml: %w", err)
+	}
+
+	cfg := Config{
+		Endpoint:         raw.Endpoint,
+		APIKey:           raw.APIKey,
+		Models:           raw.Models,
+		Concurrency:      raw.Concurrency,
+		MaxTokensDefault: raw.MaxTokensDefault,
+	}
+
+	if raw.RequestTimeout != "" {
+		d, err := time.ParseDuration(raw.RequestTimeout)
+		if err != nil {
+			return Config{}, fmt.Errorf("config: parse request_timeout %q: %w", raw.RequestTimeout, err)
+		}
+		cfg.RequestTimeout = d
+	}
+
+	if envKey := os.Getenv(apiKeyEnvVar); envKey != "" {
+		cfg.APIKey = envKey
+	}
+
+	if err := cfg.Validate(); err != nil {
+		return Config{}, err
+	}
+	return cfg, nil
+}
+
+// Validate checks that required fields are present and sane.
+func (c Config) Validate() error {
+	if c.Endpoint == "" {
+		return fmt.Errorf("config: endpoint must not be empty")
+	}
+	if len(c.Models) == 0 {
+		return fmt.Errorf("config: models must list at least one model")
+	}
+	for i, m := range c.Models {
+		if m == "" {
+			return fmt.Errorf("config: models[%d] must not be empty", i)
+		}
+	}
+	if c.Concurrency <= 0 {
+		return fmt.Errorf("config: concurrency must be > 0, got %d", c.Concurrency)
+	}
+	if c.RequestTimeout <= 0 {
+		return fmt.Errorf("config: request_timeout must be > 0, got %s", c.RequestTimeout)
+	}
+	if c.MaxTokensDefault <= 0 {
+		return fmt.Errorf("config: max_tokens_default must be > 0, got %d", c.MaxTokensDefault)
+	}
+	return nil
+}

internal/config/config_test.go

@@ -0,0 +1,155 @@
+package config
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+)
+
+const validYAML = `
+endpoint: https://llm-gateway.example.com/v1
+api_key: ""
+models:
+  - uni/deepseek-v4-flash-0731
+  - uni/qwen3.6-27b
+concurrency: 4
+request_timeout: 120s
+max_tokens_default: 4000
+`
+
+func TestParse_Valid(t *testing.T) {
+	cfg, err := parse([]byte(validYAML))
+	if err != nil {
+		t.Fatalf("parse() error = %v", err)
+	}
+	if cfg.Endpoint != "https://llm-gateway.example.com/v1" {
+		t.Errorf("Endpoint = %q", cfg.Endpoint)
+	}
+	if len(cfg.Models) != 2 {
+		t.Errorf("Models = %v, want 2 entries", cfg.Models)
+	}
+	if cfg.Concurrency != 4 {
+		t.Errorf("Concurrency = %d, want 4", cfg.Concurrency)
+	}
+	if cfg.RequestTimeout != 120*time.Second {
+		t.Errorf("RequestTimeout = %v, want 120s", cfg.RequestTimeout)
+	}
+	if cfg.MaxTokensDefault != 4000 {
+		t.Errorf("MaxTokensDefault = %d, want 4000", cfg.MaxTokensDefault)
+	}
+}
+
+func TestParse_EnvOverridesAPIKey(t *testing.T) {
+	t.Setenv(apiKeyEnvVar, "env-secret")
+	cfg, err := parse([]byte(validYAML))
+	if err != nil {
+		t.Fatalf("parse() error = %v", err)
+	}
+	if cfg.APIKey != "env-secret" {
+		t.Errorf("APIKey = %q, want env override", cfg.APIKey)
+	}
+}
+
+func TestParse_FileAPIKeyUsedWhenNoEnv(t *testing.T) {
+	// S10: guarantee no env var leakage from the operator's real shell -
+	// without this, a machine with LLMTB_API_KEY already set would make
+	// this test assert the wrong thing regardless of parse()'s behavior.
+	t.Setenv(apiKeyEnvVar, "")
+
+	yaml := `
+endpoint: https://llm-gateway.example.com/v1
+api_key: "file-key"
+models:
+  - uni/deepseek-v4-flash-0731
+concurrency: 4
+request_timeout: 120s
+max_tokens_default: 4000
+`
+	cfg, err := parse([]byte(yaml))
+	if err != nil {
+		t.Fatalf("parse() error = %v", err)
+	}
+	if cfg.APIKey != "file-key" {
+		t.Errorf("APIKey = %q, want file-key", cfg.APIKey)
+	}
+}
+
+func TestParse_InvalidYAML(t *testing.T) {
+	_, err := parse([]byte("not: [valid: yaml"))
+	if err == nil {
+		t.Fatal("parse() error = nil, want error for invalid YAML")
+	}
+}
+
+func TestParse_InvalidDuration(t *testing.T) {
+	yaml := `
+endpoint: https://x
+models: [m]
+concurrency: 1
+request_timeout: not-a-duration
+max_tokens_default: 100
+`
+	_, err := parse([]byte(yaml))
+	if err == nil {
+		t.Fatal("parse() error = nil, want error for invalid duration")
+	}
+}
+
+func TestValidate(t *testing.T) {
+	base := Config{
+		Endpoint:         "https://x",
+		Models:           []string{"m"},
+		Concurrency:      1,
+		RequestTimeout:   time.Second,
+		MaxTokensDefault: 100,
+	}
+
+	if err := base.Validate(); err != nil {
+		t.Fatalf("Validate() on valid config error = %v", err)
+	}
+
+	tests := []struct {
+		mutate func(c Config) Config
+		name   string
+	}{
+		{name: "empty endpoint", mutate: func(c Config) Config { c.Endpoint = ""; return c }},
+		{name: "no models", mutate: func(c Config) Config { c.Models = nil; return c }},
+		{name: "empty model entry", mutate: func(c Config) Config { c.Models = []string{""}; return c }},
+		{name: "zero concurrency", mutate: func(c Config) Config { c.Concurrency = 0; return c }},
+		{name: "negative concurrency", mutate: func(c Config) Config { c.Concurrency = -1; return c }},
+		{name: "zero timeout", mutate: func(c Config) Config { c.RequestTimeout = 0; return c }},
+		{name: "zero max tokens", mutate: func(c Config) Config { c.MaxTokensDefault = 0; return c }},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			c := tt.mutate(base)
+			if err := c.Validate(); err == nil {
+				t.Errorf("Validate() on %s: error = nil, want error", tt.name)
+			}
+		})
+	}
+}
+
+func TestLoad_ReadsFile(t *testing.T) {
+	dir := t.TempDir()
+	path := filepath.Join(dir, "config.yaml")
+	if err := os.WriteFile(path, []byte(validYAML), 0o600); err != nil {
+		t.Fatalf("os.WriteFile() error = %v", err)
+	}
+
+	cfg, err := Load(path)
+	if err != nil {
+		t.Fatalf("Load() error = %v", err)
+	}
+	if cfg.Endpoint == "" {
+		t.Error("Load() returned empty Endpoint")
+	}
+}
+
+func TestLoad_MissingFile(t *testing.T) {
+	_, err := Load(filepath.Join(t.TempDir(), "does-not-exist.yaml"))
+	if err == nil {
+		t.Fatal("Load() error = nil, want error for missing file")
+	}
+}

internal/eval/composite.go

@@ -0,0 +1,70 @@
+package eval
+
+import (
+	"context"
+	"fmt"
+	"strings"
+)
+
+// WeightedEvaluator pairs an Evaluator with its share of the composite score.
+type WeightedEvaluator struct {
+	Eval   Evaluator
+	Weight float64
+}
+
+// W builds a WeightedEvaluator; a convenience constructor for All.
+func W(e Evaluator, weight float64) WeightedEvaluator {
+	return WeightedEvaluator{Eval: e, Weight: weight}
+}
+
+// compositeEval computes a weighted mean over its sub-evaluators, ignoring
+// sub-scores that are Skipped.
+type compositeEval struct {
+	weighted []WeightedEvaluator
+}
+
+// All returns an Evaluator that combines weighted sub-evaluators into a
+// single weighted-mean Score. Sub-scores marked Skipped are excluded from
+// both the numerator and the weight total; if every sub-evaluator is
+// skipped, the composite itself is Skipped.
+func All(weighted ...WeightedEvaluator) Evaluator {
+	return compositeEval{weighted: weighted}
+}
+
+// Mean wraps evaluators with equal weight 1 and combines them via All. A
+// convenience for the common case where every sub-check counts the same.
+// Named Mean, not Equal, to avoid reading as a synonym of the Equals
+// evaluator (S12) - Mean computes an unweighted average of N sub-scores,
+// while Equals is a single trimmed, case-insensitive string comparison.
+func Mean(evaluators ...Evaluator) Evaluator {
+	weighted := make([]WeightedEvaluator, 0, len(evaluators))
+	for _, e := range evaluators {
+		weighted = append(weighted, W(e, 1))
+	}
+	return All(weighted...)
+}
+
+func (c compositeEval) Evaluate(ctx context.Context, response string) Score {
+	var weightedSum, activeWeight float64
+	details := make([]string, 0, len(c.weighted))
+
+	for _, we := range c.weighted {
+		s := we.Eval.Evaluate(ctx, response)
+		if s.Skipped {
+			details = append(details, fmt.Sprintf("skipped (w=%.2f): %s", we.Weight, s.Detail))
+			continue
+		}
+		activeWeight += we.Weight
+		weightedSum += s.Value * we.Weight
+		details = append(details, fmt.Sprintf("%.2f (w=%.2f): %s", s.Value, we.Weight, s.Detail))
+	}
+
+	if activeWeight == 0 {
+		return Score{Skipped: true, Detail: "all sub-evaluators skipped: " + strings.Join(details, "; ")}
+	}
+
+	return Score{
+		Value:  clamp01(weightedSum / activeWeight),
+		Detail: strings.Join(details, "; "),
+	}
+}

internal/eval/composite_test.go

@@ -0,0 +1,71 @@
+package eval
+
+import (
+	"context"
+	"testing"
+)
+
+func TestAll_WeightedMean(t *testing.T) {
+	e := All(
+		W(Equals("minor"), 2),
+		W(ContainsAny("bump"), 1),
+	)
+	// "minor bump" -> Equals fails (0, weight 2), ContainsAny succeeds (1, weight 1)
+	// weighted mean = (0*2 + 1*1) / 3 = 1/3
+	got := e.Evaluate(context.Background(), "minor bump")
+	want := 1.0 / 3.0
+	if got.Value != want {
+		t.Errorf("Evaluate() = %v, want %v (detail: %s)", got.Value, want, got.Detail)
+	}
+}
+
+func TestMean_AllPass(t *testing.T) {
+	e := Mean(
+		ContainsAll("launchd"),
+		ContainsAll("StartInterval"),
+	)
+	got := e.Evaluate(context.Background(), "use launchd with StartInterval key")
+	if got.Value != 1 {
+		t.Errorf("Evaluate() = %v, want 1 (detail: %s)", got.Value, got.Detail)
+	}
+}
+
+func TestMean_PartialCredit(t *testing.T) {
+	e := Mean(
+		ContainsAll("launchd"),
+		ContainsAll("StartInterval"),
+	)
+	// Only "launchd" present -> (1 + 0) / 2 = 0.5
+	got := e.Evaluate(context.Background(), "use launchd, not cron")
+	if got.Value != 0.5 {
+		t.Errorf("Evaluate() = %v, want 0.5 (detail: %s)", got.Value, got.Detail)
+	}
+}
+
+func TestAll_SkippedSubEvaluatorExcluded(t *testing.T) {
+	skipped := EvaluatorFunc(func(_ context.Context, _ string) Score {
+		return Score{Skipped: true, Detail: "toolchain missing: go"}
+	})
+	e := All(
+		W(Equals("x"), 1),
+		W(skipped, 5),
+	)
+	got := e.Evaluate(context.Background(), "x")
+	if got.Skipped {
+		t.Fatal("Evaluate() should not be Skipped when at least one sub-evaluator is active")
+	}
+	if got.Value != 1 {
+		t.Errorf("Evaluate() = %v, want 1 (skipped sub-evaluator excluded from weight)", got.Value)
+	}
+}
+
+func TestAll_EverythingSkipped(t *testing.T) {
+	skipped := EvaluatorFunc(func(_ context.Context, _ string) Score {
+		return Score{Skipped: true}
+	})
+	e := All(W(skipped, 1))
+	got := e.Evaluate(context.Background(), "anything")
+	if !got.Skipped {
+		t.Error("Evaluate() should be Skipped when every sub-evaluator is skipped")
+	}
+}

internal/eval/contains.go

@@ -0,0 +1,85 @@
+package eval
+
+import (
+	"context"
+	"fmt"
+	"strings"
+)
+
+// containsAll scores the fraction of substrings present in the response,
+// case-insensitive. Full credit only when every substring matches.
+type containsAll struct {
+	substrings []string
+}
+
+// ContainsAll returns an Evaluator that awards matched/total credit for how
+// many of substrings appear (case-insensitive) in the response.
+func ContainsAll(substrings ...string) Evaluator {
+	return containsAll{substrings: substrings}
+}
+
+func (c containsAll) Evaluate(_ context.Context, response string) Score {
+	if len(c.substrings) == 0 {
+		return Score{Value: 1, Detail: "no substrings required"}
+	}
+	lower := strings.ToLower(response)
+	matched := 0
+	var missing []string
+	for _, s := range c.substrings {
+		if strings.Contains(lower, strings.ToLower(s)) {
+			matched++
+		} else {
+			missing = append(missing, s)
+		}
+	}
+	value := float64(matched) / float64(len(c.substrings))
+	detail := fmt.Sprintf("matched %d/%d", matched, len(c.substrings))
+	if len(missing) > 0 {
+		detail += fmt.Sprintf("; missing: %s", strings.Join(missing, ", "))
+	}
+	return Score{Value: value, Detail: detail}
+}
+
+// containsAny awards full credit if at least one alternative matches.
+type containsAny struct {
+	substrings []string
+}
+
+// ContainsAny returns an Evaluator that awards full credit if any substring
+// (case-insensitive) appears in the response, zero otherwise.
+func ContainsAny(substrings ...string) Evaluator {
+	return containsAny{substrings: substrings}
+}
+
+func (c containsAny) Evaluate(_ context.Context, response string) Score {
+	lower := strings.ToLower(response)
+	for _, s := range c.substrings {
+		if strings.Contains(lower, strings.ToLower(s)) {
+			return Score{Value: 1, Detail: fmt.Sprintf("matched %q", s)}
+		}
+	}
+	return Score{Value: 0, Detail: fmt.Sprintf("none of %v matched", c.substrings)}
+}
+
+// notContains is a guard: full credit unless a forbidden substring appears.
+type notContains struct {
+	substrings []string
+}
+
+// NotContains returns a composable guard Evaluator: full credit unless the
+// response contains any of substrings (case-insensitive), in which case it
+// scores zero. Useful to penalize unsafe/undesired suggestions (e.g.
+// "kubectl edit").
+func NotContains(substrings ...string) Evaluator {
+	return notContains{substrings: substrings}
+}
+
+func (c notContains) Evaluate(_ context.Context, response string) Score {
+	lower := strings.ToLower(response)
+	for _, s := range c.substrings {
+		if strings.Contains(lower, strings.ToLower(s)) {
+			return Score{Value: 0, Detail: fmt.Sprintf("forbidden %q present", s)}
+		}
+	}
+	return Score{Value: 1, Detail: "no forbidden substrings present"}
+}

internal/eval/contains_test.go

@@ -0,0 +1,71 @@
+package eval
+
+import (
+	"context"
+	"testing"
+)
+
+func TestContainsAll(t *testing.T) {
+	tests := []struct {
+		name       string
+		response   string
+		substrings []string
+		want       float64
+	}{
+		{"all present", "use launchd with StartInterval", []string{"launchd", "StartInterval"}, 1},
+		{"case insensitive", "use Launchd here", []string{"LAUNCHD"}, 1},
+		{"partial credit", "alpha and beta only", []string{"alpha", "beta", "gamma", "delta"}, 0.5},
+		{"none present", "nothing matches", []string{"x", "y"}, 0},
+		{"empty list", "anything", nil, 1},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := ContainsAll(tt.substrings...).Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("ContainsAll(%v).Evaluate(%q) = %v, want %v (detail: %s)", tt.substrings, tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}
+
+func TestContainsAny(t *testing.T) {
+	tests := []struct {
+		name       string
+		response   string
+		substrings []string
+		want       float64
+	}{
+		{"one matches", "install coreutils via brew", []string{"coreutils", "gtimeout"}, 1},
+		{"case insensitive match", "use gtimeout instead", []string{"GTIMEOUT"}, 1},
+		{"none match", "totally unrelated text", []string{"foo", "bar"}, 0},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := ContainsAny(tt.substrings...).Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("ContainsAny(%v).Evaluate(%q) = %v, want %v", tt.substrings, tt.response, got.Value, tt.want)
+			}
+		})
+	}
+}
+
+func TestNotContains(t *testing.T) {
+	tests := []struct {
+		name       string
+		response   string
+		substrings []string
+		want       float64
+	}{
+		{"forbidden absent", "git commit the manifest change", []string{"kubectl edit"}, 1},
+		{"forbidden present", "just run kubectl edit deployment", []string{"kubectl edit"}, 0},
+		{"case insensitive forbidden", "run kubectl patch now", []string{"KUBECTL PATCH"}, 0},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := NotContains(tt.substrings...).Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("NotContains(%v).Evaluate(%q) = %v, want %v", tt.substrings, tt.response, got.Value, tt.want)
+			}
+		})
+	}
+}

internal/eval/equals.go

@@ -0,0 +1,27 @@
+package eval
+
+import (
+	"context"
+	"fmt"
+	"strings"
+)
+
+// equalsEval scores full credit on a trimmed, case-insensitive exact match.
+type equalsEval struct {
+	want string
+}
+
+// Equals returns an Evaluator awarding full credit when the response,
+// trimmed and lower-cased, equals want (also trimmed and lower-cased).
+func Equals(want string) Evaluator {
+	return equalsEval{want: want}
+}
+
+func (e equalsEval) Evaluate(_ context.Context, response string) Score {
+	got := strings.ToLower(strings.TrimSpace(response))
+	want := strings.ToLower(strings.TrimSpace(e.want))
+	if got == want {
+		return Score{Value: 1, Detail: fmt.Sprintf("equals %q", e.want)}
+	}
+	return Score{Value: 0, Detail: fmt.Sprintf("got %q, want %q", strings.TrimSpace(response), e.want)}
+}

internal/eval/equals_test.go

@@ -0,0 +1,29 @@
+package eval
+
+import (
+	"context"
+	"testing"
+)
+
+func TestEquals(t *testing.T) {
+	tests := []struct {
+		name     string
+		want     string
+		response string
+		wantVal  float64
+	}{
+		{"exact match", "patch", "patch", 1},
+		{"case insensitive", "PATCH", "patch", 1},
+		{"trimmed whitespace", "patch", "  patch\n", 1},
+		{"mismatch", "patch", "minor", 0},
+		{"substring is not equal", "patch", "patch bump", 0},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := Equals(tt.want).Evaluate(context.Background(), tt.response)
+			if got.Value != tt.wantVal {
+				t.Errorf("Equals(%q).Evaluate(%q) = %v, want %v", tt.want, tt.response, got.Value, tt.wantVal)
+			}
+		})
+	}
+}

internal/eval/eval.go

@@ -0,0 +1,40 @@
+// Package eval provides deterministic evaluators that score a (normalized)
+// LLM response against an expected answer. All evaluators are pure functions
+// of the response text plus evaluator configuration; none call the network.
+package eval
+
+import "context"
+
+// Score is the outcome of evaluating one response. Value is in [0, 1] unless
+// Skipped is true (e.g. a required toolchain is missing), in which case
+// Value carries no meaning and the test is excluded from aggregates.
+type Score struct {
+	Detail  string
+	Value   float64
+	Skipped bool
+}
+
+// Evaluator scores a normalized response string.
+type Evaluator interface {
+	Evaluate(ctx context.Context, response string) Score
+}
+
+// EvaluatorFunc adapts a plain function to the Evaluator interface.
+type EvaluatorFunc func(ctx context.Context, response string) Score
+
+// Evaluate implements Evaluator.
+func (f EvaluatorFunc) Evaluate(ctx context.Context, response string) Score {
+	return f(ctx, response)
+}
+
+// clamp01 constrains v to the closed interval [0, 1].
+func clamp01(v float64) float64 {
+	switch {
+	case v < 0:
+		return 0
+	case v > 1:
+		return 1
+	default:
+		return v
+	}
+}

internal/eval/exec.go

@@ -0,0 +1,331 @@
+package eval
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"regexp"
+	"strings"
+	"sync"
+	"time"
+)
+
+// leadingPackageClausePattern matches a Go source line that is nothing but
+// a package clause (optionally indented), used to defensively strip a
+// model's own "package xxx" line before splicing its code into
+// solution.go. Only the first non-blank line is ever checked/stripped; a
+// package clause appearing later (inside a string, comment, or as a
+// genuine syntax error) is left alone.
+var leadingPackageClausePattern = regexp.MustCompile(`^\s*package\s+\w+\s*$`)
+
+// stripLeadingPackageClause removes a leading "package xxx" line from code,
+// if the first non-blank line is one. This lets a model's response be
+// either a bare code fragment or a full file (package clause + imports +
+// declarations) and still compile as part of solution.go's package main.
+func stripLeadingPackageClause(code string) string {
+	lines := strings.Split(code, "\n")
+	for i, line := range lines {
+		if strings.TrimSpace(line) == "" {
+			continue
+		}
+		if leadingPackageClausePattern.MatchString(line) {
+			return strings.Join(append(lines[:i:i], lines[i+1:]...), "\n")
+		}
+		break
+	}
+	return code
+}
+
+// execTimeout bounds every compile/run step of the exec evaluators.
+const execTimeout = 30 * time.Second
+
+// CodePlaceholder marks where the extracted response code is substituted
+// into a harness template.
+const CodePlaceholder = "{{CODE}}"
+
+// PassthroughHarness is a harness that runs the extracted code verbatim,
+// with no surrounding driver. Use it when the response is expected to be a
+// complete, self-contained program (e.g. a full Python script) rather than
+// a fragment plugged into a test driver.
+const PassthroughHarness = CodePlaceholder
+
+// buildSource substitutes code into harness at CodePlaceholder.
+func buildSource(harness, code string) string {
+	return strings.ReplaceAll(harness, CodePlaceholder, code)
+}
+
+// goEnvCache memoizes `go env GOCACHE`/`go env GOMODCACHE` so every
+// exec-evaluator call does not pay for an extra subprocess: they are read
+// once per process and reused.
+var (
+	goEnvOnce     sync.Once
+	goCacheDir    string
+	goModCacheDir string
+)
+
+// goEnvValue shells out to `go env key` and returns its trimmed output, or
+// "" if the go tool is unavailable or the call fails.
+func goEnvValue(key string) string {
+	// #nosec G204 -- key is always one of the two fixed, hardcoded literals
+	// ("GOCACHE"/"GOMODCACHE") passed by minimalExecEnv below, never
+	// response/user-controlled input.
+	out, err := exec.Command("go", "env", key).Output() //nolint:gosec // key is one of two hardcoded literals, see comment above
+	if err != nil {
+		return ""
+	}
+	return strings.TrimSpace(string(out))
+}
+
+// minimalExecEnv builds a minimal environment allowlist for compiling and
+// running model-generated code, rather than inheriting the operator's full
+// environment: PATH (needed to find go/python3/cc and any tool they shell
+// out to), HOME and TMPDIR pinned to homeDir (an isolated, single-call temp
+// directory, never the operator's real home), and GOCACHE/GOMODCACHE
+// inherited from the operator's actual Go environment so `go run` reuses
+// the existing build/module cache instead of rebuilding it from scratch on
+// every call.
+func minimalExecEnv(homeDir string) []string {
+	goEnvOnce.Do(func() {
+		goCacheDir = goEnvValue("GOCACHE")
+		goModCacheDir = goEnvValue("GOMODCACHE")
+	})
+
+	env := []string{
+		"PATH=" + os.Getenv("PATH"),
+		"HOME=" + homeDir,
+		"TMPDIR=" + homeDir,
+	}
+	if goCacheDir != "" {
+		env = append(env, "GOCACHE="+goCacheDir)
+	}
+	if goModCacheDir != "" {
+		env = append(env, "GOMODCACHE="+goModCacheDir)
+	}
+	return env
+}
+
+// mkWorkDirs creates an isolated per-call temp root plus a "work"
+// subdirectory inside it. Callers write source files into work and run
+// commands with cmd.Dir = work, while pointing HOME/TMPDIR (via
+// minimalExecEnv) at root, never at work itself: the Go toolchain
+// specifically refuses to detect a go.mod when the module directory is
+// exactly equal to the resolved system temp root ("ignoring go.mod in
+// system temp root"), which fires precisely when TMPDIR is set to the same
+// directory `go run` is invoked from. Keeping root and work as distinct
+// (parent, child) directories avoids that while still confining
+// HOME/TMPDIR to a directory nothing else uses.
+func mkWorkDirs(prefix string) (root, work string, err error) {
+	root, err = os.MkdirTemp("", prefix)
+	if err != nil {
+		return "", "", err
+	}
+	work = filepath.Join(root, "work")
+	if err := os.Mkdir(work, 0o700); err != nil {
+		return "", "", err
+	}
+	return root, work, nil
+}
+
+// checkToolchain reports a Skipped score if name is not on PATH, nil
+// otherwise.
+func checkToolchain(name string) *Score {
+	if _, err := exec.LookPath(name); err != nil {
+		s := Score{Skipped: true, Detail: fmt.Sprintf("toolchain missing: %s", name)}
+		return &s
+	}
+	return nil
+}
+
+// compareStdout scores full credit when stdout, trimmed, equals want,
+// trimmed. cmdErr, when non-nil, always scores zero and surfaces stderr.
+func compareStdout(stdout, want string, cmdErr error, stderr string) Score {
+	if cmdErr != nil {
+		return Score{Value: 0, Detail: fmt.Sprintf("execution failed: %v; stderr: %s", cmdErr, strings.TrimSpace(stderr))}
+	}
+	got := strings.TrimSpace(stdout)
+	wantTrimmed := strings.TrimSpace(want)
+	if got == wantTrimmed {
+		return Score{Value: 1, Detail: "stdout matches expected"}
+	}
+	return Score{Value: 0, Detail: fmt.Sprintf("stdout = %q, want %q", got, wantTrimmed)}
+}
+
+// goRunEval compiles/runs a two-file Go module: solution.go (the model's
+// code, wrapped in package main) plus harness.go (a complete, independent
+// driver file supplied by the test author).
+type goRunEval struct {
+	harness string
+	want    string
+}
+
+// GoRun returns an Evaluator that writes the response's first fenced Go
+// code block to its own solution.go (defensively stripping a leading
+// "package xxx" line and prefixing "package main"), writes harness
+// verbatim to its own harness.go (a complete file: its own package clause,
+// imports, and func main), `go run`s both together in an isolated temp
+// module, and compares trimmed stdout to want. Splitting the model's code
+// and the harness into separate files means each may import what it needs
+// independently, instead of splicing code into a single shared import
+// block. If the `go` toolchain is unavailable, the Score is Skipped rather
+// than zero.
+func GoRun(harness, want string) Evaluator {
+	return goRunEval{harness: harness, want: want}
+}
+
+func (g goRunEval) Evaluate(ctx context.Context, response string) Score {
+	if s := checkToolchain("go"); s != nil {
+		return *s
+	}
+
+	code := ExtractCodeBlock(response, "go")
+	solution := "package main\n\n" + stripLeadingPackageClause(code)
+
+	root, work, err := mkWorkDirs("llmtb-go-*")
+	if err != nil {
+		return Score{Value: 0, Detail: fmt.Sprintf("mkdtemp: %v", err)}
+	}
+	defer func() { _ = os.RemoveAll(root) }() // best-effort cleanup; failure is not actionable
+
+	if writeErr := os.WriteFile(filepath.Join(work, "solution.go"), []byte(solution), 0o600); writeErr != nil {
+		return Score{Value: 0, Detail: fmt.Sprintf("write solution.go: %v", writeErr)}
+	}
+	if writeErr := os.WriteFile(filepath.Join(work, "harness.go"), []byte(g.harness), 0o600); writeErr != nil {
+		return Score{Value: 0, Detail: fmt.Sprintf("write harness.go: %v", writeErr)}
+	}
+	if modErr := os.WriteFile(filepath.Join(work, "go.mod"), []byte("module llmtbharness\n\ngo 1.23\n"), 0o600); modErr != nil {
+		return Score{Value: 0, Detail: fmt.Sprintf("write go.mod: %v", modErr)}
+	}
+
+	runCtx, cancel := context.WithTimeout(ctx, execTimeout)
+	defer cancel()
+
+	cmd := exec.CommandContext(runCtx, "go", "run", ".")
+	cmd.Dir = work
+	cmd.Env = minimalExecEnv(root)
+	var stdout, stderr bytes.Buffer
+	cmd.Stdout = &stdout
+	cmd.Stderr = &stderr
+	err = cmd.Run()
+
+	return compareStdout(stdout.String(), g.want, err, stderr.String())
+}
+
+// pyRunEval runs a Python program built from a harness template plus the
+// extracted Python code block from the response.
+type pyRunEval struct {
+	harness string
+	want    string
+}
+
+// PyRun returns an Evaluator that substitutes the response's first fenced
+// Python code block into harness at CodePlaceholder, runs it with python3,
+// and compares trimmed stdout to want. If python3 is unavailable, the Score
+// is Skipped rather than zero.
+func PyRun(harness, want string) Evaluator {
+	return pyRunEval{harness: harness, want: want}
+}
+
+func (p pyRunEval) Evaluate(ctx context.Context, response string) Score {
+	if s := checkToolchain("python3"); s != nil {
+		return *s
+	}
+
+	code := ExtractCodeBlock(response, "python")
+	source := buildSource(p.harness, code)
+
+	root, work, err := mkWorkDirs("llmtb-py-*")
+	if err != nil {
+		return Score{Value: 0, Detail: fmt.Sprintf("mkdtemp: %v", err)}
+	}
+	defer func() { _ = os.RemoveAll(root) }() // best-effort cleanup; failure is not actionable
+
+	scriptPath := filepath.Join(work, "script.py")
+	if writeErr := os.WriteFile(scriptPath, []byte(source), 0o600); writeErr != nil {
+		return Score{Value: 0, Detail: fmt.Sprintf("write script.py: %v", writeErr)}
+	}
+
+	runCtx, cancel := context.WithTimeout(ctx, execTimeout)
+	defer cancel()
+
+	// #nosec G204 -- this is the eval package's whole purpose: compile/run
+	// model-generated code under an isolated temp dir and a minimal env
+	// allowlist (minimalExecEnv). See README's "Requirements" section for
+	// the corresponding operator-facing warning (S13).
+	cmd := exec.CommandContext(runCtx, "python3", scriptPath) //nolint:gosec // see comment above
+	cmd.Dir = work
+	cmd.Env = minimalExecEnv(root)
+	var stdout, stderr bytes.Buffer
+	cmd.Stdout = &stdout
+	cmd.Stderr = &stderr
+	err = cmd.Run()
+
+	return compareStdout(stdout.String(), p.want, err, stderr.String())
+}
+
+// cRunEval compiles/runs a C program built from a harness template plus the
+// extracted C code block from the response.
+type cRunEval struct {
+	harness string
+	want    string
+}
+
+// CRun returns an Evaluator that substitutes the response's first fenced C
+// code block into harness at CodePlaceholder, compiles it with cc, runs the
+// binary, and compares trimmed stdout to want. If cc is unavailable, the
+// Score is Skipped rather than zero.
+func CRun(harness, want string) Evaluator {
+	return cRunEval{harness: harness, want: want}
+}
+
+func (c cRunEval) Evaluate(ctx context.Context, response string) Score {
+	if s := checkToolchain("cc"); s != nil {
+		return *s
+	}
+
+	code := ExtractCodeBlock(response, "c")
+	source := buildSource(c.harness, code)
+
+	root, work, err := mkWorkDirs("llmtb-c-*")
+	if err != nil {
+		return Score{Value: 0, Detail: fmt.Sprintf("mkdtemp: %v", err)}
+	}
+	defer func() { _ = os.RemoveAll(root) }() // best-effort cleanup; failure is not actionable
+
+	srcPath := filepath.Join(work, "main.c")
+	if writeErr := os.WriteFile(srcPath, []byte(source), 0o600); writeErr != nil {
+		return Score{Value: 0, Detail: fmt.Sprintf("write main.c: %v", writeErr)}
+	}
+	binPath := filepath.Join(work, "prog")
+
+	buildCtx, buildCancel := context.WithTimeout(ctx, execTimeout)
+	defer buildCancel()
+	// #nosec G204 -- compiling model-generated C source under an isolated
+	// temp dir and a minimal env allowlist is this evaluator's job; see the
+	// PyRun comment above and README's "Requirements" warning (S13).
+	buildCmd := exec.CommandContext(buildCtx, "cc", "-o", binPath, srcPath) //nolint:gosec // see comment above
+	buildCmd.Dir = work
+	buildCmd.Env = minimalExecEnv(root)
+	var buildStderr bytes.Buffer
+	buildCmd.Stderr = &buildStderr
+	if buildErr := buildCmd.Run(); buildErr != nil {
+		return Score{Value: 0, Detail: fmt.Sprintf("cc failed: %v; stderr: %s", buildErr, strings.TrimSpace(buildStderr.String()))}
+	}
+
+	runCtx, runCancel := context.WithTimeout(ctx, execTimeout)
+	defer runCancel()
+	// #nosec G204 -- binPath is the binary this same call just compiled,
+	// into a fresh, isolated temp dir this evaluator created; not an
+	// externally supplied executable path.
+	runCmd := exec.CommandContext(runCtx, binPath) //nolint:gosec // see comment above
+	runCmd.Dir = work
+	runCmd.Env = minimalExecEnv(root)
+	var stdout, stderr bytes.Buffer
+	runCmd.Stdout = &stdout
+	runCmd.Stderr = &stderr
+	err = runCmd.Run()
+
+	return compareStdout(stdout.String(), c.want, err, stderr.String())
+}

internal/eval/exec_test.go

@@ -0,0 +1,186 @@
+package eval
+
+import (
+	"context"
+	"os/exec"
+	"testing"
+)
+
+func requireToolchain(t *testing.T, name string) {
+	t.Helper()
+	if _, err := exec.LookPath(name); err != nil {
+		t.Skipf("toolchain %s not found on PATH, skipping", name)
+	}
+}
+
+// goHarness is a complete, standalone harness.go: its own package clause
+// and imports, calling Double (defined separately in solution.go, same
+// package) - no {{CODE}} placeholder, per GoRun's two-file contract.
+const goHarness = `package main
+
+import "fmt"
+
+func main() {
+	fmt.Println(Double(21))
+}
+`
+
+func TestGoRun_Pass(t *testing.T) {
+	requireToolchain(t, "go")
+	e := GoRun(goHarness, "42")
+	response := "```go\nfunc Double(n int) int { return n * 2 }\n```"
+	got := e.Evaluate(context.Background(), response)
+	if got.Value != 1 {
+		t.Errorf("Evaluate() = %v, want 1 (detail: %s)", got.Value, got.Detail)
+	}
+}
+
+func TestGoRun_WrongOutput(t *testing.T) {
+	requireToolchain(t, "go")
+	e := GoRun(goHarness, "42")
+	response := "```go\nfunc Double(n int) int { return n + 2 }\n```"
+	got := e.Evaluate(context.Background(), response)
+	if got.Value != 0 {
+		t.Errorf("Evaluate() = %v, want 0 (detail: %s)", got.Value, got.Detail)
+	}
+}
+
+func TestGoRun_CompileError(t *testing.T) {
+	requireToolchain(t, "go")
+	e := GoRun(goHarness, "42")
+	response := "```go\nfunc Double(n int) int { this is not valid go\n```"
+	got := e.Evaluate(context.Background(), response)
+	if got.Value != 0 {
+		t.Errorf("Evaluate() = %v, want 0 for compile error", got.Value)
+	}
+	if got.Skipped {
+		t.Error("Evaluate() should not be Skipped for a compile error")
+	}
+}
+
+func TestStripLeadingPackageClause(t *testing.T) {
+	tests := []struct {
+		name string
+		code string
+		want string
+	}{
+		{
+			name: "leading package clause is stripped",
+			code: "package main\n\nfunc F() {}",
+			want: "\nfunc F() {}",
+		},
+		{
+			name: "blank lines before package clause are tolerated",
+			code: "\n\npackage foo\nfunc F() {}",
+			want: "\n\nfunc F() {}",
+		},
+		{
+			name: "no package clause leaves code untouched",
+			code: "import \"fmt\"\n\nfunc F() {}",
+			want: "import \"fmt\"\n\nfunc F() {}",
+		},
+		{
+			name: "package clause not on the first line is left alone",
+			code: "func F() {}\n\npackage main",
+			want: "func F() {}\n\npackage main",
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := stripLeadingPackageClause(tt.code)
+			if got != tt.want {
+				t.Errorf("stripLeadingPackageClause(%q) = %q, want %q", tt.code, got, tt.want)
+			}
+		})
+	}
+}
+
+func TestCheckToolchain_MissingBinaryIsSkipped(t *testing.T) {
+	s := checkToolchain("definitely-not-a-real-toolchain-xyz")
+	if s == nil || !s.Skipped {
+		t.Fatal("checkToolchain() for missing binary should return Skipped score")
+	}
+}
+
+const pyPassthroughHarness = PassthroughHarness
+
+func TestPyRun_Pass(t *testing.T) {
+	requireToolchain(t, "python3")
+	e := PyRun(pyPassthroughHarness, "OOMKilled=2\nImagePullBackOff=1")
+	response := "```python\nprint('OOMKilled=2')\nprint('ImagePullBackOff=1')\n```"
+	got := e.Evaluate(context.Background(), response)
+	if got.Value != 1 {
+		t.Errorf("Evaluate() = %v, want 1 (detail: %s)", got.Value, got.Detail)
+	}
+}
+
+func TestPyRun_WrongOutput(t *testing.T) {
+	requireToolchain(t, "python3")
+	e := PyRun(pyPassthroughHarness, "OOMKilled=2")
+	response := "```python\nprint('OOMKilled=3')\n```"
+	got := e.Evaluate(context.Background(), response)
+	if got.Value != 0 {
+		t.Errorf("Evaluate() = %v, want 0 (detail: %s)", got.Value, got.Detail)
+	}
+}
+
+func TestPyRun_RuntimeError(t *testing.T) {
+	requireToolchain(t, "python3")
+	e := PyRun(pyPassthroughHarness, "anything")
+	response := "```python\nraise ValueError('boom')\n```"
+	got := e.Evaluate(context.Background(), response)
+	if got.Value != 0 || got.Skipped {
+		t.Errorf("Evaluate() = %+v, want zero non-skipped score for runtime error", got)
+	}
+}
+
+const cHarness = `#include <stdio.h>
+
+{{CODE}}
+
+int main(void) {
+    printf("%d\n", triple(7));
+    return 0;
+}
+`
+
+func TestCRun_Pass(t *testing.T) {
+	requireToolchain(t, "cc")
+	e := CRun(cHarness, "21")
+	response := "```c\nint triple(int n) { return n * 3; }\n```"
+	got := e.Evaluate(context.Background(), response)
+	if got.Value != 1 {
+		t.Errorf("Evaluate() = %v, want 1 (detail: %s)", got.Value, got.Detail)
+	}
+}
+
+func TestCRun_CompileError(t *testing.T) {
+	requireToolchain(t, "cc")
+	e := CRun(cHarness, "21")
+	response := "```c\nthis is not valid c code {{{\n```"
+	got := e.Evaluate(context.Background(), response)
+	if got.Value != 0 {
+		t.Errorf("Evaluate() = %v, want 0 for compile error", got.Value)
+	}
+}
+
+func TestExecEvaluators_ToolchainMissingSkipsNotFails(t *testing.T) {
+	// This test does not require any toolchain; it verifies the Skipped
+	// contract directly against a bogus PATH so it runs in every environment.
+	t.Setenv("PATH", t.TempDir())
+
+	goScore := GoRun(goHarness, "42").Evaluate(context.Background(), "```go\nfunc Double(n int) int { return n*2 }\n```")
+	if !goScore.Skipped {
+		t.Errorf("GoRun with empty PATH: Skipped = false, want true (%+v)", goScore)
+	}
+
+	pyScore := PyRun(pyPassthroughHarness, "x").Evaluate(context.Background(), "```python\nprint('x')\n```")
+	if !pyScore.Skipped {
+		t.Errorf("PyRun with empty PATH: Skipped = false, want true (%+v)", pyScore)
+	}
+
+	cScore := CRun(cHarness, "21").Evaluate(context.Background(), "```c\nint triple(int n){return n*3;}\n```")
+	if !cScore.Skipped {
+		t.Errorf("CRun with empty PATH: Skipped = false, want true (%+v)", cScore)
+	}
+}

internal/eval/extract.go

@@ -0,0 +1,122 @@
+package eval
+
+import (
+	"fmt"
+	"regexp"
+	"strings"
+)
+
+// fencePattern matches a fenced code block: ```lang\n...body...```
+var fencePattern = regexp.MustCompile("(?s)```([a-zA-Z0-9_+-]*)\\n?(.*?)```")
+
+// ExtractCodeBlock returns the first fenced code block tagged with lang
+// (case-insensitive). If none is tagged, it falls back to the first
+// untagged fenced block, then to the first fenced block of any language,
+// then to the whole trimmed response if there are no fences at all.
+func ExtractCodeBlock(response, lang string) string {
+	matches := fencePattern.FindAllStringSubmatch(response, -1)
+	wantLang := strings.ToLower(strings.TrimSpace(lang))
+
+	var untagged string
+	for _, m := range matches {
+		blockLang := strings.ToLower(strings.TrimSpace(m[1]))
+		if wantLang != "" && blockLang == wantLang {
+			return strings.TrimSpace(m[2])
+		}
+		if untagged == "" && blockLang == "" {
+			untagged = m[2]
+		}
+	}
+	if untagged != "" {
+		return strings.TrimSpace(untagged)
+	}
+	if len(matches) > 0 {
+		return strings.TrimSpace(matches[0][2])
+	}
+	return strings.TrimSpace(response)
+}
+
+// jsonFencePattern matches a fenced ```json ... ``` block specifically
+// (case-insensitive on the "json" language tag).
+var jsonFencePattern = regexp.MustCompile("(?is)```json\\s*\\n?(.*?)```")
+
+// findAllBalancedJSON scans text for every top-level, bracket-balanced JSON
+// object/array literal, in the order they appear. Brackets inside string
+// literals are ignored. An unbalanced trailing fragment (an opening
+// bracket with no matching close) is not included.
+func findAllBalancedJSON(text string) []string {
+	var out []string
+	pos := 0
+	for pos < len(text) {
+		start := -1
+		for i := pos; i < len(text); i++ {
+			if text[i] == '{' || text[i] == '[' {
+				start = i
+				break
+			}
+		}
+		if start == -1 {
+			break
+		}
+
+		depth := 0
+		inString := false
+		escaped := false
+		end := -1
+		for i := start; i < len(text); i++ {
+			c := text[i]
+			if inString {
+				switch {
+				case escaped:
+					escaped = false
+				case c == '\\':
+					escaped = true
+				case c == '"':
+					inString = false
+				}
+				continue
+			}
+			switch c {
+			case '"':
+				inString = true
+			case '{', '[':
+				depth++
+			case '}', ']':
+				depth--
+				if depth == 0 {
+					end = i
+				}
+			}
+			if end != -1 {
+				break
+			}
+		}
+		if end == -1 {
+			break
+		}
+		out = append(out, text[start:end+1])
+		pos = end + 1
+	}
+	return out
+}
+
+// ExtractJSON returns response's intended JSON object/array literal,
+// tolerant of surrounding prose and markdown code fences. It prefers the
+// content of the first ```json fenced block, if any; otherwise it takes
+// the LAST balanced JSON value anywhere in response. Taking the last
+// (rather than the first) value matters for a model that thinks out loud
+// with a candidate JSON draft before committing to a final answer: the
+// final JSON, not the draft, is the one to score.
+func ExtractJSON(response string) (string, error) {
+	if m := jsonFencePattern.FindStringSubmatch(response); m != nil {
+		if all := findAllBalancedJSON(m[1]); len(all) > 0 {
+			return all[0], nil
+		}
+	}
+
+	all := findAllBalancedJSON(response)
+	if len(all) == 0 {
+		return "", fmt.Errorf("no JSON object or array found in response")
+	}
+	return all[len(all)-1], nil
+}

internal/eval/extract_test.go

@@ -0,0 +1,51 @@
+package eval
+
+import "testing"
+
+func TestExtractCodeBlock(t *testing.T) {
+	tests := []struct {
+		name     string
+		response string
+		lang     string
+		want     string
+	}{
+		{
+			name:     "tagged block matches lang",
+			response: "Here:\n```go\nfunc F() {}\n```\nDone.",
+			lang:     "go",
+			want:     "func F() {}",
+		},
+		{
+			name:     "case insensitive lang tag",
+			response: "```GO\nfunc F() {}\n```",
+			lang:     "go",
+			want:     "func F() {}",
+		},
+		{
+			name:     "prefers matching lang over other tagged blocks",
+			response: "```json\n{\"x\":1}\n```\n```go\nfunc F() {}\n```",
+			lang:     "go",
+			want:     "func F() {}",
+		},
+		{
+			name:     "falls back to untagged block",
+			response: "```\nfunc F() {}\n```",
+			lang:     "go",
+			want:     "func F() {}",
+		},
+		{
+			name:     "no fences falls back to whole response",
+			response: "  func F() {}  ",
+			lang:     "go",
+			want:     "func F() {}",
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := ExtractCodeBlock(tt.response, tt.lang)
+			if got != tt.want {
+				t.Errorf("ExtractCodeBlock(%q, %q) = %q, want %q", tt.response, tt.lang, got, tt.want)
+			}
+		})
+	}
+}

internal/eval/jsonarray.go

@@ -0,0 +1,107 @@
+package eval
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"strings"
+)
+
+// decodeStringArray extracts the first JSON value from response and decodes
+// it as a []string. Elements that are not strings cause an error.
+func decodeStringArray(response string) ([]string, error) {
+	raw, err := ExtractJSON(response)
+	if err != nil {
+		return nil, err
+	}
+	var arr []string
+	if err := json.Unmarshal([]byte(raw), &arr); err != nil {
+		return nil, fmt.Errorf("invalid JSON string array: %w", err)
+	}
+	return arr, nil
+}
+
+// jsonStringSetEval scores a JSON array of strings as an unordered set,
+// case-insensitive and whitespace-trimmed.
+type jsonStringSetEval struct {
+	want []string
+}
+
+// JSONStringSet returns an Evaluator that parses the first JSON array in the
+// response as a set of strings (case-insensitive, trimmed) and compares it
+// to want, order-independent. Score is the Jaccard-style overlap
+// |intersection| / max(len(want), len(got)), so both missing and extraneous
+// entries cost credit; full credit requires an exact set match.
+func JSONStringSet(want []string) Evaluator {
+	return jsonStringSetEval{want: want}
+}
+
+func normalizeSet(items []string) map[string]struct{} {
+	set := make(map[string]struct{}, len(items))
+	for _, s := range items {
+		set[strings.ToLower(strings.TrimSpace(s))] = struct{}{}
+	}
+	return set
+}
+
+func (j jsonStringSetEval) Evaluate(_ context.Context, response string) Score {
+	got, err := decodeStringArray(response)
+	if err != nil {
+		return Score{Value: 0, Detail: err.Error()}
+	}
+
+	wantSet := normalizeSet(j.want)
+	gotSet := normalizeSet(got)
+
+	matched := 0
+	for k := range gotSet {
+		if _, ok := wantSet[k]; ok {
+			matched++
+		}
+	}
+
+	denom := len(wantSet)
+	if len(gotSet) > denom {
+		denom = len(gotSet)
+	}
+	if denom == 0 {
+		return Score{Value: 1, Detail: "both sets empty"}
+	}
+
+	value := float64(matched) / float64(denom)
+	return Score{
+		Value:  value,
+		Detail: fmt.Sprintf("matched %d/%d (got %v, want %v)", matched, denom, got, j.want),
+	}
+}
+
+// jsonStringArrayEqualsEval scores full credit only on an exact, in-order
+// match of a JSON string array.
+type jsonStringArrayEqualsEval struct {
+	want []string
+}
+
+// JSONStringArrayEquals returns an Evaluator that parses the first JSON
+// array in the response as []string and awards full credit only when it
+// exactly equals want element-for-element, in order (case-insensitive,
+// trimmed). Any mismatch, including reordering, scores zero.
+func JSONStringArrayEquals(want []string) Evaluator {
+	return jsonStringArrayEqualsEval{want: want}
+}
+
+func (j jsonStringArrayEqualsEval) Evaluate(_ context.Context, response string) Score {
+	got, err := decodeStringArray(response)
+	if err != nil {
+		return Score{Value: 0, Detail: err.Error()}
+	}
+
+	if len(got) != len(j.want) {
+		return Score{Value: 0, Detail: fmt.Sprintf("length %d, want %d (got %v, want %v)", len(got), len(j.want), got, j.want)}
+	}
+	for i := range got {
+		if !strings.EqualFold(strings.TrimSpace(got[i]), strings.TrimSpace(j.want[i])) {
+			return Score{Value: 0, Detail: fmt.Sprintf("index %d: got %q, want %q (got %v, want %v)", i, got[i], j.want[i], got, j.want)}
+		}
+	}
+	return Score{Value: 1, Detail: fmt.Sprintf("exact match: %v", got)}
+}

internal/eval/jsonfield.go

@@ -0,0 +1,184 @@
+package eval
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"strconv"
+	"strings"
+)
+
+// pathSegment is one step of a dot/bracket JSON path, e.g. "a.b[0]" is
+// [{key:"a"} {key:"b"} {index:0, isIndex:true}].
+type pathSegment struct {
+	key     string
+	index   int
+	isIndex bool
+}
+
+// parsePath splits a path like "a.b[0].c" into segments. An empty path
+// yields no segments (navigate returns the root value unchanged).
+func parsePath(path string) ([]pathSegment, error) {
+	var segments []pathSegment
+	if path == "" {
+		return segments, nil
+	}
+	for _, rawPart := range strings.Split(path, ".") {
+		part := rawPart
+		for part != "" {
+			if idx := strings.IndexByte(part, '['); idx >= 0 {
+				if idx > 0 {
+					segments = append(segments, pathSegment{key: part[:idx]})
+				}
+				end := strings.IndexByte(part, ']')
+				if end < idx {
+					return nil, fmt.Errorf("malformed path %q: unmatched '['", path)
+				}
+				n, err := strconv.Atoi(part[idx+1 : end])
+				if err != nil {
+					return nil, fmt.Errorf("malformed path %q: bad index: %w", path, err)
+				}
+				segments = append(segments, pathSegment{index: n, isIndex: true})
+				part = part[end+1:]
+				continue
+			}
+			segments = append(segments, pathSegment{key: part})
+			part = ""
+		}
+	}
+	return segments, nil
+}
+
+// navigate walks v (as decoded by encoding/json into any) following segments.
+func navigate(v any, segments []pathSegment) (any, error) {
+	cur := v
+	for _, seg := range segments {
+		if seg.isIndex {
+			arr, ok := cur.([]any)
+			if !ok {
+				return nil, fmt.Errorf("expected array at index [%d], got %T", seg.index, cur)
+			}
+			if seg.index < 0 || seg.index >= len(arr) {
+				return nil, fmt.Errorf("index [%d] out of range (len %d)", seg.index, len(arr))
+			}
+			cur = arr[seg.index]
+			continue
+		}
+		m, ok := cur.(map[string]any)
+		if !ok {
+			return nil, fmt.Errorf("expected object at field %q, got %T", seg.key, cur)
+		}
+		val, ok := m[seg.key]
+		if !ok {
+			return nil, fmt.Errorf("field %q not found", seg.key)
+		}
+		cur = val
+	}
+	return cur, nil
+}
+
+// jsonValueAs converts a decoded JSON value (string/float64/bool/etc.) to T,
+// dispatching on the runtime type of the zero value of T.
+func jsonValueAs[T comparable](v any) (T, error) {
+	var zero T
+	switch any(zero).(type) {
+	case string:
+		s, ok := v.(string)
+		if !ok {
+			return zero, fmt.Errorf("expected string, got %T (%v)", v, v)
+		}
+		return any(s).(T), nil
+	case bool:
+		b, ok := v.(bool)
+		if !ok {
+			return zero, fmt.Errorf("expected bool, got %T (%v)", v, v)
+		}
+		return any(b).(T), nil
+	case float64:
+		f, err := asFloat64(v)
+		if err != nil {
+			return zero, err
+		}
+		return any(f).(T), nil
+	case int:
+		f, err := asFloat64(v)
+		if err != nil {
+			return zero, err
+		}
+		return any(int(f)).(T), nil
+	default:
+		return zero, fmt.Errorf("unsupported JSONField target type %T", zero)
+	}
+}
+
+func asFloat64(v any) (float64, error) {
+	switch n := v.(type) {
+	case float64:
+		return n, nil
+	case json.Number:
+		return n.Float64()
+	default:
+		return 0, fmt.Errorf("expected number, got %T (%v)", v, v)
+	}
+}
+
+// jsonFieldEval extracts one field by path from the first JSON value in the
+// response and compares it to want.
+type jsonFieldEval[T comparable] struct {
+	want T
+	path string
+}
+
+// JSONField returns an Evaluator that extracts the first JSON object/array
+// in the response (stripping surrounding prose and code fences), navigates
+// path (dot/bracket syntax, e.g. "bump" or "steps[0]"), and awards full
+// credit when the field equals want. T is inferred from want; supported
+// types are string, bool, float64, and int. String comparison is
+// case-insensitive and trims whitespace.
+func JSONField[T comparable](path string, want T) Evaluator {
+	return jsonFieldEval[T]{path: path, want: want}
+}
+
+func (j jsonFieldEval[T]) Evaluate(_ context.Context, response string) Score {
+	raw, err := ExtractJSON(response)
+	if err != nil {
+		return Score{Value: 0, Detail: err.Error()}
+	}
+
+	dec := json.NewDecoder(strings.NewReader(raw))
+	dec.UseNumber()
+	var parsed any
+	if decodeErr := dec.Decode(&parsed); decodeErr != nil {
+		return Score{Value: 0, Detail: fmt.Sprintf("invalid JSON: %v", decodeErr)}
+	}
+
+	segments, err := parsePath(j.path)
+	if err != nil {
+		return Score{Value: 0, Detail: err.Error()}
+	}
+
+	val, err := navigate(parsed, segments)
+	if err != nil {
+		return Score{Value: 0, Detail: fmt.Sprintf("path %q: %v", j.path, err)}
+	}
+
+	got, err := jsonValueAs[T](val)
+	if err != nil {
+		return Score{Value: 0, Detail: fmt.Sprintf("path %q: %v", j.path, err)}
+	}
+
+	if equalJSONValue(got, j.want) {
+		return Score{Value: 1, Detail: fmt.Sprintf("%s = %v", j.path, got)}
+	}
+	return Score{Value: 0, Detail: fmt.Sprintf("%s: got %v, want %v", j.path, got, j.want)}
+}
+
+// equalJSONValue compares two values of the same comparable type T, folding
+// case and whitespace when T is string.
+func equalJSONValue[T comparable](got, want T) bool {
+	if gs, ok := any(got).(string); ok {
+		ws := any(want).(string)
+		return strings.EqualFold(strings.TrimSpace(gs), strings.TrimSpace(ws))
+	}
+	return got == want
+}

internal/eval/jsonfield_test.go

@@ -0,0 +1,161 @@
+package eval
+
+import (
+	"context"
+	"testing"
+)
+
+func TestExtractJSON(t *testing.T) {
+	tests := []struct {
+		name     string
+		response string
+		want     string
+		wantErr  bool
+	}{
+		{"plain object", `{"bump":"minor"}`, `{"bump":"minor"}`, false},
+		{"fenced object", "Here you go:\n```json\n{\"bump\":\"minor\"}\n```", `{"bump":"minor"}`, false},
+		{"object with prose around", `The answer is {"bump":"minor"} as computed.`, `{"bump":"minor"}`, false},
+		{"nested braces ignored inside strings", `{"note":"a { b } c","bump":"patch"}`, `{"note":"a { b } c","bump":"patch"}`, false},
+		{"array", `["ClaudeBot","Googlebot"]`, `["ClaudeBot","Googlebot"]`, false},
+		{"no json", "no json here at all", "", true},
+		// S7: a candidate JSON considered during reasoning, followed by the
+		// real final JSON, must resolve to the final one.
+		{
+			name:     "reasoning with a candidate JSON then a final JSON: final wins",
+			response: `Let me think. One option would be {"bump":"major"} but that is not right. The actual answer is {"bump":"minor"}.`,
+			want:     `{"bump":"minor"}`,
+			wantErr:  false,
+		},
+		{
+			name:     "fenced json block preferred over any other json-looking text before it",
+			response: "Draft: {\"bump\":\"major\"}\n\nFinal answer:\n```json\n{\"bump\":\"minor\"}\n```",
+			want:     `{"bump":"minor"}`,
+			wantErr:  false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := ExtractJSON(tt.response)
+			if (err != nil) != tt.wantErr {
+				t.Fatalf("ExtractJSON(%q) error = %v, wantErr %v", tt.response, err, tt.wantErr)
+			}
+			if err == nil && got != tt.want {
+				t.Errorf("ExtractJSON(%q) = %q, want %q", tt.response, got, tt.want)
+			}
+		})
+	}
+}
+
+func TestJSONField_String(t *testing.T) {
+	e := JSONField("bump", "minor")
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{"exact match", `{"bump":"minor"}`, 1},
+		{"case insensitive", `{"bump":"MINOR"}`, 1},
+		{"wrong value", `{"bump":"patch"}`, 0},
+		{"missing field", `{"other":"x"}`, 0},
+		{"fenced json", "```json\n{\"bump\": \"minor\"}\n```", 1},
+		{"invalid json", "not json at all", 0},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := e.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}
+
+func TestJSONField_Int(t *testing.T) {
+	e := JSONField("M", 16)
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{"exact match", `{"M":16,"efConstruction":200,"efSearch":50}`, 1},
+		{"wrong value", `{"M":32}`, 0},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := e.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}
+
+func TestJSONField_NestedPath(t *testing.T) {
+	e := JSONField("task1", "search_web")
+	got := e.Evaluate(context.Background(), `{"task1":"search_web","task2":"read_file"}`)
+	if got.Value != 1 {
+		t.Errorf("Evaluate() = %v, want 1 (detail: %s)", got.Value, got.Detail)
+	}
+}
+
+func TestJSONField_IndexedPath(t *testing.T) {
+	e := JSONField("steps[0]", "build")
+	got := e.Evaluate(context.Background(), `{"steps":["build","test","deploy"]}`)
+	if got.Value != 1 {
+		t.Errorf("Evaluate() = %v, want 1 (detail: %s)", got.Value, got.Detail)
+	}
+	got = e.Evaluate(context.Background(), `{"steps":["test","build","deploy"]}`)
+	if got.Value != 0 {
+		t.Errorf("Evaluate() = %v, want 0 (detail: %s)", got.Value, got.Detail)
+	}
+}
+
+func TestJSONStringSet(t *testing.T) {
+	e := JSONStringSet([]string{"ClaudeBot", "Googlebot"})
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{"exact set", `["ClaudeBot","Googlebot"]`, 1},
+		{"exact set different order", `["Googlebot","ClaudeBot"]`, 1},
+		{"case insensitive", `["claudebot","googlebot"]`, 1},
+		{"missing one", `["ClaudeBot"]`, 0.5},
+		{"extra one", `["ClaudeBot","Googlebot","GPTBot"]`, float64(2) / 3},
+		{"completely wrong", `["GPTBot","PerplexityBot"]`, 0},
+		{"invalid json", "no array here", 0},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := e.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}
+
+func TestJSONStringArrayEquals(t *testing.T) {
+	want := []string{"build", "test", "backup", "deploy", "verify"}
+	e := JSONStringArrayEquals(want)
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{"exact order", `["build","test","backup","deploy","verify"]`, 1},
+		{"case insensitive exact order", `["BUILD","TEST","BACKUP","DEPLOY","VERIFY"]`, 1},
+		{"wrong order", `["build","backup","test","deploy","verify"]`, 0},
+		{"wrong length", `["build","test"]`, 0},
+		{"invalid json", "not an array", 0},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := e.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}

internal/eval/numeric.go

@@ -0,0 +1,117 @@
+package eval
+
+import (
+	"context"
+	"fmt"
+	"regexp"
+	"strconv"
+	"strings"
+)
+
+// Number is the set of numeric primitive kinds Numeric can extract and
+// compare. Defined locally (rather than importing golang.org/x/exp/constraints)
+// since only gopkg.in/yaml.v3 and golang.org/x/sync/errgroup are allowed deps.
+type Number interface {
+	~float32 | ~float64 |
+		~int | ~int8 | ~int16 | ~int32 | ~int64 |
+		~uint | ~uint8 | ~uint16 | ~uint32 | ~uint64
+}
+
+// numberPattern matches signed integers and decimals, including a
+// leading-dot form with no integer part (".9896"). Longer alternatives are
+// listed first so the match is not truncated to just the integer part of
+// "0.9896" or just the dot-less ".9896".
+var numberPattern = regexp.MustCompile(`-?(?:\d+\.\d+|\.\d+|\d+)`)
+
+// hyphenCompoundSuffixPattern matches a hyphen directly after a number
+// joining it to a following word, e.g. the "-bit" in "64-bit" or the
+// "-byte" in "8-byte". A number immediately followed by this is part of a
+// unit/architecture descriptor, not a standalone answer value, and is
+// skipped when picking "the" number out of a line - otherwise "sizeof(...)
+// is 24 bytes on a 64-bit system" would extract the trailing 64 instead of
+// the actual answer, 24.
+var hyphenCompoundSuffixPattern = regexp.MustCompile(`^-\w`)
+
+// lastRelevantNumber returns the last number in text that is not part of a
+// "<number>-word" compound, scanning from the end.
+func lastRelevantNumber(text string) (string, bool) {
+	matches := numberPattern.FindAllStringIndex(text, -1)
+	for i := len(matches) - 1; i >= 0; i-- {
+		start, end := matches[i][0], matches[i][1]
+		if hyphenCompoundSuffixPattern.MatchString(text[end:]) {
+			continue
+		}
+		return text[start:end], true
+	}
+	return "", false
+}
+
+// ExtractLastNumber returns the model's answer number from s, converted to
+// T. It first looks only at the last non-empty line (the common case: a
+// final "24" or "0.9896" on its own line after any reasoning), and only
+// falls back to the last number anywhere in s if that line has no usable
+// number at all. Within whichever text it searches, it skips a number that
+// is part of a "<number>-word" compound (see hyphenCompoundSuffixPattern):
+// "sizeof(struct Config) is 24 bytes on a 64-bit system" must extract 24,
+// not the later, unrelated 64. It is the default extractor for Numeric.
+func ExtractLastNumber[T Number](s string) (T, error) {
+	lines := strings.Split(s, "\n")
+	for i := len(lines) - 1; i >= 0; i-- {
+		line := strings.TrimSpace(lines[i])
+		if line == "" {
+			continue
+		}
+		if lit, ok := lastRelevantNumber(line); ok {
+			return parseNumber[T](lit)
+		}
+		break
+	}
+
+	if lit, ok := lastRelevantNumber(s); ok {
+		return parseNumber[T](lit)
+	}
+	return 0, fmt.Errorf("no number found in response")
+}
+
+// parseNumber converts a numberPattern match (possibly ".9896", with no
+// leading zero) to T.
+func parseNumber[T Number](literal string) (T, error) {
+	f, err := strconv.ParseFloat(literal, 64)
+	if err != nil {
+		return 0, fmt.Errorf("parse number %q: %w", literal, err)
+	}
+	return T(f), nil
+}
+
+// numericEval scores full credit when the extracted number is within tol of
+// want, zero otherwise.
+type numericEval[T Number] struct {
+	extract func(string) (T, error)
+	want    T
+	tol     T
+}
+
+// Numeric returns an Evaluator that extracts a value of type T from the
+// response via extract and awards full credit when |got-want| <= tol.
+func Numeric[T Number](extract func(string) (T, error), want, tol T) Evaluator {
+	return numericEval[T]{extract: extract, want: want, tol: tol}
+}
+
+func (n numericEval[T]) Evaluate(_ context.Context, response string) Score {
+	got, err := n.extract(response)
+	if err != nil {
+		return Score{Value: 0, Detail: fmt.Sprintf("extract failed: %v", err)}
+	}
+
+	var diff T
+	if got >= n.want {
+		diff = got - n.want
+	} else {
+		diff = n.want - got
+	}
+
+	if diff <= n.tol {
+		return Score{Value: 1, Detail: fmt.Sprintf("got %v, want %v (tol %v)", got, n.want, n.tol)}
+	}
+	return Score{Value: 0, Detail: fmt.Sprintf("got %v, want %v (tol %v)", got, n.want, n.tol)}
+}

internal/eval/numeric_test.go

@@ -0,0 +1,96 @@
+package eval
+
+import (
+	"context"
+	"testing"
+)
+
+func TestExtractLastNumber(t *testing.T) {
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+		wantErr  bool
+	}{
+		{"plain number", "0.8785", 0.8785, false},
+		{"trailing number after prose", "The cosine similarity is 0.8785", 0.8785, false},
+		{"last of multiple numbers", "step 1 then step 2 result is 42", 42, false},
+		{"negative number", "the delta is -3.5", -3.5, false},
+		{"no number", "no digits here", 0, true},
+		// S6 regressions.
+		{"unit qualifier after the answer is skipped", "24 bytes on a 64-bit system", 24, false},
+		{"leading-dot decimal", "= .9896", 0.9896, false},
+		{"answer on last line, reasoning with numbers on earlier lines", "step 1 uses 8 items\nstep 2 uses 16 items\nfinal answer: 42", 42, false},
+		{"last line has only a unit-qualified number, falls back to whole text", "the computation used 100 iterations\nresult on a 64-bit system", 100, false},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := ExtractLastNumber[float64](tt.response)
+			if (err != nil) != tt.wantErr {
+				t.Fatalf("ExtractLastNumber(%q) error = %v, wantErr %v", tt.response, err, tt.wantErr)
+			}
+			if err == nil && got != tt.want {
+				t.Errorf("ExtractLastNumber(%q) = %v, want %v", tt.response, got, tt.want)
+			}
+		})
+	}
+}
+
+func TestNumeric_Float(t *testing.T) {
+	e := Numeric(ExtractLastNumber[float64], 0.8785, 0.0005)
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{"exact", "0.8785", 1},
+		{"within tolerance above", "0.8789", 1},
+		{"within tolerance below", "0.8781", 1},
+		{"outside tolerance", "0.9000", 0},
+		{"unparsable", "no number here", 0},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := e.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}
+
+func TestNumeric_IntExact(t *testing.T) {
+	e := Numeric(ExtractLastNumber[int], 32, 0)
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{"exact match", "32", 1},
+		{"off by one", "33", 0},
+		{"exact match with prose", "The sizeof value is 32 bytes", 1},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := e.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}
+
+func TestNumeric_UnsignedNoUnderflow(t *testing.T) {
+	// Regression: unsigned subtraction must not wrap when got < want.
+	e := Numeric(ExtractLastNumber[uint], 10, 2)
+	got := e.Evaluate(context.Background(), "9")
+	if got.Value != 1 {
+		t.Errorf("Evaluate(9) with want=10 tol=2 = %v, want 1 (detail: %s)", got.Value, got.Detail)
+	}
+	got = e.Evaluate(context.Background(), "5")
+	if got.Value != 0 {
+		t.Errorf("Evaluate(5) with want=10 tol=2 = %v, want 0 (detail: %s)", got.Value, got.Detail)
+	}
+}

internal/eval/regex.go

@@ -0,0 +1,29 @@
+package eval
+
+import (
+	"context"
+	"fmt"
+	"regexp"
+)
+
+// regexEval scores full credit when pattern matches the response.
+type regexEval struct {
+	pattern *regexp.Regexp
+	raw     string
+}
+
+// Regex returns an Evaluator awarding full credit when pattern matches
+// anywhere in the response, zero otherwise. pattern must be a valid RE2
+// expression; it panics on an invalid pattern since evaluators are built
+// once at catalog registration time, not per-request.
+func Regex(pattern string) Evaluator {
+	re := regexp.MustCompile(pattern)
+	return regexEval{pattern: re, raw: pattern}
+}
+
+func (r regexEval) Evaluate(_ context.Context, response string) Score {
+	if r.pattern.MatchString(response) {
+		return Score{Value: 1, Detail: fmt.Sprintf("matched /%s/", r.raw)}
+	}
+	return Score{Value: 0, Detail: fmt.Sprintf("no match for /%s/", r.raw)}
+}

internal/eval/regex_test.go

@@ -0,0 +1,36 @@
+package eval
+
+import (
+	"context"
+	"testing"
+)
+
+func TestRegex(t *testing.T) {
+	tests := []struct {
+		name     string
+		pattern  string
+		response string
+		want     float64
+	}{
+		{"matches", `ssh\s+root@10\.0\.0\.100`, "run: ssh [email protected] 'pct exec 251 -- systemctl status myservice'", 1},
+		{"no match", `ssh\s+root@10\.0\.0\.100`, "ssh [email protected]", 0},
+		{"anchored word boundary", `\bpct exec 251\b`, "use pct exec 251 -- cmd", 1},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := Regex(tt.pattern).Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Regex(%q).Evaluate(%q) = %v, want %v", tt.pattern, tt.response, got.Value, tt.want)
+			}
+		})
+	}
+}
+
+func TestRegex_PanicsOnInvalidPattern(t *testing.T) {
+	defer func() {
+		if r := recover(); r == nil {
+			t.Fatal("Regex() with invalid pattern did not panic")
+		}
+	}()
+	Regex(`(unclosed`)
+}

internal/llm/llm.go

@@ -0,0 +1,36 @@
+// Package llm defines the OpenAI-compatible chat client interface used to
+// exercise language models under test, plus a concrete HTTP implementation.
+package llm
+
+import (
+	"context"
+	"time"
+)
+
+// Message is a single chat message in an OpenAI-compatible conversation.
+type Message struct {
+	Role    string
+	Content string
+}
+
+// Request is one chat-completion request sent to a model.
+type Request struct {
+	Model       string
+	Messages    []Message
+	MaxTokens   int
+	Temperature float64
+}
+
+// Response is the normalized result of a chat-completion call.
+type Response struct {
+	Text             string
+	PromptTokens     int
+	CompletionTokens int
+	Latency          time.Duration
+}
+
+// Client completes chat requests against a language model backend.
+// Implementations must be safe for concurrent use.
+type Client interface {
+	Complete(ctx context.Context, req Request) (Response, error)
+}

internal/llm/openai.go

@@ -0,0 +1,179 @@
+package llm
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"time"
+)
+
+// maxRetries is the number of retry attempts after the initial request on
+// transport errors or 5xx responses, per PLAN.md ("retry w/ backoff x2").
+const maxRetries = 2
+
+// retryBaseDelay is the base backoff delay; attempt N waits N*retryBaseDelay.
+const retryBaseDelay = 250 * time.Millisecond
+
+// OpenAIClient talks to an OpenAI-compatible /v1/chat/completions endpoint.
+type OpenAIClient struct {
+	httpClient *http.Client
+	endpoint   string
+	apiKey     string
+}
+
+// NewOpenAIClient builds a client against endpoint (e.g.
+// "https://host/v1") using apiKey (may be empty for keyless gateways) and
+// per-request timeout.
+func NewOpenAIClient(endpoint, apiKey string, timeout time.Duration) *OpenAIClient {
+	return &OpenAIClient{
+		endpoint: endpoint,
+		apiKey:   apiKey,
+		httpClient: &http.Client{
+			Timeout: timeout,
+		},
+	}
+}
+
+type chatCompletionRequest struct {
+	Model       string    `json:"model"`
+	Messages    []chatMsg `json:"messages"`
+	MaxTokens   int       `json:"max_tokens,omitempty"`
+	Temperature float64   `json:"temperature"`
+}
+
+type chatMsg struct {
+	Role    string `json:"role"`
+	Content string `json:"content"`
+}
+
+type chatCompletionResponse struct {
+	Error *struct {
+		Message string `json:"message"`
+	} `json:"error"`
+	Choices []struct {
+		Message chatMsg `json:"message"`
+	} `json:"choices"`
+	Usage struct {
+		PromptTokens     int `json:"prompt_tokens"`
+		CompletionTokens int `json:"completion_tokens"`
+	} `json:"usage"`
+}
+
+// Complete implements Client. It retries transport failures and 5xx
+// responses up to maxRetries times with linear backoff.
+func (c *OpenAIClient) Complete(ctx context.Context, req Request) (Response, error) {
+	body := chatCompletionRequest{
+		Model:       req.Model,
+		MaxTokens:   req.MaxTokens,
+		Temperature: req.Temperature,
+	}
+	for _, m := range req.Messages {
+		body.Messages = append(body.Messages, chatMsg(m))
+	}
+
+	payload, err := json.Marshal(body)
+	if err != nil {
+		return Response{}, fmt.Errorf("llm: marshal request: %w", err)
+	}
+
+	var lastErr error
+	for attempt := 0; attempt <= maxRetries; attempt++ {
+		if attempt > 0 {
+			delay := time.Duration(attempt) * retryBaseDelay
+			select {
+			case <-ctx.Done():
+				return Response{}, ctx.Err()
+			case <-time.After(delay):
+			}
+		}
+
+		// Measured per-attempt, starting after any backoff delay: Latency
+		// reflects the successful attempt's own generation time, not the
+		// cumulative wait across earlier failed/retried attempts (S8).
+		attemptStart := time.Now()
+		resp, err := c.doRequest(ctx, payload)
+		if err != nil {
+			lastErr = err
+			if isRetryable(err) {
+				continue
+			}
+			return Response{}, err
+		}
+
+		latency := time.Since(attemptStart)
+		text := ""
+		if len(resp.Choices) > 0 {
+			text = resp.Choices[0].Message.Content
+		}
+		if resp.Error != nil {
+			lastErr = fmt.Errorf("llm: api error: %s", resp.Error.Message)
+			return Response{}, lastErr
+		}
+		return Response{
+			Text:             text,
+			PromptTokens:     resp.Usage.PromptTokens,
+			CompletionTokens: resp.Usage.CompletionTokens,
+			Latency:          latency,
+		}, nil
+	}
+	return Response{}, fmt.Errorf("llm: request failed after %d attempts: %w", maxRetries+1, lastErr)
+}
+
+type retryableError struct {
+	err    error
+	status int
+}
+
+func (e *retryableError) Error() string {
+	if e.status != 0 {
+		return fmt.Sprintf("llm: server returned status %d", e.status)
+	}
+	return fmt.Sprintf("llm: transport error: %v", e.err)
+}
+
+func (e *retryableError) Unwrap() error { return e.err }
+
+func isRetryable(err error) bool {
+	var re *retryableError
+	return errors.As(err, &re)
+}
+
+func (c *OpenAIClient) doRequest(ctx context.Context, payload []byte) (*chatCompletionResponse, error) {
+	url := c.endpoint + "/chat/completions"
+	httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(payload))
+	if err != nil {
+		return nil, fmt.Errorf("llm: build request: %w", err)
+	}
+	httpReq.Header.Set("Content-Type", "application/json")
+	if c.apiKey != "" {
+		httpReq.Header.Set("Authorization", "Bearer "+c.apiKey)
+	}
+
+	httpResp, err := c.httpClient.Do(httpReq)
+	if err != nil {
+		return nil, &retryableError{err: err}
+	}
+	defer func() { _ = httpResp.Body.Close() }() // best-effort; body already fully read below
+
+	respBody, err := io.ReadAll(httpResp.Body)
+	if err != nil {
+		return nil, &retryableError{err: fmt.Errorf("read body: %w", err)}
+	}
+
+	if httpResp.StatusCode >= 500 {
+		return nil, &retryableError{status: httpResp.StatusCode}
+	}
+	if httpResp.StatusCode >= 400 {
+		return nil, fmt.Errorf("llm: server returned status %d: %s", httpResp.StatusCode, string(respBody))
+	}
+
+	var out chatCompletionResponse
+	if err := json.Unmarshal(respBody, &out); err != nil {
+		return nil, fmt.Errorf("llm: decode response: %w", err)
+	}
+	return &out, nil
+}

internal/llm/openai_test.go

@@ -0,0 +1,217 @@
+package llm
+
+import (
+	"context"
+	"encoding/json"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"sync/atomic"
+	"testing"
+	"time"
+)
+
+// newTestServer spins up a local httptest server; this is not the live
+// endpoint and is safe for offline unit tests.
+func newTestServer(t *testing.T, handler http.HandlerFunc) *httptest.Server {
+	t.Helper()
+	srv := httptest.NewServer(handler)
+	t.Cleanup(srv.Close)
+	return srv
+}
+
+func TestOpenAIClient_Complete_Success(t *testing.T) {
+	srv := newTestServer(t, func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path != "/chat/completions" {
+			t.Errorf("unexpected path: %s", r.URL.Path)
+		}
+		var body chatCompletionRequest
+		if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
+			t.Fatalf("decode request: %v", err)
+		}
+		if body.Model != "test-model" {
+			t.Errorf("model = %q, want test-model", body.Model)
+		}
+		resp := chatCompletionResponse{}
+		resp.Choices = []struct {
+			Message chatMsg `json:"message"`
+		}{{Message: chatMsg{Role: "assistant", Content: "hello there"}}}
+		resp.Usage.PromptTokens = 12
+		resp.Usage.CompletionTokens = 3
+		_ = json.NewEncoder(w).Encode(resp)
+	})
+
+	client := NewOpenAIClient(srv.URL, "", time.Second)
+	resp, err := client.Complete(context.Background(), Request{
+		Model:    "test-model",
+		Messages: []Message{{Role: "user", Content: "hi"}},
+	})
+	if err != nil {
+		t.Fatalf("Complete() error = %v", err)
+	}
+	if resp.Text != "hello there" {
+		t.Errorf("Text = %q, want %q", resp.Text, "hello there")
+	}
+	if resp.PromptTokens != 12 || resp.CompletionTokens != 3 {
+		t.Errorf("tokens = %d/%d, want 12/3", resp.PromptTokens, resp.CompletionTokens)
+	}
+}
+
+// TestOpenAIClient_Complete_SendsTemperatureZero verifies Request.Temperature
+// (PLAN.md pins this to 0 for deterministic scoring, wired from main.go's
+// requestTemperature constant) actually lands on the wire as a JSON
+// "temperature":0 field, not silently omitted (chatCompletionRequest has no
+// "omitempty" on Temperature specifically so a real zero is distinguishable
+// from an absent field) (N7).
+func TestOpenAIClient_Complete_SendsTemperatureZero(t *testing.T) {
+	var rawBody []byte
+	srv := newTestServer(t, func(w http.ResponseWriter, r *http.Request) {
+		var err error
+		rawBody, err = io.ReadAll(r.Body)
+		if err != nil {
+			t.Fatalf("read request body: %v", err)
+		}
+		_ = json.NewEncoder(w).Encode(chatCompletionResponse{})
+	})
+
+	client := NewOpenAIClient(srv.URL, "", time.Second)
+	_, err := client.Complete(context.Background(), Request{
+		Model:       "m",
+		Messages:    []Message{{Role: "user", Content: "x"}},
+		Temperature: 0,
+	})
+	if err != nil {
+		t.Fatalf("Complete() error = %v", err)
+	}
+
+	var decoded map[string]any
+	if err := json.Unmarshal(rawBody, &decoded); err != nil {
+		t.Fatalf("json.Unmarshal(rawBody) error = %v; body: %s", err, rawBody)
+	}
+	got, present := decoded["temperature"]
+	if !present {
+		t.Fatalf("request body has no \"temperature\" key at all: %s", rawBody)
+	}
+	if got != float64(0) {
+		t.Errorf("temperature = %v, want 0", got)
+	}
+}
+
+func TestOpenAIClient_Complete_RetriesOn5xxThenSucceeds(t *testing.T) {
+	var calls int32
+	srv := newTestServer(t, func(w http.ResponseWriter, r *http.Request) {
+		n := atomic.AddInt32(&calls, 1)
+		if n < 3 {
+			w.WriteHeader(http.StatusInternalServerError)
+			return
+		}
+		resp := chatCompletionResponse{}
+		resp.Choices = []struct {
+			Message chatMsg `json:"message"`
+		}{{Message: chatMsg{Role: "assistant", Content: "ok"}}}
+		_ = json.NewEncoder(w).Encode(resp)
+	})
+
+	client := NewOpenAIClient(srv.URL, "", time.Second)
+	resp, err := client.Complete(context.Background(), Request{Model: "m", Messages: []Message{{Role: "user", Content: "x"}}})
+	if err != nil {
+		t.Fatalf("Complete() error = %v", err)
+	}
+	if resp.Text != "ok" {
+		t.Errorf("Text = %q, want ok", resp.Text)
+	}
+	// S8: Latency must reflect the successful attempt's own generation
+	// time, not the cumulative wait across the two failed attempts plus
+	// their backoff delays (which total 250ms+500ms=750ms for attempts 1
+	// and 2 - see retryBaseDelay/maxRetries).
+	if resp.Latency > 200*time.Millisecond {
+		t.Errorf("Latency = %v, want well under the 750ms of retry backoff (per-attempt measurement, S8)", resp.Latency)
+	}
+
+	if got := atomic.LoadInt32(&calls); got != 3 {
+		t.Errorf("calls = %d, want 3 (1 initial + 2 retries)", got)
+	}
+}
+
+func TestOpenAIClient_Complete_ExhaustsRetriesOn5xx(t *testing.T) {
+	var calls int32
+	srv := newTestServer(t, func(w http.ResponseWriter, r *http.Request) {
+		atomic.AddInt32(&calls, 1)
+		w.WriteHeader(http.StatusServiceUnavailable)
+	})
+
+	client := NewOpenAIClient(srv.URL, "", time.Second)
+	_, err := client.Complete(context.Background(), Request{Model: "m", Messages: []Message{{Role: "user", Content: "x"}}})
+	if err == nil {
+		t.Fatal("Complete() error = nil, want error after exhausting retries")
+	}
+	if got := atomic.LoadInt32(&calls); got != maxRetries+1 {
+		t.Errorf("calls = %d, want %d", got, maxRetries+1)
+	}
+}
+
+func TestOpenAIClient_Complete_NoRetryOn4xx(t *testing.T) {
+	var calls int32
+	srv := newTestServer(t, func(w http.ResponseWriter, r *http.Request) {
+		atomic.AddInt32(&calls, 1)
+		w.WriteHeader(http.StatusBadRequest)
+		_, _ = w.Write([]byte(`{"error":{"message":"bad request"}}`))
+	})
+
+	client := NewOpenAIClient(srv.URL, "", time.Second)
+	_, err := client.Complete(context.Background(), Request{Model: "m", Messages: []Message{{Role: "user", Content: "x"}}})
+	if err == nil {
+		t.Fatal("Complete() error = nil, want error")
+	}
+	if got := atomic.LoadInt32(&calls); got != 1 {
+		t.Errorf("calls = %d, want 1 (no retry on 4xx)", got)
+	}
+	if !strings.Contains(err.Error(), "400") {
+		t.Errorf("error = %v, want mention of 400", err)
+	}
+}
+
+func TestOpenAIClient_Complete_APIErrorField(t *testing.T) {
+	srv := newTestServer(t, func(w http.ResponseWriter, r *http.Request) {
+		_, _ = w.Write([]byte(`{"error":{"message":"model overloaded"}}`))
+	})
+
+	client := NewOpenAIClient(srv.URL, "", time.Second)
+	_, err := client.Complete(context.Background(), Request{Model: "m", Messages: []Message{{Role: "user", Content: "x"}}})
+	if err == nil {
+		t.Fatal("Complete() error = nil, want error")
+	}
+	if !strings.Contains(err.Error(), "model overloaded") {
+		t.Errorf("error = %v, want mention of model overloaded", err)
+	}
+}
+
+func TestOpenAIClient_Complete_ContextCanceled(t *testing.T) {
+	srv := newTestServer(t, func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(http.StatusInternalServerError)
+	})
+
+	client := NewOpenAIClient(srv.URL, "", time.Second)
+	ctx, cancel := context.WithCancel(context.Background())
+	cancel()
+	_, err := client.Complete(ctx, Request{Model: "m", Messages: []Message{{Role: "user", Content: "x"}}})
+	if err == nil {
+		t.Fatal("Complete() error = nil, want error for canceled context")
+	}
+}
+
+func TestOpenAIClient_Complete_SendsAPIKey(t *testing.T) {
+	srv := newTestServer(t, func(w http.ResponseWriter, r *http.Request) {
+		if got := r.Header.Get("Authorization"); got != "Bearer secret-key" {
+			t.Errorf("Authorization = %q, want Bearer secret-key", got)
+		}
+		_ = json.NewEncoder(w).Encode(chatCompletionResponse{})
+	})
+
+	client := NewOpenAIClient(srv.URL, "secret-key", time.Second)
+	_, err := client.Complete(context.Background(), Request{Model: "m", Messages: []Message{{Role: "user", Content: "x"}}})
+	if err != nil {
+		t.Fatalf("Complete() error = %v", err)
+	}
+}

internal/report/json.go

@@ -0,0 +1,60 @@
+package report
+
+import (
+	"encoding/json"
+	"io"
+
+	"github.com/lukaszraczylo/llm-testbench/internal/runner"
+	"github.com/lukaszraczylo/llm-testbench/internal/testkit"
+)
+
+// jsonResult is the JSON-serializable projection of a runner.Result, joined
+// with its test's category/subcategory for self-contained output.
+type jsonResult struct {
+	Model            string  `json:"model"`
+	TestID           string  `json:"test_id"`
+	Category         string  `json:"category,omitempty"`
+	Subcategory      string  `json:"subcategory,omitempty"`
+	Detail           string  `json:"detail,omitempty"`
+	Error            string  `json:"error,omitempty"`
+	Score            float64 `json:"score"`
+	LatencyMS        int64   `json:"latency_ms"`
+	PromptTokens     int     `json:"prompt_tokens"`
+	CompletionTokens int     `json:"completion_tokens"`
+	Tokens           int     `json:"tokens"`
+	Skipped          bool    `json:"skipped"`
+}
+
+// renderJSON dumps raw results as a JSON array, joined with test metadata.
+func renderJSON(w io.Writer, tests []testkit.Test, results []runner.Result) error {
+	testByID := make(map[string]testkit.Test, len(tests))
+	for _, t := range tests {
+		testByID[t.ID] = t
+	}
+
+	out := make([]jsonResult, 0, len(results))
+	for _, r := range results {
+		t := testByID[r.TestID]
+		jr := jsonResult{
+			Model:            r.Model,
+			TestID:           r.TestID,
+			Category:         t.Category,
+			Subcategory:      t.Subcategory,
+			Score:            r.Score.Value,
+			Detail:           r.Score.Detail,
+			Skipped:          r.Score.Skipped,
+			LatencyMS:        r.Latency.Milliseconds(),
+			PromptTokens:     r.PromptTokens,
+			CompletionTokens: r.CompletionTokens,
+			Tokens:           r.TotalTokens(),
+		}
+		if r.Err != nil {
+			jr.Error = r.Err.Error()
+		}
+		out = append(out, jr)
+	}
+
+	enc := json.NewEncoder(w)
+	enc.SetIndent("", "  ")
+	return enc.Encode(out)
+}

internal/report/markdown.go

@@ -0,0 +1,77 @@
+package report
+
+import (
+	"fmt"
+	"io"
+	"math"
+	"strings"
+
+	"github.com/lukaszraczylo/llm-testbench/internal/runner"
+	"github.com/lukaszraczylo/llm-testbench/internal/testkit"
+)
+
+// renderMarkdown writes the same three tables as renderTable, formatted as
+// GitHub-flavored markdown tables.
+func renderMarkdown(w io.Writer, tests []testkit.Test, models []string, results []runner.Result) error {
+	idx := indexResults(results)
+	ew := &errWriter{w: w}
+
+	ew.println("## Per-test scores")
+	ew.println()
+	scoreHeader := append([]string{"Test", "Category", "Subcategory"}, models...)
+	writeMarkdownHeader(ew, scoreHeader)
+	for _, t := range sortedTests(tests) {
+		row := []string{t.ID, t.Category, t.Subcategory}
+		for _, model := range models {
+			r, ok := idx[t.ID][model]
+			row = append(row, cellText(r, ok))
+		}
+		writeMarkdownRow(ew, row)
+	}
+
+	ew.println()
+	ew.println("## Category rollup (mean score)")
+	ew.println()
+	catHeader := append([]string{"Category"}, models...)
+	writeMarkdownHeader(ew, catHeader)
+	for _, cm := range categoryRollup(tests, models, idx) {
+		row := []string{cm.category}
+		for _, model := range models {
+			row = append(row, meanCellText(cm.means[model]))
+		}
+		writeMarkdownRow(ew, row)
+	}
+
+	ew.println()
+	ew.println("## Model summary")
+	ew.println()
+	writeMarkdownHeader(ew, []string{"Model", "Mean", "Passed", "Partial", "Failed", "Errors", "Mean latency", "Total tokens", "Tok/s"})
+	for _, ms := range summarize(models, results) {
+		writeMarkdownRow(ew, []string{
+			ms.model,
+			meanCellText(ms.overallMean),
+			fmt.Sprintf("%d", ms.passed),
+			fmt.Sprintf("%d", ms.partial),
+			fmt.Sprintf("%d", ms.failed),
+			fmt.Sprintf("%d", ms.errors),
+			durationText(ms.meanLatency, !math.IsNaN(ms.overallMean)),
+			fmt.Sprintf("%d", ms.totalTokens),
+			rateText(ms.avgTokPerSec),
+		})
+	}
+
+	return ew.err
+}
+
+func writeMarkdownHeader(ew *errWriter, cols []string) {
+	ew.println("| " + strings.Join(cols, " | ") + " |")
+	sep := make([]string, len(cols))
+	for i := range sep {
+		sep[i] = "---"
+	}
+	ew.println("| " + strings.Join(sep, " | ") + " |")
+}
+
+func writeMarkdownRow(ew *errWriter, cols []string) {
+	ew.println("| " + strings.Join(cols, " | ") + " |")
+}

internal/report/report.go

@@ -0,0 +1,238 @@
+// Package report renders runner.Result sets as a comparison table, GitHub
+// markdown tables, or raw JSON.
+package report
+
+import (
+	"fmt"
+	"io"
+	"math"
+	"sort"
+	"time"
+
+	"github.com/lukaszraczylo/llm-testbench/internal/runner"
+	"github.com/lukaszraczylo/llm-testbench/internal/testkit"
+)
+
+// Format selects the output renderer.
+type Format string
+
+// Supported Format values.
+const (
+	FormatTable    Format = "table"
+	FormatMarkdown Format = "markdown"
+	FormatJSON     Format = "json"
+)
+
+// passThreshold is the minimum score counted as a full pass in the model
+// summary table.
+const passThreshold = 0.99
+
+// Render writes tests/models/results to w in the requested format.
+func Render(w io.Writer, format Format, tests []testkit.Test, models []string, results []runner.Result) error {
+	switch format {
+	case FormatTable:
+		return renderTable(w, tests, models, results)
+	case FormatMarkdown:
+		return renderMarkdown(w, tests, models, results)
+	case FormatJSON:
+		return renderJSON(w, tests, results)
+	default:
+		return fmt.Errorf("report: unknown format %q", format)
+	}
+}
+
+// resultIndex looks up a Result by testID then model.
+type resultIndex map[string]map[string]runner.Result
+
+func indexResults(results []runner.Result) resultIndex {
+	idx := make(resultIndex)
+	for _, r := range results {
+		if idx[r.TestID] == nil {
+			idx[r.TestID] = make(map[string]runner.Result)
+		}
+		idx[r.TestID][r.Model] = r
+	}
+	return idx
+}
+
+// sortedTests returns tests ordered by Category, Subcategory, then ID, for
+// deterministic, human-grouped table output.
+func sortedTests(tests []testkit.Test) []testkit.Test {
+	out := make([]testkit.Test, len(tests))
+	copy(out, tests)
+	sort.Slice(out, func(i, j int) bool {
+		if out[i].Category != out[j].Category {
+			return out[i].Category < out[j].Category
+		}
+		if out[i].Subcategory != out[j].Subcategory {
+			return out[i].Subcategory < out[j].Subcategory
+		}
+		return out[i].ID < out[j].ID
+	})
+	return out
+}
+
+// cellText renders one (test, model) cell for the score table: "score
+// (Ntok)" for a scored result, or ERR/skip/N/A otherwise.
+func cellText(r runner.Result, found bool) string {
+	switch {
+	case !found:
+		return "N/A"
+	case r.Err != nil:
+		return "ERR"
+	case r.Score.Skipped:
+		return "skip"
+	default:
+		return fmt.Sprintf("%.2f (%dtok)", r.Score.Value, r.TotalTokens())
+	}
+}
+
+// categoryMean holds the per-model mean score for one category, computed
+// over non-skipped, non-error results only.
+type categoryMean struct {
+	means    map[string]float64
+	category string
+}
+
+func categoryRollup(tests []testkit.Test, models []string, idx resultIndex) []categoryMean {
+	order := make([]string, 0)
+	byCategory := make(map[string][]testkit.Test)
+	for _, t := range sortedTests(tests) {
+		if _, seen := byCategory[t.Category]; !seen {
+			order = append(order, t.Category)
+		}
+		byCategory[t.Category] = append(byCategory[t.Category], t)
+	}
+
+	out := make([]categoryMean, 0, len(order))
+	for _, cat := range order {
+		cm := categoryMean{category: cat, means: make(map[string]float64, len(models))}
+		for _, model := range models {
+			var sum float64
+			var n int
+			for _, t := range byCategory[cat] {
+				r, ok := idx[t.ID][model]
+				if !ok || r.Err != nil || r.Score.Skipped {
+					continue
+				}
+				sum += r.Score.Value
+				n++
+			}
+			if n > 0 {
+				cm.means[model] = sum / float64(n)
+			} else {
+				cm.means[model] = math.NaN()
+			}
+		}
+		out = append(out, cm)
+	}
+	return out
+}
+
+// meanCellText renders a categoryMean/modelSummary mean cell, showing N/A
+// when there is no scored data.
+func meanCellText(v float64) string {
+	if math.IsNaN(v) {
+		return "N/A"
+	}
+	return fmt.Sprintf("%.2f", v)
+}
+
+// rateText renders a modelSummary avgTokPerSec cell, showing N/A when there
+// is no timed, non-skipped data to compute a rate from.
+func rateText(v float64) string {
+	if math.IsNaN(v) {
+		return "N/A"
+	}
+	return fmt.Sprintf("%.1f", v)
+}
+
+// durationText renders a modelSummary meanLatency cell, showing N/A when
+// there is no scored data (N6): meanLatency's zero value, printed via
+// time.Duration.String(), would otherwise misleadingly read as a real "0s"
+// mean latency rather than "no data". hasData mirrors overallMean's NaN
+// check, since both are set together (or left unset together) in
+// summarize.
+func durationText(d time.Duration, hasData bool) string {
+	if !hasData {
+		return "N/A"
+	}
+	return d.String()
+}
+
+// modelSummary aggregates one model's results across the whole run.
+type modelSummary struct {
+	model        string
+	overallMean  float64 // math.NaN() when no scored data
+	passed       int
+	partial      int
+	failed       int
+	errors       int
+	meanLatency  time.Duration
+	totalTokens  int
+	avgTokPerSec float64 // math.NaN() when no timed, non-skipped data
+}
+
+func summarize(models []string, results []runner.Result) []modelSummary {
+	byModel := make(map[string][]runner.Result, len(models))
+	for _, r := range results {
+		byModel[r.Model] = append(byModel[r.Model], r)
+	}
+
+	out := make([]modelSummary, 0, len(models))
+	for _, model := range models {
+		ms := modelSummary{model: model}
+		var scoreSum float64
+		var scoredCount int
+		var latencySum time.Duration
+		var latencyCount int
+		var completionTokenSum int
+		var latencySecondsSum float64
+
+		for _, r := range byModel[model] {
+			ms.totalTokens += r.TotalTokens()
+			switch {
+			case r.Err != nil:
+				ms.errors++
+				continue
+			case r.Score.Skipped:
+				continue
+			}
+
+			scoreSum += r.Score.Value
+			scoredCount++
+			latencySum += r.Latency
+			latencyCount++
+			completionTokenSum += r.CompletionTokens
+			latencySecondsSum += r.Latency.Seconds()
+
+			switch {
+			case r.Score.Value >= passThreshold:
+				ms.passed++
+			case r.Score.Value <= 0:
+				ms.failed++
+			default:
+				ms.partial++
+			}
+		}
+
+		if scoredCount > 0 {
+			ms.overallMean = scoreSum / float64(scoredCount)
+			ms.meanLatency = latencySum / time.Duration(latencyCount)
+		} else {
+			ms.overallMean = math.NaN()
+		}
+
+		// Aggregate ratio (sum of completion tokens over sum of elapsed
+		// seconds), not a mean of per-test ratios: robust to a handful of
+		// very short responses skewing a per-test-averaged rate.
+		if latencySecondsSum > 0 {
+			ms.avgTokPerSec = float64(completionTokenSum) / latencySecondsSum
+		} else {
+			ms.avgTokPerSec = math.NaN()
+		}
+
+		out = append(out, ms)
+	}
+	return out
+}

internal/report/report_test.go

@@ -0,0 +1,235 @@
+package report
+
+import (
+	"bytes"
+	"encoding/json"
+	"errors"
+	"math"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/lukaszraczylo/llm-testbench/internal/eval"
+	"github.com/lukaszraczylo/llm-testbench/internal/runner"
+	"github.com/lukaszraczylo/llm-testbench/internal/testkit"
+)
+
+func sampleTests() []testkit.Test {
+	return []testkit.Test{
+		{ID: "go-1", Category: "programming", Subcategory: "golang"},
+		{ID: "py-1", Category: "programming", Subcategory: "python"},
+		{ID: "k8s-1", Category: "operations", Subcategory: "kubernetes"},
+	}
+}
+
+func sampleResults() []runner.Result {
+	return []runner.Result{
+		{Model: "m1", TestID: "go-1", Score: eval.Score{Value: 1}, Latency: 100 * time.Millisecond, PromptTokens: 6, CompletionTokens: 4},
+		{Model: "m1", TestID: "py-1", Score: eval.Score{Value: 0.5}, Latency: 200 * time.Millisecond, PromptTokens: 12, CompletionTokens: 8},
+		{Model: "m1", TestID: "k8s-1", Err: errors.New("boom")},
+		{Model: "m2", TestID: "go-1", Score: eval.Score{Value: 0}, Latency: 50 * time.Millisecond, PromptTokens: 3, CompletionTokens: 2},
+		{Model: "m2", TestID: "py-1", Score: eval.Score{Skipped: true, Detail: "toolchain missing: cc"}},
+		{Model: "m2", TestID: "k8s-1", Score: eval.Score{Value: 1}, Latency: 150 * time.Millisecond, PromptTokens: 9, CompletionTokens: 6},
+	}
+}
+
+func TestRenderTable_ContainsExpectedCells(t *testing.T) {
+	var buf bytes.Buffer
+	if err := Render(&buf, FormatTable, sampleTests(), []string{"m1", "m2"}, sampleResults()); err != nil {
+		t.Fatalf("Render() error = %v", err)
+	}
+	out := buf.String()
+
+	for _, want := range []string{
+		"== Per-test scores ==",
+		"== Category rollup (mean score) ==",
+		"== Model summary ==",
+		"1.00 (10tok)", // m1/go-1: 6 prompt + 4 completion
+		"0.50 (20tok)", // m1/py-1: 12 prompt + 8 completion
+		"ERR",          // m1/k8s-1
+		"0.00 (5tok)",  // m2/go-1: 3 prompt + 2 completion
+		"skip",         // m2/py-1
+		"TOK/S",
+	} {
+		if !strings.Contains(out, want) {
+			t.Errorf("table output missing %q\n---\n%s", want, out)
+		}
+	}
+}
+
+func TestRenderMarkdown_ProducesGitHubTables(t *testing.T) {
+	var buf bytes.Buffer
+	if err := Render(&buf, FormatMarkdown, sampleTests(), []string{"m1", "m2"}, sampleResults()); err != nil {
+		t.Fatalf("Render() error = %v", err)
+	}
+	out := buf.String()
+
+	for _, want := range []string{
+		"## Per-test scores",
+		"| Test | Category | Subcategory | m1 | m2 |",
+		"| --- | --- | --- | --- | --- |",
+		"## Category rollup (mean score)",
+		"## Model summary",
+		"Tok/s",
+		"1.00 (10tok)",
+	} {
+		if !strings.Contains(out, want) {
+			t.Errorf("markdown output missing %q\n---\n%s", want, out)
+		}
+	}
+}
+
+func TestRenderJSON_DumpsRawResults(t *testing.T) {
+	var buf bytes.Buffer
+	if err := Render(&buf, FormatJSON, sampleTests(), []string{"m1", "m2"}, sampleResults()); err != nil {
+		t.Fatalf("Render() error = %v", err)
+	}
+
+	var out []jsonResult
+	if err := json.Unmarshal(buf.Bytes(), &out); err != nil {
+		t.Fatalf("json.Unmarshal() error = %v; output:\n%s", err, buf.String())
+	}
+	if len(out) != len(sampleResults()) {
+		t.Fatalf("len(out) = %d, want %d", len(out), len(sampleResults()))
+	}
+
+	var errEntry, skipEntry, tokenEntry *jsonResult
+	for i := range out {
+		if out[i].Error != "" {
+			errEntry = &out[i]
+		}
+		if out[i].Skipped {
+			skipEntry = &out[i]
+		}
+		if out[i].Model == "m1" && out[i].TestID == "go-1" {
+			tokenEntry = &out[i]
+		}
+	}
+	if errEntry == nil || errEntry.Error != "boom" {
+		t.Errorf("expected an entry with Error = \"boom\", got %+v", errEntry)
+	}
+	if skipEntry == nil || skipEntry.Category != "programming" {
+		t.Errorf("expected skipped entry to carry Category, got %+v", skipEntry)
+	}
+	if tokenEntry == nil || tokenEntry.PromptTokens != 6 || tokenEntry.CompletionTokens != 4 || tokenEntry.Tokens != 10 {
+		t.Errorf("expected m1/go-1 to carry prompt=6 completion=4 total=10 tokens, got %+v", tokenEntry)
+	}
+}
+
+func TestRender_UnknownFormat(t *testing.T) {
+	var buf bytes.Buffer
+	err := Render(&buf, Format("bogus"), sampleTests(), []string{"m1"}, sampleResults())
+	if err == nil {
+		t.Fatal("Render() error = nil, want error for unknown format")
+	}
+}
+
+func TestSummarize_Buckets(t *testing.T) {
+	results := []runner.Result{
+		{Model: "m1", TestID: "a", Score: eval.Score{Value: 1}},
+		{Model: "m1", TestID: "b", Score: eval.Score{Value: 0.5}},
+		{Model: "m1", TestID: "c", Score: eval.Score{Value: 0}},
+		{Model: "m1", TestID: "d", Err: errors.New("fail")},
+		{Model: "m1", TestID: "e", Score: eval.Score{Skipped: true}},
+	}
+	summaries := summarize([]string{"m1"}, results)
+	if len(summaries) != 1 {
+		t.Fatalf("summarize() len = %d, want 1", len(summaries))
+	}
+	ms := summaries[0]
+	if ms.passed != 1 || ms.partial != 1 || ms.failed != 1 || ms.errors != 1 {
+		t.Errorf("buckets = passed:%d partial:%d failed:%d errors:%d, want 1,1,1,1", ms.passed, ms.partial, ms.failed, ms.errors)
+	}
+	wantMean := (1.0 + 0.5 + 0.0) / 3.0
+	if ms.overallMean != wantMean {
+		t.Errorf("overallMean = %v, want %v", ms.overallMean, wantMean)
+	}
+}
+
+func TestSummarize_AvgTokPerSec(t *testing.T) {
+	// Aggregate ratio (sum completion tokens / sum latency seconds), not a
+	// mean of per-test ratios: a fast, short response (10 tok / 0.1s = 100
+	// tok/s) and a slow, long one (100 tok / 2s = 50 tok/s) must average to
+	// 110 tok / 2.1s, not (100+50)/2.
+	results := []runner.Result{
+		{Model: "m1", TestID: "fast", Score: eval.Score{Value: 1}, Latency: 100 * time.Millisecond, CompletionTokens: 10},
+		{Model: "m1", TestID: "slow", Score: eval.Score{Value: 1}, Latency: 2 * time.Second, CompletionTokens: 100},
+		{Model: "m1", TestID: "err", Err: errors.New("fail"), Latency: time.Second, CompletionTokens: 999},
+		{Model: "m1", TestID: "skipped", Score: eval.Score{Skipped: true}, Latency: time.Second, CompletionTokens: 999},
+	}
+	summaries := summarize([]string{"m1"}, results)
+	ms := summaries[0]
+
+	want := 110.0 / 2.1
+	if math.Abs(ms.avgTokPerSec-want) > 1e-9 {
+		t.Errorf("avgTokPerSec = %v, want %v (errored/skipped results must be excluded)", ms.avgTokPerSec, want)
+	}
+}
+
+func TestSummarize_AvgTokPerSec_NoDataIsNaN(t *testing.T) {
+	results := []runner.Result{
+		{Model: "m1", TestID: "err", Err: errors.New("fail")},
+		{Model: "m1", TestID: "skipped", Score: eval.Score{Skipped: true}},
+	}
+	summaries := summarize([]string{"m1"}, results)
+	if !math.IsNaN(summaries[0].avgTokPerSec) {
+		t.Errorf("avgTokPerSec = %v, want NaN when there is no timed data", summaries[0].avgTokPerSec)
+	}
+	if rateText(summaries[0].avgTokPerSec) != "N/A" {
+		t.Errorf("rateText(NaN) = %q, want N/A", rateText(summaries[0].avgTokPerSec))
+	}
+}
+
+func TestDurationText(t *testing.T) {
+	if got := durationText(0, false); got != "N/A" {
+		t.Errorf("durationText(0, false) = %q, want N/A", got)
+	}
+	if got := durationText(0, true); got != "0s" {
+		t.Errorf("durationText(0, true) = %q, want 0s (a real zero-latency measurement, not lack of data)", got)
+	}
+	if got := durationText(42*time.Millisecond, true); got != "42ms" {
+		t.Errorf("durationText(42ms, true) = %q, want 42ms", got)
+	}
+}
+
+func TestSummarize_MeanLatency_NoDataIsNA(t *testing.T) {
+	// N6 regression: a model with only errored/skipped results (no scored
+	// data at all) must render MEAN_LATENCY as N/A, not the zero Duration's
+	// misleading "0s".
+	results := []runner.Result{
+		{Model: "m1", TestID: "err", Err: errors.New("fail")},
+		{Model: "m1", TestID: "skipped", Score: eval.Score{Skipped: true}},
+	}
+	var buf bytes.Buffer
+	if err := Render(&buf, FormatTable, []testkit.Test{{ID: "err"}, {ID: "skipped"}}, []string{"m1"}, results); err != nil {
+		t.Fatalf("Render() error = %v", err)
+	}
+	out := buf.String()
+	if !strings.Contains(out, "N/A") {
+		t.Errorf("table output missing N/A for mean latency with no scored data:\n%s", out)
+	}
+	if strings.Contains(out, "0s") {
+		t.Errorf("table output should not render a misleading 0s mean latency:\n%s", out)
+	}
+}
+
+func TestCategoryRollup_ExcludesErrorsAndSkips(t *testing.T) {
+	tests := []testkit.Test{
+		{ID: "a", Category: "cat"},
+		{ID: "b", Category: "cat"},
+		{ID: "c", Category: "cat"},
+	}
+	results := []runner.Result{
+		{Model: "m1", TestID: "a", Score: eval.Score{Value: 1}},
+		{Model: "m1", TestID: "b", Err: errors.New("fail")},
+		{Model: "m1", TestID: "c", Score: eval.Score{Skipped: true}},
+	}
+	idx := indexResults(results)
+	rollup := categoryRollup(tests, []string{"m1"}, idx)
+	if len(rollup) != 1 {
+		t.Fatalf("categoryRollup() len = %d, want 1", len(rollup))
+	}
+	if rollup[0].means["m1"] != 1 {
+		t.Errorf("mean = %v, want 1 (error/skip excluded)", rollup[0].means["m1"])
+	}
+}

internal/report/table.go

@@ -0,0 +1,96 @@
+package report
+
+import (
+	"fmt"
+	"io"
+	"math"
+	"strings"
+	"text/tabwriter"
+
+	"github.com/lukaszraczylo/llm-testbench/internal/runner"
+	"github.com/lukaszraczylo/llm-testbench/internal/testkit"
+)
+
+// errWriter wraps an io.Writer and remembers the first write error, so a
+// sequence of Fprint* calls can be written without checking after every
+// line; the caller checks ew.err once at the end.
+type errWriter struct {
+	w   io.Writer
+	err error
+}
+
+func (ew *errWriter) println(args ...any) {
+	if ew.err != nil {
+		return
+	}
+	_, ew.err = fmt.Fprintln(ew.w, args...)
+}
+
+// renderTable writes three plain-text tables: per-test scores, category
+// rollup, and model summary.
+func renderTable(w io.Writer, tests []testkit.Test, models []string, results []runner.Result) error {
+	idx := indexResults(results)
+	tw := tabwriter.NewWriter(w, 0, 4, 2, ' ', 0)
+	ew := &errWriter{w: tw}
+
+	writeScoreTable(ew, tests, models, idx)
+	ew.println()
+	writeCategoryTable(ew, tests, models, idx)
+	ew.println()
+	writeSummaryTable(ew, models, results)
+	if ew.err != nil {
+		return ew.err
+	}
+
+	return tw.Flush()
+}
+
+func writeScoreTable(ew *errWriter, tests []testkit.Test, models []string, idx resultIndex) {
+	ew.println("== Per-test scores ==")
+	header := []string{"TEST", "CATEGORY", "SUBCATEGORY"}
+	header = append(header, models...)
+	ew.println(strings.Join(header, "\t"))
+
+	for _, t := range sortedTests(tests) {
+		row := []string{t.ID, t.Category, t.Subcategory}
+		for _, model := range models {
+			r, ok := idx[t.ID][model]
+			row = append(row, cellText(r, ok))
+		}
+		ew.println(strings.Join(row, "\t"))
+	}
+}
+
+func writeCategoryTable(ew *errWriter, tests []testkit.Test, models []string, idx resultIndex) {
+	ew.println("== Category rollup (mean score) ==")
+	header := append([]string{"CATEGORY"}, models...)
+	ew.println(strings.Join(header, "\t"))
+
+	for _, cm := range categoryRollup(tests, models, idx) {
+		row := []string{cm.category}
+		for _, model := range models {
+			row = append(row, meanCellText(cm.means[model]))
+		}
+		ew.println(strings.Join(row, "\t"))
+	}
+}
+
+func writeSummaryTable(ew *errWriter, models []string, results []runner.Result) {
+	ew.println("== Model summary ==")
+	ew.println(strings.Join([]string{"MODEL", "MEAN", "PASSED", "PARTIAL", "FAILED", "ERRORS", "MEAN_LATENCY", "TOTAL_TOKENS", "TOK/S"}, "\t"))
+
+	for _, ms := range summarize(models, results) {
+		row := []string{
+			ms.model,
+			meanCellText(ms.overallMean),
+			fmt.Sprintf("%d", ms.passed),
+			fmt.Sprintf("%d", ms.partial),
+			fmt.Sprintf("%d", ms.failed),
+			fmt.Sprintf("%d", ms.errors),
+			durationText(ms.meanLatency, !math.IsNaN(ms.overallMean)),
+			fmt.Sprintf("%d", ms.totalTokens),
+			rateText(ms.avgTokPerSec),
+		}
+		ew.println(strings.Join(row, "\t"))
+	}
+}

internal/runner/runner.go

@@ -0,0 +1,132 @@
+// Package runner fans a test catalog out across models with bounded
+// concurrency and collects per-(model,test) results.
+package runner
+
+import (
+	"context"
+	"time"
+
+	"golang.org/x/sync/errgroup"
+
+	"github.com/lukaszraczylo/llm-testbench/internal/eval"
+	"github.com/lukaszraczylo/llm-testbench/internal/llm"
+	"github.com/lukaszraczylo/llm-testbench/internal/testkit"
+)
+
+// Result is the outcome of running one test against one model.
+type Result struct {
+	Err              error
+	Model            string
+	TestID           string
+	Score            eval.Score
+	Latency          time.Duration
+	PromptTokens     int
+	CompletionTokens int
+}
+
+// TotalTokens returns PromptTokens + CompletionTokens, the total token
+// usage billed for this (model, test) call.
+func (r Result) TotalTokens() int {
+	return r.PromptTokens + r.CompletionTokens
+}
+
+// Config controls how the Runner fans work out.
+type Config struct {
+	// Concurrency bounds the number of in-flight (model,test) calls.
+	Concurrency int
+	// Temperature is sent on every request; PLAN.md pins this to 0 for
+	// determinism across runs.
+	Temperature float64
+	// MaxTokensDefault is used when a Test does not set its own MaxTokens.
+	MaxTokensDefault int
+}
+
+// Runner executes a set of tests against a set of models.
+type Runner struct {
+	client llm.Client
+	cfg    Config
+}
+
+// New builds a Runner that issues requests through client.
+func New(client llm.Client, cfg Config) *Runner {
+	return &Runner{client: client, cfg: cfg}
+}
+
+// Run executes every combination of models x tests, bounded by
+// cfg.Concurrency in-flight calls, and returns one Result per combination.
+// A per-call failure is captured in Result.Err rather than aborting the run.
+func (r *Runner) Run(ctx context.Context, models []string, tests []testkit.Test) []Result {
+	type job struct {
+		model string
+		test  testkit.Test
+	}
+
+	jobs := make([]job, 0, len(models)*len(tests))
+	for _, m := range models {
+		for _, t := range tests {
+			jobs = append(jobs, job{model: m, test: t})
+		}
+	}
+
+	results := make([]Result, len(jobs))
+
+	g, gctx := errgroup.WithContext(ctx)
+	concurrency := r.cfg.Concurrency
+	if concurrency <= 0 {
+		concurrency = 1
+	}
+	g.SetLimit(concurrency)
+
+	// go.mod's go directive is 1.22+, so loop variables are already
+	// per-iteration scoped; no manual i, j := i, j shadowing needed (N4).
+	for i, j := range jobs {
+		g.Go(func() error {
+			results[i] = r.runOne(gctx, j.model, j.test)
+			return nil
+		})
+	}
+	// Errors are captured per-result, not surfaced from Run; g.Wait() only
+	// ever returns non-nil if a goroutine itself returned an error, which
+	// runOne never does.
+	_ = g.Wait()
+
+	return results
+}
+
+// runOne executes a single (model, test) combination.
+func (r *Runner) runOne(ctx context.Context, model string, test testkit.Test) Result {
+	maxTokens := test.MaxTokens
+	if maxTokens <= 0 {
+		maxTokens = r.cfg.MaxTokensDefault
+	}
+
+	messages := make([]llm.Message, 0, 2)
+	if test.System != "" {
+		messages = append(messages, llm.Message{Role: "system", Content: test.System})
+	}
+	messages = append(messages, llm.Message{Role: "user", Content: test.Prompt})
+
+	req := llm.Request{
+		Model:       model,
+		Messages:    messages,
+		MaxTokens:   maxTokens,
+		Temperature: r.cfg.Temperature,
+	}
+
+	resp, err := r.client.Complete(ctx, req)
+	if err != nil {
+		return Result{Model: model, TestID: test.ID, Err: err}
+	}
+
+	normalized := testkit.Normalize(resp.Text)
+	score := test.Eval.Evaluate(ctx, normalized)
+
+	return Result{
+		Model:            model,
+		TestID:           test.ID,
+		Score:            score,
+		Latency:          resp.Latency,
+		PromptTokens:     resp.PromptTokens,
+		CompletionTokens: resp.CompletionTokens,
+	}
+}

internal/runner/runner_test.go

@@ -0,0 +1,226 @@
+package runner
+
+import (
+	"context"
+	"errors"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/lukaszraczylo/llm-testbench/internal/eval"
+	"github.com/lukaszraczylo/llm-testbench/internal/llm"
+	"github.com/lukaszraczylo/llm-testbench/internal/testkit"
+)
+
+// mockClient is an offline llm.Client for unit tests; it never touches the
+// network.
+type mockClient struct {
+	respond func(req llm.Request) (llm.Response, error)
+
+	inFlight    int32
+	maxInFlight int32
+}
+
+func (m *mockClient) Complete(_ context.Context, req llm.Request) (llm.Response, error) {
+	cur := atomic.AddInt32(&m.inFlight, 1)
+	for {
+		max := atomic.LoadInt32(&m.maxInFlight)
+		if cur <= max || atomic.CompareAndSwapInt32(&m.maxInFlight, max, cur) {
+			break
+		}
+	}
+	defer atomic.AddInt32(&m.inFlight, -1)
+
+	if m.respond != nil {
+		return m.respond(req)
+	}
+	return llm.Response{Text: "OK"}, nil
+}
+
+func echoEval() eval.Evaluator {
+	return eval.EvaluatorFunc(func(_ context.Context, response string) eval.Score {
+		if response == "OK" {
+			return eval.Score{Value: 1}
+		}
+		return eval.Score{Value: 0, Detail: "got " + response}
+	})
+}
+
+func TestRunner_Run_FansOutAllCombinations(t *testing.T) {
+	client := &mockClient{}
+	r := New(client, Config{Concurrency: 4, MaxTokensDefault: 100})
+
+	tests := []testkit.Test{
+		{ID: "t1", Category: "c", Prompt: "p1", Eval: echoEval()},
+		{ID: "t2", Category: "c", Prompt: "p2", Eval: echoEval()},
+	}
+	models := []string{"model-a", "model-b", "model-c"}
+
+	results := r.Run(context.Background(), models, tests)
+
+	if len(results) != len(models)*len(tests) {
+		t.Fatalf("Run() returned %d results, want %d", len(results), len(models)*len(tests))
+	}
+
+	seen := make(map[string]bool)
+	for _, res := range results {
+		seen[res.Model+"/"+res.TestID] = true
+		if res.Err != nil {
+			t.Errorf("Result.Err = %v, want nil", res.Err)
+		}
+		if res.Score.Value != 1 {
+			t.Errorf("Result.Score.Value = %v, want 1", res.Score.Value)
+		}
+	}
+	for _, m := range models {
+		for _, tc := range tests {
+			key := m + "/" + tc.ID
+			if !seen[key] {
+				t.Errorf("Run() missing combination %s", key)
+			}
+		}
+	}
+}
+
+func TestRunner_Run_RespectsConcurrencyLimit(t *testing.T) {
+	client := &mockClient{
+		respond: func(_ llm.Request) (llm.Response, error) {
+			time.Sleep(20 * time.Millisecond)
+			return llm.Response{Text: "OK"}, nil
+		},
+	}
+	const limit = 2
+	r := New(client, Config{Concurrency: limit, MaxTokensDefault: 100})
+
+	var tests []testkit.Test
+	for i := range 8 {
+		tests = append(tests, testkit.Test{ID: string(rune('a' + i)), Category: "c", Prompt: "p", Eval: echoEval()})
+	}
+
+	r.Run(context.Background(), []string{"model-a"}, tests)
+
+	if got := atomic.LoadInt32(&client.maxInFlight); got > limit {
+		t.Errorf("max in-flight calls = %d, want <= %d", got, limit)
+	}
+}
+
+func TestRunner_Run_CapturesPerCallError(t *testing.T) {
+	wantErr := errors.New("transport failure")
+	client := &mockClient{
+		respond: func(_ llm.Request) (llm.Response, error) {
+			return llm.Response{}, wantErr
+		},
+	}
+	r := New(client, Config{Concurrency: 2, MaxTokensDefault: 100})
+
+	results := r.Run(context.Background(), []string{"model-a"}, []testkit.Test{
+		{ID: "t1", Category: "c", Prompt: "p", Eval: echoEval()},
+	})
+
+	if len(results) != 1 {
+		t.Fatalf("Run() returned %d results, want 1", len(results))
+	}
+	if !errors.Is(results[0].Err, wantErr) {
+		t.Errorf("Result.Err = %v, want %v", results[0].Err, wantErr)
+	}
+}
+
+func TestRunner_Run_NormalizesResponseBeforeEval(t *testing.T) {
+	client := &mockClient{
+		respond: func(_ llm.Request) (llm.Response, error) {
+			return llm.Response{Text: "<think>internal reasoning</think>\n\nOK"}, nil
+		},
+	}
+	r := New(client, Config{Concurrency: 1, MaxTokensDefault: 100})
+
+	results := r.Run(context.Background(), []string{"model-a"}, []testkit.Test{
+		{ID: "t1", Category: "c", Prompt: "p", Eval: echoEval()},
+	})
+
+	if results[0].Score.Value != 1 {
+		t.Errorf("Score.Value = %v, want 1 (detail: %s)", results[0].Score.Value, results[0].Score.Detail)
+	}
+}
+
+func TestRunner_Run_UsesTestMaxTokensOverDefault(t *testing.T) {
+	var gotMaxTokens int
+	client := &mockClient{
+		respond: func(req llm.Request) (llm.Response, error) {
+			gotMaxTokens = req.MaxTokens
+			return llm.Response{Text: "OK"}, nil
+		},
+	}
+	r := New(client, Config{Concurrency: 1, MaxTokensDefault: 100})
+
+	r.Run(context.Background(), []string{"model-a"}, []testkit.Test{
+		{ID: "t1", Category: "c", Prompt: "p", MaxTokens: 500, Eval: echoEval()},
+	})
+
+	if gotMaxTokens != 500 {
+		t.Errorf("MaxTokens = %d, want 500 (test override)", gotMaxTokens)
+	}
+}
+
+func TestRunner_Run_FallsBackToDefaultMaxTokens(t *testing.T) {
+	var gotMaxTokens int
+	client := &mockClient{
+		respond: func(req llm.Request) (llm.Response, error) {
+			gotMaxTokens = req.MaxTokens
+			return llm.Response{Text: "OK"}, nil
+		},
+	}
+	r := New(client, Config{Concurrency: 1, MaxTokensDefault: 100})
+
+	r.Run(context.Background(), []string{"model-a"}, []testkit.Test{
+		{ID: "t1", Category: "c", Prompt: "p", Eval: echoEval()},
+	})
+
+	if gotMaxTokens != 100 {
+		t.Errorf("MaxTokens = %d, want 100 (config default)", gotMaxTokens)
+	}
+}
+
+func TestRunner_Run_IncludesSystemMessageWhenSet(t *testing.T) {
+	var gotMessages []llm.Message
+	client := &mockClient{
+		respond: func(req llm.Request) (llm.Response, error) {
+			gotMessages = req.Messages
+			return llm.Response{Text: "OK"}, nil
+		},
+	}
+	r := New(client, Config{Concurrency: 1, MaxTokensDefault: 100})
+
+	r.Run(context.Background(), []string{"model-a"}, []testkit.Test{
+		{ID: "t1", Category: "c", System: "you are terse", Prompt: "p", Eval: echoEval()},
+	})
+
+	if len(gotMessages) != 2 || gotMessages[0].Role != "system" || gotMessages[1].Role != "user" {
+		t.Errorf("Messages = %+v, want [system, user]", gotMessages)
+	}
+}
+
+func TestRunner_Run_TracksLatencyAndTokens(t *testing.T) {
+	client := &mockClient{
+		respond: func(_ llm.Request) (llm.Response, error) {
+			return llm.Response{Text: "OK", PromptTokens: 10, CompletionTokens: 5, Latency: 42 * time.Millisecond}, nil
+		},
+	}
+	r := New(client, Config{Concurrency: 1, MaxTokensDefault: 100})
+
+	results := r.Run(context.Background(), []string{"model-a"}, []testkit.Test{
+		{ID: "t1", Category: "c", Prompt: "p", Eval: echoEval()},
+	})
+
+	if results[0].PromptTokens != 10 {
+		t.Errorf("PromptTokens = %d, want 10", results[0].PromptTokens)
+	}
+	if results[0].CompletionTokens != 5 {
+		t.Errorf("CompletionTokens = %d, want 5", results[0].CompletionTokens)
+	}
+	if results[0].TotalTokens() != 15 {
+		t.Errorf("TotalTokens() = %d, want 15", results[0].TotalTokens())
+	}
+	if results[0].Latency != 42*time.Millisecond {
+		t.Errorf("Latency = %v, want 42ms", results[0].Latency)
+	}
+}

internal/testkit/testkit.go

@@ -0,0 +1,133 @@
+// Package testkit defines the Test type shared by the catalog in
+// internal/tests, a Registry to collect them, and response normalization
+// applied before evaluation.
+package testkit
+
+import (
+	"fmt"
+	"regexp"
+	"sort"
+	"strings"
+
+	"github.com/lukaszraczylo/llm-testbench/internal/eval"
+)
+
+// Test is one catalog entry: a fully self-contained prompt scored by a
+// deterministic Evaluator.
+type Test struct {
+	Eval        eval.Evaluator
+	ID          string
+	Category    string
+	Subcategory string
+	Description string
+	System      string
+	Prompt      string
+	MaxTokens   int
+}
+
+// thinkBlockPattern strips a leading <think>...</think> or
+// <reasoning>...</reasoning> block, case-insensitive, before the real
+// answer. Models observed on this endpoint prepend one.
+var thinkBlockPattern = regexp.MustCompile(`(?is)^\s*<(think|reasoning)>.*?</(think|reasoning)>\s*`)
+
+// thinkCloseTagPattern and thinkOpenTagPattern locate a closing/opening
+// <think>/<reasoning> tag anywhere in the text, used to detect an orphan
+// closing tag (S7): some models on this endpoint truncate the opening tag
+// (e.g. it falls outside a context/streaming window) but still emit the
+// closing tag, leaving stray reasoning text with no opener in front of it.
+var thinkCloseTagPattern = regexp.MustCompile(`(?i)</(?:think|reasoning)>`)
+var thinkOpenTagPattern = regexp.MustCompile(`(?i)<(?:think|reasoning)>`)
+
+// stripOrphanThinkClose removes everything up to and including a closing
+// </think>/</reasoning> tag that has no matching opening tag earlier in s.
+// A tag pair with a real opener is left alone (thinkBlockPattern already
+// handles the well-formed, anchored-at-start case).
+func stripOrphanThinkClose(s string) string {
+	closeLoc := thinkCloseTagPattern.FindStringIndex(s)
+	if closeLoc == nil {
+		return s
+	}
+	if thinkOpenTagPattern.MatchString(s[:closeLoc[0]]) {
+		return s
+	}
+	return s[closeLoc[1]:]
+}
+
+// Normalize prepares a raw model response for evaluation: it strips a
+// leading <think>/<reasoning> block (or, failing that, an orphan closing
+// tag with no opener), collapses Windows line endings, and trims
+// surrounding whitespace.
+func Normalize(response string) string {
+	s := thinkBlockPattern.ReplaceAllString(response, "")
+	s = stripOrphanThinkClose(s)
+	s = strings.ReplaceAll(s, "\r\n", "\n")
+	return strings.TrimSpace(s)
+}
+
+// Registry collects Tests and provides lookup/filtering for the runner and
+// CLI.
+type Registry struct {
+	byID  map[string]Test
+	tests []Test
+}
+
+// NewRegistry builds an empty Registry.
+func NewRegistry() *Registry {
+	return &Registry{byID: make(map[string]Test)}
+}
+
+// Register adds t to the registry. It panics on a duplicate ID since the
+// catalog is assembled once at program startup via init()-time registration,
+// making a duplicate a programming error, not a runtime condition.
+func (r *Registry) Register(t Test) {
+	if t.ID == "" {
+		panic("testkit: Test.ID must not be empty")
+	}
+	if _, exists := r.byID[t.ID]; exists {
+		panic(fmt.Sprintf("testkit: duplicate test ID %q", t.ID))
+	}
+	if t.Eval == nil {
+		panic(fmt.Sprintf("testkit: test %q has a nil Evaluator", t.ID))
+	}
+	r.tests = append(r.tests, t)
+	r.byID[t.ID] = t
+}
+
+// All returns every registered test, sorted by ID for deterministic output.
+func (r *Registry) All() []Test {
+	out := make([]Test, len(r.tests))
+	copy(out, r.tests)
+	sort.Slice(out, func(i, j int) bool { return out[i].ID < out[j].ID })
+	return out
+}
+
+// Filter returns registered tests whose Category/Subcategory match the
+// given filters. Empty filters match everything.
+func (r *Registry) Filter(category, subcategory string) []Test {
+	all := r.All()
+	if category == "" && subcategory == "" {
+		return all
+	}
+	var out []Test
+	for _, t := range all {
+		if category != "" && !strings.EqualFold(t.Category, category) {
+			continue
+		}
+		if subcategory != "" && !strings.EqualFold(t.Subcategory, subcategory) {
+			continue
+		}
+		out = append(out, t)
+	}
+	return out
+}
+
+// Get returns the test with the given ID.
+func (r *Registry) Get(id string) (Test, bool) {
+	t, ok := r.byID[id]
+	return t, ok
+}
+
+// Len returns the number of registered tests.
+func (r *Registry) Len() int {
+	return len(r.tests)
+}

internal/testkit/testkit_test.go

@@ -0,0 +1,153 @@
+package testkit
+
+import (
+	"context"
+	"testing"
+
+	"github.com/lukaszraczylo/llm-testbench/internal/eval"
+)
+
+func TestNormalize(t *testing.T) {
+	tests := []struct {
+		name string
+		in   string
+		want string
+	}{
+		{"trims whitespace", "  hello  ", "hello"},
+		{"leading whitespace before answer", "\n\nOK", "OK"},
+		{"strips think block", "<think>reasoning here</think>OK", "OK"},
+		{"strips think block with whitespace after", "<think>reasoning here</think>\n\nOK", "OK"},
+		{"strips think block case insensitive", "<THINK>x</THINK>OK", "OK"},
+		{"strips reasoning block", "<reasoning>plan</reasoning>final answer", "final answer"},
+		{"collapses windows line endings", "line1\r\nline2", "line1\nline2"},
+		{"multiline think block", "<think>\nstep 1\nstep 2\n</think>\nanswer", "answer"},
+		{"no think block passthrough", "just a plain answer", "just a plain answer"},
+		{"think tag not at start is left alone", "prefix <think>x</think> suffix", "prefix <think>x</think> suffix"},
+		// S7: an orphan closing tag (opening tag truncated by the gateway)
+		// must still be stripped.
+		{"orphan think close tag with no opener", "leftover reasoning residue</think>OK", "OK"},
+		{"orphan reasoning close tag with no opener", "leftover plan</reasoning>final answer", "final answer"},
+		{"orphan close tag case insensitive", "residue</THINK>OK", "OK"},
+		{"orphan close tag with whitespace after", "residue</think>\n\nOK", "OK"},
+		{"no close tag at all passthrough", "just a plain answer with no tags", "just a plain answer with no tags"},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := Normalize(tt.in)
+			if got != tt.want {
+				t.Errorf("Normalize(%q) = %q, want %q", tt.in, got, tt.want)
+			}
+		})
+	}
+}
+
+func newDummyEval(value float64) eval.Evaluator {
+	return eval.EvaluatorFunc(func(_ context.Context, _ string) eval.Score {
+		return eval.Score{Value: value}
+	})
+}
+
+func TestRegistry_RegisterAndAll(t *testing.T) {
+	r := NewRegistry()
+	r.Register(Test{ID: "b-test", Category: "cat", Subcategory: "sub", Prompt: "p", Eval: newDummyEval(1)})
+	r.Register(Test{ID: "a-test", Category: "cat", Subcategory: "sub", Prompt: "p", Eval: newDummyEval(1)})
+
+	all := r.All()
+	if len(all) != 2 {
+		t.Fatalf("All() len = %d, want 2", len(all))
+	}
+	if all[0].ID != "a-test" || all[1].ID != "b-test" {
+		t.Errorf("All() not sorted by ID: got %q, %q", all[0].ID, all[1].ID)
+	}
+}
+
+func TestRegistry_RegisterDuplicateIDPanics(t *testing.T) {
+	r := NewRegistry()
+	r.Register(Test{ID: "dup", Category: "c", Prompt: "p", Eval: newDummyEval(1)})
+
+	defer func() {
+		if recover() == nil {
+			t.Fatal("Register() with duplicate ID did not panic")
+		}
+	}()
+	r.Register(Test{ID: "dup", Category: "c", Prompt: "p", Eval: newDummyEval(1)})
+}
+
+func TestRegistry_RegisterEmptyIDPanics(t *testing.T) {
+	r := NewRegistry()
+	defer func() {
+		if recover() == nil {
+			t.Fatal("Register() with empty ID did not panic")
+		}
+	}()
+	r.Register(Test{Category: "c", Prompt: "p", Eval: newDummyEval(1)})
+}
+
+func TestRegistry_RegisterNilEvaluatorPanics(t *testing.T) {
+	r := NewRegistry()
+	defer func() {
+		if recover() == nil {
+			t.Fatal("Register() with nil Evaluator did not panic")
+		}
+	}()
+	r.Register(Test{ID: "x", Category: "c", Prompt: "p"})
+}
+
+func TestRegistry_Filter(t *testing.T) {
+	r := NewRegistry()
+	r.Register(Test{ID: "go-1", Category: "programming", Subcategory: "golang", Prompt: "p", Eval: newDummyEval(1)})
+	r.Register(Test{ID: "py-1", Category: "programming", Subcategory: "python", Prompt: "p", Eval: newDummyEval(1)})
+	r.Register(Test{ID: "k8s-1", Category: "operations", Subcategory: "kubernetes", Prompt: "p", Eval: newDummyEval(1)})
+
+	tests := []struct {
+		name        string
+		category    string
+		subcategory string
+		wantIDs     []string
+	}{
+		{"no filter", "", "", []string{"go-1", "k8s-1", "py-1"}},
+		{"by category", "programming", "", []string{"go-1", "py-1"}},
+		{"by category case insensitive", "PROGRAMMING", "", []string{"go-1", "py-1"}},
+		{"by subcategory", "", "golang", []string{"go-1"}},
+		{"by category and subcategory", "programming", "python", []string{"py-1"}},
+		{"no match", "nonexistent", "", nil},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := r.Filter(tt.category, tt.subcategory)
+			if len(got) != len(tt.wantIDs) {
+				t.Fatalf("Filter(%q, %q) len = %d, want %d", tt.category, tt.subcategory, len(got), len(tt.wantIDs))
+			}
+			for i, want := range tt.wantIDs {
+				if got[i].ID != want {
+					t.Errorf("Filter(%q, %q)[%d].ID = %q, want %q", tt.category, tt.subcategory, i, got[i].ID, want)
+				}
+			}
+		})
+	}
+}
+
+func TestRegistry_Get(t *testing.T) {
+	r := NewRegistry()
+	r.Register(Test{ID: "x", Category: "c", Prompt: "p", Eval: newDummyEval(1)})
+
+	got, ok := r.Get("x")
+	if !ok || got.ID != "x" {
+		t.Errorf("Get(%q) = %+v, %v, want ID x, true", "x", got, ok)
+	}
+	_, ok = r.Get("missing")
+	if ok {
+		t.Error("Get() for missing ID returned ok=true")
+	}
+}
+
+func TestRegistry_Len(t *testing.T) {
+	r := NewRegistry()
+	if r.Len() != 0 {
+		t.Errorf("Len() = %d, want 0", r.Len())
+	}
+	r.Register(Test{ID: "x", Category: "c", Prompt: "p", Eval: newDummyEval(1)})
+	if r.Len() != 1 {
+		t.Errorf("Len() = %d, want 1", r.Len())
+	}
+}

internal/tests/agents.go

@@ -0,0 +1,106 @@
+package tests
+
+import (
+	"github.com/lukaszraczylo/llm-testbench/internal/eval"
+	"github.com/lukaszraczylo/llm-testbench/internal/testkit"
+)
+
+func registerAgentsTests(r *testkit.Registry) {
+	r.Register(agentToolRoutingTest())
+	r.Register(agentPlanOrderingTest())
+}
+
+// agentToolRoutingRoster is the inline tool roster for
+// agentToolRoutingTest.
+const agentToolRoutingRoster = `- search_web: Search the public web for current information not in your
+  training data.
+- read_file: Read the contents of a file from the local filesystem.
+- run_shell: Execute an arbitrary shell command on the local machine.
+- query_db: Run a read-only SQL query against the application's Postgres
+  database.
+- send_email: Send an email to a specified recipient.
+- none: No tool is needed; answer directly from what you already know.`
+
+// agentToolRoutingTest: map 4 mini-tasks to the correct tool from a 6-tool
+// roster.
+//
+// ground truth: task1 is arithmetic answerable without any tool (none);
+// task2 needs a row count from a database table (query_db); task3 asks for
+// current information that cannot be in training data (search_web); task4
+// asks to notify someone by email (send_email).
+func agentToolRoutingTest() testkit.Test {
+	prompt := `You have access to this roster of tools:
+
+` + agentToolRoutingRoster + `
+
+For each of these 4 tasks, decide which single tool (or "none") is the
+correct one to use:
+
+task1: "What is 15% of 240?"
+task2: "How many rows are in the 'orders' table?"
+task3: "What is today's top headline on a major news site?"
+task4: "Notify the on-call engineer at [email protected] that the deploy
+finished."
+
+Respond with only a JSON object mapping each task id to the tool name:
+{"task1":"...","task2":"...","task3":"...","task4":"..."}`
+
+	evaluator := eval.Mean(
+		eval.JSONField("task1", "none"),
+		eval.JSONField("task2", "query_db"),
+		eval.JSONField("task3", "search_web"),
+		eval.JSONField("task4", "send_email"),
+	)
+
+	return testkit.Test{
+		ID:          "agent-tool-routing",
+		Category:    "agents",
+		Subcategory: "tool-routing",
+		Description: "Map 4 mini-tasks to the correct tool from a 6-tool roster.",
+		Prompt:      prompt,
+		MaxTokens:   300,
+		Eval:        evaluator,
+	}
+}
+
+// agentPlanOrderingWant is the unique valid step order implied by the
+// dependency constraints in agentPlanOrderingTest's prompt.
+var agentPlanOrderingWant = []string{"build", "test", "backup", "deploy", "verify", "rollback"}
+
+// agentPlanOrderingTest: order deployment steps that have explicit
+// pairwise dependencies.
+//
+// ground truth: the prompt states build must precede test (need a built
+// artifact to test), backup must happen only after tests pass and only
+// before deploy, deploy must follow backup, verify must follow deploy, and
+// rollback is only meaningful after a deploy whose verify step failed. This
+// pins a single valid total order: build, test, backup, deploy, verify,
+// rollback.
+func agentPlanOrderingTest() testkit.Test {
+	prompt := `You are planning a deployment with these steps: build, test,
+backup, deploy, verify, rollback.
+
+Constraints:
+- You must build the artifact before you can test it.
+- You only take a backup of the current production state once tests have
+  passed - there is no point backing up before you know the new build is
+  viable.
+- Deploy only happens after the backup completes.
+- After deploy, you must verify the new deployment is healthy.
+- Rollback (restoring the pre-deploy backup) only makes sense after a
+  deploy has happened, and only if verify fails.
+
+Give the single correct step order as a JSON array of step ids, e.g.
+["build","test",...]. Include all 6 steps, listing rollback in its position
+in the sequence even though it is conditional on verify failing.`
+
+	return testkit.Test{
+		ID:          "agent-plan-ordering",
+		Category:    "agents",
+		Subcategory: "planning",
+		Description: "Order 6 deployment steps under explicit pairwise dependency constraints.",
+		Prompt:      prompt,
+		MaxTokens:   300,
+		Eval:        eval.JSONStringArrayEquals(agentPlanOrderingWant),
+	}
+}

internal/tests/agents_test.go

@@ -0,0 +1,84 @@
+package tests
+
+import (
+	"context"
+	"testing"
+)
+
+func TestAgentToolRoutingTest_Eval(t *testing.T) {
+	tc := agentToolRoutingTest()
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{
+			name:     "all correct",
+			response: `{"task1":"none","task2":"query_db","task3":"search_web","task4":"send_email"}`,
+			want:     1,
+		},
+		{
+			name:     "one wrong",
+			response: `{"task1":"run_shell","task2":"query_db","task3":"search_web","task4":"send_email"}`,
+			want:     0.75,
+		},
+		{
+			name:     "all wrong",
+			response: `{"task1":"run_shell","task2":"search_web","task3":"query_db","task4":"none"}`,
+			want:     0,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}
+
+func TestAgentPlanOrderingTest_Eval(t *testing.T) {
+	tc := agentPlanOrderingTest()
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{
+			name:     "correct order",
+			response: `["build","test","backup","deploy","verify","rollback"]`,
+			want:     1,
+		},
+		{
+			name:     "correct order fenced",
+			response: "```json\n[\"build\",\"test\",\"backup\",\"deploy\",\"verify\",\"rollback\"]\n```",
+			want:     1,
+		},
+		{
+			name:     "backup before test violates dependency",
+			response: `["build","backup","test","deploy","verify","rollback"]`,
+			want:     0,
+		},
+		{
+			name:     "rollback before verify",
+			response: `["build","test","backup","deploy","rollback","verify"]`,
+			want:     0,
+		},
+		{
+			name:     "missing a step",
+			response: `["build","test","backup","deploy","verify"]`,
+			want:     0,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}

internal/tests/c.go

@@ -0,0 +1,50 @@
+package tests
+
+import (
+	"github.com/lukaszraczylo/llm-testbench/internal/eval"
+	"github.com/lukaszraczylo/llm-testbench/internal/testkit"
+)
+
+func registerCTests(r *testkit.Registry) {
+	r.Register(cStructSizeTest())
+}
+
+// cStructSizeWant is sizeof(struct Config) on LP64 (64-bit System V / Apple
+// ARM64 ABI: natural alignment, no packing).
+//
+// ground truth: char flag(align 1, size 1) at offset 0; double ratio
+// (align 8, size 8) needs offset divisible by 8, so it starts at offset 8
+// (7 bytes padding after flag), ending at 16; int count (align 4, size 4)
+// at offset 16 (already aligned), ending at 20; short mode (align 2, size
+// 2) at offset 20, ending at 22. The struct's own alignment is the widest
+// member alignment (8, from double), so the total size rounds up from 22
+// to the next multiple of 8: 24. c_test.go cross-checks this with cc via
+// sizeof and offsetof when the cc toolchain is available.
+const cStructSizeWant = 24
+
+func cStructSizeTest() testkit.Test {
+	prompt := `Here is a C struct:
+
+` + "```c" + `
+struct Config {
+	char flag;
+	double ratio;
+	int count;
+	short mode;
+};
+` + "```" + `
+
+Assuming the LP64 data model (64-bit, no #pragma pack, no
+non-default alignment attributes), what is sizeof(struct Config) in bytes?
+Respond with only the number, nothing else.`
+
+	return testkit.Test{
+		ID:          "c-struct-size",
+		Category:    "programming",
+		Subcategory: "c",
+		Description: "Compute sizeof() for a C struct on LP64 given its member types and default alignment.",
+		Prompt:      prompt,
+		MaxTokens:   200,
+		Eval:        eval.Numeric(eval.ExtractLastNumber[int], cStructSizeWant, 0),
+	}
+}

internal/tests/c_test.go

@@ -0,0 +1,87 @@
+package tests
+
+import (
+	"context"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+	"testing"
+)
+
+// TestCStructSizeWant_GroundTruth cross-checks cStructSizeWant against the
+// real cc toolchain's sizeof(struct Config), rather than trusting the
+// hand-derived alignment arithmetic in the doc comment alone. Skips (does
+// not fail) when cc is unavailable, matching the exec-evaluator contract.
+func TestCStructSizeWant_GroundTruth(t *testing.T) {
+	if _, err := exec.LookPath("cc"); err != nil {
+		t.Skip("cc not found on PATH, skipping ground-truth cross-check")
+	}
+
+	const src = `#include <stdio.h>
+struct Config {
+	char flag;
+	double ratio;
+	int count;
+	short mode;
+};
+int main(void) {
+	printf("%zu", sizeof(struct Config));
+	return 0;
+}
+`
+	dir := t.TempDir()
+	srcPath := filepath.Join(dir, "main.c")
+	if err := os.WriteFile(srcPath, []byte(src), 0o600); err != nil {
+		t.Fatalf("write main.c: %v", err)
+	}
+	binPath := filepath.Join(dir, "prog")
+
+	// #nosec G204 -- compiling a fixed, hardcoded C source (src above) in a
+	// t.TempDir() this test just created; not external input.
+	build := exec.Command("cc", "-o", binPath, srcPath) //nolint:gosec // see comment above
+	if out, err := build.CombinedOutput(); err != nil {
+		t.Fatalf("cc failed: %v\n%s", err, out)
+	}
+
+	// #nosec G204 -- binPath is the binary this test just compiled above,
+	// in its own t.TempDir(); not an externally supplied executable path.
+	out, err := exec.Command(binPath).Output() //nolint:gosec // see comment above
+	if err != nil {
+		t.Fatalf("running compiled program failed: %v", err)
+	}
+
+	got := strings.TrimSpace(string(out))
+	if got != "24" {
+		t.Errorf("cc-computed sizeof(struct Config) = %s, want 24 (matches cStructSizeWant)", got)
+	}
+	if cStructSizeWant != 24 {
+		t.Errorf("cStructSizeWant = %d, want 24", cStructSizeWant)
+	}
+}
+
+func TestCStructSizeTest_Eval(t *testing.T) {
+	tc := cStructSizeTest()
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{"exact correct", "24", 1},
+		{"prose wrapped", "sizeof(struct Config) is 24 bytes", 1},
+		{"unpadded sum is wrong", "15", 0},
+		{"off by one", "23", 0},
+		// S6 regression: the trailing "64-bit" qualifier must not be
+		// mistaken for the answer.
+		{"answer followed by a 64-bit qualifier", "sizeof(struct Config) is 24 bytes on a 64-bit system", 1},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate() = %v, want %v (detail: %s)", got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}

internal/tests/catalog.go

@@ -0,0 +1,24 @@
+package tests
+
+import "github.com/lukaszraczylo/llm-testbench/internal/testkit"
+
+// All builds a Registry containing the full 17-test catalog: 3
+// programming/golang, 2 programming/python, 1 programming/typescript, 1
+// programming/c, 2 operations/macos, 2 operations/linux, 1
+// operations/kubernetes, 1 research/web, 1 research/whitepapers, 1
+// research/codebase, and 2 agents (tool-routing, planning).
+func All() *testkit.Registry {
+	r := testkit.NewRegistry()
+	registerGolangTests(r)
+	registerPythonTests(r)
+	registerTypeScriptTests(r)
+	registerCTests(r)
+	registerMacOSTests(r)
+	registerLinuxTests(r)
+	registerKubernetesTests(r)
+	registerWebTests(r)
+	registerWhitepapersTests(r)
+	registerCodebaseTests(r)
+	registerAgentsTests(r)
+	return r
+}

internal/tests/catalog_test.go

@@ -0,0 +1,61 @@
+package tests
+
+import "testing"
+
+// wantCatalog is the full 17-test catalog per PLAN.md, id -> {category,
+// subcategory}. TestAll_MatchesCatalog checks the registry against this
+// list exactly, so an accidental removal, duplication, or miscategorized
+// test fails loudly.
+var wantCatalog = map[string][2]string{
+	"go-struct-align":           {"programming", "golang"},
+	"go-worker-pool":            {"programming", "golang"},
+	"go-semver-classify":        {"programming", "golang"},
+	"py-log-triage":             {"programming", "python"},
+	"py-cosine":                 {"programming", "python"},
+	"ts-debounce-composable":    {"programming", "typescript"},
+	"c-struct-size":             {"programming", "c"},
+	"macos-timeout-portability": {"operations", "macos"},
+	"macos-launchd-cron":        {"operations", "macos"},
+	"linux-pct-exec":            {"operations", "linux"},
+	"linux-systemd-oneshot":     {"operations", "linux"},
+	"k8s-crashloop-gitops":      {"operations", "kubernetes"},
+	"web-robots-ai-crawlers":    {"research", "web"},
+	"paper-hnsw-params":         {"research", "whitepapers"},
+	"code-trace-go":             {"research", "codebase"},
+	"agent-tool-routing":        {"agents", "tool-routing"},
+	"agent-plan-ordering":       {"agents", "planning"},
+}
+
+func TestAll_MatchesCatalog(t *testing.T) {
+	r := All()
+
+	if r.Len() != len(wantCatalog) {
+		t.Fatalf("registry has %d tests, want %d", r.Len(), len(wantCatalog))
+	}
+
+	for id, catSub := range wantCatalog {
+		tc, ok := r.Get(id)
+		if !ok {
+			t.Errorf("missing test %q", id)
+			continue
+		}
+		if tc.Category != catSub[0] {
+			t.Errorf("%s: Category = %q, want %q", id, tc.Category, catSub[0])
+		}
+		if tc.Subcategory != catSub[1] {
+			t.Errorf("%s: Subcategory = %q, want %q", id, tc.Subcategory, catSub[1])
+		}
+		if tc.Prompt == "" {
+			t.Errorf("%s: Prompt is empty", id)
+		}
+		if tc.Eval == nil {
+			t.Errorf("%s: Eval is nil", id)
+		}
+		if tc.Description == "" {
+			t.Errorf("%s: Description is empty", id)
+		}
+		if tc.MaxTokens <= 0 {
+			t.Errorf("%s: MaxTokens = %d, want > 0", id, tc.MaxTokens)
+		}
+	}
+}

internal/tests/codebase.go

@@ -0,0 +1,66 @@
+package tests
+
+import (
+	"github.com/lukaszraczylo/llm-testbench/internal/eval"
+	"github.com/lukaszraczylo/llm-testbench/internal/testkit"
+)
+
+func registerCodebaseTests(r *testkit.Registry) {
+	r.Register(codeTraceGoTest())
+}
+
+// mix is the traced function for codeTraceGoTest: deterministic bit
+// manipulation plus recursion. It is called directly (not re-implemented)
+// to derive codeTraceGoWant, and codebase_test.go calls it again with the
+// same input to ground that value, per PLAN.md's instruction that this
+// test's ground truth be established by running the same function.
+func mix(n int, depth int) int {
+	if depth == 0 {
+		return n
+	}
+	if n%2 == 0 {
+		n = n >> 1
+	} else {
+		n = (n << 1) ^ 0x2A
+	}
+	return mix(n, depth-1) + depth
+}
+
+// codeTraceGoWant is mix(37, 4), computed by running mix itself.
+//
+// ground truth: mix(37,4): 37 is odd -> n=(37<<1)^0x2A=96, return
+// mix(96,3)+4. 96 is even -> n=48, return mix(48,2)+3. 48 is even -> n=24,
+// return mix(24,1)+2. 24 is even -> n=12, return mix(12,0)+1. depth=0 ->
+// return 12. Unwinding: 12+1=13, 13+2=15, 15+3=18, 18+4=22.
+var codeTraceGoWant = mix(37, 4)
+
+func codeTraceGoTest() testkit.Test {
+	prompt := `Here is a Go function:
+
+` + "```go" + `
+func mix(n int, depth int) int {
+	if depth == 0 {
+		return n
+	}
+	if n%2 == 0 {
+		n = n >> 1
+	} else {
+		n = (n << 1) ^ 0x2A
+	}
+	return mix(n, depth-1) + depth
+}
+` + "```" + `
+
+What is the exact value of mix(37, 4)? Respond with only the number,
+nothing else.`
+
+	return testkit.Test{
+		ID:          "code-trace-go",
+		Category:    "research",
+		Subcategory: "codebase",
+		Description: "Trace a recursive Go function combining bit manipulation and recursion to an exact numeric result.",
+		Prompt:      prompt,
+		MaxTokens:   200,
+		Eval:        eval.Numeric(eval.ExtractLastNumber[int], codeTraceGoWant, 0),
+	}
+}

internal/tests/codebase_test.go

@@ -0,0 +1,64 @@
+package tests
+
+import (
+	"context"
+	"testing"
+)
+
+// TestMix_GroundTruth runs mix itself (the traced function) to ground
+// codeTraceGoWant, per PLAN.md's instruction for this specific test.
+func TestMix_GroundTruth(t *testing.T) {
+	tests := []struct {
+		name  string
+		n     int
+		depth int
+		want  int
+	}{
+		{"base case depth 0", 12, 0, 12},
+		{"one level, even", 24, 1, 13}, // 24>>1=12; mix(12,0)+1=13
+		{"two levels", 48, 2, 15},      // 48>>1=24; mix(24,1)+2=15
+		{"three levels", 96, 3, 18},    // 96>>1=48; mix(48,2)+3=18
+		{"full trace, odd start", 37, 4, 22},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := mix(tt.n, tt.depth)
+			if got != tt.want {
+				t.Errorf("mix(%d, %d) = %d, want %d", tt.n, tt.depth, got, tt.want)
+			}
+		})
+	}
+
+	if codeTraceGoWant != 22 {
+		t.Errorf("codeTraceGoWant = %d, want 22", codeTraceGoWant)
+	}
+	if got := mix(37, 4); got != codeTraceGoWant {
+		t.Errorf("mix(37, 4) = %d, want codeTraceGoWant %d", got, codeTraceGoWant)
+	}
+}
+
+func TestCodeTraceGoTest_Eval(t *testing.T) {
+	tc := codeTraceGoTest()
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{"exact correct", "22", 1},
+		{"prose wrapped", "The result is 22.", 1},
+		{"off by one", "21", 0},
+		{"forgot to track depth in the unwind", "12", 0},
+		// S6 regression: a trailing "64-bit" qualifier after the real
+		// answer must not be mistaken for the answer itself.
+		{"answer followed by a 64-bit qualifier", "The result is 22, computed via a 64-bit accumulator.", 1},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}

internal/tests/golang.go

@@ -0,0 +1,225 @@
+// Package tests is the test catalog: one file per category, each
+// registering its Tests into a shared Registry via Register in main.go's
+// wiring (see All in catalog.go).
+package tests
+
+import (
+	"github.com/lukaszraczylo/llm-testbench/internal/eval"
+	"github.com/lukaszraczylo/llm-testbench/internal/testkit"
+)
+
+// terseCodeOnly is prepended as a system message on code-generation tests
+// to keep responses parseable by the exec/regex evaluators.
+const terseCodeOnly = "Respond with only the requested code in a single fenced code block. " +
+	"No prose before or after the code block, no explanations, no comments beyond what was asked for."
+
+func registerGolangTests(r *testkit.Registry) {
+	r.Register(goStructAlignTest())
+	r.Register(goWorkerPoolTest())
+	r.Register(goSemverClassifyTest())
+}
+
+// goStructAlignTest: reorder struct fields to minimize padding on a 64-bit
+// (LP64) system.
+//
+// ground truth: on LP64, alignment/size per field are bool(align 1, size
+// 1), int64(align 8, size 8), int32(align 4, size 4), string(align 8, size
+// 16 = 8-byte pointer + 8-byte length), byte(align 1, size 1). The Go
+// compiler lays out fields in declaration order, padding each field to its
+// own alignment, then rounds the struct size up to the widest field
+// alignment (8).
+//
+//	Wasteful order: Active bool(0..1) pad(1..8) ID int64(8..16)
+//	Priority int32(16..20) pad(20..24) Name string(24..40) Flag byte(40..41)
+//	pad(41..48) -> total 48 bytes, 18 bytes of padding.
+//
+//	Optimal order: group the two 8-byte-aligned fields first (Name string,
+//	ID int64, in either order: 0..16, 16..24), then the 4-byte-aligned
+//	int32 (24..28), then the two 1-byte fields adjacent with no padding
+//	between them (28..29, 29..30), then round up to the next multiple of 8
+//	-> total 32 bytes. No smaller layout exists: the payload is
+//	16+8+4+1+1 = 30 bytes and the struct's own alignment (8, from the
+//	64-bit-aligned fields) forces the size up to the next multiple of 8,
+//	which is 32.
+//
+// golang_test.go verifies both sizes with unsafe.Sizeof on literal struct
+// definitions, and cross-checks computeLP64StructSize itself against
+// unsafe.Sizeof for several canonical field orders. Scoring
+// (goStructAlignEval) parses the response's actual field order and
+// computes its real layout size rather than matching one hardcoded
+// ordering, so any of the several field orders that reach 32 bytes scores
+// full credit (S3).
+func goStructAlignTest() testkit.Test {
+	prompt := `Here is a Go struct definition:
+
+` + "```go" + `
+type Task struct {
+	Active   bool
+	ID       int64
+	Priority int32
+	Name     string
+	Flag     byte
+}
+` + "```" + `
+
+On a 64-bit (LP64) system, this field order wastes memory to padding.
+Reorder the fields (keep the same field names and types, change nothing
+else) to minimize the total size of the struct due to alignment padding.
+Respond with the corrected struct definition only.`
+
+	return testkit.Test{
+		ID:          "go-struct-align",
+		Category:    "programming",
+		Subcategory: "golang",
+		Description: "Reorder a Go struct's fields to minimize alignment padding on a 64-bit system.",
+		System:      terseCodeOnly,
+		Prompt:      prompt,
+		MaxTokens:   500,
+		Eval:        goStructAlignEval(),
+	}
+}
+
+// goWorkerPoolHarness is a complete, independent driver file (its own
+// package clause and imports) that eval.GoRun writes to harness.go
+// alongside the model's response in solution.go, so the model's code and
+// the harness's code never share one spliced import block. It drives the
+// response's Pool[T, R any] implementation (defined in solution.go, same
+// package) with a deliberately slow worker function and checks (a) output
+// order matches input order and (b) the pool actually parallelizes work,
+// via both an observed-concurrency counter and a wall-clock bound far
+// below the fully-sequential time.
+const goWorkerPoolHarness = `package main
+
+import (
+	"fmt"
+	"os"
+	"sync/atomic"
+	"time"
+)
+
+func main() {
+	const n = 100
+	const workers = 8
+	const perItem = 2 * time.Millisecond
+
+	in := make([]int, n)
+	for i := range in {
+		in[i] = i
+	}
+
+	var concurrent int32
+	var maxConcurrent int32
+	f := func(x int) int {
+		cur := atomic.AddInt32(&concurrent, 1)
+		for {
+			m := atomic.LoadInt32(&maxConcurrent)
+			if cur <= m || atomic.CompareAndSwapInt32(&maxConcurrent, m, cur) {
+				break
+			}
+		}
+		time.Sleep(perItem)
+		atomic.AddInt32(&concurrent, -1)
+		return x * 2
+	}
+
+	start := time.Now()
+	out := Pool(in, workers, f)
+	elapsed := time.Since(start)
+
+	if len(out) != n {
+		fmt.Fprintf(os.Stderr, "len(out)=%d, want %d\n", len(out), n)
+		os.Exit(1)
+	}
+	for i := range out {
+		if out[i] != i*2 {
+			fmt.Fprintf(os.Stderr, "out[%d]=%d, want %d: input order not preserved\n", i, out[i], i*2)
+			os.Exit(1)
+		}
+	}
+	if atomic.LoadInt32(&maxConcurrent) < 2 {
+		fmt.Fprintf(os.Stderr, "maxConcurrent=%d, want >= 2: no real concurrency observed\n", maxConcurrent)
+		os.Exit(1)
+	}
+	// Sequential execution would take n*perItem = 200ms; with workers
+	// goroutines truly running in parallel it should land well under that.
+	if elapsed > 120*time.Millisecond {
+		fmt.Fprintf(os.Stderr, "elapsed=%s, want < 120ms: looks sequential\n", elapsed)
+		os.Exit(1)
+	}
+
+	fmt.Println("PASS")
+}
+`
+
+// goWorkerPoolTest: implement a generic, order-preserving, concurrent
+// worker pool.
+//
+// ground truth: the harness above is the oracle; it directly measures
+// order preservation and real concurrency rather than a hardcoded number,
+// so there is no separate constant to derive.
+func goWorkerPoolTest() testkit.Test {
+	prompt := `Implement this Go generic function:
+
+` + "```go" + `
+func Pool[T, R any](in []T, workers int, f func(T) R) []R
+` + "```" + `
+
+Requirements:
+- Run f concurrently across exactly workers goroutines.
+- The returned slice must preserve the input order: result[i] is f(in[i]).
+- You may include whatever imports your implementation needs. Do not
+  include a package clause or a main function: respond with only the
+  Pool function definition (plus any imports it needs).`
+
+	return testkit.Test{
+		ID:          "go-worker-pool",
+		Category:    "programming",
+		Subcategory: "golang",
+		Description: "Implement a generic, order-preserving, concurrent worker pool.",
+		System:      terseCodeOnly,
+		Prompt:      prompt,
+		MaxTokens:   800,
+		Eval:        eval.GoRun(goWorkerPoolHarness, "PASS"),
+	}
+}
+
+// goSemverClassifyTest: classify the aggregate semver bump implied by a
+// batch of conventional commits.
+//
+// ground truth: per Conventional Commits, a MAJOR bump requires either a
+// "!" after the type/scope or a "BREAKING CHANGE:" footer; a MINOR bump
+// comes from any "feat:" commit; a PATCH bump comes from any "fix:" commit
+// with no feat present; chore/docs/etc. commits do not trigger a release on
+// their own. The batch below has one "feat:" commit, two "fix:" commits, a
+// "chore:" and a "docs:" commit, and zero commits with a "!" marker or a
+// "BREAKING CHANGE:" footer (commit 4's body uses the word "breaking" in
+// the ordinary-English sense "test breaking flakiness", not as a release
+// footer) -> the highest applicable bump across the batch is minor.
+func goSemverClassifyTest() testkit.Test {
+	prompt := `Here are 5 conventional commit messages from one release batch:
+
+1. "feat(cache): add TTL eviction policy for LRU cache"
+2. "fix(scheduler): prevent duplicate job dispatch on retry"
+3. "chore(ci): pin golangci-lint version in the lint config"
+4. "fix(auth): correct token expiry check off by one
+
+This resolves an intermittent test breaking flakiness in local dev but
+does not change any public behaviour."
+5. "docs(readme): clarify installation steps for macOS"
+
+Using the Conventional Commits specification, determine the single semver
+bump ("major", "minor", or "patch") that this batch of commits, taken
+together, should produce for the next release. Respond with only a JSON
+object: {"bump":"major|minor|patch"}`
+
+	return testkit.Test{
+		ID:          "go-semver-classify",
+		Category:    "programming",
+		Subcategory: "golang",
+		Description: "Classify the aggregate semver bump for a batch of conventional commits, including a 'breaking' word trap.",
+		System:      terseCodeOnly,
+		Prompt:      prompt,
+		MaxTokens:   300,
+		Eval:        eval.JSONField("bump", "minor"),
+	}
+}

internal/tests/golang_test.go

@@ -0,0 +1,259 @@
+package tests
+
+import (
+	"context"
+	"testing"
+	"unsafe"
+)
+
+// wastefulTask and optimalTask mirror the struct in goStructAlignTest's
+// prompt (wasteful field order) and its minimal-padding reordering. Sizes
+// are asserted with unsafe.Sizeof to ground the "ground truth:" derivation
+// comment in golang.go against the actual compiler layout, per PLAN.md's
+// requirement to recompute cheap ground truths in the unit test.
+//
+// wastefulTask deliberately keeps the prompt's wasteful field order
+// verbatim (the "before" layout), so fieldalignment's auto-fix must not be
+// allowed to reorder it - that would make it identical to optimalTask and
+// defeat this test's comparison.
+//
+//nolint:govet // fieldalignment: intentional, see comment above
+type wastefulTask struct {
+	Active   bool
+	ID       int64
+	Priority int32
+	Name     string
+	Flag     byte
+}
+
+type optimalTask struct {
+	Name     string
+	ID       int64
+	Priority int32
+	Active   bool
+	Flag     byte
+}
+
+func TestGoStructAlign_GroundTruthSizes(t *testing.T) {
+	if got, want := unsafe.Sizeof(wastefulTask{}), uintptr(48); got != want {
+		t.Errorf("unsafe.Sizeof(wastefulTask{}) = %d, want %d", got, want)
+	}
+	if got, want := unsafe.Sizeof(optimalTask{}), uintptr(32); got != want {
+		t.Errorf("unsafe.Sizeof(optimalTask{}) = %d, want %d", got, want)
+	}
+}
+
+func TestGoStructAlignTest_Eval(t *testing.T) {
+	tc := goStructAlignTest()
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{
+			name:     "optimal order Name then ID",
+			response: "```go\ntype Task struct {\n\tName     string\n\tID       int64\n\tPriority int32\n\tActive   bool\n\tFlag     byte\n}\n```",
+			want:     1,
+		},
+		{
+			name:     "optimal order ID then Name",
+			response: "```go\ntype Task struct {\n\tID       int64\n\tName     string\n\tPriority int32\n\tFlag     byte\n\tActive   bool\n}\n```",
+			want:     1,
+		},
+		{
+			name:     "original wasteful order",
+			response: "```go\ntype Task struct {\n\tActive   bool\n\tID       int64\n\tPriority int32\n\tName     string\n\tFlag     byte\n}\n```",
+			want:     0,
+		},
+		{
+			name:     "int32 not moved after align-8 fields",
+			response: "```go\ntype Task struct {\n\tPriority int32\n\tName     string\n\tID       int64\n\tActive   bool\n\tFlag     byte\n}\n```",
+			want:     0,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate() = %v, want %v (detail: %s)", got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}
+
+func TestGoWorkerPoolTest_Eval(t *testing.T) {
+	tc := goWorkerPoolTest()
+
+	correctPool := "```go\n" + `func Pool[T, R any](in []T, workers int, f func(T) R) []R {
+	out := make([]R, len(in))
+	sem := make(chan struct{}, workers)
+	done := make(chan struct{})
+	for i, v := range in {
+		i, v := i, v
+		sem <- struct{}{}
+		go func() {
+			defer func() { <-sem; done <- struct{}{} }()
+			out[i] = f(v)
+		}()
+	}
+	for range in {
+		<-done
+	}
+	return out
+}
+` + "```"
+
+	sequentialPool := "```go\n" + `func Pool[T, R any](in []T, workers int, f func(T) R) []R {
+	out := make([]R, len(in))
+	for i, v := range in {
+		out[i] = f(v)
+	}
+	return out
+}
+` + "```"
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{"correct concurrent pool", correctPool, 1},
+		{"sequential pool fails concurrency/timing check", sequentialPool, 0},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Skipped {
+				t.Skip("go toolchain not available, skipping exec-evaluator assertion")
+			}
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate() = %v, want %v (detail: %s)", got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}
+
+// TestGoWorkerPoolTest_Eval_ResponseShapes covers B1: the two-file
+// (solution.go + harness.go) split must accept a response that brings its
+// own imports, whether or not it also carries a package clause, since
+// eval.GoRun strips at most a leading package clause and gives the
+// response its own import block separate from the harness's.
+func TestGoWorkerPoolTest_Eval_ResponseShapes(t *testing.T) {
+	tc := goWorkerPoolTest()
+
+	// Shape 1: a bare function body with its own "import" block and no
+	// package clause - the common case when a model obeys "respond with
+	// only the function definition".
+	withImportsNoPackage := "```go\n" + `import "sync"
+
+func Pool[T, R any](in []T, workers int, f func(T) R) []R {
+	out := make([]R, len(in))
+	var wg sync.WaitGroup
+	sem := make(chan struct{}, workers)
+	for i, v := range in {
+		i, v := i, v
+		wg.Add(1)
+		sem <- struct{}{}
+		go func() {
+			defer wg.Done()
+			defer func() { <-sem }()
+			out[i] = f(v)
+		}()
+	}
+	wg.Wait()
+	return out
+}
+` + "```"
+
+	// Shape 2: a full, standalone file - package clause, imports, and the
+	// function - as some models emit even when told not to include a
+	// package clause. eval.GoRun must strip the leading "package main" line
+	// and still compile it as part of solution.go.
+	fullFileWithPackageClause := "```go\n" + `package main
+
+import "sync"
+
+func Pool[T, R any](in []T, workers int, f func(T) R) []R {
+	out := make([]R, len(in))
+	var wg sync.WaitGroup
+	sem := make(chan struct{}, workers)
+	for i, v := range in {
+		i, v := i, v
+		wg.Add(1)
+		sem <- struct{}{}
+		go func() {
+			defer wg.Done()
+			defer func() { <-sem }()
+			out[i] = f(v)
+		}()
+	}
+	wg.Wait()
+	return out
+}
+` + "```"
+
+	// Shape 3: uses sync.WaitGroup without importing "sync" - now that the
+	// prompt explicitly allows imports, this is a genuine model mistake,
+	// not a harness limitation. It must fail to compile (score 0), not be
+	// silently patched or Skipped.
+	usesSyncWithoutImport := "```go\n" + `func Pool[T, R any](in []T, workers int, f func(T) R) []R {
+	out := make([]R, len(in))
+	var wg sync.WaitGroup
+	for i, v := range in {
+		wg.Add(1)
+		i, v := i, v
+		go func() {
+			defer wg.Done()
+			out[i] = f(v)
+		}()
+	}
+	wg.Wait()
+	return out
+}
+` + "```"
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{"imports, no package clause: PASS", withImportsNoPackage, 1},
+		{"full file with package clause: PASS after strip", fullFileWithPackageClause, 1},
+		{"uses sync without importing it: fails to compile", usesSyncWithoutImport, 0},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Skipped {
+				t.Skip("go toolchain not available, skipping exec-evaluator assertion")
+			}
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate() = %v, want %v (detail: %s)", got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}
+
+func TestGoSemverClassifyTest_Eval(t *testing.T) {
+	tc := goSemverClassifyTest()
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{"correct minor", `{"bump":"minor"}`, 1},
+		{"correct minor fenced", "```json\n{\"bump\": \"minor\"}\n```", 1},
+		{"trap: falls for the word breaking and answers major", `{"bump":"major"}`, 0},
+		{"wrong: answers patch, missing the feat commit", `{"bump":"patch"}`, 0},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate() = %v, want %v (detail: %s)", got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}

internal/tests/kubernetes.go

@@ -0,0 +1,139 @@
+package tests
+
+import (
+	"context"
+	"regexp"
+	"strings"
+
+	"github.com/lukaszraczylo/llm-testbench/internal/eval"
+	"github.com/lukaszraczylo/llm-testbench/internal/testkit"
+)
+
+func registerKubernetesTests(r *testkit.Registry) {
+	r.Register(k8sCrashloopGitopsTest())
+}
+
+// k8sCrashloopGitopsDescribeExcerpt is the inline kubectl describe pod
+// excerpt for k8sCrashloopGitopsTest.
+const k8sCrashloopGitopsDescribeExcerpt = `Name:         inventory-worker-6f9b7c8d9-2xk4p
+Namespace:    inventory
+Status:       Running
+Containers:
+  worker:
+    State:          Waiting
+      Reason:       CrashLoopBackOff
+    Last State:     Terminated
+      Reason:       OOMKilled
+      Exit Code:    137
+    Ready:          False
+    Restart Count:  9
+    Limits:
+      cpu:     500m
+      memory:  128Mi
+    Requests:
+      cpu:     100m
+      memory:  64Mi
+Events:
+  Warning  BackOff  kubelet  Back-off restarting failed container worker in pod inventory-worker-6f9b7c8d9-2xk4p`
+
+// mentionsLimitIncrease scores full credit only when the response both
+// names the resource being raised (memory / the limit / the 128Mi value)
+// and uses language for raising it. Using two independent ContainsAny
+// checks combined by weighted mean would give partial credit to a response
+// that raises the wrong resource (e.g. CPU) or that mentions memory
+// without proposing to change the limit; this requires both.
+func mentionsLimitIncrease() eval.Evaluator {
+	verbs := eval.ContainsAny("increase", "raise", "bump", "higher", "raising")
+	target := eval.ContainsAny("memory limit", "limit", "128Mi")
+	return eval.EvaluatorFunc(func(ctx context.Context, response string) eval.Score {
+		v := verbs.Evaluate(ctx, response)
+		t := target.Evaluate(ctx, response)
+		if v.Value == 1 && t.Value == 1 {
+			return eval.Score{Value: 1, Detail: "proposes raising the memory limit"}
+		}
+		return eval.Score{Value: 0, Detail: "does not clearly propose raising the memory limit"}
+	})
+}
+
+// kubectlLiveMutationPattern matches any mention of "kubectl edit" or
+// "kubectl patch", negated or not.
+var kubectlLiveMutationPattern = regexp.MustCompile(`(?i)kubectl\s+(edit|patch)\b`)
+
+// kubectlImperativeMutationPattern matches a bare/imperative "kubectl
+// edit"/"kubectl patch" at the start of a line (optionally after
+// whitespace or a bullet marker) - a command a reader would actually run,
+// never acceptable regardless of negation cues elsewhere in the response.
+var kubectlImperativeMutationPattern = regexp.MustCompile(`(?im)^\s*(?:[-*]\s*)?kubectl\s+(edit|patch)\b`)
+
+// negationCuePattern matches a word that turns a mention of the forbidden
+// command into a warning against running it, rather than an instruction to
+// run it (e.g. "do not run kubectl patch...").
+var negationCuePattern = regexp.MustCompile(`(?i)\b(don'?t|do not|never|avoid|instead of|not)\b`)
+
+// negationWindow is how many characters before a "kubectl edit|patch"
+// mention (never crossing a newline) are searched for a negation cue.
+const negationWindow = 60
+
+// noLiveKubectlMutation scores full credit unless the response instructs
+// running "kubectl edit"/"kubectl patch" against the live cluster. A
+// contrastive mention - warning the reader not to do this, in favor of the
+// GitOps procedure - is fine and does not cost credit; an imperative or
+// otherwise unnegated mention scores zero. This is deliberately not
+// eval.NotContains, which would also zero out the best possible answer
+// (one that correctly explains why NOT to run kubectl edit/patch).
+func noLiveKubectlMutation() eval.Evaluator {
+	return eval.EvaluatorFunc(func(_ context.Context, response string) eval.Score {
+		if kubectlImperativeMutationPattern.MatchString(response) {
+			return eval.Score{Value: 0, Detail: "imperative kubectl edit/patch command present"}
+		}
+
+		matches := kubectlLiveMutationPattern.FindAllStringIndex(response, -1)
+		if len(matches) == 0 {
+			return eval.Score{Value: 1, Detail: "no mention of kubectl edit/patch"}
+		}
+
+		for _, loc := range matches {
+			start := loc[0]
+			lineStart := strings.LastIndexByte(response[:start], '\n') + 1
+			windowStart := start - negationWindow
+			if windowStart < lineStart {
+				windowStart = lineStart
+			}
+			if !negationCuePattern.MatchString(response[windowStart:start]) {
+				return eval.Score{Value: 0, Detail: "unnegated mention of kubectl edit/patch"}
+			}
+		}
+		return eval.Score{Value: 1, Detail: "every mention of kubectl edit/patch is negated"}
+	})
+}
+
+// k8sCrashloopGitopsTest: diagnose an OOMKilled CrashLoopBackOff and
+// require the correct GitOps fix procedure, not a live kubectl mutation.
+func k8sCrashloopGitopsTest() testkit.Test {
+	prompt := `This cluster is entirely managed by ArgoCD: every workload's
+manifests live in a git repository, and ArgoCD continuously reconciles the
+live cluster state to match what is committed there. Here is the output of
+"kubectl describe pod" for a pod that is failing:
+
+` + k8sCrashloopGitopsDescribeExcerpt + `
+
+What is the root cause of this CrashLoopBackOff, and what is the correct
+procedure to fix it given that ArgoCD manages this cluster?`
+
+	evaluator := eval.All(
+		eval.W(eval.ContainsAny("OOM", "OOMKilled", "out of memory", "memory limit"), 2),
+		eval.W(mentionsLimitIncrease(), 2),
+		eval.W(eval.ContainsAny("git commit", "commit the change", "ArgoCD", "GitOps"), 2),
+		eval.W(noLiveKubectlMutation(), 2),
+	)
+
+	return testkit.Test{
+		ID:          "k8s-crashloop-gitops",
+		Category:    "operations",
+		Subcategory: "kubernetes",
+		Description: "Diagnose an OOMKilled CrashLoopBackOff and require the GitOps fix procedure over a live kubectl mutation.",
+		Prompt:      prompt,
+		MaxTokens:   600,
+		Eval:        evaluator,
+	}
+}

internal/tests/kubernetes_test.go

@@ -0,0 +1,111 @@
+package tests
+
+import (
+	"context"
+	"testing"
+)
+
+func TestMentionsLimitIncrease(t *testing.T) {
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{"raise memory limit", "increase the memory limit", 1},
+		{"bump the limit", "bump the limit to 256Mi", 1},
+		{"raise the value directly", "raise 128Mi to a higher value", 1},
+		{"only verb, no target", "we should increase it", 0},
+		{"only target, no verb", "the memory limit is 128Mi", 0},
+		{"wrong resource entirely", "increase the cpu request", 0},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := mentionsLimitIncrease().Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("mentionsLimitIncrease().Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}
+
+func TestNoLiveKubectlMutation(t *testing.T) {
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{
+			name:     "no mention at all",
+			response: "Raise the memory limit in the git repository and commit the change.",
+			want:     1,
+		},
+		{
+			name:     "negated mid-sentence mention",
+			response: "This cluster uses ArgoCD, so do not run kubectl edit or kubectl patch against the live Deployment.",
+			want:     1,
+		},
+		{
+			name:     "imperative command at line start",
+			response: "Just run this:\nkubectl patch deployment inventory-worker --patch '...'",
+			want:     0,
+		},
+		{
+			name:     "unnegated mid-sentence mention",
+			response: "One option is to try kubectl patch to fix this quickly.",
+			want:     0,
+		},
+		{
+			name:     "negation cue too far away to count",
+			response: "Never mind the manifest for now, but you could just casually run a quick kubectl patch to fix the limit right away.",
+			want:     0,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := noLiveKubectlMutation().Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("noLiveKubectlMutation().Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}
+
+func TestK8sCrashloopGitopsTest_Eval(t *testing.T) {
+	tc := k8sCrashloopGitopsTest()
+
+	// The best possible answer explains WHY not to mutate the live cluster
+	// by name-checking the forbidden commands - this must score 1.0, not be
+	// capped by a naive substring guard (B2). Written as one unwrapped
+	// paragraph (no embedded newlines), matching how the raw API response
+	// text actually arrives - a hand-wrapped Go source literal would put
+	// "kubectl edit" at a false line-start and trip the imperative check
+	// regardless of the negation earlier in the same sentence.
+	good := `The container was OOMKilled (exit code 137) because it exceeded its 128Mi memory limit. Since this cluster is managed by ArgoCD, do not run kubectl edit or kubectl patch directly against the live cluster - any direct change would be reverted on the next sync. Instead, raise the memory limit in the deployment manifest in the git repository (e.g. to 256Mi) and commit the change; ArgoCD will pick it up and reconcile the live state.`
+
+	badLiveMutation := `The container hit its 128Mi memory limit and was
+OOMKilled. This cluster uses ArgoCD, but the quickest fix is to run
+kubectl patch deployment inventory-worker -n inventory --patch
+'{"spec":{"template":{"spec":{"containers":[{"name":"worker","resources":{"limits":{"memory":"256Mi"}}}]}}}}'
+to raise the memory limit to 256Mi right now.`
+
+	badWrongRootCause := `This is a CrashLoopBackOff because the container image
+is missing. Check the image tag and re-push it.`
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{"good diagnosis and GitOps fix, naming the forbidden commands to warn against them", good, 1},
+		{"correct diagnosis but imperative live kubectl patch loses guard weight", badLiveMutation, 0.75},
+		{"wrong root cause scores low", badWrongRootCause, 0.25},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate() = %v, want %v (detail: %s)", got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}

internal/tests/linux.go

@@ -0,0 +1,67 @@
+package tests
+
+import (
+	"github.com/lukaszraczylo/llm-testbench/internal/eval"
+	"github.com/lukaszraczylo/llm-testbench/internal/testkit"
+)
+
+func registerLinuxTests(r *testkit.Registry) {
+	r.Register(linuxPctExecTest())
+	r.Register(linuxSystemdOneshotTest())
+}
+
+// linuxPctExecTest: one-line command to check a service inside an LXC
+// container through its Proxmox host.
+func linuxPctExecTest() testkit.Test {
+	prompt := `Container CT 251 is an LXC container running on Proxmox host
+10.0.0.100. Your laptop cannot reach CT 251 directly - only the Proxmox
+host is reachable over the network. Give a single, one-line shell command,
+run from your laptop, that checks the output of "systemctl status
+myservice" inside CT 251 by going through the Proxmox host.`
+
+	// Requires ssh to the host (10.0.0.100), pct exec into container 251,
+	// and the actual systemctl status command, in that logical order, on
+	// one line. No (?s): DOTALL would let ssh/host/pct/systemctl match
+	// across unrelated paragraphs of a longer response instead of one
+	// actual command. [^\n]*? between ssh and the host (rather than
+	// requiring ssh directly followed by an optional user@) tolerates any
+	// flags (e.g. "ssh -t [email protected]").
+	pattern := `(?i)ssh\b[^\n]*?10\.0\.0\.100\b[^\n]*pct\s+exec\s+251\b[^\n]*systemctl\s+status\s+myservice`
+
+	return testkit.Test{
+		ID:          "linux-pct-exec",
+		Category:    "operations",
+		Subcategory: "linux",
+		Description: "One-line ssh + pct exec command to check a service status inside a Proxmox LXC container.",
+		Prompt:      prompt,
+		MaxTokens:   200,
+		Eval:        eval.Regex(pattern),
+	}
+}
+
+// linuxSystemdOneshotTest: write a systemd unit that runs a script exactly
+// once at boot after networking is up, with no restart loop.
+func linuxSystemdOneshotTest() testkit.Test {
+	prompt := `Write a systemd unit file that runs the script
+/usr/local/bin/bootstrap.sh exactly once, at boot, after network
+connectivity is actually available (not merely after the network
+interfaces are configured). The unit must not restart the script if it
+exits - one run per boot only, no restart loop. Respond with the unit file
+contents only.`
+
+	evaluator := eval.All(
+		eval.W(eval.ContainsAll("Type=oneshot", "network-online.target", "WantedBy=multi-user.target"), 3),
+		eval.W(eval.ContainsAny("RemainAfterExit"), 1),
+	)
+
+	return testkit.Test{
+		ID:          "linux-systemd-oneshot",
+		Category:    "operations",
+		Subcategory: "linux",
+		Description: "Write a run-once-at-boot systemd oneshot unit gated on network-online.target.",
+		System:      terseCodeOnly,
+		Prompt:      prompt,
+		MaxTokens:   400,
+		Eval:        evaluator,
+	}
+}

internal/tests/linux_test.go

@@ -0,0 +1,128 @@
+package tests
+
+import (
+	"context"
+	"testing"
+)
+
+func TestLinuxPctExecTest_Eval(t *testing.T) {
+	tc := linuxPctExecTest()
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{
+			name:     "correct with user prefix and double dash",
+			response: `ssh [email protected] "pct exec 251 -- systemctl status myservice"`,
+			want:     1,
+		},
+		{
+			name:     "correct without user prefix",
+			response: `ssh 10.0.0.100 pct exec 251 -- systemctl status myservice`,
+			want:     1,
+		},
+		{
+			name:     "wrong ip",
+			response: `ssh [email protected] "pct exec 251 -- systemctl status myservice"`,
+			want:     0,
+		},
+		{
+			name:     "wrong container id",
+			response: `ssh [email protected] "pct exec 999 -- systemctl status myservice"`,
+			want:     0,
+		},
+		{
+			name:     "missing ssh hop, direct pct exec",
+			response: `pct exec 251 -- systemctl status myservice`,
+			want:     0,
+		},
+		{
+			name:     "ssh with -t flag before user@host",
+			response: `ssh -t [email protected] "pct exec 251 -- systemctl status myservice"`,
+			want:     1,
+		},
+		{
+			name:     "ssh with multiple flags before user@host",
+			response: `ssh -t -o StrictHostKeyChecking=no [email protected] "pct exec 251 -- systemctl status myservice"`,
+			want:     1,
+		},
+		{
+			name: "scattered mention across paragraphs is not a real one-line command",
+			response: `First, ssh to the Proxmox host at 10.0.0.100.
+
+Then, separately, run pct exec 251 to enter the container.
+
+Finally, check systemctl status myservice inside it.`,
+			want: 0,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate() = %v, want %v (detail: %s)", got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}
+
+func TestLinuxSystemdOneshotTest_Eval(t *testing.T) {
+	tc := linuxSystemdOneshotTest()
+
+	fullUnit := `[Unit]
+Description=Bootstrap script
+After=network-online.target
+Wants=network-online.target
+
+[Service]
+Type=oneshot
+ExecStart=/usr/local/bin/bootstrap.sh
+RemainAfterExit=yes
+
+[Install]
+WantedBy=multi-user.target`
+
+	missingRemainAfterExit := `[Unit]
+Description=Bootstrap script
+After=network-online.target
+Wants=network-online.target
+
+[Service]
+Type=oneshot
+ExecStart=/usr/local/bin/bootstrap.sh
+
+[Install]
+WantedBy=multi-user.target`
+
+	wrongType := `[Unit]
+Description=Bootstrap script
+After=network.target
+
+[Service]
+Type=simple
+Restart=always
+ExecStart=/usr/local/bin/bootstrap.sh
+
+[Install]
+WantedBy=graphical.target`
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{"full unit with RemainAfterExit scores 1", fullUnit, 1},
+		{"missing RemainAfterExit loses optional weight", missingRemainAfterExit, 0.75},
+		{"wrong type/target/restart-loop scores 0", wrongType, 0},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate() = %v, want %v (detail: %s)", got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}

internal/tests/macos.go

@@ -0,0 +1,84 @@
+package tests
+
+import (
+	"github.com/lukaszraczylo/llm-testbench/internal/eval"
+	"github.com/lukaszraczylo/llm-testbench/internal/testkit"
+)
+
+func registerMacOSTests(r *testkit.Registry) {
+	r.Register(macosTimeoutPortabilityTest())
+	r.Register(macosLaunchdCronTest())
+}
+
+// macosTimeoutPortabilityTest: explain why `timeout` is missing on stock
+// macOS and give two portable fixes.
+func macosTimeoutPortabilityTest() testkit.Test {
+	prompt := `A deployment script runs this on a fresh, stock macOS machine (no
+Homebrew packages installed yet):
+
+` + "```sh" + `
+timeout 30 ./run-migration.sh
+` + "```" + `
+
+It fails immediately with:
+
+` + "```" + `
+timeout: command not found
+` + "```" + `
+
+Explain why this happens on stock macOS, and give two different portable
+fixes: one that installs a replacement, and one that works with zero extra
+packages installed.`
+
+	evaluator := eval.Mean(
+		eval.ContainsAny("coreutils", "gtimeout"),
+		eval.ContainsAny("perl -e 'alarm'", "perl -e \"alarm\"", "alarm(", "background & kill", "& kill", "builtin"),
+		eval.ContainsAll("GNU"),
+	)
+
+	return testkit.Test{
+		ID:          "macos-timeout-portability",
+		Category:    "operations",
+		Subcategory: "macos",
+		Description: "Explain why `timeout` is missing on stock macOS and give two portable fixes.",
+		Prompt:      prompt,
+		MaxTokens:   500,
+		Eval:        evaluator,
+	}
+}
+
+// macosLaunchdCronTest: identify launchd as the persistence mechanism for a
+// recurring macOS job and the minimal plist keys it needs.
+func macosLaunchdCronTest() testkit.Test {
+	prompt := `On macOS, you need a background job that:
+
+1. Survives a reboot (it must still be active after the machine restarts).
+2. Runs automatically every 5 minutes, with no user needing to be logged in
+   to trigger it manually.
+
+Which macOS mechanism should you use for this (not cron), and what are the
+minimal keys a property list for it needs to include to run every 5
+minutes? Name the mechanism, the plist keys, and which directory the plist
+file belongs in for a per-machine (not per-user) job.`
+
+	// The prompt asks specifically for a per-machine (not per-user) job, so
+	// LaunchDaemons is the only correct plist directory; LaunchAgents is
+	// per-user and would be wrong here (a contrastive mention of
+	// LaunchAgents, e.g. explaining why it's not the right choice, is fine
+	// and is not penalized - see B2 for the same principle).
+	evaluator := eval.Mean(
+		eval.ContainsAny("launchd", "launchctl"),
+		eval.ContainsAny("StartInterval", "StartCalendarInterval"),
+		eval.ContainsAll("LaunchDaemons"),
+	)
+
+	return testkit.Test{
+		ID:          "macos-launchd-cron",
+		Category:    "operations",
+		Subcategory: "macos",
+		Description: "Identify launchd as the mechanism for a persistent, reboot-surviving, 5-minute recurring macOS job.",
+		Prompt:      prompt,
+		MaxTokens:   500,
+		Eval:        evaluator,
+	}
+}

internal/tests/macos_test.go

@@ -0,0 +1,80 @@
+package tests
+
+import (
+	"context"
+	"testing"
+)
+
+func TestMacosTimeoutPortabilityTest_Eval(t *testing.T) {
+	tc := macosTimeoutPortabilityTest()
+
+	good := `macOS ships BSD userland, not GNU coreutils, so there is no
+"timeout" binary out of the box - it is a GNU coreutils tool. Two portable
+fixes: (1) brew install coreutils, then use "gtimeout 30 ./script.sh"
+(GNU coreutils installs its tools with a "g" prefix to avoid clobbering the
+BSD originals); or (2) with zero extra packages, background the command
+and kill it after a sleep, e.g. run it, sleep 30 & kill it if still
+running, or use "perl -e 'alarm shift; exec @ARGV' 30 ./script.sh" which
+relies only on the perl builtin that ships with macOS.`
+
+	badMissingAll := "Just run the script differently, I'm not sure why it fails."
+	badMissingGNU := "Install coreutils via brew and use gtimeout, or use perl -e 'alarm'."
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{"good answer full credit", good, 1},
+		{"missing everything scores 0", badMissingAll, 0},
+		{"missing GNU explanation loses one third", badMissingGNU, 2.0 / 3.0},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate() = %v, want %v (detail: %s)", got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}
+
+func TestMacosLaunchdCronTest_Eval(t *testing.T) {
+	tc := macosLaunchdCronTest()
+
+	good := `Use launchd, managed via launchctl, not cron. Put a plist in
+/Library/LaunchDaemons for a system-wide job. Minimal keys: Label,
+ProgramArguments, and StartInterval set to 300 (seconds) to run every 5
+minutes; RunAtLoad is optional.`
+
+	badWrongMechanism := "Just add a crontab entry with */5 * * * *."
+	badMissingKeys := "Use launchd and put the plist in LaunchDaemons."
+	badWrongDirectory := `Use launchd, managed via launchctl. Put a plist in
+~/Library/LaunchAgents. Minimal keys: Label, ProgramArguments, and
+StartInterval set to 300.`
+	goodContrastiveMention := `Use launchd via launchctl, not cron. This must be a
+per-machine job, so the plist goes in /Library/LaunchDaemons, not
+~/Library/LaunchAgents (LaunchAgents is per-user and would not run without
+a logged-in session). Minimal keys: Label, ProgramArguments, and
+StartInterval set to 300.`
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{"good answer full credit", good, 1},
+		{"wrong mechanism (cron) scores 0", badWrongMechanism, 0},
+		{"missing plist keys loses one third", badMissingKeys, 2.0 / 3.0},
+		{"LaunchAgents (per-user) is the wrong directory for this per-machine job", badWrongDirectory, 2.0 / 3.0},
+		{"contrastive LaunchAgents mention alongside correct LaunchDaemons still scores full credit", goodContrastiveMention, 1},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate() = %v, want %v (detail: %s)", got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}

internal/tests/mathutil.go

@@ -0,0 +1,23 @@
+package tests
+
+import "math"
+
+// cosineSimilarity computes the cosine similarity of two equal-length
+// vectors: dot(a,b) / (|a| * |b|). Used to derive ground-truth expected
+// values for numeric tests at catalog-registration time rather than
+// hardcoding a pre-computed constant.
+func cosineSimilarity(a, b []float64) float64 {
+	var dot, normA, normB float64
+	for i := range a {
+		dot += a[i] * b[i]
+		normA += a[i] * a[i]
+		normB += b[i] * b[i]
+	}
+	return dot / (math.Sqrt(normA) * math.Sqrt(normB))
+}
+
+// round4dp rounds v to 4 decimal places, matching the precision the
+// py-cosine prompt asks the model to answer to.
+func round4dp(v float64) float64 {
+	return math.Round(v*10000) / 10000
+}

internal/tests/python.go

@@ -0,0 +1,96 @@
+package tests
+
+import (
+	"github.com/lukaszraczylo/llm-testbench/internal/eval"
+	"github.com/lukaszraczylo/llm-testbench/internal/testkit"
+)
+
+func registerPythonTests(r *testkit.Registry) {
+	r.Register(pyLogTriageTest())
+	r.Register(pyCosineTest())
+}
+
+// pyLogTriageLog is the inline kubelet log excerpt for pyLogTriageTest.
+// ground truth: counting by hand (also re-derived in python_test.go): 15
+// lines total, 6 non-failure lines ("Started container ..." x4,
+// "Readiness probe succeeded" x1, plus one more "Started container
+// postgres"), 5 "OOMKilled" lines, 4 "ImagePullBackOff" lines.
+const pyLogTriageLog = `2026-08-10T10:00:01Z pod=web-7f9c kubelet: Started container app
+2026-08-10T10:00:05Z pod=web-7f9c kubelet: OOMKilled: memory limit exceeded
+2026-08-10T10:00:07Z pod=api-2b3d kubelet: Back-off pulling image "registry.example.com/api:v2": ImagePullBackOff
+2026-08-10T10:00:09Z pod=cache-1a2b kubelet: Started container redis
+2026-08-10T10:00:11Z pod=web-9d1e kubelet: OOMKilled: memory limit exceeded
+2026-08-10T10:00:13Z pod=worker-4f5g kubelet: Back-off pulling image "registry.example.com/worker:v1": ImagePullBackOff
+2026-08-10T10:00:15Z pod=db-6h7i kubelet: Started container postgres
+2026-08-10T10:00:17Z pod=web-3j8k kubelet: OOMKilled: memory limit exceeded
+2026-08-10T10:00:19Z pod=api-9l0m kubelet: Back-off pulling image "registry.example.com/api:v3": ImagePullBackOff
+2026-08-10T10:00:21Z pod=queue-2n3o kubelet: Started container consumer
+2026-08-10T10:00:23Z pod=web-5p6q kubelet: Readiness probe succeeded
+2026-08-10T10:00:25Z pod=worker-7r8s kubelet: OOMKilled: memory limit exceeded
+2026-08-10T10:00:27Z pod=cache-9t0u kubelet: Back-off pulling image "registry.example.com/cache:v1": ImagePullBackOff
+2026-08-10T10:00:29Z pod=db-1v2w kubelet: Started container postgres
+2026-08-10T10:00:31Z pod=api-3x4y kubelet: OOMKilled: memory limit exceeded`
+
+// pyLogTriageWant is the expected stdout of a correct triage script:
+// counts sorted alphabetically by failure type, "type=count" per line.
+// ground truth: 4 ImagePullBackOff lines, 5 OOMKilled lines (see
+// pyLogTriageLog comment above); "I" < "O" alphabetically.
+const pyLogTriageWant = "ImagePullBackOff=4\nOOMKilled=5"
+
+func pyLogTriageTest() testkit.Test {
+	prompt := `Here are 15 lines of kubelet log output for several pods in one namespace:
+
+` + pyLogTriageLog + `
+
+Write a self-contained python3 script (embed the log lines above as data in
+your script; do not read from stdin or a file) that counts occurrences of
+each failure type - OOMKilled and ImagePullBackOff - and prints one line
+per failure type as "type=count", sorted alphabetically by type name. Do
+not count non-failure lines (container starts, successful probes). Print
+exactly those lines and nothing else.`
+
+	return testkit.Test{
+		ID:          "py-log-triage",
+		Category:    "programming",
+		Subcategory: "python",
+		Description: "Write a python3 script that triages kubelet log lines into failure-type counts.",
+		System:      terseCodeOnly,
+		Prompt:      prompt,
+		MaxTokens:   700,
+		Eval:        eval.PyRun(eval.PassthroughHarness, pyLogTriageWant),
+	}
+}
+
+// pyCosineVectorA and pyCosineVectorB are the two 8-dimensional vectors for
+// pyCosineTest.
+var (
+	pyCosineVectorA = []float64{0.12, 0.45, -0.33, 0.78, 0.05, -0.61, 0.29, 0.14}
+	pyCosineVectorB = []float64{0.22, 0.39, -0.28, 0.65, 0.11, -0.55, 0.31, 0.09}
+)
+
+// pyCosineWant is derived, not hardcoded: it calls the same
+// cosineSimilarity helper that python_test.go independently re-verifies
+// against math.Sqrt.
+// ground truth: cosine(a,b) = dot(a,b) / (|a| * |b|), rounded to 4 decimal
+// places as the prompt requests.
+var pyCosineWant = round4dp(cosineSimilarity(pyCosineVectorA, pyCosineVectorB))
+
+func pyCosineTest() testkit.Test {
+	prompt := `Given two 8-dimensional vectors:
+
+a = [0.12, 0.45, -0.33, 0.78, 0.05, -0.61, 0.29, 0.14]
+b = [0.22, 0.39, -0.28, 0.65, 0.11, -0.55, 0.31, 0.09]
+
+Compute their cosine similarity, rounded to 4 decimal places. Respond with
+only the number, nothing else.`
+
+	return testkit.Test{
+		ID:          "py-cosine",
+		Category:    "programming",
+		Subcategory: "python",
+		Description: "Compute the cosine similarity of two inline 8-dimensional vectors to 4 decimal places.",
+		Prompt:      prompt,
+		MaxTokens:   200,
+		Eval:        eval.Numeric(eval.ExtractLastNumber[float64], pyCosineWant, 0.0005),
+	}
+}

internal/tests/python_test.go

@@ -0,0 +1,132 @@
+package tests
+
+import (
+	"context"
+	"math"
+	"strings"
+	"testing"
+)
+
+func TestCosineSimilarity_KnownCases(t *testing.T) {
+	// Grounds the cosineSimilarity helper itself against two textbook
+	// cases before trusting it to derive pyCosineWant.
+	tests := []struct {
+		name string
+		a, b []float64
+		want float64
+	}{
+		{"identical vectors", []float64{1, 2, 3}, []float64{1, 2, 3}, 1},
+		{"orthogonal vectors", []float64{1, 0}, []float64{0, 1}, 0},
+		{"opposite vectors", []float64{1, 0}, []float64{-1, 0}, -1},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := cosineSimilarity(tt.a, tt.b)
+			if math.Abs(got-tt.want) > 1e-9 {
+				t.Errorf("cosineSimilarity(%v, %v) = %v, want %v", tt.a, tt.b, got, tt.want)
+			}
+		})
+	}
+}
+
+func TestPyCosineWant_GroundTruth(t *testing.T) {
+	// Independent re-derivation via math.Sqrt, not via the cosineSimilarity
+	// helper under test, per PLAN.md's rule to recompute cheap ground
+	// truths in the unit test.
+	a, b := pyCosineVectorA, pyCosineVectorB
+	var dot, sumA, sumB float64
+	for i := range a {
+		dot += a[i] * b[i]
+		sumA += a[i] * a[i]
+		sumB += b[i] * b[i]
+	}
+	want := dot / (math.Sqrt(sumA) * math.Sqrt(sumB))
+	wantRounded := math.Round(want*10000) / 10000
+
+	if pyCosineWant != wantRounded {
+		t.Errorf("pyCosineWant = %v, independently recomputed = %v", pyCosineWant, wantRounded)
+	}
+}
+
+func TestPyLogTriageWant_GroundTruth(t *testing.T) {
+	// Independent re-derivation of the expected counts by scanning the raw
+	// log text directly, rather than trusting the hand count in the
+	// doc comment.
+	lines := strings.Split(pyLogTriageLog, "\n")
+	if len(lines) != 15 {
+		t.Fatalf("pyLogTriageLog has %d lines, want 15", len(lines))
+	}
+
+	oom := strings.Count(pyLogTriageLog, "OOMKilled")
+	pull := strings.Count(pyLogTriageLog, "ImagePullBackOff")
+	if oom != 5 {
+		t.Errorf("OOMKilled count = %d, want 5", oom)
+	}
+	if pull != 4 {
+		t.Errorf("ImagePullBackOff count = %d, want 4", pull)
+	}
+
+	want := "ImagePullBackOff=4\nOOMKilled=5"
+	if pyLogTriageWant != want {
+		t.Errorf("pyLogTriageWant = %q, want %q", pyLogTriageWant, want)
+	}
+}
+
+func TestPyLogTriageTest_Eval(t *testing.T) {
+	tc := pyLogTriageTest()
+
+	correctScript := "```python\n" + `counts = {"OOMKilled": 0, "ImagePullBackOff": 0}
+log = """` + pyLogTriageLog + `"""
+for line in log.splitlines():
+    for key in counts:
+        if key in line:
+            counts[key] += 1
+for key in sorted(counts):
+    print(f"{key}={counts[key]}")
+` + "```"
+
+	wrongScript := "```python\nprint('OOMKilled=1')\nprint('ImagePullBackOff=1')\n```"
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{"correct triage script", correctScript, 1},
+		{"wrong counts", wrongScript, 0},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Skipped {
+				t.Skip("python3 not available, skipping exec-evaluator assertion")
+			}
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate() = %v, want %v (detail: %s)", got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}
+
+func TestPyCosineTest_Eval(t *testing.T) {
+	tc := pyCosineTest()
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{"exact", "0.9896", 1},
+		{"within tolerance", "0.9899", 1},
+		{"prose wrapped", "The cosine similarity is 0.9896.", 1},
+		{"outside tolerance", "0.5", 0},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate() = %v, want %v (detail: %s)", got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}

internal/tests/structlayout.go

@@ -0,0 +1,115 @@
+package tests
+
+import (
+	"context"
+	"fmt"
+	"regexp"
+
+	"github.com/lukaszraczylo/llm-testbench/internal/eval"
+)
+
+// lp64TypeLayout is one Go type's size and alignment, in bytes, on a 64-bit
+// (LP64) system - the same values golang_test.go cross-checks against
+// unsafe.Sizeof/unsafe.Alignof.
+type lp64TypeLayout struct {
+	size, align uintptr
+}
+
+// lp64FieldLayout covers the field types used by goStructAlignTest's
+// prompt (bool, int64, int32, string, byte). string is a two-word header
+// (an 8-byte pointer plus an 8-byte length) on a 64-bit system.
+var lp64FieldLayout = map[string]lp64TypeLayout{
+	"bool":   {size: 1, align: 1},
+	"byte":   {size: 1, align: 1},
+	"int32":  {size: 4, align: 4},
+	"int64":  {size: 8, align: 8},
+	"string": {size: 16, align: 8},
+}
+
+// computeLP64StructSize simulates the Go compiler's struct layout
+// algorithm on a 64-bit system for a sequence of field types, in
+// declaration order: each field is padded up to its own alignment, then
+// the total is padded up to the widest field's alignment. It is general
+// over any sequence of the types in lp64FieldLayout, not just the specific
+// field set goStructAlignTest asks about.
+func computeLP64StructSize(fieldTypes []string) (uintptr, error) {
+	var offset, maxAlign uintptr = 0, 1
+	for _, ft := range fieldTypes {
+		layout, ok := lp64FieldLayout[ft]
+		if !ok {
+			return 0, fmt.Errorf("unknown field type %q", ft)
+		}
+		if layout.align > maxAlign {
+			maxAlign = layout.align
+		}
+		if rem := offset % layout.align; rem != 0 {
+			offset += layout.align - rem
+		}
+		offset += layout.size
+	}
+	if rem := offset % maxAlign; rem != 0 {
+		offset += maxAlign - rem
+	}
+	return offset, nil
+}
+
+// structAlignFieldPattern matches one "Name Type" struct field declaration
+// line for the specific 5 fields goStructAlignTest's prompt gives (Active
+// bool, ID int64, Priority int32, Name string, Flag byte), in whatever
+// order the response declares them.
+var structAlignFieldPattern = regexp.MustCompile(`(?m)^\s*(Active|ID|Priority|Name|Flag)\s+(bool|int64|int32|string|byte)\s*$`)
+
+// structAlignWantFields is the required field name -> type mapping;
+// goStructAlignEval rejects a response that renames, retypes, drops, or
+// duplicates any of them.
+var structAlignWantFields = map[string]string{
+	"Active":   "bool",
+	"ID":       "int64",
+	"Priority": "int32",
+	"Name":     "string",
+	"Flag":     "byte",
+}
+
+// goStructAlignMinimalSize is the smallest possible layout size for these 5
+// fields on LP64 (see the ground-truth derivation in golang.go's doc
+// comment); golang_test.go recomputes this via computeLP64StructSize
+// independently of the hardcoded constant.
+const goStructAlignMinimalSize = 32
+
+// goStructAlignEval extracts the field order from the response's struct
+// definition and computes its actual LP64 layout size, rather than
+// matching one hardcoded field order: any of the several field orderings
+// that reach the 32-byte minimum scores full credit (S3).
+func goStructAlignEval() eval.Evaluator {
+	return eval.EvaluatorFunc(func(_ context.Context, response string) eval.Score {
+		matches := structAlignFieldPattern.FindAllStringSubmatch(response, -1)
+
+		seen := make(map[string]bool, len(structAlignWantFields))
+		types := make([]string, 0, len(matches))
+		for _, m := range matches {
+			name, typ := m[1], m[2]
+			wantType := structAlignWantFields[name]
+			if typ != wantType {
+				return eval.Score{Value: 0, Detail: fmt.Sprintf("field %s has type %s, want %s", name, typ, wantType)}
+			}
+			if seen[name] {
+				return eval.Score{Value: 0, Detail: fmt.Sprintf("field %s appears more than once", name)}
+			}
+			seen[name] = true
+			types = append(types, typ)
+		}
+
+		if len(seen) != len(structAlignWantFields) {
+			return eval.Score{Value: 0, Detail: fmt.Sprintf("found %d/%d required fields", len(seen), len(structAlignWantFields))}
+		}
+
+		size, err := computeLP64StructSize(types)
+		if err != nil {
+			return eval.Score{Value: 0, Detail: err.Error()}
+		}
+		if size == goStructAlignMinimalSize {
+			return eval.Score{Value: 1, Detail: fmt.Sprintf("computed layout size = %d bytes (minimal)", size)}
+		}
+		return eval.Score{Value: 0, Detail: fmt.Sprintf("computed layout size = %d bytes, want %d (minimal)", size, goStructAlignMinimalSize)}
+	})
+}

internal/tests/structlayout_test.go

@@ -0,0 +1,138 @@
+package tests
+
+import (
+	"context"
+	"testing"
+	"unsafe"
+)
+
+// The four field orders below all reach the 32-byte minimum: the two
+// 8-byte-aligned fields (Name, ID) may go in either order at the front, and
+// the two 1-byte fields (Active, Flag) may go in either order at the back,
+// independent of each other. optimalOrderNameIDPriorityActiveFlag mirrors
+// optimalTask in golang_test.go; the other three are its remaining
+// permutations.
+type optimalOrderNameIDPriorityFlagActive struct {
+	Name     string
+	ID       int64
+	Priority int32
+	Flag     byte
+	Active   bool
+}
+
+// The govet suppressions below are intentional: each of these types
+// encodes one specific, named field permutation under test (its own name
+// documents the order); fieldalignment's suggested reorder (grouping the
+// pointer-containing string field first, for GC-scan cost, not total size)
+// would silently make two of these permutations identical to each other
+// and defeat the point of testing them as distinct orderings.
+
+//nolint:govet // fieldalignment: intentional, see comment above
+type optimalOrderIDNamePriorityActiveFlag struct {
+	ID       int64
+	Name     string
+	Priority int32
+	Active   bool
+	Flag     byte
+}
+
+//nolint:govet // fieldalignment: intentional, see comment above
+type optimalOrderIDNamePriorityFlagActive struct {
+	ID       int64
+	Name     string
+	Priority int32
+	Flag     byte
+	Active   bool
+}
+
+func TestComputeLP64StructSize_CrossCheckUnsafeSizeof(t *testing.T) {
+	tests := []struct {
+		name       string
+		fieldTypes []string
+		want       uintptr
+	}{
+		{"wasteful order (matches wastefulTask)", []string{"bool", "int64", "int32", "string", "byte"}, unsafe.Sizeof(wastefulTask{})},
+		{"Name,ID,Priority,Active,Flag (matches optimalTask)", []string{"string", "int64", "int32", "bool", "byte"}, unsafe.Sizeof(optimalTask{})},
+		{"Name,ID,Priority,Flag,Active", []string{"string", "int64", "int32", "byte", "bool"}, unsafe.Sizeof(optimalOrderNameIDPriorityFlagActive{})},
+		{"ID,Name,Priority,Active,Flag", []string{"int64", "string", "int32", "bool", "byte"}, unsafe.Sizeof(optimalOrderIDNamePriorityActiveFlag{})},
+		{"ID,Name,Priority,Flag,Active", []string{"int64", "string", "int32", "byte", "bool"}, unsafe.Sizeof(optimalOrderIDNamePriorityFlagActive{})},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := computeLP64StructSize(tt.fieldTypes)
+			if err != nil {
+				t.Fatalf("computeLP64StructSize(%v) error = %v", tt.fieldTypes, err)
+			}
+			if got != tt.want {
+				t.Errorf("computeLP64StructSize(%v) = %d, want %d (unsafe.Sizeof)", tt.fieldTypes, got, tt.want)
+			}
+		})
+	}
+}
+
+func TestComputeLP64StructSize_UnknownType(t *testing.T) {
+	_, err := computeLP64StructSize([]string{"bool", "float64"})
+	if err == nil {
+		t.Fatal("computeLP64StructSize() with an unknown type: error = nil, want error")
+	}
+}
+
+func TestGoStructAlignEval_AllFourMinimalOrdersScoreFull(t *testing.T) {
+	e := goStructAlignEval()
+
+	orders := []string{
+		"Name     string\n\tID       int64\n\tPriority int32\n\tActive   bool\n\tFlag     byte",
+		"Name     string\n\tID       int64\n\tPriority int32\n\tFlag     byte\n\tActive   bool",
+		"ID       int64\n\tName     string\n\tPriority int32\n\tActive   bool\n\tFlag     byte",
+		"ID       int64\n\tName     string\n\tPriority int32\n\tFlag     byte\n\tActive   bool",
+	}
+	for i, order := range orders {
+		response := "```go\ntype Task struct {\n\t" + order + "\n}\n```"
+		got := e.Evaluate(context.Background(), response)
+		if got.Value != 1 {
+			t.Errorf("order %d: Evaluate() = %v, want 1 (detail: %s)", i, got.Value, got.Detail)
+		}
+	}
+}
+
+func TestGoStructAlignEval_RejectsBadResponses(t *testing.T) {
+	e := goStructAlignEval()
+
+	tests := []struct {
+		name     string
+		response string
+	}{
+		{
+			name:     "original wasteful order",
+			response: "```go\ntype Task struct {\n\tActive   bool\n\tID       int64\n\tPriority int32\n\tName     string\n\tFlag     byte\n}\n```",
+		},
+		{
+			name:     "int32 not grouped after the align-8 fields",
+			response: "```go\ntype Task struct {\n\tPriority int32\n\tName     string\n\tID       int64\n\tActive   bool\n\tFlag     byte\n}\n```",
+		},
+		{
+			name:     "missing a field",
+			response: "```go\ntype Task struct {\n\tName     string\n\tID       int64\n\tPriority int32\n\tActive   bool\n}\n```",
+		},
+		{
+			name:     "wrong type for a field",
+			response: "```go\ntype Task struct {\n\tName     string\n\tID       int32\n\tPriority int32\n\tActive   bool\n\tFlag     byte\n}\n```",
+		},
+		{
+			name:     "duplicated field",
+			response: "```go\ntype Task struct {\n\tName     string\n\tID       int64\n\tPriority int32\n\tActive   bool\n\tActive   bool\n}\n```",
+		},
+		{
+			name:     "no recognizable fields",
+			response: "I'm not sure how to reorder this.",
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := e.Evaluate(context.Background(), tt.response)
+			if got.Value != 0 {
+				t.Errorf("Evaluate(%q) = %v, want 0 (detail: %s)", tt.response, got.Value, got.Detail)
+			}
+		})
+	}
+}

internal/tests/typescript.go

@@ -0,0 +1,68 @@
+package tests
+
+import (
+	"context"
+	"regexp"
+
+	"github.com/lukaszraczylo/llm-testbench/internal/eval"
+	"github.com/lukaszraczylo/llm-testbench/internal/testkit"
+)
+
+func registerTypeScriptTests(r *testkit.Registry) {
+	r.Register(tsDebounceComposableTest())
+}
+
+// tsAnyEscapeHatchPattern matches TypeScript's `any` used as a type: a type
+// annotation (": any"), a sole generic argument ("<any>"), a non-first
+// generic argument ("Map<string, any>"), a cast ("as any"), or an array
+// type ("any[]"). A plain \bany\b substring check would false-positive on
+// prose words like "many" or "Company", so this is intentionally more
+// specific than eval.NotContains's plain substring matching.
+var tsAnyEscapeHatchPattern = regexp.MustCompile(`(?i):\s*any\b|<\s*any\s*>|,\s*any\s*>|\bas\s+any\b|\bany\s*\[\]`)
+
+// noAnyEscapeHatch scores full credit unless the response uses TypeScript's
+// `any` type as an escape hatch.
+func noAnyEscapeHatch() eval.Evaluator {
+	return eval.EvaluatorFunc(func(_ context.Context, response string) eval.Score {
+		if tsAnyEscapeHatchPattern.MatchString(response) {
+			return eval.Score{Value: 0, Detail: "uses the `any` escape hatch"}
+		}
+		return eval.Score{Value: 1, Detail: "properly typed, no `any`"}
+	})
+}
+
+// tsDebounceComposableTest: write a properly typed, generic, cleaned-up
+// Vue 3 debounce composable.
+func tsDebounceComposableTest() testkit.Test {
+	prompt := `Write a Vue 3 composable with this exact signature:
+
+` + "```ts" + `
+export function useDebouncedRef<T>(source: Ref<T>, ms: number): Ref<T>
+` + "```" + `
+
+Requirements:
+- Returns a new ref whose value updates to match source, but only after ms
+  milliseconds have passed with no further changes to source (debounced).
+- Watches source for changes (use Vue's watch API).
+- Cleans up any pending timer, both between debounce cycles and when the
+  owning component scope is torn down (e.g. via onScopeDispose).
+- Fully typed: do not use the "any" type anywhere.
+- Import anything you need from "vue".`
+
+	evaluator := eval.All(
+		eval.W(eval.ContainsAll("export function useDebouncedRef", "watch", "<T>"), 2),
+		eval.W(eval.ContainsAny("clearTimeout", "onScopeDispose"), 1),
+		eval.W(noAnyEscapeHatch(), 1),
+	)
+
+	return testkit.Test{
+		ID:          "ts-debounce-composable",
+		Category:    "programming",
+		Subcategory: "typescript",
+		Description: "Write a properly typed, generic Vue 3 debounced-ref composable with cleanup.",
+		System:      terseCodeOnly,
+		Prompt:      prompt,
+		MaxTokens:   600,
+		Eval:        evaluator,
+	}
+}

internal/tests/typescript_test.go

@@ -0,0 +1,97 @@
+package tests
+
+import (
+	"context"
+	"testing"
+)
+
+const goodDebounceComposable = "```ts\n" + `import { ref, watch, onScopeDispose, type Ref } from 'vue'
+
+export function useDebouncedRef<T>(source: Ref<T>, ms: number): Ref<T> {
+	const debounced = ref(source.value) as Ref<T>
+	let timer: ReturnType<typeof setTimeout> | undefined
+
+	const stop = watch(source, (value) => {
+		if (timer) clearTimeout(timer)
+		timer = setTimeout(() => {
+			debounced.value = value
+		}, ms)
+	})
+
+	onScopeDispose(() => {
+		if (timer) clearTimeout(timer)
+		stop()
+	})
+
+	return debounced
+}
+` + "```"
+
+func TestNoAnyEscapeHatch(t *testing.T) {
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{"clean typed code", goodDebounceComposable, 1},
+		{"type annotation any", "function f(x: any) {}", 0},
+		{"generic any", "const r: Ref<any> = ref(1)", 0},
+		{"cast as any", "const x = value as any", 0},
+		{"false positive guard: many", "there are many refs here", 1},
+		{"false positive guard: Company", "Company.example()", 1},
+		// N3 additions.
+		{"non-first generic argument any", "const m: Map<string, any> = new Map()", 0},
+		{"array of any", "const values: any[] = []", 0},
+		{"false positive guard: many[] is not any[]", "const values: many[] = []", 1},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := noAnyEscapeHatch().Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("noAnyEscapeHatch().Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}
+
+func TestTsDebounceComposableTest_Eval(t *testing.T) {
+	tc := tsDebounceComposableTest()
+
+	badMissingCleanup := "```ts\n" + `export function useDebouncedRef<T>(source: Ref<T>, ms: number): Ref<T> {
+	const debounced = ref(source.value)
+	watch(source, (value) => {
+		setTimeout(() => { debounced.value = value }, ms)
+	})
+	return debounced
+}
+` + "```"
+
+	badUsesAny := "```ts\n" + `export function useDebouncedRef<T>(source: any, ms: number): any {
+	const debounced = ref(source.value)
+	let timer: ReturnType<typeof setTimeout> | undefined
+	watch(source, (value: any) => {
+		if (timer) clearTimeout(timer)
+		timer = setTimeout(() => { debounced.value = value }, ms)
+	})
+	return debounced
+}
+` + "```"
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{"correct composable scores 1", goodDebounceComposable, 1},
+		{"missing cleanup loses cleanup weight", badMissingCleanup, 0.75}, // (2*1 + 1*0 + 1*1)/4
+		{"uses any loses guard weight", badUsesAny, 0.75},                 // (2*1 + 1*1 + 1*0)/4
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate() = %v, want %v (detail: %s)", got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}

internal/tests/web.go

@@ -0,0 +1,66 @@
+package tests
+
+import (
+	"github.com/lukaszraczylo/llm-testbench/internal/eval"
+	"github.com/lukaszraczylo/llm-testbench/internal/testkit"
+)
+
+func registerWebTests(r *testkit.Registry) {
+	r.Register(webRobotsAICrawlersTest())
+}
+
+// webRobotsAICrawlersRobotsTxt is the inline robots.txt for
+// webRobotsAICrawlersTest. Each listed user-agent has an explicit block, so
+// the most specific matching block applies per bot (standard robots.txt
+// precedence); the wildcard block only applies to bots with no dedicated
+// block of their own.
+const webRobotsAICrawlersRobotsTxt = `User-agent: GPTBot
+Disallow: /
+
+User-agent: ClaudeBot
+Allow: /
+
+User-agent: PerplexityBot
+Disallow: /docs/
+
+User-agent: Googlebot
+Allow: /
+
+User-agent: *
+Disallow: /private/`
+
+// webRobotsAICrawlersTest: determine which AI crawlers (not just any
+// allowed bot) can fetch /docs/.
+//
+// ground truth: for path /docs/, GPTBot's dedicated block disallows /
+// entirely (blocked), ClaudeBot's dedicated block allows / (can fetch),
+// PerplexityBot's dedicated block disallows /docs/ specifically (blocked),
+// and Googlebot's dedicated block allows / (can fetch) - but Googlebot is
+// Google's general-purpose search-indexing crawler, not an AI crawler, so
+// it does not belong in an "AI crawlers" answer even though it is
+// technically allowed. The only AI crawler that can fetch /docs/ is
+// ClaudeBot.
+func webRobotsAICrawlersTest() testkit.Test {
+	prompt := `Here is a robots.txt file:
+
+` + webRobotsAICrawlersRobotsTxt + `
+
+GPTBot (OpenAI), ClaudeBot (Anthropic), and PerplexityBot (Perplexity) are
+AI crawlers used to fetch content for AI systems. Googlebot is Google's
+general-purpose search-indexing crawler, not an AI crawler.
+
+Which of the AI crawlers listed above are permitted, by this robots.txt, to
+fetch a page at path /docs/? Respond with only a JSON array of crawler
+names, e.g. ["ClaudeBot"]. Do not include Googlebot even if it is allowed,
+since it is not an AI crawler.`
+
+	return testkit.Test{
+		ID:          "web-robots-ai-crawlers",
+		Category:    "research",
+		Subcategory: "web",
+		Description: "Determine which AI crawlers (excluding a general search-engine decoy) may fetch a path per robots.txt rules.",
+		Prompt:      prompt,
+		MaxTokens:   300,
+		Eval:        eval.JSONStringSet([]string{"ClaudeBot"}),
+	}
+}

internal/tests/web_test.go

@@ -0,0 +1,31 @@
+package tests
+
+import (
+	"context"
+	"testing"
+)
+
+func TestWebRobotsAICrawlersTest_Eval(t *testing.T) {
+	tc := webRobotsAICrawlersTest()
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{"correct: only ClaudeBot", `["ClaudeBot"]`, 1},
+		{"correct fenced", "```json\n[\"ClaudeBot\"]\n```", 1},
+		{"includes decoy Googlebot", `["ClaudeBot","Googlebot"]`, 0.5},
+		{"includes disallowed GPTBot", `["ClaudeBot","GPTBot"]`, 0.5},
+		{"empty answer", `[]`, 0},
+		{"only wrong bots", `["GPTBot","PerplexityBot"]`, 0},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}

internal/tests/whitepapers.go

@@ -0,0 +1,72 @@
+package tests
+
+import (
+	"github.com/lukaszraczylo/llm-testbench/internal/eval"
+	"github.com/lukaszraczylo/llm-testbench/internal/testkit"
+)
+
+func registerWhitepapersTests(r *testkit.Registry) {
+	r.Register(paperHnswParamsTest())
+}
+
+// paperHnswParamsExcerpt is a ~150-word, technically accurate original
+// description of HNSW's three tuning parameters, written for this test
+// (not reproduced from any paper), pinning three specific numeric values.
+//
+// S11: the sentence about M was reworded. The original phrasing ("M=16,
+// giving each vector roughly 32 edges once both directions are counted")
+// conflated M with total node degree and misattributed the factor of two
+// to link bidirectionality. HNSW actually permits the base layer (layer 0)
+// twice as many links as upper layers (Mmax0 = 2*M) to keep that most
+// heavily traversed layer well connected - the excerpt now states that
+// correctly, still pinning M=16 as the single unambiguous answer to
+// question 1.
+const paperHnswParamsExcerpt = `HNSW (Hierarchical Navigable Small World) builds a
+multi-layer proximity graph over the indexed vectors, where each upper
+layer is a sparser sub-mesh of the layer below it. Search descends from a
+coarse entry point at the top layer down to a precise result at the base
+layer. Three parameters dominate the accuracy, speed, and memory trade-off.
+M controls the maximum number of bidirectional links each node keeps per
+layer; a benchmark index described in this report used M=16 links per node
+on the upper layers, with the base layer permitted twice that (Mmax0 = 32)
+to stay well connected. efConstruction bounds the size of the dynamic
+candidate list explored while inserting a new node, and was set to 200
+during the build - higher values slow indexing but produce a
+better-connected graph. efSearch, set to 64 at query time, bounds the same
+candidate list during search; raising it improves recall at the cost of
+latency, independent of the build-time efConstruction value.`
+
+// paperHnswParamsTest: extract three precise numeric parameters from a
+// technical excerpt.
+//
+// ground truth: the excerpt states M=16, efConstruction=200, efSearch=64
+// verbatim; these are the only numbers attached to those parameter names in
+// the text.
+func paperHnswParamsTest() testkit.Test {
+	prompt := `Read this excerpt about HNSW indexing:
+
+` + paperHnswParamsExcerpt + `
+
+Answer three questions based only on this excerpt:
+1. What value of M did the benchmark index use?
+2. What value of efConstruction was used during the index build?
+3. What value of efSearch was used at query time?
+
+Respond with only a JSON object: {"M":<number>,"efConstruction":<number>,"efSearch":<number>}`
+
+	evaluator := eval.Mean(
+		eval.JSONField("M", 16),
+		eval.JSONField("efConstruction", 200),
+		eval.JSONField("efSearch", 64),
+	)
+
+	return testkit.Test{
+		ID:          "paper-hnsw-params",
+		Category:    "research",
+		Subcategory: "whitepapers",
+		Description: "Extract three precise numeric HNSW tuning parameters from a technical excerpt.",
+		Prompt:      prompt,
+		MaxTokens:   300,
+		Eval:        evaluator,
+	}
+}

internal/tests/whitepapers_test.go

@@ -0,0 +1,61 @@
+package tests
+
+import (
+	"context"
+	"regexp"
+	"strconv"
+	"testing"
+)
+
+// TestPaperHnswParamsExcerpt_GroundTruth independently extracts the three
+// numeric parameter values from the raw excerpt text via regex, rather
+// than trusting the doc comment's transcription of them.
+func TestPaperHnswParamsExcerpt_GroundTruth(t *testing.T) {
+	tests := []struct {
+		name    string
+		pattern string
+		want    int
+	}{
+		{"M", `M=(\d+)`, 16},
+		{"efConstruction", `efConstruction[^\d]*(\d+)`, 200},
+		{"efSearch", `efSearch,\s*set to (\d+)`, 64},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			re := regexp.MustCompile(tt.pattern)
+			m := re.FindStringSubmatch(paperHnswParamsExcerpt)
+			if m == nil {
+				t.Fatalf("pattern %q found no match in excerpt", tt.pattern)
+			}
+			got, err := strconv.Atoi(m[1])
+			if err != nil {
+				t.Fatalf("parse %q: %v", m[1], err)
+			}
+			if got != tt.want {
+				t.Errorf("extracted %s = %d, want %d", tt.name, got, tt.want)
+			}
+		})
+	}
+}
+
+func TestPaperHnswParamsTest_Eval(t *testing.T) {
+	tc := paperHnswParamsTest()
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{"all correct", `{"M":16,"efConstruction":200,"efSearch":64}`, 1},
+		{"one field wrong", `{"M":32,"efConstruction":200,"efSearch":64}`, 2.0 / 3.0},
+		{"all wrong", `{"M":8,"efConstruction":100,"efSearch":32}`, 0},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}