feat: LLM accuracy testing framework with 17-test catalog

Framework runs a category/subcategory test catalog against multiple LLMs through an OpenAI-compatible endpoint. Deterministic evaluators score each response. Reports render as table, markdown, or JSON.
2026-10-12 00:45:19 +00:00 · 2026-08-17 01:29:45 +01:00
commit 1d51d4cbbd
68 changed files with 7089 additions and 0 deletions
@@ -0,0 +1,161 @@
+package eval
+
+import (
+	"context"
+	"testing"
+)
+
+func TestExtractJSON(t *testing.T) {
+	tests := []struct {
+		name     string
+		response string
+		want     string
+		wantErr  bool
+	}{
+		{"plain object", `{"bump":"minor"}`, `{"bump":"minor"}`, false},
+		{"fenced object", "Here you go:\n```json\n{\"bump\":\"minor\"}\n```", `{"bump":"minor"}`, false},
+		{"object with prose around", `The answer is {"bump":"minor"} as computed.`, `{"bump":"minor"}`, false},
+		{"nested braces ignored inside strings", `{"note":"a { b } c","bump":"patch"}`, `{"note":"a { b } c","bump":"patch"}`, false},
+		{"array", `["ClaudeBot","Googlebot"]`, `["ClaudeBot","Googlebot"]`, false},
+		{"no json", "no json here at all", "", true},
+		// S7: a candidate JSON considered during reasoning, followed by the
+		// real final JSON, must resolve to the final one.
+		{
+			name:     "reasoning with a candidate JSON then a final JSON: final wins",
+			response: `Let me think. One option would be {"bump":"major"} but that is not right. The actual answer is {"bump":"minor"}.`,
+			want:     `{"bump":"minor"}`,
+			wantErr:  false,
+		},
+		{
+			name:     "fenced json block preferred over any other json-looking text before it",
+			response: "Draft: {\"bump\":\"major\"}\n\nFinal answer:\n```json\n{\"bump\":\"minor\"}\n```",
+			want:     `{"bump":"minor"}`,
+			wantErr:  false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := ExtractJSON(tt.response)
+			if (err != nil) != tt.wantErr {
+				t.Fatalf("ExtractJSON(%q) error = %v, wantErr %v", tt.response, err, tt.wantErr)
+			}
+			if err == nil && got != tt.want {
+				t.Errorf("ExtractJSON(%q) = %q, want %q", tt.response, got, tt.want)
+			}
+		})
+	}
+}
+
+func TestJSONField_String(t *testing.T) {
+	e := JSONField("bump", "minor")
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{"exact match", `{"bump":"minor"}`, 1},
+		{"case insensitive", `{"bump":"MINOR"}`, 1},
+		{"wrong value", `{"bump":"patch"}`, 0},
+		{"missing field", `{"other":"x"}`, 0},
+		{"fenced json", "```json\n{\"bump\": \"minor\"}\n```", 1},
+		{"invalid json", "not json at all", 0},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := e.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}
+
+func TestJSONField_Int(t *testing.T) {
+	e := JSONField("M", 16)
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{"exact match", `{"M":16,"efConstruction":200,"efSearch":50}`, 1},
+		{"wrong value", `{"M":32}`, 0},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := e.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}
+
+func TestJSONField_NestedPath(t *testing.T) {
+	e := JSONField("task1", "search_web")
+	got := e.Evaluate(context.Background(), `{"task1":"search_web","task2":"read_file"}`)
+	if got.Value != 1 {
+		t.Errorf("Evaluate() = %v, want 1 (detail: %s)", got.Value, got.Detail)
+	}
+}
+
+func TestJSONField_IndexedPath(t *testing.T) {
+	e := JSONField("steps[0]", "build")
+	got := e.Evaluate(context.Background(), `{"steps":["build","test","deploy"]}`)
+	if got.Value != 1 {
+		t.Errorf("Evaluate() = %v, want 1 (detail: %s)", got.Value, got.Detail)
+	}
+	got = e.Evaluate(context.Background(), `{"steps":["test","build","deploy"]}`)
+	if got.Value != 0 {
+		t.Errorf("Evaluate() = %v, want 0 (detail: %s)", got.Value, got.Detail)
+	}
+}
+
+func TestJSONStringSet(t *testing.T) {
+	e := JSONStringSet([]string{"ClaudeBot", "Googlebot"})
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{"exact set", `["ClaudeBot","Googlebot"]`, 1},
+		{"exact set different order", `["Googlebot","ClaudeBot"]`, 1},
+		{"case insensitive", `["claudebot","googlebot"]`, 1},
+		{"missing one", `["ClaudeBot"]`, 0.5},
+		{"extra one", `["ClaudeBot","Googlebot","GPTBot"]`, float64(2) / 3},
+		{"completely wrong", `["GPTBot","PerplexityBot"]`, 0},
+		{"invalid json", "no array here", 0},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := e.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}
+
+func TestJSONStringArrayEquals(t *testing.T) {
+	want := []string{"build", "test", "backup", "deploy", "verify"}
+	e := JSONStringArrayEquals(want)
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{"exact order", `["build","test","backup","deploy","verify"]`, 1},
+		{"case insensitive exact order", `["BUILD","TEST","BACKUP","DEPLOY","VERIFY"]`, 1},
+		{"wrong order", `["build","backup","test","deploy","verify"]`, 0},
+		{"wrong length", `["build","test"]`, 0},
+		{"invalid json", "not an array", 0},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := e.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}