feat: LLM accuracy testing framework with 17-test catalog

Framework runs a category/subcategory test catalog against multiple LLMs through an OpenAI-compatible endpoint. Deterministic evaluators score each response. Reports render as table, markdown, or JSON.
2026-10-12 00:45:19 +00:00 · 2026-08-17 01:29:45 +01:00
commit 1d51d4cbbd
68 changed files with 7089 additions and 0 deletions
@@ -0,0 +1,36 @@
+// Package llm defines the OpenAI-compatible chat client interface used to
+// exercise language models under test, plus a concrete HTTP implementation.
+package llm
+
+import (
+	"context"
+	"time"
+)
+
+// Message is a single chat message in an OpenAI-compatible conversation.
+type Message struct {
+	Role    string
+	Content string
+}
+
+// Request is one chat-completion request sent to a model.
+type Request struct {
+	Model       string
+	Messages    []Message
+	MaxTokens   int
+	Temperature float64
+}
+
+// Response is the normalized result of a chat-completion call.
+type Response struct {
+	Text             string
+	PromptTokens     int
+	CompletionTokens int
+	Latency          time.Duration
+}
+
+// Client completes chat requests against a language model backend.
+// Implementations must be safe for concurrent use.
+type Client interface {
+	Complete(ctx context.Context, req Request) (Response, error)
+}
@@ -0,0 +1,179 @@
+package llm
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"time"
+)
+
+// maxRetries is the number of retry attempts after the initial request on
+// transport errors or 5xx responses, per PLAN.md ("retry w/ backoff x2").
+const maxRetries = 2
+
+// retryBaseDelay is the base backoff delay; attempt N waits N*retryBaseDelay.
+const retryBaseDelay = 250 * time.Millisecond
+
+// OpenAIClient talks to an OpenAI-compatible /v1/chat/completions endpoint.
+type OpenAIClient struct {
+	httpClient *http.Client
+	endpoint   string
+	apiKey     string
+}
+
+// NewOpenAIClient builds a client against endpoint (e.g.
+// "https://host/v1") using apiKey (may be empty for keyless gateways) and
+// per-request timeout.
+func NewOpenAIClient(endpoint, apiKey string, timeout time.Duration) *OpenAIClient {
+	return &OpenAIClient{
+		endpoint: endpoint,
+		apiKey:   apiKey,
+		httpClient: &http.Client{
+			Timeout: timeout,
+		},
+	}
+}
+
+type chatCompletionRequest struct {
+	Model       string    `json:"model"`
+	Messages    []chatMsg `json:"messages"`
+	MaxTokens   int       `json:"max_tokens,omitempty"`
+	Temperature float64   `json:"temperature"`
+}
+
+type chatMsg struct {
+	Role    string `json:"role"`
+	Content string `json:"content"`
+}
+
+type chatCompletionResponse struct {
+	Error *struct {
+		Message string `json:"message"`
+	} `json:"error"`
+	Choices []struct {
+		Message chatMsg `json:"message"`
+	} `json:"choices"`
+	Usage struct {
+		PromptTokens     int `json:"prompt_tokens"`
+		CompletionTokens int `json:"completion_tokens"`
+	} `json:"usage"`
+}
+
+// Complete implements Client. It retries transport failures and 5xx
+// responses up to maxRetries times with linear backoff.
+func (c *OpenAIClient) Complete(ctx context.Context, req Request) (Response, error) {
+	body := chatCompletionRequest{
+		Model:       req.Model,
+		MaxTokens:   req.MaxTokens,
+		Temperature: req.Temperature,
+	}
+	for _, m := range req.Messages {
+		body.Messages = append(body.Messages, chatMsg(m))
+	}
+
+	payload, err := json.Marshal(body)
+	if err != nil {
+		return Response{}, fmt.Errorf("llm: marshal request: %w", err)
+	}
+
+	var lastErr error
+	for attempt := 0; attempt <= maxRetries; attempt++ {
+		if attempt > 0 {
+			delay := time.Duration(attempt) * retryBaseDelay
+			select {
+			case <-ctx.Done():
+				return Response{}, ctx.Err()
+			case <-time.After(delay):
+			}
+		}
+
+		// Measured per-attempt, starting after any backoff delay: Latency
+		// reflects the successful attempt's own generation time, not the
+		// cumulative wait across earlier failed/retried attempts (S8).
+		attemptStart := time.Now()
+		resp, err := c.doRequest(ctx, payload)
+		if err != nil {
+			lastErr = err
+			if isRetryable(err) {
+				continue
+			}
+			return Response{}, err
+		}
+
+		latency := time.Since(attemptStart)
+		text := ""
+		if len(resp.Choices) > 0 {
+			text = resp.Choices[0].Message.Content
+		}
+		if resp.Error != nil {
+			lastErr = fmt.Errorf("llm: api error: %s", resp.Error.Message)
+			return Response{}, lastErr
+		}
+		return Response{
+			Text:             text,
+			PromptTokens:     resp.Usage.PromptTokens,
+			CompletionTokens: resp.Usage.CompletionTokens,
+			Latency:          latency,
+		}, nil
+	}
+	return Response{}, fmt.Errorf("llm: request failed after %d attempts: %w", maxRetries+1, lastErr)
+}
+
+type retryableError struct {
+	err    error
+	status int
+}
+
+func (e *retryableError) Error() string {
+	if e.status != 0 {
+		return fmt.Sprintf("llm: server returned status %d", e.status)
+	}
+	return fmt.Sprintf("llm: transport error: %v", e.err)
+}
+
+func (e *retryableError) Unwrap() error { return e.err }
+
+func isRetryable(err error) bool {
+	var re *retryableError
+	return errors.As(err, &re)
+}
+
+func (c *OpenAIClient) doRequest(ctx context.Context, payload []byte) (*chatCompletionResponse, error) {
+	url := c.endpoint + "/chat/completions"
+	httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(payload))
+	if err != nil {
+		return nil, fmt.Errorf("llm: build request: %w", err)
+	}
+	httpReq.Header.Set("Content-Type", "application/json")
+	if c.apiKey != "" {
+		httpReq.Header.Set("Authorization", "Bearer "+c.apiKey)
+	}
+
+	httpResp, err := c.httpClient.Do(httpReq)
+	if err != nil {
+		return nil, &retryableError{err: err}
+	}
+	defer func() { _ = httpResp.Body.Close() }() // best-effort; body already fully read below
+
+	respBody, err := io.ReadAll(httpResp.Body)
+	if err != nil {
+		return nil, &retryableError{err: fmt.Errorf("read body: %w", err)}
+	}
+
+	if httpResp.StatusCode >= 500 {
+		return nil, &retryableError{status: httpResp.StatusCode}
+	}
+	if httpResp.StatusCode >= 400 {
+		return nil, fmt.Errorf("llm: server returned status %d: %s", httpResp.StatusCode, string(respBody))
+	}
+
+	var out chatCompletionResponse
+	if err := json.Unmarshal(respBody, &out); err != nil {
+		return nil, fmt.Errorf("llm: decode response: %w", err)
+	}
+	return &out, nil
+}
@@ -0,0 +1,217 @@
+package llm
+
+import (
+	"context"
+	"encoding/json"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"sync/atomic"
+	"testing"
+	"time"
+)
+
+// newTestServer spins up a local httptest server; this is not the live
+// endpoint and is safe for offline unit tests.
+func newTestServer(t *testing.T, handler http.HandlerFunc) *httptest.Server {
+	t.Helper()
+	srv := httptest.NewServer(handler)
+	t.Cleanup(srv.Close)
+	return srv
+}
+
+func TestOpenAIClient_Complete_Success(t *testing.T) {
+	srv := newTestServer(t, func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path != "/chat/completions" {
+			t.Errorf("unexpected path: %s", r.URL.Path)
+		}
+		var body chatCompletionRequest
+		if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
+			t.Fatalf("decode request: %v", err)
+		}
+		if body.Model != "test-model" {
+			t.Errorf("model = %q, want test-model", body.Model)
+		}
+		resp := chatCompletionResponse{}
+		resp.Choices = []struct {
+			Message chatMsg `json:"message"`
+		}{{Message: chatMsg{Role: "assistant", Content: "hello there"}}}
+		resp.Usage.PromptTokens = 12
+		resp.Usage.CompletionTokens = 3
+		_ = json.NewEncoder(w).Encode(resp)
+	})
+
+	client := NewOpenAIClient(srv.URL, "", time.Second)
+	resp, err := client.Complete(context.Background(), Request{
+		Model:    "test-model",
+		Messages: []Message{{Role: "user", Content: "hi"}},
+	})
+	if err != nil {
+		t.Fatalf("Complete() error = %v", err)
+	}
+	if resp.Text != "hello there" {
+		t.Errorf("Text = %q, want %q", resp.Text, "hello there")
+	}
+	if resp.PromptTokens != 12 || resp.CompletionTokens != 3 {
+		t.Errorf("tokens = %d/%d, want 12/3", resp.PromptTokens, resp.CompletionTokens)
+	}
+}
+
+// TestOpenAIClient_Complete_SendsTemperatureZero verifies Request.Temperature
+// (PLAN.md pins this to 0 for deterministic scoring, wired from main.go's
+// requestTemperature constant) actually lands on the wire as a JSON
+// "temperature":0 field, not silently omitted (chatCompletionRequest has no
+// "omitempty" on Temperature specifically so a real zero is distinguishable
+// from an absent field) (N7).
+func TestOpenAIClient_Complete_SendsTemperatureZero(t *testing.T) {
+	var rawBody []byte
+	srv := newTestServer(t, func(w http.ResponseWriter, r *http.Request) {
+		var err error
+		rawBody, err = io.ReadAll(r.Body)
+		if err != nil {
+			t.Fatalf("read request body: %v", err)
+		}
+		_ = json.NewEncoder(w).Encode(chatCompletionResponse{})
+	})
+
+	client := NewOpenAIClient(srv.URL, "", time.Second)
+	_, err := client.Complete(context.Background(), Request{
+		Model:       "m",
+		Messages:    []Message{{Role: "user", Content: "x"}},
+		Temperature: 0,
+	})
+	if err != nil {
+		t.Fatalf("Complete() error = %v", err)
+	}
+
+	var decoded map[string]any
+	if err := json.Unmarshal(rawBody, &decoded); err != nil {
+		t.Fatalf("json.Unmarshal(rawBody) error = %v; body: %s", err, rawBody)
+	}
+	got, present := decoded["temperature"]
+	if !present {
+		t.Fatalf("request body has no \"temperature\" key at all: %s", rawBody)
+	}
+	if got != float64(0) {
+		t.Errorf("temperature = %v, want 0", got)
+	}
+}
+
+func TestOpenAIClient_Complete_RetriesOn5xxThenSucceeds(t *testing.T) {
+	var calls int32
+	srv := newTestServer(t, func(w http.ResponseWriter, r *http.Request) {
+		n := atomic.AddInt32(&calls, 1)
+		if n < 3 {
+			w.WriteHeader(http.StatusInternalServerError)
+			return
+		}
+		resp := chatCompletionResponse{}
+		resp.Choices = []struct {
+			Message chatMsg `json:"message"`
+		}{{Message: chatMsg{Role: "assistant", Content: "ok"}}}
+		_ = json.NewEncoder(w).Encode(resp)
+	})
+
+	client := NewOpenAIClient(srv.URL, "", time.Second)
+	resp, err := client.Complete(context.Background(), Request{Model: "m", Messages: []Message{{Role: "user", Content: "x"}}})
+	if err != nil {
+		t.Fatalf("Complete() error = %v", err)
+	}
+	if resp.Text != "ok" {
+		t.Errorf("Text = %q, want ok", resp.Text)
+	}
+	// S8: Latency must reflect the successful attempt's own generation
+	// time, not the cumulative wait across the two failed attempts plus
+	// their backoff delays (which total 250ms+500ms=750ms for attempts 1
+	// and 2 - see retryBaseDelay/maxRetries).
+	if resp.Latency > 200*time.Millisecond {
+		t.Errorf("Latency = %v, want well under the 750ms of retry backoff (per-attempt measurement, S8)", resp.Latency)
+	}
+
+	if got := atomic.LoadInt32(&calls); got != 3 {
+		t.Errorf("calls = %d, want 3 (1 initial + 2 retries)", got)
+	}
+}
+
+func TestOpenAIClient_Complete_ExhaustsRetriesOn5xx(t *testing.T) {
+	var calls int32
+	srv := newTestServer(t, func(w http.ResponseWriter, r *http.Request) {
+		atomic.AddInt32(&calls, 1)
+		w.WriteHeader(http.StatusServiceUnavailable)
+	})
+
+	client := NewOpenAIClient(srv.URL, "", time.Second)
+	_, err := client.Complete(context.Background(), Request{Model: "m", Messages: []Message{{Role: "user", Content: "x"}}})
+	if err == nil {
+		t.Fatal("Complete() error = nil, want error after exhausting retries")
+	}
+	if got := atomic.LoadInt32(&calls); got != maxRetries+1 {
+		t.Errorf("calls = %d, want %d", got, maxRetries+1)
+	}
+}
+
+func TestOpenAIClient_Complete_NoRetryOn4xx(t *testing.T) {
+	var calls int32
+	srv := newTestServer(t, func(w http.ResponseWriter, r *http.Request) {
+		atomic.AddInt32(&calls, 1)
+		w.WriteHeader(http.StatusBadRequest)
+		_, _ = w.Write([]byte(`{"error":{"message":"bad request"}}`))
+	})
+
+	client := NewOpenAIClient(srv.URL, "", time.Second)
+	_, err := client.Complete(context.Background(), Request{Model: "m", Messages: []Message{{Role: "user", Content: "x"}}})
+	if err == nil {
+		t.Fatal("Complete() error = nil, want error")
+	}
+	if got := atomic.LoadInt32(&calls); got != 1 {
+		t.Errorf("calls = %d, want 1 (no retry on 4xx)", got)
+	}
+	if !strings.Contains(err.Error(), "400") {
+		t.Errorf("error = %v, want mention of 400", err)
+	}
+}
+
+func TestOpenAIClient_Complete_APIErrorField(t *testing.T) {
+	srv := newTestServer(t, func(w http.ResponseWriter, r *http.Request) {
+		_, _ = w.Write([]byte(`{"error":{"message":"model overloaded"}}`))
+	})
+
+	client := NewOpenAIClient(srv.URL, "", time.Second)
+	_, err := client.Complete(context.Background(), Request{Model: "m", Messages: []Message{{Role: "user", Content: "x"}}})
+	if err == nil {
+		t.Fatal("Complete() error = nil, want error")
+	}
+	if !strings.Contains(err.Error(), "model overloaded") {
+		t.Errorf("error = %v, want mention of model overloaded", err)
+	}
+}
+
+func TestOpenAIClient_Complete_ContextCanceled(t *testing.T) {
+	srv := newTestServer(t, func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(http.StatusInternalServerError)
+	})
+
+	client := NewOpenAIClient(srv.URL, "", time.Second)
+	ctx, cancel := context.WithCancel(context.Background())
+	cancel()
+	_, err := client.Complete(ctx, Request{Model: "m", Messages: []Message{{Role: "user", Content: "x"}}})
+	if err == nil {
+		t.Fatal("Complete() error = nil, want error for canceled context")
+	}
+}
+
+func TestOpenAIClient_Complete_SendsAPIKey(t *testing.T) {
+	srv := newTestServer(t, func(w http.ResponseWriter, r *http.Request) {
+		if got := r.Header.Get("Authorization"); got != "Bearer secret-key" {
+			t.Errorf("Authorization = %q, want Bearer secret-key", got)
+		}
+		_ = json.NewEncoder(w).Encode(chatCompletionResponse{})
+	})
+
+	client := NewOpenAIClient(srv.URL, "secret-key", time.Second)
+	_, err := client.Complete(context.Background(), Request{Model: "m", Messages: []Message{{Role: "user", Content: "x"}}})
+	if err != nil {
+		t.Fatalf("Complete() error = %v", err)
+	}
+}