feat: LLM accuracy testing framework with 17-test catalog

Framework runs a category/subcategory test catalog against multiple
LLMs through an OpenAI-compatible endpoint. Deterministic evaluators
score each response. Reports render as table, markdown, or JSON.

internal/llm/openai.go

@@ -0,0 +1,179 @@
+package llm
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"time"
+)
+
+// maxRetries is the number of retry attempts after the initial request on
+// transport errors or 5xx responses, per PLAN.md ("retry w/ backoff x2").
+const maxRetries = 2
+
+// retryBaseDelay is the base backoff delay; attempt N waits N*retryBaseDelay.
+const retryBaseDelay = 250 * time.Millisecond
+
+// OpenAIClient talks to an OpenAI-compatible /v1/chat/completions endpoint.
+type OpenAIClient struct {
+	httpClient *http.Client
+	endpoint   string
+	apiKey     string
+}
+
+// NewOpenAIClient builds a client against endpoint (e.g.
+// "https://host/v1") using apiKey (may be empty for keyless gateways) and
+// per-request timeout.
+func NewOpenAIClient(endpoint, apiKey string, timeout time.Duration) *OpenAIClient {
+	return &OpenAIClient{
+		endpoint: endpoint,
+		apiKey:   apiKey,
+		httpClient: &http.Client{
+			Timeout: timeout,
+		},
+	}
+}
+
+type chatCompletionRequest struct {
+	Model       string    `json:"model"`
+	Messages    []chatMsg `json:"messages"`
+	MaxTokens   int       `json:"max_tokens,omitempty"`
+	Temperature float64   `json:"temperature"`
+}
+
+type chatMsg struct {
+	Role    string `json:"role"`
+	Content string `json:"content"`
+}
+
+type chatCompletionResponse struct {
+	Error *struct {
+		Message string `json:"message"`
+	} `json:"error"`
+	Choices []struct {
+		Message chatMsg `json:"message"`
+	} `json:"choices"`
+	Usage struct {
+		PromptTokens     int `json:"prompt_tokens"`
+		CompletionTokens int `json:"completion_tokens"`
+	} `json:"usage"`
+}
+
+// Complete implements Client. It retries transport failures and 5xx
+// responses up to maxRetries times with linear backoff.
+func (c *OpenAIClient) Complete(ctx context.Context, req Request) (Response, error) {
+	body := chatCompletionRequest{
+		Model:       req.Model,
+		MaxTokens:   req.MaxTokens,
+		Temperature: req.Temperature,
+	}
+	for _, m := range req.Messages {
+		body.Messages = append(body.Messages, chatMsg(m))
+	}
+
+	payload, err := json.Marshal(body)
+	if err != nil {
+		return Response{}, fmt.Errorf("llm: marshal request: %w", err)
+	}
+
+	var lastErr error
+	for attempt := 0; attempt <= maxRetries; attempt++ {
+		if attempt > 0 {
+			delay := time.Duration(attempt) * retryBaseDelay
+			select {
+			case <-ctx.Done():
+				return Response{}, ctx.Err()
+			case <-time.After(delay):
+			}
+		}
+
+		// Measured per-attempt, starting after any backoff delay: Latency
+		// reflects the successful attempt's own generation time, not the
+		// cumulative wait across earlier failed/retried attempts (S8).
+		attemptStart := time.Now()
+		resp, err := c.doRequest(ctx, payload)
+		if err != nil {
+			lastErr = err
+			if isRetryable(err) {
+				continue
+			}
+			return Response{}, err
+		}
+
+		latency := time.Since(attemptStart)
+		text := ""
+		if len(resp.Choices) > 0 {
+			text = resp.Choices[0].Message.Content
+		}
+		if resp.Error != nil {
+			lastErr = fmt.Errorf("llm: api error: %s", resp.Error.Message)
+			return Response{}, lastErr
+		}
+		return Response{
+			Text:             text,
+			PromptTokens:     resp.Usage.PromptTokens,
+			CompletionTokens: resp.Usage.CompletionTokens,
+			Latency:          latency,
+		}, nil
+	}
+	return Response{}, fmt.Errorf("llm: request failed after %d attempts: %w", maxRetries+1, lastErr)
+}
+
+type retryableError struct {
+	err    error
+	status int
+}
+
+func (e *retryableError) Error() string {
+	if e.status != 0 {
+		return fmt.Sprintf("llm: server returned status %d", e.status)
+	}
+	return fmt.Sprintf("llm: transport error: %v", e.err)
+}
+
+func (e *retryableError) Unwrap() error { return e.err }
+
+func isRetryable(err error) bool {
+	var re *retryableError
+	return errors.As(err, &re)
+}
+
+func (c *OpenAIClient) doRequest(ctx context.Context, payload []byte) (*chatCompletionResponse, error) {
+	url := c.endpoint + "/chat/completions"
+	httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(payload))
+	if err != nil {
+		return nil, fmt.Errorf("llm: build request: %w", err)
+	}
+	httpReq.Header.Set("Content-Type", "application/json")
+	if c.apiKey != "" {
+		httpReq.Header.Set("Authorization", "Bearer "+c.apiKey)
+	}
+
+	httpResp, err := c.httpClient.Do(httpReq)
+	if err != nil {
+		return nil, &retryableError{err: err}
+	}
+	defer func() { _ = httpResp.Body.Close() }() // best-effort; body already fully read below
+
+	respBody, err := io.ReadAll(httpResp.Body)
+	if err != nil {
+		return nil, &retryableError{err: fmt.Errorf("read body: %w", err)}
+	}
+
+	if httpResp.StatusCode >= 500 {
+		return nil, &retryableError{status: httpResp.StatusCode}
+	}
+	if httpResp.StatusCode >= 400 {
+		return nil, fmt.Errorf("llm: server returned status %d: %s", httpResp.StatusCode, string(respBody))
+	}
+
+	var out chatCompletionResponse
+	if err := json.Unmarshal(respBody, &out); err != nil {
+		return nil, fmt.Errorf("llm: decode response: %w", err)
+	}
+	return &out, nil
+}