feat: LLM accuracy testing framework with 17-test catalog

Framework runs a category/subcategory test catalog against multiple
LLMs through an OpenAI-compatible endpoint. Deterministic evaluators
score each response. Reports render as table, markdown, or JSON.

internal/eval/numeric.go

@@ -0,0 +1,117 @@
+package eval
+
+import (
+	"context"
+	"fmt"
+	"regexp"
+	"strconv"
+	"strings"
+)
+
+// Number is the set of numeric primitive kinds Numeric can extract and
+// compare. Defined locally (rather than importing golang.org/x/exp/constraints)
+// since only gopkg.in/yaml.v3 and golang.org/x/sync/errgroup are allowed deps.
+type Number interface {
+	~float32 | ~float64 |
+		~int | ~int8 | ~int16 | ~int32 | ~int64 |
+		~uint | ~uint8 | ~uint16 | ~uint32 | ~uint64
+}
+
+// numberPattern matches signed integers and decimals, including a
+// leading-dot form with no integer part (".9896"). Longer alternatives are
+// listed first so the match is not truncated to just the integer part of
+// "0.9896" or just the dot-less ".9896".
+var numberPattern = regexp.MustCompile(`-?(?:\d+\.\d+|\.\d+|\d+)`)
+
+// hyphenCompoundSuffixPattern matches a hyphen directly after a number
+// joining it to a following word, e.g. the "-bit" in "64-bit" or the
+// "-byte" in "8-byte". A number immediately followed by this is part of a
+// unit/architecture descriptor, not a standalone answer value, and is
+// skipped when picking "the" number out of a line - otherwise "sizeof(...)
+// is 24 bytes on a 64-bit system" would extract the trailing 64 instead of
+// the actual answer, 24.
+var hyphenCompoundSuffixPattern = regexp.MustCompile(`^-\w`)
+
+// lastRelevantNumber returns the last number in text that is not part of a
+// "<number>-word" compound, scanning from the end.
+func lastRelevantNumber(text string) (string, bool) {
+	matches := numberPattern.FindAllStringIndex(text, -1)
+	for i := len(matches) - 1; i >= 0; i-- {
+		start, end := matches[i][0], matches[i][1]
+		if hyphenCompoundSuffixPattern.MatchString(text[end:]) {
+			continue
+		}
+		return text[start:end], true
+	}
+	return "", false
+}
+
+// ExtractLastNumber returns the model's answer number from s, converted to
+// T. It first looks only at the last non-empty line (the common case: a
+// final "24" or "0.9896" on its own line after any reasoning), and only
+// falls back to the last number anywhere in s if that line has no usable
+// number at all. Within whichever text it searches, it skips a number that
+// is part of a "<number>-word" compound (see hyphenCompoundSuffixPattern):
+// "sizeof(struct Config) is 24 bytes on a 64-bit system" must extract 24,
+// not the later, unrelated 64. It is the default extractor for Numeric.
+func ExtractLastNumber[T Number](s string) (T, error) {
+	lines := strings.Split(s, "\n")
+	for i := len(lines) - 1; i >= 0; i-- {
+		line := strings.TrimSpace(lines[i])
+		if line == "" {
+			continue
+		}
+		if lit, ok := lastRelevantNumber(line); ok {
+			return parseNumber[T](lit)
+		}
+		break
+	}
+
+	if lit, ok := lastRelevantNumber(s); ok {
+		return parseNumber[T](lit)
+	}
+	return 0, fmt.Errorf("no number found in response")
+}
+
+// parseNumber converts a numberPattern match (possibly ".9896", with no
+// leading zero) to T.
+func parseNumber[T Number](literal string) (T, error) {
+	f, err := strconv.ParseFloat(literal, 64)
+	if err != nil {
+		return 0, fmt.Errorf("parse number %q: %w", literal, err)
+	}
+	return T(f), nil
+}
+
+// numericEval scores full credit when the extracted number is within tol of
+// want, zero otherwise.
+type numericEval[T Number] struct {
+	extract func(string) (T, error)
+	want    T
+	tol     T
+}
+
+// Numeric returns an Evaluator that extracts a value of type T from the
+// response via extract and awards full credit when |got-want| <= tol.
+func Numeric[T Number](extract func(string) (T, error), want, tol T) Evaluator {
+	return numericEval[T]{extract: extract, want: want, tol: tol}
+}
+
+func (n numericEval[T]) Evaluate(_ context.Context, response string) Score {
+	got, err := n.extract(response)
+	if err != nil {
+		return Score{Value: 0, Detail: fmt.Sprintf("extract failed: %v", err)}
+	}
+
+	var diff T
+	if got >= n.want {
+		diff = got - n.want
+	} else {
+		diff = n.want - got
+	}
+
+	if diff <= n.tol {
+		return Score{Value: 1, Detail: fmt.Sprintf("got %v, want %v (tol %v)", got, n.want, n.tol)}
+	}
+	return Score{Value: 0, Detail: fmt.Sprintf("got %v, want %v (tol %v)", got, n.want, n.tol)}
+}