mirror of
https://github.com/lukaszraczylo/llm-testbench.git
synced 2026-10-12 00:45:19 +00:00
feat: LLM accuracy testing framework with 17-test catalog
Framework runs a category/subcategory test catalog against multiple LLMs through an OpenAI-compatible endpoint. Deterministic evaluators score each response. Reports render as table, markdown, or JSON.
This commit is contained in:
@@ -0,0 +1,70 @@
|
||||
package eval
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// WeightedEvaluator pairs an Evaluator with its share of the composite score.
|
||||
type WeightedEvaluator struct {
|
||||
Eval Evaluator
|
||||
Weight float64
|
||||
}
|
||||
|
||||
// W builds a WeightedEvaluator; a convenience constructor for All.
|
||||
func W(e Evaluator, weight float64) WeightedEvaluator {
|
||||
return WeightedEvaluator{Eval: e, Weight: weight}
|
||||
}
|
||||
|
||||
// compositeEval computes a weighted mean over its sub-evaluators, ignoring
|
||||
// sub-scores that are Skipped.
|
||||
type compositeEval struct {
|
||||
weighted []WeightedEvaluator
|
||||
}
|
||||
|
||||
// All returns an Evaluator that combines weighted sub-evaluators into a
|
||||
// single weighted-mean Score. Sub-scores marked Skipped are excluded from
|
||||
// both the numerator and the weight total; if every sub-evaluator is
|
||||
// skipped, the composite itself is Skipped.
|
||||
func All(weighted ...WeightedEvaluator) Evaluator {
|
||||
return compositeEval{weighted: weighted}
|
||||
}
|
||||
|
||||
// Mean wraps evaluators with equal weight 1 and combines them via All. A
|
||||
// convenience for the common case where every sub-check counts the same.
|
||||
// Named Mean, not Equal, to avoid reading as a synonym of the Equals
|
||||
// evaluator (S12) - Mean computes an unweighted average of N sub-scores,
|
||||
// while Equals is a single trimmed, case-insensitive string comparison.
|
||||
func Mean(evaluators ...Evaluator) Evaluator {
|
||||
weighted := make([]WeightedEvaluator, 0, len(evaluators))
|
||||
for _, e := range evaluators {
|
||||
weighted = append(weighted, W(e, 1))
|
||||
}
|
||||
return All(weighted...)
|
||||
}
|
||||
|
||||
func (c compositeEval) Evaluate(ctx context.Context, response string) Score {
|
||||
var weightedSum, activeWeight float64
|
||||
details := make([]string, 0, len(c.weighted))
|
||||
|
||||
for _, we := range c.weighted {
|
||||
s := we.Eval.Evaluate(ctx, response)
|
||||
if s.Skipped {
|
||||
details = append(details, fmt.Sprintf("skipped (w=%.2f): %s", we.Weight, s.Detail))
|
||||
continue
|
||||
}
|
||||
activeWeight += we.Weight
|
||||
weightedSum += s.Value * we.Weight
|
||||
details = append(details, fmt.Sprintf("%.2f (w=%.2f): %s", s.Value, we.Weight, s.Detail))
|
||||
}
|
||||
|
||||
if activeWeight == 0 {
|
||||
return Score{Skipped: true, Detail: "all sub-evaluators skipped: " + strings.Join(details, "; ")}
|
||||
}
|
||||
|
||||
return Score{
|
||||
Value: clamp01(weightedSum / activeWeight),
|
||||
Detail: strings.Join(details, "; "),
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,71 @@
|
||||
package eval
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestAll_WeightedMean(t *testing.T) {
|
||||
e := All(
|
||||
W(Equals("minor"), 2),
|
||||
W(ContainsAny("bump"), 1),
|
||||
)
|
||||
// "minor bump" -> Equals fails (0, weight 2), ContainsAny succeeds (1, weight 1)
|
||||
// weighted mean = (0*2 + 1*1) / 3 = 1/3
|
||||
got := e.Evaluate(context.Background(), "minor bump")
|
||||
want := 1.0 / 3.0
|
||||
if got.Value != want {
|
||||
t.Errorf("Evaluate() = %v, want %v (detail: %s)", got.Value, want, got.Detail)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMean_AllPass(t *testing.T) {
|
||||
e := Mean(
|
||||
ContainsAll("launchd"),
|
||||
ContainsAll("StartInterval"),
|
||||
)
|
||||
got := e.Evaluate(context.Background(), "use launchd with StartInterval key")
|
||||
if got.Value != 1 {
|
||||
t.Errorf("Evaluate() = %v, want 1 (detail: %s)", got.Value, got.Detail)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMean_PartialCredit(t *testing.T) {
|
||||
e := Mean(
|
||||
ContainsAll("launchd"),
|
||||
ContainsAll("StartInterval"),
|
||||
)
|
||||
// Only "launchd" present -> (1 + 0) / 2 = 0.5
|
||||
got := e.Evaluate(context.Background(), "use launchd, not cron")
|
||||
if got.Value != 0.5 {
|
||||
t.Errorf("Evaluate() = %v, want 0.5 (detail: %s)", got.Value, got.Detail)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAll_SkippedSubEvaluatorExcluded(t *testing.T) {
|
||||
skipped := EvaluatorFunc(func(_ context.Context, _ string) Score {
|
||||
return Score{Skipped: true, Detail: "toolchain missing: go"}
|
||||
})
|
||||
e := All(
|
||||
W(Equals("x"), 1),
|
||||
W(skipped, 5),
|
||||
)
|
||||
got := e.Evaluate(context.Background(), "x")
|
||||
if got.Skipped {
|
||||
t.Fatal("Evaluate() should not be Skipped when at least one sub-evaluator is active")
|
||||
}
|
||||
if got.Value != 1 {
|
||||
t.Errorf("Evaluate() = %v, want 1 (skipped sub-evaluator excluded from weight)", got.Value)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAll_EverythingSkipped(t *testing.T) {
|
||||
skipped := EvaluatorFunc(func(_ context.Context, _ string) Score {
|
||||
return Score{Skipped: true}
|
||||
})
|
||||
e := All(W(skipped, 1))
|
||||
got := e.Evaluate(context.Background(), "anything")
|
||||
if !got.Skipped {
|
||||
t.Error("Evaluate() should be Skipped when every sub-evaluator is skipped")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,85 @@
|
||||
package eval
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// containsAll scores the fraction of substrings present in the response,
|
||||
// case-insensitive. Full credit only when every substring matches.
|
||||
type containsAll struct {
|
||||
substrings []string
|
||||
}
|
||||
|
||||
// ContainsAll returns an Evaluator that awards matched/total credit for how
|
||||
// many of substrings appear (case-insensitive) in the response.
|
||||
func ContainsAll(substrings ...string) Evaluator {
|
||||
return containsAll{substrings: substrings}
|
||||
}
|
||||
|
||||
func (c containsAll) Evaluate(_ context.Context, response string) Score {
|
||||
if len(c.substrings) == 0 {
|
||||
return Score{Value: 1, Detail: "no substrings required"}
|
||||
}
|
||||
lower := strings.ToLower(response)
|
||||
matched := 0
|
||||
var missing []string
|
||||
for _, s := range c.substrings {
|
||||
if strings.Contains(lower, strings.ToLower(s)) {
|
||||
matched++
|
||||
} else {
|
||||
missing = append(missing, s)
|
||||
}
|
||||
}
|
||||
value := float64(matched) / float64(len(c.substrings))
|
||||
detail := fmt.Sprintf("matched %d/%d", matched, len(c.substrings))
|
||||
if len(missing) > 0 {
|
||||
detail += fmt.Sprintf("; missing: %s", strings.Join(missing, ", "))
|
||||
}
|
||||
return Score{Value: value, Detail: detail}
|
||||
}
|
||||
|
||||
// containsAny awards full credit if at least one alternative matches.
|
||||
type containsAny struct {
|
||||
substrings []string
|
||||
}
|
||||
|
||||
// ContainsAny returns an Evaluator that awards full credit if any substring
|
||||
// (case-insensitive) appears in the response, zero otherwise.
|
||||
func ContainsAny(substrings ...string) Evaluator {
|
||||
return containsAny{substrings: substrings}
|
||||
}
|
||||
|
||||
func (c containsAny) Evaluate(_ context.Context, response string) Score {
|
||||
lower := strings.ToLower(response)
|
||||
for _, s := range c.substrings {
|
||||
if strings.Contains(lower, strings.ToLower(s)) {
|
||||
return Score{Value: 1, Detail: fmt.Sprintf("matched %q", s)}
|
||||
}
|
||||
}
|
||||
return Score{Value: 0, Detail: fmt.Sprintf("none of %v matched", c.substrings)}
|
||||
}
|
||||
|
||||
// notContains is a guard: full credit unless a forbidden substring appears.
|
||||
type notContains struct {
|
||||
substrings []string
|
||||
}
|
||||
|
||||
// NotContains returns a composable guard Evaluator: full credit unless the
|
||||
// response contains any of substrings (case-insensitive), in which case it
|
||||
// scores zero. Useful to penalize unsafe/undesired suggestions (e.g.
|
||||
// "kubectl edit").
|
||||
func NotContains(substrings ...string) Evaluator {
|
||||
return notContains{substrings: substrings}
|
||||
}
|
||||
|
||||
func (c notContains) Evaluate(_ context.Context, response string) Score {
|
||||
lower := strings.ToLower(response)
|
||||
for _, s := range c.substrings {
|
||||
if strings.Contains(lower, strings.ToLower(s)) {
|
||||
return Score{Value: 0, Detail: fmt.Sprintf("forbidden %q present", s)}
|
||||
}
|
||||
}
|
||||
return Score{Value: 1, Detail: "no forbidden substrings present"}
|
||||
}
|
||||
@@ -0,0 +1,71 @@
|
||||
package eval
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestContainsAll(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
substrings []string
|
||||
want float64
|
||||
}{
|
||||
{"all present", "use launchd with StartInterval", []string{"launchd", "StartInterval"}, 1},
|
||||
{"case insensitive", "use Launchd here", []string{"LAUNCHD"}, 1},
|
||||
{"partial credit", "alpha and beta only", []string{"alpha", "beta", "gamma", "delta"}, 0.5},
|
||||
{"none present", "nothing matches", []string{"x", "y"}, 0},
|
||||
{"empty list", "anything", nil, 1},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := ContainsAll(tt.substrings...).Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("ContainsAll(%v).Evaluate(%q) = %v, want %v (detail: %s)", tt.substrings, tt.response, got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestContainsAny(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
substrings []string
|
||||
want float64
|
||||
}{
|
||||
{"one matches", "install coreutils via brew", []string{"coreutils", "gtimeout"}, 1},
|
||||
{"case insensitive match", "use gtimeout instead", []string{"GTIMEOUT"}, 1},
|
||||
{"none match", "totally unrelated text", []string{"foo", "bar"}, 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := ContainsAny(tt.substrings...).Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("ContainsAny(%v).Evaluate(%q) = %v, want %v", tt.substrings, tt.response, got.Value, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestNotContains(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
substrings []string
|
||||
want float64
|
||||
}{
|
||||
{"forbidden absent", "git commit the manifest change", []string{"kubectl edit"}, 1},
|
||||
{"forbidden present", "just run kubectl edit deployment", []string{"kubectl edit"}, 0},
|
||||
{"case insensitive forbidden", "run kubectl patch now", []string{"KUBECTL PATCH"}, 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := NotContains(tt.substrings...).Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("NotContains(%v).Evaluate(%q) = %v, want %v", tt.substrings, tt.response, got.Value, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
package eval
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// equalsEval scores full credit on a trimmed, case-insensitive exact match.
|
||||
type equalsEval struct {
|
||||
want string
|
||||
}
|
||||
|
||||
// Equals returns an Evaluator awarding full credit when the response,
|
||||
// trimmed and lower-cased, equals want (also trimmed and lower-cased).
|
||||
func Equals(want string) Evaluator {
|
||||
return equalsEval{want: want}
|
||||
}
|
||||
|
||||
func (e equalsEval) Evaluate(_ context.Context, response string) Score {
|
||||
got := strings.ToLower(strings.TrimSpace(response))
|
||||
want := strings.ToLower(strings.TrimSpace(e.want))
|
||||
if got == want {
|
||||
return Score{Value: 1, Detail: fmt.Sprintf("equals %q", e.want)}
|
||||
}
|
||||
return Score{Value: 0, Detail: fmt.Sprintf("got %q, want %q", strings.TrimSpace(response), e.want)}
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
package eval
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestEquals(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
want string
|
||||
response string
|
||||
wantVal float64
|
||||
}{
|
||||
{"exact match", "patch", "patch", 1},
|
||||
{"case insensitive", "PATCH", "patch", 1},
|
||||
{"trimmed whitespace", "patch", " patch\n", 1},
|
||||
{"mismatch", "patch", "minor", 0},
|
||||
{"substring is not equal", "patch", "patch bump", 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := Equals(tt.want).Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.wantVal {
|
||||
t.Errorf("Equals(%q).Evaluate(%q) = %v, want %v", tt.want, tt.response, got.Value, tt.wantVal)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
// Package eval provides deterministic evaluators that score a (normalized)
|
||||
// LLM response against an expected answer. All evaluators are pure functions
|
||||
// of the response text plus evaluator configuration; none call the network.
|
||||
package eval
|
||||
|
||||
import "context"
|
||||
|
||||
// Score is the outcome of evaluating one response. Value is in [0, 1] unless
|
||||
// Skipped is true (e.g. a required toolchain is missing), in which case
|
||||
// Value carries no meaning and the test is excluded from aggregates.
|
||||
type Score struct {
|
||||
Detail string
|
||||
Value float64
|
||||
Skipped bool
|
||||
}
|
||||
|
||||
// Evaluator scores a normalized response string.
|
||||
type Evaluator interface {
|
||||
Evaluate(ctx context.Context, response string) Score
|
||||
}
|
||||
|
||||
// EvaluatorFunc adapts a plain function to the Evaluator interface.
|
||||
type EvaluatorFunc func(ctx context.Context, response string) Score
|
||||
|
||||
// Evaluate implements Evaluator.
|
||||
func (f EvaluatorFunc) Evaluate(ctx context.Context, response string) Score {
|
||||
return f(ctx, response)
|
||||
}
|
||||
|
||||
// clamp01 constrains v to the closed interval [0, 1].
|
||||
func clamp01(v float64) float64 {
|
||||
switch {
|
||||
case v < 0:
|
||||
return 0
|
||||
case v > 1:
|
||||
return 1
|
||||
default:
|
||||
return v
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,331 @@
|
||||
package eval
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// leadingPackageClausePattern matches a Go source line that is nothing but
|
||||
// a package clause (optionally indented), used to defensively strip a
|
||||
// model's own "package xxx" line before splicing its code into
|
||||
// solution.go. Only the first non-blank line is ever checked/stripped; a
|
||||
// package clause appearing later (inside a string, comment, or as a
|
||||
// genuine syntax error) is left alone.
|
||||
var leadingPackageClausePattern = regexp.MustCompile(`^\s*package\s+\w+\s*$`)
|
||||
|
||||
// stripLeadingPackageClause removes a leading "package xxx" line from code,
|
||||
// if the first non-blank line is one. This lets a model's response be
|
||||
// either a bare code fragment or a full file (package clause + imports +
|
||||
// declarations) and still compile as part of solution.go's package main.
|
||||
func stripLeadingPackageClause(code string) string {
|
||||
lines := strings.Split(code, "\n")
|
||||
for i, line := range lines {
|
||||
if strings.TrimSpace(line) == "" {
|
||||
continue
|
||||
}
|
||||
if leadingPackageClausePattern.MatchString(line) {
|
||||
return strings.Join(append(lines[:i:i], lines[i+1:]...), "\n")
|
||||
}
|
||||
break
|
||||
}
|
||||
return code
|
||||
}
|
||||
|
||||
// execTimeout bounds every compile/run step of the exec evaluators.
|
||||
const execTimeout = 30 * time.Second
|
||||
|
||||
// CodePlaceholder marks where the extracted response code is substituted
|
||||
// into a harness template.
|
||||
const CodePlaceholder = "{{CODE}}"
|
||||
|
||||
// PassthroughHarness is a harness that runs the extracted code verbatim,
|
||||
// with no surrounding driver. Use it when the response is expected to be a
|
||||
// complete, self-contained program (e.g. a full Python script) rather than
|
||||
// a fragment plugged into a test driver.
|
||||
const PassthroughHarness = CodePlaceholder
|
||||
|
||||
// buildSource substitutes code into harness at CodePlaceholder.
|
||||
func buildSource(harness, code string) string {
|
||||
return strings.ReplaceAll(harness, CodePlaceholder, code)
|
||||
}
|
||||
|
||||
// goEnvCache memoizes `go env GOCACHE`/`go env GOMODCACHE` so every
|
||||
// exec-evaluator call does not pay for an extra subprocess: they are read
|
||||
// once per process and reused.
|
||||
var (
|
||||
goEnvOnce sync.Once
|
||||
goCacheDir string
|
||||
goModCacheDir string
|
||||
)
|
||||
|
||||
// goEnvValue shells out to `go env key` and returns its trimmed output, or
|
||||
// "" if the go tool is unavailable or the call fails.
|
||||
func goEnvValue(key string) string {
|
||||
// #nosec G204 -- key is always one of the two fixed, hardcoded literals
|
||||
// ("GOCACHE"/"GOMODCACHE") passed by minimalExecEnv below, never
|
||||
// response/user-controlled input.
|
||||
out, err := exec.Command("go", "env", key).Output() //nolint:gosec // key is one of two hardcoded literals, see comment above
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
return strings.TrimSpace(string(out))
|
||||
}
|
||||
|
||||
// minimalExecEnv builds a minimal environment allowlist for compiling and
|
||||
// running model-generated code, rather than inheriting the operator's full
|
||||
// environment: PATH (needed to find go/python3/cc and any tool they shell
|
||||
// out to), HOME and TMPDIR pinned to homeDir (an isolated, single-call temp
|
||||
// directory, never the operator's real home), and GOCACHE/GOMODCACHE
|
||||
// inherited from the operator's actual Go environment so `go run` reuses
|
||||
// the existing build/module cache instead of rebuilding it from scratch on
|
||||
// every call.
|
||||
func minimalExecEnv(homeDir string) []string {
|
||||
goEnvOnce.Do(func() {
|
||||
goCacheDir = goEnvValue("GOCACHE")
|
||||
goModCacheDir = goEnvValue("GOMODCACHE")
|
||||
})
|
||||
|
||||
env := []string{
|
||||
"PATH=" + os.Getenv("PATH"),
|
||||
"HOME=" + homeDir,
|
||||
"TMPDIR=" + homeDir,
|
||||
}
|
||||
if goCacheDir != "" {
|
||||
env = append(env, "GOCACHE="+goCacheDir)
|
||||
}
|
||||
if goModCacheDir != "" {
|
||||
env = append(env, "GOMODCACHE="+goModCacheDir)
|
||||
}
|
||||
return env
|
||||
}
|
||||
|
||||
// mkWorkDirs creates an isolated per-call temp root plus a "work"
|
||||
// subdirectory inside it. Callers write source files into work and run
|
||||
// commands with cmd.Dir = work, while pointing HOME/TMPDIR (via
|
||||
// minimalExecEnv) at root, never at work itself: the Go toolchain
|
||||
// specifically refuses to detect a go.mod when the module directory is
|
||||
// exactly equal to the resolved system temp root ("ignoring go.mod in
|
||||
// system temp root"), which fires precisely when TMPDIR is set to the same
|
||||
// directory `go run` is invoked from. Keeping root and work as distinct
|
||||
// (parent, child) directories avoids that while still confining
|
||||
// HOME/TMPDIR to a directory nothing else uses.
|
||||
func mkWorkDirs(prefix string) (root, work string, err error) {
|
||||
root, err = os.MkdirTemp("", prefix)
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
work = filepath.Join(root, "work")
|
||||
if err := os.Mkdir(work, 0o700); err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
return root, work, nil
|
||||
}
|
||||
|
||||
// checkToolchain reports a Skipped score if name is not on PATH, nil
|
||||
// otherwise.
|
||||
func checkToolchain(name string) *Score {
|
||||
if _, err := exec.LookPath(name); err != nil {
|
||||
s := Score{Skipped: true, Detail: fmt.Sprintf("toolchain missing: %s", name)}
|
||||
return &s
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// compareStdout scores full credit when stdout, trimmed, equals want,
|
||||
// trimmed. cmdErr, when non-nil, always scores zero and surfaces stderr.
|
||||
func compareStdout(stdout, want string, cmdErr error, stderr string) Score {
|
||||
if cmdErr != nil {
|
||||
return Score{Value: 0, Detail: fmt.Sprintf("execution failed: %v; stderr: %s", cmdErr, strings.TrimSpace(stderr))}
|
||||
}
|
||||
got := strings.TrimSpace(stdout)
|
||||
wantTrimmed := strings.TrimSpace(want)
|
||||
if got == wantTrimmed {
|
||||
return Score{Value: 1, Detail: "stdout matches expected"}
|
||||
}
|
||||
return Score{Value: 0, Detail: fmt.Sprintf("stdout = %q, want %q", got, wantTrimmed)}
|
||||
}
|
||||
|
||||
// goRunEval compiles/runs a two-file Go module: solution.go (the model's
|
||||
// code, wrapped in package main) plus harness.go (a complete, independent
|
||||
// driver file supplied by the test author).
|
||||
type goRunEval struct {
|
||||
harness string
|
||||
want string
|
||||
}
|
||||
|
||||
// GoRun returns an Evaluator that writes the response's first fenced Go
|
||||
// code block to its own solution.go (defensively stripping a leading
|
||||
// "package xxx" line and prefixing "package main"), writes harness
|
||||
// verbatim to its own harness.go (a complete file: its own package clause,
|
||||
// imports, and func main), `go run`s both together in an isolated temp
|
||||
// module, and compares trimmed stdout to want. Splitting the model's code
|
||||
// and the harness into separate files means each may import what it needs
|
||||
// independently, instead of splicing code into a single shared import
|
||||
// block. If the `go` toolchain is unavailable, the Score is Skipped rather
|
||||
// than zero.
|
||||
func GoRun(harness, want string) Evaluator {
|
||||
return goRunEval{harness: harness, want: want}
|
||||
}
|
||||
|
||||
func (g goRunEval) Evaluate(ctx context.Context, response string) Score {
|
||||
if s := checkToolchain("go"); s != nil {
|
||||
return *s
|
||||
}
|
||||
|
||||
code := ExtractCodeBlock(response, "go")
|
||||
solution := "package main\n\n" + stripLeadingPackageClause(code)
|
||||
|
||||
root, work, err := mkWorkDirs("llmtb-go-*")
|
||||
if err != nil {
|
||||
return Score{Value: 0, Detail: fmt.Sprintf("mkdtemp: %v", err)}
|
||||
}
|
||||
defer func() { _ = os.RemoveAll(root) }() // best-effort cleanup; failure is not actionable
|
||||
|
||||
if writeErr := os.WriteFile(filepath.Join(work, "solution.go"), []byte(solution), 0o600); writeErr != nil {
|
||||
return Score{Value: 0, Detail: fmt.Sprintf("write solution.go: %v", writeErr)}
|
||||
}
|
||||
if writeErr := os.WriteFile(filepath.Join(work, "harness.go"), []byte(g.harness), 0o600); writeErr != nil {
|
||||
return Score{Value: 0, Detail: fmt.Sprintf("write harness.go: %v", writeErr)}
|
||||
}
|
||||
if modErr := os.WriteFile(filepath.Join(work, "go.mod"), []byte("module llmtbharness\n\ngo 1.23\n"), 0o600); modErr != nil {
|
||||
return Score{Value: 0, Detail: fmt.Sprintf("write go.mod: %v", modErr)}
|
||||
}
|
||||
|
||||
runCtx, cancel := context.WithTimeout(ctx, execTimeout)
|
||||
defer cancel()
|
||||
|
||||
cmd := exec.CommandContext(runCtx, "go", "run", ".")
|
||||
cmd.Dir = work
|
||||
cmd.Env = minimalExecEnv(root)
|
||||
var stdout, stderr bytes.Buffer
|
||||
cmd.Stdout = &stdout
|
||||
cmd.Stderr = &stderr
|
||||
err = cmd.Run()
|
||||
|
||||
return compareStdout(stdout.String(), g.want, err, stderr.String())
|
||||
}
|
||||
|
||||
// pyRunEval runs a Python program built from a harness template plus the
|
||||
// extracted Python code block from the response.
|
||||
type pyRunEval struct {
|
||||
harness string
|
||||
want string
|
||||
}
|
||||
|
||||
// PyRun returns an Evaluator that substitutes the response's first fenced
|
||||
// Python code block into harness at CodePlaceholder, runs it with python3,
|
||||
// and compares trimmed stdout to want. If python3 is unavailable, the Score
|
||||
// is Skipped rather than zero.
|
||||
func PyRun(harness, want string) Evaluator {
|
||||
return pyRunEval{harness: harness, want: want}
|
||||
}
|
||||
|
||||
func (p pyRunEval) Evaluate(ctx context.Context, response string) Score {
|
||||
if s := checkToolchain("python3"); s != nil {
|
||||
return *s
|
||||
}
|
||||
|
||||
code := ExtractCodeBlock(response, "python")
|
||||
source := buildSource(p.harness, code)
|
||||
|
||||
root, work, err := mkWorkDirs("llmtb-py-*")
|
||||
if err != nil {
|
||||
return Score{Value: 0, Detail: fmt.Sprintf("mkdtemp: %v", err)}
|
||||
}
|
||||
defer func() { _ = os.RemoveAll(root) }() // best-effort cleanup; failure is not actionable
|
||||
|
||||
scriptPath := filepath.Join(work, "script.py")
|
||||
if writeErr := os.WriteFile(scriptPath, []byte(source), 0o600); writeErr != nil {
|
||||
return Score{Value: 0, Detail: fmt.Sprintf("write script.py: %v", writeErr)}
|
||||
}
|
||||
|
||||
runCtx, cancel := context.WithTimeout(ctx, execTimeout)
|
||||
defer cancel()
|
||||
|
||||
// #nosec G204 -- this is the eval package's whole purpose: compile/run
|
||||
// model-generated code under an isolated temp dir and a minimal env
|
||||
// allowlist (minimalExecEnv). See README's "Requirements" section for
|
||||
// the corresponding operator-facing warning (S13).
|
||||
cmd := exec.CommandContext(runCtx, "python3", scriptPath) //nolint:gosec // see comment above
|
||||
cmd.Dir = work
|
||||
cmd.Env = minimalExecEnv(root)
|
||||
var stdout, stderr bytes.Buffer
|
||||
cmd.Stdout = &stdout
|
||||
cmd.Stderr = &stderr
|
||||
err = cmd.Run()
|
||||
|
||||
return compareStdout(stdout.String(), p.want, err, stderr.String())
|
||||
}
|
||||
|
||||
// cRunEval compiles/runs a C program built from a harness template plus the
|
||||
// extracted C code block from the response.
|
||||
type cRunEval struct {
|
||||
harness string
|
||||
want string
|
||||
}
|
||||
|
||||
// CRun returns an Evaluator that substitutes the response's first fenced C
|
||||
// code block into harness at CodePlaceholder, compiles it with cc, runs the
|
||||
// binary, and compares trimmed stdout to want. If cc is unavailable, the
|
||||
// Score is Skipped rather than zero.
|
||||
func CRun(harness, want string) Evaluator {
|
||||
return cRunEval{harness: harness, want: want}
|
||||
}
|
||||
|
||||
func (c cRunEval) Evaluate(ctx context.Context, response string) Score {
|
||||
if s := checkToolchain("cc"); s != nil {
|
||||
return *s
|
||||
}
|
||||
|
||||
code := ExtractCodeBlock(response, "c")
|
||||
source := buildSource(c.harness, code)
|
||||
|
||||
root, work, err := mkWorkDirs("llmtb-c-*")
|
||||
if err != nil {
|
||||
return Score{Value: 0, Detail: fmt.Sprintf("mkdtemp: %v", err)}
|
||||
}
|
||||
defer func() { _ = os.RemoveAll(root) }() // best-effort cleanup; failure is not actionable
|
||||
|
||||
srcPath := filepath.Join(work, "main.c")
|
||||
if writeErr := os.WriteFile(srcPath, []byte(source), 0o600); writeErr != nil {
|
||||
return Score{Value: 0, Detail: fmt.Sprintf("write main.c: %v", writeErr)}
|
||||
}
|
||||
binPath := filepath.Join(work, "prog")
|
||||
|
||||
buildCtx, buildCancel := context.WithTimeout(ctx, execTimeout)
|
||||
defer buildCancel()
|
||||
// #nosec G204 -- compiling model-generated C source under an isolated
|
||||
// temp dir and a minimal env allowlist is this evaluator's job; see the
|
||||
// PyRun comment above and README's "Requirements" warning (S13).
|
||||
buildCmd := exec.CommandContext(buildCtx, "cc", "-o", binPath, srcPath) //nolint:gosec // see comment above
|
||||
buildCmd.Dir = work
|
||||
buildCmd.Env = minimalExecEnv(root)
|
||||
var buildStderr bytes.Buffer
|
||||
buildCmd.Stderr = &buildStderr
|
||||
if buildErr := buildCmd.Run(); buildErr != nil {
|
||||
return Score{Value: 0, Detail: fmt.Sprintf("cc failed: %v; stderr: %s", buildErr, strings.TrimSpace(buildStderr.String()))}
|
||||
}
|
||||
|
||||
runCtx, runCancel := context.WithTimeout(ctx, execTimeout)
|
||||
defer runCancel()
|
||||
// #nosec G204 -- binPath is the binary this same call just compiled,
|
||||
// into a fresh, isolated temp dir this evaluator created; not an
|
||||
// externally supplied executable path.
|
||||
runCmd := exec.CommandContext(runCtx, binPath) //nolint:gosec // see comment above
|
||||
runCmd.Dir = work
|
||||
runCmd.Env = minimalExecEnv(root)
|
||||
var stdout, stderr bytes.Buffer
|
||||
runCmd.Stdout = &stdout
|
||||
runCmd.Stderr = &stderr
|
||||
err = runCmd.Run()
|
||||
|
||||
return compareStdout(stdout.String(), c.want, err, stderr.String())
|
||||
}
|
||||
@@ -0,0 +1,186 @@
|
||||
package eval
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os/exec"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func requireToolchain(t *testing.T, name string) {
|
||||
t.Helper()
|
||||
if _, err := exec.LookPath(name); err != nil {
|
||||
t.Skipf("toolchain %s not found on PATH, skipping", name)
|
||||
}
|
||||
}
|
||||
|
||||
// goHarness is a complete, standalone harness.go: its own package clause
|
||||
// and imports, calling Double (defined separately in solution.go, same
|
||||
// package) - no {{CODE}} placeholder, per GoRun's two-file contract.
|
||||
const goHarness = `package main
|
||||
|
||||
import "fmt"
|
||||
|
||||
func main() {
|
||||
fmt.Println(Double(21))
|
||||
}
|
||||
`
|
||||
|
||||
func TestGoRun_Pass(t *testing.T) {
|
||||
requireToolchain(t, "go")
|
||||
e := GoRun(goHarness, "42")
|
||||
response := "```go\nfunc Double(n int) int { return n * 2 }\n```"
|
||||
got := e.Evaluate(context.Background(), response)
|
||||
if got.Value != 1 {
|
||||
t.Errorf("Evaluate() = %v, want 1 (detail: %s)", got.Value, got.Detail)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGoRun_WrongOutput(t *testing.T) {
|
||||
requireToolchain(t, "go")
|
||||
e := GoRun(goHarness, "42")
|
||||
response := "```go\nfunc Double(n int) int { return n + 2 }\n```"
|
||||
got := e.Evaluate(context.Background(), response)
|
||||
if got.Value != 0 {
|
||||
t.Errorf("Evaluate() = %v, want 0 (detail: %s)", got.Value, got.Detail)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGoRun_CompileError(t *testing.T) {
|
||||
requireToolchain(t, "go")
|
||||
e := GoRun(goHarness, "42")
|
||||
response := "```go\nfunc Double(n int) int { this is not valid go\n```"
|
||||
got := e.Evaluate(context.Background(), response)
|
||||
if got.Value != 0 {
|
||||
t.Errorf("Evaluate() = %v, want 0 for compile error", got.Value)
|
||||
}
|
||||
if got.Skipped {
|
||||
t.Error("Evaluate() should not be Skipped for a compile error")
|
||||
}
|
||||
}
|
||||
|
||||
func TestStripLeadingPackageClause(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
code string
|
||||
want string
|
||||
}{
|
||||
{
|
||||
name: "leading package clause is stripped",
|
||||
code: "package main\n\nfunc F() {}",
|
||||
want: "\nfunc F() {}",
|
||||
},
|
||||
{
|
||||
name: "blank lines before package clause are tolerated",
|
||||
code: "\n\npackage foo\nfunc F() {}",
|
||||
want: "\n\nfunc F() {}",
|
||||
},
|
||||
{
|
||||
name: "no package clause leaves code untouched",
|
||||
code: "import \"fmt\"\n\nfunc F() {}",
|
||||
want: "import \"fmt\"\n\nfunc F() {}",
|
||||
},
|
||||
{
|
||||
name: "package clause not on the first line is left alone",
|
||||
code: "func F() {}\n\npackage main",
|
||||
want: "func F() {}\n\npackage main",
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := stripLeadingPackageClause(tt.code)
|
||||
if got != tt.want {
|
||||
t.Errorf("stripLeadingPackageClause(%q) = %q, want %q", tt.code, got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestCheckToolchain_MissingBinaryIsSkipped(t *testing.T) {
|
||||
s := checkToolchain("definitely-not-a-real-toolchain-xyz")
|
||||
if s == nil || !s.Skipped {
|
||||
t.Fatal("checkToolchain() for missing binary should return Skipped score")
|
||||
}
|
||||
}
|
||||
|
||||
const pyPassthroughHarness = PassthroughHarness
|
||||
|
||||
func TestPyRun_Pass(t *testing.T) {
|
||||
requireToolchain(t, "python3")
|
||||
e := PyRun(pyPassthroughHarness, "OOMKilled=2\nImagePullBackOff=1")
|
||||
response := "```python\nprint('OOMKilled=2')\nprint('ImagePullBackOff=1')\n```"
|
||||
got := e.Evaluate(context.Background(), response)
|
||||
if got.Value != 1 {
|
||||
t.Errorf("Evaluate() = %v, want 1 (detail: %s)", got.Value, got.Detail)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPyRun_WrongOutput(t *testing.T) {
|
||||
requireToolchain(t, "python3")
|
||||
e := PyRun(pyPassthroughHarness, "OOMKilled=2")
|
||||
response := "```python\nprint('OOMKilled=3')\n```"
|
||||
got := e.Evaluate(context.Background(), response)
|
||||
if got.Value != 0 {
|
||||
t.Errorf("Evaluate() = %v, want 0 (detail: %s)", got.Value, got.Detail)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPyRun_RuntimeError(t *testing.T) {
|
||||
requireToolchain(t, "python3")
|
||||
e := PyRun(pyPassthroughHarness, "anything")
|
||||
response := "```python\nraise ValueError('boom')\n```"
|
||||
got := e.Evaluate(context.Background(), response)
|
||||
if got.Value != 0 || got.Skipped {
|
||||
t.Errorf("Evaluate() = %+v, want zero non-skipped score for runtime error", got)
|
||||
}
|
||||
}
|
||||
|
||||
const cHarness = `#include <stdio.h>
|
||||
|
||||
{{CODE}}
|
||||
|
||||
int main(void) {
|
||||
printf("%d\n", triple(7));
|
||||
return 0;
|
||||
}
|
||||
`
|
||||
|
||||
func TestCRun_Pass(t *testing.T) {
|
||||
requireToolchain(t, "cc")
|
||||
e := CRun(cHarness, "21")
|
||||
response := "```c\nint triple(int n) { return n * 3; }\n```"
|
||||
got := e.Evaluate(context.Background(), response)
|
||||
if got.Value != 1 {
|
||||
t.Errorf("Evaluate() = %v, want 1 (detail: %s)", got.Value, got.Detail)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCRun_CompileError(t *testing.T) {
|
||||
requireToolchain(t, "cc")
|
||||
e := CRun(cHarness, "21")
|
||||
response := "```c\nthis is not valid c code {{{\n```"
|
||||
got := e.Evaluate(context.Background(), response)
|
||||
if got.Value != 0 {
|
||||
t.Errorf("Evaluate() = %v, want 0 for compile error", got.Value)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecEvaluators_ToolchainMissingSkipsNotFails(t *testing.T) {
|
||||
// This test does not require any toolchain; it verifies the Skipped
|
||||
// contract directly against a bogus PATH so it runs in every environment.
|
||||
t.Setenv("PATH", t.TempDir())
|
||||
|
||||
goScore := GoRun(goHarness, "42").Evaluate(context.Background(), "```go\nfunc Double(n int) int { return n*2 }\n```")
|
||||
if !goScore.Skipped {
|
||||
t.Errorf("GoRun with empty PATH: Skipped = false, want true (%+v)", goScore)
|
||||
}
|
||||
|
||||
pyScore := PyRun(pyPassthroughHarness, "x").Evaluate(context.Background(), "```python\nprint('x')\n```")
|
||||
if !pyScore.Skipped {
|
||||
t.Errorf("PyRun with empty PATH: Skipped = false, want true (%+v)", pyScore)
|
||||
}
|
||||
|
||||
cScore := CRun(cHarness, "21").Evaluate(context.Background(), "```c\nint triple(int n){return n*3;}\n```")
|
||||
if !cScore.Skipped {
|
||||
t.Errorf("CRun with empty PATH: Skipped = false, want true (%+v)", cScore)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,122 @@
|
||||
package eval
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"regexp"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// fencePattern matches a fenced code block: ```lang\n...body...```
|
||||
var fencePattern = regexp.MustCompile("(?s)```([a-zA-Z0-9_+-]*)\\n?(.*?)```")
|
||||
|
||||
// ExtractCodeBlock returns the first fenced code block tagged with lang
|
||||
// (case-insensitive). If none is tagged, it falls back to the first
|
||||
// untagged fenced block, then to the first fenced block of any language,
|
||||
// then to the whole trimmed response if there are no fences at all.
|
||||
func ExtractCodeBlock(response, lang string) string {
|
||||
matches := fencePattern.FindAllStringSubmatch(response, -1)
|
||||
wantLang := strings.ToLower(strings.TrimSpace(lang))
|
||||
|
||||
var untagged string
|
||||
for _, m := range matches {
|
||||
blockLang := strings.ToLower(strings.TrimSpace(m[1]))
|
||||
if wantLang != "" && blockLang == wantLang {
|
||||
return strings.TrimSpace(m[2])
|
||||
}
|
||||
if untagged == "" && blockLang == "" {
|
||||
untagged = m[2]
|
||||
}
|
||||
}
|
||||
if untagged != "" {
|
||||
return strings.TrimSpace(untagged)
|
||||
}
|
||||
if len(matches) > 0 {
|
||||
return strings.TrimSpace(matches[0][2])
|
||||
}
|
||||
return strings.TrimSpace(response)
|
||||
}
|
||||
|
||||
// jsonFencePattern matches a fenced ```json ... ``` block specifically
|
||||
// (case-insensitive on the "json" language tag).
|
||||
var jsonFencePattern = regexp.MustCompile("(?is)```json\\s*\\n?(.*?)```")
|
||||
|
||||
// findAllBalancedJSON scans text for every top-level, bracket-balanced JSON
|
||||
// object/array literal, in the order they appear. Brackets inside string
|
||||
// literals are ignored. An unbalanced trailing fragment (an opening
|
||||
// bracket with no matching close) is not included.
|
||||
func findAllBalancedJSON(text string) []string {
|
||||
var out []string
|
||||
pos := 0
|
||||
for pos < len(text) {
|
||||
start := -1
|
||||
for i := pos; i < len(text); i++ {
|
||||
if text[i] == '{' || text[i] == '[' {
|
||||
start = i
|
||||
break
|
||||
}
|
||||
}
|
||||
if start == -1 {
|
||||
break
|
||||
}
|
||||
|
||||
depth := 0
|
||||
inString := false
|
||||
escaped := false
|
||||
end := -1
|
||||
for i := start; i < len(text); i++ {
|
||||
c := text[i]
|
||||
if inString {
|
||||
switch {
|
||||
case escaped:
|
||||
escaped = false
|
||||
case c == '\\':
|
||||
escaped = true
|
||||
case c == '"':
|
||||
inString = false
|
||||
}
|
||||
continue
|
||||
}
|
||||
switch c {
|
||||
case '"':
|
||||
inString = true
|
||||
case '{', '[':
|
||||
depth++
|
||||
case '}', ']':
|
||||
depth--
|
||||
if depth == 0 {
|
||||
end = i
|
||||
}
|
||||
}
|
||||
if end != -1 {
|
||||
break
|
||||
}
|
||||
}
|
||||
if end == -1 {
|
||||
break
|
||||
}
|
||||
out = append(out, text[start:end+1])
|
||||
pos = end + 1
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// ExtractJSON returns response's intended JSON object/array literal,
|
||||
// tolerant of surrounding prose and markdown code fences. It prefers the
|
||||
// content of the first ```json fenced block, if any; otherwise it takes
|
||||
// the LAST balanced JSON value anywhere in response. Taking the last
|
||||
// (rather than the first) value matters for a model that thinks out loud
|
||||
// with a candidate JSON draft before committing to a final answer: the
|
||||
// final JSON, not the draft, is the one to score.
|
||||
func ExtractJSON(response string) (string, error) {
|
||||
if m := jsonFencePattern.FindStringSubmatch(response); m != nil {
|
||||
if all := findAllBalancedJSON(m[1]); len(all) > 0 {
|
||||
return all[0], nil
|
||||
}
|
||||
}
|
||||
|
||||
all := findAllBalancedJSON(response)
|
||||
if len(all) == 0 {
|
||||
return "", fmt.Errorf("no JSON object or array found in response")
|
||||
}
|
||||
return all[len(all)-1], nil
|
||||
}
|
||||
@@ -0,0 +1,51 @@
|
||||
package eval
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestExtractCodeBlock(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
lang string
|
||||
want string
|
||||
}{
|
||||
{
|
||||
name: "tagged block matches lang",
|
||||
response: "Here:\n```go\nfunc F() {}\n```\nDone.",
|
||||
lang: "go",
|
||||
want: "func F() {}",
|
||||
},
|
||||
{
|
||||
name: "case insensitive lang tag",
|
||||
response: "```GO\nfunc F() {}\n```",
|
||||
lang: "go",
|
||||
want: "func F() {}",
|
||||
},
|
||||
{
|
||||
name: "prefers matching lang over other tagged blocks",
|
||||
response: "```json\n{\"x\":1}\n```\n```go\nfunc F() {}\n```",
|
||||
lang: "go",
|
||||
want: "func F() {}",
|
||||
},
|
||||
{
|
||||
name: "falls back to untagged block",
|
||||
response: "```\nfunc F() {}\n```",
|
||||
lang: "go",
|
||||
want: "func F() {}",
|
||||
},
|
||||
{
|
||||
name: "no fences falls back to whole response",
|
||||
response: " func F() {} ",
|
||||
lang: "go",
|
||||
want: "func F() {}",
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := ExtractCodeBlock(tt.response, tt.lang)
|
||||
if got != tt.want {
|
||||
t.Errorf("ExtractCodeBlock(%q, %q) = %q, want %q", tt.response, tt.lang, got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,107 @@
|
||||
package eval
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// decodeStringArray extracts the first JSON value from response and decodes
|
||||
// it as a []string. Elements that are not strings cause an error.
|
||||
func decodeStringArray(response string) ([]string, error) {
|
||||
raw, err := ExtractJSON(response)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var arr []string
|
||||
if err := json.Unmarshal([]byte(raw), &arr); err != nil {
|
||||
return nil, fmt.Errorf("invalid JSON string array: %w", err)
|
||||
}
|
||||
return arr, nil
|
||||
}
|
||||
|
||||
// jsonStringSetEval scores a JSON array of strings as an unordered set,
|
||||
// case-insensitive and whitespace-trimmed.
|
||||
type jsonStringSetEval struct {
|
||||
want []string
|
||||
}
|
||||
|
||||
// JSONStringSet returns an Evaluator that parses the first JSON array in the
|
||||
// response as a set of strings (case-insensitive, trimmed) and compares it
|
||||
// to want, order-independent. Score is the Jaccard-style overlap
|
||||
// |intersection| / max(len(want), len(got)), so both missing and extraneous
|
||||
// entries cost credit; full credit requires an exact set match.
|
||||
func JSONStringSet(want []string) Evaluator {
|
||||
return jsonStringSetEval{want: want}
|
||||
}
|
||||
|
||||
func normalizeSet(items []string) map[string]struct{} {
|
||||
set := make(map[string]struct{}, len(items))
|
||||
for _, s := range items {
|
||||
set[strings.ToLower(strings.TrimSpace(s))] = struct{}{}
|
||||
}
|
||||
return set
|
||||
}
|
||||
|
||||
func (j jsonStringSetEval) Evaluate(_ context.Context, response string) Score {
|
||||
got, err := decodeStringArray(response)
|
||||
if err != nil {
|
||||
return Score{Value: 0, Detail: err.Error()}
|
||||
}
|
||||
|
||||
wantSet := normalizeSet(j.want)
|
||||
gotSet := normalizeSet(got)
|
||||
|
||||
matched := 0
|
||||
for k := range gotSet {
|
||||
if _, ok := wantSet[k]; ok {
|
||||
matched++
|
||||
}
|
||||
}
|
||||
|
||||
denom := len(wantSet)
|
||||
if len(gotSet) > denom {
|
||||
denom = len(gotSet)
|
||||
}
|
||||
if denom == 0 {
|
||||
return Score{Value: 1, Detail: "both sets empty"}
|
||||
}
|
||||
|
||||
value := float64(matched) / float64(denom)
|
||||
return Score{
|
||||
Value: value,
|
||||
Detail: fmt.Sprintf("matched %d/%d (got %v, want %v)", matched, denom, got, j.want),
|
||||
}
|
||||
}
|
||||
|
||||
// jsonStringArrayEqualsEval scores full credit only on an exact, in-order
|
||||
// match of a JSON string array.
|
||||
type jsonStringArrayEqualsEval struct {
|
||||
want []string
|
||||
}
|
||||
|
||||
// JSONStringArrayEquals returns an Evaluator that parses the first JSON
|
||||
// array in the response as []string and awards full credit only when it
|
||||
// exactly equals want element-for-element, in order (case-insensitive,
|
||||
// trimmed). Any mismatch, including reordering, scores zero.
|
||||
func JSONStringArrayEquals(want []string) Evaluator {
|
||||
return jsonStringArrayEqualsEval{want: want}
|
||||
}
|
||||
|
||||
func (j jsonStringArrayEqualsEval) Evaluate(_ context.Context, response string) Score {
|
||||
got, err := decodeStringArray(response)
|
||||
if err != nil {
|
||||
return Score{Value: 0, Detail: err.Error()}
|
||||
}
|
||||
|
||||
if len(got) != len(j.want) {
|
||||
return Score{Value: 0, Detail: fmt.Sprintf("length %d, want %d (got %v, want %v)", len(got), len(j.want), got, j.want)}
|
||||
}
|
||||
for i := range got {
|
||||
if !strings.EqualFold(strings.TrimSpace(got[i]), strings.TrimSpace(j.want[i])) {
|
||||
return Score{Value: 0, Detail: fmt.Sprintf("index %d: got %q, want %q (got %v, want %v)", i, got[i], j.want[i], got, j.want)}
|
||||
}
|
||||
}
|
||||
return Score{Value: 1, Detail: fmt.Sprintf("exact match: %v", got)}
|
||||
}
|
||||
@@ -0,0 +1,184 @@
|
||||
package eval
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// pathSegment is one step of a dot/bracket JSON path, e.g. "a.b[0]" is
|
||||
// [{key:"a"} {key:"b"} {index:0, isIndex:true}].
|
||||
type pathSegment struct {
|
||||
key string
|
||||
index int
|
||||
isIndex bool
|
||||
}
|
||||
|
||||
// parsePath splits a path like "a.b[0].c" into segments. An empty path
|
||||
// yields no segments (navigate returns the root value unchanged).
|
||||
func parsePath(path string) ([]pathSegment, error) {
|
||||
var segments []pathSegment
|
||||
if path == "" {
|
||||
return segments, nil
|
||||
}
|
||||
for _, rawPart := range strings.Split(path, ".") {
|
||||
part := rawPart
|
||||
for part != "" {
|
||||
if idx := strings.IndexByte(part, '['); idx >= 0 {
|
||||
if idx > 0 {
|
||||
segments = append(segments, pathSegment{key: part[:idx]})
|
||||
}
|
||||
end := strings.IndexByte(part, ']')
|
||||
if end < idx {
|
||||
return nil, fmt.Errorf("malformed path %q: unmatched '['", path)
|
||||
}
|
||||
n, err := strconv.Atoi(part[idx+1 : end])
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("malformed path %q: bad index: %w", path, err)
|
||||
}
|
||||
segments = append(segments, pathSegment{index: n, isIndex: true})
|
||||
part = part[end+1:]
|
||||
continue
|
||||
}
|
||||
segments = append(segments, pathSegment{key: part})
|
||||
part = ""
|
||||
}
|
||||
}
|
||||
return segments, nil
|
||||
}
|
||||
|
||||
// navigate walks v (as decoded by encoding/json into any) following segments.
|
||||
func navigate(v any, segments []pathSegment) (any, error) {
|
||||
cur := v
|
||||
for _, seg := range segments {
|
||||
if seg.isIndex {
|
||||
arr, ok := cur.([]any)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("expected array at index [%d], got %T", seg.index, cur)
|
||||
}
|
||||
if seg.index < 0 || seg.index >= len(arr) {
|
||||
return nil, fmt.Errorf("index [%d] out of range (len %d)", seg.index, len(arr))
|
||||
}
|
||||
cur = arr[seg.index]
|
||||
continue
|
||||
}
|
||||
m, ok := cur.(map[string]any)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("expected object at field %q, got %T", seg.key, cur)
|
||||
}
|
||||
val, ok := m[seg.key]
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("field %q not found", seg.key)
|
||||
}
|
||||
cur = val
|
||||
}
|
||||
return cur, nil
|
||||
}
|
||||
|
||||
// jsonValueAs converts a decoded JSON value (string/float64/bool/etc.) to T,
|
||||
// dispatching on the runtime type of the zero value of T.
|
||||
func jsonValueAs[T comparable](v any) (T, error) {
|
||||
var zero T
|
||||
switch any(zero).(type) {
|
||||
case string:
|
||||
s, ok := v.(string)
|
||||
if !ok {
|
||||
return zero, fmt.Errorf("expected string, got %T (%v)", v, v)
|
||||
}
|
||||
return any(s).(T), nil
|
||||
case bool:
|
||||
b, ok := v.(bool)
|
||||
if !ok {
|
||||
return zero, fmt.Errorf("expected bool, got %T (%v)", v, v)
|
||||
}
|
||||
return any(b).(T), nil
|
||||
case float64:
|
||||
f, err := asFloat64(v)
|
||||
if err != nil {
|
||||
return zero, err
|
||||
}
|
||||
return any(f).(T), nil
|
||||
case int:
|
||||
f, err := asFloat64(v)
|
||||
if err != nil {
|
||||
return zero, err
|
||||
}
|
||||
return any(int(f)).(T), nil
|
||||
default:
|
||||
return zero, fmt.Errorf("unsupported JSONField target type %T", zero)
|
||||
}
|
||||
}
|
||||
|
||||
func asFloat64(v any) (float64, error) {
|
||||
switch n := v.(type) {
|
||||
case float64:
|
||||
return n, nil
|
||||
case json.Number:
|
||||
return n.Float64()
|
||||
default:
|
||||
return 0, fmt.Errorf("expected number, got %T (%v)", v, v)
|
||||
}
|
||||
}
|
||||
|
||||
// jsonFieldEval extracts one field by path from the first JSON value in the
|
||||
// response and compares it to want.
|
||||
type jsonFieldEval[T comparable] struct {
|
||||
want T
|
||||
path string
|
||||
}
|
||||
|
||||
// JSONField returns an Evaluator that extracts the first JSON object/array
|
||||
// in the response (stripping surrounding prose and code fences), navigates
|
||||
// path (dot/bracket syntax, e.g. "bump" or "steps[0]"), and awards full
|
||||
// credit when the field equals want. T is inferred from want; supported
|
||||
// types are string, bool, float64, and int. String comparison is
|
||||
// case-insensitive and trims whitespace.
|
||||
func JSONField[T comparable](path string, want T) Evaluator {
|
||||
return jsonFieldEval[T]{path: path, want: want}
|
||||
}
|
||||
|
||||
func (j jsonFieldEval[T]) Evaluate(_ context.Context, response string) Score {
|
||||
raw, err := ExtractJSON(response)
|
||||
if err != nil {
|
||||
return Score{Value: 0, Detail: err.Error()}
|
||||
}
|
||||
|
||||
dec := json.NewDecoder(strings.NewReader(raw))
|
||||
dec.UseNumber()
|
||||
var parsed any
|
||||
if decodeErr := dec.Decode(&parsed); decodeErr != nil {
|
||||
return Score{Value: 0, Detail: fmt.Sprintf("invalid JSON: %v", decodeErr)}
|
||||
}
|
||||
|
||||
segments, err := parsePath(j.path)
|
||||
if err != nil {
|
||||
return Score{Value: 0, Detail: err.Error()}
|
||||
}
|
||||
|
||||
val, err := navigate(parsed, segments)
|
||||
if err != nil {
|
||||
return Score{Value: 0, Detail: fmt.Sprintf("path %q: %v", j.path, err)}
|
||||
}
|
||||
|
||||
got, err := jsonValueAs[T](val)
|
||||
if err != nil {
|
||||
return Score{Value: 0, Detail: fmt.Sprintf("path %q: %v", j.path, err)}
|
||||
}
|
||||
|
||||
if equalJSONValue(got, j.want) {
|
||||
return Score{Value: 1, Detail: fmt.Sprintf("%s = %v", j.path, got)}
|
||||
}
|
||||
return Score{Value: 0, Detail: fmt.Sprintf("%s: got %v, want %v", j.path, got, j.want)}
|
||||
}
|
||||
|
||||
// equalJSONValue compares two values of the same comparable type T, folding
|
||||
// case and whitespace when T is string.
|
||||
func equalJSONValue[T comparable](got, want T) bool {
|
||||
if gs, ok := any(got).(string); ok {
|
||||
ws := any(want).(string)
|
||||
return strings.EqualFold(strings.TrimSpace(gs), strings.TrimSpace(ws))
|
||||
}
|
||||
return got == want
|
||||
}
|
||||
@@ -0,0 +1,161 @@
|
||||
package eval
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestExtractJSON(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want string
|
||||
wantErr bool
|
||||
}{
|
||||
{"plain object", `{"bump":"minor"}`, `{"bump":"minor"}`, false},
|
||||
{"fenced object", "Here you go:\n```json\n{\"bump\":\"minor\"}\n```", `{"bump":"minor"}`, false},
|
||||
{"object with prose around", `The answer is {"bump":"minor"} as computed.`, `{"bump":"minor"}`, false},
|
||||
{"nested braces ignored inside strings", `{"note":"a { b } c","bump":"patch"}`, `{"note":"a { b } c","bump":"patch"}`, false},
|
||||
{"array", `["ClaudeBot","Googlebot"]`, `["ClaudeBot","Googlebot"]`, false},
|
||||
{"no json", "no json here at all", "", true},
|
||||
// S7: a candidate JSON considered during reasoning, followed by the
|
||||
// real final JSON, must resolve to the final one.
|
||||
{
|
||||
name: "reasoning with a candidate JSON then a final JSON: final wins",
|
||||
response: `Let me think. One option would be {"bump":"major"} but that is not right. The actual answer is {"bump":"minor"}.`,
|
||||
want: `{"bump":"minor"}`,
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "fenced json block preferred over any other json-looking text before it",
|
||||
response: "Draft: {\"bump\":\"major\"}\n\nFinal answer:\n```json\n{\"bump\":\"minor\"}\n```",
|
||||
want: `{"bump":"minor"}`,
|
||||
wantErr: false,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got, err := ExtractJSON(tt.response)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Fatalf("ExtractJSON(%q) error = %v, wantErr %v", tt.response, err, tt.wantErr)
|
||||
}
|
||||
if err == nil && got != tt.want {
|
||||
t.Errorf("ExtractJSON(%q) = %q, want %q", tt.response, got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestJSONField_String(t *testing.T) {
|
||||
e := JSONField("bump", "minor")
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{"exact match", `{"bump":"minor"}`, 1},
|
||||
{"case insensitive", `{"bump":"MINOR"}`, 1},
|
||||
{"wrong value", `{"bump":"patch"}`, 0},
|
||||
{"missing field", `{"other":"x"}`, 0},
|
||||
{"fenced json", "```json\n{\"bump\": \"minor\"}\n```", 1},
|
||||
{"invalid json", "not json at all", 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := e.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestJSONField_Int(t *testing.T) {
|
||||
e := JSONField("M", 16)
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{"exact match", `{"M":16,"efConstruction":200,"efSearch":50}`, 1},
|
||||
{"wrong value", `{"M":32}`, 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := e.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestJSONField_NestedPath(t *testing.T) {
|
||||
e := JSONField("task1", "search_web")
|
||||
got := e.Evaluate(context.Background(), `{"task1":"search_web","task2":"read_file"}`)
|
||||
if got.Value != 1 {
|
||||
t.Errorf("Evaluate() = %v, want 1 (detail: %s)", got.Value, got.Detail)
|
||||
}
|
||||
}
|
||||
|
||||
func TestJSONField_IndexedPath(t *testing.T) {
|
||||
e := JSONField("steps[0]", "build")
|
||||
got := e.Evaluate(context.Background(), `{"steps":["build","test","deploy"]}`)
|
||||
if got.Value != 1 {
|
||||
t.Errorf("Evaluate() = %v, want 1 (detail: %s)", got.Value, got.Detail)
|
||||
}
|
||||
got = e.Evaluate(context.Background(), `{"steps":["test","build","deploy"]}`)
|
||||
if got.Value != 0 {
|
||||
t.Errorf("Evaluate() = %v, want 0 (detail: %s)", got.Value, got.Detail)
|
||||
}
|
||||
}
|
||||
|
||||
func TestJSONStringSet(t *testing.T) {
|
||||
e := JSONStringSet([]string{"ClaudeBot", "Googlebot"})
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{"exact set", `["ClaudeBot","Googlebot"]`, 1},
|
||||
{"exact set different order", `["Googlebot","ClaudeBot"]`, 1},
|
||||
{"case insensitive", `["claudebot","googlebot"]`, 1},
|
||||
{"missing one", `["ClaudeBot"]`, 0.5},
|
||||
{"extra one", `["ClaudeBot","Googlebot","GPTBot"]`, float64(2) / 3},
|
||||
{"completely wrong", `["GPTBot","PerplexityBot"]`, 0},
|
||||
{"invalid json", "no array here", 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := e.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestJSONStringArrayEquals(t *testing.T) {
|
||||
want := []string{"build", "test", "backup", "deploy", "verify"}
|
||||
e := JSONStringArrayEquals(want)
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{"exact order", `["build","test","backup","deploy","verify"]`, 1},
|
||||
{"case insensitive exact order", `["BUILD","TEST","BACKUP","DEPLOY","VERIFY"]`, 1},
|
||||
{"wrong order", `["build","backup","test","deploy","verify"]`, 0},
|
||||
{"wrong length", `["build","test"]`, 0},
|
||||
{"invalid json", "not an array", 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := e.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,117 @@
|
||||
package eval
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Number is the set of numeric primitive kinds Numeric can extract and
|
||||
// compare. Defined locally (rather than importing golang.org/x/exp/constraints)
|
||||
// since only gopkg.in/yaml.v3 and golang.org/x/sync/errgroup are allowed deps.
|
||||
type Number interface {
|
||||
~float32 | ~float64 |
|
||||
~int | ~int8 | ~int16 | ~int32 | ~int64 |
|
||||
~uint | ~uint8 | ~uint16 | ~uint32 | ~uint64
|
||||
}
|
||||
|
||||
// numberPattern matches signed integers and decimals, including a
|
||||
// leading-dot form with no integer part (".9896"). Longer alternatives are
|
||||
// listed first so the match is not truncated to just the integer part of
|
||||
// "0.9896" or just the dot-less ".9896".
|
||||
var numberPattern = regexp.MustCompile(`-?(?:\d+\.\d+|\.\d+|\d+)`)
|
||||
|
||||
// hyphenCompoundSuffixPattern matches a hyphen directly after a number
|
||||
// joining it to a following word, e.g. the "-bit" in "64-bit" or the
|
||||
// "-byte" in "8-byte". A number immediately followed by this is part of a
|
||||
// unit/architecture descriptor, not a standalone answer value, and is
|
||||
// skipped when picking "the" number out of a line - otherwise "sizeof(...)
|
||||
// is 24 bytes on a 64-bit system" would extract the trailing 64 instead of
|
||||
// the actual answer, 24.
|
||||
var hyphenCompoundSuffixPattern = regexp.MustCompile(`^-\w`)
|
||||
|
||||
// lastRelevantNumber returns the last number in text that is not part of a
|
||||
// "<number>-word" compound, scanning from the end.
|
||||
func lastRelevantNumber(text string) (string, bool) {
|
||||
matches := numberPattern.FindAllStringIndex(text, -1)
|
||||
for i := len(matches) - 1; i >= 0; i-- {
|
||||
start, end := matches[i][0], matches[i][1]
|
||||
if hyphenCompoundSuffixPattern.MatchString(text[end:]) {
|
||||
continue
|
||||
}
|
||||
return text[start:end], true
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
|
||||
// ExtractLastNumber returns the model's answer number from s, converted to
|
||||
// T. It first looks only at the last non-empty line (the common case: a
|
||||
// final "24" or "0.9896" on its own line after any reasoning), and only
|
||||
// falls back to the last number anywhere in s if that line has no usable
|
||||
// number at all. Within whichever text it searches, it skips a number that
|
||||
// is part of a "<number>-word" compound (see hyphenCompoundSuffixPattern):
|
||||
// "sizeof(struct Config) is 24 bytes on a 64-bit system" must extract 24,
|
||||
// not the later, unrelated 64. It is the default extractor for Numeric.
|
||||
func ExtractLastNumber[T Number](s string) (T, error) {
|
||||
lines := strings.Split(s, "\n")
|
||||
for i := len(lines) - 1; i >= 0; i-- {
|
||||
line := strings.TrimSpace(lines[i])
|
||||
if line == "" {
|
||||
continue
|
||||
}
|
||||
if lit, ok := lastRelevantNumber(line); ok {
|
||||
return parseNumber[T](lit)
|
||||
}
|
||||
break
|
||||
}
|
||||
|
||||
if lit, ok := lastRelevantNumber(s); ok {
|
||||
return parseNumber[T](lit)
|
||||
}
|
||||
return 0, fmt.Errorf("no number found in response")
|
||||
}
|
||||
|
||||
// parseNumber converts a numberPattern match (possibly ".9896", with no
|
||||
// leading zero) to T.
|
||||
func parseNumber[T Number](literal string) (T, error) {
|
||||
f, err := strconv.ParseFloat(literal, 64)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("parse number %q: %w", literal, err)
|
||||
}
|
||||
return T(f), nil
|
||||
}
|
||||
|
||||
// numericEval scores full credit when the extracted number is within tol of
|
||||
// want, zero otherwise.
|
||||
type numericEval[T Number] struct {
|
||||
extract func(string) (T, error)
|
||||
want T
|
||||
tol T
|
||||
}
|
||||
|
||||
// Numeric returns an Evaluator that extracts a value of type T from the
|
||||
// response via extract and awards full credit when |got-want| <= tol.
|
||||
func Numeric[T Number](extract func(string) (T, error), want, tol T) Evaluator {
|
||||
return numericEval[T]{extract: extract, want: want, tol: tol}
|
||||
}
|
||||
|
||||
func (n numericEval[T]) Evaluate(_ context.Context, response string) Score {
|
||||
got, err := n.extract(response)
|
||||
if err != nil {
|
||||
return Score{Value: 0, Detail: fmt.Sprintf("extract failed: %v", err)}
|
||||
}
|
||||
|
||||
var diff T
|
||||
if got >= n.want {
|
||||
diff = got - n.want
|
||||
} else {
|
||||
diff = n.want - got
|
||||
}
|
||||
|
||||
if diff <= n.tol {
|
||||
return Score{Value: 1, Detail: fmt.Sprintf("got %v, want %v (tol %v)", got, n.want, n.tol)}
|
||||
}
|
||||
return Score{Value: 0, Detail: fmt.Sprintf("got %v, want %v (tol %v)", got, n.want, n.tol)}
|
||||
}
|
||||
@@ -0,0 +1,96 @@
|
||||
package eval
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestExtractLastNumber(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
wantErr bool
|
||||
}{
|
||||
{"plain number", "0.8785", 0.8785, false},
|
||||
{"trailing number after prose", "The cosine similarity is 0.8785", 0.8785, false},
|
||||
{"last of multiple numbers", "step 1 then step 2 result is 42", 42, false},
|
||||
{"negative number", "the delta is -3.5", -3.5, false},
|
||||
{"no number", "no digits here", 0, true},
|
||||
// S6 regressions.
|
||||
{"unit qualifier after the answer is skipped", "24 bytes on a 64-bit system", 24, false},
|
||||
{"leading-dot decimal", "= .9896", 0.9896, false},
|
||||
{"answer on last line, reasoning with numbers on earlier lines", "step 1 uses 8 items\nstep 2 uses 16 items\nfinal answer: 42", 42, false},
|
||||
{"last line has only a unit-qualified number, falls back to whole text", "the computation used 100 iterations\nresult on a 64-bit system", 100, false},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got, err := ExtractLastNumber[float64](tt.response)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Fatalf("ExtractLastNumber(%q) error = %v, wantErr %v", tt.response, err, tt.wantErr)
|
||||
}
|
||||
if err == nil && got != tt.want {
|
||||
t.Errorf("ExtractLastNumber(%q) = %v, want %v", tt.response, got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestNumeric_Float(t *testing.T) {
|
||||
e := Numeric(ExtractLastNumber[float64], 0.8785, 0.0005)
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{"exact", "0.8785", 1},
|
||||
{"within tolerance above", "0.8789", 1},
|
||||
{"within tolerance below", "0.8781", 1},
|
||||
{"outside tolerance", "0.9000", 0},
|
||||
{"unparsable", "no number here", 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := e.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestNumeric_IntExact(t *testing.T) {
|
||||
e := Numeric(ExtractLastNumber[int], 32, 0)
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{"exact match", "32", 1},
|
||||
{"off by one", "33", 0},
|
||||
{"exact match with prose", "The sizeof value is 32 bytes", 1},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := e.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestNumeric_UnsignedNoUnderflow(t *testing.T) {
|
||||
// Regression: unsigned subtraction must not wrap when got < want.
|
||||
e := Numeric(ExtractLastNumber[uint], 10, 2)
|
||||
got := e.Evaluate(context.Background(), "9")
|
||||
if got.Value != 1 {
|
||||
t.Errorf("Evaluate(9) with want=10 tol=2 = %v, want 1 (detail: %s)", got.Value, got.Detail)
|
||||
}
|
||||
got = e.Evaluate(context.Background(), "5")
|
||||
if got.Value != 0 {
|
||||
t.Errorf("Evaluate(5) with want=10 tol=2 = %v, want 0 (detail: %s)", got.Value, got.Detail)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
package eval
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"regexp"
|
||||
)
|
||||
|
||||
// regexEval scores full credit when pattern matches the response.
|
||||
type regexEval struct {
|
||||
pattern *regexp.Regexp
|
||||
raw string
|
||||
}
|
||||
|
||||
// Regex returns an Evaluator awarding full credit when pattern matches
|
||||
// anywhere in the response, zero otherwise. pattern must be a valid RE2
|
||||
// expression; it panics on an invalid pattern since evaluators are built
|
||||
// once at catalog registration time, not per-request.
|
||||
func Regex(pattern string) Evaluator {
|
||||
re := regexp.MustCompile(pattern)
|
||||
return regexEval{pattern: re, raw: pattern}
|
||||
}
|
||||
|
||||
func (r regexEval) Evaluate(_ context.Context, response string) Score {
|
||||
if r.pattern.MatchString(response) {
|
||||
return Score{Value: 1, Detail: fmt.Sprintf("matched /%s/", r.raw)}
|
||||
}
|
||||
return Score{Value: 0, Detail: fmt.Sprintf("no match for /%s/", r.raw)}
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
package eval
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestRegex(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
pattern string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{"matches", `ssh\s+root@10\.0\.0\.100`, "run: ssh [email protected] 'pct exec 251 -- systemctl status myservice'", 1},
|
||||
{"no match", `ssh\s+root@10\.0\.0\.100`, "ssh [email protected]", 0},
|
||||
{"anchored word boundary", `\bpct exec 251\b`, "use pct exec 251 -- cmd", 1},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := Regex(tt.pattern).Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Regex(%q).Evaluate(%q) = %v, want %v", tt.pattern, tt.response, got.Value, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRegex_PanicsOnInvalidPattern(t *testing.T) {
|
||||
defer func() {
|
||||
if r := recover(); r == nil {
|
||||
t.Fatal("Regex() with invalid pattern did not panic")
|
||||
}
|
||||
}()
|
||||
Regex(`(unclosed`)
|
||||
}
|
||||
Reference in New Issue
Block a user