mirror of
https://github.com/lukaszraczylo/llm-testbench.git
synced 2026-10-14 00:57:45 +00:00
feat: LLM accuracy testing framework with 17-test catalog
Framework runs a category/subcategory test catalog against multiple LLMs through an OpenAI-compatible endpoint. Deterministic evaluators score each response. Reports render as table, markdown, or JSON.
This commit is contained in:
@@ -0,0 +1,60 @@
|
||||
package report
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"io"
|
||||
|
||||
"github.com/lukaszraczylo/llm-testbench/internal/runner"
|
||||
"github.com/lukaszraczylo/llm-testbench/internal/testkit"
|
||||
)
|
||||
|
||||
// jsonResult is the JSON-serializable projection of a runner.Result, joined
|
||||
// with its test's category/subcategory for self-contained output.
|
||||
type jsonResult struct {
|
||||
Model string `json:"model"`
|
||||
TestID string `json:"test_id"`
|
||||
Category string `json:"category,omitempty"`
|
||||
Subcategory string `json:"subcategory,omitempty"`
|
||||
Detail string `json:"detail,omitempty"`
|
||||
Error string `json:"error,omitempty"`
|
||||
Score float64 `json:"score"`
|
||||
LatencyMS int64 `json:"latency_ms"`
|
||||
PromptTokens int `json:"prompt_tokens"`
|
||||
CompletionTokens int `json:"completion_tokens"`
|
||||
Tokens int `json:"tokens"`
|
||||
Skipped bool `json:"skipped"`
|
||||
}
|
||||
|
||||
// renderJSON dumps raw results as a JSON array, joined with test metadata.
|
||||
func renderJSON(w io.Writer, tests []testkit.Test, results []runner.Result) error {
|
||||
testByID := make(map[string]testkit.Test, len(tests))
|
||||
for _, t := range tests {
|
||||
testByID[t.ID] = t
|
||||
}
|
||||
|
||||
out := make([]jsonResult, 0, len(results))
|
||||
for _, r := range results {
|
||||
t := testByID[r.TestID]
|
||||
jr := jsonResult{
|
||||
Model: r.Model,
|
||||
TestID: r.TestID,
|
||||
Category: t.Category,
|
||||
Subcategory: t.Subcategory,
|
||||
Score: r.Score.Value,
|
||||
Detail: r.Score.Detail,
|
||||
Skipped: r.Score.Skipped,
|
||||
LatencyMS: r.Latency.Milliseconds(),
|
||||
PromptTokens: r.PromptTokens,
|
||||
CompletionTokens: r.CompletionTokens,
|
||||
Tokens: r.TotalTokens(),
|
||||
}
|
||||
if r.Err != nil {
|
||||
jr.Error = r.Err.Error()
|
||||
}
|
||||
out = append(out, jr)
|
||||
}
|
||||
|
||||
enc := json.NewEncoder(w)
|
||||
enc.SetIndent("", " ")
|
||||
return enc.Encode(out)
|
||||
}
|
||||
@@ -0,0 +1,77 @@
|
||||
package report
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"math"
|
||||
"strings"
|
||||
|
||||
"github.com/lukaszraczylo/llm-testbench/internal/runner"
|
||||
"github.com/lukaszraczylo/llm-testbench/internal/testkit"
|
||||
)
|
||||
|
||||
// renderMarkdown writes the same three tables as renderTable, formatted as
|
||||
// GitHub-flavored markdown tables.
|
||||
func renderMarkdown(w io.Writer, tests []testkit.Test, models []string, results []runner.Result) error {
|
||||
idx := indexResults(results)
|
||||
ew := &errWriter{w: w}
|
||||
|
||||
ew.println("## Per-test scores")
|
||||
ew.println()
|
||||
scoreHeader := append([]string{"Test", "Category", "Subcategory"}, models...)
|
||||
writeMarkdownHeader(ew, scoreHeader)
|
||||
for _, t := range sortedTests(tests) {
|
||||
row := []string{t.ID, t.Category, t.Subcategory}
|
||||
for _, model := range models {
|
||||
r, ok := idx[t.ID][model]
|
||||
row = append(row, cellText(r, ok))
|
||||
}
|
||||
writeMarkdownRow(ew, row)
|
||||
}
|
||||
|
||||
ew.println()
|
||||
ew.println("## Category rollup (mean score)")
|
||||
ew.println()
|
||||
catHeader := append([]string{"Category"}, models...)
|
||||
writeMarkdownHeader(ew, catHeader)
|
||||
for _, cm := range categoryRollup(tests, models, idx) {
|
||||
row := []string{cm.category}
|
||||
for _, model := range models {
|
||||
row = append(row, meanCellText(cm.means[model]))
|
||||
}
|
||||
writeMarkdownRow(ew, row)
|
||||
}
|
||||
|
||||
ew.println()
|
||||
ew.println("## Model summary")
|
||||
ew.println()
|
||||
writeMarkdownHeader(ew, []string{"Model", "Mean", "Passed", "Partial", "Failed", "Errors", "Mean latency", "Total tokens", "Tok/s"})
|
||||
for _, ms := range summarize(models, results) {
|
||||
writeMarkdownRow(ew, []string{
|
||||
ms.model,
|
||||
meanCellText(ms.overallMean),
|
||||
fmt.Sprintf("%d", ms.passed),
|
||||
fmt.Sprintf("%d", ms.partial),
|
||||
fmt.Sprintf("%d", ms.failed),
|
||||
fmt.Sprintf("%d", ms.errors),
|
||||
durationText(ms.meanLatency, !math.IsNaN(ms.overallMean)),
|
||||
fmt.Sprintf("%d", ms.totalTokens),
|
||||
rateText(ms.avgTokPerSec),
|
||||
})
|
||||
}
|
||||
|
||||
return ew.err
|
||||
}
|
||||
|
||||
func writeMarkdownHeader(ew *errWriter, cols []string) {
|
||||
ew.println("| " + strings.Join(cols, " | ") + " |")
|
||||
sep := make([]string, len(cols))
|
||||
for i := range sep {
|
||||
sep[i] = "---"
|
||||
}
|
||||
ew.println("| " + strings.Join(sep, " | ") + " |")
|
||||
}
|
||||
|
||||
func writeMarkdownRow(ew *errWriter, cols []string) {
|
||||
ew.println("| " + strings.Join(cols, " | ") + " |")
|
||||
}
|
||||
@@ -0,0 +1,238 @@
|
||||
// Package report renders runner.Result sets as a comparison table, GitHub
|
||||
// markdown tables, or raw JSON.
|
||||
package report
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"math"
|
||||
"sort"
|
||||
"time"
|
||||
|
||||
"github.com/lukaszraczylo/llm-testbench/internal/runner"
|
||||
"github.com/lukaszraczylo/llm-testbench/internal/testkit"
|
||||
)
|
||||
|
||||
// Format selects the output renderer.
|
||||
type Format string
|
||||
|
||||
// Supported Format values.
|
||||
const (
|
||||
FormatTable Format = "table"
|
||||
FormatMarkdown Format = "markdown"
|
||||
FormatJSON Format = "json"
|
||||
)
|
||||
|
||||
// passThreshold is the minimum score counted as a full pass in the model
|
||||
// summary table.
|
||||
const passThreshold = 0.99
|
||||
|
||||
// Render writes tests/models/results to w in the requested format.
|
||||
func Render(w io.Writer, format Format, tests []testkit.Test, models []string, results []runner.Result) error {
|
||||
switch format {
|
||||
case FormatTable:
|
||||
return renderTable(w, tests, models, results)
|
||||
case FormatMarkdown:
|
||||
return renderMarkdown(w, tests, models, results)
|
||||
case FormatJSON:
|
||||
return renderJSON(w, tests, results)
|
||||
default:
|
||||
return fmt.Errorf("report: unknown format %q", format)
|
||||
}
|
||||
}
|
||||
|
||||
// resultIndex looks up a Result by testID then model.
|
||||
type resultIndex map[string]map[string]runner.Result
|
||||
|
||||
func indexResults(results []runner.Result) resultIndex {
|
||||
idx := make(resultIndex)
|
||||
for _, r := range results {
|
||||
if idx[r.TestID] == nil {
|
||||
idx[r.TestID] = make(map[string]runner.Result)
|
||||
}
|
||||
idx[r.TestID][r.Model] = r
|
||||
}
|
||||
return idx
|
||||
}
|
||||
|
||||
// sortedTests returns tests ordered by Category, Subcategory, then ID, for
|
||||
// deterministic, human-grouped table output.
|
||||
func sortedTests(tests []testkit.Test) []testkit.Test {
|
||||
out := make([]testkit.Test, len(tests))
|
||||
copy(out, tests)
|
||||
sort.Slice(out, func(i, j int) bool {
|
||||
if out[i].Category != out[j].Category {
|
||||
return out[i].Category < out[j].Category
|
||||
}
|
||||
if out[i].Subcategory != out[j].Subcategory {
|
||||
return out[i].Subcategory < out[j].Subcategory
|
||||
}
|
||||
return out[i].ID < out[j].ID
|
||||
})
|
||||
return out
|
||||
}
|
||||
|
||||
// cellText renders one (test, model) cell for the score table: "score
|
||||
// (Ntok)" for a scored result, or ERR/skip/N/A otherwise.
|
||||
func cellText(r runner.Result, found bool) string {
|
||||
switch {
|
||||
case !found:
|
||||
return "N/A"
|
||||
case r.Err != nil:
|
||||
return "ERR"
|
||||
case r.Score.Skipped:
|
||||
return "skip"
|
||||
default:
|
||||
return fmt.Sprintf("%.2f (%dtok)", r.Score.Value, r.TotalTokens())
|
||||
}
|
||||
}
|
||||
|
||||
// categoryMean holds the per-model mean score for one category, computed
|
||||
// over non-skipped, non-error results only.
|
||||
type categoryMean struct {
|
||||
means map[string]float64
|
||||
category string
|
||||
}
|
||||
|
||||
func categoryRollup(tests []testkit.Test, models []string, idx resultIndex) []categoryMean {
|
||||
order := make([]string, 0)
|
||||
byCategory := make(map[string][]testkit.Test)
|
||||
for _, t := range sortedTests(tests) {
|
||||
if _, seen := byCategory[t.Category]; !seen {
|
||||
order = append(order, t.Category)
|
||||
}
|
||||
byCategory[t.Category] = append(byCategory[t.Category], t)
|
||||
}
|
||||
|
||||
out := make([]categoryMean, 0, len(order))
|
||||
for _, cat := range order {
|
||||
cm := categoryMean{category: cat, means: make(map[string]float64, len(models))}
|
||||
for _, model := range models {
|
||||
var sum float64
|
||||
var n int
|
||||
for _, t := range byCategory[cat] {
|
||||
r, ok := idx[t.ID][model]
|
||||
if !ok || r.Err != nil || r.Score.Skipped {
|
||||
continue
|
||||
}
|
||||
sum += r.Score.Value
|
||||
n++
|
||||
}
|
||||
if n > 0 {
|
||||
cm.means[model] = sum / float64(n)
|
||||
} else {
|
||||
cm.means[model] = math.NaN()
|
||||
}
|
||||
}
|
||||
out = append(out, cm)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// meanCellText renders a categoryMean/modelSummary mean cell, showing N/A
|
||||
// when there is no scored data.
|
||||
func meanCellText(v float64) string {
|
||||
if math.IsNaN(v) {
|
||||
return "N/A"
|
||||
}
|
||||
return fmt.Sprintf("%.2f", v)
|
||||
}
|
||||
|
||||
// rateText renders a modelSummary avgTokPerSec cell, showing N/A when there
|
||||
// is no timed, non-skipped data to compute a rate from.
|
||||
func rateText(v float64) string {
|
||||
if math.IsNaN(v) {
|
||||
return "N/A"
|
||||
}
|
||||
return fmt.Sprintf("%.1f", v)
|
||||
}
|
||||
|
||||
// durationText renders a modelSummary meanLatency cell, showing N/A when
|
||||
// there is no scored data (N6): meanLatency's zero value, printed via
|
||||
// time.Duration.String(), would otherwise misleadingly read as a real "0s"
|
||||
// mean latency rather than "no data". hasData mirrors overallMean's NaN
|
||||
// check, since both are set together (or left unset together) in
|
||||
// summarize.
|
||||
func durationText(d time.Duration, hasData bool) string {
|
||||
if !hasData {
|
||||
return "N/A"
|
||||
}
|
||||
return d.String()
|
||||
}
|
||||
|
||||
// modelSummary aggregates one model's results across the whole run.
|
||||
type modelSummary struct {
|
||||
model string
|
||||
overallMean float64 // math.NaN() when no scored data
|
||||
passed int
|
||||
partial int
|
||||
failed int
|
||||
errors int
|
||||
meanLatency time.Duration
|
||||
totalTokens int
|
||||
avgTokPerSec float64 // math.NaN() when no timed, non-skipped data
|
||||
}
|
||||
|
||||
func summarize(models []string, results []runner.Result) []modelSummary {
|
||||
byModel := make(map[string][]runner.Result, len(models))
|
||||
for _, r := range results {
|
||||
byModel[r.Model] = append(byModel[r.Model], r)
|
||||
}
|
||||
|
||||
out := make([]modelSummary, 0, len(models))
|
||||
for _, model := range models {
|
||||
ms := modelSummary{model: model}
|
||||
var scoreSum float64
|
||||
var scoredCount int
|
||||
var latencySum time.Duration
|
||||
var latencyCount int
|
||||
var completionTokenSum int
|
||||
var latencySecondsSum float64
|
||||
|
||||
for _, r := range byModel[model] {
|
||||
ms.totalTokens += r.TotalTokens()
|
||||
switch {
|
||||
case r.Err != nil:
|
||||
ms.errors++
|
||||
continue
|
||||
case r.Score.Skipped:
|
||||
continue
|
||||
}
|
||||
|
||||
scoreSum += r.Score.Value
|
||||
scoredCount++
|
||||
latencySum += r.Latency
|
||||
latencyCount++
|
||||
completionTokenSum += r.CompletionTokens
|
||||
latencySecondsSum += r.Latency.Seconds()
|
||||
|
||||
switch {
|
||||
case r.Score.Value >= passThreshold:
|
||||
ms.passed++
|
||||
case r.Score.Value <= 0:
|
||||
ms.failed++
|
||||
default:
|
||||
ms.partial++
|
||||
}
|
||||
}
|
||||
|
||||
if scoredCount > 0 {
|
||||
ms.overallMean = scoreSum / float64(scoredCount)
|
||||
ms.meanLatency = latencySum / time.Duration(latencyCount)
|
||||
} else {
|
||||
ms.overallMean = math.NaN()
|
||||
}
|
||||
|
||||
// Aggregate ratio (sum of completion tokens over sum of elapsed
|
||||
// seconds), not a mean of per-test ratios: robust to a handful of
|
||||
// very short responses skewing a per-test-averaged rate.
|
||||
if latencySecondsSum > 0 {
|
||||
ms.avgTokPerSec = float64(completionTokenSum) / latencySecondsSum
|
||||
} else {
|
||||
ms.avgTokPerSec = math.NaN()
|
||||
}
|
||||
|
||||
out = append(out, ms)
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,235 @@
|
||||
package report
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"math"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/lukaszraczylo/llm-testbench/internal/eval"
|
||||
"github.com/lukaszraczylo/llm-testbench/internal/runner"
|
||||
"github.com/lukaszraczylo/llm-testbench/internal/testkit"
|
||||
)
|
||||
|
||||
func sampleTests() []testkit.Test {
|
||||
return []testkit.Test{
|
||||
{ID: "go-1", Category: "programming", Subcategory: "golang"},
|
||||
{ID: "py-1", Category: "programming", Subcategory: "python"},
|
||||
{ID: "k8s-1", Category: "operations", Subcategory: "kubernetes"},
|
||||
}
|
||||
}
|
||||
|
||||
func sampleResults() []runner.Result {
|
||||
return []runner.Result{
|
||||
{Model: "m1", TestID: "go-1", Score: eval.Score{Value: 1}, Latency: 100 * time.Millisecond, PromptTokens: 6, CompletionTokens: 4},
|
||||
{Model: "m1", TestID: "py-1", Score: eval.Score{Value: 0.5}, Latency: 200 * time.Millisecond, PromptTokens: 12, CompletionTokens: 8},
|
||||
{Model: "m1", TestID: "k8s-1", Err: errors.New("boom")},
|
||||
{Model: "m2", TestID: "go-1", Score: eval.Score{Value: 0}, Latency: 50 * time.Millisecond, PromptTokens: 3, CompletionTokens: 2},
|
||||
{Model: "m2", TestID: "py-1", Score: eval.Score{Skipped: true, Detail: "toolchain missing: cc"}},
|
||||
{Model: "m2", TestID: "k8s-1", Score: eval.Score{Value: 1}, Latency: 150 * time.Millisecond, PromptTokens: 9, CompletionTokens: 6},
|
||||
}
|
||||
}
|
||||
|
||||
func TestRenderTable_ContainsExpectedCells(t *testing.T) {
|
||||
var buf bytes.Buffer
|
||||
if err := Render(&buf, FormatTable, sampleTests(), []string{"m1", "m2"}, sampleResults()); err != nil {
|
||||
t.Fatalf("Render() error = %v", err)
|
||||
}
|
||||
out := buf.String()
|
||||
|
||||
for _, want := range []string{
|
||||
"== Per-test scores ==",
|
||||
"== Category rollup (mean score) ==",
|
||||
"== Model summary ==",
|
||||
"1.00 (10tok)", // m1/go-1: 6 prompt + 4 completion
|
||||
"0.50 (20tok)", // m1/py-1: 12 prompt + 8 completion
|
||||
"ERR", // m1/k8s-1
|
||||
"0.00 (5tok)", // m2/go-1: 3 prompt + 2 completion
|
||||
"skip", // m2/py-1
|
||||
"TOK/S",
|
||||
} {
|
||||
if !strings.Contains(out, want) {
|
||||
t.Errorf("table output missing %q\n---\n%s", want, out)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRenderMarkdown_ProducesGitHubTables(t *testing.T) {
|
||||
var buf bytes.Buffer
|
||||
if err := Render(&buf, FormatMarkdown, sampleTests(), []string{"m1", "m2"}, sampleResults()); err != nil {
|
||||
t.Fatalf("Render() error = %v", err)
|
||||
}
|
||||
out := buf.String()
|
||||
|
||||
for _, want := range []string{
|
||||
"## Per-test scores",
|
||||
"| Test | Category | Subcategory | m1 | m2 |",
|
||||
"| --- | --- | --- | --- | --- |",
|
||||
"## Category rollup (mean score)",
|
||||
"## Model summary",
|
||||
"Tok/s",
|
||||
"1.00 (10tok)",
|
||||
} {
|
||||
if !strings.Contains(out, want) {
|
||||
t.Errorf("markdown output missing %q\n---\n%s", want, out)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRenderJSON_DumpsRawResults(t *testing.T) {
|
||||
var buf bytes.Buffer
|
||||
if err := Render(&buf, FormatJSON, sampleTests(), []string{"m1", "m2"}, sampleResults()); err != nil {
|
||||
t.Fatalf("Render() error = %v", err)
|
||||
}
|
||||
|
||||
var out []jsonResult
|
||||
if err := json.Unmarshal(buf.Bytes(), &out); err != nil {
|
||||
t.Fatalf("json.Unmarshal() error = %v; output:\n%s", err, buf.String())
|
||||
}
|
||||
if len(out) != len(sampleResults()) {
|
||||
t.Fatalf("len(out) = %d, want %d", len(out), len(sampleResults()))
|
||||
}
|
||||
|
||||
var errEntry, skipEntry, tokenEntry *jsonResult
|
||||
for i := range out {
|
||||
if out[i].Error != "" {
|
||||
errEntry = &out[i]
|
||||
}
|
||||
if out[i].Skipped {
|
||||
skipEntry = &out[i]
|
||||
}
|
||||
if out[i].Model == "m1" && out[i].TestID == "go-1" {
|
||||
tokenEntry = &out[i]
|
||||
}
|
||||
}
|
||||
if errEntry == nil || errEntry.Error != "boom" {
|
||||
t.Errorf("expected an entry with Error = \"boom\", got %+v", errEntry)
|
||||
}
|
||||
if skipEntry == nil || skipEntry.Category != "programming" {
|
||||
t.Errorf("expected skipped entry to carry Category, got %+v", skipEntry)
|
||||
}
|
||||
if tokenEntry == nil || tokenEntry.PromptTokens != 6 || tokenEntry.CompletionTokens != 4 || tokenEntry.Tokens != 10 {
|
||||
t.Errorf("expected m1/go-1 to carry prompt=6 completion=4 total=10 tokens, got %+v", tokenEntry)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRender_UnknownFormat(t *testing.T) {
|
||||
var buf bytes.Buffer
|
||||
err := Render(&buf, Format("bogus"), sampleTests(), []string{"m1"}, sampleResults())
|
||||
if err == nil {
|
||||
t.Fatal("Render() error = nil, want error for unknown format")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSummarize_Buckets(t *testing.T) {
|
||||
results := []runner.Result{
|
||||
{Model: "m1", TestID: "a", Score: eval.Score{Value: 1}},
|
||||
{Model: "m1", TestID: "b", Score: eval.Score{Value: 0.5}},
|
||||
{Model: "m1", TestID: "c", Score: eval.Score{Value: 0}},
|
||||
{Model: "m1", TestID: "d", Err: errors.New("fail")},
|
||||
{Model: "m1", TestID: "e", Score: eval.Score{Skipped: true}},
|
||||
}
|
||||
summaries := summarize([]string{"m1"}, results)
|
||||
if len(summaries) != 1 {
|
||||
t.Fatalf("summarize() len = %d, want 1", len(summaries))
|
||||
}
|
||||
ms := summaries[0]
|
||||
if ms.passed != 1 || ms.partial != 1 || ms.failed != 1 || ms.errors != 1 {
|
||||
t.Errorf("buckets = passed:%d partial:%d failed:%d errors:%d, want 1,1,1,1", ms.passed, ms.partial, ms.failed, ms.errors)
|
||||
}
|
||||
wantMean := (1.0 + 0.5 + 0.0) / 3.0
|
||||
if ms.overallMean != wantMean {
|
||||
t.Errorf("overallMean = %v, want %v", ms.overallMean, wantMean)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSummarize_AvgTokPerSec(t *testing.T) {
|
||||
// Aggregate ratio (sum completion tokens / sum latency seconds), not a
|
||||
// mean of per-test ratios: a fast, short response (10 tok / 0.1s = 100
|
||||
// tok/s) and a slow, long one (100 tok / 2s = 50 tok/s) must average to
|
||||
// 110 tok / 2.1s, not (100+50)/2.
|
||||
results := []runner.Result{
|
||||
{Model: "m1", TestID: "fast", Score: eval.Score{Value: 1}, Latency: 100 * time.Millisecond, CompletionTokens: 10},
|
||||
{Model: "m1", TestID: "slow", Score: eval.Score{Value: 1}, Latency: 2 * time.Second, CompletionTokens: 100},
|
||||
{Model: "m1", TestID: "err", Err: errors.New("fail"), Latency: time.Second, CompletionTokens: 999},
|
||||
{Model: "m1", TestID: "skipped", Score: eval.Score{Skipped: true}, Latency: time.Second, CompletionTokens: 999},
|
||||
}
|
||||
summaries := summarize([]string{"m1"}, results)
|
||||
ms := summaries[0]
|
||||
|
||||
want := 110.0 / 2.1
|
||||
if math.Abs(ms.avgTokPerSec-want) > 1e-9 {
|
||||
t.Errorf("avgTokPerSec = %v, want %v (errored/skipped results must be excluded)", ms.avgTokPerSec, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSummarize_AvgTokPerSec_NoDataIsNaN(t *testing.T) {
|
||||
results := []runner.Result{
|
||||
{Model: "m1", TestID: "err", Err: errors.New("fail")},
|
||||
{Model: "m1", TestID: "skipped", Score: eval.Score{Skipped: true}},
|
||||
}
|
||||
summaries := summarize([]string{"m1"}, results)
|
||||
if !math.IsNaN(summaries[0].avgTokPerSec) {
|
||||
t.Errorf("avgTokPerSec = %v, want NaN when there is no timed data", summaries[0].avgTokPerSec)
|
||||
}
|
||||
if rateText(summaries[0].avgTokPerSec) != "N/A" {
|
||||
t.Errorf("rateText(NaN) = %q, want N/A", rateText(summaries[0].avgTokPerSec))
|
||||
}
|
||||
}
|
||||
|
||||
func TestDurationText(t *testing.T) {
|
||||
if got := durationText(0, false); got != "N/A" {
|
||||
t.Errorf("durationText(0, false) = %q, want N/A", got)
|
||||
}
|
||||
if got := durationText(0, true); got != "0s" {
|
||||
t.Errorf("durationText(0, true) = %q, want 0s (a real zero-latency measurement, not lack of data)", got)
|
||||
}
|
||||
if got := durationText(42*time.Millisecond, true); got != "42ms" {
|
||||
t.Errorf("durationText(42ms, true) = %q, want 42ms", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSummarize_MeanLatency_NoDataIsNA(t *testing.T) {
|
||||
// N6 regression: a model with only errored/skipped results (no scored
|
||||
// data at all) must render MEAN_LATENCY as N/A, not the zero Duration's
|
||||
// misleading "0s".
|
||||
results := []runner.Result{
|
||||
{Model: "m1", TestID: "err", Err: errors.New("fail")},
|
||||
{Model: "m1", TestID: "skipped", Score: eval.Score{Skipped: true}},
|
||||
}
|
||||
var buf bytes.Buffer
|
||||
if err := Render(&buf, FormatTable, []testkit.Test{{ID: "err"}, {ID: "skipped"}}, []string{"m1"}, results); err != nil {
|
||||
t.Fatalf("Render() error = %v", err)
|
||||
}
|
||||
out := buf.String()
|
||||
if !strings.Contains(out, "N/A") {
|
||||
t.Errorf("table output missing N/A for mean latency with no scored data:\n%s", out)
|
||||
}
|
||||
if strings.Contains(out, "0s") {
|
||||
t.Errorf("table output should not render a misleading 0s mean latency:\n%s", out)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCategoryRollup_ExcludesErrorsAndSkips(t *testing.T) {
|
||||
tests := []testkit.Test{
|
||||
{ID: "a", Category: "cat"},
|
||||
{ID: "b", Category: "cat"},
|
||||
{ID: "c", Category: "cat"},
|
||||
}
|
||||
results := []runner.Result{
|
||||
{Model: "m1", TestID: "a", Score: eval.Score{Value: 1}},
|
||||
{Model: "m1", TestID: "b", Err: errors.New("fail")},
|
||||
{Model: "m1", TestID: "c", Score: eval.Score{Skipped: true}},
|
||||
}
|
||||
idx := indexResults(results)
|
||||
rollup := categoryRollup(tests, []string{"m1"}, idx)
|
||||
if len(rollup) != 1 {
|
||||
t.Fatalf("categoryRollup() len = %d, want 1", len(rollup))
|
||||
}
|
||||
if rollup[0].means["m1"] != 1 {
|
||||
t.Errorf("mean = %v, want 1 (error/skip excluded)", rollup[0].means["m1"])
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,96 @@
|
||||
package report
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"math"
|
||||
"strings"
|
||||
"text/tabwriter"
|
||||
|
||||
"github.com/lukaszraczylo/llm-testbench/internal/runner"
|
||||
"github.com/lukaszraczylo/llm-testbench/internal/testkit"
|
||||
)
|
||||
|
||||
// errWriter wraps an io.Writer and remembers the first write error, so a
|
||||
// sequence of Fprint* calls can be written without checking after every
|
||||
// line; the caller checks ew.err once at the end.
|
||||
type errWriter struct {
|
||||
w io.Writer
|
||||
err error
|
||||
}
|
||||
|
||||
func (ew *errWriter) println(args ...any) {
|
||||
if ew.err != nil {
|
||||
return
|
||||
}
|
||||
_, ew.err = fmt.Fprintln(ew.w, args...)
|
||||
}
|
||||
|
||||
// renderTable writes three plain-text tables: per-test scores, category
|
||||
// rollup, and model summary.
|
||||
func renderTable(w io.Writer, tests []testkit.Test, models []string, results []runner.Result) error {
|
||||
idx := indexResults(results)
|
||||
tw := tabwriter.NewWriter(w, 0, 4, 2, ' ', 0)
|
||||
ew := &errWriter{w: tw}
|
||||
|
||||
writeScoreTable(ew, tests, models, idx)
|
||||
ew.println()
|
||||
writeCategoryTable(ew, tests, models, idx)
|
||||
ew.println()
|
||||
writeSummaryTable(ew, models, results)
|
||||
if ew.err != nil {
|
||||
return ew.err
|
||||
}
|
||||
|
||||
return tw.Flush()
|
||||
}
|
||||
|
||||
func writeScoreTable(ew *errWriter, tests []testkit.Test, models []string, idx resultIndex) {
|
||||
ew.println("== Per-test scores ==")
|
||||
header := []string{"TEST", "CATEGORY", "SUBCATEGORY"}
|
||||
header = append(header, models...)
|
||||
ew.println(strings.Join(header, "\t"))
|
||||
|
||||
for _, t := range sortedTests(tests) {
|
||||
row := []string{t.ID, t.Category, t.Subcategory}
|
||||
for _, model := range models {
|
||||
r, ok := idx[t.ID][model]
|
||||
row = append(row, cellText(r, ok))
|
||||
}
|
||||
ew.println(strings.Join(row, "\t"))
|
||||
}
|
||||
}
|
||||
|
||||
func writeCategoryTable(ew *errWriter, tests []testkit.Test, models []string, idx resultIndex) {
|
||||
ew.println("== Category rollup (mean score) ==")
|
||||
header := append([]string{"CATEGORY"}, models...)
|
||||
ew.println(strings.Join(header, "\t"))
|
||||
|
||||
for _, cm := range categoryRollup(tests, models, idx) {
|
||||
row := []string{cm.category}
|
||||
for _, model := range models {
|
||||
row = append(row, meanCellText(cm.means[model]))
|
||||
}
|
||||
ew.println(strings.Join(row, "\t"))
|
||||
}
|
||||
}
|
||||
|
||||
func writeSummaryTable(ew *errWriter, models []string, results []runner.Result) {
|
||||
ew.println("== Model summary ==")
|
||||
ew.println(strings.Join([]string{"MODEL", "MEAN", "PASSED", "PARTIAL", "FAILED", "ERRORS", "MEAN_LATENCY", "TOTAL_TOKENS", "TOK/S"}, "\t"))
|
||||
|
||||
for _, ms := range summarize(models, results) {
|
||||
row := []string{
|
||||
ms.model,
|
||||
meanCellText(ms.overallMean),
|
||||
fmt.Sprintf("%d", ms.passed),
|
||||
fmt.Sprintf("%d", ms.partial),
|
||||
fmt.Sprintf("%d", ms.failed),
|
||||
fmt.Sprintf("%d", ms.errors),
|
||||
durationText(ms.meanLatency, !math.IsNaN(ms.overallMean)),
|
||||
fmt.Sprintf("%d", ms.totalTokens),
|
||||
rateText(ms.avgTokPerSec),
|
||||
}
|
||||
ew.println(strings.Join(row, "\t"))
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user