llm-testbench/internal/report/table.go

package report

import (
	"fmt"
	"io"
	"math"
	"strings"
	"text/tabwriter"

	"github.com/lukaszraczylo/llm-testbench/internal/runner"
	"github.com/lukaszraczylo/llm-testbench/internal/testkit"
)

// errWriter wraps an io.Writer and remembers the first write error, so a
// sequence of Fprint* calls can be written without checking after every
// line; the caller checks ew.err once at the end.
type errWriter struct {
	w   io.Writer
	err error
}

func (ew *errWriter) println(args ...any) {
	if ew.err != nil {
		return
	}
	_, ew.err = fmt.Fprintln(ew.w, args...)
}

// renderTable writes four plain-text tables: per-test scores, category
// rollup, discrimination/stability rollup, and model summary.
func renderTable(w io.Writer, tests []testkit.Test, models []string, results []runner.Result) error {
	idx := indexResults(results)
	tw := tabwriter.NewWriter(w, 0, 4, 2, ' ', 0)
	ew := &errWriter{w: tw}

	writeScoreTable(ew, tests, models, idx)
	ew.println()
	writeCategoryTable(ew, tests, models, idx)
	ew.println()
	writeDiscriminationTable(ew, testStats(tests, models, idx), len(models) > 1)
	ew.println()
	writeSummaryTable(ew, models, idx)
	if ew.err != nil {
		return ew.err
	}

	return tw.Flush()
}

func writeScoreTable(ew *errWriter, tests []testkit.Test, models []string, idx cellIndex) {
	ew.println("== Per-test scores ==")
	header := []string{"TEST", "CATEGORY", "SUBCATEGORY"}
	header = append(header, models...)
	ew.println(strings.Join(header, "\t"))

	anyTruncated := false
	for _, t := range sortedTests(tests) {
		row := []string{t.ID, t.Category, t.Subcategory}
		for _, model := range models {
			c := idx[t.ID][model]
			row = append(row, cellText(c))
			anyTruncated = anyTruncated || c.result.Truncated()
		}
		ew.println(strings.Join(row, "\t"))
	}
	if anyTruncated {
		ew.println(truncatedLegend)
	}
}

func writeCategoryTable(ew *errWriter, tests []testkit.Test, models []string, idx cellIndex) {
	ew.println("== Category rollup (mean score) ==")
	header := append([]string{"CATEGORY"}, models...)
	ew.println(strings.Join(header, "\t"))

	for _, cm := range categoryRollup(tests, models, idx) {
		row := []string{cm.category}
		for _, model := range models {
			row = append(row, meanCellText(cm.means[model]))
		}
		ew.println(strings.Join(row, "\t"))
	}
}

// writeDiscriminationTable lists the tests that actually measure
// something: cross-model spread >= DiscriminationSpread, or attempts that
// disagreed under --repeat. With a single model there is no spread to
// speak of, so the non-passing tests are listed instead.
func writeDiscriminationTable(ew *errWriter, stats []testStat, multiModel bool) {
	writeDiscriminationRows(tablePrinter{ew: ew}, stats, multiModel)
}

// discriminationSink abstracts the tab (table) vs pipe (markdown) output
// so the discrimination section is written once.
type discriminationSink interface {
	header(cells ...string)
	row(cells ...string)
	note(text string)
}

type tablePrinter struct{ ew *errWriter }

func (tp tablePrinter) header(cells ...string) { tp.ew.println(strings.Join(cells, "\t")) }
func (tp tablePrinter) row(cells ...string)    { tp.ew.println(strings.Join(cells, "\t")) }
func (tp tablePrinter) note(text string)       { tp.ew.println(text) }

func writeDiscriminationRows(sink discriminationSink, stats []testStat, multiModel bool) {
	discriminating, unstable, scored := 0, 0, 0
	for _, st := range stats {
		if !st.scored {
			continue
		}
		scored++
		if st.unstable {
			unstable++
		}
		if multiModel && !math.IsNaN(st.spread) && st.spread >= DiscriminationSpread {
			discriminating++
		}
	}

	if multiModel {
		sink.note(fmt.Sprintf("== Discrimination: %d/%d scored tests separate models by >= %.2f; %d unstable under repeat ==",
			discriminating, scored, DiscriminationSpread, unstable))
	} else {
		sink.note(fmt.Sprintf("== Discrimination: single model (no cross-model spread); %d unstable under repeat ==", unstable))
	}

	rows := interestingStats(stats, multiModel)
	if len(rows) == 0 {
		sink.note("no discriminating or unstable tests: every scored test passed identically for every model")
		return
	}

	sink.header("TEST", "CATEGORY", "SUBCATEGORY", "MEAN", "SPREAD", "UNSTABLE", "TRUNC")
	for _, st := range rows {
		sink.row(
			st.testID,
			st.category,
			st.subcategory,
			meanCellText(st.mean),
			meanCellText(st.spread),
			yesNo(st.unstable),
			yesNo(st.truncated),
		)
	}
}

func yesNo(b bool) string {
	if b {
		return "yes"
	}
	return "-"
}

func writeSummaryTable(ew *errWriter, models []string, idx cellIndex) {
	ew.println("== Model summary ==")
	ew.println(strings.Join([]string{"MODEL", "MEAN", "PASSED", "PARTIAL", "FAILED", "ERRORS", "MEAN_LATENCY", "TOTAL_TOKENS", "TOK/S"}, "\t"))

	for _, ms := range summarize(models, idx) {
		row := []string{
			ms.model,
			meanCellText(ms.overallMean),
			fmt.Sprintf("%d", ms.passed),
			fmt.Sprintf("%d", ms.partial),
			fmt.Sprintf("%d", ms.failed),
			fmt.Sprintf("%d", ms.errors),
			durationText(ms.meanLatency, !math.IsNaN(ms.overallMean)),
			fmt.Sprintf("%d", ms.totalTokens),
			rateText(ms.avgTokPerSec),
		}
		ew.println(strings.Join(row, "\t"))
	}
}