llm-testbench/internal/report/compare.go

package report

import (
	"fmt"
	"io"
	"math"
	"sort"
	"strings"
)

// Comparison is the per-(model, test) diff between two saved artifacts.
// Scores are means over attempts, so a test that flapped between runs
// shows up as a regression on one attempt basis only if the mean moved.
type Comparison struct {
	Model        string
	TestID       string
	Category     string
	Subcategory  string
	Status       string
	BaselineMean float64
	CurrentMean  float64
	Delta        float64
}

// RegressionDelta is the mean-score drop at which a test counts as a
// regression. Below this, run-to-run sampling noise dominates.
const RegressionDelta = 0.01

type attemptKey struct{ model, testID string }

// meanAttempts collapses saved attempts into per-(model,test) means and
// carries category metadata from the first attempt that has it. Truncated
// attempts are excluded from the mean when any attempt finished normally,
// mirroring the truncation policy in indexResults; when every scored
// attempt was truncated, the partial-text scores are used rather than
// reporting no data.
func meanAttempts(results []AttemptResult) map[attemptKey]Comparison {
	type agg struct {
		cat      string
		sb       string
		sum      float64
		n        int
		cleanSum float64
		cleanN   int
		errs     int
	}
	by := make(map[attemptKey]*agg)
	order := make([]attemptKey, 0, len(results))
	for _, r := range results {
		k := attemptKey{r.Model, r.TestID}
		a, seen := by[k]
		if !seen {
			a = &agg{}
			by[k] = a
			order = append(order, k)
		}
		if a.cat == "" {
			a.cat, a.sb = r.Category, r.Subcategory
		}
		if r.Error != "" {
			a.errs++
			continue
		}
		if r.Skipped {
			continue
		}
		a.sum += r.Score
		a.n++
		if !r.Truncated {
			a.cleanSum += r.Score
			a.cleanN++
		}
	}

	out := make(map[attemptKey]Comparison, len(by))
	for _, k := range order {
		a := by[k]
		c := Comparison{Model: k.model, TestID: k.testID, Category: a.cat, Subcategory: a.sb}
		switch {
		case a.n == 0:
			c.Status = "error"
		case a.cleanN > 0:
			c.BaselineMean = a.cleanSum / float64(a.cleanN)
		default:
			c.BaselineMean = a.sum / float64(a.n)
		}
		out[k] = c
	}
	return out
}

// CompareArtifacts diffs two artifacts per (model, test), sorted by
// delta ascending (worst regressions first), then test ID.
func CompareArtifacts(baseline, current Artifact) []Comparison {
	base := meanAttempts(baseline.Results)
	cur := meanAttempts(current.Results)

	keys := make(map[attemptKey]bool, len(base)+len(cur))
	for k := range base {
		keys[k] = true
	}
	for k := range cur {
		keys[k] = true
	}

	out := make([]Comparison, 0, len(keys))
	for k := range keys {
		b, hasB := base[k]
		c, hasC := cur[k]
		switch {
		case hasB && !hasC:
			b.CurrentMean = math.NaN()
			b.Delta = math.NaN()
			b.Status = "baseline-only"
			out = append(out, b)
		case !hasB && hasC:
			c.BaselineMean = math.NaN()
			c.Delta = math.NaN()
			c.Status = "current-only"
			out = append(out, c)
		default:
			m := Comparison{
				Model:        k.model,
				TestID:       k.testID,
				Category:     c.Category,
				Subcategory:  c.Subcategory,
				BaselineMean: b.BaselineMean,
				CurrentMean:  c.BaselineMean,
			}
			switch {
			case b.Status == "error" || c.Status == "error":
				m.Status = "error"
				m.Delta = math.NaN()
			default:
				m.Delta = m.CurrentMean - m.BaselineMean
				switch {
				case m.Delta <= -RegressionDelta:
					m.Status = "regression"
				case m.Delta >= RegressionDelta:
					m.Status = "improvement"
				default:
					m.Status = "stable"
				}
			}
			out = append(out, m)
		}
	}

	sort.Slice(out, func(i, j int) bool {
		di, dj := out[i].Delta, out[j].Delta
		if math.IsNaN(di) != math.IsNaN(dj) {
			return math.IsNaN(di)
		}
		if math.IsNaN(di) && math.IsNaN(dj) {
			if out[i].Status != out[j].Status {
				return out[i].Status < out[j].Status
			}
			return out[i].TestID < out[j].TestID
		}
		if di != dj {
			return di < dj
		}
		if out[i].TestID != out[j].TestID {
			return out[i].TestID < out[j].TestID
		}
		return out[i].Model < out[j].Model
	})
	return out
}

// RenderCompare writes a comparison report: headline counts, then only
// changed/missing rows (stable rows are summarized, not listed).
func RenderCompare(w io.Writer, comparisons []Comparison, warnings ...string) error {
	ew := &errWriter{w: w}
	for _, warn := range warnings {
		ew.println("WARNING: " + warn)
	}

	var reg, imp, stable, added, removed, errored int
	for _, c := range comparisons {
		switch c.Status {
		case "regression":
			reg++
		case "improvement":
			imp++
		case "baseline-only":
			removed++
		case "current-only":
			added++
		case "error":
			errored++
		default:
			stable++
		}
	}

	ew.println(fmt.Sprintf("== Compare: %d regressions, %d improvements, %d stable, %d errors, %d baseline-only, %d current-only ==",
		reg, imp, stable, errored, removed, added))
	ew.println()
	ew.println("MODEL\tTEST\tCATEGORY\tSUBCATEGORY\tBASELINE\tCURRENT\tDELTA\tSTATUS")
	shown := 0
	for _, c := range comparisons {
		if c.Status == "stable" {
			continue
		}
		ew.println(strings.Join([]string{
			c.Model, c.TestID, c.Category, c.Subcategory,
			meanCellText(c.BaselineMean), meanCellText(c.CurrentMean),
			deltaText(c.Delta), c.Status,
		}, "\t"))
		shown++
	}
	if shown == 0 {
		ew.println("(no changed, missing, or erroring tests)")
	}
	return ew.err
}

func deltaText(v float64) string {
	if math.IsNaN(v) {
		return "N/A"
	}
	return fmt.Sprintf("%+.2f", v)
}

// CompareWarnings reports reasons the two artifacts may not be comparable
// like-for-like (catalog, sampling or backend-build drift).
func CompareWarnings(baseline, current Artifact) []string {
	return compareWarnings(baseline.Meta, current.Meta)
}