Update

internal/report/compare.go

@@ -0,0 +1,221 @@
+package report
+
+import (
+	"fmt"
+	"io"
+	"math"
+	"sort"
+	"strings"
+)
+
+// Comparison is the per-(model, test) diff between two saved artifacts.
+// Scores are means over attempts, so a test that flapped between runs
+// shows up as a regression on one attempt basis only if the mean moved.
+type Comparison struct {
+	Model        string
+	TestID       string
+	Category     string
+	Subcategory  string
+	Status       string
+	BaselineMean float64
+	CurrentMean  float64
+	Delta        float64
+}
+
+// RegressionDelta is the mean-score drop at which a test counts as a
+// regression. Below this, run-to-run sampling noise dominates.
+const RegressionDelta = 0.01
+
+type attemptKey struct{ model, testID string }
+
+// meanAttempts collapses saved attempts into per-(model,test) means and
+// carries category metadata from the first attempt that has it. Truncated
+// attempts are excluded from the mean when any attempt finished normally,
+// mirroring the truncation policy in indexResults; when every scored
+// attempt was truncated, the partial-text scores are used rather than
+// reporting no data.
+func meanAttempts(results []AttemptResult) map[attemptKey]Comparison {
+	type agg struct {
+		cat      string
+		sb       string
+		sum      float64
+		n        int
+		cleanSum float64
+		cleanN   int
+		errs     int
+	}
+	by := make(map[attemptKey]*agg)
+	order := make([]attemptKey, 0, len(results))
+	for _, r := range results {
+		k := attemptKey{r.Model, r.TestID}
+		a, seen := by[k]
+		if !seen {
+			a = &agg{}
+			by[k] = a
+			order = append(order, k)
+		}
+		if a.cat == "" {
+			a.cat, a.sb = r.Category, r.Subcategory
+		}
+		if r.Error != "" {
+			a.errs++
+			continue
+		}
+		if r.Skipped {
+			continue
+		}
+		a.sum += r.Score
+		a.n++
+		if !r.Truncated {
+			a.cleanSum += r.Score
+			a.cleanN++
+		}
+	}
+
+	out := make(map[attemptKey]Comparison, len(by))
+	for _, k := range order {
+		a := by[k]
+		c := Comparison{Model: k.model, TestID: k.testID, Category: a.cat, Subcategory: a.sb}
+		switch {
+		case a.n == 0:
+			c.Status = "error"
+		case a.cleanN > 0:
+			c.BaselineMean = a.cleanSum / float64(a.cleanN)
+		default:
+			c.BaselineMean = a.sum / float64(a.n)
+		}
+		out[k] = c
+	}
+	return out
+}
+
+// CompareArtifacts diffs two artifacts per (model, test), sorted by
+// delta ascending (worst regressions first), then test ID.
+func CompareArtifacts(baseline, current Artifact) []Comparison {
+	base := meanAttempts(baseline.Results)
+	cur := meanAttempts(current.Results)
+
+	keys := make(map[attemptKey]bool, len(base)+len(cur))
+	for k := range base {
+		keys[k] = true
+	}
+	for k := range cur {
+		keys[k] = true
+	}
+
+	out := make([]Comparison, 0, len(keys))
+	for k := range keys {
+		b, hasB := base[k]
+		c, hasC := cur[k]
+		switch {
+		case hasB && !hasC:
+			b.CurrentMean = math.NaN()
+			b.Delta = math.NaN()
+			b.Status = "baseline-only"
+			out = append(out, b)
+		case !hasB && hasC:
+			c.BaselineMean = math.NaN()
+			c.Delta = math.NaN()
+			c.Status = "current-only"
+			out = append(out, c)
+		default:
+			m := Comparison{
+				Model:        k.model,
+				TestID:       k.testID,
+				Category:     c.Category,
+				Subcategory:  c.Subcategory,
+				BaselineMean: b.BaselineMean,
+				CurrentMean:  c.BaselineMean,
+			}
+			switch {
+			case b.Status == "error" || c.Status == "error":
+				m.Status = "error"
+				m.Delta = math.NaN()
+			default:
+				m.Delta = m.CurrentMean - m.BaselineMean
+				switch {
+				case m.Delta <= -RegressionDelta:
+					m.Status = "regression"
+				case m.Delta >= RegressionDelta:
+					m.Status = "improvement"
+				default:
+					m.Status = "stable"
+				}
+			}
+			out = append(out, m)
+		}
+	}
+
+	sort.Slice(out, func(i, j int) bool {
+		di, dj := out[i].Delta, out[j].Delta
+		if math.IsNaN(di) != math.IsNaN(dj) {
+			return math.IsNaN(di)
+		}
+		if math.IsNaN(di) && math.IsNaN(dj) {
+			if out[i].Status != out[j].Status {
+				return out[i].Status < out[j].Status
+			}
+			return out[i].TestID < out[j].TestID
+		}
+		if di != dj {
+			return di < dj
+		}
+		if out[i].TestID != out[j].TestID {
+			return out[i].TestID < out[j].TestID
+		}
+		return out[i].Model < out[j].Model
+	})
+	return out
+}
+
+// RenderCompare writes a comparison report: headline counts, then only
+// changed/missing rows (stable rows are summarized, not listed).
+func RenderCompare(w io.Writer, comparisons []Comparison) error {
+	ew := &errWriter{w: w}
+
+	var reg, imp, stable, added, removed, errored int
+	for _, c := range comparisons {
+		switch c.Status {
+		case "regression":
+			reg++
+		case "improvement":
+			imp++
+		case "baseline-only":
+			removed++
+		case "current-only":
+			added++
+		case "error":
+			errored++
+		default:
+			stable++
+		}
+	}
+
+	ew.println(fmt.Sprintf("== Compare: %d regressions, %d improvements, %d stable, %d errors, %d baseline-only, %d current-only ==",
+		reg, imp, stable, errored, removed, added))
+	ew.println()
+	ew.println("MODEL\tTEST\tCATEGORY\tSUBCATEGORY\tBASELINE\tCURRENT\tDELTA\tSTATUS")
+	shown := 0
+	for _, c := range comparisons {
+		if c.Status == "stable" {
+			continue
+		}
+		ew.println(strings.Join([]string{
+			c.Model, c.TestID, c.Category, c.Subcategory,
+			meanCellText(c.BaselineMean), meanCellText(c.CurrentMean),
+			deltaText(c.Delta), c.Status,
+		}, "\t"))
+		shown++
+	}
+	if shown == 0 {
+		ew.println("(no changed, missing, or erroring tests)")
+	}
+	return ew.err
+}
+
+func deltaText(v float64) string {
+	if math.IsNaN(v) {
+		return "N/A"
+	}
+	return fmt.Sprintf("%+.2f", v)
+}

internal/report/compare_test.go

@@ -0,0 +1,114 @@
+package report
+
+import (
+	"bytes"
+	"math"
+	"strings"
+	"testing"
+)
+
+func attempt(model, testID string, score float64) AttemptResult {
+	return AttemptResult{Model: model, TestID: testID, Category: "c", Subcategory: "s", Score: score}
+}
+
+func TestCompareArtifacts_ClassifiesRegressionImprovementStable(t *testing.T) {
+	baseline := Artifact{Results: []AttemptResult{
+		attempt("m1", "reg", 1.0),
+		attempt("m1", "imp", 0.2),
+		attempt("m1", "stable", 1.0),
+	}}
+	current := Artifact{Results: []AttemptResult{
+		attempt("m1", "reg", 0.4),
+		attempt("m1", "imp", 1.0),
+		attempt("m1", "stable", 0.995),
+	}}
+
+	got := CompareArtifacts(baseline, current)
+	byID := map[string]Comparison{}
+	for _, c := range got {
+		byID[c.TestID] = c
+	}
+	if byID["reg"].Status != "regression" || byID["reg"].Delta > -0.5 {
+		t.Errorf("reg = %+v, want regression delta ~ -0.6", byID["reg"])
+	}
+	if byID["imp"].Status != "improvement" {
+		t.Errorf("imp = %+v, want improvement", byID["imp"])
+	}
+	if byID["stable"].Status != "stable" {
+		t.Errorf("stable = %+v, want stable", byID["stable"])
+	}
+	// Worst delta first.
+	if got[0].TestID != "reg" {
+		t.Errorf("first row = %q, want worst regression %q", got[0].TestID, "reg")
+	}
+}
+
+func TestCompareArtifacts_AveragesAttemptsAndFlagsMembership(t *testing.T) {
+	baseline := Artifact{Results: []AttemptResult{
+		attempt("m1", "flaky", 1.0), attempt("m1", "flaky", 0.0), // mean 0.5
+		attempt("m1", "gone", 1.0),
+	}}
+	current := Artifact{Results: []AttemptResult{
+		attempt("m1", "flaky", 1.0), attempt("m1", "flaky", 1.0), // mean 1.0 => improvement
+		attempt("m1", "new", 0.5),
+	}}
+
+	got := CompareArtifacts(baseline, current)
+	byID := map[string]Comparison{}
+	for _, c := range got {
+		byID[c.TestID] = c
+	}
+	if byID["flaky"].Status != "improvement" || math.Abs(byID["flaky"].BaselineMean-0.5) > 1e-9 {
+		t.Errorf("flaky = %+v, want improvement from mean 0.5", byID["flaky"])
+	}
+	if byID["gone"].Status != "baseline-only" {
+		t.Errorf("gone = %+v, want baseline-only", byID["gone"])
+	}
+	if byID["new"].Status != "current-only" {
+		t.Errorf("new = %+v, want current-only", byID["new"])
+	}
+}
+
+func TestCompareArtifacts_TruncatedAttemptExcludedFromMean(t *testing.T) {
+	clean := func(score float64) AttemptResult {
+		return AttemptResult{Model: "m1", TestID: "t", Category: "c", Subcategory: "s", Score: score}
+	}
+	trunc := func(score float64) AttemptResult {
+		return AttemptResult{Model: "m1", TestID: "t", Category: "c", Subcategory: "s", Score: score, Truncated: true}
+	}
+	baseline := Artifact{Results: []AttemptResult{clean(1.0), trunc(0.0)}} // clean mean 1.0
+	current := Artifact{Results: []AttemptResult{trunc(0.4), trunc(0.6)}}  // all truncated => fallback 0.5
+
+	got := CompareArtifacts(baseline, current)
+	if len(got) != 1 {
+		t.Fatalf("comparisons = %d, want 1", len(got))
+	}
+	if math.Abs(got[0].BaselineMean-1.0) > 1e-9 {
+		t.Errorf("baseline mean = %v, want 1.0 (truncated attempt excluded)", got[0].BaselineMean)
+	}
+	if math.Abs(got[0].CurrentMean-0.5) > 1e-9 {
+		t.Errorf("current mean = %v, want 0.5 (all-truncated fallback)", got[0].CurrentMean)
+	}
+	if got[0].Status != "regression" {
+		t.Errorf("status = %q, want regression", got[0].Status)
+	}
+}
+
+func TestRenderCompare_HeadlineAndRows(t *testing.T) {
+	baseline := Artifact{Results: []AttemptResult{attempt("m1", "a", 1.0), attempt("m1", "b", 1.0)}}
+	current := Artifact{Results: []AttemptResult{attempt("m1", "a", 0.0), attempt("m1", "b", 1.0)}}
+
+	var buf bytes.Buffer
+	if err := RenderCompare(&buf, CompareArtifacts(baseline, current)); err != nil {
+		t.Fatalf("RenderCompare() error = %v", err)
+	}
+	out := buf.String()
+	for _, want := range []string{"1 regressions", "0 improvements", "1 stable", "a", "-1.00"} {
+		if !strings.Contains(out, want) {
+			t.Errorf("output missing %q:\n%s", want, out)
+		}
+	}
+	if strings.Contains(out, "\nb\t") {
+		t.Errorf("stable row b should be summarized, not listed:\n%s", out)
+	}
+}

internal/report/discrimination.go

@@ -0,0 +1,114 @@
+package report
+
+import (
+	"math"
+	"sort"
+
+	"github.com/lukaszraczylo/llm-testbench/internal/testkit"
+)
+
+// DiscriminationSpread is the cross-model score gap at which a test starts
+// to separate models. The last full run showed a median spread of 0.0
+// across 250 tests: nearly every test scored >=0.99 for every model, so
+// the suite measured nothing about relative capability. This number makes
+// that visible per test instead of averaged away.
+const DiscriminationSpread = 0.05
+
+// testStat is the per-test measurement rollup: cross-model discrimination
+// (spread, only meaningful with 2+ models), attempt stability (did
+// --repeat samples disagree), and aggregate score. Computed over the
+// attempt-aggregated cells so a flaky test shows as unstable rather than
+// as a random single score.
+type testStat struct {
+	testID      string
+	category    string
+	subcategory string
+	mean        float64 // mean over model cell means (scored cells only)
+	spread      float64 // max-min over model cell means; NaN with <2 scored models
+	unstable    bool    // some model's attempts disagreed beyond epsilon
+	truncated   bool    // some cell was token-budget truncated
+	scored      bool    // at least one scored cell
+}
+
+// unstableEpsilon absorbs float noise in averaged scores; a real
+// disagreement between attempts is orders of magnitude larger.
+const unstableEpsilon = 1e-9
+
+// testStats computes the per-test rollup for every test in tests, ordered
+// by category, subcategory, ID (same order as the score table).
+func testStats(tests []testkit.Test, models []string, idx cellIndex) []testStat {
+	out := make([]testStat, 0, len(tests))
+	for _, t := range sortedTests(tests) {
+		st := testStat{testID: t.ID, category: t.Category, subcategory: t.Subcategory, spread: math.NaN()}
+		var sum float64
+		var n int
+		var lo, hi float64
+		for _, model := range models {
+			c, ok := idx[t.ID][model]
+			if !ok || c.errorAll || c.skippedAll || c.scored == 0 {
+				continue
+			}
+			v := c.result.Score.Value
+			if n == 0 {
+				lo, hi = v, v
+			} else {
+				lo = math.Min(lo, v)
+				hi = math.Max(hi, v)
+			}
+			sum += v
+			n++
+			st.scored = true
+			if c.maxScore-c.minScore > unstableEpsilon {
+				st.unstable = true
+			}
+			st.truncated = st.truncated || c.result.Truncated()
+		}
+		if n > 1 {
+			st.spread = hi - lo
+			st.mean = sum / float64(n)
+		} else if n == 1 {
+			st.mean = sum
+		}
+		out = append(out, st)
+	}
+	return out
+}
+
+// interestingStats narrows testStats to rows worth reading: tests that
+// discriminate between models, tests whose repeated attempts disagreed,
+// and (when no cross-model data exists) tests that any model failed to
+// pass outright. Sorted by spread descending, unstable first within equal
+// spread.
+func interestingStats(stats []testStat, multiModel bool) []testStat {
+	var out []testStat
+	for _, st := range stats {
+		if !st.scored {
+			continue
+		}
+		switch {
+		case st.unstable:
+		case multiModel && !math.IsNaN(st.spread) && st.spread >= DiscriminationSpread:
+		case !multiModel && st.mean < passThreshold:
+		default:
+			continue
+		}
+		out = append(out, st)
+	}
+	sort.SliceStable(out, func(i, j int) bool {
+		if out[i].unstable != out[j].unstable {
+			return out[i].unstable
+		}
+		si, sj := out[i].spread, out[j].spread
+		if math.IsNaN(si) {
+			si = -1
+		}
+		if math.IsNaN(sj) {
+			sj = -1
+		}
+		if si != sj {
+			return si > sj
+		}
+		return out[i].testID < out[j].testID
+	})
+	return out
+}

internal/report/discrimination_test.go

@@ -0,0 +1,86 @@
+package report
+
+import (
+	"bytes"
+	"encoding/json"
+	"strings"
+	"testing"
+
+	"github.com/lukaszraczylo/llm-testbench/internal/eval"
+	"github.com/lukaszraczylo/llm-testbench/internal/runner"
+	"github.com/lukaszraczylo/llm-testbench/internal/testkit"
+)
+
+func TestDiscrimination_TableShowsSpreadAndInstability(t *testing.T) {
+	tests := []testkit.Test{
+		{ID: "sep", Category: "c", Subcategory: "s"},
+		{ID: "same", Category: "c", Subcategory: "s"},
+	}
+	results := []runner.Result{
+		{Model: "m1", TestID: "sep", Score: eval.Score{Value: 1}},
+		{Model: "m2", TestID: "sep", Score: eval.Score{Value: 0.2}},
+		{Model: "m1", TestID: "same", Score: eval.Score{Value: 1}},
+		{Model: "m2", TestID: "same", Score: eval.Score{Value: 1}},
+		// Flaky pair for m1/sep: a second attempt disagrees with the first.
+		{Model: "m1", TestID: "sep", Attempt: 1, Score: eval.Score{Value: 0.4}},
+	}
+
+	var buf bytes.Buffer
+	if err := Render(&buf, FormatTable, tests, []string{"m1", "m2"}, results); err != nil {
+		t.Fatalf("Render() error = %v", err)
+	}
+	out := buf.String()
+
+	for _, want := range []string{
+		"== Discrimination: 1/2 scored tests separate models by >= 0.05; 1 unstable under repeat ==",
+		"sep",
+		"[0.40-1.00]", // unstable range in the score table cell
+	} {
+		if !strings.Contains(out, want) {
+			t.Errorf("output missing %q:\n%s", want, out)
+		}
+	}
+	if strings.Contains(out, "\nsame\t") {
+		// "same" appears in the per-test table; ensure it is NOT in the
+		// discrimination rows by checking the section body only.
+		section := out[strings.Index(out, "== Discrimination"):]
+		if strings.Contains(section, "same") {
+			t.Errorf("non-discriminating test listed in discrimination section:\n%s", section)
+		}
+	}
+}
+
+func TestDiscrimination_JSONStatsAndAttemptRoundTrip(t *testing.T) {
+	tests := []testkit.Test{{ID: "t1", Category: "c", Subcategory: "s"}}
+	results := []runner.Result{
+		{Model: "m1", TestID: "t1", Score: eval.Score{Value: 1}},
+		{Model: "m2", TestID: "t1", Score: eval.Score{Value: 0.5}},
+		{Model: "m1", TestID: "t1", Attempt: 1, Score: eval.Score{Value: 0.5}},
+	}
+
+	var buf bytes.Buffer
+	if err := Render(&buf, FormatJSON, tests, []string{"m1", "m2"}, results); err != nil {
+		t.Fatalf("Render() error = %v", err)
+	}
+	var doc jsonArtifact
+	if err := json.Unmarshal(buf.Bytes(), &doc); err != nil {
+		t.Fatalf("unmarshal: %v", err)
+	}
+	if len(doc.Stats) != 1 {
+		t.Fatalf("len(stats) = %d, want 1", len(doc.Stats))
+	}
+	st := doc.Stats[0]
+	if !st.Unstable {
+		t.Error("Unstable = false, want true (attempts 1.0 vs 0.5)")
+	}
+	if st.Spread < 0.24 || st.Spread > 0.26 {
+		t.Errorf("Spread = %v, want ~0.25 (m1 mean 0.75 vs m2 0.5)", st.Spread)
+	}
+
+	// Attempt indices survive the round trip for compare.
+	var attempts []AttemptResult
+	attempts = append(attempts, doc.Results...)
+	if attempts[2].Attempt != 1 {
+		t.Errorf("attempt = %d, want 1", attempts[2].Attempt)
+	}
+}

internal/report/health.go

@@ -0,0 +1,222 @@
+package report
+
+import (
+	"fmt"
+	"io"
+	"sort"
+)
+
+// PerfectScore is the score at which a (model, test) cell counts as a
+// perfect pass. Partial-credit evaluators score between 0 and 1; anything
+// below this is not a perfect pass.
+const PerfectScore = 0.999
+
+// SaturationRate is the share of a test's scored cells that must be
+// perfect for the test to count as saturated - passed by every model it
+// was run against, every attempt. Saturated tests carry no signal about
+// relative capability; they only detect outright regressions.
+const SaturationRate = 0.9
+
+// SubHealth is the suite-health rollup for one category/subcategory pair,
+// pooled across one or more artifacts (so single-model probe runs can be
+// combined with a full multi-model run). Cells are attempt-aggregated
+// (model, test) means: a pair counts once, taking its WORST artifact -
+// health asks "can any observed run fail this test?", so pooling over
+// re-runs must not let a lucky second attempt hide a real failure.
+type SubHealth struct {
+	Category    string
+	Subcategory string
+	WeakTests   []string // non-saturated test IDs, sorted - the signal-carriers
+	Tests       int      // distinct tests with at least one scored cell
+	Cells       int      // scored (model, test) cells
+	Perfect     int      // cells at PerfectScore or above
+	Saturated   int      // tests perfect in >= SaturationRate of their cells
+	Unstable    int      // tests whose repeated attempts disagreed
+	Truncated   int      // tests with a truncated cell
+}
+
+// PerfectRate is the share of scored cells that are perfect.
+func (h SubHealth) PerfectRate() float64 {
+	if h.Cells == 0 {
+		return 0
+	}
+	return float64(h.Perfect) / float64(h.Cells)
+}
+
+// HealthReport pools every subcategory present in the artifacts.
+type HealthReport struct {
+	Subcategories []SubHealth
+	Models        []string
+	Tests         int // distinct scored tests across all subcategories
+	Cells         int
+	Perfect       int
+	Saturated     int
+	Artifacts     int
+}
+
+// healthTestID scopes a test ID by model for cell pooling.
+type healthCellID struct{ model, testID string }
+
+// healthTest accumulates one test's cells across artifacts.
+type healthTest struct {
+	cells                 map[healthCellID]float64
+	category, subcategory string
+	unstable, truncated   bool
+}
+
+// AuditHealth pools artifacts into a per-subcategory health view.
+// Errored and skipped attempts never form cells (meanAttempts already
+// separates them by Status); truncated attempts follow the same
+// exclusion policy as compare. Per-test unstable/truncated flags are
+// OR-ed across artifacts: once flaky, always flagged.
+func AuditHealth(artifacts []Artifact) HealthReport {
+	agg := map[string]*healthTest{} // key: testID
+	modelSet := map[string]bool{}
+
+	for _, a := range artifacts {
+		for _, c := range meanAttempts(a.Results) {
+			if c.Status != "" && c.Status != "scored" {
+				continue
+			}
+			modelSet[c.Model] = true
+			ht := agg[c.TestID]
+			if ht == nil {
+				ht = &healthTest{category: c.Category, subcategory: c.Subcategory, cells: map[healthCellID]float64{}}
+				if ht.category == "" {
+					ht.category = "unknown"
+				}
+				if ht.subcategory == "" {
+					ht.subcategory = "unknown"
+				}
+				agg[c.TestID] = ht
+			}
+			id := healthCellID{c.Model, c.TestID}
+			if prev, ok := ht.cells[id]; !ok || c.BaselineMean < prev {
+				// Pool the worst observed mean for this cell. On the
+				// first artifact BaselineMean is the current run's mean.
+				ht.cells[id] = c.BaselineMean
+			}
+		}
+		for _, s := range a.Stats {
+			if ht, ok := agg[s.TestID]; ok {
+				ht.unstable = ht.unstable || s.Unstable
+				ht.truncated = ht.truncated || s.Truncated
+			}
+		}
+	}
+
+	models := make([]string, 0, len(modelSet))
+	for m := range modelSet {
+		models = append(models, m)
+	}
+	sort.Strings(models)
+
+	bySub := map[string]*SubHealth{}
+	var tot, totCells, totPerfect, totSat int
+	for testID, ht := range agg {
+		sk := ht.category + "/" + ht.subcategory
+		sh := bySub[sk]
+		if sh == nil {
+			sh = &SubHealth{Category: ht.category, Subcategory: ht.subcategory}
+			bySub[sk] = sh
+		}
+		sh.Tests++
+		tot++
+		var perfect int
+		for _, s := range ht.cells {
+			sh.Cells++
+			totCells++
+			if s >= PerfectScore {
+				perfect++
+				sh.Perfect++
+				totPerfect++
+			}
+		}
+		if sh.Cells > 0 && ht.unstable {
+			sh.Unstable++
+		}
+		if ht.truncated {
+			sh.Truncated++
+		}
+		if float64(perfect)/float64(len(ht.cells)) >= SaturationRate {
+			sh.Saturated++
+			totSat++
+		} else {
+			sh.WeakTests = append(sh.WeakTests, testID)
+		}
+	}
+
+	report := HealthReport{Artifacts: len(artifacts), Models: models, Tests: tot, Cells: totCells, Perfect: totPerfect, Saturated: totSat}
+	for _, sh := range bySub {
+		sort.Strings(sh.WeakTests)
+		report.Subcategories = append(report.Subcategories, *sh)
+	}
+	sort.Slice(report.Subcategories, func(i, j int) bool {
+		a, b := report.Subcategories[i], report.Subcategories[j]
+		if a.PerfectRate() != b.PerfectRate() {
+			return a.PerfectRate() > b.PerfectRate()
+		}
+		if a.Category != b.Category {
+			return a.Category < b.Category
+		}
+		return a.Subcategory < b.Subcategory
+	})
+	return report
+}
+
+// weakCap bounds listed weak tests per subcategory in the text report;
+// the counts stay accurate regardless.
+const weakCap = 8
+
+// RenderHealth writes the audit: headline numbers, then one line per
+// subcategory sorted by perfect-cell rate descending (most saturated -
+// least informative - first), listing its non-saturated tests.
+func RenderHealth(w io.Writer, r HealthReport) error {
+	ew := &errWriter{w: w}
+	perfectRate := 0.0
+	if r.Cells > 0 {
+		perfectRate = float64(r.Perfect) / float64(r.Cells)
+	}
+	ew.println(fmt.Sprintf("== Suite health: %d tests, %d scored (model,test) cells over %d model(s), %d artifact(s) ==",
+		r.Tests, r.Cells, len(r.Models), r.Artifacts))
+	ew.println(fmt.Sprintf("perfect cells: %d/%d (%.1f%%) | saturated tests (no signal): %d/%d\n",
+		r.Perfect, r.Cells, perfectRate*100, r.Saturated, r.Tests))
+	if r.Saturated == r.Tests && r.Tests > 0 {
+		ew.println("every scored test is saturated: the suite cannot rank these models at all.")
+	}
+	ew.println(fmt.Sprintf("%-30s %5s %6s %8s %9s %8s %8s  WEAK TESTS",
+		"SUBCATEGORY", "TESTS", "CELLS", "PERFECT%", "SATURATED", "UNSTBL", "TRUNC"))
+	for _, sh := range r.Subcategories {
+		name := sh.Category + "/" + sh.Subcategory
+		weak := ""
+		if len(sh.WeakTests) > 0 {
+			weak = fmt.Sprintf("%d", len(sh.WeakTests))
+			if len(sh.WeakTests) > weakCap {
+				weak += ": " + joinN(sh.WeakTests, weakCap) + ", ..."
+			} else {
+				weak += ": " + joinAll(sh.WeakTests)
+			}
+		}
+		ew.println(fmt.Sprintf("%-30s %5d %6d %8.1f %9d %8d %8d  %s",
+			name, sh.Tests, sh.Cells, sh.PerfectRate()*100, sh.Saturated, sh.Unstable, sh.Truncated, weak))
+	}
+	return ew.err
+}
+
+func joinN(xs []string, n int) string {
+	if len(xs) <= n {
+		return joinAll(xs)
+	}
+	return joinAll(xs[:n])
+}
+
+func joinAll(xs []string) string {
+	out := ""
+	for i, x := range xs {
+		if i > 0 {
+			out += ", "
+		}
+		out += x
+	}
+	return out
+}

internal/report/health_test.go

@@ -0,0 +1,200 @@
+package report
+
+import (
+	"bytes"
+	"strings"
+	"testing"
+)
+
+// scoredAttempt builds a scored attempt in subcategory (cat, sub).
+func scoredAttempt(model, testID, cat, sub string, score float64) AttemptResult {
+	return AttemptResult{Model: model, TestID: testID, Category: cat, Subcategory: sub, Score: score}
+}
+
+func subHealth(t *testing.T, r HealthReport, cat, sub string) SubHealth {
+	t.Helper()
+	for _, sh := range r.Subcategories {
+		if sh.Category == cat && sh.Subcategory == sub {
+			return sh
+		}
+	}
+	t.Fatalf("subcategory %s/%s missing from report", cat, sub)
+	return SubHealth{}
+}
+
+func TestAuditHealth_SaturatedVersusWeakTests(t *testing.T) {
+	// a: perfect for both models => saturated, no signal.
+	// b: model m2 fails => weak, carries signal.
+	art := Artifact{Results: []AttemptResult{
+		scoredAttempt("m1", "a", "c", "s", 1.0),
+		scoredAttempt("m2", "a", "c", "s", 1.0),
+		scoredAttempt("m1", "b", "c", "s", 1.0),
+		scoredAttempt("m2", "b", "c", "s", 0.0),
+	}}
+	r := AuditHealth([]Artifact{art})
+
+	sh := subHealth(t, r, "c", "s")
+	if sh.Tests != 2 || sh.Cells != 4 {
+		t.Fatalf("tests/cells = %d/%d, want 2/4", sh.Tests, sh.Cells)
+	}
+	if sh.Perfect != 3 {
+		t.Errorf("perfect = %d, want 3", sh.Perfect)
+	}
+	if sh.Saturated != 1 {
+		t.Errorf("saturated = %d, want 1 (only test a)", sh.Saturated)
+	}
+	if len(sh.WeakTests) != 1 || sh.WeakTests[0] != "b" {
+		t.Errorf("weak = %v, want [b]", sh.WeakTests)
+	}
+	if r.Saturated != 1 || r.Tests != 2 {
+		t.Errorf("report totals = %d/%d, want 1/2", r.Saturated, r.Tests)
+	}
+	if len(r.Models) != 2 || r.Models[0] != "m1" {
+		t.Errorf("models = %v, want [m1 m2]", r.Models)
+	}
+}
+
+func TestAuditHealth_PoolsWorstCellAcrossArtifacts(t *testing.T) {
+	// A flaky test that passes on a second artifact must still count as
+	// weak: health asks whether any observed run failed it.
+	a1 := Artifact{Results: []AttemptResult{
+		scoredAttempt("m1", "flaky", "c", "s", 0.0),
+	}}
+	a2 := Artifact{Results: []AttemptResult{
+		scoredAttempt("m1", "flaky", "c", "s", 1.0),
+	}}
+	r := AuditHealth([]Artifact{a1, a2})
+
+	sh := subHealth(t, r, "c", "s")
+	if sh.Saturated != 0 || len(sh.WeakTests) != 1 {
+		t.Errorf("saturated=%d weak=%v, want 0/[flaky] (worst cell pooled)", sh.Saturated, sh.WeakTests)
+	}
+	if sh.Cells != 1 {
+		t.Errorf("cells = %d, want 1 (same model,test pooled)", sh.Cells)
+	}
+}
+
+func TestAuditHealth_AveragesRepeatedAttemptsPerCell(t *testing.T) {
+	// Two attempts of one cell: mean 0.5 < PerfectScore => not perfect.
+	art := Artifact{Results: []AttemptResult{
+		scoredAttempt("m1", "t", "c", "s", 1.0),
+		scoredAttempt("m1", "t", "c", "s", 0.0),
+	}}
+	r := AuditHealth([]Artifact{art})
+
+	sh := subHealth(t, r, "c", "s")
+	if sh.Cells != 1 || sh.Perfect != 0 {
+		t.Errorf("cells/perfect = %d/%d, want 1/0 (attempt mean 0.5)", sh.Cells, sh.Perfect)
+	}
+}
+
+func TestAuditHealth_ErrorsAndSkipsExcluded(t *testing.T) {
+	art := Artifact{Results: []AttemptResult{
+		{Model: "m1", TestID: "boom", Category: "c", Subcategory: "s", Error: "404"},
+		{Model: "m1", TestID: "skip", Category: "c", Subcategory: "s", Skipped: true},
+		scoredAttempt("m1", "ok", "c", "s", 1.0),
+	}}
+	r := AuditHealth([]Artifact{art})
+
+	sh := subHealth(t, r, "c", "s")
+	if sh.Tests != 1 || sh.Cells != 1 || sh.Saturated != 1 {
+		t.Errorf("tests/cells/saturated = %d/%d/%d, want 1/1/1 (error+skip excluded)",
+			sh.Tests, sh.Cells, sh.Saturated)
+	}
+}
+
+func TestAuditHealth_FlagsUnstableAndTruncated(t *testing.T) {
+	art := Artifact{
+		Results: []AttemptResult{
+			scoredAttempt("m1", "u", "c", "s", 1.0),
+			scoredAttempt("m1", "t", "c", "s", 0.5),
+		},
+		Stats: []jsonStatRow{
+			{TestID: "u", Unstable: true},
+			{TestID: "t", Truncated: true},
+		},
+	}
+	r := AuditHealth([]Artifact{art})
+
+	sh := subHealth(t, r, "c", "s")
+	if sh.Unstable != 1 || sh.Truncated != 1 {
+		t.Errorf("unstable/truncated = %d/%d, want 1/1", sh.Unstable, sh.Truncated)
+	}
+}
+
+func TestAuditHealth_SaturationBoundary(t *testing.T) {
+	// 9 perfect cells + 1 failed cell = 0.9 rate = at SaturationRate =>
+	// saturated. 8/9 = 0.888 < rate => weak.
+	perfect9 := Artifact{}
+	for i := range 9 {
+		perfect9.Results = append(perfect9.Results, scoredAttempt("m"+string(rune('a'+i)), "t1", "c", "s", 1.0))
+	}
+	perfect9.Results = append(perfect9.Results, scoredAttempt("mj", "t1", "c", "s", 0.0))
+
+	if r := AuditHealth([]Artifact{perfect9}); r.Saturated != 1 {
+		t.Errorf("9/10 perfect: saturated = %d, want 1 (rate == SaturationRate)", r.Saturated)
+	}
+
+	eight := Artifact{}
+	for i := range 8 {
+		eight.Results = append(eight.Results, scoredAttempt("m"+string(rune('a'+i)), "t1", "c", "s", 1.0))
+	}
+	eight.Results = append(eight.Results,
+		scoredAttempt("mi", "t1", "c", "s", 0.0),
+		scoredAttempt("mj", "t1", "c", "s", 0.0))
+
+	if r := AuditHealth([]Artifact{eight}); r.Saturated != 0 {
+		t.Errorf("8/10 perfect: saturated = %d, want 0", r.Saturated)
+	}
+}
+
+func TestAuditHealth_SortsMostSaturatedFirst(t *testing.T) {
+	art := Artifact{Results: []AttemptResult{
+		scoredAttempt("m1", "hard", "c", "low", 0.0),
+		scoredAttempt("m1", "easy1", "c", "high", 1.0),
+		scoredAttempt("m1", "easy2", "c", "high", 1.0),
+	}}
+	r := AuditHealth([]Artifact{art})
+
+	if len(r.Subcategories) != 2 {
+		t.Fatalf("subcategories = %d, want 2", len(r.Subcategories))
+	}
+	if got := r.Subcategories[0].Subcategory; got != "high" {
+		t.Errorf("first row = %q, want most saturated %q", got, "high")
+	}
+}
+
+func TestRenderHealth_HeadlineAndWeakList(t *testing.T) {
+	art := Artifact{Results: []AttemptResult{
+		scoredAttempt("m1", "a", "c", "s", 1.0),
+		scoredAttempt("m1", "b", "c", "s", 0.0),
+	}}
+	var buf bytes.Buffer
+	if err := RenderHealth(&buf, AuditHealth([]Artifact{art})); err != nil {
+		t.Fatalf("RenderHealth() error = %v", err)
+	}
+	out := buf.String()
+	for _, want := range []string{"2 tests", "2 scored", "c/s", "50.0", "1: b"} {
+		if !strings.Contains(out, want) {
+			t.Errorf("output missing %q:\n%s", want, out)
+		}
+	}
+	// "a" is saturated: it must not appear anywhere in the weak list.
+	if strings.Contains(out, ": a") || strings.Contains(out, ", a,") || strings.HasSuffix(strings.TrimSpace(out), " a") {
+		t.Errorf("saturated test a should not be listed weak:\n%s", out)
+	}
+}
+
+func TestRenderHealth_AllSaturatedVerdict(t *testing.T) {
+	art := Artifact{Results: []AttemptResult{
+		scoredAttempt("m1", "a", "c", "s", 1.0),
+		scoredAttempt("m2", "a", "c", "s", 1.0),
+	}}
+	var buf bytes.Buffer
+	if err := RenderHealth(&buf, AuditHealth([]Artifact{art})); err != nil {
+		t.Fatalf("RenderHealth() error = %v", err)
+	}
+	if !strings.Contains(buf.String(), "cannot rank these models") {
+		t.Errorf("missing all-saturated verdict:\n%s", buf.String())
+	}
+}

internal/report/json.go

@@ -2,14 +2,17 @@ package report
 
 import (
 	"encoding/json"
+	"fmt"
 	"io"
+	"math"
+	"os"
 
 	"github.com/lukaszraczylo/llm-testbench/internal/runner"
 	"github.com/lukaszraczylo/llm-testbench/internal/testkit"
 )
 
-// jsonResult is the JSON-serializable projection of a runner.Result, joined
-// with its test's category/subcategory for self-contained output.
+// jsonResult is the JSON-serializable projection of one runner.Result
+// (one attempt), joined with its test's category/subcategory.
 type jsonResult struct {
 	Model            string  `json:"model"`
 	TestID           string  `json:"test_id"`
@@ -26,16 +29,43 @@ type jsonResult struct {
 	Tokens           int     `json:"tokens"`
 	Skipped          bool    `json:"skipped"`
 	Truncated        bool    `json:"truncated"`
+	Attempt          int     `json:"attempt,omitempty"`
 }
 
-// renderJSON dumps raw results as a JSON array, joined with test metadata.
-func renderJSON(w io.Writer, tests []testkit.Test, results []runner.Result) error {
+// jsonArtifact is the --out / --format json document: every raw attempt
+// plus the per-test discrimination rollup, so downstream tooling
+// (compare) can re-derive stability without re-running the suite.
+type jsonArtifact struct {
+	Results []jsonResult  `json:"results"`
+	Stats   []jsonStatRow `json:"stats,omitempty"`
+}
+
+// jsonStatRow is the JSON projection of testStat.
+type jsonStatRow struct {
+	TestID      string  `json:"test_id"`
+	Category    string  `json:"category"`
+	Subcategory string  `json:"subcategory"`
+	Mean        float64 `json:"mean"`
+	Spread      float64 `json:"spread,omitempty"`
+	Unstable    bool    `json:"unstable,omitempty"`
+	Truncated   bool    `json:"truncated,omitempty"`
+}
+
+// renderJSON dumps the run as a JSON artifact document.
+func renderJSON(w io.Writer, tests []testkit.Test, models []string, results []runner.Result) error {
+	artifact := buildArtifact(tests, models, results)
+	enc := json.NewEncoder(w)
+	enc.SetIndent("", "  ")
+	return enc.Encode(artifact)
+}
+
+func buildArtifact(tests []testkit.Test, models []string, results []runner.Result) jsonArtifact {
 	testByID := make(map[string]testkit.Test, len(tests))
 	for _, t := range tests {
 		testByID[t.ID] = t
 	}
 
-	out := make([]jsonResult, 0, len(results))
+	out := jsonArtifact{Results: make([]jsonResult, 0, len(results))}
 	for _, r := range results {
 		t := testByID[r.TestID]
 		jr := jsonResult{
@@ -53,14 +83,79 @@ func renderJSON(w io.Writer, tests []testkit.Test, results []runner.Result) erro
 			PromptTokens:     r.PromptTokens,
 			CompletionTokens: r.CompletionTokens,
 			Tokens:           r.TotalTokens(),
+			Attempt:          r.Attempt,
 		}
 		if r.Err != nil {
 			jr.Error = r.Err.Error()
 		}
-		out = append(out, jr)
+		out.Results = append(out.Results, jr)
 	}
 
-	enc := json.NewEncoder(w)
-	enc.SetIndent("", "  ")
-	return enc.Encode(out)
+	idx := indexResults(results)
+	for _, st := range testStats(tests, models, idx) {
+		if !st.scored {
+			continue
+		}
+		row := jsonStatRow{
+			TestID:      st.testID,
+			Category:    st.category,
+			Subcategory: st.subcategory,
+			Mean:        round6(st.mean),
+			Unstable:    st.unstable,
+			Truncated:   st.truncated,
+		}
+		if !math.IsNaN(st.spread) {
+			row.Spread = round6(st.spread)
+		}
+		out.Stats = append(out.Stats, row)
+	}
+	return out
+}
+
+func round6(v float64) float64 {
+	return math.Round(v*1e6) / 1e6
+}
+
+// Artifact is a saved run document: every raw attempt plus the per-test
+// discrimination rollup. Exported so the compare command can consume it.
+type Artifact = jsonArtifact
+
+// AttemptResult is one saved attempt. Exported alias of jsonResult.
+type AttemptResult = jsonResult
+
+// LoadArtifact reads a --out JSON artifact written by a previous run.
+// Accepts both the current document shape and the legacy bare array of
+// results, so artifacts from earlier releases remain comparable.
+func LoadArtifact(path string) (jsonArtifact, error) {
+	b, err := os.ReadFile(path) //nolint:gosec // path is the operator's own `llmtest compare` argument, not external input
+	if err != nil {
+		return jsonArtifact{}, err
+	}
+
+	var probe struct {
+		Results []jsonResult `json:"results"`
+	}
+	if err := json.Unmarshal(b, &probe); err == nil && probe.Results != nil {
+		var a jsonArtifact
+		if err := json.Unmarshal(b, &a); err != nil {
+			return jsonArtifact{}, fmt.Errorf("%s: %w", path, err)
+		}
+		return a, nil
+	}
+
+	var legacy []jsonResult
+	if err := json.Unmarshal(b, &legacy); err != nil {
+		return jsonArtifact{}, fmt.Errorf("%s: not a llmtest json artifact: %w", path, err)
+	}
+	return jsonArtifact{Results: legacy}, nil
+}
+
+// WriteArtifact saves the run as a JSON artifact at path, for later
+// `llmtest compare` runs, regardless of the stdout report format.
+func WriteArtifact(path string, tests []testkit.Test, models []string, results []runner.Result) error {
+	b, err := json.MarshalIndent(buildArtifact(tests, models, results), "", "  ")
+	if err != nil {
+		return err
+	}
+	return os.WriteFile(path, append(b, '\n'), 0o644) //nolint:gosec // 0o644: the artifact is a report meant to be shared/committed, not a secret file
 }

internal/report/markdown.go

@@ -10,8 +10,8 @@ import (
 	"github.com/lukaszraczylo/llm-testbench/internal/testkit"
 )
 
-// renderMarkdown writes the same three tables as renderTable, formatted as
-// GitHub-flavored markdown tables.
+// renderMarkdown writes the same four sections as renderTable, formatted
+// as GitHub-flavored markdown tables.
 func renderMarkdown(w io.Writer, tests []testkit.Test, models []string, results []runner.Result) error {
 	idx := indexResults(results)
 	ew := &errWriter{w: w}
@@ -24,9 +24,9 @@ func renderMarkdown(w io.Writer, tests []testkit.Test, models []string, results
 	for _, t := range sortedTests(tests) {
 		row := []string{t.ID, t.Category, t.Subcategory}
 		for _, model := range models {
-			r, ok := idx[t.ID][model]
-			row = append(row, cellText(r, ok))
-			anyTruncated = anyTruncated || r.Truncated()
+			c := idx[t.ID][model]
+			row = append(row, cellText(c))
+			anyTruncated = anyTruncated || c.result.Truncated()
 		}
 		writeMarkdownRow(ew, row)
 	}
@@ -48,11 +48,16 @@ func renderMarkdown(w io.Writer, tests []testkit.Test, models []string, results
 		writeMarkdownRow(ew, row)
 	}
 
+	ew.println()
+	ew.println("## Discrimination")
+	ew.println()
+	writeDiscriminationRows(markdownPrinter{ew: ew}, testStats(tests, models, idx), len(models) > 1)
+
 	ew.println()
 	ew.println("## Model summary")
 	ew.println()
 	writeMarkdownHeader(ew, []string{"Model", "Mean", "Passed", "Partial", "Failed", "Errors", "Mean latency", "Total tokens", "Tok/s"})
-	for _, ms := range summarize(models, results) {
+	for _, ms := range summarize(models, idx) {
 		writeMarkdownRow(ew, []string{
 			ms.model,
 			meanCellText(ms.overallMean),
@@ -69,6 +74,18 @@ func renderMarkdown(w io.Writer, tests []testkit.Test, models []string, results
 	return ew.err
 }
 
+// markdownPrinter adapts writeDiscriminationRows to pipe-delimited rows.
+// GFM needs a blank line between the note paragraph and the table, so
+// header() emits one first.
+type markdownPrinter struct{ ew *errWriter }
+
+func (mp markdownPrinter) header(cells ...string) {
+	mp.ew.println()
+	writeMarkdownHeader(mp.ew, cells)
+}
+func (mp markdownPrinter) row(cells ...string) { writeMarkdownRow(mp.ew, cells) }
+func (mp markdownPrinter) note(text string)    { mp.ew.println(text) }
+
 func writeMarkdownHeader(ew *errWriter, cols []string) {
 	ew.println("| " + strings.Join(cols, " | ") + " |")
 	sep := make([]string, len(cols))

internal/report/report.go

@@ -9,6 +9,8 @@ import (
 	"sort"
 	"time"
 
+	"github.com/lukaszraczylo/llm-testbench/internal/eval"
+	"github.com/lukaszraczylo/llm-testbench/internal/llm"
 	"github.com/lukaszraczylo/llm-testbench/internal/runner"
 	"github.com/lukaszraczylo/llm-testbench/internal/testkit"
 )
@@ -35,22 +37,125 @@ func Render(w io.Writer, format Format, tests []testkit.Test, models []string, r
 	case FormatMarkdown:
 		return renderMarkdown(w, tests, models, results)
 	case FormatJSON:
-		return renderJSON(w, tests, results)
+		return renderJSON(w, tests, models, results)
 	default:
 		return fmt.Errorf("report: unknown format %q", format)
 	}
 }
 
-// resultIndex looks up a Result by testID then model.
-type resultIndex map[string]map[string]runner.Result
+// modelCell is the aggregate view of every attempt recorded for one
+// (testID, model) pair: with Config.Repeat = 1 this is the single result;
+// with repeats the Score is the mean over scored attempts, and minScore/
+// maxScore expose the within-model spread (instability) that repeats are
+// measured to find.
+type modelCell struct {
+	result     runner.Result // representative: last scored attempt, mean score patched in
+	minScore   float64
+	maxScore   float64
+	meanTokens float64
+	attempts   int
+	scored     int
+	skippedAll bool
+	errorAll   bool
+	ok         bool
+}
 
-func indexResults(results []runner.Result) resultIndex {
-	idx := make(resultIndex)
+// cellIndex looks up the aggregate cell for a testID then model.
+type cellIndex map[string]map[string]modelCell
+
+func indexResults(results []runner.Result) cellIndex {
+	type key struct{ testID, model string }
+	grouped := make(map[key][]runner.Result)
+	order := make([]key, 0, len(results))
 	for _, r := range results {
-		if idx[r.TestID] == nil {
-			idx[r.TestID] = make(map[string]runner.Result)
+		k := key{r.TestID, r.Model}
+		if _, seen := grouped[k]; !seen {
+			order = append(order, k)
 		}
-		idx[r.TestID][r.Model] = r
+		grouped[k] = append(grouped[k], r)
+	}
+
+	idx := make(cellIndex)
+	for _, k := range order {
+		attempts := grouped[k]
+		sort.Slice(attempts, func(i, j int) bool { return attempts[i].Attempt < attempts[j].Attempt })
+
+		c := modelCell{ok: true, attempts: len(attempts)}
+		// Truncation policy: a truncated attempt was scored on partial
+		// text, so its score measures the token budget as much as the
+		// model. Attempts that finished normally are therefore preferred
+		// for the cell mean (and for the min-max instability range); only
+		// when EVERY scored attempt was cut off does the mean fall back to
+		// the truncated scores. A cell is flagged "!" whenever any attempt
+		// was truncated, clean mean or not.
+		var sum, toks, cleanSum, cleanToks float64
+		var cleanN, scoredN int
+		var cleanMin, cleanMax float64
+		var anyTrunc bool
+		var last, lastClean runner.Result
+		for _, a := range attempts {
+			last = a
+			switch {
+			case a.Err != nil:
+				continue
+			case a.Score.Skipped:
+				continue
+			}
+			v := a.Score.Value
+			sum += v
+			toks += float64(a.TotalTokens())
+			scoredN++
+			if a.Truncated() {
+				anyTrunc = true
+				continue
+			}
+			if cleanN == 0 {
+				cleanMin, cleanMax = v, v
+			} else {
+				cleanMin = math.Min(cleanMin, v)
+				cleanMax = math.Max(cleanMax, v)
+			}
+			cleanSum += v
+			cleanToks += float64(a.TotalTokens())
+			cleanN++
+			lastClean = a
+		}
+		c.scored = scoredN
+
+		rep := last
+		if cleanN > 0 {
+			rep = lastClean
+		}
+		switch {
+		case scoredN > 0 && cleanN > 0:
+			rep.Score = eval.Score{
+				Value:  cleanSum / float64(cleanN),
+				Detail: lastClean.Score.Detail,
+			}
+			c.minScore, c.maxScore = cleanMin, cleanMax
+			c.meanTokens = cleanToks / float64(cleanN)
+			if anyTrunc {
+				rep.FinishReason = llm.FinishReasonLength
+			}
+		case scoredN > 0:
+			rep.Score = eval.Score{
+				Value:  sum / float64(scoredN),
+				Detail: last.Score.Detail,
+			}
+			c.minScore, c.maxScore = rep.Score.Value, rep.Score.Value
+			c.meanTokens = toks / float64(scoredN)
+			rep.FinishReason = llm.FinishReasonLength
+		case c.attempts > 0 && last.Score.Skipped:
+			c.skippedAll = true
+		case c.attempts > 0 && last.Err != nil:
+			c.errorAll = true
+		}
+
+		c.result = rep
+		if idx[k.testID] == nil {
+			idx[k.testID] = make(map[string]modelCell)
+		}
+		idx[k.testID][k.model] = c
 	}
 	return idx
 }
@@ -72,34 +177,41 @@ func sortedTests(tests []testkit.Test) []testkit.Test {
 	return out
 }
 
-// truncatedSuffix marks a cell whose underlying response was cut off by
-// the token budget (finish_reason=length): the score may not reflect what
-// the model would have answered with a larger budget. truncatedLegend is
-// the one-line explanation printed once per table/markdown report.
+// truncatedSuffix marks a cell where any attempt was cut off by the token
+// budget (finish_reason=length). Per the truncation policy in
+// indexResults, the cell mean then comes from the attempts that finished
+// normally; "!" warns that at least one sample was discarded (or, when no
+// attempt finished, that the whole mean is partial-text based).
+// truncatedLegend is the one-line explanation printed once per
+// table/markdown report.
 const (
 	truncatedSuffix = "!"
-	truncatedLegend = "! = response truncated by the token budget (finish_reason=length); increase max_tokens_default to avoid"
+	truncatedLegend = "! = some attempt was truncated by the token budget (finish_reason=length); the mean uses only attempts that finished, increase max_tokens_default if every attempt is flagged"
 )
 
 // cellText renders one (test, model) cell for the score table: "score
 // (Ntok)" for a scored result, or ERR/skip/N/A otherwise, with a
-// truncatedSuffix appended when the response was cut off by the token
-// budget.
-func cellText(r runner.Result, found bool) string {
-	if !found {
+// truncatedSuffix appended when any attempt was cut off by the token
+// budget. With repeat attempts, a "[min-max]" range is appended when the
+// attempts disagreed, marking the cell unstable.
+func cellText(c modelCell) string {
+	if !c.ok {
+		return "N/A"
+	}
+	switch {
+	case c.errorAll:
+		return "ERR"
+	case c.skippedAll:
+		return "skip"
+	case c.scored == 0:
 		return "N/A"
 	}
 
-	var text string
-	switch {
-	case r.Err != nil:
-		text = "ERR"
-	case r.Score.Skipped:
-		text = "skip"
-	default:
-		text = fmt.Sprintf("%.2f (%dtok)", r.Score.Value, r.TotalTokens())
+	text := fmt.Sprintf("%.2f (%dtok)", c.result.Score.Value, int(math.Round(c.meanTokens)))
+	if c.attempts > 1 && c.maxScore > c.minScore {
+		text += fmt.Sprintf(" [%0.2f-%0.2f]", c.minScore, c.maxScore)
 	}
-	if r.Truncated() {
+	if c.result.Truncated() {
 		text += truncatedSuffix
 	}
 	return text
@@ -112,7 +224,7 @@ type categoryMean struct {
 	category string
 }
 
-func categoryRollup(tests []testkit.Test, models []string, idx resultIndex) []categoryMean {
+func categoryRollup(tests []testkit.Test, models []string, idx cellIndex) []categoryMean {
 	order := make([]string, 0)
 	byCategory := make(map[string][]testkit.Test)
 	for _, t := range sortedTests(tests) {
@@ -129,11 +241,11 @@ func categoryRollup(tests []testkit.Test, models []string, idx resultIndex) []ca
 			var sum float64
 			var n int
 			for _, t := range byCategory[cat] {
-				r, ok := idx[t.ID][model]
-				if !ok || r.Err != nil || r.Score.Skipped {
+				c, ok := idx[t.ID][model]
+				if !ok || c.errorAll || c.skippedAll || c.scored == 0 {
 					continue
 				}
-				sum += r.Score.Value
+				sum += c.result.Score.Value
 				n++
 			}
 			if n > 0 {
@@ -191,49 +303,49 @@ type modelSummary struct {
 	avgTokPerSec float64 // math.NaN() when no timed, non-skipped data
 }
 
-func summarize(models []string, results []runner.Result) []modelSummary {
-	byModel := make(map[string][]runner.Result, len(models))
-	for _, r := range results {
-		byModel[r.Model] = append(byModel[r.Model], r)
-	}
-
+// summarize aggregates one model's results per test (attempts already
+// merged by indexResults), so pass/partial/fail counts stay per-test even
+// with Config.Repeat > 1.
+func summarize(models []string, idx cellIndex) []modelSummary {
 	out := make([]modelSummary, 0, len(models))
 	for _, model := range models {
 		ms := modelSummary{model: model}
-		var scoreSum float64
-		var scoredCount int
+		var scoreSum, latencySecondsSum, meanTokenSum float64
+		var scoredCount, latencyCount int
 		var latencySum time.Duration
-		var latencyCount int
-		var completionTokenSum int
-		var latencySecondsSum float64
 
-		for _, r := range byModel[model] {
-			ms.totalTokens += r.TotalTokens()
-			switch {
-			case r.Err != nil:
-				ms.errors++
+		for _, cells := range idx {
+			c, ok := cells[model]
+			if !ok {
 				continue
-			case r.Score.Skipped:
+			}
+			meanTokenSum += c.meanTokens * float64(max(c.scored, 1))
+			if c.errorAll {
+				ms.errors += c.attempts
+				continue
+			}
+			if c.skippedAll || c.scored == 0 {
 				continue
 			}
 
-			scoreSum += r.Score.Value
+			v := c.result.Score.Value
+			scoreSum += v
 			scoredCount++
-			latencySum += r.Latency
+			latencySum += c.result.Latency
 			latencyCount++
-			completionTokenSum += r.CompletionTokens
-			latencySecondsSum += r.Latency.Seconds()
+			latencySecondsSum += c.result.Latency.Seconds()
 
 			switch {
-			case r.Score.Value >= passThreshold:
+			case v >= passThreshold:
 				ms.passed++
-			case r.Score.Value <= 0:
+			case v <= 0:
 				ms.failed++
 			default:
 				ms.partial++
 			}
 		}
 
+		ms.totalTokens = int(math.Round(meanTokenSum))
 		if scoredCount > 0 {
 			ms.overallMean = scoreSum / float64(scoredCount)
 			ms.meanLatency = latencySum / time.Duration(latencyCount)
@@ -241,11 +353,11 @@ func summarize(models []string, results []runner.Result) []modelSummary {
 			ms.overallMean = math.NaN()
 		}
 
-		// Aggregate ratio (sum of completion tokens over sum of elapsed
-		// seconds), not a mean of per-test ratios: robust to a handful of
-		// very short responses skewing a per-test-averaged rate.
+		// Aggregate ratio (completion tokens over elapsed seconds for the
+		// representative attempt), not a mean of per-test ratios: robust to
+		// a handful of very short responses skewing a per-test-averaged rate.
 		if latencySecondsSum > 0 {
-			ms.avgTokPerSec = float64(completionTokenSum) / latencySecondsSum
+			ms.avgTokPerSec = float64(ms.totalTokens) / latencySecondsSum
 		} else {
 			ms.avgTokPerSec = math.NaN()
 		}

internal/report/report_test.go

@@ -10,6 +10,7 @@ import (
 	"time"
 
 	"github.com/lukaszraczylo/llm-testbench/internal/eval"
+	"github.com/lukaszraczylo/llm-testbench/internal/llm"
 	"github.com/lukaszraczylo/llm-testbench/internal/runner"
 	"github.com/lukaszraczylo/llm-testbench/internal/testkit"
 )
@@ -33,6 +34,47 @@ func sampleResults() []runner.Result {
 	}
 }
 
+// Truncation policy: attempts cut off by the token budget are excluded
+// from a cell's mean when any attempt finished normally; the cell is still
+// flagged "!" so the discarded sample is visible. When every scored
+// attempt was truncated, the mean falls back to the partial-text scores.
+func TestIndexResults_TruncatedAttemptExcludedFromMean(t *testing.T) {
+	idx := indexResults([]runner.Result{
+		{Model: "m1", TestID: "t", Attempt: 0, Score: eval.Score{Value: 1}, PromptTokens: 4},
+		{Model: "m1", TestID: "t", Attempt: 1, Score: eval.Score{Value: 0}, FinishReason: llm.FinishReasonLength, PromptTokens: 100},
+	})
+	c := idx["t"]["m1"]
+	if got := c.result.Score.Value; got != 1 {
+		t.Errorf("mean = %v, want 1 (clean attempts only)", got)
+	}
+	if !c.result.Truncated() {
+		t.Error("cell should be flagged truncated")
+	}
+	if c.minScore != 1 || c.maxScore != 1 {
+		t.Errorf("min/max = %v/%v, want 1/1 (truncated attempt excluded from instability range)", c.minScore, c.maxScore)
+	}
+	if got := int(c.meanTokens); got != 4 {
+		t.Errorf("meanTokens = %d, want 4 (clean attempts only)", got)
+	}
+}
+
+func TestIndexResults_AllAttemptsTruncatedFallBackToPartialScores(t *testing.T) {
+	idx := indexResults([]runner.Result{
+		{Model: "m1", TestID: "t", Attempt: 0, Score: eval.Score{Value: 0.4}, FinishReason: llm.FinishReasonLength, PromptTokens: 10},
+		{Model: "m1", TestID: "t", Attempt: 1, Score: eval.Score{Value: 0.6}, FinishReason: llm.FinishReasonLength, PromptTokens: 20},
+	})
+	c := idx["t"]["m1"]
+	if got := c.result.Score.Value; math.Abs(got-0.5) > 1e-9 {
+		t.Errorf("mean = %v, want 0.5 (fallback over all-truncated attempts)", got)
+	}
+	if !c.result.Truncated() {
+		t.Error("cell should be flagged truncated")
+	}
+	if c.scored != 2 {
+		t.Errorf("scored = %d, want 2", c.scored)
+	}
+}
+
 func TestRenderTable_ContainsExpectedCells(t *testing.T) {
 	var buf bytes.Buffer
 	if err := Render(&buf, FormatTable, sampleTests(), []string{"m1", "m2"}, sampleResults()); err != nil {
@@ -85,10 +127,11 @@ func TestRenderJSON_DumpsRawResults(t *testing.T) {
 		t.Fatalf("Render() error = %v", err)
 	}
 
-	var out []jsonResult
-	if err := json.Unmarshal(buf.Bytes(), &out); err != nil {
+	var doc jsonArtifact
+	if err := json.Unmarshal(buf.Bytes(), &doc); err != nil {
 		t.Fatalf("json.Unmarshal() error = %v; output:\n%s", err, buf.String())
 	}
+	out := doc.Results
 	if len(out) != len(sampleResults()) {
 		t.Fatalf("len(out) = %d, want %d", len(out), len(sampleResults()))
 	}
@@ -173,14 +216,14 @@ func TestRenderJSON_IncludesResponseTextFinishReasonAndTruncated(t *testing.T) {
 		t.Fatalf("Render() error = %v", err)
 	}
 
-	var out []jsonResult
-	if err := json.Unmarshal(buf.Bytes(), &out); err != nil {
+	var doc jsonArtifact
+	if err := json.Unmarshal(buf.Bytes(), &doc); err != nil {
 		t.Fatalf("json.Unmarshal() error = %v; output:\n%s", err, buf.String())
 	}
-	if len(out) != 1 {
-		t.Fatalf("len(out) = %d, want 1", len(out))
+	if len(doc.Results) != 1 {
+		t.Fatalf("len(results) = %d, want 1", len(doc.Results))
 	}
-	got := out[0]
+	got := doc.Results[0]
 	if got.ResponseText != "partial answer" {
 		t.Errorf("ResponseText = %q, want %q", got.ResponseText, "partial answer")
 	}
@@ -211,7 +254,7 @@ func TestSummarize_Buckets(t *testing.T) {
 		{Model: "m1", TestID: "d", Err: errors.New("fail")},
 		{Model: "m1", TestID: "e", Score: eval.Score{Skipped: true}},
 	}
-	summaries := summarize([]string{"m1"}, results)
+	summaries := summarize([]string{"m1"}, indexResults(results))
 	if len(summaries) != 1 {
 		t.Fatalf("summarize() len = %d, want 1", len(summaries))
 	}
@@ -236,7 +279,7 @@ func TestSummarize_AvgTokPerSec(t *testing.T) {
 		{Model: "m1", TestID: "err", Err: errors.New("fail"), Latency: time.Second, CompletionTokens: 999},
 		{Model: "m1", TestID: "skipped", Score: eval.Score{Skipped: true}, Latency: time.Second, CompletionTokens: 999},
 	}
-	summaries := summarize([]string{"m1"}, results)
+	summaries := summarize([]string{"m1"}, indexResults(results))
 	ms := summaries[0]
 
 	want := 110.0 / 2.1
@@ -250,7 +293,7 @@ func TestSummarize_AvgTokPerSec_NoDataIsNaN(t *testing.T) {
 		{Model: "m1", TestID: "err", Err: errors.New("fail")},
 		{Model: "m1", TestID: "skipped", Score: eval.Score{Skipped: true}},
 	}
-	summaries := summarize([]string{"m1"}, results)
+	summaries := summarize([]string{"m1"}, indexResults(results))
 	if !math.IsNaN(summaries[0].avgTokPerSec) {
 		t.Errorf("avgTokPerSec = %v, want NaN when there is no timed data", summaries[0].avgTokPerSec)
 	}

internal/report/table.go

@@ -26,8 +26,8 @@ func (ew *errWriter) println(args ...any) {
 	_, ew.err = fmt.Fprintln(ew.w, args...)
 }
 
-// renderTable writes three plain-text tables: per-test scores, category
-// rollup, and model summary.
+// renderTable writes four plain-text tables: per-test scores, category
+// rollup, discrimination/stability rollup, and model summary.
 func renderTable(w io.Writer, tests []testkit.Test, models []string, results []runner.Result) error {
 	idx := indexResults(results)
 	tw := tabwriter.NewWriter(w, 0, 4, 2, ' ', 0)
@@ -37,7 +37,9 @@ func renderTable(w io.Writer, tests []testkit.Test, models []string, results []r
 	ew.println()
 	writeCategoryTable(ew, tests, models, idx)
 	ew.println()
-	writeSummaryTable(ew, models, results)
+	writeDiscriminationTable(ew, testStats(tests, models, idx), len(models) > 1)
+	ew.println()
+	writeSummaryTable(ew, models, idx)
 	if ew.err != nil {
 		return ew.err
 	}
@@ -45,7 +47,7 @@ func renderTable(w io.Writer, tests []testkit.Test, models []string, results []r
 	return tw.Flush()
 }
 
-func writeScoreTable(ew *errWriter, tests []testkit.Test, models []string, idx resultIndex) {
+func writeScoreTable(ew *errWriter, tests []testkit.Test, models []string, idx cellIndex) {
 	ew.println("== Per-test scores ==")
 	header := []string{"TEST", "CATEGORY", "SUBCATEGORY"}
 	header = append(header, models...)
@@ -55,9 +57,9 @@ func writeScoreTable(ew *errWriter, tests []testkit.Test, models []string, idx r
 	for _, t := range sortedTests(tests) {
 		row := []string{t.ID, t.Category, t.Subcategory}
 		for _, model := range models {
-			r, ok := idx[t.ID][model]
-			row = append(row, cellText(r, ok))
-			anyTruncated = anyTruncated || r.Truncated()
+			c := idx[t.ID][model]
+			row = append(row, cellText(c))
+			anyTruncated = anyTruncated || c.result.Truncated()
 		}
 		ew.println(strings.Join(row, "\t"))
 	}
@@ -66,7 +68,7 @@ func writeScoreTable(ew *errWriter, tests []testkit.Test, models []string, idx r
 	}
 }
 
-func writeCategoryTable(ew *errWriter, tests []testkit.Test, models []string, idx resultIndex) {
+func writeCategoryTable(ew *errWriter, tests []testkit.Test, models []string, idx cellIndex) {
 	ew.println("== Category rollup (mean score) ==")
 	header := append([]string{"CATEGORY"}, models...)
 	ew.println(strings.Join(header, "\t"))
@@ -80,11 +82,82 @@ func writeCategoryTable(ew *errWriter, tests []testkit.Test, models []string, id
 	}
 }
 
-func writeSummaryTable(ew *errWriter, models []string, results []runner.Result) {
+// writeDiscriminationTable lists the tests that actually measure
+// something: cross-model spread >= DiscriminationSpread, or attempts that
+// disagreed under --repeat. With a single model there is no spread to
+// speak of, so the non-passing tests are listed instead.
+func writeDiscriminationTable(ew *errWriter, stats []testStat, multiModel bool) {
+	writeDiscriminationRows(tablePrinter{ew: ew}, stats, multiModel)
+}
+
+// discriminationSink abstracts the tab (table) vs pipe (markdown) output
+// so the discrimination section is written once.
+type discriminationSink interface {
+	header(cells ...string)
+	row(cells ...string)
+	note(text string)
+}
+
+type tablePrinter struct{ ew *errWriter }
+
+func (tp tablePrinter) header(cells ...string) { tp.ew.println(strings.Join(cells, "\t")) }
+func (tp tablePrinter) row(cells ...string)    { tp.ew.println(strings.Join(cells, "\t")) }
+func (tp tablePrinter) note(text string)       { tp.ew.println(text) }
+
+func writeDiscriminationRows(sink discriminationSink, stats []testStat, multiModel bool) {
+	discriminating, unstable, scored := 0, 0, 0
+	for _, st := range stats {
+		if !st.scored {
+			continue
+		}
+		scored++
+		if st.unstable {
+			unstable++
+		}
+		if multiModel && !math.IsNaN(st.spread) && st.spread >= DiscriminationSpread {
+			discriminating++
+		}
+	}
+
+	if multiModel {
+		sink.note(fmt.Sprintf("== Discrimination: %d/%d scored tests separate models by >= %.2f; %d unstable under repeat ==",
+			discriminating, scored, DiscriminationSpread, unstable))
+	} else {
+		sink.note(fmt.Sprintf("== Discrimination: single model (no cross-model spread); %d unstable under repeat ==", unstable))
+	}
+
+	rows := interestingStats(stats, multiModel)
+	if len(rows) == 0 {
+		sink.note("no discriminating or unstable tests: every scored test passed identically for every model")
+		return
+	}
+
+	sink.header("TEST", "CATEGORY", "SUBCATEGORY", "MEAN", "SPREAD", "UNSTABLE", "TRUNC")
+	for _, st := range rows {
+		sink.row(
+			st.testID,
+			st.category,
+			st.subcategory,
+			meanCellText(st.mean),
+			meanCellText(st.spread),
+			yesNo(st.unstable),
+			yesNo(st.truncated),
+		)
+	}
+}
+
+func yesNo(b bool) string {
+	if b {
+		return "yes"
+	}
+	return "-"
+}
+
+func writeSummaryTable(ew *errWriter, models []string, idx cellIndex) {
 	ew.println("== Model summary ==")
 	ew.println(strings.Join([]string{"MODEL", "MEAN", "PASSED", "PARTIAL", "FAILED", "ERRORS", "MEAN_LATENCY", "TOTAL_TOKENS", "TOK/S"}, "\t"))
 
-	for _, ms := range summarize(models, results) {
+	for _, ms := range summarize(models, idx) {
 		row := []string{
 			ms.model,
 			meanCellText(ms.overallMean),