llm-testbench/internal/report/health_test.go

package report

import (
	"bytes"
	"strings"
	"testing"
)

// scoredAttempt builds a scored attempt in subcategory (cat, sub).
func scoredAttempt(model, testID, cat, sub string, score float64) AttemptResult {
	return AttemptResult{Model: model, TestID: testID, Category: cat, Subcategory: sub, Score: score}
}

func subHealth(t *testing.T, r HealthReport, cat, sub string) SubHealth {
	t.Helper()
	for _, sh := range r.Subcategories {
		if sh.Category == cat && sh.Subcategory == sub {
			return sh
		}
	}
	t.Fatalf("subcategory %s/%s missing from report", cat, sub)
	return SubHealth{}
}

func TestAuditHealth_SaturatedVersusWeakTests(t *testing.T) {
	// a: perfect for both models => saturated, no signal.
	// b: model m2 fails => weak, carries signal.
	art := Artifact{Results: []AttemptResult{
		scoredAttempt("m1", "a", "c", "s", 1.0),
		scoredAttempt("m2", "a", "c", "s", 1.0),
		scoredAttempt("m1", "b", "c", "s", 1.0),
		scoredAttempt("m2", "b", "c", "s", 0.0),
	}}
	r := AuditHealth([]Artifact{art})

	sh := subHealth(t, r, "c", "s")
	if sh.Tests != 2 || sh.Cells != 4 {
		t.Fatalf("tests/cells = %d/%d, want 2/4", sh.Tests, sh.Cells)
	}
	if sh.Perfect != 3 {
		t.Errorf("perfect = %d, want 3", sh.Perfect)
	}
	if sh.Saturated != 1 {
		t.Errorf("saturated = %d, want 1 (only test a)", sh.Saturated)
	}
	if len(sh.WeakTests) != 1 || sh.WeakTests[0] != "b" {
		t.Errorf("weak = %v, want [b]", sh.WeakTests)
	}
	if r.Saturated != 1 || r.Tests != 2 {
		t.Errorf("report totals = %d/%d, want 1/2", r.Saturated, r.Tests)
	}
	if len(r.Models) != 2 || r.Models[0] != "m1" {
		t.Errorf("models = %v, want [m1 m2]", r.Models)
	}
}

func TestAuditHealth_PoolsWorstCellAcrossArtifacts(t *testing.T) {
	// A flaky test that passes on a second artifact must still count as
	// weak: health asks whether any observed run failed it.
	a1 := Artifact{Results: []AttemptResult{
		scoredAttempt("m1", "flaky", "c", "s", 0.0),
	}}
	a2 := Artifact{Results: []AttemptResult{
		scoredAttempt("m1", "flaky", "c", "s", 1.0),
	}}
	r := AuditHealth([]Artifact{a1, a2})

	sh := subHealth(t, r, "c", "s")
	if sh.Saturated != 0 || len(sh.WeakTests) != 1 {
		t.Errorf("saturated=%d weak=%v, want 0/[flaky] (worst cell pooled)", sh.Saturated, sh.WeakTests)
	}
	if sh.Cells != 1 {
		t.Errorf("cells = %d, want 1 (same model,test pooled)", sh.Cells)
	}
}

func TestAuditHealth_AveragesRepeatedAttemptsPerCell(t *testing.T) {
	// Two attempts of one cell: mean 0.5 < PerfectScore => not perfect.
	art := Artifact{Results: []AttemptResult{
		scoredAttempt("m1", "t", "c", "s", 1.0),
		scoredAttempt("m1", "t", "c", "s", 0.0),
	}}
	r := AuditHealth([]Artifact{art})

	sh := subHealth(t, r, "c", "s")
	if sh.Cells != 1 || sh.Perfect != 0 {
		t.Errorf("cells/perfect = %d/%d, want 1/0 (attempt mean 0.5)", sh.Cells, sh.Perfect)
	}
}

func TestAuditHealth_ErrorsAndSkipsExcluded(t *testing.T) {
	art := Artifact{Results: []AttemptResult{
		{Model: "m1", TestID: "boom", Category: "c", Subcategory: "s", Error: "404"},
		{Model: "m1", TestID: "skip", Category: "c", Subcategory: "s", Skipped: true},
		scoredAttempt("m1", "ok", "c", "s", 1.0),
	}}
	r := AuditHealth([]Artifact{art})

	sh := subHealth(t, r, "c", "s")
	if sh.Tests != 1 || sh.Cells != 1 || sh.Saturated != 1 {
		t.Errorf("tests/cells/saturated = %d/%d/%d, want 1/1/1 (error+skip excluded)",
			sh.Tests, sh.Cells, sh.Saturated)
	}
}

func TestAuditHealth_FlagsUnstableAndTruncated(t *testing.T) {
	art := Artifact{
		Results: []AttemptResult{
			scoredAttempt("m1", "u", "c", "s", 1.0),
			scoredAttempt("m1", "t", "c", "s", 0.5),
		},
		Stats: []jsonStatRow{
			{TestID: "u", Unstable: true},
			{TestID: "t", Truncated: true},
		},
	}
	r := AuditHealth([]Artifact{art})

	sh := subHealth(t, r, "c", "s")
	if sh.Unstable != 1 || sh.Truncated != 1 {
		t.Errorf("unstable/truncated = %d/%d, want 1/1", sh.Unstable, sh.Truncated)
	}
}

func TestAuditHealth_SaturationBoundary(t *testing.T) {
	// 9 perfect cells + 1 failed cell = 0.9 rate = at SaturationRate =>
	// saturated. 8/9 = 0.888 < rate => weak.
	perfect9 := Artifact{}
	for i := range 9 {
		perfect9.Results = append(perfect9.Results, scoredAttempt("m"+string(rune('a'+i)), "t1", "c", "s", 1.0))
	}
	perfect9.Results = append(perfect9.Results, scoredAttempt("mj", "t1", "c", "s", 0.0))

	if r := AuditHealth([]Artifact{perfect9}); r.Saturated != 1 {
		t.Errorf("9/10 perfect: saturated = %d, want 1 (rate == SaturationRate)", r.Saturated)
	}

	eight := Artifact{}
	for i := range 8 {
		eight.Results = append(eight.Results, scoredAttempt("m"+string(rune('a'+i)), "t1", "c", "s", 1.0))
	}
	eight.Results = append(eight.Results,
		scoredAttempt("mi", "t1", "c", "s", 0.0),
		scoredAttempt("mj", "t1", "c", "s", 0.0))

	if r := AuditHealth([]Artifact{eight}); r.Saturated != 0 {
		t.Errorf("8/10 perfect: saturated = %d, want 0", r.Saturated)
	}
}

func TestAuditHealth_SortsMostSaturatedFirst(t *testing.T) {
	art := Artifact{Results: []AttemptResult{
		scoredAttempt("m1", "hard", "c", "low", 0.0),
		scoredAttempt("m1", "easy1", "c", "high", 1.0),
		scoredAttempt("m1", "easy2", "c", "high", 1.0),
	}}
	r := AuditHealth([]Artifact{art})

	if len(r.Subcategories) != 2 {
		t.Fatalf("subcategories = %d, want 2", len(r.Subcategories))
	}
	if got := r.Subcategories[0].Subcategory; got != "high" {
		t.Errorf("first row = %q, want most saturated %q", got, "high")
	}
}

func TestRenderHealth_HeadlineAndWeakList(t *testing.T) {
	art := Artifact{Results: []AttemptResult{
		scoredAttempt("m1", "a", "c", "s", 1.0),
		scoredAttempt("m1", "b", "c", "s", 0.0),
	}}
	var buf bytes.Buffer
	if err := RenderHealth(&buf, AuditHealth([]Artifact{art})); err != nil {
		t.Fatalf("RenderHealth() error = %v", err)
	}
	out := buf.String()
	for _, want := range []string{"2 tests", "2 scored", "c/s", "50.0", "1: b"} {
		if !strings.Contains(out, want) {
			t.Errorf("output missing %q:\n%s", want, out)
		}
	}
	// "a" is saturated: it must not appear anywhere in the weak list.
	if strings.Contains(out, ": a") || strings.Contains(out, ", a,") || strings.HasSuffix(strings.TrimSpace(out), " a") {
		t.Errorf("saturated test a should not be listed weak:\n%s", out)
	}
}

func TestRenderHealth_AllSaturatedVerdict(t *testing.T) {
	art := Artifact{Results: []AttemptResult{
		scoredAttempt("m1", "a", "c", "s", 1.0),
		scoredAttempt("m2", "a", "c", "s", 1.0),
	}}
	var buf bytes.Buffer
	if err := RenderHealth(&buf, AuditHealth([]Artifact{art})); err != nil {
		t.Fatalf("RenderHealth() error = %v", err)
	}
	if !strings.Contains(buf.String(), "cannot rank these models") {
		t.Errorf("missing all-saturated verdict:\n%s", buf.String())
	}
}