Update

internal/report/compare.go

@@ -0,0 +1,221 @@
+package report
+
+import (
+	"fmt"
+	"io"
+	"math"
+	"sort"
+	"strings"
+)
+
+// Comparison is the per-(model, test) diff between two saved artifacts.
+// Scores are means over attempts, so a test that flapped between runs
+// shows up as a regression on one attempt basis only if the mean moved.
+type Comparison struct {
+	Model        string
+	TestID       string
+	Category     string
+	Subcategory  string
+	Status       string
+	BaselineMean float64
+	CurrentMean  float64
+	Delta        float64
+}
+
+// RegressionDelta is the mean-score drop at which a test counts as a
+// regression. Below this, run-to-run sampling noise dominates.
+const RegressionDelta = 0.01
+
+type attemptKey struct{ model, testID string }
+
+// meanAttempts collapses saved attempts into per-(model,test) means and
+// carries category metadata from the first attempt that has it. Truncated
+// attempts are excluded from the mean when any attempt finished normally,
+// mirroring the truncation policy in indexResults; when every scored
+// attempt was truncated, the partial-text scores are used rather than
+// reporting no data.
+func meanAttempts(results []AttemptResult) map[attemptKey]Comparison {
+	type agg struct {
+		cat      string
+		sb       string
+		sum      float64
+		n        int
+		cleanSum float64
+		cleanN   int
+		errs     int
+	}
+	by := make(map[attemptKey]*agg)
+	order := make([]attemptKey, 0, len(results))
+	for _, r := range results {
+		k := attemptKey{r.Model, r.TestID}
+		a, seen := by[k]
+		if !seen {
+			a = &agg{}
+			by[k] = a
+			order = append(order, k)
+		}
+		if a.cat == "" {
+			a.cat, a.sb = r.Category, r.Subcategory
+		}
+		if r.Error != "" {
+			a.errs++
+			continue
+		}
+		if r.Skipped {
+			continue
+		}
+		a.sum += r.Score
+		a.n++
+		if !r.Truncated {
+			a.cleanSum += r.Score
+			a.cleanN++
+		}
+	}
+
+	out := make(map[attemptKey]Comparison, len(by))
+	for _, k := range order {
+		a := by[k]
+		c := Comparison{Model: k.model, TestID: k.testID, Category: a.cat, Subcategory: a.sb}
+		switch {
+		case a.n == 0:
+			c.Status = "error"
+		case a.cleanN > 0:
+			c.BaselineMean = a.cleanSum / float64(a.cleanN)
+		default:
+			c.BaselineMean = a.sum / float64(a.n)
+		}
+		out[k] = c
+	}
+	return out
+}
+
+// CompareArtifacts diffs two artifacts per (model, test), sorted by
+// delta ascending (worst regressions first), then test ID.
+func CompareArtifacts(baseline, current Artifact) []Comparison {
+	base := meanAttempts(baseline.Results)
+	cur := meanAttempts(current.Results)
+
+	keys := make(map[attemptKey]bool, len(base)+len(cur))
+	for k := range base {
+		keys[k] = true
+	}
+	for k := range cur {
+		keys[k] = true
+	}
+
+	out := make([]Comparison, 0, len(keys))
+	for k := range keys {
+		b, hasB := base[k]
+		c, hasC := cur[k]
+		switch {
+		case hasB && !hasC:
+			b.CurrentMean = math.NaN()
+			b.Delta = math.NaN()
+			b.Status = "baseline-only"
+			out = append(out, b)
+		case !hasB && hasC:
+			c.BaselineMean = math.NaN()
+			c.Delta = math.NaN()
+			c.Status = "current-only"
+			out = append(out, c)
+		default:
+			m := Comparison{
+				Model:        k.model,
+				TestID:       k.testID,
+				Category:     c.Category,
+				Subcategory:  c.Subcategory,
+				BaselineMean: b.BaselineMean,
+				CurrentMean:  c.BaselineMean,
+			}
+			switch {
+			case b.Status == "error" || c.Status == "error":
+				m.Status = "error"
+				m.Delta = math.NaN()
+			default:
+				m.Delta = m.CurrentMean - m.BaselineMean
+				switch {
+				case m.Delta <= -RegressionDelta:
+					m.Status = "regression"
+				case m.Delta >= RegressionDelta:
+					m.Status = "improvement"
+				default:
+					m.Status = "stable"
+				}
+			}
+			out = append(out, m)
+		}
+	}
+
+	sort.Slice(out, func(i, j int) bool {
+		di, dj := out[i].Delta, out[j].Delta
+		if math.IsNaN(di) != math.IsNaN(dj) {
+			return math.IsNaN(di)
+		}
+		if math.IsNaN(di) && math.IsNaN(dj) {
+			if out[i].Status != out[j].Status {
+				return out[i].Status < out[j].Status
+			}
+			return out[i].TestID < out[j].TestID
+		}
+		if di != dj {
+			return di < dj
+		}
+		if out[i].TestID != out[j].TestID {
+			return out[i].TestID < out[j].TestID
+		}
+		return out[i].Model < out[j].Model
+	})
+	return out
+}
+
+// RenderCompare writes a comparison report: headline counts, then only
+// changed/missing rows (stable rows are summarized, not listed).
+func RenderCompare(w io.Writer, comparisons []Comparison) error {
+	ew := &errWriter{w: w}
+
+	var reg, imp, stable, added, removed, errored int
+	for _, c := range comparisons {
+		switch c.Status {
+		case "regression":
+			reg++
+		case "improvement":
+			imp++
+		case "baseline-only":
+			removed++
+		case "current-only":
+			added++
+		case "error":
+			errored++
+		default:
+			stable++
+		}
+	}
+
+	ew.println(fmt.Sprintf("== Compare: %d regressions, %d improvements, %d stable, %d errors, %d baseline-only, %d current-only ==",
+		reg, imp, stable, errored, removed, added))
+	ew.println()
+	ew.println("MODEL\tTEST\tCATEGORY\tSUBCATEGORY\tBASELINE\tCURRENT\tDELTA\tSTATUS")
+	shown := 0
+	for _, c := range comparisons {
+		if c.Status == "stable" {
+			continue
+		}
+		ew.println(strings.Join([]string{
+			c.Model, c.TestID, c.Category, c.Subcategory,
+			meanCellText(c.BaselineMean), meanCellText(c.CurrentMean),
+			deltaText(c.Delta), c.Status,
+		}, "\t"))
+		shown++
+	}
+	if shown == 0 {
+		ew.println("(no changed, missing, or erroring tests)")
+	}
+	return ew.err
+}
+
+func deltaText(v float64) string {
+	if math.IsNaN(v) {
+		return "N/A"
+	}
+	return fmt.Sprintf("%+.2f", v)
+}

internal/report/compare_test.go

@@ -0,0 +1,114 @@
+package report
+
+import (
+	"bytes"
+	"math"
+	"strings"
+	"testing"
+)
+
+func attempt(model, testID string, score float64) AttemptResult {
+	return AttemptResult{Model: model, TestID: testID, Category: "c", Subcategory: "s", Score: score}
+}
+
+func TestCompareArtifacts_ClassifiesRegressionImprovementStable(t *testing.T) {
+	baseline := Artifact{Results: []AttemptResult{
+		attempt("m1", "reg", 1.0),
+		attempt("m1", "imp", 0.2),
+		attempt("m1", "stable", 1.0),
+	}}
+	current := Artifact{Results: []AttemptResult{
+		attempt("m1", "reg", 0.4),
+		attempt("m1", "imp", 1.0),
+		attempt("m1", "stable", 0.995),
+	}}
+
+	got := CompareArtifacts(baseline, current)
+	byID := map[string]Comparison{}
+	for _, c := range got {
+		byID[c.TestID] = c
+	}
+	if byID["reg"].Status != "regression" || byID["reg"].Delta > -0.5 {
+		t.Errorf("reg = %+v, want regression delta ~ -0.6", byID["reg"])
+	}
+	if byID["imp"].Status != "improvement" {
+		t.Errorf("imp = %+v, want improvement", byID["imp"])
+	}
+	if byID["stable"].Status != "stable" {
+		t.Errorf("stable = %+v, want stable", byID["stable"])
+	}
+	// Worst delta first.
+	if got[0].TestID != "reg" {
+		t.Errorf("first row = %q, want worst regression %q", got[0].TestID, "reg")
+	}
+}
+
+func TestCompareArtifacts_AveragesAttemptsAndFlagsMembership(t *testing.T) {
+	baseline := Artifact{Results: []AttemptResult{
+		attempt("m1", "flaky", 1.0), attempt("m1", "flaky", 0.0), // mean 0.5
+		attempt("m1", "gone", 1.0),
+	}}
+	current := Artifact{Results: []AttemptResult{
+		attempt("m1", "flaky", 1.0), attempt("m1", "flaky", 1.0), // mean 1.0 => improvement
+		attempt("m1", "new", 0.5),
+	}}
+
+	got := CompareArtifacts(baseline, current)
+	byID := map[string]Comparison{}
+	for _, c := range got {
+		byID[c.TestID] = c
+	}
+	if byID["flaky"].Status != "improvement" || math.Abs(byID["flaky"].BaselineMean-0.5) > 1e-9 {
+		t.Errorf("flaky = %+v, want improvement from mean 0.5", byID["flaky"])
+	}
+	if byID["gone"].Status != "baseline-only" {
+		t.Errorf("gone = %+v, want baseline-only", byID["gone"])
+	}
+	if byID["new"].Status != "current-only" {
+		t.Errorf("new = %+v, want current-only", byID["new"])
+	}
+}
+
+func TestCompareArtifacts_TruncatedAttemptExcludedFromMean(t *testing.T) {
+	clean := func(score float64) AttemptResult {
+		return AttemptResult{Model: "m1", TestID: "t", Category: "c", Subcategory: "s", Score: score}
+	}
+	trunc := func(score float64) AttemptResult {
+		return AttemptResult{Model: "m1", TestID: "t", Category: "c", Subcategory: "s", Score: score, Truncated: true}
+	}
+	baseline := Artifact{Results: []AttemptResult{clean(1.0), trunc(0.0)}} // clean mean 1.0
+	current := Artifact{Results: []AttemptResult{trunc(0.4), trunc(0.6)}}  // all truncated => fallback 0.5
+
+	got := CompareArtifacts(baseline, current)
+	if len(got) != 1 {
+		t.Fatalf("comparisons = %d, want 1", len(got))
+	}
+	if math.Abs(got[0].BaselineMean-1.0) > 1e-9 {
+		t.Errorf("baseline mean = %v, want 1.0 (truncated attempt excluded)", got[0].BaselineMean)
+	}
+	if math.Abs(got[0].CurrentMean-0.5) > 1e-9 {
+		t.Errorf("current mean = %v, want 0.5 (all-truncated fallback)", got[0].CurrentMean)
+	}
+	if got[0].Status != "regression" {
+		t.Errorf("status = %q, want regression", got[0].Status)
+	}
+}
+
+func TestRenderCompare_HeadlineAndRows(t *testing.T) {
+	baseline := Artifact{Results: []AttemptResult{attempt("m1", "a", 1.0), attempt("m1", "b", 1.0)}}
+	current := Artifact{Results: []AttemptResult{attempt("m1", "a", 0.0), attempt("m1", "b", 1.0)}}
+
+	var buf bytes.Buffer
+	if err := RenderCompare(&buf, CompareArtifacts(baseline, current)); err != nil {
+		t.Fatalf("RenderCompare() error = %v", err)
+	}
+	out := buf.String()
+	for _, want := range []string{"1 regressions", "0 improvements", "1 stable", "a", "-1.00"} {
+		if !strings.Contains(out, want) {
+			t.Errorf("output missing %q:\n%s", want, out)
+		}
+	}
+	if strings.Contains(out, "\nb\t") {
+		t.Errorf("stable row b should be summarized, not listed:\n%s", out)
+	}
+}

internal/report/discrimination.go

@@ -0,0 +1,114 @@
+package report
+
+import (
+	"math"
+	"sort"
+
+	"github.com/lukaszraczylo/llm-testbench/internal/testkit"
+)
+
+// DiscriminationSpread is the cross-model score gap at which a test starts
+// to separate models. The last full run showed a median spread of 0.0
+// across 250 tests: nearly every test scored >=0.99 for every model, so
+// the suite measured nothing about relative capability. This number makes
+// that visible per test instead of averaged away.
+const DiscriminationSpread = 0.05
+
+// testStat is the per-test measurement rollup: cross-model discrimination
+// (spread, only meaningful with 2+ models), attempt stability (did
+// --repeat samples disagree), and aggregate score. Computed over the
+// attempt-aggregated cells so a flaky test shows as unstable rather than
+// as a random single score.
+type testStat struct {
+	testID      string
+	category    string
+	subcategory string
+	mean        float64 // mean over model cell means (scored cells only)
+	spread      float64 // max-min over model cell means; NaN with <2 scored models
+	unstable    bool    // some model's attempts disagreed beyond epsilon
+	truncated   bool    // some cell was token-budget truncated
+	scored      bool    // at least one scored cell
+}
+
+// unstableEpsilon absorbs float noise in averaged scores; a real
+// disagreement between attempts is orders of magnitude larger.
+const unstableEpsilon = 1e-9
+
+// testStats computes the per-test rollup for every test in tests, ordered
+// by category, subcategory, ID (same order as the score table).
+func testStats(tests []testkit.Test, models []string, idx cellIndex) []testStat {
+	out := make([]testStat, 0, len(tests))
+	for _, t := range sortedTests(tests) {
+		st := testStat{testID: t.ID, category: t.Category, subcategory: t.Subcategory, spread: math.NaN()}
+		var sum float64
+		var n int
+		var lo, hi float64
+		for _, model := range models {
+			c, ok := idx[t.ID][model]
+			if !ok || c.errorAll || c.skippedAll || c.scored == 0 {
+				continue
+			}
+			v := c.result.Score.Value
+			if n == 0 {
+				lo, hi = v, v
+			} else {
+				lo = math.Min(lo, v)
+				hi = math.Max(hi, v)
+			}
+			sum += v
+			n++
+			st.scored = true
+			if c.maxScore-c.minScore > unstableEpsilon {
+				st.unstable = true
+			}
+			st.truncated = st.truncated || c.result.Truncated()
+		}
+		if n > 1 {
+			st.spread = hi - lo
+			st.mean = sum / float64(n)
+		} else if n == 1 {
+			st.mean = sum
+		}
+		out = append(out, st)
+	}
+	return out
+}
+
+// interestingStats narrows testStats to rows worth reading: tests that
+// discriminate between models, tests whose repeated attempts disagreed,
+// and (when no cross-model data exists) tests that any model failed to
+// pass outright. Sorted by spread descending, unstable first within equal
+// spread.
+func interestingStats(stats []testStat, multiModel bool) []testStat {
+	var out []testStat
+	for _, st := range stats {
+		if !st.scored {
+			continue
+		}
+		switch {
+		case st.unstable:
+		case multiModel && !math.IsNaN(st.spread) && st.spread >= DiscriminationSpread:
+		case !multiModel && st.mean < passThreshold:
+		default:
+			continue
+		}
+		out = append(out, st)
+	}
+	sort.SliceStable(out, func(i, j int) bool {
+		if out[i].unstable != out[j].unstable {
+			return out[i].unstable
+		}
+		si, sj := out[i].spread, out[j].spread
+		if math.IsNaN(si) {
+			si = -1
+		}
+		if math.IsNaN(sj) {
+			sj = -1
+		}
+		if si != sj {
+			return si > sj
+		}
+		return out[i].testID < out[j].testID
+	})
+	return out
+}

internal/report/discrimination_test.go

@@ -0,0 +1,86 @@
+package report
+
+import (
+	"bytes"
+	"encoding/json"
+	"strings"
+	"testing"
+
+	"github.com/lukaszraczylo/llm-testbench/internal/eval"
+	"github.com/lukaszraczylo/llm-testbench/internal/runner"
+	"github.com/lukaszraczylo/llm-testbench/internal/testkit"
+)
+
+func TestDiscrimination_TableShowsSpreadAndInstability(t *testing.T) {
+	tests := []testkit.Test{
+		{ID: "sep", Category: "c", Subcategory: "s"},
+		{ID: "same", Category: "c", Subcategory: "s"},
+	}
+	results := []runner.Result{
+		{Model: "m1", TestID: "sep", Score: eval.Score{Value: 1}},
+		{Model: "m2", TestID: "sep", Score: eval.Score{Value: 0.2}},
+		{Model: "m1", TestID: "same", Score: eval.Score{Value: 1}},
+		{Model: "m2", TestID: "same", Score: eval.Score{Value: 1}},
+		// Flaky pair for m1/sep: a second attempt disagrees with the first.
+		{Model: "m1", TestID: "sep", Attempt: 1, Score: eval.Score{Value: 0.4}},
+	}
+
+	var buf bytes.Buffer
+	if err := Render(&buf, FormatTable, tests, []string{"m1", "m2"}, results); err != nil {
+		t.Fatalf("Render() error = %v", err)
+	}
+	out := buf.String()
+
+	for _, want := range []string{
+		"== Discrimination: 1/2 scored tests separate models by >= 0.05; 1 unstable under repeat ==",
+		"sep",
+		"[0.40-1.00]", // unstable range in the score table cell
+	} {
+		if !strings.Contains(out, want) {
+			t.Errorf("output missing %q:\n%s", want, out)
+		}
+	}
+	if strings.Contains(out, "\nsame\t") {
+		// "same" appears in the per-test table; ensure it is NOT in the
+		// discrimination rows by checking the section body only.
+		section := out[strings.Index(out, "== Discrimination"):]
+		if strings.Contains(section, "same") {
+			t.Errorf("non-discriminating test listed in discrimination section:\n%s", section)
+		}
+	}
+}
+
+func TestDiscrimination_JSONStatsAndAttemptRoundTrip(t *testing.T) {
+	tests := []testkit.Test{{ID: "t1", Category: "c", Subcategory: "s"}}
+	results := []runner.Result{
+		{Model: "m1", TestID: "t1", Score: eval.Score{Value: 1}},
+		{Model: "m2", TestID: "t1", Score: eval.Score{Value: 0.5}},
+		{Model: "m1", TestID: "t1", Attempt: 1, Score: eval.Score{Value: 0.5}},
+	}
+
+	var buf bytes.Buffer
+	if err := Render(&buf, FormatJSON, tests, []string{"m1", "m2"}, results); err != nil {
+		t.Fatalf("Render() error = %v", err)
+	}
+	var doc jsonArtifact
+	if err := json.Unmarshal(buf.Bytes(), &doc); err != nil {
+		t.Fatalf("unmarshal: %v", err)
+	}
+	if len(doc.Stats) != 1 {
+		t.Fatalf("len(stats) = %d, want 1", len(doc.Stats))
+	}
+	st := doc.Stats[0]
+	if !st.Unstable {
+		t.Error("Unstable = false, want true (attempts 1.0 vs 0.5)")
+	}
+	if st.Spread < 0.24 || st.Spread > 0.26 {
+		t.Errorf("Spread = %v, want ~0.25 (m1 mean 0.75 vs m2 0.5)", st.Spread)
+	}
+
+	// Attempt indices survive the round trip for compare.
+	var attempts []AttemptResult
+	attempts = append(attempts, doc.Results...)
+	if attempts[2].Attempt != 1 {
+		t.Errorf("attempt = %d, want 1", attempts[2].Attempt)
+	}
+}

internal/report/health.go

@@ -0,0 +1,222 @@
+package report
+
+import (
+	"fmt"
+	"io"
+	"sort"
+)
+
+// PerfectScore is the score at which a (model, test) cell counts as a
+// perfect pass. Partial-credit evaluators score between 0 and 1; anything
+// below this is not a perfect pass.
+const PerfectScore = 0.999
+
+// SaturationRate is the share of a test's scored cells that must be
+// perfect for the test to count as saturated - passed by every model it
+// was run against, every attempt. Saturated tests carry no signal about
+// relative capability; they only detect outright regressions.
+const SaturationRate = 0.9
+
+// SubHealth is the suite-health rollup for one category/subcategory pair,
+// pooled across one or more artifacts (so single-model probe runs can be
+// combined with a full multi-model run). Cells are attempt-aggregated
+// (model, test) means: a pair counts once, taking its WORST artifact -
+// health asks "can any observed run fail this test?", so pooling over
+// re-runs must not let a lucky second attempt hide a real failure.
+type SubHealth struct {
+	Category    string
+	Subcategory string
+	WeakTests   []string // non-saturated test IDs, sorted - the signal-carriers
+	Tests       int      // distinct tests with at least one scored cell
+	Cells       int      // scored (model, test) cells
+	Perfect     int      // cells at PerfectScore or above
+	Saturated   int      // tests perfect in >= SaturationRate of their cells
+	Unstable    int      // tests whose repeated attempts disagreed
+	Truncated   int      // tests with a truncated cell
+}
+
+// PerfectRate is the share of scored cells that are perfect.
+func (h SubHealth) PerfectRate() float64 {
+	if h.Cells == 0 {
+		return 0
+	}
+	return float64(h.Perfect) / float64(h.Cells)
+}
+
+// HealthReport pools every subcategory present in the artifacts.
+type HealthReport struct {
+	Subcategories []SubHealth
+	Models        []string
+	Tests         int // distinct scored tests across all subcategories
+	Cells         int
+	Perfect       int
+	Saturated     int
+	Artifacts     int
+}
+
+// healthTestID scopes a test ID by model for cell pooling.
+type healthCellID struct{ model, testID string }
+
+// healthTest accumulates one test's cells across artifacts.
+type healthTest struct {
+	cells                 map[healthCellID]float64
+	category, subcategory string
+	unstable, truncated   bool
+}
+
+// AuditHealth pools artifacts into a per-subcategory health view.
+// Errored and skipped attempts never form cells (meanAttempts already
+// separates them by Status); truncated attempts follow the same
+// exclusion policy as compare. Per-test unstable/truncated flags are
+// OR-ed across artifacts: once flaky, always flagged.
+func AuditHealth(artifacts []Artifact) HealthReport {
+	agg := map[string]*healthTest{} // key: testID
+	modelSet := map[string]bool{}
+
+	for _, a := range artifacts {
+		for _, c := range meanAttempts(a.Results) {
+			if c.Status != "" && c.Status != "scored" {
+				continue
+			}
+			modelSet[c.Model] = true
+			ht := agg[c.TestID]
+			if ht == nil {
+				ht = &healthTest{category: c.Category, subcategory: c.Subcategory, cells: map[healthCellID]float64{}}
+				if ht.category == "" {
+					ht.category = "unknown"
+				}
+				if ht.subcategory == "" {
+					ht.subcategory = "unknown"
+				}
+				agg[c.TestID] = ht
+			}
+			id := healthCellID{c.Model, c.TestID}
+			if prev, ok := ht.cells[id]; !ok || c.BaselineMean < prev {
+				// Pool the worst observed mean for this cell. On the
+				// first artifact BaselineMean is the current run's mean.
+				ht.cells[id] = c.BaselineMean
+			}
+		}
+		for _, s := range a.Stats {
+			if ht, ok := agg[s.TestID]; ok {
+				ht.unstable = ht.unstable || s.Unstable
+				ht.truncated = ht.truncated || s.Truncated
+			}
+		}
+	}
+
+	models := make([]string, 0, len(modelSet))
+	for m := range modelSet {
+		models = append(models, m)
+	}
+	sort.Strings(models)
+
+	bySub := map[string]*SubHealth{}
+	var tot, totCells, totPerfect, totSat int
+	for testID, ht := range agg {
+		sk := ht.category + "/" + ht.subcategory
+		sh := bySub[sk]
+		if sh == nil {
+			sh = &SubHealth{Category: ht.category, Subcategory: ht.subcategory}
+			bySub[sk] = sh
+		}
+		sh.Tests++
+		tot++
+		var perfect int
+		for _, s := range ht.cells {
+			sh.Cells++
+			totCells++
+			if s >= PerfectScore {
+				perfect++
+				sh.Perfect++
+				totPerfect++
+			}
+		}
+		if sh.Cells > 0 && ht.unstable {
+			sh.Unstable++
+		}
+		if ht.truncated {
+			sh.Truncated++
+		}
+		if float64(perfect)/float64(len(ht.cells)) >= SaturationRate {
+			sh.Saturated++
+			totSat++
+		} else {
+			sh.WeakTests = append(sh.WeakTests, testID)
+		}
+	}
+
+	report := HealthReport{Artifacts: len(artifacts), Models: models, Tests: tot, Cells: totCells, Perfect: totPerfect, Saturated: totSat}
+	for _, sh := range bySub {
+		sort.Strings(sh.WeakTests)
+		report.Subcategories = append(report.Subcategories, *sh)
+	}
+	sort.Slice(report.Subcategories, func(i, j int) bool {
+		a, b := report.Subcategories[i], report.Subcategories[j]
+		if a.PerfectRate() != b.PerfectRate() {
+			return a.PerfectRate() > b.PerfectRate()
+		}
+		if a.Category != b.Category {
+			return a.Category < b.Category
+		}
+		return a.Subcategory < b.Subcategory
+	})
+	return report
+}
+
+// weakCap bounds listed weak tests per subcategory in the text report;
+// the counts stay accurate regardless.
+const weakCap = 8
+
+// RenderHealth writes the audit: headline numbers, then one line per
+// subcategory sorted by perfect-cell rate descending (most saturated -
+// least informative - first), listing its non-saturated tests.
+func RenderHealth(w io.Writer, r HealthReport) error {
+	ew := &errWriter{w: w}
+	perfectRate := 0.0
+	if r.Cells > 0 {
+		perfectRate = float64(r.Perfect) / float64(r.Cells)
+	}
+	ew.println(fmt.Sprintf("== Suite health: %d tests, %d scored (model,test) cells over %d model(s), %d artifact(s) ==",
+		r.Tests, r.Cells, len(r.Models), r.Artifacts))
+	ew.println(fmt.Sprintf("perfect cells: %d/%d (%.1f%%) | saturated tests (no signal): %d/%d\n",
+		r.Perfect, r.Cells, perfectRate*100, r.Saturated, r.Tests))
+	if r.Saturated == r.Tests && r.Tests > 0 {
+		ew.println("every scored test is saturated: the suite cannot rank these models at all.")
+	}
+	ew.println(fmt.Sprintf("%-30s %5s %6s %8s %9s %8s %8s  WEAK TESTS",
+		"SUBCATEGORY", "TESTS", "CELLS", "PERFECT%", "SATURATED", "UNSTBL", "TRUNC"))
+	for _, sh := range r.Subcategories {
+		name := sh.Category + "/" + sh.Subcategory
+		weak := ""
+		if len(sh.WeakTests) > 0 {
+			weak = fmt.Sprintf("%d", len(sh.WeakTests))
+			if len(sh.WeakTests) > weakCap {
+				weak += ": " + joinN(sh.WeakTests, weakCap) + ", ..."
+			} else {
+				weak += ": " + joinAll(sh.WeakTests)
+			}
+		}
+		ew.println(fmt.Sprintf("%-30s %5d %6d %8.1f %9d %8d %8d  %s",
+			name, sh.Tests, sh.Cells, sh.PerfectRate()*100, sh.Saturated, sh.Unstable, sh.Truncated, weak))
+	}
+	return ew.err
+}
+
+func joinN(xs []string, n int) string {
+	if len(xs) <= n {
+		return joinAll(xs)
+	}
+	return joinAll(xs[:n])
+}
+
+func joinAll(xs []string) string {
+	out := ""
+	for i, x := range xs {
+		if i > 0 {
+			out += ", "
+		}
+		out += x
+	}
+	return out
+}

internal/report/health_test.go

@@ -0,0 +1,200 @@
+package report
+
+import (
+	"bytes"
+	"strings"
+	"testing"
+)
+
+// scoredAttempt builds a scored attempt in subcategory (cat, sub).
+func scoredAttempt(model, testID, cat, sub string, score float64) AttemptResult {
+	return AttemptResult{Model: model, TestID: testID, Category: cat, Subcategory: sub, Score: score}
+}
+
+func subHealth(t *testing.T, r HealthReport, cat, sub string) SubHealth {
+	t.Helper()
+	for _, sh := range r.Subcategories {
+		if sh.Category == cat && sh.Subcategory == sub {
+			return sh
+		}
+	}
+	t.Fatalf("subcategory %s/%s missing from report", cat, sub)
+	return SubHealth{}
+}
+
+func TestAuditHealth_SaturatedVersusWeakTests(t *testing.T) {
+	// a: perfect for both models => saturated, no signal.
+	// b: model m2 fails => weak, carries signal.
+	art := Artifact{Results: []AttemptResult{
+		scoredAttempt("m1", "a", "c", "s", 1.0),
+		scoredAttempt("m2", "a", "c", "s", 1.0),
+		scoredAttempt("m1", "b", "c", "s", 1.0),
+		scoredAttempt("m2", "b", "c", "s", 0.0),
+	}}
+	r := AuditHealth([]Artifact{art})
+
+	sh := subHealth(t, r, "c", "s")
+	if sh.Tests != 2 || sh.Cells != 4 {
+		t.Fatalf("tests/cells = %d/%d, want 2/4", sh.Tests, sh.Cells)
+	}
+	if sh.Perfect != 3 {
+		t.Errorf("perfect = %d, want 3", sh.Perfect)
+	}
+	if sh.Saturated != 1 {
+		t.Errorf("saturated = %d, want 1 (only test a)", sh.Saturated)
+	}
+	if len(sh.WeakTests) != 1 || sh.WeakTests[0] != "b" {
+		t.Errorf("weak = %v, want [b]", sh.WeakTests)
+	}
+	if r.Saturated != 1 || r.Tests != 2 {
+		t.Errorf("report totals = %d/%d, want 1/2", r.Saturated, r.Tests)
+	}
+	if len(r.Models) != 2 || r.Models[0] != "m1" {
+		t.Errorf("models = %v, want [m1 m2]", r.Models)
+	}
+}
+
+func TestAuditHealth_PoolsWorstCellAcrossArtifacts(t *testing.T) {
+	// A flaky test that passes on a second artifact must still count as
+	// weak: health asks whether any observed run failed it.
+	a1 := Artifact{Results: []AttemptResult{
+		scoredAttempt("m1", "flaky", "c", "s", 0.0),
+	}}
+	a2 := Artifact{Results: []AttemptResult{
+		scoredAttempt("m1", "flaky", "c", "s", 1.0),
+	}}
+	r := AuditHealth([]Artifact{a1, a2})
+
+	sh := subHealth(t, r, "c", "s")
+	if sh.Saturated != 0 || len(sh.WeakTests) != 1 {
+		t.Errorf("saturated=%d weak=%v, want 0/[flaky] (worst cell pooled)", sh.Saturated, sh.WeakTests)
+	}
+	if sh.Cells != 1 {
+		t.Errorf("cells = %d, want 1 (same model,test pooled)", sh.Cells)
+	}
+}
+
+func TestAuditHealth_AveragesRepeatedAttemptsPerCell(t *testing.T) {
+	// Two attempts of one cell: mean 0.5 < PerfectScore => not perfect.
+	art := Artifact{Results: []AttemptResult{
+		scoredAttempt("m1", "t", "c", "s", 1.0),
+		scoredAttempt("m1", "t", "c", "s", 0.0),
+	}}
+	r := AuditHealth([]Artifact{art})
+
+	sh := subHealth(t, r, "c", "s")
+	if sh.Cells != 1 || sh.Perfect != 0 {
+		t.Errorf("cells/perfect = %d/%d, want 1/0 (attempt mean 0.5)", sh.Cells, sh.Perfect)
+	}
+}
+
+func TestAuditHealth_ErrorsAndSkipsExcluded(t *testing.T) {
+	art := Artifact{Results: []AttemptResult{
+		{Model: "m1", TestID: "boom", Category: "c", Subcategory: "s", Error: "404"},
+		{Model: "m1", TestID: "skip", Category: "c", Subcategory: "s", Skipped: true},
+		scoredAttempt("m1", "ok", "c", "s", 1.0),
+	}}
+	r := AuditHealth([]Artifact{art})
+
+	sh := subHealth(t, r, "c", "s")
+	if sh.Tests != 1 || sh.Cells != 1 || sh.Saturated != 1 {
+		t.Errorf("tests/cells/saturated = %d/%d/%d, want 1/1/1 (error+skip excluded)",
+			sh.Tests, sh.Cells, sh.Saturated)
+	}
+}
+
+func TestAuditHealth_FlagsUnstableAndTruncated(t *testing.T) {
+	art := Artifact{
+		Results: []AttemptResult{
+			scoredAttempt("m1", "u", "c", "s", 1.0),
+			scoredAttempt("m1", "t", "c", "s", 0.5),
+		},
+		Stats: []jsonStatRow{
+			{TestID: "u", Unstable: true},
+			{TestID: "t", Truncated: true},
+		},
+	}
+	r := AuditHealth([]Artifact{art})
+
+	sh := subHealth(t, r, "c", "s")
+	if sh.Unstable != 1 || sh.Truncated != 1 {
+		t.Errorf("unstable/truncated = %d/%d, want 1/1", sh.Unstable, sh.Truncated)
+	}
+}
+
+func TestAuditHealth_SaturationBoundary(t *testing.T) {
+	// 9 perfect cells + 1 failed cell = 0.9 rate = at SaturationRate =>
+	// saturated. 8/9 = 0.888 < rate => weak.
+	perfect9 := Artifact{}
+	for i := range 9 {
+		perfect9.Results = append(perfect9.Results, scoredAttempt("m"+string(rune('a'+i)), "t1", "c", "s", 1.0))
+	}
+	perfect9.Results = append(perfect9.Results, scoredAttempt("mj", "t1", "c", "s", 0.0))
+
+	if r := AuditHealth([]Artifact{perfect9}); r.Saturated != 1 {
+		t.Errorf("9/10 perfect: saturated = %d, want 1 (rate == SaturationRate)", r.Saturated)
+	}
+
+	eight := Artifact{}
+	for i := range 8 {
+		eight.Results = append(eight.Results, scoredAttempt("m"+string(rune('a'+i)), "t1", "c", "s", 1.0))
+	}
+	eight.Results = append(eight.Results,
+		scoredAttempt("mi", "t1", "c", "s", 0.0),
+		scoredAttempt("mj", "t1", "c", "s", 0.0))
+
+	if r := AuditHealth([]Artifact{eight}); r.Saturated != 0 {
+		t.Errorf("8/10 perfect: saturated = %d, want 0", r.Saturated)
+	}
+}
+
+func TestAuditHealth_SortsMostSaturatedFirst(t *testing.T) {
+	art := Artifact{Results: []AttemptResult{
+		scoredAttempt("m1", "hard", "c", "low", 0.0),
+		scoredAttempt("m1", "easy1", "c", "high", 1.0),
+		scoredAttempt("m1", "easy2", "c", "high", 1.0),
+	}}
+	r := AuditHealth([]Artifact{art})
+
+	if len(r.Subcategories) != 2 {
+		t.Fatalf("subcategories = %d, want 2", len(r.Subcategories))
+	}
+	if got := r.Subcategories[0].Subcategory; got != "high" {
+		t.Errorf("first row = %q, want most saturated %q", got, "high")
+	}
+}
+
+func TestRenderHealth_HeadlineAndWeakList(t *testing.T) {
+	art := Artifact{Results: []AttemptResult{
+		scoredAttempt("m1", "a", "c", "s", 1.0),
+		scoredAttempt("m1", "b", "c", "s", 0.0),
+	}}
+	var buf bytes.Buffer
+	if err := RenderHealth(&buf, AuditHealth([]Artifact{art})); err != nil {
+		t.Fatalf("RenderHealth() error = %v", err)
+	}
+	out := buf.String()
+	for _, want := range []string{"2 tests", "2 scored", "c/s", "50.0", "1: b"} {
+		if !strings.Contains(out, want) {
+			t.Errorf("output missing %q:\n%s", want, out)
+		}
+	}
+	// "a" is saturated: it must not appear anywhere in the weak list.
+	if strings.Contains(out, ": a") || strings.Contains(out, ", a,") || strings.HasSuffix(strings.TrimSpace(out), " a") {
+		t.Errorf("saturated test a should not be listed weak:\n%s", out)
+	}
+}
+
+func TestRenderHealth_AllSaturatedVerdict(t *testing.T) {
+	art := Artifact{Results: []AttemptResult{
+		scoredAttempt("m1", "a", "c", "s", 1.0),
+		scoredAttempt("m2", "a", "c", "s", 1.0),
+	}}
+	var buf bytes.Buffer
+	if err := RenderHealth(&buf, AuditHealth([]Artifact{art})); err != nil {
+		t.Fatalf("RenderHealth() error = %v", err)
+	}
+	if !strings.Contains(buf.String(), "cannot rank these models") {
+		t.Errorf("missing all-saturated verdict:\n%s", buf.String())
+	}
+}

internal/report/json.go

@@ -2,14 +2,17 @@ package report
 
 import (
 	"encoding/json"
+	"fmt"
 	"io"
+	"math"
+	"os"
 
 	"github.com/lukaszraczylo/llm-testbench/internal/runner"
 	"github.com/lukaszraczylo/llm-testbench/internal/testkit"
 )
 
-// jsonResult is the JSON-serializable projection of a runner.Result, joined
-// with its test's category/subcategory for self-contained output.
+// jsonResult is the JSON-serializable projection of one runner.Result
+// (one attempt), joined with its test's category/subcategory.
 type jsonResult struct {
 	Model            string  `json:"model"`
 	TestID           string  `json:"test_id"`
@@ -26,16 +29,43 @@ type jsonResult struct {
 	Tokens           int     `json:"tokens"`
 	Skipped          bool    `json:"skipped"`
 	Truncated        bool    `json:"truncated"`
+	Attempt          int     `json:"attempt,omitempty"`
 }
 
-// renderJSON dumps raw results as a JSON array, joined with test metadata.
-func renderJSON(w io.Writer, tests []testkit.Test, results []runner.Result) error {
+// jsonArtifact is the --out / --format json document: every raw attempt
+// plus the per-test discrimination rollup, so downstream tooling
+// (compare) can re-derive stability without re-running the suite.
+type jsonArtifact struct {
+	Results []jsonResult  `json:"results"`
+	Stats   []jsonStatRow `json:"stats,omitempty"`
+}
+
+// jsonStatRow is the JSON projection of testStat.
+type jsonStatRow struct {
+	TestID      string  `json:"test_id"`
+	Category    string  `json:"category"`
+	Subcategory string  `json:"subcategory"`
+	Mean        float64 `json:"mean"`
+	Spread      float64 `json:"spread,omitempty"`
+	Unstable    bool    `json:"unstable,omitempty"`
+	Truncated   bool    `json:"truncated,omitempty"`
+}
+
+// renderJSON dumps the run as a JSON artifact document.
+func renderJSON(w io.Writer, tests []testkit.Test, models []string, results []runner.Result) error {
+	artifact := buildArtifact(tests, models, results)
+	enc := json.NewEncoder(w)
+	enc.SetIndent("", "  ")
+	return enc.Encode(artifact)
+}
+
+func buildArtifact(tests []testkit.Test, models []string, results []runner.Result) jsonArtifact {
 	testByID := make(map[string]testkit.Test, len(tests))
 	for _, t := range tests {
 		testByID[t.ID] = t
 	}
 
-	out := make([]jsonResult, 0, len(results))
+	out := jsonArtifact{Results: make([]jsonResult, 0, len(results))}
 	for _, r := range results {
 		t := testByID[r.TestID]
 		jr := jsonResult{
@@ -53,14 +83,79 @@ func renderJSON(w io.Writer, tests []testkit.Test, results []runner.Result) erro
 			PromptTokens:     r.PromptTokens,
 			CompletionTokens: r.CompletionTokens,
 			Tokens:           r.TotalTokens(),
+			Attempt:          r.Attempt,
 		}
 		if r.Err != nil {
 			jr.Error = r.Err.Error()
 		}
-		out = append(out, jr)
+		out.Results = append(out.Results, jr)
 	}
 
-	enc := json.NewEncoder(w)
-	enc.SetIndent("", "  ")
-	return enc.Encode(out)
+	idx := indexResults(results)
+	for _, st := range testStats(tests, models, idx) {
+		if !st.scored {
+			continue
+		}
+		row := jsonStatRow{
+			TestID:      st.testID,
+			Category:    st.category,
+			Subcategory: st.subcategory,
+			Mean:        round6(st.mean),
+			Unstable:    st.unstable,
+			Truncated:   st.truncated,
+		}
+		if !math.IsNaN(st.spread) {
+			row.Spread = round6(st.spread)
+		}
+		out.Stats = append(out.Stats, row)
+	}
+	return out
+}
+
+func round6(v float64) float64 {
+	return math.Round(v*1e6) / 1e6
+}
+
+// Artifact is a saved run document: every raw attempt plus the per-test
+// discrimination rollup. Exported so the compare command can consume it.
+type Artifact = jsonArtifact
+
+// AttemptResult is one saved attempt. Exported alias of jsonResult.
+type AttemptResult = jsonResult
+
+// LoadArtifact reads a --out JSON artifact written by a previous run.
+// Accepts both the current document shape and the legacy bare array of
+// results, so artifacts from earlier releases remain comparable.
+func LoadArtifact(path string) (jsonArtifact, error) {
+	b, err := os.ReadFile(path) //nolint:gosec // path is the operator's own `llmtest compare` argument, not external input
+	if err != nil {
+		return jsonArtifact{}, err
+	}
+
+	var probe struct {
+		Results []jsonResult `json:"results"`
+	}
+	if err := json.Unmarshal(b, &probe); err == nil && probe.Results != nil {
+		var a jsonArtifact
+		if err := json.Unmarshal(b, &a); err != nil {
+			return jsonArtifact{}, fmt.Errorf("%s: %w", path, err)
+		}
+		return a, nil
+	}
+
+	var legacy []jsonResult
+	if err := json.Unmarshal(b, &legacy); err != nil {
+		return jsonArtifact{}, fmt.Errorf("%s: not a llmtest json artifact: %w", path, err)
+	}
+	return jsonArtifact{Results: legacy}, nil
+}
+
+// WriteArtifact saves the run as a JSON artifact at path, for later
+// `llmtest compare` runs, regardless of the stdout report format.
+func WriteArtifact(path string, tests []testkit.Test, models []string, results []runner.Result) error {
+	b, err := json.MarshalIndent(buildArtifact(tests, models, results), "", "  ")
+	if err != nil {
+		return err
+	}
+	return os.WriteFile(path, append(b, '\n'), 0o644) //nolint:gosec // 0o644: the artifact is a report meant to be shared/committed, not a secret file
 }

internal/report/markdown.go

@@ -10,8 +10,8 @@ import (
 	"github.com/lukaszraczylo/llm-testbench/internal/testkit"
 )
 
-// renderMarkdown writes the same three tables as renderTable, formatted as
-// GitHub-flavored markdown tables.
+// renderMarkdown writes the same four sections as renderTable, formatted
+// as GitHub-flavored markdown tables.
 func renderMarkdown(w io.Writer, tests []testkit.Test, models []string, results []runner.Result) error {
 	idx := indexResults(results)
 	ew := &errWriter{w: w}
@@ -24,9 +24,9 @@ func renderMarkdown(w io.Writer, tests []testkit.Test, models []string, results
 	for _, t := range sortedTests(tests) {
 		row := []string{t.ID, t.Category, t.Subcategory}
 		for _, model := range models {
-			r, ok := idx[t.ID][model]
-			row = append(row, cellText(r, ok))
-			anyTruncated = anyTruncated || r.Truncated()
+			c := idx[t.ID][model]
+			row = append(row, cellText(c))
+			anyTruncated = anyTruncated || c.result.Truncated()
 		}
 		writeMarkdownRow(ew, row)
 	}
@@ -48,11 +48,16 @@ func renderMarkdown(w io.Writer, tests []testkit.Test, models []string, results
 		writeMarkdownRow(ew, row)
 	}
 
+	ew.println()
+	ew.println("## Discrimination")
+	ew.println()
+	writeDiscriminationRows(markdownPrinter{ew: ew}, testStats(tests, models, idx), len(models) > 1)
+
 	ew.println()
 	ew.println("## Model summary")
 	ew.println()
 	writeMarkdownHeader(ew, []string{"Model", "Mean", "Passed", "Partial", "Failed", "Errors", "Mean latency", "Total tokens", "Tok/s"})
-	for _, ms := range summarize(models, results) {
+	for _, ms := range summarize(models, idx) {
 		writeMarkdownRow(ew, []string{
 			ms.model,
 			meanCellText(ms.overallMean),
@@ -69,6 +74,18 @@ func renderMarkdown(w io.Writer, tests []testkit.Test, models []string, results
 	return ew.err
 }
 
+// markdownPrinter adapts writeDiscriminationRows to pipe-delimited rows.
+// GFM needs a blank line between the note paragraph and the table, so
+// header() emits one first.
+type markdownPrinter struct{ ew *errWriter }
+
+func (mp markdownPrinter) header(cells ...string) {
+	mp.ew.println()
+	writeMarkdownHeader(mp.ew, cells)
+}
+func (mp markdownPrinter) row(cells ...string) { writeMarkdownRow(mp.ew, cells) }
+func (mp markdownPrinter) note(text string)    { mp.ew.println(text) }
+
 func writeMarkdownHeader(ew *errWriter, cols []string) {
 	ew.println("| " + strings.Join(cols, " | ") + " |")
 	sep := make([]string, len(cols))

internal/report/report.go

@@ -9,6 +9,8 @@ import (
 	"sort"
 	"time"
 
+	"github.com/lukaszraczylo/llm-testbench/internal/eval"
+	"github.com/lukaszraczylo/llm-testbench/internal/llm"
 	"github.com/lukaszraczylo/llm-testbench/internal/runner"
 	"github.com/lukaszraczylo/llm-testbench/internal/testkit"
 )
@@ -35,22 +37,125 @@ func Render(w io.Writer, format Format, tests []testkit.Test, models []string, r
 	case FormatMarkdown:
 		return renderMarkdown(w, tests, models, results)
 	case FormatJSON:
-		return renderJSON(w, tests, results)
+		return renderJSON(w, tests, models, results)
 	default:
 		return fmt.Errorf("report: unknown format %q", format)
 	}
 }
 
-// resultIndex looks up a Result by testID then model.
-type resultIndex map[string]map[string]runner.Result
+// modelCell is the aggregate view of every attempt recorded for one
+// (testID, model) pair: with Config.Repeat = 1 this is the single result;
+// with repeats the Score is the mean over scored attempts, and minScore/
+// maxScore expose the within-model spread (instability) that repeats are
+// measured to find.
+type modelCell struct {
+	result     runner.Result // representative: last scored attempt, mean score patched in
+	minScore   float64
+	maxScore   float64
+	meanTokens float64
+	attempts   int
+	scored     int
+	skippedAll bool
+	errorAll   bool
+	ok         bool
+}
 
-func indexResults(results []runner.Result) resultIndex {
-	idx := make(resultIndex)
+// cellIndex looks up the aggregate cell for a testID then model.
+type cellIndex map[string]map[string]modelCell
+
+func indexResults(results []runner.Result) cellIndex {
+	type key struct{ testID, model string }
+	grouped := make(map[key][]runner.Result)
+	order := make([]key, 0, len(results))
 	for _, r := range results {
-		if idx[r.TestID] == nil {
-			idx[r.TestID] = make(map[string]runner.Result)
+		k := key{r.TestID, r.Model}
+		if _, seen := grouped[k]; !seen {
+			order = append(order, k)
 		}
-		idx[r.TestID][r.Model] = r
+		grouped[k] = append(grouped[k], r)
+	}
+
+	idx := make(cellIndex)
+	for _, k := range order {
+		attempts := grouped[k]
+		sort.Slice(attempts, func(i, j int) bool { return attempts[i].Attempt < attempts[j].Attempt })
+
+		c := modelCell{ok: true, attempts: len(attempts)}
+		// Truncation policy: a truncated attempt was scored on partial
+		// text, so its score measures the token budget as much as the
+		// model. Attempts that finished normally are therefore preferred
+		// for the cell mean (and for the min-max instability range); only
+		// when EVERY scored attempt was cut off does the mean fall back to
+		// the truncated scores. A cell is flagged "!" whenever any attempt
+		// was truncated, clean mean or not.
+		var sum, toks, cleanSum, cleanToks float64
+		var cleanN, scoredN int
+		var cleanMin, cleanMax float64
+		var anyTrunc bool
+		var last, lastClean runner.Result
+		for _, a := range attempts {
+			last = a
+			switch {
+			case a.Err != nil:
+				continue
+			case a.Score.Skipped:
+				continue
+			}
+			v := a.Score.Value
+			sum += v
+			toks += float64(a.TotalTokens())
+			scoredN++
+			if a.Truncated() {
+				anyTrunc = true
+				continue
+			}
+			if cleanN == 0 {
+				cleanMin, cleanMax = v, v
+			} else {
+				cleanMin = math.Min(cleanMin, v)
+				cleanMax = math.Max(cleanMax, v)
+			}
+			cleanSum += v
+			cleanToks += float64(a.TotalTokens())
+			cleanN++
+			lastClean = a
+		}
+		c.scored = scoredN
+
+		rep := last
+		if cleanN > 0 {
+			rep = lastClean
+		}
+		switch {
+		case scoredN > 0 && cleanN > 0:
+			rep.Score = eval.Score{
+				Value:  cleanSum / float64(cleanN),
+				Detail: lastClean.Score.Detail,
+			}
+			c.minScore, c.maxScore = cleanMin, cleanMax
+			c.meanTokens = cleanToks / float64(cleanN)
+			if anyTrunc {
+				rep.FinishReason = llm.FinishReasonLength
+			}
+		case scoredN > 0:
+			rep.Score = eval.Score{
+				Value:  sum / float64(scoredN),
+				Detail: last.Score.Detail,
+			}
+			c.minScore, c.maxScore = rep.Score.Value, rep.Score.Value
+			c.meanTokens = toks / float64(scoredN)
+			rep.FinishReason = llm.FinishReasonLength
+		case c.attempts > 0 && last.Score.Skipped:
+			c.skippedAll = true
+		case c.attempts > 0 && last.Err != nil:
+			c.errorAll = true
+		}
+
+		c.result = rep
+		if idx[k.testID] == nil {
+			idx[k.testID] = make(map[string]modelCell)
+		}
+		idx[k.testID][k.model] = c
 	}
 	return idx
 }
@@ -72,34 +177,41 @@ func sortedTests(tests []testkit.Test) []testkit.Test {
 	return out
 }
 
-// truncatedSuffix marks a cell whose underlying response was cut off by
-// the token budget (finish_reason=length): the score may not reflect what
-// the model would have answered with a larger budget. truncatedLegend is
-// the one-line explanation printed once per table/markdown report.
+// truncatedSuffix marks a cell where any attempt was cut off by the token
+// budget (finish_reason=length). Per the truncation policy in
+// indexResults, the cell mean then comes from the attempts that finished
+// normally; "!" warns that at least one sample was discarded (or, when no
+// attempt finished, that the whole mean is partial-text based).
+// truncatedLegend is the one-line explanation printed once per
+// table/markdown report.
 const (
 	truncatedSuffix = "!"
-	truncatedLegend = "! = response truncated by the token budget (finish_reason=length); increase max_tokens_default to avoid"
+	truncatedLegend = "! = some attempt was truncated by the token budget (finish_reason=length); the mean uses only attempts that finished, increase max_tokens_default if every attempt is flagged"
 )
 
 // cellText renders one (test, model) cell for the score table: "score
 // (Ntok)" for a scored result, or ERR/skip/N/A otherwise, with a
-// truncatedSuffix appended when the response was cut off by the token
-// budget.
-func cellText(r runner.Result, found bool) string {
-	if !found {
+// truncatedSuffix appended when any attempt was cut off by the token
+// budget. With repeat attempts, a "[min-max]" range is appended when the
+// attempts disagreed, marking the cell unstable.
+func cellText(c modelCell) string {
+	if !c.ok {
+		return "N/A"
+	}
+	switch {
+	case c.errorAll:
+		return "ERR"
+	case c.skippedAll:
+		return "skip"
+	case c.scored == 0:
 		return "N/A"
 	}
 
-	var text string
-	switch {
-	case r.Err != nil:
-		text = "ERR"
-	case r.Score.Skipped:
-		text = "skip"
-	default:
-		text = fmt.Sprintf("%.2f (%dtok)", r.Score.Value, r.TotalTokens())
+	text := fmt.Sprintf("%.2f (%dtok)", c.result.Score.Value, int(math.Round(c.meanTokens)))
+	if c.attempts > 1 && c.maxScore > c.minScore {
+		text += fmt.Sprintf(" [%0.2f-%0.2f]", c.minScore, c.maxScore)
 	}
-	if r.Truncated() {
+	if c.result.Truncated() {
 		text += truncatedSuffix
 	}
 	return text
@@ -112,7 +224,7 @@ type categoryMean struct {
 	category string
 }
 
-func categoryRollup(tests []testkit.Test, models []string, idx resultIndex) []categoryMean {
+func categoryRollup(tests []testkit.Test, models []string, idx cellIndex) []categoryMean {
 	order := make([]string, 0)
 	byCategory := make(map[string][]testkit.Test)
 	for _, t := range sortedTests(tests) {
@@ -129,11 +241,11 @@ func categoryRollup(tests []testkit.Test, models []string, idx resultIndex) []ca
 			var sum float64
 			var n int
 			for _, t := range byCategory[cat] {
-				r, ok := idx[t.ID][model]
-				if !ok || r.Err != nil || r.Score.Skipped {
+				c, ok := idx[t.ID][model]
+				if !ok || c.errorAll || c.skippedAll || c.scored == 0 {
 					continue
 				}
-				sum += r.Score.Value
+				sum += c.result.Score.Value
 				n++
 			}
 			if n > 0 {
@@ -191,49 +303,49 @@ type modelSummary struct {
 	avgTokPerSec float64 // math.NaN() when no timed, non-skipped data
 }
 
-func summarize(models []string, results []runner.Result) []modelSummary {
-	byModel := make(map[string][]runner.Result, len(models))
-	for _, r := range results {
-		byModel[r.Model] = append(byModel[r.Model], r)
-	}
-
+// summarize aggregates one model's results per test (attempts already
+// merged by indexResults), so pass/partial/fail counts stay per-test even
+// with Config.Repeat > 1.
+func summarize(models []string, idx cellIndex) []modelSummary {
 	out := make([]modelSummary, 0, len(models))
 	for _, model := range models {
 		ms := modelSummary{model: model}
-		var scoreSum float64
-		var scoredCount int
+		var scoreSum, latencySecondsSum, meanTokenSum float64
+		var scoredCount, latencyCount int
 		var latencySum time.Duration
-		var latencyCount int
-		var completionTokenSum int
-		var latencySecondsSum float64
 
-		for _, r := range byModel[model] {
-			ms.totalTokens += r.TotalTokens()
-			switch {
-			case r.Err != nil:
-				ms.errors++
+		for _, cells := range idx {
+			c, ok := cells[model]
+			if !ok {
 				continue
-			case r.Score.Skipped:
+			}
+			meanTokenSum += c.meanTokens * float64(max(c.scored, 1))
+			if c.errorAll {
+				ms.errors += c.attempts
+				continue
+			}
+			if c.skippedAll || c.scored == 0 {
 				continue
 			}
 
-			scoreSum += r.Score.Value
+			v := c.result.Score.Value
+			scoreSum += v
 			scoredCount++
-			latencySum += r.Latency
+			latencySum += c.result.Latency
 			latencyCount++
-			completionTokenSum += r.CompletionTokens
-			latencySecondsSum += r.Latency.Seconds()
+			latencySecondsSum += c.result.Latency.Seconds()
 
 			switch {
-			case r.Score.Value >= passThreshold:
+			case v >= passThreshold:
 				ms.passed++
-			case r.Score.Value <= 0:
+			case v <= 0:
 				ms.failed++
 			default:
 				ms.partial++
 			}
 		}
 
+		ms.totalTokens = int(math.Round(meanTokenSum))
 		if scoredCount > 0 {
 			ms.overallMean = scoreSum / float64(scoredCount)
 			ms.meanLatency = latencySum / time.Duration(latencyCount)
@@ -241,11 +353,11 @@ func summarize(models []string, results []runner.Result) []modelSummary {
 			ms.overallMean = math.NaN()
 		}
 
-		// Aggregate ratio (sum of completion tokens over sum of elapsed
-		// seconds), not a mean of per-test ratios: robust to a handful of
-		// very short responses skewing a per-test-averaged rate.
+		// Aggregate ratio (completion tokens over elapsed seconds for the
+		// representative attempt), not a mean of per-test ratios: robust to
+		// a handful of very short responses skewing a per-test-averaged rate.
 		if latencySecondsSum > 0 {
-			ms.avgTokPerSec = float64(completionTokenSum) / latencySecondsSum
+			ms.avgTokPerSec = float64(ms.totalTokens) / latencySecondsSum
 		} else {
 			ms.avgTokPerSec = math.NaN()
 		}

internal/report/report_test.go

@@ -10,6 +10,7 @@ import (
 	"time"
 
 	"github.com/lukaszraczylo/llm-testbench/internal/eval"
+	"github.com/lukaszraczylo/llm-testbench/internal/llm"
 	"github.com/lukaszraczylo/llm-testbench/internal/runner"
 	"github.com/lukaszraczylo/llm-testbench/internal/testkit"
 )
@@ -33,6 +34,47 @@ func sampleResults() []runner.Result {
 	}
 }
 
+// Truncation policy: attempts cut off by the token budget are excluded
+// from a cell's mean when any attempt finished normally; the cell is still
+// flagged "!" so the discarded sample is visible. When every scored
+// attempt was truncated, the mean falls back to the partial-text scores.
+func TestIndexResults_TruncatedAttemptExcludedFromMean(t *testing.T) {
+	idx := indexResults([]runner.Result{
+		{Model: "m1", TestID: "t", Attempt: 0, Score: eval.Score{Value: 1}, PromptTokens: 4},
+		{Model: "m1", TestID: "t", Attempt: 1, Score: eval.Score{Value: 0}, FinishReason: llm.FinishReasonLength, PromptTokens: 100},
+	})
+	c := idx["t"]["m1"]
+	if got := c.result.Score.Value; got != 1 {
+		t.Errorf("mean = %v, want 1 (clean attempts only)", got)
+	}
+	if !c.result.Truncated() {
+		t.Error("cell should be flagged truncated")
+	}
+	if c.minScore != 1 || c.maxScore != 1 {
+		t.Errorf("min/max = %v/%v, want 1/1 (truncated attempt excluded from instability range)", c.minScore, c.maxScore)
+	}
+	if got := int(c.meanTokens); got != 4 {
+		t.Errorf("meanTokens = %d, want 4 (clean attempts only)", got)
+	}
+}
+
+func TestIndexResults_AllAttemptsTruncatedFallBackToPartialScores(t *testing.T) {
+	idx := indexResults([]runner.Result{
+		{Model: "m1", TestID: "t", Attempt: 0, Score: eval.Score{Value: 0.4}, FinishReason: llm.FinishReasonLength, PromptTokens: 10},
+		{Model: "m1", TestID: "t", Attempt: 1, Score: eval.Score{Value: 0.6}, FinishReason: llm.FinishReasonLength, PromptTokens: 20},
+	})
+	c := idx["t"]["m1"]
+	if got := c.result.Score.Value; math.Abs(got-0.5) > 1e-9 {
+		t.Errorf("mean = %v, want 0.5 (fallback over all-truncated attempts)", got)
+	}
+	if !c.result.Truncated() {
+		t.Error("cell should be flagged truncated")
+	}
+	if c.scored != 2 {
+		t.Errorf("scored = %d, want 2", c.scored)
+	}
+}
+
 func TestRenderTable_ContainsExpectedCells(t *testing.T) {
 	var buf bytes.Buffer
 	if err := Render(&buf, FormatTable, sampleTests(), []string{"m1", "m2"}, sampleResults()); err != nil {
@@ -85,10 +127,11 @@ func TestRenderJSON_DumpsRawResults(t *testing.T) {
 		t.Fatalf("Render() error = %v", err)
 	}
 
-	var out []jsonResult
-	if err := json.Unmarshal(buf.Bytes(), &out); err != nil {
+	var doc jsonArtifact
+	if err := json.Unmarshal(buf.Bytes(), &doc); err != nil {
 		t.Fatalf("json.Unmarshal() error = %v; output:\n%s", err, buf.String())
 	}
+	out := doc.Results
 	if len(out) != len(sampleResults()) {
 		t.Fatalf("len(out) = %d, want %d", len(out), len(sampleResults()))
 	}
@@ -173,14 +216,14 @@ func TestRenderJSON_IncludesResponseTextFinishReasonAndTruncated(t *testing.T) {
 		t.Fatalf("Render() error = %v", err)
 	}
 
-	var out []jsonResult
-	if err := json.Unmarshal(buf.Bytes(), &out); err != nil {
+	var doc jsonArtifact
+	if err := json.Unmarshal(buf.Bytes(), &doc); err != nil {
 		t.Fatalf("json.Unmarshal() error = %v; output:\n%s", err, buf.String())
 	}
-	if len(out) != 1 {
-		t.Fatalf("len(out) = %d, want 1", len(out))
+	if len(doc.Results) != 1 {
+		t.Fatalf("len(results) = %d, want 1", len(doc.Results))
 	}
-	got := out[0]
+	got := doc.Results[0]
 	if got.ResponseText != "partial answer" {
 		t.Errorf("ResponseText = %q, want %q", got.ResponseText, "partial answer")
 	}
@@ -211,7 +254,7 @@ func TestSummarize_Buckets(t *testing.T) {
 		{Model: "m1", TestID: "d", Err: errors.New("fail")},
 		{Model: "m1", TestID: "e", Score: eval.Score{Skipped: true}},
 	}
-	summaries := summarize([]string{"m1"}, results)
+	summaries := summarize([]string{"m1"}, indexResults(results))
 	if len(summaries) != 1 {
 		t.Fatalf("summarize() len = %d, want 1", len(summaries))
 	}
@@ -236,7 +279,7 @@ func TestSummarize_AvgTokPerSec(t *testing.T) {
 		{Model: "m1", TestID: "err", Err: errors.New("fail"), Latency: time.Second, CompletionTokens: 999},
 		{Model: "m1", TestID: "skipped", Score: eval.Score{Skipped: true}, Latency: time.Second, CompletionTokens: 999},
 	}
-	summaries := summarize([]string{"m1"}, results)
+	summaries := summarize([]string{"m1"}, indexResults(results))
 	ms := summaries[0]
 
 	want := 110.0 / 2.1
@@ -250,7 +293,7 @@ func TestSummarize_AvgTokPerSec_NoDataIsNaN(t *testing.T) {
 		{Model: "m1", TestID: "err", Err: errors.New("fail")},
 		{Model: "m1", TestID: "skipped", Score: eval.Score{Skipped: true}},
 	}
-	summaries := summarize([]string{"m1"}, results)
+	summaries := summarize([]string{"m1"}, indexResults(results))
 	if !math.IsNaN(summaries[0].avgTokPerSec) {
 		t.Errorf("avgTokPerSec = %v, want NaN when there is no timed data", summaries[0].avgTokPerSec)
 	}

internal/report/table.go

@@ -26,8 +26,8 @@ func (ew *errWriter) println(args ...any) {
 	_, ew.err = fmt.Fprintln(ew.w, args...)
 }
 
-// renderTable writes three plain-text tables: per-test scores, category
-// rollup, and model summary.
+// renderTable writes four plain-text tables: per-test scores, category
+// rollup, discrimination/stability rollup, and model summary.
 func renderTable(w io.Writer, tests []testkit.Test, models []string, results []runner.Result) error {
 	idx := indexResults(results)
 	tw := tabwriter.NewWriter(w, 0, 4, 2, ' ', 0)
@@ -37,7 +37,9 @@ func renderTable(w io.Writer, tests []testkit.Test, models []string, results []r
 	ew.println()
 	writeCategoryTable(ew, tests, models, idx)
 	ew.println()
-	writeSummaryTable(ew, models, results)
+	writeDiscriminationTable(ew, testStats(tests, models, idx), len(models) > 1)
+	ew.println()
+	writeSummaryTable(ew, models, idx)
 	if ew.err != nil {
 		return ew.err
 	}
@@ -45,7 +47,7 @@ func renderTable(w io.Writer, tests []testkit.Test, models []string, results []r
 	return tw.Flush()
 }
 
-func writeScoreTable(ew *errWriter, tests []testkit.Test, models []string, idx resultIndex) {
+func writeScoreTable(ew *errWriter, tests []testkit.Test, models []string, idx cellIndex) {
 	ew.println("== Per-test scores ==")
 	header := []string{"TEST", "CATEGORY", "SUBCATEGORY"}
 	header = append(header, models...)
@@ -55,9 +57,9 @@ func writeScoreTable(ew *errWriter, tests []testkit.Test, models []string, idx r
 	for _, t := range sortedTests(tests) {
 		row := []string{t.ID, t.Category, t.Subcategory}
 		for _, model := range models {
-			r, ok := idx[t.ID][model]
-			row = append(row, cellText(r, ok))
-			anyTruncated = anyTruncated || r.Truncated()
+			c := idx[t.ID][model]
+			row = append(row, cellText(c))
+			anyTruncated = anyTruncated || c.result.Truncated()
 		}
 		ew.println(strings.Join(row, "\t"))
 	}
@@ -66,7 +68,7 @@ func writeScoreTable(ew *errWriter, tests []testkit.Test, models []string, idx r
 	}
 }
 
-func writeCategoryTable(ew *errWriter, tests []testkit.Test, models []string, idx resultIndex) {
+func writeCategoryTable(ew *errWriter, tests []testkit.Test, models []string, idx cellIndex) {
 	ew.println("== Category rollup (mean score) ==")
 	header := append([]string{"CATEGORY"}, models...)
 	ew.println(strings.Join(header, "\t"))
@@ -80,11 +82,82 @@ func writeCategoryTable(ew *errWriter, tests []testkit.Test, models []string, id
 	}
 }
 
-func writeSummaryTable(ew *errWriter, models []string, results []runner.Result) {
+// writeDiscriminationTable lists the tests that actually measure
+// something: cross-model spread >= DiscriminationSpread, or attempts that
+// disagreed under --repeat. With a single model there is no spread to
+// speak of, so the non-passing tests are listed instead.
+func writeDiscriminationTable(ew *errWriter, stats []testStat, multiModel bool) {
+	writeDiscriminationRows(tablePrinter{ew: ew}, stats, multiModel)
+}
+
+// discriminationSink abstracts the tab (table) vs pipe (markdown) output
+// so the discrimination section is written once.
+type discriminationSink interface {
+	header(cells ...string)
+	row(cells ...string)
+	note(text string)
+}
+
+type tablePrinter struct{ ew *errWriter }
+
+func (tp tablePrinter) header(cells ...string) { tp.ew.println(strings.Join(cells, "\t")) }
+func (tp tablePrinter) row(cells ...string)    { tp.ew.println(strings.Join(cells, "\t")) }
+func (tp tablePrinter) note(text string)       { tp.ew.println(text) }
+
+func writeDiscriminationRows(sink discriminationSink, stats []testStat, multiModel bool) {
+	discriminating, unstable, scored := 0, 0, 0
+	for _, st := range stats {
+		if !st.scored {
+			continue
+		}
+		scored++
+		if st.unstable {
+			unstable++
+		}
+		if multiModel && !math.IsNaN(st.spread) && st.spread >= DiscriminationSpread {
+			discriminating++
+		}
+	}
+
+	if multiModel {
+		sink.note(fmt.Sprintf("== Discrimination: %d/%d scored tests separate models by >= %.2f; %d unstable under repeat ==",
+			discriminating, scored, DiscriminationSpread, unstable))
+	} else {
+		sink.note(fmt.Sprintf("== Discrimination: single model (no cross-model spread); %d unstable under repeat ==", unstable))
+	}
+
+	rows := interestingStats(stats, multiModel)
+	if len(rows) == 0 {
+		sink.note("no discriminating or unstable tests: every scored test passed identically for every model")
+		return
+	}
+
+	sink.header("TEST", "CATEGORY", "SUBCATEGORY", "MEAN", "SPREAD", "UNSTABLE", "TRUNC")
+	for _, st := range rows {
+		sink.row(
+			st.testID,
+			st.category,
+			st.subcategory,
+			meanCellText(st.mean),
+			meanCellText(st.spread),
+			yesNo(st.unstable),
+			yesNo(st.truncated),
+		)
+	}
+}
+
+func yesNo(b bool) string {
+	if b {
+		return "yes"
+	}
+	return "-"
+}
+
+func writeSummaryTable(ew *errWriter, models []string, idx cellIndex) {
 	ew.println("== Model summary ==")
 	ew.println(strings.Join([]string{"MODEL", "MEAN", "PASSED", "PARTIAL", "FAILED", "ERRORS", "MEAN_LATENCY", "TOTAL_TOKENS", "TOK/S"}, "\t"))
 
-	for _, ms := range summarize(models, results) {
+	for _, ms := range summarize(models, idx) {
 		row := []string{
 			ms.model,
 			meanCellText(ms.overallMean),

internal/runner/runner.go

@@ -20,7 +20,9 @@ import (
 // referencing FinishReason separately.
 const truncatedDetailPrefix = "TRUNCATED: "
 
-// Result is the outcome of running one test against one model.
+// Result is the outcome of running one test against one model. Attempt is
+// the zero-based index of this sample when Config.Repeat > 1; it is 0 for
+// single-sample runs.
 type Result struct {
 	Err              error
 	Model            string
@@ -31,6 +33,7 @@ type Result struct {
 	Latency          time.Duration
 	PromptTokens     int
 	CompletionTokens int
+	Attempt          int
 }
 
 // Truncated reports whether the model's response was cut off by the token
@@ -54,6 +57,12 @@ type Config struct {
 	Concurrency      int
 	Temperature      float64
 	MaxTokensDefault int
+	// Repeat runs each (model, test) combination this many times (minimum
+	// 1). Even at temperature 0, reasoning models sample differently per
+	// call; repeats expose that instability so a score can be trusted
+	// (stable across attempts) or flagged (flaky) instead of silently
+	// treated as a capability measurement.
+	Repeat int
 }
 
 // Runner executes a set of tests against a set of models.
@@ -71,18 +80,24 @@ func New(client llm.Client, cfg Config) *Runner {
 }
 
 // Run executes every combination of models x tests, bounded by
-// cfg.Concurrency in-flight calls, and returns one Result per combination.
-// A per-call failure is captured in Result.Err rather than aborting the run.
+// cfg.Concurrency in-flight calls, and returns one Result per combination
+// (times Config.Repeat samples each). A per-call failure is captured in
+// Result.Err rather than aborting the run.
 func (r *Runner) Run(ctx context.Context, models []string, tests []testkit.Test) []Result {
 	type job struct {
-		model string
-		test  testkit.Test
+		model   string
+		test    testkit.Test
+		attempt int
 	}
 
-	jobs := make([]job, 0, len(models)*len(tests))
+	repeat := max(r.cfg.Repeat, 1)
+
+	jobs := make([]job, 0, len(models)*len(tests)*repeat)
 	for _, m := range models {
 		for _, t := range tests {
-			jobs = append(jobs, job{model: m, test: t})
+			for a := range repeat {
+				jobs = append(jobs, job{model: m, test: t, attempt: a})
+			}
 		}
 	}
 
@@ -103,6 +118,7 @@ func (r *Runner) Run(ctx context.Context, models []string, tests []testkit.Test)
 	for i, j := range jobs {
 		g.Go(func() error {
 			res := r.runOne(gctx, j.model, j.test)
+			res.Attempt = j.attempt
 			results[i] = res
 			// atomic.AddInt32 makes the completion counter safe under
 			// Config.Concurrency-many concurrent goroutines; ReportDone

internal/runner/runner_test.go

@@ -377,3 +377,36 @@ func TestRunner_Run_TracksLatencyAndTokens(t *testing.T) {
 		t.Errorf("Latency = %v, want 42ms", results[0].Latency)
 	}
 }
+
+func TestRunner_Run_RepeatSamplesEachCombination(t *testing.T) {
+	client := &mockClient{}
+	r := New(client, Config{Concurrency: 2, MaxTokensDefault: 100, Repeat: 3})
+
+	tests := []testkit.Test{{ID: "t1", Category: "c", Prompt: "p1", Eval: echoEval()}}
+	models := []string{"model-a", "model-b"}
+
+	results := r.Run(context.Background(), models, tests)
+
+	if len(results) != 6 {
+		t.Fatalf("len(results) = %d, want 6 (2 models x 1 test x 3 repeats)", len(results))
+	}
+	seen := make(map[[2]string][]int)
+	for _, res := range results {
+		k := [2]string{res.Model, res.TestID}
+		seen[k] = append(seen[k], res.Attempt)
+	}
+	for k, attempts := range seen {
+		if len(attempts) != 3 {
+			t.Errorf("%v: %d attempts, want 3", k, len(attempts))
+		}
+		got := map[int]bool{}
+		for _, a := range attempts {
+			got[a] = true
+		}
+		for a := range 3 {
+			if !got[a] {
+				t.Errorf("%v: missing attempt %d (have %v)", k, a, attempts)
+			}
+		}
+	}
+}

internal/tests/catalog.go

@@ -23,5 +23,8 @@ func All() *testkit.Registry {
 	registerSecurityTests(r)
 	registerDeliveryTests(r)
 	registerAITests(r)
+	registerHardTests(r)
+	registerScenarioTests(r)
+	registerSaturationTests(r)
 	return r
 }

internal/tests/catalog_test.go

@@ -64,8 +64,8 @@ func TestAll_SubcategoryMinimums(t *testing.T) {
 	const minTestsPerSubcategory = 10
 
 	wantSubcats := map[string][]string{
-		"programming": {"golang", "python", "typescript", "c"},
-		"operations":  {"macos", "linux", "kubernetes"},
+		"programming": {"golang", "python", "typescript", "c", "hard"},
+		"operations":  {"macos", "linux", "kubernetes", "scenario"},
 		"research":    {"web", "whitepapers", "codebase"},
 		"agents":      {"tool-routing", "planning", "delegation"},
 		"databases":   {"postgres", "redis", "sql-tuning"},

internal/tests/databases_redis.go

@@ -159,27 +159,58 @@ pipelined commands? Respond with only "yes" or "no".`
 	}
 }
 
-// dbKeysCommandPattern matches an explicit mention of the Redis KEYS
-// command actually being invoked - "KEYS *" (its most common prod-danger
-// form) or a backtick-quoted `KEYS` - not a generic use of the English
-// word "keys" elsewhere in prose ("the keys to fixing this...").
-var dbKeysCommandPattern = regexp.MustCompile("(?:`KEYS`|\\bKEYS\\s*\\*)")
+// dbKeysEndorsementPattern matches a KEYS mention in an endorsement-shaped
+// position — a verb recommending it ("use `KEYS`", "just run KEYS *"), a
+// positive predicate ("KEYS * is fine/simplest"), or a bare imperative line
+// ("run this:\nKEYS *") — rather than every mention. The guard then still
+// requires a negation cue near such a mention (see dbNoBareKeysInProd).
+//
+// Triggering on any mention is too strict: a fully correct live answer
+// (probe, 2026-08-29) opens with "`KEYS *` is dangerous..." and later says
+// "so `KEYS *` can block the server while it walks through millions of
+// keys" — harm descriptions whose words ("can block") sit outside the
+// negation cue list, yet they are exactly the explanation the prompt asks
+// for, not an endorsement.
+//
+// The predicate branch's leading (?:^|[^...]) guard stops a hyphen-joined
+// identifier from matching mid-word and keeps the match start at the term
+// area rather than arbitrary prose.
+var dbKeysEndorsementPattern = regexp.MustCompile("(?i)(?:" +
+
+	// a verb recommending it: "use `KEYS`", "just run KEYS *", "stick with KEYS *"
+
+	"(?:use|using|used|run|running|runs|execute|executing|call|calling|should(?:\\s+(?:also|just))?|better\\s+to|go\\s+with|opt\\s+for|stick\\s+with|switch(?:ing)?\\s+(?:back\\s+)?to)\\s+(?:the\\s+)?(?:redis\\s+)?(?:command\\s+)?(?:`KEYS`|\\bKEYS\\s*\\*)" +
+
+	// a positive predicate about it: "KEYS * is fine", "`KEYS` is simplest"
+
+	"|(?:^|[^A-Za-z0-9_-])(?:`KEYS`|\\bKEYS\\s*\\*)[^.;\\n]{0,30}?\\b(?:is|are|was|were|seems?|looks?|works?|stays?|remains?|would\\s+be|should\\s+be)\\s+(?:a\\s+|just\\s+|totally\\s+|still\\s+|actually\\s+)?(?:fine|good|great|safe|ok|okay|acceptable|simplest|easiest|better|fast\\s+enough)" +
+
+	// a bare command line after a colon or full stop: "run this:\\nKEYS *" -
+
+	// requiring the break keeps a line-initial mention that merely
+
+	// continues a wrapped sentence from triggering.
+
+	"|(?:[.:]|^)[ \\t]*(?:\\r?\\n)[ \\t*`>-]*(?:`KEYS`|\\bKEYS\\s*\\*)" +
+
+	")")
 
 // dbNegationWindow is how many characters around a KEYS-command mention
 // are searched for a negation cue.
 const dbNegationWindow = 60
 
-// dbNoBareKeysInProd scores full credit unless the response instructs
-// running "KEYS *" against a production instance without any negating
-// context (a warning not to, or an instruction to use SCAN instead is
-// fine). This is deliberately not eval.NotContains, which would also zero
-// out the best possible answer: one that correctly explains why NOT to run
-// KEYS in production.
+// dbNoBareKeysInProd scores full credit unless the response endorses
+// running "KEYS *" against a production instance (see
+// dbKeysEndorsementPattern) with no negating context nearby — a warning
+// not to, or an instruction to use SCAN instead, is fine. This is
+// deliberately not eval.NotContains, which would also zero out the best
+// possible answer: one that correctly explains why NOT to run KEYS in
+// production.
 //
 // Delegates to eval.NoUnnegatedMention (D5), the primitive shared with
 // kubernetes.go, security.go, and delivery_git.go's equivalent guards.
 func dbNoBareKeysInProd() eval.Evaluator {
-	return eval.NoUnnegatedMention(dbKeysCommandPattern, dbNegationWindow, nil)
+	return eval.NoUnnegatedMention(dbKeysEndorsementPattern, dbNegationWindow, nil)
 }
 
 // dbRedisScanVsKeysTest: require SCAN over KEYS for a hot-path production

internal/tests/databases_redis_test.go

@@ -147,10 +147,33 @@ func TestDBNoBareKeysInProd(t *testing.T) {
 			want:     0,
 		},
 		{
-			name:     "negation cue too far away to count",
+			// The guard now triggers on an endorsement-shaped mention only,
+			// and the negation window is measured from the endorsement
+			// phrase ("...run KEYS *"), not from the sentence start — so a
+			// hedging "never mind" that is still within 60 chars counts.
+			name:     "negation cue close to the endorsement phrase",
 			response: "Never mind the migration for now, but you could casually run KEYS * to check this quickly.",
+			want:     1,
+		},
+		{
+			// ground truth: same endorsement, but the cue is now >60 chars
+			// from "run KEYS *" (73), outside the window - still an
+			// unnegated recommendation.
+			name:     "negation cue too far away to count",
+			response: "Never mind everything else that was said in the earlier review about migrations, but you could casually run KEYS * to check this quickly.",
 			want:     0,
 		},
+		{
+			// Regression from the live probe (2026-08-29): a fully correct
+			// answer describes the harm ("can block", no cue word in the
+			// list) and only negates one mention ("avoid `KEYS *` unless").
+			// None of these is an endorsement, so the guard must stay at 1.
+			name: "correct probe answer: harm description without cue words",
+			response: "`KEYS *` is dangerous in production because it scans the entire keyspace synchronously.\n" +
+				"Redis is single-threaded for command execution, so `KEYS *` can block the server while it walks through millions of keys.\n" +
+				"For production code, use `SCAN` with a reasonable `COUNT`, and avoid `KEYS *` unless you are in development.",
+			want: 1,
+		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {

internal/tests/operations_scenario.go

@@ -0,0 +1,538 @@
+package tests
+
+import (
+	"context"
+	"encoding/json"
+	"strings"
+
+	"github.com/lukaszraczylo/llm-testbench/internal/eval"
+	"github.com/lukaszraczylo/llm-testbench/internal/testkit"
+)
+
+func registerScenarioTests(r *testkit.Registry) {
+	r.Register(scenDiskpressureFirstcmdTest())
+	r.Register(scenInodeExhaustionTest())
+	r.Register(scenCrashloopOrderTest())
+	r.Register(scenNetLayerIsolateTest())
+	r.Register(scenServiceDegradeTest())
+	r.Register(scenLogRootcauseMCQTest())
+	r.Register(scenBackupRestoreOrderTest())
+	r.Register(scenCertexpiryTriageTest())
+	r.Register(scenSystemdFailreasonTest())
+	r.Register(scenPortbindFailTest())
+	r.Register(scenLvmResizeOrderTest())
+}
+
+// scenDiskpressureDf is the inline df output for scenDiskpressureFirstcmdTest.
+const scenDiskpressureDf = `Filesystem      Size  Used Avail Use% Mounted on
+/dev/sda1       100G  100G    0G 100% /
+tmpfs           16G   1.1G   15G   8% /run
+/dev/sdb1       500G   30G  470G   6% /data`
+
+// scenFirstDiskCommand scores a response whose {"first_command"} field is a
+// safe, size-sorted du scan of the root filesystem. It demands the two
+// essential verb flags (du to measure recursive usage, sort to order the
+// result) and rejects a destructive first step (rm). A naive model that
+// jumps straight to deleting files, or that lists with ls (which does not
+// measure recursive usage) or find -size (which only matches known sizes),
+// scores zero. This is deliberately not a bare ContainsAny on the whole
+// response: the prompt forces a JSON object, and matching inside the
+// command field keeps the check grounded in an actual command rather than
+// prose that merely names du.
+func scenFirstDiskCommand() eval.Evaluator {
+	return eval.EvaluatorFunc(func(_ context.Context, response string) eval.Score {
+		raw, err := eval.ExtractJSON(response)
+		if err != nil {
+			return eval.Score{Value: 0, Detail: "no JSON object found"}
+		}
+		var obj map[string]string
+		if err := json.Unmarshal([]byte(raw), &obj); err != nil {
+			return eval.Score{Value: 0, Detail: "JSON not usable as a string map"}
+		}
+		cmd := strings.ToLower(strings.TrimSpace(obj["first_command"]))
+		if cmd == "" {
+			return eval.Score{Value: 0, Detail: "first_command empty"}
+		}
+		for _, req := range []string{"du", "sort"} {
+			if !strings.Contains(cmd, req) {
+				return eval.Score{Value: 0, Detail: "first_command missing " + req}
+			}
+		}
+		if strings.HasPrefix(cmd, "rm") || strings.Contains(cmd, " rm ") || strings.Contains(cmd, "rm -") {
+			return eval.Score{Value: 0, Detail: "first_command starts by deleting files, not measuring"}
+		}
+		return eval.Score{Value: 1, Detail: "first_command is a safe du|sort scan of the filesystem"}
+	})
+}
+
+// scenDiskpressureFirstcmdTest: give the exact one-line first command to
+// find what is filling a 100%-full root filesystem.
+//
+// ground truth: / is 100% full on /dev/sda1 while /data (/dev/sdb1) has
+// 470G free, so the scan must stay within the / filesystem and rank top-
+// level directories by size. The canonical, standard-OSS one-liner is
+// `du -xh --max-depth=1 / 2>/dev/null | sort -rh | head -20` - du with
+// -x to not descend into the nearly-empty /data mount, a depth of 1 to get
+// per-top-level-directory usage, sort -rh descending by human size, and
+// head to the top consumers. ls does not measure recursive usage and find
+// -size only matches known file sizes, so neither finds the fillers.
+func scenDiskpressureFirstcmdTest() testkit.Test {
+	prompt := `This Linux host's root filesystem /dev/sda1 is 100%
+full; /data on /dev/sdb1 has plenty of free space. You must find which
+paths under / are consuming the space, so you can free it. Here is the
+relevant df output:
+
+` + scenDiskpressureDf + `
+
+Give the exact one-line shell command you would run FIRST to identify the
+largest consumers under /, staying on the / filesystem and producing a
+size-sorted list. Respond with only a JSON object:
+{"first_command":"<the exact one-line command>"}`
+
+	return testkit.Test{
+		ID:          "scen-diskpressure-firstcmd",
+		Category:    "operations",
+		Subcategory: "scenario",
+		Description: "Give the exact first shell command (du|sort) to find what filled a 100%-full root filesystem.",
+		Prompt:      prompt,
+		Eval:        scenFirstDiskCommand(),
+	}
+}
+
+// scenInodeExhaustionDfHi is the inline df -hi output for
+// scenInodeExhaustionTest.
+const scenInodeExhaustionDfHi = `$ df -h /
+Filesystem      Size  Used Avail Use% Mounted on
+/dev/sda1       100G   35G   65G  35% /
+
+$ df -hi /
+Filesystem     Inodes IUsed IFree IUse% Mounted on
+/dev/sda1       1.0M  1.0M    0  100% /`
+
+// scenInodeExhaustionTest: diagnose "No space left on device" that is
+// actually inode exhaustion (blocks free, inodes exhausted).
+//
+// ground truth: writes fail with "No space left on device" even though
+// df -h shows 65G free, because df -hi shows the filesystem's inode
+// table is 100% used (1.0M of 1.0M). Inode exhaustion, not block
+// exhaustion, is the cause; the command that reveals it is `df -hi`
+// (df with the inode flag). The fix is deleting cheap small files
+// (often an unbounded cache dir filling the inode table with tiny files),
+// not growing the disk (which adds blocks, not inodes) or swapping mounts.
+func scenInodeExhaustionTest() testkit.Test {
+	prompt := `Deleting nothing and with the disk apparently not full,
+this host now fails writes with "No space left on device". Here are two
+diagnostic outputs:
+
+` + scenInodeExhaustionDfHi + `
+
+What is the cause, and which exact command produced the revealing
+output? Respond with only a JSON object:
+{"cause":"<one of: inodes-exhausted, disk-full, quota-exceeded>",
+"command":"<one of: df -hi, df -h, stat -f> "}`
+
+	evaluator := eval.Mean(
+		eval.JSONField("cause", "inodes-exhausted"),
+		eval.JSONField("command", "df -hi"),
+	)
+
+	return testkit.Test{
+		ID:          "scen-inode-exhaustion",
+		Category:    "operations",
+		Subcategory: "scenario",
+		Description: "Recognize ENOSPC from inode exhaustion (not disk space) and the df -hi command that reveals it.",
+		Prompt:      prompt,
+		Eval:        evaluator,
+	}
+}
+
+// scenCrashloopOrderTest: an incident where the operator must pick the
+// diagnostic steps to find why a CrashLoopBackOff container crashes,
+// before taking any action.
+//
+// ground truth: the only observation is the CrashLoopBackOff status and
+// the restart count - nothing about *why* the container dies. Both
+// read-only diagnostics are needed: B (`kubectl describe pod`, recent
+// events and the last exit reason - rules out OOM-kill, image problems,
+// probe failures) and A (`kubectl logs <pod> --previous`, the actual
+// error output of the crashed run - names the failing line). Neither
+// alone roots an app crash: describe shows the exit code but not the
+// error message; logs --previous show the error but not the node-level
+// events. Whether describe or logs comes first is a matter of taste, so
+// the evaluator scores the SET {A, B}. What the test discriminates is
+// diagnose-before-acting: C (`rollout restart`) and D (`scale
+// --replicas=0`) act without diagnosing, and E (ssh to a node for
+// journalctl) reaches for node-level tooling prematurely when the
+// container's own state and logs are one kubectl call away.
+func scenCrashloopOrderTest() testkit.Test {
+	prompt := `A pod named ingestion-5d7c9b6f8b-p2k1q in namespace data
+keeps crashing and restarting: kubectl get pods shows STATUS
+CrashLoopBackOff, RESTARTS 14. You know nothing else about why it dies,
+and you must find the root cause before changing anything. These are the
+available steps:
+
+A) kubectl logs <pod> --previous   (read the last crashed run's output)
+B) kubectl describe pod <pod>      (inspect events and last exit reason)
+C) kubectl rollout restart deployment ingestion
+D) kubectl scale deployment ingestion --replicas=0
+E) ssh to a node and run journalctl -u kubelet
+
+Respond with only a JSON array of the step letters that make up your
+diagnosis, containing exactly the diagnostic steps you would take to find
+the root cause (any order) - do not include steps that only act without
+diagnosing.`
+
+	return testkit.Test{
+		ID:          "scen-crashloop-order",
+		Category:    "operations",
+		Subcategory: "scenario",
+		Description: "Pick the diagnostic steps (describe + logs --previous) that root-cause a CrashLoopBackOff before acting.",
+		Prompt:      prompt,
+		Eval:        eval.JSONStringSet([]string{"A", "B"}),
+	}
+}
+
+// scenNetLayerSymptom is the inline diagnostic session for
+// scenNetLayerIsolateTest.
+const scenNetLayerSymptom = `$ curl -sS https://api.example.org/data
+curl: (6) Could not resolve host: api.example.org
+
+$ ping -c1 10.0.0.7
+64 bytes from 10.0.0.7: icmp_seq=1 ttl=64 time=0.8 ms
+(10.0.0.7 is a working internal host on the same subnet)
+
+$ cat /etc/resolv.conf
+nameserver 10.0.0.53`
+
+// scenNetLayerIsolateTest: isolate which network layer is broken from
+// inline symptoms and pick the strict layer among an enumerated set.
+//
+// ground truth: curl fails with "Could not resolve host" - DNS
+// resolution, not a routing or TCP problem (an ICMP packet to an internal
+// host on the same subnet succeeds, so IP reachability on the local
+// segment is fine, and curl never even reached TCP/TLS because it could
+// not translate the name). The nameserver 10.0.0.53 is configured but
+// the name does not resolve, so the failing layer is strictly dns; it is
+// not ip-routing, tcp-connect, tls, or http, none of which are reached
+// when resolution itself fails.
+func scenNetLayerIsolateTest() testkit.Test {
+	prompt := `A service on this host cannot reach https://api.example.org.
+Here is the diagnostic session:
+
+` + scenNetLayerSymptom + `
+
+At which network layer is the failure occurring? Respond with only a
+JSON object:
+{"layer":"<one of: dns, ip-routing, tcp-connect, tls, http>"}
+
+Choose exactly one layer from the enumerated list.`
+
+	return testkit.Test{
+		ID:          "scen-net-layer-isolate",
+		Category:    "operations",
+		Subcategory: "scenario",
+		Description: "Isolate a network failure to the DNS layer from inline curl/ping/resolv.conf symptoms.",
+		Prompt:      prompt,
+		Eval:        eval.JSONField("layer", "dns"),
+	}
+}
+
+// scenServiceDegradeContext is the inline telemetry + deploy timeline for
+// scenServiceDegradeTest.
+const scenServiceDegradeContext = `Canary deployed to 100% at 14:00: config file
+change lowering the app's connection-pool size (pool: 200 -> 50) and
+raising the worker count.
+
+Latency (p99) of the checkout service API:
+  13:30  210ms
+  14:05  1400ms
+  14:30  1650ms
+  15:00  1700ms
+
+Resource metrics for the checkout pods: CPU at 12%, memory at 40% -
+both low. No crash, no 5xx errors, no data loss.`
+
+// scenServiceDegradeTest: a service-degradation decision tree where the
+// regression is tracked to a config change with healthy resources.
+//
+// ground truth: the p99 latency jumped from ~210ms to ~1400ms exactly at
+// the 14:00 canary rollout, whose only meaningful change is the
+// connection-pool size drop (200 -> 50) - the obvious pattern match is the
+// config change, and CPU/memory are low so the pool is not saturated by
+// resource pressure. The correct action is to roll back / revert that
+// config change (rollback-config). scale-out is wrong (no CPU/memory
+// pressure to scale for), restart alone is wrong (it does not revert the
+// config), and restore-backup is wrong (no data loss to recover from).
+func scenServiceDegradeTest() testkit.Test {
+	prompt := `The checkout service API latency spiked and stayed high.
+Here is the context:
+
+` + scenServiceDegradeContext + `
+
+Which single action best restores service? Respond with only a JSON
+object:
+{"action":"<one of: rollback-config, scale-out, restore-backup, restart>"}`
+
+	return testkit.Test{
+		ID:          "scen-service-degrade",
+		Category:    "operations",
+		Subcategory: "scenario",
+		Description: "Pick the correct remediation (config rollback) for a canary-introduced service degradation with healthy resources.",
+		Prompt:      prompt,
+		Eval:        eval.JSONField("action", "rollback-config"),
+	}
+}
+
+// scenPgConnectionLog is the inline log excerpt for
+// scenLogRootcauseMCQTest.
+const scenPgConnectionLog = `2026-08-29 14:22:11 UTC FATAL:  remaining connection slots are reserved for non-replication superuser connections
+2026-08-29 14:22:11 UTC DETAIL:  1000 concurrent connections, 100 remaining connection slots available
+2026-08-29 14:22:12 UTC ERROR:  connection to server at "db.internal" (10.0.1.25), port 5432 failed: FATAL:  remaining connection slots are reserved for non-replication superuser connections`
+
+// scenLogRootcauseMCQTest: map a log pattern to its root cause from
+// lettered options.
+//
+// ground truth: "remaining connection slots are reserved" with a detail
+// line counting 1000 concurrent connections is the textbook PostgreSQL
+// signal that the connection slot limit (max_connections) has been
+// reached - applications are holding more connections than the server is
+// configured to allow. It is not a disk-full condition (no out-of-space
+// message), not a slow-query problem (no long query shown), and not an
+// authentication failure (no `password authentication failed` line).
+func scenLogRootcauseMCQTest() testkit.Test {
+	prompt := `Applications calling the Postgres database below now fail to
+connect. Here is a log excerpt:
+
+` + scenPgConnectionLog + `
+
+What is the root cause of these connection failures?
+
+A) connection pool exhausted (max_connections reached)
+B) disk full
+C) a single slow query blocking others
+D) authentication failure
+
+Respond with only a JSON object: {"answer":"<the single letter A-D>"}`
+
+	return testkit.Test{
+		ID:          "scen-log-rootcause-mcq",
+		Category:    "operations",
+		Subcategory: "scenario",
+		Description: "Map a PostgreSQL max_connections log pattern to its root cause via a lettered MCQ.",
+		Prompt:      prompt,
+		Eval:        eval.JSONField("answer", "A"),
+	}
+}
+
+// scenBackupRestoreOptions lists the available steps for
+// scenBackupRestoreOrderTest.
+const scenBackupRestoreOptions = `A) take a fresh logical backup (pg_dump) of the current data
+B) apply the destructive migration
+C) validate row counts and data after the migration
+D) restore last night's backup before doing anything else`
+
+// scenBackupRestoreOrderTest: order the steps for a safe data-changing
+// migration, guaranteeing recoverability.
+//
+// ground truth: to make a destructive migration recoverable you must
+// snapshot the CURRENT state first (A - a fresh pg_dump), then apply the
+// migration (B), then validate the result (C). Restoring last night's
+// backup (D) first is wrong: it is a stale snapshot that would throw
+// away today's writes before any change has even been made, and it cannot
+// be the safety net for the migration.
+func scenBackupRestoreOrderTest() testkit.Test {
+	prompt := `You are about to apply a destructive migration to a
+production Postgres database and must guarantee you can get back to the
+current state. The available steps are:
+
+` + scenBackupRestoreOptions + `
+
+Respond with only a JSON array of the step letters, in the order from
+first to last, that makes this migration recoverable - include exactly
+the steps you would run, in order.`
+
+	return testkit.Test{
+		ID:          "scen-backup-restore-order",
+		Category:    "operations",
+		Subcategory: "scenario",
+		Description: "Order the steps (fresh backup, then migrate, then validate) that make a destructive migration recoverable.",
+		Prompt:      prompt,
+		Eval:        eval.JSONStringArrayEquals([]string{"A", "B", "C"}),
+	}
+}
+
+// scenCertExpirySymptom is the inline symptom for
+// scenCertexpiryTriageTest.
+const scenCertExpirySymptom = `$ date
+Sat Aug 29 09:14:02 UTC 2026
+
+$ curl -v https://gateway.internal/
+...
+* SSL connection using TLSv1.3
+* SSL certificate verify result: certificate has expired
+
+$ openssl x509 -in /etc/ssl/svc.crt -noout -dates
+notBefore=Aug  1 12:00:00 2025 GMT
+notAfter=Aug  1 12:00:00 2026 GMT`
+
+// scenCertexpiryTriageTest: triage a TLS failure whose cause is an
+// expired certificate.
+//
+// ground truth: `date` anchors wall time at Aug 29 2026; the openssl
+// -dates readout shows the served cert valid until Aug 1 2026 - the
+// current date is past notAfter, so the certificate expired and the
+// cause category is cert-expired. Clock-skew would need the host clock
+// to disagree with reality, but the cert's own window explains the
+// failure exactly, and cipher-mismatch would show a handshake/algo error,
+// not a verify result. The standard command to inspect a certificate's
+// validity end date directly is openssl x509 -noout -enddate (the
+// readout above came from the -dates variant of the same inspection).
+func scenCertexpiryTriageTest() testkit.Test {
+	prompt := `Clients now fail to connect to gateway.internal with a TLS
+verification error. Here is the diagnostic session:
+
+` + scenCertExpirySymptom + `
+
+What is the cause, and which standard command inspects a certificate's
+validity directly? Respond with only a JSON object:
+{"cause":"<one of: cert-expired, clock-skew, cipher-mismatch>",
+"check_command":"<one of: openssl-x509-enddate, date, openssl-cipher>"}`
+
+	evaluator := eval.Mean(
+		eval.JSONField("cause", "cert-expired"),
+		eval.JSONField("check_command", "openssl-x509-enddate"),
+	)
+
+	return testkit.Test{
+		ID:          "scen-certexpiry-triage",
+		Category:    "operations",
+		Subcategory: "scenario",
+		Description: "Identify an expired certificate as the TLS failure cause and the openssl command that inspects validity.",
+		Prompt:      prompt,
+		Eval:        evaluator,
+	}
+}
+
+// scenSystemdStatus is the inline systemctl status excerpt for
+// scenSystemdFailreasonTest.
+const scenSystemdStatus = `$ systemctl status worker-queue
+● worker-queue.service - Queue worker
+     Loaded: loaded (/etc/systemd/system/worker-queue.service; enabled)
+     Active: failed (Result: exit-code) since Tue 2026-08-25 09:12:03 UTC; 12s ago
+   Main PID: 4821 (code=exited, status=1/FAILURE)
+        CPU: 120ms
+
+$ journalctl -u worker-queue -n 6
+Aug 25 09:12:03 host worker-queue[4821]: config/openapi.yaml: open: no such file or directory
+Aug 25 09:12:03 host worker-queue[4821]: fatal: failed to load configuration
+Aug 25 09:12:03 host worker-queue[4821]: exiting after startup failure`
+
+// scenSystemdFailreasonTest: reason about why a systemd unit failed from
+// an inline status + journal excerpt.
+//
+// ground truth: the journal line "config/openapi.yaml: open: no such
+// file or directory" - the binary exits because its configuration file
+// cannot be found at the expected path. The category is missing-config,
+// not permission (no "Permission denied"), not missing-binary (the
+// binary ran and printed a config-diagnostic trace), and not port-in-use
+// (no bind error). The fix is to provide or point the unit at the
+// correct config path, not to reinstall the binary or chase a port.
+func scenSystemdFailreasonTest() testkit.Test {
+	prompt := `A systemd unit is failing. Here is the status and journal:
+
+` + scenSystemdStatus + `
+
+What category of failure is this? Respond with only a JSON object:
+{"category":"<one of: missing-config, permission, missing-binary, port-in-use>"}
+
+Choose exactly one category from the enumerated list.`
+
+	return testkit.Test{
+		ID:          "scen-systemd-failreason",
+		Category:    "operations",
+		Subcategory: "scenario",
+		Description: "Classify a systemd unit failure (missing config file) from inline status and journal output.",
+		Prompt:      prompt,
+		Eval:        eval.JSONField("category", "missing-config"),
+	}
+}
+
+// scenPortbindError is the inline startup error for scenPortbindFailTest.
+const scenPortbindError = `$ service listener start
+(1) listen tcp :8080: bind: address already in use
+listener: could not bind to 0.0.0.0:8080, exiting
+
+$ ss -ltnp
+State    Recv-Q   Send-Q   Local Address:Port   Peer Address:Port  Process
+LISTEN   0      128    0.0.0.0:8080          0.0.0.0:*          users:(("listener",pid=1044,fd=3))`
+
+// scenPortbindFailTest: diagnose an "address already in use" bind
+// failure and the command that identifies the occupying process.
+//
+// ground truth: the startup error "bind: address already in use" for
+// port 8080, and the `ss -ltnp` output shows an existing listener (pid
+// 1044) already bound to 0.0.0.0:8080 - a duplicate/lingering instance
+// still holds the port. The cause category is port-in-use, and the
+// standard command that lists listening sockets with the owning process
+// is `ss -ltnp` (ss with listener, numerical, port and process flags);
+// netstat is the older equivalent but ss is the modern standard on this
+// home-cluster stack.
+func scenPortbindFailTest() testkit.Test {
+	prompt := `A service fails to start. Here is the output:
+
+` + scenPortbindError + `
+
+What is the cause, and which command identified the process holding the
+port? Respond with only a JSON object:
+{"cause":"<one of: port-in-use, missing-binary, config-error>",
+"command":"<one of: ss -ltnp, netstat -r, lsblk>"}`
+
+	evaluator := eval.Mean(
+		eval.JSONField("cause", "port-in-use"),
+		eval.JSONField("command", "ss -ltnp"),
+	)
+
+	return testkit.Test{
+		ID:          "scen-portbind-fail",
+		Category:    "operations",
+		Subcategory: "scenario",
+		Description: "Diagnose an address-in-use bind failure and the ss -ltnp command that reveals the holding process.",
+		Prompt:      prompt,
+		Eval:        evaluator,
+	}
+}
+
+// scenLvmSteps lists the LVM extend steps for scenLvmResizeOrderTest.
+const scenLvmSteps = `A) lvextend -L +20G /dev/vg0/data        (grow the logical volume)
+B) pvs                                     (check free space in the volume group)
+C) resize2fs /dev/vg0/data                 (grow the filesystem to fill the LV)
+D) umount /dev/vg0/data                    (unmount the volume)`
+
+// scenLvmResizeOrderTest: order the steps of a safe LVM logical-volume
+// resize.
+//
+// ground truth: to extend a filesystem on LVM you first confirm there is
+// free space in the volume group (B - `pvs`), then extend the logical
+// volume (A - `lvextend`), then grow the filesystem to fill the newly
+// added space (C - `resize2fs`). Doing C before A, or A before
+// confirming free space in B, fails (nothing to grow into); D (umount)
+// is not required for an online extend of a mounted ext4/xfs volume.
+func scenLvmResizeOrderTest() testkit.Test {
+	prompt := `You must safely grow the filesystem on LVM logical volume
+/dev/vg0/data while it stays online. The available steps are:
+
+` + scenLvmSteps + `
+
+Respond with only a JSON array of the step letters, in the order from
+first to last, that correctly extends this volume - include exactly the
+steps you would run, in order.`
+
+	return testkit.Test{
+		ID:          "scen-lvm-resize-order",
+		Category:    "operations",
+		Subcategory: "scenario",
+		Description: "Order the LVM resize steps (check VG space, extend LV, then grow the filesystem) for an online extend.",
+		Prompt:      prompt,
+		Eval:        eval.JSONStringArrayEquals([]string{"B", "A", "C"}),
+	}
+}

internal/tests/operations_scenario_test.go

@@ -0,0 +1,357 @@
+package tests
+
+import (
+	"context"
+	"testing"
+
+	"github.com/lukaszraczylo/llm-testbench/internal/testkit"
+)
+
+// TestRegisterScenarioTests_Wiring calls registerScenarioTests into a fresh
+// registry and checks it contributes a full operations/scenario layer with
+// scen-* IDs, mirroring the wiring-test pattern used for round-2
+// categories. It also keeps registerScenarioTests referenced so it is not
+// reported as dead code while the parent still wires catalog integration.
+func TestRegisterScenarioTests_Wiring(t *testing.T) {
+	r := testkit.NewRegistry()
+	registerScenarioTests(r)
+
+	if n := r.Len(); n < 10 || n > 12 {
+		t.Fatalf("registerScenarioTests registered %d tests, want 10-12", n)
+	}
+	for _, tc := range r.All() {
+		if tc.Category != "operations" {
+			t.Errorf("test %s has Category %q, want operations", tc.ID, tc.Category)
+		}
+		if tc.Subcategory != "scenario" {
+			t.Errorf("test %s has Subcategory %q, want scenario", tc.ID, tc.Subcategory)
+		}
+		if len(tc.ID) < 5 || tc.ID[:5] != "scen-" {
+			t.Errorf("test %s has an ID not prefixed scen-", tc.ID)
+		}
+	}
+}
+
+func TestScenDiskpressureFirstcmdTest_Eval(t *testing.T) {
+	tc := scenDiskpressureFirstcmdTest()
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{
+			name:     "correct du|sort one-liner",
+			response: `{"first_command":"du -xh --max-depth=1 / 2>/dev/null | sort -rh | head -20"}`,
+			want:     1,
+		},
+		{
+			name:     "correct fenced with prose",
+			response: "Here is the command:\n```json\n{\"first_command\":\"du -x --max-depth=1 / 2>/dev/null | sort -h\"}\n```",
+			want:     1,
+		},
+		{
+			name:     "wrong: ls does not measure recursive usage",
+			response: `{"first_command":"ls -lah /"}`,
+			want:     0,
+		},
+		{
+			name:     "wrong: destructive rm as first step",
+			response: `{"first_command":"rm -rf /var/log && rm -rf /tmp"}`,
+			want:     0,
+		},
+		{
+			name:     "wrong: du but no sort",
+			response: `{"first_command":"du -sh /var"}`,
+			want:     0,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}
+
+func TestScenInodeExhaustionTest_Eval(t *testing.T) {
+	tc := scenInodeExhaustionTest()
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{"correct", `{"cause":"inodes-exhausted","command":"df -hi"}`, 1},
+		{"correct fenced", "```json\n{\"cause\":\"inodes-exhausted\",\"command\":\"df -hi\"}\n```", 1},
+		{"wrong cause", `{"cause":"disk-full","command":"df -hi"}`, 0.5},
+		{"wrong command", `{"cause":"inodes-exhausted","command":"df -h"}`, 0.5},
+		{"both wrong", `{"cause":"quota-exceeded","command":"stat -f"}`, 0},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}
+
+func TestScenCrashloopOrderTest_Eval(t *testing.T) {
+	tc := scenCrashloopOrderTest()
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{"both steps", `["B","A"]`, 1},
+		{"both steps fenced", "```json\n[\"B\",\"A\"]\n```", 1},
+		{
+			// The correct answer discusses the rejected options (C as an
+			// unhelpful action, E as premature) at length; the JSON
+			// array still governs, so the prose cannot change the score.
+			name: "both steps with long discussion of rejected options",
+			response: "```json\n[\"B\",\"A\"]\n```\n\n`kubectl describe pod` (B) shows the last exit reason and `kubectl logs --previous` (A) shows the actual crash output; either may come first since both are read-only. " +
+				"A blind `kubectl rollout restart` (C) would not reveal the cause and `kubectl scale --replicas=0` (D) would only stop the workload; " +
+				"ssh'ing to a node for journalctl (E) is node-level tooling that is premature while the container logs are local.",
+			want: 1,
+		},
+		{"order is free", `["A","B"]`, 1},
+		{"only one diagnostic step", `["A"]`, 0.5},
+		// An action step in the answer costs credit: set overlap 2/3.
+		{"correct plus an action step", `["B","A","D"]`, 2.0 / 3.0},
+		{"all wrong", `["C","D","E"]`, 0},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}
+
+func TestScenNetLayerIsolateTest_Eval(t *testing.T) {
+	tc := scenNetLayerIsolateTest()
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{"correct", `{"layer":"dns"}`, 1},
+		{"correct fenced", "```json\n{\"layer\":\"dns\"}\n```", 1},
+		{
+			name: "correct, discussing other layers at length",
+			response: `{"layer":"dns"}
+
+This is not tls or http: curl never reached the handshake because it could not resolve the name at all; the local ping to 10.0.0.7 rules out ip-routing on the segment.`,
+			want: 1,
+		},
+		{"wrong layer", `{"layer":"tls"}`, 0},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}
+
+func TestScenServiceDegradeTest_Eval(t *testing.T) {
+	tc := scenServiceDegradeTest()
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{"correct", `{"action":"rollback-config"}`, 1},
+		{"correct fenced", "```json\n{\"action\":\"rollback-config\"}\n```", 1},
+		{
+			name: "correct, discussing rejected actions at length",
+			response: `{"action":"rollback-config"}
+
+Scaling out is wrong because CPU and memory are both low - there is no resource pressure to scale for; a restart would not revert the config file; and a backup restore is wrong because there is no data loss to recover from. The regression tracks the 14:00 canary pool-size change, so reverting the config is the fix.`,
+			want: 1,
+		},
+		{"wrong action", `{"action":"restart"}`, 0},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}
+
+func TestScenLogRootcauseMCQTest_Eval(t *testing.T) {
+	tc := scenLogRootcauseMCQTest()
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{"correct", `{"answer":"A"}`, 1},
+		{"correct fenced", "```json\n{\"answer\":\"A\"}\n```", 1},
+		{
+			name: "correct, discussing every distractor at length",
+			response: `{"answer":"A"}
+
+This is not disk full (there is no out-of-space message in the log), not a slow query (no long-running query is shown), and not an authentication failure (no password-authentication-failed line) - it is the classic max_connections exhaustion signal.`,
+			want: 1,
+		},
+		{"wrong letter", `{"answer":"C"}`, 0},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}
+
+func TestScenBackupRestoreOrderTest_Eval(t *testing.T) {
+	tc := scenBackupRestoreOrderTest()
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{"correct order", `["A","B","C"]`, 1},
+		{"correct order fenced", "```json\n[\"A\",\"B\",\"C\"]\n```", 1},
+		{
+			// The correct answer is allowed to explain at length why
+			// restoring last night's backup (D) first is wrong; the JSON
+			// array is what is scored.
+			name: "correct order, discussing rejected restore-first at length",
+			response: `["A","B","C"]
+
+Restoring last night's backup first (D) would throw away today's writes before any change is made - it is a stale snapshot, not a safety net for the migration, so I take a fresh pg_dump first.`,
+			want: 1,
+		},
+		{"wrong: restore first", `["D","A"]`, 0},
+		{"wrong: reordered", `["A","C","B"]`, 0},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}
+
+func TestScenCertexpiryTriageTest_Eval(t *testing.T) {
+	tc := scenCertexpiryTriageTest()
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{"correct", `{"cause":"cert-expired","check_command":"openssl-x509-enddate"}`, 1},
+		{"correct fenced", "```json\n{\"cause\":\"cert-expired\",\"check_command\":\"openssl-x509-enddate\"}\n```", 1},
+		{"wrong cause", `{"cause":"clock-skew","check_command":"openssl-x509-enddate"}`, 0.5},
+		{"wrong command", `{"cause":"cert-expired","check_command":"date"}`, 0.5},
+		{"both wrong", `{"cause":"cipher-mismatch","check_command":"openssl-cipher"}`, 0},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}
+
+func TestScenSystemdFailreasonTest_Eval(t *testing.T) {
+	tc := scenSystemdFailreasonTest()
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{"correct", `{"category":"missing-config"}`, 1},
+		{"correct fenced", "```json\n{\"category\":\"missing-config\"}\n```", 1},
+		{
+			name: "correct, discussing rejected categories at length",
+			response: `{"category":"missing-config"}
+
+This is not a permission problem (no Permission denied in the journal), not a missing binary (the worker ran and printed a config-diagnostic trace), and not a port in use (no bind error) - the unit fails because config/openapi.yaml cannot be opened.`,
+			want: 1,
+		},
+		{"wrong category", `{"category":"permission"}`, 0},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}
+
+func TestScenPortbindFailTest_Eval(t *testing.T) {
+	tc := scenPortbindFailTest()
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{"correct", `{"cause":"port-in-use","command":"ss -ltnp"}`, 1},
+		{"correct fenced", "```json\n{\"cause\":\"port-in-use\",\"command\":\"ss -ltnp\"}\n```", 1},
+		{"wrong cause", `{"cause":"config-error","command":"ss -ltnp"}`, 0.5},
+		{"wrong command", `{"cause":"port-in-use","command":"lsblk"}`, 0.5},
+		{"both wrong", `{"cause":"missing-binary","command":"netstat -r"}`, 0},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}
+
+func TestScenLvmResizeOrderTest_Eval(t *testing.T) {
+	tc := scenLvmResizeOrderTest()
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{"correct order", `["B","A","C"]`, 1},
+		{"correct order fenced", "```json\n[\"B\",\"A\",\"C\"]\n```", 1},
+		{"wrong: grows fs before extending LV", `["A","C","B"]`, 0},
+		{"wrong: includes umount", `["B","A","C","D"]`, 0},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}

internal/tests/programming_hard.go

@@ -0,0 +1,469 @@
+package tests
+
+import (
+	"github.com/lukaszraczylo/llm-testbench/internal/eval"
+	"github.com/lukaszraczylo/llm-testbench/internal/testkit"
+)
+
+// registerHardTests registers every programming/hard test. This subcategory
+// stresses harder daily-programming accuracy - concurrency/async reasoning,
+// subtle language traps, algorithm output tracing, and API-contract edge
+// cases - each forced to a short, machine-checkable answer.
+func registerHardTests(r *testkit.Registry) {
+	r.Register(hardFloatBinaddTest())
+	r.Register(hardNullishOverOrTest())
+	r.Register(hardGoroutineRaceTest())
+	r.Register(hardDeferNamedReturnTest())
+	r.Register(hardIntDivisionTest())
+	r.Register(hardRegexGreedyTest())
+	r.Register(hardPromiseAllSettledTest())
+	r.Register(hardContextTimeoutTest())
+	r.Register(hardJSONUnmarshalNullTest())
+	r.Register(hardVarShadowTest())
+	r.Register(hardPyVersionSortTest())
+	r.Register(hardGoParenBalanceTest())
+}
+
+// hardFloatBinaddTest: judge whether 0.1 + 0.2 === 0.3 holds in
+// JavaScript / IEEE-754 doubles.
+//
+// ground truth: neither 0.1 nor 0.2 has an exact binary representation,
+// so 0.1 + 0.2 computes to the nearest double to the mathematical sum,
+// which is 0.30000000000000004, not the double for the literal 0.3.
+// Therefore the strict equality comparison evaluates to false. This is a
+// canonical, deterministic fact of IEEE-754 double arithmetic, independent
+// of any runtime; verified with node during authoring.
+func hardFloatBinaddTest() testkit.Test {
+	prompt := `In JavaScript, evaluate this comparison:
+
+0.1 + 0.2 === 0.3
+
+Does the strict-equality comparison evaluate to true or to false?
+Respond with only a JSON object:
+{"equal": true}
+or
+{"equal": false}`
+
+	return testkit.Test{
+		ID:          "hard-float-binadd",
+		Category:    "programming",
+		Subcategory: "hard",
+		Description: "Judge IEEE-754 double equality for 0.1 + 0.2 === 0.3 in JavaScript (false).",
+		Prompt:      prompt,
+		Eval:        eval.JSONField("equal", false),
+	}
+}
+
+// hardNullishOverOrTest: trace the value of a nullish-coalescing
+// expression over a falsy (but non-null) operand.
+//
+// ground truth: ?? returns the right-hand operand only when the left-hand
+// operand is null or undefined. 0 is neither null nor undefined, so
+// n ?? 42 evaluates to 0 - unlike ||, which would have returned 42 for
+// the falsy 0. This is the core distinction between nullish coalescing
+// and logical OR. Verified with node during authoring.
+func hardNullishOverOrTest() testkit.Test {
+	prompt := `Consider this TypeScript code:
+
+const n = 0;
+const v = n ?? 42;
+
+What is the value of v?
+Respond with only a JSON object:
+{"value": ...}
+
+Give the numeric value only.`
+
+	return testkit.Test{
+		ID:          "hard-nullish-over-or",
+		Category:    "programming",
+		Subcategory: "hard",
+		Description: "Trace nullish-coalescing over a falsy-but-non-null operand: 0 ?? 42 is 0, not 42.",
+		Prompt:      prompt,
+		Eval:        eval.JSONField("value", 0),
+	}
+}
+
+// hardGoroutineRaceTest: judge whether an unsynchronized shared counter
+// is guaranteed to reach N*M exactly.
+//
+// ground truth: incrementing a shared int from multiple goroutines with no
+// synchronization is a data race (Go's race detector would flag it). A
+// read-modify-write of a shared variable is not atomic; two goroutines can
+// read the same value before either writes back, so increments are lost.
+// The final value is some value strictly between M (at best serialized,
+// one goroutine's work) and N*M, but it is NOT guaranteed to equal
+// N*M. This claim follows from the Go memory model / data-race semantics
+// and cannot be recomputed by running (the outcome is nondeterministic).
+func hardGoroutineRaceTest() testkit.Test {
+	prompt := `A Go program starts 8 goroutines, each of which performs:
+counter++
+1000 times, where counter is a shared int with NO synchronization
+(no mutex, no atomic) between the goroutines. The program then waits for
+all 8 goroutines and prints counter.
+
+Is counter's final value guaranteed to be exactly 8000?
+Respond with only a JSON object:
+{"guaranteed": true}
+or
+{"guaranteed": false}`
+
+	return testkit.Test{
+		ID:          "hard-goroutine-race",
+		Category:    "programming",
+		Subcategory: "hard",
+		Description: "Judge whether an unsynchronized shared counter is guaranteed to reach N*M (false: data race).",
+		Prompt:      prompt,
+		Eval:        eval.JSONField("guaranteed", false),
+	}
+}
+
+// hardDeferNamedReturnTest: trace the return value of a function whose
+// deferred closure writes its named return during a normal (non-panic)
+// return.
+//
+// ground truth: return 7 first assigns the named result r = 7; then,
+// still before the function returns to its caller, the deferred closure
+// runs and overwrites r = 99. The caller observes the named return after
+// all defers have run, so f() returns 99 - not 7 and not 5. This is the
+// classic named-return-plus-defer trap. Verified with go run during
+// authoring.
+func hardDeferNamedReturnTest() testkit.Test {
+	prompt := `Consider this Go code:
+
+func f() (r int) {
+	defer func() { r = 99 }()
+	r = 5
+	return 7
+}
+
+What value does f() return?
+Respond with only the number, no explanation.`
+
+	return testkit.Test{
+		ID:          "hard-defer-named-return",
+		Category:    "programming",
+		Subcategory: "hard",
+		Description: "Trace a deferred closure overwriting a named return value during a normal return (99).",
+		Prompt:      prompt,
+		Eval:        eval.Numeric(eval.ExtractLastNumber[int], hardDeferNamedReturnWant, 0),
+	}
+}
+
+// hardDeferNamedReturnWant is f()'s return value.
+//
+// ground truth: the deferred closure runs during the function's epilogue,
+// after the explicit `return 7` has set the named result r = 7, and
+// overwrites it with 99; the caller therefore receives 99 (see the full
+// derivation on hardDeferNamedReturnTest). Recomputed by running the
+// identical snippet with go run during authoring.
+const hardDeferNamedReturnWant = 99
+
+// hardIntDivisionTest: judge the result of integer division being
+// assigned to a floating-point variable.
+//
+// ground truth: a / b with two int operands performs integer division,
+// which truncates toward zero: 7 / 2 = 3 (not 3.5). That integral
+// result is then implicitly converted to double, so d = 3.0. The common
+// error is assuming floating division because the target is a double.
+// Verified with go run during authoring.
+func hardIntDivisionTest() testkit.Test {
+	prompt := `Consider this C code:
+
+int a = 7;
+int b = 2;
+double d = a / b;
+
+What is the value of d?
+Respond with only the number, no explanation.`
+
+	return testkit.Test{
+		ID:          "hard-int-division",
+		Category:    "programming",
+		Subcategory: "hard",
+		Description: "Trace integer-division-then-widen: 7 / 2 is 3 (stored as 3.0), not 3.5.",
+		Prompt:      prompt,
+		Eval:        eval.Numeric(eval.ExtractLastNumber[float64], hardIntDivisionWant, 0),
+	}
+}
+
+// hardIntDivisionWant is d's value.
+//
+// ground truth: integers divide as integers (truncating), giving 7 / 2 =
+// 3, converted to double 3.0; the float value is 3, so Numeric want is
+// 3.0. Recomputed by running the identical arithmetic during authoring.
+const hardIntDivisionWant = 3.0
+
+// hardRegexGreedyTest: trace which substrings two greedy capture groups
+// of an anchored pattern match.
+//
+// ground truth: quantifiers are greedy by default - each takes as much as
+// it can while still allowing the rest of the pattern to match. In
+// /^(\d+)(\d+)$/, the first \d+ grabs the longest prefix that leaves at
+// least one digit for the second \d+ (which needs at least one match
+// because + is one-or-more). That longest prefix of "12345" is "1234",
+// leaving "5" for group 2. A naive symmetric split like ["12","345"]
+// fails because group 1 would then still be able to grow. Verified with
+// node during authoring.
+func hardRegexGreedyTest() testkit.Test {
+	prompt := `Consider this regular expression applied to the string "12345":
+
+^(\d+)(\d+)$
+
+(^ and $ anchor to the whole string, and \d+ means one or more digits.)
+
+What are the two captured groups? Respond with only a JSON array of the
+two captured strings, in group order, for example ["a","b"]`
+
+	return testkit.Test{
+		ID:          "hard-regex-greedy",
+		Category:    "programming",
+		Subcategory: "hard",
+		Description: "Trace greedy capture-group backtracking: ^(one-or-more digits)(one-or-more digits)$ on 12345 yields 1234 and 5.",
+		Prompt:      prompt,
+		Eval:        eval.JSONStringArrayEquals([]string{"1234", "5"}),
+	}
+}
+
+// hardPromiseAllSettledTest: trace the per-result status values of
+// Promise.allSettled over a mixed resolve/reject input.
+//
+// ground truth: Promise.allSettled always resolves, and each element of
+// its result array is an object whose status is the literal "fulfilled"
+// for a resolved input and "rejected" for a rejected input, preserving
+// input order. Promise.resolve(42) yields {status:"fulfilled", value:42};
+// Promise.reject(new Error("boom")) yields {status:"rejected",
+// reason:...}. So the two status values, in order, are "fulfilled" then
+// "rejected". Verified with node during authoring.
+func hardPromiseAllSettledTest() testkit.Test {
+	prompt := `Consider this TypeScript code:
+
+const results = await Promise.allSettled([
+  Promise.resolve(42),
+  Promise.reject(new Error("boom")),
+]);
+
+Each element of results has a "status" field.
+
+List the two status values, in array order. Respond with only a JSON
+array of the two status strings, for example ["a","b"]`
+
+	return testkit.Test{
+		ID:          "hard-promise-allsettled",
+		Category:    "programming",
+		Subcategory: "hard",
+		Description: "Trace Promise.allSettled result statuses over a fulfilled+rejected input (fulfilled, rejected).",
+		Prompt:      prompt,
+		Eval:        eval.JSONStringArrayEquals([]string{"fulfilled", "rejected"}),
+	}
+}
+
+// hardContextTimeoutTest: identify the exact error string a timed-out
+// context's Err() returns.
+//
+// ground truth: when a context created by context.WithTimeout fires, its
+// Done channel closes and Err() returns the sentinel
+// context.DeadlineExceeded, whose Error() string (per the context
+// package) is exactly "context deadline exceeded" - which is distinct
+// from "context canceled", the string of context.Canceled returned when a
+// cancel function is called. Verified with go run during authoring.
+func hardContextTimeoutTest() testkit.Test {
+	prompt := `In Go, a context created with context.WithTimeout fires after its
+deadline. At that point:
+
+ctx.Err()
+
+returns a non-nil error. What is the exact string of that error?
+Respond with only a JSON object:
+{"err": "..."}
+
+with the exact string value.`
+
+	return testkit.Test{
+		ID:          "hard-context-timeout",
+		Category:    "programming",
+		Subcategory: "hard",
+		Description: "Identify ctx.Err() after a context.WithTimeout deadline fires (context deadline exceeded).",
+		Prompt:      prompt,
+		Eval:        eval.JSONField("err", "context deadline exceeded"),
+	}
+}
+
+// hardJSONUnmarshalNullTest: judge the behavior of json.Unmarshal on a
+// literal JSON null into an already-populated struct.
+//
+// ground truth: per encoding/json, a JSON null into a non-nil pointer
+// target is a no-op: Unmarshal returns nil (no error) and leaves the
+// target's existing value unchanged. It does not zero the struct and does
+// not error. Verified with go run during authoring (a struct holding
+// Items: [1] stays [1] with nil error).
+func hardJSONUnmarshalNullTest() testkit.Test {
+	prompt := `In Go:
+
+var s Struct  // s is a populated struct value (some fields set)
+err := json.Unmarshal([]byte("null"), &s)
+
+Does Unmarshal return an error, and is s left unchanged (no fields
+cleared)?
+Respond with only a JSON object:
+{"returns_error": true}
+or
+{"returns_error": false}`
+
+	return testkit.Test{
+		ID:          "hard-json-unmarshal-null",
+		Category:    "programming",
+		Subcategory: "hard",
+		Description: "Judge json.Unmarshal of a literal null into a populated struct (no error, unchanged).",
+		Prompt:      prompt,
+		Eval:        eval.JSONField("returns_error", false),
+	}
+}
+
+// hardVarShadowTest: judge whether an inner := declaration shadows an
+// outer variable without changing it.
+//
+// ground truth: `count := 5` inside the if block declares a NEW, inner
+// variable that shadows the outer count for the duration of that block
+// (Go's := always creates a new variable when at least one variable on
+// the left is new in the current scope). Incrementing the inner count
+// leaves the outer count untouched, so after the block outer count is
+// still 0. Verified with go run during authoring.
+func hardVarShadowTest() testkit.Test {
+	prompt := `Consider this Go code:
+
+count := 0
+if true {
+	count := 5
+	count++
+}
+// count here?
+
+Does the outer count remain unchanged (equal to 0) after the if block?
+Respond with only a JSON object:
+{"outer_unchanged": true}
+or
+{"outer_unchanged": false}`
+
+	return testkit.Test{
+		ID:          "hard-var-shadow",
+		Category:    "programming",
+		Subcategory: "hard",
+		Description: "Trace Go variable shadowing: an inner := leaves the outer variable unchanged (true).",
+		Prompt:      prompt,
+		Eval:        eval.JSONField("outer_unchanged", true),
+	}
+}
+
+// hardPyVersionVersions is the inline version list for
+// hardPyVersionSortTest.
+const hardPyVersionVersions = `versions = [
+    "1.9.0",
+    "1.10.0",
+    "1.2.0",
+    "2.0.0",
+    "0.9.9",
+]`
+
+// hardPyVersionSortWant is the expected stdout of a correct script: the
+// second-highest version under numeric component-by-component ordering.
+//
+// ground truth: ordering versions by numeric components puts 0.9.9 <
+// 1.2.0 < 1.9.0 < 1.10.0 < 2.0.0 (1.10.0 is newer than 1.9.0 because
+// its second component 10 > 9), so the second-highest is 1.10.0. A
+// plain lexicographic string sort (the trap) would instead report 1.9.0,
+// which is exactly the failure the test discriminates. Recomputed by
+// running the correct script with python3 during authoring.
+const hardPyVersionSortWant = "1.10.0"
+
+// hardPyVersionSortTest: write a python3 script that prints the
+// second-highest version under numeric component ordering.
+//
+// ground truth: see hardPyVersionSortWant; correctness is determined by
+// running the script (version ordering is behavior, not a text match).
+func hardPyVersionSortTest() testkit.Test {
+	prompt := `Here is an inline list of software version strings:
+
+` + "```python" + `
+` + hardPyVersionVersions + `
+` + "```" + `
+
+Write a python3 program that sorts the versions numerically, component by
+component (natural version ordering: "1.10.0" is newer than "1.9.0"
+because the second component 10 > 9), and prints only the second-highest
+version.`
+
+	return testkit.Test{
+		ID:          "hard-py-version-sort",
+		Category:    "programming",
+		Subcategory: "hard",
+		Description: "Write a python3 script printing the second-highest version by numeric component ordering.",
+		System:      terseCodeOnly,
+		Prompt:      prompt,
+		Eval:        eval.PyRun(eval.PassthroughHarness, hardPyVersionSortWant),
+	}
+}
+
+// hardGoParenBalanceHarness is a complete, independent driver file (its
+// own package clause and imports) that eval.GoRun writes to harness.go
+// alongside the model's IsBalanced in solution.go. It drives IsBalanced
+// across balanced, nested, mismatched, unclosed, and wrongly-ordered
+// cases, printing PASS only if every case matches its expected result.
+const hardGoParenBalanceHarness = `package main
+
+import (
+	"fmt"
+	"os"
+)
+
+func main() {
+	cases := []struct {
+		in   string
+		want bool
+	}{
+		{"", true},
+		{"()", true},
+		{"()[]{}", true},
+		{"([{}])", true},
+		{"(", false},
+		{"(]", false},
+		{"([)]", false},
+		{"((", false},
+		{")(", false},
+	}
+	for _, c := range cases {
+		if got := IsBalanced(c.in); got != c.want {
+			fmt.Printf("FAIL %q: got %v want %v\n", c.in, got, c.want)
+			os.Exit(1)
+		}
+	}
+	fmt.Println("PASS")
+}`
+
+// hardGoParenBalanceTest: implement a stack-based balanced-parentheses
+// checker.
+//
+// ground truth: the harness above is the oracle - it exercises empty,
+// balanced, nested, mismatched-closer, trailing-opener, and wrong-order
+// inputs, so a correct stack-based implementation prints PASS. There is no
+// separate hardcoded constant beyond the PASS sentinel.
+func hardGoParenBalanceTest() testkit.Test {
+	prompt := `Implement a Go function with exactly this signature:
+
+func IsBalanced(s string) bool
+
+It must report whether s's parentheses are balanced and correctly nested:
+'(' '[' '{' are openers, ')' ']' '}' are the matching closers. An empty
+string is balanced (return true). Return false for any unclosed opener,
+any closer with no matching open, or any wrong-nesting (e.g. "([)]").`
+
+	return testkit.Test{
+		ID:          "hard-go-paren-balance",
+		Category:    "programming",
+		Subcategory: "hard",
+		Description: "Implement a stack-based balanced-parentheses checker for (), [], and {}.",
+		System:      terseCodeOnly,
+		Prompt:      prompt,
+		Eval:        eval.GoRun(hardGoParenBalanceHarness, "PASS"),
+	}
+}

internal/tests/programming_hard_test.go

@@ -0,0 +1,360 @@
+package tests
+
+import (
+	"context"
+	"testing"
+)
+
+func TestHardFloatBinaddTest_Eval(t *testing.T) {
+	tc := hardFloatBinaddTest()
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{"correct", `{"equal":false}`, 1},
+		{"correct fenced with prose", "The answer is:\n```json\n{\"equal\":false}\n```", 1},
+		{"wrong: claims equal", `{"equal":true}`, 0},
+		{"wrong: non-boolean", `{"equal":"yes"}`, 0},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}
+
+func TestHardNullishOverOrTest_Eval(t *testing.T) {
+	tc := hardNullishOverOrTest()
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{"correct", `{"value":0}`, 1},
+		{"correct as string", `{"value":"0"}`, 1},
+		{"wrong: OR semantics", `{"value":42}`, 0},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}
+
+func TestHardGoroutineRaceTest_Eval(t *testing.T) {
+	tc := hardGoroutineRaceTest()
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{"correct", `{"guaranteed":false}`, 1},
+		{"correct fenced", "```json\n{\"guaranteed\":false}\n```", 1},
+		{"wrong: claims guaranteed", `{"guaranteed":true}`, 0},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}
+
+func TestHardDeferNamedReturnTest_Eval(t *testing.T) {
+	tc := hardDeferNamedReturnTest()
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{"correct", "99", 1},
+		{"correct prose-wrapped", "The function returns 99.", 1},
+		{"wrong: returns statement value", "7", 0},
+		{"wrong: pre-defer value", "5", 0},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}
+
+func TestHardIntDivisionTest_Eval(t *testing.T) {
+	tc := hardIntDivisionTest()
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{"correct", "3", 1},
+		{"correct as float", "3.0", 1},
+		{"correct prose-wrapped", "It is 3.", 1},
+		{"wrong: assumes float division", "3.5", 0},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}
+
+func TestHardRegexGreedyTest_Eval(t *testing.T) {
+	tc := hardRegexGreedyTest()
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{"correct", `["1234","5"]`, 1},
+		{"correct fenced", "```json\n[\"1234\",\"5\"]\n```", 1},
+		{"wrong: symmetric split", `["12","345"]`, 0},
+		{"wrong: reversed", `["5","1234"]`, 0},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}
+
+func TestHardPromiseAllSettledTest_Eval(t *testing.T) {
+	tc := hardPromiseAllSettledTest()
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{"correct", `["fulfilled","rejected"]`, 1},
+		{"correct fenced", "```json\n[\"fulfilled\",\"rejected\"]\n```", 1},
+		{"wrong: reordered", `["rejected","fulfilled"]`, 0},
+		{"wrong: both fulfilled", `["fulfilled","fulfilled"]`, 0},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}
+
+func TestHardContextTimeoutTest_Eval(t *testing.T) {
+	tc := hardContextTimeoutTest()
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{"correct", `{"err":"context deadline exceeded"}`, 1},
+		{"correct fenced", "```json\n{\"err\":\"context deadline exceeded\"}\n```", 1},
+		{"wrong: canceled sentinel", `{"err":"context canceled"}`, 0},
+		{"wrong: vague", `{"err":"some error"}`, 0},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}
+
+func TestHardJSONUnmarshalNullTest_Eval(t *testing.T) {
+	tc := hardJSONUnmarshalNullTest()
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{"correct", `{"returns_error":false}`, 1},
+		{"correct fenced", "```json\n{\"returns_error\":false}\n```", 1},
+		{"wrong: claims error", `{"returns_error":true}`, 0},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}
+
+func TestHardVarShadowTest_Eval(t *testing.T) {
+	tc := hardVarShadowTest()
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{"correct", `{"outer_unchanged":true}`, 1},
+		{"correct fenced", "```json\n{\"outer_unchanged\":true}\n```", 1},
+		{"wrong: claims outer changed", `{"outer_unchanged":false}`, 0},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}
+
+// TestHardPyVersionSortTest_Eval exercises the PyRun evaluator with a
+// hand-written correct script and a lexicographic (wrong) script.
+func TestHardPyVersionSortTest_Eval(t *testing.T) {
+	tc := hardPyVersionSortTest()
+
+	correctScript := "```python\n" + `versions = [
+    "1.9.0",
+    "1.10.0",
+    "1.2.0",
+    "2.0.0",
+    "0.9.9",
+]
+
+def key(v):
+    return tuple(int(p) for p in v.split("."))
+
+sorted_versions = sorted(versions, key=key)
+print(sorted_versions[-2])
+` + "```"
+
+	wrongLexicographic := "```python\n" + `versions = [
+    "1.9.0",
+    "1.10.0",
+    "1.2.0",
+    "2.0.0",
+    "0.9.9",
+]
+print(sorted(versions)[-2])
+` + "```"
+
+	wrongHardcoded := "```python\nprint('2.0.0')\n```"
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{"correct numeric sort", correctScript, 1},
+		{"wrong: lexicographic sort", wrongLexicographic, 0},
+		{"wrong: hardcoded highest", wrongHardcoded, 0},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Skipped {
+				t.Skip("python3 not available, skipping exec-evaluator assertion")
+			}
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}
+
+// TestHardGoParenBalanceTest_Eval exercises the GoRun evaluator with a
+// hand-written correct function and a wrong one.
+func TestHardGoParenBalanceTest_Eval(t *testing.T) {
+	tc := hardGoParenBalanceTest()
+
+	correctFunc := "```go\n" + `func IsBalanced(s string) bool {
+	pairs := map[byte]byte{')': '(', ']': '[', '}': '{'}
+	stack := make([]byte, 0, len(s))
+	for i := 0; i < len(s); i++ {
+		switch s[i] {
+		case '(', '[', '{':
+			stack = append(stack, s[i])
+		case ')', ']', '}':
+			if len(stack) == 0 || stack[len(stack)-1] != pairs[s[i]] {
+				return false
+			}
+			stack = stack[:len(stack)-1]
+		}
+	}
+	return len(stack) == 0
+}
+` + "```"
+
+	wrongFunc := "```go\n" + `func IsBalanced(s string) bool {
+	return true
+}
+` + "```"
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{"correct stack implementation", correctFunc, 1},
+		{"wrong: always true", wrongFunc, 0},
+		{"wrong: no code at all", "I would use a stack here.", 0},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Skipped {
+				t.Skip("go toolchain not available, skipping exec-evaluator assertion")
+			}
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}
+
+// TestHardDeferNamedReturnWant_GroundTruth recomputes f()'s return value
+// by running an equivalent function in-process, independent of the
+// hardDeferNamedReturnWant constant.
+func TestHardDeferNamedReturnWant_GroundTruth(t *testing.T) {
+	f := func() (r int) {
+		defer func() { r = 99 }()
+		r = 5
+		return 7
+	}
+	if got := f(); got != hardDeferNamedReturnWant {
+		t.Errorf("equivalent deferred-named-return function returned %d, want %d", got, hardDeferNamedReturnWant)
+	}
+}
+
+// TestHardIntDivisionWant_GroundTruth recomputes for a double result of
+// 7 / 2 by hand, independent of the hardIntDivisionWant constant.
+func TestHardIntDivisionWant_GroundTruth(t *testing.T) {
+	a, b := 7, 2
+	want := float64(a / b) // integer division truncates before widening
+	if want != hardIntDivisionWant {
+		t.Errorf("recomputed integer division = %v, want %v", want, hardIntDivisionWant)
+	}
+}

internal/tests/saturation.go

@@ -0,0 +1,534 @@
+// Package-level hardening tests for the five subcategories that measured
+// 100% perfect-score across all three baseline models (see the health
+// audit over the 2026-08-18 3-model run): agents/delegation,
+// databases/postgres, research/web, security/appsec, security/secrets.
+// Every test there was saturated - zero discrimination signal. These ten
+// tests are trap-shaped on purpose: each embeds a near-miss answer that
+// plausible-but-wrong reasoning produces, and every expected value carries
+// a derivation comment.
+package tests
+
+import (
+	"github.com/lukaszraczylo/llm-testbench/internal/eval"
+	"github.com/lukaszraczylo/llm-testbench/internal/testkit"
+)
+
+// registerSaturationTests adds the hardening tests to the registry. They
+// land in existing subcategories, so the per-category register functions
+// stay untouched; the subcategory's test count only grows.
+func registerSaturationTests(r *testkit.Registry) {
+	r.Register(satDelegCapacityTrapTest())
+	r.Register(satDelegNearMissRosterTest())
+	r.Register(satPGCountStarTrapTest())
+	r.Register(satPGTimeoutTrapTest())
+	r.Register(satWebRobotsUAScopeTest())
+	r.Register(satWebHreflangSelfTest())
+	r.Register(satAppsecORMTrapTest())
+	r.Register(satAppsecPOSTTrapTest())
+	r.Register(satSecretsRotationMathTest())
+	r.Register(satSecretsErrorPathTest())
+}
+
+// satDelegCapacityTrapTest: the roster's specialist for the job exists but
+// is unavailable. Tests whether the model force-fits the task to a
+// mismatched agent, over-coordinates with the orchestrator, or correctly
+// keeps it in the main thread.
+func satDelegCapacityTrapTest() testkit.Test {
+	prompt := `You are coordinating agents. Available specialist roster:
+
+` + delegRoster + `
+
+Situation: the CI pipeline for the main branch has been failing for the
+past hour on a Docker build step. It needs diagnosing now. The roster's
+infra/CI specialist is already working on an unrelated production incident
+and will not free up for several hours. There is exactly one task.
+
+Respond with only a JSON object, no other text, of the form
+{"best_next_step": "...", "specialist_this_task_belongs_to": "..."} where:
+- best_next_step is exactly one of: "dispatch_code_writer",
+  "dispatch_orchestrator", "handle_in_main_thread", "wait_for_specialist"
+- specialist_this_task_belongs_to is the roster agent name whose
+  description covers this task, regardless of their availability.`
+	return testkit.Test{
+		ID:          "agent-deleg-capacity-trap",
+		Category:    "agents",
+		Subcategory: "delegation",
+		Description: "Route a task whose specialist is busy: no force-fit to a mismatched agent, no orchestrator for a single task, and still name the correct owning specialist",
+		Prompt:      prompt,
+		// ground truth: the task is diagnosing a failing CI build, which
+		// the roster assigns to devops-debugger ("Diagnoses failing
+		// deployments, CI pipelines, and infrastructure incidents"). It is
+		// busy, so the remaining options are judged against the roster:
+		// code-writer "Implements new features or bug fixes by writing and
+		// editing source code directly" - a generalist forced onto infra
+		// diagnosis; orchestrator "Plans and sequences a multi-agent
+		// workflow" - there is one task, nothing to sequence; waiting on a
+		// specialist that is hours away while CI is down blocks delivery
+		// for no gain. A single task with its specialist unavailable is
+		// handled directly (handle_in_main_thread). The owning specialist
+		// on the roster is unambiguously devops-debugger.
+		Eval: eval.Mean(
+			eval.JSONField[string]("best_next_step", "handle_in_main_thread"),
+			eval.JSONField[string]("specialist_this_task_belongs_to", "devops-debugger"),
+		),
+	}
+}
+
+// satDelegNearMissRosterTest: 4-task mapping where every task has a
+// same-vocabulary distractor (docs vs web lookup, release notes vs prose
+// docs, diff review vs security review).
+func satDelegNearMissRosterTest() testkit.Test {
+	prompt := `You are coordinating agents. Specialist roster:
+
+` + delegRoster + `
+
+Map each task to exactly one roster agent - the one whose roster
+description covers it. Tasks:
+- task1: "Update the API guide in docs/ so it matches the endpoints as
+  they exist in the repository today."
+- task2: "Determine from the vendor's public pricing page which of their
+  plans includes SSO."
+- task3: "Produce the v2.3 release notes by generating the changelog
+  section and tagging the release."
+- task4: "Audit the auth handler for vulnerabilities before it merges."
+
+Respond with only a JSON object of the form
+{"task1": "...", "task2": "...", "task3": "...", "task4": "..."}
+where each value is exactly one roster agent name.`
+	return testkit.Test{
+		ID:          "agent-deleg-near-miss-roster",
+		Category:    "agents",
+		Subcategory: "delegation",
+		Description: "Route 4 tasks whose phrasing overlaps two roster descriptions each (repo docs vs web lookup, changelog vs prose docs, diff review vs security audit)",
+		Prompt:      prompt,
+		// ground truth, per the delegRoster descriptions:
+		// task1 -> docs-writer: it writes/updates prose documentation
+		// ("READMEs, guides, design docs") from repository content.
+		// web-researcher is wrong: "Does not read or write anything in the
+		// repository" and no external search is needed.
+		// task2 -> web-researcher: the answer lives on the vendor's public
+		// page, "Searches the public web and summarizes external
+		// information"; docs-writer is the distractor ("page" sounds like
+		// documentation) but the task reads the web, not the repo.
+		// task3 -> release-manager: "Manages version tagging, changelog
+		// generation, and release publishing" covers exactly the phrasing
+		// (generate changelog + tag). docs-writer is the distractor:
+		// release notes here are the release process, not a guide.
+		// task4 -> security-reviewer: "Audits code or config for security
+		// vulnerabilities"; code-reviewer is the distractor ("before it
+		// merges" sounds like PR review) but code-reviewer reviews
+		// "correctness, style, and risk", while the stated goal is
+		// vulnerabilities.
+		Eval: eval.Mean(
+			eval.JSONField[string]("task1", "docs-writer"),
+			eval.JSONField[string]("task2", "web-researcher"),
+			eval.JSONField[string]("task3", "release-manager"),
+			eval.JSONField[string]("task4", "security-reviewer"),
+		),
+	}
+}
+
+// satPGCountStarFixture is the orders/line-items data for the join
+// fan-out trap.
+const satPGCountStarFixture = `CREATE TABLE orders (id bigint PRIMARY KEY, region text NOT NULL);
+CREATE TABLE order_items (id bigint PRIMARY KEY,
+                          order_id bigint NOT NULL REFERENCES orders(id));
+
+INSERT INTO orders (id, region) VALUES
+  (1001, 'eu'), (1002, 'eu'), (1003, 'us'),
+  (1004, 'eu'), (1005, 'ap'), (1006, 'eu');
+
+INSERT INTO order_items (id, order_id) VALUES
+  (9001, 1001), (9002, 1001), (9003, 1001),
+  (9004, 1002),
+  (9005, 1003), (9006, 1003), (9007, 1003), (9008, 1003), (9009, 1003),
+  (9010, 1004), (9011, 1004),
+  (9012, 1005);`
+
+// satPGCountStarTrapTest: COUNT(*) over an INNER JOIN counts line items,
+// not orders, and the join silently drops the order with zero items. Every
+// number is derivable from the fixture, so the evaluator never needs to
+// re-run SQL.
+func satPGCountStarTrapTest() testkit.Test {
+	prompt := `Here is a PostgreSQL schema with data:
+
+` + "```sql\n" + satPGCountStarFixture + "\n```" + `
+
+The business question is: how many ORDERS are in region 'eu'?
+A developer writes:
+
+` + "```sql\nSELECT count(*) FROM orders o JOIN order_items oi ON oi.order_id = o.id WHERE o.region = 'eu';\n```" + `
+
+Respond with only a JSON object, no other text:
+{"orders_in_eu": <number>, "rows_the_join_query_returns": <number>,
+ "count_star_answers_the_question": true or false}`
+	return testkit.Test{
+		ID:          "pg-count-star-vs-join",
+		Category:    "databases",
+		Subcategory: "postgres",
+		Description: "Reason about COUNT(*) over an INNER JOIN against a fixed fixture: join fan-out inflates the count and the join drops the childless order",
+		Prompt:      prompt,
+		// ground truth: orders with region 'eu' are 1001, 1002, 1004, 1006
+		// (INSERTs above) = 4. The join emits one row per order_item:
+		// 1001 -> 3 items (9001-9003), 1002 -> 1 (9004), 1004 -> 2 (9010,
+		// 9011), and 1006 -> 0 items, so the INNER JOIN drops it entirely.
+		// count(*) therefore returns 3+1+2 = 6. 6 != 4, so the count(*)
+		// query does not answer the question (EXISTS or COUNT(DISTINCT
+		// o.id) would return 4).
+		Eval: eval.Mean(
+			eval.JSONField[int]("orders_in_eu", 4),
+			eval.JSONField[int]("rows_the_join_query_returns", 6),
+			eval.JSONField[bool]("count_star_answers_the_question", false),
+		),
+	}
+}
+
+// satPGTimeoutTrapTest: two sessions, two timeouts, one rule each. The
+// near-miss is applying idle_in_transaction_session_timeout to a busy
+// session (or statement_timeout to an idle one), and starting the idle
+// clock at BEGIN instead of at the last statement's completion.
+func satPGTimeoutTrapTest() testkit.Test {
+	prompt := `A PostgreSQL server is configured with:
+
+` + "```\nstatement_timeout = '30s'\nidle_in_transaction_session_timeout = '5s'\n```" + `
+
+Two sessions on this server:
+- Session A: BEGIN at 11:00:00, runs statements until 11:00:40 (the last
+  one finishes then), and has been waiting for client input ever since.
+  The transaction is still open.
+- Session B: BEGIN at 11:00:53, starts one long-running query at 11:00:55
+  which is still executing at 11:01:00.
+
+Assume no client input arrives and no query finishes after 11:01:00.
+Respond with only a JSON object, no other text:
+{"session_a_killed_at": "HH:MM:SS", "session_a_rule": "...",
+ "session_b_killed_at": "HH:MM:SS", "session_b_rule": "..."}
+where each time is in UTC and each rule is exactly one of:
+"statement_timeout", "idle_in_transaction_timeout", "not_killed".`
+	return testkit.Test{
+		ID:          "pg-timeout-trap",
+		Category:    "databases",
+		Subcategory: "postgres",
+		Description: "Apply statement_timeout and idle_in_transaction_session_timeout to the right sessions: idle clock starts at the last statement's completion, not BEGIN, and never applies to a session executing a statement",
+		Prompt:      prompt,
+		// ground truth: PostgreSQL's idle_in_transaction_session_timeout
+		// terminates a session "idle ... within an open transaction" -
+		// the clock starts when the session goes idle in transaction,
+		// which for A is 11:00:40 (its last statement finished then), so
+		// A is killed at 11:00:45 - not 11:00:05, since A was executing
+		// statements from 11:00:00 to 11:00:40. A is not subject to
+		// statement_timeout: no statement is running. B is idle in
+		// transaction only from 11:00:53 to 11:00:55 (2s, under the 5s
+		// cap) and then EXECUTES a statement, so the idle timeout no
+		// longer applies; statement_timeout bounds a single running
+		// statement: 11:00:55 + 30s = 11:01:25.
+		Eval: eval.Mean(
+			eval.JSONField[string]("session_a_killed_at", "11:00:45"),
+			eval.JSONField[string]("session_a_rule", "idle_in_transaction_timeout"),
+			eval.JSONField[string]("session_b_killed_at", "11:01:25"),
+			eval.JSONField[string]("session_b_rule", "statement_timeout"),
+		),
+	}
+}
+
+// satWebRobotsFixture carries two rule groups and a longest-match pair.
+const satWebRobotsFixture = `User-agent: *
+Disallow: /cdn-cgi/
+Allow: /cdn-cgi/style.css
+Sitemap: https://example.com/sitemap.xml
+
+User-agent: GoogleOther
+Disallow: /`
+
+// satWebRobotsUAScopeTest: GoogleOther's rules do not apply to Googlebot,
+// and within Googlebot's group the longest path match decides. The
+// near-miss is answering from the GoogleOther block alone, and answering
+// the Allow/Disallow pair by document order instead of path length.
+func satWebRobotsUAScopeTest() testkit.Test {
+	prompt := `A site serves this robots.txt at https://example.com/robots.txt:
+
+` + "```\n" + satWebRobotsFixture + "\n```" + `
+
+Consider a GET request for https://example.com/cdn-cgi/style.css.
+Respond with only a JSON object, no other text:
+{"googlebot_allowed": true or false, "googleother_allowed": true or false,
+ "deciding_rule_for_googlebot": "allow" or "disallow" or "no_rule"}`
+	return testkit.Test{
+		ID:          "web-robots-ua-scope-trap",
+		Category:    "research",
+		Subcategory: "web",
+		Description: "robots.txt group matching across two User-agent blocks plus longest-path-match precedence - a separate group's Disallow does not apply to the other crawler",
+		Prompt:      prompt,
+		// ground truth: per RFC 9309, a crawler only applies the rule
+		// group(s) whose User-agent matches it, and within a group the
+		// path pattern with the LONGEST match wins. Googlebot does not
+		// match the GoogleOther group, so only the '*' group applies:
+		// the URL matches Disallow /cdn-cgi/ (9 chars) and Allow
+		// /cdn-cgi/style.css (18 chars); 18 > 9, so Allow wins ->
+		// googlebot_allowed true, deciding_rule_for_googlebot "allow".
+		// GoogleOther matches its own group, where "Disallow: /" covers
+		// every path -> googleother_allowed false.
+		Eval: eval.Mean(
+			eval.JSONField[bool]("googlebot_allowed", true),
+			eval.JSONField[bool]("googleother_allowed", false),
+			eval.JSONField[string]("deciding_rule_for_googlebot", "allow"),
+		),
+	}
+}
+
+// satWebHreflangSelfTest: an hreflang set missing its self-reference and
+// x-default's sentinel status. The near-miss is counting the canonical
+// link as the hreflang self-reference, and treating x-default as a
+// language.
+func satWebHreflangSelfTest() testkit.Test {
+	prompt := `The page https://example.com/en-us/ serves this head section:
+
+` + "```html\n" + `<html lang="en-US">
+<head>
+  <link rel="alternate" hreflang="en-AU" href="https://example.com/en-au/">
+  <link rel="alternate" hreflang="en-GB" href="https://example.com/en-gb/">
+  <link rel="alternate" hreflang="x-default" href="https://example.com/">
+  <link rel="canonical" href="https://example.com/en-us/">
+</head>` + "\n```" + `
+
+Google's hreflang rule: an hreflang group must include a self-reference -
+an hreflang annotation whose href is the page itself - and every
+annotation it lists must point back reciprocally.
+Respond with only a JSON object, no other text:
+{"missing_self_reference_for": "the BCP 47 language tag of the page that
+has no hreflang annotation pointing at it", "self_reference_present":
+true or false, "canonical_counts_as_hreflang_self_reference": true or
+false, "x_default_is_a_language_tag": true or false}`
+	return testkit.Test{
+		ID:          "web-hreflang-self-reference-trap",
+		Category:    "research",
+		Subcategory: "web",
+		Description: "hreflang self-reference validation: the page's own language has no annotation, rel=canonical is a different mechanism, and x-default is a sentinel, not a BCP 47 tag",
+		Prompt:      prompt,
+		// ground truth: the page is en-US (lang attribute and canonical
+		// URL agree). The hreflang annotations cover en-AU, en-GB,
+		// x-default only - no annotation has href
+		// https://example.com/en-us/, so the missing self-reference tag
+		// is "en-US" and self_reference_present is false.
+		// canonical_counts_as_hreflang_self_reference is false: rel
+		// -canonical deduplicates URLs for indexing; hreflang self-
+		// reference is a separate signal on rel=alternate link elements
+		// - the canonical matching the page URL does not satisfy it.
+		// x_default_is_a_language_tag is false: x-default is Google's
+		// sentinel for "none of the specific languages", not a BCP 47
+		// tag (it carries no language subtag; the other two do).
+		Eval: eval.Mean(
+			eval.JSONField[string]("missing_self_reference_for", "en-US"),
+			eval.JSONField[bool]("self_reference_present", false),
+			eval.JSONField[bool]("canonical_counts_as_hreflang_self_reference", false),
+			eval.JSONField[bool]("x_default_is_a_language_tag", false),
+		),
+	}
+}
+
+// satAppsecORMFixture shows three SQLAlchemy call sites; only one builds
+// SQL by concatenation before wrapping it in text().
+const satAppsecORMFixture = `from sqlalchemy import text
+
+def get_user(conn, name, team):
+    sql = text("SELECT * FROM users WHERE name = :name AND team = :team")
+    return conn.execute(sql, {"name": name, "team": team}).fetchone()
+
+def search_users(conn, pattern):
+    # callers pass the raw 'q' query-parameter straight in
+    sql = text("SELECT * FROM users WHERE bio LIKE '%" + pattern + "%'")
+    return conn.execute(sql).fetchall()
+
+def list_orders(conn, status):
+    return conn.execute(
+        text("SELECT * FROM orders WHERE status = :s"), {"s": status}
+    ).fetchall()`
+
+// satAppsecORMTrapTest: SQLAlchemy text() is a wrapper, not a sanitizer -
+// the concatenation already fused attacker input into the SQL string
+// before text() ever sees it. The near-miss is "ORM == safe".
+func satAppsecORMTrapTest() testkit.Test {
+	prompt := `Flask application (the 'q' request parameter is attacker-controlled):
+
+` + "```python\n" + satAppsecORMFixture + "\n```" + `
+
+Respond with only a JSON object, no other text:
+{"injectable_function": "one of get_user | search_users | list_orders",
+ "text_wrapper_neutralizes_concatenation": true or false}`
+	return testkit.Test{
+		ID:          "sec-orm-interpolation-trap",
+		Category:    "security",
+		Subcategory: "appsec",
+		Description: "Spot SQL injection inside a SQLAlchemy text() call built by string concatenation - the ORM wrapper neutralizes nothing once the string is already fused",
+		Prompt:      prompt,
+		// ground truth: get_user and list_orders pass user values as
+		// bound parameters (:name/:team, :s) - the driver sends values
+		// out-of-band, so no injection. search_users builds the SQL
+		// string with + pattern + BEFORE text() wraps it: by the time
+		// text() sees it, the attacker's input is part of the statement
+		// (pattern = "x%' OR 1=1 -- " breaks out of the LIKE literal).
+		// text() marks a string as raw SQL for the ORM; it never
+		// parameterizes or escapes - text_wrapper_neutralizes_
+		// concatenation is false.
+		Eval: eval.Mean(
+			eval.JSONField[string]("injectable_function", "search_users"),
+			eval.JSONField[bool]("text_wrapper_neutralizes_concatenation", false),
+		),
+	}
+}
+
+// satAppsecPOSTFixture shows a logout control where the visible anchor is
+// a decoy - the actual request is the auto-submitted POST form.
+const satAppsecPOSTFixture = `<a href="#" onclick="document.getElementById('f').submit(); return false;">
+  Log out
+</a>
+<form id="f" action="/api/account/logout" method="post">
+  <input type="hidden" name="confirm" value="true">
+</form>`
+
+// satAppsecPOSTTrapTest: the app switched a token-less state-changing
+// endpoint from GET to POST as its whole CSRF mitigation. Tests whether
+// the model knows POST alone is not a defense (cross-site <form
+// method=post> auto-submits with cookies), even when the page only ever
+// submits it via JavaScript.
+func satAppsecPOSTTrapTest() testkit.Test {
+	prompt := `A session-authenticated web app renders this logout control. The session
+cookie is SameSite=None; Secure (sent with every cross-site request), and
+there is no CSRF token anywhere in the request.
+
+` + "```\n" + satAppsecPOSTFixture + "\n```" + `
+
+The developer's note: "I moved logout from GET to POST, so CSRF no longer
+applies - only JavaScript on THIS page submits the form."
+
+Assume an attacker page on evil.example hosts its own form with
+action="https://app.example/api/account/logout", method="post". The victim
+visits evil.example but clicks nothing and types nothing there.
+
+Respond with only a JSON object, no other text:
+{"attacker_form_reaches_logout": true or false,
+ "post_method_alone_prevents_csrf": true or false,
+ "attacker_needs_javascript": true or false}`
+	return testkit.Test{
+		ID:          "sec-csrf-post-trap",
+		Category:    "security",
+		Subcategory: "appsec",
+		Description: "CSRF after a GET-to-POST switch without a token: cross-site form auto-submit carries the cookie, so the method is not the defense; the attacker needs one line of JS, not knowledge of the page's DOM",
+		Prompt:      prompt,
+		// ground truth: SameSite=None means the browser attaches the
+		// session cookie to the attacker's cross-site POST, and there is
+		// no token, so the request is accepted as the victim ->
+		// attacker_form_reaches_logout true; the server authenticates by
+		// cookie, not by which page submitted the form.
+		// post_method_alone_prevents_csrf false: method only restricts
+		// bookmark/simple-link attacks; a hosted <form method=post>
+		// replays the state change. (With SameSite=Lax a cross-site POST
+		// would instead drop the cookie - the prompt pins SameSite=None
+		// so the answer is forced.)
+		// attacker_needs_javascript true: the prompt pins "clicks nothing",
+		// and without JavaScript a hosted form only submits when the victim
+		// presses its own submit button - a cross-site <img>/<a> cannot POST
+		// with the cookie context this request needs, and <meta
+		// http-equiv="refresh"> cannot carry a POST body. The silent attack
+		// requires form.submit().
+		Eval: eval.Mean(
+			eval.JSONField[bool]("attacker_form_reaches_logout", true),
+			eval.JSONField[bool]("post_method_alone_prevents_csrf", false),
+			eval.JSONField[bool]("attacker_needs_javascript", true),
+		),
+	}
+}
+
+// satRotationFixture gives the three durations for the rotation
+// deadline math.
+const satRotationFixture = `Internal CA certificate expires:   2026-09-30 18:00 UTC
+Worst realistic time to mint a replacement cert once rotation starts: 10 h
+After the new cert is deployed, the reverse proxy's graceful-reload
+window (old workers serve the old cert for up to this long after the
+reload is issued):                6 h`
+
+// satSecretsRotationMathTest: latest safe start time = expiry minus the
+// sum of both sequential windows. The near-miss is subtracting only the
+// minting time and forgetting the propagation tail.
+func satSecretsRotationMathTest() testkit.Test {
+	prompt := `Certificate rotation facts for an internal service:
+
+` + "```\n" + satRotationFixture + "\n```" + `
+
+Respond with only a JSON object, no other text:
+{"total_lead_time_hours": <number>, "latest_start": "YYYY-MM-DD HH:MM",
+ "safe_if_started_2026_09_30_08_00": true or false}
+(Use UTC throughout. latest_start is the latest time rotation can START
+and still have the old certificate fully replaced everywhere before it
+expires.)`
+	return testkit.Test{
+		ID:          "sec-rotation-window-math",
+		Category:    "security",
+		Subcategory: "secrets",
+		Description: "Certificate rotation deadline math: lead time is minting plus propagation in series, and starting late leaves the old cert serving past expiry",
+		Prompt:      prompt,
+		// ground truth: the two windows are sequential - a replacement
+		// can only start propagating once it exists: 10 h + 6 h = 16 h
+		// worst-case lead time. 2026-09-30 18:00 minus 16 h =
+		// 2026-09-30 02:00 latest start. Starting at 08:00 finishes at
+		// the worst at 08:00 + 16 h = 2026-10-01 00:00, six hours after
+		// the old cert expired -> not safe.
+		Eval: eval.Mean(
+			eval.JSONField[int]("total_lead_time_hours", 16),
+			eval.JSONField[string]("latest_start", "2026-09-30 02:00"),
+			eval.JSONField[bool]("safe_if_started_2026_09_30_08_00", false),
+		),
+	}
+}
+
+// satErrorPathFixture traces a password hash from a raise through an
+// error handler into a log line, while the client only sees a generic
+// response body.
+const satErrorPathFixture = `1| def login(user, password):
+2|     row = db.get_user(user)
+3|     if not verify(password, row.hash):
+4|         raise AuthError(f"invalid credentials for user={user} "
+5|                         f"hash={row.hash}")
+6|     return token_for(user)
+7|
+8| @app.errorhandler(AuthError)
+9| def on_auth_error(exc):
+10|     app.logger.error(str(exc))
+11|     return "login failed", 401`
+
+// satSecretsErrorPathTest: the secret (password hash) is interpolated at
+// the raise, survives the handler's str(exc), and lands in the log even
+// though the HTTP response is sanitized. The near-miss is "the client
+// never sees it, so it's fine", and the reverse trap of flagging the
+// attacker-supplied username as the secret.
+func satSecretsErrorPathTest() testkit.Test {
+	prompt := `Flask login flow (numbers are source lines):
+
+` + "```\n" + satErrorPathFixture + "\n```" + `
+
+row.hash is the stored password hash (sensitive). user comes straight
+from the login request body.
+Respond with only a JSON object, no other text:
+{"hash_reaches_server_logs": true or false, "hash_visible_in_http_response":
+true or false, "sensitive_value_in_message": "one of user | hash | none"}`
+	return testkit.Test{
+		ID:          "sec-error-path-leak-trap",
+		Category:    "security",
+		Subcategory: "secrets",
+		Description: "Trace a password hash through raise -> errorhandler -> logger.error: the client response is sanitized but the log line is not, and the echoed user field is attacker-known",
+		Prompt:      prompt,
+		// ground truth: the raise on lines 4-5 interpolates row.hash into
+		// the exception message; the handler on line 10 logs str(exc) -
+		// the full message - so the hash reaches the server logs: true.
+		// The handler returns only the literal "login failed" (line 11),
+		// never str(exc), so the hash is not visible in the HTTP
+		// response: false. Between user (attacker-supplied in the request
+		// body, already known to its owner) and hash (sensitive), the
+		// sensitive interpolated value is the hash.
+		Eval: eval.Mean(
+			eval.JSONField[bool]("hash_reaches_server_logs", true),
+			eval.JSONField[bool]("hash_visible_in_http_response", false),
+			eval.JSONField[string]("sensitive_value_in_message", "hash"),
+		),
+	}
+}

internal/tests/saturation_test.go

@@ -0,0 +1,400 @@
+package tests
+
+import (
+	"context"
+	"testing"
+)
+
+// Each saturation test gets a table pinning: the full-credit answer, the
+// trap answer (the near-miss the test exists to catch), and a garbage
+// answer. Expected partial scores follow from eval.Mean's per-field mean.
+
+func TestSatDelegCapacityTrapTest_Eval(t *testing.T) {
+	tc := satDelegCapacityTrapTest()
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{
+			name:     "correct: main thread, names busy specialist",
+			response: `{"best_next_step":"handle_in_main_thread","specialist_this_task_belongs_to":"devops-debugger"}`,
+			want:     1,
+		},
+		{
+			name:     "trap: force-fits the generalist builder",
+			response: `{"best_next_step":"dispatch_code_writer","specialist_this_task_belongs_to":"devops-debugger"}`,
+			want:     0.5,
+		},
+		{
+			name:     "trap: orchestrator for a single task",
+			response: `{"best_next_step":"dispatch_orchestrator","specialist_this_task_belongs_to":"orchestrator"}`,
+			want:     0,
+		},
+		{
+			name:     "correct step but wrong owning specialist",
+			response: `{"best_next_step":"handle_in_main_thread","specialist_this_task_belongs_to":"code-writer"}`,
+			want:     0.5,
+		},
+		{
+			name:     "wait-it-out answer",
+			response: `{"best_next_step":"wait_for_specialist","specialist_this_task_belongs_to":"devops-debugger"}`,
+			want:     0.5,
+		},
+		{name: "garbage", response: `not json`, want: 0},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}
+
+func TestSatDelegNearMissRosterTest_Eval(t *testing.T) {
+	tc := satDelegNearMissRosterTest()
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{
+			name:     "all correct",
+			response: `{"task1":"docs-writer","task2":"web-researcher","task3":"release-manager","task4":"security-reviewer"}`,
+			want:     1,
+		},
+		{
+			name:     "trap: web-researcher for the repo guide, docs-writer for the pricing page",
+			response: `{"task1":"web-researcher","task2":"docs-writer","task3":"release-manager","task4":"security-reviewer"}`,
+			want:     0.5,
+		},
+		{
+			name:     "trap: docs-writer for release notes, code-reviewer for the audit",
+			response: `{"task1":"docs-writer","task2":"web-researcher","task3":"docs-writer","task4":"code-reviewer"}`,
+			want:     0.5,
+		},
+		{
+			name:     "all swapped to the distractors",
+			response: `{"task1":"web-researcher","task2":"docs-writer","task3":"docs-writer","task4":"code-reviewer"}`,
+			want:     0,
+		},
+		{
+			name:     "case-insensitive correct",
+			response: `{"task1":"Docs-Writer","task2":"Web-Researcher","task3":"Release-Manager","task4":"Security-Reviewer"}`,
+			want:     1,
+		},
+		{name: "garbage", response: `["docs-writer"]`, want: 0},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}
+
+func TestSatPGCountStarTrapTest_Eval(t *testing.T) {
+	tc := satPGCountStarTrapTest()
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{
+			name:     "correct",
+			response: `{"orders_in_eu": 4, "rows_the_join_query_returns": 6, "count_star_answers_the_question": false}`,
+			want:     1,
+		},
+		{
+			name:     "numeric strings accepted",
+			response: `{"orders_in_eu": "4", "rows_the_join_query_returns": "6", "count_star_answers_the_question": "false"}`,
+			want:     1,
+		},
+		{
+			name:     "trap: counts joined rows as orders",
+			response: `{"orders_in_eu": 6, "rows_the_join_query_returns": 6, "count_star_answers_the_question": true}`,
+			want:     0.3333333333333333,
+		},
+		{
+			name:     "trap: forgets the childless order drops out of the inner join",
+			response: `{"orders_in_eu": 4, "rows_the_join_query_returns": 7, "count_star_answers_the_question": false}`,
+			want:     0.6666666666666666,
+		},
+		{name: "garbage", response: `4`, want: 0},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}
+
+func TestSatPGTimeoutTrapTest_Eval(t *testing.T) {
+	tc := satPGTimeoutTrapTest()
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{
+			name:     "correct",
+			response: `{"session_a_killed_at":"11:00:45","session_a_rule":"idle_in_transaction_timeout","session_b_killed_at":"11:01:25","session_b_rule":"statement_timeout"}`,
+			want:     1,
+		},
+		{
+			name:     "trap: idle clock started at BEGIN",
+			response: `{"session_a_killed_at":"11:00:05","session_a_rule":"idle_in_transaction_timeout","session_b_killed_at":"11:01:25","session_b_rule":"statement_timeout"}`,
+			want:     0.75,
+		},
+		{
+			name:     "trap: timeouts swapped between sessions",
+			response: `{"session_a_killed_at":"11:00:30","session_a_rule":"statement_timeout","session_b_killed_at":"11:00:58","session_b_rule":"idle_in_transaction_timeout"}`,
+			want:     0,
+		},
+		{
+			name:     "two fields right",
+			response: `{"session_a_killed_at":"11:00:45","session_a_rule":"idle_in_transaction_timeout","session_b_killed_at":"11:01:00","session_b_rule":"not_killed"}`,
+			want:     0.5,
+		},
+		{name: "garbage", response: `no json here`, want: 0},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}
+
+func TestSatWebRobotsUAScopeTest_Eval(t *testing.T) {
+	tc := satWebRobotsUAScopeTest()
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{
+			name:     "correct: longest match wins for googlebot, group isolation for GoogleOther",
+			response: `{"googlebot_allowed": true, "googleother_allowed": false, "deciding_rule_for_googlebot": "allow"}`,
+			want:     1,
+		},
+		{
+			name:     "trap: GoogleOther's disallow applied to googlebot too",
+			response: `{"googlebot_allowed": false, "googleother_allowed": false, "deciding_rule_for_googlebot": "disallow"}`,
+			want:     0.3333333333333333,
+		},
+		{
+			name:     "trap: wildcard group treated as absent",
+			response: `{"googlebot_allowed": true, "googleother_allowed": true, "deciding_rule_for_googlebot": "allow"}`,
+			want:     0.6666666666666666,
+		},
+		{name: "garbage", response: `allow`, want: 0},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}
+
+func TestSatWebHreflangSelfTest_Eval(t *testing.T) {
+	tc := satWebHreflangSelfTest()
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{
+			name:     "correct",
+			response: `{"missing_self_reference_for":"en-US","self_reference_present":false,"canonical_counts_as_hreflang_self_reference":false,"x_default_is_a_language_tag":false}`,
+			want:     1,
+		},
+		{
+			name:     "tag casing normalized",
+			response: `{"missing_self_reference_for":"en-us","self_reference_present":false,"canonical_counts_as_hreflang_self_reference":false,"x_default_is_a_language_tag":false}`,
+			want:     1,
+		},
+		{
+			name:     "trap: canonical counted as the self-reference",
+			response: `{"missing_self_reference_for":"none","self_reference_present":true,"canonical_counts_as_hreflang_self_reference":true,"x_default_is_a_language_tag":false}`,
+			want:     0.25,
+		},
+		{
+			name:     "trap: x-default treated as a language",
+			response: `{"missing_self_reference_for":"en-US","self_reference_present":false,"canonical_counts_as_hreflang_self_reference":false,"x_default_is_a_language_tag":true}`,
+			want:     0.75,
+		},
+		{name: "garbage", response: `en-US`, want: 0},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}
+
+func TestSatAppsecORMTrapTest_Eval(t *testing.T) {
+	tc := satAppsecORMTrapTest()
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{
+			name:     "correct",
+			response: `{"injectable_function": "search_users", "text_wrapper_neutralizes_concatenation": false}`,
+			want:     1,
+		},
+		{
+			name:     "trap: ORM assumed safe everywhere",
+			response: `{"injectable_function": "none", "text_wrapper_neutralizes_concatenation": true}`,
+			want:     0,
+		},
+		{
+			name:     "right function, wrong mechanism",
+			response: `{"injectable_function": "search_users", "text_wrapper_neutralizes_concatenation": true}`,
+			want:     0.5,
+		},
+		{name: "garbage", response: `search_users`, want: 0},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}
+
+func TestSatAppsecPOSTTrapTest_Eval(t *testing.T) {
+	tc := satAppsecPOSTTrapTest()
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{
+			name:     "correct",
+			response: `{"attacker_form_reaches_logout": true, "post_method_alone_prevents_csrf": false, "attacker_needs_javascript": true}`,
+			want:     1,
+		},
+		{
+			name:     "trap: POST assumed to prevent CSRF",
+			response: `{"attacker_form_reaches_logout": false, "post_method_alone_prevents_csrf": true, "attacker_needs_javascript": true}`,
+			want:     0.3333333333333333,
+		},
+		{
+			name:     "trap: attacker blocked for lacking the page's JS",
+			response: `{"attacker_form_reaches_logout": false, "post_method_alone_prevents_csrf": false, "attacker_needs_javascript": false}`,
+			want:     0.3333333333333333,
+		},
+		{name: "garbage", response: `{"a":1}`, want: 0},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}
+
+func TestSatSecretsRotationMathTest_Eval(t *testing.T) {
+	tc := satSecretsRotationMathTest()
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{
+			name:     "correct",
+			response: `{"total_lead_time_hours": 16, "latest_start": "2026-09-30 02:00", "safe_if_started_2026_09_30_08_00": false}`,
+			want:     1,
+		},
+		{
+			name:     "trap: minting time only, propagation forgotten",
+			response: `{"total_lead_time_hours": 10, "latest_start": "2026-09-30 08:00", "safe_if_started_2026_09_30_08_00": true}`,
+			want:     0,
+		},
+		{
+			name:     "sum right, start-time arithmetic wrong",
+			response: `{"total_lead_time_hours": 16, "latest_start": "2026-09-30 02:00", "safe_if_started_2026_09_30_08_00": true}`,
+			want:     0.6666666666666666,
+		},
+		{name: "garbage", response: `16 hours`, want: 0},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}
+
+func TestSatSecretsErrorPathTest_Eval(t *testing.T) {
+	tc := satSecretsErrorPathTest()
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{
+			name:     "correct",
+			response: `{"hash_reaches_server_logs": true, "hash_visible_in_http_response": false, "sensitive_value_in_message": "hash"}`,
+			want:     1,
+		},
+		{
+			name:     "trap: sanitized response assumed to mean no leak",
+			response: `{"hash_reaches_server_logs": false, "hash_visible_in_http_response": false, "sensitive_value_in_message": "hash"}`,
+			want:     0.6666666666666666,
+		},
+		{
+			name:     "trap: attacker-known user flagged instead of the hash",
+			response: `{"hash_reaches_server_logs": true, "hash_visible_in_http_response": false, "sensitive_value_in_message": "user"}`,
+			want:     0.6666666666666666,
+		},
+		{
+			name:     "trap: leak judged through the log AND the response",
+			response: `{"hash_reaches_server_logs": true, "hash_visible_in_http_response": true, "sensitive_value_in_message": "hash"}`,
+			want:     0.6666666666666666,
+		},
+		{name: "garbage", response: `hash`, want: 0},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}

internal/tests/security_crypto.go

@@ -23,29 +23,49 @@ func registerSecurityCryptoTests(r *testkit.Registry) {
 	r.Register(secHashVsEncryptPIITest())
 }
 
-// secWeakPasswordHashPattern matches a mention of a fast, general-purpose
-// hash (MD5, plain SHA-1/256/512) - the ones that must never be used
-// directly for password storage because they lack a tunable work factor.
-var secWeakPasswordHashPattern = regexp.MustCompile(`(?i)\b(md5|sha-?1|sha-?256|sha-?512)\b`)
+// secWeakHashEndorsementPattern matches a weak-hash mention in an
+// endorsement-shaped position — a verb recommending it ("hash with MD5",
+// "just hash the password with SHA-256"), or a positive predicate about it
+// ("SHA-256 is a fine choice") — rather than every mention. The guard then
+// still requires a negation cue near such a mention (see
+// secNoUnnegatedMention).
+//
+// Triggering on any mention is too strict: a fully correct live answer
+// (probe, 2026-08-29) lists the discouraged hashes as bare bullets under a
+// negated heading ("Do **not** use fast general-purpose ... including:") —
+// the cue sits on the heading, but the negation window stops at the line
+// break and the heading is more than a window-width away, so each bullet
+// read as an unnegated endorsement and cost the model half the score.
+// A bare bullet names, it does not recommend — the predicate branch still
+// catches an endorsed bullet ("- **MD5** is fine for passwords").
+var secWeakHashEndorsementPattern = regexp.MustCompile("(?i)(?:" +
+	// a verb recommending it: "hash with MD5", "store passwords hashed with
+	// MD5", "just hash the password with SHA-256" — the filler loop only
+	// bridges recommendation boilerplate, never another algorithm name, so
+	// "prefer argon2 over md5" (a correct answer) is not an endorsement.
+	"(?:hash(?:es|ed|ing)?|store|stores|stored|crypt|encrypt|use|using|used|recommend|recommends|prefer|prefers|should(?:\\s+(?:also|just))?|want)" +
+	"\\s+(?:(?:it|its|the|a|an|password|passwords|user|users|hash|hashed|with|using|as|via|by|plain|raw|fast|for|directly|simply|just|them|to)[ \\t]*\\n?[ \\t]*(?:-[ \\t]*)?){0,6}(?:md5|sha-?1|sha-?256|sha-?512)" +
+	`|(?:^|[^A-Za-z0-9_-])(?:md5|sha-?1|sha-?256|sha-?512)[^.;\n]{0,25}?\b(?:is|are|was|were|seems?|looks?|works?|would\s+be|should\s+be)\s+(?:a\s+|just\s+|still\s+|totally\s+|actually\s+)?(?:fine|good|great|safe|ok|okay|acceptable|perfect|sufficient|enough|best|better|the\s+(?:right|best)\s+choice)` +
+	")")
 
 // secWeakHashNamedAndNegated scores full credit only when the response BOTH
-// safely negates every mention of a weak/fast general-purpose hash (or
-// never mentions one) AND actually names at least one of them by name
-// (md5/sha-1/sha-256/sha-512) - C11's fix folded into a single AND rather
-// than an independent extra ContainsAny term, so a wrong answer that
+// avoids endorsing any weak/fast general-purpose hash with no nearby
+// negation (or never mentions one) AND actually names at least one of them
+// by name (md5/sha-1/sha-256/sha-512) - C11's fix folded into a single AND
+// rather than an independent extra ContainsAny term, so a wrong answer that
 // recommends SHA-256 outright cannot earn partial credit merely for having
 // typed "SHA-256" while doing so.
 func secWeakHashNamedAndNegated() eval.Evaluator {
 	named := eval.ContainsAny("md5", "sha-1", "sha1", "sha-256", "sha256")
-	negated := secNoUnnegatedMention(secWeakPasswordHashPattern)
+	negated := secNoUnnegatedMention(secWeakHashEndorsementPattern)
 	return eval.EvaluatorFunc(func(ctx context.Context, response string) eval.Score {
 		if n := negated.Evaluate(ctx, response); n.Value != 1 {
-			return eval.Score{Value: 0, Detail: "unnegated mention of a discouraged hash: " + n.Detail}
+			return eval.Score{Value: 0, Detail: "endorses a discouraged hash without negating it: " + n.Detail}
 		}
 		if m := named.Evaluate(ctx, response); m.Value != 1 {
 			return eval.Score{Value: 0, Detail: "never names a weak hash by name"}
 		}
-		return eval.Score{Value: 1, Detail: "a weak hash is named and every mention is safely negated"}
+		return eval.Score{Value: 1, Detail: "a weak hash is named and none is endorsed without negation"}
 	})
 }
 

internal/tests/security_crypto_test.go

@@ -28,6 +28,20 @@ func TestSecPasswordHashChoiceTest_Eval(t *testing.T) {
 			response: "bcrypt is fine. Don't use SHA-256 alone, it lacks a work factor.",
 			want:     1,
 		},
+		{
+			// Regression from the live probe (2026-08-29): a fully correct
+			// answer names the discouraged hashes as bullets under a negated
+			// heading ("Do **not** use ... including:"). The heading ends in
+			// ":" - not a clause boundary - so its cue must reach every bullet.
+			name: "correct probe answer: discouraged hashes as bullets under a negated heading",
+			response: "Use Argon2id (or bcrypt with cost >= 12) to store passwords.\n\n" +
+				"Do **not** use fast general-purpose cryptographic hashes for storing passwords, including:\n" +
+				"- **MD5**\n" +
+				"- **SHA-1**\n" +
+				"- **SHA-256**\n" +
+				"- **SHA-512**",
+			want: 1,
+		},
 		{
 			name:     "wrong: recommends SHA-256 unnegated",
 			response: "Just hash the password with SHA-256 before storing it.",