Update

internal/runner/runner.go

@@ -20,7 +20,9 @@ import (
 // referencing FinishReason separately.
 const truncatedDetailPrefix = "TRUNCATED: "
 
-// Result is the outcome of running one test against one model.
+// Result is the outcome of running one test against one model. Attempt is
+// the zero-based index of this sample when Config.Repeat > 1; it is 0 for
+// single-sample runs.
 type Result struct {
 	Err              error
 	Model            string
@@ -31,6 +33,7 @@ type Result struct {
 	Latency          time.Duration
 	PromptTokens     int
 	CompletionTokens int
+	Attempt          int
 }
 
 // Truncated reports whether the model's response was cut off by the token
@@ -54,6 +57,12 @@ type Config struct {
 	Concurrency      int
 	Temperature      float64
 	MaxTokensDefault int
+	// Repeat runs each (model, test) combination this many times (minimum
+	// 1). Even at temperature 0, reasoning models sample differently per
+	// call; repeats expose that instability so a score can be trusted
+	// (stable across attempts) or flagged (flaky) instead of silently
+	// treated as a capability measurement.
+	Repeat int
 }
 
 // Runner executes a set of tests against a set of models.
@@ -71,18 +80,24 @@ func New(client llm.Client, cfg Config) *Runner {
 }
 
 // Run executes every combination of models x tests, bounded by
-// cfg.Concurrency in-flight calls, and returns one Result per combination.
-// A per-call failure is captured in Result.Err rather than aborting the run.
+// cfg.Concurrency in-flight calls, and returns one Result per combination
+// (times Config.Repeat samples each). A per-call failure is captured in
+// Result.Err rather than aborting the run.
 func (r *Runner) Run(ctx context.Context, models []string, tests []testkit.Test) []Result {
 	type job struct {
-		model string
-		test  testkit.Test
+		model   string
+		test    testkit.Test
+		attempt int
 	}
 
-	jobs := make([]job, 0, len(models)*len(tests))
+	repeat := max(r.cfg.Repeat, 1)
+
+	jobs := make([]job, 0, len(models)*len(tests)*repeat)
 	for _, m := range models {
 		for _, t := range tests {
-			jobs = append(jobs, job{model: m, test: t})
+			for a := range repeat {
+				jobs = append(jobs, job{model: m, test: t, attempt: a})
+			}
 		}
 	}
 
@@ -103,6 +118,7 @@ func (r *Runner) Run(ctx context.Context, models []string, tests []testkit.Test)
 	for i, j := range jobs {
 		g.Go(func() error {
 			res := r.runOne(gctx, j.model, j.test)
+			res.Attempt = j.attempt
 			results[i] = res
 			// atomic.AddInt32 makes the completion counter safe under
 			// Config.Concurrency-many concurrent goroutines; ReportDone

internal/runner/runner_test.go

@@ -377,3 +377,36 @@ func TestRunner_Run_TracksLatencyAndTokens(t *testing.T) {
 		t.Errorf("Latency = %v, want 42ms", results[0].Latency)
 	}
 }
+
+func TestRunner_Run_RepeatSamplesEachCombination(t *testing.T) {
+	client := &mockClient{}
+	r := New(client, Config{Concurrency: 2, MaxTokensDefault: 100, Repeat: 3})
+
+	tests := []testkit.Test{{ID: "t1", Category: "c", Prompt: "p1", Eval: echoEval()}}
+	models := []string{"model-a", "model-b"}
+
+	results := r.Run(context.Background(), models, tests)
+
+	if len(results) != 6 {
+		t.Fatalf("len(results) = %d, want 6 (2 models x 1 test x 3 repeats)", len(results))
+	}
+	seen := make(map[[2]string][]int)
+	for _, res := range results {
+		k := [2]string{res.Model, res.TestID}
+		seen[k] = append(seen[k], res.Attempt)
+	}
+	for k, attempts := range seen {
+		if len(attempts) != 3 {
+			t.Errorf("%v: %d attempts, want 3", k, len(attempts))
+		}
+		got := map[int]bool{}
+		for _, a := range attempts {
+			got[a] = true
+		}
+		for a := range 3 {
+			if !got[a] {
+				t.Errorf("%v: missing attempt %d (have %v)", k, a, attempts)
+			}
+		}
+	}
+}