feat(llm): make runs resilient and reproducible on slow backends

Retry with exponential backoff and jitter, and honour Retry-After. Retry empty-choice responses and HTTP 408.
Add max_retries, retry_timeouts, model_timeouts and model_concurrency settings for slow local models.
Add opt-in SSE streaming with an idle timeout. A hung server fails early, and a slow but active generation continues.
Send an optional seed and record the backend fingerprint of each answer.
Stamp saved artifacts with run metadata and write them atomically. Warn in compare when two runs differ.
Save partial results on Ctrl-C. Turn an evaluator panic into an error row.
Reject duplicate models and non-http endpoints in the config.

internal/runner/runner.go

@@ -5,6 +5,7 @@ package runner
 import (
 	"context"
 	"encoding/json"
+	"fmt"
 	"sync/atomic"
 	"time"
 
@@ -59,6 +60,8 @@ type Result struct {
 	TestID           string
 	ResponseText     string
 	FinishReason     string
+	Fingerprint      string
+	ServedModel      string
 	Score            eval.Score
 	Latency          time.Duration
 	PromptTokens     int
@@ -83,7 +86,13 @@ func (r Result) TotalTokens() int {
 
 // Config controls how the Runner fans work out.
 type Config struct {
-	Reporter         ProgressReporter
+	Reporter ProgressReporter
+	// Seed, when non-nil, is sent with every request for backends that
+	// honor it.
+	Seed *int
+	// ModelConcurrency caps in-flight calls per model (on top of the global
+	// Concurrency), so one slow local model is not flooded into timeouts.
+	ModelConcurrency map[string]int
 	Concurrency      int
 	Temperature      float64
 	MaxTokensDefault int
@@ -138,15 +147,32 @@ func (r *Runner) Run(ctx context.Context, models []string, tests []testkit.Test)
 	if concurrency <= 0 {
 		concurrency = 1
 	}
-	g.SetLimit(concurrency)
+	// Jobs wait for their per-model slot first, then a global slot, so a
+	// capped slow model's queue never occupies global slots other models
+	// need.
+	global := make(chan struct{}, concurrency)
 
 	r.cfg.Reporter.ReportStart(len(tests), len(models), concurrency)
 
 	var completed int32
 	// go.mod's go directive is 1.22+, so loop variables are already
 	// per-iteration scoped; no manual i, j := i, j shadowing needed (N4).
+	sems := make(map[string]chan struct{}, len(r.cfg.ModelConcurrency))
+	for m, n := range r.cfg.ModelConcurrency {
+		if n > 0 {
+			sems[m] = make(chan struct{}, n)
+		}
+	}
 	for i, j := range jobs {
 		g.Go(func() error {
+			if sem, ok := sems[j.model]; ok {
+				if acquire(gctx, sem) {
+					defer func() { <-sem }()
+				}
+			}
+			if acquire(gctx, global) {
+				defer func() { <-global }()
+			}
 			res := r.runOne(gctx, j.model, j.test)
 			res.Attempt = j.attempt
 			results[i] = res
@@ -167,8 +193,32 @@ func (r *Runner) Run(ctx context.Context, models []string, tests []testkit.Test)
 	return results
 }
 
-// runOne executes a single (model, test) combination.
-func (r *Runner) runOne(ctx context.Context, model string, test testkit.Test) Result {
+// acquire takes a slot from sem, reporting false if ctx ended first.
+func acquire(ctx context.Context, sem chan struct{}) bool {
+	select {
+	case sem <- struct{}{}:
+		return true
+	case <-ctx.Done():
+		return false
+	}
+}
+
+// runOne executes a single (model, test) combination. A cancelled context
+// short-circuits to an error result, and a panic in an evaluator becomes an
+// error result instead of killing a multi-hour run.
+func (r *Runner) runOne(ctx context.Context, model string, test testkit.Test) (res Result) {
+	if err := ctx.Err(); err != nil {
+		return Result{Model: model, TestID: test.ID, Err: err}
+	}
+	defer func() {
+		if p := recover(); p != nil {
+			res = Result{Model: model, TestID: test.ID, Err: fmt.Errorf("runner: panic scoring test: %v", p)}
+		}
+	}()
+	return r.execute(ctx, model, test)
+}
+
+func (r *Runner) execute(ctx context.Context, model string, test testkit.Test) Result {
 	// cfg.MaxTokensDefault is a floor, not a fallback used only when the
 	// test leaves MaxTokens unset: a per-test value only ever raises the
 	// budget, it never lowers it below the default. A reasoning model can
@@ -190,6 +240,7 @@ func (r *Runner) runOne(ctx context.Context, model string, test testkit.Test) Re
 		Tools:       test.Tools,
 		MaxTokens:   maxTokens,
 		Temperature: r.cfg.Temperature,
+		Seed:        r.cfg.Seed,
 	}
 
 	// Wall-clock across all attempts, recorded even on error: an error row
@@ -223,6 +274,8 @@ func (r *Runner) runOne(ctx context.Context, model string, test testkit.Test) Re
 		Score:            score,
 		ResponseText:     scored,
 		FinishReason:     resp.FinishReason,
+		Fingerprint:      resp.Fingerprint,
+		ServedModel:      resp.ServedModel,
 		Latency:          resp.Latency,
 		PromptTokens:     resp.PromptTokens,
 		CompletionTokens: resp.CompletionTokens,

internal/runner/runner_test.go

@@ -410,3 +410,89 @@ func TestRunner_Run_RepeatSamplesEachCombination(t *testing.T) {
 		}
 	}
 }
+
+func TestRunner_Run_PassesSeedAndRecordsFingerprint(t *testing.T) {
+	var gotSeed *int
+	client := &mockClient{respond: func(req llm.Request) (llm.Response, error) {
+		gotSeed = req.Seed
+		return llm.Response{Text: "OK", Fingerprint: "fp-1", ServedModel: "served"}, nil
+	}}
+	seed := 7
+	r := New(client, Config{Concurrency: 1, MaxTokensDefault: 100, Seed: &seed})
+	res := r.Run(context.Background(), []string{"m"}, []testkit.Test{{ID: "t", Category: "c", Prompt: "p", Eval: echoEval()}})
+	if gotSeed == nil || *gotSeed != 7 {
+		t.Errorf("request seed = %v, want 7", gotSeed)
+	}
+	if res[0].Fingerprint != "fp-1" || res[0].ServedModel != "served" {
+		t.Errorf("fingerprint/model = %q/%q, want fp-1/served", res[0].Fingerprint, res[0].ServedModel)
+	}
+}
+
+func TestRunner_Run_EvaluatorPanicBecomesErrorResult(t *testing.T) {
+	boom := eval.EvaluatorFunc(func(context.Context, string) eval.Score { panic("boom") })
+	r := New(&mockClient{}, Config{Concurrency: 2, MaxTokensDefault: 100})
+	res := r.Run(context.Background(), []string{"m"}, []testkit.Test{
+		{ID: "bad", Category: "c", Prompt: "p", Eval: boom},
+		{ID: "good", Category: "c", Prompt: "p", Eval: echoEval()},
+	})
+	if res[0].Err == nil || !strings.Contains(res[0].Err.Error(), "boom") {
+		t.Errorf("panicking test Err = %v, want panic error", res[0].Err)
+	}
+	if res[1].Err != nil || res[1].Score.Value != 1 {
+		t.Errorf("sibling test = %+v, want unaffected pass", res[1])
+	}
+}
+
+func TestRunner_Run_CancelledContextSkipsCalls(t *testing.T) {
+	var calls int32
+	client := &mockClient{respond: func(llm.Request) (llm.Response, error) {
+		atomic.AddInt32(&calls, 1)
+		return llm.Response{Text: "OK"}, nil
+	}}
+	ctx, cancel := context.WithCancel(context.Background())
+	cancel()
+	r := New(client, Config{Concurrency: 2, MaxTokensDefault: 100})
+	res := r.Run(ctx, []string{"m"}, []testkit.Test{{ID: "t", Category: "c", Prompt: "p", Eval: echoEval()}})
+	if atomic.LoadInt32(&calls) != 0 {
+		t.Errorf("client called %d times after cancel, want 0", calls)
+	}
+	if !errors.Is(res[0].Err, context.Canceled) {
+		t.Errorf("Err = %v, want context.Canceled", res[0].Err)
+	}
+}
+
+func TestRunner_Run_ModelConcurrencyCapDoesNotStarveOthers(t *testing.T) {
+	var slowIn, slowMax int32
+	fastDone := make(chan struct{}, 8)
+	client := &mockClient{respond: func(req llm.Request) (llm.Response, error) {
+		if req.Model == "slow" {
+			cur := atomic.AddInt32(&slowIn, 1)
+			for {
+				m := atomic.LoadInt32(&slowMax)
+				if cur <= m || atomic.CompareAndSwapInt32(&slowMax, m, cur) {
+					break
+				}
+			}
+			time.Sleep(30 * time.Millisecond)
+			atomic.AddInt32(&slowIn, -1)
+		} else {
+			fastDone <- struct{}{}
+		}
+		return llm.Response{Text: "OK"}, nil
+	}}
+	r := New(client, Config{Concurrency: 2, MaxTokensDefault: 100, ModelConcurrency: map[string]int{"slow": 1}})
+	tests := make([]testkit.Test, 6)
+	for i := range tests {
+		tests[i] = testkit.Test{ID: string(rune('a' + i)), Category: "c", Prompt: "p", Eval: echoEval()}
+	}
+	res := r.Run(context.Background(), []string{"slow", "fast"}, tests)
+	if len(res) != 12 {
+		t.Fatalf("results = %d, want 12", len(res))
+	}
+	if got := atomic.LoadInt32(&slowMax); got != 1 {
+		t.Errorf("slow model max in-flight = %d, want 1", got)
+	}
+	if len(fastDone) != 6 {
+		t.Errorf("fast calls = %d, want 6", len(fastDone))
+	}
+}