feat(llm): make runs resilient and reproducible on slow backends

Retry with exponential backoff and jitter, and honour Retry-After. Retry empty-choice responses and HTTP 408.
Add max_retries, retry_timeouts, model_timeouts and model_concurrency settings for slow local models.
Add opt-in SSE streaming with an idle timeout. A hung server fails early, and a slow but active generation continues.
Send an optional seed and record the backend fingerprint of each answer.
Stamp saved artifacts with run metadata and write them atomically. Warn in compare when two runs differ.
Save partial results on Ctrl-C. Turn an evaluator panic into an error row.
Reject duplicate models and non-http endpoints in the config.

internal/report/compare.go

@@ -170,8 +170,11 @@ func CompareArtifacts(baseline, current Artifact) []Comparison {
 
 // RenderCompare writes a comparison report: headline counts, then only
 // changed/missing rows (stable rows are summarized, not listed).
-func RenderCompare(w io.Writer, comparisons []Comparison) error {
+func RenderCompare(w io.Writer, comparisons []Comparison, warnings ...string) error {
 	ew := &errWriter{w: w}
+	for _, warn := range warnings {
+		ew.println("WARNING: " + warn)
+	}
 
 	var reg, imp, stable, added, removed, errored int
 	for _, c := range comparisons {
@@ -219,3 +222,9 @@ func deltaText(v float64) string {
 	}
 	return fmt.Sprintf("%+.2f", v)
 }
+
+// CompareWarnings reports reasons the two artifacts may not be comparable
+// like-for-like (catalog, sampling or backend-build drift).
+func CompareWarnings(baseline, current Artifact) []string {
+	return compareWarnings(baseline.Meta, current.Meta)
+}

internal/report/json.go

@@ -6,6 +6,7 @@ import (
 	"io"
 	"math"
 	"os"
+	"path/filepath"
 
 	"github.com/lukaszraczylo/llm-testbench/internal/runner"
 	"github.com/lukaszraczylo/llm-testbench/internal/testkit"
@@ -22,6 +23,7 @@ type jsonResult struct {
 	Error            string  `json:"error,omitempty"`
 	ResponseText     string  `json:"response_text"`
 	FinishReason     string  `json:"finish_reason,omitempty"`
+	Fingerprint      string  `json:"fingerprint,omitempty"`
 	Score            float64 `json:"score"`
 	LatencyMS        int64   `json:"latency_ms"`
 	PromptTokens     int     `json:"prompt_tokens"`
@@ -36,6 +38,7 @@ type jsonResult struct {
 // plus the per-test discrimination rollup, so downstream tooling
 // (compare) can re-derive stability without re-running the suite.
 type jsonArtifact struct {
+	Meta    *RunMeta      `json:"meta,omitempty"`
 	Results []jsonResult  `json:"results"`
 	Stats   []jsonStatRow `json:"stats,omitempty"`
 }
@@ -77,6 +80,7 @@ func buildArtifact(tests []testkit.Test, models []string, results []runner.Resul
 			Detail:           r.Score.Detail,
 			ResponseText:     r.ResponseText,
 			FinishReason:     r.FinishReason,
+			Fingerprint:      r.Fingerprint,
 			Skipped:          r.Score.Skipped,
 			Truncated:        r.Truncated(),
 			LatencyMS:        r.Latency.Milliseconds(),
@@ -150,12 +154,37 @@ func LoadArtifact(path string) (jsonArtifact, error) {
 	return jsonArtifact{Results: legacy}, nil
 }
 
-// WriteArtifact saves the run as a JSON artifact at path, for later
-// `llmtest compare` runs, regardless of the stdout report format.
-func WriteArtifact(path string, tests []testkit.Test, models []string, results []runner.Result) error {
-	b, err := json.MarshalIndent(buildArtifact(tests, models, results), "", "  ")
+// WriteArtifact saves the run, stamped with meta, as a JSON artifact at
+// path for later `llmtest compare` runs, regardless of the stdout report
+// format. The write is atomic (temp file + rename), so an interrupted
+// process never leaves a truncated artifact that replaced a good one.
+func WriteArtifact(path string, meta RunMeta, tests []testkit.Test, models []string, results []runner.Result) error {
+	art := buildArtifact(tests, models, results)
+	meta.CatalogHash = CatalogHash(tests)
+	meta.TestCount = len(tests)
+	meta.Fingerprints = servedFingerprints(results)
+	art.Meta = &meta
+
+	b, err := json.MarshalIndent(art, "", "  ")
 	if err != nil {
 		return err
 	}
-	return os.WriteFile(path, append(b, '\n'), 0o644) //nolint:gosec // 0o644: the artifact is a report meant to be shared/committed, not a secret file
+
+	tmp, err := os.CreateTemp(filepath.Dir(path), ".llmtest-artifact-*")
+	if err != nil {
+		return err
+	}
+	defer func() { _ = os.Remove(tmp.Name()) }() // no-op after a successful rename
+	if _, err := tmp.Write(append(b, '\n')); err != nil {
+		_ = tmp.Close()
+		return err
+	}
+	if err := tmp.Close(); err != nil {
+		return err
+	}
+	// 0o644: the artifact is a report meant to be shared/committed, not a secret file.
+	if err := os.Chmod(tmp.Name(), 0o644); err != nil { //nolint:gosec // see comment above
+		return err
+	}
+	return os.Rename(tmp.Name(), path)
 }

internal/report/meta.go

@@ -0,0 +1,109 @@
+package report
+
+import (
+	"crypto/sha256"
+	"encoding/hex"
+	"fmt"
+	"sort"
+	"strconv"
+	"strings"
+
+	"github.com/lukaszraczylo/llm-testbench/internal/runner"
+	"github.com/lukaszraczylo/llm-testbench/internal/testkit"
+)
+
+// RunMeta records the settings and environment a run used, so two
+// artifacts can be judged comparable and a result reproduced.
+type RunMeta struct {
+	Fingerprints     map[string][]string `json:"fingerprints,omitempty"`
+	Seed             *int                `json:"seed,omitempty"`
+	Version          string              `json:"version"`
+	StartedAt        string              `json:"started_at"`
+	Endpoint         string              `json:"endpoint,omitempty"`
+	CatalogHash      string              `json:"catalog_hash"`
+	Temperature      float64             `json:"temperature"`
+	MaxTokensDefault int                 `json:"max_tokens_default"`
+	Repeat           int                 `json:"repeat"`
+	TestCount        int                 `json:"test_count"`
+	Interrupted      bool                `json:"interrupted,omitempty"`
+}
+
+// CatalogHash returns a stable digest of the test inputs that determine a
+// model's answer (id, system prompt, prompt, token floor, tool names).
+func CatalogHash(tests []testkit.Test) string {
+	sorted := append([]testkit.Test(nil), tests...)
+	sort.Slice(sorted, func(i, j int) bool { return sorted[i].ID < sorted[j].ID })
+	h := sha256.New()
+	for _, t := range sorted {
+		names := make([]string, 0, len(t.Tools))
+		for _, tl := range t.Tools {
+			names = append(names, tl.Name)
+		}
+		_, _ = fmt.Fprintf(h, "%s\x00%s\x00%s\x00%d\x00%s\x01", t.ID, t.System, t.Prompt, t.MaxTokens, strings.Join(names, ","))
+	}
+	return hex.EncodeToString(h.Sum(nil))[:16]
+}
+
+// servedFingerprints collects the distinct backend fingerprints each model
+// answered with; more than one means the serving build changed mid-run.
+func servedFingerprints(results []runner.Result) map[string][]string {
+	seen := make(map[string]map[string]bool)
+	for _, r := range results {
+		if r.Fingerprint == "" {
+			continue
+		}
+		if seen[r.Model] == nil {
+			seen[r.Model] = make(map[string]bool)
+		}
+		seen[r.Model][r.Fingerprint] = true
+	}
+	if len(seen) == 0 {
+		return nil
+	}
+	out := make(map[string][]string, len(seen))
+	for m, fps := range seen {
+		for fp := range fps {
+			out[m] = append(out[m], fp)
+		}
+		sort.Strings(out[m])
+	}
+	return out
+}
+
+// compareWarnings lists reasons two artifacts may not be like-for-like.
+// Artifacts without metadata (older runs) produce no warnings.
+func compareWarnings(base, cur *RunMeta) []string {
+	if base == nil || cur == nil {
+		return nil
+	}
+	var w []string
+	if base.CatalogHash != cur.CatalogHash {
+		w = append(w, fmt.Sprintf("catalog differs (%s vs %s): prompts changed between runs", base.CatalogHash, cur.CatalogHash))
+	}
+	if base.Temperature != cur.Temperature {
+		w = append(w, fmt.Sprintf("temperature differs (%g vs %g)", base.Temperature, cur.Temperature))
+	}
+	if base.MaxTokensDefault != cur.MaxTokensDefault {
+		w = append(w, fmt.Sprintf("max_tokens_default differs (%d vs %d)", base.MaxTokensDefault, cur.MaxTokensDefault))
+	}
+	if fmtSeed(base.Seed) != fmtSeed(cur.Seed) {
+		w = append(w, fmt.Sprintf("seed differs (%s vs %s)", fmtSeed(base.Seed), fmtSeed(cur.Seed)))
+	}
+	if base.Interrupted || cur.Interrupted {
+		w = append(w, "an artifact is from an interrupted run: missing results are not regressions")
+	}
+	for m, cfps := range cur.Fingerprints {
+		if bfps, ok := base.Fingerprints[m]; ok && strings.Join(bfps, ",") != strings.Join(cfps, ",") {
+			w = append(w, fmt.Sprintf("%s served by a different backend build (%s vs %s)", m, strings.Join(bfps, "+"), strings.Join(cfps, "+")))
+		}
+	}
+	sort.Strings(w)
+	return w
+}
+
+func fmtSeed(s *int) string {
+	if s == nil {
+		return "unset"
+	}
+	return strconv.Itoa(*s)
+}

internal/report/meta_test.go

@@ -0,0 +1,71 @@
+package report
+
+import (
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/lukaszraczylo/llm-testbench/internal/runner"
+	"github.com/lukaszraczylo/llm-testbench/internal/testkit"
+)
+
+func TestCatalogHash_StableAndSensitive(t *testing.T) {
+	a := testkit.Test{ID: "a", Prompt: "p1"}
+	b := testkit.Test{ID: "b", Prompt: "p2"}
+	if CatalogHash([]testkit.Test{a, b}) != CatalogHash([]testkit.Test{b, a}) {
+		t.Error("hash depends on input order, want stable")
+	}
+	b2 := b
+	b2.Prompt = "p2 changed"
+	if CatalogHash([]testkit.Test{a, b}) == CatalogHash([]testkit.Test{a, b2}) {
+		t.Error("hash ignores a prompt change")
+	}
+}
+
+func TestCompareWarnings(t *testing.T) {
+	seed := 1
+	base := &RunMeta{CatalogHash: "x", Temperature: 0, MaxTokensDefault: 100, Fingerprints: map[string][]string{"m": {"fp1"}}}
+	if w := compareWarnings(base, base); len(w) != 0 {
+		t.Errorf("identical meta warnings = %v, want none", w)
+	}
+	if w := compareWarnings(nil, base); len(w) != 0 {
+		t.Errorf("legacy artifact warnings = %v, want none", w)
+	}
+	cur := &RunMeta{CatalogHash: "y", Temperature: 0, MaxTokensDefault: 200, Seed: &seed, Interrupted: true,
+		Fingerprints: map[string][]string{"m": {"fp2"}}}
+	got := strings.Join(compareWarnings(base, cur), "\n")
+	for _, want := range []string{"catalog differs", "max_tokens_default differs", "seed differs", "interrupted", "different backend build"} {
+		if !strings.Contains(got, want) {
+			t.Errorf("warnings missing %q:\n%s", want, got)
+		}
+	}
+}
+
+func TestWriteArtifact_AtomicWithMetaRoundTrip(t *testing.T) {
+	dir := t.TempDir()
+	path := filepath.Join(dir, "run.json")
+	if err := os.WriteFile(path, []byte("previous"), 0o600); err != nil {
+		t.Fatal(err)
+	}
+	tests := []testkit.Test{{ID: "t1", Category: "c", Prompt: "p"}}
+	results := []runner.Result{{Model: "m", TestID: "t1", Fingerprint: "fp"}}
+
+	if err := WriteArtifact(path, RunMeta{Version: "v1", Repeat: 1}, tests, []string{"m"}, results); err != nil {
+		t.Fatalf("WriteArtifact() error = %v", err)
+	}
+	art, err := LoadArtifact(path)
+	if err != nil {
+		t.Fatalf("LoadArtifact() error = %v", err)
+	}
+	if art.Meta == nil || art.Meta.Version != "v1" || art.Meta.TestCount != 1 || art.Meta.CatalogHash == "" {
+		t.Errorf("Meta = %+v, want version, test count and catalog hash set", art.Meta)
+	}
+	if got := art.Meta.Fingerprints["m"]; len(got) != 1 || got[0] != "fp" {
+		t.Errorf("Fingerprints = %v, want m:[fp]", art.Meta.Fingerprints)
+	}
+	entries, _ := os.ReadDir(dir)
+	if len(entries) != 1 {
+		t.Errorf("dir has %d entries after write, want only the artifact (no temp leftovers)", len(entries))
+	}
+}