mirror of
https://github.com/lukaszraczylo/llm-testbench.git
synced 2026-10-18 01:17:48 +00:00
feat(llm): make runs resilient and reproducible on slow backends
Retry with exponential backoff and jitter, and honour Retry-After. Retry empty-choice responses and HTTP 408. Add max_retries, retry_timeouts, model_timeouts and model_concurrency settings for slow local models. Add opt-in SSE streaming with an idle timeout. A hung server fails early, and a slow but active generation continues. Send an optional seed and record the backend fingerprint of each answer. Stamp saved artifacts with run metadata and write them atomically. Warn in compare when two runs differ. Save partial results on Ctrl-C. Turn an evaluator panic into an error row. Reject duplicate models and non-http endpoints in the config.
This commit is contained in:
@@ -170,8 +170,11 @@ func CompareArtifacts(baseline, current Artifact) []Comparison {
|
||||
|
||||
// RenderCompare writes a comparison report: headline counts, then only
|
||||
// changed/missing rows (stable rows are summarized, not listed).
|
||||
func RenderCompare(w io.Writer, comparisons []Comparison) error {
|
||||
func RenderCompare(w io.Writer, comparisons []Comparison, warnings ...string) error {
|
||||
ew := &errWriter{w: w}
|
||||
for _, warn := range warnings {
|
||||
ew.println("WARNING: " + warn)
|
||||
}
|
||||
|
||||
var reg, imp, stable, added, removed, errored int
|
||||
for _, c := range comparisons {
|
||||
@@ -219,3 +222,9 @@ func deltaText(v float64) string {
|
||||
}
|
||||
return fmt.Sprintf("%+.2f", v)
|
||||
}
|
||||
|
||||
// CompareWarnings reports reasons the two artifacts may not be comparable
|
||||
// like-for-like (catalog, sampling or backend-build drift).
|
||||
func CompareWarnings(baseline, current Artifact) []string {
|
||||
return compareWarnings(baseline.Meta, current.Meta)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user