lukaszraczylo/llm-testbench
1
0
Fork 0
mirror of https://github.com/lukaszraczylo/llm-testbench.git synced 2026-10-17 01:17:33 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(llm): make runs resilient and reproducible on slow backends

Browse Source 
Retry with exponential backoff and jitter, and honour Retry-After. Retry empty-choice responses and HTTP 408.
Add max_retries, retry_timeouts, model_timeouts and model_concurrency settings for slow local models.
Add opt-in SSE streaming with an idle timeout. A hung server fails early, and a slow but active generation continues.
Send an optional seed and record the backend fingerprint of each answer.
Stamp saved artifacts with run metadata and write them atomically. Warn in compare when two runs differ.
Save partial results on Ctrl-C. Turn an evaluator panic into an error row.
Reject duplicate models and non-http endpoints in the config.
This commit is contained in:
lukaszraczylo
2026-10-02 01:01:14 +01:00
parent 7b791a78e7
commit 1310c5bebe
17 changed files with 1367 additions and 67 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/config/config.go
+78 -12
View File
@@ -3,6 +3,7 @@ package config
import (
"fmt"
"net/url"
"os"
"time"
@@ -27,27 +28,42 @@ const (
defaultConcurrency = 8
defaultRequestTimeout = 300 * time.Second
defaultMaxTokens = 40000
defaultMaxRetries = 2
)
// Config is the top-level configuration loaded from config.yaml.
type Config struct {
Endpoint string `yaml:"endpoint"`
APIKey string `yaml:"api_key"`
Models []string `yaml:"models"`
Concurrency int `yaml:"concurrency"`
RequestTimeout time.Duration `yaml:"request_timeout"`
MaxTokensDefault int `yaml:"max_tokens_default"`
Seed *int `yaml:"seed"`
ModelTimeouts map[string]time.Duration `yaml:"-"`
ModelConcurrency map[string]int `yaml:"model_concurrency"`
Endpoint string `yaml:"endpoint"`
APIKey string `yaml:"api_key"`
Models []string `yaml:"models"`
Concurrency int `yaml:"concurrency"`
RequestTimeout time.Duration `yaml:"request_timeout"`
MaxTokensDefault int `yaml:"max_tokens_default"`
MaxRetries int `yaml:"max_retries"`
IdleTimeout time.Duration `yaml:"-"`
RetryTimeouts bool `yaml:"retry_timeouts"`
Stream bool `yaml:"stream"`
}
// rawConfig mirrors Config but with RequestTimeout as a string, since
// yaml.v3 does not natively decode Go duration strings.
type rawConfig struct {
Endpoint string `yaml:"endpoint"`
APIKey string `yaml:"api_key"`
RequestTimeout string `yaml:"request_timeout"`
Models []string `yaml:"models"`
Concurrency int `yaml:"concurrency"`
MaxTokensDefault int `yaml:"max_tokens_default"`
MaxRetries *int `yaml:"max_retries"`
Seed *int `yaml:"seed"`
RetryTimeouts *bool `yaml:"retry_timeouts"`
ModelTimeouts map[string]string `yaml:"model_timeouts"`
ModelConcurrency map[string]int `yaml:"model_concurrency"`
APIKey string `yaml:"api_key"`
Endpoint string `yaml:"endpoint"`
RequestTimeout string `yaml:"request_timeout"`
IdleTimeout string `yaml:"idle_timeout"`
Models []string `yaml:"models"`
Concurrency int `yaml:"concurrency"`
MaxTokensDefault int `yaml:"max_tokens_default"`
Stream bool `yaml:"stream"`
}
// Load reads and parses the YAML config at path, applies the LLMTB_API_KEY
@@ -78,6 +94,31 @@ func parse(data []byte) (Config, error) {
Models: raw.Models,
Concurrency: raw.Concurrency,
MaxTokensDefault: raw.MaxTokensDefault,
MaxRetries: defaultMaxRetries,
Seed: raw.Seed,
}
if raw.MaxRetries != nil {
cfg.MaxRetries = *raw.MaxRetries
}
cfg.RetryTimeouts = raw.RetryTimeouts == nil || *raw.RetryTimeouts
cfg.ModelConcurrency = raw.ModelConcurrency
cfg.Stream = raw.Stream
if raw.IdleTimeout != "" {
d, err := time.ParseDuration(raw.IdleTimeout)
if err != nil {
return Config{}, fmt.Errorf("config: parse idle_timeout %q: %w", raw.IdleTimeout, err)
}
cfg.IdleTimeout = d
}
if len(raw.ModelTimeouts) > 0 {
cfg.ModelTimeouts = make(map[string]time.Duration, len(raw.ModelTimeouts))
for m, v := range raw.ModelTimeouts {
d, err := time.ParseDuration(v)
if err != nil {
return Config{}, fmt.Errorf("config: parse model_timeouts[%q] %q: %w", m, v, err)
}
cfg.ModelTimeouts[m] = d
}
}
if raw.RequestTimeout != "" {
@@ -123,10 +164,35 @@ func (c Config) Validate() error {
if len(c.Models) == 0 {
return fmt.Errorf("config: models must list at least one model")
}
u, err := url.Parse(c.Endpoint)
if err != nil || (u.Scheme != "http" && u.Scheme != "https") || u.Host == "" {
return fmt.Errorf("config: endpoint %q must be an http(s) URL", c.Endpoint)
}
seen := make(map[string]bool, len(c.Models))
for i, m := range c.Models {
if m == "" {
return fmt.Errorf("config: models[%d] must not be empty", i)
}
if seen[m] {
return fmt.Errorf("config: models[%d] duplicates %q", i, m)
}
seen[m] = true
}
for m, d := range c.ModelTimeouts {
if d <= 0 {
return fmt.Errorf("config: model_timeouts[%q] must be > 0, got %s", m, d)
}
}
for m, n := range c.ModelConcurrency {
if n <= 0 {
return fmt.Errorf("config: model_concurrency[%q] must be > 0, got %d", m, n)
}
}
if c.IdleTimeout < 0 {
return fmt.Errorf("config: idle_timeout must be >= 0, got %s", c.IdleTimeout)
}
if c.MaxRetries < 0 {
return fmt.Errorf("config: max_retries must be >= 0, got %d", c.MaxRetries)
}
if c.Concurrency <= 0 {
return fmt.Errorf("config: concurrency must be > 0, got %d", c.Concurrency)
internal/config/config_test.go
+65
View File
@@ -160,6 +160,9 @@ func TestValidate(t *testing.T) {
{name: "zero concurrency", mutate: func(c Config) Config { c.Concurrency = 0; return c }},
{name: "negative concurrency", mutate: func(c Config) Config { c.Concurrency = -1; return c }},
{name: "zero timeout", mutate: func(c Config) Config { c.RequestTimeout = 0; return c }},
{name: "non-http endpoint", mutate: func(c Config) Config { c.Endpoint = "llm-gateway.example.com/v1"; return c }},
{name: "duplicate model", mutate: func(c Config) Config { c.Models = []string{"m", "m"}; return c }},
{name: "negative retries", mutate: func(c Config) Config { c.MaxRetries = -1; return c }},
{name: "zero max tokens", mutate: func(c Config) Config { c.MaxTokensDefault = 0; return c }},
}
for _, tt := range tests {
@@ -194,3 +197,65 @@ func TestLoad_MissingFile(t *testing.T) {
t.Fatal("Load() error = nil, want error for missing file")
}
}
func TestParse_RetriesAndSeed(t *testing.T) {
base := "endpoint: https://x\nmodels: [m]\n"
cfg, err := parse([]byte(base))
if err != nil {
t.Fatalf("parse() error = %v", err)
}
if cfg.MaxRetries != defaultMaxRetries || cfg.Seed != nil {
t.Errorf("defaults = retries %d seed %v, want %d nil", cfg.MaxRetries, cfg.Seed, defaultMaxRetries)
}
cfg, err = parse([]byte(base + "max_retries: 0\nseed: 7\n"))
if err != nil {
t.Fatalf("parse() error = %v", err)
}
if cfg.MaxRetries != 0 {
t.Errorf("explicit max_retries 0 = %d, want 0 (retrying disabled)", cfg.MaxRetries)
}
if cfg.Seed == nil || *cfg.Seed != 7 {
t.Errorf("Seed = %v, want 7", cfg.Seed)
}
}
func TestParse_ModelOverrides(t *testing.T) {
cfg, err := parse([]byte(`
endpoint: https://x
models: [a, b]
retry_timeouts: false
model_timeouts: {a: 45m}
model_concurrency: {a: 1}
`))
if err != nil {
t.Fatalf("parse() error = %v", err)
}
if cfg.RetryTimeouts || cfg.ModelTimeouts["a"] != 45*time.Minute || cfg.ModelConcurrency["a"] != 1 {
t.Errorf("got %+v", cfg)
}
def, _ := parse([]byte("endpoint: https://x\nmodels: [a]\n"))
if !def.RetryTimeouts {
t.Error("retry_timeouts default = false, want true (legacy behavior)")
}
for _, bad := range []string{"model_timeouts: {a: nope}", "model_timeouts: {a: 0s}", "model_concurrency: {a: 0}"} {
if _, err := parse([]byte("endpoint: https://x\nmodels: [a]\n" + bad)); err == nil {
t.Errorf("parse(%q) error = nil, want error", bad)
}
}
}
func TestParse_Streaming(t *testing.T) {
cfg, err := parse([]byte("endpoint: https://x\nmodels: [a]\nstream: true\nidle_timeout: 10m\n"))
if err != nil || !cfg.Stream || cfg.IdleTimeout != 10*time.Minute {
t.Fatalf("cfg = %+v, err = %v", cfg, err)
}
def, _ := parse([]byte("endpoint: https://x\nmodels: [a]\n"))
if def.Stream {
t.Error("stream default = true, want false")
}
if _, err := parse([]byte("endpoint: https://x\nmodels: [a]\nidle_timeout: nope\n")); err == nil {
t.Error("bad idle_timeout accepted")
}
}