llm-testbench/internal/config/config.go

// Package config loads and validates the llm-testbench YAML configuration.
package config

import (
	"fmt"
	"net/url"
	"os"
	"time"

	"gopkg.in/yaml.v3"
)

// apiKeyEnvVar names the environment variable that overrides Config.APIKey.
//
//nolint:gosec // this is an env var *name*, not a credential value
const apiKeyEnvVar = "LLMTB_API_KEY" // #nosec G101 -- this is an env var *name*, not a credential value

// Defaults applied when the corresponding YAML key is omitted (or explicit
// zero). max_tokens_default was raised from 4000 to 12000, then to 40000
// (2026-08-30: at 25000 the heaviest thinking-model tests still exhausted
// the whole budget on reasoning and scored 0 with an empty answer, e.g.
// ts-eslint-flat-config on qwen3.8-flash-next), after a live run
// showed reasoning models spend thousands of completion tokens on
// reasoning_content before writing an answer to content: a small
// per-test/default budget truncates the answer (finish_reason=length)
// before it is ever written, scoring a correct model as 0.
const (
	defaultConcurrency    = 8
	defaultRequestTimeout = 300 * time.Second
	defaultMaxTokens      = 40000
	defaultMaxRetries     = 2
)

// Config is the top-level configuration loaded from config.yaml.
type Config struct {
	Seed             *int                     `yaml:"seed"`
	ModelTimeouts    map[string]time.Duration `yaml:"-"`
	ModelConcurrency map[string]int           `yaml:"model_concurrency"`
	Endpoint         string                   `yaml:"endpoint"`
	APIKey           string                   `yaml:"api_key"`
	Models           []string                 `yaml:"models"`
	Concurrency      int                      `yaml:"concurrency"`
	RequestTimeout   time.Duration            `yaml:"request_timeout"`
	MaxTokensDefault int                      `yaml:"max_tokens_default"`
	MaxRetries       int                      `yaml:"max_retries"`
	IdleTimeout      time.Duration            `yaml:"-"`
	RetryTimeouts    bool                     `yaml:"retry_timeouts"`
	Stream           bool                     `yaml:"stream"`
}

// rawConfig mirrors Config but with RequestTimeout as a string, since
// yaml.v3 does not natively decode Go duration strings.
type rawConfig struct {
	MaxRetries       *int              `yaml:"max_retries"`
	Seed             *int              `yaml:"seed"`
	RetryTimeouts    *bool             `yaml:"retry_timeouts"`
	ModelTimeouts    map[string]string `yaml:"model_timeouts"`
	ModelConcurrency map[string]int    `yaml:"model_concurrency"`
	APIKey           string            `yaml:"api_key"`
	Endpoint         string            `yaml:"endpoint"`
	RequestTimeout   string            `yaml:"request_timeout"`
	IdleTimeout      string            `yaml:"idle_timeout"`
	Models           []string          `yaml:"models"`
	Concurrency      int               `yaml:"concurrency"`
	MaxTokensDefault int               `yaml:"max_tokens_default"`
	Stream           bool              `yaml:"stream"`
}

// Load reads and parses the YAML config at path, applies the LLMTB_API_KEY
// environment override (which always wins over the file value), and
// validates the result.
func Load(path string) (Config, error) {
	// #nosec G304 -- path is the operator's own --config flag value (a
	// local CLI tool reading its own config file), not externally supplied
	// or network input.
	data, err := os.ReadFile(path) //nolint:gosec // path is the operator's own --config flag value, not external input
	if err != nil {
		return Config{}, fmt.Errorf("config: read %s: %w", path, err)
	}
	return parse(data)
}

// parse decodes YAML bytes into a validated Config. Split out from Load for
// direct, file-free unit testing.
func parse(data []byte) (Config, error) {
	var raw rawConfig
	if err := yaml.Unmarshal(data, &raw); err != nil {
		return Config{}, fmt.Errorf("config: parse yaml: %w", err)
	}

	cfg := Config{
		Endpoint:         raw.Endpoint,
		APIKey:           raw.APIKey,
		Models:           raw.Models,
		Concurrency:      raw.Concurrency,
		MaxTokensDefault: raw.MaxTokensDefault,
		MaxRetries:       defaultMaxRetries,
		Seed:             raw.Seed,
	}
	if raw.MaxRetries != nil {
		cfg.MaxRetries = *raw.MaxRetries
	}
	cfg.RetryTimeouts = raw.RetryTimeouts == nil || *raw.RetryTimeouts
	cfg.ModelConcurrency = raw.ModelConcurrency
	cfg.Stream = raw.Stream
	if raw.IdleTimeout != "" {
		d, err := time.ParseDuration(raw.IdleTimeout)
		if err != nil {
			return Config{}, fmt.Errorf("config: parse idle_timeout %q: %w", raw.IdleTimeout, err)
		}
		cfg.IdleTimeout = d
	}
	if len(raw.ModelTimeouts) > 0 {
		cfg.ModelTimeouts = make(map[string]time.Duration, len(raw.ModelTimeouts))
		for m, v := range raw.ModelTimeouts {
			d, err := time.ParseDuration(v)
			if err != nil {
				return Config{}, fmt.Errorf("config: parse model_timeouts[%q] %q: %w", m, v, err)
			}
			cfg.ModelTimeouts[m] = d
		}
	}

	if raw.RequestTimeout != "" {
		d, err := time.ParseDuration(raw.RequestTimeout)
		if err != nil {
			return Config{}, fmt.Errorf("config: parse request_timeout %q: %w", raw.RequestTimeout, err)
		}
		cfg.RequestTimeout = d
	}

	applyDefaults(&cfg)

	if envKey := os.Getenv(apiKeyEnvVar); envKey != "" {
		cfg.APIKey = envKey
	}

	if err := cfg.Validate(); err != nil {
		return Config{}, err
	}
	return cfg, nil
}

// applyDefaults fills in Concurrency, RequestTimeout, and MaxTokensDefault
// when the YAML omitted them (or gave an explicit zero), so a minimal
// config.yaml still runs with sane, generous-enough values.
func applyDefaults(cfg *Config) {
	if cfg.Concurrency <= 0 {
		cfg.Concurrency = defaultConcurrency
	}
	if cfg.RequestTimeout <= 0 {
		cfg.RequestTimeout = defaultRequestTimeout
	}
	if cfg.MaxTokensDefault <= 0 {
		cfg.MaxTokensDefault = defaultMaxTokens
	}
}

// Validate checks that required fields are present and sane.
func (c Config) Validate() error {
	if c.Endpoint == "" {
		return fmt.Errorf("config: endpoint must not be empty")
	}
	if len(c.Models) == 0 {
		return fmt.Errorf("config: models must list at least one model")
	}
	u, err := url.Parse(c.Endpoint)
	if err != nil || (u.Scheme != "http" && u.Scheme != "https") || u.Host == "" {
		return fmt.Errorf("config: endpoint %q must be an http(s) URL", c.Endpoint)
	}
	seen := make(map[string]bool, len(c.Models))
	for i, m := range c.Models {
		if m == "" {
			return fmt.Errorf("config: models[%d] must not be empty", i)
		}
		if seen[m] {
			return fmt.Errorf("config: models[%d] duplicates %q", i, m)
		}
		seen[m] = true
	}
	for m, d := range c.ModelTimeouts {
		if d <= 0 {
			return fmt.Errorf("config: model_timeouts[%q] must be > 0, got %s", m, d)
		}
	}
	for m, n := range c.ModelConcurrency {
		if n <= 0 {
			return fmt.Errorf("config: model_concurrency[%q] must be > 0, got %d", m, n)
		}
	}
	if c.IdleTimeout < 0 {
		return fmt.Errorf("config: idle_timeout must be >= 0, got %s", c.IdleTimeout)
	}
	if c.MaxRetries < 0 {
		return fmt.Errorf("config: max_retries must be >= 0, got %d", c.MaxRetries)
	}
	if c.Concurrency <= 0 {
		return fmt.Errorf("config: concurrency must be > 0, got %d", c.Concurrency)
	}
	if c.RequestTimeout <= 0 {
		return fmt.Errorf("config: request_timeout must be > 0, got %s", c.RequestTimeout)
	}
	if c.MaxTokensDefault <= 0 {
		return fmt.Errorf("config: max_tokens_default must be > 0, got %d", c.MaxTokensDefault)
	}
	return nil
}