mirror of
https://github.com/lukaszraczylo/llm-testbench.git
synced 2026-10-12 00:45:19 +00:00
feat: LLM accuracy testing framework with 17-test catalog
Framework runs a category/subcategory test catalog against multiple LLMs through an OpenAI-compatible endpoint. Deterministic evaluators score each response. Reports render as table, markdown, or JSON.
This commit is contained in:
@@ -0,0 +1,109 @@
|
||||
// Package config loads and validates the llm-testbench YAML configuration.
|
||||
package config
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
// apiKeyEnvVar names the environment variable that overrides Config.APIKey.
|
||||
//
|
||||
//nolint:gosec // this is an env var *name*, not a credential value
|
||||
const apiKeyEnvVar = "LLMTB_API_KEY" // #nosec G101 -- this is an env var *name*, not a credential value
|
||||
|
||||
// Config is the top-level configuration loaded from config.yaml.
|
||||
type Config struct {
|
||||
Endpoint string `yaml:"endpoint"`
|
||||
APIKey string `yaml:"api_key"`
|
||||
Models []string `yaml:"models"`
|
||||
Concurrency int `yaml:"concurrency"`
|
||||
RequestTimeout time.Duration `yaml:"request_timeout"`
|
||||
MaxTokensDefault int `yaml:"max_tokens_default"`
|
||||
}
|
||||
|
||||
// rawConfig mirrors Config but with RequestTimeout as a string, since
|
||||
// yaml.v3 does not natively decode Go duration strings.
|
||||
type rawConfig struct {
|
||||
Endpoint string `yaml:"endpoint"`
|
||||
APIKey string `yaml:"api_key"`
|
||||
RequestTimeout string `yaml:"request_timeout"`
|
||||
Models []string `yaml:"models"`
|
||||
Concurrency int `yaml:"concurrency"`
|
||||
MaxTokensDefault int `yaml:"max_tokens_default"`
|
||||
}
|
||||
|
||||
// Load reads and parses the YAML config at path, applies the LLMTB_API_KEY
|
||||
// environment override (which always wins over the file value), and
|
||||
// validates the result.
|
||||
func Load(path string) (Config, error) {
|
||||
// #nosec G304 -- path is the operator's own --config flag value (a
|
||||
// local CLI tool reading its own config file), not externally supplied
|
||||
// or network input.
|
||||
data, err := os.ReadFile(path) //nolint:gosec // path is the operator's own --config flag value, not external input
|
||||
if err != nil {
|
||||
return Config{}, fmt.Errorf("config: read %s: %w", path, err)
|
||||
}
|
||||
return parse(data)
|
||||
}
|
||||
|
||||
// parse decodes YAML bytes into a validated Config. Split out from Load for
|
||||
// direct, file-free unit testing.
|
||||
func parse(data []byte) (Config, error) {
|
||||
var raw rawConfig
|
||||
if err := yaml.Unmarshal(data, &raw); err != nil {
|
||||
return Config{}, fmt.Errorf("config: parse yaml: %w", err)
|
||||
}
|
||||
|
||||
cfg := Config{
|
||||
Endpoint: raw.Endpoint,
|
||||
APIKey: raw.APIKey,
|
||||
Models: raw.Models,
|
||||
Concurrency: raw.Concurrency,
|
||||
MaxTokensDefault: raw.MaxTokensDefault,
|
||||
}
|
||||
|
||||
if raw.RequestTimeout != "" {
|
||||
d, err := time.ParseDuration(raw.RequestTimeout)
|
||||
if err != nil {
|
||||
return Config{}, fmt.Errorf("config: parse request_timeout %q: %w", raw.RequestTimeout, err)
|
||||
}
|
||||
cfg.RequestTimeout = d
|
||||
}
|
||||
|
||||
if envKey := os.Getenv(apiKeyEnvVar); envKey != "" {
|
||||
cfg.APIKey = envKey
|
||||
}
|
||||
|
||||
if err := cfg.Validate(); err != nil {
|
||||
return Config{}, err
|
||||
}
|
||||
return cfg, nil
|
||||
}
|
||||
|
||||
// Validate checks that required fields are present and sane.
|
||||
func (c Config) Validate() error {
|
||||
if c.Endpoint == "" {
|
||||
return fmt.Errorf("config: endpoint must not be empty")
|
||||
}
|
||||
if len(c.Models) == 0 {
|
||||
return fmt.Errorf("config: models must list at least one model")
|
||||
}
|
||||
for i, m := range c.Models {
|
||||
if m == "" {
|
||||
return fmt.Errorf("config: models[%d] must not be empty", i)
|
||||
}
|
||||
}
|
||||
if c.Concurrency <= 0 {
|
||||
return fmt.Errorf("config: concurrency must be > 0, got %d", c.Concurrency)
|
||||
}
|
||||
if c.RequestTimeout <= 0 {
|
||||
return fmt.Errorf("config: request_timeout must be > 0, got %s", c.RequestTimeout)
|
||||
}
|
||||
if c.MaxTokensDefault <= 0 {
|
||||
return fmt.Errorf("config: max_tokens_default must be > 0, got %d", c.MaxTokensDefault)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,155 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
const validYAML = `
|
||||
endpoint: https://llm-gateway.example.com/v1
|
||||
api_key: ""
|
||||
models:
|
||||
- uni/deepseek-v4-flash-0731
|
||||
- uni/qwen3.6-27b
|
||||
concurrency: 4
|
||||
request_timeout: 120s
|
||||
max_tokens_default: 4000
|
||||
`
|
||||
|
||||
func TestParse_Valid(t *testing.T) {
|
||||
cfg, err := parse([]byte(validYAML))
|
||||
if err != nil {
|
||||
t.Fatalf("parse() error = %v", err)
|
||||
}
|
||||
if cfg.Endpoint != "https://llm-gateway.example.com/v1" {
|
||||
t.Errorf("Endpoint = %q", cfg.Endpoint)
|
||||
}
|
||||
if len(cfg.Models) != 2 {
|
||||
t.Errorf("Models = %v, want 2 entries", cfg.Models)
|
||||
}
|
||||
if cfg.Concurrency != 4 {
|
||||
t.Errorf("Concurrency = %d, want 4", cfg.Concurrency)
|
||||
}
|
||||
if cfg.RequestTimeout != 120*time.Second {
|
||||
t.Errorf("RequestTimeout = %v, want 120s", cfg.RequestTimeout)
|
||||
}
|
||||
if cfg.MaxTokensDefault != 4000 {
|
||||
t.Errorf("MaxTokensDefault = %d, want 4000", cfg.MaxTokensDefault)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParse_EnvOverridesAPIKey(t *testing.T) {
|
||||
t.Setenv(apiKeyEnvVar, "env-secret")
|
||||
cfg, err := parse([]byte(validYAML))
|
||||
if err != nil {
|
||||
t.Fatalf("parse() error = %v", err)
|
||||
}
|
||||
if cfg.APIKey != "env-secret" {
|
||||
t.Errorf("APIKey = %q, want env override", cfg.APIKey)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParse_FileAPIKeyUsedWhenNoEnv(t *testing.T) {
|
||||
// S10: guarantee no env var leakage from the operator's real shell -
|
||||
// without this, a machine with LLMTB_API_KEY already set would make
|
||||
// this test assert the wrong thing regardless of parse()'s behavior.
|
||||
t.Setenv(apiKeyEnvVar, "")
|
||||
|
||||
yaml := `
|
||||
endpoint: https://llm-gateway.example.com/v1
|
||||
api_key: "file-key"
|
||||
models:
|
||||
- uni/deepseek-v4-flash-0731
|
||||
concurrency: 4
|
||||
request_timeout: 120s
|
||||
max_tokens_default: 4000
|
||||
`
|
||||
cfg, err := parse([]byte(yaml))
|
||||
if err != nil {
|
||||
t.Fatalf("parse() error = %v", err)
|
||||
}
|
||||
if cfg.APIKey != "file-key" {
|
||||
t.Errorf("APIKey = %q, want file-key", cfg.APIKey)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParse_InvalidYAML(t *testing.T) {
|
||||
_, err := parse([]byte("not: [valid: yaml"))
|
||||
if err == nil {
|
||||
t.Fatal("parse() error = nil, want error for invalid YAML")
|
||||
}
|
||||
}
|
||||
|
||||
func TestParse_InvalidDuration(t *testing.T) {
|
||||
yaml := `
|
||||
endpoint: https://x
|
||||
models: [m]
|
||||
concurrency: 1
|
||||
request_timeout: not-a-duration
|
||||
max_tokens_default: 100
|
||||
`
|
||||
_, err := parse([]byte(yaml))
|
||||
if err == nil {
|
||||
t.Fatal("parse() error = nil, want error for invalid duration")
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidate(t *testing.T) {
|
||||
base := Config{
|
||||
Endpoint: "https://x",
|
||||
Models: []string{"m"},
|
||||
Concurrency: 1,
|
||||
RequestTimeout: time.Second,
|
||||
MaxTokensDefault: 100,
|
||||
}
|
||||
|
||||
if err := base.Validate(); err != nil {
|
||||
t.Fatalf("Validate() on valid config error = %v", err)
|
||||
}
|
||||
|
||||
tests := []struct {
|
||||
mutate func(c Config) Config
|
||||
name string
|
||||
}{
|
||||
{name: "empty endpoint", mutate: func(c Config) Config { c.Endpoint = ""; return c }},
|
||||
{name: "no models", mutate: func(c Config) Config { c.Models = nil; return c }},
|
||||
{name: "empty model entry", mutate: func(c Config) Config { c.Models = []string{""}; return c }},
|
||||
{name: "zero concurrency", mutate: func(c Config) Config { c.Concurrency = 0; return c }},
|
||||
{name: "negative concurrency", mutate: func(c Config) Config { c.Concurrency = -1; return c }},
|
||||
{name: "zero timeout", mutate: func(c Config) Config { c.RequestTimeout = 0; return c }},
|
||||
{name: "zero max tokens", mutate: func(c Config) Config { c.MaxTokensDefault = 0; return c }},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
c := tt.mutate(base)
|
||||
if err := c.Validate(); err == nil {
|
||||
t.Errorf("Validate() on %s: error = nil, want error", tt.name)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoad_ReadsFile(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, "config.yaml")
|
||||
if err := os.WriteFile(path, []byte(validYAML), 0o600); err != nil {
|
||||
t.Fatalf("os.WriteFile() error = %v", err)
|
||||
}
|
||||
|
||||
cfg, err := Load(path)
|
||||
if err != nil {
|
||||
t.Fatalf("Load() error = %v", err)
|
||||
}
|
||||
if cfg.Endpoint == "" {
|
||||
t.Error("Load() returned empty Endpoint")
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoad_MissingFile(t *testing.T) {
|
||||
_, err := Load(filepath.Join(t.TempDir(), "does-not-exist.yaml"))
|
||||
if err == nil {
|
||||
t.Fatal("Load() error = nil, want error for missing file")
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user