mirror of
https://github.com/lukaszraczylo/llm-testbench.git
synced 2026-10-12 00:45:19 +00:00
fix: floor token budgets and surface truncation, add live progress
Reasoning models burn thousands of completion tokens before the answer. Small per-test caps caused silent truncation and zero scores. The runner now floors max_tokens at the config default (12000). The client parses finish_reason. Reports mark truncated cells and the JSON format carries response text for debugging. A stderr progress reporter prints one line per completed request; --quiet disables it.
This commit is contained in:
@@ -14,6 +14,18 @@ import (
|
||||
//nolint:gosec // this is an env var *name*, not a credential value
|
||||
const apiKeyEnvVar = "LLMTB_API_KEY" // #nosec G101 -- this is an env var *name*, not a credential value
|
||||
|
||||
// Defaults applied when the corresponding YAML key is omitted (or explicit
|
||||
// zero). max_tokens_default was raised from 4000 to 12000 after a live run
|
||||
// showed reasoning models spend thousands of completion tokens on
|
||||
// reasoning_content before writing an answer to content: a small
|
||||
// per-test/default budget truncates the answer (finish_reason=length)
|
||||
// before it is ever written, scoring a correct model as 0.
|
||||
const (
|
||||
defaultConcurrency = 8
|
||||
defaultRequestTimeout = 300 * time.Second
|
||||
defaultMaxTokens = 12000
|
||||
)
|
||||
|
||||
// Config is the top-level configuration loaded from config.yaml.
|
||||
type Config struct {
|
||||
Endpoint string `yaml:"endpoint"`
|
||||
@@ -73,6 +85,8 @@ func parse(data []byte) (Config, error) {
|
||||
cfg.RequestTimeout = d
|
||||
}
|
||||
|
||||
applyDefaults(&cfg)
|
||||
|
||||
if envKey := os.Getenv(apiKeyEnvVar); envKey != "" {
|
||||
cfg.APIKey = envKey
|
||||
}
|
||||
@@ -83,6 +97,21 @@ func parse(data []byte) (Config, error) {
|
||||
return cfg, nil
|
||||
}
|
||||
|
||||
// applyDefaults fills in Concurrency, RequestTimeout, and MaxTokensDefault
|
||||
// when the YAML omitted them (or gave an explicit zero), so a minimal
|
||||
// config.yaml still runs with sane, generous-enough values.
|
||||
func applyDefaults(cfg *Config) {
|
||||
if cfg.Concurrency <= 0 {
|
||||
cfg.Concurrency = defaultConcurrency
|
||||
}
|
||||
if cfg.RequestTimeout <= 0 {
|
||||
cfg.RequestTimeout = defaultRequestTimeout
|
||||
}
|
||||
if cfg.MaxTokensDefault <= 0 {
|
||||
cfg.MaxTokensDefault = defaultMaxTokens
|
||||
}
|
||||
}
|
||||
|
||||
// Validate checks that required fields are present and sane.
|
||||
func (c Config) Validate() error {
|
||||
if c.Endpoint == "" {
|
||||
|
||||
@@ -75,6 +75,47 @@ max_tokens_default: 4000
|
||||
}
|
||||
}
|
||||
|
||||
func TestParse_DefaultsAppliedWhenOmitted(t *testing.T) {
|
||||
yaml := `
|
||||
endpoint: https://llm-gateway.example.com/v1
|
||||
models:
|
||||
- uni/deepseek-v4-flash-0731
|
||||
`
|
||||
cfg, err := parse([]byte(yaml))
|
||||
if err != nil {
|
||||
t.Fatalf("parse() error = %v", err)
|
||||
}
|
||||
if cfg.Concurrency != defaultConcurrency {
|
||||
t.Errorf("Concurrency = %d, want default %d", cfg.Concurrency, defaultConcurrency)
|
||||
}
|
||||
if cfg.RequestTimeout != defaultRequestTimeout {
|
||||
t.Errorf("RequestTimeout = %v, want default %v", cfg.RequestTimeout, defaultRequestTimeout)
|
||||
}
|
||||
if cfg.MaxTokensDefault != defaultMaxTokens {
|
||||
t.Errorf("MaxTokensDefault = %d, want default %d", cfg.MaxTokensDefault, defaultMaxTokens)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParse_DefaultsDoNotOverrideExplicitValues(t *testing.T) {
|
||||
cfg, err := parse([]byte(validYAML))
|
||||
if err != nil {
|
||||
t.Fatalf("parse() error = %v", err)
|
||||
}
|
||||
// validYAML sets concurrency=4, request_timeout=120s,
|
||||
// max_tokens_default=4000, all below the new defaults; an explicit
|
||||
// value, even a small one, must still win over the default floor
|
||||
// applied by applyDefaults.
|
||||
if cfg.Concurrency != 4 {
|
||||
t.Errorf("Concurrency = %d, want the explicit 4, not the default", cfg.Concurrency)
|
||||
}
|
||||
if cfg.RequestTimeout != 120*time.Second {
|
||||
t.Errorf("RequestTimeout = %v, want the explicit 120s, not the default", cfg.RequestTimeout)
|
||||
}
|
||||
if cfg.MaxTokensDefault != 4000 {
|
||||
t.Errorf("MaxTokensDefault = %d, want the explicit 4000, not the default", cfg.MaxTokensDefault)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParse_InvalidYAML(t *testing.T) {
|
||||
_, err := parse([]byte("not: [valid: yaml"))
|
||||
if err == nil {
|
||||
|
||||
Reference in New Issue
Block a user