fix: floor token budgets and surface truncation, add live progress

Reasoning models burn thousands of completion tokens before the
answer. Small per-test caps caused silent truncation and zero scores.
The runner now floors max_tokens at the config default (12000). The
client parses finish_reason. Reports mark truncated cells and the
JSON format carries response text for debugging. A stderr progress
reporter prints one line per completed request; --quiet disables it.
This commit is contained in:
2026-08-17 01:55:14 +01:00
parent 764fe990fd
commit ace50757e4
30 changed files with 1063 additions and 184 deletions
+8 -28
View File
@@ -9,13 +9,11 @@ import (
"os"
"sort"
"strings"
"time"
"github.com/lukaszraczylo/llm-testbench/internal/config"
"github.com/lukaszraczylo/llm-testbench/internal/llm"
"github.com/lukaszraczylo/llm-testbench/internal/report"
"github.com/lukaszraczylo/llm-testbench/internal/runner"
"github.com/lukaszraczylo/llm-testbench/internal/testkit"
"github.com/lukaszraczylo/llm-testbench/internal/tests"
)
@@ -100,7 +98,7 @@ func runCommand(args []string) error {
format := fs.String("format", "table", "output format: table|markdown|json")
concurrency := fs.Int("concurrency", 0, "override config's concurrency (0 = use config)")
timeout := fs.Duration("timeout", 0, "override config's request_timeout (0 = use config)")
verbose := fs.Bool("verbose", false, "print progress to stderr as tests complete")
quiet := fs.Bool("quiet", false, "suppress progress output on stderr")
if err := fs.Parse(args); err != nil {
return err
}
@@ -133,14 +131,20 @@ func runCommand(args []string) error {
return fmt.Errorf("no tests matched category=%q subcategory=%q", shared.category, shared.subcategory)
}
var reporter runner.ProgressReporter = runner.NoopProgressReporter{}
if !*quiet {
reporter = newStderrProgressReporter(os.Stderr)
}
r := runner.New(client, runner.Config{
Concurrency: conc,
Temperature: requestTemperature,
MaxTokensDefault: cfg.MaxTokensDefault,
Reporter: reporter,
})
ctx := context.Background()
results := runWithProgress(ctx, r, models, selected, *verbose)
results := r.Run(ctx, models, selected)
f, err := validateFormat(*format)
if err != nil {
@@ -163,30 +167,6 @@ func validateFormat(format string) (report.Format, error) {
}
}
// runWithProgress runs the given tests/models and, when verbose, prints one
// progress line per completed (model, test) pair to stderr.
func runWithProgress(ctx context.Context, r *runner.Runner, models []string, selected []testkit.Test, verbose bool) []runner.Result {
if !verbose {
return r.Run(ctx, models, selected)
}
fmt.Fprintf(os.Stderr, "running %d tests x %d models = %d calls...\n", len(selected), len(models), len(selected)*len(models))
start := time.Now()
results := r.Run(ctx, models, selected)
for _, res := range results {
status := fmt.Sprintf("%.2f", res.Score.Value)
switch {
case res.Err != nil:
status = "ERR: " + res.Err.Error()
case res.Score.Skipped:
status = "skip: " + res.Score.Detail
}
fmt.Fprintf(os.Stderr, " %-14s %-28s %s (%s)\n", res.Model, res.TestID, status, res.Latency)
}
fmt.Fprintf(os.Stderr, "done in %s\n", time.Since(start))
return results
}
func splitCSV(s string) []string {
parts := strings.Split(s, ",")
out := make([]string, 0, len(parts))