fix: floor token budgets and surface truncation, add live progress

Reasoning models burn thousands of completion tokens before the
answer. Small per-test caps caused silent truncation and zero scores.
The runner now floors max_tokens at the config default (12000). The
client parses finish_reason. Reports mark truncated cells and the
JSON format carries response text for debugging. A stderr progress
reporter prints one line per completed request; --quiet disables it.

cmd/llmtest/main.go

@@ -9,13 +9,11 @@ import (
 	"os"
 	"sort"
 	"strings"
-	"time"
 
 	"github.com/lukaszraczylo/llm-testbench/internal/config"
 	"github.com/lukaszraczylo/llm-testbench/internal/llm"
 	"github.com/lukaszraczylo/llm-testbench/internal/report"
 	"github.com/lukaszraczylo/llm-testbench/internal/runner"
-	"github.com/lukaszraczylo/llm-testbench/internal/testkit"
 	"github.com/lukaszraczylo/llm-testbench/internal/tests"
 )
 
@@ -100,7 +98,7 @@ func runCommand(args []string) error {
 	format := fs.String("format", "table", "output format: table|markdown|json")
 	concurrency := fs.Int("concurrency", 0, "override config's concurrency (0 = use config)")
 	timeout := fs.Duration("timeout", 0, "override config's request_timeout (0 = use config)")
-	verbose := fs.Bool("verbose", false, "print progress to stderr as tests complete")
+	quiet := fs.Bool("quiet", false, "suppress progress output on stderr")
 	if err := fs.Parse(args); err != nil {
 		return err
 	}
@@ -133,14 +131,20 @@ func runCommand(args []string) error {
 		return fmt.Errorf("no tests matched category=%q subcategory=%q", shared.category, shared.subcategory)
 	}
 
+	var reporter runner.ProgressReporter = runner.NoopProgressReporter{}
+	if !*quiet {
+		reporter = newStderrProgressReporter(os.Stderr)
+	}
+
 	r := runner.New(client, runner.Config{
 		Concurrency:      conc,
 		Temperature:      requestTemperature,
 		MaxTokensDefault: cfg.MaxTokensDefault,
+		Reporter:         reporter,
 	})
 
 	ctx := context.Background()
-	results := runWithProgress(ctx, r, models, selected, *verbose)
+	results := r.Run(ctx, models, selected)
 
 	f, err := validateFormat(*format)
 	if err != nil {
@@ -163,30 +167,6 @@ func validateFormat(format string) (report.Format, error) {
 	}
 }
 
-// runWithProgress runs the given tests/models and, when verbose, prints one
-// progress line per completed (model, test) pair to stderr.
-func runWithProgress(ctx context.Context, r *runner.Runner, models []string, selected []testkit.Test, verbose bool) []runner.Result {
-	if !verbose {
-		return r.Run(ctx, models, selected)
-	}
-
-	fmt.Fprintf(os.Stderr, "running %d tests x %d models = %d calls...\n", len(selected), len(models), len(selected)*len(models))
-	start := time.Now()
-	results := r.Run(ctx, models, selected)
-	for _, res := range results {
-		status := fmt.Sprintf("%.2f", res.Score.Value)
-		switch {
-		case res.Err != nil:
-			status = "ERR: " + res.Err.Error()
-		case res.Score.Skipped:
-			status = "skip: " + res.Score.Detail
-		}
-		fmt.Fprintf(os.Stderr, "  %-14s %-28s %s (%s)\n", res.Model, res.TestID, status, res.Latency)
-	}
-	fmt.Fprintf(os.Stderr, "done in %s\n", time.Since(start))
-	return results
-}
-
 func splitCSV(s string) []string {
 	parts := strings.Split(s, ",")
 	out := make([]string, 0, len(parts))

cmd/llmtest/progress.go

@@ -0,0 +1,63 @@
+package main
+
+import (
+	"fmt"
+	"io"
+	"sync"
+
+	"github.com/lukaszraczylo/llm-testbench/internal/runner"
+)
+
+// stderrProgressReporter implements runner.ProgressReporter, writing one
+// line per event to w (os.Stderr in production). Run's stdout stays
+// pipeable (table/markdown/json only): all progress goes here instead.
+type stderrProgressReporter struct {
+	w  io.Writer
+	mu sync.Mutex
+}
+
+// newStderrProgressReporter builds a stderrProgressReporter writing to w.
+func newStderrProgressReporter(w io.Writer) *stderrProgressReporter {
+	return &stderrProgressReporter{w: w}
+}
+
+// ReportStart implements runner.ProgressReporter.
+func (r *stderrProgressReporter) ReportStart(totalTests, totalModels, concurrency int) {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	// A write failure on this side-channel progress stream (a closed pipe,
+	// a full/broken stderr) is not actionable and must not abort the run
+	// that produces the actual report on stdout.
+	_, _ = fmt.Fprintf(r.w, "running %d tests x %d models = %d requests, concurrency %d\n",
+		totalTests, totalModels, totalTests*totalModels, concurrency)
+}
+
+// ReportDone implements runner.ProgressReporter. Safe for concurrent calls
+// from up to Config.Concurrency goroutines at once: the mutex serializes
+// each printed line so concurrent completions cannot interleave mid-line.
+func (r *stderrProgressReporter) ReportDone(done, total int, result runner.Result) {
+	status := resultStatus(result)
+	truncated := ""
+	if result.Truncated() {
+		truncated = " TRUNCATED"
+	}
+
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	// See ReportStart: a write failure here is not actionable.
+	_, _ = fmt.Fprintf(r.w, "[%d/%d] model=%s test=%s score=%s tokens=%d (%s)%s\n",
+		done, total, result.Model, result.TestID, status, result.TotalTokens(), result.Latency, truncated)
+}
+
+// resultStatus renders a Result's score for one progress line: the numeric
+// score, or ERR/skip.
+func resultStatus(result runner.Result) string {
+	switch {
+	case result.Err != nil:
+		return "ERR"
+	case result.Score.Skipped:
+		return "skip"
+	default:
+		return fmt.Sprintf("%.2f", result.Score.Value)
+	}
+}

cmd/llmtest/progress_test.go

@@ -0,0 +1,113 @@
+package main
+
+import (
+	"bytes"
+	"errors"
+	"regexp"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/lukaszraczylo/llm-testbench/internal/eval"
+	"github.com/lukaszraczylo/llm-testbench/internal/runner"
+)
+
+func TestStderrProgressReporter_ReportStart(t *testing.T) {
+	var buf bytes.Buffer
+	r := newStderrProgressReporter(&buf)
+
+	r.ReportStart(113, 3, 8)
+
+	got := buf.String()
+	want := "running 113 tests x 3 models = 339 requests, concurrency 8\n"
+	if got != want {
+		t.Errorf("ReportStart() output = %q, want %q", got, want)
+	}
+}
+
+func TestStderrProgressReporter_ReportDone(t *testing.T) {
+	tests := []struct {
+		name    string
+		wantSub []string
+		result  runner.Result
+	}{
+		{
+			name:    "scored result",
+			result:  runner.Result{Model: "m1", TestID: "t1", Score: eval.Score{Value: 0.85}, Latency: 2 * time.Second},
+			wantSub: []string{"model=m1", "test=t1", "score=0.85", "(2s)"},
+		},
+		{
+			name:    "error result",
+			result:  runner.Result{Model: "m1", TestID: "t1", Err: errors.New("boom")},
+			wantSub: []string{"score=ERR"},
+		},
+		{
+			name:    "skipped result",
+			result:  runner.Result{Model: "m1", TestID: "t1", Score: eval.Score{Skipped: true}},
+			wantSub: []string{"score=skip"},
+		},
+		{
+			name:    "truncated result carries the TRUNCATED marker",
+			result:  runner.Result{Model: "m1", TestID: "t1", Score: eval.Score{Value: 0}, FinishReason: "length"},
+			wantSub: []string{"score=0.00", "TRUNCATED"},
+		},
+		{
+			name:    "non-truncated result has no marker",
+			result:  runner.Result{Model: "m1", TestID: "t1", Score: eval.Score{Value: 1}, FinishReason: "stop"},
+			wantSub: []string{"score=1.00"},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			var buf bytes.Buffer
+			r := newStderrProgressReporter(&buf)
+			r.ReportDone(3, 10, tt.result)
+
+			got := buf.String()
+			if !strings.HasPrefix(got, "[3/10] ") {
+				t.Errorf("ReportDone() output = %q, want prefix [3/10] ", got)
+			}
+			for _, want := range tt.wantSub {
+				if !strings.Contains(got, want) {
+					t.Errorf("ReportDone() output = %q, want substring %q", got, want)
+				}
+			}
+			if tt.name == "non-truncated result has no marker" && strings.Contains(got, "TRUNCATED") {
+				t.Errorf("ReportDone() output = %q, must not contain TRUNCATED", got)
+			}
+		})
+	}
+}
+
+// TestStderrProgressReporter_ReportDone_ConcurrencySafe writes many
+// concurrent ReportDone calls (as Runner.Run's goroutines would) and
+// verifies every line is well-formed and none interleaved mid-line - the
+// reporter's own mutex must serialize writes, not just the counter.
+func TestStderrProgressReporter_ReportDone_ConcurrencySafe(t *testing.T) {
+	var buf bytes.Buffer
+	r := newStderrProgressReporter(&buf)
+
+	const n = 50
+	var wg sync.WaitGroup
+	for i := 1; i <= n; i++ {
+		wg.Add(1)
+		go func(i int) {
+			defer wg.Done()
+			r.ReportDone(i, n, runner.Result{Model: "m", TestID: "t", Score: eval.Score{Value: 1}})
+		}(i)
+	}
+	wg.Wait()
+
+	lines := strings.Split(strings.TrimRight(buf.String(), "\n"), "\n")
+	if len(lines) != n {
+		t.Fatalf("got %d lines, want %d (a corrupted/interleaved write would produce a different count)", len(lines), n)
+	}
+
+	linePattern := regexp.MustCompile(`^\[\d+/\d+\] model=m test=t score=1\.00 tokens=0 \(0s\)$`)
+	for _, line := range lines {
+		if !linePattern.MatchString(line) {
+			t.Errorf("malformed or interleaved line: %q", line)
+		}
+	}
+}