llm-testbench/cmd/llmtest/progress_test.go

package main

import (
	"bytes"
	"regexp"
	"strings"
	"sync"
	"testing"

	"github.com/lukaszraczylo/llm-testbench/internal/eval"
	"github.com/lukaszraczylo/llm-testbench/internal/runner"
)

func TestStderrProgressReporter_ReportStart(t *testing.T) {
	var buf bytes.Buffer
	r := newStderrProgressReporter(&buf)

	r.ReportStart(113, 3, 8)

	got := buf.String()
	want := "running 113 tests x 3 models = 339 requests, concurrency 8\n"
	if got != want {
		t.Errorf("ReportStart() output = %q, want %q", got, want)
	}
}

// A bytes.Buffer is not a character device, so newStderrProgressReporter
// selects piped mode: a plain line every pipedProgressEvery completions
// plus the final one, nothing else.
func TestStderrProgressReporter_ReportDone_Piped(t *testing.T) {
	tests := []struct {
		name  string
		want  string
		done  int
		total int
	}{
		{name: "intermediate completion stays silent", done: 3, total: 10, want: ""},
		{name: "final completion always prints", done: 10, total: 10, want: "test 10 of 10\n"},
		{name: "every pipedProgressEvery-th completion prints", done: pipedProgressEvery, total: 100, want: "test 25 of 100\n"},
		{name: "multiple of pipedProgressEvery prints", done: 2 * pipedProgressEvery, total: 100, want: "test 50 of 100\n"},
		{name: "off-interval completion stays silent", done: 2*pipedProgressEvery + 1, total: 100, want: ""},
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			var buf bytes.Buffer
			r := newStderrProgressReporter(&buf)
			r.ReportDone(tt.done, tt.total, runner.Result{Model: "m", TestID: "t", Score: eval.Score{Value: 1}})

			if got := buf.String(); got != tt.want {
				t.Errorf("ReportDone(%d, %d) output = %q, want %q", tt.done, tt.total, got, tt.want)
			}
		})
	}
}

// TTY mode rewrites one counter line in place; the test forces isTTY since
// a bytes.Buffer can never be a terminal.
func TestStderrProgressReporter_ReportDone_TTY(t *testing.T) {
	tests := []struct {
		name  string
		want  string
		done  int
		total int
	}{
		{name: "intermediate rewrites in place without a newline", done: 3, total: 10, want: "\rtest 3 of 10\x1b[K"},
		{name: "final completion ends the line", done: 10, total: 10, want: "\rtest 10 of 10\x1b[K\n"},
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			var buf bytes.Buffer
			r := newStderrProgressReporter(&buf)
			r.isTTY = true
			r.ReportDone(tt.done, tt.total, runner.Result{Model: "m", TestID: "t", Score: eval.Score{Value: 1}})

			if got := buf.String(); got != tt.want {
				t.Errorf("ReportDone(%d, %d) output = %q, want %q", tt.done, tt.total, got, tt.want)
			}
		})
	}
}

// TestStderrProgressReporter_ReportDone_ConcurrencySafe writes many
// concurrent ReportDone calls (as Runner.Run's goroutines would) and
// verifies no write interleaved mid-line - the reporter's own mutex must
// serialize writes, not just the counter.
func TestStderrProgressReporter_ReportDone_ConcurrencySafe(t *testing.T) {
	var buf bytes.Buffer
	r := newStderrProgressReporter(&buf)

	const n = 2 * pipedProgressEvery
	var wg sync.WaitGroup
	for i := 1; i <= n; i++ {
		wg.Add(1)
		go func(i int) {
			defer wg.Done()
			r.ReportDone(i, n, runner.Result{Model: "m", TestID: "t", Score: eval.Score{Value: 1}})
		}(i)
	}
	wg.Wait()

	// In piped mode exactly the 25th and 50th completions print.
	lines := strings.Split(strings.TrimRight(buf.String(), "\n"), "\n")
	if len(lines) != 2 {
		t.Fatalf("got %d lines (%q), want 2 (a corrupted/interleaved write would change the count)", len(lines), buf.String())
	}

	linePattern := regexp.MustCompile(`^test \d+ of \d+$`)
	for _, line := range lines {
		if !linePattern.MatchString(line) {
			t.Errorf("malformed or interleaved line: %q", line)
		}
	}
}