llm-testbench/cmd/llmtest/progress_test.go

package main

import (
	"bytes"
	"errors"
	"regexp"
	"strings"
	"sync"
	"testing"
	"time"

	"github.com/lukaszraczylo/llm-testbench/internal/eval"
	"github.com/lukaszraczylo/llm-testbench/internal/runner"
)

func TestStderrProgressReporter_ReportStart(t *testing.T) {
	var buf bytes.Buffer
	r := newStderrProgressReporter(&buf)

	r.ReportStart(113, 3, 8)

	got := buf.String()
	want := "running 113 tests x 3 models = 339 requests, concurrency 8\n"
	if got != want {
		t.Errorf("ReportStart() output = %q, want %q", got, want)
	}
}

func TestStderrProgressReporter_ReportDone(t *testing.T) {
	tests := []struct {
		name    string
		wantSub []string
		result  runner.Result
	}{
		{
			name:    "scored result",
			result:  runner.Result{Model: "m1", TestID: "t1", Score: eval.Score{Value: 0.85}, Latency: 2 * time.Second},
			wantSub: []string{"model=m1", "test=t1", "score=0.85", "(2s)"},
		},
		{
			name:    "error result",
			result:  runner.Result{Model: "m1", TestID: "t1", Err: errors.New("boom")},
			wantSub: []string{"score=ERR"},
		},
		{
			name:    "skipped result",
			result:  runner.Result{Model: "m1", TestID: "t1", Score: eval.Score{Skipped: true}},
			wantSub: []string{"score=skip"},
		},
		{
			name:    "truncated result carries the TRUNCATED marker",
			result:  runner.Result{Model: "m1", TestID: "t1", Score: eval.Score{Value: 0}, FinishReason: "length"},
			wantSub: []string{"score=0.00", "TRUNCATED"},
		},
		{
			name:    "non-truncated result has no marker",
			result:  runner.Result{Model: "m1", TestID: "t1", Score: eval.Score{Value: 1}, FinishReason: "stop"},
			wantSub: []string{"score=1.00"},
		},
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			var buf bytes.Buffer
			r := newStderrProgressReporter(&buf)
			r.ReportDone(3, 10, tt.result)

			got := buf.String()
			if !strings.HasPrefix(got, "[3/10] ") {
				t.Errorf("ReportDone() output = %q, want prefix [3/10] ", got)
			}
			for _, want := range tt.wantSub {
				if !strings.Contains(got, want) {
					t.Errorf("ReportDone() output = %q, want substring %q", got, want)
				}
			}
			if tt.name == "non-truncated result has no marker" && strings.Contains(got, "TRUNCATED") {
				t.Errorf("ReportDone() output = %q, must not contain TRUNCATED", got)
			}
		})
	}
}

// TestStderrProgressReporter_ReportDone_ConcurrencySafe writes many
// concurrent ReportDone calls (as Runner.Run's goroutines would) and
// verifies every line is well-formed and none interleaved mid-line - the
// reporter's own mutex must serialize writes, not just the counter.
func TestStderrProgressReporter_ReportDone_ConcurrencySafe(t *testing.T) {
	var buf bytes.Buffer
	r := newStderrProgressReporter(&buf)

	const n = 50
	var wg sync.WaitGroup
	for i := 1; i <= n; i++ {
		wg.Add(1)
		go func(i int) {
			defer wg.Done()
			r.ReportDone(i, n, runner.Result{Model: "m", TestID: "t", Score: eval.Score{Value: 1}})
		}(i)
	}
	wg.Wait()

	lines := strings.Split(strings.TrimRight(buf.String(), "\n"), "\n")
	if len(lines) != n {
		t.Fatalf("got %d lines, want %d (a corrupted/interleaved write would produce a different count)", len(lines), n)
	}

	linePattern := regexp.MustCompile(`^\[\d+/\d+\] model=m test=t score=1\.00 tokens=0 \(0s\)$`)
	for _, line := range lines {
		if !linePattern.MatchString(line) {
			t.Errorf("malformed or interleaved line: %q", line)
		}
	}
}