mirror of
https://github.com/lukaszraczylo/llm-testbench.git
synced 2026-10-12 00:45:19 +00:00
fix: floor token budgets and surface truncation, add live progress
Reasoning models burn thousands of completion tokens before the answer. Small per-test caps caused silent truncation and zero scores. The runner now floors max_tokens at the config default (12000). The client parses finish_reason. Reports mark truncated cells and the JSON format carries response text for debugging. A stderr progress reporter prints one line per completed request; --quiet disables it.
This commit is contained in:
@@ -0,0 +1,31 @@
|
||||
package runner
|
||||
|
||||
// ProgressReporter receives progress events as Run executes, so a caller
|
||||
// (typically the CLI) can show the operator something is happening across
|
||||
// a run that may issue hundreds of requests at 30-50s each. Implementations
|
||||
// must be safe for concurrent use: ReportDone is called from up to
|
||||
// Config.Concurrency goroutines at once.
|
||||
type ProgressReporter interface {
|
||||
// ReportStart is called once, before any request is issued.
|
||||
// totalTests and totalModels are the catalog and model-list sizes
|
||||
// being fanned out (totalTests*totalModels is the total request
|
||||
// count); concurrency is the bound Run is applying.
|
||||
ReportStart(totalTests, totalModels, concurrency int)
|
||||
|
||||
// ReportDone is called once per completed (model, test) call, after
|
||||
// Run has already recorded result into its output slice. done is this
|
||||
// call's 1-based completion index; total is the overall request
|
||||
// count (totalTests*totalModels from ReportStart).
|
||||
ReportDone(done, total int, result Result)
|
||||
}
|
||||
|
||||
// NoopProgressReporter discards every event. It is the default Runner uses
|
||||
// when Config.Reporter is nil (unit tests, and any caller that does not
|
||||
// want progress output, e.g. the CLI's --quiet flag).
|
||||
type NoopProgressReporter struct{}
|
||||
|
||||
// ReportStart implements ProgressReporter.
|
||||
func (NoopProgressReporter) ReportStart(_, _, _ int) {}
|
||||
|
||||
// ReportDone implements ProgressReporter.
|
||||
func (NoopProgressReporter) ReportDone(_, _ int, _ Result) {}
|
||||
+48
-11
@@ -4,6 +4,7 @@ package runner
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"golang.org/x/sync/errgroup"
|
||||
@@ -13,17 +14,34 @@ import (
|
||||
"github.com/lukaszraczylo/llm-testbench/internal/testkit"
|
||||
)
|
||||
|
||||
// truncatedDetailPrefix is prepended to Score.Detail when the model's
|
||||
// response was cut off by the token budget (llm.FinishReasonLength), so
|
||||
// the reason a test scored the way it did is visible without cross-
|
||||
// referencing FinishReason separately.
|
||||
const truncatedDetailPrefix = "TRUNCATED: "
|
||||
|
||||
// Result is the outcome of running one test against one model.
|
||||
type Result struct {
|
||||
Err error
|
||||
Model string
|
||||
TestID string
|
||||
ResponseText string
|
||||
FinishReason string
|
||||
Score eval.Score
|
||||
Latency time.Duration
|
||||
PromptTokens int
|
||||
CompletionTokens int
|
||||
}
|
||||
|
||||
// Truncated reports whether the model's response was cut off by the token
|
||||
// budget before it finished (finish_reason=length), rather than completing
|
||||
// normally. A truncated response's Score is still whatever the evaluator
|
||||
// computed on the partial text; Truncated is the signal that the score may
|
||||
// not reflect what the model would have said with a larger budget.
|
||||
func (r Result) Truncated() bool {
|
||||
return r.FinishReason == llm.FinishReasonLength
|
||||
}
|
||||
|
||||
// TotalTokens returns PromptTokens + CompletionTokens, the total token
|
||||
// usage billed for this (model, test) call.
|
||||
func (r Result) TotalTokens() int {
|
||||
@@ -32,12 +50,9 @@ func (r Result) TotalTokens() int {
|
||||
|
||||
// Config controls how the Runner fans work out.
|
||||
type Config struct {
|
||||
// Concurrency bounds the number of in-flight (model,test) calls.
|
||||
Concurrency int
|
||||
// Temperature is sent on every request; PLAN.md pins this to 0 for
|
||||
// determinism across runs.
|
||||
Temperature float64
|
||||
// MaxTokensDefault is used when a Test does not set its own MaxTokens.
|
||||
Reporter ProgressReporter
|
||||
Concurrency int
|
||||
Temperature float64
|
||||
MaxTokensDefault int
|
||||
}
|
||||
|
||||
@@ -49,6 +64,9 @@ type Runner struct {
|
||||
|
||||
// New builds a Runner that issues requests through client.
|
||||
func New(client llm.Client, cfg Config) *Runner {
|
||||
if cfg.Reporter == nil {
|
||||
cfg.Reporter = NoopProgressReporter{}
|
||||
}
|
||||
return &Runner{client: client, cfg: cfg}
|
||||
}
|
||||
|
||||
@@ -77,11 +95,21 @@ func (r *Runner) Run(ctx context.Context, models []string, tests []testkit.Test)
|
||||
}
|
||||
g.SetLimit(concurrency)
|
||||
|
||||
r.cfg.Reporter.ReportStart(len(tests), len(models), concurrency)
|
||||
|
||||
var completed int32
|
||||
// go.mod's go directive is 1.22+, so loop variables are already
|
||||
// per-iteration scoped; no manual i, j := i, j shadowing needed (N4).
|
||||
for i, j := range jobs {
|
||||
g.Go(func() error {
|
||||
results[i] = r.runOne(gctx, j.model, j.test)
|
||||
res := r.runOne(gctx, j.model, j.test)
|
||||
results[i] = res
|
||||
// atomic.AddInt32 makes the completion counter safe under
|
||||
// Config.Concurrency-many concurrent goroutines; ReportDone
|
||||
// itself must also be concurrency-safe per the ProgressReporter
|
||||
// contract.
|
||||
done := atomic.AddInt32(&completed, 1)
|
||||
r.cfg.Reporter.ReportDone(int(done), len(jobs), res)
|
||||
return nil
|
||||
})
|
||||
}
|
||||
@@ -95,10 +123,14 @@ func (r *Runner) Run(ctx context.Context, models []string, tests []testkit.Test)
|
||||
|
||||
// runOne executes a single (model, test) combination.
|
||||
func (r *Runner) runOne(ctx context.Context, model string, test testkit.Test) Result {
|
||||
maxTokens := test.MaxTokens
|
||||
if maxTokens <= 0 {
|
||||
maxTokens = r.cfg.MaxTokensDefault
|
||||
}
|
||||
// cfg.MaxTokensDefault is a floor, not a fallback used only when the
|
||||
// test leaves MaxTokens unset: a per-test value only ever raises the
|
||||
// budget, it never lowers it below the default. A reasoning model can
|
||||
// burn thousands of tokens on reasoning_content before ever writing an
|
||||
// answer to content, so a small per-test MaxTokens (sized for the
|
||||
// answer alone) previously starved the model into finish_reason=length
|
||||
// with an empty answer.
|
||||
maxTokens := max(test.MaxTokens, r.cfg.MaxTokensDefault)
|
||||
|
||||
messages := make([]llm.Message, 0, 2)
|
||||
if test.System != "" {
|
||||
@@ -120,11 +152,16 @@ func (r *Runner) runOne(ctx context.Context, model string, test testkit.Test) Re
|
||||
|
||||
normalized := testkit.Normalize(resp.Text)
|
||||
score := test.Eval.Evaluate(ctx, normalized)
|
||||
if resp.FinishReason == llm.FinishReasonLength {
|
||||
score.Detail = truncatedDetailPrefix + score.Detail
|
||||
}
|
||||
|
||||
return Result{
|
||||
Model: model,
|
||||
TestID: test.ID,
|
||||
Score: score,
|
||||
ResponseText: normalized,
|
||||
FinishReason: resp.FinishReason,
|
||||
Latency: resp.Latency,
|
||||
PromptTokens: resp.PromptTokens,
|
||||
CompletionTokens: resp.CompletionTokens,
|
||||
|
||||
@@ -3,6 +3,8 @@ package runner
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
@@ -180,6 +182,74 @@ func TestRunner_Run_FallsBackToDefaultMaxTokens(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestRunner_Run_DefaultIsAFloorNotJustAFallback is the regression test for
|
||||
// the systemic live-run bug: cfg.MaxTokensDefault must raise a too-small
|
||||
// per-test MaxTokens, not just fill in for a zero one.
|
||||
func TestRunner_Run_DefaultIsAFloorNotJustAFallback(t *testing.T) {
|
||||
var gotMaxTokens int
|
||||
client := &mockClient{
|
||||
respond: func(req llm.Request) (llm.Response, error) {
|
||||
gotMaxTokens = req.MaxTokens
|
||||
return llm.Response{Text: "OK"}, nil
|
||||
},
|
||||
}
|
||||
r := New(client, Config{Concurrency: 1, MaxTokensDefault: 6000})
|
||||
|
||||
r.Run(context.Background(), []string{"model-a"}, []testkit.Test{
|
||||
{ID: "t1", Category: "c", Prompt: "p", MaxTokens: 200, Eval: echoEval()},
|
||||
})
|
||||
|
||||
if gotMaxTokens != 6000 {
|
||||
t.Errorf("MaxTokens = %d, want 6000 (config default is a floor; a smaller per-test 200 must not lower it)", gotMaxTokens)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunner_Run_TracksFinishReasonAndResponseText(t *testing.T) {
|
||||
client := &mockClient{
|
||||
respond: func(_ llm.Request) (llm.Response, error) {
|
||||
return llm.Response{Text: " OK ", FinishReason: "stop"}, nil
|
||||
},
|
||||
}
|
||||
r := New(client, Config{Concurrency: 1, MaxTokensDefault: 100})
|
||||
|
||||
results := r.Run(context.Background(), []string{"model-a"}, []testkit.Test{
|
||||
{ID: "t1", Category: "c", Prompt: "p", Eval: echoEval()},
|
||||
})
|
||||
|
||||
if results[0].FinishReason != "stop" {
|
||||
t.Errorf("FinishReason = %q, want stop", results[0].FinishReason)
|
||||
}
|
||||
if results[0].ResponseText != "OK" {
|
||||
t.Errorf("ResponseText = %q, want the normalized \"OK\"", results[0].ResponseText)
|
||||
}
|
||||
if results[0].Truncated() {
|
||||
t.Error("Truncated() = true, want false for finish_reason=stop")
|
||||
}
|
||||
}
|
||||
|
||||
// TestRunner_Run_TruncatedPrefixesScoreDetail is the regression test for
|
||||
// fix 4: a finish_reason=length response must be visibly flagged in
|
||||
// Score.Detail, not just silently scored.
|
||||
func TestRunner_Run_TruncatedPrefixesScoreDetail(t *testing.T) {
|
||||
client := &mockClient{
|
||||
respond: func(_ llm.Request) (llm.Response, error) {
|
||||
return llm.Response{Text: "partial", FinishReason: llm.FinishReasonLength}, nil
|
||||
},
|
||||
}
|
||||
r := New(client, Config{Concurrency: 1, MaxTokensDefault: 100})
|
||||
|
||||
results := r.Run(context.Background(), []string{"model-a"}, []testkit.Test{
|
||||
{ID: "t1", Category: "c", Prompt: "p", Eval: echoEval()},
|
||||
})
|
||||
|
||||
if !results[0].Truncated() {
|
||||
t.Fatal("Truncated() = false, want true for finish_reason=length")
|
||||
}
|
||||
if !strings.HasPrefix(results[0].Score.Detail, "TRUNCATED: ") {
|
||||
t.Errorf("Score.Detail = %q, want a TRUNCATED: prefix", results[0].Score.Detail)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunner_Run_IncludesSystemMessageWhenSet(t *testing.T) {
|
||||
var gotMessages []llm.Message
|
||||
client := &mockClient{
|
||||
@@ -199,6 +269,89 @@ func TestRunner_Run_IncludesSystemMessageWhenSet(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// recordingReporter is a ProgressReporter that records every event under a
|
||||
// mutex, so tests can assert on it after a concurrent Run.
|
||||
type recordingReporter struct {
|
||||
doneEvents []doneEvent
|
||||
startCalls int
|
||||
startTotalTests int
|
||||
startTotalModels int
|
||||
startConcurrency int
|
||||
mu sync.Mutex
|
||||
}
|
||||
|
||||
type doneEvent struct {
|
||||
testID string
|
||||
done int
|
||||
total int
|
||||
}
|
||||
|
||||
func (r *recordingReporter) ReportStart(totalTests, totalModels, concurrency int) {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
r.startCalls++
|
||||
r.startTotalTests = totalTests
|
||||
r.startTotalModels = totalModels
|
||||
r.startConcurrency = concurrency
|
||||
}
|
||||
|
||||
func (r *recordingReporter) ReportDone(done, total int, result Result) {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
r.doneEvents = append(r.doneEvents, doneEvent{done: done, total: total, testID: result.TestID})
|
||||
}
|
||||
|
||||
func TestRunner_Run_ReportsProgress(t *testing.T) {
|
||||
client := &mockClient{}
|
||||
reporter := &recordingReporter{}
|
||||
r := New(client, Config{Concurrency: 3, MaxTokensDefault: 100, Reporter: reporter})
|
||||
|
||||
tests := []testkit.Test{
|
||||
{ID: "t1", Category: "c", Prompt: "p", Eval: echoEval()},
|
||||
{ID: "t2", Category: "c", Prompt: "p", Eval: echoEval()},
|
||||
}
|
||||
models := []string{"model-a", "model-b", "model-c"}
|
||||
|
||||
r.Run(context.Background(), models, tests)
|
||||
|
||||
if reporter.startCalls != 1 {
|
||||
t.Errorf("ReportStart called %d times, want 1", reporter.startCalls)
|
||||
}
|
||||
if reporter.startTotalTests != len(tests) || reporter.startTotalModels != len(models) || reporter.startConcurrency != 3 {
|
||||
t.Errorf("ReportStart(totalTests=%d, totalModels=%d, concurrency=%d), want (%d, %d, 3)",
|
||||
reporter.startTotalTests, reporter.startTotalModels, reporter.startConcurrency, len(tests), len(models))
|
||||
}
|
||||
|
||||
total := len(tests) * len(models)
|
||||
if len(reporter.doneEvents) != total {
|
||||
t.Fatalf("ReportDone called %d times, want %d", len(reporter.doneEvents), total)
|
||||
}
|
||||
|
||||
seenDone := make(map[int]bool, total)
|
||||
for _, ev := range reporter.doneEvents {
|
||||
if ev.total != total {
|
||||
t.Errorf("ReportDone total = %d, want %d", ev.total, total)
|
||||
}
|
||||
if ev.done < 1 || ev.done > total {
|
||||
t.Errorf("ReportDone done = %d, want in [1, %d]", ev.done, total)
|
||||
}
|
||||
if seenDone[ev.done] {
|
||||
t.Errorf("ReportDone done=%d reported more than once: the completion counter is not concurrency-safe", ev.done)
|
||||
}
|
||||
seenDone[ev.done] = true
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunner_Run_NilReporterDefaultsToNoop(t *testing.T) {
|
||||
client := &mockClient{}
|
||||
r := New(client, Config{Concurrency: 2, MaxTokensDefault: 100})
|
||||
|
||||
// Must not panic when Config.Reporter is left nil.
|
||||
r.Run(context.Background(), []string{"model-a"}, []testkit.Test{
|
||||
{ID: "t1", Category: "c", Prompt: "p", Eval: echoEval()},
|
||||
})
|
||||
}
|
||||
|
||||
func TestRunner_Run_TracksLatencyAndTokens(t *testing.T) {
|
||||
client := &mockClient{
|
||||
respond: func(_ llm.Request) (llm.Response, error) {
|
||||
|
||||
Reference in New Issue
Block a user