From 6ad9dfbe42a9a8239077eae77501467cf7913273 Mon Sep 17 00:00:00 2001 From: Lukasz Raczylo Date: Sun, 30 Aug 2026 16:19:21 +0100 Subject: [PATCH] fix(runner): record wall-clock latency on error rows An error row with zero latency reads as an instant rejection when it can be three exhausted request timeouts. A 651-second thinking-model generation looked instant in the 2026-08-30 run and misdirected the diagnosis toward gateway rejection. The error path now records the wall clock across all attempts. --- config.example.yaml | 2 +- internal/runner/runner.go | 7 ++++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/config.example.yaml b/config.example.yaml index 0c043e8..c6e1754 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -5,7 +5,7 @@ models: - uni/qwen3.6-27b - uni/btl-4 concurrency: 8 # default when omitted; matches internal/config's default -request_timeout: 300s # default when omitted +request_timeout: 900s # default when omitted max_tokens_default: 12000 # default when omitted; a reasoning model can spend # thousands of tokens on reasoning_content before # writing an answer, so this must stay generous diff --git a/internal/runner/runner.go b/internal/runner/runner.go index dec116f..65cbe67 100644 --- a/internal/runner/runner.go +++ b/internal/runner/runner.go @@ -161,9 +161,14 @@ func (r *Runner) runOne(ctx context.Context, model string, test testkit.Test) Re Temperature: r.cfg.Temperature, } + // Wall-clock across all attempts, recorded even on error: an error row + // with zero latency reads as an instant rejection when it may be three + // exhausted timeouts (a 651-second generation looked "instant" in the + // 2026-08-30 run and misdirected the diagnosis). + callStart := time.Now() resp, err := r.client.Complete(ctx, req) if err != nil { - return Result{Model: model, TestID: test.ID, Err: err} + return Result{Model: model, TestID: test.ID, Err: err, Latency: time.Since(callStart)} } normalized := testkit.Normalize(resp.Text)