fix: floor token budgets and surface truncation, add live progress

Reasoning models burn thousands of completion tokens before the
answer. Small per-test caps caused silent truncation and zero scores.
The runner now floors max_tokens at the config default (12000). The
client parses finish_reason. Reports mark truncated cells and the
JSON format carries response text for debugging. A stderr progress
reporter prints one line per completed request; --quiet disables it.
This commit is contained in:
2026-08-17 01:55:14 +01:00
parent 764fe990fd
commit ace50757e4
30 changed files with 1063 additions and 184 deletions
+6
View File
@@ -17,12 +17,15 @@ type jsonResult struct {
Subcategory string `json:"subcategory,omitempty"`
Detail string `json:"detail,omitempty"`
Error string `json:"error,omitempty"`
ResponseText string `json:"response_text"`
FinishReason string `json:"finish_reason,omitempty"`
Score float64 `json:"score"`
LatencyMS int64 `json:"latency_ms"`
PromptTokens int `json:"prompt_tokens"`
CompletionTokens int `json:"completion_tokens"`
Tokens int `json:"tokens"`
Skipped bool `json:"skipped"`
Truncated bool `json:"truncated"`
}
// renderJSON dumps raw results as a JSON array, joined with test metadata.
@@ -42,7 +45,10 @@ func renderJSON(w io.Writer, tests []testkit.Test, results []runner.Result) erro
Subcategory: t.Subcategory,
Score: r.Score.Value,
Detail: r.Score.Detail,
ResponseText: r.ResponseText,
FinishReason: r.FinishReason,
Skipped: r.Score.Skipped,
Truncated: r.Truncated(),
LatencyMS: r.Latency.Milliseconds(),
PromptTokens: r.PromptTokens,
CompletionTokens: r.CompletionTokens,