fix: floor token budgets and surface truncation, add live progress

Reasoning models burn thousands of completion tokens before the
answer. Small per-test caps caused silent truncation and zero scores.
The runner now floors max_tokens at the config default (12000). The
client parses finish_reason. Reports mark truncated cells and the
JSON format carries response text for debugging. A stderr progress
reporter prints one line per completed request; --quiet disables it.
This commit is contained in:
2026-08-17 01:55:14 +01:00
parent 764fe990fd
commit ace50757e4
30 changed files with 1063 additions and 184 deletions
+6
View File
@@ -20,14 +20,20 @@ func renderMarkdown(w io.Writer, tests []testkit.Test, models []string, results
ew.println()
scoreHeader := append([]string{"Test", "Category", "Subcategory"}, models...)
writeMarkdownHeader(ew, scoreHeader)
anyTruncated := false
for _, t := range sortedTests(tests) {
row := []string{t.ID, t.Category, t.Subcategory}
for _, model := range models {
r, ok := idx[t.ID][model]
row = append(row, cellText(r, ok))
anyTruncated = anyTruncated || r.Truncated()
}
writeMarkdownRow(ew, row)
}
if anyTruncated {
ew.println()
ew.println(truncatedLegend)
}
ew.println()
ew.println("## Category rollup (mean score)")