fix: floor token budgets and surface truncation, add live progress

Reasoning models burn thousands of completion tokens before the
answer. Small per-test caps caused silent truncation and zero scores.
The runner now floors max_tokens at the config default (12000). The
client parses finish_reason. Reports mark truncated cells and the
JSON format carries response text for debugging. A stderr progress
reporter prints one line per completed request; --quiet disables it.
This commit is contained in:
2026-08-17 01:55:14 +01:00
parent 764fe990fd
commit ace50757e4
30 changed files with 1063 additions and 184 deletions
+5
View File
@@ -51,14 +51,19 @@ func writeScoreTable(ew *errWriter, tests []testkit.Test, models []string, idx r
header = append(header, models...)
ew.println(strings.Join(header, "\t"))
anyTruncated := false
for _, t := range sortedTests(tests) {
row := []string{t.ID, t.Category, t.Subcategory}
for _, model := range models {
r, ok := idx[t.ID][model]
row = append(row, cellText(r, ok))
anyTruncated = anyTruncated || r.Truncated()
}
ew.println(strings.Join(row, "\t"))
}
if anyTruncated {
ew.println(truncatedLegend)
}
}
func writeCategoryTable(ew *errWriter, tests []testkit.Test, models []string, idx resultIndex) {