mirror of
https://github.com/lukaszraczylo/llm-testbench.git
synced 2026-10-12 00:45:19 +00:00
fix: floor token budgets and surface truncation, add live progress
Reasoning models burn thousands of completion tokens before the answer. Small per-test caps caused silent truncation and zero scores. The runner now floors max_tokens at the config default (12000). The client parses finish_reason. Reports mark truncated cells and the JSON format carries response text for debugging. A stderr progress reporter prints one line per completed request; --quiet disables it.
This commit is contained in:
@@ -20,14 +20,20 @@ func renderMarkdown(w io.Writer, tests []testkit.Test, models []string, results
|
||||
ew.println()
|
||||
scoreHeader := append([]string{"Test", "Category", "Subcategory"}, models...)
|
||||
writeMarkdownHeader(ew, scoreHeader)
|
||||
anyTruncated := false
|
||||
for _, t := range sortedTests(tests) {
|
||||
row := []string{t.ID, t.Category, t.Subcategory}
|
||||
for _, model := range models {
|
||||
r, ok := idx[t.ID][model]
|
||||
row = append(row, cellText(r, ok))
|
||||
anyTruncated = anyTruncated || r.Truncated()
|
||||
}
|
||||
writeMarkdownRow(ew, row)
|
||||
}
|
||||
if anyTruncated {
|
||||
ew.println()
|
||||
ew.println(truncatedLegend)
|
||||
}
|
||||
|
||||
ew.println()
|
||||
ew.println("## Category rollup (mean score)")
|
||||
|
||||
Reference in New Issue
Block a user