mirror of
https://github.com/lukaszraczylo/llm-testbench.git
synced 2026-10-14 00:57:45 +00:00
fix: floor token budgets and surface truncation, add live progress
Reasoning models burn thousands of completion tokens before the answer. Small per-test caps caused silent truncation and zero scores. The runner now floors max_tokens at the config default (12000). The client parses finish_reason. Reports mark truncated cells and the JSON format carries response text for debugging. A stderr progress reporter prints one line per completed request; --quiet disables it.
This commit is contained in:
@@ -72,19 +72,37 @@ func sortedTests(tests []testkit.Test) []testkit.Test {
|
||||
return out
|
||||
}
|
||||
|
||||
// truncatedSuffix marks a cell whose underlying response was cut off by
|
||||
// the token budget (finish_reason=length): the score may not reflect what
|
||||
// the model would have answered with a larger budget. truncatedLegend is
|
||||
// the one-line explanation printed once per table/markdown report.
|
||||
const (
|
||||
truncatedSuffix = "!"
|
||||
truncatedLegend = "! = response truncated by the token budget (finish_reason=length); increase max_tokens_default to avoid"
|
||||
)
|
||||
|
||||
// cellText renders one (test, model) cell for the score table: "score
|
||||
// (Ntok)" for a scored result, or ERR/skip/N/A otherwise.
|
||||
// (Ntok)" for a scored result, or ERR/skip/N/A otherwise, with a
|
||||
// truncatedSuffix appended when the response was cut off by the token
|
||||
// budget.
|
||||
func cellText(r runner.Result, found bool) string {
|
||||
switch {
|
||||
case !found:
|
||||
if !found {
|
||||
return "N/A"
|
||||
case r.Err != nil:
|
||||
return "ERR"
|
||||
case r.Score.Skipped:
|
||||
return "skip"
|
||||
default:
|
||||
return fmt.Sprintf("%.2f (%dtok)", r.Score.Value, r.TotalTokens())
|
||||
}
|
||||
|
||||
var text string
|
||||
switch {
|
||||
case r.Err != nil:
|
||||
text = "ERR"
|
||||
case r.Score.Skipped:
|
||||
text = "skip"
|
||||
default:
|
||||
text = fmt.Sprintf("%.2f (%dtok)", r.Score.Value, r.TotalTokens())
|
||||
}
|
||||
if r.Truncated() {
|
||||
text += truncatedSuffix
|
||||
}
|
||||
return text
|
||||
}
|
||||
|
||||
// categoryMean holds the per-model mean score for one category, computed
|
||||
|
||||
Reference in New Issue
Block a user