mirror of
https://github.com/lukaszraczylo/llm-testbench.git
synced 2026-10-13 00:48:29 +00:00
fix: floor token budgets and surface truncation, add live progress
Reasoning models burn thousands of completion tokens before the answer. Small per-test caps caused silent truncation and zero scores. The runner now floors max_tokens at the config default (12000). The client parses finish_reason. Reports mark truncated cells and the JSON format carries response text for debugging. A stderr progress reporter prints one line per completed request; --quiet disables it.
This commit is contained in:
@@ -22,6 +22,16 @@ func TestExtractLastNumber(t *testing.T) {
|
||||
{"leading-dot decimal", "= .9896", 0.9896, false},
|
||||
{"answer on last line, reasoning with numbers on earlier lines", "step 1 uses 8 items\nstep 2 uses 16 items\nfinal answer: 42", 42, false},
|
||||
{"last line has only a unit-qualified number, falls back to whole text", "the computation used 100 iterations\nresult on a 64-bit system", 100, false},
|
||||
// 5b regressions: opus round 2's failing set.
|
||||
{"platform qualifier with parenthetical acronym", "24 bytes on a 64-bit (LP64) platform.", 24, false},
|
||||
{"digits glued into an identifier via underscore", "sizeof = 24 bytes on x86_64", 24, false},
|
||||
{"digits glued into an identifier via hyphen (bogus sign)", "sizeof = 24 bytes on x86-64", 24, false},
|
||||
{"hyphen-compound number accepted as a last resort", "It is a 24-byte struct.", 24, false},
|
||||
// Documented limitation, not a regression: the heuristic has no
|
||||
// notion of "the total" vs. "a breakdown component" and returns
|
||||
// the last standalone number, which here is not the answer a
|
||||
// human would pick. Prompts should ask for the number alone.
|
||||
{"ambiguous parenthetical breakdown returns the wrong number by design", "Total: 24 bytes (22 bytes of members plus 2 tail padding)", 2, false},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
|
||||
Reference in New Issue
Block a user