fix: floor token budgets and surface truncation, add live progress

Reasoning models burn thousands of completion tokens before the
answer. Small per-test caps caused silent truncation and zero scores.
The runner now floors max_tokens at the config default (12000). The
client parses finish_reason. Reports mark truncated cells and the
JSON format carries response text for debugging. A stderr progress
reporter prints one line per completed request; --quiet disables it.

internal/report/json.go

@@ -17,12 +17,15 @@ type jsonResult struct {
 	Subcategory      string  `json:"subcategory,omitempty"`
 	Detail           string  `json:"detail,omitempty"`
 	Error            string  `json:"error,omitempty"`
+	ResponseText     string  `json:"response_text"`
+	FinishReason     string  `json:"finish_reason,omitempty"`
 	Score            float64 `json:"score"`
 	LatencyMS        int64   `json:"latency_ms"`
 	PromptTokens     int     `json:"prompt_tokens"`
 	CompletionTokens int     `json:"completion_tokens"`
 	Tokens           int     `json:"tokens"`
 	Skipped          bool    `json:"skipped"`
+	Truncated        bool    `json:"truncated"`
 }
 
 // renderJSON dumps raw results as a JSON array, joined with test metadata.
@@ -42,7 +45,10 @@ func renderJSON(w io.Writer, tests []testkit.Test, results []runner.Result) erro
 			Subcategory:      t.Subcategory,
 			Score:            r.Score.Value,
 			Detail:           r.Score.Detail,
+			ResponseText:     r.ResponseText,
+			FinishReason:     r.FinishReason,
 			Skipped:          r.Score.Skipped,
+			Truncated:        r.Truncated(),
 			LatencyMS:        r.Latency.Milliseconds(),
 			PromptTokens:     r.PromptTokens,
 			CompletionTokens: r.CompletionTokens,

internal/report/markdown.go

@@ -20,14 +20,20 @@ func renderMarkdown(w io.Writer, tests []testkit.Test, models []string, results
 	ew.println()
 	scoreHeader := append([]string{"Test", "Category", "Subcategory"}, models...)
 	writeMarkdownHeader(ew, scoreHeader)
+	anyTruncated := false
 	for _, t := range sortedTests(tests) {
 		row := []string{t.ID, t.Category, t.Subcategory}
 		for _, model := range models {
 			r, ok := idx[t.ID][model]
 			row = append(row, cellText(r, ok))
+			anyTruncated = anyTruncated || r.Truncated()
 		}
 		writeMarkdownRow(ew, row)
 	}
+	if anyTruncated {
+		ew.println()
+		ew.println(truncatedLegend)
+	}
 
 	ew.println()
 	ew.println("## Category rollup (mean score)")

internal/report/report.go

@@ -72,19 +72,37 @@ func sortedTests(tests []testkit.Test) []testkit.Test {
 	return out
 }
 
+// truncatedSuffix marks a cell whose underlying response was cut off by
+// the token budget (finish_reason=length): the score may not reflect what
+// the model would have answered with a larger budget. truncatedLegend is
+// the one-line explanation printed once per table/markdown report.
+const (
+	truncatedSuffix = "!"
+	truncatedLegend = "! = response truncated by the token budget (finish_reason=length); increase max_tokens_default to avoid"
+)
+
 // cellText renders one (test, model) cell for the score table: "score
-// (Ntok)" for a scored result, or ERR/skip/N/A otherwise.
+// (Ntok)" for a scored result, or ERR/skip/N/A otherwise, with a
+// truncatedSuffix appended when the response was cut off by the token
+// budget.
 func cellText(r runner.Result, found bool) string {
-	switch {
-	case !found:
+	if !found {
 		return "N/A"
-	case r.Err != nil:
-		return "ERR"
-	case r.Score.Skipped:
-		return "skip"
-	default:
-		return fmt.Sprintf("%.2f (%dtok)", r.Score.Value, r.TotalTokens())
 	}
+
+	var text string
+	switch {
+	case r.Err != nil:
+		text = "ERR"
+	case r.Score.Skipped:
+		text = "skip"
+	default:
+		text = fmt.Sprintf("%.2f (%dtok)", r.Score.Value, r.TotalTokens())
+	}
+	if r.Truncated() {
+		text += truncatedSuffix
+	}
+	return text
 }
 
 // categoryMean holds the per-model mean score for one category, computed

internal/report/report_test.go

@@ -116,6 +116,85 @@ func TestRenderJSON_DumpsRawResults(t *testing.T) {
 	}
 }
 
+func TestRenderTable_TruncatedCellsGetSuffixAndLegend(t *testing.T) {
+	tests := []testkit.Test{{ID: "t1", Category: "c", Subcategory: "s"}}
+	results := []runner.Result{
+		{Model: "m1", TestID: "t1", Score: eval.Score{Value: 0, Detail: "TRUNCATED: got \"\""}, FinishReason: "length"},
+	}
+	var buf bytes.Buffer
+	if err := Render(&buf, FormatTable, tests, []string{"m1"}, results); err != nil {
+		t.Fatalf("Render() error = %v", err)
+	}
+	out := buf.String()
+	if !strings.Contains(out, "0.00 (0tok)!") {
+		t.Errorf("table output missing truncated \"!\" suffix:\n%s", out)
+	}
+	if !strings.Contains(out, truncatedLegend) {
+		t.Errorf("table output missing the truncation legend:\n%s", out)
+	}
+}
+
+func TestRenderTable_NoLegendWhenNothingTruncated(t *testing.T) {
+	var buf bytes.Buffer
+	if err := Render(&buf, FormatTable, sampleTests(), []string{"m1", "m2"}, sampleResults()); err != nil {
+		t.Fatalf("Render() error = %v", err)
+	}
+	out := buf.String()
+	if strings.Contains(out, truncatedLegend) {
+		t.Errorf("table output should not print the truncation legend when nothing truncated:\n%s", out)
+	}
+}
+
+func TestRenderMarkdown_TruncatedCellsGetSuffixAndLegend(t *testing.T) {
+	tests := []testkit.Test{{ID: "t1", Category: "c", Subcategory: "s"}}
+	results := []runner.Result{
+		{Model: "m1", TestID: "t1", Score: eval.Score{Value: 0}, FinishReason: "length"},
+	}
+	var buf bytes.Buffer
+	if err := Render(&buf, FormatMarkdown, tests, []string{"m1"}, results); err != nil {
+		t.Fatalf("Render() error = %v", err)
+	}
+	out := buf.String()
+	if !strings.Contains(out, "0.00 (0tok)!") {
+		t.Errorf("markdown output missing truncated \"!\" suffix:\n%s", out)
+	}
+	if !strings.Contains(out, truncatedLegend) {
+		t.Errorf("markdown output missing the truncation legend:\n%s", out)
+	}
+}
+
+func TestRenderJSON_IncludesResponseTextFinishReasonAndTruncated(t *testing.T) {
+	tests := []testkit.Test{{ID: "t1", Category: "c"}}
+	results := []runner.Result{
+		{Model: "m1", TestID: "t1", Score: eval.Score{Value: 0, Detail: "TRUNCATED: got \"\""}, ResponseText: "partial answer", FinishReason: "length"},
+	}
+	var buf bytes.Buffer
+	if err := Render(&buf, FormatJSON, tests, []string{"m1"}, results); err != nil {
+		t.Fatalf("Render() error = %v", err)
+	}
+
+	var out []jsonResult
+	if err := json.Unmarshal(buf.Bytes(), &out); err != nil {
+		t.Fatalf("json.Unmarshal() error = %v; output:\n%s", err, buf.String())
+	}
+	if len(out) != 1 {
+		t.Fatalf("len(out) = %d, want 1", len(out))
+	}
+	got := out[0]
+	if got.ResponseText != "partial answer" {
+		t.Errorf("ResponseText = %q, want %q", got.ResponseText, "partial answer")
+	}
+	if got.FinishReason != "length" {
+		t.Errorf("FinishReason = %q, want length", got.FinishReason)
+	}
+	if !got.Truncated {
+		t.Error("Truncated = false, want true")
+	}
+	if !strings.HasPrefix(got.Detail, "TRUNCATED: ") {
+		t.Errorf("Detail = %q, want a TRUNCATED: prefix", got.Detail)
+	}
+}
+
 func TestRender_UnknownFormat(t *testing.T) {
 	var buf bytes.Buffer
 	err := Render(&buf, Format("bogus"), sampleTests(), []string{"m1"}, sampleResults())

internal/report/table.go

@@ -51,14 +51,19 @@ func writeScoreTable(ew *errWriter, tests []testkit.Test, models []string, idx r
 	header = append(header, models...)
 	ew.println(strings.Join(header, "\t"))
 
+	anyTruncated := false
 	for _, t := range sortedTests(tests) {
 		row := []string{t.ID, t.Category, t.Subcategory}
 		for _, model := range models {
 			r, ok := idx[t.ID][model]
 			row = append(row, cellText(r, ok))
+			anyTruncated = anyTruncated || r.Truncated()
 		}
 		ew.println(strings.Join(row, "\t"))
 	}
+	if anyTruncated {
+		ew.println(truncatedLegend)
+	}
 }
 
 func writeCategoryTable(ew *errWriter, tests []testkit.Test, models []string, idx resultIndex) {