fix: floor token budgets and surface truncation, add live progress

Reasoning models burn thousands of completion tokens before the
answer. Small per-test caps caused silent truncation and zero scores.
The runner now floors max_tokens at the config default (12000). The
client parses finish_reason. Reports mark truncated cells and the
JSON format carries response text for debugging. A stderr progress
reporter prints one line per completed request; --quiet disables it.

internal/llm/llm.go

@@ -24,11 +24,19 @@ type Request struct {
 // Response is the normalized result of a chat-completion call.
 type Response struct {
 	Text             string
+	FinishReason     string
 	PromptTokens     int
 	CompletionTokens int
 	Latency          time.Duration
 }
 
+// FinishReasonLength is the OpenAI-compatible finish_reason value meaning
+// the model hit its token budget (MaxTokens) before it finished: the
+// response, and Text in particular, may be truncated or empty. A reasoning
+// model that spends its whole budget on internal reasoning before writing
+// anything to the answer channel reports this with Text == "".
+const FinishReasonLength = "length"
+
 // Client completes chat requests against a language model backend.
 // Implementations must be safe for concurrent use.
 type Client interface {

internal/llm/openai.go

@@ -50,12 +50,36 @@ type chatMsg struct {
 	Content string `json:"content"`
 }
 
+// responseMsg is the assistant message inside one response choice. Content
+// is a pointer, not a plain string: some OpenAI-compatible backends return
+// "content": null when generation was cut off before any answer token was
+// emitted (for example finish_reason=length on a reasoning model that
+// spent its whole token budget on reasoning_content and never reached
+// content). Text reports that case as "", explicitly, rather than relying
+// on encoding/json's implicit null-into-string zero-value behavior.
+//
+// Deliberately not evaluated: reasoning_content. The catalog's evaluators
+// score the answer a user would actually see, which is content only.
+type responseMsg struct {
+	Content *string `json:"content"`
+	Role    string  `json:"role"`
+}
+
+// Text returns m.Content, or "" if the backend sent "content": null.
+func (m responseMsg) Text() string {
+	if m.Content == nil {
+		return ""
+	}
+	return *m.Content
+}
+
 type chatCompletionResponse struct {
 	Error *struct {
 		Message string `json:"message"`
 	} `json:"error"`
 	Choices []struct {
-		Message chatMsg `json:"message"`
+		Message      responseMsg `json:"message"`
+		FinishReason string      `json:"finish_reason"`
 	} `json:"choices"`
 	Usage struct {
 		PromptTokens     int `json:"prompt_tokens"`
@@ -105,9 +129,10 @@ func (c *OpenAIClient) Complete(ctx context.Context, req Request) (Response, err
 		}
 
 		latency := time.Since(attemptStart)
-		text := ""
+		var text, finishReason string
 		if len(resp.Choices) > 0 {
-			text = resp.Choices[0].Message.Content
+			text = resp.Choices[0].Message.Text()
+			finishReason = resp.Choices[0].FinishReason
 		}
 		if resp.Error != nil {
 			lastErr = fmt.Errorf("llm: api error: %s", resp.Error.Message)
@@ -115,6 +140,7 @@ func (c *OpenAIClient) Complete(ctx context.Context, req Request) (Response, err
 		}
 		return Response{
 			Text:             text,
+			FinishReason:     finishReason,
 			PromptTokens:     resp.Usage.PromptTokens,
 			CompletionTokens: resp.Usage.CompletionTokens,
 			Latency:          latency,

internal/llm/openai_test.go

@@ -21,6 +21,23 @@ func newTestServer(t *testing.T, handler http.HandlerFunc) *httptest.Server {
 	return srv
 }
 
+// testChoice builds one response choice with a non-null content string, the
+// common case. Its return type is exactly chatCompletionResponse.Choices's
+// element type, so callers can append() it without repeating the anonymous
+// struct type at each call site.
+func testChoice(content, finishReason string) struct {
+	Message      responseMsg `json:"message"`
+	FinishReason string      `json:"finish_reason"`
+} {
+	return struct {
+		Message      responseMsg `json:"message"`
+		FinishReason string      `json:"finish_reason"`
+	}{
+		Message:      responseMsg{Role: "assistant", Content: &content},
+		FinishReason: finishReason,
+	}
+}
+
 func TestOpenAIClient_Complete_Success(t *testing.T) {
 	srv := newTestServer(t, func(w http.ResponseWriter, r *http.Request) {
 		if r.URL.Path != "/chat/completions" {
@@ -34,9 +51,7 @@ func TestOpenAIClient_Complete_Success(t *testing.T) {
 			t.Errorf("model = %q, want test-model", body.Model)
 		}
 		resp := chatCompletionResponse{}
-		resp.Choices = []struct {
-			Message chatMsg `json:"message"`
-		}{{Message: chatMsg{Role: "assistant", Content: "hello there"}}}
+		resp.Choices = append(resp.Choices, testChoice("hello there", "stop"))
 		resp.Usage.PromptTokens = 12
 		resp.Usage.CompletionTokens = 3
 		_ = json.NewEncoder(w).Encode(resp)
@@ -53,11 +68,61 @@ func TestOpenAIClient_Complete_Success(t *testing.T) {
 	if resp.Text != "hello there" {
 		t.Errorf("Text = %q, want %q", resp.Text, "hello there")
 	}
+	if resp.FinishReason != "stop" {
+		t.Errorf("FinishReason = %q, want stop", resp.FinishReason)
+	}
 	if resp.PromptTokens != 12 || resp.CompletionTokens != 3 {
 		t.Errorf("tokens = %d/%d, want 12/3", resp.PromptTokens, resp.CompletionTokens)
 	}
 }
 
+// TestOpenAIClient_Complete_ParsesFinishReasonLength verifies
+// choices[0].finish_reason lands in Response.FinishReason == "length", the
+// signal that generation was cut off by the token budget (root cause of the
+// live-run incident: a truncated reasoning-model response silently scored
+// 0 with no visible signal that it was ever cut off).
+func TestOpenAIClient_Complete_ParsesFinishReasonLength(t *testing.T) {
+	srv := newTestServer(t, func(w http.ResponseWriter, r *http.Request) {
+		resp := chatCompletionResponse{}
+		resp.Choices = append(resp.Choices, testChoice("partial ans", FinishReasonLength))
+		_ = json.NewEncoder(w).Encode(resp)
+	})
+
+	client := NewOpenAIClient(srv.URL, "", time.Second)
+	resp, err := client.Complete(context.Background(), Request{Model: "m", Messages: []Message{{Role: "user", Content: "x"}}})
+	if err != nil {
+		t.Fatalf("Complete() error = %v", err)
+	}
+	if resp.FinishReason != FinishReasonLength {
+		t.Errorf("FinishReason = %q, want %q", resp.FinishReason, FinishReasonLength)
+	}
+}
+
+// TestOpenAIClient_Complete_NullContentIsEmptyString verifies a backend
+// sending "content": null (a reasoning model that spent its whole token
+// budget on reasoning_content and never wrote to content) decodes to
+// Response.Text == "", not a decode error or a panic.
+func TestOpenAIClient_Complete_NullContentIsEmptyString(t *testing.T) {
+	srv := newTestServer(t, func(w http.ResponseWriter, r *http.Request) {
+		_, _ = w.Write([]byte(`{"choices":[{"message":{"role":"assistant","content":null},"finish_reason":"length"}],"usage":{"prompt_tokens":50,"completion_tokens":6000}}`))
+	})
+
+	client := NewOpenAIClient(srv.URL, "", time.Second)
+	resp, err := client.Complete(context.Background(), Request{Model: "m", Messages: []Message{{Role: "user", Content: "x"}}})
+	if err != nil {
+		t.Fatalf("Complete() error = %v", err)
+	}
+	if resp.Text != "" {
+		t.Errorf("Text = %q, want empty string for null content", resp.Text)
+	}
+	if resp.FinishReason != FinishReasonLength {
+		t.Errorf("FinishReason = %q, want %q", resp.FinishReason, FinishReasonLength)
+	}
+	if resp.CompletionTokens != 6000 {
+		t.Errorf("CompletionTokens = %d, want 6000 (tokens were spent on reasoning_content, not content)", resp.CompletionTokens)
+	}
+}
+
 // TestOpenAIClient_Complete_SendsTemperatureZero verifies Request.Temperature
 // (PLAN.md pins this to 0 for deterministic scoring, wired from main.go's
 // requestTemperature constant) actually lands on the wire as a JSON
@@ -107,9 +172,7 @@ func TestOpenAIClient_Complete_RetriesOn5xxThenSucceeds(t *testing.T) {
 			return
 		}
 		resp := chatCompletionResponse{}
-		resp.Choices = []struct {
-			Message chatMsg `json:"message"`
-		}{{Message: chatMsg{Role: "assistant", Content: "ok"}}}
+		resp.Choices = append(resp.Choices, testChoice("ok", "stop"))
 		_ = json.NewEncoder(w).Encode(resp)
 	})