fix: floor token budgets and surface truncation, add live progress

Reasoning models burn thousands of completion tokens before the
answer. Small per-test caps caused silent truncation and zero scores.
The runner now floors max_tokens at the config default (12000). The
client parses finish_reason. Reports mark truncated cells and the
JSON format carries response text for debugging. A stderr progress
reporter prints one line per completed request; --quiet disables it.

internal/tests/catalog_test.go

@@ -2,60 +2,53 @@ package tests
 
 import "testing"
 
-// wantCatalog is the full 17-test catalog per PLAN.md, id -> {category,
-// subcategory}. TestAll_MatchesCatalog checks the registry against this
-// list exactly, so an accidental removal, duplication, or miscategorized
-// test fails loudly.
-var wantCatalog = map[string][2]string{
-	"go-struct-align":           {"programming", "golang"},
-	"go-worker-pool":            {"programming", "golang"},
-	"go-semver-classify":        {"programming", "golang"},
-	"py-log-triage":             {"programming", "python"},
-	"py-cosine":                 {"programming", "python"},
-	"ts-debounce-composable":    {"programming", "typescript"},
-	"c-struct-size":             {"programming", "c"},
-	"macos-timeout-portability": {"operations", "macos"},
-	"macos-launchd-cron":        {"operations", "macos"},
-	"linux-pct-exec":            {"operations", "linux"},
-	"linux-systemd-oneshot":     {"operations", "linux"},
-	"k8s-crashloop-gitops":      {"operations", "kubernetes"},
-	"web-robots-ai-crawlers":    {"research", "web"},
-	"paper-hnsw-params":         {"research", "whitepapers"},
-	"code-trace-go":             {"research", "codebase"},
-	"agent-tool-routing":        {"agents", "tool-routing"},
-	"agent-plan-ordering":       {"agents", "planning"},
-}
-
-func TestAll_MatchesCatalog(t *testing.T) {
+// TestAll_Invariants checks structural invariants across the whole
+// catalog, rather than an exact id/category/count map (fix 6): four other
+// authors are adding roughly 113 more tests to internal/tests in parallel
+// worktrees, so a fixed catalog size or an exhaustive id list would break
+// on every integration merge for reasons unrelated to a real bug. An
+// exact-catalog check belongs at integration time, once the merge is
+// final, not in a file every parallel author's branch also compiles.
+func TestAll_Invariants(t *testing.T) {
 	r := All()
+	all := r.All()
 
-	if r.Len() != len(wantCatalog) {
-		t.Fatalf("registry has %d tests, want %d", r.Len(), len(wantCatalog))
+	if len(all) == 0 {
+		t.Fatal("catalog is empty")
 	}
 
-	for id, catSub := range wantCatalog {
-		tc, ok := r.Get(id)
-		if !ok {
-			t.Errorf("missing test %q", id)
+	seenIDs := make(map[string]bool, len(all))
+	for _, tc := range all {
+		if tc.ID == "" {
+			t.Error("found a test with an empty ID")
 			continue
 		}
-		if tc.Category != catSub[0] {
-			t.Errorf("%s: Category = %q, want %q", id, tc.Category, catSub[0])
+		if seenIDs[tc.ID] {
+			t.Errorf("duplicate test ID %q", tc.ID)
 		}
-		if tc.Subcategory != catSub[1] {
-			t.Errorf("%s: Subcategory = %q, want %q", id, tc.Subcategory, catSub[1])
+		seenIDs[tc.ID] = true
+
+		if tc.Category == "" {
+			t.Errorf("%s: Category is empty", tc.ID)
+		}
+		if tc.Subcategory == "" {
+			t.Errorf("%s: Subcategory is empty", tc.ID)
 		}
 		if tc.Prompt == "" {
-			t.Errorf("%s: Prompt is empty", id)
-		}
-		if tc.Eval == nil {
-			t.Errorf("%s: Eval is nil", id)
+			t.Errorf("%s: Prompt is empty", tc.ID)
 		}
 		if tc.Description == "" {
-			t.Errorf("%s: Description is empty", id)
+			t.Errorf("%s: Description is empty", tc.ID)
 		}
-		if tc.MaxTokens <= 0 {
-			t.Errorf("%s: MaxTokens = %d, want > 0", id, tc.MaxTokens)
+		if tc.Eval == nil {
+			t.Errorf("%s: Eval is nil", tc.ID)
+		}
+		// MaxTokens == 0 is valid: runner.Runner floors it with
+		// cfg.MaxTokensDefault (max(test.MaxTokens, cfg.MaxTokensDefault)),
+		// so a test need not set its own MaxTokens at all. Only a
+		// negative value is a bug.
+		if tc.MaxTokens < 0 {
+			t.Errorf("%s: MaxTokens = %d, want >= 0", tc.ID, tc.MaxTokens)
 		}
 	}
 }

internal/tests/golang.go

@@ -86,8 +86,11 @@ Respond with the corrected struct definition only.`
 // response's Pool[T, R any] implementation (defined in solution.go, same
 // package) with a deliberately slow worker function and checks (a) output
 // order matches input order and (b) the pool actually parallelizes work,
-// via both an observed-concurrency counter and a wall-clock bound far
-// below the fully-sequential time.
+// via an observed-concurrency counter (maxConcurrent >= 2). There is
+// deliberately no wall-clock bound (5e): a CI runner under load can make
+// even genuinely concurrent work take longer than any fixed millisecond
+// budget, and maxConcurrent >= 2 already proves real parallelism without
+// that flake risk.
 const goWorkerPoolHarness = `package main
 
 import (
@@ -122,9 +125,7 @@ func main() {
 		return x * 2
 	}
 
-	start := time.Now()
 	out := Pool(in, workers, f)
-	elapsed := time.Since(start)
 
 	if len(out) != n {
 		fmt.Fprintf(os.Stderr, "len(out)=%d, want %d\n", len(out), n)
@@ -140,12 +141,6 @@ func main() {
 		fmt.Fprintf(os.Stderr, "maxConcurrent=%d, want >= 2: no real concurrency observed\n", maxConcurrent)
 		os.Exit(1)
 	}
-	// Sequential execution would take n*perItem = 200ms; with workers
-	// goroutines truly running in parallel it should land well under that.
-	if elapsed > 120*time.Millisecond {
-		fmt.Fprintf(os.Stderr, "elapsed=%s, want < 120ms: looks sequential\n", elapsed)
-		os.Exit(1)
-	}
 
 	fmt.Println("PASS")
 }

internal/tests/golang_test.go

@@ -119,7 +119,7 @@ func TestGoWorkerPoolTest_Eval(t *testing.T) {
 		want     float64
 	}{
 		{"correct concurrent pool", correctPool, 1},
-		{"sequential pool fails concurrency/timing check", sequentialPool, 0},
+		{"sequential pool fails the maxConcurrent>=2 check", sequentialPool, 0},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {

internal/tests/kubernetes.go

@@ -56,37 +56,59 @@ func mentionsLimitIncrease() eval.Evaluator {
 }
 
 // kubectlLiveMutationPattern matches any mention of "kubectl edit" or
-// "kubectl patch", negated or not.
+// "kubectl patch", negated or not - whether mid-sentence, at a bare line
+// start, or as a bulleted list item.
 var kubectlLiveMutationPattern = regexp.MustCompile(`(?i)kubectl\s+(edit|patch)\b`)
 
-// kubectlImperativeMutationPattern matches a bare/imperative "kubectl
-// edit"/"kubectl patch" at the start of a line (optionally after
-// whitespace or a bullet marker) - a command a reader would actually run,
-// never acceptable regardless of negation cues elsewhere in the response.
-var kubectlImperativeMutationPattern = regexp.MustCompile(`(?im)^\s*(?:[-*]\s*)?kubectl\s+(edit|patch)\b`)
-
 // negationCuePattern matches a word that turns a mention of the forbidden
 // command into a warning against running it, rather than an instruction to
 // run it (e.g. "do not run kubectl patch...").
-var negationCuePattern = regexp.MustCompile(`(?i)\b(don'?t|do not|never|avoid|instead of|not)\b`)
+var negationCuePattern = regexp.MustCompile(`(?i)\b(don'?t|do not|never|avoid|instead of|not|cannot|can'?t|rather than|without|no need)\b`)
 
 // negationWindow is how many characters before a "kubectl edit|patch"
-// mention (never crossing a newline) are searched for a negation cue.
+// mention are searched for a negation cue.
 const negationWindow = 60
 
+// negationWindowStart returns the earliest byte offset in response to
+// search for a negation cue before a "kubectl edit|patch" match starting
+// at start. The window is the current line, extended back to the start of
+// the immediately preceding line when that line is non-empty: a
+// hard-wrapped sentence whose negation cue landed on the line above (e.g.
+// "...do not run\nkubectl edit...") would otherwise score 0 even though it
+// is the correct, negated answer. Either way, the window never reaches
+// more than negationWindow characters before start.
+func negationWindowStart(response string, start int) int {
+	curLineStart := strings.LastIndexByte(response[:start], '\n') + 1
+
+	windowFloor := curLineStart
+	if curLineStart > 0 {
+		prevLineEnd := curLineStart - 1 // the newline separating the two lines
+		prevLineStart := strings.LastIndexByte(response[:prevLineEnd], '\n') + 1
+		if strings.TrimSpace(response[prevLineStart:prevLineEnd]) != "" {
+			windowFloor = prevLineStart
+		}
+	}
+
+	return max(start-negationWindow, windowFloor)
+}
+
 // noLiveKubectlMutation scores full credit unless the response instructs
 // running "kubectl edit"/"kubectl patch" against the live cluster. A
 // contrastive mention - warning the reader not to do this, in favor of the
-// GitOps procedure - is fine and does not cost credit; an imperative or
-// otherwise unnegated mention scores zero. This is deliberately not
-// eval.NotContains, which would also zero out the best possible answer
-// (one that correctly explains why NOT to run kubectl edit/patch).
+// GitOps procedure - is fine and does not cost credit; an unnegated
+// mention scores zero. This is deliberately not eval.NotContains, which
+// would also zero out the best possible answer (one that correctly
+// explains why NOT to run kubectl edit/patch).
+//
+// Every occurrence goes through the same negation-window check, including
+// one at the start of a line (a bulleted "don't do this" list item, or a
+// hard-wrapped sentence whose negation cue landed on the line above): a
+// response is not penalized just because word-wrap or list formatting put
+// the command text at a line start. A genuinely bare imperative still
+// fails, because negationWindowStart never finds a negation cue in its
+// (possibly line-extended) window.
 func noLiveKubectlMutation() eval.Evaluator {
 	return eval.EvaluatorFunc(func(_ context.Context, response string) eval.Score {
-		if kubectlImperativeMutationPattern.MatchString(response) {
-			return eval.Score{Value: 0, Detail: "imperative kubectl edit/patch command present"}
-		}
-
 		matches := kubectlLiveMutationPattern.FindAllStringIndex(response, -1)
 		if len(matches) == 0 {
 			return eval.Score{Value: 1, Detail: "no mention of kubectl edit/patch"}
@@ -94,11 +116,7 @@ func noLiveKubectlMutation() eval.Evaluator {
 
 		for _, loc := range matches {
 			start := loc[0]
-			lineStart := strings.LastIndexByte(response[:start], '\n') + 1
-			windowStart := start - negationWindow
-			if windowStart < lineStart {
-				windowStart = lineStart
-			}
+			windowStart := negationWindowStart(response, start)
 			if !negationCuePattern.MatchString(response[windowStart:start]) {
 				return eval.Score{Value: 0, Detail: "unnegated mention of kubectl edit/patch"}
 			}

internal/tests/kubernetes_test.go

@@ -59,6 +59,50 @@ func TestNoLiveKubectlMutation(t *testing.T) {
 			response: "Never mind the manifest for now, but you could just casually run a quick kubectl patch to fix the limit right away.",
 			want:     0,
 		},
+		{
+			// Opus round 2, 5a: extending the negation window to the
+			// previous line (and routing every occurrence, including a
+			// line-start one, through that same window) fixes this - it
+			// used to score 0 because "kubectl edit" landed at a hard-wrap
+			// line start and the old code hard-vetoed any line-start
+			// occurrence regardless of the negation on the line above.
+			name:     "hard-wrapped negation: cue on the previous line",
+			response: "Since this is GitOps-managed, you must not\nkubectl edit the deployment directly - instead commit the fix to git.",
+			want:     1,
+		},
+		{
+			// Single bullet, not a multi-item list: the window extends back
+			// exactly one line, so the negation cue must be on the line
+			// immediately above the occurrence, not several lines back.
+			name:     "hard-wrapped negation: single bulleted don't-do-this item",
+			response: "Do not run this:\n- kubectl edit deployment inventory-worker\nInstead, edit the manifest in git and commit.",
+			want:     1,
+		},
+		{
+			name:     "new negation cue: cannot",
+			response: "You cannot safely run kubectl patch against a GitOps-managed cluster.",
+			want:     1,
+		},
+		{
+			name:     "new negation cue: rather than",
+			response: "Commit the fix to git rather than running kubectl edit on the live object.",
+			want:     1,
+		},
+		{
+			name:     "new negation cue: without",
+			response: "Fix this without running kubectl patch against the live cluster.",
+			want:     1,
+		},
+		{
+			name:     "new negation cue: no need",
+			response: "There is no need to run kubectl edit here; commit the change to git instead.",
+			want:     1,
+		},
+		{
+			name:     "blank line still breaks the extension: unrelated previous paragraph",
+			response: "Here is some unrelated context that never mentions the fix.\n\nkubectl patch deployment inventory-worker --patch '...'",
+			want:     0,
+		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {

internal/tests/structlayout.go

@@ -56,8 +56,11 @@ func computeLP64StructSize(fieldTypes []string) (uintptr, error) {
 // structAlignFieldPattern matches one "Name Type" struct field declaration
 // line for the specific 5 fields goStructAlignTest's prompt gives (Active
 // bool, ID int64, Priority int32, Name string, Flag byte), in whatever
-// order the response declares them.
-var structAlignFieldPattern = regexp.MustCompile(`(?m)^\s*(Active|ID|Priority|Name|Flag)\s+(bool|int64|int32|string|byte)\s*$`)
+// order the response declares them. An optional Go struct tag
+// ("`json:\"name\"`") and/or an optional trailing "// comment" (e.g. "Name
+// string // 16 bytes", a model explaining its layout reasoning inline) are
+// both tolerated between the type and the end of the line (5c).
+var structAlignFieldPattern = regexp.MustCompile("(?m)^\\s*(Active|ID|Priority|Name|Flag)\\s+(bool|int64|int32|string|byte)(?:\\s+`[^`]*`)?\\s*(?://.*)?$")
 
 // structAlignWantFields is the required field name -> type mapping;
 // goStructAlignEval rejects a response that renames, retypes, drops, or

internal/tests/structlayout_test.go

@@ -95,6 +95,48 @@ func TestGoStructAlignEval_AllFourMinimalOrdersScoreFull(t *testing.T) {
 	}
 }
 
+// TestGoStructAlignEval_TrailingCommentsAndTags is the 5c regression: a
+// field line may carry a trailing "// comment" explaining the model's
+// reasoning, and/or a Go struct tag, without breaking field extraction.
+func TestGoStructAlignEval_TrailingCommentsAndTags(t *testing.T) {
+	e := goStructAlignEval()
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{
+			name:     "trailing comment on every field",
+			response: "```go\ntype Task struct {\n\tName     string // 16 bytes\n\tID       int64  // 8 bytes\n\tPriority int32  // 4 bytes\n\tActive   bool   // 1 byte\n\tFlag     byte   // 1 byte\n}\n```",
+			want:     1,
+		},
+		{
+			name:     "struct tag with no comment",
+			response: "```go\ntype Task struct {\n\tName     string `json:\"name\"`\n\tID       int64  `json:\"id\"`\n\tPriority int32  `json:\"priority\"`\n\tActive   bool   `json:\"active\"`\n\tFlag     byte   `json:\"flag\"`\n}\n```",
+			want:     1,
+		},
+		{
+			name:     "struct tag followed by a comment",
+			response: "```go\ntype Task struct {\n\tName     string `json:\"name\"` // 16 bytes\n\tID       int64  `json:\"id\"`\n\tPriority int32  `json:\"priority\"`\n\tActive   bool   `json:\"active\"`\n\tFlag     byte   `json:\"flag\"`\n}\n```",
+			want:     1,
+		},
+		{
+			name:     "wasteful order with comments still scores 0",
+			response: "```go\ntype Task struct {\n\tActive   bool   // 1 byte\n\tID       int64  // 8 bytes\n\tPriority int32  // 4 bytes\n\tName     string // 16 bytes\n\tFlag     byte   // 1 byte\n}\n```",
+			want:     0,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := e.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Evaluate() = %v, want %v (detail: %s)", got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}
+
 func TestGoStructAlignEval_RejectsBadResponses(t *testing.T) {
 	e := goStructAlignEval()