fix: floor token budgets and surface truncation, add live progress

Reasoning models burn thousands of completion tokens before the
answer. Small per-test caps caused silent truncation and zero scores.
The runner now floors max_tokens at the config default (12000). The
client parses finish_reason. Reports mark truncated cells and the
JSON format carries response text for debugging. A stderr progress
reporter prints one line per completed request; --quiet disables it.
This commit is contained in:
2026-08-17 01:55:14 +01:00
parent 764fe990fd
commit ace50757e4
30 changed files with 1063 additions and 184 deletions
+42
View File
@@ -95,6 +95,48 @@ func TestGoStructAlignEval_AllFourMinimalOrdersScoreFull(t *testing.T) {
}
}
// TestGoStructAlignEval_TrailingCommentsAndTags is the 5c regression: a
// field line may carry a trailing "// comment" explaining the model's
// reasoning, and/or a Go struct tag, without breaking field extraction.
func TestGoStructAlignEval_TrailingCommentsAndTags(t *testing.T) {
e := goStructAlignEval()
tests := []struct {
name string
response string
want float64
}{
{
name: "trailing comment on every field",
response: "```go\ntype Task struct {\n\tName string // 16 bytes\n\tID int64 // 8 bytes\n\tPriority int32 // 4 bytes\n\tActive bool // 1 byte\n\tFlag byte // 1 byte\n}\n```",
want: 1,
},
{
name: "struct tag with no comment",
response: "```go\ntype Task struct {\n\tName string `json:\"name\"`\n\tID int64 `json:\"id\"`\n\tPriority int32 `json:\"priority\"`\n\tActive bool `json:\"active\"`\n\tFlag byte `json:\"flag\"`\n}\n```",
want: 1,
},
{
name: "struct tag followed by a comment",
response: "```go\ntype Task struct {\n\tName string `json:\"name\"` // 16 bytes\n\tID int64 `json:\"id\"`\n\tPriority int32 `json:\"priority\"`\n\tActive bool `json:\"active\"`\n\tFlag byte `json:\"flag\"`\n}\n```",
want: 1,
},
{
name: "wasteful order with comments still scores 0",
response: "```go\ntype Task struct {\n\tActive bool // 1 byte\n\tID int64 // 8 bytes\n\tPriority int32 // 4 bytes\n\tName string // 16 bytes\n\tFlag byte // 1 byte\n}\n```",
want: 0,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := e.Evaluate(context.Background(), tt.response)
if got.Value != tt.want {
t.Errorf("Evaluate() = %v, want %v (detail: %s)", got.Value, tt.want, got.Detail)
}
})
}
}
func TestGoStructAlignEval_RejectsBadResponses(t *testing.T) {
e := goStructAlignEval()