mirror of
https://github.com/lukaszraczylo/llm-testbench.git
synced 2026-10-12 00:45:19 +00:00
lukaszraczylo
7264fe81b7
Both reviewers found evaluators that scored correct answers as zero: hardcoded flag order, quote-style rigidity, missing multiline anchors, comma-grouped numbers, ratio notation, and numeric strings in JSON. A shared ExactToken evaluator now normalizes fences, quotes, and trailing periods for exact-answer tests. The obsolete tmutil snapshot reference now accepts the current localsnapshot verb.
737 lines
20 KiB
Go
737 lines
20 KiB
Go
package tests
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"os"
|
|
"os/exec"
|
|
"path/filepath"
|
|
"strings"
|
|
"sync"
|
|
"sync/atomic"
|
|
"testing"
|
|
"time"
|
|
"unsafe"
|
|
)
|
|
|
|
// wastefulTask and optimalTask mirror the struct in goStructAlignTest's
|
|
// prompt (wasteful field order) and its minimal-padding reordering. Sizes
|
|
// are asserted with unsafe.Sizeof to ground the "ground truth:" derivation
|
|
// comment in golang.go against the actual compiler layout, per PLAN.md's
|
|
// requirement to recompute cheap ground truths in the unit test.
|
|
//
|
|
// wastefulTask deliberately keeps the prompt's wasteful field order
|
|
// verbatim (the "before" layout), so fieldalignment's auto-fix must not be
|
|
// allowed to reorder it - that would make it identical to optimalTask and
|
|
// defeat this test's comparison.
|
|
//
|
|
//nolint:govet // fieldalignment: intentional, see comment above
|
|
type wastefulTask struct {
|
|
Active bool
|
|
ID int64
|
|
Priority int32
|
|
Name string
|
|
Flag byte
|
|
}
|
|
|
|
type optimalTask struct {
|
|
Name string
|
|
ID int64
|
|
Priority int32
|
|
Active bool
|
|
Flag byte
|
|
}
|
|
|
|
func TestGoStructAlign_GroundTruthSizes(t *testing.T) {
|
|
if got, want := unsafe.Sizeof(wastefulTask{}), uintptr(48); got != want {
|
|
t.Errorf("unsafe.Sizeof(wastefulTask{}) = %d, want %d", got, want)
|
|
}
|
|
if got, want := unsafe.Sizeof(optimalTask{}), uintptr(32); got != want {
|
|
t.Errorf("unsafe.Sizeof(optimalTask{}) = %d, want %d", got, want)
|
|
}
|
|
}
|
|
|
|
func TestGoStructAlignTest_Eval(t *testing.T) {
|
|
tc := goStructAlignTest()
|
|
|
|
tests := []struct {
|
|
name string
|
|
response string
|
|
want float64
|
|
}{
|
|
{
|
|
name: "optimal order Name then ID",
|
|
response: "```go\ntype Task struct {\n\tName string\n\tID int64\n\tPriority int32\n\tActive bool\n\tFlag byte\n}\n```",
|
|
want: 1,
|
|
},
|
|
{
|
|
name: "optimal order ID then Name",
|
|
response: "```go\ntype Task struct {\n\tID int64\n\tName string\n\tPriority int32\n\tFlag byte\n\tActive bool\n}\n```",
|
|
want: 1,
|
|
},
|
|
{
|
|
name: "original wasteful order",
|
|
response: "```go\ntype Task struct {\n\tActive bool\n\tID int64\n\tPriority int32\n\tName string\n\tFlag byte\n}\n```",
|
|
want: 0,
|
|
},
|
|
{
|
|
name: "int32 not moved after align-8 fields",
|
|
response: "```go\ntype Task struct {\n\tPriority int32\n\tName string\n\tID int64\n\tActive bool\n\tFlag byte\n}\n```",
|
|
want: 0,
|
|
},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
|
if got.Value != tt.want {
|
|
t.Errorf("Eval.Evaluate() = %v, want %v (detail: %s)", got.Value, tt.want, got.Detail)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestGoWorkerPoolTest_Eval(t *testing.T) {
|
|
tc := goWorkerPoolTest()
|
|
|
|
correctPool := "```go\n" + `func Pool[T, R any](in []T, workers int, f func(T) R) []R {
|
|
out := make([]R, len(in))
|
|
sem := make(chan struct{}, workers)
|
|
done := make(chan struct{})
|
|
for i, v := range in {
|
|
i, v := i, v
|
|
sem <- struct{}{}
|
|
go func() {
|
|
defer func() { <-sem; done <- struct{}{} }()
|
|
out[i] = f(v)
|
|
}()
|
|
}
|
|
for range in {
|
|
<-done
|
|
}
|
|
return out
|
|
}
|
|
` + "```"
|
|
|
|
sequentialPool := "```go\n" + `func Pool[T, R any](in []T, workers int, f func(T) R) []R {
|
|
out := make([]R, len(in))
|
|
for i, v := range in {
|
|
out[i] = f(v)
|
|
}
|
|
return out
|
|
}
|
|
` + "```"
|
|
|
|
tests := []struct {
|
|
name string
|
|
response string
|
|
want float64
|
|
}{
|
|
{"correct concurrent pool", correctPool, 1},
|
|
{"sequential pool fails the maxConcurrent>=2 check", sequentialPool, 0},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
|
if got.Skipped {
|
|
t.Skip("go toolchain not available, skipping exec-evaluator assertion")
|
|
}
|
|
if got.Value != tt.want {
|
|
t.Errorf("Eval.Evaluate() = %v, want %v (detail: %s)", got.Value, tt.want, got.Detail)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// TestGoWorkerPoolTest_Eval_ResponseShapes covers B1: the two-file
|
|
// (solution.go + harness.go) split must accept a response that brings its
|
|
// own imports, whether or not it also carries a package clause, since
|
|
// eval.GoRun strips at most a leading package clause and gives the
|
|
// response its own import block separate from the harness's.
|
|
func TestGoWorkerPoolTest_Eval_ResponseShapes(t *testing.T) {
|
|
tc := goWorkerPoolTest()
|
|
|
|
// Shape 1: a bare function body with its own "import" block and no
|
|
// package clause - the common case when a model obeys "respond with
|
|
// only the function definition".
|
|
withImportsNoPackage := "```go\n" + `import "sync"
|
|
|
|
func Pool[T, R any](in []T, workers int, f func(T) R) []R {
|
|
out := make([]R, len(in))
|
|
var wg sync.WaitGroup
|
|
sem := make(chan struct{}, workers)
|
|
for i, v := range in {
|
|
i, v := i, v
|
|
wg.Add(1)
|
|
sem <- struct{}{}
|
|
go func() {
|
|
defer wg.Done()
|
|
defer func() { <-sem }()
|
|
out[i] = f(v)
|
|
}()
|
|
}
|
|
wg.Wait()
|
|
return out
|
|
}
|
|
` + "```"
|
|
|
|
// Shape 2: a full, standalone file - package clause, imports, and the
|
|
// function - as some models emit even when told not to include a
|
|
// package clause. eval.GoRun must strip the leading "package main" line
|
|
// and still compile it as part of solution.go.
|
|
fullFileWithPackageClause := "```go\n" + `package main
|
|
|
|
import "sync"
|
|
|
|
func Pool[T, R any](in []T, workers int, f func(T) R) []R {
|
|
out := make([]R, len(in))
|
|
var wg sync.WaitGroup
|
|
sem := make(chan struct{}, workers)
|
|
for i, v := range in {
|
|
i, v := i, v
|
|
wg.Add(1)
|
|
sem <- struct{}{}
|
|
go func() {
|
|
defer wg.Done()
|
|
defer func() { <-sem }()
|
|
out[i] = f(v)
|
|
}()
|
|
}
|
|
wg.Wait()
|
|
return out
|
|
}
|
|
` + "```"
|
|
|
|
// Shape 3: uses sync.WaitGroup without importing "sync" - now that the
|
|
// prompt explicitly allows imports, this is a genuine model mistake,
|
|
// not a harness limitation. It must fail to compile (score 0), not be
|
|
// silently patched or Skipped.
|
|
usesSyncWithoutImport := "```go\n" + `func Pool[T, R any](in []T, workers int, f func(T) R) []R {
|
|
out := make([]R, len(in))
|
|
var wg sync.WaitGroup
|
|
for i, v := range in {
|
|
wg.Add(1)
|
|
i, v := i, v
|
|
go func() {
|
|
defer wg.Done()
|
|
out[i] = f(v)
|
|
}()
|
|
}
|
|
wg.Wait()
|
|
return out
|
|
}
|
|
` + "```"
|
|
|
|
tests := []struct {
|
|
name string
|
|
response string
|
|
want float64
|
|
}{
|
|
{"imports, no package clause: PASS", withImportsNoPackage, 1},
|
|
{"full file with package clause: PASS after strip", fullFileWithPackageClause, 1},
|
|
{"uses sync without importing it: fails to compile", usesSyncWithoutImport, 0},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
|
if got.Skipped {
|
|
t.Skip("go toolchain not available, skipping exec-evaluator assertion")
|
|
}
|
|
if got.Value != tt.want {
|
|
t.Errorf("Eval.Evaluate() = %v, want %v (detail: %s)", got.Value, tt.want, got.Detail)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestGoSemverClassifyTest_Eval(t *testing.T) {
|
|
tc := goSemverClassifyTest()
|
|
|
|
tests := []struct {
|
|
name string
|
|
response string
|
|
want float64
|
|
}{
|
|
{"correct minor", `{"bump":"minor"}`, 1},
|
|
{"correct minor fenced", "```json\n{\"bump\": \"minor\"}\n```", 1},
|
|
{"trap: falls for the word breaking and answers major", `{"bump":"major"}`, 0},
|
|
{"wrong: answers patch, missing the feat commit", `{"bump":"patch"}`, 0},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
|
if got.Value != tt.want {
|
|
t.Errorf("Eval.Evaluate() = %v, want %v (detail: %s)", got.Value, tt.want, got.Detail)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestGoChannelDeadlockTest_Eval(t *testing.T) {
|
|
tc := goChannelDeadlockTest()
|
|
|
|
tests := []struct {
|
|
name string
|
|
response string
|
|
want float64
|
|
}{
|
|
{"exact correct", `{"line": 5, "reason": "unbuffered-send-no-receiver"}`, 1},
|
|
{"correct fenced json", "```json\n{\"line\": 5, \"reason\": \"unbuffered-send-no-receiver\"}\n```", 1},
|
|
{"correct with prose wrapper", `The answer is: {"line": 5, "reason": "unbuffered-send-no-receiver"}`, 1},
|
|
{"wrong line", `{"line": 6, "reason": "unbuffered-send-no-receiver"}`, 0.5},
|
|
{"wrong reason", `{"line": 5, "reason": "nil-channel-receive"}`, 0.5},
|
|
{"both wrong", `{"line": 6, "reason": "nil-channel-receive"}`, 0},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
|
if got.Value != tt.want {
|
|
t.Errorf("Eval.Evaluate() = %v, want %v (detail: %s)", got.Value, tt.want, got.Detail)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// computeDeferRecover mirrors goDeferRecoverTraceCode's compute() exactly,
|
|
// so its return value grounds the "ground truth:" derivation in golang.go
|
|
// by actually running the same logic, per PLAN.md's recomputation rule.
|
|
func computeDeferRecover() (result int) {
|
|
defer func() {
|
|
if r := recover(); r != nil {
|
|
result = -1
|
|
}
|
|
}()
|
|
defer func() {
|
|
result += 10
|
|
}()
|
|
panic("boom")
|
|
}
|
|
|
|
func TestGoDeferRecoverTrace_GroundTruth(t *testing.T) {
|
|
if got, want := computeDeferRecover(), -1; got != want {
|
|
t.Errorf("computeDeferRecover() = %d, want %d", got, want)
|
|
}
|
|
}
|
|
|
|
func TestGoDeferRecoverTraceTest_Eval(t *testing.T) {
|
|
tc := goDeferRecoverTraceTest()
|
|
|
|
tests := []struct {
|
|
name string
|
|
response string
|
|
want float64
|
|
}{
|
|
{"exact correct", "-1", 1},
|
|
{"correct with prose", "The program prints -1.", 1},
|
|
{"correct with label", "Output: -1", 1},
|
|
{"wrong: ignores recover overwrite", "10", 0},
|
|
{"wrong: sign error", "1", 0},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
|
if got.Value != tt.want {
|
|
t.Errorf("Eval.Evaluate() = %v, want %v (detail: %s)", got.Value, tt.want, got.Detail)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// sliceAppendAliasCheck mirrors goSliceAppendAliasingCode's main() body,
|
|
// returning a[2] instead of printing it, so the ground truth is recomputed
|
|
// in-process rather than trusted from the doc comment alone.
|
|
func sliceAppendAliasCheck() int {
|
|
a := make([]int, 3, 5)
|
|
a[0], a[1], a[2] = 1, 2, 3
|
|
b := a[:2]
|
|
b = append(b, 99)
|
|
_ = b
|
|
return a[2]
|
|
}
|
|
|
|
func TestGoSliceAppendAliasing_GroundTruth(t *testing.T) {
|
|
if got, want := sliceAppendAliasCheck(), 99; got != want {
|
|
t.Errorf("sliceAppendAliasCheck() = %d, want %d", got, want)
|
|
}
|
|
}
|
|
|
|
func TestGoSliceAppendAliasingTest_Eval(t *testing.T) {
|
|
tc := goSliceAppendAliasingTest()
|
|
|
|
tests := []struct {
|
|
name string
|
|
response string
|
|
want float64
|
|
}{
|
|
{"exact correct", "99", 1},
|
|
{"correct with prose", "a[2] is 99.", 1},
|
|
{"correct with label", "The result is 99", 1},
|
|
{"wrong: ignores aliasing, keeps original value", "3", 0},
|
|
{"wrong: off-by-index confusion", "2", 0},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
|
if got.Value != tt.want {
|
|
t.Errorf("Eval.Evaluate() = %v, want %v (detail: %s)", got.Value, tt.want, got.Detail)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// TestGoMethodSetInterface_GroundTruth cross-checks the value_compiles=false,
|
|
// pointer_compiles=true claims in golang.go's doc comment by actually
|
|
// running `go build` on each snippet, mirroring c_test.go's
|
|
// TestCStructSizeWant_GroundTruth pattern of shelling out to the real
|
|
// toolchain rather than trusting hand analysis alone.
|
|
func TestGoMethodSetInterface_GroundTruth(t *testing.T) {
|
|
const preamble = `package main
|
|
|
|
` + goMethodSetInterfaceCode + `
|
|
|
|
`
|
|
tests := []struct {
|
|
name string
|
|
mainLine string
|
|
wantBuilds bool
|
|
}{
|
|
{"value type does not implement Stringer", `func main() { var _ Stringer = Wrapper{val: "hi"} }`, false},
|
|
{"pointer type implements Stringer", `func main() { var _ Stringer = &Wrapper{val: "hi"} }`, true},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
dir := t.TempDir()
|
|
src := preamble + tt.mainLine + "\n"
|
|
srcPath := filepath.Join(dir, "main.go")
|
|
if err := os.WriteFile(srcPath, []byte(src), 0o600); err != nil {
|
|
t.Fatalf("write main.go: %v", err)
|
|
}
|
|
// #nosec G204 -- srcPath is a fixed filename this test just wrote
|
|
// into its own t.TempDir(); not external input.
|
|
cmd := exec.Command("go", "build", "-o", filepath.Join(dir, "out"), srcPath) //nolint:gosec // see comment above
|
|
out, err := cmd.CombinedOutput()
|
|
builds := err == nil
|
|
if builds != tt.wantBuilds {
|
|
t.Errorf("go build success = %v, want %v (output: %s)", builds, tt.wantBuilds, out)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestGoMethodSetInterfaceTest_Eval(t *testing.T) {
|
|
tc := goMethodSetInterfaceTest()
|
|
|
|
tests := []struct {
|
|
name string
|
|
response string
|
|
want float64
|
|
}{
|
|
{"exact correct", `{"value_compiles": false, "pointer_compiles": true}`, 1},
|
|
{"correct fenced json", "```json\n{\"value_compiles\": false, \"pointer_compiles\": true}\n```", 1},
|
|
{"correct with prose wrapper", `Answer: {"value_compiles": false, "pointer_compiles": true}`, 1},
|
|
{"wrong: both true", `{"value_compiles": true, "pointer_compiles": true}`, 0.5},
|
|
{"wrong: both false", `{"value_compiles": false, "pointer_compiles": false}`, 0.5},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
|
if got.Value != tt.want {
|
|
t.Errorf("Eval.Evaluate() = %v, want %v (detail: %s)", got.Value, tt.want, got.Detail)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// contextCancellationTraceCheck mirrors goContextCancellationTraceCode's
|
|
// main() body, returning the text it would print instead of printing it, to
|
|
// recompute the ground truth in-process. It also grounds the "no
|
|
// goroutine/timing dependency" claim in golang.go's doc comment: this
|
|
// function has no goroutines or sleeps at all, so a deterministic result
|
|
// here demonstrates the propagation is synchronous.
|
|
func contextCancellationTraceCheck() string {
|
|
parent, cancelParent := context.WithCancel(context.Background())
|
|
child, cancelChild := context.WithCancel(parent)
|
|
defer cancelChild()
|
|
|
|
cancelParent()
|
|
|
|
select {
|
|
case <-child.Done():
|
|
return child.Err().Error()
|
|
default:
|
|
return "not done"
|
|
}
|
|
}
|
|
|
|
func TestGoContextCancellationTrace_GroundTruth(t *testing.T) {
|
|
if got, want := contextCancellationTraceCheck(), "context canceled"; got != want {
|
|
t.Errorf("contextCancellationTraceCheck() = %q, want %q", got, want)
|
|
}
|
|
}
|
|
|
|
func TestGoContextCancellationTraceTest_Eval(t *testing.T) {
|
|
tc := goContextCancellationTraceTest()
|
|
|
|
tests := []struct {
|
|
name string
|
|
response string
|
|
want float64
|
|
}{
|
|
{"exact correct", "context canceled", 1},
|
|
{"correct case-insensitive", "CONTEXT CANCELED", 1},
|
|
{"correct with surrounding whitespace", " context canceled ", 1},
|
|
// A7 regression: fenced/quoted decoration on the correct answer
|
|
// must still score full credit.
|
|
{"fenced", "```\ncontext canceled\n```", 1},
|
|
{"quoted with trailing period", `"context canceled".`, 1},
|
|
{"wrong: assumes select falls to default", "not done", 0},
|
|
{"wrong: wrong sentinel error", "context deadline exceeded", 0},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
|
if got.Value != tt.want {
|
|
t.Errorf("Eval.Evaluate() = %v, want %v (detail: %s)", got.Value, tt.want, got.Detail)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestGoGenericsConstraintFixTest_Eval(t *testing.T) {
|
|
tc := goGenericsConstraintFixTest()
|
|
|
|
fullOrderedConstraint := "```go\n" + `type Ordered interface {
|
|
~int | ~int8 | ~int16 | ~int32 | ~int64 |
|
|
~uint | ~uint8 | ~uint16 | ~uint32 | ~uint64 |
|
|
~float32 | ~float64 | ~string
|
|
}
|
|
|
|
func Max[T Ordered](a, b T) T {
|
|
if a > b {
|
|
return a
|
|
}
|
|
return b
|
|
}
|
|
` + "```"
|
|
|
|
minimalConstraint := "```go\n" + `type Ordered interface {
|
|
~int | ~float64 | ~string
|
|
}
|
|
|
|
func Max[T Ordered](a, b T) T {
|
|
if a > b {
|
|
return a
|
|
}
|
|
return b
|
|
}
|
|
` + "```"
|
|
|
|
stillBroken := "```go\n" + `func Max[T any](a, b T) T {
|
|
if a > b {
|
|
return a
|
|
}
|
|
return b
|
|
}
|
|
` + "```"
|
|
|
|
tests := []struct {
|
|
name string
|
|
response string
|
|
want float64
|
|
}{
|
|
{"full Ordered constraint covering all integer widths", fullOrderedConstraint, 1},
|
|
{"minimal constraint covering only the exercised types", minimalConstraint, 1},
|
|
{"still broken: constraint left as any", stillBroken, 0},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
|
if got.Skipped {
|
|
t.Skip("go toolchain not available, skipping exec-evaluator assertion")
|
|
}
|
|
if got.Value != tt.want {
|
|
t.Errorf("Eval.Evaluate() = %v, want %v (detail: %s)", got.Value, tt.want, got.Detail)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestGoBoundedFanoutTest_Eval(t *testing.T) {
|
|
tc := goBoundedFanoutTest()
|
|
|
|
correctBoundedRun := "```go\n" + `import "sync"
|
|
|
|
func BoundedRun(tasks []func() error, limit int) error {
|
|
sem := make(chan struct{}, limit)
|
|
var wg sync.WaitGroup
|
|
var mu sync.Mutex
|
|
var firstErr error
|
|
|
|
for _, task := range tasks {
|
|
task := task
|
|
sem <- struct{}{}
|
|
wg.Add(1)
|
|
go func() {
|
|
defer wg.Done()
|
|
defer func() { <-sem }()
|
|
if err := task(); err != nil {
|
|
mu.Lock()
|
|
if firstErr == nil {
|
|
firstErr = err
|
|
}
|
|
mu.Unlock()
|
|
}
|
|
}()
|
|
}
|
|
wg.Wait()
|
|
return firstErr
|
|
}
|
|
` + "```"
|
|
|
|
sequentialBoundedRun := "```go\n" + `func BoundedRun(tasks []func() error, limit int) error {
|
|
for _, task := range tasks {
|
|
if err := task(); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
` + "```"
|
|
|
|
unboundedBoundedRun := "```go\n" + `import "sync"
|
|
|
|
func BoundedRun(tasks []func() error, limit int) error {
|
|
var wg sync.WaitGroup
|
|
var mu sync.Mutex
|
|
var firstErr error
|
|
for _, task := range tasks {
|
|
task := task
|
|
wg.Add(1)
|
|
go func() {
|
|
defer wg.Done()
|
|
if err := task(); err != nil {
|
|
mu.Lock()
|
|
if firstErr == nil {
|
|
firstErr = err
|
|
}
|
|
mu.Unlock()
|
|
}
|
|
}()
|
|
}
|
|
wg.Wait()
|
|
return firstErr
|
|
}
|
|
` + "```"
|
|
|
|
tests := []struct {
|
|
name string
|
|
response string
|
|
want float64
|
|
}{
|
|
{"correct semaphore-bounded concurrency", correctBoundedRun, 1},
|
|
{"wrong: sequential, no concurrency", sequentialBoundedRun, 0},
|
|
{"wrong: unbounded, ignores limit", unboundedBoundedRun, 0},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
|
if got.Skipped {
|
|
t.Skip("go toolchain not available, skipping exec-evaluator assertion")
|
|
}
|
|
if got.Value != tt.want {
|
|
t.Errorf("Eval.Evaluate() = %v, want %v (detail: %s)", got.Value, tt.want, got.Detail)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// boundedRunReference is a package-local reference implementation of the
|
|
// same bounded-fan-out shape the harness expects (semaphore-limited
|
|
// concurrency, first-error propagation), independent of the fenced
|
|
// test-string copy in TestGoBoundedFanoutTest_Eval, so the harness's
|
|
// concurrency-bound assertion is grounded against real, in-process
|
|
// goroutines rather than only against exec.Command output.
|
|
func boundedRunReference(tasks []func() error, limit int) error {
|
|
sem := make(chan struct{}, limit)
|
|
var wg sync.WaitGroup
|
|
var mu sync.Mutex
|
|
var firstErr error
|
|
|
|
for _, task := range tasks {
|
|
task := task
|
|
sem <- struct{}{}
|
|
wg.Add(1)
|
|
go func() {
|
|
defer wg.Done()
|
|
defer func() { <-sem }()
|
|
if err := task(); err != nil {
|
|
mu.Lock()
|
|
if firstErr == nil {
|
|
firstErr = err
|
|
}
|
|
mu.Unlock()
|
|
}
|
|
}()
|
|
}
|
|
wg.Wait()
|
|
return firstErr
|
|
}
|
|
|
|
func TestBoundedRunReference_GroundTruth(t *testing.T) {
|
|
const limit = 4
|
|
const n = 20
|
|
|
|
var concurrent, maxConcurrent int32
|
|
makeTask := func(shouldFail bool) func() error {
|
|
return func() error {
|
|
cur := atomic.AddInt32(&concurrent, 1)
|
|
for {
|
|
m := atomic.LoadInt32(&maxConcurrent)
|
|
if cur <= m || atomic.CompareAndSwapInt32(&maxConcurrent, m, cur) {
|
|
break
|
|
}
|
|
}
|
|
time.Sleep(time.Millisecond)
|
|
atomic.AddInt32(&concurrent, -1)
|
|
if shouldFail {
|
|
return errors.New("boom")
|
|
}
|
|
return nil
|
|
}
|
|
}
|
|
|
|
okTasks := make([]func() error, n)
|
|
for i := range okTasks {
|
|
okTasks[i] = makeTask(false)
|
|
}
|
|
if err := boundedRunReference(okTasks, limit); err != nil {
|
|
t.Errorf("boundedRunReference(all ok) = %v, want nil", err)
|
|
}
|
|
if got := atomic.LoadInt32(&maxConcurrent); got < 2 || got > limit {
|
|
t.Errorf("maxConcurrent = %d, want in [2, %d]", got, limit)
|
|
}
|
|
|
|
atomic.StoreInt32(&concurrent, 0)
|
|
atomic.StoreInt32(&maxConcurrent, 0)
|
|
failTasks := make([]func() error, n)
|
|
for i := range failTasks {
|
|
failTasks[i] = makeTask(i%3 == 0)
|
|
}
|
|
if err := boundedRunReference(failTasks, limit); err == nil {
|
|
t.Error("boundedRunReference(some fail) = nil, want an error")
|
|
}
|
|
if got := atomic.LoadInt32(&maxConcurrent); got < 2 || got > limit {
|
|
t.Errorf("maxConcurrent = %d, want in [2, %d]", got, limit)
|
|
}
|
|
}
|
|
|
|
// TestGoChannelDeadlockCode_ContainsExpectedLines grounds the line-number
|
|
// claim in goChannelDeadlockTest's doc comment against the actual prompt
|
|
// text used at registration time, rather than trusting the doc comment's
|
|
// count independently of the real prompt.
|
|
func TestGoChannelDeadlockCode_ContainsExpectedLines(t *testing.T) {
|
|
tc := goChannelDeadlockTest()
|
|
if !strings.Contains(tc.Prompt, "5: \tch <- 1") {
|
|
t.Error("goChannelDeadlockTest prompt no longer has 'ch <- 1' on line 5; update the ground-truth line number")
|
|
}
|
|
}
|