mirror of
https://github.com/lukaszraczylo/llm-testbench.git
synced 2026-10-12 00:45:19 +00:00
test(catalog): expand programming to 10 tests per subcategory
Add 7 golang tests: channel deadlock spotting, defer/recover tracing, slice-append aliasing, method-set interface satisfaction, context cancellation propagation, a generics constraint fix, and bounded fan-out with first-error propagation. Add 8 python tests: softmax, asyncio.gather ordering, the mutable default argument bug, a dict comprehension trace, a pathlib rewrite, generator exhaustion, and JSON and regex log-processing scripts. Add 9 typescript tests: discriminated union narrowing, a recursive generic utility type, Promise.allSettled tracing, Vue 3 ref unwrapping, computed versus watch, ESLint flat-config migration, a tsconfig strict flag, and array-chain and nullish-coalescing tracing. Add 9 c tests: pointer arithmetic, bitmask operations, union sizing, a string-function program, integer promotion, array decay, a macro pitfall, an undefined-behavior spot, and bitfield packing. Each test carries a ground-truth derivation comment. Each unit test recomputes or cross-checks that ground truth against the real Go, Python, Node, or C toolchain where execution is possible. Golang, python, typescript, and c subcategories now register exactly 10 tests each. catalog_test.go's exact-map assertion now fails because it still expects 17 total tests; this is expected and out of scope, since that file is being replaced centrally.
This commit is contained in:
@@ -2,7 +2,15 @@ package tests
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
"unsafe"
|
||||
)
|
||||
|
||||
@@ -257,3 +265,468 @@ func TestGoSemverClassifyTest_Eval(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestGoChannelDeadlockTest_Eval(t *testing.T) {
|
||||
tc := goChannelDeadlockTest()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{"exact correct", `{"line": 5, "reason": "unbuffered-send-no-receiver"}`, 1},
|
||||
{"correct fenced json", "```json\n{\"line\": 5, \"reason\": \"unbuffered-send-no-receiver\"}\n```", 1},
|
||||
{"correct with prose wrapper", `The answer is: {"line": 5, "reason": "unbuffered-send-no-receiver"}`, 1},
|
||||
{"wrong line", `{"line": 6, "reason": "unbuffered-send-no-receiver"}`, 0.5},
|
||||
{"wrong reason", `{"line": 5, "reason": "nil-channel-receive"}`, 0.5},
|
||||
{"both wrong", `{"line": 6, "reason": "nil-channel-receive"}`, 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate() = %v, want %v (detail: %s)", got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// computeDeferRecover mirrors goDeferRecoverTraceCode's compute() exactly,
|
||||
// so its return value grounds the "ground truth:" derivation in golang.go
|
||||
// by actually running the same logic, per PLAN.md's recomputation rule.
|
||||
func computeDeferRecover() (result int) {
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
result = -1
|
||||
}
|
||||
}()
|
||||
defer func() {
|
||||
result += 10
|
||||
}()
|
||||
panic("boom")
|
||||
}
|
||||
|
||||
func TestGoDeferRecoverTrace_GroundTruth(t *testing.T) {
|
||||
if got, want := computeDeferRecover(), -1; got != want {
|
||||
t.Errorf("computeDeferRecover() = %d, want %d", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGoDeferRecoverTraceTest_Eval(t *testing.T) {
|
||||
tc := goDeferRecoverTraceTest()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{"exact correct", "-1", 1},
|
||||
{"correct with prose", "The program prints -1.", 1},
|
||||
{"correct with label", "Output: -1", 1},
|
||||
{"wrong: ignores recover overwrite", "10", 0},
|
||||
{"wrong: sign error", "1", 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate() = %v, want %v (detail: %s)", got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// sliceAppendAliasCheck mirrors goSliceAppendAliasingCode's main() body,
|
||||
// returning a[2] instead of printing it, so the ground truth is recomputed
|
||||
// in-process rather than trusted from the doc comment alone.
|
||||
func sliceAppendAliasCheck() int {
|
||||
a := make([]int, 3, 5)
|
||||
a[0], a[1], a[2] = 1, 2, 3
|
||||
b := a[:2]
|
||||
b = append(b, 99)
|
||||
_ = b
|
||||
return a[2]
|
||||
}
|
||||
|
||||
func TestGoSliceAppendAliasing_GroundTruth(t *testing.T) {
|
||||
if got, want := sliceAppendAliasCheck(), 99; got != want {
|
||||
t.Errorf("sliceAppendAliasCheck() = %d, want %d", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGoSliceAppendAliasingTest_Eval(t *testing.T) {
|
||||
tc := goSliceAppendAliasingTest()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{"exact correct", "99", 1},
|
||||
{"correct with prose", "a[2] is 99.", 1},
|
||||
{"correct with label", "The result is 99", 1},
|
||||
{"wrong: ignores aliasing, keeps original value", "3", 0},
|
||||
{"wrong: off-by-index confusion", "2", 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate() = %v, want %v (detail: %s)", got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestGoMethodSetInterface_GroundTruth cross-checks the value_compiles=false,
|
||||
// pointer_compiles=true claims in golang.go's doc comment by actually
|
||||
// running `go build` on each snippet, mirroring c_test.go's
|
||||
// TestCStructSizeWant_GroundTruth pattern of shelling out to the real
|
||||
// toolchain rather than trusting hand analysis alone.
|
||||
func TestGoMethodSetInterface_GroundTruth(t *testing.T) {
|
||||
const preamble = `package main
|
||||
|
||||
` + goMethodSetInterfaceCode + `
|
||||
|
||||
`
|
||||
tests := []struct {
|
||||
name string
|
||||
mainLine string
|
||||
wantBuilds bool
|
||||
}{
|
||||
{"value type does not implement Stringer", `func main() { var _ Stringer = Wrapper{val: "hi"} }`, false},
|
||||
{"pointer type implements Stringer", `func main() { var _ Stringer = &Wrapper{val: "hi"} }`, true},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
src := preamble + tt.mainLine + "\n"
|
||||
srcPath := filepath.Join(dir, "main.go")
|
||||
if err := os.WriteFile(srcPath, []byte(src), 0o600); err != nil {
|
||||
t.Fatalf("write main.go: %v", err)
|
||||
}
|
||||
// #nosec G204 -- srcPath is a fixed filename this test just wrote
|
||||
// into its own t.TempDir(); not external input.
|
||||
cmd := exec.Command("go", "build", "-o", filepath.Join(dir, "out"), srcPath) //nolint:gosec // see comment above
|
||||
out, err := cmd.CombinedOutput()
|
||||
builds := err == nil
|
||||
if builds != tt.wantBuilds {
|
||||
t.Errorf("go build success = %v, want %v (output: %s)", builds, tt.wantBuilds, out)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestGoMethodSetInterfaceTest_Eval(t *testing.T) {
|
||||
tc := goMethodSetInterfaceTest()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{"exact correct", `{"value_compiles": false, "pointer_compiles": true}`, 1},
|
||||
{"correct fenced json", "```json\n{\"value_compiles\": false, \"pointer_compiles\": true}\n```", 1},
|
||||
{"correct with prose wrapper", `Answer: {"value_compiles": false, "pointer_compiles": true}`, 1},
|
||||
{"wrong: both true", `{"value_compiles": true, "pointer_compiles": true}`, 0.5},
|
||||
{"wrong: both false", `{"value_compiles": false, "pointer_compiles": false}`, 0.5},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate() = %v, want %v (detail: %s)", got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// contextCancellationTraceCheck mirrors goContextCancellationTraceCode's
|
||||
// main() body, returning the text it would print instead of printing it, to
|
||||
// recompute the ground truth in-process. It also grounds the "no
|
||||
// goroutine/timing dependency" claim in golang.go's doc comment: this
|
||||
// function has no goroutines or sleeps at all, so a deterministic result
|
||||
// here demonstrates the propagation is synchronous.
|
||||
func contextCancellationTraceCheck() string {
|
||||
parent, cancelParent := context.WithCancel(context.Background())
|
||||
child, cancelChild := context.WithCancel(parent)
|
||||
defer cancelChild()
|
||||
|
||||
cancelParent()
|
||||
|
||||
select {
|
||||
case <-child.Done():
|
||||
return child.Err().Error()
|
||||
default:
|
||||
return "not done"
|
||||
}
|
||||
}
|
||||
|
||||
func TestGoContextCancellationTrace_GroundTruth(t *testing.T) {
|
||||
if got, want := contextCancellationTraceCheck(), "context canceled"; got != want {
|
||||
t.Errorf("contextCancellationTraceCheck() = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGoContextCancellationTraceTest_Eval(t *testing.T) {
|
||||
tc := goContextCancellationTraceTest()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{"exact correct", "context canceled", 1},
|
||||
{"correct case-insensitive", "CONTEXT CANCELED", 1},
|
||||
{"correct with surrounding whitespace", " context canceled ", 1},
|
||||
{"wrong: assumes select falls to default", "not done", 0},
|
||||
{"wrong: wrong sentinel error", "context deadline exceeded", 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate() = %v, want %v (detail: %s)", got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestGoGenericsConstraintFixTest_Eval(t *testing.T) {
|
||||
tc := goGenericsConstraintFixTest()
|
||||
|
||||
fullOrderedConstraint := "```go\n" + `type Ordered interface {
|
||||
~int | ~int8 | ~int16 | ~int32 | ~int64 |
|
||||
~uint | ~uint8 | ~uint16 | ~uint32 | ~uint64 |
|
||||
~float32 | ~float64 | ~string
|
||||
}
|
||||
|
||||
func Max[T Ordered](a, b T) T {
|
||||
if a > b {
|
||||
return a
|
||||
}
|
||||
return b
|
||||
}
|
||||
` + "```"
|
||||
|
||||
minimalConstraint := "```go\n" + `type Ordered interface {
|
||||
~int | ~float64 | ~string
|
||||
}
|
||||
|
||||
func Max[T Ordered](a, b T) T {
|
||||
if a > b {
|
||||
return a
|
||||
}
|
||||
return b
|
||||
}
|
||||
` + "```"
|
||||
|
||||
stillBroken := "```go\n" + `func Max[T any](a, b T) T {
|
||||
if a > b {
|
||||
return a
|
||||
}
|
||||
return b
|
||||
}
|
||||
` + "```"
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{"full Ordered constraint covering all integer widths", fullOrderedConstraint, 1},
|
||||
{"minimal constraint covering only the exercised types", minimalConstraint, 1},
|
||||
{"still broken: constraint left as any", stillBroken, 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Skipped {
|
||||
t.Skip("go toolchain not available, skipping exec-evaluator assertion")
|
||||
}
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate() = %v, want %v (detail: %s)", got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestGoBoundedFanoutTest_Eval(t *testing.T) {
|
||||
tc := goBoundedFanoutTest()
|
||||
|
||||
correctBoundedRun := "```go\n" + `import "sync"
|
||||
|
||||
func BoundedRun(tasks []func() error, limit int) error {
|
||||
sem := make(chan struct{}, limit)
|
||||
var wg sync.WaitGroup
|
||||
var mu sync.Mutex
|
||||
var firstErr error
|
||||
|
||||
for _, task := range tasks {
|
||||
task := task
|
||||
sem <- struct{}{}
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
defer func() { <-sem }()
|
||||
if err := task(); err != nil {
|
||||
mu.Lock()
|
||||
if firstErr == nil {
|
||||
firstErr = err
|
||||
}
|
||||
mu.Unlock()
|
||||
}
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
return firstErr
|
||||
}
|
||||
` + "```"
|
||||
|
||||
sequentialBoundedRun := "```go\n" + `func BoundedRun(tasks []func() error, limit int) error {
|
||||
for _, task := range tasks {
|
||||
if err := task(); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
` + "```"
|
||||
|
||||
unboundedBoundedRun := "```go\n" + `import "sync"
|
||||
|
||||
func BoundedRun(tasks []func() error, limit int) error {
|
||||
var wg sync.WaitGroup
|
||||
var mu sync.Mutex
|
||||
var firstErr error
|
||||
for _, task := range tasks {
|
||||
task := task
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
if err := task(); err != nil {
|
||||
mu.Lock()
|
||||
if firstErr == nil {
|
||||
firstErr = err
|
||||
}
|
||||
mu.Unlock()
|
||||
}
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
return firstErr
|
||||
}
|
||||
` + "```"
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{"correct semaphore-bounded concurrency", correctBoundedRun, 1},
|
||||
{"wrong: sequential, no concurrency", sequentialBoundedRun, 0},
|
||||
{"wrong: unbounded, ignores limit", unboundedBoundedRun, 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Skipped {
|
||||
t.Skip("go toolchain not available, skipping exec-evaluator assertion")
|
||||
}
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate() = %v, want %v (detail: %s)", got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// boundedRunReference is a package-local reference implementation of the
|
||||
// same bounded-fan-out shape the harness expects (semaphore-limited
|
||||
// concurrency, first-error propagation), independent of the fenced
|
||||
// test-string copy in TestGoBoundedFanoutTest_Eval, so the harness's
|
||||
// concurrency-bound assertion is grounded against real, in-process
|
||||
// goroutines rather than only against exec.Command output.
|
||||
func boundedRunReference(tasks []func() error, limit int) error {
|
||||
sem := make(chan struct{}, limit)
|
||||
var wg sync.WaitGroup
|
||||
var mu sync.Mutex
|
||||
var firstErr error
|
||||
|
||||
for _, task := range tasks {
|
||||
task := task
|
||||
sem <- struct{}{}
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
defer func() { <-sem }()
|
||||
if err := task(); err != nil {
|
||||
mu.Lock()
|
||||
if firstErr == nil {
|
||||
firstErr = err
|
||||
}
|
||||
mu.Unlock()
|
||||
}
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
return firstErr
|
||||
}
|
||||
|
||||
func TestBoundedRunReference_GroundTruth(t *testing.T) {
|
||||
const limit = 4
|
||||
const n = 20
|
||||
|
||||
var concurrent, maxConcurrent int32
|
||||
makeTask := func(shouldFail bool) func() error {
|
||||
return func() error {
|
||||
cur := atomic.AddInt32(&concurrent, 1)
|
||||
for {
|
||||
m := atomic.LoadInt32(&maxConcurrent)
|
||||
if cur <= m || atomic.CompareAndSwapInt32(&maxConcurrent, m, cur) {
|
||||
break
|
||||
}
|
||||
}
|
||||
time.Sleep(time.Millisecond)
|
||||
atomic.AddInt32(&concurrent, -1)
|
||||
if shouldFail {
|
||||
return errors.New("boom")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
okTasks := make([]func() error, n)
|
||||
for i := range okTasks {
|
||||
okTasks[i] = makeTask(false)
|
||||
}
|
||||
if err := boundedRunReference(okTasks, limit); err != nil {
|
||||
t.Errorf("boundedRunReference(all ok) = %v, want nil", err)
|
||||
}
|
||||
if got := atomic.LoadInt32(&maxConcurrent); got < 2 || got > limit {
|
||||
t.Errorf("maxConcurrent = %d, want in [2, %d]", got, limit)
|
||||
}
|
||||
|
||||
atomic.StoreInt32(&concurrent, 0)
|
||||
atomic.StoreInt32(&maxConcurrent, 0)
|
||||
failTasks := make([]func() error, n)
|
||||
for i := range failTasks {
|
||||
failTasks[i] = makeTask(i%3 == 0)
|
||||
}
|
||||
if err := boundedRunReference(failTasks, limit); err == nil {
|
||||
t.Error("boundedRunReference(some fail) = nil, want an error")
|
||||
}
|
||||
if got := atomic.LoadInt32(&maxConcurrent); got < 2 || got > limit {
|
||||
t.Errorf("maxConcurrent = %d, want in [2, %d]", got, limit)
|
||||
}
|
||||
}
|
||||
|
||||
// TestGoChannelDeadlockCode_ContainsExpectedLines grounds the line-number
|
||||
// claim in goChannelDeadlockTest's doc comment against the actual prompt
|
||||
// text used at registration time, rather than trusting the doc comment's
|
||||
// count independently of the real prompt.
|
||||
func TestGoChannelDeadlockCode_ContainsExpectedLines(t *testing.T) {
|
||||
tc := goChannelDeadlockTest()
|
||||
if !strings.Contains(tc.Prompt, "5: \tch <- 1") {
|
||||
t.Error("goChannelDeadlockTest prompt no longer has 'ch <- 1' on line 5; update the ground-truth line number")
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user