mirror of
https://github.com/lukaszraczylo/llm-testbench.git
synced 2026-10-16 01:07:47 +00:00
test(catalog): expand programming to 10 tests per subcategory
Add 7 golang tests: channel deadlock spotting, defer/recover tracing, slice-append aliasing, method-set interface satisfaction, context cancellation propagation, a generics constraint fix, and bounded fan-out with first-error propagation. Add 8 python tests: softmax, asyncio.gather ordering, the mutable default argument bug, a dict comprehension trace, a pathlib rewrite, generator exhaustion, and JSON and regex log-processing scripts. Add 9 typescript tests: discriminated union narrowing, a recursive generic utility type, Promise.allSettled tracing, Vue 3 ref unwrapping, computed versus watch, ESLint flat-config migration, a tsconfig strict flag, and array-chain and nullish-coalescing tracing. Add 9 c tests: pointer arithmetic, bitmask operations, union sizing, a string-function program, integer promotion, array decay, a macro pitfall, an undefined-behavior spot, and bitfield packing. Each test carries a ground-truth derivation comment. Each unit test recomputes or cross-checks that ground truth against the real Go, Python, Node, or C toolchain where execution is possible. Golang, python, typescript, and c subcategories now register exactly 10 tests each. catalog_test.go's exact-map assertion now fails because it still expects 17 total tests; this is expected and out of scope, since that file is being replaced centrally.
This commit is contained in:
@@ -17,6 +17,13 @@ func registerGolangTests(r *testkit.Registry) {
|
||||
r.Register(goStructAlignTest())
|
||||
r.Register(goWorkerPoolTest())
|
||||
r.Register(goSemverClassifyTest())
|
||||
r.Register(goChannelDeadlockTest())
|
||||
r.Register(goDeferRecoverTraceTest())
|
||||
r.Register(goSliceAppendAliasingTest())
|
||||
r.Register(goMethodSetInterfaceTest())
|
||||
r.Register(goContextCancellationTraceTest())
|
||||
r.Register(goGenericsConstraintFixTest())
|
||||
r.Register(goBoundedFanoutTest())
|
||||
}
|
||||
|
||||
// goStructAlignTest: reorder struct fields to minimize padding on a 64-bit
|
||||
@@ -223,3 +230,457 @@ object: {"bump":"major|minor|patch"}`
|
||||
Eval: eval.JSONField("bump", "minor"),
|
||||
}
|
||||
}
|
||||
|
||||
// goChannelDeadlockReason is the fixed vocabulary of "why" answers
|
||||
// goChannelDeadlockTest allows, so the JSON "reason" field has exactly one
|
||||
// canonical correct literal instead of open-ended prose.
|
||||
const (
|
||||
goChannelDeadlockReasonUnbufferedNoReceiver = "unbuffered-send-no-receiver"
|
||||
)
|
||||
|
||||
// goChannelDeadlockTest: identify which line of an inline snippet blocks
|
||||
// forever, and why.
|
||||
//
|
||||
// ground truth: `ch := make(chan int)` creates an unbuffered channel, whose
|
||||
// send blocks until another goroutine is ready to receive concurrently. Line
|
||||
// 5 (`ch <- 1`) runs on main's single goroutine with no other goroutine
|
||||
// started, so the send blocks forever - the receive on line 6 can never run
|
||||
// because line 5 never returns. This is a single-goroutine self-deadlock,
|
||||
// not (for example) a nil-channel receive or a full buffered channel, so the
|
||||
// enumerated reason is "unbuffered-send-no-receiver". Because a genuine
|
||||
// deadlock hangs the process, golang_test.go cannot recompute this by
|
||||
// running the snippet (per PLAN.md, ground truth is derived in the doc
|
||||
// comment instead, as for goStructAlignTest's alignment arithmetic).
|
||||
func goChannelDeadlockTest() testkit.Test {
|
||||
prompt := `Here is Go code, with line numbers prefixed:
|
||||
|
||||
1: package main
|
||||
2:
|
||||
3: func main() {
|
||||
4: ch := make(chan int)
|
||||
5: ch <- 1
|
||||
6: v := <-ch
|
||||
7: println(v)
|
||||
8: }
|
||||
|
||||
Which line number blocks forever? Pick exactly one reason from this list:
|
||||
"unbuffered-send-no-receiver", "closed-channel-send", "nil-channel-receive",
|
||||
"buffered-channel-full". Respond with only a JSON object:
|
||||
{"line": <line number as an integer>, "reason": "<one of the four reasons above, exactly as written>"}`
|
||||
|
||||
evaluator := eval.All(
|
||||
eval.W(eval.JSONField("line", 5), 1),
|
||||
eval.W(eval.JSONField("reason", goChannelDeadlockReasonUnbufferedNoReceiver), 1),
|
||||
)
|
||||
|
||||
return testkit.Test{
|
||||
ID: "go-channel-deadlock",
|
||||
Category: "programming",
|
||||
Subcategory: "golang",
|
||||
Description: "Identify the line and reason a single-goroutine unbuffered-channel send deadlocks forever.",
|
||||
System: terseCodeOnly,
|
||||
Prompt: prompt,
|
||||
Eval: evaluator,
|
||||
}
|
||||
}
|
||||
|
||||
// goDeferRecoverTraceCode is the inline snippet for goDeferRecoverTraceTest.
|
||||
// It is duplicated (not shared via a Go source string constant referencing
|
||||
// a compiled function) so the prompt text is self-contained per EXPANSION.md
|
||||
// rule 1; golang_test.go defines an equivalent compute() and calls it
|
||||
// in-process to recompute the ground truth independently of this comment.
|
||||
const goDeferRecoverTraceCode = `package main
|
||||
|
||||
import "fmt"
|
||||
|
||||
func compute() (result int) {
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
result = -1
|
||||
}
|
||||
}()
|
||||
defer func() {
|
||||
result += 10
|
||||
}()
|
||||
panic("boom")
|
||||
}
|
||||
|
||||
func main() {
|
||||
fmt.Println(compute())
|
||||
}`
|
||||
|
||||
// goDeferRecoverTraceTest: trace the exact printed output of a function
|
||||
// combining defer, recover, and a named return value.
|
||||
//
|
||||
// ground truth: deferred functions run in LIFO (last-registered-first)
|
||||
// order. compute() panics with "boom" before returning, so both defers run
|
||||
// during unwinding: first (LIFO) the second-registered defer runs,
|
||||
// `result += 10`, taking the named return `result` from its zero value 0 to
|
||||
// 10 - the panic is still active at this point, since this defer never
|
||||
// calls recover. Then the first-registered defer runs, calls recover()
|
||||
// (which returns the non-nil panic value "boom", stopping the panic), and
|
||||
// overwrites `result = -1`. The named return is read after all defers
|
||||
// finish, so compute() returns -1 and main prints "-1". golang_test.go
|
||||
// verified this by literally running an equivalent compute() with `go run`
|
||||
// during authoring (see the ground-truth test below, which re-verifies it
|
||||
// in-process).
|
||||
func goDeferRecoverTraceTest() testkit.Test {
|
||||
prompt := `Here is a Go program:
|
||||
|
||||
` + "```go" + `
|
||||
` + goDeferRecoverTraceCode + `
|
||||
` + "```" + `
|
||||
|
||||
What does this program print when run? Respond with only the number,
|
||||
nothing else.`
|
||||
|
||||
return testkit.Test{
|
||||
ID: "go-defer-recover-trace",
|
||||
Category: "programming",
|
||||
Subcategory: "golang",
|
||||
Description: "Trace the exact output of a defer/recover chain that mutates a named return value.",
|
||||
Prompt: prompt,
|
||||
Eval: eval.Numeric(eval.ExtractLastNumber[int], -1, 0),
|
||||
}
|
||||
}
|
||||
|
||||
// goSliceAppendAliasingCode is the inline snippet for
|
||||
// goSliceAppendAliasingTest.
|
||||
const goSliceAppendAliasingCode = `package main
|
||||
|
||||
import "fmt"
|
||||
|
||||
func main() {
|
||||
a := make([]int, 3, 5)
|
||||
a[0], a[1], a[2] = 1, 2, 3
|
||||
b := a[:2]
|
||||
b = append(b, 99)
|
||||
_ = b
|
||||
fmt.Println(a[2])
|
||||
}`
|
||||
|
||||
// goSliceAppendAliasingTest: trace the exact printed output of a slice
|
||||
// aliasing gotcha where append writes into a shared backing array.
|
||||
//
|
||||
// ground truth: cap(a[:2]) = cap(a) - 0 = 5 (slicing a[low:high] gives
|
||||
// cap = cap(a) - low), so b := a[:2] has len 2 but cap 5, still sharing a's
|
||||
// backing array. append(b, 99) has spare capacity, so it does not allocate a
|
||||
// new array: it writes 99 into backing-array index 2 (len(b) before the
|
||||
// append) and returns a length-3 slice. Backing-array index 2 is also a[2],
|
||||
// so a[2] becomes 99 even though a itself was never explicitly assigned to.
|
||||
// golang_test.go re-verifies this in-process.
|
||||
func goSliceAppendAliasingTest() testkit.Test {
|
||||
prompt := `Here is a Go program:
|
||||
|
||||
` + "```go" + `
|
||||
` + goSliceAppendAliasingCode + `
|
||||
` + "```" + `
|
||||
|
||||
What does this program print when run? Respond with only the number,
|
||||
nothing else.`
|
||||
|
||||
return testkit.Test{
|
||||
ID: "go-slice-append-aliasing",
|
||||
Category: "programming",
|
||||
Subcategory: "golang",
|
||||
Description: "Trace the exact output of a slice-append aliasing gotcha on a shared backing array.",
|
||||
Prompt: prompt,
|
||||
Eval: eval.Numeric(eval.ExtractLastNumber[int], 99, 0),
|
||||
}
|
||||
}
|
||||
|
||||
// goMethodSetInterfaceCode is the inline snippet for
|
||||
// goMethodSetInterfaceTest: a type whose only interface method has a
|
||||
// pointer receiver, so only *Wrapper (not Wrapper) satisfies Stringer.
|
||||
const goMethodSetInterfaceCode = `type Stringer interface {
|
||||
String() string
|
||||
}
|
||||
|
||||
type Wrapper struct {
|
||||
val string
|
||||
}
|
||||
|
||||
func (w *Wrapper) String() string {
|
||||
return "Wrapper(" + w.val + ")"
|
||||
}`
|
||||
|
||||
// goMethodSetInterfaceTest: decide whether a value type and a pointer type
|
||||
// each satisfy an interface whose method has a pointer receiver.
|
||||
//
|
||||
// ground truth: a pointer receiver method belongs to the method set of the
|
||||
// pointer type only, not the value type (Go spec, "Method sets"). So
|
||||
// `Wrapper{...}` (a value) does not implement Stringer -
|
||||
// "Wrapper does not implement Stringer (method String has pointer
|
||||
// receiver)" - while `&Wrapper{...}` (a *Wrapper) does. golang_test.go
|
||||
// cross-checks both compile-time facts by running `go build` on each
|
||||
// snippet, guarded by go-toolchain presence (as c_test.go does for cc).
|
||||
func goMethodSetInterfaceTest() testkit.Test {
|
||||
prompt := `Here is Go code:
|
||||
|
||||
` + "```go" + `
|
||||
` + goMethodSetInterfaceCode + `
|
||||
` + "```" + `
|
||||
|
||||
Does this line compile: ` + "`var _ Stringer = Wrapper{val: \"hi\"}`" + `?
|
||||
Does this line compile: ` + "`var _ Stringer = &Wrapper{val: \"hi\"}`" + `?
|
||||
|
||||
Respond with only a JSON object:
|
||||
{"value_compiles": true|false, "pointer_compiles": true|false}`
|
||||
|
||||
evaluator := eval.All(
|
||||
eval.W(eval.JSONField("value_compiles", false), 1),
|
||||
eval.W(eval.JSONField("pointer_compiles", true), 1),
|
||||
)
|
||||
|
||||
return testkit.Test{
|
||||
ID: "go-method-set-interface",
|
||||
Category: "programming",
|
||||
Subcategory: "golang",
|
||||
Description: "Decide whether a value type and its pointer type satisfy an interface with a pointer-receiver method.",
|
||||
System: terseCodeOnly,
|
||||
Prompt: prompt,
|
||||
Eval: evaluator,
|
||||
}
|
||||
}
|
||||
|
||||
// goContextCancellationTraceCode is the inline snippet for
|
||||
// goContextCancellationTraceTest.
|
||||
const goContextCancellationTraceCode = `package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
)
|
||||
|
||||
func main() {
|
||||
parent, cancelParent := context.WithCancel(context.Background())
|
||||
child, cancelChild := context.WithCancel(parent)
|
||||
defer cancelChild()
|
||||
|
||||
cancelParent()
|
||||
|
||||
select {
|
||||
case <-child.Done():
|
||||
fmt.Println(child.Err())
|
||||
default:
|
||||
fmt.Println("not done")
|
||||
}
|
||||
}`
|
||||
|
||||
// goContextCancellationTraceTest: trace the exact printed output of
|
||||
// cancellation propagating from a parent context to a context derived from
|
||||
// it.
|
||||
//
|
||||
// ground truth: context.WithCancel propagates cancellation synchronously
|
||||
// down the derivation tree - cancelParent() closes parent's Done channel
|
||||
// and, in the same call, closes every context derived from it (including
|
||||
// child), before cancelParent() returns. So by the time the select runs,
|
||||
// child.Done() is already closed and takes the first case, printing
|
||||
// child.Err(). Err() returns the sentinel context.Canceled, whose message
|
||||
// (per the context package source) is "context canceled". golang_test.go
|
||||
// re-verifies this in-process, including that the propagation really is
|
||||
// synchronous (no goroutine/timing dependency, so the trace is
|
||||
// deterministic).
|
||||
func goContextCancellationTraceTest() testkit.Test {
|
||||
prompt := `Here is a Go program:
|
||||
|
||||
` + "```go" + `
|
||||
` + goContextCancellationTraceCode + `
|
||||
` + "```" + `
|
||||
|
||||
What does this program print when run? Respond with only the exact text
|
||||
printed, nothing else.`
|
||||
|
||||
return testkit.Test{
|
||||
ID: "go-context-cancellation-trace",
|
||||
Category: "programming",
|
||||
Subcategory: "golang",
|
||||
Description: "Trace the exact output of a context cancellation propagating synchronously from a parent to a derived context.",
|
||||
Prompt: prompt,
|
||||
Eval: eval.Equals("context canceled"),
|
||||
}
|
||||
}
|
||||
|
||||
// goGenericsConstraintFixHarness calls the response's fixed Max function
|
||||
// across int, float64, and string, verifying both the return values and
|
||||
// that it compiles at all (a broken constraint fails to compile, which
|
||||
// eval.GoRun scores as 0 via its "execution failed" / build-error path).
|
||||
const goGenericsConstraintFixHarness = `package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
)
|
||||
|
||||
func main() {
|
||||
if got := Max(3, 5); got != 5 {
|
||||
fmt.Fprintf(os.Stderr, "Max(3,5) = %v, want 5\n", got)
|
||||
os.Exit(1)
|
||||
}
|
||||
if got := Max(5, 3); got != 5 {
|
||||
fmt.Fprintf(os.Stderr, "Max(5,3) = %v, want 5\n", got)
|
||||
os.Exit(1)
|
||||
}
|
||||
if got := Max(2.5, 1.5); got != 2.5 {
|
||||
fmt.Fprintf(os.Stderr, "Max(2.5,1.5) = %v, want 2.5\n", got)
|
||||
os.Exit(1)
|
||||
}
|
||||
if got := Max("apple", "banana"); got != "banana" {
|
||||
fmt.Fprintf(os.Stderr, "Max(apple,banana) = %v, want banana\n", got)
|
||||
os.Exit(1)
|
||||
}
|
||||
fmt.Println("PASS")
|
||||
}
|
||||
`
|
||||
|
||||
// goGenericsConstraintFixTest: repair a generic function whose type
|
||||
// parameter constraint (any) does not support the operator (>) its body
|
||||
// uses, without depending on any external package.
|
||||
//
|
||||
// ground truth: the harness above is the oracle - it exercises Max across
|
||||
// three concrete types the fixed constraint must accept (int, float64,
|
||||
// string), so there is no separate hardcoded constant to derive.
|
||||
func goGenericsConstraintFixTest() testkit.Test {
|
||||
prompt := `This Go generic function does not compile:
|
||||
|
||||
` + "```go" + `
|
||||
func Max[T any](a, b T) T {
|
||||
if a > b {
|
||||
return a
|
||||
}
|
||||
return b
|
||||
}
|
||||
` + "```" + `
|
||||
|
||||
The compiler rejects it because type parameter T (constrained only by any)
|
||||
does not support the > operator. Fix it with the smallest change that makes
|
||||
it compile and work correctly for int, float64, and string arguments. Define
|
||||
any constraint interface you need yourself, using only the standard library
|
||||
- do not import golang.org/x/exp/constraints or any other third-party
|
||||
package. Respond with the complete corrected code (constraint interface, if
|
||||
any, plus the Max function).`
|
||||
|
||||
return testkit.Test{
|
||||
ID: "go-generics-constraint-fix",
|
||||
Category: "programming",
|
||||
Subcategory: "golang",
|
||||
Description: "Fix a generic function's constraint so > compiles and works for int, float64, and string.",
|
||||
System: terseCodeOnly,
|
||||
Prompt: prompt,
|
||||
Eval: eval.GoRun(goGenericsConstraintFixHarness, "PASS"),
|
||||
}
|
||||
}
|
||||
|
||||
// goBoundedFanoutHarness drives the response's BoundedRun with two batches
|
||||
// of tasks (all-succeed, then some-fail) over an instrumented task function
|
||||
// that tracks observed concurrency via an atomic counter, the same pattern
|
||||
// goWorkerPoolHarness uses. It checks: (a) the all-succeed batch returns
|
||||
// nil, (b) the some-fail batch returns a non-nil error, and (c) in both
|
||||
// batches, observed concurrency stayed within [2, limit] - never sequential
|
||||
// (>=2, evidencing real concurrency), never over the requested bound.
|
||||
const goBoundedFanoutHarness = `package main
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
)
|
||||
|
||||
func main() {
|
||||
const limit = 4
|
||||
const n = 20
|
||||
|
||||
var concurrent, maxConcurrent int32
|
||||
makeTask := func(shouldFail bool) func() error {
|
||||
return func() error {
|
||||
cur := atomic.AddInt32(&concurrent, 1)
|
||||
for {
|
||||
m := atomic.LoadInt32(&maxConcurrent)
|
||||
if cur <= m || atomic.CompareAndSwapInt32(&maxConcurrent, m, cur) {
|
||||
break
|
||||
}
|
||||
}
|
||||
time.Sleep(5 * time.Millisecond)
|
||||
atomic.AddInt32(&concurrent, -1)
|
||||
if shouldFail {
|
||||
return errors.New("boom")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
okTasks := make([]func() error, n)
|
||||
for i := range okTasks {
|
||||
okTasks[i] = makeTask(false)
|
||||
}
|
||||
if err := BoundedRun(okTasks, limit); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "BoundedRun(all ok) = %v, want nil\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
if got := atomic.LoadInt32(&maxConcurrent); got < 2 || got > limit {
|
||||
fmt.Fprintf(os.Stderr, "maxConcurrent=%d, want in [2, %d]\n", got, limit)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
atomic.StoreInt32(&concurrent, 0)
|
||||
atomic.StoreInt32(&maxConcurrent, 0)
|
||||
failTasks := make([]func() error, n)
|
||||
for i := range failTasks {
|
||||
failTasks[i] = makeTask(i%3 == 0)
|
||||
}
|
||||
err := BoundedRun(failTasks, limit)
|
||||
if err == nil {
|
||||
fmt.Fprintln(os.Stderr, "BoundedRun(some fail) = nil, want an error")
|
||||
os.Exit(1)
|
||||
}
|
||||
if got := atomic.LoadInt32(&maxConcurrent); got < 2 || got > limit {
|
||||
fmt.Fprintf(os.Stderr, "maxConcurrent=%d, want in [2, %d]\n", got, limit)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
fmt.Println("PASS")
|
||||
}
|
||||
`
|
||||
|
||||
// goBoundedFanoutTest: implement bounded-concurrency fan-out with
|
||||
// first-error propagation, the same shape as
|
||||
// golang.org/x/sync/errgroup.Group.SetLimit but implemented directly, so
|
||||
// the exec evaluator never depends on an external module being resolvable
|
||||
// in the sandbox (EXPANSION.md permits substituting an equivalent-difficulty
|
||||
// topic when the literal one cannot be made deterministic offline).
|
||||
//
|
||||
// ground truth: the harness above is the oracle, exercising both
|
||||
// correctness (first error propagates, nil on all-success) and the
|
||||
// concurrency bound via the same observed-concurrency counter pattern as
|
||||
// goWorkerPoolHarness; there is no separate hardcoded constant to derive.
|
||||
func goBoundedFanoutTest() testkit.Test {
|
||||
prompt := `Implement this Go function:
|
||||
|
||||
` + "```go" + `
|
||||
func BoundedRun(tasks []func() error, limit int) error
|
||||
` + "```" + `
|
||||
|
||||
Requirements:
|
||||
- Run the tasks concurrently, but never more than limit at the same time.
|
||||
- If one or more tasks return a non-nil error, BoundedRun must return a
|
||||
non-nil error (any one of the failing tasks' errors is acceptable).
|
||||
- If every task returns nil, BoundedRun must return nil.
|
||||
- Use only the standard library (for example sync, sync/atomic, or
|
||||
channels). Do not import golang.org/x/sync/errgroup or any other
|
||||
third-party package.
|
||||
- You may include whatever standard-library imports your implementation
|
||||
needs. Do not include a package clause or a main function: respond with
|
||||
only the BoundedRun function definition (plus any imports it needs).`
|
||||
|
||||
return testkit.Test{
|
||||
ID: "go-bounded-fanout",
|
||||
Category: "programming",
|
||||
Subcategory: "golang",
|
||||
Description: "Implement bounded-concurrency fan-out with first-error propagation, using only the standard library.",
|
||||
System: terseCodeOnly,
|
||||
Prompt: prompt,
|
||||
Eval: eval.GoRun(goBoundedFanoutHarness, "PASS"),
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user