mirror of
https://github.com/lukaszraczylo/llm-testbench.git
synced 2026-10-14 00:57:45 +00:00
test(catalog): expand programming to 10 tests per subcategory
Add 7 golang tests: channel deadlock spotting, defer/recover tracing, slice-append aliasing, method-set interface satisfaction, context cancellation propagation, a generics constraint fix, and bounded fan-out with first-error propagation. Add 8 python tests: softmax, asyncio.gather ordering, the mutable default argument bug, a dict comprehension trace, a pathlib rewrite, generator exhaustion, and JSON and regex log-processing scripts. Add 9 typescript tests: discriminated union narrowing, a recursive generic utility type, Promise.allSettled tracing, Vue 3 ref unwrapping, computed versus watch, ESLint flat-config migration, a tsconfig strict flag, and array-chain and nullish-coalescing tracing. Add 9 c tests: pointer arithmetic, bitmask operations, union sizing, a string-function program, integer promotion, array decay, a macro pitfall, an undefined-behavior spot, and bitfield packing. Each test carries a ground-truth derivation comment. Each unit test recomputes or cross-checks that ground truth against the real Go, Python, Node, or C toolchain where execution is possible. Golang, python, typescript, and c subcategories now register exactly 10 tests each. catalog_test.go's exact-map assertion now fails because it still expects 17 total tests; this is expected and out of scope, since that file is being replaced centrally.
This commit is contained in:
@@ -3,8 +3,14 @@ package tests
|
||||
import (
|
||||
"context"
|
||||
"math"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/lukaszraczylo/llm-testbench/internal/eval"
|
||||
)
|
||||
|
||||
func TestCosineSimilarity_KnownCases(t *testing.T) {
|
||||
@@ -108,6 +114,398 @@ for key in sorted(counts):
|
||||
}
|
||||
}
|
||||
|
||||
func TestSoftmaxFirst_KnownCases(t *testing.T) {
|
||||
// Grounds the softmaxFirst helper itself before trusting it to derive
|
||||
// pySoftmaxWant.
|
||||
tests := []struct {
|
||||
name string
|
||||
v []float64
|
||||
want float64
|
||||
}{
|
||||
{"uniform vector splits evenly", []float64{1, 1, 1}, 1.0 / 3.0},
|
||||
{"single element gets all the mass", []float64{5}, 1},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := softmaxFirst(tt.v)
|
||||
if math.Abs(got-tt.want) > 1e-9 {
|
||||
t.Errorf("softmaxFirst(%v) = %v, want %v", tt.v, got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestPySoftmaxWant_GroundTruth(t *testing.T) {
|
||||
// Independent re-derivation via math.Exp, not via the softmaxFirst
|
||||
// helper under test, per PLAN.md's rule to recompute cheap ground
|
||||
// truths in the unit test.
|
||||
v := pySoftmaxVector
|
||||
var sum float64
|
||||
for _, x := range v {
|
||||
sum += math.Exp(x)
|
||||
}
|
||||
want := math.Exp(v[0]) / sum
|
||||
wantRounded := math.Round(want*10000) / 10000
|
||||
|
||||
if pySoftmaxWant != wantRounded {
|
||||
t.Errorf("pySoftmaxWant = %v, independently recomputed = %v", pySoftmaxWant, wantRounded)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPySoftmaxTest_Eval(t *testing.T) {
|
||||
tc := pySoftmaxTest()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{"exact", "0.5585", 1},
|
||||
{"within tolerance", "0.5588", 1},
|
||||
{"prose wrapped", "The softmax probability is 0.5585.", 1},
|
||||
{"outside tolerance", "0.2", 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate() = %v, want %v (detail: %s)", got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestPyAsyncioGatherTrace_GroundTruth(t *testing.T) {
|
||||
// Runs the identical Python snippet embedded in pyAsyncioGatherTraceCode
|
||||
// via python3 and checks the print order is exactly B, A, C - the same
|
||||
// oracle mechanism as eval.PyRun, independent of the doc-comment
|
||||
// reasoning in python.go.
|
||||
if _, err := exec.LookPath("python3"); err != nil {
|
||||
t.Skip("python3 not found on PATH, skipping ground-truth cross-check")
|
||||
}
|
||||
|
||||
dir := t.TempDir()
|
||||
scriptPath := filepath.Join(dir, "gather.py")
|
||||
if err := os.WriteFile(scriptPath, []byte(pyAsyncioGatherTraceCode), 0o600); err != nil {
|
||||
t.Fatalf("write gather.py: %v", err)
|
||||
}
|
||||
|
||||
// #nosec G204 -- scriptPath is a fixed filename this test just wrote
|
||||
// into its own t.TempDir(); not external input.
|
||||
out, err := exec.Command("python3", scriptPath).Output() //nolint:gosec // see comment above
|
||||
if err != nil {
|
||||
t.Fatalf("running gather.py failed: %v", err)
|
||||
}
|
||||
|
||||
got := strings.TrimSpace(string(out))
|
||||
want := "B\nA\nC"
|
||||
if got != want {
|
||||
t.Errorf("python3 gather.py stdout = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPyAsyncioGatherTraceTest_Eval(t *testing.T) {
|
||||
tc := pyAsyncioGatherTraceTest()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{"exact correct", `["B","A","C"]`, 1},
|
||||
{"correct fenced json", "```json\n[\"B\", \"A\", \"C\"]\n```", 1},
|
||||
{"correct with prose wrapper", `The order is: ["B","A","C"]`, 1},
|
||||
{"wrong: source order, ignores delays", `["A","B","C"]`, 0},
|
||||
{"wrong: reversed", `["C","A","B"]`, 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate() = %v, want %v (detail: %s)", got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestPyMutableDefaultArg_GroundTruth runs the identical snippet via PyRun
|
||||
// with a passthrough harness, so the doc-comment claim in python.go is
|
||||
// cross-checked against the real python3 interpreter, not just reasoning.
|
||||
func TestPyMutableDefaultArg_GroundTruth(t *testing.T) {
|
||||
score := eval.PyRun(eval.PassthroughHarness, "[1, 2, 3]").Evaluate(
|
||||
context.Background(), "```python\n"+pyMutableDefaultArgCode+"\n```",
|
||||
)
|
||||
if score.Skipped {
|
||||
t.Skip("python3 not available, skipping ground-truth cross-check")
|
||||
}
|
||||
if score.Value != 1 {
|
||||
t.Errorf("running pyMutableDefaultArgCode via PyRun scored %v, want 1 (detail: %s)", score.Value, score.Detail)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPyMutableDefaultArgTest_Eval(t *testing.T) {
|
||||
tc := pyMutableDefaultArgTest()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{"exact correct", "[1, 2, 3]", 1},
|
||||
{"correct with surrounding whitespace", " [1, 2, 3] ", 1},
|
||||
{"correct with trailing newline", "[1, 2, 3]\n", 1},
|
||||
{"wrong: assumes a fresh list per call", "[3]", 0},
|
||||
{"wrong: only counts explicit-arg calls", "[1, 2]", 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate() = %v, want %v (detail: %s)", got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestPyDictComprehensionTrace_GroundTruth runs the identical snippet via
|
||||
// PyRun with a passthrough harness.
|
||||
func TestPyDictComprehensionTrace_GroundTruth(t *testing.T) {
|
||||
want := "[('apple', 5), ('banana', 6), ('cherry', 6), ('date', 4)]"
|
||||
score := eval.PyRun(eval.PassthroughHarness, want).Evaluate(
|
||||
context.Background(), "```python\n"+pyDictComprehensionTraceCode+"\n```",
|
||||
)
|
||||
if score.Skipped {
|
||||
t.Skip("python3 not available, skipping ground-truth cross-check")
|
||||
}
|
||||
if score.Value != 1 {
|
||||
t.Errorf("running pyDictComprehensionTraceCode via PyRun scored %v, want 1 (detail: %s)", score.Value, score.Detail)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPyDictComprehensionTraceTest_Eval(t *testing.T) {
|
||||
tc := pyDictComprehensionTraceTest()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{"exact correct", "[('apple', 5), ('banana', 6), ('cherry', 6), ('date', 4)]", 1},
|
||||
{"correct with surrounding whitespace", " [('apple', 5), ('banana', 6), ('cherry', 6), ('date', 4)] ", 1},
|
||||
{"correct uppercase-folded", "[('APPLE', 5), ('BANANA', 6), ('CHERRY', 6), ('DATE', 4)]", 1},
|
||||
{"wrong: includes fig, forgets the length filter", "[('apple', 5), ('banana', 6), ('cherry', 6), ('date', 4), ('fig', 3)]", 0},
|
||||
{"wrong: unsorted, dict insertion order", "[('apple', 5), ('banana', 6), ('cherry', 6), ('fig', 3), ('date', 4)]", 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate() = %v, want %v (detail: %s)", got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestPyGeneratorExhaustionTrace_GroundTruth runs the identical snippet via
|
||||
// PyRun with a passthrough harness.
|
||||
func TestPyGeneratorExhaustionTrace_GroundTruth(t *testing.T) {
|
||||
score := eval.PyRun(eval.PassthroughHarness, "[0, 1, 2] []").Evaluate(
|
||||
context.Background(), "```python\n"+pyGeneratorExhaustionTraceCode+"\n```",
|
||||
)
|
||||
if score.Skipped {
|
||||
t.Skip("python3 not available, skipping ground-truth cross-check")
|
||||
}
|
||||
if score.Value != 1 {
|
||||
t.Errorf("running pyGeneratorExhaustionTraceCode via PyRun scored %v, want 1 (detail: %s)", score.Value, score.Detail)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPyGeneratorExhaustionTraceTest_Eval(t *testing.T) {
|
||||
tc := pyGeneratorExhaustionTraceTest()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{"exact correct", "[0, 1, 2] []", 1},
|
||||
{"correct with surrounding whitespace", " [0, 1, 2] [] ", 1},
|
||||
{"correct with trailing newline", "[0, 1, 2] []\n", 1},
|
||||
{"wrong: assumes the generator restarts", "[0, 1, 2] [0, 1, 2]", 0},
|
||||
{"wrong: assumes an error is raised and printed", "StopIteration", 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate() = %v, want %v (detail: %s)", got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestPyPathlibRewriteTest_Eval(t *testing.T) {
|
||||
tc := pyPathlibRewriteTest()
|
||||
|
||||
goodRewrite := "```python\n" + `from pathlib import Path
|
||||
|
||||
|
||||
def find_configs(base_dir):
|
||||
return [str(p) for p in Path(base_dir).rglob("*.yaml")]
|
||||
` + "```"
|
||||
|
||||
goodRewriteAltImport := "```python\n" + `import pathlib
|
||||
|
||||
|
||||
def find_configs(base_dir):
|
||||
return [str(p) for p in pathlib.Path(base_dir).rglob("*.yaml")]
|
||||
` + "```"
|
||||
|
||||
stillUsesOsPath := "```python\n" + pyOsPathCode + "\n```"
|
||||
|
||||
missingRglob := "```python\n" + `from pathlib import Path
|
||||
|
||||
|
||||
def find_configs(base_dir):
|
||||
return [str(p) for p in Path(base_dir).glob("*.yaml")]
|
||||
` + "```"
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{"good rewrite: from pathlib import Path", goodRewrite, 1},
|
||||
{"good rewrite: import pathlib", goodRewriteAltImport, 1},
|
||||
{"wrong: unchanged os.path/os.walk code", stillUsesOsPath, 0},
|
||||
{"partial: uses pathlib but non-recursive glob, not rglob", missingRglob, 2.0 / 3.0}, // (1*1 + 2*0.5)/3: ContainsAny hits, ContainsAll("rglob","*.yaml") matches only "*.yaml"
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate() = %v, want %v (detail: %s)", got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestPyJSONTransformWant_GroundTruth(t *testing.T) {
|
||||
// Independent re-derivation of the expected per-department totals by
|
||||
// summing hardcoded literals matching pyJSONTransformRecords, rather
|
||||
// than trusting the hand sum in the doc comment.
|
||||
totals := map[string]int{
|
||||
"eng": 95000 + 105000 + 88000,
|
||||
"sales": 60000 + 58000,
|
||||
"ops": 72000,
|
||||
}
|
||||
want := "eng=288000\nops=72000\nsales=118000"
|
||||
if pyJSONTransformWant != want {
|
||||
t.Errorf("pyJSONTransformWant = %q, want %q", pyJSONTransformWant, want)
|
||||
}
|
||||
if totals["eng"] != 288000 || totals["sales"] != 118000 || totals["ops"] != 72000 {
|
||||
t.Errorf("independently recomputed totals = %v, want eng=288000 sales=118000 ops=72000", totals)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPyJSONTransformTest_Eval(t *testing.T) {
|
||||
tc := pyJSONTransformTest()
|
||||
|
||||
correctScript := "```python\n" + pyJSONTransformRecords + `
|
||||
|
||||
totals = {}
|
||||
for r in records:
|
||||
totals[r["dept"]] = totals.get(r["dept"], 0) + r["salary"]
|
||||
|
||||
for dept in sorted(totals):
|
||||
print(f"{dept}={totals[dept]}")
|
||||
` + "```"
|
||||
|
||||
wrongScript := "```python\nprint('eng=1')\nprint('ops=1')\nprint('sales=1')\n```"
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{"correct transform script", correctScript, 1},
|
||||
{"wrong totals", wrongScript, 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Skipped {
|
||||
t.Skip("python3 not available, skipping exec-evaluator assertion")
|
||||
}
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate() = %v, want %v (detail: %s)", got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestPyRegexLogExtractionWant_GroundTruth(t *testing.T) {
|
||||
// Independent re-derivation of the expected counts by scanning the raw
|
||||
// log text directly with Go's regexp, rather than trusting the hand
|
||||
// count in the doc comment.
|
||||
statusPattern := regexp.MustCompile(`"\s*(\d{3})\s+\d+`)
|
||||
matches := statusPattern.FindAllStringSubmatch(pyRegexLogLines, -1)
|
||||
if len(matches) != 7 {
|
||||
t.Fatalf("found %d status codes in pyRegexLogLines, want 7", len(matches))
|
||||
}
|
||||
counts := map[string]int{}
|
||||
for _, m := range matches {
|
||||
counts[m[1]]++
|
||||
}
|
||||
if counts["200"] != 3 || counts["404"] != 2 || counts["500"] != 2 {
|
||||
t.Errorf("independently recomputed counts = %v, want 200=3 404=2 500=2", counts)
|
||||
}
|
||||
want := "200=3\n404=2\n500=2"
|
||||
if pyRegexLogExtractionWant != want {
|
||||
t.Errorf("pyRegexLogExtractionWant = %q, want %q", pyRegexLogExtractionWant, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPyRegexLogExtractionTest_Eval(t *testing.T) {
|
||||
tc := pyRegexLogExtractionTest()
|
||||
|
||||
correctScript := "```python\nimport re\n\n" + pyRegexLogLines + `
|
||||
|
||||
pattern = re.compile(r'"\s*(\d{3})\s+\d+$')
|
||||
counts = {}
|
||||
for line in log_lines:
|
||||
m = pattern.search(line)
|
||||
if m:
|
||||
code = m.group(1)
|
||||
counts[code] = counts.get(code, 0) + 1
|
||||
|
||||
for code in sorted(counts):
|
||||
print(f"{code}={counts[code]}")
|
||||
` + "```"
|
||||
|
||||
wrongScript := "```python\nprint('200=1')\nprint('404=1')\nprint('500=1')\n```"
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{"correct regex extraction script", correctScript, 1},
|
||||
{"wrong counts", wrongScript, 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Skipped {
|
||||
t.Skip("python3 not available, skipping exec-evaluator assertion")
|
||||
}
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate() = %v, want %v (detail: %s)", got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestPyCosineTest_Eval(t *testing.T) {
|
||||
tc := pyCosineTest()
|
||||
|
||||
|
||||
Reference in New Issue
Block a user