From 0816dd61862b8f986d4f55d4060a2ab40e8ecd47 Mon Sep 17 00:00:00 2001 From: Lukasz Raczylo Date: Mon, 17 Aug 2026 01:54:39 +0100 Subject: [PATCH] test(catalog): expand programming to 10 tests per subcategory Add 7 golang tests: channel deadlock spotting, defer/recover tracing, slice-append aliasing, method-set interface satisfaction, context cancellation propagation, a generics constraint fix, and bounded fan-out with first-error propagation. Add 8 python tests: softmax, asyncio.gather ordering, the mutable default argument bug, a dict comprehension trace, a pathlib rewrite, generator exhaustion, and JSON and regex log-processing scripts. Add 9 typescript tests: discriminated union narrowing, a recursive generic utility type, Promise.allSettled tracing, Vue 3 ref unwrapping, computed versus watch, ESLint flat-config migration, a tsconfig strict flag, and array-chain and nullish-coalescing tracing. Add 9 c tests: pointer arithmetic, bitmask operations, union sizing, a string-function program, integer promotion, array decay, a macro pitfall, an undefined-behavior spot, and bitfield packing. Each test carries a ground-truth derivation comment. Each unit test recomputes or cross-checks that ground truth against the real Go, Python, Node, or C toolchain where execution is possible. Golang, python, typescript, and c subcategories now register exactly 10 tests each. catalog_test.go's exact-map assertion now fails because it still expects 17 total tests; this is expected and out of scope, since that file is being replaced centrally. --- internal/tests/c.go | 381 ++++++++++++++++++++++++ internal/tests/c_test.go | 422 ++++++++++++++++++++++++++ internal/tests/golang.go | 461 +++++++++++++++++++++++++++++ internal/tests/golang_test.go | 473 ++++++++++++++++++++++++++++++ internal/tests/python.go | 352 ++++++++++++++++++++++ internal/tests/python_test.go | 398 +++++++++++++++++++++++++ internal/tests/typescript.go | 384 ++++++++++++++++++++++++ internal/tests/typescript_test.go | 359 +++++++++++++++++++++++ 8 files changed, 3230 insertions(+) diff --git a/internal/tests/c.go b/internal/tests/c.go index 22bb260..0a7ef26 100644 --- a/internal/tests/c.go +++ b/internal/tests/c.go @@ -7,6 +7,15 @@ import ( func registerCTests(r *testkit.Registry) { r.Register(cStructSizeTest()) + r.Register(cPointerArithmeticTest()) + r.Register(cBitmaskOpsTest()) + r.Register(cUnionSizeLP64Test()) + r.Register(cStringFunctionOutputTest()) + r.Register(cIntegerPromotionOverflowTest()) + r.Register(cArrayDecaySizeofTest()) + r.Register(cMacroExpansionPitfallTest()) + r.Register(cUndefinedBehaviorSpotTest()) + r.Register(cStructBitfieldPackingTest()) } // cStructSizeWant is sizeof(struct Config) on LP64 (64-bit System V / Apple @@ -48,3 +57,375 @@ Respond with only the number, nothing else.` Eval: eval.Numeric(eval.ExtractLastNumber[int], cStructSizeWant, 0), } } + +// cPointerArithmeticCode is the inline snippet for cPointerArithmeticTest. +const cPointerArithmeticCode = `int arr[6] = {10, 20, 30, 40, 50, 60}; +int *p = arr; +p += 2; +int *q = &arr[5]; +long diff = q - p; +int val = *(p + 1) + diff;` + +// cPointerArithmeticWant is the value of val. +// +// ground truth: p starts at &arr[0], then p += 2 advances it by 2 ints +// (pointer arithmetic scales by the pointee's size), landing on &arr[2] +// (value 30). q = &arr[5] (value 60). Pointer subtraction (q - p) yields the +// number of elements between them, not bytes: (&arr[5] - &arr[2]) = 3. *(p + +// 1) dereferences arr[3] (value 40). val = 40 + 3 = 43. c_test.go +// cross-checks this by compiling and running the identical snippet with cc. +const cPointerArithmeticWant = 43 + +func cPointerArithmeticTest() testkit.Test { + prompt := `Here is C code: + +` + "```c" + ` +` + cPointerArithmeticCode + ` +` + "```" + ` + +Assuming a 6-element int array laid out contiguously with no padding +between elements, what is the value of val? Respond with only the number, +nothing else.` + + return testkit.Test{ + ID: "c-pointer-arithmetic", + Category: "programming", + Subcategory: "c", + Description: "Trace the exact result of pointer arithmetic and pointer subtraction over an inline int array.", + Prompt: prompt, + Eval: eval.Numeric(eval.ExtractLastNumber[int], cPointerArithmeticWant, 0), + } +} + +// cBitmaskOpsCode is the inline snippet for cBitmaskOpsTest. +const cBitmaskOpsCode = `unsigned int flags = 0; +flags |= (1 << 2); +flags |= (1 << 5); +flags &= ~(1 << 2); +flags ^= (1 << 0);` + +// cBitmaskOpsWant is the final value of flags. +// +// ground truth: starting from 0: |= (1<<2) sets bit 2 -> 0b000100 (4); |= +// (1<<5) sets bit 5 -> 0b100100 (36); &= ~(1<<2) clears bit 2 -> 0b100000 +// (32); ^= (1<<0) toggles bit 0, which is currently 0, so it becomes 1 -> +// 0b100001 (33). c_test.go cross-checks this by compiling and running the +// identical snippet with cc. +const cBitmaskOpsWant = 33 + +func cBitmaskOpsTest() testkit.Test { + prompt := `Here is C code: + +` + "```c" + ` +` + cBitmaskOpsCode + ` +` + "```" + ` + +What is the final value of flags, as a decimal number? Respond with only +the number, nothing else.` + + return testkit.Test{ + ID: "c-bitmask-ops", + Category: "programming", + Subcategory: "c", + Description: "Trace the final decimal value of a chain of bit set/clear/toggle operations.", + Prompt: prompt, + Eval: eval.Numeric(eval.ExtractLastNumber[int], cBitmaskOpsWant, 0), + } +} + +// cUnionSizeLP64Code is the inline snippet for cUnionSizeLP64Test. +const cUnionSizeLP64Code = `union Packet { + double timestamp; + char raw[3]; + struct { int a; short b; } fields; +};` + +// cUnionSizeLP64Want is sizeof(union Packet) on LP64. +// +// ground truth: a union's size is at least the largest member's size, +// rounded up to the union's own alignment (the largest member alignment). +// timestamp: size 8, align 8. raw: size 3, align 1. fields (struct{int a; +// short b;}): a at offset 0 (size 4), b at offset 4 (size 2, already +// aligned), giving 6 bytes of payload padded up to the struct's own +// alignment (4, from int) -> struct size 8, align 4. The union's size is +// max(8, 3, 8) = 8, and its alignment is max(8, 1, 4) = 8; 8 is already a +// multiple of 8, so sizeof(union Packet) = 8. c_test.go cross-checks this +// with cc. +const cUnionSizeLP64Want = 8 + +func cUnionSizeLP64Test() testkit.Test { + prompt := `Here is a C union: + +` + "```c" + ` +` + cUnionSizeLP64Code + ` +` + "```" + ` + +Assuming the LP64 data model (64-bit, no #pragma pack, no non-default +alignment attributes), what is sizeof(union Packet) in bytes? Respond with +only the number, nothing else.` + + return testkit.Test{ + ID: "c-union-size-lp64", + Category: "programming", + Subcategory: "c", + Description: "Compute sizeof() for a C union on LP64, accounting for its largest member and its alignment.", + Prompt: prompt, + Eval: eval.Numeric(eval.ExtractLastNumber[int], cUnionSizeLP64Want, 0), + } +} + +// cStringFunctionOutputKeys is the inline dataset for +// cStringFunctionOutputTest. +// ground truth: keys starting with "srv_" (checked with strncmp(key, +// "srv_", 4) == 0): srv_web, srv_db, srv_api, srv_auth = 4 matches (also +// re-derived in c_test.go by scanning the same literal data with Go's +// strings.HasPrefix). +const cStringFunctionOutputKeys = `"srv_web", "srv_db", "cache_1", "srv_api", "queue_2", "srv_auth"` + +// cStringFunctionOutputWant is the expected stdout: the count of keys with +// the "srv_" prefix. +const cStringFunctionOutputWant = "4" + +func cStringFunctionOutputTest() testkit.Test { + prompt := `Here are 6 inline C string literals (configuration key names): + +` + "```c" + ` +const char *keys[] = {` + cStringFunctionOutputKeys + `}; +` + "```" + ` + +Write a complete, self-contained C program (embed the array above exactly +as shown; do not read stdin or a file) that uses strncmp from to +count how many of these 6 keys start with the prefix "srv_", and prints +just that count as a bare integer via printf, followed by a newline, and +nothing else.` + + return testkit.Test{ + ID: "c-string-function-output", + Category: "programming", + Subcategory: "c", + Description: "Write a C program using strncmp to count inline strings matching a prefix.", + System: terseCodeOnly, + Prompt: prompt, + Eval: eval.CRun(eval.PassthroughHarness, cStringFunctionOutputWant), + } +} + +// cIntegerPromotionOverflowCode is the inline snippet for +// cIntegerPromotionOverflowTest. +const cIntegerPromotionOverflowCode = `unsigned char a = 200; +unsigned char b = 100; +unsigned char sum = a + b;` + +// cIntegerPromotionOverflowWant is the value of sum. +// +// ground truth: in a + b, both unsigned char operands undergo the usual +// arithmetic conversions ("integer promotion"): each is promoted to int +// before the addition, so the addition computes 200 + 100 = 300 as a full +// int, with no overflow at that step. Assigning that int back to the +// unsigned char sum truncates it: unsigned integer types always wrap modulo +// 2^n (n=8 for unsigned char), which is well-defined behavior (not +// undefined, unlike signed overflow) - 300 mod 256 = 44. c_test.go +// cross-checks this by compiling and running the identical snippet with cc. +const cIntegerPromotionOverflowWant = 44 + +func cIntegerPromotionOverflowTest() testkit.Test { + prompt := `Here is C code: + +` + "```c" + ` +` + cIntegerPromotionOverflowCode + ` +` + "```" + ` + +What is the value of sum, printed as a decimal int (%d)? Respond with only +the number, nothing else.` + + return testkit.Test{ + ID: "c-integer-promotion-overflow", + Category: "programming", + Subcategory: "c", + Description: "Trace unsigned char addition through integer promotion to int and truncation back to 8 bits.", + Prompt: prompt, + Eval: eval.Numeric(eval.ExtractLastNumber[int], cIntegerPromotionOverflowWant, 0), + } +} + +// cArrayDecaySizeofCode is the inline snippet for cArrayDecaySizeofTest. +const cArrayDecaySizeofCode = `void check(int arr[10]) { + printf("%zu\n", sizeof(arr)); +} + +int main(void) { + int arr[10]; + printf("%zu\n", sizeof(arr)); + check(arr); + return 0; +}` + +// cArrayDecaySizeofWant is the sum of the two printed sizeof values. +// +// ground truth: in main, arr is a genuine 10-element int array in scope, so +// sizeof(arr) is the full array size: 10 * sizeof(int) = 40 (int is 4 bytes +// on LP64). Inside check, the parameter "int arr[10]" is adjusted by the +// compiler to "int *arr" (array-to-pointer decay in a function parameter +// declaration, per the C standard) - sizeof(arr) there measures the +// pointer, not the array: sizeof(int *) = 8 on LP64. Sum = 40 + 8 = 48. +// c_test.go cross-checks both individual values (and their sum) by +// compiling and running the identical snippet with cc. +const cArrayDecaySizeofWant = 48 + +func cArrayDecaySizeofTest() testkit.Test { + prompt := `Here is a C program: + +` + "```c" + ` +#include + +` + cArrayDecaySizeofCode + ` +` + "```" + ` + +Assuming LP64 (int is 4 bytes, pointers are 8 bytes), what is the sum of +the two numbers this program prints? Respond with only the number, nothing +else.` + + return testkit.Test{ + ID: "c-array-decay-sizeof", + Category: "programming", + Subcategory: "c", + Description: "Compute the sum of sizeof(array) in its defining scope versus sizeof(array-parameter) after decay to a pointer.", + Prompt: prompt, + Eval: eval.Numeric(eval.ExtractLastNumber[int], cArrayDecaySizeofWant, 0), + } +} + +// cMacroExpansionPitfallWant is the expected stdout: the result of the +// classic unparenthesized function-like macro pitfall. +// +// ground truth: #define SQUARE(x) x * x is a pure textual substitution with +// no parentheses around x or around the whole expansion, so +// SQUARE(2 + 3) expands to literally 2 + 3 * 2 + 3, not (2 + 3) * (2 + 3). +// Operator precedence then evaluates * before +: 2 + 3 * 2 + 3 +// = 2 + 6 + 3 = 11, not the "squared" answer 25 the macro's name suggests. +// c_test.go cross-checks this by compiling and running the identical macro +// and call with cc. +const cMacroExpansionPitfallWant = "11" + +func cMacroExpansionPitfallTest() testkit.Test { + prompt := `Here is a C macro definition: + +` + "```c" + ` +#define SQUARE(x) x * x +` + "```" + ` + +Write a complete, self-contained C program that defines SQUARE exactly as +given above, then prints the result of SQUARE(2 + 3) as a bare integer via +printf("%d\n", ...), and nothing else.` + + return testkit.Test{ + ID: "c-macro-expansion-pitfall", + Category: "programming", + Subcategory: "c", + Description: "Write a C program exposing the classic unparenthesized-macro operator-precedence pitfall.", + System: terseCodeOnly, + Prompt: prompt, + Eval: eval.CRun(eval.PassthroughHarness, cMacroExpansionPitfallWant), + } +} + +// cUndefinedBehaviorSpotCode is the inline, line-numbered snippet for +// cUndefinedBehaviorSpotTest. +const cUndefinedBehaviorSpotCode = `1: #include +2: #include +3: +4: int main(void) { +5: int x = INT_MAX; +6: int y = x + 1; +7: printf("%d\n", y); +8: return 0; +9: }` + +// cUndefinedBehaviorSpotKindSignedOverflow is the fixed literal +// cUndefinedBehaviorSpotTest requires for the "kind" JSON field. +const cUndefinedBehaviorSpotKindSignedOverflow = "signed-integer-overflow" + +// cUndefinedBehaviorSpotTest: identify the line and kind of undefined +// behavior in an inline snippet. +// +// ground truth: x holds INT_MAX (the largest representable int). Per the C +// standard (C11 6.5p5), an operation whose mathematical result is not +// representable in its result type is undefined behavior for signed +// integer types (unlike unsigned types, which are defined to wrap). Line 6 +// (x + 1) computes INT_MAX + 1, which overflows int - undefined behavior, +// not a well-defined wraparound. This cannot be "verified by execution": +// UB means the standard imposes no requirement on the observed result (in +// practice compilers may wrap, trap, or optimize the check away entirely), +// so c_test.go documents this ground truth from the standard rather than +// asserting a specific runtime output, the same approach golang.go's +// go-channel-deadlock test uses for its non-recomputable deadlock claim. +func cUndefinedBehaviorSpotTest() testkit.Test { + prompt := `Here is C code, with line numbers prefixed: + +` + "```c" + ` +` + cUndefinedBehaviorSpotCode + ` +` + "```" + ` + +Which line number contains undefined behavior? Pick exactly one kind from +this list: "signed-integer-overflow", "use-after-free", "uninitialized-read", +"buffer-overflow". Respond with only a JSON object: +{"line": , "kind": ""}` + + evaluator := eval.All( + eval.W(eval.JSONField("line", 6), 1), + eval.W(eval.JSONField("kind", cUndefinedBehaviorSpotKindSignedOverflow), 1), + ) + + return testkit.Test{ + ID: "c-undefined-behavior-spot", + Category: "programming", + Subcategory: "c", + Description: "Identify the line and kind of undefined behavior in an inline C snippet (signed integer overflow).", + System: terseCodeOnly, + Prompt: prompt, + Eval: evaluator, + } +} + +// cStructBitfieldPackingCode is the inline snippet for +// cStructBitfieldPackingTest. +const cStructBitfieldPackingCode = `struct Flags { + unsigned int a : 3; + unsigned int b : 5; + unsigned int c : 24; +};` + +// cStructBitfieldPackingWant is sizeof(struct Flags) on a typical LP64 +// System V / Apple ARM64 ABI. +// +// ground truth: consecutive bitfields declared with the same base type +// (unsigned int, a 4-byte/32-bit storage unit) are packed into shared +// storage units by mainstream compilers (gcc/clang on both x86-64 System V +// and Apple's ARM64 ABI) when they fit: a (3 bits) + b (5 bits) + c (24 +// bits) = 32 bits exactly, filling one 4-byte unsigned int storage unit +// with no padding needed. So sizeof(struct Flags) = 4. Exact bitfield +// packing is implementation-defined by the C standard (not mandated), so +// c_test.go cross-checks this empirically against the real cc toolchain +// rather than relying on the derivation alone. +const cStructBitfieldPackingWant = 4 + +func cStructBitfieldPackingTest() testkit.Test { + prompt := `Here is a C struct with bitfields: + +` + "```c" + ` +` + cStructBitfieldPackingCode + ` +` + "```" + ` + +Assuming a typical LP64 System V or Apple ARM64 ABI (gcc/clang, no +#pragma pack), what is sizeof(struct Flags) in bytes? Respond with only the +number, nothing else.` + + return testkit.Test{ + ID: "c-struct-bitfield-packing", + Category: "programming", + Subcategory: "c", + Description: "Compute sizeof() for a C struct whose bitfields exactly fill one storage unit with no padding.", + Prompt: prompt, + Eval: eval.Numeric(eval.ExtractLastNumber[int], cStructBitfieldPackingWant, 0), + } +} diff --git a/internal/tests/c_test.go b/internal/tests/c_test.go index 48f9399..49ab9a2 100644 --- a/internal/tests/c_test.go +++ b/internal/tests/c_test.go @@ -9,6 +9,39 @@ import ( "testing" ) +// ccRun compiles and runs a complete C program via cc, returning trimmed +// stdout. It skips the calling test (does not fail it) when cc is +// unavailable, mirroring TestCStructSizeWant_GroundTruth's existing +// toolchain-detection contract. +func ccRun(t *testing.T, src string) string { + t.Helper() + if _, err := exec.LookPath("cc"); err != nil { + t.Skip("cc not found on PATH, skipping ground-truth cross-check") + } + + dir := t.TempDir() + srcPath := filepath.Join(dir, "main.c") + if err := os.WriteFile(srcPath, []byte(src), 0o600); err != nil { + t.Fatalf("write main.c: %v", err) + } + binPath := filepath.Join(dir, "prog") + + // #nosec G204 -- srcPath/binPath are fixed filenames this test just + // wrote/will write into its own t.TempDir(); not external input. + build := exec.Command("cc", "-o", binPath, srcPath) //nolint:gosec // see comment above + if out, err := build.CombinedOutput(); err != nil { + t.Fatalf("cc failed: %v\n%s", err, out) + } + + // #nosec G204 -- binPath is the binary this test just compiled above, + // in its own t.TempDir(); not an externally supplied executable path. + out, err := exec.Command(binPath).Output() //nolint:gosec // see comment above + if err != nil { + t.Fatalf("running compiled program failed: %v", err) + } + return strings.TrimSpace(string(out)) +} + // TestCStructSizeWant_GroundTruth cross-checks cStructSizeWant against the // real cc toolchain's sizeof(struct Config), rather than trusting the // hand-derived alignment arithmetic in the doc comment alone. Skips (does @@ -60,6 +93,395 @@ int main(void) { } } +func TestCPointerArithmetic_GroundTruth(t *testing.T) { + src := `#include +int main(void) { + ` + cPointerArithmeticCode + ` + printf("%d", val); + return 0; +} +` + got := ccRun(t, src) + if want := "43"; got != want { + t.Errorf("cc output = %q, want %q", got, want) + } +} + +func TestCPointerArithmeticTest_Eval(t *testing.T) { + tc := cPointerArithmeticTest() + + tests := []struct { + name string + response string + want float64 + }{ + {"exact correct", "43", 1}, + {"prose wrapped", "val is 43", 1}, + {"answer followed by qualifier", "val = 43 on a 64-bit system", 1}, + {"wrong: forgets pointer arithmetic scales by element size", "13", 0}, + {"wrong: byte-based pointer diff", "val = 52", 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Value != tt.want { + t.Errorf("Eval.Evaluate() = %v, want %v (detail: %s)", got.Value, tt.want, got.Detail) + } + }) + } +} + +func TestCBitmaskOps_GroundTruth(t *testing.T) { + src := `#include +int main(void) { + ` + cBitmaskOpsCode + ` + printf("%u", flags); + return 0; +} +` + got := ccRun(t, src) + if want := "33"; got != want { + t.Errorf("cc output = %q, want %q", got, want) + } +} + +func TestCBitmaskOpsTest_Eval(t *testing.T) { + tc := cBitmaskOpsTest() + + tests := []struct { + name string + response string + want float64 + }{ + {"exact correct", "33", 1}, + {"prose wrapped", "flags is 33", 1}, + {"binary explained then decimal answer", "0b100001 = 33", 1}, + {"wrong: forgets to clear bit 2", "37", 0}, + {"wrong: forgets to toggle bit 0", "32", 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Value != tt.want { + t.Errorf("Eval.Evaluate() = %v, want %v (detail: %s)", got.Value, tt.want, got.Detail) + } + }) + } +} + +func TestCUnionSizeLP64_GroundTruth(t *testing.T) { + src := `#include +` + cUnionSizeLP64Code + ` +int main(void) { + printf("%zu", sizeof(union Packet)); + return 0; +} +` + got := ccRun(t, src) + if want := "8"; got != want { + t.Errorf("cc output = %q, want %q", got, want) + } +} + +func TestCUnionSizeLP64Test_Eval(t *testing.T) { + tc := cUnionSizeLP64Test() + + tests := []struct { + name string + response string + want float64 + }{ + {"exact correct", "8", 1}, + {"prose wrapped", "sizeof(union Packet) is 8 bytes", 1}, + {"answer followed by 64-bit qualifier", "8 bytes on a 64-bit system", 1}, + {"wrong: sums members instead of taking the max", "19", 0}, + {"wrong: forgets struct-member alignment padding", "6", 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Value != tt.want { + t.Errorf("Eval.Evaluate() = %v, want %v (detail: %s)", got.Value, tt.want, got.Detail) + } + }) + } +} + +// TestCStringFunctionOutputWant_GroundTruth independently re-derives the +// expected count by scanning the same literal key list with Go's +// strings.HasPrefix, rather than trusting the hand count in the doc +// comment. +func TestCStringFunctionOutputWant_GroundTruth(t *testing.T) { + keys := []string{"srv_web", "srv_db", "cache_1", "srv_api", "queue_2", "srv_auth"} + count := 0 + for _, k := range keys { + if strings.HasPrefix(k, "srv_") { + count++ + } + } + if count != 4 { + t.Errorf("independently recomputed count = %d, want 4", count) + } + if cStringFunctionOutputWant != "4" { + t.Errorf("cStringFunctionOutputWant = %q, want %q", cStringFunctionOutputWant, "4") + } +} + +func TestCStringFunctionOutputTest_Eval(t *testing.T) { + tc := cStringFunctionOutputTest() + + correctProgram := "```c\n" + `#include +#include + +int main(void) { + const char *keys[] = {` + cStringFunctionOutputKeys + `}; + int count = 0; + for (int i = 0; i < 6; i++) { + if (strncmp(keys[i], "srv_", 4) == 0) { + count++; + } + } + printf("%d\n", count); + return 0; +} +` + "```" + + wrongProgram := "```c\n#include \nint main(void) { printf(\"6\\n\"); return 0; }\n```" + + tests := []struct { + name string + response string + want float64 + }{ + {"correct strncmp counting program", correctProgram, 1}, + {"wrong: counts every key regardless of prefix", wrongProgram, 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Skipped { + t.Skip("cc not available, skipping exec-evaluator assertion") + } + if got.Value != tt.want { + t.Errorf("Eval.Evaluate() = %v, want %v (detail: %s)", got.Value, tt.want, got.Detail) + } + }) + } +} + +func TestCIntegerPromotionOverflow_GroundTruth(t *testing.T) { + src := `#include +int main(void) { + ` + cIntegerPromotionOverflowCode + ` + printf("%d", sum); + return 0; +} +` + got := ccRun(t, src) + if want := "44"; got != want { + t.Errorf("cc output = %q, want %q", got, want) + } +} + +func TestCIntegerPromotionOverflowTest_Eval(t *testing.T) { + tc := cIntegerPromotionOverflowTest() + + tests := []struct { + name string + response string + want float64 + }{ + {"exact correct", "44", 1}, + {"prose wrapped", "sum is 44", 1}, + {"answer with modulo explanation", "300 mod 256 = 44", 1}, + {"wrong: assumes saturation instead of wraparound", "255", 0}, + {"wrong: forgets truncation back to 8 bits", "300", 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Value != tt.want { + t.Errorf("Eval.Evaluate() = %v, want %v (detail: %s)", got.Value, tt.want, got.Detail) + } + }) + } +} + +func TestCArrayDecaySizeof_GroundTruth(t *testing.T) { + src := `#include + +` + cArrayDecaySizeofCode + ` +` + got := ccRun(t, src) + lines := strings.Split(got, "\n") + if len(lines) != 2 { + t.Fatalf("cc output = %q, want 2 lines", got) + } + if lines[0] != "40" { + t.Errorf("sizeof(arr) in main = %q, want %q", lines[0], "40") + } + if lines[1] != "8" { + t.Errorf("sizeof(arr) in function = %q, want %q", lines[1], "8") + } + if cArrayDecaySizeofWant != 48 { + t.Errorf("cArrayDecaySizeofWant = %d, want 48 (= 40 + 8)", cArrayDecaySizeofWant) + } +} + +func TestCArrayDecaySizeofTest_Eval(t *testing.T) { + tc := cArrayDecaySizeofTest() + + tests := []struct { + name string + response string + want float64 + }{ + {"exact correct", "48", 1}, + {"prose wrapped", "The sum is 48.", 1}, + {"answer with breakdown", "40 + 8 = 48", 1}, + {"wrong: assumes both report the full array size", "80", 0}, + {"wrong: assumes both decay to pointer size", "16", 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Value != tt.want { + t.Errorf("Eval.Evaluate() = %v, want %v (detail: %s)", got.Value, tt.want, got.Detail) + } + }) + } +} + +func TestCMacroExpansionPitfallWant_GroundTruth(t *testing.T) { + src := `#include + +#define SQUARE(x) x * x + +int main(void) { + printf("%d\n", SQUARE(2 + 3)); + return 0; +} +` + got := ccRun(t, src) + if cMacroExpansionPitfallWant != "11" { + t.Errorf("cMacroExpansionPitfallWant = %q, want %q", cMacroExpansionPitfallWant, "11") + } + if got != "11" { + t.Errorf("cc output = %q, want %q", got, "11") + } +} + +func TestCMacroExpansionPitfallTest_Eval(t *testing.T) { + tc := cMacroExpansionPitfallTest() + + correctProgram := "```c\n" + `#include + +#define SQUARE(x) x * x + +int main(void) { + printf("%d\n", SQUARE(2 + 3)); + return 0; +} +` + "```" + + // A model that "fixes" the macro with parentheses no longer reproduces + // the pitfall's actual (unparenthesized) expansion, so it must score 0 + // against this test's want ("11"), even though (2+3)*(2+3)=25 is what a + // naive reading of "SQUARE" would suggest. + parenthesizedProgram := "```c\n" + `#include + +#define SQUARE(x) ((x) * (x)) + +int main(void) { + printf("%d\n", SQUARE(2 + 3)); + return 0; +} +` + "```" + + tests := []struct { + name string + response string + want float64 + }{ + {"correct: reproduces the macro exactly as given", correctProgram, 1}, + {"wrong: silently parenthesizes the macro, changing its output", parenthesizedProgram, 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Skipped { + t.Skip("cc not available, skipping exec-evaluator assertion") + } + if got.Value != tt.want { + t.Errorf("Eval.Evaluate() = %v, want %v (detail: %s)", got.Value, tt.want, got.Detail) + } + }) + } +} + +func TestCUndefinedBehaviorSpotTest_Eval(t *testing.T) { + tc := cUndefinedBehaviorSpotTest() + + tests := []struct { + name string + response string + want float64 + }{ + {"exact correct", `{"line": 6, "kind": "signed-integer-overflow"}`, 1}, + {"correct fenced json", "```json\n{\"line\": 6, \"kind\": \"signed-integer-overflow\"}\n```", 1}, + {"correct with prose wrapper", `The answer is: {"line": 6, "kind": "signed-integer-overflow"}`, 1}, + {"wrong line", `{"line": 5, "kind": "signed-integer-overflow"}`, 0.5}, + {"wrong kind", `{"line": 6, "kind": "uninitialized-read"}`, 0.5}, + {"both wrong", `{"line": 7, "kind": "buffer-overflow"}`, 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Value != tt.want { + t.Errorf("Eval.Evaluate() = %v, want %v (detail: %s)", got.Value, tt.want, got.Detail) + } + }) + } +} + +func TestCStructBitfieldPacking_GroundTruth(t *testing.T) { + src := `#include +` + cStructBitfieldPackingCode + ` +int main(void) { + printf("%zu", sizeof(struct Flags)); + return 0; +} +` + got := ccRun(t, src) + if want := "4"; got != want { + t.Errorf("cc output = %q, want %q", got, want) + } +} + +func TestCStructBitfieldPackingTest_Eval(t *testing.T) { + tc := cStructBitfieldPackingTest() + + tests := []struct { + name string + response string + want float64 + }{ + {"exact correct", "4", 1}, + {"prose wrapped", "sizeof(struct Flags) is 4 bytes", 1}, + {"bit count then byte answer", "3+5+24 = 32 bits = 4 bytes", 1}, + {"wrong: one byte per bitfield member", "3", 0}, + {"wrong: assumes each bitfield gets its own storage unit", "12", 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Value != tt.want { + t.Errorf("Eval.Evaluate() = %v, want %v (detail: %s)", got.Value, tt.want, got.Detail) + } + }) + } +} + func TestCStructSizeTest_Eval(t *testing.T) { tc := cStructSizeTest() diff --git a/internal/tests/golang.go b/internal/tests/golang.go index e22c3d4..103134e 100644 --- a/internal/tests/golang.go +++ b/internal/tests/golang.go @@ -17,6 +17,13 @@ func registerGolangTests(r *testkit.Registry) { r.Register(goStructAlignTest()) r.Register(goWorkerPoolTest()) r.Register(goSemverClassifyTest()) + r.Register(goChannelDeadlockTest()) + r.Register(goDeferRecoverTraceTest()) + r.Register(goSliceAppendAliasingTest()) + r.Register(goMethodSetInterfaceTest()) + r.Register(goContextCancellationTraceTest()) + r.Register(goGenericsConstraintFixTest()) + r.Register(goBoundedFanoutTest()) } // goStructAlignTest: reorder struct fields to minimize padding on a 64-bit @@ -223,3 +230,457 @@ object: {"bump":"major|minor|patch"}` Eval: eval.JSONField("bump", "minor"), } } + +// goChannelDeadlockReason is the fixed vocabulary of "why" answers +// goChannelDeadlockTest allows, so the JSON "reason" field has exactly one +// canonical correct literal instead of open-ended prose. +const ( + goChannelDeadlockReasonUnbufferedNoReceiver = "unbuffered-send-no-receiver" +) + +// goChannelDeadlockTest: identify which line of an inline snippet blocks +// forever, and why. +// +// ground truth: `ch := make(chan int)` creates an unbuffered channel, whose +// send blocks until another goroutine is ready to receive concurrently. Line +// 5 (`ch <- 1`) runs on main's single goroutine with no other goroutine +// started, so the send blocks forever - the receive on line 6 can never run +// because line 5 never returns. This is a single-goroutine self-deadlock, +// not (for example) a nil-channel receive or a full buffered channel, so the +// enumerated reason is "unbuffered-send-no-receiver". Because a genuine +// deadlock hangs the process, golang_test.go cannot recompute this by +// running the snippet (per PLAN.md, ground truth is derived in the doc +// comment instead, as for goStructAlignTest's alignment arithmetic). +func goChannelDeadlockTest() testkit.Test { + prompt := `Here is Go code, with line numbers prefixed: + +1: package main +2: +3: func main() { +4: ch := make(chan int) +5: ch <- 1 +6: v := <-ch +7: println(v) +8: } + +Which line number blocks forever? Pick exactly one reason from this list: +"unbuffered-send-no-receiver", "closed-channel-send", "nil-channel-receive", +"buffered-channel-full". Respond with only a JSON object: +{"line": , "reason": ""}` + + evaluator := eval.All( + eval.W(eval.JSONField("line", 5), 1), + eval.W(eval.JSONField("reason", goChannelDeadlockReasonUnbufferedNoReceiver), 1), + ) + + return testkit.Test{ + ID: "go-channel-deadlock", + Category: "programming", + Subcategory: "golang", + Description: "Identify the line and reason a single-goroutine unbuffered-channel send deadlocks forever.", + System: terseCodeOnly, + Prompt: prompt, + Eval: evaluator, + } +} + +// goDeferRecoverTraceCode is the inline snippet for goDeferRecoverTraceTest. +// It is duplicated (not shared via a Go source string constant referencing +// a compiled function) so the prompt text is self-contained per EXPANSION.md +// rule 1; golang_test.go defines an equivalent compute() and calls it +// in-process to recompute the ground truth independently of this comment. +const goDeferRecoverTraceCode = `package main + +import "fmt" + +func compute() (result int) { + defer func() { + if r := recover(); r != nil { + result = -1 + } + }() + defer func() { + result += 10 + }() + panic("boom") +} + +func main() { + fmt.Println(compute()) +}` + +// goDeferRecoverTraceTest: trace the exact printed output of a function +// combining defer, recover, and a named return value. +// +// ground truth: deferred functions run in LIFO (last-registered-first) +// order. compute() panics with "boom" before returning, so both defers run +// during unwinding: first (LIFO) the second-registered defer runs, +// `result += 10`, taking the named return `result` from its zero value 0 to +// 10 - the panic is still active at this point, since this defer never +// calls recover. Then the first-registered defer runs, calls recover() +// (which returns the non-nil panic value "boom", stopping the panic), and +// overwrites `result = -1`. The named return is read after all defers +// finish, so compute() returns -1 and main prints "-1". golang_test.go +// verified this by literally running an equivalent compute() with `go run` +// during authoring (see the ground-truth test below, which re-verifies it +// in-process). +func goDeferRecoverTraceTest() testkit.Test { + prompt := `Here is a Go program: + +` + "```go" + ` +` + goDeferRecoverTraceCode + ` +` + "```" + ` + +What does this program print when run? Respond with only the number, +nothing else.` + + return testkit.Test{ + ID: "go-defer-recover-trace", + Category: "programming", + Subcategory: "golang", + Description: "Trace the exact output of a defer/recover chain that mutates a named return value.", + Prompt: prompt, + Eval: eval.Numeric(eval.ExtractLastNumber[int], -1, 0), + } +} + +// goSliceAppendAliasingCode is the inline snippet for +// goSliceAppendAliasingTest. +const goSliceAppendAliasingCode = `package main + +import "fmt" + +func main() { + a := make([]int, 3, 5) + a[0], a[1], a[2] = 1, 2, 3 + b := a[:2] + b = append(b, 99) + _ = b + fmt.Println(a[2]) +}` + +// goSliceAppendAliasingTest: trace the exact printed output of a slice +// aliasing gotcha where append writes into a shared backing array. +// +// ground truth: cap(a[:2]) = cap(a) - 0 = 5 (slicing a[low:high] gives +// cap = cap(a) - low), so b := a[:2] has len 2 but cap 5, still sharing a's +// backing array. append(b, 99) has spare capacity, so it does not allocate a +// new array: it writes 99 into backing-array index 2 (len(b) before the +// append) and returns a length-3 slice. Backing-array index 2 is also a[2], +// so a[2] becomes 99 even though a itself was never explicitly assigned to. +// golang_test.go re-verifies this in-process. +func goSliceAppendAliasingTest() testkit.Test { + prompt := `Here is a Go program: + +` + "```go" + ` +` + goSliceAppendAliasingCode + ` +` + "```" + ` + +What does this program print when run? Respond with only the number, +nothing else.` + + return testkit.Test{ + ID: "go-slice-append-aliasing", + Category: "programming", + Subcategory: "golang", + Description: "Trace the exact output of a slice-append aliasing gotcha on a shared backing array.", + Prompt: prompt, + Eval: eval.Numeric(eval.ExtractLastNumber[int], 99, 0), + } +} + +// goMethodSetInterfaceCode is the inline snippet for +// goMethodSetInterfaceTest: a type whose only interface method has a +// pointer receiver, so only *Wrapper (not Wrapper) satisfies Stringer. +const goMethodSetInterfaceCode = `type Stringer interface { + String() string +} + +type Wrapper struct { + val string +} + +func (w *Wrapper) String() string { + return "Wrapper(" + w.val + ")" +}` + +// goMethodSetInterfaceTest: decide whether a value type and a pointer type +// each satisfy an interface whose method has a pointer receiver. +// +// ground truth: a pointer receiver method belongs to the method set of the +// pointer type only, not the value type (Go spec, "Method sets"). So +// `Wrapper{...}` (a value) does not implement Stringer - +// "Wrapper does not implement Stringer (method String has pointer +// receiver)" - while `&Wrapper{...}` (a *Wrapper) does. golang_test.go +// cross-checks both compile-time facts by running `go build` on each +// snippet, guarded by go-toolchain presence (as c_test.go does for cc). +func goMethodSetInterfaceTest() testkit.Test { + prompt := `Here is Go code: + +` + "```go" + ` +` + goMethodSetInterfaceCode + ` +` + "```" + ` + +Does this line compile: ` + "`var _ Stringer = Wrapper{val: \"hi\"}`" + `? +Does this line compile: ` + "`var _ Stringer = &Wrapper{val: \"hi\"}`" + `? + +Respond with only a JSON object: +{"value_compiles": true|false, "pointer_compiles": true|false}` + + evaluator := eval.All( + eval.W(eval.JSONField("value_compiles", false), 1), + eval.W(eval.JSONField("pointer_compiles", true), 1), + ) + + return testkit.Test{ + ID: "go-method-set-interface", + Category: "programming", + Subcategory: "golang", + Description: "Decide whether a value type and its pointer type satisfy an interface with a pointer-receiver method.", + System: terseCodeOnly, + Prompt: prompt, + Eval: evaluator, + } +} + +// goContextCancellationTraceCode is the inline snippet for +// goContextCancellationTraceTest. +const goContextCancellationTraceCode = `package main + +import ( + "context" + "fmt" +) + +func main() { + parent, cancelParent := context.WithCancel(context.Background()) + child, cancelChild := context.WithCancel(parent) + defer cancelChild() + + cancelParent() + + select { + case <-child.Done(): + fmt.Println(child.Err()) + default: + fmt.Println("not done") + } +}` + +// goContextCancellationTraceTest: trace the exact printed output of +// cancellation propagating from a parent context to a context derived from +// it. +// +// ground truth: context.WithCancel propagates cancellation synchronously +// down the derivation tree - cancelParent() closes parent's Done channel +// and, in the same call, closes every context derived from it (including +// child), before cancelParent() returns. So by the time the select runs, +// child.Done() is already closed and takes the first case, printing +// child.Err(). Err() returns the sentinel context.Canceled, whose message +// (per the context package source) is "context canceled". golang_test.go +// re-verifies this in-process, including that the propagation really is +// synchronous (no goroutine/timing dependency, so the trace is +// deterministic). +func goContextCancellationTraceTest() testkit.Test { + prompt := `Here is a Go program: + +` + "```go" + ` +` + goContextCancellationTraceCode + ` +` + "```" + ` + +What does this program print when run? Respond with only the exact text +printed, nothing else.` + + return testkit.Test{ + ID: "go-context-cancellation-trace", + Category: "programming", + Subcategory: "golang", + Description: "Trace the exact output of a context cancellation propagating synchronously from a parent to a derived context.", + Prompt: prompt, + Eval: eval.Equals("context canceled"), + } +} + +// goGenericsConstraintFixHarness calls the response's fixed Max function +// across int, float64, and string, verifying both the return values and +// that it compiles at all (a broken constraint fails to compile, which +// eval.GoRun scores as 0 via its "execution failed" / build-error path). +const goGenericsConstraintFixHarness = `package main + +import ( + "fmt" + "os" +) + +func main() { + if got := Max(3, 5); got != 5 { + fmt.Fprintf(os.Stderr, "Max(3,5) = %v, want 5\n", got) + os.Exit(1) + } + if got := Max(5, 3); got != 5 { + fmt.Fprintf(os.Stderr, "Max(5,3) = %v, want 5\n", got) + os.Exit(1) + } + if got := Max(2.5, 1.5); got != 2.5 { + fmt.Fprintf(os.Stderr, "Max(2.5,1.5) = %v, want 2.5\n", got) + os.Exit(1) + } + if got := Max("apple", "banana"); got != "banana" { + fmt.Fprintf(os.Stderr, "Max(apple,banana) = %v, want banana\n", got) + os.Exit(1) + } + fmt.Println("PASS") +} +` + +// goGenericsConstraintFixTest: repair a generic function whose type +// parameter constraint (any) does not support the operator (>) its body +// uses, without depending on any external package. +// +// ground truth: the harness above is the oracle - it exercises Max across +// three concrete types the fixed constraint must accept (int, float64, +// string), so there is no separate hardcoded constant to derive. +func goGenericsConstraintFixTest() testkit.Test { + prompt := `This Go generic function does not compile: + +` + "```go" + ` +func Max[T any](a, b T) T { + if a > b { + return a + } + return b +} +` + "```" + ` + +The compiler rejects it because type parameter T (constrained only by any) +does not support the > operator. Fix it with the smallest change that makes +it compile and work correctly for int, float64, and string arguments. Define +any constraint interface you need yourself, using only the standard library +- do not import golang.org/x/exp/constraints or any other third-party +package. Respond with the complete corrected code (constraint interface, if +any, plus the Max function).` + + return testkit.Test{ + ID: "go-generics-constraint-fix", + Category: "programming", + Subcategory: "golang", + Description: "Fix a generic function's constraint so > compiles and works for int, float64, and string.", + System: terseCodeOnly, + Prompt: prompt, + Eval: eval.GoRun(goGenericsConstraintFixHarness, "PASS"), + } +} + +// goBoundedFanoutHarness drives the response's BoundedRun with two batches +// of tasks (all-succeed, then some-fail) over an instrumented task function +// that tracks observed concurrency via an atomic counter, the same pattern +// goWorkerPoolHarness uses. It checks: (a) the all-succeed batch returns +// nil, (b) the some-fail batch returns a non-nil error, and (c) in both +// batches, observed concurrency stayed within [2, limit] - never sequential +// (>=2, evidencing real concurrency), never over the requested bound. +const goBoundedFanoutHarness = `package main + +import ( + "errors" + "fmt" + "os" + "sync/atomic" + "time" +) + +func main() { + const limit = 4 + const n = 20 + + var concurrent, maxConcurrent int32 + makeTask := func(shouldFail bool) func() error { + return func() error { + cur := atomic.AddInt32(&concurrent, 1) + for { + m := atomic.LoadInt32(&maxConcurrent) + if cur <= m || atomic.CompareAndSwapInt32(&maxConcurrent, m, cur) { + break + } + } + time.Sleep(5 * time.Millisecond) + atomic.AddInt32(&concurrent, -1) + if shouldFail { + return errors.New("boom") + } + return nil + } + } + + okTasks := make([]func() error, n) + for i := range okTasks { + okTasks[i] = makeTask(false) + } + if err := BoundedRun(okTasks, limit); err != nil { + fmt.Fprintf(os.Stderr, "BoundedRun(all ok) = %v, want nil\n", err) + os.Exit(1) + } + if got := atomic.LoadInt32(&maxConcurrent); got < 2 || got > limit { + fmt.Fprintf(os.Stderr, "maxConcurrent=%d, want in [2, %d]\n", got, limit) + os.Exit(1) + } + + atomic.StoreInt32(&concurrent, 0) + atomic.StoreInt32(&maxConcurrent, 0) + failTasks := make([]func() error, n) + for i := range failTasks { + failTasks[i] = makeTask(i%3 == 0) + } + err := BoundedRun(failTasks, limit) + if err == nil { + fmt.Fprintln(os.Stderr, "BoundedRun(some fail) = nil, want an error") + os.Exit(1) + } + if got := atomic.LoadInt32(&maxConcurrent); got < 2 || got > limit { + fmt.Fprintf(os.Stderr, "maxConcurrent=%d, want in [2, %d]\n", got, limit) + os.Exit(1) + } + + fmt.Println("PASS") +} +` + +// goBoundedFanoutTest: implement bounded-concurrency fan-out with +// first-error propagation, the same shape as +// golang.org/x/sync/errgroup.Group.SetLimit but implemented directly, so +// the exec evaluator never depends on an external module being resolvable +// in the sandbox (EXPANSION.md permits substituting an equivalent-difficulty +// topic when the literal one cannot be made deterministic offline). +// +// ground truth: the harness above is the oracle, exercising both +// correctness (first error propagates, nil on all-success) and the +// concurrency bound via the same observed-concurrency counter pattern as +// goWorkerPoolHarness; there is no separate hardcoded constant to derive. +func goBoundedFanoutTest() testkit.Test { + prompt := `Implement this Go function: + +` + "```go" + ` +func BoundedRun(tasks []func() error, limit int) error +` + "```" + ` + +Requirements: +- Run the tasks concurrently, but never more than limit at the same time. +- If one or more tasks return a non-nil error, BoundedRun must return a + non-nil error (any one of the failing tasks' errors is acceptable). +- If every task returns nil, BoundedRun must return nil. +- Use only the standard library (for example sync, sync/atomic, or + channels). Do not import golang.org/x/sync/errgroup or any other + third-party package. +- You may include whatever standard-library imports your implementation + needs. Do not include a package clause or a main function: respond with + only the BoundedRun function definition (plus any imports it needs).` + + return testkit.Test{ + ID: "go-bounded-fanout", + Category: "programming", + Subcategory: "golang", + Description: "Implement bounded-concurrency fan-out with first-error propagation, using only the standard library.", + System: terseCodeOnly, + Prompt: prompt, + Eval: eval.GoRun(goBoundedFanoutHarness, "PASS"), + } +} diff --git a/internal/tests/golang_test.go b/internal/tests/golang_test.go index 525e9a4..baca161 100644 --- a/internal/tests/golang_test.go +++ b/internal/tests/golang_test.go @@ -2,7 +2,15 @@ package tests import ( "context" + "errors" + "os" + "os/exec" + "path/filepath" + "strings" + "sync" + "sync/atomic" "testing" + "time" "unsafe" ) @@ -257,3 +265,468 @@ func TestGoSemverClassifyTest_Eval(t *testing.T) { }) } } + +func TestGoChannelDeadlockTest_Eval(t *testing.T) { + tc := goChannelDeadlockTest() + + tests := []struct { + name string + response string + want float64 + }{ + {"exact correct", `{"line": 5, "reason": "unbuffered-send-no-receiver"}`, 1}, + {"correct fenced json", "```json\n{\"line\": 5, \"reason\": \"unbuffered-send-no-receiver\"}\n```", 1}, + {"correct with prose wrapper", `The answer is: {"line": 5, "reason": "unbuffered-send-no-receiver"}`, 1}, + {"wrong line", `{"line": 6, "reason": "unbuffered-send-no-receiver"}`, 0.5}, + {"wrong reason", `{"line": 5, "reason": "nil-channel-receive"}`, 0.5}, + {"both wrong", `{"line": 6, "reason": "nil-channel-receive"}`, 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Value != tt.want { + t.Errorf("Eval.Evaluate() = %v, want %v (detail: %s)", got.Value, tt.want, got.Detail) + } + }) + } +} + +// computeDeferRecover mirrors goDeferRecoverTraceCode's compute() exactly, +// so its return value grounds the "ground truth:" derivation in golang.go +// by actually running the same logic, per PLAN.md's recomputation rule. +func computeDeferRecover() (result int) { + defer func() { + if r := recover(); r != nil { + result = -1 + } + }() + defer func() { + result += 10 + }() + panic("boom") +} + +func TestGoDeferRecoverTrace_GroundTruth(t *testing.T) { + if got, want := computeDeferRecover(), -1; got != want { + t.Errorf("computeDeferRecover() = %d, want %d", got, want) + } +} + +func TestGoDeferRecoverTraceTest_Eval(t *testing.T) { + tc := goDeferRecoverTraceTest() + + tests := []struct { + name string + response string + want float64 + }{ + {"exact correct", "-1", 1}, + {"correct with prose", "The program prints -1.", 1}, + {"correct with label", "Output: -1", 1}, + {"wrong: ignores recover overwrite", "10", 0}, + {"wrong: sign error", "1", 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Value != tt.want { + t.Errorf("Eval.Evaluate() = %v, want %v (detail: %s)", got.Value, tt.want, got.Detail) + } + }) + } +} + +// sliceAppendAliasCheck mirrors goSliceAppendAliasingCode's main() body, +// returning a[2] instead of printing it, so the ground truth is recomputed +// in-process rather than trusted from the doc comment alone. +func sliceAppendAliasCheck() int { + a := make([]int, 3, 5) + a[0], a[1], a[2] = 1, 2, 3 + b := a[:2] + b = append(b, 99) + _ = b + return a[2] +} + +func TestGoSliceAppendAliasing_GroundTruth(t *testing.T) { + if got, want := sliceAppendAliasCheck(), 99; got != want { + t.Errorf("sliceAppendAliasCheck() = %d, want %d", got, want) + } +} + +func TestGoSliceAppendAliasingTest_Eval(t *testing.T) { + tc := goSliceAppendAliasingTest() + + tests := []struct { + name string + response string + want float64 + }{ + {"exact correct", "99", 1}, + {"correct with prose", "a[2] is 99.", 1}, + {"correct with label", "The result is 99", 1}, + {"wrong: ignores aliasing, keeps original value", "3", 0}, + {"wrong: off-by-index confusion", "2", 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Value != tt.want { + t.Errorf("Eval.Evaluate() = %v, want %v (detail: %s)", got.Value, tt.want, got.Detail) + } + }) + } +} + +// TestGoMethodSetInterface_GroundTruth cross-checks the value_compiles=false, +// pointer_compiles=true claims in golang.go's doc comment by actually +// running `go build` on each snippet, mirroring c_test.go's +// TestCStructSizeWant_GroundTruth pattern of shelling out to the real +// toolchain rather than trusting hand analysis alone. +func TestGoMethodSetInterface_GroundTruth(t *testing.T) { + const preamble = `package main + +` + goMethodSetInterfaceCode + ` + +` + tests := []struct { + name string + mainLine string + wantBuilds bool + }{ + {"value type does not implement Stringer", `func main() { var _ Stringer = Wrapper{val: "hi"} }`, false}, + {"pointer type implements Stringer", `func main() { var _ Stringer = &Wrapper{val: "hi"} }`, true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + dir := t.TempDir() + src := preamble + tt.mainLine + "\n" + srcPath := filepath.Join(dir, "main.go") + if err := os.WriteFile(srcPath, []byte(src), 0o600); err != nil { + t.Fatalf("write main.go: %v", err) + } + // #nosec G204 -- srcPath is a fixed filename this test just wrote + // into its own t.TempDir(); not external input. + cmd := exec.Command("go", "build", "-o", filepath.Join(dir, "out"), srcPath) //nolint:gosec // see comment above + out, err := cmd.CombinedOutput() + builds := err == nil + if builds != tt.wantBuilds { + t.Errorf("go build success = %v, want %v (output: %s)", builds, tt.wantBuilds, out) + } + }) + } +} + +func TestGoMethodSetInterfaceTest_Eval(t *testing.T) { + tc := goMethodSetInterfaceTest() + + tests := []struct { + name string + response string + want float64 + }{ + {"exact correct", `{"value_compiles": false, "pointer_compiles": true}`, 1}, + {"correct fenced json", "```json\n{\"value_compiles\": false, \"pointer_compiles\": true}\n```", 1}, + {"correct with prose wrapper", `Answer: {"value_compiles": false, "pointer_compiles": true}`, 1}, + {"wrong: both true", `{"value_compiles": true, "pointer_compiles": true}`, 0.5}, + {"wrong: both false", `{"value_compiles": false, "pointer_compiles": false}`, 0.5}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Value != tt.want { + t.Errorf("Eval.Evaluate() = %v, want %v (detail: %s)", got.Value, tt.want, got.Detail) + } + }) + } +} + +// contextCancellationTraceCheck mirrors goContextCancellationTraceCode's +// main() body, returning the text it would print instead of printing it, to +// recompute the ground truth in-process. It also grounds the "no +// goroutine/timing dependency" claim in golang.go's doc comment: this +// function has no goroutines or sleeps at all, so a deterministic result +// here demonstrates the propagation is synchronous. +func contextCancellationTraceCheck() string { + parent, cancelParent := context.WithCancel(context.Background()) + child, cancelChild := context.WithCancel(parent) + defer cancelChild() + + cancelParent() + + select { + case <-child.Done(): + return child.Err().Error() + default: + return "not done" + } +} + +func TestGoContextCancellationTrace_GroundTruth(t *testing.T) { + if got, want := contextCancellationTraceCheck(), "context canceled"; got != want { + t.Errorf("contextCancellationTraceCheck() = %q, want %q", got, want) + } +} + +func TestGoContextCancellationTraceTest_Eval(t *testing.T) { + tc := goContextCancellationTraceTest() + + tests := []struct { + name string + response string + want float64 + }{ + {"exact correct", "context canceled", 1}, + {"correct case-insensitive", "CONTEXT CANCELED", 1}, + {"correct with surrounding whitespace", " context canceled ", 1}, + {"wrong: assumes select falls to default", "not done", 0}, + {"wrong: wrong sentinel error", "context deadline exceeded", 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Value != tt.want { + t.Errorf("Eval.Evaluate() = %v, want %v (detail: %s)", got.Value, tt.want, got.Detail) + } + }) + } +} + +func TestGoGenericsConstraintFixTest_Eval(t *testing.T) { + tc := goGenericsConstraintFixTest() + + fullOrderedConstraint := "```go\n" + `type Ordered interface { + ~int | ~int8 | ~int16 | ~int32 | ~int64 | + ~uint | ~uint8 | ~uint16 | ~uint32 | ~uint64 | + ~float32 | ~float64 | ~string +} + +func Max[T Ordered](a, b T) T { + if a > b { + return a + } + return b +} +` + "```" + + minimalConstraint := "```go\n" + `type Ordered interface { + ~int | ~float64 | ~string +} + +func Max[T Ordered](a, b T) T { + if a > b { + return a + } + return b +} +` + "```" + + stillBroken := "```go\n" + `func Max[T any](a, b T) T { + if a > b { + return a + } + return b +} +` + "```" + + tests := []struct { + name string + response string + want float64 + }{ + {"full Ordered constraint covering all integer widths", fullOrderedConstraint, 1}, + {"minimal constraint covering only the exercised types", minimalConstraint, 1}, + {"still broken: constraint left as any", stillBroken, 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Skipped { + t.Skip("go toolchain not available, skipping exec-evaluator assertion") + } + if got.Value != tt.want { + t.Errorf("Eval.Evaluate() = %v, want %v (detail: %s)", got.Value, tt.want, got.Detail) + } + }) + } +} + +func TestGoBoundedFanoutTest_Eval(t *testing.T) { + tc := goBoundedFanoutTest() + + correctBoundedRun := "```go\n" + `import "sync" + +func BoundedRun(tasks []func() error, limit int) error { + sem := make(chan struct{}, limit) + var wg sync.WaitGroup + var mu sync.Mutex + var firstErr error + + for _, task := range tasks { + task := task + sem <- struct{}{} + wg.Add(1) + go func() { + defer wg.Done() + defer func() { <-sem }() + if err := task(); err != nil { + mu.Lock() + if firstErr == nil { + firstErr = err + } + mu.Unlock() + } + }() + } + wg.Wait() + return firstErr +} +` + "```" + + sequentialBoundedRun := "```go\n" + `func BoundedRun(tasks []func() error, limit int) error { + for _, task := range tasks { + if err := task(); err != nil { + return err + } + } + return nil +} +` + "```" + + unboundedBoundedRun := "```go\n" + `import "sync" + +func BoundedRun(tasks []func() error, limit int) error { + var wg sync.WaitGroup + var mu sync.Mutex + var firstErr error + for _, task := range tasks { + task := task + wg.Add(1) + go func() { + defer wg.Done() + if err := task(); err != nil { + mu.Lock() + if firstErr == nil { + firstErr = err + } + mu.Unlock() + } + }() + } + wg.Wait() + return firstErr +} +` + "```" + + tests := []struct { + name string + response string + want float64 + }{ + {"correct semaphore-bounded concurrency", correctBoundedRun, 1}, + {"wrong: sequential, no concurrency", sequentialBoundedRun, 0}, + {"wrong: unbounded, ignores limit", unboundedBoundedRun, 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Skipped { + t.Skip("go toolchain not available, skipping exec-evaluator assertion") + } + if got.Value != tt.want { + t.Errorf("Eval.Evaluate() = %v, want %v (detail: %s)", got.Value, tt.want, got.Detail) + } + }) + } +} + +// boundedRunReference is a package-local reference implementation of the +// same bounded-fan-out shape the harness expects (semaphore-limited +// concurrency, first-error propagation), independent of the fenced +// test-string copy in TestGoBoundedFanoutTest_Eval, so the harness's +// concurrency-bound assertion is grounded against real, in-process +// goroutines rather than only against exec.Command output. +func boundedRunReference(tasks []func() error, limit int) error { + sem := make(chan struct{}, limit) + var wg sync.WaitGroup + var mu sync.Mutex + var firstErr error + + for _, task := range tasks { + task := task + sem <- struct{}{} + wg.Add(1) + go func() { + defer wg.Done() + defer func() { <-sem }() + if err := task(); err != nil { + mu.Lock() + if firstErr == nil { + firstErr = err + } + mu.Unlock() + } + }() + } + wg.Wait() + return firstErr +} + +func TestBoundedRunReference_GroundTruth(t *testing.T) { + const limit = 4 + const n = 20 + + var concurrent, maxConcurrent int32 + makeTask := func(shouldFail bool) func() error { + return func() error { + cur := atomic.AddInt32(&concurrent, 1) + for { + m := atomic.LoadInt32(&maxConcurrent) + if cur <= m || atomic.CompareAndSwapInt32(&maxConcurrent, m, cur) { + break + } + } + time.Sleep(time.Millisecond) + atomic.AddInt32(&concurrent, -1) + if shouldFail { + return errors.New("boom") + } + return nil + } + } + + okTasks := make([]func() error, n) + for i := range okTasks { + okTasks[i] = makeTask(false) + } + if err := boundedRunReference(okTasks, limit); err != nil { + t.Errorf("boundedRunReference(all ok) = %v, want nil", err) + } + if got := atomic.LoadInt32(&maxConcurrent); got < 2 || got > limit { + t.Errorf("maxConcurrent = %d, want in [2, %d]", got, limit) + } + + atomic.StoreInt32(&concurrent, 0) + atomic.StoreInt32(&maxConcurrent, 0) + failTasks := make([]func() error, n) + for i := range failTasks { + failTasks[i] = makeTask(i%3 == 0) + } + if err := boundedRunReference(failTasks, limit); err == nil { + t.Error("boundedRunReference(some fail) = nil, want an error") + } + if got := atomic.LoadInt32(&maxConcurrent); got < 2 || got > limit { + t.Errorf("maxConcurrent = %d, want in [2, %d]", got, limit) + } +} + +// TestGoChannelDeadlockCode_ContainsExpectedLines grounds the line-number +// claim in goChannelDeadlockTest's doc comment against the actual prompt +// text used at registration time, rather than trusting the doc comment's +// count independently of the real prompt. +func TestGoChannelDeadlockCode_ContainsExpectedLines(t *testing.T) { + tc := goChannelDeadlockTest() + if !strings.Contains(tc.Prompt, "5: \tch <- 1") { + t.Error("goChannelDeadlockTest prompt no longer has 'ch <- 1' on line 5; update the ground-truth line number") + } +} diff --git a/internal/tests/python.go b/internal/tests/python.go index 50da4b6..b9e448c 100644 --- a/internal/tests/python.go +++ b/internal/tests/python.go @@ -1,6 +1,8 @@ package tests import ( + "math" + "github.com/lukaszraczylo/llm-testbench/internal/eval" "github.com/lukaszraczylo/llm-testbench/internal/testkit" ) @@ -8,6 +10,14 @@ import ( func registerPythonTests(r *testkit.Registry) { r.Register(pyLogTriageTest()) r.Register(pyCosineTest()) + r.Register(pySoftmaxTest()) + r.Register(pyAsyncioGatherTraceTest()) + r.Register(pyMutableDefaultArgTest()) + r.Register(pyDictComprehensionTraceTest()) + r.Register(pyGeneratorExhaustionTraceTest()) + r.Register(pyPathlibRewriteTest()) + r.Register(pyJSONTransformTest()) + r.Register(pyRegexLogExtractionTest()) } // pyLogTriageLog is the inline kubelet log excerpt for pyLogTriageTest. @@ -94,3 +104,345 @@ only the number, nothing else.` Eval: eval.Numeric(eval.ExtractLastNumber[float64], pyCosineWant, 0.0005), } } + +// pySoftmaxVector is the inline 5-dimensional vector for pySoftmaxTest. +var pySoftmaxVector = []float64{2.0, 1.0, 0.1, -1.0, 0.5} + +// softmaxFirst computes the softmax probability of index 0 of v: +// exp(v[0]) / sum(exp(v[i]) for all i). Used to derive pySoftmaxWant at +// catalog-registration time rather than hardcoding a pre-computed constant, +// mirroring pyCosineWant's use of cosineSimilarity. +func softmaxFirst(v []float64) float64 { + var sum float64 + for _, x := range v { + sum += math.Exp(x) + } + return math.Exp(v[0]) / sum +} + +// pySoftmaxWant is derived, not hardcoded: python_test.go independently +// re-verifies it against math.Exp. +// ground truth: softmax(v)[0] = exp(v[0]) / sum(exp(v[i])), rounded to 4 +// decimal places as the prompt requests. +var pySoftmaxWant = round4dp(softmaxFirst(pySoftmaxVector)) + +func pySoftmaxTest() testkit.Test { + prompt := `Given the 5-dimensional vector: + +v = [2.0, 1.0, 0.1, -1.0, 0.5] + +Compute the softmax of v, and report only the probability assigned to the +first element (index 0), rounded to 4 decimal places. Respond with only the +number, nothing else.` + + return testkit.Test{ + ID: "py-softmax", + Category: "programming", + Subcategory: "python", + Description: "Compute the softmax probability of one element of an inline 5-dimensional vector to 4 decimal places.", + Prompt: prompt, + Eval: eval.Numeric(eval.ExtractLastNumber[float64], pySoftmaxWant, 0.0005), + } +} + +// pyAsyncioGatherTraceCode is the inline snippet for +// pyAsyncioGatherTraceTest. +const pyAsyncioGatherTraceCode = `import asyncio + + +async def worker(name, delay): + await asyncio.sleep(delay) + print(name) + + +async def main(): + await asyncio.gather(worker("A", 0.02), worker("B", 0.01), worker("C", 0.03)) + + +asyncio.run(main())` + +// pyAsyncioGatherTraceTest: trace the print order of three coroutines +// started together via asyncio.gather with different sleep delays. +// +// ground truth: asyncio.gather starts all three coroutines concurrently on +// the single-threaded event loop; each immediately hits its +// asyncio.sleep(delay) and the loop resumes them in order of scheduled wake +// time (current loop time + delay), not source order. worker("B", 0.01) has +// the shortest delay, so it wakes and prints first, then worker("A", 0.02), +// then worker("C", 0.03). Because the event loop is single-threaded and +// wake times are computed from fixed delays far apart (10ms increments), +// this ordering has no OS-scheduling race and is deterministic - confirmed +// by running the exact snippet 5 times during authoring (always B, A, C) +// and independently by python_test.go's PyRun-based ground-truth test. +func pyAsyncioGatherTraceTest() testkit.Test { + prompt := `Here is a Python program: + +` + "```python" + ` +` + pyAsyncioGatherTraceCode + ` +` + "```" + ` + +In what order are the three names printed when this program runs? Respond +with only a JSON array of the three names in print order, for example +["X","Y","Z"]` + + return testkit.Test{ + ID: "py-asyncio-gather-trace", + Category: "programming", + Subcategory: "python", + Description: "Trace the print order of three asyncio.gather coroutines with different sleep delays.", + System: terseCodeOnly, + Prompt: prompt, + Eval: eval.JSONStringArrayEquals([]string{"B", "A", "C"}), + } +} + +// pyMutableDefaultArgCode is the inline snippet for pyMutableDefaultArgTest. +const pyMutableDefaultArgCode = `def add_item(item, bucket=[]): + bucket.append(item) + return bucket + + +add_item(1) +add_item(2) +result = add_item(3) +print(result)` + +// pyMutableDefaultArgTest: trace the classic mutable-default-argument bug. +// +// ground truth: a default argument value (here, the empty list literal) is +// evaluated once, at function-definition time, and the same list object is +// reused as the default on every call that omits bucket - it is not +// recreated per call. The three calls all omit bucket, so they share and +// mutate one underlying list: after add_item(1) it holds [1], after +// add_item(2) it holds [1, 2], and result = add_item(3) returns the same +// list now holding [1, 2, 3]. print(result) prints Python's list repr, +// "[1, 2, 3]". python_test.go re-verifies this via PyRun on the identical +// snippet. +func pyMutableDefaultArgTest() testkit.Test { + prompt := `Here is a Python program: + +` + "```python" + ` +` + pyMutableDefaultArgCode + ` +` + "```" + ` + +What does this program print when run? Respond with only the exact text +printed, nothing else.` + + return testkit.Test{ + ID: "py-mutable-default-arg", + Category: "programming", + Subcategory: "python", + Description: "Trace the classic mutable-default-argument bug across three calls sharing one list.", + Prompt: prompt, + Eval: eval.Equals("[1, 2, 3]"), + } +} + +// pyDictComprehensionTraceCode is the inline snippet for +// pyDictComprehensionTraceTest. +const pyDictComprehensionTraceCode = `words = ["apple", "banana", "cherry", "date", "fig"] +lengths = {w: len(w) for w in words if len(w) > 3} +print(sorted(lengths.items()))` + +// pyDictComprehensionTraceTest: trace the exact output of a filtered dict +// comprehension whose items are then sorted. +// +// ground truth: the comprehension keeps words with len > 3 - apple(5), +// banana(6), cherry(6), date(4) - and drops fig(3). sorted() on a dict's +// .items() sorts the (key, value) tuples by key first (Python tuple +// ordering compares element-by-element), giving alphabetical key order: +// apple, banana, cherry, date. print() renders that list of tuples with +// Python's repr, producing the exact text below. python_test.go +// re-verifies this via PyRun on the identical snippet. +func pyDictComprehensionTraceTest() testkit.Test { + prompt := `Here is a Python program: + +` + "```python" + ` +` + pyDictComprehensionTraceCode + ` +` + "```" + ` + +What does this program print when run? Respond with only the exact text +printed, nothing else.` + + return testkit.Test{ + ID: "py-dict-comprehension-trace", + Category: "programming", + Subcategory: "python", + Description: "Trace the exact output of a length-filtered dict comprehension whose items are sorted and printed.", + Prompt: prompt, + Eval: eval.Equals("[('apple', 5), ('banana', 6), ('cherry', 6), ('date', 4)]"), + } +} + +// pyGeneratorExhaustionTraceCode is the inline snippet for +// pyGeneratorExhaustionTraceTest. +const pyGeneratorExhaustionTraceCode = `def gen(): + for i in range(3): + yield i + + +g = gen() +first_pass = list(g) +second_pass = list(g) +print(first_pass, second_pass)` + +// pyGeneratorExhaustionTraceTest: trace the exact output of iterating an +// already-exhausted generator a second time. +// +// ground truth: a Python generator object is a single-pass iterator - once +// list(g) has driven it to completion (StopIteration), the same generator +// object has no more items to yield. first_pass = list(g) consumes all +// three values, giving [0, 1, 2]; second_pass = list(g) on the same, +// now-exhausted g immediately raises StopIteration inside list(), which +// list() treats as "no items", giving []. print(a, b) separates its +// arguments with a single space, producing the exact text below. +// python_test.go re-verifies this via PyRun on the identical snippet. +func pyGeneratorExhaustionTraceTest() testkit.Test { + prompt := `Here is a Python program: + +` + "```python" + ` +` + pyGeneratorExhaustionTraceCode + ` +` + "```" + ` + +What does this program print when run? Respond with only the exact text +printed, nothing else.` + + return testkit.Test{ + ID: "py-generator-exhaustion-trace", + Category: "programming", + Subcategory: "python", + Description: "Trace the exact output of iterating a generator a second time after it is already exhausted.", + Prompt: prompt, + Eval: eval.Equals("[0, 1, 2] []"), + } +} + +// pyOsPathCode is the inline os.path-based function pyPathlibRewriteTest +// asks the model to rewrite using pathlib. +const pyOsPathCode = `import os + + +def find_configs(base_dir): + results = [] + for root, dirs, files in os.walk(base_dir): + for f in files: + if f.endswith(".yaml"): + results.append(os.path.join(root, f)) + return results` + +// pyPathlibRewriteTest: rewrite an os.path/os.walk function using pathlib +// idioms with equivalent behavior. +func pyPathlibRewriteTest() testkit.Test { + prompt := `Here is a Python function that recursively finds all ".yaml" files +under base_dir and returns their full paths: + +` + "```python" + ` +` + pyOsPathCode + ` +` + "```" + ` + +Rewrite this function using pathlib idioms instead of os.path/os.walk, +keeping the same behavior: recursively find every ".yaml" file under +base_dir and return their full paths. Respond with only the rewritten +function.` + + evaluator := eval.All( + eval.W(eval.ContainsAny("from pathlib import Path", "import pathlib"), 1), + eval.W(eval.ContainsAll("rglob", "*.yaml"), 2), + ) + + return testkit.Test{ + ID: "py-pathlib-rewrite", + Category: "programming", + Subcategory: "python", + Description: "Rewrite an os.path/os.walk recursive file finder using pathlib idioms.", + System: terseCodeOnly, + Prompt: prompt, + Eval: evaluator, + } +} + +// pyJSONTransformRecords is the inline dataset for pyJSONTransformTest. +// ground truth: totals by dept, hand-summed (also re-derived in +// python_test.go by scanning the same literal data): eng = 95000 + 105000 + +// 88000 = 288000; ops = 72000; sales = 60000 + 58000 = 118000. +const pyJSONTransformRecords = `records = [ + {"dept": "eng", "salary": 95000}, + {"dept": "sales", "salary": 60000}, + {"dept": "eng", "salary": 105000}, + {"dept": "ops", "salary": 72000}, + {"dept": "sales", "salary": 58000}, + {"dept": "eng", "salary": 88000}, +]` + +// pyJSONTransformWant is the expected stdout of a correct transform script: +// total salary per department, sorted alphabetically by department, as +// "dept=total" per line. +const pyJSONTransformWant = "eng=288000\nops=72000\nsales=118000" + +func pyJSONTransformTest() testkit.Test { + prompt := `Here is a Python list of employee records: + +` + "```python" + ` +` + pyJSONTransformRecords + ` +` + "```" + ` + +Write a self-contained python3 script (embed the records above as data in +your script; do not read from stdin or a file) that sums salary per dept +and prints one line per department as "dept=total", sorted alphabetically by +department name. Print exactly those lines and nothing else.` + + return testkit.Test{ + ID: "py-json-transform", + Category: "programming", + Subcategory: "python", + Description: "Write a python3 script that sums an inline list of salary records by department.", + System: terseCodeOnly, + Prompt: prompt, + Eval: eval.PyRun(eval.PassthroughHarness, pyJSONTransformWant), + } +} + +// pyRegexLogLines is the inline nginx-style access-log fixture for +// pyRegexLogExtractionTest. +// ground truth: HTTP status codes by line, in order: 200, 404, 200, 500, +// 200, 404, 500 - giving 200=3, 404=2, 500=2 (also re-derived in +// python_test.go by scanning the same literal data with Go's regexp). +const pyRegexLogLines = `log_lines = [ + '203.0.113.5 - - [10/Aug/2026:10:00:01 +0000] "GET /api/health HTTP/1.1" 200 512', + '203.0.113.7 - - [10/Aug/2026:10:00:02 +0000] "GET /api/users HTTP/1.1" 404 128', + '203.0.113.5 - - [10/Aug/2026:10:00:03 +0000] "POST /api/login HTTP/1.1" 200 256', + '203.0.113.9 - - [10/Aug/2026:10:00:04 +0000] "GET /api/orders HTTP/1.1" 500 96', + '203.0.113.5 - - [10/Aug/2026:10:00:05 +0000] "GET /api/health HTTP/1.1" 200 512', + '203.0.113.7 - - [10/Aug/2026:10:00:06 +0000] "GET /api/missing HTTP/1.1" 404 128', + '203.0.113.9 - - [10/Aug/2026:10:00:07 +0000] "GET /api/orders HTTP/1.1" 500 96', +]` + +// pyRegexLogExtractionWant is the expected stdout: counts per HTTP status +// code, sorted ascending by the numeric code, as "code=count" per line. +const pyRegexLogExtractionWant = "200=3\n404=2\n500=2" + +func pyRegexLogExtractionTest() testkit.Test { + prompt := `Here are 7 lines of nginx-style access log output: + +` + "```python" + ` +` + pyRegexLogLines + ` +` + "```" + ` + +Write a self-contained python3 script (embed the log_lines list above as +data in your script; do not read from stdin or a file) that uses the re +module to extract the 3-digit HTTP status code from each line (the number +that appears right after the quoted request, e.g. right after +"GET ... HTTP/1.1"), counts occurrences of each status code, and prints one +line per status code as "code=count", sorted ascending by the numeric code. +Print exactly those lines and nothing else.` + + return testkit.Test{ + ID: "py-regex-log-extraction", + Category: "programming", + Subcategory: "python", + Description: "Write a python3 script that uses re to extract and count HTTP status codes from an inline access log.", + System: terseCodeOnly, + Prompt: prompt, + Eval: eval.PyRun(eval.PassthroughHarness, pyRegexLogExtractionWant), + } +} diff --git a/internal/tests/python_test.go b/internal/tests/python_test.go index 39acbed..01fb2fc 100644 --- a/internal/tests/python_test.go +++ b/internal/tests/python_test.go @@ -3,8 +3,14 @@ package tests import ( "context" "math" + "os" + "os/exec" + "path/filepath" + "regexp" "strings" "testing" + + "github.com/lukaszraczylo/llm-testbench/internal/eval" ) func TestCosineSimilarity_KnownCases(t *testing.T) { @@ -108,6 +114,398 @@ for key in sorted(counts): } } +func TestSoftmaxFirst_KnownCases(t *testing.T) { + // Grounds the softmaxFirst helper itself before trusting it to derive + // pySoftmaxWant. + tests := []struct { + name string + v []float64 + want float64 + }{ + {"uniform vector splits evenly", []float64{1, 1, 1}, 1.0 / 3.0}, + {"single element gets all the mass", []float64{5}, 1}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := softmaxFirst(tt.v) + if math.Abs(got-tt.want) > 1e-9 { + t.Errorf("softmaxFirst(%v) = %v, want %v", tt.v, got, tt.want) + } + }) + } +} + +func TestPySoftmaxWant_GroundTruth(t *testing.T) { + // Independent re-derivation via math.Exp, not via the softmaxFirst + // helper under test, per PLAN.md's rule to recompute cheap ground + // truths in the unit test. + v := pySoftmaxVector + var sum float64 + for _, x := range v { + sum += math.Exp(x) + } + want := math.Exp(v[0]) / sum + wantRounded := math.Round(want*10000) / 10000 + + if pySoftmaxWant != wantRounded { + t.Errorf("pySoftmaxWant = %v, independently recomputed = %v", pySoftmaxWant, wantRounded) + } +} + +func TestPySoftmaxTest_Eval(t *testing.T) { + tc := pySoftmaxTest() + + tests := []struct { + name string + response string + want float64 + }{ + {"exact", "0.5585", 1}, + {"within tolerance", "0.5588", 1}, + {"prose wrapped", "The softmax probability is 0.5585.", 1}, + {"outside tolerance", "0.2", 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Value != tt.want { + t.Errorf("Eval.Evaluate() = %v, want %v (detail: %s)", got.Value, tt.want, got.Detail) + } + }) + } +} + +func TestPyAsyncioGatherTrace_GroundTruth(t *testing.T) { + // Runs the identical Python snippet embedded in pyAsyncioGatherTraceCode + // via python3 and checks the print order is exactly B, A, C - the same + // oracle mechanism as eval.PyRun, independent of the doc-comment + // reasoning in python.go. + if _, err := exec.LookPath("python3"); err != nil { + t.Skip("python3 not found on PATH, skipping ground-truth cross-check") + } + + dir := t.TempDir() + scriptPath := filepath.Join(dir, "gather.py") + if err := os.WriteFile(scriptPath, []byte(pyAsyncioGatherTraceCode), 0o600); err != nil { + t.Fatalf("write gather.py: %v", err) + } + + // #nosec G204 -- scriptPath is a fixed filename this test just wrote + // into its own t.TempDir(); not external input. + out, err := exec.Command("python3", scriptPath).Output() //nolint:gosec // see comment above + if err != nil { + t.Fatalf("running gather.py failed: %v", err) + } + + got := strings.TrimSpace(string(out)) + want := "B\nA\nC" + if got != want { + t.Errorf("python3 gather.py stdout = %q, want %q", got, want) + } +} + +func TestPyAsyncioGatherTraceTest_Eval(t *testing.T) { + tc := pyAsyncioGatherTraceTest() + + tests := []struct { + name string + response string + want float64 + }{ + {"exact correct", `["B","A","C"]`, 1}, + {"correct fenced json", "```json\n[\"B\", \"A\", \"C\"]\n```", 1}, + {"correct with prose wrapper", `The order is: ["B","A","C"]`, 1}, + {"wrong: source order, ignores delays", `["A","B","C"]`, 0}, + {"wrong: reversed", `["C","A","B"]`, 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Value != tt.want { + t.Errorf("Eval.Evaluate() = %v, want %v (detail: %s)", got.Value, tt.want, got.Detail) + } + }) + } +} + +// TestPyMutableDefaultArg_GroundTruth runs the identical snippet via PyRun +// with a passthrough harness, so the doc-comment claim in python.go is +// cross-checked against the real python3 interpreter, not just reasoning. +func TestPyMutableDefaultArg_GroundTruth(t *testing.T) { + score := eval.PyRun(eval.PassthroughHarness, "[1, 2, 3]").Evaluate( + context.Background(), "```python\n"+pyMutableDefaultArgCode+"\n```", + ) + if score.Skipped { + t.Skip("python3 not available, skipping ground-truth cross-check") + } + if score.Value != 1 { + t.Errorf("running pyMutableDefaultArgCode via PyRun scored %v, want 1 (detail: %s)", score.Value, score.Detail) + } +} + +func TestPyMutableDefaultArgTest_Eval(t *testing.T) { + tc := pyMutableDefaultArgTest() + + tests := []struct { + name string + response string + want float64 + }{ + {"exact correct", "[1, 2, 3]", 1}, + {"correct with surrounding whitespace", " [1, 2, 3] ", 1}, + {"correct with trailing newline", "[1, 2, 3]\n", 1}, + {"wrong: assumes a fresh list per call", "[3]", 0}, + {"wrong: only counts explicit-arg calls", "[1, 2]", 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Value != tt.want { + t.Errorf("Eval.Evaluate() = %v, want %v (detail: %s)", got.Value, tt.want, got.Detail) + } + }) + } +} + +// TestPyDictComprehensionTrace_GroundTruth runs the identical snippet via +// PyRun with a passthrough harness. +func TestPyDictComprehensionTrace_GroundTruth(t *testing.T) { + want := "[('apple', 5), ('banana', 6), ('cherry', 6), ('date', 4)]" + score := eval.PyRun(eval.PassthroughHarness, want).Evaluate( + context.Background(), "```python\n"+pyDictComprehensionTraceCode+"\n```", + ) + if score.Skipped { + t.Skip("python3 not available, skipping ground-truth cross-check") + } + if score.Value != 1 { + t.Errorf("running pyDictComprehensionTraceCode via PyRun scored %v, want 1 (detail: %s)", score.Value, score.Detail) + } +} + +func TestPyDictComprehensionTraceTest_Eval(t *testing.T) { + tc := pyDictComprehensionTraceTest() + + tests := []struct { + name string + response string + want float64 + }{ + {"exact correct", "[('apple', 5), ('banana', 6), ('cherry', 6), ('date', 4)]", 1}, + {"correct with surrounding whitespace", " [('apple', 5), ('banana', 6), ('cherry', 6), ('date', 4)] ", 1}, + {"correct uppercase-folded", "[('APPLE', 5), ('BANANA', 6), ('CHERRY', 6), ('DATE', 4)]", 1}, + {"wrong: includes fig, forgets the length filter", "[('apple', 5), ('banana', 6), ('cherry', 6), ('date', 4), ('fig', 3)]", 0}, + {"wrong: unsorted, dict insertion order", "[('apple', 5), ('banana', 6), ('cherry', 6), ('fig', 3), ('date', 4)]", 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Value != tt.want { + t.Errorf("Eval.Evaluate() = %v, want %v (detail: %s)", got.Value, tt.want, got.Detail) + } + }) + } +} + +// TestPyGeneratorExhaustionTrace_GroundTruth runs the identical snippet via +// PyRun with a passthrough harness. +func TestPyGeneratorExhaustionTrace_GroundTruth(t *testing.T) { + score := eval.PyRun(eval.PassthroughHarness, "[0, 1, 2] []").Evaluate( + context.Background(), "```python\n"+pyGeneratorExhaustionTraceCode+"\n```", + ) + if score.Skipped { + t.Skip("python3 not available, skipping ground-truth cross-check") + } + if score.Value != 1 { + t.Errorf("running pyGeneratorExhaustionTraceCode via PyRun scored %v, want 1 (detail: %s)", score.Value, score.Detail) + } +} + +func TestPyGeneratorExhaustionTraceTest_Eval(t *testing.T) { + tc := pyGeneratorExhaustionTraceTest() + + tests := []struct { + name string + response string + want float64 + }{ + {"exact correct", "[0, 1, 2] []", 1}, + {"correct with surrounding whitespace", " [0, 1, 2] [] ", 1}, + {"correct with trailing newline", "[0, 1, 2] []\n", 1}, + {"wrong: assumes the generator restarts", "[0, 1, 2] [0, 1, 2]", 0}, + {"wrong: assumes an error is raised and printed", "StopIteration", 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Value != tt.want { + t.Errorf("Eval.Evaluate() = %v, want %v (detail: %s)", got.Value, tt.want, got.Detail) + } + }) + } +} + +func TestPyPathlibRewriteTest_Eval(t *testing.T) { + tc := pyPathlibRewriteTest() + + goodRewrite := "```python\n" + `from pathlib import Path + + +def find_configs(base_dir): + return [str(p) for p in Path(base_dir).rglob("*.yaml")] +` + "```" + + goodRewriteAltImport := "```python\n" + `import pathlib + + +def find_configs(base_dir): + return [str(p) for p in pathlib.Path(base_dir).rglob("*.yaml")] +` + "```" + + stillUsesOsPath := "```python\n" + pyOsPathCode + "\n```" + + missingRglob := "```python\n" + `from pathlib import Path + + +def find_configs(base_dir): + return [str(p) for p in Path(base_dir).glob("*.yaml")] +` + "```" + + tests := []struct { + name string + response string + want float64 + }{ + {"good rewrite: from pathlib import Path", goodRewrite, 1}, + {"good rewrite: import pathlib", goodRewriteAltImport, 1}, + {"wrong: unchanged os.path/os.walk code", stillUsesOsPath, 0}, + {"partial: uses pathlib but non-recursive glob, not rglob", missingRglob, 2.0 / 3.0}, // (1*1 + 2*0.5)/3: ContainsAny hits, ContainsAll("rglob","*.yaml") matches only "*.yaml" + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Value != tt.want { + t.Errorf("Eval.Evaluate() = %v, want %v (detail: %s)", got.Value, tt.want, got.Detail) + } + }) + } +} + +func TestPyJSONTransformWant_GroundTruth(t *testing.T) { + // Independent re-derivation of the expected per-department totals by + // summing hardcoded literals matching pyJSONTransformRecords, rather + // than trusting the hand sum in the doc comment. + totals := map[string]int{ + "eng": 95000 + 105000 + 88000, + "sales": 60000 + 58000, + "ops": 72000, + } + want := "eng=288000\nops=72000\nsales=118000" + if pyJSONTransformWant != want { + t.Errorf("pyJSONTransformWant = %q, want %q", pyJSONTransformWant, want) + } + if totals["eng"] != 288000 || totals["sales"] != 118000 || totals["ops"] != 72000 { + t.Errorf("independently recomputed totals = %v, want eng=288000 sales=118000 ops=72000", totals) + } +} + +func TestPyJSONTransformTest_Eval(t *testing.T) { + tc := pyJSONTransformTest() + + correctScript := "```python\n" + pyJSONTransformRecords + ` + +totals = {} +for r in records: + totals[r["dept"]] = totals.get(r["dept"], 0) + r["salary"] + +for dept in sorted(totals): + print(f"{dept}={totals[dept]}") +` + "```" + + wrongScript := "```python\nprint('eng=1')\nprint('ops=1')\nprint('sales=1')\n```" + + tests := []struct { + name string + response string + want float64 + }{ + {"correct transform script", correctScript, 1}, + {"wrong totals", wrongScript, 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Skipped { + t.Skip("python3 not available, skipping exec-evaluator assertion") + } + if got.Value != tt.want { + t.Errorf("Eval.Evaluate() = %v, want %v (detail: %s)", got.Value, tt.want, got.Detail) + } + }) + } +} + +func TestPyRegexLogExtractionWant_GroundTruth(t *testing.T) { + // Independent re-derivation of the expected counts by scanning the raw + // log text directly with Go's regexp, rather than trusting the hand + // count in the doc comment. + statusPattern := regexp.MustCompile(`"\s*(\d{3})\s+\d+`) + matches := statusPattern.FindAllStringSubmatch(pyRegexLogLines, -1) + if len(matches) != 7 { + t.Fatalf("found %d status codes in pyRegexLogLines, want 7", len(matches)) + } + counts := map[string]int{} + for _, m := range matches { + counts[m[1]]++ + } + if counts["200"] != 3 || counts["404"] != 2 || counts["500"] != 2 { + t.Errorf("independently recomputed counts = %v, want 200=3 404=2 500=2", counts) + } + want := "200=3\n404=2\n500=2" + if pyRegexLogExtractionWant != want { + t.Errorf("pyRegexLogExtractionWant = %q, want %q", pyRegexLogExtractionWant, want) + } +} + +func TestPyRegexLogExtractionTest_Eval(t *testing.T) { + tc := pyRegexLogExtractionTest() + + correctScript := "```python\nimport re\n\n" + pyRegexLogLines + ` + +pattern = re.compile(r'"\s*(\d{3})\s+\d+$') +counts = {} +for line in log_lines: + m = pattern.search(line) + if m: + code = m.group(1) + counts[code] = counts.get(code, 0) + 1 + +for code in sorted(counts): + print(f"{code}={counts[code]}") +` + "```" + + wrongScript := "```python\nprint('200=1')\nprint('404=1')\nprint('500=1')\n```" + + tests := []struct { + name string + response string + want float64 + }{ + {"correct regex extraction script", correctScript, 1}, + {"wrong counts", wrongScript, 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Skipped { + t.Skip("python3 not available, skipping exec-evaluator assertion") + } + if got.Value != tt.want { + t.Errorf("Eval.Evaluate() = %v, want %v (detail: %s)", got.Value, tt.want, got.Detail) + } + }) + } +} + func TestPyCosineTest_Eval(t *testing.T) { tc := pyCosineTest() diff --git a/internal/tests/typescript.go b/internal/tests/typescript.go index 5766b14..f1bf338 100644 --- a/internal/tests/typescript.go +++ b/internal/tests/typescript.go @@ -10,6 +10,15 @@ import ( func registerTypeScriptTests(r *testkit.Registry) { r.Register(tsDebounceComposableTest()) + r.Register(tsDiscriminatedUnionAreaTest()) + r.Register(tsGenericUtilityTypeTest()) + r.Register(tsPromiseAllSettledTraceTest()) + r.Register(tsVueRefReactiveUnwrapTest()) + r.Register(tsComputedVsWatchTest()) + r.Register(tsESLintFlatConfigTest()) + r.Register(tsTsconfigStrictFlagTest()) + r.Register(tsArrayMethodChainTraceTest()) + r.Register(tsOptionalChainingNullishTraceTest()) } // tsAnyEscapeHatchPattern matches TypeScript's `any` used as a type: a type @@ -66,3 +75,378 @@ Requirements: Eval: evaluator, } } + +// tsDiscriminatedUnionAreaCode is the inline snippet for +// tsDiscriminatedUnionAreaTest: a discriminated union narrowed with a +// switch on its "kind" tag. +const tsDiscriminatedUnionAreaCode = `type Shape = + | { kind: "circle"; radius: number } + | { kind: "rectangle"; width: number; height: number }; + +function area(s: Shape): number { + switch (s.kind) { + case "circle": + return Math.PI * s.radius ** 2; + case "rectangle": + return s.width * s.height; + } +} + +const shapes: Shape[] = [ + { kind: "rectangle", width: 4, height: 5 }, + { kind: "circle", radius: 3 }, +]; + +const total = shapes.map(area).reduce((a, b) => a + b, 0); +console.log(Math.round(total * 100) / 100);` + +// tsDiscriminatedUnionAreaTest: trace the exact output of code that narrows +// a discriminated union in a switch and sums the per-branch results. +// +// ground truth: switching on the "kind" tag narrows each branch to its +// specific member of the Shape union, so s.radius is only accessible in the +// "circle" case and s.width/s.height only in "rectangle". area({kind: +// "rectangle", width: 4, height: 5}) = 4*5 = 20. area({kind: "circle", +// radius: 3}) = pi*3^2 = 28.274333882308138. Their sum, rounded to 2 +// decimal places (Math.round(total*100)/100), is 48.27. typescript_test.go +// cross-checks this by running the equivalent JavaScript (the same +// arithmetic, with TypeScript's type annotations erased, since switch-based +// narrowing is a compile-time-only construct with no runtime effect on +// these values) via `node`, guarded by node's presence. +func tsDiscriminatedUnionAreaTest() testkit.Test { + prompt := `Here is a TypeScript program: + +` + "```ts" + ` +` + tsDiscriminatedUnionAreaCode + ` +` + "```" + ` + +What number does this program print? Respond with only the number, nothing +else.` + + return testkit.Test{ + ID: "ts-discriminated-union-narrowing", + Category: "programming", + Subcategory: "typescript", + Description: "Trace the exact output of a discriminated union narrowed with a switch, summed across variants.", + Prompt: prompt, + Eval: eval.Numeric(eval.ExtractLastNumber[float64], 48.27, 0.005), + } +} + +// tsGenericUtilityTypeTest: author a recursive generic utility type +// (DeepPartial) using a mapped type and a conditional type, without any. +func tsGenericUtilityTypeTest() testkit.Test { + prompt := `Write a generic TypeScript utility type ` + "`DeepPartial`" + ` that +makes T and all of its nested object properties optional, recursively (so +given an interface with a nested object property, that nested object's own +properties are also optional). Do not use the "any" type anywhere. Respond +with only the type definition.` + + evaluator := eval.All( + eval.W(eval.ContainsAll("DeepPartial", "keyof T"), 2), + eval.W(eval.ContainsAny("DeepPartial"), 1), + eval.W(noAnyEscapeHatch(), 1), + ) + + return testkit.Test{ + ID: "ts-generic-utility-type", + Category: "programming", + Subcategory: "typescript", + Description: "Author a recursive generic DeepPartial utility type using a mapped and conditional type, without any.", + System: terseCodeOnly, + Prompt: prompt, + Eval: evaluator, + } +} + +// tsPromiseAllSettledTraceCode is the inline snippet for +// tsPromiseAllSettledTraceTest. +const tsPromiseAllSettledTraceCode = `async function main() { + const results = await Promise.allSettled([ + Promise.resolve(1), + Promise.reject("fail"), + Promise.resolve(3), + ]); + console.log(results.map((r) => r.status)); +} + +main();` + +// tsPromiseAllSettledTraceTest: trace the exact "status" values +// Promise.allSettled reports for a mix of resolved and rejected promises. +// +// ground truth: unlike Promise.all, Promise.allSettled never short-circuits +// on a rejection - it waits for every input promise to settle and reports +// each one's outcome as {status: "fulfilled", value} or {status: "rejected", +// reason}, in the same order as the input array. The three inputs settle as +// fulfilled, rejected, fulfilled respectively (the middle one rejects with +// "fail"), so results.map(r => r.status) is +// ["fulfilled", "rejected", "fulfilled"]. typescript_test.go cross-checks +// this by running the equivalent JavaScript via `node`. +func tsPromiseAllSettledTraceTest() testkit.Test { + prompt := `Here is a TypeScript program: + +` + "```ts" + ` +` + tsPromiseAllSettledTraceCode + ` +` + "```" + ` + +What array of statuses does this program log? Respond with only a JSON +array of the three status strings in order, for example +["fulfilled","fulfilled","fulfilled"]` + + return testkit.Test{ + ID: "ts-promise-allsettled-trace", + Category: "programming", + Subcategory: "typescript", + Description: "Trace the exact per-promise status array Promise.allSettled reports for a mixed resolve/reject batch.", + System: terseCodeOnly, + Prompt: prompt, + Eval: eval.JSONStringArrayEquals([]string{"fulfilled", "rejected", "fulfilled"}), + } +} + +// tsVueRefReactiveUnwrapTest: decide whether Vue 3's ref-auto-unwrapping +// applies to a property access on a reactive object versus an element +// access on a reactive array. +// +// ground truth: Vue 3's reactive() unwraps a nested ref when it is accessed +// as a property of the reactive object (state.count reads/writes as a plain +// number, not a Ref), but explicitly does NOT unwrap a ref when it is +// accessed as an element of a reactive array or a native collection like +// Map (list[0] stays a Ref object). Verified empirically for this test (not +// from memory): `reactive({count: ref(0)}).count` has typeof "number" and +// isRef() false, while `reactive([ref(0)])[0]` has isRef() true, using +// vue@3.5.41's actual reactivity runtime. +func tsVueRefReactiveUnwrapTest() testkit.Test { + prompt := `In Vue 3's Composition API: + +` + "```ts" + ` +const state = reactive({ count: ref(0) }); +const list = reactive([ref(0)]); +` + "```" + ` + +Does accessing ` + "`state.count`" + ` (a ref nested as a property of a +reactive object) auto-unwrap to a plain number? Does accessing +` + "`list[0]`" + ` (a ref nested as an element of a reactive array) +auto-unwrap to a plain number? Respond with only a JSON object: +{"object_property_unwraps": true|false, "array_element_unwraps": true|false}` + + evaluator := eval.All( + eval.W(eval.JSONField("object_property_unwraps", true), 1), + eval.W(eval.JSONField("array_element_unwraps", false), 1), + ) + + return testkit.Test{ + ID: "ts-vue-ref-reactive-unwrap", + Category: "programming", + Subcategory: "typescript", + Description: "Decide whether Vue 3 ref auto-unwrapping applies to a reactive object property versus a reactive array element.", + System: terseCodeOnly, + Prompt: prompt, + Eval: evaluator, + } +} + +// tsComputedVsWatchChoice is the fixed vocabulary tsComputedVsWatchTest +// allows for each scenario's answer. +const ( + tsComputedVsWatchChoiceComputed = "computed" + tsComputedVsWatchChoiceWatch = "watch" +) + +// tsComputedVsWatchTest: choose computed() versus watch() for two Vue 3 +// Composition API scenarios, one pure derivation and one side effect. +// +// ground truth: per Vue's own guidance, computed() is for deriving a value +// from reactive state with no side effects - Vue's docs recommend it +// "whenever possible" for exactly this - while watch() is for running a +// side effect (an API call, a DOM mutation, a storage write, logging) in +// reaction to a state change. Scenario A only derives a formatted string +// for template display (no side effect) -> computed. Scenario B persists a +// value to localStorage whenever it changes (a side effect) -> watch. +func tsComputedVsWatchTest() testkit.Test { + prompt := `Two Vue 3 Composition API scenarios: + +Scenario A: derive a formatted currency string from a "price" ref, used +only for display in the template. Deriving it has no side effects. + +Scenario B: whenever a "settings" ref changes, persist the new value to +localStorage. This has a side effect (writing to storage). + +For each scenario, should you use computed() or watch()? Respond with only +a JSON object: {"scenario_a": "computed"|"watch", "scenario_b": "computed"|"watch"}` + + evaluator := eval.All( + eval.W(eval.JSONField("scenario_a", tsComputedVsWatchChoiceComputed), 1), + eval.W(eval.JSONField("scenario_b", tsComputedVsWatchChoiceWatch), 1), + ) + + return testkit.Test{ + ID: "ts-computed-vs-watch", + Category: "programming", + Subcategory: "typescript", + Description: "Choose computed() versus watch() for a pure-derivation scenario and a side-effect scenario.", + System: terseCodeOnly, + Prompt: prompt, + Eval: evaluator, + } +} + +// tsOldEslintrcConfig is the inline legacy config tsESLintFlatConfigTest +// asks the model to migrate. +const tsOldEslintrcConfig = `{ + "env": { "browser": true, "es2021": true }, + "extends": ["eslint:recommended"], + "parserOptions": { "ecmaVersion": "latest", "sourceType": "module" }, + "rules": { "semi": "error", "prefer-const": "error" } +}` + +// tsESLintFlatConfigTest: migrate a legacy .eslintrc.json to ESLint v9's +// flat config format. +func tsESLintFlatConfigTest() testkit.Test { + prompt := `Here is a legacy .eslintrc.json: + +` + "```json" + ` +` + tsOldEslintrcConfig + ` +` + "```" + ` + +ESLint v9 uses the flat config format (eslint.config.js) by default; the +legacy .eslintrc format is no longer picked up automatically. Rewrite this +configuration as an eslint.config.js flat config, preserving the same rules +(semi: error, prefer-const: error) and the ecmaVersion/sourceType settings +via languageOptions. Respond with only the eslint.config.js file content.` + + evaluator := eval.All( + eval.W(eval.ContainsAll("export default"), 2), + eval.W(eval.ContainsAny("defineConfig(", "languageOptions"), 1), + ) + + return testkit.Test{ + ID: "ts-eslint-flat-config", + Category: "programming", + Subcategory: "typescript", + Description: "Migrate a legacy .eslintrc.json to ESLint v9's flat eslint.config.js format.", + System: terseCodeOnly, + Prompt: prompt, + Eval: evaluator, + } +} + +// tsImplicitAnyBugCode is the inline snippet for tsTsconfigStrictFlagTest: +// a function whose parameter has no type annotation. +const tsImplicitAnyBugCode = `function greet(name) { + return "Hello, " + name.toUpperCase(); +}` + +// tsTsconfigStrictFlagTest: identify which tsconfig strict-family compiler +// flag would catch an implicit-any parameter at compile time. +// +// ground truth: `name` has no type annotation and TypeScript cannot infer +// one from usage, so under the "noImplicitAny" compiler option (part of the +// "strict" family, also implied by "strict": true) the compiler reports +// "Parameter 'name' implicitly has an 'any' type." Verified empirically for +// this test (not from memory): compiling this exact snippet with +// typescript@5's tsc --noImplicitAny reports that error (exit code 2); +// without the flag it compiles cleanly (exit code 0). +func tsTsconfigStrictFlagTest() testkit.Test { + prompt := `Here is a TypeScript function: + +` + "```ts" + ` +` + tsImplicitAnyBugCode + ` +` + "```" + ` + +This compiles without error under TypeScript's default (non-strict) +settings, but name has no type annotation and none can be inferred from its +usage. Which single tsconfig strict-family compiler option, when enabled, +makes the compiler reject this with an error? Respond with only a JSON +object: {"flag": ""}` + + return testkit.Test{ + ID: "ts-tsconfig-strict-flag", + Category: "programming", + Subcategory: "typescript", + Description: "Identify the tsconfig strict-family flag that catches an implicit-any function parameter.", + System: terseCodeOnly, + Prompt: prompt, + Eval: eval.JSONField("flag", "noImplicitAny"), + } +} + +// tsArrayMethodChainCode is the inline snippet for +// tsArrayMethodChainTraceTest. +const tsArrayMethodChainCode = `const nums = [3, 8, 1, 9, 4, 6, 2, 7]; +const result = nums + .filter((n) => n % 2 === 0) + .map((n) => n * n) + .reduce((acc, n) => acc + n, 0); +console.log(result);` + +// tsArrayMethodChainTraceTest: trace the exact numeric result of a +// filter/map/reduce chain over an inline array. +// +// ground truth: filter keeps the even numbers, in original order: 8, 4, 6, +// 2. map squares each: 64, 16, 36, 4. reduce sums them with an initial +// accumulator of 0: 64+16+36+4 = 120. typescript_test.go cross-checks this +// by running the equivalent JavaScript via `node`. +func tsArrayMethodChainTraceTest() testkit.Test { + prompt := `Here is a TypeScript program: + +` + "```ts" + ` +` + tsArrayMethodChainCode + ` +` + "```" + ` + +What number does this program print? Respond with only the number, nothing +else.` + + return testkit.Test{ + ID: "ts-array-method-chain", + Category: "programming", + Subcategory: "typescript", + Description: "Trace the exact output of a filter/map/reduce chain over an inline number array.", + Prompt: prompt, + Eval: eval.Numeric(eval.ExtractLastNumber[int], 120, 0), + } +} + +// tsOptionalChainingNullishCode is the inline snippet for +// tsOptionalChainingNullishTraceTest. +const tsOptionalChainingNullishCode = `type Config = { server?: { host?: string; port?: number } }; + +function describe(c: Config): string { + const host = c.server?.host ?? "localhost"; + const port = c.server?.port ?? 8080; + return ` + "`${host}:${port}`" + `; +} + +console.log(describe({ server: { port: 0 } }));` + +// tsOptionalChainingNullishTraceTest: trace the exact output of code +// combining optional chaining with nullish coalescing, on an input whose +// present value is falsy but not nullish. +// +// ground truth: c.server?.host is undefined (server has no host property), +// so ?? falls back to "localhost". c.server?.port is 0 - present and not +// null/undefined - so ?? does NOT fall back (unlike ||, ?? only treats +// null/undefined as "missing", not other falsy values like 0, "", or +// false), leaving port as 0. The template literal produces "localhost:0". +// typescript_test.go cross-checks this by running the equivalent +// JavaScript via `node`. +func tsOptionalChainingNullishTraceTest() testkit.Test { + prompt := `Here is a TypeScript program: + +` + "```ts" + ` +` + tsOptionalChainingNullishCode + ` +` + "```" + ` + +What does this program print when run? Respond with only the exact text +printed, nothing else.` + + return testkit.Test{ + ID: "ts-optional-chaining-nullish", + Category: "programming", + Subcategory: "typescript", + Description: "Trace the exact output of optional chaining combined with nullish coalescing on a falsy-but-present value.", + Prompt: prompt, + Eval: eval.Equals("localhost:0"), + } +} diff --git a/internal/tests/typescript_test.go b/internal/tests/typescript_test.go index 41726e7..e58a17d 100644 --- a/internal/tests/typescript_test.go +++ b/internal/tests/typescript_test.go @@ -2,9 +2,40 @@ package tests import ( "context" + "os" + "os/exec" + "path/filepath" + "strings" "testing" ) +// runNodeJS writes js (plain JavaScript - TypeScript syntax already erased +// by the caller) to a temp file and runs it with `node`, returning trimmed +// stdout. It skips the calling test (does not fail it) when node is not on +// PATH, mirroring the toolchain-detection contract eval.PyRun/GoRun/CRun +// use, since node is a ground-truth verification tool here, not a catalog +// exec.Evaluator (typescript.go owns no eval.Evaluator that shells out). +func runNodeJS(t *testing.T, js string) string { + t.Helper() + if _, err := exec.LookPath("node"); err != nil { + t.Skip("node not found on PATH, skipping ground-truth cross-check") + } + + dir := t.TempDir() + scriptPath := filepath.Join(dir, "check.js") + if err := os.WriteFile(scriptPath, []byte(js), 0o600); err != nil { + t.Fatalf("write check.js: %v", err) + } + + // #nosec G204 -- scriptPath is a fixed filename this test just wrote + // into its own t.TempDir(); not external input. + out, err := exec.Command("node", scriptPath).Output() //nolint:gosec // see comment above + if err != nil { + t.Fatalf("running check.js failed: %v", err) + } + return strings.TrimSpace(string(out)) +} + const goodDebounceComposable = "```ts\n" + `import { ref, watch, onScopeDispose, type Ref } from 'vue' export function useDebouncedRef(source: Ref, ms: number): Ref { @@ -54,6 +85,334 @@ func TestNoAnyEscapeHatch(t *testing.T) { } } +// tsDiscriminatedUnionAreaJS is a plain-JavaScript mirror of +// tsDiscriminatedUnionAreaCode (TypeScript's type annotations erased, since +// they have no runtime effect), used to cross-check the ground truth with +// `node` rather than trusting the doc comment's arithmetic alone. +const tsDiscriminatedUnionAreaJS = `function area(s) { + switch (s.kind) { + case "circle": + return Math.PI * s.radius ** 2; + case "rectangle": + return s.width * s.height; + } +} + +const shapes = [ + { kind: "rectangle", width: 4, height: 5 }, + { kind: "circle", radius: 3 }, +]; + +const total = shapes.map(area).reduce((a, b) => a + b, 0); +console.log(Math.round(total * 100) / 100);` + +func TestTsDiscriminatedUnionArea_GroundTruth(t *testing.T) { + got := runNodeJS(t, tsDiscriminatedUnionAreaJS) + if want := "48.27"; got != want { + t.Errorf("node output = %q, want %q", got, want) + } +} + +func TestTsDiscriminatedUnionAreaTest_Eval(t *testing.T) { + tc := tsDiscriminatedUnionAreaTest() + + tests := []struct { + name string + response string + want float64 + }{ + {"exact", "48.27", 1}, + {"within tolerance", "48.272", 1}, + {"prose wrapped", "The total area is 48.27.", 1}, + {"wrong: circle area only", "28.27", 0}, + {"wrong: unrounded-ish but off", "48.3", 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Value != tt.want { + t.Errorf("Eval.Evaluate() = %v, want %v (detail: %s)", got.Value, tt.want, got.Detail) + } + }) + } +} + +func TestTsGenericUtilityTypeTest_Eval(t *testing.T) { + tc := tsGenericUtilityTypeTest() + + goodDeepPartial := "```ts\n" + `type DeepPartial = { + [K in keyof T]?: T[K] extends object ? DeepPartial : T[K]; +}; +` + "```" + + missingRecursion := "```ts\n" + `type DeepPartial = { + [K in keyof T]?: T[K]; +}; +` + "```" + + usesAny := "```ts\n" + `type DeepPartial = { + [K in keyof T]?: any; +}; +` + "```" + + tests := []struct { + name string + response string + want float64 + }{ + {"correct recursive DeepPartial", goodDeepPartial, 1}, + {"wrong: shallow only, no recursion into nested objects", missingRecursion, 0.75}, // (2*1 + 1*0 + 1*1)/4: mapped type ok, no DeepPartial reference, no any + {"wrong: uses any, defeating the point", usesAny, 0.5}, // (2*1 + 1*0 + 1*0)/4: mapped type ok, no recursion, uses any + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Value != tt.want { + t.Errorf("Eval.Evaluate() = %v, want %v (detail: %s)", got.Value, tt.want, got.Detail) + } + }) + } +} + +// tsPromiseAllSettledTraceJS is a plain-JavaScript mirror of +// tsPromiseAllSettledTraceCode. +const tsPromiseAllSettledTraceJS = `async function main() { + const results = await Promise.allSettled([ + Promise.resolve(1), + Promise.reject("fail"), + Promise.resolve(3), + ]); + console.log(JSON.stringify(results.map((r) => r.status))); +} + +main();` + +func TestTsPromiseAllSettledTrace_GroundTruth(t *testing.T) { + got := runNodeJS(t, tsPromiseAllSettledTraceJS) + if want := `["fulfilled","rejected","fulfilled"]`; got != want { + t.Errorf("node output = %q, want %q", got, want) + } +} + +func TestTsPromiseAllSettledTraceTest_Eval(t *testing.T) { + tc := tsPromiseAllSettledTraceTest() + + tests := []struct { + name string + response string + want float64 + }{ + {"exact correct", `["fulfilled","rejected","fulfilled"]`, 1}, + {"correct fenced json", "```json\n[\"fulfilled\", \"rejected\", \"fulfilled\"]\n```", 1}, + {"correct with prose wrapper", `The statuses are: ["fulfilled","rejected","fulfilled"]`, 1}, + {"wrong: assumes allSettled short-circuits like Promise.all", `["fulfilled"]`, 0}, + {"wrong: assumes rejection order shifts", `["rejected","fulfilled","fulfilled"]`, 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Value != tt.want { + t.Errorf("Eval.Evaluate() = %v, want %v (detail: %s)", got.Value, tt.want, got.Detail) + } + }) + } +} + +func TestTsVueRefReactiveUnwrapTest_Eval(t *testing.T) { + tc := tsVueRefReactiveUnwrapTest() + + tests := []struct { + name string + response string + want float64 + }{ + {"exact correct", `{"object_property_unwraps": true, "array_element_unwraps": false}`, 1}, + {"correct fenced json", "```json\n{\"object_property_unwraps\": true, \"array_element_unwraps\": false}\n```", 1}, + {"correct with prose wrapper", `Answer: {"object_property_unwraps": true, "array_element_unwraps": false}`, 1}, + {"wrong: assumes both unwrap", `{"object_property_unwraps": true, "array_element_unwraps": true}`, 0.5}, + {"wrong: assumes neither unwraps", `{"object_property_unwraps": false, "array_element_unwraps": false}`, 0.5}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Value != tt.want { + t.Errorf("Eval.Evaluate() = %v, want %v (detail: %s)", got.Value, tt.want, got.Detail) + } + }) + } +} + +func TestTsComputedVsWatchTest_Eval(t *testing.T) { + tc := tsComputedVsWatchTest() + + tests := []struct { + name string + response string + want float64 + }{ + {"exact correct", `{"scenario_a": "computed", "scenario_b": "watch"}`, 1}, + {"correct fenced json", "```json\n{\"scenario_a\": \"computed\", \"scenario_b\": \"watch\"}\n```", 1}, + {"correct with prose wrapper", `Answer: {"scenario_a": "computed", "scenario_b": "watch"}`, 1}, + {"wrong: both watch", `{"scenario_a": "watch", "scenario_b": "watch"}`, 0.5}, + {"wrong: swapped", `{"scenario_a": "watch", "scenario_b": "computed"}`, 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Value != tt.want { + t.Errorf("Eval.Evaluate() = %v, want %v (detail: %s)", got.Value, tt.want, got.Detail) + } + }) + } +} + +func TestTsESLintFlatConfigTest_Eval(t *testing.T) { + tc := tsESLintFlatConfigTest() + + goodFlatConfig := "```js\n" + `export default [ + { + languageOptions: { ecmaVersion: "latest", sourceType: "module" }, + rules: { semi: "error", "prefer-const": "error" }, + }, +]; +` + "```" + + goodFlatConfigWithHelper := "```js\n" + `import { defineConfig } from "eslint/config"; + +export default defineConfig([ + { + languageOptions: { ecmaVersion: "latest", sourceType: "module" }, + rules: { semi: "error", "prefer-const": "error" }, + }, +]); +` + "```" + + stillLegacyFormat := "```json\n" + tsOldEslintrcConfig + "\n```" + + tests := []struct { + name string + response string + want float64 + }{ + {"good: plain array export default", goodFlatConfig, 1}, + {"good: defineConfig helper", goodFlatConfigWithHelper, 1}, + {"wrong: left as legacy .eslintrc JSON", stillLegacyFormat, 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Value != tt.want { + t.Errorf("Eval.Evaluate() = %v, want %v (detail: %s)", got.Value, tt.want, got.Detail) + } + }) + } +} + +func TestTsTsconfigStrictFlagTest_Eval(t *testing.T) { + tc := tsTsconfigStrictFlagTest() + + tests := []struct { + name string + response string + want float64 + }{ + {"exact correct", `{"flag": "noImplicitAny"}`, 1}, + {"correct fenced json", "```json\n{\"flag\": \"noImplicitAny\"}\n```", 1}, + {"correct with prose wrapper", `The answer is: {"flag": "noImplicitAny"}`, 1}, + {"wrong: unrelated strict flag", `{"flag": "strictNullChecks"}`, 0}, + {"wrong: not a real tsconfig flag", `{"flag": "strict-types"}`, 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Value != tt.want { + t.Errorf("Eval.Evaluate() = %v, want %v (detail: %s)", got.Value, tt.want, got.Detail) + } + }) + } +} + +// tsArrayMethodChainJS is a plain-JavaScript mirror of +// tsArrayMethodChainCode. +const tsArrayMethodChainJS = `const nums = [3, 8, 1, 9, 4, 6, 2, 7]; +const result = nums + .filter((n) => n % 2 === 0) + .map((n) => n * n) + .reduce((acc, n) => acc + n, 0); +console.log(result);` + +func TestTsArrayMethodChain_GroundTruth(t *testing.T) { + got := runNodeJS(t, tsArrayMethodChainJS) + if want := "120"; got != want { + t.Errorf("node output = %q, want %q", got, want) + } +} + +func TestTsArrayMethodChainTraceTest_Eval(t *testing.T) { + tc := tsArrayMethodChainTraceTest() + + tests := []struct { + name string + response string + want float64 + }{ + {"exact correct", "120", 1}, + {"correct with prose", "The result is 120.", 1}, + {"correct with label", "Output: 120", 1}, + {"wrong: filters odd instead of even", "30", 0}, + {"wrong: forgets to square before summing", "20", 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Value != tt.want { + t.Errorf("Eval.Evaluate() = %v, want %v (detail: %s)", got.Value, tt.want, got.Detail) + } + }) + } +} + +// tsOptionalChainingNullishJS is a plain-JavaScript mirror of +// tsOptionalChainingNullishCode (TypeScript's type annotations erased). +const tsOptionalChainingNullishJS = `function describe(c) { + const host = c.server?.host ?? "localhost"; + const port = c.server?.port ?? 8080; + return ` + "`${host}:${port}`" + `; +} + +console.log(describe({ server: { port: 0 } }));` + +func TestTsOptionalChainingNullishTrace_GroundTruth(t *testing.T) { + got := runNodeJS(t, tsOptionalChainingNullishJS) + if want := "localhost:0"; got != want { + t.Errorf("node output = %q, want %q", got, want) + } +} + +func TestTsOptionalChainingNullishTraceTest_Eval(t *testing.T) { + tc := tsOptionalChainingNullishTraceTest() + + tests := []struct { + name string + response string + want float64 + }{ + {"exact correct", "localhost:0", 1}, + {"correct with surrounding whitespace", " localhost:0 ", 1}, + {"correct case-insensitive", "LOCALHOST:0", 1}, + {"wrong: treats ?? like || and replaces the falsy 0", "localhost:8080", 0}, + {"wrong: assumes host is present", "example.com:0", 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Value != tt.want { + t.Errorf("Eval.Evaluate() = %v, want %v (detail: %s)", got.Value, tt.want, got.Detail) + } + }) + } +} + func TestTsDebounceComposableTest_Eval(t *testing.T) { tc := tsDebounceComposableTest()