package tests import ( "github.com/lukaszraczylo/llm-testbench/internal/eval" "github.com/lukaszraczylo/llm-testbench/internal/testkit" ) func registerCTests(r *testkit.Registry) { r.Register(cStructSizeTest()) r.Register(cPointerArithmeticTest()) r.Register(cBitmaskOpsTest()) r.Register(cUnionSizeLP64Test()) r.Register(cStringFunctionOutputTest()) r.Register(cIntegerPromotionOverflowTest()) r.Register(cArrayDecaySizeofTest()) r.Register(cMacroExpansionPitfallTest()) r.Register(cUndefinedBehaviorSpotTest()) r.Register(cStructBitfieldPackingTest()) } // cStructSizeWant is sizeof(struct Config) on LP64 (64-bit System V / Apple // ARM64 ABI: natural alignment, no packing). // // ground truth: char flag(align 1, size 1) at offset 0; double ratio // (align 8, size 8) needs offset divisible by 8, so it starts at offset 8 // (7 bytes padding after flag), ending at 16; int count (align 4, size 4) // at offset 16 (already aligned), ending at 20; short mode (align 2, size // 2) at offset 20, ending at 22. The struct's own alignment is the widest // member alignment (8, from double), so the total size rounds up from 22 // to the next multiple of 8: 24. c_test.go cross-checks this with cc via // sizeof and offsetof when the cc toolchain is available. const cStructSizeWant = 24 func cStructSizeTest() testkit.Test { prompt := `Here is a C struct: ` + "```c" + ` struct Config { char flag; double ratio; int count; short mode; }; ` + "```" + ` Assuming the LP64 data model (64-bit, no #pragma pack, no non-default alignment attributes), what is sizeof(struct Config) in bytes? Respond with only the number, nothing else.` return testkit.Test{ ID: "c-struct-size", Category: "programming", Subcategory: "c", Description: "Compute sizeof() for a C struct on LP64 given its member types and default alignment.", Prompt: prompt, MaxTokens: 200, Eval: eval.Numeric(eval.ExtractLastNumber[int], cStructSizeWant, 0), } } // cPointerArithmeticCode is the inline snippet for cPointerArithmeticTest. const cPointerArithmeticCode = `int arr[6] = {10, 20, 30, 40, 50, 60}; int *p = arr; p += 2; int *q = &arr[5]; long diff = q - p; int val = *(p + 1) + diff;` // cPointerArithmeticWant is the value of val. // // ground truth: p starts at &arr[0], then p += 2 advances it by 2 ints // (pointer arithmetic scales by the pointee's size), landing on &arr[2] // (value 30). q = &arr[5] (value 60). Pointer subtraction (q - p) yields the // number of elements between them, not bytes: (&arr[5] - &arr[2]) = 3. *(p + // 1) dereferences arr[3] (value 40). val = 40 + 3 = 43. c_test.go // cross-checks this by compiling and running the identical snippet with cc. const cPointerArithmeticWant = 43 func cPointerArithmeticTest() testkit.Test { prompt := `Here is C code: ` + "```c" + ` ` + cPointerArithmeticCode + ` ` + "```" + ` Assuming a 6-element int array laid out contiguously with no padding between elements, what is the value of val? Respond with only the number, nothing else.` return testkit.Test{ ID: "c-pointer-arithmetic", Category: "programming", Subcategory: "c", Description: "Trace the exact result of pointer arithmetic and pointer subtraction over an inline int array.", Prompt: prompt, Eval: eval.Numeric(eval.ExtractLastNumber[int], cPointerArithmeticWant, 0), } } // cBitmaskOpsCode is the inline snippet for cBitmaskOpsTest. const cBitmaskOpsCode = `unsigned int flags = 0; flags |= (1 << 2); flags |= (1 << 5); flags &= ~(1 << 2); flags ^= (1 << 0);` // cBitmaskOpsWant is the final value of flags. // // ground truth: starting from 0: |= (1<<2) sets bit 2 -> 0b000100 (4); |= // (1<<5) sets bit 5 -> 0b100100 (36); &= ~(1<<2) clears bit 2 -> 0b100000 // (32); ^= (1<<0) toggles bit 0, which is currently 0, so it becomes 1 -> // 0b100001 (33). c_test.go cross-checks this by compiling and running the // identical snippet with cc. const cBitmaskOpsWant = 33 func cBitmaskOpsTest() testkit.Test { prompt := `Here is C code: ` + "```c" + ` ` + cBitmaskOpsCode + ` ` + "```" + ` What is the final value of flags, as a decimal number? Respond with only the number, nothing else.` return testkit.Test{ ID: "c-bitmask-ops", Category: "programming", Subcategory: "c", Description: "Trace the final decimal value of a chain of bit set/clear/toggle operations.", Prompt: prompt, Eval: eval.Numeric(eval.ExtractLastNumber[int], cBitmaskOpsWant, 0), } } // cUnionSizeLP64Code is the inline snippet for cUnionSizeLP64Test. const cUnionSizeLP64Code = `union Packet { double timestamp; char raw[3]; struct { int a; short b; } fields; };` // cUnionSizeLP64Want is sizeof(union Packet) on LP64. // // ground truth: a union's size is at least the largest member's size, // rounded up to the union's own alignment (the largest member alignment). // timestamp: size 8, align 8. raw: size 3, align 1. fields (struct{int a; // short b;}): a at offset 0 (size 4), b at offset 4 (size 2, already // aligned), giving 6 bytes of payload padded up to the struct's own // alignment (4, from int) -> struct size 8, align 4. The union's size is // max(8, 3, 8) = 8, and its alignment is max(8, 1, 4) = 8; 8 is already a // multiple of 8, so sizeof(union Packet) = 8. c_test.go cross-checks this // with cc. const cUnionSizeLP64Want = 8 func cUnionSizeLP64Test() testkit.Test { prompt := `Here is a C union: ` + "```c" + ` ` + cUnionSizeLP64Code + ` ` + "```" + ` Assuming the LP64 data model (64-bit, no #pragma pack, no non-default alignment attributes), what is sizeof(union Packet) in bytes? Respond with only the number, nothing else.` return testkit.Test{ ID: "c-union-size-lp64", Category: "programming", Subcategory: "c", Description: "Compute sizeof() for a C union on LP64, accounting for its largest member and its alignment.", Prompt: prompt, Eval: eval.Numeric(eval.ExtractLastNumber[int], cUnionSizeLP64Want, 0), } } // cStringFunctionOutputKeys is the inline dataset for // cStringFunctionOutputTest. // ground truth: keys starting with "srv_" (checked with strncmp(key, // "srv_", 4) == 0): srv_web, srv_db, srv_api, srv_auth = 4 matches (also // re-derived in c_test.go by scanning the same literal data with Go's // strings.HasPrefix). const cStringFunctionOutputKeys = `"srv_web", "srv_db", "cache_1", "srv_api", "queue_2", "srv_auth"` // cStringFunctionOutputWant is the expected stdout: the count of keys with // the "srv_" prefix. const cStringFunctionOutputWant = "4" func cStringFunctionOutputTest() testkit.Test { prompt := `Here are 6 inline C string literals (configuration key names): ` + "```c" + ` const char *keys[] = {` + cStringFunctionOutputKeys + `}; ` + "```" + ` Write a complete, self-contained C program (embed the array above exactly as shown; do not read stdin or a file) that uses strncmp from to count how many of these 6 keys start with the prefix "srv_", and prints just that count as a bare integer via printf, followed by a newline, and nothing else.` return testkit.Test{ ID: "c-string-function-output", Category: "programming", Subcategory: "c", Description: "Write a C program using strncmp to count inline strings matching a prefix.", System: terseCodeOnly, Prompt: prompt, Eval: eval.CRun(eval.PassthroughHarness, cStringFunctionOutputWant), } } // cIntegerPromotionOverflowCode is the inline snippet for // cIntegerPromotionOverflowTest. const cIntegerPromotionOverflowCode = `unsigned char a = 200; unsigned char b = 100; unsigned char sum = a + b;` // cIntegerPromotionOverflowWant is the value of sum. // // ground truth: in a + b, both unsigned char operands undergo the usual // arithmetic conversions ("integer promotion"): each is promoted to int // before the addition, so the addition computes 200 + 100 = 300 as a full // int, with no overflow at that step. Assigning that int back to the // unsigned char sum truncates it: unsigned integer types always wrap modulo // 2^n (n=8 for unsigned char), which is well-defined behavior (not // undefined, unlike signed overflow) - 300 mod 256 = 44. c_test.go // cross-checks this by compiling and running the identical snippet with cc. const cIntegerPromotionOverflowWant = 44 func cIntegerPromotionOverflowTest() testkit.Test { prompt := `Here is C code: ` + "```c" + ` ` + cIntegerPromotionOverflowCode + ` ` + "```" + ` What is the value of sum, printed as a decimal int (%d)? Respond with only the number, nothing else.` return testkit.Test{ ID: "c-integer-promotion-overflow", Category: "programming", Subcategory: "c", Description: "Trace unsigned char addition through integer promotion to int and truncation back to 8 bits.", Prompt: prompt, Eval: eval.Numeric(eval.ExtractLastNumber[int], cIntegerPromotionOverflowWant, 0), } } // cArrayDecaySizeofCode is the inline snippet for cArrayDecaySizeofTest. const cArrayDecaySizeofCode = `void check(int arr[10]) { printf("%zu\n", sizeof(arr)); } int main(void) { int arr[10]; printf("%zu\n", sizeof(arr)); check(arr); return 0; }` // cArrayDecaySizeofWant is the sum of the two printed sizeof values. // // ground truth: in main, arr is a genuine 10-element int array in scope, so // sizeof(arr) is the full array size: 10 * sizeof(int) = 40 (int is 4 bytes // on LP64). Inside check, the parameter "int arr[10]" is adjusted by the // compiler to "int *arr" (array-to-pointer decay in a function parameter // declaration, per the C standard) - sizeof(arr) there measures the // pointer, not the array: sizeof(int *) = 8 on LP64. Sum = 40 + 8 = 48. // c_test.go cross-checks both individual values (and their sum) by // compiling and running the identical snippet with cc. const cArrayDecaySizeofWant = 48 func cArrayDecaySizeofTest() testkit.Test { prompt := `Here is a C program: ` + "```c" + ` #include ` + cArrayDecaySizeofCode + ` ` + "```" + ` Assuming LP64 (int is 4 bytes, pointers are 8 bytes), what is the sum of the two numbers this program prints? Respond with only the number, nothing else.` return testkit.Test{ ID: "c-array-decay-sizeof", Category: "programming", Subcategory: "c", Description: "Compute the sum of sizeof(array) in its defining scope versus sizeof(array-parameter) after decay to a pointer.", Prompt: prompt, Eval: eval.Numeric(eval.ExtractLastNumber[int], cArrayDecaySizeofWant, 0), } } // cMacroExpansionPitfallWant is the expected stdout: the result of the // classic unparenthesized function-like macro pitfall. // // ground truth: #define SQUARE(x) x * x is a pure textual substitution with // no parentheses around x or around the whole expansion, so // SQUARE(2 + 3) expands to literally 2 + 3 * 2 + 3, not (2 + 3) * (2 + 3). // Operator precedence then evaluates * before +: 2 + 3 * 2 + 3 // = 2 + 6 + 3 = 11, not the "squared" answer 25 the macro's name suggests. // c_test.go cross-checks this by compiling and running the identical macro // and call with cc. const cMacroExpansionPitfallWant = "11" func cMacroExpansionPitfallTest() testkit.Test { prompt := `Here is a C macro definition: ` + "```c" + ` #define SQUARE(x) x * x ` + "```" + ` Write a complete, self-contained C program that defines SQUARE exactly as given above, then prints the result of SQUARE(2 + 3) as a bare integer via printf("%d\n", ...), and nothing else.` return testkit.Test{ ID: "c-macro-expansion-pitfall", Category: "programming", Subcategory: "c", Description: "Write a C program exposing the classic unparenthesized-macro operator-precedence pitfall.", System: terseCodeOnly, Prompt: prompt, Eval: eval.CRun(eval.PassthroughHarness, cMacroExpansionPitfallWant), } } // cUndefinedBehaviorSpotCode is the inline, line-numbered snippet for // cUndefinedBehaviorSpotTest. const cUndefinedBehaviorSpotCode = `1: #include 2: #include 3: 4: int main(void) { 5: int x = INT_MAX; 6: int y = x + 1; 7: printf("%d\n", y); 8: return 0; 9: }` // cUndefinedBehaviorSpotKindSignedOverflow is the fixed literal // cUndefinedBehaviorSpotTest requires for the "kind" JSON field. const cUndefinedBehaviorSpotKindSignedOverflow = "signed-integer-overflow" // cUndefinedBehaviorSpotTest: identify the line and kind of undefined // behavior in an inline snippet. // // ground truth: x holds INT_MAX (the largest representable int). Per the C // standard (C11 6.5p5), an operation whose mathematical result is not // representable in its result type is undefined behavior for signed // integer types (unlike unsigned types, which are defined to wrap). Line 6 // (x + 1) computes INT_MAX + 1, which overflows int - undefined behavior, // not a well-defined wraparound. This cannot be "verified by execution": // UB means the standard imposes no requirement on the observed result (in // practice compilers may wrap, trap, or optimize the check away entirely), // so c_test.go documents this ground truth from the standard rather than // asserting a specific runtime output, the same approach golang.go's // go-channel-deadlock test uses for its non-recomputable deadlock claim. func cUndefinedBehaviorSpotTest() testkit.Test { prompt := `Here is C code, with line numbers prefixed: ` + "```c" + ` ` + cUndefinedBehaviorSpotCode + ` ` + "```" + ` Which line number contains undefined behavior? Pick exactly one kind from this list: "signed-integer-overflow", "use-after-free", "uninitialized-read", "buffer-overflow". Respond with only a JSON object: {"line": , "kind": ""}` evaluator := eval.All( eval.W(eval.JSONField("line", 6), 1), eval.W(eval.JSONField("kind", cUndefinedBehaviorSpotKindSignedOverflow), 1), ) return testkit.Test{ ID: "c-undefined-behavior-spot", Category: "programming", Subcategory: "c", Description: "Identify the line and kind of undefined behavior in an inline C snippet (signed integer overflow).", System: terseCodeOnly, Prompt: prompt, Eval: evaluator, } } // cStructBitfieldPackingCode is the inline snippet for // cStructBitfieldPackingTest. const cStructBitfieldPackingCode = `struct Flags { unsigned int a : 3; unsigned int b : 5; unsigned int c : 24; };` // cStructBitfieldPackingWant is sizeof(struct Flags) on a typical LP64 // System V / Apple ARM64 ABI. // // ground truth: consecutive bitfields declared with the same base type // (unsigned int, a 4-byte/32-bit storage unit) are packed into shared // storage units by mainstream compilers (gcc/clang on both x86-64 System V // and Apple's ARM64 ABI) when they fit: a (3 bits) + b (5 bits) + c (24 // bits) = 32 bits exactly, filling one 4-byte unsigned int storage unit // with no padding needed. So sizeof(struct Flags) = 4. Exact bitfield // packing is implementation-defined by the C standard (not mandated), so // c_test.go cross-checks this empirically against the real cc toolchain // rather than relying on the derivation alone. const cStructBitfieldPackingWant = 4 func cStructBitfieldPackingTest() testkit.Test { prompt := `Here is a C struct with bitfields: ` + "```c" + ` ` + cStructBitfieldPackingCode + ` ` + "```" + ` Assuming a typical LP64 System V or Apple ARM64 ABI (gcc/clang, no #pragma pack), what is sizeof(struct Flags) in bytes? Respond with only the number, nothing else.` return testkit.Test{ ID: "c-struct-bitfield-packing", Category: "programming", Subcategory: "c", Description: "Compute sizeof() for a C struct whose bitfields exactly fill one storage unit with no padding.", Prompt: prompt, Eval: eval.Numeric(eval.ExtractLastNumber[int], cStructBitfieldPackingWant, 0), } }