mirror of
https://github.com/lukaszraczylo/llm-testbench.git
synced 2026-10-12 00:45:19 +00:00
Merge branch 'worktree-agent-a752fbca389ae50df'
This commit is contained in:
@@ -7,6 +7,15 @@ import (
|
||||
|
||||
func registerCTests(r *testkit.Registry) {
|
||||
r.Register(cStructSizeTest())
|
||||
r.Register(cPointerArithmeticTest())
|
||||
r.Register(cBitmaskOpsTest())
|
||||
r.Register(cUnionSizeLP64Test())
|
||||
r.Register(cStringFunctionOutputTest())
|
||||
r.Register(cIntegerPromotionOverflowTest())
|
||||
r.Register(cArrayDecaySizeofTest())
|
||||
r.Register(cMacroExpansionPitfallTest())
|
||||
r.Register(cUndefinedBehaviorSpotTest())
|
||||
r.Register(cStructBitfieldPackingTest())
|
||||
}
|
||||
|
||||
// cStructSizeWant is sizeof(struct Config) on LP64 (64-bit System V / Apple
|
||||
@@ -48,3 +57,375 @@ Respond with only the number, nothing else.`
|
||||
Eval: eval.Numeric(eval.ExtractLastNumber[int], cStructSizeWant, 0),
|
||||
}
|
||||
}
|
||||
|
||||
// cPointerArithmeticCode is the inline snippet for cPointerArithmeticTest.
|
||||
const cPointerArithmeticCode = `int arr[6] = {10, 20, 30, 40, 50, 60};
|
||||
int *p = arr;
|
||||
p += 2;
|
||||
int *q = &arr[5];
|
||||
long diff = q - p;
|
||||
int val = *(p + 1) + diff;`
|
||||
|
||||
// cPointerArithmeticWant is the value of val.
|
||||
//
|
||||
// ground truth: p starts at &arr[0], then p += 2 advances it by 2 ints
|
||||
// (pointer arithmetic scales by the pointee's size), landing on &arr[2]
|
||||
// (value 30). q = &arr[5] (value 60). Pointer subtraction (q - p) yields the
|
||||
// number of elements between them, not bytes: (&arr[5] - &arr[2]) = 3. *(p +
|
||||
// 1) dereferences arr[3] (value 40). val = 40 + 3 = 43. c_test.go
|
||||
// cross-checks this by compiling and running the identical snippet with cc.
|
||||
const cPointerArithmeticWant = 43
|
||||
|
||||
func cPointerArithmeticTest() testkit.Test {
|
||||
prompt := `Here is C code:
|
||||
|
||||
` + "```c" + `
|
||||
` + cPointerArithmeticCode + `
|
||||
` + "```" + `
|
||||
|
||||
Assuming a 6-element int array laid out contiguously with no padding
|
||||
between elements, what is the value of val? Respond with only the number,
|
||||
nothing else.`
|
||||
|
||||
return testkit.Test{
|
||||
ID: "c-pointer-arithmetic",
|
||||
Category: "programming",
|
||||
Subcategory: "c",
|
||||
Description: "Trace the exact result of pointer arithmetic and pointer subtraction over an inline int array.",
|
||||
Prompt: prompt,
|
||||
Eval: eval.Numeric(eval.ExtractLastNumber[int], cPointerArithmeticWant, 0),
|
||||
}
|
||||
}
|
||||
|
||||
// cBitmaskOpsCode is the inline snippet for cBitmaskOpsTest.
|
||||
const cBitmaskOpsCode = `unsigned int flags = 0;
|
||||
flags |= (1 << 2);
|
||||
flags |= (1 << 5);
|
||||
flags &= ~(1 << 2);
|
||||
flags ^= (1 << 0);`
|
||||
|
||||
// cBitmaskOpsWant is the final value of flags.
|
||||
//
|
||||
// ground truth: starting from 0: |= (1<<2) sets bit 2 -> 0b000100 (4); |=
|
||||
// (1<<5) sets bit 5 -> 0b100100 (36); &= ~(1<<2) clears bit 2 -> 0b100000
|
||||
// (32); ^= (1<<0) toggles bit 0, which is currently 0, so it becomes 1 ->
|
||||
// 0b100001 (33). c_test.go cross-checks this by compiling and running the
|
||||
// identical snippet with cc.
|
||||
const cBitmaskOpsWant = 33
|
||||
|
||||
func cBitmaskOpsTest() testkit.Test {
|
||||
prompt := `Here is C code:
|
||||
|
||||
` + "```c" + `
|
||||
` + cBitmaskOpsCode + `
|
||||
` + "```" + `
|
||||
|
||||
What is the final value of flags, as a decimal number? Respond with only
|
||||
the number, nothing else.`
|
||||
|
||||
return testkit.Test{
|
||||
ID: "c-bitmask-ops",
|
||||
Category: "programming",
|
||||
Subcategory: "c",
|
||||
Description: "Trace the final decimal value of a chain of bit set/clear/toggle operations.",
|
||||
Prompt: prompt,
|
||||
Eval: eval.Numeric(eval.ExtractLastNumber[int], cBitmaskOpsWant, 0),
|
||||
}
|
||||
}
|
||||
|
||||
// cUnionSizeLP64Code is the inline snippet for cUnionSizeLP64Test.
|
||||
const cUnionSizeLP64Code = `union Packet {
|
||||
double timestamp;
|
||||
char raw[3];
|
||||
struct { int a; short b; } fields;
|
||||
};`
|
||||
|
||||
// cUnionSizeLP64Want is sizeof(union Packet) on LP64.
|
||||
//
|
||||
// ground truth: a union's size is at least the largest member's size,
|
||||
// rounded up to the union's own alignment (the largest member alignment).
|
||||
// timestamp: size 8, align 8. raw: size 3, align 1. fields (struct{int a;
|
||||
// short b;}): a at offset 0 (size 4), b at offset 4 (size 2, already
|
||||
// aligned), giving 6 bytes of payload padded up to the struct's own
|
||||
// alignment (4, from int) -> struct size 8, align 4. The union's size is
|
||||
// max(8, 3, 8) = 8, and its alignment is max(8, 1, 4) = 8; 8 is already a
|
||||
// multiple of 8, so sizeof(union Packet) = 8. c_test.go cross-checks this
|
||||
// with cc.
|
||||
const cUnionSizeLP64Want = 8
|
||||
|
||||
func cUnionSizeLP64Test() testkit.Test {
|
||||
prompt := `Here is a C union:
|
||||
|
||||
` + "```c" + `
|
||||
` + cUnionSizeLP64Code + `
|
||||
` + "```" + `
|
||||
|
||||
Assuming the LP64 data model (64-bit, no #pragma pack, no non-default
|
||||
alignment attributes), what is sizeof(union Packet) in bytes? Respond with
|
||||
only the number, nothing else.`
|
||||
|
||||
return testkit.Test{
|
||||
ID: "c-union-size-lp64",
|
||||
Category: "programming",
|
||||
Subcategory: "c",
|
||||
Description: "Compute sizeof() for a C union on LP64, accounting for its largest member and its alignment.",
|
||||
Prompt: prompt,
|
||||
Eval: eval.Numeric(eval.ExtractLastNumber[int], cUnionSizeLP64Want, 0),
|
||||
}
|
||||
}
|
||||
|
||||
// cStringFunctionOutputKeys is the inline dataset for
|
||||
// cStringFunctionOutputTest.
|
||||
// ground truth: keys starting with "srv_" (checked with strncmp(key,
|
||||
// "srv_", 4) == 0): srv_web, srv_db, srv_api, srv_auth = 4 matches (also
|
||||
// re-derived in c_test.go by scanning the same literal data with Go's
|
||||
// strings.HasPrefix).
|
||||
const cStringFunctionOutputKeys = `"srv_web", "srv_db", "cache_1", "srv_api", "queue_2", "srv_auth"`
|
||||
|
||||
// cStringFunctionOutputWant is the expected stdout: the count of keys with
|
||||
// the "srv_" prefix.
|
||||
const cStringFunctionOutputWant = "4"
|
||||
|
||||
func cStringFunctionOutputTest() testkit.Test {
|
||||
prompt := `Here are 6 inline C string literals (configuration key names):
|
||||
|
||||
` + "```c" + `
|
||||
const char *keys[] = {` + cStringFunctionOutputKeys + `};
|
||||
` + "```" + `
|
||||
|
||||
Write a complete, self-contained C program (embed the array above exactly
|
||||
as shown; do not read stdin or a file) that uses strncmp from <string.h> to
|
||||
count how many of these 6 keys start with the prefix "srv_", and prints
|
||||
just that count as a bare integer via printf, followed by a newline, and
|
||||
nothing else.`
|
||||
|
||||
return testkit.Test{
|
||||
ID: "c-string-function-output",
|
||||
Category: "programming",
|
||||
Subcategory: "c",
|
||||
Description: "Write a C program using strncmp to count inline strings matching a prefix.",
|
||||
System: terseCodeOnly,
|
||||
Prompt: prompt,
|
||||
Eval: eval.CRun(eval.PassthroughHarness, cStringFunctionOutputWant),
|
||||
}
|
||||
}
|
||||
|
||||
// cIntegerPromotionOverflowCode is the inline snippet for
|
||||
// cIntegerPromotionOverflowTest.
|
||||
const cIntegerPromotionOverflowCode = `unsigned char a = 200;
|
||||
unsigned char b = 100;
|
||||
unsigned char sum = a + b;`
|
||||
|
||||
// cIntegerPromotionOverflowWant is the value of sum.
|
||||
//
|
||||
// ground truth: in a + b, both unsigned char operands undergo the usual
|
||||
// arithmetic conversions ("integer promotion"): each is promoted to int
|
||||
// before the addition, so the addition computes 200 + 100 = 300 as a full
|
||||
// int, with no overflow at that step. Assigning that int back to the
|
||||
// unsigned char sum truncates it: unsigned integer types always wrap modulo
|
||||
// 2^n (n=8 for unsigned char), which is well-defined behavior (not
|
||||
// undefined, unlike signed overflow) - 300 mod 256 = 44. c_test.go
|
||||
// cross-checks this by compiling and running the identical snippet with cc.
|
||||
const cIntegerPromotionOverflowWant = 44
|
||||
|
||||
func cIntegerPromotionOverflowTest() testkit.Test {
|
||||
prompt := `Here is C code:
|
||||
|
||||
` + "```c" + `
|
||||
` + cIntegerPromotionOverflowCode + `
|
||||
` + "```" + `
|
||||
|
||||
What is the value of sum, printed as a decimal int (%d)? Respond with only
|
||||
the number, nothing else.`
|
||||
|
||||
return testkit.Test{
|
||||
ID: "c-integer-promotion-overflow",
|
||||
Category: "programming",
|
||||
Subcategory: "c",
|
||||
Description: "Trace unsigned char addition through integer promotion to int and truncation back to 8 bits.",
|
||||
Prompt: prompt,
|
||||
Eval: eval.Numeric(eval.ExtractLastNumber[int], cIntegerPromotionOverflowWant, 0),
|
||||
}
|
||||
}
|
||||
|
||||
// cArrayDecaySizeofCode is the inline snippet for cArrayDecaySizeofTest.
|
||||
const cArrayDecaySizeofCode = `void check(int arr[10]) {
|
||||
printf("%zu\n", sizeof(arr));
|
||||
}
|
||||
|
||||
int main(void) {
|
||||
int arr[10];
|
||||
printf("%zu\n", sizeof(arr));
|
||||
check(arr);
|
||||
return 0;
|
||||
}`
|
||||
|
||||
// cArrayDecaySizeofWant is the sum of the two printed sizeof values.
|
||||
//
|
||||
// ground truth: in main, arr is a genuine 10-element int array in scope, so
|
||||
// sizeof(arr) is the full array size: 10 * sizeof(int) = 40 (int is 4 bytes
|
||||
// on LP64). Inside check, the parameter "int arr[10]" is adjusted by the
|
||||
// compiler to "int *arr" (array-to-pointer decay in a function parameter
|
||||
// declaration, per the C standard) - sizeof(arr) there measures the
|
||||
// pointer, not the array: sizeof(int *) = 8 on LP64. Sum = 40 + 8 = 48.
|
||||
// c_test.go cross-checks both individual values (and their sum) by
|
||||
// compiling and running the identical snippet with cc.
|
||||
const cArrayDecaySizeofWant = 48
|
||||
|
||||
func cArrayDecaySizeofTest() testkit.Test {
|
||||
prompt := `Here is a C program:
|
||||
|
||||
` + "```c" + `
|
||||
#include <stdio.h>
|
||||
|
||||
` + cArrayDecaySizeofCode + `
|
||||
` + "```" + `
|
||||
|
||||
Assuming LP64 (int is 4 bytes, pointers are 8 bytes), what is the sum of
|
||||
the two numbers this program prints? Respond with only the number, nothing
|
||||
else.`
|
||||
|
||||
return testkit.Test{
|
||||
ID: "c-array-decay-sizeof",
|
||||
Category: "programming",
|
||||
Subcategory: "c",
|
||||
Description: "Compute the sum of sizeof(array) in its defining scope versus sizeof(array-parameter) after decay to a pointer.",
|
||||
Prompt: prompt,
|
||||
Eval: eval.Numeric(eval.ExtractLastNumber[int], cArrayDecaySizeofWant, 0),
|
||||
}
|
||||
}
|
||||
|
||||
// cMacroExpansionPitfallWant is the expected stdout: the result of the
|
||||
// classic unparenthesized function-like macro pitfall.
|
||||
//
|
||||
// ground truth: #define SQUARE(x) x * x is a pure textual substitution with
|
||||
// no parentheses around x or around the whole expansion, so
|
||||
// SQUARE(2 + 3) expands to literally 2 + 3 * 2 + 3, not (2 + 3) * (2 + 3).
|
||||
// Operator precedence then evaluates * before +: 2 + 3 * 2 + 3
|
||||
// = 2 + 6 + 3 = 11, not the "squared" answer 25 the macro's name suggests.
|
||||
// c_test.go cross-checks this by compiling and running the identical macro
|
||||
// and call with cc.
|
||||
const cMacroExpansionPitfallWant = "11"
|
||||
|
||||
func cMacroExpansionPitfallTest() testkit.Test {
|
||||
prompt := `Here is a C macro definition:
|
||||
|
||||
` + "```c" + `
|
||||
#define SQUARE(x) x * x
|
||||
` + "```" + `
|
||||
|
||||
Write a complete, self-contained C program that defines SQUARE exactly as
|
||||
given above, then prints the result of SQUARE(2 + 3) as a bare integer via
|
||||
printf("%d\n", ...), and nothing else.`
|
||||
|
||||
return testkit.Test{
|
||||
ID: "c-macro-expansion-pitfall",
|
||||
Category: "programming",
|
||||
Subcategory: "c",
|
||||
Description: "Write a C program exposing the classic unparenthesized-macro operator-precedence pitfall.",
|
||||
System: terseCodeOnly,
|
||||
Prompt: prompt,
|
||||
Eval: eval.CRun(eval.PassthroughHarness, cMacroExpansionPitfallWant),
|
||||
}
|
||||
}
|
||||
|
||||
// cUndefinedBehaviorSpotCode is the inline, line-numbered snippet for
|
||||
// cUndefinedBehaviorSpotTest.
|
||||
const cUndefinedBehaviorSpotCode = `1: #include <limits.h>
|
||||
2: #include <stdio.h>
|
||||
3:
|
||||
4: int main(void) {
|
||||
5: int x = INT_MAX;
|
||||
6: int y = x + 1;
|
||||
7: printf("%d\n", y);
|
||||
8: return 0;
|
||||
9: }`
|
||||
|
||||
// cUndefinedBehaviorSpotKindSignedOverflow is the fixed literal
|
||||
// cUndefinedBehaviorSpotTest requires for the "kind" JSON field.
|
||||
const cUndefinedBehaviorSpotKindSignedOverflow = "signed-integer-overflow"
|
||||
|
||||
// cUndefinedBehaviorSpotTest: identify the line and kind of undefined
|
||||
// behavior in an inline snippet.
|
||||
//
|
||||
// ground truth: x holds INT_MAX (the largest representable int). Per the C
|
||||
// standard (C11 6.5p5), an operation whose mathematical result is not
|
||||
// representable in its result type is undefined behavior for signed
|
||||
// integer types (unlike unsigned types, which are defined to wrap). Line 6
|
||||
// (x + 1) computes INT_MAX + 1, which overflows int - undefined behavior,
|
||||
// not a well-defined wraparound. This cannot be "verified by execution":
|
||||
// UB means the standard imposes no requirement on the observed result (in
|
||||
// practice compilers may wrap, trap, or optimize the check away entirely),
|
||||
// so c_test.go documents this ground truth from the standard rather than
|
||||
// asserting a specific runtime output, the same approach golang.go's
|
||||
// go-channel-deadlock test uses for its non-recomputable deadlock claim.
|
||||
func cUndefinedBehaviorSpotTest() testkit.Test {
|
||||
prompt := `Here is C code, with line numbers prefixed:
|
||||
|
||||
` + "```c" + `
|
||||
` + cUndefinedBehaviorSpotCode + `
|
||||
` + "```" + `
|
||||
|
||||
Which line number contains undefined behavior? Pick exactly one kind from
|
||||
this list: "signed-integer-overflow", "use-after-free", "uninitialized-read",
|
||||
"buffer-overflow". Respond with only a JSON object:
|
||||
{"line": <line number as an integer>, "kind": "<one of the four kinds above, exactly as written>"}`
|
||||
|
||||
evaluator := eval.All(
|
||||
eval.W(eval.JSONField("line", 6), 1),
|
||||
eval.W(eval.JSONField("kind", cUndefinedBehaviorSpotKindSignedOverflow), 1),
|
||||
)
|
||||
|
||||
return testkit.Test{
|
||||
ID: "c-undefined-behavior-spot",
|
||||
Category: "programming",
|
||||
Subcategory: "c",
|
||||
Description: "Identify the line and kind of undefined behavior in an inline C snippet (signed integer overflow).",
|
||||
System: terseCodeOnly,
|
||||
Prompt: prompt,
|
||||
Eval: evaluator,
|
||||
}
|
||||
}
|
||||
|
||||
// cStructBitfieldPackingCode is the inline snippet for
|
||||
// cStructBitfieldPackingTest.
|
||||
const cStructBitfieldPackingCode = `struct Flags {
|
||||
unsigned int a : 3;
|
||||
unsigned int b : 5;
|
||||
unsigned int c : 24;
|
||||
};`
|
||||
|
||||
// cStructBitfieldPackingWant is sizeof(struct Flags) on a typical LP64
|
||||
// System V / Apple ARM64 ABI.
|
||||
//
|
||||
// ground truth: consecutive bitfields declared with the same base type
|
||||
// (unsigned int, a 4-byte/32-bit storage unit) are packed into shared
|
||||
// storage units by mainstream compilers (gcc/clang on both x86-64 System V
|
||||
// and Apple's ARM64 ABI) when they fit: a (3 bits) + b (5 bits) + c (24
|
||||
// bits) = 32 bits exactly, filling one 4-byte unsigned int storage unit
|
||||
// with no padding needed. So sizeof(struct Flags) = 4. Exact bitfield
|
||||
// packing is implementation-defined by the C standard (not mandated), so
|
||||
// c_test.go cross-checks this empirically against the real cc toolchain
|
||||
// rather than relying on the derivation alone.
|
||||
const cStructBitfieldPackingWant = 4
|
||||
|
||||
func cStructBitfieldPackingTest() testkit.Test {
|
||||
prompt := `Here is a C struct with bitfields:
|
||||
|
||||
` + "```c" + `
|
||||
` + cStructBitfieldPackingCode + `
|
||||
` + "```" + `
|
||||
|
||||
Assuming a typical LP64 System V or Apple ARM64 ABI (gcc/clang, no
|
||||
#pragma pack), what is sizeof(struct Flags) in bytes? Respond with only the
|
||||
number, nothing else.`
|
||||
|
||||
return testkit.Test{
|
||||
ID: "c-struct-bitfield-packing",
|
||||
Category: "programming",
|
||||
Subcategory: "c",
|
||||
Description: "Compute sizeof() for a C struct whose bitfields exactly fill one storage unit with no padding.",
|
||||
Prompt: prompt,
|
||||
Eval: eval.Numeric(eval.ExtractLastNumber[int], cStructBitfieldPackingWant, 0),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -9,6 +9,39 @@ import (
|
||||
"testing"
|
||||
)
|
||||
|
||||
// ccRun compiles and runs a complete C program via cc, returning trimmed
|
||||
// stdout. It skips the calling test (does not fail it) when cc is
|
||||
// unavailable, mirroring TestCStructSizeWant_GroundTruth's existing
|
||||
// toolchain-detection contract.
|
||||
func ccRun(t *testing.T, src string) string {
|
||||
t.Helper()
|
||||
if _, err := exec.LookPath("cc"); err != nil {
|
||||
t.Skip("cc not found on PATH, skipping ground-truth cross-check")
|
||||
}
|
||||
|
||||
dir := t.TempDir()
|
||||
srcPath := filepath.Join(dir, "main.c")
|
||||
if err := os.WriteFile(srcPath, []byte(src), 0o600); err != nil {
|
||||
t.Fatalf("write main.c: %v", err)
|
||||
}
|
||||
binPath := filepath.Join(dir, "prog")
|
||||
|
||||
// #nosec G204 -- srcPath/binPath are fixed filenames this test just
|
||||
// wrote/will write into its own t.TempDir(); not external input.
|
||||
build := exec.Command("cc", "-o", binPath, srcPath) //nolint:gosec // see comment above
|
||||
if out, err := build.CombinedOutput(); err != nil {
|
||||
t.Fatalf("cc failed: %v\n%s", err, out)
|
||||
}
|
||||
|
||||
// #nosec G204 -- binPath is the binary this test just compiled above,
|
||||
// in its own t.TempDir(); not an externally supplied executable path.
|
||||
out, err := exec.Command(binPath).Output() //nolint:gosec // see comment above
|
||||
if err != nil {
|
||||
t.Fatalf("running compiled program failed: %v", err)
|
||||
}
|
||||
return strings.TrimSpace(string(out))
|
||||
}
|
||||
|
||||
// TestCStructSizeWant_GroundTruth cross-checks cStructSizeWant against the
|
||||
// real cc toolchain's sizeof(struct Config), rather than trusting the
|
||||
// hand-derived alignment arithmetic in the doc comment alone. Skips (does
|
||||
@@ -60,6 +93,395 @@ int main(void) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestCPointerArithmetic_GroundTruth(t *testing.T) {
|
||||
src := `#include <stdio.h>
|
||||
int main(void) {
|
||||
` + cPointerArithmeticCode + `
|
||||
printf("%d", val);
|
||||
return 0;
|
||||
}
|
||||
`
|
||||
got := ccRun(t, src)
|
||||
if want := "43"; got != want {
|
||||
t.Errorf("cc output = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCPointerArithmeticTest_Eval(t *testing.T) {
|
||||
tc := cPointerArithmeticTest()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{"exact correct", "43", 1},
|
||||
{"prose wrapped", "val is 43", 1},
|
||||
{"answer followed by qualifier", "val = 43 on a 64-bit system", 1},
|
||||
{"wrong: forgets pointer arithmetic scales by element size", "13", 0},
|
||||
{"wrong: byte-based pointer diff", "val = 52", 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate() = %v, want %v (detail: %s)", got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestCBitmaskOps_GroundTruth(t *testing.T) {
|
||||
src := `#include <stdio.h>
|
||||
int main(void) {
|
||||
` + cBitmaskOpsCode + `
|
||||
printf("%u", flags);
|
||||
return 0;
|
||||
}
|
||||
`
|
||||
got := ccRun(t, src)
|
||||
if want := "33"; got != want {
|
||||
t.Errorf("cc output = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCBitmaskOpsTest_Eval(t *testing.T) {
|
||||
tc := cBitmaskOpsTest()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{"exact correct", "33", 1},
|
||||
{"prose wrapped", "flags is 33", 1},
|
||||
{"binary explained then decimal answer", "0b100001 = 33", 1},
|
||||
{"wrong: forgets to clear bit 2", "37", 0},
|
||||
{"wrong: forgets to toggle bit 0", "32", 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate() = %v, want %v (detail: %s)", got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestCUnionSizeLP64_GroundTruth(t *testing.T) {
|
||||
src := `#include <stdio.h>
|
||||
` + cUnionSizeLP64Code + `
|
||||
int main(void) {
|
||||
printf("%zu", sizeof(union Packet));
|
||||
return 0;
|
||||
}
|
||||
`
|
||||
got := ccRun(t, src)
|
||||
if want := "8"; got != want {
|
||||
t.Errorf("cc output = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCUnionSizeLP64Test_Eval(t *testing.T) {
|
||||
tc := cUnionSizeLP64Test()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{"exact correct", "8", 1},
|
||||
{"prose wrapped", "sizeof(union Packet) is 8 bytes", 1},
|
||||
{"answer followed by 64-bit qualifier", "8 bytes on a 64-bit system", 1},
|
||||
{"wrong: sums members instead of taking the max", "19", 0},
|
||||
{"wrong: forgets struct-member alignment padding", "6", 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate() = %v, want %v (detail: %s)", got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestCStringFunctionOutputWant_GroundTruth independently re-derives the
|
||||
// expected count by scanning the same literal key list with Go's
|
||||
// strings.HasPrefix, rather than trusting the hand count in the doc
|
||||
// comment.
|
||||
func TestCStringFunctionOutputWant_GroundTruth(t *testing.T) {
|
||||
keys := []string{"srv_web", "srv_db", "cache_1", "srv_api", "queue_2", "srv_auth"}
|
||||
count := 0
|
||||
for _, k := range keys {
|
||||
if strings.HasPrefix(k, "srv_") {
|
||||
count++
|
||||
}
|
||||
}
|
||||
if count != 4 {
|
||||
t.Errorf("independently recomputed count = %d, want 4", count)
|
||||
}
|
||||
if cStringFunctionOutputWant != "4" {
|
||||
t.Errorf("cStringFunctionOutputWant = %q, want %q", cStringFunctionOutputWant, "4")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCStringFunctionOutputTest_Eval(t *testing.T) {
|
||||
tc := cStringFunctionOutputTest()
|
||||
|
||||
correctProgram := "```c\n" + `#include <stdio.h>
|
||||
#include <string.h>
|
||||
|
||||
int main(void) {
|
||||
const char *keys[] = {` + cStringFunctionOutputKeys + `};
|
||||
int count = 0;
|
||||
for (int i = 0; i < 6; i++) {
|
||||
if (strncmp(keys[i], "srv_", 4) == 0) {
|
||||
count++;
|
||||
}
|
||||
}
|
||||
printf("%d\n", count);
|
||||
return 0;
|
||||
}
|
||||
` + "```"
|
||||
|
||||
wrongProgram := "```c\n#include <stdio.h>\nint main(void) { printf(\"6\\n\"); return 0; }\n```"
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{"correct strncmp counting program", correctProgram, 1},
|
||||
{"wrong: counts every key regardless of prefix", wrongProgram, 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Skipped {
|
||||
t.Skip("cc not available, skipping exec-evaluator assertion")
|
||||
}
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate() = %v, want %v (detail: %s)", got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestCIntegerPromotionOverflow_GroundTruth(t *testing.T) {
|
||||
src := `#include <stdio.h>
|
||||
int main(void) {
|
||||
` + cIntegerPromotionOverflowCode + `
|
||||
printf("%d", sum);
|
||||
return 0;
|
||||
}
|
||||
`
|
||||
got := ccRun(t, src)
|
||||
if want := "44"; got != want {
|
||||
t.Errorf("cc output = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCIntegerPromotionOverflowTest_Eval(t *testing.T) {
|
||||
tc := cIntegerPromotionOverflowTest()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{"exact correct", "44", 1},
|
||||
{"prose wrapped", "sum is 44", 1},
|
||||
{"answer with modulo explanation", "300 mod 256 = 44", 1},
|
||||
{"wrong: assumes saturation instead of wraparound", "255", 0},
|
||||
{"wrong: forgets truncation back to 8 bits", "300", 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate() = %v, want %v (detail: %s)", got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestCArrayDecaySizeof_GroundTruth(t *testing.T) {
|
||||
src := `#include <stdio.h>
|
||||
|
||||
` + cArrayDecaySizeofCode + `
|
||||
`
|
||||
got := ccRun(t, src)
|
||||
lines := strings.Split(got, "\n")
|
||||
if len(lines) != 2 {
|
||||
t.Fatalf("cc output = %q, want 2 lines", got)
|
||||
}
|
||||
if lines[0] != "40" {
|
||||
t.Errorf("sizeof(arr) in main = %q, want %q", lines[0], "40")
|
||||
}
|
||||
if lines[1] != "8" {
|
||||
t.Errorf("sizeof(arr) in function = %q, want %q", lines[1], "8")
|
||||
}
|
||||
if cArrayDecaySizeofWant != 48 {
|
||||
t.Errorf("cArrayDecaySizeofWant = %d, want 48 (= 40 + 8)", cArrayDecaySizeofWant)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCArrayDecaySizeofTest_Eval(t *testing.T) {
|
||||
tc := cArrayDecaySizeofTest()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{"exact correct", "48", 1},
|
||||
{"prose wrapped", "The sum is 48.", 1},
|
||||
{"answer with breakdown", "40 + 8 = 48", 1},
|
||||
{"wrong: assumes both report the full array size", "80", 0},
|
||||
{"wrong: assumes both decay to pointer size", "16", 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate() = %v, want %v (detail: %s)", got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestCMacroExpansionPitfallWant_GroundTruth(t *testing.T) {
|
||||
src := `#include <stdio.h>
|
||||
|
||||
#define SQUARE(x) x * x
|
||||
|
||||
int main(void) {
|
||||
printf("%d\n", SQUARE(2 + 3));
|
||||
return 0;
|
||||
}
|
||||
`
|
||||
got := ccRun(t, src)
|
||||
if cMacroExpansionPitfallWant != "11" {
|
||||
t.Errorf("cMacroExpansionPitfallWant = %q, want %q", cMacroExpansionPitfallWant, "11")
|
||||
}
|
||||
if got != "11" {
|
||||
t.Errorf("cc output = %q, want %q", got, "11")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCMacroExpansionPitfallTest_Eval(t *testing.T) {
|
||||
tc := cMacroExpansionPitfallTest()
|
||||
|
||||
correctProgram := "```c\n" + `#include <stdio.h>
|
||||
|
||||
#define SQUARE(x) x * x
|
||||
|
||||
int main(void) {
|
||||
printf("%d\n", SQUARE(2 + 3));
|
||||
return 0;
|
||||
}
|
||||
` + "```"
|
||||
|
||||
// A model that "fixes" the macro with parentheses no longer reproduces
|
||||
// the pitfall's actual (unparenthesized) expansion, so it must score 0
|
||||
// against this test's want ("11"), even though (2+3)*(2+3)=25 is what a
|
||||
// naive reading of "SQUARE" would suggest.
|
||||
parenthesizedProgram := "```c\n" + `#include <stdio.h>
|
||||
|
||||
#define SQUARE(x) ((x) * (x))
|
||||
|
||||
int main(void) {
|
||||
printf("%d\n", SQUARE(2 + 3));
|
||||
return 0;
|
||||
}
|
||||
` + "```"
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{"correct: reproduces the macro exactly as given", correctProgram, 1},
|
||||
{"wrong: silently parenthesizes the macro, changing its output", parenthesizedProgram, 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Skipped {
|
||||
t.Skip("cc not available, skipping exec-evaluator assertion")
|
||||
}
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate() = %v, want %v (detail: %s)", got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestCUndefinedBehaviorSpotTest_Eval(t *testing.T) {
|
||||
tc := cUndefinedBehaviorSpotTest()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{"exact correct", `{"line": 6, "kind": "signed-integer-overflow"}`, 1},
|
||||
{"correct fenced json", "```json\n{\"line\": 6, \"kind\": \"signed-integer-overflow\"}\n```", 1},
|
||||
{"correct with prose wrapper", `The answer is: {"line": 6, "kind": "signed-integer-overflow"}`, 1},
|
||||
{"wrong line", `{"line": 5, "kind": "signed-integer-overflow"}`, 0.5},
|
||||
{"wrong kind", `{"line": 6, "kind": "uninitialized-read"}`, 0.5},
|
||||
{"both wrong", `{"line": 7, "kind": "buffer-overflow"}`, 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate() = %v, want %v (detail: %s)", got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestCStructBitfieldPacking_GroundTruth(t *testing.T) {
|
||||
src := `#include <stdio.h>
|
||||
` + cStructBitfieldPackingCode + `
|
||||
int main(void) {
|
||||
printf("%zu", sizeof(struct Flags));
|
||||
return 0;
|
||||
}
|
||||
`
|
||||
got := ccRun(t, src)
|
||||
if want := "4"; got != want {
|
||||
t.Errorf("cc output = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCStructBitfieldPackingTest_Eval(t *testing.T) {
|
||||
tc := cStructBitfieldPackingTest()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{"exact correct", "4", 1},
|
||||
{"prose wrapped", "sizeof(struct Flags) is 4 bytes", 1},
|
||||
{"bit count then byte answer", "3+5+24 = 32 bits = 4 bytes", 1},
|
||||
{"wrong: one byte per bitfield member", "3", 0},
|
||||
{"wrong: assumes each bitfield gets its own storage unit", "12", 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate() = %v, want %v (detail: %s)", got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestCStructSizeTest_Eval(t *testing.T) {
|
||||
tc := cStructSizeTest()
|
||||
|
||||
|
||||
@@ -17,6 +17,13 @@ func registerGolangTests(r *testkit.Registry) {
|
||||
r.Register(goStructAlignTest())
|
||||
r.Register(goWorkerPoolTest())
|
||||
r.Register(goSemverClassifyTest())
|
||||
r.Register(goChannelDeadlockTest())
|
||||
r.Register(goDeferRecoverTraceTest())
|
||||
r.Register(goSliceAppendAliasingTest())
|
||||
r.Register(goMethodSetInterfaceTest())
|
||||
r.Register(goContextCancellationTraceTest())
|
||||
r.Register(goGenericsConstraintFixTest())
|
||||
r.Register(goBoundedFanoutTest())
|
||||
}
|
||||
|
||||
// goStructAlignTest: reorder struct fields to minimize padding on a 64-bit
|
||||
@@ -218,3 +225,457 @@ object: {"bump":"major|minor|patch"}`
|
||||
Eval: eval.JSONField("bump", "minor"),
|
||||
}
|
||||
}
|
||||
|
||||
// goChannelDeadlockReason is the fixed vocabulary of "why" answers
|
||||
// goChannelDeadlockTest allows, so the JSON "reason" field has exactly one
|
||||
// canonical correct literal instead of open-ended prose.
|
||||
const (
|
||||
goChannelDeadlockReasonUnbufferedNoReceiver = "unbuffered-send-no-receiver"
|
||||
)
|
||||
|
||||
// goChannelDeadlockTest: identify which line of an inline snippet blocks
|
||||
// forever, and why.
|
||||
//
|
||||
// ground truth: `ch := make(chan int)` creates an unbuffered channel, whose
|
||||
// send blocks until another goroutine is ready to receive concurrently. Line
|
||||
// 5 (`ch <- 1`) runs on main's single goroutine with no other goroutine
|
||||
// started, so the send blocks forever - the receive on line 6 can never run
|
||||
// because line 5 never returns. This is a single-goroutine self-deadlock,
|
||||
// not (for example) a nil-channel receive or a full buffered channel, so the
|
||||
// enumerated reason is "unbuffered-send-no-receiver". Because a genuine
|
||||
// deadlock hangs the process, golang_test.go cannot recompute this by
|
||||
// running the snippet (per PLAN.md, ground truth is derived in the doc
|
||||
// comment instead, as for goStructAlignTest's alignment arithmetic).
|
||||
func goChannelDeadlockTest() testkit.Test {
|
||||
prompt := `Here is Go code, with line numbers prefixed:
|
||||
|
||||
1: package main
|
||||
2:
|
||||
3: func main() {
|
||||
4: ch := make(chan int)
|
||||
5: ch <- 1
|
||||
6: v := <-ch
|
||||
7: println(v)
|
||||
8: }
|
||||
|
||||
Which line number blocks forever? Pick exactly one reason from this list:
|
||||
"unbuffered-send-no-receiver", "closed-channel-send", "nil-channel-receive",
|
||||
"buffered-channel-full". Respond with only a JSON object:
|
||||
{"line": <line number as an integer>, "reason": "<one of the four reasons above, exactly as written>"}`
|
||||
|
||||
evaluator := eval.All(
|
||||
eval.W(eval.JSONField("line", 5), 1),
|
||||
eval.W(eval.JSONField("reason", goChannelDeadlockReasonUnbufferedNoReceiver), 1),
|
||||
)
|
||||
|
||||
return testkit.Test{
|
||||
ID: "go-channel-deadlock",
|
||||
Category: "programming",
|
||||
Subcategory: "golang",
|
||||
Description: "Identify the line and reason a single-goroutine unbuffered-channel send deadlocks forever.",
|
||||
System: terseCodeOnly,
|
||||
Prompt: prompt,
|
||||
Eval: evaluator,
|
||||
}
|
||||
}
|
||||
|
||||
// goDeferRecoverTraceCode is the inline snippet for goDeferRecoverTraceTest.
|
||||
// It is duplicated (not shared via a Go source string constant referencing
|
||||
// a compiled function) so the prompt text is self-contained per EXPANSION.md
|
||||
// rule 1; golang_test.go defines an equivalent compute() and calls it
|
||||
// in-process to recompute the ground truth independently of this comment.
|
||||
const goDeferRecoverTraceCode = `package main
|
||||
|
||||
import "fmt"
|
||||
|
||||
func compute() (result int) {
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
result = -1
|
||||
}
|
||||
}()
|
||||
defer func() {
|
||||
result += 10
|
||||
}()
|
||||
panic("boom")
|
||||
}
|
||||
|
||||
func main() {
|
||||
fmt.Println(compute())
|
||||
}`
|
||||
|
||||
// goDeferRecoverTraceTest: trace the exact printed output of a function
|
||||
// combining defer, recover, and a named return value.
|
||||
//
|
||||
// ground truth: deferred functions run in LIFO (last-registered-first)
|
||||
// order. compute() panics with "boom" before returning, so both defers run
|
||||
// during unwinding: first (LIFO) the second-registered defer runs,
|
||||
// `result += 10`, taking the named return `result` from its zero value 0 to
|
||||
// 10 - the panic is still active at this point, since this defer never
|
||||
// calls recover. Then the first-registered defer runs, calls recover()
|
||||
// (which returns the non-nil panic value "boom", stopping the panic), and
|
||||
// overwrites `result = -1`. The named return is read after all defers
|
||||
// finish, so compute() returns -1 and main prints "-1". golang_test.go
|
||||
// verified this by literally running an equivalent compute() with `go run`
|
||||
// during authoring (see the ground-truth test below, which re-verifies it
|
||||
// in-process).
|
||||
func goDeferRecoverTraceTest() testkit.Test {
|
||||
prompt := `Here is a Go program:
|
||||
|
||||
` + "```go" + `
|
||||
` + goDeferRecoverTraceCode + `
|
||||
` + "```" + `
|
||||
|
||||
What does this program print when run? Respond with only the number,
|
||||
nothing else.`
|
||||
|
||||
return testkit.Test{
|
||||
ID: "go-defer-recover-trace",
|
||||
Category: "programming",
|
||||
Subcategory: "golang",
|
||||
Description: "Trace the exact output of a defer/recover chain that mutates a named return value.",
|
||||
Prompt: prompt,
|
||||
Eval: eval.Numeric(eval.ExtractLastNumber[int], -1, 0),
|
||||
}
|
||||
}
|
||||
|
||||
// goSliceAppendAliasingCode is the inline snippet for
|
||||
// goSliceAppendAliasingTest.
|
||||
const goSliceAppendAliasingCode = `package main
|
||||
|
||||
import "fmt"
|
||||
|
||||
func main() {
|
||||
a := make([]int, 3, 5)
|
||||
a[0], a[1], a[2] = 1, 2, 3
|
||||
b := a[:2]
|
||||
b = append(b, 99)
|
||||
_ = b
|
||||
fmt.Println(a[2])
|
||||
}`
|
||||
|
||||
// goSliceAppendAliasingTest: trace the exact printed output of a slice
|
||||
// aliasing gotcha where append writes into a shared backing array.
|
||||
//
|
||||
// ground truth: cap(a[:2]) = cap(a) - 0 = 5 (slicing a[low:high] gives
|
||||
// cap = cap(a) - low), so b := a[:2] has len 2 but cap 5, still sharing a's
|
||||
// backing array. append(b, 99) has spare capacity, so it does not allocate a
|
||||
// new array: it writes 99 into backing-array index 2 (len(b) before the
|
||||
// append) and returns a length-3 slice. Backing-array index 2 is also a[2],
|
||||
// so a[2] becomes 99 even though a itself was never explicitly assigned to.
|
||||
// golang_test.go re-verifies this in-process.
|
||||
func goSliceAppendAliasingTest() testkit.Test {
|
||||
prompt := `Here is a Go program:
|
||||
|
||||
` + "```go" + `
|
||||
` + goSliceAppendAliasingCode + `
|
||||
` + "```" + `
|
||||
|
||||
What does this program print when run? Respond with only the number,
|
||||
nothing else.`
|
||||
|
||||
return testkit.Test{
|
||||
ID: "go-slice-append-aliasing",
|
||||
Category: "programming",
|
||||
Subcategory: "golang",
|
||||
Description: "Trace the exact output of a slice-append aliasing gotcha on a shared backing array.",
|
||||
Prompt: prompt,
|
||||
Eval: eval.Numeric(eval.ExtractLastNumber[int], 99, 0),
|
||||
}
|
||||
}
|
||||
|
||||
// goMethodSetInterfaceCode is the inline snippet for
|
||||
// goMethodSetInterfaceTest: a type whose only interface method has a
|
||||
// pointer receiver, so only *Wrapper (not Wrapper) satisfies Stringer.
|
||||
const goMethodSetInterfaceCode = `type Stringer interface {
|
||||
String() string
|
||||
}
|
||||
|
||||
type Wrapper struct {
|
||||
val string
|
||||
}
|
||||
|
||||
func (w *Wrapper) String() string {
|
||||
return "Wrapper(" + w.val + ")"
|
||||
}`
|
||||
|
||||
// goMethodSetInterfaceTest: decide whether a value type and a pointer type
|
||||
// each satisfy an interface whose method has a pointer receiver.
|
||||
//
|
||||
// ground truth: a pointer receiver method belongs to the method set of the
|
||||
// pointer type only, not the value type (Go spec, "Method sets"). So
|
||||
// `Wrapper{...}` (a value) does not implement Stringer -
|
||||
// "Wrapper does not implement Stringer (method String has pointer
|
||||
// receiver)" - while `&Wrapper{...}` (a *Wrapper) does. golang_test.go
|
||||
// cross-checks both compile-time facts by running `go build` on each
|
||||
// snippet, guarded by go-toolchain presence (as c_test.go does for cc).
|
||||
func goMethodSetInterfaceTest() testkit.Test {
|
||||
prompt := `Here is Go code:
|
||||
|
||||
` + "```go" + `
|
||||
` + goMethodSetInterfaceCode + `
|
||||
` + "```" + `
|
||||
|
||||
Does this line compile: ` + "`var _ Stringer = Wrapper{val: \"hi\"}`" + `?
|
||||
Does this line compile: ` + "`var _ Stringer = &Wrapper{val: \"hi\"}`" + `?
|
||||
|
||||
Respond with only a JSON object:
|
||||
{"value_compiles": true|false, "pointer_compiles": true|false}`
|
||||
|
||||
evaluator := eval.All(
|
||||
eval.W(eval.JSONField("value_compiles", false), 1),
|
||||
eval.W(eval.JSONField("pointer_compiles", true), 1),
|
||||
)
|
||||
|
||||
return testkit.Test{
|
||||
ID: "go-method-set-interface",
|
||||
Category: "programming",
|
||||
Subcategory: "golang",
|
||||
Description: "Decide whether a value type and its pointer type satisfy an interface with a pointer-receiver method.",
|
||||
System: terseCodeOnly,
|
||||
Prompt: prompt,
|
||||
Eval: evaluator,
|
||||
}
|
||||
}
|
||||
|
||||
// goContextCancellationTraceCode is the inline snippet for
|
||||
// goContextCancellationTraceTest.
|
||||
const goContextCancellationTraceCode = `package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
)
|
||||
|
||||
func main() {
|
||||
parent, cancelParent := context.WithCancel(context.Background())
|
||||
child, cancelChild := context.WithCancel(parent)
|
||||
defer cancelChild()
|
||||
|
||||
cancelParent()
|
||||
|
||||
select {
|
||||
case <-child.Done():
|
||||
fmt.Println(child.Err())
|
||||
default:
|
||||
fmt.Println("not done")
|
||||
}
|
||||
}`
|
||||
|
||||
// goContextCancellationTraceTest: trace the exact printed output of
|
||||
// cancellation propagating from a parent context to a context derived from
|
||||
// it.
|
||||
//
|
||||
// ground truth: context.WithCancel propagates cancellation synchronously
|
||||
// down the derivation tree - cancelParent() closes parent's Done channel
|
||||
// and, in the same call, closes every context derived from it (including
|
||||
// child), before cancelParent() returns. So by the time the select runs,
|
||||
// child.Done() is already closed and takes the first case, printing
|
||||
// child.Err(). Err() returns the sentinel context.Canceled, whose message
|
||||
// (per the context package source) is "context canceled". golang_test.go
|
||||
// re-verifies this in-process, including that the propagation really is
|
||||
// synchronous (no goroutine/timing dependency, so the trace is
|
||||
// deterministic).
|
||||
func goContextCancellationTraceTest() testkit.Test {
|
||||
prompt := `Here is a Go program:
|
||||
|
||||
` + "```go" + `
|
||||
` + goContextCancellationTraceCode + `
|
||||
` + "```" + `
|
||||
|
||||
What does this program print when run? Respond with only the exact text
|
||||
printed, nothing else.`
|
||||
|
||||
return testkit.Test{
|
||||
ID: "go-context-cancellation-trace",
|
||||
Category: "programming",
|
||||
Subcategory: "golang",
|
||||
Description: "Trace the exact output of a context cancellation propagating synchronously from a parent to a derived context.",
|
||||
Prompt: prompt,
|
||||
Eval: eval.Equals("context canceled"),
|
||||
}
|
||||
}
|
||||
|
||||
// goGenericsConstraintFixHarness calls the response's fixed Max function
|
||||
// across int, float64, and string, verifying both the return values and
|
||||
// that it compiles at all (a broken constraint fails to compile, which
|
||||
// eval.GoRun scores as 0 via its "execution failed" / build-error path).
|
||||
const goGenericsConstraintFixHarness = `package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
)
|
||||
|
||||
func main() {
|
||||
if got := Max(3, 5); got != 5 {
|
||||
fmt.Fprintf(os.Stderr, "Max(3,5) = %v, want 5\n", got)
|
||||
os.Exit(1)
|
||||
}
|
||||
if got := Max(5, 3); got != 5 {
|
||||
fmt.Fprintf(os.Stderr, "Max(5,3) = %v, want 5\n", got)
|
||||
os.Exit(1)
|
||||
}
|
||||
if got := Max(2.5, 1.5); got != 2.5 {
|
||||
fmt.Fprintf(os.Stderr, "Max(2.5,1.5) = %v, want 2.5\n", got)
|
||||
os.Exit(1)
|
||||
}
|
||||
if got := Max("apple", "banana"); got != "banana" {
|
||||
fmt.Fprintf(os.Stderr, "Max(apple,banana) = %v, want banana\n", got)
|
||||
os.Exit(1)
|
||||
}
|
||||
fmt.Println("PASS")
|
||||
}
|
||||
`
|
||||
|
||||
// goGenericsConstraintFixTest: repair a generic function whose type
|
||||
// parameter constraint (any) does not support the operator (>) its body
|
||||
// uses, without depending on any external package.
|
||||
//
|
||||
// ground truth: the harness above is the oracle - it exercises Max across
|
||||
// three concrete types the fixed constraint must accept (int, float64,
|
||||
// string), so there is no separate hardcoded constant to derive.
|
||||
func goGenericsConstraintFixTest() testkit.Test {
|
||||
prompt := `This Go generic function does not compile:
|
||||
|
||||
` + "```go" + `
|
||||
func Max[T any](a, b T) T {
|
||||
if a > b {
|
||||
return a
|
||||
}
|
||||
return b
|
||||
}
|
||||
` + "```" + `
|
||||
|
||||
The compiler rejects it because type parameter T (constrained only by any)
|
||||
does not support the > operator. Fix it with the smallest change that makes
|
||||
it compile and work correctly for int, float64, and string arguments. Define
|
||||
any constraint interface you need yourself, using only the standard library
|
||||
- do not import golang.org/x/exp/constraints or any other third-party
|
||||
package. Respond with the complete corrected code (constraint interface, if
|
||||
any, plus the Max function).`
|
||||
|
||||
return testkit.Test{
|
||||
ID: "go-generics-constraint-fix",
|
||||
Category: "programming",
|
||||
Subcategory: "golang",
|
||||
Description: "Fix a generic function's constraint so > compiles and works for int, float64, and string.",
|
||||
System: terseCodeOnly,
|
||||
Prompt: prompt,
|
||||
Eval: eval.GoRun(goGenericsConstraintFixHarness, "PASS"),
|
||||
}
|
||||
}
|
||||
|
||||
// goBoundedFanoutHarness drives the response's BoundedRun with two batches
|
||||
// of tasks (all-succeed, then some-fail) over an instrumented task function
|
||||
// that tracks observed concurrency via an atomic counter, the same pattern
|
||||
// goWorkerPoolHarness uses. It checks: (a) the all-succeed batch returns
|
||||
// nil, (b) the some-fail batch returns a non-nil error, and (c) in both
|
||||
// batches, observed concurrency stayed within [2, limit] - never sequential
|
||||
// (>=2, evidencing real concurrency), never over the requested bound.
|
||||
const goBoundedFanoutHarness = `package main
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
)
|
||||
|
||||
func main() {
|
||||
const limit = 4
|
||||
const n = 20
|
||||
|
||||
var concurrent, maxConcurrent int32
|
||||
makeTask := func(shouldFail bool) func() error {
|
||||
return func() error {
|
||||
cur := atomic.AddInt32(&concurrent, 1)
|
||||
for {
|
||||
m := atomic.LoadInt32(&maxConcurrent)
|
||||
if cur <= m || atomic.CompareAndSwapInt32(&maxConcurrent, m, cur) {
|
||||
break
|
||||
}
|
||||
}
|
||||
time.Sleep(5 * time.Millisecond)
|
||||
atomic.AddInt32(&concurrent, -1)
|
||||
if shouldFail {
|
||||
return errors.New("boom")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
okTasks := make([]func() error, n)
|
||||
for i := range okTasks {
|
||||
okTasks[i] = makeTask(false)
|
||||
}
|
||||
if err := BoundedRun(okTasks, limit); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "BoundedRun(all ok) = %v, want nil\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
if got := atomic.LoadInt32(&maxConcurrent); got < 2 || got > limit {
|
||||
fmt.Fprintf(os.Stderr, "maxConcurrent=%d, want in [2, %d]\n", got, limit)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
atomic.StoreInt32(&concurrent, 0)
|
||||
atomic.StoreInt32(&maxConcurrent, 0)
|
||||
failTasks := make([]func() error, n)
|
||||
for i := range failTasks {
|
||||
failTasks[i] = makeTask(i%3 == 0)
|
||||
}
|
||||
err := BoundedRun(failTasks, limit)
|
||||
if err == nil {
|
||||
fmt.Fprintln(os.Stderr, "BoundedRun(some fail) = nil, want an error")
|
||||
os.Exit(1)
|
||||
}
|
||||
if got := atomic.LoadInt32(&maxConcurrent); got < 2 || got > limit {
|
||||
fmt.Fprintf(os.Stderr, "maxConcurrent=%d, want in [2, %d]\n", got, limit)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
fmt.Println("PASS")
|
||||
}
|
||||
`
|
||||
|
||||
// goBoundedFanoutTest: implement bounded-concurrency fan-out with
|
||||
// first-error propagation, the same shape as
|
||||
// golang.org/x/sync/errgroup.Group.SetLimit but implemented directly, so
|
||||
// the exec evaluator never depends on an external module being resolvable
|
||||
// in the sandbox (EXPANSION.md permits substituting an equivalent-difficulty
|
||||
// topic when the literal one cannot be made deterministic offline).
|
||||
//
|
||||
// ground truth: the harness above is the oracle, exercising both
|
||||
// correctness (first error propagates, nil on all-success) and the
|
||||
// concurrency bound via the same observed-concurrency counter pattern as
|
||||
// goWorkerPoolHarness; there is no separate hardcoded constant to derive.
|
||||
func goBoundedFanoutTest() testkit.Test {
|
||||
prompt := `Implement this Go function:
|
||||
|
||||
` + "```go" + `
|
||||
func BoundedRun(tasks []func() error, limit int) error
|
||||
` + "```" + `
|
||||
|
||||
Requirements:
|
||||
- Run the tasks concurrently, but never more than limit at the same time.
|
||||
- If one or more tasks return a non-nil error, BoundedRun must return a
|
||||
non-nil error (any one of the failing tasks' errors is acceptable).
|
||||
- If every task returns nil, BoundedRun must return nil.
|
||||
- Use only the standard library (for example sync, sync/atomic, or
|
||||
channels). Do not import golang.org/x/sync/errgroup or any other
|
||||
third-party package.
|
||||
- You may include whatever standard-library imports your implementation
|
||||
needs. Do not include a package clause or a main function: respond with
|
||||
only the BoundedRun function definition (plus any imports it needs).`
|
||||
|
||||
return testkit.Test{
|
||||
ID: "go-bounded-fanout",
|
||||
Category: "programming",
|
||||
Subcategory: "golang",
|
||||
Description: "Implement bounded-concurrency fan-out with first-error propagation, using only the standard library.",
|
||||
System: terseCodeOnly,
|
||||
Prompt: prompt,
|
||||
Eval: eval.GoRun(goBoundedFanoutHarness, "PASS"),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2,7 +2,15 @@ package tests
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
"unsafe"
|
||||
)
|
||||
|
||||
@@ -257,3 +265,468 @@ func TestGoSemverClassifyTest_Eval(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestGoChannelDeadlockTest_Eval(t *testing.T) {
|
||||
tc := goChannelDeadlockTest()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{"exact correct", `{"line": 5, "reason": "unbuffered-send-no-receiver"}`, 1},
|
||||
{"correct fenced json", "```json\n{\"line\": 5, \"reason\": \"unbuffered-send-no-receiver\"}\n```", 1},
|
||||
{"correct with prose wrapper", `The answer is: {"line": 5, "reason": "unbuffered-send-no-receiver"}`, 1},
|
||||
{"wrong line", `{"line": 6, "reason": "unbuffered-send-no-receiver"}`, 0.5},
|
||||
{"wrong reason", `{"line": 5, "reason": "nil-channel-receive"}`, 0.5},
|
||||
{"both wrong", `{"line": 6, "reason": "nil-channel-receive"}`, 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate() = %v, want %v (detail: %s)", got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// computeDeferRecover mirrors goDeferRecoverTraceCode's compute() exactly,
|
||||
// so its return value grounds the "ground truth:" derivation in golang.go
|
||||
// by actually running the same logic, per PLAN.md's recomputation rule.
|
||||
func computeDeferRecover() (result int) {
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
result = -1
|
||||
}
|
||||
}()
|
||||
defer func() {
|
||||
result += 10
|
||||
}()
|
||||
panic("boom")
|
||||
}
|
||||
|
||||
func TestGoDeferRecoverTrace_GroundTruth(t *testing.T) {
|
||||
if got, want := computeDeferRecover(), -1; got != want {
|
||||
t.Errorf("computeDeferRecover() = %d, want %d", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGoDeferRecoverTraceTest_Eval(t *testing.T) {
|
||||
tc := goDeferRecoverTraceTest()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{"exact correct", "-1", 1},
|
||||
{"correct with prose", "The program prints -1.", 1},
|
||||
{"correct with label", "Output: -1", 1},
|
||||
{"wrong: ignores recover overwrite", "10", 0},
|
||||
{"wrong: sign error", "1", 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate() = %v, want %v (detail: %s)", got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// sliceAppendAliasCheck mirrors goSliceAppendAliasingCode's main() body,
|
||||
// returning a[2] instead of printing it, so the ground truth is recomputed
|
||||
// in-process rather than trusted from the doc comment alone.
|
||||
func sliceAppendAliasCheck() int {
|
||||
a := make([]int, 3, 5)
|
||||
a[0], a[1], a[2] = 1, 2, 3
|
||||
b := a[:2]
|
||||
b = append(b, 99)
|
||||
_ = b
|
||||
return a[2]
|
||||
}
|
||||
|
||||
func TestGoSliceAppendAliasing_GroundTruth(t *testing.T) {
|
||||
if got, want := sliceAppendAliasCheck(), 99; got != want {
|
||||
t.Errorf("sliceAppendAliasCheck() = %d, want %d", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGoSliceAppendAliasingTest_Eval(t *testing.T) {
|
||||
tc := goSliceAppendAliasingTest()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{"exact correct", "99", 1},
|
||||
{"correct with prose", "a[2] is 99.", 1},
|
||||
{"correct with label", "The result is 99", 1},
|
||||
{"wrong: ignores aliasing, keeps original value", "3", 0},
|
||||
{"wrong: off-by-index confusion", "2", 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate() = %v, want %v (detail: %s)", got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestGoMethodSetInterface_GroundTruth cross-checks the value_compiles=false,
|
||||
// pointer_compiles=true claims in golang.go's doc comment by actually
|
||||
// running `go build` on each snippet, mirroring c_test.go's
|
||||
// TestCStructSizeWant_GroundTruth pattern of shelling out to the real
|
||||
// toolchain rather than trusting hand analysis alone.
|
||||
func TestGoMethodSetInterface_GroundTruth(t *testing.T) {
|
||||
const preamble = `package main
|
||||
|
||||
` + goMethodSetInterfaceCode + `
|
||||
|
||||
`
|
||||
tests := []struct {
|
||||
name string
|
||||
mainLine string
|
||||
wantBuilds bool
|
||||
}{
|
||||
{"value type does not implement Stringer", `func main() { var _ Stringer = Wrapper{val: "hi"} }`, false},
|
||||
{"pointer type implements Stringer", `func main() { var _ Stringer = &Wrapper{val: "hi"} }`, true},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
src := preamble + tt.mainLine + "\n"
|
||||
srcPath := filepath.Join(dir, "main.go")
|
||||
if err := os.WriteFile(srcPath, []byte(src), 0o600); err != nil {
|
||||
t.Fatalf("write main.go: %v", err)
|
||||
}
|
||||
// #nosec G204 -- srcPath is a fixed filename this test just wrote
|
||||
// into its own t.TempDir(); not external input.
|
||||
cmd := exec.Command("go", "build", "-o", filepath.Join(dir, "out"), srcPath) //nolint:gosec // see comment above
|
||||
out, err := cmd.CombinedOutput()
|
||||
builds := err == nil
|
||||
if builds != tt.wantBuilds {
|
||||
t.Errorf("go build success = %v, want %v (output: %s)", builds, tt.wantBuilds, out)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestGoMethodSetInterfaceTest_Eval(t *testing.T) {
|
||||
tc := goMethodSetInterfaceTest()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{"exact correct", `{"value_compiles": false, "pointer_compiles": true}`, 1},
|
||||
{"correct fenced json", "```json\n{\"value_compiles\": false, \"pointer_compiles\": true}\n```", 1},
|
||||
{"correct with prose wrapper", `Answer: {"value_compiles": false, "pointer_compiles": true}`, 1},
|
||||
{"wrong: both true", `{"value_compiles": true, "pointer_compiles": true}`, 0.5},
|
||||
{"wrong: both false", `{"value_compiles": false, "pointer_compiles": false}`, 0.5},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate() = %v, want %v (detail: %s)", got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// contextCancellationTraceCheck mirrors goContextCancellationTraceCode's
|
||||
// main() body, returning the text it would print instead of printing it, to
|
||||
// recompute the ground truth in-process. It also grounds the "no
|
||||
// goroutine/timing dependency" claim in golang.go's doc comment: this
|
||||
// function has no goroutines or sleeps at all, so a deterministic result
|
||||
// here demonstrates the propagation is synchronous.
|
||||
func contextCancellationTraceCheck() string {
|
||||
parent, cancelParent := context.WithCancel(context.Background())
|
||||
child, cancelChild := context.WithCancel(parent)
|
||||
defer cancelChild()
|
||||
|
||||
cancelParent()
|
||||
|
||||
select {
|
||||
case <-child.Done():
|
||||
return child.Err().Error()
|
||||
default:
|
||||
return "not done"
|
||||
}
|
||||
}
|
||||
|
||||
func TestGoContextCancellationTrace_GroundTruth(t *testing.T) {
|
||||
if got, want := contextCancellationTraceCheck(), "context canceled"; got != want {
|
||||
t.Errorf("contextCancellationTraceCheck() = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGoContextCancellationTraceTest_Eval(t *testing.T) {
|
||||
tc := goContextCancellationTraceTest()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{"exact correct", "context canceled", 1},
|
||||
{"correct case-insensitive", "CONTEXT CANCELED", 1},
|
||||
{"correct with surrounding whitespace", " context canceled ", 1},
|
||||
{"wrong: assumes select falls to default", "not done", 0},
|
||||
{"wrong: wrong sentinel error", "context deadline exceeded", 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate() = %v, want %v (detail: %s)", got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestGoGenericsConstraintFixTest_Eval(t *testing.T) {
|
||||
tc := goGenericsConstraintFixTest()
|
||||
|
||||
fullOrderedConstraint := "```go\n" + `type Ordered interface {
|
||||
~int | ~int8 | ~int16 | ~int32 | ~int64 |
|
||||
~uint | ~uint8 | ~uint16 | ~uint32 | ~uint64 |
|
||||
~float32 | ~float64 | ~string
|
||||
}
|
||||
|
||||
func Max[T Ordered](a, b T) T {
|
||||
if a > b {
|
||||
return a
|
||||
}
|
||||
return b
|
||||
}
|
||||
` + "```"
|
||||
|
||||
minimalConstraint := "```go\n" + `type Ordered interface {
|
||||
~int | ~float64 | ~string
|
||||
}
|
||||
|
||||
func Max[T Ordered](a, b T) T {
|
||||
if a > b {
|
||||
return a
|
||||
}
|
||||
return b
|
||||
}
|
||||
` + "```"
|
||||
|
||||
stillBroken := "```go\n" + `func Max[T any](a, b T) T {
|
||||
if a > b {
|
||||
return a
|
||||
}
|
||||
return b
|
||||
}
|
||||
` + "```"
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{"full Ordered constraint covering all integer widths", fullOrderedConstraint, 1},
|
||||
{"minimal constraint covering only the exercised types", minimalConstraint, 1},
|
||||
{"still broken: constraint left as any", stillBroken, 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Skipped {
|
||||
t.Skip("go toolchain not available, skipping exec-evaluator assertion")
|
||||
}
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate() = %v, want %v (detail: %s)", got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestGoBoundedFanoutTest_Eval(t *testing.T) {
|
||||
tc := goBoundedFanoutTest()
|
||||
|
||||
correctBoundedRun := "```go\n" + `import "sync"
|
||||
|
||||
func BoundedRun(tasks []func() error, limit int) error {
|
||||
sem := make(chan struct{}, limit)
|
||||
var wg sync.WaitGroup
|
||||
var mu sync.Mutex
|
||||
var firstErr error
|
||||
|
||||
for _, task := range tasks {
|
||||
task := task
|
||||
sem <- struct{}{}
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
defer func() { <-sem }()
|
||||
if err := task(); err != nil {
|
||||
mu.Lock()
|
||||
if firstErr == nil {
|
||||
firstErr = err
|
||||
}
|
||||
mu.Unlock()
|
||||
}
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
return firstErr
|
||||
}
|
||||
` + "```"
|
||||
|
||||
sequentialBoundedRun := "```go\n" + `func BoundedRun(tasks []func() error, limit int) error {
|
||||
for _, task := range tasks {
|
||||
if err := task(); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
` + "```"
|
||||
|
||||
unboundedBoundedRun := "```go\n" + `import "sync"
|
||||
|
||||
func BoundedRun(tasks []func() error, limit int) error {
|
||||
var wg sync.WaitGroup
|
||||
var mu sync.Mutex
|
||||
var firstErr error
|
||||
for _, task := range tasks {
|
||||
task := task
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
if err := task(); err != nil {
|
||||
mu.Lock()
|
||||
if firstErr == nil {
|
||||
firstErr = err
|
||||
}
|
||||
mu.Unlock()
|
||||
}
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
return firstErr
|
||||
}
|
||||
` + "```"
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{"correct semaphore-bounded concurrency", correctBoundedRun, 1},
|
||||
{"wrong: sequential, no concurrency", sequentialBoundedRun, 0},
|
||||
{"wrong: unbounded, ignores limit", unboundedBoundedRun, 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Skipped {
|
||||
t.Skip("go toolchain not available, skipping exec-evaluator assertion")
|
||||
}
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate() = %v, want %v (detail: %s)", got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// boundedRunReference is a package-local reference implementation of the
|
||||
// same bounded-fan-out shape the harness expects (semaphore-limited
|
||||
// concurrency, first-error propagation), independent of the fenced
|
||||
// test-string copy in TestGoBoundedFanoutTest_Eval, so the harness's
|
||||
// concurrency-bound assertion is grounded against real, in-process
|
||||
// goroutines rather than only against exec.Command output.
|
||||
func boundedRunReference(tasks []func() error, limit int) error {
|
||||
sem := make(chan struct{}, limit)
|
||||
var wg sync.WaitGroup
|
||||
var mu sync.Mutex
|
||||
var firstErr error
|
||||
|
||||
for _, task := range tasks {
|
||||
task := task
|
||||
sem <- struct{}{}
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
defer func() { <-sem }()
|
||||
if err := task(); err != nil {
|
||||
mu.Lock()
|
||||
if firstErr == nil {
|
||||
firstErr = err
|
||||
}
|
||||
mu.Unlock()
|
||||
}
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
return firstErr
|
||||
}
|
||||
|
||||
func TestBoundedRunReference_GroundTruth(t *testing.T) {
|
||||
const limit = 4
|
||||
const n = 20
|
||||
|
||||
var concurrent, maxConcurrent int32
|
||||
makeTask := func(shouldFail bool) func() error {
|
||||
return func() error {
|
||||
cur := atomic.AddInt32(&concurrent, 1)
|
||||
for {
|
||||
m := atomic.LoadInt32(&maxConcurrent)
|
||||
if cur <= m || atomic.CompareAndSwapInt32(&maxConcurrent, m, cur) {
|
||||
break
|
||||
}
|
||||
}
|
||||
time.Sleep(time.Millisecond)
|
||||
atomic.AddInt32(&concurrent, -1)
|
||||
if shouldFail {
|
||||
return errors.New("boom")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
okTasks := make([]func() error, n)
|
||||
for i := range okTasks {
|
||||
okTasks[i] = makeTask(false)
|
||||
}
|
||||
if err := boundedRunReference(okTasks, limit); err != nil {
|
||||
t.Errorf("boundedRunReference(all ok) = %v, want nil", err)
|
||||
}
|
||||
if got := atomic.LoadInt32(&maxConcurrent); got < 2 || got > limit {
|
||||
t.Errorf("maxConcurrent = %d, want in [2, %d]", got, limit)
|
||||
}
|
||||
|
||||
atomic.StoreInt32(&concurrent, 0)
|
||||
atomic.StoreInt32(&maxConcurrent, 0)
|
||||
failTasks := make([]func() error, n)
|
||||
for i := range failTasks {
|
||||
failTasks[i] = makeTask(i%3 == 0)
|
||||
}
|
||||
if err := boundedRunReference(failTasks, limit); err == nil {
|
||||
t.Error("boundedRunReference(some fail) = nil, want an error")
|
||||
}
|
||||
if got := atomic.LoadInt32(&maxConcurrent); got < 2 || got > limit {
|
||||
t.Errorf("maxConcurrent = %d, want in [2, %d]", got, limit)
|
||||
}
|
||||
}
|
||||
|
||||
// TestGoChannelDeadlockCode_ContainsExpectedLines grounds the line-number
|
||||
// claim in goChannelDeadlockTest's doc comment against the actual prompt
|
||||
// text used at registration time, rather than trusting the doc comment's
|
||||
// count independently of the real prompt.
|
||||
func TestGoChannelDeadlockCode_ContainsExpectedLines(t *testing.T) {
|
||||
tc := goChannelDeadlockTest()
|
||||
if !strings.Contains(tc.Prompt, "5: \tch <- 1") {
|
||||
t.Error("goChannelDeadlockTest prompt no longer has 'ch <- 1' on line 5; update the ground-truth line number")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
package tests
|
||||
|
||||
import (
|
||||
"math"
|
||||
|
||||
"github.com/lukaszraczylo/llm-testbench/internal/eval"
|
||||
"github.com/lukaszraczylo/llm-testbench/internal/testkit"
|
||||
)
|
||||
@@ -8,6 +10,14 @@ import (
|
||||
func registerPythonTests(r *testkit.Registry) {
|
||||
r.Register(pyLogTriageTest())
|
||||
r.Register(pyCosineTest())
|
||||
r.Register(pySoftmaxTest())
|
||||
r.Register(pyAsyncioGatherTraceTest())
|
||||
r.Register(pyMutableDefaultArgTest())
|
||||
r.Register(pyDictComprehensionTraceTest())
|
||||
r.Register(pyGeneratorExhaustionTraceTest())
|
||||
r.Register(pyPathlibRewriteTest())
|
||||
r.Register(pyJSONTransformTest())
|
||||
r.Register(pyRegexLogExtractionTest())
|
||||
}
|
||||
|
||||
// pyLogTriageLog is the inline kubelet log excerpt for pyLogTriageTest.
|
||||
@@ -94,3 +104,345 @@ only the number, nothing else.`
|
||||
Eval: eval.Numeric(eval.ExtractLastNumber[float64], pyCosineWant, 0.0005),
|
||||
}
|
||||
}
|
||||
|
||||
// pySoftmaxVector is the inline 5-dimensional vector for pySoftmaxTest.
|
||||
var pySoftmaxVector = []float64{2.0, 1.0, 0.1, -1.0, 0.5}
|
||||
|
||||
// softmaxFirst computes the softmax probability of index 0 of v:
|
||||
// exp(v[0]) / sum(exp(v[i]) for all i). Used to derive pySoftmaxWant at
|
||||
// catalog-registration time rather than hardcoding a pre-computed constant,
|
||||
// mirroring pyCosineWant's use of cosineSimilarity.
|
||||
func softmaxFirst(v []float64) float64 {
|
||||
var sum float64
|
||||
for _, x := range v {
|
||||
sum += math.Exp(x)
|
||||
}
|
||||
return math.Exp(v[0]) / sum
|
||||
}
|
||||
|
||||
// pySoftmaxWant is derived, not hardcoded: python_test.go independently
|
||||
// re-verifies it against math.Exp.
|
||||
// ground truth: softmax(v)[0] = exp(v[0]) / sum(exp(v[i])), rounded to 4
|
||||
// decimal places as the prompt requests.
|
||||
var pySoftmaxWant = round4dp(softmaxFirst(pySoftmaxVector))
|
||||
|
||||
func pySoftmaxTest() testkit.Test {
|
||||
prompt := `Given the 5-dimensional vector:
|
||||
|
||||
v = [2.0, 1.0, 0.1, -1.0, 0.5]
|
||||
|
||||
Compute the softmax of v, and report only the probability assigned to the
|
||||
first element (index 0), rounded to 4 decimal places. Respond with only the
|
||||
number, nothing else.`
|
||||
|
||||
return testkit.Test{
|
||||
ID: "py-softmax",
|
||||
Category: "programming",
|
||||
Subcategory: "python",
|
||||
Description: "Compute the softmax probability of one element of an inline 5-dimensional vector to 4 decimal places.",
|
||||
Prompt: prompt,
|
||||
Eval: eval.Numeric(eval.ExtractLastNumber[float64], pySoftmaxWant, 0.0005),
|
||||
}
|
||||
}
|
||||
|
||||
// pyAsyncioGatherTraceCode is the inline snippet for
|
||||
// pyAsyncioGatherTraceTest.
|
||||
const pyAsyncioGatherTraceCode = `import asyncio
|
||||
|
||||
|
||||
async def worker(name, delay):
|
||||
await asyncio.sleep(delay)
|
||||
print(name)
|
||||
|
||||
|
||||
async def main():
|
||||
await asyncio.gather(worker("A", 0.02), worker("B", 0.01), worker("C", 0.03))
|
||||
|
||||
|
||||
asyncio.run(main())`
|
||||
|
||||
// pyAsyncioGatherTraceTest: trace the print order of three coroutines
|
||||
// started together via asyncio.gather with different sleep delays.
|
||||
//
|
||||
// ground truth: asyncio.gather starts all three coroutines concurrently on
|
||||
// the single-threaded event loop; each immediately hits its
|
||||
// asyncio.sleep(delay) and the loop resumes them in order of scheduled wake
|
||||
// time (current loop time + delay), not source order. worker("B", 0.01) has
|
||||
// the shortest delay, so it wakes and prints first, then worker("A", 0.02),
|
||||
// then worker("C", 0.03). Because the event loop is single-threaded and
|
||||
// wake times are computed from fixed delays far apart (10ms increments),
|
||||
// this ordering has no OS-scheduling race and is deterministic - confirmed
|
||||
// by running the exact snippet 5 times during authoring (always B, A, C)
|
||||
// and independently by python_test.go's PyRun-based ground-truth test.
|
||||
func pyAsyncioGatherTraceTest() testkit.Test {
|
||||
prompt := `Here is a Python program:
|
||||
|
||||
` + "```python" + `
|
||||
` + pyAsyncioGatherTraceCode + `
|
||||
` + "```" + `
|
||||
|
||||
In what order are the three names printed when this program runs? Respond
|
||||
with only a JSON array of the three names in print order, for example
|
||||
["X","Y","Z"]`
|
||||
|
||||
return testkit.Test{
|
||||
ID: "py-asyncio-gather-trace",
|
||||
Category: "programming",
|
||||
Subcategory: "python",
|
||||
Description: "Trace the print order of three asyncio.gather coroutines with different sleep delays.",
|
||||
System: terseCodeOnly,
|
||||
Prompt: prompt,
|
||||
Eval: eval.JSONStringArrayEquals([]string{"B", "A", "C"}),
|
||||
}
|
||||
}
|
||||
|
||||
// pyMutableDefaultArgCode is the inline snippet for pyMutableDefaultArgTest.
|
||||
const pyMutableDefaultArgCode = `def add_item(item, bucket=[]):
|
||||
bucket.append(item)
|
||||
return bucket
|
||||
|
||||
|
||||
add_item(1)
|
||||
add_item(2)
|
||||
result = add_item(3)
|
||||
print(result)`
|
||||
|
||||
// pyMutableDefaultArgTest: trace the classic mutable-default-argument bug.
|
||||
//
|
||||
// ground truth: a default argument value (here, the empty list literal) is
|
||||
// evaluated once, at function-definition time, and the same list object is
|
||||
// reused as the default on every call that omits bucket - it is not
|
||||
// recreated per call. The three calls all omit bucket, so they share and
|
||||
// mutate one underlying list: after add_item(1) it holds [1], after
|
||||
// add_item(2) it holds [1, 2], and result = add_item(3) returns the same
|
||||
// list now holding [1, 2, 3]. print(result) prints Python's list repr,
|
||||
// "[1, 2, 3]". python_test.go re-verifies this via PyRun on the identical
|
||||
// snippet.
|
||||
func pyMutableDefaultArgTest() testkit.Test {
|
||||
prompt := `Here is a Python program:
|
||||
|
||||
` + "```python" + `
|
||||
` + pyMutableDefaultArgCode + `
|
||||
` + "```" + `
|
||||
|
||||
What does this program print when run? Respond with only the exact text
|
||||
printed, nothing else.`
|
||||
|
||||
return testkit.Test{
|
||||
ID: "py-mutable-default-arg",
|
||||
Category: "programming",
|
||||
Subcategory: "python",
|
||||
Description: "Trace the classic mutable-default-argument bug across three calls sharing one list.",
|
||||
Prompt: prompt,
|
||||
Eval: eval.Equals("[1, 2, 3]"),
|
||||
}
|
||||
}
|
||||
|
||||
// pyDictComprehensionTraceCode is the inline snippet for
|
||||
// pyDictComprehensionTraceTest.
|
||||
const pyDictComprehensionTraceCode = `words = ["apple", "banana", "cherry", "date", "fig"]
|
||||
lengths = {w: len(w) for w in words if len(w) > 3}
|
||||
print(sorted(lengths.items()))`
|
||||
|
||||
// pyDictComprehensionTraceTest: trace the exact output of a filtered dict
|
||||
// comprehension whose items are then sorted.
|
||||
//
|
||||
// ground truth: the comprehension keeps words with len > 3 - apple(5),
|
||||
// banana(6), cherry(6), date(4) - and drops fig(3). sorted() on a dict's
|
||||
// .items() sorts the (key, value) tuples by key first (Python tuple
|
||||
// ordering compares element-by-element), giving alphabetical key order:
|
||||
// apple, banana, cherry, date. print() renders that list of tuples with
|
||||
// Python's repr, producing the exact text below. python_test.go
|
||||
// re-verifies this via PyRun on the identical snippet.
|
||||
func pyDictComprehensionTraceTest() testkit.Test {
|
||||
prompt := `Here is a Python program:
|
||||
|
||||
` + "```python" + `
|
||||
` + pyDictComprehensionTraceCode + `
|
||||
` + "```" + `
|
||||
|
||||
What does this program print when run? Respond with only the exact text
|
||||
printed, nothing else.`
|
||||
|
||||
return testkit.Test{
|
||||
ID: "py-dict-comprehension-trace",
|
||||
Category: "programming",
|
||||
Subcategory: "python",
|
||||
Description: "Trace the exact output of a length-filtered dict comprehension whose items are sorted and printed.",
|
||||
Prompt: prompt,
|
||||
Eval: eval.Equals("[('apple', 5), ('banana', 6), ('cherry', 6), ('date', 4)]"),
|
||||
}
|
||||
}
|
||||
|
||||
// pyGeneratorExhaustionTraceCode is the inline snippet for
|
||||
// pyGeneratorExhaustionTraceTest.
|
||||
const pyGeneratorExhaustionTraceCode = `def gen():
|
||||
for i in range(3):
|
||||
yield i
|
||||
|
||||
|
||||
g = gen()
|
||||
first_pass = list(g)
|
||||
second_pass = list(g)
|
||||
print(first_pass, second_pass)`
|
||||
|
||||
// pyGeneratorExhaustionTraceTest: trace the exact output of iterating an
|
||||
// already-exhausted generator a second time.
|
||||
//
|
||||
// ground truth: a Python generator object is a single-pass iterator - once
|
||||
// list(g) has driven it to completion (StopIteration), the same generator
|
||||
// object has no more items to yield. first_pass = list(g) consumes all
|
||||
// three values, giving [0, 1, 2]; second_pass = list(g) on the same,
|
||||
// now-exhausted g immediately raises StopIteration inside list(), which
|
||||
// list() treats as "no items", giving []. print(a, b) separates its
|
||||
// arguments with a single space, producing the exact text below.
|
||||
// python_test.go re-verifies this via PyRun on the identical snippet.
|
||||
func pyGeneratorExhaustionTraceTest() testkit.Test {
|
||||
prompt := `Here is a Python program:
|
||||
|
||||
` + "```python" + `
|
||||
` + pyGeneratorExhaustionTraceCode + `
|
||||
` + "```" + `
|
||||
|
||||
What does this program print when run? Respond with only the exact text
|
||||
printed, nothing else.`
|
||||
|
||||
return testkit.Test{
|
||||
ID: "py-generator-exhaustion-trace",
|
||||
Category: "programming",
|
||||
Subcategory: "python",
|
||||
Description: "Trace the exact output of iterating a generator a second time after it is already exhausted.",
|
||||
Prompt: prompt,
|
||||
Eval: eval.Equals("[0, 1, 2] []"),
|
||||
}
|
||||
}
|
||||
|
||||
// pyOsPathCode is the inline os.path-based function pyPathlibRewriteTest
|
||||
// asks the model to rewrite using pathlib.
|
||||
const pyOsPathCode = `import os
|
||||
|
||||
|
||||
def find_configs(base_dir):
|
||||
results = []
|
||||
for root, dirs, files in os.walk(base_dir):
|
||||
for f in files:
|
||||
if f.endswith(".yaml"):
|
||||
results.append(os.path.join(root, f))
|
||||
return results`
|
||||
|
||||
// pyPathlibRewriteTest: rewrite an os.path/os.walk function using pathlib
|
||||
// idioms with equivalent behavior.
|
||||
func pyPathlibRewriteTest() testkit.Test {
|
||||
prompt := `Here is a Python function that recursively finds all ".yaml" files
|
||||
under base_dir and returns their full paths:
|
||||
|
||||
` + "```python" + `
|
||||
` + pyOsPathCode + `
|
||||
` + "```" + `
|
||||
|
||||
Rewrite this function using pathlib idioms instead of os.path/os.walk,
|
||||
keeping the same behavior: recursively find every ".yaml" file under
|
||||
base_dir and return their full paths. Respond with only the rewritten
|
||||
function.`
|
||||
|
||||
evaluator := eval.All(
|
||||
eval.W(eval.ContainsAny("from pathlib import Path", "import pathlib"), 1),
|
||||
eval.W(eval.ContainsAll("rglob", "*.yaml"), 2),
|
||||
)
|
||||
|
||||
return testkit.Test{
|
||||
ID: "py-pathlib-rewrite",
|
||||
Category: "programming",
|
||||
Subcategory: "python",
|
||||
Description: "Rewrite an os.path/os.walk recursive file finder using pathlib idioms.",
|
||||
System: terseCodeOnly,
|
||||
Prompt: prompt,
|
||||
Eval: evaluator,
|
||||
}
|
||||
}
|
||||
|
||||
// pyJSONTransformRecords is the inline dataset for pyJSONTransformTest.
|
||||
// ground truth: totals by dept, hand-summed (also re-derived in
|
||||
// python_test.go by scanning the same literal data): eng = 95000 + 105000 +
|
||||
// 88000 = 288000; ops = 72000; sales = 60000 + 58000 = 118000.
|
||||
const pyJSONTransformRecords = `records = [
|
||||
{"dept": "eng", "salary": 95000},
|
||||
{"dept": "sales", "salary": 60000},
|
||||
{"dept": "eng", "salary": 105000},
|
||||
{"dept": "ops", "salary": 72000},
|
||||
{"dept": "sales", "salary": 58000},
|
||||
{"dept": "eng", "salary": 88000},
|
||||
]`
|
||||
|
||||
// pyJSONTransformWant is the expected stdout of a correct transform script:
|
||||
// total salary per department, sorted alphabetically by department, as
|
||||
// "dept=total" per line.
|
||||
const pyJSONTransformWant = "eng=288000\nops=72000\nsales=118000"
|
||||
|
||||
func pyJSONTransformTest() testkit.Test {
|
||||
prompt := `Here is a Python list of employee records:
|
||||
|
||||
` + "```python" + `
|
||||
` + pyJSONTransformRecords + `
|
||||
` + "```" + `
|
||||
|
||||
Write a self-contained python3 script (embed the records above as data in
|
||||
your script; do not read from stdin or a file) that sums salary per dept
|
||||
and prints one line per department as "dept=total", sorted alphabetically by
|
||||
department name. Print exactly those lines and nothing else.`
|
||||
|
||||
return testkit.Test{
|
||||
ID: "py-json-transform",
|
||||
Category: "programming",
|
||||
Subcategory: "python",
|
||||
Description: "Write a python3 script that sums an inline list of salary records by department.",
|
||||
System: terseCodeOnly,
|
||||
Prompt: prompt,
|
||||
Eval: eval.PyRun(eval.PassthroughHarness, pyJSONTransformWant),
|
||||
}
|
||||
}
|
||||
|
||||
// pyRegexLogLines is the inline nginx-style access-log fixture for
|
||||
// pyRegexLogExtractionTest.
|
||||
// ground truth: HTTP status codes by line, in order: 200, 404, 200, 500,
|
||||
// 200, 404, 500 - giving 200=3, 404=2, 500=2 (also re-derived in
|
||||
// python_test.go by scanning the same literal data with Go's regexp).
|
||||
const pyRegexLogLines = `log_lines = [
|
||||
'203.0.113.5 - - [10/Aug/2026:10:00:01 +0000] "GET /api/health HTTP/1.1" 200 512',
|
||||
'203.0.113.7 - - [10/Aug/2026:10:00:02 +0000] "GET /api/users HTTP/1.1" 404 128',
|
||||
'203.0.113.5 - - [10/Aug/2026:10:00:03 +0000] "POST /api/login HTTP/1.1" 200 256',
|
||||
'203.0.113.9 - - [10/Aug/2026:10:00:04 +0000] "GET /api/orders HTTP/1.1" 500 96',
|
||||
'203.0.113.5 - - [10/Aug/2026:10:00:05 +0000] "GET /api/health HTTP/1.1" 200 512',
|
||||
'203.0.113.7 - - [10/Aug/2026:10:00:06 +0000] "GET /api/missing HTTP/1.1" 404 128',
|
||||
'203.0.113.9 - - [10/Aug/2026:10:00:07 +0000] "GET /api/orders HTTP/1.1" 500 96',
|
||||
]`
|
||||
|
||||
// pyRegexLogExtractionWant is the expected stdout: counts per HTTP status
|
||||
// code, sorted ascending by the numeric code, as "code=count" per line.
|
||||
const pyRegexLogExtractionWant = "200=3\n404=2\n500=2"
|
||||
|
||||
func pyRegexLogExtractionTest() testkit.Test {
|
||||
prompt := `Here are 7 lines of nginx-style access log output:
|
||||
|
||||
` + "```python" + `
|
||||
` + pyRegexLogLines + `
|
||||
` + "```" + `
|
||||
|
||||
Write a self-contained python3 script (embed the log_lines list above as
|
||||
data in your script; do not read from stdin or a file) that uses the re
|
||||
module to extract the 3-digit HTTP status code from each line (the number
|
||||
that appears right after the quoted request, e.g. right after
|
||||
"GET ... HTTP/1.1"), counts occurrences of each status code, and prints one
|
||||
line per status code as "code=count", sorted ascending by the numeric code.
|
||||
Print exactly those lines and nothing else.`
|
||||
|
||||
return testkit.Test{
|
||||
ID: "py-regex-log-extraction",
|
||||
Category: "programming",
|
||||
Subcategory: "python",
|
||||
Description: "Write a python3 script that uses re to extract and count HTTP status codes from an inline access log.",
|
||||
System: terseCodeOnly,
|
||||
Prompt: prompt,
|
||||
Eval: eval.PyRun(eval.PassthroughHarness, pyRegexLogExtractionWant),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3,8 +3,14 @@ package tests
|
||||
import (
|
||||
"context"
|
||||
"math"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/lukaszraczylo/llm-testbench/internal/eval"
|
||||
)
|
||||
|
||||
func TestCosineSimilarity_KnownCases(t *testing.T) {
|
||||
@@ -108,6 +114,398 @@ for key in sorted(counts):
|
||||
}
|
||||
}
|
||||
|
||||
func TestSoftmaxFirst_KnownCases(t *testing.T) {
|
||||
// Grounds the softmaxFirst helper itself before trusting it to derive
|
||||
// pySoftmaxWant.
|
||||
tests := []struct {
|
||||
name string
|
||||
v []float64
|
||||
want float64
|
||||
}{
|
||||
{"uniform vector splits evenly", []float64{1, 1, 1}, 1.0 / 3.0},
|
||||
{"single element gets all the mass", []float64{5}, 1},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := softmaxFirst(tt.v)
|
||||
if math.Abs(got-tt.want) > 1e-9 {
|
||||
t.Errorf("softmaxFirst(%v) = %v, want %v", tt.v, got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestPySoftmaxWant_GroundTruth(t *testing.T) {
|
||||
// Independent re-derivation via math.Exp, not via the softmaxFirst
|
||||
// helper under test, per PLAN.md's rule to recompute cheap ground
|
||||
// truths in the unit test.
|
||||
v := pySoftmaxVector
|
||||
var sum float64
|
||||
for _, x := range v {
|
||||
sum += math.Exp(x)
|
||||
}
|
||||
want := math.Exp(v[0]) / sum
|
||||
wantRounded := math.Round(want*10000) / 10000
|
||||
|
||||
if pySoftmaxWant != wantRounded {
|
||||
t.Errorf("pySoftmaxWant = %v, independently recomputed = %v", pySoftmaxWant, wantRounded)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPySoftmaxTest_Eval(t *testing.T) {
|
||||
tc := pySoftmaxTest()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{"exact", "0.5585", 1},
|
||||
{"within tolerance", "0.5588", 1},
|
||||
{"prose wrapped", "The softmax probability is 0.5585.", 1},
|
||||
{"outside tolerance", "0.2", 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate() = %v, want %v (detail: %s)", got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestPyAsyncioGatherTrace_GroundTruth(t *testing.T) {
|
||||
// Runs the identical Python snippet embedded in pyAsyncioGatherTraceCode
|
||||
// via python3 and checks the print order is exactly B, A, C - the same
|
||||
// oracle mechanism as eval.PyRun, independent of the doc-comment
|
||||
// reasoning in python.go.
|
||||
if _, err := exec.LookPath("python3"); err != nil {
|
||||
t.Skip("python3 not found on PATH, skipping ground-truth cross-check")
|
||||
}
|
||||
|
||||
dir := t.TempDir()
|
||||
scriptPath := filepath.Join(dir, "gather.py")
|
||||
if err := os.WriteFile(scriptPath, []byte(pyAsyncioGatherTraceCode), 0o600); err != nil {
|
||||
t.Fatalf("write gather.py: %v", err)
|
||||
}
|
||||
|
||||
// #nosec G204 -- scriptPath is a fixed filename this test just wrote
|
||||
// into its own t.TempDir(); not external input.
|
||||
out, err := exec.Command("python3", scriptPath).Output() //nolint:gosec // see comment above
|
||||
if err != nil {
|
||||
t.Fatalf("running gather.py failed: %v", err)
|
||||
}
|
||||
|
||||
got := strings.TrimSpace(string(out))
|
||||
want := "B\nA\nC"
|
||||
if got != want {
|
||||
t.Errorf("python3 gather.py stdout = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPyAsyncioGatherTraceTest_Eval(t *testing.T) {
|
||||
tc := pyAsyncioGatherTraceTest()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{"exact correct", `["B","A","C"]`, 1},
|
||||
{"correct fenced json", "```json\n[\"B\", \"A\", \"C\"]\n```", 1},
|
||||
{"correct with prose wrapper", `The order is: ["B","A","C"]`, 1},
|
||||
{"wrong: source order, ignores delays", `["A","B","C"]`, 0},
|
||||
{"wrong: reversed", `["C","A","B"]`, 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate() = %v, want %v (detail: %s)", got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestPyMutableDefaultArg_GroundTruth runs the identical snippet via PyRun
|
||||
// with a passthrough harness, so the doc-comment claim in python.go is
|
||||
// cross-checked against the real python3 interpreter, not just reasoning.
|
||||
func TestPyMutableDefaultArg_GroundTruth(t *testing.T) {
|
||||
score := eval.PyRun(eval.PassthroughHarness, "[1, 2, 3]").Evaluate(
|
||||
context.Background(), "```python\n"+pyMutableDefaultArgCode+"\n```",
|
||||
)
|
||||
if score.Skipped {
|
||||
t.Skip("python3 not available, skipping ground-truth cross-check")
|
||||
}
|
||||
if score.Value != 1 {
|
||||
t.Errorf("running pyMutableDefaultArgCode via PyRun scored %v, want 1 (detail: %s)", score.Value, score.Detail)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPyMutableDefaultArgTest_Eval(t *testing.T) {
|
||||
tc := pyMutableDefaultArgTest()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{"exact correct", "[1, 2, 3]", 1},
|
||||
{"correct with surrounding whitespace", " [1, 2, 3] ", 1},
|
||||
{"correct with trailing newline", "[1, 2, 3]\n", 1},
|
||||
{"wrong: assumes a fresh list per call", "[3]", 0},
|
||||
{"wrong: only counts explicit-arg calls", "[1, 2]", 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate() = %v, want %v (detail: %s)", got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestPyDictComprehensionTrace_GroundTruth runs the identical snippet via
|
||||
// PyRun with a passthrough harness.
|
||||
func TestPyDictComprehensionTrace_GroundTruth(t *testing.T) {
|
||||
want := "[('apple', 5), ('banana', 6), ('cherry', 6), ('date', 4)]"
|
||||
score := eval.PyRun(eval.PassthroughHarness, want).Evaluate(
|
||||
context.Background(), "```python\n"+pyDictComprehensionTraceCode+"\n```",
|
||||
)
|
||||
if score.Skipped {
|
||||
t.Skip("python3 not available, skipping ground-truth cross-check")
|
||||
}
|
||||
if score.Value != 1 {
|
||||
t.Errorf("running pyDictComprehensionTraceCode via PyRun scored %v, want 1 (detail: %s)", score.Value, score.Detail)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPyDictComprehensionTraceTest_Eval(t *testing.T) {
|
||||
tc := pyDictComprehensionTraceTest()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{"exact correct", "[('apple', 5), ('banana', 6), ('cherry', 6), ('date', 4)]", 1},
|
||||
{"correct with surrounding whitespace", " [('apple', 5), ('banana', 6), ('cherry', 6), ('date', 4)] ", 1},
|
||||
{"correct uppercase-folded", "[('APPLE', 5), ('BANANA', 6), ('CHERRY', 6), ('DATE', 4)]", 1},
|
||||
{"wrong: includes fig, forgets the length filter", "[('apple', 5), ('banana', 6), ('cherry', 6), ('date', 4), ('fig', 3)]", 0},
|
||||
{"wrong: unsorted, dict insertion order", "[('apple', 5), ('banana', 6), ('cherry', 6), ('fig', 3), ('date', 4)]", 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate() = %v, want %v (detail: %s)", got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestPyGeneratorExhaustionTrace_GroundTruth runs the identical snippet via
|
||||
// PyRun with a passthrough harness.
|
||||
func TestPyGeneratorExhaustionTrace_GroundTruth(t *testing.T) {
|
||||
score := eval.PyRun(eval.PassthroughHarness, "[0, 1, 2] []").Evaluate(
|
||||
context.Background(), "```python\n"+pyGeneratorExhaustionTraceCode+"\n```",
|
||||
)
|
||||
if score.Skipped {
|
||||
t.Skip("python3 not available, skipping ground-truth cross-check")
|
||||
}
|
||||
if score.Value != 1 {
|
||||
t.Errorf("running pyGeneratorExhaustionTraceCode via PyRun scored %v, want 1 (detail: %s)", score.Value, score.Detail)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPyGeneratorExhaustionTraceTest_Eval(t *testing.T) {
|
||||
tc := pyGeneratorExhaustionTraceTest()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{"exact correct", "[0, 1, 2] []", 1},
|
||||
{"correct with surrounding whitespace", " [0, 1, 2] [] ", 1},
|
||||
{"correct with trailing newline", "[0, 1, 2] []\n", 1},
|
||||
{"wrong: assumes the generator restarts", "[0, 1, 2] [0, 1, 2]", 0},
|
||||
{"wrong: assumes an error is raised and printed", "StopIteration", 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate() = %v, want %v (detail: %s)", got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestPyPathlibRewriteTest_Eval(t *testing.T) {
|
||||
tc := pyPathlibRewriteTest()
|
||||
|
||||
goodRewrite := "```python\n" + `from pathlib import Path
|
||||
|
||||
|
||||
def find_configs(base_dir):
|
||||
return [str(p) for p in Path(base_dir).rglob("*.yaml")]
|
||||
` + "```"
|
||||
|
||||
goodRewriteAltImport := "```python\n" + `import pathlib
|
||||
|
||||
|
||||
def find_configs(base_dir):
|
||||
return [str(p) for p in pathlib.Path(base_dir).rglob("*.yaml")]
|
||||
` + "```"
|
||||
|
||||
stillUsesOsPath := "```python\n" + pyOsPathCode + "\n```"
|
||||
|
||||
missingRglob := "```python\n" + `from pathlib import Path
|
||||
|
||||
|
||||
def find_configs(base_dir):
|
||||
return [str(p) for p in Path(base_dir).glob("*.yaml")]
|
||||
` + "```"
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{"good rewrite: from pathlib import Path", goodRewrite, 1},
|
||||
{"good rewrite: import pathlib", goodRewriteAltImport, 1},
|
||||
{"wrong: unchanged os.path/os.walk code", stillUsesOsPath, 0},
|
||||
{"partial: uses pathlib but non-recursive glob, not rglob", missingRglob, 2.0 / 3.0}, // (1*1 + 2*0.5)/3: ContainsAny hits, ContainsAll("rglob","*.yaml") matches only "*.yaml"
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate() = %v, want %v (detail: %s)", got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestPyJSONTransformWant_GroundTruth(t *testing.T) {
|
||||
// Independent re-derivation of the expected per-department totals by
|
||||
// summing hardcoded literals matching pyJSONTransformRecords, rather
|
||||
// than trusting the hand sum in the doc comment.
|
||||
totals := map[string]int{
|
||||
"eng": 95000 + 105000 + 88000,
|
||||
"sales": 60000 + 58000,
|
||||
"ops": 72000,
|
||||
}
|
||||
want := "eng=288000\nops=72000\nsales=118000"
|
||||
if pyJSONTransformWant != want {
|
||||
t.Errorf("pyJSONTransformWant = %q, want %q", pyJSONTransformWant, want)
|
||||
}
|
||||
if totals["eng"] != 288000 || totals["sales"] != 118000 || totals["ops"] != 72000 {
|
||||
t.Errorf("independently recomputed totals = %v, want eng=288000 sales=118000 ops=72000", totals)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPyJSONTransformTest_Eval(t *testing.T) {
|
||||
tc := pyJSONTransformTest()
|
||||
|
||||
correctScript := "```python\n" + pyJSONTransformRecords + `
|
||||
|
||||
totals = {}
|
||||
for r in records:
|
||||
totals[r["dept"]] = totals.get(r["dept"], 0) + r["salary"]
|
||||
|
||||
for dept in sorted(totals):
|
||||
print(f"{dept}={totals[dept]}")
|
||||
` + "```"
|
||||
|
||||
wrongScript := "```python\nprint('eng=1')\nprint('ops=1')\nprint('sales=1')\n```"
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{"correct transform script", correctScript, 1},
|
||||
{"wrong totals", wrongScript, 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Skipped {
|
||||
t.Skip("python3 not available, skipping exec-evaluator assertion")
|
||||
}
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate() = %v, want %v (detail: %s)", got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestPyRegexLogExtractionWant_GroundTruth(t *testing.T) {
|
||||
// Independent re-derivation of the expected counts by scanning the raw
|
||||
// log text directly with Go's regexp, rather than trusting the hand
|
||||
// count in the doc comment.
|
||||
statusPattern := regexp.MustCompile(`"\s*(\d{3})\s+\d+`)
|
||||
matches := statusPattern.FindAllStringSubmatch(pyRegexLogLines, -1)
|
||||
if len(matches) != 7 {
|
||||
t.Fatalf("found %d status codes in pyRegexLogLines, want 7", len(matches))
|
||||
}
|
||||
counts := map[string]int{}
|
||||
for _, m := range matches {
|
||||
counts[m[1]]++
|
||||
}
|
||||
if counts["200"] != 3 || counts["404"] != 2 || counts["500"] != 2 {
|
||||
t.Errorf("independently recomputed counts = %v, want 200=3 404=2 500=2", counts)
|
||||
}
|
||||
want := "200=3\n404=2\n500=2"
|
||||
if pyRegexLogExtractionWant != want {
|
||||
t.Errorf("pyRegexLogExtractionWant = %q, want %q", pyRegexLogExtractionWant, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPyRegexLogExtractionTest_Eval(t *testing.T) {
|
||||
tc := pyRegexLogExtractionTest()
|
||||
|
||||
correctScript := "```python\nimport re\n\n" + pyRegexLogLines + `
|
||||
|
||||
pattern = re.compile(r'"\s*(\d{3})\s+\d+$')
|
||||
counts = {}
|
||||
for line in log_lines:
|
||||
m = pattern.search(line)
|
||||
if m:
|
||||
code = m.group(1)
|
||||
counts[code] = counts.get(code, 0) + 1
|
||||
|
||||
for code in sorted(counts):
|
||||
print(f"{code}={counts[code]}")
|
||||
` + "```"
|
||||
|
||||
wrongScript := "```python\nprint('200=1')\nprint('404=1')\nprint('500=1')\n```"
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{"correct regex extraction script", correctScript, 1},
|
||||
{"wrong counts", wrongScript, 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Skipped {
|
||||
t.Skip("python3 not available, skipping exec-evaluator assertion")
|
||||
}
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate() = %v, want %v (detail: %s)", got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestPyCosineTest_Eval(t *testing.T) {
|
||||
tc := pyCosineTest()
|
||||
|
||||
|
||||
@@ -10,6 +10,15 @@ import (
|
||||
|
||||
func registerTypeScriptTests(r *testkit.Registry) {
|
||||
r.Register(tsDebounceComposableTest())
|
||||
r.Register(tsDiscriminatedUnionAreaTest())
|
||||
r.Register(tsGenericUtilityTypeTest())
|
||||
r.Register(tsPromiseAllSettledTraceTest())
|
||||
r.Register(tsVueRefReactiveUnwrapTest())
|
||||
r.Register(tsComputedVsWatchTest())
|
||||
r.Register(tsESLintFlatConfigTest())
|
||||
r.Register(tsTsconfigStrictFlagTest())
|
||||
r.Register(tsArrayMethodChainTraceTest())
|
||||
r.Register(tsOptionalChainingNullishTraceTest())
|
||||
}
|
||||
|
||||
// tsAnyEscapeHatchPattern matches TypeScript's `any` used as a type: a type
|
||||
@@ -66,3 +75,378 @@ Requirements:
|
||||
Eval: evaluator,
|
||||
}
|
||||
}
|
||||
|
||||
// tsDiscriminatedUnionAreaCode is the inline snippet for
|
||||
// tsDiscriminatedUnionAreaTest: a discriminated union narrowed with a
|
||||
// switch on its "kind" tag.
|
||||
const tsDiscriminatedUnionAreaCode = `type Shape =
|
||||
| { kind: "circle"; radius: number }
|
||||
| { kind: "rectangle"; width: number; height: number };
|
||||
|
||||
function area(s: Shape): number {
|
||||
switch (s.kind) {
|
||||
case "circle":
|
||||
return Math.PI * s.radius ** 2;
|
||||
case "rectangle":
|
||||
return s.width * s.height;
|
||||
}
|
||||
}
|
||||
|
||||
const shapes: Shape[] = [
|
||||
{ kind: "rectangle", width: 4, height: 5 },
|
||||
{ kind: "circle", radius: 3 },
|
||||
];
|
||||
|
||||
const total = shapes.map(area).reduce((a, b) => a + b, 0);
|
||||
console.log(Math.round(total * 100) / 100);`
|
||||
|
||||
// tsDiscriminatedUnionAreaTest: trace the exact output of code that narrows
|
||||
// a discriminated union in a switch and sums the per-branch results.
|
||||
//
|
||||
// ground truth: switching on the "kind" tag narrows each branch to its
|
||||
// specific member of the Shape union, so s.radius is only accessible in the
|
||||
// "circle" case and s.width/s.height only in "rectangle". area({kind:
|
||||
// "rectangle", width: 4, height: 5}) = 4*5 = 20. area({kind: "circle",
|
||||
// radius: 3}) = pi*3^2 = 28.274333882308138. Their sum, rounded to 2
|
||||
// decimal places (Math.round(total*100)/100), is 48.27. typescript_test.go
|
||||
// cross-checks this by running the equivalent JavaScript (the same
|
||||
// arithmetic, with TypeScript's type annotations erased, since switch-based
|
||||
// narrowing is a compile-time-only construct with no runtime effect on
|
||||
// these values) via `node`, guarded by node's presence.
|
||||
func tsDiscriminatedUnionAreaTest() testkit.Test {
|
||||
prompt := `Here is a TypeScript program:
|
||||
|
||||
` + "```ts" + `
|
||||
` + tsDiscriminatedUnionAreaCode + `
|
||||
` + "```" + `
|
||||
|
||||
What number does this program print? Respond with only the number, nothing
|
||||
else.`
|
||||
|
||||
return testkit.Test{
|
||||
ID: "ts-discriminated-union-narrowing",
|
||||
Category: "programming",
|
||||
Subcategory: "typescript",
|
||||
Description: "Trace the exact output of a discriminated union narrowed with a switch, summed across variants.",
|
||||
Prompt: prompt,
|
||||
Eval: eval.Numeric(eval.ExtractLastNumber[float64], 48.27, 0.005),
|
||||
}
|
||||
}
|
||||
|
||||
// tsGenericUtilityTypeTest: author a recursive generic utility type
|
||||
// (DeepPartial<T>) using a mapped type and a conditional type, without any.
|
||||
func tsGenericUtilityTypeTest() testkit.Test {
|
||||
prompt := `Write a generic TypeScript utility type ` + "`DeepPartial<T>`" + ` that
|
||||
makes T and all of its nested object properties optional, recursively (so
|
||||
given an interface with a nested object property, that nested object's own
|
||||
properties are also optional). Do not use the "any" type anywhere. Respond
|
||||
with only the type definition.`
|
||||
|
||||
evaluator := eval.All(
|
||||
eval.W(eval.ContainsAll("DeepPartial<T>", "keyof T"), 2),
|
||||
eval.W(eval.ContainsAny("DeepPartial<T[K]>"), 1),
|
||||
eval.W(noAnyEscapeHatch(), 1),
|
||||
)
|
||||
|
||||
return testkit.Test{
|
||||
ID: "ts-generic-utility-type",
|
||||
Category: "programming",
|
||||
Subcategory: "typescript",
|
||||
Description: "Author a recursive generic DeepPartial<T> utility type using a mapped and conditional type, without any.",
|
||||
System: terseCodeOnly,
|
||||
Prompt: prompt,
|
||||
Eval: evaluator,
|
||||
}
|
||||
}
|
||||
|
||||
// tsPromiseAllSettledTraceCode is the inline snippet for
|
||||
// tsPromiseAllSettledTraceTest.
|
||||
const tsPromiseAllSettledTraceCode = `async function main() {
|
||||
const results = await Promise.allSettled([
|
||||
Promise.resolve(1),
|
||||
Promise.reject("fail"),
|
||||
Promise.resolve(3),
|
||||
]);
|
||||
console.log(results.map((r) => r.status));
|
||||
}
|
||||
|
||||
main();`
|
||||
|
||||
// tsPromiseAllSettledTraceTest: trace the exact "status" values
|
||||
// Promise.allSettled reports for a mix of resolved and rejected promises.
|
||||
//
|
||||
// ground truth: unlike Promise.all, Promise.allSettled never short-circuits
|
||||
// on a rejection - it waits for every input promise to settle and reports
|
||||
// each one's outcome as {status: "fulfilled", value} or {status: "rejected",
|
||||
// reason}, in the same order as the input array. The three inputs settle as
|
||||
// fulfilled, rejected, fulfilled respectively (the middle one rejects with
|
||||
// "fail"), so results.map(r => r.status) is
|
||||
// ["fulfilled", "rejected", "fulfilled"]. typescript_test.go cross-checks
|
||||
// this by running the equivalent JavaScript via `node`.
|
||||
func tsPromiseAllSettledTraceTest() testkit.Test {
|
||||
prompt := `Here is a TypeScript program:
|
||||
|
||||
` + "```ts" + `
|
||||
` + tsPromiseAllSettledTraceCode + `
|
||||
` + "```" + `
|
||||
|
||||
What array of statuses does this program log? Respond with only a JSON
|
||||
array of the three status strings in order, for example
|
||||
["fulfilled","fulfilled","fulfilled"]`
|
||||
|
||||
return testkit.Test{
|
||||
ID: "ts-promise-allsettled-trace",
|
||||
Category: "programming",
|
||||
Subcategory: "typescript",
|
||||
Description: "Trace the exact per-promise status array Promise.allSettled reports for a mixed resolve/reject batch.",
|
||||
System: terseCodeOnly,
|
||||
Prompt: prompt,
|
||||
Eval: eval.JSONStringArrayEquals([]string{"fulfilled", "rejected", "fulfilled"}),
|
||||
}
|
||||
}
|
||||
|
||||
// tsVueRefReactiveUnwrapTest: decide whether Vue 3's ref-auto-unwrapping
|
||||
// applies to a property access on a reactive object versus an element
|
||||
// access on a reactive array.
|
||||
//
|
||||
// ground truth: Vue 3's reactive() unwraps a nested ref when it is accessed
|
||||
// as a property of the reactive object (state.count reads/writes as a plain
|
||||
// number, not a Ref), but explicitly does NOT unwrap a ref when it is
|
||||
// accessed as an element of a reactive array or a native collection like
|
||||
// Map (list[0] stays a Ref object). Verified empirically for this test (not
|
||||
// from memory): `reactive({count: ref(0)}).count` has typeof "number" and
|
||||
// isRef() false, while `reactive([ref(0)])[0]` has isRef() true, using
|
||||
// [email protected]'s actual reactivity runtime.
|
||||
func tsVueRefReactiveUnwrapTest() testkit.Test {
|
||||
prompt := `In Vue 3's Composition API:
|
||||
|
||||
` + "```ts" + `
|
||||
const state = reactive({ count: ref(0) });
|
||||
const list = reactive([ref(0)]);
|
||||
` + "```" + `
|
||||
|
||||
Does accessing ` + "`state.count`" + ` (a ref nested as a property of a
|
||||
reactive object) auto-unwrap to a plain number? Does accessing
|
||||
` + "`list[0]`" + ` (a ref nested as an element of a reactive array)
|
||||
auto-unwrap to a plain number? Respond with only a JSON object:
|
||||
{"object_property_unwraps": true|false, "array_element_unwraps": true|false}`
|
||||
|
||||
evaluator := eval.All(
|
||||
eval.W(eval.JSONField("object_property_unwraps", true), 1),
|
||||
eval.W(eval.JSONField("array_element_unwraps", false), 1),
|
||||
)
|
||||
|
||||
return testkit.Test{
|
||||
ID: "ts-vue-ref-reactive-unwrap",
|
||||
Category: "programming",
|
||||
Subcategory: "typescript",
|
||||
Description: "Decide whether Vue 3 ref auto-unwrapping applies to a reactive object property versus a reactive array element.",
|
||||
System: terseCodeOnly,
|
||||
Prompt: prompt,
|
||||
Eval: evaluator,
|
||||
}
|
||||
}
|
||||
|
||||
// tsComputedVsWatchChoice is the fixed vocabulary tsComputedVsWatchTest
|
||||
// allows for each scenario's answer.
|
||||
const (
|
||||
tsComputedVsWatchChoiceComputed = "computed"
|
||||
tsComputedVsWatchChoiceWatch = "watch"
|
||||
)
|
||||
|
||||
// tsComputedVsWatchTest: choose computed() versus watch() for two Vue 3
|
||||
// Composition API scenarios, one pure derivation and one side effect.
|
||||
//
|
||||
// ground truth: per Vue's own guidance, computed() is for deriving a value
|
||||
// from reactive state with no side effects - Vue's docs recommend it
|
||||
// "whenever possible" for exactly this - while watch() is for running a
|
||||
// side effect (an API call, a DOM mutation, a storage write, logging) in
|
||||
// reaction to a state change. Scenario A only derives a formatted string
|
||||
// for template display (no side effect) -> computed. Scenario B persists a
|
||||
// value to localStorage whenever it changes (a side effect) -> watch.
|
||||
func tsComputedVsWatchTest() testkit.Test {
|
||||
prompt := `Two Vue 3 Composition API scenarios:
|
||||
|
||||
Scenario A: derive a formatted currency string from a "price" ref, used
|
||||
only for display in the template. Deriving it has no side effects.
|
||||
|
||||
Scenario B: whenever a "settings" ref changes, persist the new value to
|
||||
localStorage. This has a side effect (writing to storage).
|
||||
|
||||
For each scenario, should you use computed() or watch()? Respond with only
|
||||
a JSON object: {"scenario_a": "computed"|"watch", "scenario_b": "computed"|"watch"}`
|
||||
|
||||
evaluator := eval.All(
|
||||
eval.W(eval.JSONField("scenario_a", tsComputedVsWatchChoiceComputed), 1),
|
||||
eval.W(eval.JSONField("scenario_b", tsComputedVsWatchChoiceWatch), 1),
|
||||
)
|
||||
|
||||
return testkit.Test{
|
||||
ID: "ts-computed-vs-watch",
|
||||
Category: "programming",
|
||||
Subcategory: "typescript",
|
||||
Description: "Choose computed() versus watch() for a pure-derivation scenario and a side-effect scenario.",
|
||||
System: terseCodeOnly,
|
||||
Prompt: prompt,
|
||||
Eval: evaluator,
|
||||
}
|
||||
}
|
||||
|
||||
// tsOldEslintrcConfig is the inline legacy config tsESLintFlatConfigTest
|
||||
// asks the model to migrate.
|
||||
const tsOldEslintrcConfig = `{
|
||||
"env": { "browser": true, "es2021": true },
|
||||
"extends": ["eslint:recommended"],
|
||||
"parserOptions": { "ecmaVersion": "latest", "sourceType": "module" },
|
||||
"rules": { "semi": "error", "prefer-const": "error" }
|
||||
}`
|
||||
|
||||
// tsESLintFlatConfigTest: migrate a legacy .eslintrc.json to ESLint v9's
|
||||
// flat config format.
|
||||
func tsESLintFlatConfigTest() testkit.Test {
|
||||
prompt := `Here is a legacy .eslintrc.json:
|
||||
|
||||
` + "```json" + `
|
||||
` + tsOldEslintrcConfig + `
|
||||
` + "```" + `
|
||||
|
||||
ESLint v9 uses the flat config format (eslint.config.js) by default; the
|
||||
legacy .eslintrc format is no longer picked up automatically. Rewrite this
|
||||
configuration as an eslint.config.js flat config, preserving the same rules
|
||||
(semi: error, prefer-const: error) and the ecmaVersion/sourceType settings
|
||||
via languageOptions. Respond with only the eslint.config.js file content.`
|
||||
|
||||
evaluator := eval.All(
|
||||
eval.W(eval.ContainsAll("export default"), 2),
|
||||
eval.W(eval.ContainsAny("defineConfig(", "languageOptions"), 1),
|
||||
)
|
||||
|
||||
return testkit.Test{
|
||||
ID: "ts-eslint-flat-config",
|
||||
Category: "programming",
|
||||
Subcategory: "typescript",
|
||||
Description: "Migrate a legacy .eslintrc.json to ESLint v9's flat eslint.config.js format.",
|
||||
System: terseCodeOnly,
|
||||
Prompt: prompt,
|
||||
Eval: evaluator,
|
||||
}
|
||||
}
|
||||
|
||||
// tsImplicitAnyBugCode is the inline snippet for tsTsconfigStrictFlagTest:
|
||||
// a function whose parameter has no type annotation.
|
||||
const tsImplicitAnyBugCode = `function greet(name) {
|
||||
return "Hello, " + name.toUpperCase();
|
||||
}`
|
||||
|
||||
// tsTsconfigStrictFlagTest: identify which tsconfig strict-family compiler
|
||||
// flag would catch an implicit-any parameter at compile time.
|
||||
//
|
||||
// ground truth: `name` has no type annotation and TypeScript cannot infer
|
||||
// one from usage, so under the "noImplicitAny" compiler option (part of the
|
||||
// "strict" family, also implied by "strict": true) the compiler reports
|
||||
// "Parameter 'name' implicitly has an 'any' type." Verified empirically for
|
||||
// this test (not from memory): compiling this exact snippet with
|
||||
// typescript@5's tsc --noImplicitAny reports that error (exit code 2);
|
||||
// without the flag it compiles cleanly (exit code 0).
|
||||
func tsTsconfigStrictFlagTest() testkit.Test {
|
||||
prompt := `Here is a TypeScript function:
|
||||
|
||||
` + "```ts" + `
|
||||
` + tsImplicitAnyBugCode + `
|
||||
` + "```" + `
|
||||
|
||||
This compiles without error under TypeScript's default (non-strict)
|
||||
settings, but name has no type annotation and none can be inferred from its
|
||||
usage. Which single tsconfig strict-family compiler option, when enabled,
|
||||
makes the compiler reject this with an error? Respond with only a JSON
|
||||
object: {"flag": "<the exact tsconfig option name>"}`
|
||||
|
||||
return testkit.Test{
|
||||
ID: "ts-tsconfig-strict-flag",
|
||||
Category: "programming",
|
||||
Subcategory: "typescript",
|
||||
Description: "Identify the tsconfig strict-family flag that catches an implicit-any function parameter.",
|
||||
System: terseCodeOnly,
|
||||
Prompt: prompt,
|
||||
Eval: eval.JSONField("flag", "noImplicitAny"),
|
||||
}
|
||||
}
|
||||
|
||||
// tsArrayMethodChainCode is the inline snippet for
|
||||
// tsArrayMethodChainTraceTest.
|
||||
const tsArrayMethodChainCode = `const nums = [3, 8, 1, 9, 4, 6, 2, 7];
|
||||
const result = nums
|
||||
.filter((n) => n % 2 === 0)
|
||||
.map((n) => n * n)
|
||||
.reduce((acc, n) => acc + n, 0);
|
||||
console.log(result);`
|
||||
|
||||
// tsArrayMethodChainTraceTest: trace the exact numeric result of a
|
||||
// filter/map/reduce chain over an inline array.
|
||||
//
|
||||
// ground truth: filter keeps the even numbers, in original order: 8, 4, 6,
|
||||
// 2. map squares each: 64, 16, 36, 4. reduce sums them with an initial
|
||||
// accumulator of 0: 64+16+36+4 = 120. typescript_test.go cross-checks this
|
||||
// by running the equivalent JavaScript via `node`.
|
||||
func tsArrayMethodChainTraceTest() testkit.Test {
|
||||
prompt := `Here is a TypeScript program:
|
||||
|
||||
` + "```ts" + `
|
||||
` + tsArrayMethodChainCode + `
|
||||
` + "```" + `
|
||||
|
||||
What number does this program print? Respond with only the number, nothing
|
||||
else.`
|
||||
|
||||
return testkit.Test{
|
||||
ID: "ts-array-method-chain",
|
||||
Category: "programming",
|
||||
Subcategory: "typescript",
|
||||
Description: "Trace the exact output of a filter/map/reduce chain over an inline number array.",
|
||||
Prompt: prompt,
|
||||
Eval: eval.Numeric(eval.ExtractLastNumber[int], 120, 0),
|
||||
}
|
||||
}
|
||||
|
||||
// tsOptionalChainingNullishCode is the inline snippet for
|
||||
// tsOptionalChainingNullishTraceTest.
|
||||
const tsOptionalChainingNullishCode = `type Config = { server?: { host?: string; port?: number } };
|
||||
|
||||
function describe(c: Config): string {
|
||||
const host = c.server?.host ?? "localhost";
|
||||
const port = c.server?.port ?? 8080;
|
||||
return ` + "`${host}:${port}`" + `;
|
||||
}
|
||||
|
||||
console.log(describe({ server: { port: 0 } }));`
|
||||
|
||||
// tsOptionalChainingNullishTraceTest: trace the exact output of code
|
||||
// combining optional chaining with nullish coalescing, on an input whose
|
||||
// present value is falsy but not nullish.
|
||||
//
|
||||
// ground truth: c.server?.host is undefined (server has no host property),
|
||||
// so ?? falls back to "localhost". c.server?.port is 0 - present and not
|
||||
// null/undefined - so ?? does NOT fall back (unlike ||, ?? only treats
|
||||
// null/undefined as "missing", not other falsy values like 0, "", or
|
||||
// false), leaving port as 0. The template literal produces "localhost:0".
|
||||
// typescript_test.go cross-checks this by running the equivalent
|
||||
// JavaScript via `node`.
|
||||
func tsOptionalChainingNullishTraceTest() testkit.Test {
|
||||
prompt := `Here is a TypeScript program:
|
||||
|
||||
` + "```ts" + `
|
||||
` + tsOptionalChainingNullishCode + `
|
||||
` + "```" + `
|
||||
|
||||
What does this program print when run? Respond with only the exact text
|
||||
printed, nothing else.`
|
||||
|
||||
return testkit.Test{
|
||||
ID: "ts-optional-chaining-nullish",
|
||||
Category: "programming",
|
||||
Subcategory: "typescript",
|
||||
Description: "Trace the exact output of optional chaining combined with nullish coalescing on a falsy-but-present value.",
|
||||
Prompt: prompt,
|
||||
Eval: eval.Equals("localhost:0"),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2,9 +2,40 @@ package tests
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// runNodeJS writes js (plain JavaScript - TypeScript syntax already erased
|
||||
// by the caller) to a temp file and runs it with `node`, returning trimmed
|
||||
// stdout. It skips the calling test (does not fail it) when node is not on
|
||||
// PATH, mirroring the toolchain-detection contract eval.PyRun/GoRun/CRun
|
||||
// use, since node is a ground-truth verification tool here, not a catalog
|
||||
// exec.Evaluator (typescript.go owns no eval.Evaluator that shells out).
|
||||
func runNodeJS(t *testing.T, js string) string {
|
||||
t.Helper()
|
||||
if _, err := exec.LookPath("node"); err != nil {
|
||||
t.Skip("node not found on PATH, skipping ground-truth cross-check")
|
||||
}
|
||||
|
||||
dir := t.TempDir()
|
||||
scriptPath := filepath.Join(dir, "check.js")
|
||||
if err := os.WriteFile(scriptPath, []byte(js), 0o600); err != nil {
|
||||
t.Fatalf("write check.js: %v", err)
|
||||
}
|
||||
|
||||
// #nosec G204 -- scriptPath is a fixed filename this test just wrote
|
||||
// into its own t.TempDir(); not external input.
|
||||
out, err := exec.Command("node", scriptPath).Output() //nolint:gosec // see comment above
|
||||
if err != nil {
|
||||
t.Fatalf("running check.js failed: %v", err)
|
||||
}
|
||||
return strings.TrimSpace(string(out))
|
||||
}
|
||||
|
||||
const goodDebounceComposable = "```ts\n" + `import { ref, watch, onScopeDispose, type Ref } from 'vue'
|
||||
|
||||
export function useDebouncedRef<T>(source: Ref<T>, ms: number): Ref<T> {
|
||||
@@ -54,6 +85,334 @@ func TestNoAnyEscapeHatch(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// tsDiscriminatedUnionAreaJS is a plain-JavaScript mirror of
|
||||
// tsDiscriminatedUnionAreaCode (TypeScript's type annotations erased, since
|
||||
// they have no runtime effect), used to cross-check the ground truth with
|
||||
// `node` rather than trusting the doc comment's arithmetic alone.
|
||||
const tsDiscriminatedUnionAreaJS = `function area(s) {
|
||||
switch (s.kind) {
|
||||
case "circle":
|
||||
return Math.PI * s.radius ** 2;
|
||||
case "rectangle":
|
||||
return s.width * s.height;
|
||||
}
|
||||
}
|
||||
|
||||
const shapes = [
|
||||
{ kind: "rectangle", width: 4, height: 5 },
|
||||
{ kind: "circle", radius: 3 },
|
||||
];
|
||||
|
||||
const total = shapes.map(area).reduce((a, b) => a + b, 0);
|
||||
console.log(Math.round(total * 100) / 100);`
|
||||
|
||||
func TestTsDiscriminatedUnionArea_GroundTruth(t *testing.T) {
|
||||
got := runNodeJS(t, tsDiscriminatedUnionAreaJS)
|
||||
if want := "48.27"; got != want {
|
||||
t.Errorf("node output = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTsDiscriminatedUnionAreaTest_Eval(t *testing.T) {
|
||||
tc := tsDiscriminatedUnionAreaTest()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{"exact", "48.27", 1},
|
||||
{"within tolerance", "48.272", 1},
|
||||
{"prose wrapped", "The total area is 48.27.", 1},
|
||||
{"wrong: circle area only", "28.27", 0},
|
||||
{"wrong: unrounded-ish but off", "48.3", 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate() = %v, want %v (detail: %s)", got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestTsGenericUtilityTypeTest_Eval(t *testing.T) {
|
||||
tc := tsGenericUtilityTypeTest()
|
||||
|
||||
goodDeepPartial := "```ts\n" + `type DeepPartial<T> = {
|
||||
[K in keyof T]?: T[K] extends object ? DeepPartial<T[K]> : T[K];
|
||||
};
|
||||
` + "```"
|
||||
|
||||
missingRecursion := "```ts\n" + `type DeepPartial<T> = {
|
||||
[K in keyof T]?: T[K];
|
||||
};
|
||||
` + "```"
|
||||
|
||||
usesAny := "```ts\n" + `type DeepPartial<T> = {
|
||||
[K in keyof T]?: any;
|
||||
};
|
||||
` + "```"
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{"correct recursive DeepPartial", goodDeepPartial, 1},
|
||||
{"wrong: shallow only, no recursion into nested objects", missingRecursion, 0.75}, // (2*1 + 1*0 + 1*1)/4: mapped type ok, no DeepPartial<T[K]> reference, no any
|
||||
{"wrong: uses any, defeating the point", usesAny, 0.5}, // (2*1 + 1*0 + 1*0)/4: mapped type ok, no recursion, uses any
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate() = %v, want %v (detail: %s)", got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// tsPromiseAllSettledTraceJS is a plain-JavaScript mirror of
|
||||
// tsPromiseAllSettledTraceCode.
|
||||
const tsPromiseAllSettledTraceJS = `async function main() {
|
||||
const results = await Promise.allSettled([
|
||||
Promise.resolve(1),
|
||||
Promise.reject("fail"),
|
||||
Promise.resolve(3),
|
||||
]);
|
||||
console.log(JSON.stringify(results.map((r) => r.status)));
|
||||
}
|
||||
|
||||
main();`
|
||||
|
||||
func TestTsPromiseAllSettledTrace_GroundTruth(t *testing.T) {
|
||||
got := runNodeJS(t, tsPromiseAllSettledTraceJS)
|
||||
if want := `["fulfilled","rejected","fulfilled"]`; got != want {
|
||||
t.Errorf("node output = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTsPromiseAllSettledTraceTest_Eval(t *testing.T) {
|
||||
tc := tsPromiseAllSettledTraceTest()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{"exact correct", `["fulfilled","rejected","fulfilled"]`, 1},
|
||||
{"correct fenced json", "```json\n[\"fulfilled\", \"rejected\", \"fulfilled\"]\n```", 1},
|
||||
{"correct with prose wrapper", `The statuses are: ["fulfilled","rejected","fulfilled"]`, 1},
|
||||
{"wrong: assumes allSettled short-circuits like Promise.all", `["fulfilled"]`, 0},
|
||||
{"wrong: assumes rejection order shifts", `["rejected","fulfilled","fulfilled"]`, 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate() = %v, want %v (detail: %s)", got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestTsVueRefReactiveUnwrapTest_Eval(t *testing.T) {
|
||||
tc := tsVueRefReactiveUnwrapTest()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{"exact correct", `{"object_property_unwraps": true, "array_element_unwraps": false}`, 1},
|
||||
{"correct fenced json", "```json\n{\"object_property_unwraps\": true, \"array_element_unwraps\": false}\n```", 1},
|
||||
{"correct with prose wrapper", `Answer: {"object_property_unwraps": true, "array_element_unwraps": false}`, 1},
|
||||
{"wrong: assumes both unwrap", `{"object_property_unwraps": true, "array_element_unwraps": true}`, 0.5},
|
||||
{"wrong: assumes neither unwraps", `{"object_property_unwraps": false, "array_element_unwraps": false}`, 0.5},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate() = %v, want %v (detail: %s)", got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestTsComputedVsWatchTest_Eval(t *testing.T) {
|
||||
tc := tsComputedVsWatchTest()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{"exact correct", `{"scenario_a": "computed", "scenario_b": "watch"}`, 1},
|
||||
{"correct fenced json", "```json\n{\"scenario_a\": \"computed\", \"scenario_b\": \"watch\"}\n```", 1},
|
||||
{"correct with prose wrapper", `Answer: {"scenario_a": "computed", "scenario_b": "watch"}`, 1},
|
||||
{"wrong: both watch", `{"scenario_a": "watch", "scenario_b": "watch"}`, 0.5},
|
||||
{"wrong: swapped", `{"scenario_a": "watch", "scenario_b": "computed"}`, 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate() = %v, want %v (detail: %s)", got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestTsESLintFlatConfigTest_Eval(t *testing.T) {
|
||||
tc := tsESLintFlatConfigTest()
|
||||
|
||||
goodFlatConfig := "```js\n" + `export default [
|
||||
{
|
||||
languageOptions: { ecmaVersion: "latest", sourceType: "module" },
|
||||
rules: { semi: "error", "prefer-const": "error" },
|
||||
},
|
||||
];
|
||||
` + "```"
|
||||
|
||||
goodFlatConfigWithHelper := "```js\n" + `import { defineConfig } from "eslint/config";
|
||||
|
||||
export default defineConfig([
|
||||
{
|
||||
languageOptions: { ecmaVersion: "latest", sourceType: "module" },
|
||||
rules: { semi: "error", "prefer-const": "error" },
|
||||
},
|
||||
]);
|
||||
` + "```"
|
||||
|
||||
stillLegacyFormat := "```json\n" + tsOldEslintrcConfig + "\n```"
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{"good: plain array export default", goodFlatConfig, 1},
|
||||
{"good: defineConfig helper", goodFlatConfigWithHelper, 1},
|
||||
{"wrong: left as legacy .eslintrc JSON", stillLegacyFormat, 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate() = %v, want %v (detail: %s)", got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestTsTsconfigStrictFlagTest_Eval(t *testing.T) {
|
||||
tc := tsTsconfigStrictFlagTest()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{"exact correct", `{"flag": "noImplicitAny"}`, 1},
|
||||
{"correct fenced json", "```json\n{\"flag\": \"noImplicitAny\"}\n```", 1},
|
||||
{"correct with prose wrapper", `The answer is: {"flag": "noImplicitAny"}`, 1},
|
||||
{"wrong: unrelated strict flag", `{"flag": "strictNullChecks"}`, 0},
|
||||
{"wrong: not a real tsconfig flag", `{"flag": "strict-types"}`, 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate() = %v, want %v (detail: %s)", got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// tsArrayMethodChainJS is a plain-JavaScript mirror of
|
||||
// tsArrayMethodChainCode.
|
||||
const tsArrayMethodChainJS = `const nums = [3, 8, 1, 9, 4, 6, 2, 7];
|
||||
const result = nums
|
||||
.filter((n) => n % 2 === 0)
|
||||
.map((n) => n * n)
|
||||
.reduce((acc, n) => acc + n, 0);
|
||||
console.log(result);`
|
||||
|
||||
func TestTsArrayMethodChain_GroundTruth(t *testing.T) {
|
||||
got := runNodeJS(t, tsArrayMethodChainJS)
|
||||
if want := "120"; got != want {
|
||||
t.Errorf("node output = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTsArrayMethodChainTraceTest_Eval(t *testing.T) {
|
||||
tc := tsArrayMethodChainTraceTest()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{"exact correct", "120", 1},
|
||||
{"correct with prose", "The result is 120.", 1},
|
||||
{"correct with label", "Output: 120", 1},
|
||||
{"wrong: filters odd instead of even", "30", 0},
|
||||
{"wrong: forgets to square before summing", "20", 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate() = %v, want %v (detail: %s)", got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// tsOptionalChainingNullishJS is a plain-JavaScript mirror of
|
||||
// tsOptionalChainingNullishCode (TypeScript's type annotations erased).
|
||||
const tsOptionalChainingNullishJS = `function describe(c) {
|
||||
const host = c.server?.host ?? "localhost";
|
||||
const port = c.server?.port ?? 8080;
|
||||
return ` + "`${host}:${port}`" + `;
|
||||
}
|
||||
|
||||
console.log(describe({ server: { port: 0 } }));`
|
||||
|
||||
func TestTsOptionalChainingNullishTrace_GroundTruth(t *testing.T) {
|
||||
got := runNodeJS(t, tsOptionalChainingNullishJS)
|
||||
if want := "localhost:0"; got != want {
|
||||
t.Errorf("node output = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTsOptionalChainingNullishTraceTest_Eval(t *testing.T) {
|
||||
tc := tsOptionalChainingNullishTraceTest()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{"exact correct", "localhost:0", 1},
|
||||
{"correct with surrounding whitespace", " localhost:0 ", 1},
|
||||
{"correct case-insensitive", "LOCALHOST:0", 1},
|
||||
{"wrong: treats ?? like || and replaces the falsy 0", "localhost:8080", 0},
|
||||
{"wrong: assumes host is present", "example.com:0", 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate() = %v, want %v (detail: %s)", got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestTsDebounceComposableTest_Eval(t *testing.T) {
|
||||
tc := tsDebounceComposableTest()
|
||||
|
||||
|
||||
Reference in New Issue
Block a user