mirror of
https://github.com/lukaszraczylo/llm-testbench.git
synced 2026-10-14 00:57:45 +00:00
Update
This commit is contained in:
@@ -0,0 +1,360 @@
|
||||
package tests
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestHardFloatBinaddTest_Eval(t *testing.T) {
|
||||
tc := hardFloatBinaddTest()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{"correct", `{"equal":false}`, 1},
|
||||
{"correct fenced with prose", "The answer is:\n```json\n{\"equal\":false}\n```", 1},
|
||||
{"wrong: claims equal", `{"equal":true}`, 0},
|
||||
{"wrong: non-boolean", `{"equal":"yes"}`, 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestHardNullishOverOrTest_Eval(t *testing.T) {
|
||||
tc := hardNullishOverOrTest()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{"correct", `{"value":0}`, 1},
|
||||
{"correct as string", `{"value":"0"}`, 1},
|
||||
{"wrong: OR semantics", `{"value":42}`, 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestHardGoroutineRaceTest_Eval(t *testing.T) {
|
||||
tc := hardGoroutineRaceTest()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{"correct", `{"guaranteed":false}`, 1},
|
||||
{"correct fenced", "```json\n{\"guaranteed\":false}\n```", 1},
|
||||
{"wrong: claims guaranteed", `{"guaranteed":true}`, 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestHardDeferNamedReturnTest_Eval(t *testing.T) {
|
||||
tc := hardDeferNamedReturnTest()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{"correct", "99", 1},
|
||||
{"correct prose-wrapped", "The function returns 99.", 1},
|
||||
{"wrong: returns statement value", "7", 0},
|
||||
{"wrong: pre-defer value", "5", 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestHardIntDivisionTest_Eval(t *testing.T) {
|
||||
tc := hardIntDivisionTest()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{"correct", "3", 1},
|
||||
{"correct as float", "3.0", 1},
|
||||
{"correct prose-wrapped", "It is 3.", 1},
|
||||
{"wrong: assumes float division", "3.5", 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestHardRegexGreedyTest_Eval(t *testing.T) {
|
||||
tc := hardRegexGreedyTest()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{"correct", `["1234","5"]`, 1},
|
||||
{"correct fenced", "```json\n[\"1234\",\"5\"]\n```", 1},
|
||||
{"wrong: symmetric split", `["12","345"]`, 0},
|
||||
{"wrong: reversed", `["5","1234"]`, 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestHardPromiseAllSettledTest_Eval(t *testing.T) {
|
||||
tc := hardPromiseAllSettledTest()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{"correct", `["fulfilled","rejected"]`, 1},
|
||||
{"correct fenced", "```json\n[\"fulfilled\",\"rejected\"]\n```", 1},
|
||||
{"wrong: reordered", `["rejected","fulfilled"]`, 0},
|
||||
{"wrong: both fulfilled", `["fulfilled","fulfilled"]`, 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestHardContextTimeoutTest_Eval(t *testing.T) {
|
||||
tc := hardContextTimeoutTest()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{"correct", `{"err":"context deadline exceeded"}`, 1},
|
||||
{"correct fenced", "```json\n{\"err\":\"context deadline exceeded\"}\n```", 1},
|
||||
{"wrong: canceled sentinel", `{"err":"context canceled"}`, 0},
|
||||
{"wrong: vague", `{"err":"some error"}`, 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestHardJSONUnmarshalNullTest_Eval(t *testing.T) {
|
||||
tc := hardJSONUnmarshalNullTest()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{"correct", `{"returns_error":false}`, 1},
|
||||
{"correct fenced", "```json\n{\"returns_error\":false}\n```", 1},
|
||||
{"wrong: claims error", `{"returns_error":true}`, 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestHardVarShadowTest_Eval(t *testing.T) {
|
||||
tc := hardVarShadowTest()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{"correct", `{"outer_unchanged":true}`, 1},
|
||||
{"correct fenced", "```json\n{\"outer_unchanged\":true}\n```", 1},
|
||||
{"wrong: claims outer changed", `{"outer_unchanged":false}`, 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestHardPyVersionSortTest_Eval exercises the PyRun evaluator with a
|
||||
// hand-written correct script and a lexicographic (wrong) script.
|
||||
func TestHardPyVersionSortTest_Eval(t *testing.T) {
|
||||
tc := hardPyVersionSortTest()
|
||||
|
||||
correctScript := "```python\n" + `versions = [
|
||||
"1.9.0",
|
||||
"1.10.0",
|
||||
"1.2.0",
|
||||
"2.0.0",
|
||||
"0.9.9",
|
||||
]
|
||||
|
||||
def key(v):
|
||||
return tuple(int(p) for p in v.split("."))
|
||||
|
||||
sorted_versions = sorted(versions, key=key)
|
||||
print(sorted_versions[-2])
|
||||
` + "```"
|
||||
|
||||
wrongLexicographic := "```python\n" + `versions = [
|
||||
"1.9.0",
|
||||
"1.10.0",
|
||||
"1.2.0",
|
||||
"2.0.0",
|
||||
"0.9.9",
|
||||
]
|
||||
print(sorted(versions)[-2])
|
||||
` + "```"
|
||||
|
||||
wrongHardcoded := "```python\nprint('2.0.0')\n```"
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{"correct numeric sort", correctScript, 1},
|
||||
{"wrong: lexicographic sort", wrongLexicographic, 0},
|
||||
{"wrong: hardcoded highest", wrongHardcoded, 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Skipped {
|
||||
t.Skip("python3 not available, skipping exec-evaluator assertion")
|
||||
}
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestHardGoParenBalanceTest_Eval exercises the GoRun evaluator with a
|
||||
// hand-written correct function and a wrong one.
|
||||
func TestHardGoParenBalanceTest_Eval(t *testing.T) {
|
||||
tc := hardGoParenBalanceTest()
|
||||
|
||||
correctFunc := "```go\n" + `func IsBalanced(s string) bool {
|
||||
pairs := map[byte]byte{')': '(', ']': '[', '}': '{'}
|
||||
stack := make([]byte, 0, len(s))
|
||||
for i := 0; i < len(s); i++ {
|
||||
switch s[i] {
|
||||
case '(', '[', '{':
|
||||
stack = append(stack, s[i])
|
||||
case ')', ']', '}':
|
||||
if len(stack) == 0 || stack[len(stack)-1] != pairs[s[i]] {
|
||||
return false
|
||||
}
|
||||
stack = stack[:len(stack)-1]
|
||||
}
|
||||
}
|
||||
return len(stack) == 0
|
||||
}
|
||||
` + "```"
|
||||
|
||||
wrongFunc := "```go\n" + `func IsBalanced(s string) bool {
|
||||
return true
|
||||
}
|
||||
` + "```"
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{"correct stack implementation", correctFunc, 1},
|
||||
{"wrong: always true", wrongFunc, 0},
|
||||
{"wrong: no code at all", "I would use a stack here.", 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Skipped {
|
||||
t.Skip("go toolchain not available, skipping exec-evaluator assertion")
|
||||
}
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestHardDeferNamedReturnWant_GroundTruth recomputes f()'s return value
|
||||
// by running an equivalent function in-process, independent of the
|
||||
// hardDeferNamedReturnWant constant.
|
||||
func TestHardDeferNamedReturnWant_GroundTruth(t *testing.T) {
|
||||
f := func() (r int) {
|
||||
defer func() { r = 99 }()
|
||||
r = 5
|
||||
return 7
|
||||
}
|
||||
if got := f(); got != hardDeferNamedReturnWant {
|
||||
t.Errorf("equivalent deferred-named-return function returned %d, want %d", got, hardDeferNamedReturnWant)
|
||||
}
|
||||
}
|
||||
|
||||
// TestHardIntDivisionWant_GroundTruth recomputes for a double result of
|
||||
// 7 / 2 by hand, independent of the hardIntDivisionWant constant.
|
||||
func TestHardIntDivisionWant_GroundTruth(t *testing.T) {
|
||||
a, b := 7, 2
|
||||
want := float64(a / b) // integer division truncates before widening
|
||||
if want != hardIntDivisionWant {
|
||||
t.Errorf("recomputed integer division = %v, want %v", want, hardIntDivisionWant)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user