mirror of
https://github.com/lukaszraczylo/llm-testbench.git
synced 2026-10-12 00:45:19 +00:00
feat: LLM accuracy testing framework with 17-test catalog
Framework runs a category/subcategory test catalog against multiple LLMs through an OpenAI-compatible endpoint. Deterministic evaluators score each response. Reports render as table, markdown, or JSON.
This commit is contained in:
@@ -0,0 +1,106 @@
|
||||
package tests
|
||||
|
||||
import (
|
||||
"github.com/lukaszraczylo/llm-testbench/internal/eval"
|
||||
"github.com/lukaszraczylo/llm-testbench/internal/testkit"
|
||||
)
|
||||
|
||||
func registerAgentsTests(r *testkit.Registry) {
|
||||
r.Register(agentToolRoutingTest())
|
||||
r.Register(agentPlanOrderingTest())
|
||||
}
|
||||
|
||||
// agentToolRoutingRoster is the inline tool roster for
|
||||
// agentToolRoutingTest.
|
||||
const agentToolRoutingRoster = `- search_web: Search the public web for current information not in your
|
||||
training data.
|
||||
- read_file: Read the contents of a file from the local filesystem.
|
||||
- run_shell: Execute an arbitrary shell command on the local machine.
|
||||
- query_db: Run a read-only SQL query against the application's Postgres
|
||||
database.
|
||||
- send_email: Send an email to a specified recipient.
|
||||
- none: No tool is needed; answer directly from what you already know.`
|
||||
|
||||
// agentToolRoutingTest: map 4 mini-tasks to the correct tool from a 6-tool
|
||||
// roster.
|
||||
//
|
||||
// ground truth: task1 is arithmetic answerable without any tool (none);
|
||||
// task2 needs a row count from a database table (query_db); task3 asks for
|
||||
// current information that cannot be in training data (search_web); task4
|
||||
// asks to notify someone by email (send_email).
|
||||
func agentToolRoutingTest() testkit.Test {
|
||||
prompt := `You have access to this roster of tools:
|
||||
|
||||
` + agentToolRoutingRoster + `
|
||||
|
||||
For each of these 4 tasks, decide which single tool (or "none") is the
|
||||
correct one to use:
|
||||
|
||||
task1: "What is 15% of 240?"
|
||||
task2: "How many rows are in the 'orders' table?"
|
||||
task3: "What is today's top headline on a major news site?"
|
||||
task4: "Notify the on-call engineer at [email protected] that the deploy
|
||||
finished."
|
||||
|
||||
Respond with only a JSON object mapping each task id to the tool name:
|
||||
{"task1":"...","task2":"...","task3":"...","task4":"..."}`
|
||||
|
||||
evaluator := eval.Mean(
|
||||
eval.JSONField("task1", "none"),
|
||||
eval.JSONField("task2", "query_db"),
|
||||
eval.JSONField("task3", "search_web"),
|
||||
eval.JSONField("task4", "send_email"),
|
||||
)
|
||||
|
||||
return testkit.Test{
|
||||
ID: "agent-tool-routing",
|
||||
Category: "agents",
|
||||
Subcategory: "tool-routing",
|
||||
Description: "Map 4 mini-tasks to the correct tool from a 6-tool roster.",
|
||||
Prompt: prompt,
|
||||
MaxTokens: 300,
|
||||
Eval: evaluator,
|
||||
}
|
||||
}
|
||||
|
||||
// agentPlanOrderingWant is the unique valid step order implied by the
|
||||
// dependency constraints in agentPlanOrderingTest's prompt.
|
||||
var agentPlanOrderingWant = []string{"build", "test", "backup", "deploy", "verify", "rollback"}
|
||||
|
||||
// agentPlanOrderingTest: order deployment steps that have explicit
|
||||
// pairwise dependencies.
|
||||
//
|
||||
// ground truth: the prompt states build must precede test (need a built
|
||||
// artifact to test), backup must happen only after tests pass and only
|
||||
// before deploy, deploy must follow backup, verify must follow deploy, and
|
||||
// rollback is only meaningful after a deploy whose verify step failed. This
|
||||
// pins a single valid total order: build, test, backup, deploy, verify,
|
||||
// rollback.
|
||||
func agentPlanOrderingTest() testkit.Test {
|
||||
prompt := `You are planning a deployment with these steps: build, test,
|
||||
backup, deploy, verify, rollback.
|
||||
|
||||
Constraints:
|
||||
- You must build the artifact before you can test it.
|
||||
- You only take a backup of the current production state once tests have
|
||||
passed - there is no point backing up before you know the new build is
|
||||
viable.
|
||||
- Deploy only happens after the backup completes.
|
||||
- After deploy, you must verify the new deployment is healthy.
|
||||
- Rollback (restoring the pre-deploy backup) only makes sense after a
|
||||
deploy has happened, and only if verify fails.
|
||||
|
||||
Give the single correct step order as a JSON array of step ids, e.g.
|
||||
["build","test",...]. Include all 6 steps, listing rollback in its position
|
||||
in the sequence even though it is conditional on verify failing.`
|
||||
|
||||
return testkit.Test{
|
||||
ID: "agent-plan-ordering",
|
||||
Category: "agents",
|
||||
Subcategory: "planning",
|
||||
Description: "Order 6 deployment steps under explicit pairwise dependency constraints.",
|
||||
Prompt: prompt,
|
||||
MaxTokens: 300,
|
||||
Eval: eval.JSONStringArrayEquals(agentPlanOrderingWant),
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,84 @@
|
||||
package tests
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestAgentToolRoutingTest_Eval(t *testing.T) {
|
||||
tc := agentToolRoutingTest()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{
|
||||
name: "all correct",
|
||||
response: `{"task1":"none","task2":"query_db","task3":"search_web","task4":"send_email"}`,
|
||||
want: 1,
|
||||
},
|
||||
{
|
||||
name: "one wrong",
|
||||
response: `{"task1":"run_shell","task2":"query_db","task3":"search_web","task4":"send_email"}`,
|
||||
want: 0.75,
|
||||
},
|
||||
{
|
||||
name: "all wrong",
|
||||
response: `{"task1":"run_shell","task2":"search_web","task3":"query_db","task4":"none"}`,
|
||||
want: 0,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestAgentPlanOrderingTest_Eval(t *testing.T) {
|
||||
tc := agentPlanOrderingTest()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{
|
||||
name: "correct order",
|
||||
response: `["build","test","backup","deploy","verify","rollback"]`,
|
||||
want: 1,
|
||||
},
|
||||
{
|
||||
name: "correct order fenced",
|
||||
response: "```json\n[\"build\",\"test\",\"backup\",\"deploy\",\"verify\",\"rollback\"]\n```",
|
||||
want: 1,
|
||||
},
|
||||
{
|
||||
name: "backup before test violates dependency",
|
||||
response: `["build","backup","test","deploy","verify","rollback"]`,
|
||||
want: 0,
|
||||
},
|
||||
{
|
||||
name: "rollback before verify",
|
||||
response: `["build","test","backup","deploy","rollback","verify"]`,
|
||||
want: 0,
|
||||
},
|
||||
{
|
||||
name: "missing a step",
|
||||
response: `["build","test","backup","deploy","verify"]`,
|
||||
want: 0,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,50 @@
|
||||
package tests
|
||||
|
||||
import (
|
||||
"github.com/lukaszraczylo/llm-testbench/internal/eval"
|
||||
"github.com/lukaszraczylo/llm-testbench/internal/testkit"
|
||||
)
|
||||
|
||||
func registerCTests(r *testkit.Registry) {
|
||||
r.Register(cStructSizeTest())
|
||||
}
|
||||
|
||||
// cStructSizeWant is sizeof(struct Config) on LP64 (64-bit System V / Apple
|
||||
// ARM64 ABI: natural alignment, no packing).
|
||||
//
|
||||
// ground truth: char flag(align 1, size 1) at offset 0; double ratio
|
||||
// (align 8, size 8) needs offset divisible by 8, so it starts at offset 8
|
||||
// (7 bytes padding after flag), ending at 16; int count (align 4, size 4)
|
||||
// at offset 16 (already aligned), ending at 20; short mode (align 2, size
|
||||
// 2) at offset 20, ending at 22. The struct's own alignment is the widest
|
||||
// member alignment (8, from double), so the total size rounds up from 22
|
||||
// to the next multiple of 8: 24. c_test.go cross-checks this with cc via
|
||||
// sizeof and offsetof when the cc toolchain is available.
|
||||
const cStructSizeWant = 24
|
||||
|
||||
func cStructSizeTest() testkit.Test {
|
||||
prompt := `Here is a C struct:
|
||||
|
||||
` + "```c" + `
|
||||
struct Config {
|
||||
char flag;
|
||||
double ratio;
|
||||
int count;
|
||||
short mode;
|
||||
};
|
||||
` + "```" + `
|
||||
|
||||
Assuming the LP64 data model (64-bit, no #pragma pack, no
|
||||
non-default alignment attributes), what is sizeof(struct Config) in bytes?
|
||||
Respond with only the number, nothing else.`
|
||||
|
||||
return testkit.Test{
|
||||
ID: "c-struct-size",
|
||||
Category: "programming",
|
||||
Subcategory: "c",
|
||||
Description: "Compute sizeof() for a C struct on LP64 given its member types and default alignment.",
|
||||
Prompt: prompt,
|
||||
MaxTokens: 200,
|
||||
Eval: eval.Numeric(eval.ExtractLastNumber[int], cStructSizeWant, 0),
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,87 @@
|
||||
package tests
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// TestCStructSizeWant_GroundTruth cross-checks cStructSizeWant against the
|
||||
// real cc toolchain's sizeof(struct Config), rather than trusting the
|
||||
// hand-derived alignment arithmetic in the doc comment alone. Skips (does
|
||||
// not fail) when cc is unavailable, matching the exec-evaluator contract.
|
||||
func TestCStructSizeWant_GroundTruth(t *testing.T) {
|
||||
if _, err := exec.LookPath("cc"); err != nil {
|
||||
t.Skip("cc not found on PATH, skipping ground-truth cross-check")
|
||||
}
|
||||
|
||||
const src = `#include <stdio.h>
|
||||
struct Config {
|
||||
char flag;
|
||||
double ratio;
|
||||
int count;
|
||||
short mode;
|
||||
};
|
||||
int main(void) {
|
||||
printf("%zu", sizeof(struct Config));
|
||||
return 0;
|
||||
}
|
||||
`
|
||||
dir := t.TempDir()
|
||||
srcPath := filepath.Join(dir, "main.c")
|
||||
if err := os.WriteFile(srcPath, []byte(src), 0o600); err != nil {
|
||||
t.Fatalf("write main.c: %v", err)
|
||||
}
|
||||
binPath := filepath.Join(dir, "prog")
|
||||
|
||||
// #nosec G204 -- compiling a fixed, hardcoded C source (src above) in a
|
||||
// t.TempDir() this test just created; not external input.
|
||||
build := exec.Command("cc", "-o", binPath, srcPath) //nolint:gosec // see comment above
|
||||
if out, err := build.CombinedOutput(); err != nil {
|
||||
t.Fatalf("cc failed: %v\n%s", err, out)
|
||||
}
|
||||
|
||||
// #nosec G204 -- binPath is the binary this test just compiled above,
|
||||
// in its own t.TempDir(); not an externally supplied executable path.
|
||||
out, err := exec.Command(binPath).Output() //nolint:gosec // see comment above
|
||||
if err != nil {
|
||||
t.Fatalf("running compiled program failed: %v", err)
|
||||
}
|
||||
|
||||
got := strings.TrimSpace(string(out))
|
||||
if got != "24" {
|
||||
t.Errorf("cc-computed sizeof(struct Config) = %s, want 24 (matches cStructSizeWant)", got)
|
||||
}
|
||||
if cStructSizeWant != 24 {
|
||||
t.Errorf("cStructSizeWant = %d, want 24", cStructSizeWant)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCStructSizeTest_Eval(t *testing.T) {
|
||||
tc := cStructSizeTest()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{"exact correct", "24", 1},
|
||||
{"prose wrapped", "sizeof(struct Config) is 24 bytes", 1},
|
||||
{"unpadded sum is wrong", "15", 0},
|
||||
{"off by one", "23", 0},
|
||||
// S6 regression: the trailing "64-bit" qualifier must not be
|
||||
// mistaken for the answer.
|
||||
{"answer followed by a 64-bit qualifier", "sizeof(struct Config) is 24 bytes on a 64-bit system", 1},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate() = %v, want %v (detail: %s)", got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
package tests
|
||||
|
||||
import "github.com/lukaszraczylo/llm-testbench/internal/testkit"
|
||||
|
||||
// All builds a Registry containing the full 17-test catalog: 3
|
||||
// programming/golang, 2 programming/python, 1 programming/typescript, 1
|
||||
// programming/c, 2 operations/macos, 2 operations/linux, 1
|
||||
// operations/kubernetes, 1 research/web, 1 research/whitepapers, 1
|
||||
// research/codebase, and 2 agents (tool-routing, planning).
|
||||
func All() *testkit.Registry {
|
||||
r := testkit.NewRegistry()
|
||||
registerGolangTests(r)
|
||||
registerPythonTests(r)
|
||||
registerTypeScriptTests(r)
|
||||
registerCTests(r)
|
||||
registerMacOSTests(r)
|
||||
registerLinuxTests(r)
|
||||
registerKubernetesTests(r)
|
||||
registerWebTests(r)
|
||||
registerWhitepapersTests(r)
|
||||
registerCodebaseTests(r)
|
||||
registerAgentsTests(r)
|
||||
return r
|
||||
}
|
||||
@@ -0,0 +1,61 @@
|
||||
package tests
|
||||
|
||||
import "testing"
|
||||
|
||||
// wantCatalog is the full 17-test catalog per PLAN.md, id -> {category,
|
||||
// subcategory}. TestAll_MatchesCatalog checks the registry against this
|
||||
// list exactly, so an accidental removal, duplication, or miscategorized
|
||||
// test fails loudly.
|
||||
var wantCatalog = map[string][2]string{
|
||||
"go-struct-align": {"programming", "golang"},
|
||||
"go-worker-pool": {"programming", "golang"},
|
||||
"go-semver-classify": {"programming", "golang"},
|
||||
"py-log-triage": {"programming", "python"},
|
||||
"py-cosine": {"programming", "python"},
|
||||
"ts-debounce-composable": {"programming", "typescript"},
|
||||
"c-struct-size": {"programming", "c"},
|
||||
"macos-timeout-portability": {"operations", "macos"},
|
||||
"macos-launchd-cron": {"operations", "macos"},
|
||||
"linux-pct-exec": {"operations", "linux"},
|
||||
"linux-systemd-oneshot": {"operations", "linux"},
|
||||
"k8s-crashloop-gitops": {"operations", "kubernetes"},
|
||||
"web-robots-ai-crawlers": {"research", "web"},
|
||||
"paper-hnsw-params": {"research", "whitepapers"},
|
||||
"code-trace-go": {"research", "codebase"},
|
||||
"agent-tool-routing": {"agents", "tool-routing"},
|
||||
"agent-plan-ordering": {"agents", "planning"},
|
||||
}
|
||||
|
||||
func TestAll_MatchesCatalog(t *testing.T) {
|
||||
r := All()
|
||||
|
||||
if r.Len() != len(wantCatalog) {
|
||||
t.Fatalf("registry has %d tests, want %d", r.Len(), len(wantCatalog))
|
||||
}
|
||||
|
||||
for id, catSub := range wantCatalog {
|
||||
tc, ok := r.Get(id)
|
||||
if !ok {
|
||||
t.Errorf("missing test %q", id)
|
||||
continue
|
||||
}
|
||||
if tc.Category != catSub[0] {
|
||||
t.Errorf("%s: Category = %q, want %q", id, tc.Category, catSub[0])
|
||||
}
|
||||
if tc.Subcategory != catSub[1] {
|
||||
t.Errorf("%s: Subcategory = %q, want %q", id, tc.Subcategory, catSub[1])
|
||||
}
|
||||
if tc.Prompt == "" {
|
||||
t.Errorf("%s: Prompt is empty", id)
|
||||
}
|
||||
if tc.Eval == nil {
|
||||
t.Errorf("%s: Eval is nil", id)
|
||||
}
|
||||
if tc.Description == "" {
|
||||
t.Errorf("%s: Description is empty", id)
|
||||
}
|
||||
if tc.MaxTokens <= 0 {
|
||||
t.Errorf("%s: MaxTokens = %d, want > 0", id, tc.MaxTokens)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,66 @@
|
||||
package tests
|
||||
|
||||
import (
|
||||
"github.com/lukaszraczylo/llm-testbench/internal/eval"
|
||||
"github.com/lukaszraczylo/llm-testbench/internal/testkit"
|
||||
)
|
||||
|
||||
func registerCodebaseTests(r *testkit.Registry) {
|
||||
r.Register(codeTraceGoTest())
|
||||
}
|
||||
|
||||
// mix is the traced function for codeTraceGoTest: deterministic bit
|
||||
// manipulation plus recursion. It is called directly (not re-implemented)
|
||||
// to derive codeTraceGoWant, and codebase_test.go calls it again with the
|
||||
// same input to ground that value, per PLAN.md's instruction that this
|
||||
// test's ground truth be established by running the same function.
|
||||
func mix(n int, depth int) int {
|
||||
if depth == 0 {
|
||||
return n
|
||||
}
|
||||
if n%2 == 0 {
|
||||
n = n >> 1
|
||||
} else {
|
||||
n = (n << 1) ^ 0x2A
|
||||
}
|
||||
return mix(n, depth-1) + depth
|
||||
}
|
||||
|
||||
// codeTraceGoWant is mix(37, 4), computed by running mix itself.
|
||||
//
|
||||
// ground truth: mix(37,4): 37 is odd -> n=(37<<1)^0x2A=96, return
|
||||
// mix(96,3)+4. 96 is even -> n=48, return mix(48,2)+3. 48 is even -> n=24,
|
||||
// return mix(24,1)+2. 24 is even -> n=12, return mix(12,0)+1. depth=0 ->
|
||||
// return 12. Unwinding: 12+1=13, 13+2=15, 15+3=18, 18+4=22.
|
||||
var codeTraceGoWant = mix(37, 4)
|
||||
|
||||
func codeTraceGoTest() testkit.Test {
|
||||
prompt := `Here is a Go function:
|
||||
|
||||
` + "```go" + `
|
||||
func mix(n int, depth int) int {
|
||||
if depth == 0 {
|
||||
return n
|
||||
}
|
||||
if n%2 == 0 {
|
||||
n = n >> 1
|
||||
} else {
|
||||
n = (n << 1) ^ 0x2A
|
||||
}
|
||||
return mix(n, depth-1) + depth
|
||||
}
|
||||
` + "```" + `
|
||||
|
||||
What is the exact value of mix(37, 4)? Respond with only the number,
|
||||
nothing else.`
|
||||
|
||||
return testkit.Test{
|
||||
ID: "code-trace-go",
|
||||
Category: "research",
|
||||
Subcategory: "codebase",
|
||||
Description: "Trace a recursive Go function combining bit manipulation and recursion to an exact numeric result.",
|
||||
Prompt: prompt,
|
||||
MaxTokens: 200,
|
||||
Eval: eval.Numeric(eval.ExtractLastNumber[int], codeTraceGoWant, 0),
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,64 @@
|
||||
package tests
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// TestMix_GroundTruth runs mix itself (the traced function) to ground
|
||||
// codeTraceGoWant, per PLAN.md's instruction for this specific test.
|
||||
func TestMix_GroundTruth(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
n int
|
||||
depth int
|
||||
want int
|
||||
}{
|
||||
{"base case depth 0", 12, 0, 12},
|
||||
{"one level, even", 24, 1, 13}, // 24>>1=12; mix(12,0)+1=13
|
||||
{"two levels", 48, 2, 15}, // 48>>1=24; mix(24,1)+2=15
|
||||
{"three levels", 96, 3, 18}, // 96>>1=48; mix(48,2)+3=18
|
||||
{"full trace, odd start", 37, 4, 22},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := mix(tt.n, tt.depth)
|
||||
if got != tt.want {
|
||||
t.Errorf("mix(%d, %d) = %d, want %d", tt.n, tt.depth, got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
if codeTraceGoWant != 22 {
|
||||
t.Errorf("codeTraceGoWant = %d, want 22", codeTraceGoWant)
|
||||
}
|
||||
if got := mix(37, 4); got != codeTraceGoWant {
|
||||
t.Errorf("mix(37, 4) = %d, want codeTraceGoWant %d", got, codeTraceGoWant)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCodeTraceGoTest_Eval(t *testing.T) {
|
||||
tc := codeTraceGoTest()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{"exact correct", "22", 1},
|
||||
{"prose wrapped", "The result is 22.", 1},
|
||||
{"off by one", "21", 0},
|
||||
{"forgot to track depth in the unwind", "12", 0},
|
||||
// S6 regression: a trailing "64-bit" qualifier after the real
|
||||
// answer must not be mistaken for the answer itself.
|
||||
{"answer followed by a 64-bit qualifier", "The result is 22, computed via a 64-bit accumulator.", 1},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,225 @@
|
||||
// Package tests is the test catalog: one file per category, each
|
||||
// registering its Tests into a shared Registry via Register in main.go's
|
||||
// wiring (see All in catalog.go).
|
||||
package tests
|
||||
|
||||
import (
|
||||
"github.com/lukaszraczylo/llm-testbench/internal/eval"
|
||||
"github.com/lukaszraczylo/llm-testbench/internal/testkit"
|
||||
)
|
||||
|
||||
// terseCodeOnly is prepended as a system message on code-generation tests
|
||||
// to keep responses parseable by the exec/regex evaluators.
|
||||
const terseCodeOnly = "Respond with only the requested code in a single fenced code block. " +
|
||||
"No prose before or after the code block, no explanations, no comments beyond what was asked for."
|
||||
|
||||
func registerGolangTests(r *testkit.Registry) {
|
||||
r.Register(goStructAlignTest())
|
||||
r.Register(goWorkerPoolTest())
|
||||
r.Register(goSemverClassifyTest())
|
||||
}
|
||||
|
||||
// goStructAlignTest: reorder struct fields to minimize padding on a 64-bit
|
||||
// (LP64) system.
|
||||
//
|
||||
// ground truth: on LP64, alignment/size per field are bool(align 1, size
|
||||
// 1), int64(align 8, size 8), int32(align 4, size 4), string(align 8, size
|
||||
// 16 = 8-byte pointer + 8-byte length), byte(align 1, size 1). The Go
|
||||
// compiler lays out fields in declaration order, padding each field to its
|
||||
// own alignment, then rounds the struct size up to the widest field
|
||||
// alignment (8).
|
||||
//
|
||||
// Wasteful order: Active bool(0..1) pad(1..8) ID int64(8..16)
|
||||
// Priority int32(16..20) pad(20..24) Name string(24..40) Flag byte(40..41)
|
||||
// pad(41..48) -> total 48 bytes, 18 bytes of padding.
|
||||
//
|
||||
// Optimal order: group the two 8-byte-aligned fields first (Name string,
|
||||
// ID int64, in either order: 0..16, 16..24), then the 4-byte-aligned
|
||||
// int32 (24..28), then the two 1-byte fields adjacent with no padding
|
||||
// between them (28..29, 29..30), then round up to the next multiple of 8
|
||||
// -> total 32 bytes. No smaller layout exists: the payload is
|
||||
// 16+8+4+1+1 = 30 bytes and the struct's own alignment (8, from the
|
||||
// 64-bit-aligned fields) forces the size up to the next multiple of 8,
|
||||
// which is 32.
|
||||
//
|
||||
// golang_test.go verifies both sizes with unsafe.Sizeof on literal struct
|
||||
// definitions, and cross-checks computeLP64StructSize itself against
|
||||
// unsafe.Sizeof for several canonical field orders. Scoring
|
||||
// (goStructAlignEval) parses the response's actual field order and
|
||||
// computes its real layout size rather than matching one hardcoded
|
||||
// ordering, so any of the several field orders that reach 32 bytes scores
|
||||
// full credit (S3).
|
||||
func goStructAlignTest() testkit.Test {
|
||||
prompt := `Here is a Go struct definition:
|
||||
|
||||
` + "```go" + `
|
||||
type Task struct {
|
||||
Active bool
|
||||
ID int64
|
||||
Priority int32
|
||||
Name string
|
||||
Flag byte
|
||||
}
|
||||
` + "```" + `
|
||||
|
||||
On a 64-bit (LP64) system, this field order wastes memory to padding.
|
||||
Reorder the fields (keep the same field names and types, change nothing
|
||||
else) to minimize the total size of the struct due to alignment padding.
|
||||
Respond with the corrected struct definition only.`
|
||||
|
||||
return testkit.Test{
|
||||
ID: "go-struct-align",
|
||||
Category: "programming",
|
||||
Subcategory: "golang",
|
||||
Description: "Reorder a Go struct's fields to minimize alignment padding on a 64-bit system.",
|
||||
System: terseCodeOnly,
|
||||
Prompt: prompt,
|
||||
MaxTokens: 500,
|
||||
Eval: goStructAlignEval(),
|
||||
}
|
||||
}
|
||||
|
||||
// goWorkerPoolHarness is a complete, independent driver file (its own
|
||||
// package clause and imports) that eval.GoRun writes to harness.go
|
||||
// alongside the model's response in solution.go, so the model's code and
|
||||
// the harness's code never share one spliced import block. It drives the
|
||||
// response's Pool[T, R any] implementation (defined in solution.go, same
|
||||
// package) with a deliberately slow worker function and checks (a) output
|
||||
// order matches input order and (b) the pool actually parallelizes work,
|
||||
// via both an observed-concurrency counter and a wall-clock bound far
|
||||
// below the fully-sequential time.
|
||||
const goWorkerPoolHarness = `package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
)
|
||||
|
||||
func main() {
|
||||
const n = 100
|
||||
const workers = 8
|
||||
const perItem = 2 * time.Millisecond
|
||||
|
||||
in := make([]int, n)
|
||||
for i := range in {
|
||||
in[i] = i
|
||||
}
|
||||
|
||||
var concurrent int32
|
||||
var maxConcurrent int32
|
||||
f := func(x int) int {
|
||||
cur := atomic.AddInt32(&concurrent, 1)
|
||||
for {
|
||||
m := atomic.LoadInt32(&maxConcurrent)
|
||||
if cur <= m || atomic.CompareAndSwapInt32(&maxConcurrent, m, cur) {
|
||||
break
|
||||
}
|
||||
}
|
||||
time.Sleep(perItem)
|
||||
atomic.AddInt32(&concurrent, -1)
|
||||
return x * 2
|
||||
}
|
||||
|
||||
start := time.Now()
|
||||
out := Pool(in, workers, f)
|
||||
elapsed := time.Since(start)
|
||||
|
||||
if len(out) != n {
|
||||
fmt.Fprintf(os.Stderr, "len(out)=%d, want %d\n", len(out), n)
|
||||
os.Exit(1)
|
||||
}
|
||||
for i := range out {
|
||||
if out[i] != i*2 {
|
||||
fmt.Fprintf(os.Stderr, "out[%d]=%d, want %d: input order not preserved\n", i, out[i], i*2)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
if atomic.LoadInt32(&maxConcurrent) < 2 {
|
||||
fmt.Fprintf(os.Stderr, "maxConcurrent=%d, want >= 2: no real concurrency observed\n", maxConcurrent)
|
||||
os.Exit(1)
|
||||
}
|
||||
// Sequential execution would take n*perItem = 200ms; with workers
|
||||
// goroutines truly running in parallel it should land well under that.
|
||||
if elapsed > 120*time.Millisecond {
|
||||
fmt.Fprintf(os.Stderr, "elapsed=%s, want < 120ms: looks sequential\n", elapsed)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
fmt.Println("PASS")
|
||||
}
|
||||
`
|
||||
|
||||
// goWorkerPoolTest: implement a generic, order-preserving, concurrent
|
||||
// worker pool.
|
||||
//
|
||||
// ground truth: the harness above is the oracle; it directly measures
|
||||
// order preservation and real concurrency rather than a hardcoded number,
|
||||
// so there is no separate constant to derive.
|
||||
func goWorkerPoolTest() testkit.Test {
|
||||
prompt := `Implement this Go generic function:
|
||||
|
||||
` + "```go" + `
|
||||
func Pool[T, R any](in []T, workers int, f func(T) R) []R
|
||||
` + "```" + `
|
||||
|
||||
Requirements:
|
||||
- Run f concurrently across exactly workers goroutines.
|
||||
- The returned slice must preserve the input order: result[i] is f(in[i]).
|
||||
- You may include whatever imports your implementation needs. Do not
|
||||
include a package clause or a main function: respond with only the
|
||||
Pool function definition (plus any imports it needs).`
|
||||
|
||||
return testkit.Test{
|
||||
ID: "go-worker-pool",
|
||||
Category: "programming",
|
||||
Subcategory: "golang",
|
||||
Description: "Implement a generic, order-preserving, concurrent worker pool.",
|
||||
System: terseCodeOnly,
|
||||
Prompt: prompt,
|
||||
MaxTokens: 800,
|
||||
Eval: eval.GoRun(goWorkerPoolHarness, "PASS"),
|
||||
}
|
||||
}
|
||||
|
||||
// goSemverClassifyTest: classify the aggregate semver bump implied by a
|
||||
// batch of conventional commits.
|
||||
//
|
||||
// ground truth: per Conventional Commits, a MAJOR bump requires either a
|
||||
// "!" after the type/scope or a "BREAKING CHANGE:" footer; a MINOR bump
|
||||
// comes from any "feat:" commit; a PATCH bump comes from any "fix:" commit
|
||||
// with no feat present; chore/docs/etc. commits do not trigger a release on
|
||||
// their own. The batch below has one "feat:" commit, two "fix:" commits, a
|
||||
// "chore:" and a "docs:" commit, and zero commits with a "!" marker or a
|
||||
// "BREAKING CHANGE:" footer (commit 4's body uses the word "breaking" in
|
||||
// the ordinary-English sense "test breaking flakiness", not as a release
|
||||
// footer) -> the highest applicable bump across the batch is minor.
|
||||
func goSemverClassifyTest() testkit.Test {
|
||||
prompt := `Here are 5 conventional commit messages from one release batch:
|
||||
|
||||
1. "feat(cache): add TTL eviction policy for LRU cache"
|
||||
2. "fix(scheduler): prevent duplicate job dispatch on retry"
|
||||
3. "chore(ci): pin golangci-lint version in the lint config"
|
||||
4. "fix(auth): correct token expiry check off by one
|
||||
|
||||
This resolves an intermittent test breaking flakiness in local dev but
|
||||
does not change any public behaviour."
|
||||
5. "docs(readme): clarify installation steps for macOS"
|
||||
|
||||
Using the Conventional Commits specification, determine the single semver
|
||||
bump ("major", "minor", or "patch") that this batch of commits, taken
|
||||
together, should produce for the next release. Respond with only a JSON
|
||||
object: {"bump":"major|minor|patch"}`
|
||||
|
||||
return testkit.Test{
|
||||
ID: "go-semver-classify",
|
||||
Category: "programming",
|
||||
Subcategory: "golang",
|
||||
Description: "Classify the aggregate semver bump for a batch of conventional commits, including a 'breaking' word trap.",
|
||||
System: terseCodeOnly,
|
||||
Prompt: prompt,
|
||||
MaxTokens: 300,
|
||||
Eval: eval.JSONField("bump", "minor"),
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,259 @@
|
||||
package tests
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"unsafe"
|
||||
)
|
||||
|
||||
// wastefulTask and optimalTask mirror the struct in goStructAlignTest's
|
||||
// prompt (wasteful field order) and its minimal-padding reordering. Sizes
|
||||
// are asserted with unsafe.Sizeof to ground the "ground truth:" derivation
|
||||
// comment in golang.go against the actual compiler layout, per PLAN.md's
|
||||
// requirement to recompute cheap ground truths in the unit test.
|
||||
//
|
||||
// wastefulTask deliberately keeps the prompt's wasteful field order
|
||||
// verbatim (the "before" layout), so fieldalignment's auto-fix must not be
|
||||
// allowed to reorder it - that would make it identical to optimalTask and
|
||||
// defeat this test's comparison.
|
||||
//
|
||||
//nolint:govet // fieldalignment: intentional, see comment above
|
||||
type wastefulTask struct {
|
||||
Active bool
|
||||
ID int64
|
||||
Priority int32
|
||||
Name string
|
||||
Flag byte
|
||||
}
|
||||
|
||||
type optimalTask struct {
|
||||
Name string
|
||||
ID int64
|
||||
Priority int32
|
||||
Active bool
|
||||
Flag byte
|
||||
}
|
||||
|
||||
func TestGoStructAlign_GroundTruthSizes(t *testing.T) {
|
||||
if got, want := unsafe.Sizeof(wastefulTask{}), uintptr(48); got != want {
|
||||
t.Errorf("unsafe.Sizeof(wastefulTask{}) = %d, want %d", got, want)
|
||||
}
|
||||
if got, want := unsafe.Sizeof(optimalTask{}), uintptr(32); got != want {
|
||||
t.Errorf("unsafe.Sizeof(optimalTask{}) = %d, want %d", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGoStructAlignTest_Eval(t *testing.T) {
|
||||
tc := goStructAlignTest()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{
|
||||
name: "optimal order Name then ID",
|
||||
response: "```go\ntype Task struct {\n\tName string\n\tID int64\n\tPriority int32\n\tActive bool\n\tFlag byte\n}\n```",
|
||||
want: 1,
|
||||
},
|
||||
{
|
||||
name: "optimal order ID then Name",
|
||||
response: "```go\ntype Task struct {\n\tID int64\n\tName string\n\tPriority int32\n\tFlag byte\n\tActive bool\n}\n```",
|
||||
want: 1,
|
||||
},
|
||||
{
|
||||
name: "original wasteful order",
|
||||
response: "```go\ntype Task struct {\n\tActive bool\n\tID int64\n\tPriority int32\n\tName string\n\tFlag byte\n}\n```",
|
||||
want: 0,
|
||||
},
|
||||
{
|
||||
name: "int32 not moved after align-8 fields",
|
||||
response: "```go\ntype Task struct {\n\tPriority int32\n\tName string\n\tID int64\n\tActive bool\n\tFlag byte\n}\n```",
|
||||
want: 0,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate() = %v, want %v (detail: %s)", got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestGoWorkerPoolTest_Eval(t *testing.T) {
|
||||
tc := goWorkerPoolTest()
|
||||
|
||||
correctPool := "```go\n" + `func Pool[T, R any](in []T, workers int, f func(T) R) []R {
|
||||
out := make([]R, len(in))
|
||||
sem := make(chan struct{}, workers)
|
||||
done := make(chan struct{})
|
||||
for i, v := range in {
|
||||
i, v := i, v
|
||||
sem <- struct{}{}
|
||||
go func() {
|
||||
defer func() { <-sem; done <- struct{}{} }()
|
||||
out[i] = f(v)
|
||||
}()
|
||||
}
|
||||
for range in {
|
||||
<-done
|
||||
}
|
||||
return out
|
||||
}
|
||||
` + "```"
|
||||
|
||||
sequentialPool := "```go\n" + `func Pool[T, R any](in []T, workers int, f func(T) R) []R {
|
||||
out := make([]R, len(in))
|
||||
for i, v := range in {
|
||||
out[i] = f(v)
|
||||
}
|
||||
return out
|
||||
}
|
||||
` + "```"
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{"correct concurrent pool", correctPool, 1},
|
||||
{"sequential pool fails concurrency/timing check", sequentialPool, 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Skipped {
|
||||
t.Skip("go toolchain not available, skipping exec-evaluator assertion")
|
||||
}
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate() = %v, want %v (detail: %s)", got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestGoWorkerPoolTest_Eval_ResponseShapes covers B1: the two-file
|
||||
// (solution.go + harness.go) split must accept a response that brings its
|
||||
// own imports, whether or not it also carries a package clause, since
|
||||
// eval.GoRun strips at most a leading package clause and gives the
|
||||
// response its own import block separate from the harness's.
|
||||
func TestGoWorkerPoolTest_Eval_ResponseShapes(t *testing.T) {
|
||||
tc := goWorkerPoolTest()
|
||||
|
||||
// Shape 1: a bare function body with its own "import" block and no
|
||||
// package clause - the common case when a model obeys "respond with
|
||||
// only the function definition".
|
||||
withImportsNoPackage := "```go\n" + `import "sync"
|
||||
|
||||
func Pool[T, R any](in []T, workers int, f func(T) R) []R {
|
||||
out := make([]R, len(in))
|
||||
var wg sync.WaitGroup
|
||||
sem := make(chan struct{}, workers)
|
||||
for i, v := range in {
|
||||
i, v := i, v
|
||||
wg.Add(1)
|
||||
sem <- struct{}{}
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
defer func() { <-sem }()
|
||||
out[i] = f(v)
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
return out
|
||||
}
|
||||
` + "```"
|
||||
|
||||
// Shape 2: a full, standalone file - package clause, imports, and the
|
||||
// function - as some models emit even when told not to include a
|
||||
// package clause. eval.GoRun must strip the leading "package main" line
|
||||
// and still compile it as part of solution.go.
|
||||
fullFileWithPackageClause := "```go\n" + `package main
|
||||
|
||||
import "sync"
|
||||
|
||||
func Pool[T, R any](in []T, workers int, f func(T) R) []R {
|
||||
out := make([]R, len(in))
|
||||
var wg sync.WaitGroup
|
||||
sem := make(chan struct{}, workers)
|
||||
for i, v := range in {
|
||||
i, v := i, v
|
||||
wg.Add(1)
|
||||
sem <- struct{}{}
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
defer func() { <-sem }()
|
||||
out[i] = f(v)
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
return out
|
||||
}
|
||||
` + "```"
|
||||
|
||||
// Shape 3: uses sync.WaitGroup without importing "sync" - now that the
|
||||
// prompt explicitly allows imports, this is a genuine model mistake,
|
||||
// not a harness limitation. It must fail to compile (score 0), not be
|
||||
// silently patched or Skipped.
|
||||
usesSyncWithoutImport := "```go\n" + `func Pool[T, R any](in []T, workers int, f func(T) R) []R {
|
||||
out := make([]R, len(in))
|
||||
var wg sync.WaitGroup
|
||||
for i, v := range in {
|
||||
wg.Add(1)
|
||||
i, v := i, v
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
out[i] = f(v)
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
return out
|
||||
}
|
||||
` + "```"
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{"imports, no package clause: PASS", withImportsNoPackage, 1},
|
||||
{"full file with package clause: PASS after strip", fullFileWithPackageClause, 1},
|
||||
{"uses sync without importing it: fails to compile", usesSyncWithoutImport, 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Skipped {
|
||||
t.Skip("go toolchain not available, skipping exec-evaluator assertion")
|
||||
}
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate() = %v, want %v (detail: %s)", got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestGoSemverClassifyTest_Eval(t *testing.T) {
|
||||
tc := goSemverClassifyTest()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{"correct minor", `{"bump":"minor"}`, 1},
|
||||
{"correct minor fenced", "```json\n{\"bump\": \"minor\"}\n```", 1},
|
||||
{"trap: falls for the word breaking and answers major", `{"bump":"major"}`, 0},
|
||||
{"wrong: answers patch, missing the feat commit", `{"bump":"patch"}`, 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate() = %v, want %v (detail: %s)", got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,139 @@
|
||||
package tests
|
||||
|
||||
import (
|
||||
"context"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/lukaszraczylo/llm-testbench/internal/eval"
|
||||
"github.com/lukaszraczylo/llm-testbench/internal/testkit"
|
||||
)
|
||||
|
||||
func registerKubernetesTests(r *testkit.Registry) {
|
||||
r.Register(k8sCrashloopGitopsTest())
|
||||
}
|
||||
|
||||
// k8sCrashloopGitopsDescribeExcerpt is the inline kubectl describe pod
|
||||
// excerpt for k8sCrashloopGitopsTest.
|
||||
const k8sCrashloopGitopsDescribeExcerpt = `Name: inventory-worker-6f9b7c8d9-2xk4p
|
||||
Namespace: inventory
|
||||
Status: Running
|
||||
Containers:
|
||||
worker:
|
||||
State: Waiting
|
||||
Reason: CrashLoopBackOff
|
||||
Last State: Terminated
|
||||
Reason: OOMKilled
|
||||
Exit Code: 137
|
||||
Ready: False
|
||||
Restart Count: 9
|
||||
Limits:
|
||||
cpu: 500m
|
||||
memory: 128Mi
|
||||
Requests:
|
||||
cpu: 100m
|
||||
memory: 64Mi
|
||||
Events:
|
||||
Warning BackOff kubelet Back-off restarting failed container worker in pod inventory-worker-6f9b7c8d9-2xk4p`
|
||||
|
||||
// mentionsLimitIncrease scores full credit only when the response both
|
||||
// names the resource being raised (memory / the limit / the 128Mi value)
|
||||
// and uses language for raising it. Using two independent ContainsAny
|
||||
// checks combined by weighted mean would give partial credit to a response
|
||||
// that raises the wrong resource (e.g. CPU) or that mentions memory
|
||||
// without proposing to change the limit; this requires both.
|
||||
func mentionsLimitIncrease() eval.Evaluator {
|
||||
verbs := eval.ContainsAny("increase", "raise", "bump", "higher", "raising")
|
||||
target := eval.ContainsAny("memory limit", "limit", "128Mi")
|
||||
return eval.EvaluatorFunc(func(ctx context.Context, response string) eval.Score {
|
||||
v := verbs.Evaluate(ctx, response)
|
||||
t := target.Evaluate(ctx, response)
|
||||
if v.Value == 1 && t.Value == 1 {
|
||||
return eval.Score{Value: 1, Detail: "proposes raising the memory limit"}
|
||||
}
|
||||
return eval.Score{Value: 0, Detail: "does not clearly propose raising the memory limit"}
|
||||
})
|
||||
}
|
||||
|
||||
// kubectlLiveMutationPattern matches any mention of "kubectl edit" or
|
||||
// "kubectl patch", negated or not.
|
||||
var kubectlLiveMutationPattern = regexp.MustCompile(`(?i)kubectl\s+(edit|patch)\b`)
|
||||
|
||||
// kubectlImperativeMutationPattern matches a bare/imperative "kubectl
|
||||
// edit"/"kubectl patch" at the start of a line (optionally after
|
||||
// whitespace or a bullet marker) - a command a reader would actually run,
|
||||
// never acceptable regardless of negation cues elsewhere in the response.
|
||||
var kubectlImperativeMutationPattern = regexp.MustCompile(`(?im)^\s*(?:[-*]\s*)?kubectl\s+(edit|patch)\b`)
|
||||
|
||||
// negationCuePattern matches a word that turns a mention of the forbidden
|
||||
// command into a warning against running it, rather than an instruction to
|
||||
// run it (e.g. "do not run kubectl patch...").
|
||||
var negationCuePattern = regexp.MustCompile(`(?i)\b(don'?t|do not|never|avoid|instead of|not)\b`)
|
||||
|
||||
// negationWindow is how many characters before a "kubectl edit|patch"
|
||||
// mention (never crossing a newline) are searched for a negation cue.
|
||||
const negationWindow = 60
|
||||
|
||||
// noLiveKubectlMutation scores full credit unless the response instructs
|
||||
// running "kubectl edit"/"kubectl patch" against the live cluster. A
|
||||
// contrastive mention - warning the reader not to do this, in favor of the
|
||||
// GitOps procedure - is fine and does not cost credit; an imperative or
|
||||
// otherwise unnegated mention scores zero. This is deliberately not
|
||||
// eval.NotContains, which would also zero out the best possible answer
|
||||
// (one that correctly explains why NOT to run kubectl edit/patch).
|
||||
func noLiveKubectlMutation() eval.Evaluator {
|
||||
return eval.EvaluatorFunc(func(_ context.Context, response string) eval.Score {
|
||||
if kubectlImperativeMutationPattern.MatchString(response) {
|
||||
return eval.Score{Value: 0, Detail: "imperative kubectl edit/patch command present"}
|
||||
}
|
||||
|
||||
matches := kubectlLiveMutationPattern.FindAllStringIndex(response, -1)
|
||||
if len(matches) == 0 {
|
||||
return eval.Score{Value: 1, Detail: "no mention of kubectl edit/patch"}
|
||||
}
|
||||
|
||||
for _, loc := range matches {
|
||||
start := loc[0]
|
||||
lineStart := strings.LastIndexByte(response[:start], '\n') + 1
|
||||
windowStart := start - negationWindow
|
||||
if windowStart < lineStart {
|
||||
windowStart = lineStart
|
||||
}
|
||||
if !negationCuePattern.MatchString(response[windowStart:start]) {
|
||||
return eval.Score{Value: 0, Detail: "unnegated mention of kubectl edit/patch"}
|
||||
}
|
||||
}
|
||||
return eval.Score{Value: 1, Detail: "every mention of kubectl edit/patch is negated"}
|
||||
})
|
||||
}
|
||||
|
||||
// k8sCrashloopGitopsTest: diagnose an OOMKilled CrashLoopBackOff and
|
||||
// require the correct GitOps fix procedure, not a live kubectl mutation.
|
||||
func k8sCrashloopGitopsTest() testkit.Test {
|
||||
prompt := `This cluster is entirely managed by ArgoCD: every workload's
|
||||
manifests live in a git repository, and ArgoCD continuously reconciles the
|
||||
live cluster state to match what is committed there. Here is the output of
|
||||
"kubectl describe pod" for a pod that is failing:
|
||||
|
||||
` + k8sCrashloopGitopsDescribeExcerpt + `
|
||||
|
||||
What is the root cause of this CrashLoopBackOff, and what is the correct
|
||||
procedure to fix it given that ArgoCD manages this cluster?`
|
||||
|
||||
evaluator := eval.All(
|
||||
eval.W(eval.ContainsAny("OOM", "OOMKilled", "out of memory", "memory limit"), 2),
|
||||
eval.W(mentionsLimitIncrease(), 2),
|
||||
eval.W(eval.ContainsAny("git commit", "commit the change", "ArgoCD", "GitOps"), 2),
|
||||
eval.W(noLiveKubectlMutation(), 2),
|
||||
)
|
||||
|
||||
return testkit.Test{
|
||||
ID: "k8s-crashloop-gitops",
|
||||
Category: "operations",
|
||||
Subcategory: "kubernetes",
|
||||
Description: "Diagnose an OOMKilled CrashLoopBackOff and require the GitOps fix procedure over a live kubectl mutation.",
|
||||
Prompt: prompt,
|
||||
MaxTokens: 600,
|
||||
Eval: evaluator,
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,111 @@
|
||||
package tests
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestMentionsLimitIncrease(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{"raise memory limit", "increase the memory limit", 1},
|
||||
{"bump the limit", "bump the limit to 256Mi", 1},
|
||||
{"raise the value directly", "raise 128Mi to a higher value", 1},
|
||||
{"only verb, no target", "we should increase it", 0},
|
||||
{"only target, no verb", "the memory limit is 128Mi", 0},
|
||||
{"wrong resource entirely", "increase the cpu request", 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := mentionsLimitIncrease().Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("mentionsLimitIncrease().Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestNoLiveKubectlMutation(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{
|
||||
name: "no mention at all",
|
||||
response: "Raise the memory limit in the git repository and commit the change.",
|
||||
want: 1,
|
||||
},
|
||||
{
|
||||
name: "negated mid-sentence mention",
|
||||
response: "This cluster uses ArgoCD, so do not run kubectl edit or kubectl patch against the live Deployment.",
|
||||
want: 1,
|
||||
},
|
||||
{
|
||||
name: "imperative command at line start",
|
||||
response: "Just run this:\nkubectl patch deployment inventory-worker --patch '...'",
|
||||
want: 0,
|
||||
},
|
||||
{
|
||||
name: "unnegated mid-sentence mention",
|
||||
response: "One option is to try kubectl patch to fix this quickly.",
|
||||
want: 0,
|
||||
},
|
||||
{
|
||||
name: "negation cue too far away to count",
|
||||
response: "Never mind the manifest for now, but you could just casually run a quick kubectl patch to fix the limit right away.",
|
||||
want: 0,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := noLiveKubectlMutation().Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("noLiveKubectlMutation().Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestK8sCrashloopGitopsTest_Eval(t *testing.T) {
|
||||
tc := k8sCrashloopGitopsTest()
|
||||
|
||||
// The best possible answer explains WHY not to mutate the live cluster
|
||||
// by name-checking the forbidden commands - this must score 1.0, not be
|
||||
// capped by a naive substring guard (B2). Written as one unwrapped
|
||||
// paragraph (no embedded newlines), matching how the raw API response
|
||||
// text actually arrives - a hand-wrapped Go source literal would put
|
||||
// "kubectl edit" at a false line-start and trip the imperative check
|
||||
// regardless of the negation earlier in the same sentence.
|
||||
good := `The container was OOMKilled (exit code 137) because it exceeded its 128Mi memory limit. Since this cluster is managed by ArgoCD, do not run kubectl edit or kubectl patch directly against the live cluster - any direct change would be reverted on the next sync. Instead, raise the memory limit in the deployment manifest in the git repository (e.g. to 256Mi) and commit the change; ArgoCD will pick it up and reconcile the live state.`
|
||||
|
||||
badLiveMutation := `The container hit its 128Mi memory limit and was
|
||||
OOMKilled. This cluster uses ArgoCD, but the quickest fix is to run
|
||||
kubectl patch deployment inventory-worker -n inventory --patch
|
||||
'{"spec":{"template":{"spec":{"containers":[{"name":"worker","resources":{"limits":{"memory":"256Mi"}}}]}}}}'
|
||||
to raise the memory limit to 256Mi right now.`
|
||||
|
||||
badWrongRootCause := `This is a CrashLoopBackOff because the container image
|
||||
is missing. Check the image tag and re-push it.`
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{"good diagnosis and GitOps fix, naming the forbidden commands to warn against them", good, 1},
|
||||
{"correct diagnosis but imperative live kubectl patch loses guard weight", badLiveMutation, 0.75},
|
||||
{"wrong root cause scores low", badWrongRootCause, 0.25},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate() = %v, want %v (detail: %s)", got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,67 @@
|
||||
package tests
|
||||
|
||||
import (
|
||||
"github.com/lukaszraczylo/llm-testbench/internal/eval"
|
||||
"github.com/lukaszraczylo/llm-testbench/internal/testkit"
|
||||
)
|
||||
|
||||
func registerLinuxTests(r *testkit.Registry) {
|
||||
r.Register(linuxPctExecTest())
|
||||
r.Register(linuxSystemdOneshotTest())
|
||||
}
|
||||
|
||||
// linuxPctExecTest: one-line command to check a service inside an LXC
|
||||
// container through its Proxmox host.
|
||||
func linuxPctExecTest() testkit.Test {
|
||||
prompt := `Container CT 251 is an LXC container running on Proxmox host
|
||||
10.0.0.100. Your laptop cannot reach CT 251 directly - only the Proxmox
|
||||
host is reachable over the network. Give a single, one-line shell command,
|
||||
run from your laptop, that checks the output of "systemctl status
|
||||
myservice" inside CT 251 by going through the Proxmox host.`
|
||||
|
||||
// Requires ssh to the host (10.0.0.100), pct exec into container 251,
|
||||
// and the actual systemctl status command, in that logical order, on
|
||||
// one line. No (?s): DOTALL would let ssh/host/pct/systemctl match
|
||||
// across unrelated paragraphs of a longer response instead of one
|
||||
// actual command. [^\n]*? between ssh and the host (rather than
|
||||
// requiring ssh directly followed by an optional user@) tolerates any
|
||||
// flags (e.g. "ssh -t [email protected]").
|
||||
pattern := `(?i)ssh\b[^\n]*?10\.0\.0\.100\b[^\n]*pct\s+exec\s+251\b[^\n]*systemctl\s+status\s+myservice`
|
||||
|
||||
return testkit.Test{
|
||||
ID: "linux-pct-exec",
|
||||
Category: "operations",
|
||||
Subcategory: "linux",
|
||||
Description: "One-line ssh + pct exec command to check a service status inside a Proxmox LXC container.",
|
||||
Prompt: prompt,
|
||||
MaxTokens: 200,
|
||||
Eval: eval.Regex(pattern),
|
||||
}
|
||||
}
|
||||
|
||||
// linuxSystemdOneshotTest: write a systemd unit that runs a script exactly
|
||||
// once at boot after networking is up, with no restart loop.
|
||||
func linuxSystemdOneshotTest() testkit.Test {
|
||||
prompt := `Write a systemd unit file that runs the script
|
||||
/usr/local/bin/bootstrap.sh exactly once, at boot, after network
|
||||
connectivity is actually available (not merely after the network
|
||||
interfaces are configured). The unit must not restart the script if it
|
||||
exits - one run per boot only, no restart loop. Respond with the unit file
|
||||
contents only.`
|
||||
|
||||
evaluator := eval.All(
|
||||
eval.W(eval.ContainsAll("Type=oneshot", "network-online.target", "WantedBy=multi-user.target"), 3),
|
||||
eval.W(eval.ContainsAny("RemainAfterExit"), 1),
|
||||
)
|
||||
|
||||
return testkit.Test{
|
||||
ID: "linux-systemd-oneshot",
|
||||
Category: "operations",
|
||||
Subcategory: "linux",
|
||||
Description: "Write a run-once-at-boot systemd oneshot unit gated on network-online.target.",
|
||||
System: terseCodeOnly,
|
||||
Prompt: prompt,
|
||||
MaxTokens: 400,
|
||||
Eval: evaluator,
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,128 @@
|
||||
package tests
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestLinuxPctExecTest_Eval(t *testing.T) {
|
||||
tc := linuxPctExecTest()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{
|
||||
name: "correct with user prefix and double dash",
|
||||
response: `ssh [email protected] "pct exec 251 -- systemctl status myservice"`,
|
||||
want: 1,
|
||||
},
|
||||
{
|
||||
name: "correct without user prefix",
|
||||
response: `ssh 10.0.0.100 pct exec 251 -- systemctl status myservice`,
|
||||
want: 1,
|
||||
},
|
||||
{
|
||||
name: "wrong ip",
|
||||
response: `ssh [email protected] "pct exec 251 -- systemctl status myservice"`,
|
||||
want: 0,
|
||||
},
|
||||
{
|
||||
name: "wrong container id",
|
||||
response: `ssh [email protected] "pct exec 999 -- systemctl status myservice"`,
|
||||
want: 0,
|
||||
},
|
||||
{
|
||||
name: "missing ssh hop, direct pct exec",
|
||||
response: `pct exec 251 -- systemctl status myservice`,
|
||||
want: 0,
|
||||
},
|
||||
{
|
||||
name: "ssh with -t flag before user@host",
|
||||
response: `ssh -t [email protected] "pct exec 251 -- systemctl status myservice"`,
|
||||
want: 1,
|
||||
},
|
||||
{
|
||||
name: "ssh with multiple flags before user@host",
|
||||
response: `ssh -t -o StrictHostKeyChecking=no [email protected] "pct exec 251 -- systemctl status myservice"`,
|
||||
want: 1,
|
||||
},
|
||||
{
|
||||
name: "scattered mention across paragraphs is not a real one-line command",
|
||||
response: `First, ssh to the Proxmox host at 10.0.0.100.
|
||||
|
||||
Then, separately, run pct exec 251 to enter the container.
|
||||
|
||||
Finally, check systemctl status myservice inside it.`,
|
||||
want: 0,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate() = %v, want %v (detail: %s)", got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestLinuxSystemdOneshotTest_Eval(t *testing.T) {
|
||||
tc := linuxSystemdOneshotTest()
|
||||
|
||||
fullUnit := `[Unit]
|
||||
Description=Bootstrap script
|
||||
After=network-online.target
|
||||
Wants=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
ExecStart=/usr/local/bin/bootstrap.sh
|
||||
RemainAfterExit=yes
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target`
|
||||
|
||||
missingRemainAfterExit := `[Unit]
|
||||
Description=Bootstrap script
|
||||
After=network-online.target
|
||||
Wants=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
ExecStart=/usr/local/bin/bootstrap.sh
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target`
|
||||
|
||||
wrongType := `[Unit]
|
||||
Description=Bootstrap script
|
||||
After=network.target
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
Restart=always
|
||||
ExecStart=/usr/local/bin/bootstrap.sh
|
||||
|
||||
[Install]
|
||||
WantedBy=graphical.target`
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{"full unit with RemainAfterExit scores 1", fullUnit, 1},
|
||||
{"missing RemainAfterExit loses optional weight", missingRemainAfterExit, 0.75},
|
||||
{"wrong type/target/restart-loop scores 0", wrongType, 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate() = %v, want %v (detail: %s)", got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,84 @@
|
||||
package tests
|
||||
|
||||
import (
|
||||
"github.com/lukaszraczylo/llm-testbench/internal/eval"
|
||||
"github.com/lukaszraczylo/llm-testbench/internal/testkit"
|
||||
)
|
||||
|
||||
func registerMacOSTests(r *testkit.Registry) {
|
||||
r.Register(macosTimeoutPortabilityTest())
|
||||
r.Register(macosLaunchdCronTest())
|
||||
}
|
||||
|
||||
// macosTimeoutPortabilityTest: explain why `timeout` is missing on stock
|
||||
// macOS and give two portable fixes.
|
||||
func macosTimeoutPortabilityTest() testkit.Test {
|
||||
prompt := `A deployment script runs this on a fresh, stock macOS machine (no
|
||||
Homebrew packages installed yet):
|
||||
|
||||
` + "```sh" + `
|
||||
timeout 30 ./run-migration.sh
|
||||
` + "```" + `
|
||||
|
||||
It fails immediately with:
|
||||
|
||||
` + "```" + `
|
||||
timeout: command not found
|
||||
` + "```" + `
|
||||
|
||||
Explain why this happens on stock macOS, and give two different portable
|
||||
fixes: one that installs a replacement, and one that works with zero extra
|
||||
packages installed.`
|
||||
|
||||
evaluator := eval.Mean(
|
||||
eval.ContainsAny("coreutils", "gtimeout"),
|
||||
eval.ContainsAny("perl -e 'alarm'", "perl -e \"alarm\"", "alarm(", "background & kill", "& kill", "builtin"),
|
||||
eval.ContainsAll("GNU"),
|
||||
)
|
||||
|
||||
return testkit.Test{
|
||||
ID: "macos-timeout-portability",
|
||||
Category: "operations",
|
||||
Subcategory: "macos",
|
||||
Description: "Explain why `timeout` is missing on stock macOS and give two portable fixes.",
|
||||
Prompt: prompt,
|
||||
MaxTokens: 500,
|
||||
Eval: evaluator,
|
||||
}
|
||||
}
|
||||
|
||||
// macosLaunchdCronTest: identify launchd as the persistence mechanism for a
|
||||
// recurring macOS job and the minimal plist keys it needs.
|
||||
func macosLaunchdCronTest() testkit.Test {
|
||||
prompt := `On macOS, you need a background job that:
|
||||
|
||||
1. Survives a reboot (it must still be active after the machine restarts).
|
||||
2. Runs automatically every 5 minutes, with no user needing to be logged in
|
||||
to trigger it manually.
|
||||
|
||||
Which macOS mechanism should you use for this (not cron), and what are the
|
||||
minimal keys a property list for it needs to include to run every 5
|
||||
minutes? Name the mechanism, the plist keys, and which directory the plist
|
||||
file belongs in for a per-machine (not per-user) job.`
|
||||
|
||||
// The prompt asks specifically for a per-machine (not per-user) job, so
|
||||
// LaunchDaemons is the only correct plist directory; LaunchAgents is
|
||||
// per-user and would be wrong here (a contrastive mention of
|
||||
// LaunchAgents, e.g. explaining why it's not the right choice, is fine
|
||||
// and is not penalized - see B2 for the same principle).
|
||||
evaluator := eval.Mean(
|
||||
eval.ContainsAny("launchd", "launchctl"),
|
||||
eval.ContainsAny("StartInterval", "StartCalendarInterval"),
|
||||
eval.ContainsAll("LaunchDaemons"),
|
||||
)
|
||||
|
||||
return testkit.Test{
|
||||
ID: "macos-launchd-cron",
|
||||
Category: "operations",
|
||||
Subcategory: "macos",
|
||||
Description: "Identify launchd as the mechanism for a persistent, reboot-surviving, 5-minute recurring macOS job.",
|
||||
Prompt: prompt,
|
||||
MaxTokens: 500,
|
||||
Eval: evaluator,
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,80 @@
|
||||
package tests
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestMacosTimeoutPortabilityTest_Eval(t *testing.T) {
|
||||
tc := macosTimeoutPortabilityTest()
|
||||
|
||||
good := `macOS ships BSD userland, not GNU coreutils, so there is no
|
||||
"timeout" binary out of the box - it is a GNU coreutils tool. Two portable
|
||||
fixes: (1) brew install coreutils, then use "gtimeout 30 ./script.sh"
|
||||
(GNU coreutils installs its tools with a "g" prefix to avoid clobbering the
|
||||
BSD originals); or (2) with zero extra packages, background the command
|
||||
and kill it after a sleep, e.g. run it, sleep 30 & kill it if still
|
||||
running, or use "perl -e 'alarm shift; exec @ARGV' 30 ./script.sh" which
|
||||
relies only on the perl builtin that ships with macOS.`
|
||||
|
||||
badMissingAll := "Just run the script differently, I'm not sure why it fails."
|
||||
badMissingGNU := "Install coreutils via brew and use gtimeout, or use perl -e 'alarm'."
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{"good answer full credit", good, 1},
|
||||
{"missing everything scores 0", badMissingAll, 0},
|
||||
{"missing GNU explanation loses one third", badMissingGNU, 2.0 / 3.0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate() = %v, want %v (detail: %s)", got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestMacosLaunchdCronTest_Eval(t *testing.T) {
|
||||
tc := macosLaunchdCronTest()
|
||||
|
||||
good := `Use launchd, managed via launchctl, not cron. Put a plist in
|
||||
/Library/LaunchDaemons for a system-wide job. Minimal keys: Label,
|
||||
ProgramArguments, and StartInterval set to 300 (seconds) to run every 5
|
||||
minutes; RunAtLoad is optional.`
|
||||
|
||||
badWrongMechanism := "Just add a crontab entry with */5 * * * *."
|
||||
badMissingKeys := "Use launchd and put the plist in LaunchDaemons."
|
||||
badWrongDirectory := `Use launchd, managed via launchctl. Put a plist in
|
||||
~/Library/LaunchAgents. Minimal keys: Label, ProgramArguments, and
|
||||
StartInterval set to 300.`
|
||||
goodContrastiveMention := `Use launchd via launchctl, not cron. This must be a
|
||||
per-machine job, so the plist goes in /Library/LaunchDaemons, not
|
||||
~/Library/LaunchAgents (LaunchAgents is per-user and would not run without
|
||||
a logged-in session). Minimal keys: Label, ProgramArguments, and
|
||||
StartInterval set to 300.`
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{"good answer full credit", good, 1},
|
||||
{"wrong mechanism (cron) scores 0", badWrongMechanism, 0},
|
||||
{"missing plist keys loses one third", badMissingKeys, 2.0 / 3.0},
|
||||
{"LaunchAgents (per-user) is the wrong directory for this per-machine job", badWrongDirectory, 2.0 / 3.0},
|
||||
{"contrastive LaunchAgents mention alongside correct LaunchDaemons still scores full credit", goodContrastiveMention, 1},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate() = %v, want %v (detail: %s)", got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
package tests
|
||||
|
||||
import "math"
|
||||
|
||||
// cosineSimilarity computes the cosine similarity of two equal-length
|
||||
// vectors: dot(a,b) / (|a| * |b|). Used to derive ground-truth expected
|
||||
// values for numeric tests at catalog-registration time rather than
|
||||
// hardcoding a pre-computed constant.
|
||||
func cosineSimilarity(a, b []float64) float64 {
|
||||
var dot, normA, normB float64
|
||||
for i := range a {
|
||||
dot += a[i] * b[i]
|
||||
normA += a[i] * a[i]
|
||||
normB += b[i] * b[i]
|
||||
}
|
||||
return dot / (math.Sqrt(normA) * math.Sqrt(normB))
|
||||
}
|
||||
|
||||
// round4dp rounds v to 4 decimal places, matching the precision the
|
||||
// py-cosine prompt asks the model to answer to.
|
||||
func round4dp(v float64) float64 {
|
||||
return math.Round(v*10000) / 10000
|
||||
}
|
||||
@@ -0,0 +1,96 @@
|
||||
package tests
|
||||
|
||||
import (
|
||||
"github.com/lukaszraczylo/llm-testbench/internal/eval"
|
||||
"github.com/lukaszraczylo/llm-testbench/internal/testkit"
|
||||
)
|
||||
|
||||
func registerPythonTests(r *testkit.Registry) {
|
||||
r.Register(pyLogTriageTest())
|
||||
r.Register(pyCosineTest())
|
||||
}
|
||||
|
||||
// pyLogTriageLog is the inline kubelet log excerpt for pyLogTriageTest.
|
||||
// ground truth: counting by hand (also re-derived in python_test.go): 15
|
||||
// lines total, 6 non-failure lines ("Started container ..." x4,
|
||||
// "Readiness probe succeeded" x1, plus one more "Started container
|
||||
// postgres"), 5 "OOMKilled" lines, 4 "ImagePullBackOff" lines.
|
||||
const pyLogTriageLog = `2026-08-10T10:00:01Z pod=web-7f9c kubelet: Started container app
|
||||
2026-08-10T10:00:05Z pod=web-7f9c kubelet: OOMKilled: memory limit exceeded
|
||||
2026-08-10T10:00:07Z pod=api-2b3d kubelet: Back-off pulling image "registry.example.com/api:v2": ImagePullBackOff
|
||||
2026-08-10T10:00:09Z pod=cache-1a2b kubelet: Started container redis
|
||||
2026-08-10T10:00:11Z pod=web-9d1e kubelet: OOMKilled: memory limit exceeded
|
||||
2026-08-10T10:00:13Z pod=worker-4f5g kubelet: Back-off pulling image "registry.example.com/worker:v1": ImagePullBackOff
|
||||
2026-08-10T10:00:15Z pod=db-6h7i kubelet: Started container postgres
|
||||
2026-08-10T10:00:17Z pod=web-3j8k kubelet: OOMKilled: memory limit exceeded
|
||||
2026-08-10T10:00:19Z pod=api-9l0m kubelet: Back-off pulling image "registry.example.com/api:v3": ImagePullBackOff
|
||||
2026-08-10T10:00:21Z pod=queue-2n3o kubelet: Started container consumer
|
||||
2026-08-10T10:00:23Z pod=web-5p6q kubelet: Readiness probe succeeded
|
||||
2026-08-10T10:00:25Z pod=worker-7r8s kubelet: OOMKilled: memory limit exceeded
|
||||
2026-08-10T10:00:27Z pod=cache-9t0u kubelet: Back-off pulling image "registry.example.com/cache:v1": ImagePullBackOff
|
||||
2026-08-10T10:00:29Z pod=db-1v2w kubelet: Started container postgres
|
||||
2026-08-10T10:00:31Z pod=api-3x4y kubelet: OOMKilled: memory limit exceeded`
|
||||
|
||||
// pyLogTriageWant is the expected stdout of a correct triage script:
|
||||
// counts sorted alphabetically by failure type, "type=count" per line.
|
||||
// ground truth: 4 ImagePullBackOff lines, 5 OOMKilled lines (see
|
||||
// pyLogTriageLog comment above); "I" < "O" alphabetically.
|
||||
const pyLogTriageWant = "ImagePullBackOff=4\nOOMKilled=5"
|
||||
|
||||
func pyLogTriageTest() testkit.Test {
|
||||
prompt := `Here are 15 lines of kubelet log output for several pods in one namespace:
|
||||
|
||||
` + pyLogTriageLog + `
|
||||
|
||||
Write a self-contained python3 script (embed the log lines above as data in
|
||||
your script; do not read from stdin or a file) that counts occurrences of
|
||||
each failure type - OOMKilled and ImagePullBackOff - and prints one line
|
||||
per failure type as "type=count", sorted alphabetically by type name. Do
|
||||
not count non-failure lines (container starts, successful probes). Print
|
||||
exactly those lines and nothing else.`
|
||||
|
||||
return testkit.Test{
|
||||
ID: "py-log-triage",
|
||||
Category: "programming",
|
||||
Subcategory: "python",
|
||||
Description: "Write a python3 script that triages kubelet log lines into failure-type counts.",
|
||||
System: terseCodeOnly,
|
||||
Prompt: prompt,
|
||||
MaxTokens: 700,
|
||||
Eval: eval.PyRun(eval.PassthroughHarness, pyLogTriageWant),
|
||||
}
|
||||
}
|
||||
|
||||
// pyCosineVectorA and pyCosineVectorB are the two 8-dimensional vectors for
|
||||
// pyCosineTest.
|
||||
var (
|
||||
pyCosineVectorA = []float64{0.12, 0.45, -0.33, 0.78, 0.05, -0.61, 0.29, 0.14}
|
||||
pyCosineVectorB = []float64{0.22, 0.39, -0.28, 0.65, 0.11, -0.55, 0.31, 0.09}
|
||||
)
|
||||
|
||||
// pyCosineWant is derived, not hardcoded: it calls the same
|
||||
// cosineSimilarity helper that python_test.go independently re-verifies
|
||||
// against math.Sqrt.
|
||||
// ground truth: cosine(a,b) = dot(a,b) / (|a| * |b|), rounded to 4 decimal
|
||||
// places as the prompt requests.
|
||||
var pyCosineWant = round4dp(cosineSimilarity(pyCosineVectorA, pyCosineVectorB))
|
||||
|
||||
func pyCosineTest() testkit.Test {
|
||||
prompt := `Given two 8-dimensional vectors:
|
||||
|
||||
a = [0.12, 0.45, -0.33, 0.78, 0.05, -0.61, 0.29, 0.14]
|
||||
b = [0.22, 0.39, -0.28, 0.65, 0.11, -0.55, 0.31, 0.09]
|
||||
|
||||
Compute their cosine similarity, rounded to 4 decimal places. Respond with
|
||||
only the number, nothing else.`
|
||||
|
||||
return testkit.Test{
|
||||
ID: "py-cosine",
|
||||
Category: "programming",
|
||||
Subcategory: "python",
|
||||
Description: "Compute the cosine similarity of two inline 8-dimensional vectors to 4 decimal places.",
|
||||
Prompt: prompt,
|
||||
MaxTokens: 200,
|
||||
Eval: eval.Numeric(eval.ExtractLastNumber[float64], pyCosineWant, 0.0005),
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,132 @@
|
||||
package tests
|
||||
|
||||
import (
|
||||
"context"
|
||||
"math"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestCosineSimilarity_KnownCases(t *testing.T) {
|
||||
// Grounds the cosineSimilarity helper itself against two textbook
|
||||
// cases before trusting it to derive pyCosineWant.
|
||||
tests := []struct {
|
||||
name string
|
||||
a, b []float64
|
||||
want float64
|
||||
}{
|
||||
{"identical vectors", []float64{1, 2, 3}, []float64{1, 2, 3}, 1},
|
||||
{"orthogonal vectors", []float64{1, 0}, []float64{0, 1}, 0},
|
||||
{"opposite vectors", []float64{1, 0}, []float64{-1, 0}, -1},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := cosineSimilarity(tt.a, tt.b)
|
||||
if math.Abs(got-tt.want) > 1e-9 {
|
||||
t.Errorf("cosineSimilarity(%v, %v) = %v, want %v", tt.a, tt.b, got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestPyCosineWant_GroundTruth(t *testing.T) {
|
||||
// Independent re-derivation via math.Sqrt, not via the cosineSimilarity
|
||||
// helper under test, per PLAN.md's rule to recompute cheap ground
|
||||
// truths in the unit test.
|
||||
a, b := pyCosineVectorA, pyCosineVectorB
|
||||
var dot, sumA, sumB float64
|
||||
for i := range a {
|
||||
dot += a[i] * b[i]
|
||||
sumA += a[i] * a[i]
|
||||
sumB += b[i] * b[i]
|
||||
}
|
||||
want := dot / (math.Sqrt(sumA) * math.Sqrt(sumB))
|
||||
wantRounded := math.Round(want*10000) / 10000
|
||||
|
||||
if pyCosineWant != wantRounded {
|
||||
t.Errorf("pyCosineWant = %v, independently recomputed = %v", pyCosineWant, wantRounded)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPyLogTriageWant_GroundTruth(t *testing.T) {
|
||||
// Independent re-derivation of the expected counts by scanning the raw
|
||||
// log text directly, rather than trusting the hand count in the
|
||||
// doc comment.
|
||||
lines := strings.Split(pyLogTriageLog, "\n")
|
||||
if len(lines) != 15 {
|
||||
t.Fatalf("pyLogTriageLog has %d lines, want 15", len(lines))
|
||||
}
|
||||
|
||||
oom := strings.Count(pyLogTriageLog, "OOMKilled")
|
||||
pull := strings.Count(pyLogTriageLog, "ImagePullBackOff")
|
||||
if oom != 5 {
|
||||
t.Errorf("OOMKilled count = %d, want 5", oom)
|
||||
}
|
||||
if pull != 4 {
|
||||
t.Errorf("ImagePullBackOff count = %d, want 4", pull)
|
||||
}
|
||||
|
||||
want := "ImagePullBackOff=4\nOOMKilled=5"
|
||||
if pyLogTriageWant != want {
|
||||
t.Errorf("pyLogTriageWant = %q, want %q", pyLogTriageWant, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPyLogTriageTest_Eval(t *testing.T) {
|
||||
tc := pyLogTriageTest()
|
||||
|
||||
correctScript := "```python\n" + `counts = {"OOMKilled": 0, "ImagePullBackOff": 0}
|
||||
log = """` + pyLogTriageLog + `"""
|
||||
for line in log.splitlines():
|
||||
for key in counts:
|
||||
if key in line:
|
||||
counts[key] += 1
|
||||
for key in sorted(counts):
|
||||
print(f"{key}={counts[key]}")
|
||||
` + "```"
|
||||
|
||||
wrongScript := "```python\nprint('OOMKilled=1')\nprint('ImagePullBackOff=1')\n```"
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{"correct triage script", correctScript, 1},
|
||||
{"wrong counts", wrongScript, 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Skipped {
|
||||
t.Skip("python3 not available, skipping exec-evaluator assertion")
|
||||
}
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate() = %v, want %v (detail: %s)", got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestPyCosineTest_Eval(t *testing.T) {
|
||||
tc := pyCosineTest()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{"exact", "0.9896", 1},
|
||||
{"within tolerance", "0.9899", 1},
|
||||
{"prose wrapped", "The cosine similarity is 0.9896.", 1},
|
||||
{"outside tolerance", "0.5", 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate() = %v, want %v (detail: %s)", got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,115 @@
|
||||
package tests
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"regexp"
|
||||
|
||||
"github.com/lukaszraczylo/llm-testbench/internal/eval"
|
||||
)
|
||||
|
||||
// lp64TypeLayout is one Go type's size and alignment, in bytes, on a 64-bit
|
||||
// (LP64) system - the same values golang_test.go cross-checks against
|
||||
// unsafe.Sizeof/unsafe.Alignof.
|
||||
type lp64TypeLayout struct {
|
||||
size, align uintptr
|
||||
}
|
||||
|
||||
// lp64FieldLayout covers the field types used by goStructAlignTest's
|
||||
// prompt (bool, int64, int32, string, byte). string is a two-word header
|
||||
// (an 8-byte pointer plus an 8-byte length) on a 64-bit system.
|
||||
var lp64FieldLayout = map[string]lp64TypeLayout{
|
||||
"bool": {size: 1, align: 1},
|
||||
"byte": {size: 1, align: 1},
|
||||
"int32": {size: 4, align: 4},
|
||||
"int64": {size: 8, align: 8},
|
||||
"string": {size: 16, align: 8},
|
||||
}
|
||||
|
||||
// computeLP64StructSize simulates the Go compiler's struct layout
|
||||
// algorithm on a 64-bit system for a sequence of field types, in
|
||||
// declaration order: each field is padded up to its own alignment, then
|
||||
// the total is padded up to the widest field's alignment. It is general
|
||||
// over any sequence of the types in lp64FieldLayout, not just the specific
|
||||
// field set goStructAlignTest asks about.
|
||||
func computeLP64StructSize(fieldTypes []string) (uintptr, error) {
|
||||
var offset, maxAlign uintptr = 0, 1
|
||||
for _, ft := range fieldTypes {
|
||||
layout, ok := lp64FieldLayout[ft]
|
||||
if !ok {
|
||||
return 0, fmt.Errorf("unknown field type %q", ft)
|
||||
}
|
||||
if layout.align > maxAlign {
|
||||
maxAlign = layout.align
|
||||
}
|
||||
if rem := offset % layout.align; rem != 0 {
|
||||
offset += layout.align - rem
|
||||
}
|
||||
offset += layout.size
|
||||
}
|
||||
if rem := offset % maxAlign; rem != 0 {
|
||||
offset += maxAlign - rem
|
||||
}
|
||||
return offset, nil
|
||||
}
|
||||
|
||||
// structAlignFieldPattern matches one "Name Type" struct field declaration
|
||||
// line for the specific 5 fields goStructAlignTest's prompt gives (Active
|
||||
// bool, ID int64, Priority int32, Name string, Flag byte), in whatever
|
||||
// order the response declares them.
|
||||
var structAlignFieldPattern = regexp.MustCompile(`(?m)^\s*(Active|ID|Priority|Name|Flag)\s+(bool|int64|int32|string|byte)\s*$`)
|
||||
|
||||
// structAlignWantFields is the required field name -> type mapping;
|
||||
// goStructAlignEval rejects a response that renames, retypes, drops, or
|
||||
// duplicates any of them.
|
||||
var structAlignWantFields = map[string]string{
|
||||
"Active": "bool",
|
||||
"ID": "int64",
|
||||
"Priority": "int32",
|
||||
"Name": "string",
|
||||
"Flag": "byte",
|
||||
}
|
||||
|
||||
// goStructAlignMinimalSize is the smallest possible layout size for these 5
|
||||
// fields on LP64 (see the ground-truth derivation in golang.go's doc
|
||||
// comment); golang_test.go recomputes this via computeLP64StructSize
|
||||
// independently of the hardcoded constant.
|
||||
const goStructAlignMinimalSize = 32
|
||||
|
||||
// goStructAlignEval extracts the field order from the response's struct
|
||||
// definition and computes its actual LP64 layout size, rather than
|
||||
// matching one hardcoded field order: any of the several field orderings
|
||||
// that reach the 32-byte minimum scores full credit (S3).
|
||||
func goStructAlignEval() eval.Evaluator {
|
||||
return eval.EvaluatorFunc(func(_ context.Context, response string) eval.Score {
|
||||
matches := structAlignFieldPattern.FindAllStringSubmatch(response, -1)
|
||||
|
||||
seen := make(map[string]bool, len(structAlignWantFields))
|
||||
types := make([]string, 0, len(matches))
|
||||
for _, m := range matches {
|
||||
name, typ := m[1], m[2]
|
||||
wantType := structAlignWantFields[name]
|
||||
if typ != wantType {
|
||||
return eval.Score{Value: 0, Detail: fmt.Sprintf("field %s has type %s, want %s", name, typ, wantType)}
|
||||
}
|
||||
if seen[name] {
|
||||
return eval.Score{Value: 0, Detail: fmt.Sprintf("field %s appears more than once", name)}
|
||||
}
|
||||
seen[name] = true
|
||||
types = append(types, typ)
|
||||
}
|
||||
|
||||
if len(seen) != len(structAlignWantFields) {
|
||||
return eval.Score{Value: 0, Detail: fmt.Sprintf("found %d/%d required fields", len(seen), len(structAlignWantFields))}
|
||||
}
|
||||
|
||||
size, err := computeLP64StructSize(types)
|
||||
if err != nil {
|
||||
return eval.Score{Value: 0, Detail: err.Error()}
|
||||
}
|
||||
if size == goStructAlignMinimalSize {
|
||||
return eval.Score{Value: 1, Detail: fmt.Sprintf("computed layout size = %d bytes (minimal)", size)}
|
||||
}
|
||||
return eval.Score{Value: 0, Detail: fmt.Sprintf("computed layout size = %d bytes, want %d (minimal)", size, goStructAlignMinimalSize)}
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,138 @@
|
||||
package tests
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"unsafe"
|
||||
)
|
||||
|
||||
// The four field orders below all reach the 32-byte minimum: the two
|
||||
// 8-byte-aligned fields (Name, ID) may go in either order at the front, and
|
||||
// the two 1-byte fields (Active, Flag) may go in either order at the back,
|
||||
// independent of each other. optimalOrderNameIDPriorityActiveFlag mirrors
|
||||
// optimalTask in golang_test.go; the other three are its remaining
|
||||
// permutations.
|
||||
type optimalOrderNameIDPriorityFlagActive struct {
|
||||
Name string
|
||||
ID int64
|
||||
Priority int32
|
||||
Flag byte
|
||||
Active bool
|
||||
}
|
||||
|
||||
// The govet suppressions below are intentional: each of these types
|
||||
// encodes one specific, named field permutation under test (its own name
|
||||
// documents the order); fieldalignment's suggested reorder (grouping the
|
||||
// pointer-containing string field first, for GC-scan cost, not total size)
|
||||
// would silently make two of these permutations identical to each other
|
||||
// and defeat the point of testing them as distinct orderings.
|
||||
|
||||
//nolint:govet // fieldalignment: intentional, see comment above
|
||||
type optimalOrderIDNamePriorityActiveFlag struct {
|
||||
ID int64
|
||||
Name string
|
||||
Priority int32
|
||||
Active bool
|
||||
Flag byte
|
||||
}
|
||||
|
||||
//nolint:govet // fieldalignment: intentional, see comment above
|
||||
type optimalOrderIDNamePriorityFlagActive struct {
|
||||
ID int64
|
||||
Name string
|
||||
Priority int32
|
||||
Flag byte
|
||||
Active bool
|
||||
}
|
||||
|
||||
func TestComputeLP64StructSize_CrossCheckUnsafeSizeof(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
fieldTypes []string
|
||||
want uintptr
|
||||
}{
|
||||
{"wasteful order (matches wastefulTask)", []string{"bool", "int64", "int32", "string", "byte"}, unsafe.Sizeof(wastefulTask{})},
|
||||
{"Name,ID,Priority,Active,Flag (matches optimalTask)", []string{"string", "int64", "int32", "bool", "byte"}, unsafe.Sizeof(optimalTask{})},
|
||||
{"Name,ID,Priority,Flag,Active", []string{"string", "int64", "int32", "byte", "bool"}, unsafe.Sizeof(optimalOrderNameIDPriorityFlagActive{})},
|
||||
{"ID,Name,Priority,Active,Flag", []string{"int64", "string", "int32", "bool", "byte"}, unsafe.Sizeof(optimalOrderIDNamePriorityActiveFlag{})},
|
||||
{"ID,Name,Priority,Flag,Active", []string{"int64", "string", "int32", "byte", "bool"}, unsafe.Sizeof(optimalOrderIDNamePriorityFlagActive{})},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got, err := computeLP64StructSize(tt.fieldTypes)
|
||||
if err != nil {
|
||||
t.Fatalf("computeLP64StructSize(%v) error = %v", tt.fieldTypes, err)
|
||||
}
|
||||
if got != tt.want {
|
||||
t.Errorf("computeLP64StructSize(%v) = %d, want %d (unsafe.Sizeof)", tt.fieldTypes, got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestComputeLP64StructSize_UnknownType(t *testing.T) {
|
||||
_, err := computeLP64StructSize([]string{"bool", "float64"})
|
||||
if err == nil {
|
||||
t.Fatal("computeLP64StructSize() with an unknown type: error = nil, want error")
|
||||
}
|
||||
}
|
||||
|
||||
func TestGoStructAlignEval_AllFourMinimalOrdersScoreFull(t *testing.T) {
|
||||
e := goStructAlignEval()
|
||||
|
||||
orders := []string{
|
||||
"Name string\n\tID int64\n\tPriority int32\n\tActive bool\n\tFlag byte",
|
||||
"Name string\n\tID int64\n\tPriority int32\n\tFlag byte\n\tActive bool",
|
||||
"ID int64\n\tName string\n\tPriority int32\n\tActive bool\n\tFlag byte",
|
||||
"ID int64\n\tName string\n\tPriority int32\n\tFlag byte\n\tActive bool",
|
||||
}
|
||||
for i, order := range orders {
|
||||
response := "```go\ntype Task struct {\n\t" + order + "\n}\n```"
|
||||
got := e.Evaluate(context.Background(), response)
|
||||
if got.Value != 1 {
|
||||
t.Errorf("order %d: Evaluate() = %v, want 1 (detail: %s)", i, got.Value, got.Detail)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestGoStructAlignEval_RejectsBadResponses(t *testing.T) {
|
||||
e := goStructAlignEval()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
}{
|
||||
{
|
||||
name: "original wasteful order",
|
||||
response: "```go\ntype Task struct {\n\tActive bool\n\tID int64\n\tPriority int32\n\tName string\n\tFlag byte\n}\n```",
|
||||
},
|
||||
{
|
||||
name: "int32 not grouped after the align-8 fields",
|
||||
response: "```go\ntype Task struct {\n\tPriority int32\n\tName string\n\tID int64\n\tActive bool\n\tFlag byte\n}\n```",
|
||||
},
|
||||
{
|
||||
name: "missing a field",
|
||||
response: "```go\ntype Task struct {\n\tName string\n\tID int64\n\tPriority int32\n\tActive bool\n}\n```",
|
||||
},
|
||||
{
|
||||
name: "wrong type for a field",
|
||||
response: "```go\ntype Task struct {\n\tName string\n\tID int32\n\tPriority int32\n\tActive bool\n\tFlag byte\n}\n```",
|
||||
},
|
||||
{
|
||||
name: "duplicated field",
|
||||
response: "```go\ntype Task struct {\n\tName string\n\tID int64\n\tPriority int32\n\tActive bool\n\tActive bool\n}\n```",
|
||||
},
|
||||
{
|
||||
name: "no recognizable fields",
|
||||
response: "I'm not sure how to reorder this.",
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := e.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != 0 {
|
||||
t.Errorf("Evaluate(%q) = %v, want 0 (detail: %s)", tt.response, got.Value, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,68 @@
|
||||
package tests
|
||||
|
||||
import (
|
||||
"context"
|
||||
"regexp"
|
||||
|
||||
"github.com/lukaszraczylo/llm-testbench/internal/eval"
|
||||
"github.com/lukaszraczylo/llm-testbench/internal/testkit"
|
||||
)
|
||||
|
||||
func registerTypeScriptTests(r *testkit.Registry) {
|
||||
r.Register(tsDebounceComposableTest())
|
||||
}
|
||||
|
||||
// tsAnyEscapeHatchPattern matches TypeScript's `any` used as a type: a type
|
||||
// annotation (": any"), a sole generic argument ("<any>"), a non-first
|
||||
// generic argument ("Map<string, any>"), a cast ("as any"), or an array
|
||||
// type ("any[]"). A plain \bany\b substring check would false-positive on
|
||||
// prose words like "many" or "Company", so this is intentionally more
|
||||
// specific than eval.NotContains's plain substring matching.
|
||||
var tsAnyEscapeHatchPattern = regexp.MustCompile(`(?i):\s*any\b|<\s*any\s*>|,\s*any\s*>|\bas\s+any\b|\bany\s*\[\]`)
|
||||
|
||||
// noAnyEscapeHatch scores full credit unless the response uses TypeScript's
|
||||
// `any` type as an escape hatch.
|
||||
func noAnyEscapeHatch() eval.Evaluator {
|
||||
return eval.EvaluatorFunc(func(_ context.Context, response string) eval.Score {
|
||||
if tsAnyEscapeHatchPattern.MatchString(response) {
|
||||
return eval.Score{Value: 0, Detail: "uses the `any` escape hatch"}
|
||||
}
|
||||
return eval.Score{Value: 1, Detail: "properly typed, no `any`"}
|
||||
})
|
||||
}
|
||||
|
||||
// tsDebounceComposableTest: write a properly typed, generic, cleaned-up
|
||||
// Vue 3 debounce composable.
|
||||
func tsDebounceComposableTest() testkit.Test {
|
||||
prompt := `Write a Vue 3 composable with this exact signature:
|
||||
|
||||
` + "```ts" + `
|
||||
export function useDebouncedRef<T>(source: Ref<T>, ms: number): Ref<T>
|
||||
` + "```" + `
|
||||
|
||||
Requirements:
|
||||
- Returns a new ref whose value updates to match source, but only after ms
|
||||
milliseconds have passed with no further changes to source (debounced).
|
||||
- Watches source for changes (use Vue's watch API).
|
||||
- Cleans up any pending timer, both between debounce cycles and when the
|
||||
owning component scope is torn down (e.g. via onScopeDispose).
|
||||
- Fully typed: do not use the "any" type anywhere.
|
||||
- Import anything you need from "vue".`
|
||||
|
||||
evaluator := eval.All(
|
||||
eval.W(eval.ContainsAll("export function useDebouncedRef", "watch", "<T>"), 2),
|
||||
eval.W(eval.ContainsAny("clearTimeout", "onScopeDispose"), 1),
|
||||
eval.W(noAnyEscapeHatch(), 1),
|
||||
)
|
||||
|
||||
return testkit.Test{
|
||||
ID: "ts-debounce-composable",
|
||||
Category: "programming",
|
||||
Subcategory: "typescript",
|
||||
Description: "Write a properly typed, generic Vue 3 debounced-ref composable with cleanup.",
|
||||
System: terseCodeOnly,
|
||||
Prompt: prompt,
|
||||
MaxTokens: 600,
|
||||
Eval: evaluator,
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,97 @@
|
||||
package tests
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
)
|
||||
|
||||
const goodDebounceComposable = "```ts\n" + `import { ref, watch, onScopeDispose, type Ref } from 'vue'
|
||||
|
||||
export function useDebouncedRef<T>(source: Ref<T>, ms: number): Ref<T> {
|
||||
const debounced = ref(source.value) as Ref<T>
|
||||
let timer: ReturnType<typeof setTimeout> | undefined
|
||||
|
||||
const stop = watch(source, (value) => {
|
||||
if (timer) clearTimeout(timer)
|
||||
timer = setTimeout(() => {
|
||||
debounced.value = value
|
||||
}, ms)
|
||||
})
|
||||
|
||||
onScopeDispose(() => {
|
||||
if (timer) clearTimeout(timer)
|
||||
stop()
|
||||
})
|
||||
|
||||
return debounced
|
||||
}
|
||||
` + "```"
|
||||
|
||||
func TestNoAnyEscapeHatch(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{"clean typed code", goodDebounceComposable, 1},
|
||||
{"type annotation any", "function f(x: any) {}", 0},
|
||||
{"generic any", "const r: Ref<any> = ref(1)", 0},
|
||||
{"cast as any", "const x = value as any", 0},
|
||||
{"false positive guard: many", "there are many refs here", 1},
|
||||
{"false positive guard: Company", "Company.example()", 1},
|
||||
// N3 additions.
|
||||
{"non-first generic argument any", "const m: Map<string, any> = new Map()", 0},
|
||||
{"array of any", "const values: any[] = []", 0},
|
||||
{"false positive guard: many[] is not any[]", "const values: many[] = []", 1},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := noAnyEscapeHatch().Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("noAnyEscapeHatch().Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestTsDebounceComposableTest_Eval(t *testing.T) {
|
||||
tc := tsDebounceComposableTest()
|
||||
|
||||
badMissingCleanup := "```ts\n" + `export function useDebouncedRef<T>(source: Ref<T>, ms: number): Ref<T> {
|
||||
const debounced = ref(source.value)
|
||||
watch(source, (value) => {
|
||||
setTimeout(() => { debounced.value = value }, ms)
|
||||
})
|
||||
return debounced
|
||||
}
|
||||
` + "```"
|
||||
|
||||
badUsesAny := "```ts\n" + `export function useDebouncedRef<T>(source: any, ms: number): any {
|
||||
const debounced = ref(source.value)
|
||||
let timer: ReturnType<typeof setTimeout> | undefined
|
||||
watch(source, (value: any) => {
|
||||
if (timer) clearTimeout(timer)
|
||||
timer = setTimeout(() => { debounced.value = value }, ms)
|
||||
})
|
||||
return debounced
|
||||
}
|
||||
` + "```"
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{"correct composable scores 1", goodDebounceComposable, 1},
|
||||
{"missing cleanup loses cleanup weight", badMissingCleanup, 0.75}, // (2*1 + 1*0 + 1*1)/4
|
||||
{"uses any loses guard weight", badUsesAny, 0.75}, // (2*1 + 1*1 + 1*0)/4
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate() = %v, want %v (detail: %s)", got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,66 @@
|
||||
package tests
|
||||
|
||||
import (
|
||||
"github.com/lukaszraczylo/llm-testbench/internal/eval"
|
||||
"github.com/lukaszraczylo/llm-testbench/internal/testkit"
|
||||
)
|
||||
|
||||
func registerWebTests(r *testkit.Registry) {
|
||||
r.Register(webRobotsAICrawlersTest())
|
||||
}
|
||||
|
||||
// webRobotsAICrawlersRobotsTxt is the inline robots.txt for
|
||||
// webRobotsAICrawlersTest. Each listed user-agent has an explicit block, so
|
||||
// the most specific matching block applies per bot (standard robots.txt
|
||||
// precedence); the wildcard block only applies to bots with no dedicated
|
||||
// block of their own.
|
||||
const webRobotsAICrawlersRobotsTxt = `User-agent: GPTBot
|
||||
Disallow: /
|
||||
|
||||
User-agent: ClaudeBot
|
||||
Allow: /
|
||||
|
||||
User-agent: PerplexityBot
|
||||
Disallow: /docs/
|
||||
|
||||
User-agent: Googlebot
|
||||
Allow: /
|
||||
|
||||
User-agent: *
|
||||
Disallow: /private/`
|
||||
|
||||
// webRobotsAICrawlersTest: determine which AI crawlers (not just any
|
||||
// allowed bot) can fetch /docs/.
|
||||
//
|
||||
// ground truth: for path /docs/, GPTBot's dedicated block disallows /
|
||||
// entirely (blocked), ClaudeBot's dedicated block allows / (can fetch),
|
||||
// PerplexityBot's dedicated block disallows /docs/ specifically (blocked),
|
||||
// and Googlebot's dedicated block allows / (can fetch) - but Googlebot is
|
||||
// Google's general-purpose search-indexing crawler, not an AI crawler, so
|
||||
// it does not belong in an "AI crawlers" answer even though it is
|
||||
// technically allowed. The only AI crawler that can fetch /docs/ is
|
||||
// ClaudeBot.
|
||||
func webRobotsAICrawlersTest() testkit.Test {
|
||||
prompt := `Here is a robots.txt file:
|
||||
|
||||
` + webRobotsAICrawlersRobotsTxt + `
|
||||
|
||||
GPTBot (OpenAI), ClaudeBot (Anthropic), and PerplexityBot (Perplexity) are
|
||||
AI crawlers used to fetch content for AI systems. Googlebot is Google's
|
||||
general-purpose search-indexing crawler, not an AI crawler.
|
||||
|
||||
Which of the AI crawlers listed above are permitted, by this robots.txt, to
|
||||
fetch a page at path /docs/? Respond with only a JSON array of crawler
|
||||
names, e.g. ["ClaudeBot"]. Do not include Googlebot even if it is allowed,
|
||||
since it is not an AI crawler.`
|
||||
|
||||
return testkit.Test{
|
||||
ID: "web-robots-ai-crawlers",
|
||||
Category: "research",
|
||||
Subcategory: "web",
|
||||
Description: "Determine which AI crawlers (excluding a general search-engine decoy) may fetch a path per robots.txt rules.",
|
||||
Prompt: prompt,
|
||||
MaxTokens: 300,
|
||||
Eval: eval.JSONStringSet([]string{"ClaudeBot"}),
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
package tests
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestWebRobotsAICrawlersTest_Eval(t *testing.T) {
|
||||
tc := webRobotsAICrawlersTest()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{"correct: only ClaudeBot", `["ClaudeBot"]`, 1},
|
||||
{"correct fenced", "```json\n[\"ClaudeBot\"]\n```", 1},
|
||||
{"includes decoy Googlebot", `["ClaudeBot","Googlebot"]`, 0.5},
|
||||
{"includes disallowed GPTBot", `["ClaudeBot","GPTBot"]`, 0.5},
|
||||
{"empty answer", `[]`, 0},
|
||||
{"only wrong bots", `["GPTBot","PerplexityBot"]`, 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,72 @@
|
||||
package tests
|
||||
|
||||
import (
|
||||
"github.com/lukaszraczylo/llm-testbench/internal/eval"
|
||||
"github.com/lukaszraczylo/llm-testbench/internal/testkit"
|
||||
)
|
||||
|
||||
func registerWhitepapersTests(r *testkit.Registry) {
|
||||
r.Register(paperHnswParamsTest())
|
||||
}
|
||||
|
||||
// paperHnswParamsExcerpt is a ~150-word, technically accurate original
|
||||
// description of HNSW's three tuning parameters, written for this test
|
||||
// (not reproduced from any paper), pinning three specific numeric values.
|
||||
//
|
||||
// S11: the sentence about M was reworded. The original phrasing ("M=16,
|
||||
// giving each vector roughly 32 edges once both directions are counted")
|
||||
// conflated M with total node degree and misattributed the factor of two
|
||||
// to link bidirectionality. HNSW actually permits the base layer (layer 0)
|
||||
// twice as many links as upper layers (Mmax0 = 2*M) to keep that most
|
||||
// heavily traversed layer well connected - the excerpt now states that
|
||||
// correctly, still pinning M=16 as the single unambiguous answer to
|
||||
// question 1.
|
||||
const paperHnswParamsExcerpt = `HNSW (Hierarchical Navigable Small World) builds a
|
||||
multi-layer proximity graph over the indexed vectors, where each upper
|
||||
layer is a sparser sub-mesh of the layer below it. Search descends from a
|
||||
coarse entry point at the top layer down to a precise result at the base
|
||||
layer. Three parameters dominate the accuracy, speed, and memory trade-off.
|
||||
M controls the maximum number of bidirectional links each node keeps per
|
||||
layer; a benchmark index described in this report used M=16 links per node
|
||||
on the upper layers, with the base layer permitted twice that (Mmax0 = 32)
|
||||
to stay well connected. efConstruction bounds the size of the dynamic
|
||||
candidate list explored while inserting a new node, and was set to 200
|
||||
during the build - higher values slow indexing but produce a
|
||||
better-connected graph. efSearch, set to 64 at query time, bounds the same
|
||||
candidate list during search; raising it improves recall at the cost of
|
||||
latency, independent of the build-time efConstruction value.`
|
||||
|
||||
// paperHnswParamsTest: extract three precise numeric parameters from a
|
||||
// technical excerpt.
|
||||
//
|
||||
// ground truth: the excerpt states M=16, efConstruction=200, efSearch=64
|
||||
// verbatim; these are the only numbers attached to those parameter names in
|
||||
// the text.
|
||||
func paperHnswParamsTest() testkit.Test {
|
||||
prompt := `Read this excerpt about HNSW indexing:
|
||||
|
||||
` + paperHnswParamsExcerpt + `
|
||||
|
||||
Answer three questions based only on this excerpt:
|
||||
1. What value of M did the benchmark index use?
|
||||
2. What value of efConstruction was used during the index build?
|
||||
3. What value of efSearch was used at query time?
|
||||
|
||||
Respond with only a JSON object: {"M":<number>,"efConstruction":<number>,"efSearch":<number>}`
|
||||
|
||||
evaluator := eval.Mean(
|
||||
eval.JSONField("M", 16),
|
||||
eval.JSONField("efConstruction", 200),
|
||||
eval.JSONField("efSearch", 64),
|
||||
)
|
||||
|
||||
return testkit.Test{
|
||||
ID: "paper-hnsw-params",
|
||||
Category: "research",
|
||||
Subcategory: "whitepapers",
|
||||
Description: "Extract three precise numeric HNSW tuning parameters from a technical excerpt.",
|
||||
Prompt: prompt,
|
||||
MaxTokens: 300,
|
||||
Eval: evaluator,
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,61 @@
|
||||
package tests
|
||||
|
||||
import (
|
||||
"context"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// TestPaperHnswParamsExcerpt_GroundTruth independently extracts the three
|
||||
// numeric parameter values from the raw excerpt text via regex, rather
|
||||
// than trusting the doc comment's transcription of them.
|
||||
func TestPaperHnswParamsExcerpt_GroundTruth(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
pattern string
|
||||
want int
|
||||
}{
|
||||
{"M", `M=(\d+)`, 16},
|
||||
{"efConstruction", `efConstruction[^\d]*(\d+)`, 200},
|
||||
{"efSearch", `efSearch,\s*set to (\d+)`, 64},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
re := regexp.MustCompile(tt.pattern)
|
||||
m := re.FindStringSubmatch(paperHnswParamsExcerpt)
|
||||
if m == nil {
|
||||
t.Fatalf("pattern %q found no match in excerpt", tt.pattern)
|
||||
}
|
||||
got, err := strconv.Atoi(m[1])
|
||||
if err != nil {
|
||||
t.Fatalf("parse %q: %v", m[1], err)
|
||||
}
|
||||
if got != tt.want {
|
||||
t.Errorf("extracted %s = %d, want %d", tt.name, got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestPaperHnswParamsTest_Eval(t *testing.T) {
|
||||
tc := paperHnswParamsTest()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{"all correct", `{"M":16,"efConstruction":200,"efSearch":64}`, 1},
|
||||
{"one field wrong", `{"M":32,"efConstruction":200,"efSearch":64}`, 2.0 / 3.0},
|
||||
{"all wrong", `{"M":8,"efConstruction":100,"efSearch":32}`, 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user