mirror of
https://github.com/lukaszraczylo/llm-testbench.git
synced 2026-10-12 00:45:19 +00:00
test(catalog): expand operations to 10 tests per subcategory
Add 8 macOS tests, 8 Linux tests, and 9 Kubernetes tests to internal/tests. Each operations subcategory now holds 10 deterministic tests, up from 2, 2, and 1. Tests cover BSD-versus-GNU tooling, launchd, Proxmox pct exec, systemd timers, cgroup v2 diagnosis, and ArgoCD GitOps scenarios. Evaluators use ContainsAll, ContainsAny, Regex, JSONField, Numeric, and weighted composites, matching the existing catalog style. Every evaluator unit test asserts three correct phrasings and two wrong ones, per the expansion contract. Ground truth for numeric and ordered answers carries a derivation comment. catalog_test.go still expects 17 tests and fails as documented. This commit does not modify that file.
This commit is contained in:
@@ -78,3 +78,227 @@ StartInterval set to 300.`
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestMacosSedInplaceTest_Eval(t *testing.T) {
|
||||
tc := macosSedInplaceTest()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{"single-quote empty extension", `sed -i '' 's/foo/bar/' file.txt`, 1},
|
||||
{"double-quote empty extension with -e flag", `On macOS: sed -i "" -e 's/foo/bar/' file.txt`, 1},
|
||||
{"global replace flag still matches", `sed -i '' 's/foo/bar/g' file.txt`, 1},
|
||||
{"GNU-style with no extension argument is wrong on macOS", `sed -i 's/foo/bar/' file.txt`, 0},
|
||||
{"backup extension creates a file, contradicting the requirement", `sed -i.bak 's/foo/bar/' file.txt`, 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestMacosLaunchctlBootstrapTest_Eval(t *testing.T) {
|
||||
tc := macosLaunchctlBootstrapTest()
|
||||
|
||||
good := `Use "sudo launchctl bootstrap system /Library/LaunchDaemons/com.example.myjob.plist".
|
||||
The older "launchctl load" command is deprecated because it has no explicit
|
||||
domain target.`
|
||||
goodReordered := `The modern subcommand is bootstrap, targeting the system
|
||||
domain: sudo launchctl bootstrap system /Library/LaunchDaemons/com.example.myjob.plist.
|
||||
launchctl load is deprecated.`
|
||||
goodPhrasing := `Load it with launchctl bootstrap, using the system domain
|
||||
target since this is a system-wide daemon. Avoid launchctl load, which
|
||||
Apple deprecated for lacking an explicit domain target.`
|
||||
badWrongSubcommand := `Just run "sudo launchctl load /Library/LaunchDaemons/com.example.myjob.plist".`
|
||||
badMissingDomain := `Use launchctl bootstrap to load the daemon.`
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{"good answer full credit", good, 1},
|
||||
{"good answer, reordered phrasing", goodReordered, 1},
|
||||
{"good answer, alternate phrasing", goodPhrasing, 1},
|
||||
{"wrong subcommand (deprecated load only) scores 0", badWrongSubcommand, 0},
|
||||
{"missing domain target and deprecation note loses two thirds", badMissingDomain, 1.0 / 3.0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestMacosPlutilPlistReadTest_Eval(t *testing.T) {
|
||||
tc := macosPlutilPlistReadTest()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{"correct plutil -p", `plutil -p /Library/Preferences/com.example.app.plist`, 1},
|
||||
{"prose wrapped", `Run: plutil -p /Library/Preferences/com.example.app.plist to pretty-print it.`, 1},
|
||||
{"quoted path", `plutil -p "/Library/Preferences/com.example.app.plist"`, 1},
|
||||
{"wrong tool entirely (cat prints binary garbage)", `cat /Library/Preferences/com.example.app.plist`, 0},
|
||||
{"plutil without -p just validates, does not print", `plutil /Library/Preferences/com.example.app.plist`, 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestMacosMdfindSpotlightTest_Eval(t *testing.T) {
|
||||
tc := macosMdfindSpotlightTest()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{"onlyin with query", `mdfind -onlyin ~ "invoice"`, 1},
|
||||
{"name flag", `mdfind -onlyin ~ -name invoice`, 1},
|
||||
{"prose wrapped", `Run mdfind -onlyin ~ invoice to search the Spotlight index.`, 1},
|
||||
{"uses find instead of mdfind, walks the filesystem", `find ~ -iname "*invoice*"`, 1.0 / 3.0},
|
||||
{"mdfind used but searching for the wrong term entirely", `mdfind -onlyin ~ "receipt"`, 2.0 / 3.0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestMacosAPFSSnapshotTest_Eval(t *testing.T) {
|
||||
tc := macosAPFSSnapshotTest()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{"both commands", "Create: tmutil snapshot\nList: tmutil listlocalsnapshots /", 1},
|
||||
{"prose wrapped", "Run `tmutil snapshot` to create one, then `tmutil listlocalsnapshots /` to list them.", 1},
|
||||
{"listlocalsnapshots without explicit tmutil prefix restated", "First tmutil snapshot, then check with listlocalsnapshots /.", 1},
|
||||
{"only creates, never lists", "Just run tmutil snapshot.", 0.5},
|
||||
{"wrong tool (diskutil) mentioned instead of tmutil", "Use diskutil apfs snapshot / to create one.", 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestMacosSSHIdentityAgentTest_Eval(t *testing.T) {
|
||||
tc := macosSSHIdentityAgentTest()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{
|
||||
name: "correct directive and socket",
|
||||
response: `IdentityAgent "~/Library/Group Containers/2BUA8C4S2C.com.1password/t/agent.sock"`,
|
||||
want: 1,
|
||||
},
|
||||
{
|
||||
name: "inside a Host block, prose wrapped",
|
||||
response: "Add this under your Host block:\nHost *\n IdentityAgent ~/Library/Group Containers/2BUA8C4S2C.com.1password/t/agent.sock",
|
||||
want: 1,
|
||||
},
|
||||
{
|
||||
name: "shortened socket mention still credited",
|
||||
response: "Set IdentityAgent to point at 1Password's agent.sock in its Group Containers path.",
|
||||
want: 1,
|
||||
},
|
||||
{
|
||||
name: "sets SSH_AUTH_SOCK env var instead of ssh_config directive",
|
||||
response: `export SSH_AUTH_SOCK=~/Library/Group Containers/2BUA8C4S2C.com.1password/t/agent.sock`,
|
||||
want: 0.5,
|
||||
},
|
||||
{
|
||||
name: "wrong directive entirely",
|
||||
response: `Add IdentityFile ~/.ssh/id_ed25519 to your config.`,
|
||||
want: 0,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestMacosCaffeinateTest_Eval(t *testing.T) {
|
||||
tc := macosCaffeinateTest()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{"bare invocation", `caffeinate ./migrate.sh`, 1},
|
||||
{"with system-sleep flag", `caffeinate -s ./migrate.sh`, 1},
|
||||
{"prose wrapped", "Run `caffeinate -i ./migrate.sh` and it stops holding the assertion once migrate.sh exits.", 1},
|
||||
{"wrong script named, not the target", `caffeinate ./other-script.sh`, 0.5},
|
||||
{"unrelated tool, does not auto-release on exit", `Just run pmset noidle before starting your script.`, 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestMacosStatBSDGNUTest_Eval(t *testing.T) {
|
||||
tc := macosStatBSDGNUTest()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{"correct BSD form and GNU flag", `stat -f%z file.txt` + "\nGNU stat uses -c instead.", 1},
|
||||
{"quoted format string", `stat -f "%z" file.txt, and on Linux GNU stat uses -c.`, 1},
|
||||
{"prose wrapped", "Run stat -f%z file.txt on macOS; the GNU equivalent flag is -c.", 1},
|
||||
{"correct BSD command but never names the GNU flag", `stat -f%z file.txt`, 0.5},
|
||||
{"GNU -c%s syntax used on macOS scores half: -c token matches but -f BSD form is missing", `stat -c%s file.txt`, 0.5},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user