llm-testbench/internal/tests/macos_test.go

package tests

import (
	"context"
	"testing"
)

func TestMacosTimeoutPortabilityTest_Eval(t *testing.T) {
	tc := macosTimeoutPortabilityTest()

	good := `macOS ships BSD userland, not GNU coreutils, so there is no
"timeout" binary out of the box - it is a GNU coreutils tool. Two portable
fixes: (1) brew install coreutils, then use "gtimeout 30 ./script.sh"
(GNU coreutils installs its tools with a "g" prefix to avoid clobbering the
BSD originals); or (2) with zero extra packages, background the command
and kill it after a sleep, e.g. run it, sleep 30 & kill it if still
running, or use "perl -e 'alarm shift; exec @ARGV' 30 ./script.sh" which
relies only on the perl builtin that ships with macOS.`

	badMissingAll := "Just run the script differently, I'm not sure why it fails."
	badMissingGNU := "Install coreutils via brew and use gtimeout, or use perl -e 'alarm'."

	tests := []struct {
		name     string
		response string
		want     float64
	}{
		{"good answer full credit", good, 1},
		{"missing everything scores 0", badMissingAll, 0},
		{"missing GNU explanation loses one third", badMissingGNU, 2.0 / 3.0},
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			got := tc.Eval.Evaluate(context.Background(), tt.response)
			if got.Value != tt.want {
				t.Errorf("Eval.Evaluate() = %v, want %v (detail: %s)", got.Value, tt.want, got.Detail)
			}
		})
	}
}

func TestMacosLaunchdCronTest_Eval(t *testing.T) {
	tc := macosLaunchdCronTest()

	good := `Use launchd, managed via launchctl, not cron. Put a plist in
/Library/LaunchDaemons for a system-wide job. Minimal keys: Label,
ProgramArguments, and StartInterval set to 300 (seconds) to run every 5
minutes; RunAtLoad is optional.`

	badWrongMechanism := "Just add a crontab entry with */5 * * * *."
	badMissingKeys := "Use launchd and put the plist in LaunchDaemons."
	badWrongDirectory := `Use launchd, managed via launchctl. Put a plist in
~/Library/LaunchAgents. Minimal keys: Label, ProgramArguments, and
StartInterval set to 300.`
	goodContrastiveMention := `Use launchd via launchctl, not cron. This must be a
per-machine job, so the plist goes in /Library/LaunchDaemons, not
~/Library/LaunchAgents (LaunchAgents is per-user and would not run without
a logged-in session). Minimal keys: Label, ProgramArguments, and
StartInterval set to 300.`

	tests := []struct {
		name     string
		response string
		want     float64
	}{
		{"good answer full credit", good, 1},
		{"wrong mechanism (cron) scores 0", badWrongMechanism, 0},
		{"missing plist keys loses one third", badMissingKeys, 2.0 / 3.0},
		{"LaunchAgents (per-user) is the wrong directory for this per-machine job", badWrongDirectory, 2.0 / 3.0},
		{"contrastive LaunchAgents mention alongside correct LaunchDaemons still scores full credit", goodContrastiveMention, 1},
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			got := tc.Eval.Evaluate(context.Background(), tt.response)
			if got.Value != tt.want {
				t.Errorf("Eval.Evaluate() = %v, want %v (detail: %s)", got.Value, tt.want, got.Detail)
			}
		})
	}
}

func TestMacosSedInplaceTest_Eval(t *testing.T) {
	tc := macosSedInplaceTest()

	tests := []struct {
		name     string
		response string
		want     float64
	}{
		{"single-quote empty extension", `sed -i '' 's/foo/bar/' file.txt`, 1},
		{"double-quote empty extension with -e flag", `On macOS: sed -i "" -e 's/foo/bar/' file.txt`, 1},
		{"global replace flag still matches", `sed -i '' 's/foo/bar/g' file.txt`, 1},
		{"GNU-style with no extension argument is wrong on macOS", `sed -i 's/foo/bar/' file.txt`, 0},
		{"backup extension creates a file, contradicting the requirement", `sed -i.bak 's/foo/bar/' file.txt`, 0},
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			got := tc.Eval.Evaluate(context.Background(), tt.response)
			if got.Value != tt.want {
				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
			}
		})
	}
}

func TestMacosLaunchctlBootstrapTest_Eval(t *testing.T) {
	tc := macosLaunchctlBootstrapTest()

	good := `Use "sudo launchctl bootstrap system /Library/LaunchDaemons/com.example.myjob.plist".
The older "launchctl load" command is deprecated because it has no explicit
domain target.`
	goodReordered := `The modern subcommand is bootstrap, targeting the system
domain: sudo launchctl bootstrap system /Library/LaunchDaemons/com.example.myjob.plist.
launchctl load is deprecated.`
	goodPhrasing := `Load it with launchctl bootstrap, using the system domain
target since this is a system-wide daemon. Avoid launchctl load, which
Apple deprecated for lacking an explicit domain target.`
	badWrongSubcommand := `Just run "sudo launchctl load /Library/LaunchDaemons/com.example.myjob.plist".`
	badMissingDomain := `Use launchctl bootstrap to load the daemon.`

	tests := []struct {
		name     string
		response string
		want     float64
	}{
		{"good answer full credit", good, 1},
		{"good answer, reordered phrasing", goodReordered, 1},
		{"good answer, alternate phrasing", goodPhrasing, 1},
		{"wrong subcommand (deprecated load only) scores 0", badWrongSubcommand, 0},
		{"missing domain target and deprecation note loses two thirds", badMissingDomain, 1.0 / 3.0},
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			got := tc.Eval.Evaluate(context.Background(), tt.response)
			if got.Value != tt.want {
				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
			}
		})
	}
}

func TestMacosPlutilPlistReadTest_Eval(t *testing.T) {
	tc := macosPlutilPlistReadTest()

	tests := []struct {
		name     string
		response string
		want     float64
	}{
		{"correct plutil -p", `plutil -p /Library/Preferences/com.example.app.plist`, 1},
		{"prose wrapped", `Run: plutil -p /Library/Preferences/com.example.app.plist to pretty-print it.`, 1},
		{"quoted path", `plutil -p "/Library/Preferences/com.example.app.plist"`, 1},
		{"wrong tool entirely (cat prints binary garbage)", `cat /Library/Preferences/com.example.app.plist`, 0},
		{"plutil without -p just validates, does not print", `plutil /Library/Preferences/com.example.app.plist`, 0},
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			got := tc.Eval.Evaluate(context.Background(), tt.response)
			if got.Value != tt.want {
				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
			}
		})
	}
}

func TestMacosMdfindSpotlightTest_Eval(t *testing.T) {
	tc := macosMdfindSpotlightTest()

	tests := []struct {
		name     string
		response string
		want     float64
	}{
		{"onlyin with query", `mdfind -onlyin ~ "invoice"`, 1},
		{"name flag", `mdfind -onlyin ~ -name invoice`, 1},
		{"prose wrapped", `Run mdfind -onlyin ~ invoice to search the Spotlight index.`, 1},
		{"uses find instead of mdfind, walks the filesystem", `find ~ -iname "*invoice*"`, 1.0 / 3.0},
		{"mdfind used but searching for the wrong term entirely", `mdfind -onlyin ~ "receipt"`, 2.0 / 3.0},
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			got := tc.Eval.Evaluate(context.Background(), tt.response)
			if got.Value != tt.want {
				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
			}
		})
	}
}

func TestMacosAPFSSnapshotTest_Eval(t *testing.T) {
	tc := macosAPFSSnapshotTest()

	tests := []struct {
		name     string
		response string
		want     float64
	}{
		{"both commands", "Create: tmutil snapshot\nList: tmutil listlocalsnapshots /", 1},
		{"prose wrapped", "Run `tmutil snapshot` to create one, then `tmutil listlocalsnapshots /` to list them.", 1},
		{"listlocalsnapshots without explicit tmutil prefix restated", "First tmutil snapshot, then check with listlocalsnapshots /.", 1},
		{"only creates, never lists", "Just run tmutil snapshot.", 0.5},
		{"wrong tool (diskutil) mentioned instead of tmutil", "Use diskutil apfs snapshot / to create one.", 0},
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			got := tc.Eval.Evaluate(context.Background(), tt.response)
			if got.Value != tt.want {
				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
			}
		})
	}
}

func TestMacosSSHIdentityAgentTest_Eval(t *testing.T) {
	tc := macosSSHIdentityAgentTest()

	tests := []struct {
		name     string
		response string
		want     float64
	}{
		{
			name:     "correct directive and socket",
			response: `IdentityAgent "~/Library/Group Containers/2BUA8C4S2C.com.1password/t/agent.sock"`,
			want:     1,
		},
		{
			name:     "inside a Host block, prose wrapped",
			response: "Add this under your Host block:\nHost *\n  IdentityAgent ~/Library/Group Containers/2BUA8C4S2C.com.1password/t/agent.sock",
			want:     1,
		},
		{
			name:     "shortened socket mention still credited",
			response: "Set IdentityAgent to point at 1Password's agent.sock in its Group Containers path.",
			want:     1,
		},
		{
			name:     "sets SSH_AUTH_SOCK env var instead of ssh_config directive",
			response: `export SSH_AUTH_SOCK=~/Library/Group Containers/2BUA8C4S2C.com.1password/t/agent.sock`,
			want:     0.5,
		},
		{
			name:     "wrong directive entirely",
			response: `Add IdentityFile ~/.ssh/id_ed25519 to your config.`,
			want:     0,
		},
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			got := tc.Eval.Evaluate(context.Background(), tt.response)
			if got.Value != tt.want {
				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
			}
		})
	}
}

func TestMacosCaffeinateTest_Eval(t *testing.T) {
	tc := macosCaffeinateTest()

	tests := []struct {
		name     string
		response string
		want     float64
	}{
		{"bare invocation", `caffeinate ./migrate.sh`, 1},
		{"with system-sleep flag", `caffeinate -s ./migrate.sh`, 1},
		{"prose wrapped", "Run `caffeinate -i ./migrate.sh` and it stops holding the assertion once migrate.sh exits.", 1},
		{"wrong script named, not the target", `caffeinate ./other-script.sh`, 0.5},
		{"unrelated tool, does not auto-release on exit", `Just run pmset noidle before starting your script.`, 0},
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			got := tc.Eval.Evaluate(context.Background(), tt.response)
			if got.Value != tt.want {
				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
			}
		})
	}
}

func TestMacosStatBSDGNUTest_Eval(t *testing.T) {
	tc := macosStatBSDGNUTest()

	tests := []struct {
		name     string
		response string
		want     float64
	}{
		{"correct BSD form and GNU flag", `stat -f%z file.txt` + "\nGNU stat uses -c instead.", 1},
		{"quoted format string", `stat -f "%z" file.txt, and on Linux GNU stat uses -c.`, 1},
		{"prose wrapped", "Run stat -f%z file.txt on macOS; the GNU equivalent flag is -c.", 1},
		{"correct BSD command but never names the GNU flag", `stat -f%z file.txt`, 0.5},
		{"GNU -c%s syntax used on macOS scores half: -c token matches but -f BSD form is missing", `stat -c%s file.txt`, 0.5},
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			got := tc.Eval.Evaluate(context.Background(), tt.response)
			if got.Value != tt.want {
				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
			}
		})
	}
}