lukaszraczylo/llm-testbench
1
0
Fork 0
mirror of https://github.com/lukaszraczylo/llm-testbench.git synced 2026-10-12 00:45:19 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'worktree-agent-a51a5d5173d0ab0f0'

Browse Source
This commit is contained in:
lukaszraczylo
2026-08-17 01:55:48 +01:00
parent 3e3e0d4f01 3f9fed6f02
commit 8b70503f87
6 changed files with 1665 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/tests/kubernetes.go
+460
View File
@@ -11,6 +11,15 @@ import (
func registerKubernetesTests(r *testkit.Registry) {
r.Register(k8sCrashloopGitopsTest())
r.Register(k8sImagePullBackoffTest())
r.Register(k8sPendingTaintsTest())
r.Register(k8sPVCStorageclassMismatchTest())
r.Register(k8sServiceSelectorMismatchTest())
r.Register(k8sNetworkPolicyDNSBlockTest())
r.Register(k8sArgoCDOutOfSyncTest())
r.Register(k8sTraefikIngressRouteHostTest())
r.Register(k8sQoSClassTest())
r.Register(k8sCNPGPDBTest())
}
// k8sCrashloopGitopsDescribeExcerpt is the inline kubectl describe pod
@@ -155,3 +164,454 @@ procedure to fix it given that ArgoCD manages this cluster?`
Eval: evaluator,
}
}
// k8sImagePullBackoffDescribeExcerpt is the inline kubectl describe pod
// excerpt for k8sImagePullBackoffTest.
const k8sImagePullBackoffDescribeExcerpt = `Name: checkout-api-7d8f9c6b5d-4mnbv
Namespace: checkout
Status: Pending
Containers:
api:
Image: registry.internal.example.com/checkout-api:v1.2.3
State: Waiting
Reason: ImagePullBackOff
Events:
Warning Failed kubelet Failed to pull image "registry.internal.example.com/checkout-api:v1.2.3": rpc error: code = Unknown desc = failed to pull and unpack image: failed to resolve reference: unexpected status code 401 Unauthorized: authentication required
Warning Failed kubelet Error: ErrImagePull
Normal BackOff kubelet Back-off pulling image "registry.internal.example.com/checkout-api:v1.2.3"`
// k8sImagePullBackoffTest: classify an ImagePullBackOff's root cause from
// the HTTP status embedded in the pull error, and name the matching fix.
//
// ground truth: the pull error is "401 Unauthorized: authentication
// required" - the registry rejected the request for lacking credentials,
// not because the image/tag is missing (that would be a 404/"not found")
// or because the registry is unreachable (that would be a network/DNS/TLS
// error). The fix is to give the pod credentials for the private registry
// via imagePullSecrets, not to touch the image tag or check connectivity.
func k8sImagePullBackoffTest() testkit.Test {
prompt := `Here is the output of "kubectl describe pod" for a pod stuck
in ImagePullBackOff:
` + k8sImagePullBackoffDescribeExcerpt + `
What category of root cause is this, and what category of fix addresses
it? Respond with only a JSON object:
{"cause":"<one of: auth, not-found, network>","fix":"<one of: imagePullSecrets, fix-image-tag, check-network>"}`
evaluator := eval.Mean(
eval.JSONField("cause", "auth"),
eval.JSONField("fix", "imagePullSecrets"),
)
return testkit.Test{
ID: "k8s-imagepullbackoff",
Category: "operations",
Subcategory: "kubernetes",
Description: "Classify an ImagePullBackOff's root cause (401 auth failure, not a missing tag or network issue) and its fix category.",
Prompt: prompt,
Eval: evaluator,
}
}
// k8sPendingTaintsDescribeExcerpt is the inline kubectl describe pod
// excerpt for k8sPendingTaintsTest.
const k8sPendingTaintsDescribeExcerpt = `Name: batch-etl-59c8d8c9f7-k2p8q
Namespace: data
Status: Pending
Spec (excerpt):
tolerations: <none>
nodeSelector: <none>
Events:
Warning FailedScheduling default-scheduler 0/3 nodes are available: 3 node(s) had untolerated taint {dedicated: gpu-only: NoSchedule}. preemption: 0/3 nodes are available: 3 Preemption is not helpful for scheduling.`
// k8sPendingTaintsTest: identify an untolerated node taint as the reason a
// pod cannot be scheduled, and name the pod-spec field that fixes it.
//
// ground truth: the FailedScheduling event names the exact blocker -
// "untolerated taint {dedicated: gpu-only: NoSchedule}" on all 3
// candidate nodes - and the pod spec confirms it has no tolerations at
// all. Adding a matching entry under the pod's tolerations field (not
// nodeSelector, which restricts placement rather than tolerating a taint,
// and not resources, which is unrelated) is the fix.
func k8sPendingTaintsTest() testkit.Test {
prompt := `Here is the output of "kubectl describe pod" for a pod stuck
Pending:
` + k8sPendingTaintsDescribeExcerpt + `
Why can the scheduler not place this pod on any of the 3 candidate nodes,
and which pod spec field must be added to fix it? Respond with only a
JSON object:
{"cause":"<one of: taint, resource-limit, affinity>","fix_field":"<one of: tolerations, nodeSelector, resources>"}`
evaluator := eval.Mean(
eval.JSONField("cause", "taint"),
eval.JSONField("fix_field", "tolerations"),
)
return testkit.Test{
ID: "k8s-pending-taints",
Category: "operations",
Subcategory: "kubernetes",
Description: "Identify an untolerated node taint blocking scheduling and the tolerations field that fixes it.",
Prompt: prompt,
Eval: evaluator,
}
}
// k8sPVCStorageclassExcerpt is the inline kubectl describe pvc + kubectl
// get storageclass excerpt for k8sPVCStorageclassMismatchTest.
const k8sPVCStorageclassExcerpt = `$ kubectl describe pvc data-pvc -n analytics
Name: data-pvc
Namespace: analytics
StorageClass: fast-ssd
Status: Pending
Events:
Warning ProvisioningFailed persistentvolume-controller storageclass.storage.k8s.io "fast-ssd" not found
$ kubectl get storageclass
NAME PROVISIONER
standard csi.example.com`
// k8sPVCStorageclassMismatchTest: diagnose a Pending PVC referencing a
// StorageClass that does not exist in the cluster, and pick the existing
// class to switch to.
//
// ground truth: the ProvisioningFailed event states plainly that
// StorageClass "fast-ssd" does not exist; "kubectl get storageclass" shows
// only "standard" exists. Since this cluster's StorageClasses are managed
// centrally via GitOps and creating one ad hoc is ruled out by the prompt,
// the only available fix is to point the PVC at the StorageClass that
// already exists: "standard".
func k8sPVCStorageclassMismatchTest() testkit.Test {
prompt := `This cluster's StorageClasses are defined centrally via
GitOps; creating a new StorageClass ad hoc is not an option here. Here is
the PVC and the StorageClasses that actually exist in the cluster:
` + k8sPVCStorageclassExcerpt + `
What caused this PVC to stay Pending, and which existing StorageClass name
should it be changed to reference so it can be provisioned? Respond with
only a JSON object:
{"cause":"<one of: missing-storageclass, provisioner-down, quota-exceeded>","use_storageclass":"<name>"}`
evaluator := eval.Mean(
eval.JSONField("cause", "missing-storageclass"),
eval.JSONField("use_storageclass", "standard"),
)
return testkit.Test{
ID: "k8s-pvc-storageclass-mismatch",
Category: "operations",
Subcategory: "kubernetes",
Description: "Diagnose a Pending PVC referencing a nonexistent StorageClass and pick the one existing class to use instead.",
Prompt: prompt,
Eval: evaluator,
}
}
// k8sServiceSelectorMismatchYAML is the inline Deployment + Service YAML
// excerpt for k8sServiceSelectorMismatchTest.
const k8sServiceSelectorMismatchYAML = `# Deployment (excerpt)
apiVersion: apps/v1
kind: Deployment
metadata:
name: inventory-api
namespace: inventory
spec:
template:
metadata:
labels:
app: inventory-api
tier: backend
---
# Service
apiVersion: v1
kind: Service
metadata:
name: inventory-api
namespace: inventory
spec:
selector:
app: inventory-api-svc
ports:
- port: 80
targetPort: 8080`
// k8sServiceSelectorMismatchTest: spot a Service selector that does not
// match the Deployment's pod-template labels, leaving the Service with no
// Endpoints.
//
// ground truth: the pods carry label app=inventory-api (from the
// Deployment's pod template), but the Service's selector is
// app=inventory-api-svc - a value that matches no pod's labels, so the
// Service has zero Endpoints. The single wrong field is the Service's
// spec.selector.app value; it must be changed to "inventory-api" to match
// the pods.
func k8sServiceSelectorMismatchTest() testkit.Test {
prompt := `Requests to this Service time out with no available
endpoints. Here is the Deployment's pod template and the Service:
` + k8sServiceSelectorMismatchYAML + `
Which single field in the Service YAML is wrong, and what value should it
be changed to so the Service's Endpoints include these pods? Respond with
only a JSON object: {"field":"selector.app","value":"<correct value>"}`
evaluator := eval.Mean(
eval.JSONField("field", "selector.app"),
eval.JSONField("value", "inventory-api"),
)
return testkit.Test{
ID: "k8s-service-selector-mismatch",
Category: "operations",
Subcategory: "kubernetes",
Description: "Spot a Service selector.app value that does not match the Deployment's pod-template labels.",
Prompt: prompt,
Eval: evaluator,
}
}
// k8sNetworkPolicyDNSYAML is the inline default-deny-egress NetworkPolicy
// excerpt for k8sNetworkPolicyDNSBlockTest.
const k8sNetworkPolicyDNSYAML = `apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: payments-egress
namespace: payments
spec:
podSelector:
matchLabels:
app: payments
policyTypes:
- Egress
egress:
- to:
- podSelector:
matchLabels:
app: payments-db
ports:
- protocol: TCP
port: 5432`
// k8sNetworkPolicyDNSBlockTest: explain why a default-deny-egress
// NetworkPolicy that only allows one destination breaks DNS resolution,
// and what egress rule fixes it.
//
// ground truth: this NetworkPolicy selects pods labeled app=payments,
// applies to Egress, and lists exactly one allowed destination (the
// payments-db pods on 5432/TCP) - meaning every other egress, including
// DNS lookups to CoreDNS/kube-dns on port 53, is implicitly denied. The
// fix is an additional egress rule allowing UDP (and typically TCP) port
// 53 to the cluster's DNS pods (conventionally labeled k8s-app=kube-dns in
// the kube-system namespace).
func k8sNetworkPolicyDNSBlockTest() testkit.Test {
prompt := `Pods labeled app=payments in the payments namespace can reach
the payments database but cannot resolve ANY DNS name, including external
ones. Here is the only NetworkPolicy selecting these pods:
` + k8sNetworkPolicyDNSYAML + `
Why does this NetworkPolicy break DNS resolution for these pods, and what
additional egress rule (name the port, protocol, and typical target label)
must be added to fix it?`
evaluator := eval.All(
eval.W(eval.ContainsAll("53"), 2),
eval.W(eval.ContainsAny("udp", "UDP"), 2),
eval.W(eval.ContainsAny("kube-dns", "coredns", "kube-system"), 2),
)
return testkit.Test{
ID: "k8s-networkpolicy-dns-block",
Category: "operations",
Subcategory: "kubernetes",
Description: "Explain why a single-destination-allow egress NetworkPolicy implicitly blocks DNS, and the port-53/UDP fix.",
Prompt: prompt,
Eval: evaluator,
}
}
// k8sArgoCDOutOfSyncStatus is the inline "argocd app get" excerpt for
// k8sArgoCDOutOfSyncTest.
const k8sArgoCDOutOfSyncStatus = `$ argocd app get inventory-api
Name: inventory-api
Namespace: inventory
Health Status: Healthy
Sync Status: OutOfSync
Resources:
Kind Name Status
Deployment inventory-api OutOfSync
Diff (live vs target, Deployment/inventory-api):
live: spec.replicas: 5
target: spec.replicas: 2
Note: a HorizontalPodAutoscaler named inventory-api-hpa also targets this
Deployment and actively manages its replica count based on CPU load.
Nobody has manually run "kubectl scale", "kubectl edit", or
"argocd app sync" recently.`
// k8sArgoCDOutOfSyncTest: identify an HPA-managed field as the reason an
// ArgoCD Application stays OutOfSync, and the idiomatic fix.
//
// ground truth: the only diff is spec.replicas, and an HPA legitimately
// changes that field outside of git as it scales the Deployment - so the
// Application will drift back OutOfSync after every "argocd app sync"
// without addressing the underlying cause. ArgoCD's own mechanism for this
// exact situation is configuring spec.ignoreDifferences on the Application
// for the HPA-managed field, not repeatedly re-syncing or reverting a
// change nobody made.
func k8sArgoCDOutOfSyncTest() testkit.Test {
prompt := `Here is the status of an ArgoCD Application that keeps
showing OutOfSync:
` + k8sArgoCDOutOfSyncStatus + `
Why is this Application stuck OutOfSync on spec.replicas specifically, and
what is the correct ArgoCD-idiomatic fix so it stops flapping between
Synced and OutOfSync? Respond with only a JSON object:
{"cause":"<one of: hpa-managed-field, manual-drift, stale-git>","fix":"<one of: ignoreDifferences, revert-manual-change, sync-git>"}`
evaluator := eval.Mean(
eval.JSONField("cause", "hpa-managed-field"),
eval.JSONField("fix", "ignoreDifferences"),
)
return testkit.Test{
ID: "k8s-argocd-outofsync",
Category: "operations",
Subcategory: "kubernetes",
Description: "Identify an HPA-managed replicas field as the cause of a flapping ArgoCD OutOfSync and the ignoreDifferences fix.",
Prompt: prompt,
Eval: evaluator,
}
}
// k8sTraefikIngressRouteYAML is the inline IngressRoute excerpt for
// k8sTraefikIngressRouteHostTest.
const k8sTraefikIngressRouteYAML = "apiVersion: traefik.io/v1alpha1\n" +
"kind: IngressRoute\n" +
"metadata:\n" +
" name: app-route\n" +
" namespace: web\n" +
"spec:\n" +
" entryPoints:\n" +
" - websecure\n" +
" routes:\n" +
" - match: Host(`app.example.com`)\n" +
" kind: Rule\n" +
" services:\n" +
" - name: app-svc\n" +
" port: 80"
// k8sTraefikIngressRoutePattern requires the corrected match expression,
// with the backtick-quoted domain exactly matching app.raczylo.com.
const k8sTraefikIngressRoutePattern = "(?i)host\\(`app\\.raczylo\\.com`\\)"
// k8sTraefikIngressRouteHostTest: correct a Traefik IngressRoute's Host()
// match rule to the actual domain the service must answer for.
//
// ground truth: the IngressRoute's only match rule is
// Host(`app.example.com`), but the domain that must route to this service
// is app.raczylo.com - a different rule value entirely, not a change to
// entryPoints, the service name, or the port.
func k8sTraefikIngressRouteHostTest() testkit.Test {
prompt := `This Traefik IngressRoute is not receiving any traffic
because it is matching the wrong hostname. The actual domain this service
must answer for is app.raczylo.com. Here is the IngressRoute:
` + "```yaml\n" + k8sTraefikIngressRouteYAML + "\n```" + `
Give the single corrected "match" rule line (the full match expression,
using the same Host() syntax) for this IngressRoute.`
return testkit.Test{
ID: "k8s-traefik-ingressroute-host",
Category: "operations",
Subcategory: "kubernetes",
Description: "Correct a Traefik IngressRoute's Host() match rule to the actual domain the service must serve.",
Prompt: prompt,
Eval: eval.Regex(k8sTraefikIngressRoutePattern),
}
}
// k8sQoSClassPodSpec is the inline pod spec excerpt for k8sQoSClassTest.
const k8sQoSClassPodSpec = `apiVersion: v1
kind: Pod
metadata:
name: report-generator
spec:
containers:
- name: report-generator
image: registry.internal.example.com/report-generator:v3
resources:
requests:
cpu: "250m"
memory: "256Mi"
limits:
cpu: "500m"
memory: "512Mi"`
// k8sQoSClassTest: derive the scheduler-assigned QoS class from a pod
// spec's requests/limits.
//
// ground truth: every container defines both requests and limits for both
// cpu and memory, ruling out BestEffort (which requires no requests or
// limits at all). Since requests (250m/256Mi) differ from limits
// (500m/512Mi) rather than being exactly equal, the class is Burstable,
// not Guaranteed (which additionally requires requests == limits for
// every resource on every container).
func k8sQoSClassTest() testkit.Test {
prompt := `Here is a pod spec:
` + "```yaml\n" + k8sQoSClassPodSpec + "\n```" + `
What Quality of Service (QoS) class does the Kubernetes scheduler assign
to this pod? Respond with only one word: Guaranteed, Burstable, or
BestEffort.`
return testkit.Test{
ID: "k8s-qos-class",
Category: "operations",
Subcategory: "kubernetes",
Description: "Derive a pod's QoS class (Burstable) from requests that are set but not equal to limits.",
Prompt: prompt,
Eval: eval.Equals("Burstable"),
}
}
// k8sCNPGPDBTest: pick PodDisruptionBudget with minAvailable as the
// mechanism guaranteeing quorum survives voluntary node maintenance for a
// CNPG-managed Postgres cluster.
func k8sCNPGPDBTest() testkit.Test {
prompt := `A CNPG (CloudNativePG)-managed Postgres cluster runs as 3
pods (1 primary, 2 replicas) in the shared-resources namespace. The team
wants a guarantee that at least 2 of the 3 Postgres pods stay available
during voluntary node maintenance (e.g. "kubectl drain"). Which Kubernetes
resource type provides this guarantee, and what field on it must be set,
with what minimum value, to enforce "at least 2 available"?`
// ground truth: a PodDisruptionBudget (PDB) is the resource that
// bounds voluntary disruptions (like a node drain evicting pods) for a
// selected set of pods. Setting its minAvailable field to 2 enforces
// that at least 2 of the 3 Postgres pods must remain available before
// the eviction API will permit another one to be drained.
evaluator := eval.Mean(
eval.ContainsAll("PodDisruptionBudget"),
eval.ContainsAny("minAvailable: 2", "minAvailable=2", "minAvailable 2", "minAvailable of 2"),
)
return testkit.Test{
ID: "k8s-cnpg-pdb",
Category: "operations",
Subcategory: "kubernetes",
Description: "Identify PodDisruptionBudget with minAvailable=2 as the guarantee for a 3-pod CNPG cluster during node drains.",
Prompt: prompt,
Eval: evaluator,
}
}
internal/tests/kubernetes_test.go
+232
View File
@@ -153,3 +153,235 @@ is missing. Check the image tag and re-push it.`
})
}
}
func TestK8sImagePullBackoffTest_Eval(t *testing.T) {
tc := k8sImagePullBackoffTest()
tests := []struct {
name string
response string
want float64
}{
{"correct object", `{"cause":"auth","fix":"imagePullSecrets"}`, 1},
{"prose then JSON", "This is a 401, an auth failure.\n" + `{"cause":"auth","fix":"imagePullSecrets"}`, 1},
{"fenced JSON", "```json\n" + `{"cause":"auth","fix":"imagePullSecrets"}` + "\n```", 1},
{"misreads it as a missing tag", `{"cause":"not-found","fix":"fix-image-tag"}`, 0},
{"right cause, wrong fix category", `{"cause":"auth","fix":"check-network"}`, 0.5},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := tc.Eval.Evaluate(context.Background(), tt.response)
if got.Value != tt.want {
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
}
})
}
}
func TestK8sPendingTaintsTest_Eval(t *testing.T) {
tc := k8sPendingTaintsTest()
tests := []struct {
name string
response string
want float64
}{
{"correct object", `{"cause":"taint","fix_field":"tolerations"}`, 1},
{"prose then JSON", "The nodes have an untolerated taint.\n" + `{"cause":"taint","fix_field":"tolerations"}`, 1},
{"fenced JSON", "```json\n" + `{"cause":"taint","fix_field":"tolerations"}` + "\n```", 1},
{"misdiagnoses as a resource limit issue", `{"cause":"resource-limit","fix_field":"resources"}`, 0},
{"right cause, wrong fix field", `{"cause":"taint","fix_field":"nodeSelector"}`, 0.5},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := tc.Eval.Evaluate(context.Background(), tt.response)
if got.Value != tt.want {
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
}
})
}
}
func TestK8sPVCStorageclassMismatchTest_Eval(t *testing.T) {
tc := k8sPVCStorageclassMismatchTest()
tests := []struct {
name string
response string
want float64
}{
{"correct object", `{"cause":"missing-storageclass","use_storageclass":"standard"}`, 1},
{"prose then JSON", "The fast-ssd class does not exist.\n" + `{"cause":"missing-storageclass","use_storageclass":"standard"}`, 1},
{"fenced JSON", "```json\n" + `{"cause":"missing-storageclass","use_storageclass":"standard"}` + "\n```", 1},
{"misdiagnoses as the provisioner being down", `{"cause":"provisioner-down","use_storageclass":"standard"}`, 0.5},
{"right cause but names a class that does not exist", `{"cause":"missing-storageclass","use_storageclass":"fast-ssd"}`, 0.5},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := tc.Eval.Evaluate(context.Background(), tt.response)
if got.Value != tt.want {
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
}
})
}
}
func TestK8sServiceSelectorMismatchTest_Eval(t *testing.T) {
tc := k8sServiceSelectorMismatchTest()
tests := []struct {
name string
response string
want float64
}{
{"correct object", `{"field":"selector.app","value":"inventory-api"}`, 1},
{"prose then JSON", "The selector doesn't match the pod labels.\n" + `{"field":"selector.app","value":"inventory-api"}`, 1},
{"fenced JSON", "```json\n" + `{"field":"selector.app","value":"inventory-api"}` + "\n```", 1},
{"points at the wrong field (targetPort instead of selector)", `{"field":"targetPort","value":"8080"}`, 0},
{"right field, wrong replacement value", `{"field":"selector.app","value":"inventory-api-svc"}`, 0.5},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := tc.Eval.Evaluate(context.Background(), tt.response)
if got.Value != tt.want {
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
}
})
}
}
func TestK8sNetworkPolicyDNSBlockTest_Eval(t *testing.T) {
tc := k8sNetworkPolicyDNSBlockTest()
good := `This NetworkPolicy allows Egress only to payments-db pods on
5432/TCP, so every other egress - including DNS lookups on port 53/UDP -
is implicitly denied. Add an egress rule allowing UDP port 53 (and
typically TCP 53) to pods labeled k8s-app=kube-dns in kube-system.`
goodAlt := `The policy default-denies all egress except the one listed
destination, and DNS needs to reach CoreDNS on UDP port 53, which is not
allowed here. Add an egress rule permitting port 53/udp to the coredns
pods.`
goodTerse := `Add an egress allow rule for port 53 UDP targeting the
kube-system namespace's kube-dns pods; everything else is denied by
default here.`
badWrongPort := `Add an egress rule allowing port 80/TCP to the ingress
controller.`
badWrongProtocol := `Add an egress rule allowing TCP port 53 only to
kube-system, since DNS always uses TCP.`
tests := []struct {
name string
response string
want float64
}{
{"good answer, kube-dns phrasing", good, 1},
{"good answer, coredns phrasing", goodAlt, 1},
{"good answer, terse phrasing", goodTerse, 1},
{"wrong port and protocol entirely", badWrongPort, 0},
{"right port, missing udp mention", badWrongProtocol, 2.0 / 3.0},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := tc.Eval.Evaluate(context.Background(), tt.response)
if got.Value != tt.want {
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
}
})
}
}
func TestK8sArgoCDOutOfSyncTest_Eval(t *testing.T) {
tc := k8sArgoCDOutOfSyncTest()
tests := []struct {
name string
response string
want float64
}{
{"correct object", `{"cause":"hpa-managed-field","fix":"ignoreDifferences"}`, 1},
{"prose then JSON", "The HPA owns replicas here.\n" + `{"cause":"hpa-managed-field","fix":"ignoreDifferences"}`, 1},
{"fenced JSON", "```json\n" + `{"cause":"hpa-managed-field","fix":"ignoreDifferences"}` + "\n```", 1},
{"wrongly blames manual drift nobody caused", `{"cause":"manual-drift","fix":"revert-manual-change"}`, 0},
{"right cause, wrong fix (fighting the HPA by re-syncing)", `{"cause":"hpa-managed-field","fix":"sync-git"}`, 0.5},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := tc.Eval.Evaluate(context.Background(), tt.response)
if got.Value != tt.want {
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
}
})
}
}
func TestK8sTraefikIngressRouteHostTest_Eval(t *testing.T) {
tc := k8sTraefikIngressRouteHostTest()
tests := []struct {
name string
response string
want float64
}{
{"bare corrected line", "match: Host(`app.raczylo.com`)", 1},
{"prose wrapped", "The corrected line is:\nmatch: Host(`app.raczylo.com`)", 1},
{"uppercase Host keyword variant", "MATCH: HOST(`app.raczylo.com`)", 1},
{"still the old domain", "match: Host(`app.example.com`)", 0},
{"right function, wrong domain typo", "match: Host(`app.raczlo.com`)", 0},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := tc.Eval.Evaluate(context.Background(), tt.response)
if got.Value != tt.want {
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
}
})
}
}
func TestK8sQoSClassTest_Eval(t *testing.T) {
tc := k8sQoSClassTest()
tests := []struct {
name string
response string
want float64
}{
{"exact word", "Burstable", 1},
{"lowercase", "burstable", 1},
{"trailing whitespace", "Burstable\n", 1},
{"wrong class, mistakes it for Guaranteed", "Guaranteed", 0},
{"wrong class, mistakes it for BestEffort", "BestEffort", 0},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := tc.Eval.Evaluate(context.Background(), tt.response)
if got.Value != tt.want {
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
}
})
}
}
func TestK8sCNPGPDBTest_Eval(t *testing.T) {
tc := k8sCNPGPDBTest()
tests := []struct {
name string
response string
want float64
}{
{"correct resource and value", "Use a PodDisruptionBudget with minAvailable: 2.", 1},
{"equals-sign phrasing", "A PodDisruptionBudget (PDB) with minAvailable=2 guarantees this.", 1},
{"english phrasing of the value", "Create a PodDisruptionBudget with a minAvailable of 2.", 1},
{"wrong resource entirely (a NetworkPolicy cannot do this)", "Use a NetworkPolicy with minAvailable: 2.", 0.5},
{"right resource, wrong/missing value", "Use a PodDisruptionBudget, but leave the fields at their defaults.", 0.5},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := tc.Eval.Evaluate(context.Background(), tt.response)
if got.Value != tt.want {
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
}
})
}
}
internal/tests/linux.go
+250
View File
@@ -8,6 +8,14 @@ import (
func registerLinuxTests(r *testkit.Registry) {
r.Register(linuxPctExecTest())
r.Register(linuxSystemdOneshotTest())
r.Register(linuxJournalctlFilterTest())
r.Register(linuxCgroupOOMDiagnosisTest())
r.Register(linuxNftablesDNATTest())
r.Register(linuxLVMExtendOrderTest())
r.Register(linuxSSHPortForwardTest())
r.Register(linuxCronExpressionTest())
r.Register(linuxSystemdTimerTest())
r.Register(linuxProcMeminfoAvailableTest())
}
// linuxPctExecTest: one-line command to check a service inside an LXC
@@ -65,3 +73,245 @@ contents only.`
Eval: evaluator,
}
}
// linuxJournalctlFilterPattern requires journalctl filtered to unit
// nginx.service and a since-window of 2 hours, in the canonical
// "-u UNIT --since WINDOW" order (the overwhelmingly common idiom), on a
// single line so a scattered mention across paragraphs does not count.
const linuxJournalctlFilterPattern = `(?i)\bjournalctl\b[^\n]*?-u\s+nginx\.service\b[^\n]*?--since\s+["']?-?2\s*h(?:ours?)?(?:\s+ago)?["']?`
// linuxJournalctlFilterTest: one-line journalctl command filtered to one
// unit over a relative time window.
func linuxJournalctlFilterTest() testkit.Test {
prompt := `Give a single, one-line journalctl command that shows only
log entries for the systemd unit "nginx.service" from the last 2 hours.`
return testkit.Test{
ID: "linux-journalctl-filter",
Category: "operations",
Subcategory: "linux",
Description: "One-line journalctl command filtered by unit and a relative 2-hour time window.",
Prompt: prompt,
Eval: eval.Regex(linuxJournalctlFilterPattern),
}
}
// linuxCgroupMemoryEvents is the inline cgroup v2 memory.events fixture for
// linuxCgroupOOMDiagnosisTest.
const linuxCgroupMemoryEvents = `low 0
high 0
max 47
oom 3
oom_kill 3`
// linuxCgroupOOMDiagnosisTest: read a cgroup v2 memory.events file and
// report whether the kernel actually OOM-killed the cgroup's processes.
//
// ground truth: cgroup v2's memory.events "oom" counts memory allocation
// failures that triggered the OOM killer to run, while "oom_kill" counts
// processes the kernel actually killed as a result. Here oom_kill is 3
// (not 0), so the cgroup's processes were in fact OOM-killed, 3 times.
func linuxCgroupOOMDiagnosisTest() testkit.Test {
prompt := `Here is the contents of a container's cgroup v2
memory.events file:
` + linuxCgroupMemoryEvents + `
Based only on this file, was any process in this cgroup actually killed by
the kernel's OOM killer, and if so, how many times? Respond with only a
JSON object: {"oom_killed": true or false, "oom_kill_count": <number>}.`
evaluator := eval.Mean(
eval.JSONField("oom_killed", true),
eval.JSONField("oom_kill_count", 3),
)
return testkit.Test{
ID: "linux-cgroup-oom-diagnosis",
Category: "operations",
Subcategory: "linux",
Description: "Diagnose an actual OOM-kill count from cgroup v2 memory.events (oom_kill, not the broader oom counter).",
Prompt: prompt,
Eval: evaluator,
}
}
// linuxNftablesDNATPattern requires a DNAT rule (iptables or nft form) on
// the nat table's prerouting chain, matching TCP port 8080 and forwarding
// to 10.0.5.20:80, on a single line. iptables and nft flags/keywords follow
// a fixed canonical order (table/chain, then match, then target), so this
// uses the same non-greedy same-line anchoring as the pct-exec lesson
// rather than a permutation of independently-orderable flags.
const linuxNftablesDNATPattern = `(?i)(?:iptables|nft)\b[^\n]*?nat[^\n]*?prerouting[^\n]*?(?:--dport|dport)\s*8080[^\n]*?dnat[^\n]*?10\.0\.5\.20:80`
// linuxNftablesDNATTest: one-line DNAT rule forwarding an external port to
// an internal host:port.
func linuxNftablesDNATTest() testkit.Test {
prompt := `Give a single, one-line firewall command (iptables or nft,
your choice) that DNATs incoming TCP traffic on this host's eth0 interface,
port 8080, to internal address 10.0.5.20, port 80.`
return testkit.Test{
ID: "linux-nftables-dnat",
Category: "operations",
Subcategory: "linux",
Description: "One-line DNAT rule (iptables or nft) forwarding TCP 8080 on eth0 to 10.0.5.20:80.",
Prompt: prompt,
Eval: eval.Regex(linuxNftablesDNATPattern),
}
}
// linuxLVMExtendWant is the unique correct 4-step LVM growth sequence for
// linuxLVMExtendOrderTest.
var linuxLVMExtendWant = []string{"pvcreate", "vgextend", "lvextend", "resize2fs"}
// linuxLVMExtendOrderTest: order the LVM commands that grow a logical
// volume and its filesystem onto a newly added disk.
//
// ground truth: a new disk must first become an LVM physical volume
// (pvcreate), then be added to the volume group (vgextend), before the
// logical volume can grow into that new space (lvextend), after which the
// filesystem itself must be grown to use it (resize2fs, for ext4). mkfs is
// never used here - it would format over, and destroy, the existing data.
func linuxLVMExtendOrderTest() testkit.Test {
prompt := `The logical volume data-lv on volume group data-vg (ext4
filesystem) is full. You have already partitioned a newly added disk as
/dev/sdb1, but done nothing else with it.
From this list of steps - pvcreate, vgextend, lvextend, resize2fs, mkfs -
choose exactly the 4 steps needed, in the correct order, to grow data-lv
and its filesystem onto the new space without destroying the existing
data. Respond with only a JSON array of the step ids, e.g.
["step1","step2",...].`
return testkit.Test{
ID: "linux-lvm-extend-order",
Category: "operations",
Subcategory: "linux",
Description: "Order the 4 LVM/filesystem commands (of 5 candidates) that grow a logical volume onto a newly added disk.",
Prompt: prompt,
Eval: eval.JSONStringArrayEquals(linuxLVMExtendWant),
}
}
// linuxSSHPortForwardPattern requires an ssh local port-forward (-L) from
// local port 15432 to the remote Postgres, connecting to the correct host.
const linuxSSHPortForwardPattern = `(?i)\bssh\b[^\n]*?-L\s*15432:(?:127\.0\.0\.1|localhost):5432[^\n]*?ops@db1\.internal`
// linuxSSHPortForwardTest: ssh -L local port-forward command derived from a
// scenario.
func linuxSSHPortForwardTest() testkit.Test {
prompt := `A Postgres database only listens on 127.0.0.1:5432 on remote
host db1.internal, which you can reach over SSH as user "ops". Give a
single, one-line ssh command, run from your laptop, that forwards your
laptop's local port 15432 to that remote Postgres instance, so a local
client can connect to localhost:15432.`
return testkit.Test{
ID: "linux-ssh-port-forward",
Category: "operations",
Subcategory: "linux",
Description: "One-line ssh -L local port-forward command reaching a Postgres instance that only listens on remote loopback.",
Prompt: prompt,
Eval: eval.Regex(linuxSSHPortForwardPattern),
}
}
// linuxCronExpressionPattern requires the exact 5-field cron schedule for
// "03:30 every Monday", tolerant only of whitespace width between fields -
// unlike eval.Equals, this stays correct even if the model adds a leading
// or trailing space, since the fields themselves have no valid rephrasing.
const linuxCronExpressionPattern = `^\s*30\s+3\s+\*\s+\*\s+1\s*$`
// linuxCronExpressionTest: produce the 5-field cron expression matching a
// stated schedule.
//
// ground truth: cron fields are minute hour day-of-month month
// day-of-week. 03:30 is minute=30, hour=3; day-of-month and month are
// unrestricted (*); Monday is day-of-week 1 in standard cron numbering
// (0 and 7 both mean Sunday).
func linuxCronExpressionTest() testkit.Test {
prompt := `Give the single 5-field cron expression that runs a job at
03:30 every Monday only. Use numeric day-of-week (Monday = 1). Respond
with only the expression, nothing else.`
return testkit.Test{
ID: "linux-cron-expression",
Category: "operations",
Subcategory: "linux",
Description: "Produce the 5-field cron expression for a 03:30-every-Monday schedule.",
Prompt: prompt,
Eval: eval.Regex(linuxCronExpressionPattern),
}
}
// linuxSystemdTimerTest: name the systemd timer+service unit pair (not
// cron) for a fixed daily schedule, with the required timer keys.
func linuxSystemdTimerTest() testkit.Test {
prompt := `A backup script must run every day at 02:00, managed by
systemd (not cron), using a timer unit plus a service unit. Name the two
unit file types this requires, give the key line in the .timer file that
sets the daily 02:00 schedule, and give the [Install] target the .timer
unit's WantedBy needs.`
// ground truth: a systemd timer job is a .timer unit (the schedule)
// paired with a same-named .service unit (the work). The .timer's
// OnCalendar= key sets the schedule (e.g. "*-*-* 02:00:00" for daily at
// 02:00), and its [Install] section needs WantedBy=timers.target for
// systemctl enable to activate it.
evaluator := eval.Mean(
eval.ContainsAll(".service", ".timer"),
eval.ContainsAny("OnCalendar"),
eval.ContainsAll("WantedBy=timers.target"),
)
return testkit.Test{
ID: "linux-systemd-timer",
Category: "operations",
Subcategory: "linux",
Description: "Name the systemd .timer/.service unit pair, OnCalendar schedule key, and timers.target for a daily job.",
Prompt: prompt,
Eval: evaluator,
}
}
// linuxProcMeminfo is the inline /proc/meminfo fixture for
// linuxProcMeminfoAvailableTest.
const linuxProcMeminfo = `MemTotal: 16384000 kB
MemFree: 1024000 kB
MemAvailable: 9876000 kB
Buffers: 512000 kB
Cached: 3200000 kB`
// linuxProcMeminfoAvailableWant is MemAvailable (9876000 kB) converted to
// whole megabytes, floored: 9876000 / 1024 = 9644.53125 -> 9644.
var linuxProcMeminfoAvailableWant = 9876000 / 1024
// linuxProcMeminfoAvailableTest: interpret /proc/meminfo's MemAvailable
// field (not MemFree) as the memory actually available for new processes.
//
// ground truth: MemFree undercounts usable memory because it excludes
// reclaimable page cache and buffers; MemAvailable is the kernel's own
// estimate of memory available for new allocations without swapping,
// already accounting for reclaimable cache. /proc/meminfo's "kB" is always
// 1024-byte kibibytes, so converting to MB divides by 1024, not 1000.
func linuxProcMeminfoAvailableTest() testkit.Test {
prompt := `Here is a container host's /proc/meminfo:
` + linuxProcMeminfo + `
How much memory, in whole megabytes (rounded down), is actually available
for new processes to allocate without triggering swap? Use the correct
field for this - not MemFree, which excludes reclaimable cache and
buffers. Note that /proc/meminfo's "kB" unit is always 1024-byte kibibytes,
so convert by dividing by 1024, not 1000. Respond with only the number.`
return testkit.Test{
ID: "linux-proc-meminfo-available",
Category: "operations",
Subcategory: "linux",
Description: "Convert /proc/meminfo's MemAvailable field (not MemFree) to whole megabytes.",
Prompt: prompt,
Eval: eval.Numeric(eval.ExtractLastNumber[int], linuxProcMeminfoAvailableWant, 1),
}
}
internal/tests/linux_test.go
+234
View File
@@ -126,3 +126,237 @@ WantedBy=graphical.target`
})
}
}
func TestLinuxJournalctlFilterTest_Eval(t *testing.T) {
tc := linuxJournalctlFilterTest()
tests := []struct {
name string
response string
want float64
}{
{"quoted negative window", `journalctl -u nginx.service --since "-2 hours"`, 1},
{"english phrasing window", `journalctl -u nginx.service --since "2 hours ago"`, 1},
{"compact -2h form, no quotes", `Run: journalctl -u nginx.service --since -2h`, 1},
{"wrong unit", `journalctl -u apache.service --since "-2 hours"`, 0},
{"missing unit filter entirely", `journalctl --since "-2 hours"`, 0},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := tc.Eval.Evaluate(context.Background(), tt.response)
if got.Value != tt.want {
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
}
})
}
}
func TestLinuxCgroupOOMDiagnosisTest_Eval(t *testing.T) {
tc := linuxCgroupOOMDiagnosisTest()
tests := []struct {
name string
response string
want float64
}{
{"correct object", `{"oom_killed": true, "oom_kill_count": 3}`, 1},
{"prose then JSON", "Yes, it was OOM-killed.\n" + `{"oom_killed": true, "oom_kill_count": 3}`, 1},
{"fenced JSON", "```json\n" + `{"oom_killed": true, "oom_kill_count": 3}` + "\n```", 1},
{"confuses oom counter with oom_kill counter", `{"oom_killed": true, "oom_kill_count": 47}`, 0.5},
{"wrongly claims no kill happened", `{"oom_killed": false, "oom_kill_count": 0}`, 0},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := tc.Eval.Evaluate(context.Background(), tt.response)
if got.Value != tt.want {
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
}
})
}
}
func TestLinuxNftablesDNATTest_Eval(t *testing.T) {
tc := linuxNftablesDNATTest()
tests := []struct {
name string
response string
want float64
}{
{
name: "iptables form",
response: `iptables -t nat -A PREROUTING -i eth0 -p tcp --dport 8080 -j DNAT --to-destination 10.0.5.20:80`,
want: 1,
},
{
name: "iptables form with flags reordered around the match",
response: `sudo iptables -t nat -A PREROUTING -p tcp -i eth0 --dport 8080 -j DNAT --to-destination 10.0.5.20:80`,
want: 1,
},
{
name: "nft form",
response: `nft add rule ip nat prerouting tcp dport 8080 dnat to 10.0.5.20:80`,
want: 1,
},
{
name: "wrong destination IP",
response: `iptables -t nat -A PREROUTING -p tcp --dport 8080 -j DNAT --to-destination 10.0.5.21:80`,
want: 0,
},
{
name: "wrong chain (OUTPUT instead of PREROUTING)",
response: `iptables -t nat -A OUTPUT -p tcp --dport 8080 -j DNAT --to-destination 10.0.5.20:80`,
want: 0,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := tc.Eval.Evaluate(context.Background(), tt.response)
if got.Value != tt.want {
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
}
})
}
}
func TestLinuxLVMExtendOrderTest_Eval(t *testing.T) {
tc := linuxLVMExtendOrderTest()
tests := []struct {
name string
response string
want float64
}{
{"correct order", `["pvcreate","vgextend","lvextend","resize2fs"]`, 1},
{"correct order, prose wrapped", "Steps:\n" + `["pvcreate","vgextend","lvextend","resize2fs"]`, 1},
{"correct order, fenced JSON", "```json\n" + `["pvcreate","vgextend","lvextend","resize2fs"]` + "\n```", 1},
{"wrong order (lvextend before vgextend)", `["pvcreate","lvextend","vgextend","resize2fs"]`, 0},
{"includes destructive mkfs instead of resize2fs", `["pvcreate","vgextend","lvextend","mkfs"]`, 0},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := tc.Eval.Evaluate(context.Background(), tt.response)
if got.Value != tt.want {
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
}
})
}
}
func TestLinuxSSHPortForwardTest_Eval(t *testing.T) {
tc := linuxSSHPortForwardTest()
tests := []struct {
name string
response string
want float64
}{
{"loopback IP form", `ssh -L 15432:127.0.0.1:5432 [email protected]`, 1},
{"localhost form", `ssh -L 15432:localhost:5432 [email protected]`, 1},
{"extra flags before -L", `ssh -N -L 15432:127.0.0.1:5432 [email protected]`, 1},
{"wrong local port", `ssh -L 5432:127.0.0.1:5432 [email protected]`, 0},
{"wrong host", `ssh -L 15432:127.0.0.1:5432 [email protected]`, 0},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := tc.Eval.Evaluate(context.Background(), tt.response)
if got.Value != tt.want {
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
}
})
}
}
func TestLinuxCronExpressionTest_Eval(t *testing.T) {
tc := linuxCronExpressionTest()
tests := []struct {
name string
response string
want float64
}{
{"canonical single spacing", "30 3 * * 1", 1},
{"extra spaces between fields", "30 3 * * 1", 1},
{"tab-separated fields", "30\t3\t*\t*\t1", 1},
{"wrong day of week", "30 3 * * 2", 0},
{"wrong time", "45 3 * * 1", 0},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := tc.Eval.Evaluate(context.Background(), tt.response)
if got.Value != tt.want {
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
}
})
}
}
func TestLinuxSystemdTimerTest_Eval(t *testing.T) {
tc := linuxSystemdTimerTest()
tests := []struct {
name string
response string
want float64
}{
{
name: "full correct answer",
response: `Create backup.service and backup.timer. In backup.timer's
[Timer] section: OnCalendar=*-*-* 02:00:00. In [Install]:
WantedBy=timers.target.`,
want: 1,
},
{
name: "reordered phrasing, service named first then timer",
response: `You need a .service unit for the work and a .timer unit for the schedule. Set OnCalendar=02:00 in the timer, and WantedBy=timers.target under [Install].`,
want: 1,
},
{
name: "minimal but complete",
response: `backup.service + backup.timer, OnCalendar=*-*-* 02:00:00, WantedBy=timers.target`,
want: 1,
},
{
name: "missing OnCalendar key entirely",
response: `Use backup.service and backup.timer, with WantedBy=timers.target in [Install].`,
want: 2.0 / 3.0,
},
{
name: "wrong install target and no OnCalendar (cron-style thinking)",
response: `Just add a backup.service and backup.timer, and enable it with WantedBy=multi-user.target.`,
want: 1.0 / 3.0,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := tc.Eval.Evaluate(context.Background(), tt.response)
if got.Value != tt.want {
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
}
})
}
}
func TestLinuxProcMeminfoAvailableTest_Eval(t *testing.T) {
tc := linuxProcMeminfoAvailableTest()
tests := []struct {
name string
response string
want float64
}{
{"exact number", "9644", 1},
{"prose wrapped", "The available memory is 9644 MB.", 1},
{"with unit suffix", "9644 MB", 1},
{"divided by 1000 instead of 1024", "9876", 0},
{"used MemFree instead of MemAvailable", "1000", 0},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := tc.Eval.Evaluate(context.Background(), tt.response)
if got.Value != tt.want {
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
}
})
}
}
internal/tests/macos.go
+265
View File
@@ -8,6 +8,14 @@ import (
func registerMacOSTests(r *testkit.Registry) {
r.Register(macosTimeoutPortabilityTest())
r.Register(macosLaunchdCronTest())
r.Register(macosSedInplaceTest())
r.Register(macosLaunchctlBootstrapTest())
r.Register(macosPlutilPlistReadTest())
r.Register(macosMdfindSpotlightTest())
r.Register(macosAPFSSnapshotTest())
r.Register(macosSSHIdentityAgentTest())
r.Register(macosCaffeinateTest())
r.Register(macosStatBSDGNUTest())
}
// macosTimeoutPortabilityTest: explain why `timeout` is missing on stock
@@ -82,3 +90,260 @@ file belongs in for a per-machine (not per-user) job.`
Eval: evaluator,
}
}
// macosSedPattern requires the BSD-form "sed -i" with an empty
// single-quoted or double-quoted extension argument, invoking a foo->bar
// substitution on file.txt, on a single line (no DOTALL) so a scattered,
// multi-paragraph mention of the same tokens does not count as a real
// one-line command.
//
// ground truth: BSD sed's -i flag takes a mandatory in-place-suffix
// argument (used for backup files); passing an empty single-quoted or
// double-quoted string means "no backup file". GNU sed's -i takes an
// OPTIONAL argument, so
// "sed -i 's/foo/bar/' file.txt" (no argument to -i) works on Linux but
// errors on macOS, where it is parsed as "-i 's/foo/bar/'" (using the sed
// script itself as the backup suffix) followed by a missing script operand.
const macosSedPattern = `(?i)\bsed\b[^\n]*-i\s*(''|"")[^\n]*s/foo/bar/g?[^\n]*file\.txt`
// macosSedInplaceTest: give the portable BSD sed one-liner for in-place
// substitution with no backup file, contrasted with the GNU-only form that
// fails on stock macOS.
func macosSedInplaceTest() testkit.Test {
prompt := `On a stock macOS machine (BSD sed, not GNU sed), this command:
` + "```sh" + `
sed -i 's/foo/bar/' file.txt
` + "```" + `
fails with an error about a missing extension or a "no such file"
complaint. Give the corrected one-line BSD sed command that replaces "foo"
with "bar" in file.txt, edits the file in place, and creates NO backup
file.`
return testkit.Test{
ID: "macos-sed-inplace",
Category: "operations",
Subcategory: "macos",
Description: "Give the BSD sed -i '' in-place substitution syntax that works on stock macOS with no backup file.",
Prompt: prompt,
Eval: eval.Regex(macosSedPattern),
}
}
// macosLaunchctlBootstrapTest: identify launchctl bootstrap with an
// explicit system domain target as the modern replacement for the
// deprecated launchctl load, for loading a LaunchDaemon.
func macosLaunchctlBootstrapTest() testkit.Test {
prompt := `A new LaunchDaemon plist has just been written to
/Library/LaunchDaemons/com.example.myjob.plist on a current macOS release.
On modern macOS (10.11 El Capitan and later), which launchctl subcommand
should you use to load and start this LaunchDaemon, and which domain
target must you pass it (this is a system-wide daemon, not a per-user
agent)? Also name the older launchctl subcommand for loading jobs that
Apple deprecated in favor of this one, and briefly say why (it lacked
explicit domain targeting).`
// ground truth: Apple deprecated the session-based "launchctl load" /
// "launchctl unload" pair starting in OS X 10.11, replacing them with
// "launchctl bootstrap" / "launchctl bootout", which require an
// explicit domain target (system/, gui/<uid>, user/<uid>). A
// system-wide LaunchDaemon uses the "system" domain, giving
// "sudo launchctl bootstrap system /Library/LaunchDaemons/....plist".
// Component 3 deliberately requires an explicit deprecation cue rather
// than a bare "launchctl load" mention: a response that just uses
// launchctl load (as its own recommended command, not something to
// avoid) must not get credit for correctly flagging it as deprecated.
evaluator := eval.Mean(
eval.ContainsAny("bootstrap"),
eval.ContainsAny("bootstrap system", "system domain", "domain system", "target system", "system target"),
eval.ContainsAny("deprecated", "avoid launchctl load", "no longer recommended", "outdated"),
)
return testkit.Test{
ID: "macos-launchctl-bootstrap",
Category: "operations",
Subcategory: "macos",
Description: "Identify launchctl bootstrap+system domain as the modern replacement for the deprecated launchctl load.",
Prompt: prompt,
Eval: evaluator,
}
}
// macosPlutilPattern requires the built-in plutil -p pretty-printer
// invoked directly on the plist path, on one line.
const macosPlutilPattern = `(?i)\bplutil\b[^\n]*-p[^\n]*/Library/Preferences/com\.example\.app\.plist`
// macosPlutilPlistReadTest: pretty-print a plist file's contents using a
// tool that ships with macOS (no Homebrew installs).
func macosPlutilPlistReadTest() testkit.Test {
prompt := `You need to inspect the raw contents of this plist file on a
stock macOS machine, with no Homebrew packages installed:
/Library/Preferences/com.example.app.plist
Give a single command, using only a tool that ships with macOS by default,
that prints the plist's contents to your terminal in a readable
(non-binary) form.`
// ground truth: plutil ships with macOS and its -p flag pretty-prints a
// plist (binary or XML) to stdout in a human-readable form, taking the
// file path directly - no domain-name lookup or Homebrew install
// needed, unlike "defaults read <domain>".
return testkit.Test{
ID: "macos-plutil-plist-read",
Category: "operations",
Subcategory: "macos",
Description: "Pretty-print a plist file's contents using the built-in plutil -p command.",
Prompt: prompt,
Eval: eval.Regex(macosPlutilPattern),
}
}
// macosMdfindSpotlightTest: use mdfind (Spotlight's metadata index) instead
// of a full filesystem walk to find files by name.
func macosMdfindSpotlightTest() testkit.Test {
prompt := `On macOS, give a single command that searches Spotlight's
existing metadata index (not a full filesystem walk with "find") for files
whose name contains "invoice", scoped only to your home directory (~).`
// ground truth: mdfind queries the already-built Spotlight index rather
// than walking the filesystem; -onlyin scopes the search to a
// directory, and either -name or a plain query string matches on file
// name.
evaluator := eval.Mean(
eval.ContainsAll("mdfind"),
eval.ContainsAny("-onlyin", "-name"),
eval.ContainsAny("invoice"),
)
return testkit.Test{
ID: "macos-mdfind-spotlight",
Category: "operations",
Subcategory: "macos",
Description: "Use mdfind against Spotlight's index, scoped to a directory, instead of a full filesystem walk.",
Prompt: prompt,
Eval: evaluator,
}
}
// macosAPFSSnapshotTest: create and list local APFS snapshots via tmutil,
// which works even with no external Time Machine disk attached.
func macosAPFSSnapshotTest() testkit.Test {
prompt := `Before a risky system change, you want to create a local APFS
snapshot of the root volume as a quick rollback point - with no external
Time Machine disk attached (APFS local snapshots do not need one). Give
the command to create a local snapshot now, and the separate command to
list the local snapshots that currently exist.`
// ground truth: tmutil manages local APFS snapshots independently of
// having a Time Machine backup destination attached. "tmutil snapshot"
// creates one immediately; "tmutil listlocalsnapshots /" lists the
// local snapshots that exist for the root volume.
evaluator := eval.Mean(
eval.ContainsAny("tmutil snapshot"),
eval.ContainsAny("tmutil listlocalsnapshots", "listlocalsnapshots"),
)
return testkit.Test{
ID: "macos-apfs-snapshot",
Category: "operations",
Subcategory: "macos",
Description: "Create and list local APFS snapshots via tmutil with no external Time Machine disk attached.",
Prompt: prompt,
Eval: evaluator,
}
}
// macosSSHIdentityAgentTest: point ssh at 1Password's SSH agent socket via
// the IdentityAgent ssh_config directive.
func macosSSHIdentityAgentTest() testkit.Test {
prompt := `You use the 1Password app as your SSH agent instead of the
default ssh-agent, and your shell environment does NOT set SSH_AUTH_SOCK.
1Password's agent socket lives at:
~/Library/Group Containers/2BUA8C4S2C.com.1password/t/agent.sock
Give the single ssh_config line to add (to ~/.ssh/config, either globally
or under a Host block) that points ssh at this socket, so ssh and git-over-
ssh commands can find your keys through 1Password.`
// ground truth: ssh_config's IdentityAgent directive overrides which
// agent socket ssh (and anything using libssh's config, like git)
// connects to; 1Password's own setup docs specify this exact directive
// and socket path for its SSH agent integration.
evaluator := eval.Mean(
eval.ContainsAny("IdentityAgent"),
eval.ContainsAny("1password/t/agent.sock", "agent.sock"),
)
return testkit.Test{
ID: "macos-ssh-identity-agent",
Category: "operations",
Subcategory: "macos",
Description: "Point ssh at 1Password's SSH agent socket via the IdentityAgent ssh_config directive.",
Prompt: prompt,
Eval: evaluator,
}
}
// macosCaffeinateTest: use caffeinate, built into macOS, to keep the
// machine awake only for the duration of a given command.
func macosCaffeinateTest() testkit.Test {
prompt := `A long-running data migration script, ./migrate.sh, needs to
run on a MacBook without the machine going to sleep partway through - and
you want sleep prevention to end automatically the moment the script
finishes, with no separate cleanup step. Give a single command, using a
tool built into macOS, that does this.`
// ground truth: caffeinate, bundled with macOS, takes a power-management
// assertion for as long as a given utility runs when you pass that
// utility as its argument, and releases the assertion automatically
// when the utility exits - no manual cleanup needed.
evaluator := eval.Mean(
eval.ContainsAll("caffeinate"),
eval.ContainsAny("migrate.sh"),
)
return testkit.Test{
ID: "macos-caffeinate",
Category: "operations",
Subcategory: "macos",
Description: "Use caffeinate to prevent sleep only for the duration of a given script.",
Prompt: prompt,
Eval: evaluator,
}
}
// macosStatPattern requires the BSD stat -f format-string form printing
// only the byte size (%z) of file.txt.
const macosStatPattern = `(?i)\bstat\b[^\n]*-f\s*"?%z"?[^\n]*file\.txt`
// macosStatBSDGNUTest: give the BSD stat command (macOS's built-in stat)
// for a file's size in bytes, and name the differing GNU stat flag.
func macosStatBSDGNUTest() testkit.Test {
prompt := `On macOS (BSD stat, not GNU stat), give a single command that
prints ONLY the size of file.txt in bytes, nothing else. Then name the
single-letter flag GNU stat (as found on Linux) uses in place of macOS's
-f for its own format-string argument.`
// ground truth: BSD stat takes its format string via -f, and %z is the
// byte-size format specifier, giving "stat -f%z file.txt". GNU stat
// instead uses -c for its format string (with %s as the byte-size
// specifier) - the flag letter itself, not just the format string
// syntax, differs between the two implementations.
evaluator := eval.Mean(
eval.Regex(macosStatPattern),
eval.ContainsAny("-c"),
)
return testkit.Test{
ID: "macos-stat-bsd-gnu",
Category: "operations",
Subcategory: "macos",
Description: "Give the BSD stat -f%z byte-size command and name the differing GNU stat flag (-c).",
Prompt: prompt,
Eval: evaluator,
}
}
internal/tests/macos_test.go
+224
View File
@@ -78,3 +78,227 @@ StartInterval set to 300.`
})
}
}
func TestMacosSedInplaceTest_Eval(t *testing.T) {
tc := macosSedInplaceTest()
tests := []struct {
name string
response string
want float64
}{
{"single-quote empty extension", `sed -i '' 's/foo/bar/' file.txt`, 1},
{"double-quote empty extension with -e flag", `On macOS: sed -i "" -e 's/foo/bar/' file.txt`, 1},
{"global replace flag still matches", `sed -i '' 's/foo/bar/g' file.txt`, 1},
{"GNU-style with no extension argument is wrong on macOS", `sed -i 's/foo/bar/' file.txt`, 0},
{"backup extension creates a file, contradicting the requirement", `sed -i.bak 's/foo/bar/' file.txt`, 0},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := tc.Eval.Evaluate(context.Background(), tt.response)
if got.Value != tt.want {
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
}
})
}
}
func TestMacosLaunchctlBootstrapTest_Eval(t *testing.T) {
tc := macosLaunchctlBootstrapTest()
good := `Use "sudo launchctl bootstrap system /Library/LaunchDaemons/com.example.myjob.plist".
The older "launchctl load" command is deprecated because it has no explicit
domain target.`
goodReordered := `The modern subcommand is bootstrap, targeting the system
domain: sudo launchctl bootstrap system /Library/LaunchDaemons/com.example.myjob.plist.
launchctl load is deprecated.`
goodPhrasing := `Load it with launchctl bootstrap, using the system domain
target since this is a system-wide daemon. Avoid launchctl load, which
Apple deprecated for lacking an explicit domain target.`
badWrongSubcommand := `Just run "sudo launchctl load /Library/LaunchDaemons/com.example.myjob.plist".`
badMissingDomain := `Use launchctl bootstrap to load the daemon.`
tests := []struct {
name string
response string
want float64
}{
{"good answer full credit", good, 1},
{"good answer, reordered phrasing", goodReordered, 1},
{"good answer, alternate phrasing", goodPhrasing, 1},
{"wrong subcommand (deprecated load only) scores 0", badWrongSubcommand, 0},
{"missing domain target and deprecation note loses two thirds", badMissingDomain, 1.0 / 3.0},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := tc.Eval.Evaluate(context.Background(), tt.response)
if got.Value != tt.want {
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
}
})
}
}
func TestMacosPlutilPlistReadTest_Eval(t *testing.T) {
tc := macosPlutilPlistReadTest()
tests := []struct {
name string
response string
want float64
}{
{"correct plutil -p", `plutil -p /Library/Preferences/com.example.app.plist`, 1},
{"prose wrapped", `Run: plutil -p /Library/Preferences/com.example.app.plist to pretty-print it.`, 1},
{"quoted path", `plutil -p "/Library/Preferences/com.example.app.plist"`, 1},
{"wrong tool entirely (cat prints binary garbage)", `cat /Library/Preferences/com.example.app.plist`, 0},
{"plutil without -p just validates, does not print", `plutil /Library/Preferences/com.example.app.plist`, 0},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := tc.Eval.Evaluate(context.Background(), tt.response)
if got.Value != tt.want {
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
}
})
}
}
func TestMacosMdfindSpotlightTest_Eval(t *testing.T) {
tc := macosMdfindSpotlightTest()
tests := []struct {
name string
response string
want float64
}{
{"onlyin with query", `mdfind -onlyin ~ "invoice"`, 1},
{"name flag", `mdfind -onlyin ~ -name invoice`, 1},
{"prose wrapped", `Run mdfind -onlyin ~ invoice to search the Spotlight index.`, 1},
{"uses find instead of mdfind, walks the filesystem", `find ~ -iname "*invoice*"`, 1.0 / 3.0},
{"mdfind used but searching for the wrong term entirely", `mdfind -onlyin ~ "receipt"`, 2.0 / 3.0},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := tc.Eval.Evaluate(context.Background(), tt.response)
if got.Value != tt.want {
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
}
})
}
}
func TestMacosAPFSSnapshotTest_Eval(t *testing.T) {
tc := macosAPFSSnapshotTest()
tests := []struct {
name string
response string
want float64
}{
{"both commands", "Create: tmutil snapshot\nList: tmutil listlocalsnapshots /", 1},
{"prose wrapped", "Run `tmutil snapshot` to create one, then `tmutil listlocalsnapshots /` to list them.", 1},
{"listlocalsnapshots without explicit tmutil prefix restated", "First tmutil snapshot, then check with listlocalsnapshots /.", 1},
{"only creates, never lists", "Just run tmutil snapshot.", 0.5},
{"wrong tool (diskutil) mentioned instead of tmutil", "Use diskutil apfs snapshot / to create one.", 0},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := tc.Eval.Evaluate(context.Background(), tt.response)
if got.Value != tt.want {
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
}
})
}
}
func TestMacosSSHIdentityAgentTest_Eval(t *testing.T) {
tc := macosSSHIdentityAgentTest()
tests := []struct {
name string
response string
want float64
}{
{
name: "correct directive and socket",
response: `IdentityAgent "~/Library/Group Containers/2BUA8C4S2C.com.1password/t/agent.sock"`,
want: 1,
},
{
name: "inside a Host block, prose wrapped",
response: "Add this under your Host block:\nHost *\n IdentityAgent ~/Library/Group Containers/2BUA8C4S2C.com.1password/t/agent.sock",
want: 1,
},
{
name: "shortened socket mention still credited",
response: "Set IdentityAgent to point at 1Password's agent.sock in its Group Containers path.",
want: 1,
},
{
name: "sets SSH_AUTH_SOCK env var instead of ssh_config directive",
response: `export SSH_AUTH_SOCK=~/Library/Group Containers/2BUA8C4S2C.com.1password/t/agent.sock`,
want: 0.5,
},
{
name: "wrong directive entirely",
response: `Add IdentityFile ~/.ssh/id_ed25519 to your config.`,
want: 0,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := tc.Eval.Evaluate(context.Background(), tt.response)
if got.Value != tt.want {
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
}
})
}
}
func TestMacosCaffeinateTest_Eval(t *testing.T) {
tc := macosCaffeinateTest()
tests := []struct {
name string
response string
want float64
}{
{"bare invocation", `caffeinate ./migrate.sh`, 1},
{"with system-sleep flag", `caffeinate -s ./migrate.sh`, 1},
{"prose wrapped", "Run `caffeinate -i ./migrate.sh` and it stops holding the assertion once migrate.sh exits.", 1},
{"wrong script named, not the target", `caffeinate ./other-script.sh`, 0.5},
{"unrelated tool, does not auto-release on exit", `Just run pmset noidle before starting your script.`, 0},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := tc.Eval.Evaluate(context.Background(), tt.response)
if got.Value != tt.want {
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
}
})
}
}
func TestMacosStatBSDGNUTest_Eval(t *testing.T) {
tc := macosStatBSDGNUTest()
tests := []struct {
name string
response string
want float64
}{
{"correct BSD form and GNU flag", `stat -f%z file.txt` + "\nGNU stat uses -c instead.", 1},
{"quoted format string", `stat -f "%z" file.txt, and on Linux GNU stat uses -c.`, 1},
{"prose wrapped", "Run stat -f%z file.txt on macOS; the GNU equivalent flag is -c.", 1},
{"correct BSD command but never names the GNU flag", `stat -f%z file.txt`, 0.5},
{"GNU -c%s syntax used on macOS scores half: -c token matches but -f BSD form is missing", `stat -c%s file.txt`, 0.5},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := tc.Eval.Evaluate(context.Background(), tt.response)
if got.Value != tt.want {
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
}
})
}
}