diff --git a/internal/tests/kubernetes.go b/internal/tests/kubernetes.go index 327e4a9..89805bb 100644 --- a/internal/tests/kubernetes.go +++ b/internal/tests/kubernetes.go @@ -11,6 +11,15 @@ import ( func registerKubernetesTests(r *testkit.Registry) { r.Register(k8sCrashloopGitopsTest()) + r.Register(k8sImagePullBackoffTest()) + r.Register(k8sPendingTaintsTest()) + r.Register(k8sPVCStorageclassMismatchTest()) + r.Register(k8sServiceSelectorMismatchTest()) + r.Register(k8sNetworkPolicyDNSBlockTest()) + r.Register(k8sArgoCDOutOfSyncTest()) + r.Register(k8sTraefikIngressRouteHostTest()) + r.Register(k8sQoSClassTest()) + r.Register(k8sCNPGPDBTest()) } // k8sCrashloopGitopsDescribeExcerpt is the inline kubectl describe pod @@ -137,3 +146,454 @@ procedure to fix it given that ArgoCD manages this cluster?` Eval: evaluator, } } + +// k8sImagePullBackoffDescribeExcerpt is the inline kubectl describe pod +// excerpt for k8sImagePullBackoffTest. +const k8sImagePullBackoffDescribeExcerpt = `Name: checkout-api-7d8f9c6b5d-4mnbv +Namespace: checkout +Status: Pending +Containers: + api: + Image: registry.internal.example.com/checkout-api:v1.2.3 + State: Waiting + Reason: ImagePullBackOff +Events: + Warning Failed kubelet Failed to pull image "registry.internal.example.com/checkout-api:v1.2.3": rpc error: code = Unknown desc = failed to pull and unpack image: failed to resolve reference: unexpected status code 401 Unauthorized: authentication required + Warning Failed kubelet Error: ErrImagePull + Normal BackOff kubelet Back-off pulling image "registry.internal.example.com/checkout-api:v1.2.3"` + +// k8sImagePullBackoffTest: classify an ImagePullBackOff's root cause from +// the HTTP status embedded in the pull error, and name the matching fix. +// +// ground truth: the pull error is "401 Unauthorized: authentication +// required" - the registry rejected the request for lacking credentials, +// not because the image/tag is missing (that would be a 404/"not found") +// or because the registry is unreachable (that would be a network/DNS/TLS +// error). The fix is to give the pod credentials for the private registry +// via imagePullSecrets, not to touch the image tag or check connectivity. +func k8sImagePullBackoffTest() testkit.Test { + prompt := `Here is the output of "kubectl describe pod" for a pod stuck +in ImagePullBackOff: + +` + k8sImagePullBackoffDescribeExcerpt + ` + +What category of root cause is this, and what category of fix addresses +it? Respond with only a JSON object: +{"cause":"","fix":""}` + + evaluator := eval.Mean( + eval.JSONField("cause", "auth"), + eval.JSONField("fix", "imagePullSecrets"), + ) + + return testkit.Test{ + ID: "k8s-imagepullbackoff", + Category: "operations", + Subcategory: "kubernetes", + Description: "Classify an ImagePullBackOff's root cause (401 auth failure, not a missing tag or network issue) and its fix category.", + Prompt: prompt, + Eval: evaluator, + } +} + +// k8sPendingTaintsDescribeExcerpt is the inline kubectl describe pod +// excerpt for k8sPendingTaintsTest. +const k8sPendingTaintsDescribeExcerpt = `Name: batch-etl-59c8d8c9f7-k2p8q +Namespace: data +Status: Pending +Spec (excerpt): + tolerations: + nodeSelector: +Events: + Warning FailedScheduling default-scheduler 0/3 nodes are available: 3 node(s) had untolerated taint {dedicated: gpu-only: NoSchedule}. preemption: 0/3 nodes are available: 3 Preemption is not helpful for scheduling.` + +// k8sPendingTaintsTest: identify an untolerated node taint as the reason a +// pod cannot be scheduled, and name the pod-spec field that fixes it. +// +// ground truth: the FailedScheduling event names the exact blocker - +// "untolerated taint {dedicated: gpu-only: NoSchedule}" on all 3 +// candidate nodes - and the pod spec confirms it has no tolerations at +// all. Adding a matching entry under the pod's tolerations field (not +// nodeSelector, which restricts placement rather than tolerating a taint, +// and not resources, which is unrelated) is the fix. +func k8sPendingTaintsTest() testkit.Test { + prompt := `Here is the output of "kubectl describe pod" for a pod stuck +Pending: + +` + k8sPendingTaintsDescribeExcerpt + ` + +Why can the scheduler not place this pod on any of the 3 candidate nodes, +and which pod spec field must be added to fix it? Respond with only a +JSON object: +{"cause":"","fix_field":""}` + + evaluator := eval.Mean( + eval.JSONField("cause", "taint"), + eval.JSONField("fix_field", "tolerations"), + ) + + return testkit.Test{ + ID: "k8s-pending-taints", + Category: "operations", + Subcategory: "kubernetes", + Description: "Identify an untolerated node taint blocking scheduling and the tolerations field that fixes it.", + Prompt: prompt, + Eval: evaluator, + } +} + +// k8sPVCStorageclassExcerpt is the inline kubectl describe pvc + kubectl +// get storageclass excerpt for k8sPVCStorageclassMismatchTest. +const k8sPVCStorageclassExcerpt = `$ kubectl describe pvc data-pvc -n analytics +Name: data-pvc +Namespace: analytics +StorageClass: fast-ssd +Status: Pending +Events: + Warning ProvisioningFailed persistentvolume-controller storageclass.storage.k8s.io "fast-ssd" not found + +$ kubectl get storageclass +NAME PROVISIONER +standard csi.example.com` + +// k8sPVCStorageclassMismatchTest: diagnose a Pending PVC referencing a +// StorageClass that does not exist in the cluster, and pick the existing +// class to switch to. +// +// ground truth: the ProvisioningFailed event states plainly that +// StorageClass "fast-ssd" does not exist; "kubectl get storageclass" shows +// only "standard" exists. Since this cluster's StorageClasses are managed +// centrally via GitOps and creating one ad hoc is ruled out by the prompt, +// the only available fix is to point the PVC at the StorageClass that +// already exists: "standard". +func k8sPVCStorageclassMismatchTest() testkit.Test { + prompt := `This cluster's StorageClasses are defined centrally via +GitOps; creating a new StorageClass ad hoc is not an option here. Here is +the PVC and the StorageClasses that actually exist in the cluster: + +` + k8sPVCStorageclassExcerpt + ` + +What caused this PVC to stay Pending, and which existing StorageClass name +should it be changed to reference so it can be provisioned? Respond with +only a JSON object: +{"cause":"","use_storageclass":""}` + + evaluator := eval.Mean( + eval.JSONField("cause", "missing-storageclass"), + eval.JSONField("use_storageclass", "standard"), + ) + + return testkit.Test{ + ID: "k8s-pvc-storageclass-mismatch", + Category: "operations", + Subcategory: "kubernetes", + Description: "Diagnose a Pending PVC referencing a nonexistent StorageClass and pick the one existing class to use instead.", + Prompt: prompt, + Eval: evaluator, + } +} + +// k8sServiceSelectorMismatchYAML is the inline Deployment + Service YAML +// excerpt for k8sServiceSelectorMismatchTest. +const k8sServiceSelectorMismatchYAML = `# Deployment (excerpt) +apiVersion: apps/v1 +kind: Deployment +metadata: + name: inventory-api + namespace: inventory +spec: + template: + metadata: + labels: + app: inventory-api + tier: backend + +--- +# Service +apiVersion: v1 +kind: Service +metadata: + name: inventory-api + namespace: inventory +spec: + selector: + app: inventory-api-svc + ports: + - port: 80 + targetPort: 8080` + +// k8sServiceSelectorMismatchTest: spot a Service selector that does not +// match the Deployment's pod-template labels, leaving the Service with no +// Endpoints. +// +// ground truth: the pods carry label app=inventory-api (from the +// Deployment's pod template), but the Service's selector is +// app=inventory-api-svc - a value that matches no pod's labels, so the +// Service has zero Endpoints. The single wrong field is the Service's +// spec.selector.app value; it must be changed to "inventory-api" to match +// the pods. +func k8sServiceSelectorMismatchTest() testkit.Test { + prompt := `Requests to this Service time out with no available +endpoints. Here is the Deployment's pod template and the Service: + +` + k8sServiceSelectorMismatchYAML + ` + +Which single field in the Service YAML is wrong, and what value should it +be changed to so the Service's Endpoints include these pods? Respond with +only a JSON object: {"field":"selector.app","value":""}` + + evaluator := eval.Mean( + eval.JSONField("field", "selector.app"), + eval.JSONField("value", "inventory-api"), + ) + + return testkit.Test{ + ID: "k8s-service-selector-mismatch", + Category: "operations", + Subcategory: "kubernetes", + Description: "Spot a Service selector.app value that does not match the Deployment's pod-template labels.", + Prompt: prompt, + Eval: evaluator, + } +} + +// k8sNetworkPolicyDNSYAML is the inline default-deny-egress NetworkPolicy +// excerpt for k8sNetworkPolicyDNSBlockTest. +const k8sNetworkPolicyDNSYAML = `apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: payments-egress + namespace: payments +spec: + podSelector: + matchLabels: + app: payments + policyTypes: + - Egress + egress: + - to: + - podSelector: + matchLabels: + app: payments-db + ports: + - protocol: TCP + port: 5432` + +// k8sNetworkPolicyDNSBlockTest: explain why a default-deny-egress +// NetworkPolicy that only allows one destination breaks DNS resolution, +// and what egress rule fixes it. +// +// ground truth: this NetworkPolicy selects pods labeled app=payments, +// applies to Egress, and lists exactly one allowed destination (the +// payments-db pods on 5432/TCP) - meaning every other egress, including +// DNS lookups to CoreDNS/kube-dns on port 53, is implicitly denied. The +// fix is an additional egress rule allowing UDP (and typically TCP) port +// 53 to the cluster's DNS pods (conventionally labeled k8s-app=kube-dns in +// the kube-system namespace). +func k8sNetworkPolicyDNSBlockTest() testkit.Test { + prompt := `Pods labeled app=payments in the payments namespace can reach +the payments database but cannot resolve ANY DNS name, including external +ones. Here is the only NetworkPolicy selecting these pods: + +` + k8sNetworkPolicyDNSYAML + ` + +Why does this NetworkPolicy break DNS resolution for these pods, and what +additional egress rule (name the port, protocol, and typical target label) +must be added to fix it?` + + evaluator := eval.All( + eval.W(eval.ContainsAll("53"), 2), + eval.W(eval.ContainsAny("udp", "UDP"), 2), + eval.W(eval.ContainsAny("kube-dns", "coredns", "kube-system"), 2), + ) + + return testkit.Test{ + ID: "k8s-networkpolicy-dns-block", + Category: "operations", + Subcategory: "kubernetes", + Description: "Explain why a single-destination-allow egress NetworkPolicy implicitly blocks DNS, and the port-53/UDP fix.", + Prompt: prompt, + Eval: evaluator, + } +} + +// k8sArgoCDOutOfSyncStatus is the inline "argocd app get" excerpt for +// k8sArgoCDOutOfSyncTest. +const k8sArgoCDOutOfSyncStatus = `$ argocd app get inventory-api +Name: inventory-api +Namespace: inventory +Health Status: Healthy +Sync Status: OutOfSync + +Resources: + Kind Name Status + Deployment inventory-api OutOfSync + +Diff (live vs target, Deployment/inventory-api): + live: spec.replicas: 5 + target: spec.replicas: 2 + +Note: a HorizontalPodAutoscaler named inventory-api-hpa also targets this +Deployment and actively manages its replica count based on CPU load. +Nobody has manually run "kubectl scale", "kubectl edit", or +"argocd app sync" recently.` + +// k8sArgoCDOutOfSyncTest: identify an HPA-managed field as the reason an +// ArgoCD Application stays OutOfSync, and the idiomatic fix. +// +// ground truth: the only diff is spec.replicas, and an HPA legitimately +// changes that field outside of git as it scales the Deployment - so the +// Application will drift back OutOfSync after every "argocd app sync" +// without addressing the underlying cause. ArgoCD's own mechanism for this +// exact situation is configuring spec.ignoreDifferences on the Application +// for the HPA-managed field, not repeatedly re-syncing or reverting a +// change nobody made. +func k8sArgoCDOutOfSyncTest() testkit.Test { + prompt := `Here is the status of an ArgoCD Application that keeps +showing OutOfSync: + +` + k8sArgoCDOutOfSyncStatus + ` + +Why is this Application stuck OutOfSync on spec.replicas specifically, and +what is the correct ArgoCD-idiomatic fix so it stops flapping between +Synced and OutOfSync? Respond with only a JSON object: +{"cause":"","fix":""}` + + evaluator := eval.Mean( + eval.JSONField("cause", "hpa-managed-field"), + eval.JSONField("fix", "ignoreDifferences"), + ) + + return testkit.Test{ + ID: "k8s-argocd-outofsync", + Category: "operations", + Subcategory: "kubernetes", + Description: "Identify an HPA-managed replicas field as the cause of a flapping ArgoCD OutOfSync and the ignoreDifferences fix.", + Prompt: prompt, + Eval: evaluator, + } +} + +// k8sTraefikIngressRouteYAML is the inline IngressRoute excerpt for +// k8sTraefikIngressRouteHostTest. +const k8sTraefikIngressRouteYAML = "apiVersion: traefik.io/v1alpha1\n" + + "kind: IngressRoute\n" + + "metadata:\n" + + " name: app-route\n" + + " namespace: web\n" + + "spec:\n" + + " entryPoints:\n" + + " - websecure\n" + + " routes:\n" + + " - match: Host(`app.example.com`)\n" + + " kind: Rule\n" + + " services:\n" + + " - name: app-svc\n" + + " port: 80" + +// k8sTraefikIngressRoutePattern requires the corrected match expression, +// with the backtick-quoted domain exactly matching app.raczylo.com. +const k8sTraefikIngressRoutePattern = "(?i)host\\(`app\\.raczylo\\.com`\\)" + +// k8sTraefikIngressRouteHostTest: correct a Traefik IngressRoute's Host() +// match rule to the actual domain the service must answer for. +// +// ground truth: the IngressRoute's only match rule is +// Host(`app.example.com`), but the domain that must route to this service +// is app.raczylo.com - a different rule value entirely, not a change to +// entryPoints, the service name, or the port. +func k8sTraefikIngressRouteHostTest() testkit.Test { + prompt := `This Traefik IngressRoute is not receiving any traffic +because it is matching the wrong hostname. The actual domain this service +must answer for is app.raczylo.com. Here is the IngressRoute: + +` + "```yaml\n" + k8sTraefikIngressRouteYAML + "\n```" + ` + +Give the single corrected "match" rule line (the full match expression, +using the same Host() syntax) for this IngressRoute.` + + return testkit.Test{ + ID: "k8s-traefik-ingressroute-host", + Category: "operations", + Subcategory: "kubernetes", + Description: "Correct a Traefik IngressRoute's Host() match rule to the actual domain the service must serve.", + Prompt: prompt, + Eval: eval.Regex(k8sTraefikIngressRoutePattern), + } +} + +// k8sQoSClassPodSpec is the inline pod spec excerpt for k8sQoSClassTest. +const k8sQoSClassPodSpec = `apiVersion: v1 +kind: Pod +metadata: + name: report-generator +spec: + containers: + - name: report-generator + image: registry.internal.example.com/report-generator:v3 + resources: + requests: + cpu: "250m" + memory: "256Mi" + limits: + cpu: "500m" + memory: "512Mi"` + +// k8sQoSClassTest: derive the scheduler-assigned QoS class from a pod +// spec's requests/limits. +// +// ground truth: every container defines both requests and limits for both +// cpu and memory, ruling out BestEffort (which requires no requests or +// limits at all). Since requests (250m/256Mi) differ from limits +// (500m/512Mi) rather than being exactly equal, the class is Burstable, +// not Guaranteed (which additionally requires requests == limits for +// every resource on every container). +func k8sQoSClassTest() testkit.Test { + prompt := `Here is a pod spec: + +` + "```yaml\n" + k8sQoSClassPodSpec + "\n```" + ` + +What Quality of Service (QoS) class does the Kubernetes scheduler assign +to this pod? Respond with only one word: Guaranteed, Burstable, or +BestEffort.` + + return testkit.Test{ + ID: "k8s-qos-class", + Category: "operations", + Subcategory: "kubernetes", + Description: "Derive a pod's QoS class (Burstable) from requests that are set but not equal to limits.", + Prompt: prompt, + Eval: eval.Equals("Burstable"), + } +} + +// k8sCNPGPDBTest: pick PodDisruptionBudget with minAvailable as the +// mechanism guaranteeing quorum survives voluntary node maintenance for a +// CNPG-managed Postgres cluster. +func k8sCNPGPDBTest() testkit.Test { + prompt := `A CNPG (CloudNativePG)-managed Postgres cluster runs as 3 +pods (1 primary, 2 replicas) in the shared-resources namespace. The team +wants a guarantee that at least 2 of the 3 Postgres pods stay available +during voluntary node maintenance (e.g. "kubectl drain"). Which Kubernetes +resource type provides this guarantee, and what field on it must be set, +with what minimum value, to enforce "at least 2 available"?` + + // ground truth: a PodDisruptionBudget (PDB) is the resource that + // bounds voluntary disruptions (like a node drain evicting pods) for a + // selected set of pods. Setting its minAvailable field to 2 enforces + // that at least 2 of the 3 Postgres pods must remain available before + // the eviction API will permit another one to be drained. + evaluator := eval.Mean( + eval.ContainsAll("PodDisruptionBudget"), + eval.ContainsAny("minAvailable: 2", "minAvailable=2", "minAvailable 2", "minAvailable of 2"), + ) + + return testkit.Test{ + ID: "k8s-cnpg-pdb", + Category: "operations", + Subcategory: "kubernetes", + Description: "Identify PodDisruptionBudget with minAvailable=2 as the guarantee for a 3-pod CNPG cluster during node drains.", + Prompt: prompt, + Eval: evaluator, + } +} diff --git a/internal/tests/kubernetes_test.go b/internal/tests/kubernetes_test.go index 7182593..647534e 100644 --- a/internal/tests/kubernetes_test.go +++ b/internal/tests/kubernetes_test.go @@ -109,3 +109,235 @@ is missing. Check the image tag and re-push it.` }) } } + +func TestK8sImagePullBackoffTest_Eval(t *testing.T) { + tc := k8sImagePullBackoffTest() + + tests := []struct { + name string + response string + want float64 + }{ + {"correct object", `{"cause":"auth","fix":"imagePullSecrets"}`, 1}, + {"prose then JSON", "This is a 401, an auth failure.\n" + `{"cause":"auth","fix":"imagePullSecrets"}`, 1}, + {"fenced JSON", "```json\n" + `{"cause":"auth","fix":"imagePullSecrets"}` + "\n```", 1}, + {"misreads it as a missing tag", `{"cause":"not-found","fix":"fix-image-tag"}`, 0}, + {"right cause, wrong fix category", `{"cause":"auth","fix":"check-network"}`, 0.5}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Value != tt.want { + t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail) + } + }) + } +} + +func TestK8sPendingTaintsTest_Eval(t *testing.T) { + tc := k8sPendingTaintsTest() + + tests := []struct { + name string + response string + want float64 + }{ + {"correct object", `{"cause":"taint","fix_field":"tolerations"}`, 1}, + {"prose then JSON", "The nodes have an untolerated taint.\n" + `{"cause":"taint","fix_field":"tolerations"}`, 1}, + {"fenced JSON", "```json\n" + `{"cause":"taint","fix_field":"tolerations"}` + "\n```", 1}, + {"misdiagnoses as a resource limit issue", `{"cause":"resource-limit","fix_field":"resources"}`, 0}, + {"right cause, wrong fix field", `{"cause":"taint","fix_field":"nodeSelector"}`, 0.5}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Value != tt.want { + t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail) + } + }) + } +} + +func TestK8sPVCStorageclassMismatchTest_Eval(t *testing.T) { + tc := k8sPVCStorageclassMismatchTest() + + tests := []struct { + name string + response string + want float64 + }{ + {"correct object", `{"cause":"missing-storageclass","use_storageclass":"standard"}`, 1}, + {"prose then JSON", "The fast-ssd class does not exist.\n" + `{"cause":"missing-storageclass","use_storageclass":"standard"}`, 1}, + {"fenced JSON", "```json\n" + `{"cause":"missing-storageclass","use_storageclass":"standard"}` + "\n```", 1}, + {"misdiagnoses as the provisioner being down", `{"cause":"provisioner-down","use_storageclass":"standard"}`, 0.5}, + {"right cause but names a class that does not exist", `{"cause":"missing-storageclass","use_storageclass":"fast-ssd"}`, 0.5}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Value != tt.want { + t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail) + } + }) + } +} + +func TestK8sServiceSelectorMismatchTest_Eval(t *testing.T) { + tc := k8sServiceSelectorMismatchTest() + + tests := []struct { + name string + response string + want float64 + }{ + {"correct object", `{"field":"selector.app","value":"inventory-api"}`, 1}, + {"prose then JSON", "The selector doesn't match the pod labels.\n" + `{"field":"selector.app","value":"inventory-api"}`, 1}, + {"fenced JSON", "```json\n" + `{"field":"selector.app","value":"inventory-api"}` + "\n```", 1}, + {"points at the wrong field (targetPort instead of selector)", `{"field":"targetPort","value":"8080"}`, 0}, + {"right field, wrong replacement value", `{"field":"selector.app","value":"inventory-api-svc"}`, 0.5}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Value != tt.want { + t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail) + } + }) + } +} + +func TestK8sNetworkPolicyDNSBlockTest_Eval(t *testing.T) { + tc := k8sNetworkPolicyDNSBlockTest() + + good := `This NetworkPolicy allows Egress only to payments-db pods on +5432/TCP, so every other egress - including DNS lookups on port 53/UDP - +is implicitly denied. Add an egress rule allowing UDP port 53 (and +typically TCP 53) to pods labeled k8s-app=kube-dns in kube-system.` + goodAlt := `The policy default-denies all egress except the one listed +destination, and DNS needs to reach CoreDNS on UDP port 53, which is not +allowed here. Add an egress rule permitting port 53/udp to the coredns +pods.` + goodTerse := `Add an egress allow rule for port 53 UDP targeting the +kube-system namespace's kube-dns pods; everything else is denied by +default here.` + badWrongPort := `Add an egress rule allowing port 80/TCP to the ingress +controller.` + badWrongProtocol := `Add an egress rule allowing TCP port 53 only to +kube-system, since DNS always uses TCP.` + + tests := []struct { + name string + response string + want float64 + }{ + {"good answer, kube-dns phrasing", good, 1}, + {"good answer, coredns phrasing", goodAlt, 1}, + {"good answer, terse phrasing", goodTerse, 1}, + {"wrong port and protocol entirely", badWrongPort, 0}, + {"right port, missing udp mention", badWrongProtocol, 2.0 / 3.0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Value != tt.want { + t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail) + } + }) + } +} + +func TestK8sArgoCDOutOfSyncTest_Eval(t *testing.T) { + tc := k8sArgoCDOutOfSyncTest() + + tests := []struct { + name string + response string + want float64 + }{ + {"correct object", `{"cause":"hpa-managed-field","fix":"ignoreDifferences"}`, 1}, + {"prose then JSON", "The HPA owns replicas here.\n" + `{"cause":"hpa-managed-field","fix":"ignoreDifferences"}`, 1}, + {"fenced JSON", "```json\n" + `{"cause":"hpa-managed-field","fix":"ignoreDifferences"}` + "\n```", 1}, + {"wrongly blames manual drift nobody caused", `{"cause":"manual-drift","fix":"revert-manual-change"}`, 0}, + {"right cause, wrong fix (fighting the HPA by re-syncing)", `{"cause":"hpa-managed-field","fix":"sync-git"}`, 0.5}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Value != tt.want { + t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail) + } + }) + } +} + +func TestK8sTraefikIngressRouteHostTest_Eval(t *testing.T) { + tc := k8sTraefikIngressRouteHostTest() + + tests := []struct { + name string + response string + want float64 + }{ + {"bare corrected line", "match: Host(`app.raczylo.com`)", 1}, + {"prose wrapped", "The corrected line is:\nmatch: Host(`app.raczylo.com`)", 1}, + {"uppercase Host keyword variant", "MATCH: HOST(`app.raczylo.com`)", 1}, + {"still the old domain", "match: Host(`app.example.com`)", 0}, + {"right function, wrong domain typo", "match: Host(`app.raczlo.com`)", 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Value != tt.want { + t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail) + } + }) + } +} + +func TestK8sQoSClassTest_Eval(t *testing.T) { + tc := k8sQoSClassTest() + + tests := []struct { + name string + response string + want float64 + }{ + {"exact word", "Burstable", 1}, + {"lowercase", "burstable", 1}, + {"trailing whitespace", "Burstable\n", 1}, + {"wrong class, mistakes it for Guaranteed", "Guaranteed", 0}, + {"wrong class, mistakes it for BestEffort", "BestEffort", 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Value != tt.want { + t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail) + } + }) + } +} + +func TestK8sCNPGPDBTest_Eval(t *testing.T) { + tc := k8sCNPGPDBTest() + + tests := []struct { + name string + response string + want float64 + }{ + {"correct resource and value", "Use a PodDisruptionBudget with minAvailable: 2.", 1}, + {"equals-sign phrasing", "A PodDisruptionBudget (PDB) with minAvailable=2 guarantees this.", 1}, + {"english phrasing of the value", "Create a PodDisruptionBudget with a minAvailable of 2.", 1}, + {"wrong resource entirely (a NetworkPolicy cannot do this)", "Use a NetworkPolicy with minAvailable: 2.", 0.5}, + {"right resource, wrong/missing value", "Use a PodDisruptionBudget, but leave the fields at their defaults.", 0.5}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Value != tt.want { + t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail) + } + }) + } +} diff --git a/internal/tests/linux.go b/internal/tests/linux.go index 779a574..e3b63b9 100644 --- a/internal/tests/linux.go +++ b/internal/tests/linux.go @@ -8,6 +8,14 @@ import ( func registerLinuxTests(r *testkit.Registry) { r.Register(linuxPctExecTest()) r.Register(linuxSystemdOneshotTest()) + r.Register(linuxJournalctlFilterTest()) + r.Register(linuxCgroupOOMDiagnosisTest()) + r.Register(linuxNftablesDNATTest()) + r.Register(linuxLVMExtendOrderTest()) + r.Register(linuxSSHPortForwardTest()) + r.Register(linuxCronExpressionTest()) + r.Register(linuxSystemdTimerTest()) + r.Register(linuxProcMeminfoAvailableTest()) } // linuxPctExecTest: one-line command to check a service inside an LXC @@ -65,3 +73,245 @@ contents only.` Eval: evaluator, } } + +// linuxJournalctlFilterPattern requires journalctl filtered to unit +// nginx.service and a since-window of 2 hours, in the canonical +// "-u UNIT --since WINDOW" order (the overwhelmingly common idiom), on a +// single line so a scattered mention across paragraphs does not count. +const linuxJournalctlFilterPattern = `(?i)\bjournalctl\b[^\n]*?-u\s+nginx\.service\b[^\n]*?--since\s+["']?-?2\s*h(?:ours?)?(?:\s+ago)?["']?` + +// linuxJournalctlFilterTest: one-line journalctl command filtered to one +// unit over a relative time window. +func linuxJournalctlFilterTest() testkit.Test { + prompt := `Give a single, one-line journalctl command that shows only +log entries for the systemd unit "nginx.service" from the last 2 hours.` + + return testkit.Test{ + ID: "linux-journalctl-filter", + Category: "operations", + Subcategory: "linux", + Description: "One-line journalctl command filtered by unit and a relative 2-hour time window.", + Prompt: prompt, + Eval: eval.Regex(linuxJournalctlFilterPattern), + } +} + +// linuxCgroupMemoryEvents is the inline cgroup v2 memory.events fixture for +// linuxCgroupOOMDiagnosisTest. +const linuxCgroupMemoryEvents = `low 0 +high 0 +max 47 +oom 3 +oom_kill 3` + +// linuxCgroupOOMDiagnosisTest: read a cgroup v2 memory.events file and +// report whether the kernel actually OOM-killed the cgroup's processes. +// +// ground truth: cgroup v2's memory.events "oom" counts memory allocation +// failures that triggered the OOM killer to run, while "oom_kill" counts +// processes the kernel actually killed as a result. Here oom_kill is 3 +// (not 0), so the cgroup's processes were in fact OOM-killed, 3 times. +func linuxCgroupOOMDiagnosisTest() testkit.Test { + prompt := `Here is the contents of a container's cgroup v2 +memory.events file: + +` + linuxCgroupMemoryEvents + ` + +Based only on this file, was any process in this cgroup actually killed by +the kernel's OOM killer, and if so, how many times? Respond with only a +JSON object: {"oom_killed": true or false, "oom_kill_count": }.` + + evaluator := eval.Mean( + eval.JSONField("oom_killed", true), + eval.JSONField("oom_kill_count", 3), + ) + + return testkit.Test{ + ID: "linux-cgroup-oom-diagnosis", + Category: "operations", + Subcategory: "linux", + Description: "Diagnose an actual OOM-kill count from cgroup v2 memory.events (oom_kill, not the broader oom counter).", + Prompt: prompt, + Eval: evaluator, + } +} + +// linuxNftablesDNATPattern requires a DNAT rule (iptables or nft form) on +// the nat table's prerouting chain, matching TCP port 8080 and forwarding +// to 10.0.5.20:80, on a single line. iptables and nft flags/keywords follow +// a fixed canonical order (table/chain, then match, then target), so this +// uses the same non-greedy same-line anchoring as the pct-exec lesson +// rather than a permutation of independently-orderable flags. +const linuxNftablesDNATPattern = `(?i)(?:iptables|nft)\b[^\n]*?nat[^\n]*?prerouting[^\n]*?(?:--dport|dport)\s*8080[^\n]*?dnat[^\n]*?10\.0\.5\.20:80` + +// linuxNftablesDNATTest: one-line DNAT rule forwarding an external port to +// an internal host:port. +func linuxNftablesDNATTest() testkit.Test { + prompt := `Give a single, one-line firewall command (iptables or nft, +your choice) that DNATs incoming TCP traffic on this host's eth0 interface, +port 8080, to internal address 10.0.5.20, port 80.` + + return testkit.Test{ + ID: "linux-nftables-dnat", + Category: "operations", + Subcategory: "linux", + Description: "One-line DNAT rule (iptables or nft) forwarding TCP 8080 on eth0 to 10.0.5.20:80.", + Prompt: prompt, + Eval: eval.Regex(linuxNftablesDNATPattern), + } +} + +// linuxLVMExtendWant is the unique correct 4-step LVM growth sequence for +// linuxLVMExtendOrderTest. +var linuxLVMExtendWant = []string{"pvcreate", "vgextend", "lvextend", "resize2fs"} + +// linuxLVMExtendOrderTest: order the LVM commands that grow a logical +// volume and its filesystem onto a newly added disk. +// +// ground truth: a new disk must first become an LVM physical volume +// (pvcreate), then be added to the volume group (vgextend), before the +// logical volume can grow into that new space (lvextend), after which the +// filesystem itself must be grown to use it (resize2fs, for ext4). mkfs is +// never used here - it would format over, and destroy, the existing data. +func linuxLVMExtendOrderTest() testkit.Test { + prompt := `The logical volume data-lv on volume group data-vg (ext4 +filesystem) is full. You have already partitioned a newly added disk as +/dev/sdb1, but done nothing else with it. + +From this list of steps - pvcreate, vgextend, lvextend, resize2fs, mkfs - +choose exactly the 4 steps needed, in the correct order, to grow data-lv +and its filesystem onto the new space without destroying the existing +data. Respond with only a JSON array of the step ids, e.g. +["step1","step2",...].` + + return testkit.Test{ + ID: "linux-lvm-extend-order", + Category: "operations", + Subcategory: "linux", + Description: "Order the 4 LVM/filesystem commands (of 5 candidates) that grow a logical volume onto a newly added disk.", + Prompt: prompt, + Eval: eval.JSONStringArrayEquals(linuxLVMExtendWant), + } +} + +// linuxSSHPortForwardPattern requires an ssh local port-forward (-L) from +// local port 15432 to the remote Postgres, connecting to the correct host. +const linuxSSHPortForwardPattern = `(?i)\bssh\b[^\n]*?-L\s*15432:(?:127\.0\.0\.1|localhost):5432[^\n]*?ops@db1\.internal` + +// linuxSSHPortForwardTest: ssh -L local port-forward command derived from a +// scenario. +func linuxSSHPortForwardTest() testkit.Test { + prompt := `A Postgres database only listens on 127.0.0.1:5432 on remote +host db1.internal, which you can reach over SSH as user "ops". Give a +single, one-line ssh command, run from your laptop, that forwards your +laptop's local port 15432 to that remote Postgres instance, so a local +client can connect to localhost:15432.` + + return testkit.Test{ + ID: "linux-ssh-port-forward", + Category: "operations", + Subcategory: "linux", + Description: "One-line ssh -L local port-forward command reaching a Postgres instance that only listens on remote loopback.", + Prompt: prompt, + Eval: eval.Regex(linuxSSHPortForwardPattern), + } +} + +// linuxCronExpressionPattern requires the exact 5-field cron schedule for +// "03:30 every Monday", tolerant only of whitespace width between fields - +// unlike eval.Equals, this stays correct even if the model adds a leading +// or trailing space, since the fields themselves have no valid rephrasing. +const linuxCronExpressionPattern = `^\s*30\s+3\s+\*\s+\*\s+1\s*$` + +// linuxCronExpressionTest: produce the 5-field cron expression matching a +// stated schedule. +// +// ground truth: cron fields are minute hour day-of-month month +// day-of-week. 03:30 is minute=30, hour=3; day-of-month and month are +// unrestricted (*); Monday is day-of-week 1 in standard cron numbering +// (0 and 7 both mean Sunday). +func linuxCronExpressionTest() testkit.Test { + prompt := `Give the single 5-field cron expression that runs a job at +03:30 every Monday only. Use numeric day-of-week (Monday = 1). Respond +with only the expression, nothing else.` + + return testkit.Test{ + ID: "linux-cron-expression", + Category: "operations", + Subcategory: "linux", + Description: "Produce the 5-field cron expression for a 03:30-every-Monday schedule.", + Prompt: prompt, + Eval: eval.Regex(linuxCronExpressionPattern), + } +} + +// linuxSystemdTimerTest: name the systemd timer+service unit pair (not +// cron) for a fixed daily schedule, with the required timer keys. +func linuxSystemdTimerTest() testkit.Test { + prompt := `A backup script must run every day at 02:00, managed by +systemd (not cron), using a timer unit plus a service unit. Name the two +unit file types this requires, give the key line in the .timer file that +sets the daily 02:00 schedule, and give the [Install] target the .timer +unit's WantedBy needs.` + + // ground truth: a systemd timer job is a .timer unit (the schedule) + // paired with a same-named .service unit (the work). The .timer's + // OnCalendar= key sets the schedule (e.g. "*-*-* 02:00:00" for daily at + // 02:00), and its [Install] section needs WantedBy=timers.target for + // systemctl enable to activate it. + evaluator := eval.Mean( + eval.ContainsAll(".service", ".timer"), + eval.ContainsAny("OnCalendar"), + eval.ContainsAll("WantedBy=timers.target"), + ) + + return testkit.Test{ + ID: "linux-systemd-timer", + Category: "operations", + Subcategory: "linux", + Description: "Name the systemd .timer/.service unit pair, OnCalendar schedule key, and timers.target for a daily job.", + Prompt: prompt, + Eval: evaluator, + } +} + +// linuxProcMeminfo is the inline /proc/meminfo fixture for +// linuxProcMeminfoAvailableTest. +const linuxProcMeminfo = `MemTotal: 16384000 kB +MemFree: 1024000 kB +MemAvailable: 9876000 kB +Buffers: 512000 kB +Cached: 3200000 kB` + +// linuxProcMeminfoAvailableWant is MemAvailable (9876000 kB) converted to +// whole megabytes, floored: 9876000 / 1024 = 9644.53125 -> 9644. +var linuxProcMeminfoAvailableWant = 9876000 / 1024 + +// linuxProcMeminfoAvailableTest: interpret /proc/meminfo's MemAvailable +// field (not MemFree) as the memory actually available for new processes. +// +// ground truth: MemFree undercounts usable memory because it excludes +// reclaimable page cache and buffers; MemAvailable is the kernel's own +// estimate of memory available for new allocations without swapping, +// already accounting for reclaimable cache. /proc/meminfo's "kB" is always +// 1024-byte kibibytes, so converting to MB divides by 1024, not 1000. +func linuxProcMeminfoAvailableTest() testkit.Test { + prompt := `Here is a container host's /proc/meminfo: + +` + linuxProcMeminfo + ` + +How much memory, in whole megabytes (rounded down), is actually available +for new processes to allocate without triggering swap? Use the correct +field for this - not MemFree, which excludes reclaimable cache and +buffers. Note that /proc/meminfo's "kB" unit is always 1024-byte kibibytes, +so convert by dividing by 1024, not 1000. Respond with only the number.` + + return testkit.Test{ + ID: "linux-proc-meminfo-available", + Category: "operations", + Subcategory: "linux", + Description: "Convert /proc/meminfo's MemAvailable field (not MemFree) to whole megabytes.", + Prompt: prompt, + Eval: eval.Numeric(eval.ExtractLastNumber[int], linuxProcMeminfoAvailableWant, 1), + } +} diff --git a/internal/tests/linux_test.go b/internal/tests/linux_test.go index b673607..b335483 100644 --- a/internal/tests/linux_test.go +++ b/internal/tests/linux_test.go @@ -126,3 +126,237 @@ WantedBy=graphical.target` }) } } + +func TestLinuxJournalctlFilterTest_Eval(t *testing.T) { + tc := linuxJournalctlFilterTest() + + tests := []struct { + name string + response string + want float64 + }{ + {"quoted negative window", `journalctl -u nginx.service --since "-2 hours"`, 1}, + {"english phrasing window", `journalctl -u nginx.service --since "2 hours ago"`, 1}, + {"compact -2h form, no quotes", `Run: journalctl -u nginx.service --since -2h`, 1}, + {"wrong unit", `journalctl -u apache.service --since "-2 hours"`, 0}, + {"missing unit filter entirely", `journalctl --since "-2 hours"`, 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Value != tt.want { + t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail) + } + }) + } +} + +func TestLinuxCgroupOOMDiagnosisTest_Eval(t *testing.T) { + tc := linuxCgroupOOMDiagnosisTest() + + tests := []struct { + name string + response string + want float64 + }{ + {"correct object", `{"oom_killed": true, "oom_kill_count": 3}`, 1}, + {"prose then JSON", "Yes, it was OOM-killed.\n" + `{"oom_killed": true, "oom_kill_count": 3}`, 1}, + {"fenced JSON", "```json\n" + `{"oom_killed": true, "oom_kill_count": 3}` + "\n```", 1}, + {"confuses oom counter with oom_kill counter", `{"oom_killed": true, "oom_kill_count": 47}`, 0.5}, + {"wrongly claims no kill happened", `{"oom_killed": false, "oom_kill_count": 0}`, 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Value != tt.want { + t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail) + } + }) + } +} + +func TestLinuxNftablesDNATTest_Eval(t *testing.T) { + tc := linuxNftablesDNATTest() + + tests := []struct { + name string + response string + want float64 + }{ + { + name: "iptables form", + response: `iptables -t nat -A PREROUTING -i eth0 -p tcp --dport 8080 -j DNAT --to-destination 10.0.5.20:80`, + want: 1, + }, + { + name: "iptables form with flags reordered around the match", + response: `sudo iptables -t nat -A PREROUTING -p tcp -i eth0 --dport 8080 -j DNAT --to-destination 10.0.5.20:80`, + want: 1, + }, + { + name: "nft form", + response: `nft add rule ip nat prerouting tcp dport 8080 dnat to 10.0.5.20:80`, + want: 1, + }, + { + name: "wrong destination IP", + response: `iptables -t nat -A PREROUTING -p tcp --dport 8080 -j DNAT --to-destination 10.0.5.21:80`, + want: 0, + }, + { + name: "wrong chain (OUTPUT instead of PREROUTING)", + response: `iptables -t nat -A OUTPUT -p tcp --dport 8080 -j DNAT --to-destination 10.0.5.20:80`, + want: 0, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Value != tt.want { + t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail) + } + }) + } +} + +func TestLinuxLVMExtendOrderTest_Eval(t *testing.T) { + tc := linuxLVMExtendOrderTest() + + tests := []struct { + name string + response string + want float64 + }{ + {"correct order", `["pvcreate","vgextend","lvextend","resize2fs"]`, 1}, + {"correct order, prose wrapped", "Steps:\n" + `["pvcreate","vgextend","lvextend","resize2fs"]`, 1}, + {"correct order, fenced JSON", "```json\n" + `["pvcreate","vgextend","lvextend","resize2fs"]` + "\n```", 1}, + {"wrong order (lvextend before vgextend)", `["pvcreate","lvextend","vgextend","resize2fs"]`, 0}, + {"includes destructive mkfs instead of resize2fs", `["pvcreate","vgextend","lvextend","mkfs"]`, 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Value != tt.want { + t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail) + } + }) + } +} + +func TestLinuxSSHPortForwardTest_Eval(t *testing.T) { + tc := linuxSSHPortForwardTest() + + tests := []struct { + name string + response string + want float64 + }{ + {"loopback IP form", `ssh -L 15432:127.0.0.1:5432 ops@db1.internal`, 1}, + {"localhost form", `ssh -L 15432:localhost:5432 ops@db1.internal`, 1}, + {"extra flags before -L", `ssh -N -L 15432:127.0.0.1:5432 ops@db1.internal`, 1}, + {"wrong local port", `ssh -L 5432:127.0.0.1:5432 ops@db1.internal`, 0}, + {"wrong host", `ssh -L 15432:127.0.0.1:5432 ops@db2.internal`, 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Value != tt.want { + t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail) + } + }) + } +} + +func TestLinuxCronExpressionTest_Eval(t *testing.T) { + tc := linuxCronExpressionTest() + + tests := []struct { + name string + response string + want float64 + }{ + {"canonical single spacing", "30 3 * * 1", 1}, + {"extra spaces between fields", "30 3 * * 1", 1}, + {"tab-separated fields", "30\t3\t*\t*\t1", 1}, + {"wrong day of week", "30 3 * * 2", 0}, + {"wrong time", "45 3 * * 1", 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Value != tt.want { + t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail) + } + }) + } +} + +func TestLinuxSystemdTimerTest_Eval(t *testing.T) { + tc := linuxSystemdTimerTest() + + tests := []struct { + name string + response string + want float64 + }{ + { + name: "full correct answer", + response: `Create backup.service and backup.timer. In backup.timer's +[Timer] section: OnCalendar=*-*-* 02:00:00. In [Install]: +WantedBy=timers.target.`, + want: 1, + }, + { + name: "reordered phrasing, service named first then timer", + response: `You need a .service unit for the work and a .timer unit for the schedule. Set OnCalendar=02:00 in the timer, and WantedBy=timers.target under [Install].`, + want: 1, + }, + { + name: "minimal but complete", + response: `backup.service + backup.timer, OnCalendar=*-*-* 02:00:00, WantedBy=timers.target`, + want: 1, + }, + { + name: "missing OnCalendar key entirely", + response: `Use backup.service and backup.timer, with WantedBy=timers.target in [Install].`, + want: 2.0 / 3.0, + }, + { + name: "wrong install target and no OnCalendar (cron-style thinking)", + response: `Just add a backup.service and backup.timer, and enable it with WantedBy=multi-user.target.`, + want: 1.0 / 3.0, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Value != tt.want { + t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail) + } + }) + } +} + +func TestLinuxProcMeminfoAvailableTest_Eval(t *testing.T) { + tc := linuxProcMeminfoAvailableTest() + + tests := []struct { + name string + response string + want float64 + }{ + {"exact number", "9644", 1}, + {"prose wrapped", "The available memory is 9644 MB.", 1}, + {"with unit suffix", "9644 MB", 1}, + {"divided by 1000 instead of 1024", "9876", 0}, + {"used MemFree instead of MemAvailable", "1000", 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Value != tt.want { + t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail) + } + }) + } +} diff --git a/internal/tests/macos.go b/internal/tests/macos.go index afeee17..61984a7 100644 --- a/internal/tests/macos.go +++ b/internal/tests/macos.go @@ -8,6 +8,14 @@ import ( func registerMacOSTests(r *testkit.Registry) { r.Register(macosTimeoutPortabilityTest()) r.Register(macosLaunchdCronTest()) + r.Register(macosSedInplaceTest()) + r.Register(macosLaunchctlBootstrapTest()) + r.Register(macosPlutilPlistReadTest()) + r.Register(macosMdfindSpotlightTest()) + r.Register(macosAPFSSnapshotTest()) + r.Register(macosSSHIdentityAgentTest()) + r.Register(macosCaffeinateTest()) + r.Register(macosStatBSDGNUTest()) } // macosTimeoutPortabilityTest: explain why `timeout` is missing on stock @@ -82,3 +90,260 @@ file belongs in for a per-machine (not per-user) job.` Eval: evaluator, } } + +// macosSedPattern requires the BSD-form "sed -i" with an empty +// single-quoted or double-quoted extension argument, invoking a foo->bar +// substitution on file.txt, on a single line (no DOTALL) so a scattered, +// multi-paragraph mention of the same tokens does not count as a real +// one-line command. +// +// ground truth: BSD sed's -i flag takes a mandatory in-place-suffix +// argument (used for backup files); passing an empty single-quoted or +// double-quoted string means "no backup file". GNU sed's -i takes an +// OPTIONAL argument, so +// "sed -i 's/foo/bar/' file.txt" (no argument to -i) works on Linux but +// errors on macOS, where it is parsed as "-i 's/foo/bar/'" (using the sed +// script itself as the backup suffix) followed by a missing script operand. +const macosSedPattern = `(?i)\bsed\b[^\n]*-i\s*(''|"")[^\n]*s/foo/bar/g?[^\n]*file\.txt` + +// macosSedInplaceTest: give the portable BSD sed one-liner for in-place +// substitution with no backup file, contrasted with the GNU-only form that +// fails on stock macOS. +func macosSedInplaceTest() testkit.Test { + prompt := `On a stock macOS machine (BSD sed, not GNU sed), this command: + +` + "```sh" + ` +sed -i 's/foo/bar/' file.txt +` + "```" + ` + +fails with an error about a missing extension or a "no such file" +complaint. Give the corrected one-line BSD sed command that replaces "foo" +with "bar" in file.txt, edits the file in place, and creates NO backup +file.` + + return testkit.Test{ + ID: "macos-sed-inplace", + Category: "operations", + Subcategory: "macos", + Description: "Give the BSD sed -i '' in-place substitution syntax that works on stock macOS with no backup file.", + Prompt: prompt, + Eval: eval.Regex(macosSedPattern), + } +} + +// macosLaunchctlBootstrapTest: identify launchctl bootstrap with an +// explicit system domain target as the modern replacement for the +// deprecated launchctl load, for loading a LaunchDaemon. +func macosLaunchctlBootstrapTest() testkit.Test { + prompt := `A new LaunchDaemon plist has just been written to +/Library/LaunchDaemons/com.example.myjob.plist on a current macOS release. + +On modern macOS (10.11 El Capitan and later), which launchctl subcommand +should you use to load and start this LaunchDaemon, and which domain +target must you pass it (this is a system-wide daemon, not a per-user +agent)? Also name the older launchctl subcommand for loading jobs that +Apple deprecated in favor of this one, and briefly say why (it lacked +explicit domain targeting).` + + // ground truth: Apple deprecated the session-based "launchctl load" / + // "launchctl unload" pair starting in OS X 10.11, replacing them with + // "launchctl bootstrap" / "launchctl bootout", which require an + // explicit domain target (system/, gui/, user/). A + // system-wide LaunchDaemon uses the "system" domain, giving + // "sudo launchctl bootstrap system /Library/LaunchDaemons/....plist". + // Component 3 deliberately requires an explicit deprecation cue rather + // than a bare "launchctl load" mention: a response that just uses + // launchctl load (as its own recommended command, not something to + // avoid) must not get credit for correctly flagging it as deprecated. + evaluator := eval.Mean( + eval.ContainsAny("bootstrap"), + eval.ContainsAny("bootstrap system", "system domain", "domain system", "target system", "system target"), + eval.ContainsAny("deprecated", "avoid launchctl load", "no longer recommended", "outdated"), + ) + + return testkit.Test{ + ID: "macos-launchctl-bootstrap", + Category: "operations", + Subcategory: "macos", + Description: "Identify launchctl bootstrap+system domain as the modern replacement for the deprecated launchctl load.", + Prompt: prompt, + Eval: evaluator, + } +} + +// macosPlutilPattern requires the built-in plutil -p pretty-printer +// invoked directly on the plist path, on one line. +const macosPlutilPattern = `(?i)\bplutil\b[^\n]*-p[^\n]*/Library/Preferences/com\.example\.app\.plist` + +// macosPlutilPlistReadTest: pretty-print a plist file's contents using a +// tool that ships with macOS (no Homebrew installs). +func macosPlutilPlistReadTest() testkit.Test { + prompt := `You need to inspect the raw contents of this plist file on a +stock macOS machine, with no Homebrew packages installed: + +/Library/Preferences/com.example.app.plist + +Give a single command, using only a tool that ships with macOS by default, +that prints the plist's contents to your terminal in a readable +(non-binary) form.` + + // ground truth: plutil ships with macOS and its -p flag pretty-prints a + // plist (binary or XML) to stdout in a human-readable form, taking the + // file path directly - no domain-name lookup or Homebrew install + // needed, unlike "defaults read ". + return testkit.Test{ + ID: "macos-plutil-plist-read", + Category: "operations", + Subcategory: "macos", + Description: "Pretty-print a plist file's contents using the built-in plutil -p command.", + Prompt: prompt, + Eval: eval.Regex(macosPlutilPattern), + } +} + +// macosMdfindSpotlightTest: use mdfind (Spotlight's metadata index) instead +// of a full filesystem walk to find files by name. +func macosMdfindSpotlightTest() testkit.Test { + prompt := `On macOS, give a single command that searches Spotlight's +existing metadata index (not a full filesystem walk with "find") for files +whose name contains "invoice", scoped only to your home directory (~).` + + // ground truth: mdfind queries the already-built Spotlight index rather + // than walking the filesystem; -onlyin scopes the search to a + // directory, and either -name or a plain query string matches on file + // name. + evaluator := eval.Mean( + eval.ContainsAll("mdfind"), + eval.ContainsAny("-onlyin", "-name"), + eval.ContainsAny("invoice"), + ) + + return testkit.Test{ + ID: "macos-mdfind-spotlight", + Category: "operations", + Subcategory: "macos", + Description: "Use mdfind against Spotlight's index, scoped to a directory, instead of a full filesystem walk.", + Prompt: prompt, + Eval: evaluator, + } +} + +// macosAPFSSnapshotTest: create and list local APFS snapshots via tmutil, +// which works even with no external Time Machine disk attached. +func macosAPFSSnapshotTest() testkit.Test { + prompt := `Before a risky system change, you want to create a local APFS +snapshot of the root volume as a quick rollback point - with no external +Time Machine disk attached (APFS local snapshots do not need one). Give +the command to create a local snapshot now, and the separate command to +list the local snapshots that currently exist.` + + // ground truth: tmutil manages local APFS snapshots independently of + // having a Time Machine backup destination attached. "tmutil snapshot" + // creates one immediately; "tmutil listlocalsnapshots /" lists the + // local snapshots that exist for the root volume. + evaluator := eval.Mean( + eval.ContainsAny("tmutil snapshot"), + eval.ContainsAny("tmutil listlocalsnapshots", "listlocalsnapshots"), + ) + + return testkit.Test{ + ID: "macos-apfs-snapshot", + Category: "operations", + Subcategory: "macos", + Description: "Create and list local APFS snapshots via tmutil with no external Time Machine disk attached.", + Prompt: prompt, + Eval: evaluator, + } +} + +// macosSSHIdentityAgentTest: point ssh at 1Password's SSH agent socket via +// the IdentityAgent ssh_config directive. +func macosSSHIdentityAgentTest() testkit.Test { + prompt := `You use the 1Password app as your SSH agent instead of the +default ssh-agent, and your shell environment does NOT set SSH_AUTH_SOCK. +1Password's agent socket lives at: + +~/Library/Group Containers/2BUA8C4S2C.com.1password/t/agent.sock + +Give the single ssh_config line to add (to ~/.ssh/config, either globally +or under a Host block) that points ssh at this socket, so ssh and git-over- +ssh commands can find your keys through 1Password.` + + // ground truth: ssh_config's IdentityAgent directive overrides which + // agent socket ssh (and anything using libssh's config, like git) + // connects to; 1Password's own setup docs specify this exact directive + // and socket path for its SSH agent integration. + evaluator := eval.Mean( + eval.ContainsAny("IdentityAgent"), + eval.ContainsAny("1password/t/agent.sock", "agent.sock"), + ) + + return testkit.Test{ + ID: "macos-ssh-identity-agent", + Category: "operations", + Subcategory: "macos", + Description: "Point ssh at 1Password's SSH agent socket via the IdentityAgent ssh_config directive.", + Prompt: prompt, + Eval: evaluator, + } +} + +// macosCaffeinateTest: use caffeinate, built into macOS, to keep the +// machine awake only for the duration of a given command. +func macosCaffeinateTest() testkit.Test { + prompt := `A long-running data migration script, ./migrate.sh, needs to +run on a MacBook without the machine going to sleep partway through - and +you want sleep prevention to end automatically the moment the script +finishes, with no separate cleanup step. Give a single command, using a +tool built into macOS, that does this.` + + // ground truth: caffeinate, bundled with macOS, takes a power-management + // assertion for as long as a given utility runs when you pass that + // utility as its argument, and releases the assertion automatically + // when the utility exits - no manual cleanup needed. + evaluator := eval.Mean( + eval.ContainsAll("caffeinate"), + eval.ContainsAny("migrate.sh"), + ) + + return testkit.Test{ + ID: "macos-caffeinate", + Category: "operations", + Subcategory: "macos", + Description: "Use caffeinate to prevent sleep only for the duration of a given script.", + Prompt: prompt, + Eval: evaluator, + } +} + +// macosStatPattern requires the BSD stat -f format-string form printing +// only the byte size (%z) of file.txt. +const macosStatPattern = `(?i)\bstat\b[^\n]*-f\s*"?%z"?[^\n]*file\.txt` + +// macosStatBSDGNUTest: give the BSD stat command (macOS's built-in stat) +// for a file's size in bytes, and name the differing GNU stat flag. +func macosStatBSDGNUTest() testkit.Test { + prompt := `On macOS (BSD stat, not GNU stat), give a single command that +prints ONLY the size of file.txt in bytes, nothing else. Then name the +single-letter flag GNU stat (as found on Linux) uses in place of macOS's +-f for its own format-string argument.` + + // ground truth: BSD stat takes its format string via -f, and %z is the + // byte-size format specifier, giving "stat -f%z file.txt". GNU stat + // instead uses -c for its format string (with %s as the byte-size + // specifier) - the flag letter itself, not just the format string + // syntax, differs between the two implementations. + evaluator := eval.Mean( + eval.Regex(macosStatPattern), + eval.ContainsAny("-c"), + ) + + return testkit.Test{ + ID: "macos-stat-bsd-gnu", + Category: "operations", + Subcategory: "macos", + Description: "Give the BSD stat -f%z byte-size command and name the differing GNU stat flag (-c).", + Prompt: prompt, + Eval: evaluator, + } +} diff --git a/internal/tests/macos_test.go b/internal/tests/macos_test.go index edf674e..3aaa953 100644 --- a/internal/tests/macos_test.go +++ b/internal/tests/macos_test.go @@ -78,3 +78,227 @@ StartInterval set to 300.` }) } } + +func TestMacosSedInplaceTest_Eval(t *testing.T) { + tc := macosSedInplaceTest() + + tests := []struct { + name string + response string + want float64 + }{ + {"single-quote empty extension", `sed -i '' 's/foo/bar/' file.txt`, 1}, + {"double-quote empty extension with -e flag", `On macOS: sed -i "" -e 's/foo/bar/' file.txt`, 1}, + {"global replace flag still matches", `sed -i '' 's/foo/bar/g' file.txt`, 1}, + {"GNU-style with no extension argument is wrong on macOS", `sed -i 's/foo/bar/' file.txt`, 0}, + {"backup extension creates a file, contradicting the requirement", `sed -i.bak 's/foo/bar/' file.txt`, 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Value != tt.want { + t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail) + } + }) + } +} + +func TestMacosLaunchctlBootstrapTest_Eval(t *testing.T) { + tc := macosLaunchctlBootstrapTest() + + good := `Use "sudo launchctl bootstrap system /Library/LaunchDaemons/com.example.myjob.plist". +The older "launchctl load" command is deprecated because it has no explicit +domain target.` + goodReordered := `The modern subcommand is bootstrap, targeting the system +domain: sudo launchctl bootstrap system /Library/LaunchDaemons/com.example.myjob.plist. +launchctl load is deprecated.` + goodPhrasing := `Load it with launchctl bootstrap, using the system domain +target since this is a system-wide daemon. Avoid launchctl load, which +Apple deprecated for lacking an explicit domain target.` + badWrongSubcommand := `Just run "sudo launchctl load /Library/LaunchDaemons/com.example.myjob.plist".` + badMissingDomain := `Use launchctl bootstrap to load the daemon.` + + tests := []struct { + name string + response string + want float64 + }{ + {"good answer full credit", good, 1}, + {"good answer, reordered phrasing", goodReordered, 1}, + {"good answer, alternate phrasing", goodPhrasing, 1}, + {"wrong subcommand (deprecated load only) scores 0", badWrongSubcommand, 0}, + {"missing domain target and deprecation note loses two thirds", badMissingDomain, 1.0 / 3.0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Value != tt.want { + t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail) + } + }) + } +} + +func TestMacosPlutilPlistReadTest_Eval(t *testing.T) { + tc := macosPlutilPlistReadTest() + + tests := []struct { + name string + response string + want float64 + }{ + {"correct plutil -p", `plutil -p /Library/Preferences/com.example.app.plist`, 1}, + {"prose wrapped", `Run: plutil -p /Library/Preferences/com.example.app.plist to pretty-print it.`, 1}, + {"quoted path", `plutil -p "/Library/Preferences/com.example.app.plist"`, 1}, + {"wrong tool entirely (cat prints binary garbage)", `cat /Library/Preferences/com.example.app.plist`, 0}, + {"plutil without -p just validates, does not print", `plutil /Library/Preferences/com.example.app.plist`, 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Value != tt.want { + t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail) + } + }) + } +} + +func TestMacosMdfindSpotlightTest_Eval(t *testing.T) { + tc := macosMdfindSpotlightTest() + + tests := []struct { + name string + response string + want float64 + }{ + {"onlyin with query", `mdfind -onlyin ~ "invoice"`, 1}, + {"name flag", `mdfind -onlyin ~ -name invoice`, 1}, + {"prose wrapped", `Run mdfind -onlyin ~ invoice to search the Spotlight index.`, 1}, + {"uses find instead of mdfind, walks the filesystem", `find ~ -iname "*invoice*"`, 1.0 / 3.0}, + {"mdfind used but searching for the wrong term entirely", `mdfind -onlyin ~ "receipt"`, 2.0 / 3.0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Value != tt.want { + t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail) + } + }) + } +} + +func TestMacosAPFSSnapshotTest_Eval(t *testing.T) { + tc := macosAPFSSnapshotTest() + + tests := []struct { + name string + response string + want float64 + }{ + {"both commands", "Create: tmutil snapshot\nList: tmutil listlocalsnapshots /", 1}, + {"prose wrapped", "Run `tmutil snapshot` to create one, then `tmutil listlocalsnapshots /` to list them.", 1}, + {"listlocalsnapshots without explicit tmutil prefix restated", "First tmutil snapshot, then check with listlocalsnapshots /.", 1}, + {"only creates, never lists", "Just run tmutil snapshot.", 0.5}, + {"wrong tool (diskutil) mentioned instead of tmutil", "Use diskutil apfs snapshot / to create one.", 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Value != tt.want { + t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail) + } + }) + } +} + +func TestMacosSSHIdentityAgentTest_Eval(t *testing.T) { + tc := macosSSHIdentityAgentTest() + + tests := []struct { + name string + response string + want float64 + }{ + { + name: "correct directive and socket", + response: `IdentityAgent "~/Library/Group Containers/2BUA8C4S2C.com.1password/t/agent.sock"`, + want: 1, + }, + { + name: "inside a Host block, prose wrapped", + response: "Add this under your Host block:\nHost *\n IdentityAgent ~/Library/Group Containers/2BUA8C4S2C.com.1password/t/agent.sock", + want: 1, + }, + { + name: "shortened socket mention still credited", + response: "Set IdentityAgent to point at 1Password's agent.sock in its Group Containers path.", + want: 1, + }, + { + name: "sets SSH_AUTH_SOCK env var instead of ssh_config directive", + response: `export SSH_AUTH_SOCK=~/Library/Group Containers/2BUA8C4S2C.com.1password/t/agent.sock`, + want: 0.5, + }, + { + name: "wrong directive entirely", + response: `Add IdentityFile ~/.ssh/id_ed25519 to your config.`, + want: 0, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Value != tt.want { + t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail) + } + }) + } +} + +func TestMacosCaffeinateTest_Eval(t *testing.T) { + tc := macosCaffeinateTest() + + tests := []struct { + name string + response string + want float64 + }{ + {"bare invocation", `caffeinate ./migrate.sh`, 1}, + {"with system-sleep flag", `caffeinate -s ./migrate.sh`, 1}, + {"prose wrapped", "Run `caffeinate -i ./migrate.sh` and it stops holding the assertion once migrate.sh exits.", 1}, + {"wrong script named, not the target", `caffeinate ./other-script.sh`, 0.5}, + {"unrelated tool, does not auto-release on exit", `Just run pmset noidle before starting your script.`, 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Value != tt.want { + t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail) + } + }) + } +} + +func TestMacosStatBSDGNUTest_Eval(t *testing.T) { + tc := macosStatBSDGNUTest() + + tests := []struct { + name string + response string + want float64 + }{ + {"correct BSD form and GNU flag", `stat -f%z file.txt` + "\nGNU stat uses -c instead.", 1}, + {"quoted format string", `stat -f "%z" file.txt, and on Linux GNU stat uses -c.`, 1}, + {"prose wrapped", "Run stat -f%z file.txt on macOS; the GNU equivalent flag is -c.", 1}, + {"correct BSD command but never names the GNU flag", `stat -f%z file.txt`, 0.5}, + {"GNU -c%s syntax used on macOS scores half: -c token matches but -f BSD form is missing", `stat -c%s file.txt`, 0.5}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Value != tt.want { + t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail) + } + }) + } +}