mirror of
https://github.com/lukaszraczylo/llm-testbench.git
synced 2026-10-19 01:27:32 +00:00
test(catalog): expand operations to 10 tests per subcategory
Add 8 macOS tests, 8 Linux tests, and 9 Kubernetes tests to internal/tests. Each operations subcategory now holds 10 deterministic tests, up from 2, 2, and 1. Tests cover BSD-versus-GNU tooling, launchd, Proxmox pct exec, systemd timers, cgroup v2 diagnosis, and ArgoCD GitOps scenarios. Evaluators use ContainsAll, ContainsAny, Regex, JSONField, Numeric, and weighted composites, matching the existing catalog style. Every evaluator unit test asserts three correct phrasings and two wrong ones, per the expansion contract. Ground truth for numeric and ordered answers carries a derivation comment. catalog_test.go still expects 17 tests and fails as documented. This commit does not modify that file.
This commit is contained in:
@@ -109,3 +109,235 @@ is missing. Check the image tag and re-push it.`
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestK8sImagePullBackoffTest_Eval(t *testing.T) {
|
||||
tc := k8sImagePullBackoffTest()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{"correct object", `{"cause":"auth","fix":"imagePullSecrets"}`, 1},
|
||||
{"prose then JSON", "This is a 401, an auth failure.\n" + `{"cause":"auth","fix":"imagePullSecrets"}`, 1},
|
||||
{"fenced JSON", "```json\n" + `{"cause":"auth","fix":"imagePullSecrets"}` + "\n```", 1},
|
||||
{"misreads it as a missing tag", `{"cause":"not-found","fix":"fix-image-tag"}`, 0},
|
||||
{"right cause, wrong fix category", `{"cause":"auth","fix":"check-network"}`, 0.5},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestK8sPendingTaintsTest_Eval(t *testing.T) {
|
||||
tc := k8sPendingTaintsTest()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{"correct object", `{"cause":"taint","fix_field":"tolerations"}`, 1},
|
||||
{"prose then JSON", "The nodes have an untolerated taint.\n" + `{"cause":"taint","fix_field":"tolerations"}`, 1},
|
||||
{"fenced JSON", "```json\n" + `{"cause":"taint","fix_field":"tolerations"}` + "\n```", 1},
|
||||
{"misdiagnoses as a resource limit issue", `{"cause":"resource-limit","fix_field":"resources"}`, 0},
|
||||
{"right cause, wrong fix field", `{"cause":"taint","fix_field":"nodeSelector"}`, 0.5},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestK8sPVCStorageclassMismatchTest_Eval(t *testing.T) {
|
||||
tc := k8sPVCStorageclassMismatchTest()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{"correct object", `{"cause":"missing-storageclass","use_storageclass":"standard"}`, 1},
|
||||
{"prose then JSON", "The fast-ssd class does not exist.\n" + `{"cause":"missing-storageclass","use_storageclass":"standard"}`, 1},
|
||||
{"fenced JSON", "```json\n" + `{"cause":"missing-storageclass","use_storageclass":"standard"}` + "\n```", 1},
|
||||
{"misdiagnoses as the provisioner being down", `{"cause":"provisioner-down","use_storageclass":"standard"}`, 0.5},
|
||||
{"right cause but names a class that does not exist", `{"cause":"missing-storageclass","use_storageclass":"fast-ssd"}`, 0.5},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestK8sServiceSelectorMismatchTest_Eval(t *testing.T) {
|
||||
tc := k8sServiceSelectorMismatchTest()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{"correct object", `{"field":"selector.app","value":"inventory-api"}`, 1},
|
||||
{"prose then JSON", "The selector doesn't match the pod labels.\n" + `{"field":"selector.app","value":"inventory-api"}`, 1},
|
||||
{"fenced JSON", "```json\n" + `{"field":"selector.app","value":"inventory-api"}` + "\n```", 1},
|
||||
{"points at the wrong field (targetPort instead of selector)", `{"field":"targetPort","value":"8080"}`, 0},
|
||||
{"right field, wrong replacement value", `{"field":"selector.app","value":"inventory-api-svc"}`, 0.5},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestK8sNetworkPolicyDNSBlockTest_Eval(t *testing.T) {
|
||||
tc := k8sNetworkPolicyDNSBlockTest()
|
||||
|
||||
good := `This NetworkPolicy allows Egress only to payments-db pods on
|
||||
5432/TCP, so every other egress - including DNS lookups on port 53/UDP -
|
||||
is implicitly denied. Add an egress rule allowing UDP port 53 (and
|
||||
typically TCP 53) to pods labeled k8s-app=kube-dns in kube-system.`
|
||||
goodAlt := `The policy default-denies all egress except the one listed
|
||||
destination, and DNS needs to reach CoreDNS on UDP port 53, which is not
|
||||
allowed here. Add an egress rule permitting port 53/udp to the coredns
|
||||
pods.`
|
||||
goodTerse := `Add an egress allow rule for port 53 UDP targeting the
|
||||
kube-system namespace's kube-dns pods; everything else is denied by
|
||||
default here.`
|
||||
badWrongPort := `Add an egress rule allowing port 80/TCP to the ingress
|
||||
controller.`
|
||||
badWrongProtocol := `Add an egress rule allowing TCP port 53 only to
|
||||
kube-system, since DNS always uses TCP.`
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{"good answer, kube-dns phrasing", good, 1},
|
||||
{"good answer, coredns phrasing", goodAlt, 1},
|
||||
{"good answer, terse phrasing", goodTerse, 1},
|
||||
{"wrong port and protocol entirely", badWrongPort, 0},
|
||||
{"right port, missing udp mention", badWrongProtocol, 2.0 / 3.0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestK8sArgoCDOutOfSyncTest_Eval(t *testing.T) {
|
||||
tc := k8sArgoCDOutOfSyncTest()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{"correct object", `{"cause":"hpa-managed-field","fix":"ignoreDifferences"}`, 1},
|
||||
{"prose then JSON", "The HPA owns replicas here.\n" + `{"cause":"hpa-managed-field","fix":"ignoreDifferences"}`, 1},
|
||||
{"fenced JSON", "```json\n" + `{"cause":"hpa-managed-field","fix":"ignoreDifferences"}` + "\n```", 1},
|
||||
{"wrongly blames manual drift nobody caused", `{"cause":"manual-drift","fix":"revert-manual-change"}`, 0},
|
||||
{"right cause, wrong fix (fighting the HPA by re-syncing)", `{"cause":"hpa-managed-field","fix":"sync-git"}`, 0.5},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestK8sTraefikIngressRouteHostTest_Eval(t *testing.T) {
|
||||
tc := k8sTraefikIngressRouteHostTest()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{"bare corrected line", "match: Host(`app.raczylo.com`)", 1},
|
||||
{"prose wrapped", "The corrected line is:\nmatch: Host(`app.raczylo.com`)", 1},
|
||||
{"uppercase Host keyword variant", "MATCH: HOST(`app.raczylo.com`)", 1},
|
||||
{"still the old domain", "match: Host(`app.example.com`)", 0},
|
||||
{"right function, wrong domain typo", "match: Host(`app.raczlo.com`)", 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestK8sQoSClassTest_Eval(t *testing.T) {
|
||||
tc := k8sQoSClassTest()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{"exact word", "Burstable", 1},
|
||||
{"lowercase", "burstable", 1},
|
||||
{"trailing whitespace", "Burstable\n", 1},
|
||||
{"wrong class, mistakes it for Guaranteed", "Guaranteed", 0},
|
||||
{"wrong class, mistakes it for BestEffort", "BestEffort", 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestK8sCNPGPDBTest_Eval(t *testing.T) {
|
||||
tc := k8sCNPGPDBTest()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{"correct resource and value", "Use a PodDisruptionBudget with minAvailable: 2.", 1},
|
||||
{"equals-sign phrasing", "A PodDisruptionBudget (PDB) with minAvailable=2 guarantees this.", 1},
|
||||
{"english phrasing of the value", "Create a PodDisruptionBudget with a minAvailable of 2.", 1},
|
||||
{"wrong resource entirely (a NetworkPolicy cannot do this)", "Use a NetworkPolicy with minAvailable: 2.", 0.5},
|
||||
{"right resource, wrong/missing value", "Use a PodDisruptionBudget, but leave the fields at their defaults.", 0.5},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user