llm-testbench/internal/tests/kubernetes.go

package tests

import (
	"context"
	"regexp"
	"strings"

	"github.com/lukaszraczylo/llm-testbench/internal/eval"
	"github.com/lukaszraczylo/llm-testbench/internal/testkit"
)

func registerKubernetesTests(r *testkit.Registry) {
	r.Register(k8sCrashloopGitopsTest())
}

// k8sCrashloopGitopsDescribeExcerpt is the inline kubectl describe pod
// excerpt for k8sCrashloopGitopsTest.
const k8sCrashloopGitopsDescribeExcerpt = `Name:         inventory-worker-6f9b7c8d9-2xk4p
Namespace:    inventory
Status:       Running
Containers:
  worker:
    State:          Waiting
      Reason:       CrashLoopBackOff
    Last State:     Terminated
      Reason:       OOMKilled
      Exit Code:    137
    Ready:          False
    Restart Count:  9
    Limits:
      cpu:     500m
      memory:  128Mi
    Requests:
      cpu:     100m
      memory:  64Mi
Events:
  Warning  BackOff  kubelet  Back-off restarting failed container worker in pod inventory-worker-6f9b7c8d9-2xk4p`

// mentionsLimitIncrease scores full credit only when the response both
// names the resource being raised (memory / the limit / the 128Mi value)
// and uses language for raising it. Using two independent ContainsAny
// checks combined by weighted mean would give partial credit to a response
// that raises the wrong resource (e.g. CPU) or that mentions memory
// without proposing to change the limit; this requires both.
func mentionsLimitIncrease() eval.Evaluator {
	verbs := eval.ContainsAny("increase", "raise", "bump", "higher", "raising")
	target := eval.ContainsAny("memory limit", "limit", "128Mi")
	return eval.EvaluatorFunc(func(ctx context.Context, response string) eval.Score {
		v := verbs.Evaluate(ctx, response)
		t := target.Evaluate(ctx, response)
		if v.Value == 1 && t.Value == 1 {
			return eval.Score{Value: 1, Detail: "proposes raising the memory limit"}
		}
		return eval.Score{Value: 0, Detail: "does not clearly propose raising the memory limit"}
	})
}

// kubectlLiveMutationPattern matches any mention of "kubectl edit" or
// "kubectl patch", negated or not - whether mid-sentence, at a bare line
// start, or as a bulleted list item.
var kubectlLiveMutationPattern = regexp.MustCompile(`(?i)kubectl\s+(edit|patch)\b`)

// negationCuePattern matches a word that turns a mention of the forbidden
// command into a warning against running it, rather than an instruction to
// run it (e.g. "do not run kubectl patch...").
var negationCuePattern = regexp.MustCompile(`(?i)\b(don'?t|do not|never|avoid|instead of|not|cannot|can'?t|rather than|without|no need)\b`)

// negationWindow is how many characters before a "kubectl edit|patch"
// mention are searched for a negation cue.
const negationWindow = 60

// negationWindowStart returns the earliest byte offset in response to
// search for a negation cue before a "kubectl edit|patch" match starting
// at start. The window is the current line, extended back to the start of
// the immediately preceding line when that line is non-empty: a
// hard-wrapped sentence whose negation cue landed on the line above (e.g.
// "...do not run\nkubectl edit...") would otherwise score 0 even though it
// is the correct, negated answer. Either way, the window never reaches
// more than negationWindow characters before start.
func negationWindowStart(response string, start int) int {
	curLineStart := strings.LastIndexByte(response[:start], '\n') + 1

	windowFloor := curLineStart
	if curLineStart > 0 {
		prevLineEnd := curLineStart - 1 // the newline separating the two lines
		prevLineStart := strings.LastIndexByte(response[:prevLineEnd], '\n') + 1
		if strings.TrimSpace(response[prevLineStart:prevLineEnd]) != "" {
			windowFloor = prevLineStart
		}
	}

	return max(start-negationWindow, windowFloor)
}

// noLiveKubectlMutation scores full credit unless the response instructs
// running "kubectl edit"/"kubectl patch" against the live cluster. A
// contrastive mention - warning the reader not to do this, in favor of the
// GitOps procedure - is fine and does not cost credit; an unnegated
// mention scores zero. This is deliberately not eval.NotContains, which
// would also zero out the best possible answer (one that correctly
// explains why NOT to run kubectl edit/patch).
//
// Every occurrence goes through the same negation-window check, including
// one at the start of a line (a bulleted "don't do this" list item, or a
// hard-wrapped sentence whose negation cue landed on the line above): a
// response is not penalized just because word-wrap or list formatting put
// the command text at a line start. A genuinely bare imperative still
// fails, because negationWindowStart never finds a negation cue in its
// (possibly line-extended) window.
func noLiveKubectlMutation() eval.Evaluator {
	return eval.EvaluatorFunc(func(_ context.Context, response string) eval.Score {
		matches := kubectlLiveMutationPattern.FindAllStringIndex(response, -1)
		if len(matches) == 0 {
			return eval.Score{Value: 1, Detail: "no mention of kubectl edit/patch"}
		}

		for _, loc := range matches {
			start := loc[0]
			windowStart := negationWindowStart(response, start)
			if !negationCuePattern.MatchString(response[windowStart:start]) {
				return eval.Score{Value: 0, Detail: "unnegated mention of kubectl edit/patch"}
			}
		}
		return eval.Score{Value: 1, Detail: "every mention of kubectl edit/patch is negated"}
	})
}

// k8sCrashloopGitopsTest: diagnose an OOMKilled CrashLoopBackOff and
// require the correct GitOps fix procedure, not a live kubectl mutation.
func k8sCrashloopGitopsTest() testkit.Test {
	prompt := `This cluster is entirely managed by ArgoCD: every workload's
manifests live in a git repository, and ArgoCD continuously reconciles the
live cluster state to match what is committed there. Here is the output of
"kubectl describe pod" for a pod that is failing:

` + k8sCrashloopGitopsDescribeExcerpt + `

What is the root cause of this CrashLoopBackOff, and what is the correct
procedure to fix it given that ArgoCD manages this cluster?`

	evaluator := eval.All(
		eval.W(eval.ContainsAny("OOM", "OOMKilled", "out of memory", "memory limit"), 2),
		eval.W(mentionsLimitIncrease(), 2),
		eval.W(eval.ContainsAny("git commit", "commit the change", "ArgoCD", "GitOps"), 2),
		eval.W(noLiveKubectlMutation(), 2),
	)

	return testkit.Test{
		ID:          "k8s-crashloop-gitops",
		Category:    "operations",
		Subcategory: "kubernetes",
		Description: "Diagnose an OOMKilled CrashLoopBackOff and require the GitOps fix procedure over a live kubectl mutation.",
		Prompt:      prompt,
		MaxTokens:   600,
		Eval:        evaluator,
	}
}