mirror of
https://github.com/lukaszraczylo/llm-testbench.git
synced 2026-10-12 00:45:19 +00:00
test(catalog): add delivery category with 30 tests
Add a new delivery category with three subcategories: git, containers, and release-engineering. Each subcategory holds 10 self-contained, deterministically-scored tests, grounded in real git, Docker, and release-engineering behaviour. The git subcategory covers bisect step derivation, rebase versus merge, conventional-commit classification, git worktree, force-with-lease versus bare force, hook choice, detached-HEAD recovery, cherry-pick versus revert, a .gitignore negation trace, and stash-pop conflict behaviour. Two of these tests recompute their ground truth against a real, throwaway git repository created in t.TempDir, and skip if git is not installed. The containers subcategory covers Dockerfile layer-cache busting, ENTRYPOINT and CMD interaction, multi-stage image size, non-root users and capabilities, HEALTHCHECK semantics, .dockerignore effects, COPY versus ADD, build-arg versus runtime env, and layer counting. The release-engineering subcategory covers semver bump derivation from a changelog, the clean-tree requirement, rollback ordering, canary versus blue-green, tag-to-release sequencing, commit-to-changelog mapping, checksum and signing purpose, pipeline stage ordering, pin versus range dependency policy, and hotfix flow ordering. A shared negation-aware evaluator helper, local to this category's files, scores a forbidden phrase only when it appears unnegated, so a response correctly warning against bare --force or a silent tracked-file mutation still scores full credit. registerDeliveryTests wires the three subcategories together and is exercised only by a new wiring test in this worktree, since catalog.go and catalog_test.go stay untouched until the orchestrator integrates this category centrally.
This commit is contained in:
@@ -0,0 +1,477 @@
|
||||
package tests
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"math"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// TestDelGitBisectStepsWant_GroundTruth recomputes delGitBisectStepsWant
|
||||
// from first principles (ceil(log2(N))) rather than trusting the hardcoded
|
||||
// literal in delivery_git.go.
|
||||
func TestDelGitBisectStepsWant_GroundTruth(t *testing.T) {
|
||||
got := int(math.Ceil(math.Log2(float64(delGitBisectCommitCount))))
|
||||
if got != delGitBisectStepsWant {
|
||||
t.Fatalf("ceil(log2(%d)) = %d, want %d", delGitBisectCommitCount, got, delGitBisectStepsWant)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDelGitBisectStepsTest_Eval(t *testing.T) {
|
||||
tc := delGitBisectStepsTest()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{name: "bare number", response: "8", want: 1},
|
||||
{name: "sentence form", response: "The answer is 8.", want: 1},
|
||||
{name: "different sentence form", response: "There are 8 bisect steps needed.", want: 1},
|
||||
{name: "wrong: repeats commit count", response: "137", want: 0},
|
||||
{name: "wrong: off by one", response: "7", want: 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDelGitRebaseVsMergeTest_Eval(t *testing.T) {
|
||||
tc := delGitRebaseVsMergeTest()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{name: "all correct", response: `{"scenario_a":"rebase","scenario_b":"merge"}`, want: 1},
|
||||
{name: "all correct fenced", response: "```json\n{\"scenario_a\":\"rebase\",\"scenario_b\":\"merge\"}\n```", want: 1},
|
||||
{name: "scenario_a wrong", response: `{"scenario_a":"merge","scenario_b":"merge"}`, want: 0.5},
|
||||
{name: "scenario_b wrong", response: `{"scenario_a":"rebase","scenario_b":"rebase"}`, want: 0.5},
|
||||
{name: "both swapped", response: `{"scenario_a":"merge","scenario_b":"rebase"}`, want: 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDelGitConventionalCommitClassifyTest_Eval(t *testing.T) {
|
||||
tc := delGitConventionalCommitClassifyTest()
|
||||
|
||||
allCorrect := `{"commit1":"feat","commit2":"fix","commit3":"chore","commit4":"docs","commit5":"refactor","commit6":"test"}`
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{name: "all correct", response: allCorrect, want: 1},
|
||||
{name: "all correct fenced with prose", response: "Here is my classification:\n```json\n" + allCorrect + "\n```", want: 1},
|
||||
{
|
||||
name: "one wrong: commit1 misclassified as chore",
|
||||
response: `{"commit1":"chore","commit2":"fix","commit3":"chore","commit4":"docs","commit5":"refactor","commit6":"test"}`,
|
||||
want: 5.0 / 6.0,
|
||||
},
|
||||
{
|
||||
name: "two wrong: commit3 and commit5 swapped types",
|
||||
response: `{"commit1":"feat","commit2":"fix","commit3":"refactor","commit4":"docs","commit5":"chore","commit6":"test"}`,
|
||||
want: 4.0 / 6.0,
|
||||
},
|
||||
{
|
||||
name: "all wrong",
|
||||
response: `{"commit1":"docs","commit2":"chore","commit3":"feat","commit4":"test","commit5":"fix","commit6":"refactor"}`,
|
||||
want: 0,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDelGitWorktreeUseCaseTest_Eval(t *testing.T) {
|
||||
tc := delGitWorktreeUseCaseTest()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{
|
||||
name: "correct: exact command",
|
||||
response: "Use `git worktree add ../v1.4.0-check v1.4.0` to create a second, linked working directory.",
|
||||
want: 1,
|
||||
},
|
||||
{
|
||||
name: "correct: relaxed prose",
|
||||
response: "Run git worktree add ../v1.4.0-check v1.4.0 to build the old tag in another directory.",
|
||||
want: 1,
|
||||
},
|
||||
{
|
||||
name: "correct: alternate phrasing",
|
||||
response: "git worktree is exactly this: git worktree add ../v1.4.0-check v1.4.0 links a second checkout to the same repo.",
|
||||
want: 1,
|
||||
},
|
||||
{
|
||||
name: "wrong: suggests re-cloning, mentions the tag only",
|
||||
response: "Just run git clone again into a new folder and checkout v1.4.0 there.",
|
||||
want: 1.0 / 3.0,
|
||||
},
|
||||
{
|
||||
name: "wrong: suggests stash, no worktree or tag path mentioned",
|
||||
response: "Use git stash to save your changes, then checkout the old tag in a scratch directory, then stash pop when done.",
|
||||
want: 0,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDelGitForceWithLeaseVsForceTest_Eval(t *testing.T) {
|
||||
tc := delGitForceWithLeaseVsForceTest()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{
|
||||
name: "correct: recommends force-with-lease, negates bare force",
|
||||
response: "Use git push --force-with-lease on the shared branch; never use a bare --force because it would silently overwrite the remote branch if it moved.",
|
||||
want: 1,
|
||||
},
|
||||
{
|
||||
name: "correct: only ever mentions the safe form",
|
||||
response: "On a shared branch always push with --force-with-lease; it aborts if the remote moved since your last fetch, unlike a plain overwrite.",
|
||||
want: 1,
|
||||
},
|
||||
{
|
||||
name: "correct: bare force mentioned but clearly negated",
|
||||
response: "Don't ever run a bare --force on release/2.4; instead use --force-with-lease, which fails safely if the remote changed.",
|
||||
want: 1,
|
||||
},
|
||||
{
|
||||
name: "wrong: recommends bare force outright",
|
||||
response: "Just use git push --force on the shared branch to make sure your rewritten history replaces theirs.",
|
||||
want: 0,
|
||||
},
|
||||
{
|
||||
name: "wrong: recommends force-with-lease but also an unnegated bare-force fallback",
|
||||
response: "Push with --force-with-lease normally, but if the lease check fails just fall back to --force to get it through.",
|
||||
want: 0.5,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDelGitHookChoiceTest_Eval(t *testing.T) {
|
||||
tc := delGitHookChoiceTest()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{name: "exact", response: "commit-msg", want: 1},
|
||||
{name: "different case", response: "Commit-Msg", want: 1},
|
||||
{name: "surrounding whitespace", response: " commit-msg \n", want: 1},
|
||||
{name: "wrong: pre-commit", response: "pre-commit", want: 0},
|
||||
{name: "wrong: pre-push", response: "pre-push", want: 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDelGitDetachedHeadRecoveryTest_Eval(t *testing.T) {
|
||||
tc := delGitDetachedHeadRecoveryTest()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{name: "correct order", response: `["branch","checkout"]`, want: 1},
|
||||
{name: "correct order fenced", response: "```json\n[\"branch\",\"checkout\"]\n```", want: 1},
|
||||
{name: "correct order different case", response: `["Branch","Checkout"]`, want: 1},
|
||||
{name: "reversed order", response: `["checkout","branch"]`, want: 0},
|
||||
{name: "wrong second subcommand", response: `["branch","switch"]`, want: 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDelGitCherryPickVsRevertTest_Eval(t *testing.T) {
|
||||
tc := delGitCherryPickVsRevertTest()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{name: "exact", response: "revert", want: 1},
|
||||
{name: "different case", response: "Revert", want: 1},
|
||||
{name: "surrounding whitespace", response: " revert \n", want: 1},
|
||||
{name: "wrong: cherry-pick", response: "cherry-pick", want: 0},
|
||||
{name: "wrong: reset", response: "reset", want: 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDelGitGitignoreNegationTraceTest_Eval(t *testing.T) {
|
||||
tc := delGitGitignoreNegationTraceTest()
|
||||
|
||||
correctSet := `["build/keep.txt","build/output.bin","debug.log","docs/notes.tmp"]`
|
||||
correctSetReordered := `["docs/notes.tmp","debug.log","build/output.bin","build/keep.txt"]`
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{name: "exact correct set", response: correctSet, want: 1},
|
||||
{name: "correct set, different order", response: correctSetReordered, want: 1},
|
||||
{name: "correct set, fenced", response: "```json\n" + correctSet + "\n```", want: 1},
|
||||
{
|
||||
name: "wrong: falls for the negation trap, includes docs/important.tmp",
|
||||
response: `["build/keep.txt","build/output.bin","debug.log","docs/notes.tmp","docs/important.tmp"]`,
|
||||
want: 0.8,
|
||||
},
|
||||
{
|
||||
name: "wrong: assumes negation always works, drops build/keep.txt",
|
||||
response: `["build/output.bin","debug.log","docs/notes.tmp"]`,
|
||||
want: 0.75,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDelGitStashPopConflictTest_Eval(t *testing.T) {
|
||||
tc := delGitStashPopConflictTest()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{name: "exact", response: "kept", want: 1},
|
||||
{name: "different case", response: "Kept", want: 1},
|
||||
{name: "surrounding whitespace", response: " kept \n", want: 1},
|
||||
{name: "wrong: dropped", response: "dropped", want: 0},
|
||||
{name: "wrong: removed", response: "removed", want: 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// delGitTestEnv builds a minimal, isolated environment for exec.Command
|
||||
// calls against a throwaway git repository: PATH (to find git), HOME
|
||||
// pointed at the isolated dir (never the operator's real home), and
|
||||
// GIT_CONFIG_NOSYSTEM/GIT_CONFIG_GLOBAL disabled so neither the system nor
|
||||
// the operator's real ~/.gitconfig (which may set core.hooksPath to a
|
||||
// global pre-commit hook) is consulted. GIT_AUTHOR_*/GIT_COMMITTER_* supply
|
||||
// a commit identity directly, so no "git config user.email" step is needed
|
||||
// either. This keeps the test hermetic: no network, no dependency on the
|
||||
// operator's actual git configuration.
|
||||
func delGitTestEnv(dir string) []string {
|
||||
return []string{
|
||||
"PATH=" + os.Getenv("PATH"),
|
||||
"HOME=" + dir,
|
||||
"GIT_CONFIG_NOSYSTEM=1",
|
||||
"GIT_CONFIG_GLOBAL=/dev/null",
|
||||
"GIT_AUTHOR_NAME=llmtb-test",
|
||||
"[email protected]",
|
||||
"GIT_COMMITTER_NAME=llmtb-test",
|
||||
"[email protected]",
|
||||
}
|
||||
}
|
||||
|
||||
// delRunGit runs `git args...` inside dir with env, returning combined
|
||||
// stdout+stderr and any error.
|
||||
func delRunGit(t *testing.T, dir string, env []string, args ...string) (string, error) {
|
||||
t.Helper()
|
||||
// #nosec G204 -- args are always fixed literals or the fixed
|
||||
// delGitignoreTreeFiles list defined in delivery_git.go, never
|
||||
// response/external/user-controlled input.
|
||||
cmd := exec.Command("git", args...) //nolint:gosec // see comment above
|
||||
cmd.Dir = dir
|
||||
cmd.Env = env
|
||||
var out bytes.Buffer
|
||||
cmd.Stdout = &out
|
||||
cmd.Stderr = &out
|
||||
err := cmd.Run()
|
||||
return out.String(), err
|
||||
}
|
||||
|
||||
// TestDelGitGitignoreNegationTrace_GroundTruth verifies delGitignoreIgnoredWant
|
||||
// against a real, throwaway git repository (git.md's "verify empirically
|
||||
// where cheap" guidance) rather than trusting delivery_git.go's derivation
|
||||
// comment alone. Skips if git is not installed.
|
||||
func TestDelGitGitignoreNegationTrace_GroundTruth(t *testing.T) {
|
||||
if _, err := exec.LookPath("git"); err != nil {
|
||||
t.Skip("git not installed")
|
||||
}
|
||||
|
||||
dir := t.TempDir()
|
||||
env := delGitTestEnv(dir)
|
||||
|
||||
if err := os.MkdirAll(filepath.Join(dir, "build"), 0o700); err != nil {
|
||||
t.Fatalf("mkdir build: %v", err)
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Join(dir, "docs"), 0o700); err != nil {
|
||||
t.Fatalf("mkdir docs: %v", err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(dir, ".gitignore"), []byte(delGitignoreFile+"\n"), 0o600); err != nil {
|
||||
t.Fatalf("write .gitignore: %v", err)
|
||||
}
|
||||
for _, f := range delGitignoreTreeFiles {
|
||||
if err := os.WriteFile(filepath.Join(dir, f), []byte("x\n"), 0o600); err != nil {
|
||||
t.Fatalf("write %s: %v", f, err)
|
||||
}
|
||||
}
|
||||
|
||||
if out, err := delRunGit(t, dir, env, "init", "-q"); err != nil {
|
||||
t.Fatalf("git init: %v (%s)", err, out)
|
||||
}
|
||||
|
||||
args := append([]string{"check-ignore"}, delGitignoreTreeFiles...)
|
||||
out, err := delRunGit(t, dir, env, args...)
|
||||
if err != nil {
|
||||
var exitErr *exec.ExitError
|
||||
if !errors.As(err, &exitErr) || exitErr.ExitCode() != 1 {
|
||||
t.Fatalf("git check-ignore: %v (%s)", err, out)
|
||||
}
|
||||
// exit code 1 just means "none of the given paths are ignored",
|
||||
// which is not the case here but is a valid, non-fatal outcome of
|
||||
// the command in general.
|
||||
}
|
||||
|
||||
var got []string
|
||||
for _, line := range strings.Split(strings.TrimSpace(out), "\n") {
|
||||
if line != "" {
|
||||
got = append(got, line)
|
||||
}
|
||||
}
|
||||
sort.Strings(got)
|
||||
|
||||
want := append([]string(nil), delGitignoreIgnoredWant...)
|
||||
sort.Strings(want)
|
||||
|
||||
if !slices.Equal(got, want) {
|
||||
t.Fatalf("git check-ignore ignored set = %v, want %v", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
// TestDelGitStashPopConflict_GroundTruth verifies, against a real,
|
||||
// throwaway git repository, that a conflicting `git stash pop` keeps the
|
||||
// stash entry rather than dropping it. Skips if git is not installed.
|
||||
func TestDelGitStashPopConflict_GroundTruth(t *testing.T) {
|
||||
if _, err := exec.LookPath("git"); err != nil {
|
||||
t.Skip("git not installed")
|
||||
}
|
||||
|
||||
dir := t.TempDir()
|
||||
env := delGitTestEnv(dir)
|
||||
fpath := filepath.Join(dir, "f.txt")
|
||||
|
||||
write := func(content string) {
|
||||
t.Helper()
|
||||
if err := os.WriteFile(fpath, []byte(content), 0o600); err != nil {
|
||||
t.Fatalf("write f.txt: %v", err)
|
||||
}
|
||||
}
|
||||
mustGit := func(args ...string) string {
|
||||
t.Helper()
|
||||
out, err := delRunGit(t, dir, env, args...)
|
||||
if err != nil {
|
||||
t.Fatalf("git %v: %v (%s)", args, err, out)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
mustGit("init", "-q")
|
||||
write("line1\nline2\nline3\n")
|
||||
mustGit("add", "f.txt")
|
||||
mustGit("commit", "-q", "-m", "init")
|
||||
|
||||
write("line1\nline2-stashed\nline3\n")
|
||||
mustGit("stash", "push", "-q", "-m", "work")
|
||||
|
||||
write("line1\nline2-committed\nline3\n")
|
||||
mustGit("add", "f.txt")
|
||||
mustGit("commit", "-q", "-m", "conflicting")
|
||||
|
||||
if _, err := delRunGit(t, dir, env, "stash", "pop"); err == nil {
|
||||
t.Fatal("git stash pop: want a conflict (non-nil error) given the two changes touch the same line, got nil")
|
||||
}
|
||||
|
||||
out := mustGit("stash", "list")
|
||||
if strings.TrimSpace(out) == "" {
|
||||
t.Fatal("git stash list: want the stash entry kept after a conflicting pop, got an empty list")
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user