mirror of
https://github.com/lukaszraczylo/llm-testbench.git
synced 2026-10-15 01:07:26 +00:00
test(catalog): add delivery category with 30 tests
Add a new delivery category with three subcategories: git, containers, and release-engineering. Each subcategory holds 10 self-contained, deterministically-scored tests, grounded in real git, Docker, and release-engineering behaviour. The git subcategory covers bisect step derivation, rebase versus merge, conventional-commit classification, git worktree, force-with-lease versus bare force, hook choice, detached-HEAD recovery, cherry-pick versus revert, a .gitignore negation trace, and stash-pop conflict behaviour. Two of these tests recompute their ground truth against a real, throwaway git repository created in t.TempDir, and skip if git is not installed. The containers subcategory covers Dockerfile layer-cache busting, ENTRYPOINT and CMD interaction, multi-stage image size, non-root users and capabilities, HEALTHCHECK semantics, .dockerignore effects, COPY versus ADD, build-arg versus runtime env, and layer counting. The release-engineering subcategory covers semver bump derivation from a changelog, the clean-tree requirement, rollback ordering, canary versus blue-green, tag-to-release sequencing, commit-to-changelog mapping, checksum and signing purpose, pipeline stage ordering, pin versus range dependency policy, and hotfix flow ordering. A shared negation-aware evaluator helper, local to this category's files, scores a forbidden phrase only when it appears unnegated, so a response correctly warning against bare --force or a silent tracked-file mutation still scores full credit. registerDeliveryTests wires the three subcategories together and is exercised only by a new wiring test in this worktree, since catalog.go and catalog_test.go stay untouched until the orchestrator integrates this category centrally.
This commit is contained in:
@@ -0,0 +1,476 @@
|
||||
package tests
|
||||
|
||||
import (
|
||||
"context"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/lukaszraczylo/llm-testbench/internal/eval"
|
||||
"github.com/lukaszraczylo/llm-testbench/internal/testkit"
|
||||
)
|
||||
|
||||
func registerDeliveryGitTests(r *testkit.Registry) {
|
||||
r.Register(delGitBisectStepsTest())
|
||||
r.Register(delGitRebaseVsMergeTest())
|
||||
r.Register(delGitConventionalCommitClassifyTest())
|
||||
r.Register(delGitWorktreeUseCaseTest())
|
||||
r.Register(delGitForceWithLeaseVsForceTest())
|
||||
r.Register(delGitHookChoiceTest())
|
||||
r.Register(delGitDetachedHeadRecoveryTest())
|
||||
r.Register(delGitCherryPickVsRevertTest())
|
||||
r.Register(delGitGitignoreNegationTraceTest())
|
||||
r.Register(delGitStashPopConflictTest())
|
||||
}
|
||||
|
||||
// --- negation-aware helpers, shared across delivery_git.go and
|
||||
// delivery_release.go (same package). Duplicated in spirit from
|
||||
// operations/kubernetes's noLiveKubectlMutation/negationCuePattern rather
|
||||
// than imported from kubernetes.go, so this category's files stay
|
||||
// self-contained: round-2 authors work in isolated worktrees on
|
||||
// databases/security/delivery/ai in parallel, and depending on another
|
||||
// category's unexported symbols would create a merge-time coupling this
|
||||
// worktree cannot see or coordinate on. ---
|
||||
|
||||
// delNegationCuePattern matches a word that turns a mention of a forbidden
|
||||
// phrase into a warning against doing it, rather than an instruction to do
|
||||
// it (e.g. "never use bare --force", "must not silently modify").
|
||||
var delNegationCuePattern = regexp.MustCompile(`(?i)\b(don'?t|do not|never|avoid|instead of|not|cannot|can'?t|rather than|without|no need|must not|should not|shouldn'?t)\b`)
|
||||
|
||||
// delNegationWindow is how many characters before a forbidden-phrase match
|
||||
// are searched for a negation cue.
|
||||
const delNegationWindow = 60
|
||||
|
||||
// delNegationWindowStart returns the earliest byte offset in response to
|
||||
// search for a negation cue before a forbidden-phrase match starting at
|
||||
// start. The window is the current line, extended back to the start of the
|
||||
// immediately preceding line when that line is non-empty, so a hard-wrapped
|
||||
// sentence whose negation cue landed on the line above still counts. Either
|
||||
// way the window never reaches more than delNegationWindow characters
|
||||
// before start.
|
||||
func delNegationWindowStart(response string, start int) int {
|
||||
curLineStart := strings.LastIndexByte(response[:start], '\n') + 1
|
||||
|
||||
windowFloor := curLineStart
|
||||
if curLineStart > 0 {
|
||||
prevLineEnd := curLineStart - 1
|
||||
prevLineStart := strings.LastIndexByte(response[:prevLineEnd], '\n') + 1
|
||||
if strings.TrimSpace(response[prevLineStart:prevLineEnd]) != "" {
|
||||
windowFloor = prevLineStart
|
||||
}
|
||||
}
|
||||
|
||||
return max(start-delNegationWindow, windowFloor)
|
||||
}
|
||||
|
||||
// delNoUnnegatedMention returns an Evaluator scoring full credit unless
|
||||
// forbidden matches response at some position with no negation cue in the
|
||||
// preceding window - i.e. the response recommends the forbidden thing
|
||||
// outright rather than warning against it. This is deliberately not
|
||||
// eval.NotContains, which would also zero out the best possible answer (one
|
||||
// that correctly explains why NOT to do the forbidden thing).
|
||||
func delNoUnnegatedMention(forbidden *regexp.Regexp, safeDetail string) eval.Evaluator {
|
||||
return eval.EvaluatorFunc(func(_ context.Context, response string) eval.Score {
|
||||
matches := forbidden.FindAllStringIndex(response, -1)
|
||||
if len(matches) == 0 {
|
||||
return eval.Score{Value: 1, Detail: "no forbidden mention: " + safeDetail}
|
||||
}
|
||||
for _, loc := range matches {
|
||||
start := loc[0]
|
||||
windowStart := delNegationWindowStart(response, start)
|
||||
if !delNegationCuePattern.MatchString(response[windowStart:start]) {
|
||||
return eval.Score{Value: 0, Detail: "unnegated forbidden mention"}
|
||||
}
|
||||
}
|
||||
return eval.Score{Value: 1, Detail: "every forbidden mention is negated"}
|
||||
})
|
||||
}
|
||||
|
||||
// delForceFlagPattern matches every "--force" mention. delForceWithLeaseSuffixPattern
|
||||
// checks whether one of those matches is immediately followed by
|
||||
// "-with-lease" - the safe form, exempt from the bare-force check entirely
|
||||
// rather than needing a negation cue.
|
||||
var delForceFlagPattern = regexp.MustCompile(`(?i)--force\b`)
|
||||
var delForceWithLeaseSuffixPattern = regexp.MustCompile(`(?i)^-with-lease`)
|
||||
|
||||
// delNoBareForcePush scores full credit unless the response recommends a
|
||||
// bare "--force" push (not "--force-with-lease") with no negation cue
|
||||
// nearby. A response that only ever mentions "--force-with-lease" never
|
||||
// matches delForceFlagPattern's bare form (the suffix check exempts it), so
|
||||
// it always scores full credit; a response that recommends bare --force
|
||||
// outright scores zero.
|
||||
func delNoBareForcePush() eval.Evaluator {
|
||||
return eval.EvaluatorFunc(func(_ context.Context, response string) eval.Score {
|
||||
matches := delForceFlagPattern.FindAllStringIndex(response, -1)
|
||||
if len(matches) == 0 {
|
||||
return eval.Score{Value: 1, Detail: "no --force mention"}
|
||||
}
|
||||
for _, loc := range matches {
|
||||
start, end := loc[0], loc[1]
|
||||
if delForceWithLeaseSuffixPattern.MatchString(response[end:]) {
|
||||
continue // "--force-with-lease": the safe form, not bare force
|
||||
}
|
||||
windowStart := delNegationWindowStart(response, start)
|
||||
if !delNegationCuePattern.MatchString(response[windowStart:start]) {
|
||||
return eval.Score{Value: 0, Detail: "unnegated bare --force mention"}
|
||||
}
|
||||
}
|
||||
return eval.Score{Value: 1, Detail: "every bare --force mention is negated or absent"}
|
||||
})
|
||||
}
|
||||
|
||||
// delGitBisectCommitCount is the number of candidate commits between the
|
||||
// last known-good commit and the current bad HEAD in delGitBisectStepsTest.
|
||||
const delGitBisectCommitCount = 137
|
||||
|
||||
// delGitBisectStepsWant is the worst-case number of `git bisect` steps
|
||||
// needed to find the single first-bad commit among
|
||||
// delGitBisectCommitCount candidates by binary search.
|
||||
//
|
||||
// ground truth: `git bisect` performs a binary search over the candidate
|
||||
// commit range, halving the remaining range on each step until exactly one
|
||||
// commit remains. The worst-case step count for N candidates is
|
||||
// ceil(log2(N)): 2^7 = 128 < 137 <= 256 = 2^8, so ceil(log2(137)) = 8.
|
||||
// delivery_git_test.go recomputes this with math.Ceil(math.Log2(137))
|
||||
// rather than trusting the hardcoded literal alone.
|
||||
var delGitBisectStepsWant = 8
|
||||
|
||||
// delGitBisectStepsTest: derive the worst-case number of `git bisect`
|
||||
// binary-search steps needed to find one bad commit among N candidates.
|
||||
func delGitBisectStepsTest() testkit.Test {
|
||||
prompt := `A regression was introduced somewhere among the last 137
|
||||
commits on main: the commit at the tip of main is bad, and the commit 137
|
||||
commits back (the last tagged release) is known good. Using "git bisect"
|
||||
(binary search over the candidate commits, testing one commit per step),
|
||||
what is the maximum number of "git bisect" steps needed to find the exact
|
||||
first commit that introduced the regression? Respond with only the integer
|
||||
number of steps.`
|
||||
|
||||
return testkit.Test{
|
||||
ID: "git-bisect-steps",
|
||||
Category: "delivery",
|
||||
Subcategory: "git",
|
||||
Description: "Derive the worst-case git bisect step count (ceil(log2(N))) for 137 candidate commits.",
|
||||
Prompt: prompt,
|
||||
Eval: eval.Numeric(eval.ExtractLastNumber[int], delGitBisectStepsWant, 0),
|
||||
}
|
||||
}
|
||||
|
||||
// delGitRebaseVsMergeTest: pick rebase or merge for a solo short-lived
|
||||
// branch versus a shared long-lived branch.
|
||||
//
|
||||
// ground truth: rebasing a branch nobody else has pulled is safe and
|
||||
// produces a clean, linear, single-author history before review - exactly
|
||||
// what interactive rebase is for. Rewriting history on a branch other
|
||||
// people have already pulled and built work on top of forces everyone else
|
||||
// to reconcile diverged history; merging (which adds a new commit rather
|
||||
// than rewriting existing ones) is the only one of the two that does not
|
||||
// rewrite commits other people already depend on.
|
||||
func delGitRebaseVsMergeTest() testkit.Test {
|
||||
prompt := `For each of these two situations, should you use "git rebase"
|
||||
or "git merge" to bring in the latest changes?
|
||||
|
||||
scenario_a: "You are the sole author of a short-lived feature branch that
|
||||
has never been pushed anywhere, and you want a clean, linear, single-author
|
||||
history before opening a pull request."
|
||||
scenario_b: "You are integrating a long-lived, shared release branch that
|
||||
several other people have already pulled and built further commits on top
|
||||
of; you must not rewrite any commit they already depend on."
|
||||
|
||||
Respond with only a JSON object:
|
||||
{"scenario_a":"rebase"|"merge","scenario_b":"rebase"|"merge"}`
|
||||
|
||||
evaluator := eval.Mean(
|
||||
eval.JSONField("scenario_a", "rebase"),
|
||||
eval.JSONField("scenario_b", "merge"),
|
||||
)
|
||||
|
||||
return testkit.Test{
|
||||
ID: "git-rebase-vs-merge",
|
||||
Category: "delivery",
|
||||
Subcategory: "git",
|
||||
Description: "Choose rebase for a solo unshared branch and merge for a shared branch others already depend on.",
|
||||
Prompt: prompt,
|
||||
Eval: evaluator,
|
||||
}
|
||||
}
|
||||
|
||||
// delGitConventionalCommits is the inline set of 6 commit messages for
|
||||
// delGitConventionalCommitClassifyTest, one per standard conventional-commit
|
||||
// type, grounded in this repository's own real history (agents.go's
|
||||
// per-subcategory file split, ExtractLastNumber's dot-prefixed-decimal
|
||||
// handling, exec.go's toolchain-missing skip behavior, and the errgroup
|
||||
// dependency named in PLAN.md).
|
||||
const delGitConventionalCommits = `commit1: "feat(runner): add bounded concurrency to the model x test fan-out via errgroup"
|
||||
commit2: "fix(eval): correct ExtractLastNumber rejecting a standalone dot-prefixed decimal like \".9896\""
|
||||
commit3: "chore(deps): bump golang.org/x/sync to v0.22.0"
|
||||
commit4: "docs: document how to add a new evaluator in README.md"
|
||||
commit5: "refactor(tests): split agents.go into per-subcategory files to stay under 600 lines"
|
||||
commit6: "test(exec): add toolchain-missing skip cases for GoRun/PyRun/CRun"`
|
||||
|
||||
// delGitConventionalCommitClassifyTest: classify 6 commit messages, one per
|
||||
// standard conventional-commit type, by their type prefix.
|
||||
func delGitConventionalCommitClassifyTest() testkit.Test {
|
||||
prompt := `Here are 6 commit messages from a project using Conventional
|
||||
Commits:
|
||||
|
||||
` + delGitConventionalCommits + `
|
||||
|
||||
Classify each commit by its conventional-commit type (one of: feat, fix,
|
||||
chore, docs, refactor, test). Respond with only a JSON object:
|
||||
{"commit1":"...","commit2":"...","commit3":"...","commit4":"...","commit5":"...","commit6":"..."}`
|
||||
|
||||
evaluator := eval.Mean(
|
||||
eval.JSONField("commit1", "feat"),
|
||||
eval.JSONField("commit2", "fix"),
|
||||
eval.JSONField("commit3", "chore"),
|
||||
eval.JSONField("commit4", "docs"),
|
||||
eval.JSONField("commit5", "refactor"),
|
||||
eval.JSONField("commit6", "test"),
|
||||
)
|
||||
|
||||
return testkit.Test{
|
||||
ID: "git-conventional-commit-classify",
|
||||
Category: "delivery",
|
||||
Subcategory: "git",
|
||||
Description: "Classify 6 commit messages into their conventional-commit type from the message prefix.",
|
||||
Prompt: prompt,
|
||||
Eval: evaluator,
|
||||
}
|
||||
}
|
||||
|
||||
// delGitWorktreeUseCaseTest: name git worktree as the mechanism for a
|
||||
// second, independently checked-out working directory from the same
|
||||
// repository.
|
||||
//
|
||||
// ground truth: "git worktree add <path> <ref>" creates a second, linked
|
||||
// working directory backed by the same .git repository, checked out to a
|
||||
// different ref, without stashing, committing, or cloning again - exactly
|
||||
// the described need (an in-progress feature branch with uncommitted
|
||||
// changes stays untouched in the main working directory while an old tag is
|
||||
// built and tested elsewhere).
|
||||
func delGitWorktreeUseCaseTest() testkit.Test {
|
||||
prompt := `You are mid-way through an in-progress feature branch with
|
||||
uncommitted changes in your main working directory. You need to build and
|
||||
test the old release tag v1.4.0 at the same time, without stashing your
|
||||
changes and without cloning the repository again. Name the git subcommand
|
||||
that creates a second, linked working directory from the same repository
|
||||
checked out to a different ref, and give the exact command to create one at
|
||||
path ../v1.4.0-check checked out to tag v1.4.0.`
|
||||
|
||||
return testkit.Test{
|
||||
ID: "git-worktree-use-case",
|
||||
Category: "delivery",
|
||||
Subcategory: "git",
|
||||
Description: "Name git worktree as the way to check out a second ref from the same repo without stashing or re-cloning.",
|
||||
Prompt: prompt,
|
||||
Eval: eval.ContainsAll("git worktree add", "v1.4.0-check", "v1.4.0"),
|
||||
}
|
||||
}
|
||||
|
||||
// delGitForceWithLeaseVsForceTest: require --force-with-lease on a shared
|
||||
// branch, and require any mention of bare --force to be negated.
|
||||
//
|
||||
// ground truth: "--force-with-lease" refuses the push if the remote branch
|
||||
// has moved since your last fetch (someone else pushed), failing safely
|
||||
// instead of silently overwriting their work; bare "--force" overwrites
|
||||
// whatever is on the remote unconditionally, with no such check, so it must
|
||||
// never be recommended unnegated for a push to a branch other people also
|
||||
// push to.
|
||||
func delGitForceWithLeaseVsForceTest() testkit.Test {
|
||||
prompt := `Two teammates also push to the shared branch "release/2.4".
|
||||
After an interactive rebase, you rewrote your local commits and must
|
||||
force-push your branch. Which git push flag should you use on this shared
|
||||
branch, and why must a bare "--force" push never be used there instead?`
|
||||
|
||||
evaluator := eval.All(
|
||||
eval.W(eval.ContainsAny("--force-with-lease", "force-with-lease"), 2),
|
||||
eval.W(delNoBareForcePush(), 2),
|
||||
)
|
||||
|
||||
return testkit.Test{
|
||||
ID: "git-force-with-lease-vs-force",
|
||||
Category: "delivery",
|
||||
Subcategory: "git",
|
||||
Description: "Require --force-with-lease over bare --force for a force-push to a branch shared with teammates.",
|
||||
Prompt: prompt,
|
||||
Eval: evaluator,
|
||||
}
|
||||
}
|
||||
|
||||
// delGitHookChoiceTest: pick commit-msg as the hook that enforces a commit
|
||||
// message format at commit time, on the developer's own machine.
|
||||
//
|
||||
// ground truth: commit-msg receives the path to the drafted commit message
|
||||
// file and runs (and can reject it) at the moment of commit, before the
|
||||
// commit is created - exactly when a message-format policy must be
|
||||
// enforced. pre-commit inspects staged file contents, not the message; and
|
||||
// pre-push runs too late, after commits (and any bad messages) already
|
||||
// exist locally.
|
||||
func delGitHookChoiceTest() testkit.Test {
|
||||
prompt := `Your team wants to reject any commit whose message does not
|
||||
follow the Conventional Commits format, at the moment the commit is
|
||||
created, before it ever leaves the developer's machine. Name the single git
|
||||
hook (its exact filename under .git/hooks/) that enforces this. Respond
|
||||
with only the hook name.`
|
||||
|
||||
return testkit.Test{
|
||||
ID: "git-hook-choice",
|
||||
Category: "delivery",
|
||||
Subcategory: "git",
|
||||
Description: "Pick commit-msg as the hook that enforces a commit message format at commit time.",
|
||||
Prompt: prompt,
|
||||
Eval: eval.Equals("commit-msg"),
|
||||
}
|
||||
}
|
||||
|
||||
// delGitDetachedHeadRecoveryTest: recover 3 commits made in detached HEAD
|
||||
// state onto a new branch, in the one order that works.
|
||||
//
|
||||
// ground truth: while still on the detached HEAD (pointing at the 3 new
|
||||
// commits), "git branch <name>" creates a new branch ref at the current
|
||||
// commit without moving HEAD, capturing the work; only then does "git
|
||||
// checkout <name>" move HEAD onto that branch so it stops being detached.
|
||||
// Reversing the order (checkout first) would move HEAD away from the
|
||||
// detached commits before a branch ref exists to save them, losing the
|
||||
// work to garbage collection.
|
||||
func delGitDetachedHeadRecoveryTest() testkit.Test {
|
||||
prompt := `You ran "git checkout <old-commit-sha>" directly (not a
|
||||
branch name), then made 3 new commits while HEAD was detached. You only now
|
||||
realize you need to keep that work, and HEAD is still detached. Give the
|
||||
ordered list of git subcommands (just the subcommand names, e.g. "branch")
|
||||
you must run, in order, to save the 3 commits onto a new branch without
|
||||
losing them. Respond with only a JSON array of subcommand names, e.g.
|
||||
["a","b"].`
|
||||
|
||||
return testkit.Test{
|
||||
ID: "git-detached-head-recovery",
|
||||
Category: "delivery",
|
||||
Subcategory: "git",
|
||||
Description: "Order git branch then git checkout to save 3 detached-HEAD commits onto a new branch.",
|
||||
Prompt: prompt,
|
||||
Eval: eval.JSONStringArrayEquals([]string{"branch", "checkout"}),
|
||||
}
|
||||
}
|
||||
|
||||
// delGitCherryPickVsRevertTest: pick revert for undoing an already-shared
|
||||
// commit without rewriting public history.
|
||||
//
|
||||
// ground truth: "git revert" creates a new commit that undoes the target
|
||||
// commit's changes, leaving all existing history (including the original
|
||||
// commit) intact - safe once a commit has been pulled by other people.
|
||||
// "git cherry-pick" applies an existing commit's changes onto another
|
||||
// branch; it does not undo anything, so it cannot be the answer to "undo
|
||||
// its effect".
|
||||
func delGitCherryPickVsRevertTest() testkit.Test {
|
||||
prompt := `Commit abc1234 was merged into the shared "main" branch three
|
||||
days ago and has already been pulled by other teammates. You must undo its
|
||||
effect without rewriting any commit history that others have already
|
||||
pulled. Which single git operation do you use: cherry-pick or revert?
|
||||
Respond with only one word: cherry-pick or revert.`
|
||||
|
||||
return testkit.Test{
|
||||
ID: "git-cherry-pick-vs-revert",
|
||||
Category: "delivery",
|
||||
Subcategory: "git",
|
||||
Description: "Pick git revert (not cherry-pick) to undo an already-shared commit without rewriting history.",
|
||||
Prompt: prompt,
|
||||
Eval: eval.Equals("revert"),
|
||||
}
|
||||
}
|
||||
|
||||
// delGitignoreFile is the inline .gitignore content for
|
||||
// delGitGitignoreNegationTraceTest.
|
||||
const delGitignoreFile = `*.log
|
||||
build/
|
||||
!build/keep.txt
|
||||
docs/*.tmp
|
||||
!docs/important.tmp`
|
||||
|
||||
// delGitignoreTreeFiles is every file path in the inline tree for
|
||||
// delGitGitignoreNegationTraceTest, in prompt display order.
|
||||
var delGitignoreTreeFiles = []string{
|
||||
"app.go",
|
||||
"debug.log",
|
||||
"build/output.bin",
|
||||
"build/keep.txt",
|
||||
"docs/notes.tmp",
|
||||
"docs/important.tmp",
|
||||
"docs/readme.md",
|
||||
}
|
||||
|
||||
// delGitignoreIgnoredWant is the set of paths from delGitignoreTreeFiles
|
||||
// that git actually treats as ignored under delGitignoreFile.
|
||||
//
|
||||
// ground truth: "*.log" ignores debug.log. "build/" ignores the whole
|
||||
// build/ directory, which ignores build/output.bin AND build/keep.txt -
|
||||
// git's documented behavior is that a negation pattern cannot re-include a
|
||||
// file whose parent directory is itself excluded, since git does not
|
||||
// descend into an excluded directory to evaluate file-level patterns at
|
||||
// all, so "!build/keep.txt" has no effect. "docs/*.tmp" ignores
|
||||
// docs/notes.tmp; "!docs/important.tmp" DOES re-include docs/important.tmp,
|
||||
// because docs/ itself was never excluded as a directory (only files
|
||||
// matching *.tmp within it were), so git does descend into docs/ and the
|
||||
// later, more specific negation pattern wins. app.go and docs/readme.md
|
||||
// match no pattern. delivery_git_test.go verifies this exact set against a
|
||||
// real throwaway git repository (git status --ignored / git check-ignore)
|
||||
// rather than trusting this comment alone, skipping if git is unavailable.
|
||||
var delGitignoreIgnoredWant = []string{
|
||||
"build/keep.txt",
|
||||
"build/output.bin",
|
||||
"debug.log",
|
||||
"docs/notes.tmp",
|
||||
}
|
||||
|
||||
// delGitGitignoreNegationTraceTest: trace which files a .gitignore with
|
||||
// mixed exclude/negate rules actually ignores, including the
|
||||
// excluded-parent-directory negation trap.
|
||||
func delGitGitignoreNegationTraceTest() testkit.Test {
|
||||
prompt := `Here is a repository's .gitignore:
|
||||
|
||||
` + "```gitignore\n" + delGitignoreFile + "\n```" + `
|
||||
|
||||
And here is the full list of files that exist in the working tree:
|
||||
app.go, debug.log, build/output.bin, build/keep.txt, docs/notes.tmp,
|
||||
docs/important.tmp, docs/readme.md
|
||||
|
||||
Which of these files does git actually treat as ignored? Respond with only
|
||||
a JSON array of the ignored file paths.`
|
||||
|
||||
return testkit.Test{
|
||||
ID: "git-gitignore-negation-trace",
|
||||
Category: "delivery",
|
||||
Subcategory: "git",
|
||||
Description: "Trace which files a .gitignore with mixed exclude/negate rules ignores, including the excluded-parent-directory negation trap.",
|
||||
Prompt: prompt,
|
||||
Eval: eval.JSONStringSet(delGitignoreIgnoredWant),
|
||||
}
|
||||
}
|
||||
|
||||
// delGitStashPopConflictTest: state whether a conflicting `git stash pop`
|
||||
// drops the stash entry.
|
||||
//
|
||||
// ground truth: "git stash pop" only drops the stash entry after a clean
|
||||
// apply. On a conflict it leaves the working tree with conflict markers AND
|
||||
// explicitly keeps the stash entry in the stash list (git itself prints
|
||||
// "The stash entry is kept in case you need it again"), so no work is lost;
|
||||
// the user must resolve the conflict and then run "git stash drop"
|
||||
// manually. delivery_git_test.go verifies this against a real throwaway git
|
||||
// repository by staging a genuine stash/commit conflict and checking `git
|
||||
// stash list` afterward, skipping if git is unavailable.
|
||||
func delGitStashPopConflictTest() testkit.Test {
|
||||
prompt := `You run "git stash push" to save uncommitted changes to a
|
||||
file, then make and commit a new change to that same file, and later run
|
||||
"git stash pop". The stashed change conflicts with your new commit on that
|
||||
file. Immediately after "git stash pop" reports the conflict, is the stash
|
||||
entry dropped from the stash list, or kept? Respond with only one word:
|
||||
dropped or kept.`
|
||||
|
||||
return testkit.Test{
|
||||
ID: "git-stash-pop-conflict",
|
||||
Category: "delivery",
|
||||
Subcategory: "git",
|
||||
Description: "State that a conflicting git stash pop keeps the stash entry rather than dropping it.",
|
||||
Prompt: prompt,
|
||||
Eval: eval.Equals("kept"),
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user