mirror of
https://github.com/lukaszraczylo/llm-testbench.git
synced 2026-10-12 00:45:19 +00:00
test(catalog): add delivery category with 30 tests
Add a new delivery category with three subcategories: git, containers, and release-engineering. Each subcategory holds 10 self-contained, deterministically-scored tests, grounded in real git, Docker, and release-engineering behaviour. The git subcategory covers bisect step derivation, rebase versus merge, conventional-commit classification, git worktree, force-with-lease versus bare force, hook choice, detached-HEAD recovery, cherry-pick versus revert, a .gitignore negation trace, and stash-pop conflict behaviour. Two of these tests recompute their ground truth against a real, throwaway git repository created in t.TempDir, and skip if git is not installed. The containers subcategory covers Dockerfile layer-cache busting, ENTRYPOINT and CMD interaction, multi-stage image size, non-root users and capabilities, HEALTHCHECK semantics, .dockerignore effects, COPY versus ADD, build-arg versus runtime env, and layer counting. The release-engineering subcategory covers semver bump derivation from a changelog, the clean-tree requirement, rollback ordering, canary versus blue-green, tag-to-release sequencing, commit-to-changelog mapping, checksum and signing purpose, pipeline stage ordering, pin versus range dependency policy, and hotfix flow ordering. A shared negation-aware evaluator helper, local to this category's files, scores a forbidden phrase only when it appears unnegated, so a response correctly warning against bare --force or a silent tracked-file mutation still scores full credit. registerDeliveryTests wires the three subcategories together and is exercised only by a new wiring test in this worktree, since catalog.go and catalog_test.go stay untouched until the orchestrator integrates this category centrally.
This commit is contained in:
@@ -0,0 +1,16 @@
|
||||
package tests
|
||||
|
||||
import "github.com/lukaszraczylo/llm-testbench/internal/testkit"
|
||||
|
||||
// registerDeliveryTests registers every delivery-category test. Each
|
||||
// subcategory - git, containers, and release-engineering - has its own
|
||||
// register function and source file (delivery_git.go,
|
||||
// delivery_containers.go, delivery_release.go) to keep any one file from
|
||||
// growing past a few hundred lines, mirroring the agents category's split
|
||||
// (agents.go / agents_toolrouting.go / agents_planning.go /
|
||||
// agents_delegation.go).
|
||||
func registerDeliveryTests(r *testkit.Registry) {
|
||||
registerDeliveryGitTests(r)
|
||||
registerDeliveryContainersTests(r)
|
||||
registerDeliveryReleaseTests(r)
|
||||
}
|
||||
@@ -0,0 +1,371 @@
|
||||
package tests
|
||||
|
||||
import (
|
||||
"github.com/lukaszraczylo/llm-testbench/internal/eval"
|
||||
"github.com/lukaszraczylo/llm-testbench/internal/testkit"
|
||||
)
|
||||
|
||||
func registerDeliveryContainersTests(r *testkit.Registry) {
|
||||
r.Register(delDockerLayerCacheBustTest())
|
||||
r.Register(delDockerEntrypointCmdTraceTest())
|
||||
r.Register(delDockerMultistageSizeBenefitTest())
|
||||
r.Register(delDockerImageSizeMathTest())
|
||||
r.Register(delDockerNonrootUserCapsTest())
|
||||
r.Register(delDockerHealthcheckSemanticsTest())
|
||||
r.Register(delDockerDockerignoreEffectTest())
|
||||
r.Register(delDockerCopyVsAddTest())
|
||||
r.Register(delDockerBuildArgVsEnvTest())
|
||||
r.Register(delDockerLayerCountTest())
|
||||
}
|
||||
|
||||
// delDockerLayerCacheBustDockerfile is the inline numbered Dockerfile for
|
||||
// delDockerLayerCacheBustTest.
|
||||
const delDockerLayerCacheBustDockerfile = `1: FROM golang:1.25
|
||||
2: WORKDIR /app
|
||||
3: COPY . .
|
||||
4: RUN go mod download
|
||||
5: RUN go build -o app .
|
||||
6: CMD ["./app"]`
|
||||
|
||||
// delDockerLayerCacheBustTest: spot the line whose placement busts the
|
||||
// Docker layer cache on every source change.
|
||||
//
|
||||
// ground truth: line 3 (COPY . .) copies the entire source tree, including
|
||||
// files that change on nearly every commit, before line 4 (go mod
|
||||
// download). Docker invalidates a layer's cache, and every layer after it,
|
||||
// as soon as any input to that layer changes - so any single source-file
|
||||
// edit busts the cache starting at line 3, forcing "go mod download" to
|
||||
// rerun on every build even when go.mod/go.sum never changed. Copying only
|
||||
// go.mod/go.sum first, downloading dependencies, and copying the rest of
|
||||
// the source afterward would let dependency downloads stay cached across
|
||||
// source-only changes.
|
||||
func delDockerLayerCacheBustTest() testkit.Test {
|
||||
prompt := `Here is a Dockerfile, with line numbers prefixed:
|
||||
|
||||
` + delDockerLayerCacheBustDockerfile + `
|
||||
|
||||
This Dockerfile reruns "go mod download" on every single build, even when
|
||||
go.mod and go.sum have not changed. Which line number's placement busts the
|
||||
Docker layer cache on every source-code change, forcing "go mod download"
|
||||
to rerun unnecessarily? Respond with only a JSON object: {"line": <line
|
||||
number as an integer>}`
|
||||
|
||||
return testkit.Test{
|
||||
ID: "docker-layer-cache-bust",
|
||||
Category: "delivery",
|
||||
Subcategory: "containers",
|
||||
Description: "Spot the COPY . . line placed before dependency download that busts the Docker layer cache on every source change.",
|
||||
Prompt: prompt,
|
||||
Eval: eval.JSONField("line", 3),
|
||||
}
|
||||
}
|
||||
|
||||
// delDockerEntrypointCmdTest: trace the exact resulting startup command
|
||||
// when both ENTRYPOINT and CMD are set in exec form.
|
||||
//
|
||||
// ground truth: with both instructions in exec (JSON array) form, CMD's
|
||||
// array is not run on its own - it supplies the default arguments appended
|
||||
// after ENTRYPOINT's array, used only when "docker run" is given no
|
||||
// trailing command. Run with no extra arguments, the process actually
|
||||
// executed is "python3 app.py --port 8080".
|
||||
func delDockerEntrypointCmdTraceTest() testkit.Test {
|
||||
prompt := `A Dockerfile contains exactly these two instructions:
|
||||
|
||||
` + "```dockerfile\n" + `ENTRYPOINT ["python3", "app.py"]
|
||||
CMD ["--port", "8080"]
|
||||
` + "```" + `
|
||||
|
||||
If the container is run with no extra arguments ("docker run image"), what
|
||||
is the exact resulting command line executed inside the container? Respond
|
||||
with only the exact command line, nothing else.`
|
||||
|
||||
return testkit.Test{
|
||||
ID: "docker-entrypoint-cmd-trace",
|
||||
Category: "delivery",
|
||||
Subcategory: "containers",
|
||||
Description: "Trace ENTRYPOINT+CMD exec-form interaction to the exact resulting startup command line.",
|
||||
Prompt: prompt,
|
||||
Eval: eval.Equals("python3 app.py --port 8080"),
|
||||
}
|
||||
}
|
||||
|
||||
// delDockerMultistageSizeBenefitTest: explain how a multi-stage build
|
||||
// shrinks the final image and name a minimal final-stage base.
|
||||
func delDockerMultistageSizeBenefitTest() testkit.Test {
|
||||
prompt := `A Go service's Dockerfile currently uses a single
|
||||
"golang:1.25" base image (full Go toolchain, apt package caches, and build
|
||||
tools) both to compile the binary AND to run it, producing a final image
|
||||
over 900MB. Explain how switching to a multi-stage build reduces the final
|
||||
image size, and name the kind of minimal base image you would copy the
|
||||
compiled binary into for the final stage.`
|
||||
|
||||
evaluator := eval.All(
|
||||
eval.W(eval.ContainsAll("multi-stage"), 2),
|
||||
eval.W(eval.ContainsAny("scratch", "distroless", "alpine"), 2),
|
||||
)
|
||||
|
||||
return testkit.Test{
|
||||
ID: "docker-multistage-size-benefit",
|
||||
Category: "delivery",
|
||||
Subcategory: "containers",
|
||||
Description: "Explain the multi-stage build size benefit and name a minimal final-stage base (scratch/distroless/alpine).",
|
||||
Prompt: prompt,
|
||||
Eval: evaluator,
|
||||
}
|
||||
}
|
||||
|
||||
// delDockerImageSizeMathWant is the total size, in MB, of only the final
|
||||
// stage's layers in delDockerImageSizeMathTest.
|
||||
//
|
||||
// ground truth: only layers in the FINAL stage of a multi-stage build ship
|
||||
// in the resulting image. The build stage's numbers (72 + 148 + 4 + 310)
|
||||
// are discarded distractors; the final stage is alpine base (8MB) plus the
|
||||
// compiled-binary COPY layer (22MB): 8 + 22 = 30.
|
||||
var delDockerImageSizeMathWant = 8 + 22
|
||||
|
||||
// delDockerImageSizeMathTest: sum only the final stage's layer sizes in a
|
||||
// multi-stage build, ignoring the discarded build-stage layers.
|
||||
func delDockerImageSizeMathTest() testkit.Test {
|
||||
prompt := `A Docker image is built with a multi-stage build:
|
||||
|
||||
Build stage (discarded, not part of the final image): base image 72MB, an
|
||||
apt-get install layer adds 148MB, a COPY of application source adds 4MB,
|
||||
and a RUN go build layer adds 310MB.
|
||||
|
||||
Final stage (this is what actually ships): base image alpine at 8MB, plus a
|
||||
COPY of the compiled binary from the build stage adds 22MB.
|
||||
|
||||
What is the total size, in MB, of the final shipped image? Respond with
|
||||
only the number.`
|
||||
|
||||
return testkit.Test{
|
||||
ID: "docker-image-size-math",
|
||||
Category: "delivery",
|
||||
Subcategory: "containers",
|
||||
Description: "Sum only the final stage's layer sizes (8+22=30MB) in a multi-stage build, ignoring discarded build-stage layers.",
|
||||
Prompt: prompt,
|
||||
Eval: eval.Numeric(eval.ExtractLastNumber[int], delDockerImageSizeMathWant, 0),
|
||||
}
|
||||
}
|
||||
|
||||
// delDockerNonrootUserCapsTest: require a non-root USER and a fully-dropped
|
||||
// capability set for a container that needs no privileged syscalls.
|
||||
//
|
||||
// ground truth: adding a USER instruction (a numeric non-root UID, or a
|
||||
// user created via adduser/useradd) stops the container's process from
|
||||
// running as root. Binding to a port above 1024 needs no special Linux
|
||||
// capability (only binding below 1024 needs CAP_NET_BIND_SERVICE), so with
|
||||
// no other privileged operation required, every capability should be
|
||||
// dropped at runtime (--cap-drop=ALL, with no --cap-add).
|
||||
func delDockerNonrootUserCapsTest() testkit.Test {
|
||||
prompt := `This Dockerfile currently has no USER instruction, so its
|
||||
process runs as root inside the container. The entrypoint binary only needs
|
||||
to bind to a port above 1024 - nothing else privileged. Give the Dockerfile
|
||||
instruction that makes the container run as a non-root user, and name the
|
||||
Linux capability set it should end up running with at runtime, given it
|
||||
needs no special privileges.`
|
||||
|
||||
evaluator := eval.All(
|
||||
eval.W(eval.ContainsAll("USER"), 2),
|
||||
eval.W(eval.ContainsAny("cap-drop=all", "cap-drop all", "drop all capabilities", "capdrop=all", "drop all"), 2),
|
||||
)
|
||||
|
||||
return testkit.Test{
|
||||
ID: "docker-nonroot-user-caps",
|
||||
Category: "delivery",
|
||||
Subcategory: "containers",
|
||||
Description: "Require a USER instruction plus a fully-dropped capability set for a container needing no privileged syscalls.",
|
||||
Prompt: prompt,
|
||||
Eval: evaluator,
|
||||
}
|
||||
}
|
||||
|
||||
// delDockerHealthcheckRetriesWant is the number of consecutive failed
|
||||
// probes before Docker marks the container unhealthy in
|
||||
// delDockerHealthcheckSemanticsTest, taken directly from the instruction's
|
||||
// own --retries=3 value.
|
||||
var delDockerHealthcheckRetriesWant = 3
|
||||
|
||||
// delDockerHealthcheckSemanticsTest: read the HEALTHCHECK retries value
|
||||
// correctly rather than the interval or timeout values.
|
||||
func delDockerHealthcheckSemanticsTest() testkit.Test {
|
||||
prompt := `Given this Dockerfile instruction:
|
||||
|
||||
` + "```dockerfile\n" + `HEALTHCHECK --interval=30s --timeout=3s --retries=3 CMD curl -f http://localhost:8080/health || exit 1
|
||||
` + "```" + `
|
||||
|
||||
Assuming every probe run so far has failed, after how many consecutive
|
||||
FAILED probes does Docker mark the container's health status as
|
||||
"unhealthy"? Respond with only the integer.`
|
||||
|
||||
return testkit.Test{
|
||||
ID: "docker-healthcheck-semantics",
|
||||
Category: "delivery",
|
||||
Subcategory: "containers",
|
||||
Description: "Read a HEALTHCHECK instruction's --retries value as the consecutive-failure count before unhealthy status.",
|
||||
Prompt: prompt,
|
||||
Eval: eval.Numeric(eval.ExtractLastNumber[int], delDockerHealthcheckRetriesWant, 0),
|
||||
}
|
||||
}
|
||||
|
||||
// delDockerignoreFile is the inline .dockerignore content for
|
||||
// delDockerDockerignoreEffectTest. Deliberately uses no negation patterns
|
||||
// (unlike delGitignoreFile's git-specific negation trap): Docker's own
|
||||
// negation-after-directory-exclusion behavior differs between the classic
|
||||
// builder and BuildKit and is not as uniformly documented as git's, so
|
||||
// relying on it here would not be a deterministic, single-answer test.
|
||||
const delDockerignoreFile = `node_modules
|
||||
*.log
|
||||
.git`
|
||||
|
||||
// delDockerDockerignoreEffectWant is the sorted set of files from
|
||||
// delDockerDockerignoreEffectTest's inline file list that are NOT excluded
|
||||
// by delDockerignoreFile and so are sent to the Docker build context.
|
||||
//
|
||||
// ground truth: "node_modules" excludes node_modules/pkg/index.js. "*.log"
|
||||
// excludes debug.log. ".git" excludes .git/config. app.js and README.md
|
||||
// match no pattern and are included.
|
||||
var delDockerDockerignoreEffectWant = []string{"README.md", "app.js"}
|
||||
|
||||
// delDockerDockerignoreEffectTest: identify which files are NOT excluded by
|
||||
// a .dockerignore and so are sent to the Docker daemon's build context.
|
||||
func delDockerDockerignoreEffectTest() testkit.Test {
|
||||
prompt := `Here is a repository's .dockerignore:
|
||||
|
||||
` + "```\n" + delDockerignoreFile + "\n```" + `
|
||||
|
||||
And here is the full list of files that exist in the build directory:
|
||||
app.js, node_modules/pkg/index.js, debug.log, README.md, .git/config
|
||||
|
||||
Which of these files WILL be sent to the Docker daemon as part of the build
|
||||
context (i.e. are NOT excluded by .dockerignore)? Respond with only a JSON
|
||||
array of the included file paths.`
|
||||
|
||||
return testkit.Test{
|
||||
ID: "docker-dockerignore-effect",
|
||||
Category: "delivery",
|
||||
Subcategory: "containers",
|
||||
Description: "Identify which files a .dockerignore leaves in the build context that gets sent to the Docker daemon.",
|
||||
Prompt: prompt,
|
||||
Eval: eval.JSONStringSet(delDockerDockerignoreEffectWant),
|
||||
}
|
||||
}
|
||||
|
||||
// delDockerCopyVsAddTest: pick COPY for plain local files, ADD for the one
|
||||
// feature COPY lacks - local tar auto-extraction.
|
||||
//
|
||||
// ground truth: COPY is Docker's own recommended default - transparent,
|
||||
// predictable, does only a file copy. ADD additionally auto-extracts a
|
||||
// local tar/gzip archive into the destination directory as part of the same
|
||||
// instruction (and can also fetch remote URLs, which Docker's own best
|
||||
// practices discourage) - the tar-extraction behavior is the one thing ADD
|
||||
// does that COPY cannot, making it the correct choice specifically for
|
||||
// scenario_b.
|
||||
func delDockerCopyVsAddTest() testkit.Test {
|
||||
prompt := `For each of these two situations, should you use "COPY" or
|
||||
"ADD" in the Dockerfile?
|
||||
|
||||
scenario_a: "Copy a local directory of static assets into the image, with
|
||||
no extraction or URL-fetching behavior needed - just the files as they are
|
||||
on disk."
|
||||
scenario_b: "Add a local .tar.gz archive to the image and have it
|
||||
automatically extracted into the destination directory as part of the same
|
||||
instruction."
|
||||
|
||||
Respond with only a JSON object:
|
||||
{"scenario_a":"COPY"|"ADD","scenario_b":"COPY"|"ADD"}`
|
||||
|
||||
evaluator := eval.Mean(
|
||||
eval.JSONField("scenario_a", "COPY"),
|
||||
eval.JSONField("scenario_b", "ADD"),
|
||||
)
|
||||
|
||||
return testkit.Test{
|
||||
ID: "docker-copy-vs-add",
|
||||
Category: "delivery",
|
||||
Subcategory: "containers",
|
||||
Description: "Choose COPY for a plain file copy and ADD for its unique local-tar-auto-extraction behavior.",
|
||||
Prompt: prompt,
|
||||
Eval: evaluator,
|
||||
}
|
||||
}
|
||||
|
||||
// delDockerBuildArgVsEnvTest: pick ARG for a build-time-only value and ENV
|
||||
// for a value the running container must expose.
|
||||
//
|
||||
// ground truth: ARG values exist only during the build and are not present
|
||||
// in the final image's runtime environment or in "docker inspect"'s Env
|
||||
// list, matching scenario_a's requirement. ENV values are baked into the
|
||||
// image's runtime config and are what "docker run -e" overrides and what a
|
||||
// running process reads via getenv, matching scenario_b.
|
||||
func delDockerBuildArgVsEnvTest() testkit.Test {
|
||||
prompt := `For each of these two situations, should the value be set
|
||||
with "ARG" or "ENV" in the Dockerfile?
|
||||
|
||||
scenario_a: "A value is needed only at image-build time (for example,
|
||||
selecting which internal build number to embed into a version label), and
|
||||
must NOT be present in the final running container's environment or in
|
||||
'docker inspect' output."
|
||||
scenario_b: "A value must be configurable at container start time (for
|
||||
example by an operator running 'docker run -e ...'), without rebuilding the
|
||||
image, and must be readable by the running process."
|
||||
|
||||
Respond with only a JSON object:
|
||||
{"scenario_a":"ARG"|"ENV","scenario_b":"ARG"|"ENV"}`
|
||||
|
||||
evaluator := eval.Mean(
|
||||
eval.JSONField("scenario_a", "ARG"),
|
||||
eval.JSONField("scenario_b", "ENV"),
|
||||
)
|
||||
|
||||
return testkit.Test{
|
||||
ID: "docker-build-arg-vs-env",
|
||||
Category: "delivery",
|
||||
Subcategory: "containers",
|
||||
Description: "Choose ARG for a build-time-only value and ENV for a value the running container must expose.",
|
||||
Prompt: prompt,
|
||||
Eval: evaluator,
|
||||
}
|
||||
}
|
||||
|
||||
// delDockerLayerCountDockerfile is the inline numbered Dockerfile for
|
||||
// delDockerLayerCountTest.
|
||||
const delDockerLayerCountDockerfile = `1: FROM alpine:3.20
|
||||
2: WORKDIR /app
|
||||
3: COPY app /app/app
|
||||
4: RUN chmod +x /app/app
|
||||
5: ENV PORT=8080
|
||||
6: EXPOSE 8080
|
||||
7: USER 1000
|
||||
8: ENTRYPOINT ["/app/app"]`
|
||||
|
||||
// delDockerLayerCountWant is the number of new filesystem layers this
|
||||
// Dockerfile adds on top of its base image.
|
||||
//
|
||||
// ground truth: only RUN, COPY, and ADD instructions create a new
|
||||
// filesystem layer; WORKDIR, ENV, EXPOSE, USER, ENTRYPOINT, CMD, and LABEL
|
||||
// only update image metadata/config and add no layer. This Dockerfile has
|
||||
// exactly one COPY (line 3) and one RUN (line 4): 2 new layers.
|
||||
var delDockerLayerCountWant = 2
|
||||
|
||||
// delDockerLayerCountTest: count only the new filesystem layers a
|
||||
// Dockerfile adds on top of its base image, distinguishing layer-creating
|
||||
// instructions (RUN/COPY/ADD) from metadata-only ones.
|
||||
func delDockerLayerCountTest() testkit.Test {
|
||||
prompt := `Here is a Dockerfile, with line numbers prefixed:
|
||||
|
||||
` + delDockerLayerCountDockerfile + `
|
||||
|
||||
Not counting whatever layers already exist inside the alpine:3.20 base
|
||||
image itself, how many NEW filesystem layers does building this Dockerfile
|
||||
add on top of that base? Respond with only the number.`
|
||||
|
||||
return testkit.Test{
|
||||
ID: "docker-layer-count",
|
||||
Category: "delivery",
|
||||
Subcategory: "containers",
|
||||
Description: "Count only the layer-creating instructions (RUN/COPY/ADD) in a Dockerfile, excluding metadata-only ones.",
|
||||
Prompt: prompt,
|
||||
Eval: eval.Numeric(eval.ExtractLastNumber[int], delDockerLayerCountWant, 0),
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,294 @@
|
||||
package tests
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestDelDockerLayerCacheBustTest_Eval(t *testing.T) {
|
||||
tc := delDockerLayerCacheBustTest()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{name: "bare object", response: `{"line": 3}`, want: 1},
|
||||
{name: "fenced with prose", response: "The culprit is:\n```json\n{\"line\": 3}\n```", want: 1},
|
||||
{name: "extra whitespace inside the object", response: `{ "line" : 3 }`, want: 1},
|
||||
{name: "wrong: blames the FROM line", response: `{"line": 1}`, want: 0},
|
||||
{name: "wrong: blames go build instead", response: `{"line": 5}`, want: 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDelDockerEntrypointCmdTraceTest_Eval(t *testing.T) {
|
||||
tc := delDockerEntrypointCmdTraceTest()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{name: "exact", response: "python3 app.py --port 8080", want: 1},
|
||||
{name: "different case", response: "Python3 App.py --Port 8080", want: 1},
|
||||
{name: "surrounding whitespace", response: " python3 app.py --port 8080\n", want: 1},
|
||||
{name: "wrong: only CMD's args", response: "--port 8080", want: 0},
|
||||
{name: "wrong: only ENTRYPOINT, drops CMD's default args", response: "python3 app.py", want: 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDelDockerMultistageSizeBenefitTest_Eval(t *testing.T) {
|
||||
tc := delDockerMultistageSizeBenefitTest()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{
|
||||
name: "correct: multi-stage + scratch",
|
||||
response: "Use a multi-stage build: compile in the golang stage, then copy only the binary into a final scratch image with no toolchain at all.",
|
||||
want: 1,
|
||||
},
|
||||
{
|
||||
name: "correct: multi-stage + distroless",
|
||||
response: "A multi-stage build lets you copy just the compiled binary into a distroless final image, dropping the entire build toolchain layer.",
|
||||
want: 1,
|
||||
},
|
||||
{
|
||||
name: "correct: multi-stage + alpine",
|
||||
response: "With a multi-stage build the final stage can be a minimal alpine base holding only the binary, instead of the full golang image.",
|
||||
want: 1,
|
||||
},
|
||||
{
|
||||
name: "wrong: neither term present",
|
||||
response: "Just delete unused files with a RUN rm -rf step at the end of the same image.",
|
||||
want: 0,
|
||||
},
|
||||
{
|
||||
name: "wrong: names the base but never says multi-stage",
|
||||
response: "Switch the final image to alpine instead of golang.",
|
||||
want: 0.5,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDelDockerImageSizeMathTest_Eval(t *testing.T) {
|
||||
tc := delDockerImageSizeMathTest()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{name: "bare number", response: "30", want: 1},
|
||||
{name: "sentence form", response: "The final image is 30 MB.", want: 1},
|
||||
{name: "with units spelled out", response: "8 + 22 = 30 megabytes.", want: 1},
|
||||
{name: "wrong: sums every layer including the discarded build stage", response: "534", want: 0},
|
||||
{name: "wrong: only the alpine base, forgets the binary layer", response: "8", want: 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDelDockerNonrootUserCapsTest_Eval(t *testing.T) {
|
||||
tc := delDockerNonrootUserCapsTest()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{
|
||||
name: "correct: USER + cap-drop=all",
|
||||
response: "Add `USER 1000` to the Dockerfile, and run the container with --cap-drop=ALL since it needs no special capabilities.",
|
||||
want: 1,
|
||||
},
|
||||
{
|
||||
name: "correct: USER + drop all capabilities phrasing",
|
||||
response: "Create a non-root user with adduser and add a USER instruction; at runtime, drop all capabilities.",
|
||||
want: 1,
|
||||
},
|
||||
{
|
||||
name: "correct: USER + cap-drop all spaced form",
|
||||
response: "Set USER app in the image, and pass --cap-drop all at runtime since no privileged syscalls are needed.",
|
||||
want: 1,
|
||||
},
|
||||
{
|
||||
name: "wrong: neither addressed",
|
||||
response: "This is fine as-is, since the process only listens on a port.",
|
||||
want: 0,
|
||||
},
|
||||
{
|
||||
name: "wrong: only addresses the user, not capabilities",
|
||||
response: "Add a USER instruction pointing at a non-root UID.",
|
||||
want: 0.5,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDelDockerHealthcheckSemanticsTest_Eval(t *testing.T) {
|
||||
tc := delDockerHealthcheckSemanticsTest()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{name: "bare number", response: "3", want: 1},
|
||||
{name: "sentence form", response: "After 3 consecutive failures.", want: 1},
|
||||
{name: "labelled form", response: "retries: 3", want: 1},
|
||||
{name: "wrong: confuses timeout with retries", response: "3s", want: 0}, // "3s" glues the 3 to a unit letter, so extraction fails and scores 0
|
||||
{name: "wrong: confuses interval with retries", response: "30", want: 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDelDockerDockerignoreEffectTest_Eval(t *testing.T) {
|
||||
tc := delDockerDockerignoreEffectTest()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{name: "exact correct set", response: `["app.js","README.md"]`, want: 1},
|
||||
{name: "correct set, different order", response: `["README.md","app.js"]`, want: 1},
|
||||
{name: "correct set, fenced", response: "```json\n[\"app.js\",\"README.md\"]\n```", want: 1},
|
||||
{
|
||||
name: "wrong: includes an excluded file",
|
||||
response: `["app.js","README.md","debug.log"]`,
|
||||
want: 2.0 / 3.0,
|
||||
},
|
||||
{
|
||||
name: "wrong: also excludes a file that should be included",
|
||||
response: `["app.js"]`,
|
||||
want: 0.5,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDelDockerCopyVsAddTest_Eval(t *testing.T) {
|
||||
tc := delDockerCopyVsAddTest()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{name: "all correct", response: `{"scenario_a":"COPY","scenario_b":"ADD"}`, want: 1},
|
||||
{name: "all correct fenced", response: "```json\n{\"scenario_a\":\"COPY\",\"scenario_b\":\"ADD\"}\n```", want: 1},
|
||||
{name: "all correct lowercase", response: `{"scenario_a":"copy","scenario_b":"add"}`, want: 1},
|
||||
{name: "scenario_a wrong", response: `{"scenario_a":"ADD","scenario_b":"ADD"}`, want: 0.5},
|
||||
{name: "both swapped", response: `{"scenario_a":"ADD","scenario_b":"COPY"}`, want: 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDelDockerBuildArgVsEnvTest_Eval(t *testing.T) {
|
||||
tc := delDockerBuildArgVsEnvTest()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{name: "all correct", response: `{"scenario_a":"ARG","scenario_b":"ENV"}`, want: 1},
|
||||
{name: "all correct fenced", response: "```json\n{\"scenario_a\":\"ARG\",\"scenario_b\":\"ENV\"}\n```", want: 1},
|
||||
{name: "all correct lowercase", response: `{"scenario_a":"arg","scenario_b":"env"}`, want: 1},
|
||||
{name: "scenario_b wrong", response: `{"scenario_a":"ARG","scenario_b":"ARG"}`, want: 0.5},
|
||||
{name: "both swapped", response: `{"scenario_a":"ENV","scenario_b":"ARG"}`, want: 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDelDockerLayerCountTest_Eval(t *testing.T) {
|
||||
tc := delDockerLayerCountTest()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{name: "bare number", response: "2", want: 1},
|
||||
{name: "sentence form", response: "This Dockerfile adds 2 new layers.", want: 1},
|
||||
{name: "explains then answers", response: "Only COPY and RUN create layers here, so the answer is 2.", want: 1},
|
||||
{name: "wrong: counts every instruction", response: "8", want: 0},
|
||||
{name: "wrong: forgets the RUN layer", response: "1", want: 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,476 @@
|
||||
package tests
|
||||
|
||||
import (
|
||||
"context"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/lukaszraczylo/llm-testbench/internal/eval"
|
||||
"github.com/lukaszraczylo/llm-testbench/internal/testkit"
|
||||
)
|
||||
|
||||
func registerDeliveryGitTests(r *testkit.Registry) {
|
||||
r.Register(delGitBisectStepsTest())
|
||||
r.Register(delGitRebaseVsMergeTest())
|
||||
r.Register(delGitConventionalCommitClassifyTest())
|
||||
r.Register(delGitWorktreeUseCaseTest())
|
||||
r.Register(delGitForceWithLeaseVsForceTest())
|
||||
r.Register(delGitHookChoiceTest())
|
||||
r.Register(delGitDetachedHeadRecoveryTest())
|
||||
r.Register(delGitCherryPickVsRevertTest())
|
||||
r.Register(delGitGitignoreNegationTraceTest())
|
||||
r.Register(delGitStashPopConflictTest())
|
||||
}
|
||||
|
||||
// --- negation-aware helpers, shared across delivery_git.go and
|
||||
// delivery_release.go (same package). Duplicated in spirit from
|
||||
// operations/kubernetes's noLiveKubectlMutation/negationCuePattern rather
|
||||
// than imported from kubernetes.go, so this category's files stay
|
||||
// self-contained: round-2 authors work in isolated worktrees on
|
||||
// databases/security/delivery/ai in parallel, and depending on another
|
||||
// category's unexported symbols would create a merge-time coupling this
|
||||
// worktree cannot see or coordinate on. ---
|
||||
|
||||
// delNegationCuePattern matches a word that turns a mention of a forbidden
|
||||
// phrase into a warning against doing it, rather than an instruction to do
|
||||
// it (e.g. "never use bare --force", "must not silently modify").
|
||||
var delNegationCuePattern = regexp.MustCompile(`(?i)\b(don'?t|do not|never|avoid|instead of|not|cannot|can'?t|rather than|without|no need|must not|should not|shouldn'?t)\b`)
|
||||
|
||||
// delNegationWindow is how many characters before a forbidden-phrase match
|
||||
// are searched for a negation cue.
|
||||
const delNegationWindow = 60
|
||||
|
||||
// delNegationWindowStart returns the earliest byte offset in response to
|
||||
// search for a negation cue before a forbidden-phrase match starting at
|
||||
// start. The window is the current line, extended back to the start of the
|
||||
// immediately preceding line when that line is non-empty, so a hard-wrapped
|
||||
// sentence whose negation cue landed on the line above still counts. Either
|
||||
// way the window never reaches more than delNegationWindow characters
|
||||
// before start.
|
||||
func delNegationWindowStart(response string, start int) int {
|
||||
curLineStart := strings.LastIndexByte(response[:start], '\n') + 1
|
||||
|
||||
windowFloor := curLineStart
|
||||
if curLineStart > 0 {
|
||||
prevLineEnd := curLineStart - 1
|
||||
prevLineStart := strings.LastIndexByte(response[:prevLineEnd], '\n') + 1
|
||||
if strings.TrimSpace(response[prevLineStart:prevLineEnd]) != "" {
|
||||
windowFloor = prevLineStart
|
||||
}
|
||||
}
|
||||
|
||||
return max(start-delNegationWindow, windowFloor)
|
||||
}
|
||||
|
||||
// delNoUnnegatedMention returns an Evaluator scoring full credit unless
|
||||
// forbidden matches response at some position with no negation cue in the
|
||||
// preceding window - i.e. the response recommends the forbidden thing
|
||||
// outright rather than warning against it. This is deliberately not
|
||||
// eval.NotContains, which would also zero out the best possible answer (one
|
||||
// that correctly explains why NOT to do the forbidden thing).
|
||||
func delNoUnnegatedMention(forbidden *regexp.Regexp, safeDetail string) eval.Evaluator {
|
||||
return eval.EvaluatorFunc(func(_ context.Context, response string) eval.Score {
|
||||
matches := forbidden.FindAllStringIndex(response, -1)
|
||||
if len(matches) == 0 {
|
||||
return eval.Score{Value: 1, Detail: "no forbidden mention: " + safeDetail}
|
||||
}
|
||||
for _, loc := range matches {
|
||||
start := loc[0]
|
||||
windowStart := delNegationWindowStart(response, start)
|
||||
if !delNegationCuePattern.MatchString(response[windowStart:start]) {
|
||||
return eval.Score{Value: 0, Detail: "unnegated forbidden mention"}
|
||||
}
|
||||
}
|
||||
return eval.Score{Value: 1, Detail: "every forbidden mention is negated"}
|
||||
})
|
||||
}
|
||||
|
||||
// delForceFlagPattern matches every "--force" mention. delForceWithLeaseSuffixPattern
|
||||
// checks whether one of those matches is immediately followed by
|
||||
// "-with-lease" - the safe form, exempt from the bare-force check entirely
|
||||
// rather than needing a negation cue.
|
||||
var delForceFlagPattern = regexp.MustCompile(`(?i)--force\b`)
|
||||
var delForceWithLeaseSuffixPattern = regexp.MustCompile(`(?i)^-with-lease`)
|
||||
|
||||
// delNoBareForcePush scores full credit unless the response recommends a
|
||||
// bare "--force" push (not "--force-with-lease") with no negation cue
|
||||
// nearby. A response that only ever mentions "--force-with-lease" never
|
||||
// matches delForceFlagPattern's bare form (the suffix check exempts it), so
|
||||
// it always scores full credit; a response that recommends bare --force
|
||||
// outright scores zero.
|
||||
func delNoBareForcePush() eval.Evaluator {
|
||||
return eval.EvaluatorFunc(func(_ context.Context, response string) eval.Score {
|
||||
matches := delForceFlagPattern.FindAllStringIndex(response, -1)
|
||||
if len(matches) == 0 {
|
||||
return eval.Score{Value: 1, Detail: "no --force mention"}
|
||||
}
|
||||
for _, loc := range matches {
|
||||
start, end := loc[0], loc[1]
|
||||
if delForceWithLeaseSuffixPattern.MatchString(response[end:]) {
|
||||
continue // "--force-with-lease": the safe form, not bare force
|
||||
}
|
||||
windowStart := delNegationWindowStart(response, start)
|
||||
if !delNegationCuePattern.MatchString(response[windowStart:start]) {
|
||||
return eval.Score{Value: 0, Detail: "unnegated bare --force mention"}
|
||||
}
|
||||
}
|
||||
return eval.Score{Value: 1, Detail: "every bare --force mention is negated or absent"}
|
||||
})
|
||||
}
|
||||
|
||||
// delGitBisectCommitCount is the number of candidate commits between the
|
||||
// last known-good commit and the current bad HEAD in delGitBisectStepsTest.
|
||||
const delGitBisectCommitCount = 137
|
||||
|
||||
// delGitBisectStepsWant is the worst-case number of `git bisect` steps
|
||||
// needed to find the single first-bad commit among
|
||||
// delGitBisectCommitCount candidates by binary search.
|
||||
//
|
||||
// ground truth: `git bisect` performs a binary search over the candidate
|
||||
// commit range, halving the remaining range on each step until exactly one
|
||||
// commit remains. The worst-case step count for N candidates is
|
||||
// ceil(log2(N)): 2^7 = 128 < 137 <= 256 = 2^8, so ceil(log2(137)) = 8.
|
||||
// delivery_git_test.go recomputes this with math.Ceil(math.Log2(137))
|
||||
// rather than trusting the hardcoded literal alone.
|
||||
var delGitBisectStepsWant = 8
|
||||
|
||||
// delGitBisectStepsTest: derive the worst-case number of `git bisect`
|
||||
// binary-search steps needed to find one bad commit among N candidates.
|
||||
func delGitBisectStepsTest() testkit.Test {
|
||||
prompt := `A regression was introduced somewhere among the last 137
|
||||
commits on main: the commit at the tip of main is bad, and the commit 137
|
||||
commits back (the last tagged release) is known good. Using "git bisect"
|
||||
(binary search over the candidate commits, testing one commit per step),
|
||||
what is the maximum number of "git bisect" steps needed to find the exact
|
||||
first commit that introduced the regression? Respond with only the integer
|
||||
number of steps.`
|
||||
|
||||
return testkit.Test{
|
||||
ID: "git-bisect-steps",
|
||||
Category: "delivery",
|
||||
Subcategory: "git",
|
||||
Description: "Derive the worst-case git bisect step count (ceil(log2(N))) for 137 candidate commits.",
|
||||
Prompt: prompt,
|
||||
Eval: eval.Numeric(eval.ExtractLastNumber[int], delGitBisectStepsWant, 0),
|
||||
}
|
||||
}
|
||||
|
||||
// delGitRebaseVsMergeTest: pick rebase or merge for a solo short-lived
|
||||
// branch versus a shared long-lived branch.
|
||||
//
|
||||
// ground truth: rebasing a branch nobody else has pulled is safe and
|
||||
// produces a clean, linear, single-author history before review - exactly
|
||||
// what interactive rebase is for. Rewriting history on a branch other
|
||||
// people have already pulled and built work on top of forces everyone else
|
||||
// to reconcile diverged history; merging (which adds a new commit rather
|
||||
// than rewriting existing ones) is the only one of the two that does not
|
||||
// rewrite commits other people already depend on.
|
||||
func delGitRebaseVsMergeTest() testkit.Test {
|
||||
prompt := `For each of these two situations, should you use "git rebase"
|
||||
or "git merge" to bring in the latest changes?
|
||||
|
||||
scenario_a: "You are the sole author of a short-lived feature branch that
|
||||
has never been pushed anywhere, and you want a clean, linear, single-author
|
||||
history before opening a pull request."
|
||||
scenario_b: "You are integrating a long-lived, shared release branch that
|
||||
several other people have already pulled and built further commits on top
|
||||
of; you must not rewrite any commit they already depend on."
|
||||
|
||||
Respond with only a JSON object:
|
||||
{"scenario_a":"rebase"|"merge","scenario_b":"rebase"|"merge"}`
|
||||
|
||||
evaluator := eval.Mean(
|
||||
eval.JSONField("scenario_a", "rebase"),
|
||||
eval.JSONField("scenario_b", "merge"),
|
||||
)
|
||||
|
||||
return testkit.Test{
|
||||
ID: "git-rebase-vs-merge",
|
||||
Category: "delivery",
|
||||
Subcategory: "git",
|
||||
Description: "Choose rebase for a solo unshared branch and merge for a shared branch others already depend on.",
|
||||
Prompt: prompt,
|
||||
Eval: evaluator,
|
||||
}
|
||||
}
|
||||
|
||||
// delGitConventionalCommits is the inline set of 6 commit messages for
|
||||
// delGitConventionalCommitClassifyTest, one per standard conventional-commit
|
||||
// type, grounded in this repository's own real history (agents.go's
|
||||
// per-subcategory file split, ExtractLastNumber's dot-prefixed-decimal
|
||||
// handling, exec.go's toolchain-missing skip behavior, and the errgroup
|
||||
// dependency named in PLAN.md).
|
||||
const delGitConventionalCommits = `commit1: "feat(runner): add bounded concurrency to the model x test fan-out via errgroup"
|
||||
commit2: "fix(eval): correct ExtractLastNumber rejecting a standalone dot-prefixed decimal like \".9896\""
|
||||
commit3: "chore(deps): bump golang.org/x/sync to v0.22.0"
|
||||
commit4: "docs: document how to add a new evaluator in README.md"
|
||||
commit5: "refactor(tests): split agents.go into per-subcategory files to stay under 600 lines"
|
||||
commit6: "test(exec): add toolchain-missing skip cases for GoRun/PyRun/CRun"`
|
||||
|
||||
// delGitConventionalCommitClassifyTest: classify 6 commit messages, one per
|
||||
// standard conventional-commit type, by their type prefix.
|
||||
func delGitConventionalCommitClassifyTest() testkit.Test {
|
||||
prompt := `Here are 6 commit messages from a project using Conventional
|
||||
Commits:
|
||||
|
||||
` + delGitConventionalCommits + `
|
||||
|
||||
Classify each commit by its conventional-commit type (one of: feat, fix,
|
||||
chore, docs, refactor, test). Respond with only a JSON object:
|
||||
{"commit1":"...","commit2":"...","commit3":"...","commit4":"...","commit5":"...","commit6":"..."}`
|
||||
|
||||
evaluator := eval.Mean(
|
||||
eval.JSONField("commit1", "feat"),
|
||||
eval.JSONField("commit2", "fix"),
|
||||
eval.JSONField("commit3", "chore"),
|
||||
eval.JSONField("commit4", "docs"),
|
||||
eval.JSONField("commit5", "refactor"),
|
||||
eval.JSONField("commit6", "test"),
|
||||
)
|
||||
|
||||
return testkit.Test{
|
||||
ID: "git-conventional-commit-classify",
|
||||
Category: "delivery",
|
||||
Subcategory: "git",
|
||||
Description: "Classify 6 commit messages into their conventional-commit type from the message prefix.",
|
||||
Prompt: prompt,
|
||||
Eval: evaluator,
|
||||
}
|
||||
}
|
||||
|
||||
// delGitWorktreeUseCaseTest: name git worktree as the mechanism for a
|
||||
// second, independently checked-out working directory from the same
|
||||
// repository.
|
||||
//
|
||||
// ground truth: "git worktree add <path> <ref>" creates a second, linked
|
||||
// working directory backed by the same .git repository, checked out to a
|
||||
// different ref, without stashing, committing, or cloning again - exactly
|
||||
// the described need (an in-progress feature branch with uncommitted
|
||||
// changes stays untouched in the main working directory while an old tag is
|
||||
// built and tested elsewhere).
|
||||
func delGitWorktreeUseCaseTest() testkit.Test {
|
||||
prompt := `You are mid-way through an in-progress feature branch with
|
||||
uncommitted changes in your main working directory. You need to build and
|
||||
test the old release tag v1.4.0 at the same time, without stashing your
|
||||
changes and without cloning the repository again. Name the git subcommand
|
||||
that creates a second, linked working directory from the same repository
|
||||
checked out to a different ref, and give the exact command to create one at
|
||||
path ../v1.4.0-check checked out to tag v1.4.0.`
|
||||
|
||||
return testkit.Test{
|
||||
ID: "git-worktree-use-case",
|
||||
Category: "delivery",
|
||||
Subcategory: "git",
|
||||
Description: "Name git worktree as the way to check out a second ref from the same repo without stashing or re-cloning.",
|
||||
Prompt: prompt,
|
||||
Eval: eval.ContainsAll("git worktree add", "v1.4.0-check", "v1.4.0"),
|
||||
}
|
||||
}
|
||||
|
||||
// delGitForceWithLeaseVsForceTest: require --force-with-lease on a shared
|
||||
// branch, and require any mention of bare --force to be negated.
|
||||
//
|
||||
// ground truth: "--force-with-lease" refuses the push if the remote branch
|
||||
// has moved since your last fetch (someone else pushed), failing safely
|
||||
// instead of silently overwriting their work; bare "--force" overwrites
|
||||
// whatever is on the remote unconditionally, with no such check, so it must
|
||||
// never be recommended unnegated for a push to a branch other people also
|
||||
// push to.
|
||||
func delGitForceWithLeaseVsForceTest() testkit.Test {
|
||||
prompt := `Two teammates also push to the shared branch "release/2.4".
|
||||
After an interactive rebase, you rewrote your local commits and must
|
||||
force-push your branch. Which git push flag should you use on this shared
|
||||
branch, and why must a bare "--force" push never be used there instead?`
|
||||
|
||||
evaluator := eval.All(
|
||||
eval.W(eval.ContainsAny("--force-with-lease", "force-with-lease"), 2),
|
||||
eval.W(delNoBareForcePush(), 2),
|
||||
)
|
||||
|
||||
return testkit.Test{
|
||||
ID: "git-force-with-lease-vs-force",
|
||||
Category: "delivery",
|
||||
Subcategory: "git",
|
||||
Description: "Require --force-with-lease over bare --force for a force-push to a branch shared with teammates.",
|
||||
Prompt: prompt,
|
||||
Eval: evaluator,
|
||||
}
|
||||
}
|
||||
|
||||
// delGitHookChoiceTest: pick commit-msg as the hook that enforces a commit
|
||||
// message format at commit time, on the developer's own machine.
|
||||
//
|
||||
// ground truth: commit-msg receives the path to the drafted commit message
|
||||
// file and runs (and can reject it) at the moment of commit, before the
|
||||
// commit is created - exactly when a message-format policy must be
|
||||
// enforced. pre-commit inspects staged file contents, not the message; and
|
||||
// pre-push runs too late, after commits (and any bad messages) already
|
||||
// exist locally.
|
||||
func delGitHookChoiceTest() testkit.Test {
|
||||
prompt := `Your team wants to reject any commit whose message does not
|
||||
follow the Conventional Commits format, at the moment the commit is
|
||||
created, before it ever leaves the developer's machine. Name the single git
|
||||
hook (its exact filename under .git/hooks/) that enforces this. Respond
|
||||
with only the hook name.`
|
||||
|
||||
return testkit.Test{
|
||||
ID: "git-hook-choice",
|
||||
Category: "delivery",
|
||||
Subcategory: "git",
|
||||
Description: "Pick commit-msg as the hook that enforces a commit message format at commit time.",
|
||||
Prompt: prompt,
|
||||
Eval: eval.Equals("commit-msg"),
|
||||
}
|
||||
}
|
||||
|
||||
// delGitDetachedHeadRecoveryTest: recover 3 commits made in detached HEAD
|
||||
// state onto a new branch, in the one order that works.
|
||||
//
|
||||
// ground truth: while still on the detached HEAD (pointing at the 3 new
|
||||
// commits), "git branch <name>" creates a new branch ref at the current
|
||||
// commit without moving HEAD, capturing the work; only then does "git
|
||||
// checkout <name>" move HEAD onto that branch so it stops being detached.
|
||||
// Reversing the order (checkout first) would move HEAD away from the
|
||||
// detached commits before a branch ref exists to save them, losing the
|
||||
// work to garbage collection.
|
||||
func delGitDetachedHeadRecoveryTest() testkit.Test {
|
||||
prompt := `You ran "git checkout <old-commit-sha>" directly (not a
|
||||
branch name), then made 3 new commits while HEAD was detached. You only now
|
||||
realize you need to keep that work, and HEAD is still detached. Give the
|
||||
ordered list of git subcommands (just the subcommand names, e.g. "branch")
|
||||
you must run, in order, to save the 3 commits onto a new branch without
|
||||
losing them. Respond with only a JSON array of subcommand names, e.g.
|
||||
["a","b"].`
|
||||
|
||||
return testkit.Test{
|
||||
ID: "git-detached-head-recovery",
|
||||
Category: "delivery",
|
||||
Subcategory: "git",
|
||||
Description: "Order git branch then git checkout to save 3 detached-HEAD commits onto a new branch.",
|
||||
Prompt: prompt,
|
||||
Eval: eval.JSONStringArrayEquals([]string{"branch", "checkout"}),
|
||||
}
|
||||
}
|
||||
|
||||
// delGitCherryPickVsRevertTest: pick revert for undoing an already-shared
|
||||
// commit without rewriting public history.
|
||||
//
|
||||
// ground truth: "git revert" creates a new commit that undoes the target
|
||||
// commit's changes, leaving all existing history (including the original
|
||||
// commit) intact - safe once a commit has been pulled by other people.
|
||||
// "git cherry-pick" applies an existing commit's changes onto another
|
||||
// branch; it does not undo anything, so it cannot be the answer to "undo
|
||||
// its effect".
|
||||
func delGitCherryPickVsRevertTest() testkit.Test {
|
||||
prompt := `Commit abc1234 was merged into the shared "main" branch three
|
||||
days ago and has already been pulled by other teammates. You must undo its
|
||||
effect without rewriting any commit history that others have already
|
||||
pulled. Which single git operation do you use: cherry-pick or revert?
|
||||
Respond with only one word: cherry-pick or revert.`
|
||||
|
||||
return testkit.Test{
|
||||
ID: "git-cherry-pick-vs-revert",
|
||||
Category: "delivery",
|
||||
Subcategory: "git",
|
||||
Description: "Pick git revert (not cherry-pick) to undo an already-shared commit without rewriting history.",
|
||||
Prompt: prompt,
|
||||
Eval: eval.Equals("revert"),
|
||||
}
|
||||
}
|
||||
|
||||
// delGitignoreFile is the inline .gitignore content for
|
||||
// delGitGitignoreNegationTraceTest.
|
||||
const delGitignoreFile = `*.log
|
||||
build/
|
||||
!build/keep.txt
|
||||
docs/*.tmp
|
||||
!docs/important.tmp`
|
||||
|
||||
// delGitignoreTreeFiles is every file path in the inline tree for
|
||||
// delGitGitignoreNegationTraceTest, in prompt display order.
|
||||
var delGitignoreTreeFiles = []string{
|
||||
"app.go",
|
||||
"debug.log",
|
||||
"build/output.bin",
|
||||
"build/keep.txt",
|
||||
"docs/notes.tmp",
|
||||
"docs/important.tmp",
|
||||
"docs/readme.md",
|
||||
}
|
||||
|
||||
// delGitignoreIgnoredWant is the set of paths from delGitignoreTreeFiles
|
||||
// that git actually treats as ignored under delGitignoreFile.
|
||||
//
|
||||
// ground truth: "*.log" ignores debug.log. "build/" ignores the whole
|
||||
// build/ directory, which ignores build/output.bin AND build/keep.txt -
|
||||
// git's documented behavior is that a negation pattern cannot re-include a
|
||||
// file whose parent directory is itself excluded, since git does not
|
||||
// descend into an excluded directory to evaluate file-level patterns at
|
||||
// all, so "!build/keep.txt" has no effect. "docs/*.tmp" ignores
|
||||
// docs/notes.tmp; "!docs/important.tmp" DOES re-include docs/important.tmp,
|
||||
// because docs/ itself was never excluded as a directory (only files
|
||||
// matching *.tmp within it were), so git does descend into docs/ and the
|
||||
// later, more specific negation pattern wins. app.go and docs/readme.md
|
||||
// match no pattern. delivery_git_test.go verifies this exact set against a
|
||||
// real throwaway git repository (git status --ignored / git check-ignore)
|
||||
// rather than trusting this comment alone, skipping if git is unavailable.
|
||||
var delGitignoreIgnoredWant = []string{
|
||||
"build/keep.txt",
|
||||
"build/output.bin",
|
||||
"debug.log",
|
||||
"docs/notes.tmp",
|
||||
}
|
||||
|
||||
// delGitGitignoreNegationTraceTest: trace which files a .gitignore with
|
||||
// mixed exclude/negate rules actually ignores, including the
|
||||
// excluded-parent-directory negation trap.
|
||||
func delGitGitignoreNegationTraceTest() testkit.Test {
|
||||
prompt := `Here is a repository's .gitignore:
|
||||
|
||||
` + "```gitignore\n" + delGitignoreFile + "\n```" + `
|
||||
|
||||
And here is the full list of files that exist in the working tree:
|
||||
app.go, debug.log, build/output.bin, build/keep.txt, docs/notes.tmp,
|
||||
docs/important.tmp, docs/readme.md
|
||||
|
||||
Which of these files does git actually treat as ignored? Respond with only
|
||||
a JSON array of the ignored file paths.`
|
||||
|
||||
return testkit.Test{
|
||||
ID: "git-gitignore-negation-trace",
|
||||
Category: "delivery",
|
||||
Subcategory: "git",
|
||||
Description: "Trace which files a .gitignore with mixed exclude/negate rules ignores, including the excluded-parent-directory negation trap.",
|
||||
Prompt: prompt,
|
||||
Eval: eval.JSONStringSet(delGitignoreIgnoredWant),
|
||||
}
|
||||
}
|
||||
|
||||
// delGitStashPopConflictTest: state whether a conflicting `git stash pop`
|
||||
// drops the stash entry.
|
||||
//
|
||||
// ground truth: "git stash pop" only drops the stash entry after a clean
|
||||
// apply. On a conflict it leaves the working tree with conflict markers AND
|
||||
// explicitly keeps the stash entry in the stash list (git itself prints
|
||||
// "The stash entry is kept in case you need it again"), so no work is lost;
|
||||
// the user must resolve the conflict and then run "git stash drop"
|
||||
// manually. delivery_git_test.go verifies this against a real throwaway git
|
||||
// repository by staging a genuine stash/commit conflict and checking `git
|
||||
// stash list` afterward, skipping if git is unavailable.
|
||||
func delGitStashPopConflictTest() testkit.Test {
|
||||
prompt := `You run "git stash push" to save uncommitted changes to a
|
||||
file, then make and commit a new change to that same file, and later run
|
||||
"git stash pop". The stashed change conflicts with your new commit on that
|
||||
file. Immediately after "git stash pop" reports the conflict, is the stash
|
||||
entry dropped from the stash list, or kept? Respond with only one word:
|
||||
dropped or kept.`
|
||||
|
||||
return testkit.Test{
|
||||
ID: "git-stash-pop-conflict",
|
||||
Category: "delivery",
|
||||
Subcategory: "git",
|
||||
Description: "State that a conflicting git stash pop keeps the stash entry rather than dropping it.",
|
||||
Prompt: prompt,
|
||||
Eval: eval.Equals("kept"),
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,477 @@
|
||||
package tests
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"math"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// TestDelGitBisectStepsWant_GroundTruth recomputes delGitBisectStepsWant
|
||||
// from first principles (ceil(log2(N))) rather than trusting the hardcoded
|
||||
// literal in delivery_git.go.
|
||||
func TestDelGitBisectStepsWant_GroundTruth(t *testing.T) {
|
||||
got := int(math.Ceil(math.Log2(float64(delGitBisectCommitCount))))
|
||||
if got != delGitBisectStepsWant {
|
||||
t.Fatalf("ceil(log2(%d)) = %d, want %d", delGitBisectCommitCount, got, delGitBisectStepsWant)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDelGitBisectStepsTest_Eval(t *testing.T) {
|
||||
tc := delGitBisectStepsTest()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{name: "bare number", response: "8", want: 1},
|
||||
{name: "sentence form", response: "The answer is 8.", want: 1},
|
||||
{name: "different sentence form", response: "There are 8 bisect steps needed.", want: 1},
|
||||
{name: "wrong: repeats commit count", response: "137", want: 0},
|
||||
{name: "wrong: off by one", response: "7", want: 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDelGitRebaseVsMergeTest_Eval(t *testing.T) {
|
||||
tc := delGitRebaseVsMergeTest()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{name: "all correct", response: `{"scenario_a":"rebase","scenario_b":"merge"}`, want: 1},
|
||||
{name: "all correct fenced", response: "```json\n{\"scenario_a\":\"rebase\",\"scenario_b\":\"merge\"}\n```", want: 1},
|
||||
{name: "scenario_a wrong", response: `{"scenario_a":"merge","scenario_b":"merge"}`, want: 0.5},
|
||||
{name: "scenario_b wrong", response: `{"scenario_a":"rebase","scenario_b":"rebase"}`, want: 0.5},
|
||||
{name: "both swapped", response: `{"scenario_a":"merge","scenario_b":"rebase"}`, want: 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDelGitConventionalCommitClassifyTest_Eval(t *testing.T) {
|
||||
tc := delGitConventionalCommitClassifyTest()
|
||||
|
||||
allCorrect := `{"commit1":"feat","commit2":"fix","commit3":"chore","commit4":"docs","commit5":"refactor","commit6":"test"}`
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{name: "all correct", response: allCorrect, want: 1},
|
||||
{name: "all correct fenced with prose", response: "Here is my classification:\n```json\n" + allCorrect + "\n```", want: 1},
|
||||
{
|
||||
name: "one wrong: commit1 misclassified as chore",
|
||||
response: `{"commit1":"chore","commit2":"fix","commit3":"chore","commit4":"docs","commit5":"refactor","commit6":"test"}`,
|
||||
want: 5.0 / 6.0,
|
||||
},
|
||||
{
|
||||
name: "two wrong: commit3 and commit5 swapped types",
|
||||
response: `{"commit1":"feat","commit2":"fix","commit3":"refactor","commit4":"docs","commit5":"chore","commit6":"test"}`,
|
||||
want: 4.0 / 6.0,
|
||||
},
|
||||
{
|
||||
name: "all wrong",
|
||||
response: `{"commit1":"docs","commit2":"chore","commit3":"feat","commit4":"test","commit5":"fix","commit6":"refactor"}`,
|
||||
want: 0,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDelGitWorktreeUseCaseTest_Eval(t *testing.T) {
|
||||
tc := delGitWorktreeUseCaseTest()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{
|
||||
name: "correct: exact command",
|
||||
response: "Use `git worktree add ../v1.4.0-check v1.4.0` to create a second, linked working directory.",
|
||||
want: 1,
|
||||
},
|
||||
{
|
||||
name: "correct: relaxed prose",
|
||||
response: "Run git worktree add ../v1.4.0-check v1.4.0 to build the old tag in another directory.",
|
||||
want: 1,
|
||||
},
|
||||
{
|
||||
name: "correct: alternate phrasing",
|
||||
response: "git worktree is exactly this: git worktree add ../v1.4.0-check v1.4.0 links a second checkout to the same repo.",
|
||||
want: 1,
|
||||
},
|
||||
{
|
||||
name: "wrong: suggests re-cloning, mentions the tag only",
|
||||
response: "Just run git clone again into a new folder and checkout v1.4.0 there.",
|
||||
want: 1.0 / 3.0,
|
||||
},
|
||||
{
|
||||
name: "wrong: suggests stash, no worktree or tag path mentioned",
|
||||
response: "Use git stash to save your changes, then checkout the old tag in a scratch directory, then stash pop when done.",
|
||||
want: 0,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDelGitForceWithLeaseVsForceTest_Eval(t *testing.T) {
|
||||
tc := delGitForceWithLeaseVsForceTest()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{
|
||||
name: "correct: recommends force-with-lease, negates bare force",
|
||||
response: "Use git push --force-with-lease on the shared branch; never use a bare --force because it would silently overwrite the remote branch if it moved.",
|
||||
want: 1,
|
||||
},
|
||||
{
|
||||
name: "correct: only ever mentions the safe form",
|
||||
response: "On a shared branch always push with --force-with-lease; it aborts if the remote moved since your last fetch, unlike a plain overwrite.",
|
||||
want: 1,
|
||||
},
|
||||
{
|
||||
name: "correct: bare force mentioned but clearly negated",
|
||||
response: "Don't ever run a bare --force on release/2.4; instead use --force-with-lease, which fails safely if the remote changed.",
|
||||
want: 1,
|
||||
},
|
||||
{
|
||||
name: "wrong: recommends bare force outright",
|
||||
response: "Just use git push --force on the shared branch to make sure your rewritten history replaces theirs.",
|
||||
want: 0,
|
||||
},
|
||||
{
|
||||
name: "wrong: recommends force-with-lease but also an unnegated bare-force fallback",
|
||||
response: "Push with --force-with-lease normally, but if the lease check fails just fall back to --force to get it through.",
|
||||
want: 0.5,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDelGitHookChoiceTest_Eval(t *testing.T) {
|
||||
tc := delGitHookChoiceTest()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{name: "exact", response: "commit-msg", want: 1},
|
||||
{name: "different case", response: "Commit-Msg", want: 1},
|
||||
{name: "surrounding whitespace", response: " commit-msg \n", want: 1},
|
||||
{name: "wrong: pre-commit", response: "pre-commit", want: 0},
|
||||
{name: "wrong: pre-push", response: "pre-push", want: 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDelGitDetachedHeadRecoveryTest_Eval(t *testing.T) {
|
||||
tc := delGitDetachedHeadRecoveryTest()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{name: "correct order", response: `["branch","checkout"]`, want: 1},
|
||||
{name: "correct order fenced", response: "```json\n[\"branch\",\"checkout\"]\n```", want: 1},
|
||||
{name: "correct order different case", response: `["Branch","Checkout"]`, want: 1},
|
||||
{name: "reversed order", response: `["checkout","branch"]`, want: 0},
|
||||
{name: "wrong second subcommand", response: `["branch","switch"]`, want: 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDelGitCherryPickVsRevertTest_Eval(t *testing.T) {
|
||||
tc := delGitCherryPickVsRevertTest()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{name: "exact", response: "revert", want: 1},
|
||||
{name: "different case", response: "Revert", want: 1},
|
||||
{name: "surrounding whitespace", response: " revert \n", want: 1},
|
||||
{name: "wrong: cherry-pick", response: "cherry-pick", want: 0},
|
||||
{name: "wrong: reset", response: "reset", want: 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDelGitGitignoreNegationTraceTest_Eval(t *testing.T) {
|
||||
tc := delGitGitignoreNegationTraceTest()
|
||||
|
||||
correctSet := `["build/keep.txt","build/output.bin","debug.log","docs/notes.tmp"]`
|
||||
correctSetReordered := `["docs/notes.tmp","debug.log","build/output.bin","build/keep.txt"]`
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{name: "exact correct set", response: correctSet, want: 1},
|
||||
{name: "correct set, different order", response: correctSetReordered, want: 1},
|
||||
{name: "correct set, fenced", response: "```json\n" + correctSet + "\n```", want: 1},
|
||||
{
|
||||
name: "wrong: falls for the negation trap, includes docs/important.tmp",
|
||||
response: `["build/keep.txt","build/output.bin","debug.log","docs/notes.tmp","docs/important.tmp"]`,
|
||||
want: 0.8,
|
||||
},
|
||||
{
|
||||
name: "wrong: assumes negation always works, drops build/keep.txt",
|
||||
response: `["build/output.bin","debug.log","docs/notes.tmp"]`,
|
||||
want: 0.75,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDelGitStashPopConflictTest_Eval(t *testing.T) {
|
||||
tc := delGitStashPopConflictTest()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{name: "exact", response: "kept", want: 1},
|
||||
{name: "different case", response: "Kept", want: 1},
|
||||
{name: "surrounding whitespace", response: " kept \n", want: 1},
|
||||
{name: "wrong: dropped", response: "dropped", want: 0},
|
||||
{name: "wrong: removed", response: "removed", want: 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// delGitTestEnv builds a minimal, isolated environment for exec.Command
|
||||
// calls against a throwaway git repository: PATH (to find git), HOME
|
||||
// pointed at the isolated dir (never the operator's real home), and
|
||||
// GIT_CONFIG_NOSYSTEM/GIT_CONFIG_GLOBAL disabled so neither the system nor
|
||||
// the operator's real ~/.gitconfig (which may set core.hooksPath to a
|
||||
// global pre-commit hook) is consulted. GIT_AUTHOR_*/GIT_COMMITTER_* supply
|
||||
// a commit identity directly, so no "git config user.email" step is needed
|
||||
// either. This keeps the test hermetic: no network, no dependency on the
|
||||
// operator's actual git configuration.
|
||||
func delGitTestEnv(dir string) []string {
|
||||
return []string{
|
||||
"PATH=" + os.Getenv("PATH"),
|
||||
"HOME=" + dir,
|
||||
"GIT_CONFIG_NOSYSTEM=1",
|
||||
"GIT_CONFIG_GLOBAL=/dev/null",
|
||||
"GIT_AUTHOR_NAME=llmtb-test",
|
||||
"[email protected]",
|
||||
"GIT_COMMITTER_NAME=llmtb-test",
|
||||
"[email protected]",
|
||||
}
|
||||
}
|
||||
|
||||
// delRunGit runs `git args...` inside dir with env, returning combined
|
||||
// stdout+stderr and any error.
|
||||
func delRunGit(t *testing.T, dir string, env []string, args ...string) (string, error) {
|
||||
t.Helper()
|
||||
// #nosec G204 -- args are always fixed literals or the fixed
|
||||
// delGitignoreTreeFiles list defined in delivery_git.go, never
|
||||
// response/external/user-controlled input.
|
||||
cmd := exec.Command("git", args...) //nolint:gosec // see comment above
|
||||
cmd.Dir = dir
|
||||
cmd.Env = env
|
||||
var out bytes.Buffer
|
||||
cmd.Stdout = &out
|
||||
cmd.Stderr = &out
|
||||
err := cmd.Run()
|
||||
return out.String(), err
|
||||
}
|
||||
|
||||
// TestDelGitGitignoreNegationTrace_GroundTruth verifies delGitignoreIgnoredWant
|
||||
// against a real, throwaway git repository (git.md's "verify empirically
|
||||
// where cheap" guidance) rather than trusting delivery_git.go's derivation
|
||||
// comment alone. Skips if git is not installed.
|
||||
func TestDelGitGitignoreNegationTrace_GroundTruth(t *testing.T) {
|
||||
if _, err := exec.LookPath("git"); err != nil {
|
||||
t.Skip("git not installed")
|
||||
}
|
||||
|
||||
dir := t.TempDir()
|
||||
env := delGitTestEnv(dir)
|
||||
|
||||
if err := os.MkdirAll(filepath.Join(dir, "build"), 0o700); err != nil {
|
||||
t.Fatalf("mkdir build: %v", err)
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Join(dir, "docs"), 0o700); err != nil {
|
||||
t.Fatalf("mkdir docs: %v", err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(dir, ".gitignore"), []byte(delGitignoreFile+"\n"), 0o600); err != nil {
|
||||
t.Fatalf("write .gitignore: %v", err)
|
||||
}
|
||||
for _, f := range delGitignoreTreeFiles {
|
||||
if err := os.WriteFile(filepath.Join(dir, f), []byte("x\n"), 0o600); err != nil {
|
||||
t.Fatalf("write %s: %v", f, err)
|
||||
}
|
||||
}
|
||||
|
||||
if out, err := delRunGit(t, dir, env, "init", "-q"); err != nil {
|
||||
t.Fatalf("git init: %v (%s)", err, out)
|
||||
}
|
||||
|
||||
args := append([]string{"check-ignore"}, delGitignoreTreeFiles...)
|
||||
out, err := delRunGit(t, dir, env, args...)
|
||||
if err != nil {
|
||||
var exitErr *exec.ExitError
|
||||
if !errors.As(err, &exitErr) || exitErr.ExitCode() != 1 {
|
||||
t.Fatalf("git check-ignore: %v (%s)", err, out)
|
||||
}
|
||||
// exit code 1 just means "none of the given paths are ignored",
|
||||
// which is not the case here but is a valid, non-fatal outcome of
|
||||
// the command in general.
|
||||
}
|
||||
|
||||
var got []string
|
||||
for _, line := range strings.Split(strings.TrimSpace(out), "\n") {
|
||||
if line != "" {
|
||||
got = append(got, line)
|
||||
}
|
||||
}
|
||||
sort.Strings(got)
|
||||
|
||||
want := append([]string(nil), delGitignoreIgnoredWant...)
|
||||
sort.Strings(want)
|
||||
|
||||
if !slices.Equal(got, want) {
|
||||
t.Fatalf("git check-ignore ignored set = %v, want %v", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
// TestDelGitStashPopConflict_GroundTruth verifies, against a real,
|
||||
// throwaway git repository, that a conflicting `git stash pop` keeps the
|
||||
// stash entry rather than dropping it. Skips if git is not installed.
|
||||
func TestDelGitStashPopConflict_GroundTruth(t *testing.T) {
|
||||
if _, err := exec.LookPath("git"); err != nil {
|
||||
t.Skip("git not installed")
|
||||
}
|
||||
|
||||
dir := t.TempDir()
|
||||
env := delGitTestEnv(dir)
|
||||
fpath := filepath.Join(dir, "f.txt")
|
||||
|
||||
write := func(content string) {
|
||||
t.Helper()
|
||||
if err := os.WriteFile(fpath, []byte(content), 0o600); err != nil {
|
||||
t.Fatalf("write f.txt: %v", err)
|
||||
}
|
||||
}
|
||||
mustGit := func(args ...string) string {
|
||||
t.Helper()
|
||||
out, err := delRunGit(t, dir, env, args...)
|
||||
if err != nil {
|
||||
t.Fatalf("git %v: %v (%s)", args, err, out)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
mustGit("init", "-q")
|
||||
write("line1\nline2\nline3\n")
|
||||
mustGit("add", "f.txt")
|
||||
mustGit("commit", "-q", "-m", "init")
|
||||
|
||||
write("line1\nline2-stashed\nline3\n")
|
||||
mustGit("stash", "push", "-q", "-m", "work")
|
||||
|
||||
write("line1\nline2-committed\nline3\n")
|
||||
mustGit("add", "f.txt")
|
||||
mustGit("commit", "-q", "-m", "conflicting")
|
||||
|
||||
if _, err := delRunGit(t, dir, env, "stash", "pop"); err == nil {
|
||||
t.Fatal("git stash pop: want a conflict (non-nil error) given the two changes touch the same line, got nil")
|
||||
}
|
||||
|
||||
out := mustGit("stash", "list")
|
||||
if strings.TrimSpace(out) == "" {
|
||||
t.Fatal("git stash list: want the stash entry kept after a conflicting pop, got an empty list")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,417 @@
|
||||
package tests
|
||||
|
||||
import (
|
||||
"regexp"
|
||||
|
||||
"github.com/lukaszraczylo/llm-testbench/internal/eval"
|
||||
"github.com/lukaszraczylo/llm-testbench/internal/testkit"
|
||||
)
|
||||
|
||||
func registerDeliveryReleaseTests(r *testkit.Registry) {
|
||||
r.Register(delRelSemverBumpChangelogTest())
|
||||
r.Register(delRelCleanTreeRequirementTest())
|
||||
r.Register(delRelRollbackOrderingTest())
|
||||
r.Register(delRelCanaryVsBlueGreenTest())
|
||||
r.Register(delRelTagToReleaseSequenceTest())
|
||||
r.Register(delRelCommitChangelogMappingTest())
|
||||
r.Register(delRelChecksumsSigningPurposeTest())
|
||||
r.Register(delRelPipelineStageOrderingTest())
|
||||
r.Register(delRelPinVsRangePolicyTest())
|
||||
r.Register(delRelHotfixFlowOrderingTest())
|
||||
}
|
||||
|
||||
// delRelSemverBumpChangelogTest: derive the next semver version from a
|
||||
// merged-change list dominated by one feat commit.
|
||||
//
|
||||
// ground truth: of the four merged changes, the fix is a patch-level
|
||||
// change, the chore and docs entries are not release-relevant on their own,
|
||||
// and the feat entry adds a new, backward-compatible, optional query
|
||||
// parameter - a minor-level change under Semantic Versioning. The highest
|
||||
// severity present (feat) determines the bump: v2.3.1 with a minor bump
|
||||
// becomes 2.4.0 (MINOR increments, PATCH resets to 0).
|
||||
func delRelSemverBumpChangelogTest() testkit.Test {
|
||||
prompt := `Since the last release v2.3.1, these changes have merged:
|
||||
|
||||
- fix(auth): correct token refresh race condition
|
||||
- feat(api): add optional "include_deleted" query parameter to GET /items (backward compatible)
|
||||
- chore(ci): cache go build output between runs
|
||||
- docs: fix typo in CONTRIBUTING.md
|
||||
|
||||
None of these changes removes or renames any existing field, endpoint, or
|
||||
behavior. Per Semantic Versioning, what is the next version number?
|
||||
Respond with only a JSON object: {"version":"X.Y.Z"}`
|
||||
|
||||
return testkit.Test{
|
||||
ID: "rel-semver-bump-changelog",
|
||||
Category: "delivery",
|
||||
Subcategory: "release-engineering",
|
||||
Description: "Derive the next semver version (minor bump to 2.4.0) from a merged-change list dominated by one backward-compatible feat.",
|
||||
Prompt: prompt,
|
||||
Eval: eval.JSONField("version", "2.4.0"),
|
||||
}
|
||||
}
|
||||
|
||||
// delRelSilentlyModifyPattern matches "silently modif(y|ies|ying)" -
|
||||
// delRelCleanTreeRequirementTest's forbidden phrase, deliberately quoted
|
||||
// from the prompt's own wording so a response that engages with the
|
||||
// question at all is likely to reuse it, whether to recommend it (wrong,
|
||||
// unnegated) or warn against it (correct, negated).
|
||||
var delRelSilentlyModifyPattern = regexp.MustCompile(`(?i)silently modif\w*`)
|
||||
|
||||
// delRelCleanTreeRequirementTest: explain why release tooling requires a
|
||||
// clean git tree, and require that a hook silently mutating tracked files
|
||||
// mid-release is only ever mentioned in a negated (warned-against) form.
|
||||
//
|
||||
// ground truth: release tooling (e.g. goreleaser) refuses to run against a
|
||||
// dirty tree because the published artifact, its checksums, and its
|
||||
// signature must correspond exactly to a known, reviewed commit; a hook
|
||||
// that silently modifies tracked files right before packaging would ship
|
||||
// code that was never actually committed or reviewed, breaking that
|
||||
// guarantee, so it must never be allowed - not merely worked around.
|
||||
func delRelCleanTreeRequirementTest() testkit.Test {
|
||||
prompt := `Your release pipeline runs a release tool (for example
|
||||
goreleaser) against a tagged commit. A teammate proposes adding a pre-build
|
||||
hook that would silently modify tracked source files (auto-formatting them)
|
||||
immediately before the tool packages the release, with no separate commit
|
||||
for the change.
|
||||
|
||||
In 2-3 sentences: explain why release tooling requires a clean git working
|
||||
tree before it runs, and say whether a pre-build hook should be allowed to
|
||||
silently modify tracked files this way.`
|
||||
|
||||
evaluator := eval.All(
|
||||
eval.W(eval.ContainsAny("clean tree", "clean working tree", "clean git tree", "uncommitted", "dirty"), 2),
|
||||
eval.W(delNoUnnegatedMention(delRelSilentlyModifyPattern, "no unsafe recommendation to silently modify tracked files"), 2),
|
||||
)
|
||||
|
||||
return testkit.Test{
|
||||
ID: "rel-clean-tree-requirement",
|
||||
Category: "delivery",
|
||||
Subcategory: "release-engineering",
|
||||
Description: "Explain the clean-tree requirement and require any mention of a hook silently mutating tracked files to be negated.",
|
||||
Prompt: prompt,
|
||||
Eval: evaluator,
|
||||
}
|
||||
}
|
||||
|
||||
// delRelRollbackOrderingWant is the one defensible incident-response order
|
||||
// for delRelRollbackOrderingTest.
|
||||
//
|
||||
// ground truth: page-oncall first, so the response is coordinated and no
|
||||
// one else takes a conflicting action at the same time; then
|
||||
// stop-new-traffic-if-needed, to bound ongoing user harm before the actual
|
||||
// fix lands; then redeploy-previous-image, the fix itself; then
|
||||
// confirm-health-checks-pass, verifying the fix actually worked before
|
||||
// declaring the incident over; then post-incident-notes, written up only
|
||||
// once the incident is resolved.
|
||||
var delRelRollbackOrderingWant = []string{
|
||||
"page-oncall",
|
||||
"stop-new-traffic-if-needed",
|
||||
"redeploy-previous-image",
|
||||
"confirm-health-checks-pass",
|
||||
"post-incident-notes",
|
||||
}
|
||||
|
||||
// delRelRollbackOrderingTest: order the steps of a production rollback
|
||||
// after a failed deploy.
|
||||
func delRelRollbackOrderingTest() testkit.Test {
|
||||
prompt := `Release v3.1.0 was just deployed and is now failing health
|
||||
checks in production. The previous release v3.0.4 was healthy. Give the
|
||||
ordered list of steps to safely roll back to the last known-good release,
|
||||
choosing from and ordering exactly these labels:
|
||||
["redeploy-previous-image", "page-oncall", "confirm-health-checks-pass",
|
||||
"stop-new-traffic-if-needed", "post-incident-notes"]
|
||||
|
||||
Respond with only a JSON array of the labels, in the order you would
|
||||
perform them.`
|
||||
|
||||
return testkit.Test{
|
||||
ID: "rel-rollback-ordering",
|
||||
Category: "delivery",
|
||||
Subcategory: "release-engineering",
|
||||
Description: "Order a production rollback: page-oncall, stop traffic, redeploy, confirm health, then post-incident notes.",
|
||||
Prompt: prompt,
|
||||
Eval: eval.JSONStringArrayEquals(delRelRollbackOrderingWant),
|
||||
}
|
||||
}
|
||||
|
||||
// delRelCanaryVsBlueGreenTest: pick canary for gradual percentage-based
|
||||
// rollout, blue-green for instant all-or-nothing cutover.
|
||||
//
|
||||
// ground truth: canary is specifically the strategy that exposes a change
|
||||
// to a small, gradually increasing percentage of real traffic while
|
||||
// watching metrics, with the ability to halt at any percentage - exactly
|
||||
// scenario_a. blue-green is specifically the strategy with two full,
|
||||
// independently warmed environments and an instant all-or-nothing traffic
|
||||
// switch that can flip back immediately - exactly scenario_b.
|
||||
func delRelCanaryVsBlueGreenTest() testkit.Test {
|
||||
prompt := `For each of these two situations, is "canary" or "blue-green"
|
||||
the matching deployment strategy?
|
||||
|
||||
scenario_a: "Expose a risky new feature to a small percentage (e.g. 5%) of
|
||||
real production traffic first, gradually increasing that percentage while
|
||||
watching error-rate metrics, with the ability to automatically halt the
|
||||
rollout at any percentage if metrics regress."
|
||||
scenario_b: "An instant, all-or-nothing cutover from the old version to the
|
||||
new version, using a full, previously warmed standby environment, so 100%
|
||||
of traffic switches at once and can be flipped back to the old environment
|
||||
instantly if anything goes wrong."
|
||||
|
||||
Respond with only a JSON object:
|
||||
{"scenario_a":"canary"|"blue-green","scenario_b":"canary"|"blue-green"}`
|
||||
|
||||
evaluator := eval.Mean(
|
||||
eval.JSONField("scenario_a", "canary"),
|
||||
eval.JSONField("scenario_b", "blue-green"),
|
||||
)
|
||||
|
||||
return testkit.Test{
|
||||
ID: "rel-canary-vs-bluegreen",
|
||||
Category: "delivery",
|
||||
Subcategory: "release-engineering",
|
||||
Description: "Choose canary for a gradual percentage rollout and blue-green for an instant all-or-nothing cutover.",
|
||||
Prompt: prompt,
|
||||
Eval: evaluator,
|
||||
}
|
||||
}
|
||||
|
||||
// delRelTagToReleaseWant is the one defensible order from a verified main
|
||||
// branch to a published GitHub release with notes.
|
||||
//
|
||||
// ground truth: run-tests verifies main is releasable before it is tagged
|
||||
// at all. create-git-tag then tags that verified commit, and push-tag
|
||||
// publishes the tag to the remote - a prerequisite for anything that
|
||||
// references it. generate-changelog builds release notes from the commits
|
||||
// since the last tag. create-github-release publishes the GitHub release
|
||||
// against the now-pushed tag, using those generated notes. announce comes
|
||||
// last, only once the release genuinely exists.
|
||||
var delRelTagToReleaseWant = []string{
|
||||
"run-tests",
|
||||
"create-git-tag",
|
||||
"push-tag",
|
||||
"generate-changelog",
|
||||
"create-github-release",
|
||||
"announce",
|
||||
}
|
||||
|
||||
// delRelTagToReleaseSequenceTest: order the steps from a verified main
|
||||
// branch to a published, announced GitHub release.
|
||||
func delRelTagToReleaseSequenceTest() testkit.Test {
|
||||
prompt := `Starting from an already-merged, untested-since-merge main
|
||||
branch, give the ordered sequence of steps to reach a published,
|
||||
announced GitHub release with generated release notes, choosing from and
|
||||
ordering exactly these labels:
|
||||
["run-tests", "create-git-tag", "push-tag", "generate-changelog",
|
||||
"create-github-release", "announce"]
|
||||
|
||||
Respond with only a JSON array of the labels, in the order you would
|
||||
perform them.`
|
||||
|
||||
return testkit.Test{
|
||||
ID: "rel-tag-to-release-sequence",
|
||||
Category: "delivery",
|
||||
Subcategory: "release-engineering",
|
||||
Description: "Order tests, tag, push-tag, changelog, GitHub release, and announcement from a merged main branch to a published release.",
|
||||
Prompt: prompt,
|
||||
Eval: eval.JSONStringArrayEquals(delRelTagToReleaseWant),
|
||||
}
|
||||
}
|
||||
|
||||
// delRelCommitChangelogMappingTest: map 4 commits to their Keep a Changelog
|
||||
// section.
|
||||
//
|
||||
// ground truth (Keep a Changelog sections: Added, Changed, Deprecated,
|
||||
// Removed, Fixed, Security): a plain "feat" commit adds new behavior ->
|
||||
// Added. A plain "fix" commit -> Fixed. A "perf" commit changes existing
|
||||
// behavior's characteristics without adding or fixing a defect -> Changed.
|
||||
// A "fix(security)" commit specifically patches a vulnerability, which Keep
|
||||
// a Changelog calls out as its own Security section rather than the
|
||||
// generic Fixed section.
|
||||
func delRelCommitChangelogMappingTest() testkit.Test {
|
||||
prompt := `Using the Keep a Changelog section names (Added, Changed,
|
||||
Deprecated, Removed, Fixed, Security), map each of these 4 commits to the
|
||||
single changelog section it belongs in:
|
||||
|
||||
commit_a: "feat(runner): add bounded concurrency to the test fan-out via errgroup"
|
||||
commit_b: "fix(eval): correct off-by-one in numeric extraction"
|
||||
commit_c: "perf(exec): reuse the go env cache across evaluator calls"
|
||||
commit_d: "fix(security): patch a path traversal vulnerability in the template loader"
|
||||
|
||||
Respond with only a JSON object:
|
||||
{"commit_a":"...","commit_b":"...","commit_c":"...","commit_d":"..."}`
|
||||
|
||||
evaluator := eval.Mean(
|
||||
eval.JSONField("commit_a", "Added"),
|
||||
eval.JSONField("commit_b", "Fixed"),
|
||||
eval.JSONField("commit_c", "Changed"),
|
||||
eval.JSONField("commit_d", "Security"),
|
||||
)
|
||||
|
||||
return testkit.Test{
|
||||
ID: "rel-commit-changelog-mapping",
|
||||
Category: "delivery",
|
||||
Subcategory: "release-engineering",
|
||||
Description: "Map feat/fix/perf/security-fix commits to their Keep a Changelog section, including the Security carve-out.",
|
||||
Prompt: prompt,
|
||||
Eval: evaluator,
|
||||
}
|
||||
}
|
||||
|
||||
// delRelChecksumsSigningPurposeTest: explain what a checksums file and its
|
||||
// signature each protect against.
|
||||
//
|
||||
// ground truth: a checksums (e.g. SHA256) file lets a downloader verify the
|
||||
// binary was not corrupted or truncated in transit - an integrity check.
|
||||
// On its own, though, an attacker who swaps the binary can just regenerate
|
||||
// a matching checksum, so a signature over the checksums file is what ties
|
||||
// it back to a trusted publisher identity - an authenticity/provenance
|
||||
// check that a bare checksum cannot provide by itself.
|
||||
func delRelChecksumsSigningPurposeTest() testkit.Test {
|
||||
prompt := `A release pipeline publishes SHA256 checksums alongside each
|
||||
binary artifact, and additionally signs the checksums file with a GPG or
|
||||
cosign key. In 1-2 sentences, explain what problem each of these two things
|
||||
solves for someone downloading the release: the checksums file, and the
|
||||
signature over it.`
|
||||
|
||||
evaluator := eval.All(
|
||||
eval.W(eval.ContainsAny("integrity", "corrupt", "tamper", "not been altered", "not been modified"), 2),
|
||||
eval.W(eval.ContainsAny("authentic", "provenance", "came from", "verify the publisher", "signed by", "identity"), 2),
|
||||
)
|
||||
|
||||
return testkit.Test{
|
||||
ID: "rel-checksums-signing-purpose",
|
||||
Category: "delivery",
|
||||
Subcategory: "release-engineering",
|
||||
Description: "Explain checksums as an integrity check and the signature over them as an authenticity/provenance check.",
|
||||
Prompt: prompt,
|
||||
Eval: evaluator,
|
||||
}
|
||||
}
|
||||
|
||||
// delRelPipelineStageOrderingWant is the one defensible pipeline stage
|
||||
// order for delRelPipelineStageOrderingTest.
|
||||
//
|
||||
// ground truth: lint runs first because it is the fastest check and fails
|
||||
// fast on style/static issues before spending time on anything else; then
|
||||
// unit-test (fast, isolated); then build, compiling/packaging only once the
|
||||
// code is known-good; then integration-test, which needs the built
|
||||
// artifact; then publish-artifact, only after both test tiers pass;
|
||||
// deploy-staging promotes the published artifact to a pre-prod
|
||||
// environment; deploy-production is the final promotion, only after
|
||||
// staging is verified.
|
||||
var delRelPipelineStageOrderingWant = []string{
|
||||
"lint",
|
||||
"unit-test",
|
||||
"build",
|
||||
"integration-test",
|
||||
"publish-artifact",
|
||||
"deploy-staging",
|
||||
"deploy-production",
|
||||
}
|
||||
|
||||
// delRelPipelineStageOrderingTest: order a CI/CD pipeline's stages.
|
||||
func delRelPipelineStageOrderingTest() testkit.Test {
|
||||
prompt := `Order these CI/CD pipeline stages into the sequence a
|
||||
well-designed pipeline would run them in, fastest/cheapest checks first and
|
||||
each stage only running once its prerequisites have passed:
|
||||
["deploy-production", "build", "lint", "publish-artifact",
|
||||
"integration-test", "deploy-staging", "unit-test"]
|
||||
|
||||
Respond with only a JSON array of the labels, in the order they should
|
||||
run.`
|
||||
|
||||
return testkit.Test{
|
||||
ID: "rel-pipeline-stage-ordering",
|
||||
Category: "delivery",
|
||||
Subcategory: "release-engineering",
|
||||
Description: "Order pipeline stages fail-fast-first: lint, unit-test, build, integration-test, publish, staging, production.",
|
||||
Prompt: prompt,
|
||||
Eval: eval.JSONStringArrayEquals(delRelPipelineStageOrderingWant),
|
||||
}
|
||||
}
|
||||
|
||||
// delRelPinVsRangePolicyTest: pick exact pinning for an application's own
|
||||
// dependencies, and a version range for a published library.
|
||||
//
|
||||
// ground truth: an application that is deployed (not consumed by other
|
||||
// projects) should pin exact versions so every environment resolves the
|
||||
// identical dependency graph, changing only via an explicit, reviewed bump
|
||||
// - exactly scenario_a. A published library should instead declare a
|
||||
// version range, because pinning an exact version would force every
|
||||
// consumer into dependency-resolution conflicts whenever two libraries in
|
||||
// their own dependency tree pin different exact versions of the same
|
||||
// shared dependency - exactly scenario_b.
|
||||
func delRelPinVsRangePolicyTest() testkit.Test {
|
||||
prompt := `For each of these two situations, should dependencies be
|
||||
declared as an exact pinned version, or as a version range?
|
||||
|
||||
scenario_a: "A production application's own direct dependencies, where
|
||||
every environment (dev, CI, staging, production) must build against
|
||||
exactly the same resolved dependency graph, with nothing changing
|
||||
underneath it between releases without an explicit, reviewed bump."
|
||||
scenario_b: "A publicly published library that other projects will import
|
||||
as a dependency, where being overly strict would force every consumer into
|
||||
dependency-resolution conflicts whenever two libraries in their own tree
|
||||
pin different exact versions of the same shared dependency."
|
||||
|
||||
Respond with only a JSON object:
|
||||
{"scenario_a":"pin"|"range","scenario_b":"pin"|"range"}`
|
||||
|
||||
evaluator := eval.Mean(
|
||||
eval.JSONField("scenario_a", "pin"),
|
||||
eval.JSONField("scenario_b", "range"),
|
||||
)
|
||||
|
||||
return testkit.Test{
|
||||
ID: "rel-pin-vs-range-policy",
|
||||
Category: "delivery",
|
||||
Subcategory: "release-engineering",
|
||||
Description: "Choose exact pinning for an application's own dependencies and a version range for a published library.",
|
||||
Prompt: prompt,
|
||||
Eval: evaluator,
|
||||
}
|
||||
}
|
||||
|
||||
// delRelHotfixFlowOrderingWant is the one defensible order for an urgent
|
||||
// production hotfix when main has diverged with unrelated unfinished work.
|
||||
//
|
||||
// ground truth: branch-from-production-tag starts from exactly the code
|
||||
// currently running in production, not from main, which carries unrelated
|
||||
// in-progress work that must not ship early. apply-minimal-fix makes the
|
||||
// smallest change that resolves the incident, nothing else. run-tests
|
||||
// verifies the fix before shipping it under time pressure.
|
||||
// tag-hotfix-release cuts the release from this branch. deploy ships the
|
||||
// fix to production immediately, the urgent action. merge-back-to-main
|
||||
// comes last: it matters so the fix is not lost on the next regular
|
||||
// release, but it does not block getting the fix into production first.
|
||||
var delRelHotfixFlowOrderingWant = []string{
|
||||
"branch-from-production-tag",
|
||||
"apply-minimal-fix",
|
||||
"run-tests",
|
||||
"tag-hotfix-release",
|
||||
"deploy",
|
||||
"merge-back-to-main",
|
||||
}
|
||||
|
||||
// delRelHotfixFlowOrderingTest: order an urgent hotfix flow when main has
|
||||
// diverged with unrelated in-progress work that must not ship yet.
|
||||
func delRelHotfixFlowOrderingTest() testkit.Test {
|
||||
prompt := `Production is on fire and needs an urgent hotfix. The "main"
|
||||
branch has since diverged with unrelated, unfinished work that must NOT
|
||||
ship as part of this fix. Order these steps into the sequence you would
|
||||
actually perform them in:
|
||||
["merge-back-to-main", "tag-hotfix-release", "branch-from-production-tag",
|
||||
"deploy", "run-tests", "apply-minimal-fix"]
|
||||
|
||||
Respond with only a JSON array of the labels, in the order you would
|
||||
perform them.`
|
||||
|
||||
return testkit.Test{
|
||||
ID: "rel-hotfix-flow-ordering",
|
||||
Category: "delivery",
|
||||
Subcategory: "release-engineering",
|
||||
Description: "Order an urgent hotfix flow branched from production (not diverged main), through deploy, merging back to main last.",
|
||||
Prompt: prompt,
|
||||
Eval: eval.JSONStringArrayEquals(delRelHotfixFlowOrderingWant),
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,356 @@
|
||||
package tests
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestDelRelSemverBumpChangelogTest_Eval(t *testing.T) {
|
||||
tc := delRelSemverBumpChangelogTest()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{name: "bare object", response: `{"version":"2.4.0"}`, want: 1},
|
||||
{name: "fenced with prose", response: "The feat entry dominates, so:\n```json\n{\"version\":\"2.4.0\"}\n```", want: 1},
|
||||
{name: "extra whitespace inside the object", response: `{ "version" : "2.4.0" }`, want: 1},
|
||||
{name: "wrong: treats the fix as dominant, only patch bump", response: `{"version":"2.3.2"}`, want: 0},
|
||||
{name: "wrong: major bump for no breaking change", response: `{"version":"3.0.0"}`, want: 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDelRelCleanTreeRequirementTest_Eval(t *testing.T) {
|
||||
tc := delRelCleanTreeRequirementTest()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{
|
||||
name: "correct: explains reproducibility, negates the hook with must never",
|
||||
response: "Release tooling requires a clean working tree so the published artifact matches exactly what was committed and reviewed. A pre-build hook must never be allowed to silently modify tracked files mid-release.",
|
||||
want: 1,
|
||||
},
|
||||
{
|
||||
name: "correct: explains dirty tree, negates the hook with should not",
|
||||
response: "A dirty git tree means the shipped code was never actually committed, so the tool refuses it. Such a hook should not be allowed to silently modify tracked files.",
|
||||
want: 1,
|
||||
},
|
||||
{
|
||||
name: "correct: explains uncommitted changes, negates the hook with no",
|
||||
response: "It requires a clean git tree because uncommitted changes would ship code nobody reviewed. No, a hook must not silently modify tracked source files before packaging.",
|
||||
want: 1,
|
||||
},
|
||||
{
|
||||
name: "wrong: recommends the unsafe hook outright, no clean-tree reason given",
|
||||
response: "It's fine for the hook to silently modify tracked files right before packaging, since the formatting is harmless.",
|
||||
want: 0,
|
||||
},
|
||||
{
|
||||
name: "wrong: gives the reason but still endorses the unsafe hook",
|
||||
response: "A clean working tree matters because the artifact must match a known commit. That said, it's fine here for the hook to silently modify tracked files, since only formatting changes.",
|
||||
want: 0.5,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDelRelRollbackOrderingTest_Eval(t *testing.T) {
|
||||
tc := delRelRollbackOrderingTest()
|
||||
|
||||
correct := `["page-oncall","stop-new-traffic-if-needed","redeploy-previous-image","confirm-health-checks-pass","post-incident-notes"]`
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{name: "correct order", response: correct, want: 1},
|
||||
{name: "correct order fenced", response: "```json\n" + correct + "\n```", want: 1},
|
||||
{
|
||||
name: "correct order with extra whitespace",
|
||||
response: `[ "page-oncall", "stop-new-traffic-if-needed", "redeploy-previous-image", "confirm-health-checks-pass", "post-incident-notes" ]`,
|
||||
want: 1,
|
||||
},
|
||||
{
|
||||
name: "wrong: redeploys before paging anyone",
|
||||
response: `["redeploy-previous-image","page-oncall","stop-new-traffic-if-needed","confirm-health-checks-pass","post-incident-notes"]`,
|
||||
want: 0,
|
||||
},
|
||||
{
|
||||
name: "wrong: writes post-incident notes before confirming health",
|
||||
response: `["page-oncall","stop-new-traffic-if-needed","redeploy-previous-image","post-incident-notes","confirm-health-checks-pass"]`,
|
||||
want: 0,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDelRelCanaryVsBlueGreenTest_Eval(t *testing.T) {
|
||||
tc := delRelCanaryVsBlueGreenTest()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{name: "all correct", response: `{"scenario_a":"canary","scenario_b":"blue-green"}`, want: 1},
|
||||
{name: "all correct fenced", response: "```json\n{\"scenario_a\":\"canary\",\"scenario_b\":\"blue-green\"}\n```", want: 1},
|
||||
{name: "all correct different case", response: `{"scenario_a":"Canary","scenario_b":"Blue-Green"}`, want: 1},
|
||||
{name: "scenario_a wrong", response: `{"scenario_a":"blue-green","scenario_b":"blue-green"}`, want: 0.5},
|
||||
{name: "both swapped", response: `{"scenario_a":"blue-green","scenario_b":"canary"}`, want: 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDelRelTagToReleaseSequenceTest_Eval(t *testing.T) {
|
||||
tc := delRelTagToReleaseSequenceTest()
|
||||
|
||||
correct := `["run-tests","create-git-tag","push-tag","generate-changelog","create-github-release","announce"]`
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{name: "correct order", response: correct, want: 1},
|
||||
{name: "correct order fenced", response: "```json\n" + correct + "\n```", want: 1},
|
||||
{
|
||||
name: "correct order different case",
|
||||
response: `["Run-Tests","Create-Git-Tag","Push-Tag","Generate-Changelog","Create-Github-Release","Announce"]`,
|
||||
want: 1,
|
||||
},
|
||||
{
|
||||
name: "wrong: tags before running tests",
|
||||
response: `["create-git-tag","run-tests","push-tag","generate-changelog","create-github-release","announce"]`,
|
||||
want: 0,
|
||||
},
|
||||
{
|
||||
name: "wrong: creates the GitHub release before pushing the tag",
|
||||
response: `["run-tests","create-git-tag","generate-changelog","create-github-release","push-tag","announce"]`,
|
||||
want: 0,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDelRelCommitChangelogMappingTest_Eval(t *testing.T) {
|
||||
tc := delRelCommitChangelogMappingTest()
|
||||
|
||||
allCorrect := `{"commit_a":"Added","commit_b":"Fixed","commit_c":"Changed","commit_d":"Security"}`
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{name: "all correct", response: allCorrect, want: 1},
|
||||
{name: "all correct fenced with prose", response: "Here is the mapping:\n```json\n" + allCorrect + "\n```", want: 1},
|
||||
{
|
||||
name: "one wrong: security fix filed as generic Fixed",
|
||||
response: `{"commit_a":"Added","commit_b":"Fixed","commit_c":"Changed","commit_d":"Fixed"}`,
|
||||
want: 3.0 / 4.0,
|
||||
},
|
||||
{
|
||||
name: "two wrong: perf and security both misfiled",
|
||||
response: `{"commit_a":"Added","commit_b":"Fixed","commit_c":"Fixed","commit_d":"Fixed"}`,
|
||||
want: 2.0 / 4.0,
|
||||
},
|
||||
{
|
||||
name: "all wrong",
|
||||
response: `{"commit_a":"Fixed","commit_b":"Added","commit_c":"Removed","commit_d":"Deprecated"}`,
|
||||
want: 0,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDelRelChecksumsSigningPurposeTest_Eval(t *testing.T) {
|
||||
tc := delRelChecksumsSigningPurposeTest()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{
|
||||
name: "correct: integrity + authenticity",
|
||||
response: "The checksum lets you confirm the binary was not corrupted in transit; the signature proves the checksums file itself is authentic and came from the real publisher.",
|
||||
want: 1,
|
||||
},
|
||||
{
|
||||
name: "correct: alternate integrity + provenance phrasing",
|
||||
response: "Checksums detect if the download was tampered with or truncated. The signature adds provenance: it ties the checksums back to a trusted identity.",
|
||||
want: 1,
|
||||
},
|
||||
{
|
||||
name: "correct: alternate wording",
|
||||
response: "The SHA256 file verifies the binary has not been altered. Signing the checksums file verifies it was actually signed by the maintainer, not swapped in by an attacker.",
|
||||
want: 1,
|
||||
},
|
||||
{
|
||||
name: "wrong: neither concept present",
|
||||
response: "Both files just make the release page look more professional.",
|
||||
want: 0,
|
||||
},
|
||||
{
|
||||
name: "wrong: only explains the checksum, not the signature",
|
||||
response: "The checksum lets you confirm the binary was not corrupted in transit.",
|
||||
want: 0.5,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDelRelPipelineStageOrderingTest_Eval(t *testing.T) {
|
||||
tc := delRelPipelineStageOrderingTest()
|
||||
|
||||
correct := `["lint","unit-test","build","integration-test","publish-artifact","deploy-staging","deploy-production"]`
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{name: "correct order", response: correct, want: 1},
|
||||
{name: "correct order fenced", response: "```json\n" + correct + "\n```", want: 1},
|
||||
{
|
||||
name: "correct order different case",
|
||||
response: `["Lint","Unit-Test","Build","Integration-Test","Publish-Artifact","Deploy-Staging","Deploy-Production"]`,
|
||||
want: 1,
|
||||
},
|
||||
{
|
||||
name: "wrong: builds before linting",
|
||||
response: `["build","lint","unit-test","integration-test","publish-artifact","deploy-staging","deploy-production"]`,
|
||||
want: 0,
|
||||
},
|
||||
{
|
||||
name: "wrong: deploys production before staging",
|
||||
response: `["lint","unit-test","build","integration-test","publish-artifact","deploy-production","deploy-staging"]`,
|
||||
want: 0,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDelRelPinVsRangePolicyTest_Eval(t *testing.T) {
|
||||
tc := delRelPinVsRangePolicyTest()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{name: "all correct", response: `{"scenario_a":"pin","scenario_b":"range"}`, want: 1},
|
||||
{name: "all correct fenced", response: "```json\n{\"scenario_a\":\"pin\",\"scenario_b\":\"range\"}\n```", want: 1},
|
||||
{name: "all correct different case", response: `{"scenario_a":"Pin","scenario_b":"Range"}`, want: 1},
|
||||
{name: "scenario_b wrong", response: `{"scenario_a":"pin","scenario_b":"pin"}`, want: 0.5},
|
||||
{name: "both swapped", response: `{"scenario_a":"range","scenario_b":"pin"}`, want: 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDelRelHotfixFlowOrderingTest_Eval(t *testing.T) {
|
||||
tc := delRelHotfixFlowOrderingTest()
|
||||
|
||||
correct := `["branch-from-production-tag","apply-minimal-fix","run-tests","tag-hotfix-release","deploy","merge-back-to-main"]`
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{name: "correct order", response: correct, want: 1},
|
||||
{name: "correct order fenced", response: "```json\n" + correct + "\n```", want: 1},
|
||||
{
|
||||
name: "correct order different case",
|
||||
response: `["Branch-From-Production-Tag","Apply-Minimal-Fix","Run-Tests","Tag-Hotfix-Release","Deploy","Merge-Back-To-Main"]`,
|
||||
want: 1,
|
||||
},
|
||||
{
|
||||
name: "wrong: branches from diverged main instead of the production tag",
|
||||
response: `["apply-minimal-fix","run-tests","tag-hotfix-release","deploy","merge-back-to-main","branch-from-production-tag"]`,
|
||||
want: 0,
|
||||
},
|
||||
{
|
||||
name: "wrong: merges back to main before deploying the fix",
|
||||
response: `["branch-from-production-tag","apply-minimal-fix","run-tests","tag-hotfix-release","merge-back-to-main","deploy"]`,
|
||||
want: 0,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,51 @@
|
||||
package tests
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/lukaszraczylo/llm-testbench/internal/testkit"
|
||||
)
|
||||
|
||||
// TestRegisterDeliveryTests_Wiring checks registerDeliveryTests wires up all
|
||||
// three delivery subcategories - git, containers, release-engineering - with
|
||||
// 10 tests each, all categorized "delivery", and no duplicate IDs across the
|
||||
// three sibling files (testkit.Registry.Register panics on a duplicate ID,
|
||||
// which would fail this test outright). Mirrors
|
||||
// TestRegisterAgentsTests_Wiring in agents_test.go; catalog.go and
|
||||
// catalog_test.go are not touched by this worktree, so this is the only
|
||||
// place registerDeliveryTests is exercised until the orchestrator wires it
|
||||
// into All() at integration.
|
||||
func TestRegisterDeliveryTests_Wiring(t *testing.T) {
|
||||
r := testkit.NewRegistry()
|
||||
registerDeliveryTests(r)
|
||||
|
||||
const wantTotal = 30
|
||||
if r.Len() != wantTotal {
|
||||
t.Fatalf("registerDeliveryTests: registry has %d tests, want %d", r.Len(), wantTotal)
|
||||
}
|
||||
|
||||
counts := map[string]int{}
|
||||
for _, tc := range r.All() {
|
||||
if tc.Category != "delivery" {
|
||||
t.Errorf("test %q: Category = %q, want %q", tc.ID, tc.Category, "delivery")
|
||||
}
|
||||
if tc.Prompt == "" {
|
||||
t.Errorf("test %q: Prompt is empty", tc.ID)
|
||||
}
|
||||
if tc.Eval == nil {
|
||||
t.Errorf("test %q: Eval is nil", tc.ID)
|
||||
}
|
||||
if tc.Description == "" {
|
||||
t.Errorf("test %q: Description is empty", tc.ID)
|
||||
}
|
||||
if tc.MaxTokens < 0 {
|
||||
t.Errorf("test %q: MaxTokens = %d, want >= 0", tc.ID, tc.MaxTokens)
|
||||
}
|
||||
counts[tc.Subcategory]++
|
||||
}
|
||||
for _, sub := range []string{"git", "containers", "release-engineering"} {
|
||||
if counts[sub] != 10 {
|
||||
t.Errorf("subcategory %q: got %d tests, want 10", sub, counts[sub])
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user