diff --git a/internal/tests/delivery.go b/internal/tests/delivery.go new file mode 100644 index 0000000..f9c19d7 --- /dev/null +++ b/internal/tests/delivery.go @@ -0,0 +1,16 @@ +package tests + +import "github.com/lukaszraczylo/llm-testbench/internal/testkit" + +// registerDeliveryTests registers every delivery-category test. Each +// subcategory - git, containers, and release-engineering - has its own +// register function and source file (delivery_git.go, +// delivery_containers.go, delivery_release.go) to keep any one file from +// growing past a few hundred lines, mirroring the agents category's split +// (agents.go / agents_toolrouting.go / agents_planning.go / +// agents_delegation.go). +func registerDeliveryTests(r *testkit.Registry) { + registerDeliveryGitTests(r) + registerDeliveryContainersTests(r) + registerDeliveryReleaseTests(r) +} diff --git a/internal/tests/delivery_containers.go b/internal/tests/delivery_containers.go new file mode 100644 index 0000000..78d795a --- /dev/null +++ b/internal/tests/delivery_containers.go @@ -0,0 +1,371 @@ +package tests + +import ( + "github.com/lukaszraczylo/llm-testbench/internal/eval" + "github.com/lukaszraczylo/llm-testbench/internal/testkit" +) + +func registerDeliveryContainersTests(r *testkit.Registry) { + r.Register(delDockerLayerCacheBustTest()) + r.Register(delDockerEntrypointCmdTraceTest()) + r.Register(delDockerMultistageSizeBenefitTest()) + r.Register(delDockerImageSizeMathTest()) + r.Register(delDockerNonrootUserCapsTest()) + r.Register(delDockerHealthcheckSemanticsTest()) + r.Register(delDockerDockerignoreEffectTest()) + r.Register(delDockerCopyVsAddTest()) + r.Register(delDockerBuildArgVsEnvTest()) + r.Register(delDockerLayerCountTest()) +} + +// delDockerLayerCacheBustDockerfile is the inline numbered Dockerfile for +// delDockerLayerCacheBustTest. +const delDockerLayerCacheBustDockerfile = `1: FROM golang:1.25 +2: WORKDIR /app +3: COPY . . +4: RUN go mod download +5: RUN go build -o app . +6: CMD ["./app"]` + +// delDockerLayerCacheBustTest: spot the line whose placement busts the +// Docker layer cache on every source change. +// +// ground truth: line 3 (COPY . .) copies the entire source tree, including +// files that change on nearly every commit, before line 4 (go mod +// download). Docker invalidates a layer's cache, and every layer after it, +// as soon as any input to that layer changes - so any single source-file +// edit busts the cache starting at line 3, forcing "go mod download" to +// rerun on every build even when go.mod/go.sum never changed. Copying only +// go.mod/go.sum first, downloading dependencies, and copying the rest of +// the source afterward would let dependency downloads stay cached across +// source-only changes. +func delDockerLayerCacheBustTest() testkit.Test { + prompt := `Here is a Dockerfile, with line numbers prefixed: + +` + delDockerLayerCacheBustDockerfile + ` + +This Dockerfile reruns "go mod download" on every single build, even when +go.mod and go.sum have not changed. Which line number's placement busts the +Docker layer cache on every source-code change, forcing "go mod download" +to rerun unnecessarily? Respond with only a JSON object: {"line": }` + + return testkit.Test{ + ID: "docker-layer-cache-bust", + Category: "delivery", + Subcategory: "containers", + Description: "Spot the COPY . . line placed before dependency download that busts the Docker layer cache on every source change.", + Prompt: prompt, + Eval: eval.JSONField("line", 3), + } +} + +// delDockerEntrypointCmdTest: trace the exact resulting startup command +// when both ENTRYPOINT and CMD are set in exec form. +// +// ground truth: with both instructions in exec (JSON array) form, CMD's +// array is not run on its own - it supplies the default arguments appended +// after ENTRYPOINT's array, used only when "docker run" is given no +// trailing command. Run with no extra arguments, the process actually +// executed is "python3 app.py --port 8080". +func delDockerEntrypointCmdTraceTest() testkit.Test { + prompt := `A Dockerfile contains exactly these two instructions: + +` + "```dockerfile\n" + `ENTRYPOINT ["python3", "app.py"] +CMD ["--port", "8080"] +` + "```" + ` + +If the container is run with no extra arguments ("docker run image"), what +is the exact resulting command line executed inside the container? Respond +with only the exact command line, nothing else.` + + return testkit.Test{ + ID: "docker-entrypoint-cmd-trace", + Category: "delivery", + Subcategory: "containers", + Description: "Trace ENTRYPOINT+CMD exec-form interaction to the exact resulting startup command line.", + Prompt: prompt, + Eval: eval.Equals("python3 app.py --port 8080"), + } +} + +// delDockerMultistageSizeBenefitTest: explain how a multi-stage build +// shrinks the final image and name a minimal final-stage base. +func delDockerMultistageSizeBenefitTest() testkit.Test { + prompt := `A Go service's Dockerfile currently uses a single +"golang:1.25" base image (full Go toolchain, apt package caches, and build +tools) both to compile the binary AND to run it, producing a final image +over 900MB. Explain how switching to a multi-stage build reduces the final +image size, and name the kind of minimal base image you would copy the +compiled binary into for the final stage.` + + evaluator := eval.All( + eval.W(eval.ContainsAll("multi-stage"), 2), + eval.W(eval.ContainsAny("scratch", "distroless", "alpine"), 2), + ) + + return testkit.Test{ + ID: "docker-multistage-size-benefit", + Category: "delivery", + Subcategory: "containers", + Description: "Explain the multi-stage build size benefit and name a minimal final-stage base (scratch/distroless/alpine).", + Prompt: prompt, + Eval: evaluator, + } +} + +// delDockerImageSizeMathWant is the total size, in MB, of only the final +// stage's layers in delDockerImageSizeMathTest. +// +// ground truth: only layers in the FINAL stage of a multi-stage build ship +// in the resulting image. The build stage's numbers (72 + 148 + 4 + 310) +// are discarded distractors; the final stage is alpine base (8MB) plus the +// compiled-binary COPY layer (22MB): 8 + 22 = 30. +var delDockerImageSizeMathWant = 8 + 22 + +// delDockerImageSizeMathTest: sum only the final stage's layer sizes in a +// multi-stage build, ignoring the discarded build-stage layers. +func delDockerImageSizeMathTest() testkit.Test { + prompt := `A Docker image is built with a multi-stage build: + +Build stage (discarded, not part of the final image): base image 72MB, an +apt-get install layer adds 148MB, a COPY of application source adds 4MB, +and a RUN go build layer adds 310MB. + +Final stage (this is what actually ships): base image alpine at 8MB, plus a +COPY of the compiled binary from the build stage adds 22MB. + +What is the total size, in MB, of the final shipped image? Respond with +only the number.` + + return testkit.Test{ + ID: "docker-image-size-math", + Category: "delivery", + Subcategory: "containers", + Description: "Sum only the final stage's layer sizes (8+22=30MB) in a multi-stage build, ignoring discarded build-stage layers.", + Prompt: prompt, + Eval: eval.Numeric(eval.ExtractLastNumber[int], delDockerImageSizeMathWant, 0), + } +} + +// delDockerNonrootUserCapsTest: require a non-root USER and a fully-dropped +// capability set for a container that needs no privileged syscalls. +// +// ground truth: adding a USER instruction (a numeric non-root UID, or a +// user created via adduser/useradd) stops the container's process from +// running as root. Binding to a port above 1024 needs no special Linux +// capability (only binding below 1024 needs CAP_NET_BIND_SERVICE), so with +// no other privileged operation required, every capability should be +// dropped at runtime (--cap-drop=ALL, with no --cap-add). +func delDockerNonrootUserCapsTest() testkit.Test { + prompt := `This Dockerfile currently has no USER instruction, so its +process runs as root inside the container. The entrypoint binary only needs +to bind to a port above 1024 - nothing else privileged. Give the Dockerfile +instruction that makes the container run as a non-root user, and name the +Linux capability set it should end up running with at runtime, given it +needs no special privileges.` + + evaluator := eval.All( + eval.W(eval.ContainsAll("USER"), 2), + eval.W(eval.ContainsAny("cap-drop=all", "cap-drop all", "drop all capabilities", "capdrop=all", "drop all"), 2), + ) + + return testkit.Test{ + ID: "docker-nonroot-user-caps", + Category: "delivery", + Subcategory: "containers", + Description: "Require a USER instruction plus a fully-dropped capability set for a container needing no privileged syscalls.", + Prompt: prompt, + Eval: evaluator, + } +} + +// delDockerHealthcheckRetriesWant is the number of consecutive failed +// probes before Docker marks the container unhealthy in +// delDockerHealthcheckSemanticsTest, taken directly from the instruction's +// own --retries=3 value. +var delDockerHealthcheckRetriesWant = 3 + +// delDockerHealthcheckSemanticsTest: read the HEALTHCHECK retries value +// correctly rather than the interval or timeout values. +func delDockerHealthcheckSemanticsTest() testkit.Test { + prompt := `Given this Dockerfile instruction: + +` + "```dockerfile\n" + `HEALTHCHECK --interval=30s --timeout=3s --retries=3 CMD curl -f http://localhost:8080/health || exit 1 +` + "```" + ` + +Assuming every probe run so far has failed, after how many consecutive +FAILED probes does Docker mark the container's health status as +"unhealthy"? Respond with only the integer.` + + return testkit.Test{ + ID: "docker-healthcheck-semantics", + Category: "delivery", + Subcategory: "containers", + Description: "Read a HEALTHCHECK instruction's --retries value as the consecutive-failure count before unhealthy status.", + Prompt: prompt, + Eval: eval.Numeric(eval.ExtractLastNumber[int], delDockerHealthcheckRetriesWant, 0), + } +} + +// delDockerignoreFile is the inline .dockerignore content for +// delDockerDockerignoreEffectTest. Deliberately uses no negation patterns +// (unlike delGitignoreFile's git-specific negation trap): Docker's own +// negation-after-directory-exclusion behavior differs between the classic +// builder and BuildKit and is not as uniformly documented as git's, so +// relying on it here would not be a deterministic, single-answer test. +const delDockerignoreFile = `node_modules +*.log +.git` + +// delDockerDockerignoreEffectWant is the sorted set of files from +// delDockerDockerignoreEffectTest's inline file list that are NOT excluded +// by delDockerignoreFile and so are sent to the Docker build context. +// +// ground truth: "node_modules" excludes node_modules/pkg/index.js. "*.log" +// excludes debug.log. ".git" excludes .git/config. app.js and README.md +// match no pattern and are included. +var delDockerDockerignoreEffectWant = []string{"README.md", "app.js"} + +// delDockerDockerignoreEffectTest: identify which files are NOT excluded by +// a .dockerignore and so are sent to the Docker daemon's build context. +func delDockerDockerignoreEffectTest() testkit.Test { + prompt := `Here is a repository's .dockerignore: + +` + "```\n" + delDockerignoreFile + "\n```" + ` + +And here is the full list of files that exist in the build directory: +app.js, node_modules/pkg/index.js, debug.log, README.md, .git/config + +Which of these files WILL be sent to the Docker daemon as part of the build +context (i.e. are NOT excluded by .dockerignore)? Respond with only a JSON +array of the included file paths.` + + return testkit.Test{ + ID: "docker-dockerignore-effect", + Category: "delivery", + Subcategory: "containers", + Description: "Identify which files a .dockerignore leaves in the build context that gets sent to the Docker daemon.", + Prompt: prompt, + Eval: eval.JSONStringSet(delDockerDockerignoreEffectWant), + } +} + +// delDockerCopyVsAddTest: pick COPY for plain local files, ADD for the one +// feature COPY lacks - local tar auto-extraction. +// +// ground truth: COPY is Docker's own recommended default - transparent, +// predictable, does only a file copy. ADD additionally auto-extracts a +// local tar/gzip archive into the destination directory as part of the same +// instruction (and can also fetch remote URLs, which Docker's own best +// practices discourage) - the tar-extraction behavior is the one thing ADD +// does that COPY cannot, making it the correct choice specifically for +// scenario_b. +func delDockerCopyVsAddTest() testkit.Test { + prompt := `For each of these two situations, should you use "COPY" or +"ADD" in the Dockerfile? + +scenario_a: "Copy a local directory of static assets into the image, with +no extraction or URL-fetching behavior needed - just the files as they are +on disk." +scenario_b: "Add a local .tar.gz archive to the image and have it +automatically extracted into the destination directory as part of the same +instruction." + +Respond with only a JSON object: +{"scenario_a":"COPY"|"ADD","scenario_b":"COPY"|"ADD"}` + + evaluator := eval.Mean( + eval.JSONField("scenario_a", "COPY"), + eval.JSONField("scenario_b", "ADD"), + ) + + return testkit.Test{ + ID: "docker-copy-vs-add", + Category: "delivery", + Subcategory: "containers", + Description: "Choose COPY for a plain file copy and ADD for its unique local-tar-auto-extraction behavior.", + Prompt: prompt, + Eval: evaluator, + } +} + +// delDockerBuildArgVsEnvTest: pick ARG for a build-time-only value and ENV +// for a value the running container must expose. +// +// ground truth: ARG values exist only during the build and are not present +// in the final image's runtime environment or in "docker inspect"'s Env +// list, matching scenario_a's requirement. ENV values are baked into the +// image's runtime config and are what "docker run -e" overrides and what a +// running process reads via getenv, matching scenario_b. +func delDockerBuildArgVsEnvTest() testkit.Test { + prompt := `For each of these two situations, should the value be set +with "ARG" or "ENV" in the Dockerfile? + +scenario_a: "A value is needed only at image-build time (for example, +selecting which internal build number to embed into a version label), and +must NOT be present in the final running container's environment or in +'docker inspect' output." +scenario_b: "A value must be configurable at container start time (for +example by an operator running 'docker run -e ...'), without rebuilding the +image, and must be readable by the running process." + +Respond with only a JSON object: +{"scenario_a":"ARG"|"ENV","scenario_b":"ARG"|"ENV"}` + + evaluator := eval.Mean( + eval.JSONField("scenario_a", "ARG"), + eval.JSONField("scenario_b", "ENV"), + ) + + return testkit.Test{ + ID: "docker-build-arg-vs-env", + Category: "delivery", + Subcategory: "containers", + Description: "Choose ARG for a build-time-only value and ENV for a value the running container must expose.", + Prompt: prompt, + Eval: evaluator, + } +} + +// delDockerLayerCountDockerfile is the inline numbered Dockerfile for +// delDockerLayerCountTest. +const delDockerLayerCountDockerfile = `1: FROM alpine:3.20 +2: WORKDIR /app +3: COPY app /app/app +4: RUN chmod +x /app/app +5: ENV PORT=8080 +6: EXPOSE 8080 +7: USER 1000 +8: ENTRYPOINT ["/app/app"]` + +// delDockerLayerCountWant is the number of new filesystem layers this +// Dockerfile adds on top of its base image. +// +// ground truth: only RUN, COPY, and ADD instructions create a new +// filesystem layer; WORKDIR, ENV, EXPOSE, USER, ENTRYPOINT, CMD, and LABEL +// only update image metadata/config and add no layer. This Dockerfile has +// exactly one COPY (line 3) and one RUN (line 4): 2 new layers. +var delDockerLayerCountWant = 2 + +// delDockerLayerCountTest: count only the new filesystem layers a +// Dockerfile adds on top of its base image, distinguishing layer-creating +// instructions (RUN/COPY/ADD) from metadata-only ones. +func delDockerLayerCountTest() testkit.Test { + prompt := `Here is a Dockerfile, with line numbers prefixed: + +` + delDockerLayerCountDockerfile + ` + +Not counting whatever layers already exist inside the alpine:3.20 base +image itself, how many NEW filesystem layers does building this Dockerfile +add on top of that base? Respond with only the number.` + + return testkit.Test{ + ID: "docker-layer-count", + Category: "delivery", + Subcategory: "containers", + Description: "Count only the layer-creating instructions (RUN/COPY/ADD) in a Dockerfile, excluding metadata-only ones.", + Prompt: prompt, + Eval: eval.Numeric(eval.ExtractLastNumber[int], delDockerLayerCountWant, 0), + } +} diff --git a/internal/tests/delivery_containers_test.go b/internal/tests/delivery_containers_test.go new file mode 100644 index 0000000..f62be5d --- /dev/null +++ b/internal/tests/delivery_containers_test.go @@ -0,0 +1,294 @@ +package tests + +import ( + "context" + "testing" +) + +func TestDelDockerLayerCacheBustTest_Eval(t *testing.T) { + tc := delDockerLayerCacheBustTest() + + tests := []struct { + name string + response string + want float64 + }{ + {name: "bare object", response: `{"line": 3}`, want: 1}, + {name: "fenced with prose", response: "The culprit is:\n```json\n{\"line\": 3}\n```", want: 1}, + {name: "extra whitespace inside the object", response: `{ "line" : 3 }`, want: 1}, + {name: "wrong: blames the FROM line", response: `{"line": 1}`, want: 0}, + {name: "wrong: blames go build instead", response: `{"line": 5}`, want: 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Value != tt.want { + t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail) + } + }) + } +} + +func TestDelDockerEntrypointCmdTraceTest_Eval(t *testing.T) { + tc := delDockerEntrypointCmdTraceTest() + + tests := []struct { + name string + response string + want float64 + }{ + {name: "exact", response: "python3 app.py --port 8080", want: 1}, + {name: "different case", response: "Python3 App.py --Port 8080", want: 1}, + {name: "surrounding whitespace", response: " python3 app.py --port 8080\n", want: 1}, + {name: "wrong: only CMD's args", response: "--port 8080", want: 0}, + {name: "wrong: only ENTRYPOINT, drops CMD's default args", response: "python3 app.py", want: 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Value != tt.want { + t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail) + } + }) + } +} + +func TestDelDockerMultistageSizeBenefitTest_Eval(t *testing.T) { + tc := delDockerMultistageSizeBenefitTest() + + tests := []struct { + name string + response string + want float64 + }{ + { + name: "correct: multi-stage + scratch", + response: "Use a multi-stage build: compile in the golang stage, then copy only the binary into a final scratch image with no toolchain at all.", + want: 1, + }, + { + name: "correct: multi-stage + distroless", + response: "A multi-stage build lets you copy just the compiled binary into a distroless final image, dropping the entire build toolchain layer.", + want: 1, + }, + { + name: "correct: multi-stage + alpine", + response: "With a multi-stage build the final stage can be a minimal alpine base holding only the binary, instead of the full golang image.", + want: 1, + }, + { + name: "wrong: neither term present", + response: "Just delete unused files with a RUN rm -rf step at the end of the same image.", + want: 0, + }, + { + name: "wrong: names the base but never says multi-stage", + response: "Switch the final image to alpine instead of golang.", + want: 0.5, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Value != tt.want { + t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail) + } + }) + } +} + +func TestDelDockerImageSizeMathTest_Eval(t *testing.T) { + tc := delDockerImageSizeMathTest() + + tests := []struct { + name string + response string + want float64 + }{ + {name: "bare number", response: "30", want: 1}, + {name: "sentence form", response: "The final image is 30 MB.", want: 1}, + {name: "with units spelled out", response: "8 + 22 = 30 megabytes.", want: 1}, + {name: "wrong: sums every layer including the discarded build stage", response: "534", want: 0}, + {name: "wrong: only the alpine base, forgets the binary layer", response: "8", want: 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Value != tt.want { + t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail) + } + }) + } +} + +func TestDelDockerNonrootUserCapsTest_Eval(t *testing.T) { + tc := delDockerNonrootUserCapsTest() + + tests := []struct { + name string + response string + want float64 + }{ + { + name: "correct: USER + cap-drop=all", + response: "Add `USER 1000` to the Dockerfile, and run the container with --cap-drop=ALL since it needs no special capabilities.", + want: 1, + }, + { + name: "correct: USER + drop all capabilities phrasing", + response: "Create a non-root user with adduser and add a USER instruction; at runtime, drop all capabilities.", + want: 1, + }, + { + name: "correct: USER + cap-drop all spaced form", + response: "Set USER app in the image, and pass --cap-drop all at runtime since no privileged syscalls are needed.", + want: 1, + }, + { + name: "wrong: neither addressed", + response: "This is fine as-is, since the process only listens on a port.", + want: 0, + }, + { + name: "wrong: only addresses the user, not capabilities", + response: "Add a USER instruction pointing at a non-root UID.", + want: 0.5, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Value != tt.want { + t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail) + } + }) + } +} + +func TestDelDockerHealthcheckSemanticsTest_Eval(t *testing.T) { + tc := delDockerHealthcheckSemanticsTest() + + tests := []struct { + name string + response string + want float64 + }{ + {name: "bare number", response: "3", want: 1}, + {name: "sentence form", response: "After 3 consecutive failures.", want: 1}, + {name: "labelled form", response: "retries: 3", want: 1}, + {name: "wrong: confuses timeout with retries", response: "3s", want: 0}, // "3s" glues the 3 to a unit letter, so extraction fails and scores 0 + {name: "wrong: confuses interval with retries", response: "30", want: 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Value != tt.want { + t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail) + } + }) + } +} + +func TestDelDockerDockerignoreEffectTest_Eval(t *testing.T) { + tc := delDockerDockerignoreEffectTest() + + tests := []struct { + name string + response string + want float64 + }{ + {name: "exact correct set", response: `["app.js","README.md"]`, want: 1}, + {name: "correct set, different order", response: `["README.md","app.js"]`, want: 1}, + {name: "correct set, fenced", response: "```json\n[\"app.js\",\"README.md\"]\n```", want: 1}, + { + name: "wrong: includes an excluded file", + response: `["app.js","README.md","debug.log"]`, + want: 2.0 / 3.0, + }, + { + name: "wrong: also excludes a file that should be included", + response: `["app.js"]`, + want: 0.5, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Value != tt.want { + t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail) + } + }) + } +} + +func TestDelDockerCopyVsAddTest_Eval(t *testing.T) { + tc := delDockerCopyVsAddTest() + + tests := []struct { + name string + response string + want float64 + }{ + {name: "all correct", response: `{"scenario_a":"COPY","scenario_b":"ADD"}`, want: 1}, + {name: "all correct fenced", response: "```json\n{\"scenario_a\":\"COPY\",\"scenario_b\":\"ADD\"}\n```", want: 1}, + {name: "all correct lowercase", response: `{"scenario_a":"copy","scenario_b":"add"}`, want: 1}, + {name: "scenario_a wrong", response: `{"scenario_a":"ADD","scenario_b":"ADD"}`, want: 0.5}, + {name: "both swapped", response: `{"scenario_a":"ADD","scenario_b":"COPY"}`, want: 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Value != tt.want { + t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail) + } + }) + } +} + +func TestDelDockerBuildArgVsEnvTest_Eval(t *testing.T) { + tc := delDockerBuildArgVsEnvTest() + + tests := []struct { + name string + response string + want float64 + }{ + {name: "all correct", response: `{"scenario_a":"ARG","scenario_b":"ENV"}`, want: 1}, + {name: "all correct fenced", response: "```json\n{\"scenario_a\":\"ARG\",\"scenario_b\":\"ENV\"}\n```", want: 1}, + {name: "all correct lowercase", response: `{"scenario_a":"arg","scenario_b":"env"}`, want: 1}, + {name: "scenario_b wrong", response: `{"scenario_a":"ARG","scenario_b":"ARG"}`, want: 0.5}, + {name: "both swapped", response: `{"scenario_a":"ENV","scenario_b":"ARG"}`, want: 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Value != tt.want { + t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail) + } + }) + } +} + +func TestDelDockerLayerCountTest_Eval(t *testing.T) { + tc := delDockerLayerCountTest() + + tests := []struct { + name string + response string + want float64 + }{ + {name: "bare number", response: "2", want: 1}, + {name: "sentence form", response: "This Dockerfile adds 2 new layers.", want: 1}, + {name: "explains then answers", response: "Only COPY and RUN create layers here, so the answer is 2.", want: 1}, + {name: "wrong: counts every instruction", response: "8", want: 0}, + {name: "wrong: forgets the RUN layer", response: "1", want: 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Value != tt.want { + t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail) + } + }) + } +} diff --git a/internal/tests/delivery_git.go b/internal/tests/delivery_git.go new file mode 100644 index 0000000..b8f13ce --- /dev/null +++ b/internal/tests/delivery_git.go @@ -0,0 +1,476 @@ +package tests + +import ( + "context" + "regexp" + "strings" + + "github.com/lukaszraczylo/llm-testbench/internal/eval" + "github.com/lukaszraczylo/llm-testbench/internal/testkit" +) + +func registerDeliveryGitTests(r *testkit.Registry) { + r.Register(delGitBisectStepsTest()) + r.Register(delGitRebaseVsMergeTest()) + r.Register(delGitConventionalCommitClassifyTest()) + r.Register(delGitWorktreeUseCaseTest()) + r.Register(delGitForceWithLeaseVsForceTest()) + r.Register(delGitHookChoiceTest()) + r.Register(delGitDetachedHeadRecoveryTest()) + r.Register(delGitCherryPickVsRevertTest()) + r.Register(delGitGitignoreNegationTraceTest()) + r.Register(delGitStashPopConflictTest()) +} + +// --- negation-aware helpers, shared across delivery_git.go and +// delivery_release.go (same package). Duplicated in spirit from +// operations/kubernetes's noLiveKubectlMutation/negationCuePattern rather +// than imported from kubernetes.go, so this category's files stay +// self-contained: round-2 authors work in isolated worktrees on +// databases/security/delivery/ai in parallel, and depending on another +// category's unexported symbols would create a merge-time coupling this +// worktree cannot see or coordinate on. --- + +// delNegationCuePattern matches a word that turns a mention of a forbidden +// phrase into a warning against doing it, rather than an instruction to do +// it (e.g. "never use bare --force", "must not silently modify"). +var delNegationCuePattern = regexp.MustCompile(`(?i)\b(don'?t|do not|never|avoid|instead of|not|cannot|can'?t|rather than|without|no need|must not|should not|shouldn'?t)\b`) + +// delNegationWindow is how many characters before a forbidden-phrase match +// are searched for a negation cue. +const delNegationWindow = 60 + +// delNegationWindowStart returns the earliest byte offset in response to +// search for a negation cue before a forbidden-phrase match starting at +// start. The window is the current line, extended back to the start of the +// immediately preceding line when that line is non-empty, so a hard-wrapped +// sentence whose negation cue landed on the line above still counts. Either +// way the window never reaches more than delNegationWindow characters +// before start. +func delNegationWindowStart(response string, start int) int { + curLineStart := strings.LastIndexByte(response[:start], '\n') + 1 + + windowFloor := curLineStart + if curLineStart > 0 { + prevLineEnd := curLineStart - 1 + prevLineStart := strings.LastIndexByte(response[:prevLineEnd], '\n') + 1 + if strings.TrimSpace(response[prevLineStart:prevLineEnd]) != "" { + windowFloor = prevLineStart + } + } + + return max(start-delNegationWindow, windowFloor) +} + +// delNoUnnegatedMention returns an Evaluator scoring full credit unless +// forbidden matches response at some position with no negation cue in the +// preceding window - i.e. the response recommends the forbidden thing +// outright rather than warning against it. This is deliberately not +// eval.NotContains, which would also zero out the best possible answer (one +// that correctly explains why NOT to do the forbidden thing). +func delNoUnnegatedMention(forbidden *regexp.Regexp, safeDetail string) eval.Evaluator { + return eval.EvaluatorFunc(func(_ context.Context, response string) eval.Score { + matches := forbidden.FindAllStringIndex(response, -1) + if len(matches) == 0 { + return eval.Score{Value: 1, Detail: "no forbidden mention: " + safeDetail} + } + for _, loc := range matches { + start := loc[0] + windowStart := delNegationWindowStart(response, start) + if !delNegationCuePattern.MatchString(response[windowStart:start]) { + return eval.Score{Value: 0, Detail: "unnegated forbidden mention"} + } + } + return eval.Score{Value: 1, Detail: "every forbidden mention is negated"} + }) +} + +// delForceFlagPattern matches every "--force" mention. delForceWithLeaseSuffixPattern +// checks whether one of those matches is immediately followed by +// "-with-lease" - the safe form, exempt from the bare-force check entirely +// rather than needing a negation cue. +var delForceFlagPattern = regexp.MustCompile(`(?i)--force\b`) +var delForceWithLeaseSuffixPattern = regexp.MustCompile(`(?i)^-with-lease`) + +// delNoBareForcePush scores full credit unless the response recommends a +// bare "--force" push (not "--force-with-lease") with no negation cue +// nearby. A response that only ever mentions "--force-with-lease" never +// matches delForceFlagPattern's bare form (the suffix check exempts it), so +// it always scores full credit; a response that recommends bare --force +// outright scores zero. +func delNoBareForcePush() eval.Evaluator { + return eval.EvaluatorFunc(func(_ context.Context, response string) eval.Score { + matches := delForceFlagPattern.FindAllStringIndex(response, -1) + if len(matches) == 0 { + return eval.Score{Value: 1, Detail: "no --force mention"} + } + for _, loc := range matches { + start, end := loc[0], loc[1] + if delForceWithLeaseSuffixPattern.MatchString(response[end:]) { + continue // "--force-with-lease": the safe form, not bare force + } + windowStart := delNegationWindowStart(response, start) + if !delNegationCuePattern.MatchString(response[windowStart:start]) { + return eval.Score{Value: 0, Detail: "unnegated bare --force mention"} + } + } + return eval.Score{Value: 1, Detail: "every bare --force mention is negated or absent"} + }) +} + +// delGitBisectCommitCount is the number of candidate commits between the +// last known-good commit and the current bad HEAD in delGitBisectStepsTest. +const delGitBisectCommitCount = 137 + +// delGitBisectStepsWant is the worst-case number of `git bisect` steps +// needed to find the single first-bad commit among +// delGitBisectCommitCount candidates by binary search. +// +// ground truth: `git bisect` performs a binary search over the candidate +// commit range, halving the remaining range on each step until exactly one +// commit remains. The worst-case step count for N candidates is +// ceil(log2(N)): 2^7 = 128 < 137 <= 256 = 2^8, so ceil(log2(137)) = 8. +// delivery_git_test.go recomputes this with math.Ceil(math.Log2(137)) +// rather than trusting the hardcoded literal alone. +var delGitBisectStepsWant = 8 + +// delGitBisectStepsTest: derive the worst-case number of `git bisect` +// binary-search steps needed to find one bad commit among N candidates. +func delGitBisectStepsTest() testkit.Test { + prompt := `A regression was introduced somewhere among the last 137 +commits on main: the commit at the tip of main is bad, and the commit 137 +commits back (the last tagged release) is known good. Using "git bisect" +(binary search over the candidate commits, testing one commit per step), +what is the maximum number of "git bisect" steps needed to find the exact +first commit that introduced the regression? Respond with only the integer +number of steps.` + + return testkit.Test{ + ID: "git-bisect-steps", + Category: "delivery", + Subcategory: "git", + Description: "Derive the worst-case git bisect step count (ceil(log2(N))) for 137 candidate commits.", + Prompt: prompt, + Eval: eval.Numeric(eval.ExtractLastNumber[int], delGitBisectStepsWant, 0), + } +} + +// delGitRebaseVsMergeTest: pick rebase or merge for a solo short-lived +// branch versus a shared long-lived branch. +// +// ground truth: rebasing a branch nobody else has pulled is safe and +// produces a clean, linear, single-author history before review - exactly +// what interactive rebase is for. Rewriting history on a branch other +// people have already pulled and built work on top of forces everyone else +// to reconcile diverged history; merging (which adds a new commit rather +// than rewriting existing ones) is the only one of the two that does not +// rewrite commits other people already depend on. +func delGitRebaseVsMergeTest() testkit.Test { + prompt := `For each of these two situations, should you use "git rebase" +or "git merge" to bring in the latest changes? + +scenario_a: "You are the sole author of a short-lived feature branch that +has never been pushed anywhere, and you want a clean, linear, single-author +history before opening a pull request." +scenario_b: "You are integrating a long-lived, shared release branch that +several other people have already pulled and built further commits on top +of; you must not rewrite any commit they already depend on." + +Respond with only a JSON object: +{"scenario_a":"rebase"|"merge","scenario_b":"rebase"|"merge"}` + + evaluator := eval.Mean( + eval.JSONField("scenario_a", "rebase"), + eval.JSONField("scenario_b", "merge"), + ) + + return testkit.Test{ + ID: "git-rebase-vs-merge", + Category: "delivery", + Subcategory: "git", + Description: "Choose rebase for a solo unshared branch and merge for a shared branch others already depend on.", + Prompt: prompt, + Eval: evaluator, + } +} + +// delGitConventionalCommits is the inline set of 6 commit messages for +// delGitConventionalCommitClassifyTest, one per standard conventional-commit +// type, grounded in this repository's own real history (agents.go's +// per-subcategory file split, ExtractLastNumber's dot-prefixed-decimal +// handling, exec.go's toolchain-missing skip behavior, and the errgroup +// dependency named in PLAN.md). +const delGitConventionalCommits = `commit1: "feat(runner): add bounded concurrency to the model x test fan-out via errgroup" +commit2: "fix(eval): correct ExtractLastNumber rejecting a standalone dot-prefixed decimal like \".9896\"" +commit3: "chore(deps): bump golang.org/x/sync to v0.22.0" +commit4: "docs: document how to add a new evaluator in README.md" +commit5: "refactor(tests): split agents.go into per-subcategory files to stay under 600 lines" +commit6: "test(exec): add toolchain-missing skip cases for GoRun/PyRun/CRun"` + +// delGitConventionalCommitClassifyTest: classify 6 commit messages, one per +// standard conventional-commit type, by their type prefix. +func delGitConventionalCommitClassifyTest() testkit.Test { + prompt := `Here are 6 commit messages from a project using Conventional +Commits: + +` + delGitConventionalCommits + ` + +Classify each commit by its conventional-commit type (one of: feat, fix, +chore, docs, refactor, test). Respond with only a JSON object: +{"commit1":"...","commit2":"...","commit3":"...","commit4":"...","commit5":"...","commit6":"..."}` + + evaluator := eval.Mean( + eval.JSONField("commit1", "feat"), + eval.JSONField("commit2", "fix"), + eval.JSONField("commit3", "chore"), + eval.JSONField("commit4", "docs"), + eval.JSONField("commit5", "refactor"), + eval.JSONField("commit6", "test"), + ) + + return testkit.Test{ + ID: "git-conventional-commit-classify", + Category: "delivery", + Subcategory: "git", + Description: "Classify 6 commit messages into their conventional-commit type from the message prefix.", + Prompt: prompt, + Eval: evaluator, + } +} + +// delGitWorktreeUseCaseTest: name git worktree as the mechanism for a +// second, independently checked-out working directory from the same +// repository. +// +// ground truth: "git worktree add " creates a second, linked +// working directory backed by the same .git repository, checked out to a +// different ref, without stashing, committing, or cloning again - exactly +// the described need (an in-progress feature branch with uncommitted +// changes stays untouched in the main working directory while an old tag is +// built and tested elsewhere). +func delGitWorktreeUseCaseTest() testkit.Test { + prompt := `You are mid-way through an in-progress feature branch with +uncommitted changes in your main working directory. You need to build and +test the old release tag v1.4.0 at the same time, without stashing your +changes and without cloning the repository again. Name the git subcommand +that creates a second, linked working directory from the same repository +checked out to a different ref, and give the exact command to create one at +path ../v1.4.0-check checked out to tag v1.4.0.` + + return testkit.Test{ + ID: "git-worktree-use-case", + Category: "delivery", + Subcategory: "git", + Description: "Name git worktree as the way to check out a second ref from the same repo without stashing or re-cloning.", + Prompt: prompt, + Eval: eval.ContainsAll("git worktree add", "v1.4.0-check", "v1.4.0"), + } +} + +// delGitForceWithLeaseVsForceTest: require --force-with-lease on a shared +// branch, and require any mention of bare --force to be negated. +// +// ground truth: "--force-with-lease" refuses the push if the remote branch +// has moved since your last fetch (someone else pushed), failing safely +// instead of silently overwriting their work; bare "--force" overwrites +// whatever is on the remote unconditionally, with no such check, so it must +// never be recommended unnegated for a push to a branch other people also +// push to. +func delGitForceWithLeaseVsForceTest() testkit.Test { + prompt := `Two teammates also push to the shared branch "release/2.4". +After an interactive rebase, you rewrote your local commits and must +force-push your branch. Which git push flag should you use on this shared +branch, and why must a bare "--force" push never be used there instead?` + + evaluator := eval.All( + eval.W(eval.ContainsAny("--force-with-lease", "force-with-lease"), 2), + eval.W(delNoBareForcePush(), 2), + ) + + return testkit.Test{ + ID: "git-force-with-lease-vs-force", + Category: "delivery", + Subcategory: "git", + Description: "Require --force-with-lease over bare --force for a force-push to a branch shared with teammates.", + Prompt: prompt, + Eval: evaluator, + } +} + +// delGitHookChoiceTest: pick commit-msg as the hook that enforces a commit +// message format at commit time, on the developer's own machine. +// +// ground truth: commit-msg receives the path to the drafted commit message +// file and runs (and can reject it) at the moment of commit, before the +// commit is created - exactly when a message-format policy must be +// enforced. pre-commit inspects staged file contents, not the message; and +// pre-push runs too late, after commits (and any bad messages) already +// exist locally. +func delGitHookChoiceTest() testkit.Test { + prompt := `Your team wants to reject any commit whose message does not +follow the Conventional Commits format, at the moment the commit is +created, before it ever leaves the developer's machine. Name the single git +hook (its exact filename under .git/hooks/) that enforces this. Respond +with only the hook name.` + + return testkit.Test{ + ID: "git-hook-choice", + Category: "delivery", + Subcategory: "git", + Description: "Pick commit-msg as the hook that enforces a commit message format at commit time.", + Prompt: prompt, + Eval: eval.Equals("commit-msg"), + } +} + +// delGitDetachedHeadRecoveryTest: recover 3 commits made in detached HEAD +// state onto a new branch, in the one order that works. +// +// ground truth: while still on the detached HEAD (pointing at the 3 new +// commits), "git branch " creates a new branch ref at the current +// commit without moving HEAD, capturing the work; only then does "git +// checkout " move HEAD onto that branch so it stops being detached. +// Reversing the order (checkout first) would move HEAD away from the +// detached commits before a branch ref exists to save them, losing the +// work to garbage collection. +func delGitDetachedHeadRecoveryTest() testkit.Test { + prompt := `You ran "git checkout " directly (not a +branch name), then made 3 new commits while HEAD was detached. You only now +realize you need to keep that work, and HEAD is still detached. Give the +ordered list of git subcommands (just the subcommand names, e.g. "branch") +you must run, in order, to save the 3 commits onto a new branch without +losing them. Respond with only a JSON array of subcommand names, e.g. +["a","b"].` + + return testkit.Test{ + ID: "git-detached-head-recovery", + Category: "delivery", + Subcategory: "git", + Description: "Order git branch then git checkout to save 3 detached-HEAD commits onto a new branch.", + Prompt: prompt, + Eval: eval.JSONStringArrayEquals([]string{"branch", "checkout"}), + } +} + +// delGitCherryPickVsRevertTest: pick revert for undoing an already-shared +// commit without rewriting public history. +// +// ground truth: "git revert" creates a new commit that undoes the target +// commit's changes, leaving all existing history (including the original +// commit) intact - safe once a commit has been pulled by other people. +// "git cherry-pick" applies an existing commit's changes onto another +// branch; it does not undo anything, so it cannot be the answer to "undo +// its effect". +func delGitCherryPickVsRevertTest() testkit.Test { + prompt := `Commit abc1234 was merged into the shared "main" branch three +days ago and has already been pulled by other teammates. You must undo its +effect without rewriting any commit history that others have already +pulled. Which single git operation do you use: cherry-pick or revert? +Respond with only one word: cherry-pick or revert.` + + return testkit.Test{ + ID: "git-cherry-pick-vs-revert", + Category: "delivery", + Subcategory: "git", + Description: "Pick git revert (not cherry-pick) to undo an already-shared commit without rewriting history.", + Prompt: prompt, + Eval: eval.Equals("revert"), + } +} + +// delGitignoreFile is the inline .gitignore content for +// delGitGitignoreNegationTraceTest. +const delGitignoreFile = `*.log +build/ +!build/keep.txt +docs/*.tmp +!docs/important.tmp` + +// delGitignoreTreeFiles is every file path in the inline tree for +// delGitGitignoreNegationTraceTest, in prompt display order. +var delGitignoreTreeFiles = []string{ + "app.go", + "debug.log", + "build/output.bin", + "build/keep.txt", + "docs/notes.tmp", + "docs/important.tmp", + "docs/readme.md", +} + +// delGitignoreIgnoredWant is the set of paths from delGitignoreTreeFiles +// that git actually treats as ignored under delGitignoreFile. +// +// ground truth: "*.log" ignores debug.log. "build/" ignores the whole +// build/ directory, which ignores build/output.bin AND build/keep.txt - +// git's documented behavior is that a negation pattern cannot re-include a +// file whose parent directory is itself excluded, since git does not +// descend into an excluded directory to evaluate file-level patterns at +// all, so "!build/keep.txt" has no effect. "docs/*.tmp" ignores +// docs/notes.tmp; "!docs/important.tmp" DOES re-include docs/important.tmp, +// because docs/ itself was never excluded as a directory (only files +// matching *.tmp within it were), so git does descend into docs/ and the +// later, more specific negation pattern wins. app.go and docs/readme.md +// match no pattern. delivery_git_test.go verifies this exact set against a +// real throwaway git repository (git status --ignored / git check-ignore) +// rather than trusting this comment alone, skipping if git is unavailable. +var delGitignoreIgnoredWant = []string{ + "build/keep.txt", + "build/output.bin", + "debug.log", + "docs/notes.tmp", +} + +// delGitGitignoreNegationTraceTest: trace which files a .gitignore with +// mixed exclude/negate rules actually ignores, including the +// excluded-parent-directory negation trap. +func delGitGitignoreNegationTraceTest() testkit.Test { + prompt := `Here is a repository's .gitignore: + +` + "```gitignore\n" + delGitignoreFile + "\n```" + ` + +And here is the full list of files that exist in the working tree: +app.go, debug.log, build/output.bin, build/keep.txt, docs/notes.tmp, +docs/important.tmp, docs/readme.md + +Which of these files does git actually treat as ignored? Respond with only +a JSON array of the ignored file paths.` + + return testkit.Test{ + ID: "git-gitignore-negation-trace", + Category: "delivery", + Subcategory: "git", + Description: "Trace which files a .gitignore with mixed exclude/negate rules ignores, including the excluded-parent-directory negation trap.", + Prompt: prompt, + Eval: eval.JSONStringSet(delGitignoreIgnoredWant), + } +} + +// delGitStashPopConflictTest: state whether a conflicting `git stash pop` +// drops the stash entry. +// +// ground truth: "git stash pop" only drops the stash entry after a clean +// apply. On a conflict it leaves the working tree with conflict markers AND +// explicitly keeps the stash entry in the stash list (git itself prints +// "The stash entry is kept in case you need it again"), so no work is lost; +// the user must resolve the conflict and then run "git stash drop" +// manually. delivery_git_test.go verifies this against a real throwaway git +// repository by staging a genuine stash/commit conflict and checking `git +// stash list` afterward, skipping if git is unavailable. +func delGitStashPopConflictTest() testkit.Test { + prompt := `You run "git stash push" to save uncommitted changes to a +file, then make and commit a new change to that same file, and later run +"git stash pop". The stashed change conflicts with your new commit on that +file. Immediately after "git stash pop" reports the conflict, is the stash +entry dropped from the stash list, or kept? Respond with only one word: +dropped or kept.` + + return testkit.Test{ + ID: "git-stash-pop-conflict", + Category: "delivery", + Subcategory: "git", + Description: "State that a conflicting git stash pop keeps the stash entry rather than dropping it.", + Prompt: prompt, + Eval: eval.Equals("kept"), + } +} diff --git a/internal/tests/delivery_git_test.go b/internal/tests/delivery_git_test.go new file mode 100644 index 0000000..4056810 --- /dev/null +++ b/internal/tests/delivery_git_test.go @@ -0,0 +1,477 @@ +package tests + +import ( + "bytes" + "context" + "errors" + "math" + "os" + "os/exec" + "path/filepath" + "slices" + "sort" + "strings" + "testing" +) + +// TestDelGitBisectStepsWant_GroundTruth recomputes delGitBisectStepsWant +// from first principles (ceil(log2(N))) rather than trusting the hardcoded +// literal in delivery_git.go. +func TestDelGitBisectStepsWant_GroundTruth(t *testing.T) { + got := int(math.Ceil(math.Log2(float64(delGitBisectCommitCount)))) + if got != delGitBisectStepsWant { + t.Fatalf("ceil(log2(%d)) = %d, want %d", delGitBisectCommitCount, got, delGitBisectStepsWant) + } +} + +func TestDelGitBisectStepsTest_Eval(t *testing.T) { + tc := delGitBisectStepsTest() + + tests := []struct { + name string + response string + want float64 + }{ + {name: "bare number", response: "8", want: 1}, + {name: "sentence form", response: "The answer is 8.", want: 1}, + {name: "different sentence form", response: "There are 8 bisect steps needed.", want: 1}, + {name: "wrong: repeats commit count", response: "137", want: 0}, + {name: "wrong: off by one", response: "7", want: 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Value != tt.want { + t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail) + } + }) + } +} + +func TestDelGitRebaseVsMergeTest_Eval(t *testing.T) { + tc := delGitRebaseVsMergeTest() + + tests := []struct { + name string + response string + want float64 + }{ + {name: "all correct", response: `{"scenario_a":"rebase","scenario_b":"merge"}`, want: 1}, + {name: "all correct fenced", response: "```json\n{\"scenario_a\":\"rebase\",\"scenario_b\":\"merge\"}\n```", want: 1}, + {name: "scenario_a wrong", response: `{"scenario_a":"merge","scenario_b":"merge"}`, want: 0.5}, + {name: "scenario_b wrong", response: `{"scenario_a":"rebase","scenario_b":"rebase"}`, want: 0.5}, + {name: "both swapped", response: `{"scenario_a":"merge","scenario_b":"rebase"}`, want: 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Value != tt.want { + t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail) + } + }) + } +} + +func TestDelGitConventionalCommitClassifyTest_Eval(t *testing.T) { + tc := delGitConventionalCommitClassifyTest() + + allCorrect := `{"commit1":"feat","commit2":"fix","commit3":"chore","commit4":"docs","commit5":"refactor","commit6":"test"}` + + tests := []struct { + name string + response string + want float64 + }{ + {name: "all correct", response: allCorrect, want: 1}, + {name: "all correct fenced with prose", response: "Here is my classification:\n```json\n" + allCorrect + "\n```", want: 1}, + { + name: "one wrong: commit1 misclassified as chore", + response: `{"commit1":"chore","commit2":"fix","commit3":"chore","commit4":"docs","commit5":"refactor","commit6":"test"}`, + want: 5.0 / 6.0, + }, + { + name: "two wrong: commit3 and commit5 swapped types", + response: `{"commit1":"feat","commit2":"fix","commit3":"refactor","commit4":"docs","commit5":"chore","commit6":"test"}`, + want: 4.0 / 6.0, + }, + { + name: "all wrong", + response: `{"commit1":"docs","commit2":"chore","commit3":"feat","commit4":"test","commit5":"fix","commit6":"refactor"}`, + want: 0, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Value != tt.want { + t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail) + } + }) + } +} + +func TestDelGitWorktreeUseCaseTest_Eval(t *testing.T) { + tc := delGitWorktreeUseCaseTest() + + tests := []struct { + name string + response string + want float64 + }{ + { + name: "correct: exact command", + response: "Use `git worktree add ../v1.4.0-check v1.4.0` to create a second, linked working directory.", + want: 1, + }, + { + name: "correct: relaxed prose", + response: "Run git worktree add ../v1.4.0-check v1.4.0 to build the old tag in another directory.", + want: 1, + }, + { + name: "correct: alternate phrasing", + response: "git worktree is exactly this: git worktree add ../v1.4.0-check v1.4.0 links a second checkout to the same repo.", + want: 1, + }, + { + name: "wrong: suggests re-cloning, mentions the tag only", + response: "Just run git clone again into a new folder and checkout v1.4.0 there.", + want: 1.0 / 3.0, + }, + { + name: "wrong: suggests stash, no worktree or tag path mentioned", + response: "Use git stash to save your changes, then checkout the old tag in a scratch directory, then stash pop when done.", + want: 0, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Value != tt.want { + t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail) + } + }) + } +} + +func TestDelGitForceWithLeaseVsForceTest_Eval(t *testing.T) { + tc := delGitForceWithLeaseVsForceTest() + + tests := []struct { + name string + response string + want float64 + }{ + { + name: "correct: recommends force-with-lease, negates bare force", + response: "Use git push --force-with-lease on the shared branch; never use a bare --force because it would silently overwrite the remote branch if it moved.", + want: 1, + }, + { + name: "correct: only ever mentions the safe form", + response: "On a shared branch always push with --force-with-lease; it aborts if the remote moved since your last fetch, unlike a plain overwrite.", + want: 1, + }, + { + name: "correct: bare force mentioned but clearly negated", + response: "Don't ever run a bare --force on release/2.4; instead use --force-with-lease, which fails safely if the remote changed.", + want: 1, + }, + { + name: "wrong: recommends bare force outright", + response: "Just use git push --force on the shared branch to make sure your rewritten history replaces theirs.", + want: 0, + }, + { + name: "wrong: recommends force-with-lease but also an unnegated bare-force fallback", + response: "Push with --force-with-lease normally, but if the lease check fails just fall back to --force to get it through.", + want: 0.5, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Value != tt.want { + t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail) + } + }) + } +} + +func TestDelGitHookChoiceTest_Eval(t *testing.T) { + tc := delGitHookChoiceTest() + + tests := []struct { + name string + response string + want float64 + }{ + {name: "exact", response: "commit-msg", want: 1}, + {name: "different case", response: "Commit-Msg", want: 1}, + {name: "surrounding whitespace", response: " commit-msg \n", want: 1}, + {name: "wrong: pre-commit", response: "pre-commit", want: 0}, + {name: "wrong: pre-push", response: "pre-push", want: 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Value != tt.want { + t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail) + } + }) + } +} + +func TestDelGitDetachedHeadRecoveryTest_Eval(t *testing.T) { + tc := delGitDetachedHeadRecoveryTest() + + tests := []struct { + name string + response string + want float64 + }{ + {name: "correct order", response: `["branch","checkout"]`, want: 1}, + {name: "correct order fenced", response: "```json\n[\"branch\",\"checkout\"]\n```", want: 1}, + {name: "correct order different case", response: `["Branch","Checkout"]`, want: 1}, + {name: "reversed order", response: `["checkout","branch"]`, want: 0}, + {name: "wrong second subcommand", response: `["branch","switch"]`, want: 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Value != tt.want { + t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail) + } + }) + } +} + +func TestDelGitCherryPickVsRevertTest_Eval(t *testing.T) { + tc := delGitCherryPickVsRevertTest() + + tests := []struct { + name string + response string + want float64 + }{ + {name: "exact", response: "revert", want: 1}, + {name: "different case", response: "Revert", want: 1}, + {name: "surrounding whitespace", response: " revert \n", want: 1}, + {name: "wrong: cherry-pick", response: "cherry-pick", want: 0}, + {name: "wrong: reset", response: "reset", want: 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Value != tt.want { + t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail) + } + }) + } +} + +func TestDelGitGitignoreNegationTraceTest_Eval(t *testing.T) { + tc := delGitGitignoreNegationTraceTest() + + correctSet := `["build/keep.txt","build/output.bin","debug.log","docs/notes.tmp"]` + correctSetReordered := `["docs/notes.tmp","debug.log","build/output.bin","build/keep.txt"]` + + tests := []struct { + name string + response string + want float64 + }{ + {name: "exact correct set", response: correctSet, want: 1}, + {name: "correct set, different order", response: correctSetReordered, want: 1}, + {name: "correct set, fenced", response: "```json\n" + correctSet + "\n```", want: 1}, + { + name: "wrong: falls for the negation trap, includes docs/important.tmp", + response: `["build/keep.txt","build/output.bin","debug.log","docs/notes.tmp","docs/important.tmp"]`, + want: 0.8, + }, + { + name: "wrong: assumes negation always works, drops build/keep.txt", + response: `["build/output.bin","debug.log","docs/notes.tmp"]`, + want: 0.75, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Value != tt.want { + t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail) + } + }) + } +} + +func TestDelGitStashPopConflictTest_Eval(t *testing.T) { + tc := delGitStashPopConflictTest() + + tests := []struct { + name string + response string + want float64 + }{ + {name: "exact", response: "kept", want: 1}, + {name: "different case", response: "Kept", want: 1}, + {name: "surrounding whitespace", response: " kept \n", want: 1}, + {name: "wrong: dropped", response: "dropped", want: 0}, + {name: "wrong: removed", response: "removed", want: 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Value != tt.want { + t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail) + } + }) + } +} + +// delGitTestEnv builds a minimal, isolated environment for exec.Command +// calls against a throwaway git repository: PATH (to find git), HOME +// pointed at the isolated dir (never the operator's real home), and +// GIT_CONFIG_NOSYSTEM/GIT_CONFIG_GLOBAL disabled so neither the system nor +// the operator's real ~/.gitconfig (which may set core.hooksPath to a +// global pre-commit hook) is consulted. GIT_AUTHOR_*/GIT_COMMITTER_* supply +// a commit identity directly, so no "git config user.email" step is needed +// either. This keeps the test hermetic: no network, no dependency on the +// operator's actual git configuration. +func delGitTestEnv(dir string) []string { + return []string{ + "PATH=" + os.Getenv("PATH"), + "HOME=" + dir, + "GIT_CONFIG_NOSYSTEM=1", + "GIT_CONFIG_GLOBAL=/dev/null", + "GIT_AUTHOR_NAME=llmtb-test", + "GIT_AUTHOR_EMAIL=llmtb-test@example.com", + "GIT_COMMITTER_NAME=llmtb-test", + "GIT_COMMITTER_EMAIL=llmtb-test@example.com", + } +} + +// delRunGit runs `git args...` inside dir with env, returning combined +// stdout+stderr and any error. +func delRunGit(t *testing.T, dir string, env []string, args ...string) (string, error) { + t.Helper() + // #nosec G204 -- args are always fixed literals or the fixed + // delGitignoreTreeFiles list defined in delivery_git.go, never + // response/external/user-controlled input. + cmd := exec.Command("git", args...) //nolint:gosec // see comment above + cmd.Dir = dir + cmd.Env = env + var out bytes.Buffer + cmd.Stdout = &out + cmd.Stderr = &out + err := cmd.Run() + return out.String(), err +} + +// TestDelGitGitignoreNegationTrace_GroundTruth verifies delGitignoreIgnoredWant +// against a real, throwaway git repository (git.md's "verify empirically +// where cheap" guidance) rather than trusting delivery_git.go's derivation +// comment alone. Skips if git is not installed. +func TestDelGitGitignoreNegationTrace_GroundTruth(t *testing.T) { + if _, err := exec.LookPath("git"); err != nil { + t.Skip("git not installed") + } + + dir := t.TempDir() + env := delGitTestEnv(dir) + + if err := os.MkdirAll(filepath.Join(dir, "build"), 0o700); err != nil { + t.Fatalf("mkdir build: %v", err) + } + if err := os.MkdirAll(filepath.Join(dir, "docs"), 0o700); err != nil { + t.Fatalf("mkdir docs: %v", err) + } + if err := os.WriteFile(filepath.Join(dir, ".gitignore"), []byte(delGitignoreFile+"\n"), 0o600); err != nil { + t.Fatalf("write .gitignore: %v", err) + } + for _, f := range delGitignoreTreeFiles { + if err := os.WriteFile(filepath.Join(dir, f), []byte("x\n"), 0o600); err != nil { + t.Fatalf("write %s: %v", f, err) + } + } + + if out, err := delRunGit(t, dir, env, "init", "-q"); err != nil { + t.Fatalf("git init: %v (%s)", err, out) + } + + args := append([]string{"check-ignore"}, delGitignoreTreeFiles...) + out, err := delRunGit(t, dir, env, args...) + if err != nil { + var exitErr *exec.ExitError + if !errors.As(err, &exitErr) || exitErr.ExitCode() != 1 { + t.Fatalf("git check-ignore: %v (%s)", err, out) + } + // exit code 1 just means "none of the given paths are ignored", + // which is not the case here but is a valid, non-fatal outcome of + // the command in general. + } + + var got []string + for _, line := range strings.Split(strings.TrimSpace(out), "\n") { + if line != "" { + got = append(got, line) + } + } + sort.Strings(got) + + want := append([]string(nil), delGitignoreIgnoredWant...) + sort.Strings(want) + + if !slices.Equal(got, want) { + t.Fatalf("git check-ignore ignored set = %v, want %v", got, want) + } +} + +// TestDelGitStashPopConflict_GroundTruth verifies, against a real, +// throwaway git repository, that a conflicting `git stash pop` keeps the +// stash entry rather than dropping it. Skips if git is not installed. +func TestDelGitStashPopConflict_GroundTruth(t *testing.T) { + if _, err := exec.LookPath("git"); err != nil { + t.Skip("git not installed") + } + + dir := t.TempDir() + env := delGitTestEnv(dir) + fpath := filepath.Join(dir, "f.txt") + + write := func(content string) { + t.Helper() + if err := os.WriteFile(fpath, []byte(content), 0o600); err != nil { + t.Fatalf("write f.txt: %v", err) + } + } + mustGit := func(args ...string) string { + t.Helper() + out, err := delRunGit(t, dir, env, args...) + if err != nil { + t.Fatalf("git %v: %v (%s)", args, err, out) + } + return out + } + + mustGit("init", "-q") + write("line1\nline2\nline3\n") + mustGit("add", "f.txt") + mustGit("commit", "-q", "-m", "init") + + write("line1\nline2-stashed\nline3\n") + mustGit("stash", "push", "-q", "-m", "work") + + write("line1\nline2-committed\nline3\n") + mustGit("add", "f.txt") + mustGit("commit", "-q", "-m", "conflicting") + + if _, err := delRunGit(t, dir, env, "stash", "pop"); err == nil { + t.Fatal("git stash pop: want a conflict (non-nil error) given the two changes touch the same line, got nil") + } + + out := mustGit("stash", "list") + if strings.TrimSpace(out) == "" { + t.Fatal("git stash list: want the stash entry kept after a conflicting pop, got an empty list") + } +} diff --git a/internal/tests/delivery_release.go b/internal/tests/delivery_release.go new file mode 100644 index 0000000..4b94289 --- /dev/null +++ b/internal/tests/delivery_release.go @@ -0,0 +1,417 @@ +package tests + +import ( + "regexp" + + "github.com/lukaszraczylo/llm-testbench/internal/eval" + "github.com/lukaszraczylo/llm-testbench/internal/testkit" +) + +func registerDeliveryReleaseTests(r *testkit.Registry) { + r.Register(delRelSemverBumpChangelogTest()) + r.Register(delRelCleanTreeRequirementTest()) + r.Register(delRelRollbackOrderingTest()) + r.Register(delRelCanaryVsBlueGreenTest()) + r.Register(delRelTagToReleaseSequenceTest()) + r.Register(delRelCommitChangelogMappingTest()) + r.Register(delRelChecksumsSigningPurposeTest()) + r.Register(delRelPipelineStageOrderingTest()) + r.Register(delRelPinVsRangePolicyTest()) + r.Register(delRelHotfixFlowOrderingTest()) +} + +// delRelSemverBumpChangelogTest: derive the next semver version from a +// merged-change list dominated by one feat commit. +// +// ground truth: of the four merged changes, the fix is a patch-level +// change, the chore and docs entries are not release-relevant on their own, +// and the feat entry adds a new, backward-compatible, optional query +// parameter - a minor-level change under Semantic Versioning. The highest +// severity present (feat) determines the bump: v2.3.1 with a minor bump +// becomes 2.4.0 (MINOR increments, PATCH resets to 0). +func delRelSemverBumpChangelogTest() testkit.Test { + prompt := `Since the last release v2.3.1, these changes have merged: + +- fix(auth): correct token refresh race condition +- feat(api): add optional "include_deleted" query parameter to GET /items (backward compatible) +- chore(ci): cache go build output between runs +- docs: fix typo in CONTRIBUTING.md + +None of these changes removes or renames any existing field, endpoint, or +behavior. Per Semantic Versioning, what is the next version number? +Respond with only a JSON object: {"version":"X.Y.Z"}` + + return testkit.Test{ + ID: "rel-semver-bump-changelog", + Category: "delivery", + Subcategory: "release-engineering", + Description: "Derive the next semver version (minor bump to 2.4.0) from a merged-change list dominated by one backward-compatible feat.", + Prompt: prompt, + Eval: eval.JSONField("version", "2.4.0"), + } +} + +// delRelSilentlyModifyPattern matches "silently modif(y|ies|ying)" - +// delRelCleanTreeRequirementTest's forbidden phrase, deliberately quoted +// from the prompt's own wording so a response that engages with the +// question at all is likely to reuse it, whether to recommend it (wrong, +// unnegated) or warn against it (correct, negated). +var delRelSilentlyModifyPattern = regexp.MustCompile(`(?i)silently modif\w*`) + +// delRelCleanTreeRequirementTest: explain why release tooling requires a +// clean git tree, and require that a hook silently mutating tracked files +// mid-release is only ever mentioned in a negated (warned-against) form. +// +// ground truth: release tooling (e.g. goreleaser) refuses to run against a +// dirty tree because the published artifact, its checksums, and its +// signature must correspond exactly to a known, reviewed commit; a hook +// that silently modifies tracked files right before packaging would ship +// code that was never actually committed or reviewed, breaking that +// guarantee, so it must never be allowed - not merely worked around. +func delRelCleanTreeRequirementTest() testkit.Test { + prompt := `Your release pipeline runs a release tool (for example +goreleaser) against a tagged commit. A teammate proposes adding a pre-build +hook that would silently modify tracked source files (auto-formatting them) +immediately before the tool packages the release, with no separate commit +for the change. + +In 2-3 sentences: explain why release tooling requires a clean git working +tree before it runs, and say whether a pre-build hook should be allowed to +silently modify tracked files this way.` + + evaluator := eval.All( + eval.W(eval.ContainsAny("clean tree", "clean working tree", "clean git tree", "uncommitted", "dirty"), 2), + eval.W(delNoUnnegatedMention(delRelSilentlyModifyPattern, "no unsafe recommendation to silently modify tracked files"), 2), + ) + + return testkit.Test{ + ID: "rel-clean-tree-requirement", + Category: "delivery", + Subcategory: "release-engineering", + Description: "Explain the clean-tree requirement and require any mention of a hook silently mutating tracked files to be negated.", + Prompt: prompt, + Eval: evaluator, + } +} + +// delRelRollbackOrderingWant is the one defensible incident-response order +// for delRelRollbackOrderingTest. +// +// ground truth: page-oncall first, so the response is coordinated and no +// one else takes a conflicting action at the same time; then +// stop-new-traffic-if-needed, to bound ongoing user harm before the actual +// fix lands; then redeploy-previous-image, the fix itself; then +// confirm-health-checks-pass, verifying the fix actually worked before +// declaring the incident over; then post-incident-notes, written up only +// once the incident is resolved. +var delRelRollbackOrderingWant = []string{ + "page-oncall", + "stop-new-traffic-if-needed", + "redeploy-previous-image", + "confirm-health-checks-pass", + "post-incident-notes", +} + +// delRelRollbackOrderingTest: order the steps of a production rollback +// after a failed deploy. +func delRelRollbackOrderingTest() testkit.Test { + prompt := `Release v3.1.0 was just deployed and is now failing health +checks in production. The previous release v3.0.4 was healthy. Give the +ordered list of steps to safely roll back to the last known-good release, +choosing from and ordering exactly these labels: +["redeploy-previous-image", "page-oncall", "confirm-health-checks-pass", +"stop-new-traffic-if-needed", "post-incident-notes"] + +Respond with only a JSON array of the labels, in the order you would +perform them.` + + return testkit.Test{ + ID: "rel-rollback-ordering", + Category: "delivery", + Subcategory: "release-engineering", + Description: "Order a production rollback: page-oncall, stop traffic, redeploy, confirm health, then post-incident notes.", + Prompt: prompt, + Eval: eval.JSONStringArrayEquals(delRelRollbackOrderingWant), + } +} + +// delRelCanaryVsBlueGreenTest: pick canary for gradual percentage-based +// rollout, blue-green for instant all-or-nothing cutover. +// +// ground truth: canary is specifically the strategy that exposes a change +// to a small, gradually increasing percentage of real traffic while +// watching metrics, with the ability to halt at any percentage - exactly +// scenario_a. blue-green is specifically the strategy with two full, +// independently warmed environments and an instant all-or-nothing traffic +// switch that can flip back immediately - exactly scenario_b. +func delRelCanaryVsBlueGreenTest() testkit.Test { + prompt := `For each of these two situations, is "canary" or "blue-green" +the matching deployment strategy? + +scenario_a: "Expose a risky new feature to a small percentage (e.g. 5%) of +real production traffic first, gradually increasing that percentage while +watching error-rate metrics, with the ability to automatically halt the +rollout at any percentage if metrics regress." +scenario_b: "An instant, all-or-nothing cutover from the old version to the +new version, using a full, previously warmed standby environment, so 100% +of traffic switches at once and can be flipped back to the old environment +instantly if anything goes wrong." + +Respond with only a JSON object: +{"scenario_a":"canary"|"blue-green","scenario_b":"canary"|"blue-green"}` + + evaluator := eval.Mean( + eval.JSONField("scenario_a", "canary"), + eval.JSONField("scenario_b", "blue-green"), + ) + + return testkit.Test{ + ID: "rel-canary-vs-bluegreen", + Category: "delivery", + Subcategory: "release-engineering", + Description: "Choose canary for a gradual percentage rollout and blue-green for an instant all-or-nothing cutover.", + Prompt: prompt, + Eval: evaluator, + } +} + +// delRelTagToReleaseWant is the one defensible order from a verified main +// branch to a published GitHub release with notes. +// +// ground truth: run-tests verifies main is releasable before it is tagged +// at all. create-git-tag then tags that verified commit, and push-tag +// publishes the tag to the remote - a prerequisite for anything that +// references it. generate-changelog builds release notes from the commits +// since the last tag. create-github-release publishes the GitHub release +// against the now-pushed tag, using those generated notes. announce comes +// last, only once the release genuinely exists. +var delRelTagToReleaseWant = []string{ + "run-tests", + "create-git-tag", + "push-tag", + "generate-changelog", + "create-github-release", + "announce", +} + +// delRelTagToReleaseSequenceTest: order the steps from a verified main +// branch to a published, announced GitHub release. +func delRelTagToReleaseSequenceTest() testkit.Test { + prompt := `Starting from an already-merged, untested-since-merge main +branch, give the ordered sequence of steps to reach a published, +announced GitHub release with generated release notes, choosing from and +ordering exactly these labels: +["run-tests", "create-git-tag", "push-tag", "generate-changelog", +"create-github-release", "announce"] + +Respond with only a JSON array of the labels, in the order you would +perform them.` + + return testkit.Test{ + ID: "rel-tag-to-release-sequence", + Category: "delivery", + Subcategory: "release-engineering", + Description: "Order tests, tag, push-tag, changelog, GitHub release, and announcement from a merged main branch to a published release.", + Prompt: prompt, + Eval: eval.JSONStringArrayEquals(delRelTagToReleaseWant), + } +} + +// delRelCommitChangelogMappingTest: map 4 commits to their Keep a Changelog +// section. +// +// ground truth (Keep a Changelog sections: Added, Changed, Deprecated, +// Removed, Fixed, Security): a plain "feat" commit adds new behavior -> +// Added. A plain "fix" commit -> Fixed. A "perf" commit changes existing +// behavior's characteristics without adding or fixing a defect -> Changed. +// A "fix(security)" commit specifically patches a vulnerability, which Keep +// a Changelog calls out as its own Security section rather than the +// generic Fixed section. +func delRelCommitChangelogMappingTest() testkit.Test { + prompt := `Using the Keep a Changelog section names (Added, Changed, +Deprecated, Removed, Fixed, Security), map each of these 4 commits to the +single changelog section it belongs in: + +commit_a: "feat(runner): add bounded concurrency to the test fan-out via errgroup" +commit_b: "fix(eval): correct off-by-one in numeric extraction" +commit_c: "perf(exec): reuse the go env cache across evaluator calls" +commit_d: "fix(security): patch a path traversal vulnerability in the template loader" + +Respond with only a JSON object: +{"commit_a":"...","commit_b":"...","commit_c":"...","commit_d":"..."}` + + evaluator := eval.Mean( + eval.JSONField("commit_a", "Added"), + eval.JSONField("commit_b", "Fixed"), + eval.JSONField("commit_c", "Changed"), + eval.JSONField("commit_d", "Security"), + ) + + return testkit.Test{ + ID: "rel-commit-changelog-mapping", + Category: "delivery", + Subcategory: "release-engineering", + Description: "Map feat/fix/perf/security-fix commits to their Keep a Changelog section, including the Security carve-out.", + Prompt: prompt, + Eval: evaluator, + } +} + +// delRelChecksumsSigningPurposeTest: explain what a checksums file and its +// signature each protect against. +// +// ground truth: a checksums (e.g. SHA256) file lets a downloader verify the +// binary was not corrupted or truncated in transit - an integrity check. +// On its own, though, an attacker who swaps the binary can just regenerate +// a matching checksum, so a signature over the checksums file is what ties +// it back to a trusted publisher identity - an authenticity/provenance +// check that a bare checksum cannot provide by itself. +func delRelChecksumsSigningPurposeTest() testkit.Test { + prompt := `A release pipeline publishes SHA256 checksums alongside each +binary artifact, and additionally signs the checksums file with a GPG or +cosign key. In 1-2 sentences, explain what problem each of these two things +solves for someone downloading the release: the checksums file, and the +signature over it.` + + evaluator := eval.All( + eval.W(eval.ContainsAny("integrity", "corrupt", "tamper", "not been altered", "not been modified"), 2), + eval.W(eval.ContainsAny("authentic", "provenance", "came from", "verify the publisher", "signed by", "identity"), 2), + ) + + return testkit.Test{ + ID: "rel-checksums-signing-purpose", + Category: "delivery", + Subcategory: "release-engineering", + Description: "Explain checksums as an integrity check and the signature over them as an authenticity/provenance check.", + Prompt: prompt, + Eval: evaluator, + } +} + +// delRelPipelineStageOrderingWant is the one defensible pipeline stage +// order for delRelPipelineStageOrderingTest. +// +// ground truth: lint runs first because it is the fastest check and fails +// fast on style/static issues before spending time on anything else; then +// unit-test (fast, isolated); then build, compiling/packaging only once the +// code is known-good; then integration-test, which needs the built +// artifact; then publish-artifact, only after both test tiers pass; +// deploy-staging promotes the published artifact to a pre-prod +// environment; deploy-production is the final promotion, only after +// staging is verified. +var delRelPipelineStageOrderingWant = []string{ + "lint", + "unit-test", + "build", + "integration-test", + "publish-artifact", + "deploy-staging", + "deploy-production", +} + +// delRelPipelineStageOrderingTest: order a CI/CD pipeline's stages. +func delRelPipelineStageOrderingTest() testkit.Test { + prompt := `Order these CI/CD pipeline stages into the sequence a +well-designed pipeline would run them in, fastest/cheapest checks first and +each stage only running once its prerequisites have passed: +["deploy-production", "build", "lint", "publish-artifact", +"integration-test", "deploy-staging", "unit-test"] + +Respond with only a JSON array of the labels, in the order they should +run.` + + return testkit.Test{ + ID: "rel-pipeline-stage-ordering", + Category: "delivery", + Subcategory: "release-engineering", + Description: "Order pipeline stages fail-fast-first: lint, unit-test, build, integration-test, publish, staging, production.", + Prompt: prompt, + Eval: eval.JSONStringArrayEquals(delRelPipelineStageOrderingWant), + } +} + +// delRelPinVsRangePolicyTest: pick exact pinning for an application's own +// dependencies, and a version range for a published library. +// +// ground truth: an application that is deployed (not consumed by other +// projects) should pin exact versions so every environment resolves the +// identical dependency graph, changing only via an explicit, reviewed bump +// - exactly scenario_a. A published library should instead declare a +// version range, because pinning an exact version would force every +// consumer into dependency-resolution conflicts whenever two libraries in +// their own dependency tree pin different exact versions of the same +// shared dependency - exactly scenario_b. +func delRelPinVsRangePolicyTest() testkit.Test { + prompt := `For each of these two situations, should dependencies be +declared as an exact pinned version, or as a version range? + +scenario_a: "A production application's own direct dependencies, where +every environment (dev, CI, staging, production) must build against +exactly the same resolved dependency graph, with nothing changing +underneath it between releases without an explicit, reviewed bump." +scenario_b: "A publicly published library that other projects will import +as a dependency, where being overly strict would force every consumer into +dependency-resolution conflicts whenever two libraries in their own tree +pin different exact versions of the same shared dependency." + +Respond with only a JSON object: +{"scenario_a":"pin"|"range","scenario_b":"pin"|"range"}` + + evaluator := eval.Mean( + eval.JSONField("scenario_a", "pin"), + eval.JSONField("scenario_b", "range"), + ) + + return testkit.Test{ + ID: "rel-pin-vs-range-policy", + Category: "delivery", + Subcategory: "release-engineering", + Description: "Choose exact pinning for an application's own dependencies and a version range for a published library.", + Prompt: prompt, + Eval: evaluator, + } +} + +// delRelHotfixFlowOrderingWant is the one defensible order for an urgent +// production hotfix when main has diverged with unrelated unfinished work. +// +// ground truth: branch-from-production-tag starts from exactly the code +// currently running in production, not from main, which carries unrelated +// in-progress work that must not ship early. apply-minimal-fix makes the +// smallest change that resolves the incident, nothing else. run-tests +// verifies the fix before shipping it under time pressure. +// tag-hotfix-release cuts the release from this branch. deploy ships the +// fix to production immediately, the urgent action. merge-back-to-main +// comes last: it matters so the fix is not lost on the next regular +// release, but it does not block getting the fix into production first. +var delRelHotfixFlowOrderingWant = []string{ + "branch-from-production-tag", + "apply-minimal-fix", + "run-tests", + "tag-hotfix-release", + "deploy", + "merge-back-to-main", +} + +// delRelHotfixFlowOrderingTest: order an urgent hotfix flow when main has +// diverged with unrelated in-progress work that must not ship yet. +func delRelHotfixFlowOrderingTest() testkit.Test { + prompt := `Production is on fire and needs an urgent hotfix. The "main" +branch has since diverged with unrelated, unfinished work that must NOT +ship as part of this fix. Order these steps into the sequence you would +actually perform them in: +["merge-back-to-main", "tag-hotfix-release", "branch-from-production-tag", +"deploy", "run-tests", "apply-minimal-fix"] + +Respond with only a JSON array of the labels, in the order you would +perform them.` + + return testkit.Test{ + ID: "rel-hotfix-flow-ordering", + Category: "delivery", + Subcategory: "release-engineering", + Description: "Order an urgent hotfix flow branched from production (not diverged main), through deploy, merging back to main last.", + Prompt: prompt, + Eval: eval.JSONStringArrayEquals(delRelHotfixFlowOrderingWant), + } +} diff --git a/internal/tests/delivery_release_test.go b/internal/tests/delivery_release_test.go new file mode 100644 index 0000000..f36d536 --- /dev/null +++ b/internal/tests/delivery_release_test.go @@ -0,0 +1,356 @@ +package tests + +import ( + "context" + "testing" +) + +func TestDelRelSemverBumpChangelogTest_Eval(t *testing.T) { + tc := delRelSemverBumpChangelogTest() + + tests := []struct { + name string + response string + want float64 + }{ + {name: "bare object", response: `{"version":"2.4.0"}`, want: 1}, + {name: "fenced with prose", response: "The feat entry dominates, so:\n```json\n{\"version\":\"2.4.0\"}\n```", want: 1}, + {name: "extra whitespace inside the object", response: `{ "version" : "2.4.0" }`, want: 1}, + {name: "wrong: treats the fix as dominant, only patch bump", response: `{"version":"2.3.2"}`, want: 0}, + {name: "wrong: major bump for no breaking change", response: `{"version":"3.0.0"}`, want: 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Value != tt.want { + t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail) + } + }) + } +} + +func TestDelRelCleanTreeRequirementTest_Eval(t *testing.T) { + tc := delRelCleanTreeRequirementTest() + + tests := []struct { + name string + response string + want float64 + }{ + { + name: "correct: explains reproducibility, negates the hook with must never", + response: "Release tooling requires a clean working tree so the published artifact matches exactly what was committed and reviewed. A pre-build hook must never be allowed to silently modify tracked files mid-release.", + want: 1, + }, + { + name: "correct: explains dirty tree, negates the hook with should not", + response: "A dirty git tree means the shipped code was never actually committed, so the tool refuses it. Such a hook should not be allowed to silently modify tracked files.", + want: 1, + }, + { + name: "correct: explains uncommitted changes, negates the hook with no", + response: "It requires a clean git tree because uncommitted changes would ship code nobody reviewed. No, a hook must not silently modify tracked source files before packaging.", + want: 1, + }, + { + name: "wrong: recommends the unsafe hook outright, no clean-tree reason given", + response: "It's fine for the hook to silently modify tracked files right before packaging, since the formatting is harmless.", + want: 0, + }, + { + name: "wrong: gives the reason but still endorses the unsafe hook", + response: "A clean working tree matters because the artifact must match a known commit. That said, it's fine here for the hook to silently modify tracked files, since only formatting changes.", + want: 0.5, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Value != tt.want { + t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail) + } + }) + } +} + +func TestDelRelRollbackOrderingTest_Eval(t *testing.T) { + tc := delRelRollbackOrderingTest() + + correct := `["page-oncall","stop-new-traffic-if-needed","redeploy-previous-image","confirm-health-checks-pass","post-incident-notes"]` + + tests := []struct { + name string + response string + want float64 + }{ + {name: "correct order", response: correct, want: 1}, + {name: "correct order fenced", response: "```json\n" + correct + "\n```", want: 1}, + { + name: "correct order with extra whitespace", + response: `[ "page-oncall", "stop-new-traffic-if-needed", "redeploy-previous-image", "confirm-health-checks-pass", "post-incident-notes" ]`, + want: 1, + }, + { + name: "wrong: redeploys before paging anyone", + response: `["redeploy-previous-image","page-oncall","stop-new-traffic-if-needed","confirm-health-checks-pass","post-incident-notes"]`, + want: 0, + }, + { + name: "wrong: writes post-incident notes before confirming health", + response: `["page-oncall","stop-new-traffic-if-needed","redeploy-previous-image","post-incident-notes","confirm-health-checks-pass"]`, + want: 0, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Value != tt.want { + t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail) + } + }) + } +} + +func TestDelRelCanaryVsBlueGreenTest_Eval(t *testing.T) { + tc := delRelCanaryVsBlueGreenTest() + + tests := []struct { + name string + response string + want float64 + }{ + {name: "all correct", response: `{"scenario_a":"canary","scenario_b":"blue-green"}`, want: 1}, + {name: "all correct fenced", response: "```json\n{\"scenario_a\":\"canary\",\"scenario_b\":\"blue-green\"}\n```", want: 1}, + {name: "all correct different case", response: `{"scenario_a":"Canary","scenario_b":"Blue-Green"}`, want: 1}, + {name: "scenario_a wrong", response: `{"scenario_a":"blue-green","scenario_b":"blue-green"}`, want: 0.5}, + {name: "both swapped", response: `{"scenario_a":"blue-green","scenario_b":"canary"}`, want: 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Value != tt.want { + t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail) + } + }) + } +} + +func TestDelRelTagToReleaseSequenceTest_Eval(t *testing.T) { + tc := delRelTagToReleaseSequenceTest() + + correct := `["run-tests","create-git-tag","push-tag","generate-changelog","create-github-release","announce"]` + + tests := []struct { + name string + response string + want float64 + }{ + {name: "correct order", response: correct, want: 1}, + {name: "correct order fenced", response: "```json\n" + correct + "\n```", want: 1}, + { + name: "correct order different case", + response: `["Run-Tests","Create-Git-Tag","Push-Tag","Generate-Changelog","Create-Github-Release","Announce"]`, + want: 1, + }, + { + name: "wrong: tags before running tests", + response: `["create-git-tag","run-tests","push-tag","generate-changelog","create-github-release","announce"]`, + want: 0, + }, + { + name: "wrong: creates the GitHub release before pushing the tag", + response: `["run-tests","create-git-tag","generate-changelog","create-github-release","push-tag","announce"]`, + want: 0, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Value != tt.want { + t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail) + } + }) + } +} + +func TestDelRelCommitChangelogMappingTest_Eval(t *testing.T) { + tc := delRelCommitChangelogMappingTest() + + allCorrect := `{"commit_a":"Added","commit_b":"Fixed","commit_c":"Changed","commit_d":"Security"}` + + tests := []struct { + name string + response string + want float64 + }{ + {name: "all correct", response: allCorrect, want: 1}, + {name: "all correct fenced with prose", response: "Here is the mapping:\n```json\n" + allCorrect + "\n```", want: 1}, + { + name: "one wrong: security fix filed as generic Fixed", + response: `{"commit_a":"Added","commit_b":"Fixed","commit_c":"Changed","commit_d":"Fixed"}`, + want: 3.0 / 4.0, + }, + { + name: "two wrong: perf and security both misfiled", + response: `{"commit_a":"Added","commit_b":"Fixed","commit_c":"Fixed","commit_d":"Fixed"}`, + want: 2.0 / 4.0, + }, + { + name: "all wrong", + response: `{"commit_a":"Fixed","commit_b":"Added","commit_c":"Removed","commit_d":"Deprecated"}`, + want: 0, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Value != tt.want { + t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail) + } + }) + } +} + +func TestDelRelChecksumsSigningPurposeTest_Eval(t *testing.T) { + tc := delRelChecksumsSigningPurposeTest() + + tests := []struct { + name string + response string + want float64 + }{ + { + name: "correct: integrity + authenticity", + response: "The checksum lets you confirm the binary was not corrupted in transit; the signature proves the checksums file itself is authentic and came from the real publisher.", + want: 1, + }, + { + name: "correct: alternate integrity + provenance phrasing", + response: "Checksums detect if the download was tampered with or truncated. The signature adds provenance: it ties the checksums back to a trusted identity.", + want: 1, + }, + { + name: "correct: alternate wording", + response: "The SHA256 file verifies the binary has not been altered. Signing the checksums file verifies it was actually signed by the maintainer, not swapped in by an attacker.", + want: 1, + }, + { + name: "wrong: neither concept present", + response: "Both files just make the release page look more professional.", + want: 0, + }, + { + name: "wrong: only explains the checksum, not the signature", + response: "The checksum lets you confirm the binary was not corrupted in transit.", + want: 0.5, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Value != tt.want { + t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail) + } + }) + } +} + +func TestDelRelPipelineStageOrderingTest_Eval(t *testing.T) { + tc := delRelPipelineStageOrderingTest() + + correct := `["lint","unit-test","build","integration-test","publish-artifact","deploy-staging","deploy-production"]` + + tests := []struct { + name string + response string + want float64 + }{ + {name: "correct order", response: correct, want: 1}, + {name: "correct order fenced", response: "```json\n" + correct + "\n```", want: 1}, + { + name: "correct order different case", + response: `["Lint","Unit-Test","Build","Integration-Test","Publish-Artifact","Deploy-Staging","Deploy-Production"]`, + want: 1, + }, + { + name: "wrong: builds before linting", + response: `["build","lint","unit-test","integration-test","publish-artifact","deploy-staging","deploy-production"]`, + want: 0, + }, + { + name: "wrong: deploys production before staging", + response: `["lint","unit-test","build","integration-test","publish-artifact","deploy-production","deploy-staging"]`, + want: 0, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Value != tt.want { + t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail) + } + }) + } +} + +func TestDelRelPinVsRangePolicyTest_Eval(t *testing.T) { + tc := delRelPinVsRangePolicyTest() + + tests := []struct { + name string + response string + want float64 + }{ + {name: "all correct", response: `{"scenario_a":"pin","scenario_b":"range"}`, want: 1}, + {name: "all correct fenced", response: "```json\n{\"scenario_a\":\"pin\",\"scenario_b\":\"range\"}\n```", want: 1}, + {name: "all correct different case", response: `{"scenario_a":"Pin","scenario_b":"Range"}`, want: 1}, + {name: "scenario_b wrong", response: `{"scenario_a":"pin","scenario_b":"pin"}`, want: 0.5}, + {name: "both swapped", response: `{"scenario_a":"range","scenario_b":"pin"}`, want: 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Value != tt.want { + t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail) + } + }) + } +} + +func TestDelRelHotfixFlowOrderingTest_Eval(t *testing.T) { + tc := delRelHotfixFlowOrderingTest() + + correct := `["branch-from-production-tag","apply-minimal-fix","run-tests","tag-hotfix-release","deploy","merge-back-to-main"]` + + tests := []struct { + name string + response string + want float64 + }{ + {name: "correct order", response: correct, want: 1}, + {name: "correct order fenced", response: "```json\n" + correct + "\n```", want: 1}, + { + name: "correct order different case", + response: `["Branch-From-Production-Tag","Apply-Minimal-Fix","Run-Tests","Tag-Hotfix-Release","Deploy","Merge-Back-To-Main"]`, + want: 1, + }, + { + name: "wrong: branches from diverged main instead of the production tag", + response: `["apply-minimal-fix","run-tests","tag-hotfix-release","deploy","merge-back-to-main","branch-from-production-tag"]`, + want: 0, + }, + { + name: "wrong: merges back to main before deploying the fix", + response: `["branch-from-production-tag","apply-minimal-fix","run-tests","tag-hotfix-release","merge-back-to-main","deploy"]`, + want: 0, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Value != tt.want { + t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail) + } + }) + } +} diff --git a/internal/tests/delivery_test.go b/internal/tests/delivery_test.go new file mode 100644 index 0000000..860e7bf --- /dev/null +++ b/internal/tests/delivery_test.go @@ -0,0 +1,51 @@ +package tests + +import ( + "testing" + + "github.com/lukaszraczylo/llm-testbench/internal/testkit" +) + +// TestRegisterDeliveryTests_Wiring checks registerDeliveryTests wires up all +// three delivery subcategories - git, containers, release-engineering - with +// 10 tests each, all categorized "delivery", and no duplicate IDs across the +// three sibling files (testkit.Registry.Register panics on a duplicate ID, +// which would fail this test outright). Mirrors +// TestRegisterAgentsTests_Wiring in agents_test.go; catalog.go and +// catalog_test.go are not touched by this worktree, so this is the only +// place registerDeliveryTests is exercised until the orchestrator wires it +// into All() at integration. +func TestRegisterDeliveryTests_Wiring(t *testing.T) { + r := testkit.NewRegistry() + registerDeliveryTests(r) + + const wantTotal = 30 + if r.Len() != wantTotal { + t.Fatalf("registerDeliveryTests: registry has %d tests, want %d", r.Len(), wantTotal) + } + + counts := map[string]int{} + for _, tc := range r.All() { + if tc.Category != "delivery" { + t.Errorf("test %q: Category = %q, want %q", tc.ID, tc.Category, "delivery") + } + if tc.Prompt == "" { + t.Errorf("test %q: Prompt is empty", tc.ID) + } + if tc.Eval == nil { + t.Errorf("test %q: Eval is nil", tc.ID) + } + if tc.Description == "" { + t.Errorf("test %q: Description is empty", tc.ID) + } + if tc.MaxTokens < 0 { + t.Errorf("test %q: MaxTokens = %d, want >= 0", tc.ID, tc.MaxTokens) + } + counts[tc.Subcategory]++ + } + for _, sub := range []string{"git", "containers", "release-engineering"} { + if counts[sub] != 10 { + t.Errorf("subcategory %q: got %d tests, want 10", sub, counts[sub]) + } + } +}