mirror of
https://github.com/lukaszraczylo/llm-testbench.git
synced 2026-10-12 00:45:19 +00:00
lukaszraczylo
9d1d4d89e9
A shared NoUnnegatedMention primitive replaces four per-category implementations, with a bidirectional clause-scoped window and an adjectival cue set. The docker layer fixture drops WORKDIR, which creates a real layer and made the pinned answer wrong. The replica failover test now forces a JSON verdict so a negated answer cannot score. Remaining exact-answer tests migrate to ExactToken. Ordering prompts shuffle their candidate lists so echoing scores zero.
409 lines
14 KiB
Go
409 lines
14 KiB
Go
package tests
|
|
|
|
import (
|
|
"context"
|
|
"os"
|
|
"os/exec"
|
|
"path/filepath"
|
|
"strconv"
|
|
"strings"
|
|
"testing"
|
|
)
|
|
|
|
func TestDelDockerLayerCacheBustTest_Eval(t *testing.T) {
|
|
tc := delDockerLayerCacheBustTest()
|
|
|
|
tests := []struct {
|
|
name string
|
|
response string
|
|
want float64
|
|
}{
|
|
{name: "bare object", response: `{"line": 3}`, want: 1},
|
|
{name: "fenced with prose", response: "The culprit is:\n```json\n{\"line\": 3}\n```", want: 1},
|
|
{name: "extra whitespace inside the object", response: `{ "line" : 3 }`, want: 1},
|
|
{name: "wrong: blames the FROM line", response: `{"line": 1}`, want: 0},
|
|
{name: "wrong: blames go build instead", response: `{"line": 5}`, want: 0},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
|
if got.Value != tt.want {
|
|
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestDelDockerEntrypointCmdTraceTest_Eval(t *testing.T) {
|
|
tc := delDockerEntrypointCmdTraceTest()
|
|
|
|
tests := []struct {
|
|
name string
|
|
response string
|
|
want float64
|
|
}{
|
|
{name: "exact", response: "python3 app.py --port 8080", want: 1},
|
|
{name: "different case", response: "Python3 App.py --Port 8080", want: 1},
|
|
{name: "surrounding whitespace", response: " python3 app.py --port 8080\n", want: 1},
|
|
{name: "wrong: only CMD's args", response: "--port 8080", want: 0},
|
|
{name: "wrong: only ENTRYPOINT, drops CMD's default args", response: "python3 app.py", want: 0},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
|
if got.Value != tt.want {
|
|
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestDelDockerMultistageSizeBenefitTest_Eval(t *testing.T) {
|
|
tc := delDockerMultistageSizeBenefitTest()
|
|
|
|
tests := []struct {
|
|
name string
|
|
response string
|
|
want float64
|
|
}{
|
|
{
|
|
name: "correct: multi-stage + scratch",
|
|
response: "Use a multi-stage build: compile in the golang stage, then copy only the binary into a final scratch image with no toolchain at all.",
|
|
want: 1,
|
|
},
|
|
{
|
|
name: "correct: multi-stage + distroless",
|
|
response: "A multi-stage build lets you copy just the compiled binary into a distroless final image, dropping the entire build toolchain layer.",
|
|
want: 1,
|
|
},
|
|
{
|
|
name: "correct: multi-stage + alpine",
|
|
response: "With a multi-stage build the final stage can be a minimal alpine base holding only the binary, instead of the full golang image.",
|
|
want: 1,
|
|
},
|
|
{
|
|
name: "wrong: neither term present",
|
|
response: "Just delete unused files with a RUN rm -rf step at the end of the same image.",
|
|
want: 0,
|
|
},
|
|
{
|
|
name: "wrong: names the base but never says multi-stage",
|
|
response: "Switch the final image to alpine instead of golang.",
|
|
want: 0.5,
|
|
},
|
|
{
|
|
// D8 bug probe: a terse, technical-syntax answer that never
|
|
// says the literal word "multi-stage" but genuinely
|
|
// demonstrates the mechanism (COPY --from a named build
|
|
// stage) must still score full credit.
|
|
name: "correct: technical syntax only, never says 'multi-stage' literally (D8 bug probe)",
|
|
response: "In the final stage: COPY --from=builder /app /app, using scratch as the base.",
|
|
want: 1,
|
|
},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
|
if got.Value != tt.want {
|
|
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestDelDockerImageSizeMathTest_Eval(t *testing.T) {
|
|
tc := delDockerImageSizeMathTest()
|
|
|
|
tests := []struct {
|
|
name string
|
|
response string
|
|
want float64
|
|
}{
|
|
{name: "bare number", response: "30", want: 1},
|
|
{name: "sentence form", response: "The final image is 30 MB.", want: 1},
|
|
{name: "with units spelled out", response: "8 + 22 = 30 megabytes.", want: 1},
|
|
{name: "wrong: sums every layer including the discarded build stage", response: "534", want: 0},
|
|
{name: "wrong: only the alpine base, forgets the binary layer", response: "8", want: 0},
|
|
{
|
|
// DC1 bug probe: a glued unit suffix ("30MB", no space) must
|
|
// extract the same as the spaced form.
|
|
name: "glued unit suffix, no space (DC1 bug probe)",
|
|
response: "The final image is 30MB.",
|
|
want: 1,
|
|
},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
|
if got.Value != tt.want {
|
|
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestDelDockerNonrootUserCapsTest_Eval(t *testing.T) {
|
|
tc := delDockerNonrootUserCapsTest()
|
|
|
|
tests := []struct {
|
|
name string
|
|
response string
|
|
want float64
|
|
}{
|
|
{
|
|
name: "correct: USER + cap-drop=all",
|
|
response: "Add `USER 1000` to the Dockerfile, and run the container with --cap-drop=ALL since it needs no special capabilities.",
|
|
want: 1,
|
|
},
|
|
{
|
|
name: "correct: USER + drop all capabilities phrasing",
|
|
response: "Create a non-root user with adduser and add a USER instruction; at runtime, drop all capabilities.",
|
|
want: 1,
|
|
},
|
|
{
|
|
name: "correct: USER + cap-drop all spaced form",
|
|
response: "Set USER app in the image, and pass --cap-drop all at runtime since no privileged syscalls are needed.",
|
|
want: 1,
|
|
},
|
|
{
|
|
name: "wrong: neither addressed",
|
|
response: "This is fine as-is, since the process only listens on a port.",
|
|
want: 0,
|
|
},
|
|
{
|
|
// D8: a generic "add a USER instruction" mention with no
|
|
// concrete UID/username/adduser artifact is exactly the
|
|
// vacuous echo of the prompt's own wording ("this Dockerfile
|
|
// currently has no USER instruction") that this fix targets -
|
|
// it no longer earns partial credit on its own.
|
|
name: "wrong: generic USER mention with no concrete artifact, still misses capabilities",
|
|
response: "Add a USER instruction pointing at a non-root UID.",
|
|
want: 0,
|
|
},
|
|
{
|
|
name: "partial: concrete USER artifact but capabilities still missing",
|
|
response: "Add `USER 1000` to the Dockerfile.",
|
|
want: 0.5,
|
|
},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
|
if got.Value != tt.want {
|
|
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestDelDockerHealthcheckSemanticsTest_Eval(t *testing.T) {
|
|
tc := delDockerHealthcheckSemanticsTest()
|
|
|
|
tests := []struct {
|
|
name string
|
|
response string
|
|
want float64
|
|
}{
|
|
{name: "bare number", response: "3", want: 1},
|
|
{name: "sentence form", response: "After 3 consecutive failures.", want: 1},
|
|
{name: "labelled form", response: "retries: 3", want: 1},
|
|
{name: "wrong: confuses timeout with retries", response: "3s", want: 0}, // "3s" glues the 3 to a unit letter, so extraction fails and scores 0
|
|
{name: "wrong: confuses interval with retries", response: "30", want: 0},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
|
if got.Value != tt.want {
|
|
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestDelDockerDockerignoreEffectTest_Eval(t *testing.T) {
|
|
tc := delDockerDockerignoreEffectTest()
|
|
|
|
tests := []struct {
|
|
name string
|
|
response string
|
|
want float64
|
|
}{
|
|
{name: "exact correct set", response: `["app.js","README.md"]`, want: 1},
|
|
{name: "correct set, different order", response: `["README.md","app.js"]`, want: 1},
|
|
{name: "correct set, fenced", response: "```json\n[\"app.js\",\"README.md\"]\n```", want: 1},
|
|
{
|
|
name: "wrong: includes an excluded file",
|
|
response: `["app.js","README.md","debug.log"]`,
|
|
want: 2.0 / 3.0,
|
|
},
|
|
{
|
|
name: "wrong: also excludes a file that should be included",
|
|
response: `["app.js"]`,
|
|
want: 0.5,
|
|
},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
|
if got.Value != tt.want {
|
|
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestDelDockerCopyVsAddTest_Eval(t *testing.T) {
|
|
tc := delDockerCopyVsAddTest()
|
|
|
|
tests := []struct {
|
|
name string
|
|
response string
|
|
want float64
|
|
}{
|
|
{name: "all correct", response: `{"scenario_a":"COPY","scenario_b":"ADD"}`, want: 1},
|
|
{name: "all correct fenced", response: "```json\n{\"scenario_a\":\"COPY\",\"scenario_b\":\"ADD\"}\n```", want: 1},
|
|
{name: "all correct lowercase", response: `{"scenario_a":"copy","scenario_b":"add"}`, want: 1},
|
|
{name: "scenario_a wrong", response: `{"scenario_a":"ADD","scenario_b":"ADD"}`, want: 0.5},
|
|
{name: "both swapped", response: `{"scenario_a":"ADD","scenario_b":"COPY"}`, want: 0},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
|
if got.Value != tt.want {
|
|
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestDelDockerBuildArgVsEnvTest_Eval(t *testing.T) {
|
|
tc := delDockerBuildArgVsEnvTest()
|
|
|
|
tests := []struct {
|
|
name string
|
|
response string
|
|
want float64
|
|
}{
|
|
{name: "all correct", response: `{"scenario_a":"ARG","scenario_b":"ENV"}`, want: 1},
|
|
{name: "all correct fenced", response: "```json\n{\"scenario_a\":\"ARG\",\"scenario_b\":\"ENV\"}\n```", want: 1},
|
|
{name: "all correct lowercase", response: `{"scenario_a":"arg","scenario_b":"env"}`, want: 1},
|
|
{name: "scenario_b wrong", response: `{"scenario_a":"ARG","scenario_b":"ARG"}`, want: 0.5},
|
|
{name: "both swapped", response: `{"scenario_a":"ENV","scenario_b":"ARG"}`, want: 0},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
|
if got.Value != tt.want {
|
|
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestDelDockerLayerCountTest_Eval(t *testing.T) {
|
|
tc := delDockerLayerCountTest()
|
|
|
|
tests := []struct {
|
|
name string
|
|
response string
|
|
want float64
|
|
}{
|
|
{name: "bare number", response: "2", want: 1},
|
|
{name: "sentence form", response: "This Dockerfile adds 2 new layers.", want: 1},
|
|
{name: "explains then answers", response: "Only COPY and RUN create layers here, so the answer is 2.", want: 1},
|
|
{name: "wrong: counts every instruction", response: "7", want: 0},
|
|
{name: "wrong: forgets the RUN layer", response: "1", want: 0},
|
|
{
|
|
// D1 bug probe: WORKDIR is not in this fixture at all, but a
|
|
// wrong answer that (incorrectly) assumes it's present AND
|
|
// counts it as a layer-creating instruction must still score 0
|
|
// (want stays 2 either way, since WORKDIR is genuinely absent).
|
|
name: "wrong: assumes a WORKDIR layer that is not in this fixture (D1 bug probe)",
|
|
response: "3",
|
|
want: 0,
|
|
},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
|
if got.Value != tt.want {
|
|
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// delDockerStripLineNumbers removes the "N: " prompt-display prefix from
|
|
// each line of a numbered Dockerfile fixture, producing real Dockerfile
|
|
// syntax that can actually be built.
|
|
func delDockerStripLineNumbers(text string) string {
|
|
lines := strings.Split(text, "\n")
|
|
for i, line := range lines {
|
|
if idx := strings.Index(line, ": "); idx >= 0 {
|
|
lines[i] = line[idx+2:]
|
|
}
|
|
}
|
|
return strings.Join(lines, "\n")
|
|
}
|
|
|
|
// delDockerInspectLayerCount runs `docker inspect <ref> --format
|
|
// '{{len .RootFS.Layers}}'` and returns the parsed layer count.
|
|
func delDockerInspectLayerCount(t *testing.T, ref string) int {
|
|
t.Helper()
|
|
// #nosec G204 -- ref is always a fixed literal image tag/reference
|
|
// built earlier in this same test, never external/user-controlled
|
|
// input.
|
|
out, err := exec.Command("docker", "inspect", ref, "--format", "{{len .RootFS.Layers}}").Output() //nolint:gosec // see comment above
|
|
if err != nil {
|
|
t.Fatalf("docker inspect %s: %v", ref, err)
|
|
}
|
|
n, err := strconv.Atoi(strings.TrimSpace(string(out)))
|
|
if err != nil {
|
|
t.Fatalf("docker inspect %s: unparseable layer count %q: %v", ref, string(out), err)
|
|
}
|
|
return n
|
|
}
|
|
|
|
// TestDelDockerLayerCountWant_GroundTruth verifies delDockerLayerCountWant
|
|
// against a real Docker build (D1) rather than trusting the derivation
|
|
// comment alone: it builds delDockerLayerCountDockerfile (line numbers
|
|
// stripped) with a dummy "app" binary, then diffs the built image's
|
|
// RootFS.Layers count against the alpine:3.20 base image's own layer
|
|
// count. Skips if docker is not installed or its daemon is unreachable.
|
|
func TestDelDockerLayerCountWant_GroundTruth(t *testing.T) {
|
|
if _, err := exec.LookPath("docker"); err != nil {
|
|
t.Skip("docker not installed")
|
|
}
|
|
if err := exec.Command("docker", "info").Run(); err != nil {
|
|
t.Skip("docker daemon not reachable")
|
|
}
|
|
|
|
dir := t.TempDir()
|
|
dockerfile := delDockerStripLineNumbers(delDockerLayerCountDockerfile)
|
|
if err := os.WriteFile(filepath.Join(dir, "Dockerfile"), []byte(dockerfile), 0o600); err != nil {
|
|
t.Fatalf("write Dockerfile: %v", err)
|
|
}
|
|
if err := os.WriteFile(filepath.Join(dir, "app"), []byte("#!/bin/sh\necho hi\n"), 0o600); err != nil {
|
|
t.Fatalf("write app: %v", err)
|
|
}
|
|
|
|
const tag = "llmtestbench-layercount-groundtruth:test"
|
|
// #nosec G204 -- tag is a fixed literal and dir is a t.TempDir() path,
|
|
// never external/user-controlled input.
|
|
build := exec.Command("docker", "build", "-t", tag, dir) //nolint:gosec // see comment above
|
|
if out, err := build.CombinedOutput(); err != nil {
|
|
t.Fatalf("docker build: %v\n%s", err, out)
|
|
}
|
|
t.Cleanup(func() {
|
|
_ = exec.Command("docker", "rmi", tag).Run()
|
|
})
|
|
|
|
totalLayers := delDockerInspectLayerCount(t, tag)
|
|
baseLayers := delDockerInspectLayerCount(t, "alpine:3.20")
|
|
newLayers := totalLayers - baseLayers
|
|
|
|
if newLayers != delDockerLayerCountWant {
|
|
t.Fatalf("docker-verified new layer count = %d (total %d, base %d), want %d", newLayers, totalLayers, baseLayers, delDockerLayerCountWant)
|
|
}
|
|
}
|