mirror of
https://github.com/lukaszraczylo/llm-testbench.git
synced 2026-10-12 00:45:19 +00:00
lukaszraczylo
e455eeecc7
Add a new delivery category with three subcategories: git, containers, and release-engineering. Each subcategory holds 10 self-contained, deterministically-scored tests, grounded in real git, Docker, and release-engineering behaviour. The git subcategory covers bisect step derivation, rebase versus merge, conventional-commit classification, git worktree, force-with-lease versus bare force, hook choice, detached-HEAD recovery, cherry-pick versus revert, a .gitignore negation trace, and stash-pop conflict behaviour. Two of these tests recompute their ground truth against a real, throwaway git repository created in t.TempDir, and skip if git is not installed. The containers subcategory covers Dockerfile layer-cache busting, ENTRYPOINT and CMD interaction, multi-stage image size, non-root users and capabilities, HEALTHCHECK semantics, .dockerignore effects, COPY versus ADD, build-arg versus runtime env, and layer counting. The release-engineering subcategory covers semver bump derivation from a changelog, the clean-tree requirement, rollback ordering, canary versus blue-green, tag-to-release sequencing, commit-to-changelog mapping, checksum and signing purpose, pipeline stage ordering, pin versus range dependency policy, and hotfix flow ordering. A shared negation-aware evaluator helper, local to this category's files, scores a forbidden phrase only when it appears unnegated, so a response correctly warning against bare --force or a silent tracked-file mutation still scores full credit. registerDeliveryTests wires the three subcategories together and is exercised only by a new wiring test in this worktree, since catalog.go and catalog_test.go stay untouched until the orchestrator integrates this category centrally.
357 lines
13 KiB
Go
357 lines
13 KiB
Go
package tests
|
|
|
|
import (
|
|
"context"
|
|
"testing"
|
|
)
|
|
|
|
func TestDelRelSemverBumpChangelogTest_Eval(t *testing.T) {
|
|
tc := delRelSemverBumpChangelogTest()
|
|
|
|
tests := []struct {
|
|
name string
|
|
response string
|
|
want float64
|
|
}{
|
|
{name: "bare object", response: `{"version":"2.4.0"}`, want: 1},
|
|
{name: "fenced with prose", response: "The feat entry dominates, so:\n```json\n{\"version\":\"2.4.0\"}\n```", want: 1},
|
|
{name: "extra whitespace inside the object", response: `{ "version" : "2.4.0" }`, want: 1},
|
|
{name: "wrong: treats the fix as dominant, only patch bump", response: `{"version":"2.3.2"}`, want: 0},
|
|
{name: "wrong: major bump for no breaking change", response: `{"version":"3.0.0"}`, want: 0},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
|
if got.Value != tt.want {
|
|
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestDelRelCleanTreeRequirementTest_Eval(t *testing.T) {
|
|
tc := delRelCleanTreeRequirementTest()
|
|
|
|
tests := []struct {
|
|
name string
|
|
response string
|
|
want float64
|
|
}{
|
|
{
|
|
name: "correct: explains reproducibility, negates the hook with must never",
|
|
response: "Release tooling requires a clean working tree so the published artifact matches exactly what was committed and reviewed. A pre-build hook must never be allowed to silently modify tracked files mid-release.",
|
|
want: 1,
|
|
},
|
|
{
|
|
name: "correct: explains dirty tree, negates the hook with should not",
|
|
response: "A dirty git tree means the shipped code was never actually committed, so the tool refuses it. Such a hook should not be allowed to silently modify tracked files.",
|
|
want: 1,
|
|
},
|
|
{
|
|
name: "correct: explains uncommitted changes, negates the hook with no",
|
|
response: "It requires a clean git tree because uncommitted changes would ship code nobody reviewed. No, a hook must not silently modify tracked source files before packaging.",
|
|
want: 1,
|
|
},
|
|
{
|
|
name: "wrong: recommends the unsafe hook outright, no clean-tree reason given",
|
|
response: "It's fine for the hook to silently modify tracked files right before packaging, since the formatting is harmless.",
|
|
want: 0,
|
|
},
|
|
{
|
|
name: "wrong: gives the reason but still endorses the unsafe hook",
|
|
response: "A clean working tree matters because the artifact must match a known commit. That said, it's fine here for the hook to silently modify tracked files, since only formatting changes.",
|
|
want: 0.5,
|
|
},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
|
if got.Value != tt.want {
|
|
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestDelRelRollbackOrderingTest_Eval(t *testing.T) {
|
|
tc := delRelRollbackOrderingTest()
|
|
|
|
correct := `["page-oncall","stop-new-traffic-if-needed","redeploy-previous-image","confirm-health-checks-pass","post-incident-notes"]`
|
|
|
|
tests := []struct {
|
|
name string
|
|
response string
|
|
want float64
|
|
}{
|
|
{name: "correct order", response: correct, want: 1},
|
|
{name: "correct order fenced", response: "```json\n" + correct + "\n```", want: 1},
|
|
{
|
|
name: "correct order with extra whitespace",
|
|
response: `[ "page-oncall", "stop-new-traffic-if-needed", "redeploy-previous-image", "confirm-health-checks-pass", "post-incident-notes" ]`,
|
|
want: 1,
|
|
},
|
|
{
|
|
name: "wrong: redeploys before paging anyone",
|
|
response: `["redeploy-previous-image","page-oncall","stop-new-traffic-if-needed","confirm-health-checks-pass","post-incident-notes"]`,
|
|
want: 0,
|
|
},
|
|
{
|
|
name: "wrong: writes post-incident notes before confirming health",
|
|
response: `["page-oncall","stop-new-traffic-if-needed","redeploy-previous-image","post-incident-notes","confirm-health-checks-pass"]`,
|
|
want: 0,
|
|
},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
|
if got.Value != tt.want {
|
|
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestDelRelCanaryVsBlueGreenTest_Eval(t *testing.T) {
|
|
tc := delRelCanaryVsBlueGreenTest()
|
|
|
|
tests := []struct {
|
|
name string
|
|
response string
|
|
want float64
|
|
}{
|
|
{name: "all correct", response: `{"scenario_a":"canary","scenario_b":"blue-green"}`, want: 1},
|
|
{name: "all correct fenced", response: "```json\n{\"scenario_a\":\"canary\",\"scenario_b\":\"blue-green\"}\n```", want: 1},
|
|
{name: "all correct different case", response: `{"scenario_a":"Canary","scenario_b":"Blue-Green"}`, want: 1},
|
|
{name: "scenario_a wrong", response: `{"scenario_a":"blue-green","scenario_b":"blue-green"}`, want: 0.5},
|
|
{name: "both swapped", response: `{"scenario_a":"blue-green","scenario_b":"canary"}`, want: 0},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
|
if got.Value != tt.want {
|
|
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestDelRelTagToReleaseSequenceTest_Eval(t *testing.T) {
|
|
tc := delRelTagToReleaseSequenceTest()
|
|
|
|
correct := `["run-tests","create-git-tag","push-tag","generate-changelog","create-github-release","announce"]`
|
|
|
|
tests := []struct {
|
|
name string
|
|
response string
|
|
want float64
|
|
}{
|
|
{name: "correct order", response: correct, want: 1},
|
|
{name: "correct order fenced", response: "```json\n" + correct + "\n```", want: 1},
|
|
{
|
|
name: "correct order different case",
|
|
response: `["Run-Tests","Create-Git-Tag","Push-Tag","Generate-Changelog","Create-Github-Release","Announce"]`,
|
|
want: 1,
|
|
},
|
|
{
|
|
name: "wrong: tags before running tests",
|
|
response: `["create-git-tag","run-tests","push-tag","generate-changelog","create-github-release","announce"]`,
|
|
want: 0,
|
|
},
|
|
{
|
|
name: "wrong: creates the GitHub release before pushing the tag",
|
|
response: `["run-tests","create-git-tag","generate-changelog","create-github-release","push-tag","announce"]`,
|
|
want: 0,
|
|
},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
|
if got.Value != tt.want {
|
|
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestDelRelCommitChangelogMappingTest_Eval(t *testing.T) {
|
|
tc := delRelCommitChangelogMappingTest()
|
|
|
|
allCorrect := `{"commit_a":"Added","commit_b":"Fixed","commit_c":"Changed","commit_d":"Security"}`
|
|
|
|
tests := []struct {
|
|
name string
|
|
response string
|
|
want float64
|
|
}{
|
|
{name: "all correct", response: allCorrect, want: 1},
|
|
{name: "all correct fenced with prose", response: "Here is the mapping:\n```json\n" + allCorrect + "\n```", want: 1},
|
|
{
|
|
name: "one wrong: security fix filed as generic Fixed",
|
|
response: `{"commit_a":"Added","commit_b":"Fixed","commit_c":"Changed","commit_d":"Fixed"}`,
|
|
want: 3.0 / 4.0,
|
|
},
|
|
{
|
|
name: "two wrong: perf and security both misfiled",
|
|
response: `{"commit_a":"Added","commit_b":"Fixed","commit_c":"Fixed","commit_d":"Fixed"}`,
|
|
want: 2.0 / 4.0,
|
|
},
|
|
{
|
|
name: "all wrong",
|
|
response: `{"commit_a":"Fixed","commit_b":"Added","commit_c":"Removed","commit_d":"Deprecated"}`,
|
|
want: 0,
|
|
},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
|
if got.Value != tt.want {
|
|
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestDelRelChecksumsSigningPurposeTest_Eval(t *testing.T) {
|
|
tc := delRelChecksumsSigningPurposeTest()
|
|
|
|
tests := []struct {
|
|
name string
|
|
response string
|
|
want float64
|
|
}{
|
|
{
|
|
name: "correct: integrity + authenticity",
|
|
response: "The checksum lets you confirm the binary was not corrupted in transit; the signature proves the checksums file itself is authentic and came from the real publisher.",
|
|
want: 1,
|
|
},
|
|
{
|
|
name: "correct: alternate integrity + provenance phrasing",
|
|
response: "Checksums detect if the download was tampered with or truncated. The signature adds provenance: it ties the checksums back to a trusted identity.",
|
|
want: 1,
|
|
},
|
|
{
|
|
name: "correct: alternate wording",
|
|
response: "The SHA256 file verifies the binary has not been altered. Signing the checksums file verifies it was actually signed by the maintainer, not swapped in by an attacker.",
|
|
want: 1,
|
|
},
|
|
{
|
|
name: "wrong: neither concept present",
|
|
response: "Both files just make the release page look more professional.",
|
|
want: 0,
|
|
},
|
|
{
|
|
name: "wrong: only explains the checksum, not the signature",
|
|
response: "The checksum lets you confirm the binary was not corrupted in transit.",
|
|
want: 0.5,
|
|
},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
|
if got.Value != tt.want {
|
|
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestDelRelPipelineStageOrderingTest_Eval(t *testing.T) {
|
|
tc := delRelPipelineStageOrderingTest()
|
|
|
|
correct := `["lint","unit-test","build","integration-test","publish-artifact","deploy-staging","deploy-production"]`
|
|
|
|
tests := []struct {
|
|
name string
|
|
response string
|
|
want float64
|
|
}{
|
|
{name: "correct order", response: correct, want: 1},
|
|
{name: "correct order fenced", response: "```json\n" + correct + "\n```", want: 1},
|
|
{
|
|
name: "correct order different case",
|
|
response: `["Lint","Unit-Test","Build","Integration-Test","Publish-Artifact","Deploy-Staging","Deploy-Production"]`,
|
|
want: 1,
|
|
},
|
|
{
|
|
name: "wrong: builds before linting",
|
|
response: `["build","lint","unit-test","integration-test","publish-artifact","deploy-staging","deploy-production"]`,
|
|
want: 0,
|
|
},
|
|
{
|
|
name: "wrong: deploys production before staging",
|
|
response: `["lint","unit-test","build","integration-test","publish-artifact","deploy-production","deploy-staging"]`,
|
|
want: 0,
|
|
},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
|
if got.Value != tt.want {
|
|
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestDelRelPinVsRangePolicyTest_Eval(t *testing.T) {
|
|
tc := delRelPinVsRangePolicyTest()
|
|
|
|
tests := []struct {
|
|
name string
|
|
response string
|
|
want float64
|
|
}{
|
|
{name: "all correct", response: `{"scenario_a":"pin","scenario_b":"range"}`, want: 1},
|
|
{name: "all correct fenced", response: "```json\n{\"scenario_a\":\"pin\",\"scenario_b\":\"range\"}\n```", want: 1},
|
|
{name: "all correct different case", response: `{"scenario_a":"Pin","scenario_b":"Range"}`, want: 1},
|
|
{name: "scenario_b wrong", response: `{"scenario_a":"pin","scenario_b":"pin"}`, want: 0.5},
|
|
{name: "both swapped", response: `{"scenario_a":"range","scenario_b":"pin"}`, want: 0},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
|
if got.Value != tt.want {
|
|
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestDelRelHotfixFlowOrderingTest_Eval(t *testing.T) {
|
|
tc := delRelHotfixFlowOrderingTest()
|
|
|
|
correct := `["branch-from-production-tag","apply-minimal-fix","run-tests","tag-hotfix-release","deploy","merge-back-to-main"]`
|
|
|
|
tests := []struct {
|
|
name string
|
|
response string
|
|
want float64
|
|
}{
|
|
{name: "correct order", response: correct, want: 1},
|
|
{name: "correct order fenced", response: "```json\n" + correct + "\n```", want: 1},
|
|
{
|
|
name: "correct order different case",
|
|
response: `["Branch-From-Production-Tag","Apply-Minimal-Fix","Run-Tests","Tag-Hotfix-Release","Deploy","Merge-Back-To-Main"]`,
|
|
want: 1,
|
|
},
|
|
{
|
|
name: "wrong: branches from diverged main instead of the production tag",
|
|
response: `["apply-minimal-fix","run-tests","tag-hotfix-release","deploy","merge-back-to-main","branch-from-production-tag"]`,
|
|
want: 0,
|
|
},
|
|
{
|
|
name: "wrong: merges back to main before deploying the fix",
|
|
response: `["branch-from-production-tag","apply-minimal-fix","run-tests","tag-hotfix-release","merge-back-to-main","deploy"]`,
|
|
want: 0,
|
|
},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
|
if got.Value != tt.want {
|
|
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
|
}
|
|
})
|
|
}
|
|
}
|