mirror of
https://github.com/lukaszraczylo/llm-testbench.git
synced 2026-10-12 00:45:19 +00:00
lukaszraczylo
e455eeecc7
Add a new delivery category with three subcategories: git, containers, and release-engineering. Each subcategory holds 10 self-contained, deterministically-scored tests, grounded in real git, Docker, and release-engineering behaviour. The git subcategory covers bisect step derivation, rebase versus merge, conventional-commit classification, git worktree, force-with-lease versus bare force, hook choice, detached-HEAD recovery, cherry-pick versus revert, a .gitignore negation trace, and stash-pop conflict behaviour. Two of these tests recompute their ground truth against a real, throwaway git repository created in t.TempDir, and skip if git is not installed. The containers subcategory covers Dockerfile layer-cache busting, ENTRYPOINT and CMD interaction, multi-stage image size, non-root users and capabilities, HEALTHCHECK semantics, .dockerignore effects, COPY versus ADD, build-arg versus runtime env, and layer counting. The release-engineering subcategory covers semver bump derivation from a changelog, the clean-tree requirement, rollback ordering, canary versus blue-green, tag-to-release sequencing, commit-to-changelog mapping, checksum and signing purpose, pipeline stage ordering, pin versus range dependency policy, and hotfix flow ordering. A shared negation-aware evaluator helper, local to this category's files, scores a forbidden phrase only when it appears unnegated, so a response correctly warning against bare --force or a silent tracked-file mutation still scores full credit. registerDeliveryTests wires the three subcategories together and is exercised only by a new wiring test in this worktree, since catalog.go and catalog_test.go stay untouched until the orchestrator integrates this category centrally.
52 lines
1.6 KiB
Go
52 lines
1.6 KiB
Go
package tests
|
|
|
|
import (
|
|
"testing"
|
|
|
|
"github.com/lukaszraczylo/llm-testbench/internal/testkit"
|
|
)
|
|
|
|
// TestRegisterDeliveryTests_Wiring checks registerDeliveryTests wires up all
|
|
// three delivery subcategories - git, containers, release-engineering - with
|
|
// 10 tests each, all categorized "delivery", and no duplicate IDs across the
|
|
// three sibling files (testkit.Registry.Register panics on a duplicate ID,
|
|
// which would fail this test outright). Mirrors
|
|
// TestRegisterAgentsTests_Wiring in agents_test.go; catalog.go and
|
|
// catalog_test.go are not touched by this worktree, so this is the only
|
|
// place registerDeliveryTests is exercised until the orchestrator wires it
|
|
// into All() at integration.
|
|
func TestRegisterDeliveryTests_Wiring(t *testing.T) {
|
|
r := testkit.NewRegistry()
|
|
registerDeliveryTests(r)
|
|
|
|
const wantTotal = 30
|
|
if r.Len() != wantTotal {
|
|
t.Fatalf("registerDeliveryTests: registry has %d tests, want %d", r.Len(), wantTotal)
|
|
}
|
|
|
|
counts := map[string]int{}
|
|
for _, tc := range r.All() {
|
|
if tc.Category != "delivery" {
|
|
t.Errorf("test %q: Category = %q, want %q", tc.ID, tc.Category, "delivery")
|
|
}
|
|
if tc.Prompt == "" {
|
|
t.Errorf("test %q: Prompt is empty", tc.ID)
|
|
}
|
|
if tc.Eval == nil {
|
|
t.Errorf("test %q: Eval is nil", tc.ID)
|
|
}
|
|
if tc.Description == "" {
|
|
t.Errorf("test %q: Description is empty", tc.ID)
|
|
}
|
|
if tc.MaxTokens < 0 {
|
|
t.Errorf("test %q: MaxTokens = %d, want >= 0", tc.ID, tc.MaxTokens)
|
|
}
|
|
counts[tc.Subcategory]++
|
|
}
|
|
for _, sub := range []string{"git", "containers", "release-engineering"} {
|
|
if counts[sub] != 10 {
|
|
t.Errorf("subcategory %q: got %d tests, want 10", sub, counts[sub])
|
|
}
|
|
}
|
|
}
|