mirror of
https://github.com/lukaszraczylo/llm-testbench.git
synced 2026-10-12 00:45:19 +00:00
test(catalog): assert at least 10 tests per subcategory
This commit is contained in:
@@ -1,6 +1,9 @@
|
|||||||
package tests
|
package tests
|
||||||
|
|
||||||
import "testing"
|
import (
|
||||||
|
"slices"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
// TestAll_Invariants checks structural invariants across the whole
|
// TestAll_Invariants checks structural invariants across the whole
|
||||||
// catalog, rather than an exact id/category/count map (fix 6): four other
|
// catalog, rather than an exact id/category/count map (fix 6): four other
|
||||||
@@ -52,3 +55,47 @@ func TestAll_Invariants(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TestAll_SubcategoryMinimums pins the post-expansion catalog shape: every
|
||||||
|
// subcategory holds at least minTestsPerSubcategory tests, and exactly the
|
||||||
|
// expected category/subcategory pairs exist (no strays from a typo in a
|
||||||
|
// registration).
|
||||||
|
func TestAll_SubcategoryMinimums(t *testing.T) {
|
||||||
|
const minTestsPerSubcategory = 10
|
||||||
|
|
||||||
|
wantSubcats := map[string][]string{
|
||||||
|
"programming": {"golang", "python", "typescript", "c"},
|
||||||
|
"operations": {"macos", "linux", "kubernetes"},
|
||||||
|
"research": {"web", "whitepapers", "codebase"},
|
||||||
|
"agents": {"tool-routing", "planning", "delegation"},
|
||||||
|
}
|
||||||
|
|
||||||
|
counts := make(map[string]map[string]int)
|
||||||
|
for _, tc := range All().All() {
|
||||||
|
if counts[tc.Category] == nil {
|
||||||
|
counts[tc.Category] = make(map[string]int)
|
||||||
|
}
|
||||||
|
counts[tc.Category][tc.Subcategory]++
|
||||||
|
}
|
||||||
|
|
||||||
|
for cat, subcats := range wantSubcats {
|
||||||
|
for _, sub := range subcats {
|
||||||
|
if n := counts[cat][sub]; n < minTestsPerSubcategory {
|
||||||
|
t.Errorf("%s/%s: %d tests, want >= %d", cat, sub, n, minTestsPerSubcategory)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
for cat, subs := range counts {
|
||||||
|
want, ok := wantSubcats[cat]
|
||||||
|
if !ok {
|
||||||
|
t.Errorf("unexpected category %q", cat)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
for sub := range subs {
|
||||||
|
if !slices.Contains(want, sub) {
|
||||||
|
t.Errorf("unexpected subcategory %s/%s", cat, sub)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user