mirror of
https://github.com/lukaszraczylo/llm-testbench.git
synced 2026-10-14 00:57:45 +00:00
lukaszraczylo
d778bb6c17
Add a new security category with three subcategories: appsec, crypto, and secrets. Each subcategory holds 10 self-contained, deterministic tests grounded in real vulnerability classes and remediation procedures. Appsec tests spot line-numbered injection, XSS, path traversal, SSRF, IDOR, and open-redirect flaws in synthetic fixtures, plus CSRF, rate limiting, input validation, and log-secret placement questions. Crypto tests cover password hashing, constant-time comparison, JWT alg=none, TLS floor version, AES-GCM nonce reuse, random source choice, key rotation ordering, HMAC versus asymmetric signing, certificate chain validation, and hash-versus-encrypt choices for PII. Secrets tests cover hardcoded-secret spotting, remediation ordering (rotate before history rewrite), vault tradeoffs, Kubernetes Secret base64 nuance, zero-downtime rotation ordering, least-privilege scoping, diff secret spotting, fork-PR CI exposure, SSH agent identity handling, and gitignore-versus-history nuance. A shared security.go file exposes registerSecurityTests and two reusable helpers: secNoUnnegatedMention, a bidirectional negation-aware guard adapted from kubernetes.go so a correct warning against a discouraged term is never penalized, and secExactAnswer, a forgiving exact-match evaluator for single-token answers. This change touches only new internal/tests/security*.go files. It does not modify catalog.go or catalog_test.go; the orchestrator wires registerSecurityTests at integration. A wiring test in security_test.go verifies the registration directly, mirroring the agents_test.go pattern. Every test carries a ground-truth comment, and every evaluator's unit test covers at least three correct phrasings and two incorrect ones.
48 lines
1.4 KiB
Go
48 lines
1.4 KiB
Go
package tests
|
|
|
|
import (
|
|
"testing"
|
|
|
|
"github.com/lukaszraczylo/llm-testbench/internal/testkit"
|
|
)
|
|
|
|
// TestRegisterSecurityTests_Wiring checks registerSecurityTests wires up
|
|
// all three security subcategories - appsec, crypto, secrets - with 10
|
|
// tests each, all categorized "security", and no duplicate IDs across the
|
|
// three sibling files (testkit.Registry.Register panics on a duplicate ID,
|
|
// which would fail this test outright).
|
|
func TestRegisterSecurityTests_Wiring(t *testing.T) {
|
|
r := testkit.NewRegistry()
|
|
registerSecurityTests(r)
|
|
|
|
const wantTotal = 30
|
|
if r.Len() != wantTotal {
|
|
t.Fatalf("registerSecurityTests: registry has %d tests, want %d", r.Len(), wantTotal)
|
|
}
|
|
|
|
counts := map[string]int{}
|
|
for _, tc := range r.All() {
|
|
if tc.Category != "security" {
|
|
t.Errorf("test %q: Category = %q, want %q", tc.ID, tc.Category, "security")
|
|
}
|
|
if tc.Prompt == "" {
|
|
t.Errorf("test %q: Prompt is empty", tc.ID)
|
|
}
|
|
if tc.Eval == nil {
|
|
t.Errorf("test %q: Eval is nil", tc.ID)
|
|
}
|
|
if tc.Description == "" {
|
|
t.Errorf("test %q: Description is empty", tc.ID)
|
|
}
|
|
if tc.MaxTokens < 0 {
|
|
t.Errorf("test %q: MaxTokens = %d, want >= 0", tc.ID, tc.MaxTokens)
|
|
}
|
|
counts[tc.Subcategory]++
|
|
}
|
|
for _, sub := range []string{"appsec", "crypto", "secrets"} {
|
|
if counts[sub] != 10 {
|
|
t.Errorf("subcategory %q: got %d tests, want 10", sub, counts[sub])
|
|
}
|
|
}
|
|
}
|