mirror of
https://github.com/lukaszraczylo/llm-testbench.git
synced 2026-10-12 00:45:19 +00:00
lukaszraczylo
d778bb6c17
Add a new security category with three subcategories: appsec, crypto, and secrets. Each subcategory holds 10 self-contained, deterministic tests grounded in real vulnerability classes and remediation procedures. Appsec tests spot line-numbered injection, XSS, path traversal, SSRF, IDOR, and open-redirect flaws in synthetic fixtures, plus CSRF, rate limiting, input validation, and log-secret placement questions. Crypto tests cover password hashing, constant-time comparison, JWT alg=none, TLS floor version, AES-GCM nonce reuse, random source choice, key rotation ordering, HMAC versus asymmetric signing, certificate chain validation, and hash-versus-encrypt choices for PII. Secrets tests cover hardcoded-secret spotting, remediation ordering (rotate before history rewrite), vault tradeoffs, Kubernetes Secret base64 nuance, zero-downtime rotation ordering, least-privilege scoping, diff secret spotting, fork-PR CI exposure, SSH agent identity handling, and gitignore-versus-history nuance. A shared security.go file exposes registerSecurityTests and two reusable helpers: secNoUnnegatedMention, a bidirectional negation-aware guard adapted from kubernetes.go so a correct warning against a discouraged term is never penalized, and secExactAnswer, a forgiving exact-match evaluator for single-token answers. This change touches only new internal/tests/security*.go files. It does not modify catalog.go or catalog_test.go; the orchestrator wires registerSecurityTests at integration. A wiring test in security_test.go verifies the registration directly, mirroring the agents_test.go pattern. Every test carries a ground-truth comment, and every evaluator's unit test covers at least three correct phrasings and two incorrect ones.
267 lines
8.9 KiB
Go
267 lines
8.9 KiB
Go
package tests
|
|
|
|
import (
|
|
"context"
|
|
"testing"
|
|
)
|
|
|
|
func TestSecSQLiSpotTest_Eval(t *testing.T) {
|
|
tc := secSQLiSpotTest()
|
|
|
|
tests := []struct {
|
|
name string
|
|
response string
|
|
want float64
|
|
}{
|
|
{name: "correct", response: `{"line":2,"fix":"parameterized-query"}`, want: 1},
|
|
{name: "correct fenced with prose", response: "The injection is here:\n```json\n{\"line\":2,\"fix\":\"parameterized-query\"}\n```", want: 1},
|
|
{name: "correct with spacing", response: `{ "line": 2, "fix": "parameterized-query" }`, want: 1},
|
|
{name: "wrong line", response: `{"line":3,"fix":"parameterized-query"}`, want: 0.5},
|
|
{name: "wrong fix", response: `{"line":2,"fix":"escape-output"}`, want: 0.5},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
|
if got.Value != tt.want {
|
|
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestSecXSSSinkSpotTest_Eval(t *testing.T) {
|
|
tc := secXSSSinkSpotTest()
|
|
|
|
tests := []struct {
|
|
name string
|
|
response string
|
|
want float64
|
|
}{
|
|
{name: "correct", response: `{"line":4,"fix":"escape-output"}`, want: 1},
|
|
{name: "correct fenced with prose", response: "```json\n{\"line\":4,\"fix\":\"escape-output\"}\n```", want: 1},
|
|
{name: "correct with spacing", response: `{ "line": 4, "fix": "escape-output" }`, want: 1},
|
|
{name: "wrong line", response: `{"line":2,"fix":"escape-output"}`, want: 0.5},
|
|
{name: "wrong fix", response: `{"line":4,"fix":"validate-input"}`, want: 0.5},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
|
if got.Value != tt.want {
|
|
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestSecPathTraversalSpotTest_Eval(t *testing.T) {
|
|
tc := secPathTraversalSpotTest()
|
|
|
|
tests := []struct {
|
|
name string
|
|
response string
|
|
want float64
|
|
}{
|
|
{name: "correct", response: `{"line":3,"fix":"sanitize-path"}`, want: 1},
|
|
{name: "correct fenced with prose", response: "```json\n{\"line\":3,\"fix\":\"sanitize-path\"}\n```", want: 1},
|
|
{name: "correct with spacing", response: `{ "line": 3, "fix": "sanitize-path" }`, want: 1},
|
|
{name: "wrong line", response: `{"line":4,"fix":"sanitize-path"}`, want: 0.5},
|
|
{name: "wrong fix", response: `{"line":3,"fix":"escape-output"}`, want: 0.5},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
|
if got.Value != tt.want {
|
|
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestSecSSRFSpotTest_Eval(t *testing.T) {
|
|
tc := secSSRFSpotTest()
|
|
|
|
tests := []struct {
|
|
name string
|
|
response string
|
|
want float64
|
|
}{
|
|
{name: "correct", response: `{"line":3,"fix":"allowlist-hosts"}`, want: 1},
|
|
{name: "correct fenced with prose", response: "```json\n{\"line\":3,\"fix\":\"allowlist-hosts\"}\n```", want: 1},
|
|
{name: "correct with spacing", response: `{ "line": 3, "fix": "allowlist-hosts" }`, want: 1},
|
|
{name: "wrong line", response: `{"line":2,"fix":"allowlist-hosts"}`, want: 0.5},
|
|
{name: "wrong fix", response: `{"line":3,"fix":"sanitize-path"}`, want: 0.5},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
|
if got.Value != tt.want {
|
|
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestSecIDORSpotTest_Eval(t *testing.T) {
|
|
tc := secIDORSpotTest()
|
|
|
|
tests := []struct {
|
|
name string
|
|
response string
|
|
want float64
|
|
}{
|
|
{name: "correct", response: `{"line":4,"fix":"authorize-owner"}`, want: 1},
|
|
{name: "correct fenced with prose", response: "```json\n{\"line\":4,\"fix\":\"authorize-owner\"}\n```", want: 1},
|
|
{name: "correct with spacing", response: `{ "line": 4, "fix": "authorize-owner" }`, want: 1},
|
|
{name: "wrong line", response: `{"line":9,"fix":"authorize-owner"}`, want: 0.5},
|
|
{name: "wrong fix", response: `{"line":4,"fix":"rate-limit"}`, want: 0.5},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
|
if got.Value != tt.want {
|
|
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestSecOpenRedirectSpotTest_Eval(t *testing.T) {
|
|
tc := secOpenRedirectSpotTest()
|
|
|
|
tests := []struct {
|
|
name string
|
|
response string
|
|
want float64
|
|
}{
|
|
{name: "correct", response: `{"line":4}`, want: 1},
|
|
{name: "correct fenced with prose", response: "```json\n{\"line\":4}\n```", want: 1},
|
|
{name: "correct with spacing", response: `{ "line": 4 }`, want: 1},
|
|
{name: "wrong line: username param", response: `{"line":2}`, want: 0},
|
|
{name: "wrong line: signature comment", response: `{"line":1}`, want: 0},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
|
if got.Value != tt.want {
|
|
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestSecCSRFRequirementTest_Eval(t *testing.T) {
|
|
tc := secCSRFRequirementTest()
|
|
|
|
tests := []struct {
|
|
name string
|
|
response string
|
|
want float64
|
|
}{
|
|
{name: "correct bare", response: "no", want: 1},
|
|
{name: "correct with period", response: "No.", want: 1},
|
|
{name: "correct quoted", response: `'no'`, want: 1},
|
|
{name: "wrong bare", response: "yes", want: 0},
|
|
{name: "wrong with sentence", response: "Yes, it needs CSRF protection.", want: 0},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
|
if got.Value != tt.want {
|
|
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestSecRateLimitPlacementTest_Eval(t *testing.T) {
|
|
tc := secRateLimitPlacementTest()
|
|
|
|
tests := []struct {
|
|
name string
|
|
response string
|
|
want float64
|
|
}{
|
|
{name: "correct", response: `{"stage":"before-auth-check","key":"ip-and-account"}`, want: 1},
|
|
{name: "correct fenced with prose", response: "```json\n{\"stage\":\"before-auth-check\",\"key\":\"ip-and-account\"}\n```", want: 1},
|
|
{name: "correct with spacing", response: `{ "stage": "before-auth-check", "key": "ip-and-account" }`, want: 1},
|
|
{name: "wrong stage", response: `{"stage":"after-auth-check","key":"ip-and-account"}`, want: 0.5},
|
|
{name: "wrong key", response: `{"stage":"before-auth-check","key":"ip-only"}`, want: 0.5},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
|
if got.Value != tt.want {
|
|
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestSecInputValidationBoundaryTest_Eval(t *testing.T) {
|
|
tc := secInputValidationBoundaryTest()
|
|
|
|
tests := []struct {
|
|
name string
|
|
response string
|
|
want float64
|
|
}{
|
|
{
|
|
name: "correct: negative and cap phrasing",
|
|
response: "Reject quantity <= 0 (negative or zero) and cap it at a sane maximum to avoid overflow.",
|
|
want: 1,
|
|
},
|
|
{
|
|
name: "correct: greater-than-zero and upper-bound phrasing",
|
|
response: "Quantity must be greater than zero, and there must be an upper bound to prevent an excessively large value.",
|
|
want: 1,
|
|
},
|
|
{
|
|
name: "correct: isn't-negative and maximum phrasing",
|
|
response: "Check that quantity isn't negative and add a maximum limit on it.",
|
|
want: 1,
|
|
},
|
|
{
|
|
name: "wrong: no boundary checks named",
|
|
response: "Just make sure quantity is an integer.",
|
|
want: 0,
|
|
},
|
|
{
|
|
name: "wrong: only lower bound named",
|
|
response: "Reject negative quantities.",
|
|
want: 0.5,
|
|
},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
|
if got.Value != tt.want {
|
|
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestSecSecretLogSpotTest_Eval(t *testing.T) {
|
|
tc := secSecretLogSpotTest()
|
|
|
|
tests := []struct {
|
|
name string
|
|
response string
|
|
want float64
|
|
}{
|
|
{name: "correct", response: `{"line":3}`, want: 1},
|
|
{name: "correct fenced with prose", response: "```json\n{\"line\":3}\n```", want: 1},
|
|
{name: "correct with spacing", response: `{ "line": 3 }`, want: 1},
|
|
{name: "wrong line: header read", response: `{"line":2}`, want: 0},
|
|
{name: "wrong line: charge call", response: `{"line":4}`, want: 0},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
|
if got.Value != tt.want {
|
|
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
|
}
|
|
})
|
|
}
|
|
}
|