package tests import ( "context" "testing" ) func TestSecPasswordHashChoiceTest_Eval(t *testing.T) { tc := secPasswordHashChoiceTest() tests := []struct { name string response string want float64 }{ { name: "correct: bcrypt, negated MD5/SHA-256", response: "Use bcrypt (cost >= 12) to hash passwords. Never use plain MD5 or SHA-256 for password storage.", want: 1, }, { name: "correct: argon2id, negated SHA-1/MD5", response: "argon2id is the right choice here; avoid SHA-1/MD5, they are far too fast for password hashing.", want: 1, }, { name: "correct: bcrypt, negated SHA-256 after the term", response: "bcrypt is fine. Don't use SHA-256 alone, it lacks a work factor.", want: 1, }, { name: "wrong: recommends SHA-256 unnegated", response: "Just hash the password with SHA-256 before storing it.", want: 0, }, { name: "wrong: recommends MD5 unnegated", response: "Store passwords hashed with MD5 for speed.", want: 0, }, { // C11 bug probe: naming a weak hash while recommending it // (not warning against it) must not earn any credit for // "naming" the discouraged term - the naming credit is folded // into the same AND as the negation check, not an independent // term. name: "wrong: names SHA-256 by name but still recommends it (C11 bug probe)", response: "SHA-256 is a fine choice for hashing user passwords.", want: 0, }, { name: "wrong: warns against a hash never actually named by name (C11 bug probe)", response: "Use bcrypt, and avoid weak, fast, general-purpose hashing algorithms for passwords.", want: 0.5, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { got := tc.Eval.Evaluate(context.Background(), tt.response) if got.Value != tt.want { t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail) } }) } } func TestSecConstantTimeCompareTest_Eval(t *testing.T) { tc := secConstantTimeCompareTest() tests := []struct { name string response string want float64 }{ { name: "correct: timing attack, hmac.Equal", response: "This is vulnerable to a timing attack; use hmac.Equal instead of ==.", want: 1, }, { name: "correct: timing side channel, subtle.ConstantTimeCompare", response: "That's a timing side channel. Use subtle.ConstantTimeCompare to fix it.", want: 1, }, { // CC1: "constant-time" describes the FIX, not the defect - a // response that only ever says "constant-time" (never "timing // attack"/"timing side channel"/"time-based") does not // independently name the defect class the prompt asks for, so // this now earns only the fix-group's credit, not both. name: "partial: only fix-side 'constant-time' phrasing, never names the defect class (CC1)", response: "Comparing with == leaks timing information (constant-time is required). Use crypto/subtle's ConstantTimeCompare function.", want: 0.5, }, { name: "correct: names the defect AND uses 'constant-time' for the fix (CC1 bug probe)", response: "This is a timing attack. Fix it by doing a constant-time comparison via crypto/subtle.", want: 1, }, { name: "wrong: claims == is fine", response: "This looks fine, == works for byte slice comparison.", want: 0, }, { name: "wrong: bytes.Equal is still not constant-time", response: "Use bytes.Equal instead of ==.", want: 0, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { got := tc.Eval.Evaluate(context.Background(), tt.response) if got.Value != tt.want { t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail) } }) } } func TestSecJWTAlgNoneTest_Eval(t *testing.T) { tc := secJWTAlgNoneTest() tests := []struct { name string response string want float64 }{ {name: "correct", response: `{"vuln":"alg-none-bypass","fix":"reject-alg-none"}`, want: 1}, {name: "correct fenced with prose", response: "```json\n{\"vuln\":\"alg-none-bypass\",\"fix\":\"reject-alg-none\"}\n```", want: 1}, {name: "correct with spacing", response: `{ "vuln": "alg-none-bypass", "fix": "reject-alg-none" }`, want: 1}, {name: "wrong vuln", response: `{"vuln":"algorithm-confusion","fix":"reject-alg-none"}`, want: 0.5}, {name: "wrong fix", response: `{"vuln":"alg-none-bypass","fix":"use-rs256"}`, want: 0.5}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { got := tc.Eval.Evaluate(context.Background(), tt.response) if got.Value != tt.want { t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail) } }) } } func TestSecTLSFloorVersionTest_Eval(t *testing.T) { tc := secTLSFloorVersionTest() tests := []struct { name string response string want float64 }{ {name: "correct bare", response: "1.2", want: 1}, {name: "correct with TLS prefix", response: "TLS 1.2", want: 1}, {name: "correct with trailing period", response: "1.2.", want: 1}, { // C1: "TLSv1.2" (no space) has no word boundary between "v" // and "1" - a plain \b1\.2\b regex rejected it. The // normalize-then-strip-prefix design correctly accepts it. name: "correct: TLSv prefix with no space (C1 bug probe)", response: "TLSv1.2", want: 1, }, { // This response is no longer accepted (C1): the prompt asks to // "respond with only the version number", and a full // sentence-wrapped answer is the same category of over-permissive // substring match that let "TLS 1.3 only; 1.2 is legacy" (below) // wrongly score 1 under the old \b1\.2\b substring search - // consistent with how every other eval.ExactToken-style forced // single-token test in this codebase treats full-sentence // wrapping. name: "wrong: sentence-wrapped, not a bare/quoted/fenced token", response: "The minimum should be 1.2.", want: 0, }, { // C1 bug probe: the actual chosen floor is 1.3, but the old // substring search matched "1.2" anyway because it appears // later in the same response, in an unrelated aside. name: "wrong: chooses 1.3 but mentions 1.2 in an aside (C1 bug probe)", response: "TLS 1.3 only; 1.2 is legacy.", want: 0, }, {name: "wrong: TLS 1.3", response: "1.3", want: 0}, {name: "wrong: TLS 1.0", response: "1.0", want: 0}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { got := tc.Eval.Evaluate(context.Background(), tt.response) if got.Value != tt.want { t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail) } }) } } func TestSecAESGCMNonceReuseTest_Eval(t *testing.T) { tc := secAESGCMNonceReuseTest() tests := []struct { name string response string want float64 }{ { name: "correct: forge, random nonce", response: "This lets an attacker forge ciphertexts because the authentication tag's key material repeats; use a fresh random nonce per encryption.", want: 1, }, { name: "correct: authentication/forgery, unique nonce", response: "It breaks GCM's authentication guarantee, allowing forgery. Generate a unique nonce for every encryption call.", want: 1, }, { name: "correct: keystream, never reuse plus crypto/rand", response: "An attacker can recover the plaintext via keystream reuse. Never reuse a nonce; use crypto/rand to generate one each time.", want: 1, }, { name: "wrong: claims reuse is fine", response: "This is fine as long as the key is secret.", want: 0, }, { name: "wrong: irrelevant fix suggested", response: "Just use a bigger nonce size.", want: 0, }, { // CC2 bug probe: bare "authentication" (with no forge/forgery/ // keystream/auth-tag consequence term) no longer triggers the // defect-naming group on its own. name: "wrong: bare 'authentication' mention with no actual consequence term (CC2 bug probe)", response: "This affects authentication somehow. Use a fresh random nonce per encryption.", want: 0.5, }, { name: "correct: specific 'auth tag' consequence term (CC2 bug probe)", response: "The auth tag can be forged if the nonce repeats. Use a fresh random nonce per encryption.", want: 1, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { got := tc.Eval.Evaluate(context.Background(), tt.response) if got.Value != tt.want { t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail) } }) } } func TestSecRandSourceChoiceTest_Eval(t *testing.T) { tc := secRandSourceChoiceTest() tests := []struct { name string response string want float64 }{ { name: "correct: crypto/rand, negated math/rand after the term", response: "Use crypto/rand instead; math/rand must never be used for a security token like this.", want: 1, }, { name: "correct: crypto/rand, negated math/rand before the term", response: "crypto/rand is correct here. Avoid math/rand - it's a deterministic PRNG predictable by an attacker.", want: 1, }, { name: "correct: crypto/rand, math/rand not suitable phrasing", response: "Switch to crypto/rand. math/rand is not suitable for security-sensitive tokens since it's predictable.", want: 1, }, { name: "wrong: endorses math/rand unnegated", response: "math/rand works well for this use case.", want: 0, }, { name: "wrong: endorses math/rand as good enough", response: "math/rand is good enough for a reset token.", want: 0, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { got := tc.Eval.Evaluate(context.Background(), tt.response) if got.Value != tt.want { t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail) } }) } } func TestSecRotateOrderTest_Eval(t *testing.T) { tc := secRotateOrderTest() tests := []struct { name string response string want float64 }{ { name: "correct order", response: `["add-new-key-to-verify-set","start-signing-with-new-key","wait-for-old-tokens-to-expire","remove-old-key-from-verify-set"]`, want: 1, }, { name: "correct order fenced with prose", response: "Here is the order:\n```json\n[\"add-new-key-to-verify-set\",\"start-signing-with-new-key\",\"wait-for-old-tokens-to-expire\",\"remove-old-key-from-verify-set\"]\n```", want: 1, }, { name: "correct order different case", response: `["Add-New-Key-To-Verify-Set","Start-Signing-With-New-Key","Wait-For-Old-Tokens-To-Expire","Remove-Old-Key-From-Verify-Set"]`, want: 1, }, { name: "wrong: fully reversed", response: `["remove-old-key-from-verify-set","wait-for-old-tokens-to-expire","start-signing-with-new-key","add-new-key-to-verify-set"]`, want: 0, }, { name: "wrong: removes old key too early", response: `["add-new-key-to-verify-set","remove-old-key-from-verify-set","start-signing-with-new-key","wait-for-old-tokens-to-expire"]`, want: 0, }, { // C4 bug probe: the prompt's presented list is now // hand-shuffled to a different order than the correct answer, // so an echo of the prompt's own list order must score 0, not 1. name: "wrong: echoes the prompt's shuffled presentation order verbatim (C4 bug probe)", response: `["remove-old-key-from-verify-set","add-new-key-to-verify-set","wait-for-old-tokens-to-expire","start-signing-with-new-key"]`, want: 0, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { got := tc.Eval.Evaluate(context.Background(), tt.response) if got.Value != tt.want { t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail) } }) } } func TestSecHMACVsSignatureTest_Eval(t *testing.T) { tc := secHMACVsSignatureTest() tests := []struct { name string response string want float64 }{ {name: "correct bare", response: "asymmetric", want: 1}, {name: "correct with period", response: "Asymmetric.", want: 1}, {name: "correct quoted", response: `'asymmetric'`, want: 1}, {name: "wrong bare", response: "hmac", want: 0}, {name: "wrong uppercase", response: "HMAC", want: 0}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { got := tc.Eval.Evaluate(context.Background(), tt.response) if got.Value != tt.want { t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail) } }) } } func TestSecCertChainValidationTest_Eval(t *testing.T) { tc := secCertChainValidationTest() tests := []struct { name string response string want float64 }{ {name: "correct hyphenated", response: "man-in-the-middle", want: 1}, {name: "correct spaced in a sentence", response: "This exposes the client to a man in the middle attack.", want: 1}, {name: "correct capitalized", response: "MITM stands for Man-In-The-Middle, which is the attack here.", want: 1}, {name: "correct: bare MITM abbreviation (C6 bug probe)", response: "MITM", want: 1}, {name: "correct: AiTM abbreviation (C6 bug probe)", response: "AiTM (adversary-in-the-middle)", want: 1}, {name: "correct: on-path attacker (C6 bug probe)", response: "This is an on-path attacker.", want: 1}, {name: "correct: adversary-in-the-middle phrasing (C6 bug probe)", response: "adversary-in-the-middle", want: 1}, {name: "correct: machine-in-the-middle phrasing (C6 bug probe)", response: "machine-in-the-middle", want: 1}, {name: "wrong: replay attack", response: "replay attack", want: 0}, {name: "wrong: denial of service", response: "denial of service", want: 0}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { got := tc.Eval.Evaluate(context.Background(), tt.response) if got.Value != tt.want { t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail) } }) } } func TestSecHashVsEncryptPIITest_Eval(t *testing.T) { tc := secHashVsEncryptPIITest() tests := []struct { name string response string want float64 }{ {name: "correct", response: `{"field_a":"hash","field_b":"encrypt"}`, want: 1}, {name: "correct fenced with prose", response: "```json\n{\"field_a\":\"hash\",\"field_b\":\"encrypt\"}\n```", want: 1}, {name: "correct with spacing", response: `{ "field_a": "hash", "field_b": "encrypt" }`, want: 1}, {name: "wrong field_a", response: `{"field_a":"encrypt","field_b":"encrypt"}`, want: 0.5}, {name: "wrong field_b", response: `{"field_a":"hash","field_b":"hash"}`, want: 0.5}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { got := tc.Eval.Evaluate(context.Background(), tt.response) if got.Value != tt.want { t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail) } }) } }