mirror of
https://github.com/lukaszraczylo/llm-testbench.git
synced 2026-10-12 00:45:19 +00:00
401 lines
13 KiB
Go
401 lines
13 KiB
Go
package tests
|
|
|
|
import (
|
|
"context"
|
|
"testing"
|
|
)
|
|
|
|
// Each saturation test gets a table pinning: the full-credit answer, the
|
|
// trap answer (the near-miss the test exists to catch), and a garbage
|
|
// answer. Expected partial scores follow from eval.Mean's per-field mean.
|
|
|
|
func TestSatDelegCapacityTrapTest_Eval(t *testing.T) {
|
|
tc := satDelegCapacityTrapTest()
|
|
|
|
tests := []struct {
|
|
name string
|
|
response string
|
|
want float64
|
|
}{
|
|
{
|
|
name: "correct: main thread, names busy specialist",
|
|
response: `{"best_next_step":"handle_in_main_thread","specialist_this_task_belongs_to":"devops-debugger"}`,
|
|
want: 1,
|
|
},
|
|
{
|
|
name: "trap: force-fits the generalist builder",
|
|
response: `{"best_next_step":"dispatch_code_writer","specialist_this_task_belongs_to":"devops-debugger"}`,
|
|
want: 0.5,
|
|
},
|
|
{
|
|
name: "trap: orchestrator for a single task",
|
|
response: `{"best_next_step":"dispatch_orchestrator","specialist_this_task_belongs_to":"orchestrator"}`,
|
|
want: 0,
|
|
},
|
|
{
|
|
name: "correct step but wrong owning specialist",
|
|
response: `{"best_next_step":"handle_in_main_thread","specialist_this_task_belongs_to":"code-writer"}`,
|
|
want: 0.5,
|
|
},
|
|
{
|
|
name: "wait-it-out answer",
|
|
response: `{"best_next_step":"wait_for_specialist","specialist_this_task_belongs_to":"devops-debugger"}`,
|
|
want: 0.5,
|
|
},
|
|
{name: "garbage", response: `not json`, want: 0},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
|
if got.Value != tt.want {
|
|
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestSatDelegNearMissRosterTest_Eval(t *testing.T) {
|
|
tc := satDelegNearMissRosterTest()
|
|
|
|
tests := []struct {
|
|
name string
|
|
response string
|
|
want float64
|
|
}{
|
|
{
|
|
name: "all correct",
|
|
response: `{"task1":"docs-writer","task2":"web-researcher","task3":"release-manager","task4":"security-reviewer"}`,
|
|
want: 1,
|
|
},
|
|
{
|
|
name: "trap: web-researcher for the repo guide, docs-writer for the pricing page",
|
|
response: `{"task1":"web-researcher","task2":"docs-writer","task3":"release-manager","task4":"security-reviewer"}`,
|
|
want: 0.5,
|
|
},
|
|
{
|
|
name: "trap: docs-writer for release notes, code-reviewer for the audit",
|
|
response: `{"task1":"docs-writer","task2":"web-researcher","task3":"docs-writer","task4":"code-reviewer"}`,
|
|
want: 0.5,
|
|
},
|
|
{
|
|
name: "all swapped to the distractors",
|
|
response: `{"task1":"web-researcher","task2":"docs-writer","task3":"docs-writer","task4":"code-reviewer"}`,
|
|
want: 0,
|
|
},
|
|
{
|
|
name: "case-insensitive correct",
|
|
response: `{"task1":"Docs-Writer","task2":"Web-Researcher","task3":"Release-Manager","task4":"Security-Reviewer"}`,
|
|
want: 1,
|
|
},
|
|
{name: "garbage", response: `["docs-writer"]`, want: 0},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
|
if got.Value != tt.want {
|
|
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestSatPGCountStarTrapTest_Eval(t *testing.T) {
|
|
tc := satPGCountStarTrapTest()
|
|
|
|
tests := []struct {
|
|
name string
|
|
response string
|
|
want float64
|
|
}{
|
|
{
|
|
name: "correct",
|
|
response: `{"orders_in_eu": 4, "rows_the_join_query_returns": 6, "count_star_answers_the_question": false}`,
|
|
want: 1,
|
|
},
|
|
{
|
|
name: "numeric strings accepted",
|
|
response: `{"orders_in_eu": "4", "rows_the_join_query_returns": "6", "count_star_answers_the_question": "false"}`,
|
|
want: 1,
|
|
},
|
|
{
|
|
name: "trap: counts joined rows as orders",
|
|
response: `{"orders_in_eu": 6, "rows_the_join_query_returns": 6, "count_star_answers_the_question": true}`,
|
|
want: 0.3333333333333333,
|
|
},
|
|
{
|
|
name: "trap: forgets the childless order drops out of the inner join",
|
|
response: `{"orders_in_eu": 4, "rows_the_join_query_returns": 7, "count_star_answers_the_question": false}`,
|
|
want: 0.6666666666666666,
|
|
},
|
|
{name: "garbage", response: `4`, want: 0},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
|
if got.Value != tt.want {
|
|
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestSatPGTimeoutTrapTest_Eval(t *testing.T) {
|
|
tc := satPGTimeoutTrapTest()
|
|
|
|
tests := []struct {
|
|
name string
|
|
response string
|
|
want float64
|
|
}{
|
|
{
|
|
name: "correct",
|
|
response: `{"session_a_killed_at":"11:00:45","session_a_rule":"idle_in_transaction_timeout","session_b_killed_at":"11:01:25","session_b_rule":"statement_timeout"}`,
|
|
want: 1,
|
|
},
|
|
{
|
|
name: "trap: idle clock started at BEGIN",
|
|
response: `{"session_a_killed_at":"11:00:05","session_a_rule":"idle_in_transaction_timeout","session_b_killed_at":"11:01:25","session_b_rule":"statement_timeout"}`,
|
|
want: 0.75,
|
|
},
|
|
{
|
|
name: "trap: timeouts swapped between sessions",
|
|
response: `{"session_a_killed_at":"11:00:30","session_a_rule":"statement_timeout","session_b_killed_at":"11:00:58","session_b_rule":"idle_in_transaction_timeout"}`,
|
|
want: 0,
|
|
},
|
|
{
|
|
name: "two fields right",
|
|
response: `{"session_a_killed_at":"11:00:45","session_a_rule":"idle_in_transaction_timeout","session_b_killed_at":"11:01:00","session_b_rule":"not_killed"}`,
|
|
want: 0.5,
|
|
},
|
|
{name: "garbage", response: `no json here`, want: 0},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
|
if got.Value != tt.want {
|
|
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestSatWebRobotsUAScopeTest_Eval(t *testing.T) {
|
|
tc := satWebRobotsUAScopeTest()
|
|
|
|
tests := []struct {
|
|
name string
|
|
response string
|
|
want float64
|
|
}{
|
|
{
|
|
name: "correct: longest match wins for googlebot, group isolation for GoogleOther",
|
|
response: `{"googlebot_allowed": true, "googleother_allowed": false, "deciding_rule_for_googlebot": "allow"}`,
|
|
want: 1,
|
|
},
|
|
{
|
|
name: "trap: GoogleOther's disallow applied to googlebot too",
|
|
response: `{"googlebot_allowed": false, "googleother_allowed": false, "deciding_rule_for_googlebot": "disallow"}`,
|
|
want: 0.3333333333333333,
|
|
},
|
|
{
|
|
name: "trap: wildcard group treated as absent",
|
|
response: `{"googlebot_allowed": true, "googleother_allowed": true, "deciding_rule_for_googlebot": "allow"}`,
|
|
want: 0.6666666666666666,
|
|
},
|
|
{name: "garbage", response: `allow`, want: 0},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
|
if got.Value != tt.want {
|
|
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestSatWebHreflangSelfTest_Eval(t *testing.T) {
|
|
tc := satWebHreflangSelfTest()
|
|
|
|
tests := []struct {
|
|
name string
|
|
response string
|
|
want float64
|
|
}{
|
|
{
|
|
name: "correct",
|
|
response: `{"missing_self_reference_for":"en-US","self_reference_present":false,"canonical_counts_as_hreflang_self_reference":false,"x_default_is_a_language_tag":false}`,
|
|
want: 1,
|
|
},
|
|
{
|
|
name: "tag casing normalized",
|
|
response: `{"missing_self_reference_for":"en-us","self_reference_present":false,"canonical_counts_as_hreflang_self_reference":false,"x_default_is_a_language_tag":false}`,
|
|
want: 1,
|
|
},
|
|
{
|
|
name: "trap: canonical counted as the self-reference",
|
|
response: `{"missing_self_reference_for":"none","self_reference_present":true,"canonical_counts_as_hreflang_self_reference":true,"x_default_is_a_language_tag":false}`,
|
|
want: 0.25,
|
|
},
|
|
{
|
|
name: "trap: x-default treated as a language",
|
|
response: `{"missing_self_reference_for":"en-US","self_reference_present":false,"canonical_counts_as_hreflang_self_reference":false,"x_default_is_a_language_tag":true}`,
|
|
want: 0.75,
|
|
},
|
|
{name: "garbage", response: `en-US`, want: 0},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
|
if got.Value != tt.want {
|
|
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestSatAppsecORMTrapTest_Eval(t *testing.T) {
|
|
tc := satAppsecORMTrapTest()
|
|
|
|
tests := []struct {
|
|
name string
|
|
response string
|
|
want float64
|
|
}{
|
|
{
|
|
name: "correct",
|
|
response: `{"injectable_function": "search_users", "text_wrapper_neutralizes_concatenation": false}`,
|
|
want: 1,
|
|
},
|
|
{
|
|
name: "trap: ORM assumed safe everywhere",
|
|
response: `{"injectable_function": "none", "text_wrapper_neutralizes_concatenation": true}`,
|
|
want: 0,
|
|
},
|
|
{
|
|
name: "right function, wrong mechanism",
|
|
response: `{"injectable_function": "search_users", "text_wrapper_neutralizes_concatenation": true}`,
|
|
want: 0.5,
|
|
},
|
|
{name: "garbage", response: `search_users`, want: 0},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
|
if got.Value != tt.want {
|
|
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestSatAppsecPOSTTrapTest_Eval(t *testing.T) {
|
|
tc := satAppsecPOSTTrapTest()
|
|
|
|
tests := []struct {
|
|
name string
|
|
response string
|
|
want float64
|
|
}{
|
|
{
|
|
name: "correct",
|
|
response: `{"attacker_form_reaches_logout": true, "post_method_alone_prevents_csrf": false, "attacker_needs_javascript": true}`,
|
|
want: 1,
|
|
},
|
|
{
|
|
name: "trap: POST assumed to prevent CSRF",
|
|
response: `{"attacker_form_reaches_logout": false, "post_method_alone_prevents_csrf": true, "attacker_needs_javascript": true}`,
|
|
want: 0.3333333333333333,
|
|
},
|
|
{
|
|
name: "trap: attacker blocked for lacking the page's JS",
|
|
response: `{"attacker_form_reaches_logout": false, "post_method_alone_prevents_csrf": false, "attacker_needs_javascript": false}`,
|
|
want: 0.3333333333333333,
|
|
},
|
|
{name: "garbage", response: `{"a":1}`, want: 0},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
|
if got.Value != tt.want {
|
|
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestSatSecretsRotationMathTest_Eval(t *testing.T) {
|
|
tc := satSecretsRotationMathTest()
|
|
|
|
tests := []struct {
|
|
name string
|
|
response string
|
|
want float64
|
|
}{
|
|
{
|
|
name: "correct",
|
|
response: `{"total_lead_time_hours": 16, "latest_start": "2026-09-30 02:00", "safe_if_started_2026_09_30_08_00": false}`,
|
|
want: 1,
|
|
},
|
|
{
|
|
name: "trap: minting time only, propagation forgotten",
|
|
response: `{"total_lead_time_hours": 10, "latest_start": "2026-09-30 08:00", "safe_if_started_2026_09_30_08_00": true}`,
|
|
want: 0,
|
|
},
|
|
{
|
|
name: "sum right, start-time arithmetic wrong",
|
|
response: `{"total_lead_time_hours": 16, "latest_start": "2026-09-30 02:00", "safe_if_started_2026_09_30_08_00": true}`,
|
|
want: 0.6666666666666666,
|
|
},
|
|
{name: "garbage", response: `16 hours`, want: 0},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
|
if got.Value != tt.want {
|
|
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestSatSecretsErrorPathTest_Eval(t *testing.T) {
|
|
tc := satSecretsErrorPathTest()
|
|
|
|
tests := []struct {
|
|
name string
|
|
response string
|
|
want float64
|
|
}{
|
|
{
|
|
name: "correct",
|
|
response: `{"hash_reaches_server_logs": true, "hash_visible_in_http_response": false, "sensitive_value_in_message": "hash"}`,
|
|
want: 1,
|
|
},
|
|
{
|
|
name: "trap: sanitized response assumed to mean no leak",
|
|
response: `{"hash_reaches_server_logs": false, "hash_visible_in_http_response": false, "sensitive_value_in_message": "hash"}`,
|
|
want: 0.6666666666666666,
|
|
},
|
|
{
|
|
name: "trap: attacker-known user flagged instead of the hash",
|
|
response: `{"hash_reaches_server_logs": true, "hash_visible_in_http_response": false, "sensitive_value_in_message": "user"}`,
|
|
want: 0.6666666666666666,
|
|
},
|
|
{
|
|
name: "trap: leak judged through the log AND the response",
|
|
response: `{"hash_reaches_server_logs": true, "hash_visible_in_http_response": true, "sensitive_value_in_message": "hash"}`,
|
|
want: 0.6666666666666666,
|
|
},
|
|
{name: "garbage", response: `hash`, want: 0},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
|
if got.Value != tt.want {
|
|
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
|
}
|
|
})
|
|
}
|
|
}
|