mirror of
https://github.com/lukaszraczylo/llm-testbench.git
synced 2026-10-12 00:45:19 +00:00
Update
This commit is contained in:
@@ -23,5 +23,8 @@ func All() *testkit.Registry {
|
||||
registerSecurityTests(r)
|
||||
registerDeliveryTests(r)
|
||||
registerAITests(r)
|
||||
registerHardTests(r)
|
||||
registerScenarioTests(r)
|
||||
registerSaturationTests(r)
|
||||
return r
|
||||
}
|
||||
|
||||
@@ -64,8 +64,8 @@ func TestAll_SubcategoryMinimums(t *testing.T) {
|
||||
const minTestsPerSubcategory = 10
|
||||
|
||||
wantSubcats := map[string][]string{
|
||||
"programming": {"golang", "python", "typescript", "c"},
|
||||
"operations": {"macos", "linux", "kubernetes"},
|
||||
"programming": {"golang", "python", "typescript", "c", "hard"},
|
||||
"operations": {"macos", "linux", "kubernetes", "scenario"},
|
||||
"research": {"web", "whitepapers", "codebase"},
|
||||
"agents": {"tool-routing", "planning", "delegation"},
|
||||
"databases": {"postgres", "redis", "sql-tuning"},
|
||||
|
||||
@@ -159,27 +159,58 @@ pipelined commands? Respond with only "yes" or "no".`
|
||||
}
|
||||
}
|
||||
|
||||
// dbKeysCommandPattern matches an explicit mention of the Redis KEYS
|
||||
// command actually being invoked - "KEYS *" (its most common prod-danger
|
||||
// form) or a backtick-quoted `KEYS` - not a generic use of the English
|
||||
// word "keys" elsewhere in prose ("the keys to fixing this...").
|
||||
var dbKeysCommandPattern = regexp.MustCompile("(?:`KEYS`|\\bKEYS\\s*\\*)")
|
||||
// dbKeysEndorsementPattern matches a KEYS mention in an endorsement-shaped
|
||||
// position — a verb recommending it ("use `KEYS`", "just run KEYS *"), a
|
||||
// positive predicate ("KEYS * is fine/simplest"), or a bare imperative line
|
||||
// ("run this:\nKEYS *") — rather than every mention. The guard then still
|
||||
// requires a negation cue near such a mention (see dbNoBareKeysInProd).
|
||||
//
|
||||
// Triggering on any mention is too strict: a fully correct live answer
|
||||
// (probe, 2026-08-29) opens with "`KEYS *` is dangerous..." and later says
|
||||
// "so `KEYS *` can block the server while it walks through millions of
|
||||
// keys" — harm descriptions whose words ("can block") sit outside the
|
||||
// negation cue list, yet they are exactly the explanation the prompt asks
|
||||
// for, not an endorsement.
|
||||
//
|
||||
// The predicate branch's leading (?:^|[^...]) guard stops a hyphen-joined
|
||||
// identifier from matching mid-word and keeps the match start at the term
|
||||
// area rather than arbitrary prose.
|
||||
var dbKeysEndorsementPattern = regexp.MustCompile("(?i)(?:" +
|
||||
|
||||
// a verb recommending it: "use `KEYS`", "just run KEYS *", "stick with KEYS *"
|
||||
|
||||
"(?:use|using|used|run|running|runs|execute|executing|call|calling|should(?:\\s+(?:also|just))?|better\\s+to|go\\s+with|opt\\s+for|stick\\s+with|switch(?:ing)?\\s+(?:back\\s+)?to)\\s+(?:the\\s+)?(?:redis\\s+)?(?:command\\s+)?(?:`KEYS`|\\bKEYS\\s*\\*)" +
|
||||
|
||||
// a positive predicate about it: "KEYS * is fine", "`KEYS` is simplest"
|
||||
|
||||
"|(?:^|[^A-Za-z0-9_-])(?:`KEYS`|\\bKEYS\\s*\\*)[^.;\\n]{0,30}?\\b(?:is|are|was|were|seems?|looks?|works?|stays?|remains?|would\\s+be|should\\s+be)\\s+(?:a\\s+|just\\s+|totally\\s+|still\\s+|actually\\s+)?(?:fine|good|great|safe|ok|okay|acceptable|simplest|easiest|better|fast\\s+enough)" +
|
||||
|
||||
// a bare command line after a colon or full stop: "run this:\\nKEYS *" -
|
||||
|
||||
// requiring the break keeps a line-initial mention that merely
|
||||
|
||||
// continues a wrapped sentence from triggering.
|
||||
|
||||
"|(?:[.:]|^)[ \\t]*(?:\\r?\\n)[ \\t*`>-]*(?:`KEYS`|\\bKEYS\\s*\\*)" +
|
||||
|
||||
")")
|
||||
|
||||
// dbNegationWindow is how many characters around a KEYS-command mention
|
||||
// are searched for a negation cue.
|
||||
const dbNegationWindow = 60
|
||||
|
||||
// dbNoBareKeysInProd scores full credit unless the response instructs
|
||||
// running "KEYS *" against a production instance without any negating
|
||||
// context (a warning not to, or an instruction to use SCAN instead is
|
||||
// fine). This is deliberately not eval.NotContains, which would also zero
|
||||
// out the best possible answer: one that correctly explains why NOT to run
|
||||
// KEYS in production.
|
||||
// dbNoBareKeysInProd scores full credit unless the response endorses
|
||||
// running "KEYS *" against a production instance (see
|
||||
// dbKeysEndorsementPattern) with no negating context nearby — a warning
|
||||
// not to, or an instruction to use SCAN instead, is fine. This is
|
||||
// deliberately not eval.NotContains, which would also zero out the best
|
||||
// possible answer: one that correctly explains why NOT to run KEYS in
|
||||
// production.
|
||||
//
|
||||
// Delegates to eval.NoUnnegatedMention (D5), the primitive shared with
|
||||
// kubernetes.go, security.go, and delivery_git.go's equivalent guards.
|
||||
func dbNoBareKeysInProd() eval.Evaluator {
|
||||
return eval.NoUnnegatedMention(dbKeysCommandPattern, dbNegationWindow, nil)
|
||||
return eval.NoUnnegatedMention(dbKeysEndorsementPattern, dbNegationWindow, nil)
|
||||
}
|
||||
|
||||
// dbRedisScanVsKeysTest: require SCAN over KEYS for a hot-path production
|
||||
|
||||
@@ -147,10 +147,33 @@ func TestDBNoBareKeysInProd(t *testing.T) {
|
||||
want: 0,
|
||||
},
|
||||
{
|
||||
name: "negation cue too far away to count",
|
||||
// The guard now triggers on an endorsement-shaped mention only,
|
||||
// and the negation window is measured from the endorsement
|
||||
// phrase ("...run KEYS *"), not from the sentence start — so a
|
||||
// hedging "never mind" that is still within 60 chars counts.
|
||||
name: "negation cue close to the endorsement phrase",
|
||||
response: "Never mind the migration for now, but you could casually run KEYS * to check this quickly.",
|
||||
want: 1,
|
||||
},
|
||||
{
|
||||
// ground truth: same endorsement, but the cue is now >60 chars
|
||||
// from "run KEYS *" (73), outside the window - still an
|
||||
// unnegated recommendation.
|
||||
name: "negation cue too far away to count",
|
||||
response: "Never mind everything else that was said in the earlier review about migrations, but you could casually run KEYS * to check this quickly.",
|
||||
want: 0,
|
||||
},
|
||||
{
|
||||
// Regression from the live probe (2026-08-29): a fully correct
|
||||
// answer describes the harm ("can block", no cue word in the
|
||||
// list) and only negates one mention ("avoid `KEYS *` unless").
|
||||
// None of these is an endorsement, so the guard must stay at 1.
|
||||
name: "correct probe answer: harm description without cue words",
|
||||
response: "`KEYS *` is dangerous in production because it scans the entire keyspace synchronously.\n" +
|
||||
"Redis is single-threaded for command execution, so `KEYS *` can block the server while it walks through millions of keys.\n" +
|
||||
"For production code, use `SCAN` with a reasonable `COUNT`, and avoid `KEYS *` unless you are in development.",
|
||||
want: 1,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
|
||||
@@ -0,0 +1,538 @@
|
||||
package tests
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"strings"
|
||||
|
||||
"github.com/lukaszraczylo/llm-testbench/internal/eval"
|
||||
"github.com/lukaszraczylo/llm-testbench/internal/testkit"
|
||||
)
|
||||
|
||||
func registerScenarioTests(r *testkit.Registry) {
|
||||
r.Register(scenDiskpressureFirstcmdTest())
|
||||
r.Register(scenInodeExhaustionTest())
|
||||
r.Register(scenCrashloopOrderTest())
|
||||
r.Register(scenNetLayerIsolateTest())
|
||||
r.Register(scenServiceDegradeTest())
|
||||
r.Register(scenLogRootcauseMCQTest())
|
||||
r.Register(scenBackupRestoreOrderTest())
|
||||
r.Register(scenCertexpiryTriageTest())
|
||||
r.Register(scenSystemdFailreasonTest())
|
||||
r.Register(scenPortbindFailTest())
|
||||
r.Register(scenLvmResizeOrderTest())
|
||||
}
|
||||
|
||||
// scenDiskpressureDf is the inline df output for scenDiskpressureFirstcmdTest.
|
||||
const scenDiskpressureDf = `Filesystem Size Used Avail Use% Mounted on
|
||||
/dev/sda1 100G 100G 0G 100% /
|
||||
tmpfs 16G 1.1G 15G 8% /run
|
||||
/dev/sdb1 500G 30G 470G 6% /data`
|
||||
|
||||
// scenFirstDiskCommand scores a response whose {"first_command"} field is a
|
||||
// safe, size-sorted du scan of the root filesystem. It demands the two
|
||||
// essential verb flags (du to measure recursive usage, sort to order the
|
||||
// result) and rejects a destructive first step (rm). A naive model that
|
||||
// jumps straight to deleting files, or that lists with ls (which does not
|
||||
// measure recursive usage) or find -size (which only matches known sizes),
|
||||
// scores zero. This is deliberately not a bare ContainsAny on the whole
|
||||
// response: the prompt forces a JSON object, and matching inside the
|
||||
// command field keeps the check grounded in an actual command rather than
|
||||
// prose that merely names du.
|
||||
func scenFirstDiskCommand() eval.Evaluator {
|
||||
return eval.EvaluatorFunc(func(_ context.Context, response string) eval.Score {
|
||||
raw, err := eval.ExtractJSON(response)
|
||||
if err != nil {
|
||||
return eval.Score{Value: 0, Detail: "no JSON object found"}
|
||||
}
|
||||
var obj map[string]string
|
||||
if err := json.Unmarshal([]byte(raw), &obj); err != nil {
|
||||
return eval.Score{Value: 0, Detail: "JSON not usable as a string map"}
|
||||
}
|
||||
cmd := strings.ToLower(strings.TrimSpace(obj["first_command"]))
|
||||
if cmd == "" {
|
||||
return eval.Score{Value: 0, Detail: "first_command empty"}
|
||||
}
|
||||
for _, req := range []string{"du", "sort"} {
|
||||
if !strings.Contains(cmd, req) {
|
||||
return eval.Score{Value: 0, Detail: "first_command missing " + req}
|
||||
}
|
||||
}
|
||||
if strings.HasPrefix(cmd, "rm") || strings.Contains(cmd, " rm ") || strings.Contains(cmd, "rm -") {
|
||||
return eval.Score{Value: 0, Detail: "first_command starts by deleting files, not measuring"}
|
||||
}
|
||||
return eval.Score{Value: 1, Detail: "first_command is a safe du|sort scan of the filesystem"}
|
||||
})
|
||||
}
|
||||
|
||||
// scenDiskpressureFirstcmdTest: give the exact one-line first command to
|
||||
// find what is filling a 100%-full root filesystem.
|
||||
//
|
||||
// ground truth: / is 100% full on /dev/sda1 while /data (/dev/sdb1) has
|
||||
// 470G free, so the scan must stay within the / filesystem and rank top-
|
||||
// level directories by size. The canonical, standard-OSS one-liner is
|
||||
// `du -xh --max-depth=1 / 2>/dev/null | sort -rh | head -20` - du with
|
||||
// -x to not descend into the nearly-empty /data mount, a depth of 1 to get
|
||||
// per-top-level-directory usage, sort -rh descending by human size, and
|
||||
// head to the top consumers. ls does not measure recursive usage and find
|
||||
// -size only matches known file sizes, so neither finds the fillers.
|
||||
func scenDiskpressureFirstcmdTest() testkit.Test {
|
||||
prompt := `This Linux host's root filesystem /dev/sda1 is 100%
|
||||
full; /data on /dev/sdb1 has plenty of free space. You must find which
|
||||
paths under / are consuming the space, so you can free it. Here is the
|
||||
relevant df output:
|
||||
|
||||
` + scenDiskpressureDf + `
|
||||
|
||||
Give the exact one-line shell command you would run FIRST to identify the
|
||||
largest consumers under /, staying on the / filesystem and producing a
|
||||
size-sorted list. Respond with only a JSON object:
|
||||
{"first_command":"<the exact one-line command>"}`
|
||||
|
||||
return testkit.Test{
|
||||
ID: "scen-diskpressure-firstcmd",
|
||||
Category: "operations",
|
||||
Subcategory: "scenario",
|
||||
Description: "Give the exact first shell command (du|sort) to find what filled a 100%-full root filesystem.",
|
||||
Prompt: prompt,
|
||||
Eval: scenFirstDiskCommand(),
|
||||
}
|
||||
}
|
||||
|
||||
// scenInodeExhaustionDfHi is the inline df -hi output for
|
||||
// scenInodeExhaustionTest.
|
||||
const scenInodeExhaustionDfHi = `$ df -h /
|
||||
Filesystem Size Used Avail Use% Mounted on
|
||||
/dev/sda1 100G 35G 65G 35% /
|
||||
|
||||
$ df -hi /
|
||||
Filesystem Inodes IUsed IFree IUse% Mounted on
|
||||
/dev/sda1 1.0M 1.0M 0 100% /`
|
||||
|
||||
// scenInodeExhaustionTest: diagnose "No space left on device" that is
|
||||
// actually inode exhaustion (blocks free, inodes exhausted).
|
||||
//
|
||||
// ground truth: writes fail with "No space left on device" even though
|
||||
// df -h shows 65G free, because df -hi shows the filesystem's inode
|
||||
// table is 100% used (1.0M of 1.0M). Inode exhaustion, not block
|
||||
// exhaustion, is the cause; the command that reveals it is `df -hi`
|
||||
// (df with the inode flag). The fix is deleting cheap small files
|
||||
// (often an unbounded cache dir filling the inode table with tiny files),
|
||||
// not growing the disk (which adds blocks, not inodes) or swapping mounts.
|
||||
func scenInodeExhaustionTest() testkit.Test {
|
||||
prompt := `Deleting nothing and with the disk apparently not full,
|
||||
this host now fails writes with "No space left on device". Here are two
|
||||
diagnostic outputs:
|
||||
|
||||
` + scenInodeExhaustionDfHi + `
|
||||
|
||||
What is the cause, and which exact command produced the revealing
|
||||
output? Respond with only a JSON object:
|
||||
{"cause":"<one of: inodes-exhausted, disk-full, quota-exceeded>",
|
||||
"command":"<one of: df -hi, df -h, stat -f> "}`
|
||||
|
||||
evaluator := eval.Mean(
|
||||
eval.JSONField("cause", "inodes-exhausted"),
|
||||
eval.JSONField("command", "df -hi"),
|
||||
)
|
||||
|
||||
return testkit.Test{
|
||||
ID: "scen-inode-exhaustion",
|
||||
Category: "operations",
|
||||
Subcategory: "scenario",
|
||||
Description: "Recognize ENOSPC from inode exhaustion (not disk space) and the df -hi command that reveals it.",
|
||||
Prompt: prompt,
|
||||
Eval: evaluator,
|
||||
}
|
||||
}
|
||||
|
||||
// scenCrashloopOrderTest: an incident where the operator must pick the
|
||||
// diagnostic steps to find why a CrashLoopBackOff container crashes,
|
||||
// before taking any action.
|
||||
//
|
||||
// ground truth: the only observation is the CrashLoopBackOff status and
|
||||
// the restart count - nothing about *why* the container dies. Both
|
||||
// read-only diagnostics are needed: B (`kubectl describe pod`, recent
|
||||
// events and the last exit reason - rules out OOM-kill, image problems,
|
||||
// probe failures) and A (`kubectl logs <pod> --previous`, the actual
|
||||
// error output of the crashed run - names the failing line). Neither
|
||||
// alone roots an app crash: describe shows the exit code but not the
|
||||
// error message; logs --previous show the error but not the node-level
|
||||
// events. Whether describe or logs comes first is a matter of taste, so
|
||||
// the evaluator scores the SET {A, B}. What the test discriminates is
|
||||
// diagnose-before-acting: C (`rollout restart`) and D (`scale
|
||||
// --replicas=0`) act without diagnosing, and E (ssh to a node for
|
||||
// journalctl) reaches for node-level tooling prematurely when the
|
||||
// container's own state and logs are one kubectl call away.
|
||||
func scenCrashloopOrderTest() testkit.Test {
|
||||
prompt := `A pod named ingestion-5d7c9b6f8b-p2k1q in namespace data
|
||||
keeps crashing and restarting: kubectl get pods shows STATUS
|
||||
CrashLoopBackOff, RESTARTS 14. You know nothing else about why it dies,
|
||||
and you must find the root cause before changing anything. These are the
|
||||
available steps:
|
||||
|
||||
A) kubectl logs <pod> --previous (read the last crashed run's output)
|
||||
B) kubectl describe pod <pod> (inspect events and last exit reason)
|
||||
C) kubectl rollout restart deployment ingestion
|
||||
D) kubectl scale deployment ingestion --replicas=0
|
||||
E) ssh to a node and run journalctl -u kubelet
|
||||
|
||||
Respond with only a JSON array of the step letters that make up your
|
||||
diagnosis, containing exactly the diagnostic steps you would take to find
|
||||
the root cause (any order) - do not include steps that only act without
|
||||
diagnosing.`
|
||||
|
||||
return testkit.Test{
|
||||
ID: "scen-crashloop-order",
|
||||
Category: "operations",
|
||||
Subcategory: "scenario",
|
||||
Description: "Pick the diagnostic steps (describe + logs --previous) that root-cause a CrashLoopBackOff before acting.",
|
||||
Prompt: prompt,
|
||||
Eval: eval.JSONStringSet([]string{"A", "B"}),
|
||||
}
|
||||
}
|
||||
|
||||
// scenNetLayerSymptom is the inline diagnostic session for
|
||||
// scenNetLayerIsolateTest.
|
||||
const scenNetLayerSymptom = `$ curl -sS https://api.example.org/data
|
||||
curl: (6) Could not resolve host: api.example.org
|
||||
|
||||
$ ping -c1 10.0.0.7
|
||||
64 bytes from 10.0.0.7: icmp_seq=1 ttl=64 time=0.8 ms
|
||||
(10.0.0.7 is a working internal host on the same subnet)
|
||||
|
||||
$ cat /etc/resolv.conf
|
||||
nameserver 10.0.0.53`
|
||||
|
||||
// scenNetLayerIsolateTest: isolate which network layer is broken from
|
||||
// inline symptoms and pick the strict layer among an enumerated set.
|
||||
//
|
||||
// ground truth: curl fails with "Could not resolve host" - DNS
|
||||
// resolution, not a routing or TCP problem (an ICMP packet to an internal
|
||||
// host on the same subnet succeeds, so IP reachability on the local
|
||||
// segment is fine, and curl never even reached TCP/TLS because it could
|
||||
// not translate the name). The nameserver 10.0.0.53 is configured but
|
||||
// the name does not resolve, so the failing layer is strictly dns; it is
|
||||
// not ip-routing, tcp-connect, tls, or http, none of which are reached
|
||||
// when resolution itself fails.
|
||||
func scenNetLayerIsolateTest() testkit.Test {
|
||||
prompt := `A service on this host cannot reach https://api.example.org.
|
||||
Here is the diagnostic session:
|
||||
|
||||
` + scenNetLayerSymptom + `
|
||||
|
||||
At which network layer is the failure occurring? Respond with only a
|
||||
JSON object:
|
||||
{"layer":"<one of: dns, ip-routing, tcp-connect, tls, http>"}
|
||||
|
||||
Choose exactly one layer from the enumerated list.`
|
||||
|
||||
return testkit.Test{
|
||||
ID: "scen-net-layer-isolate",
|
||||
Category: "operations",
|
||||
Subcategory: "scenario",
|
||||
Description: "Isolate a network failure to the DNS layer from inline curl/ping/resolv.conf symptoms.",
|
||||
Prompt: prompt,
|
||||
Eval: eval.JSONField("layer", "dns"),
|
||||
}
|
||||
}
|
||||
|
||||
// scenServiceDegradeContext is the inline telemetry + deploy timeline for
|
||||
// scenServiceDegradeTest.
|
||||
const scenServiceDegradeContext = `Canary deployed to 100% at 14:00: config file
|
||||
change lowering the app's connection-pool size (pool: 200 -> 50) and
|
||||
raising the worker count.
|
||||
|
||||
Latency (p99) of the checkout service API:
|
||||
13:30 210ms
|
||||
14:05 1400ms
|
||||
14:30 1650ms
|
||||
15:00 1700ms
|
||||
|
||||
Resource metrics for the checkout pods: CPU at 12%, memory at 40% -
|
||||
both low. No crash, no 5xx errors, no data loss.`
|
||||
|
||||
// scenServiceDegradeTest: a service-degradation decision tree where the
|
||||
// regression is tracked to a config change with healthy resources.
|
||||
//
|
||||
// ground truth: the p99 latency jumped from ~210ms to ~1400ms exactly at
|
||||
// the 14:00 canary rollout, whose only meaningful change is the
|
||||
// connection-pool size drop (200 -> 50) - the obvious pattern match is the
|
||||
// config change, and CPU/memory are low so the pool is not saturated by
|
||||
// resource pressure. The correct action is to roll back / revert that
|
||||
// config change (rollback-config). scale-out is wrong (no CPU/memory
|
||||
// pressure to scale for), restart alone is wrong (it does not revert the
|
||||
// config), and restore-backup is wrong (no data loss to recover from).
|
||||
func scenServiceDegradeTest() testkit.Test {
|
||||
prompt := `The checkout service API latency spiked and stayed high.
|
||||
Here is the context:
|
||||
|
||||
` + scenServiceDegradeContext + `
|
||||
|
||||
Which single action best restores service? Respond with only a JSON
|
||||
object:
|
||||
{"action":"<one of: rollback-config, scale-out, restore-backup, restart>"}`
|
||||
|
||||
return testkit.Test{
|
||||
ID: "scen-service-degrade",
|
||||
Category: "operations",
|
||||
Subcategory: "scenario",
|
||||
Description: "Pick the correct remediation (config rollback) for a canary-introduced service degradation with healthy resources.",
|
||||
Prompt: prompt,
|
||||
Eval: eval.JSONField("action", "rollback-config"),
|
||||
}
|
||||
}
|
||||
|
||||
// scenPgConnectionLog is the inline log excerpt for
|
||||
// scenLogRootcauseMCQTest.
|
||||
const scenPgConnectionLog = `2026-08-29 14:22:11 UTC FATAL: remaining connection slots are reserved for non-replication superuser connections
|
||||
2026-08-29 14:22:11 UTC DETAIL: 1000 concurrent connections, 100 remaining connection slots available
|
||||
2026-08-29 14:22:12 UTC ERROR: connection to server at "db.internal" (10.0.1.25), port 5432 failed: FATAL: remaining connection slots are reserved for non-replication superuser connections`
|
||||
|
||||
// scenLogRootcauseMCQTest: map a log pattern to its root cause from
|
||||
// lettered options.
|
||||
//
|
||||
// ground truth: "remaining connection slots are reserved" with a detail
|
||||
// line counting 1000 concurrent connections is the textbook PostgreSQL
|
||||
// signal that the connection slot limit (max_connections) has been
|
||||
// reached - applications are holding more connections than the server is
|
||||
// configured to allow. It is not a disk-full condition (no out-of-space
|
||||
// message), not a slow-query problem (no long query shown), and not an
|
||||
// authentication failure (no `password authentication failed` line).
|
||||
func scenLogRootcauseMCQTest() testkit.Test {
|
||||
prompt := `Applications calling the Postgres database below now fail to
|
||||
connect. Here is a log excerpt:
|
||||
|
||||
` + scenPgConnectionLog + `
|
||||
|
||||
What is the root cause of these connection failures?
|
||||
|
||||
A) connection pool exhausted (max_connections reached)
|
||||
B) disk full
|
||||
C) a single slow query blocking others
|
||||
D) authentication failure
|
||||
|
||||
Respond with only a JSON object: {"answer":"<the single letter A-D>"}`
|
||||
|
||||
return testkit.Test{
|
||||
ID: "scen-log-rootcause-mcq",
|
||||
Category: "operations",
|
||||
Subcategory: "scenario",
|
||||
Description: "Map a PostgreSQL max_connections log pattern to its root cause via a lettered MCQ.",
|
||||
Prompt: prompt,
|
||||
Eval: eval.JSONField("answer", "A"),
|
||||
}
|
||||
}
|
||||
|
||||
// scenBackupRestoreOptions lists the available steps for
|
||||
// scenBackupRestoreOrderTest.
|
||||
const scenBackupRestoreOptions = `A) take a fresh logical backup (pg_dump) of the current data
|
||||
B) apply the destructive migration
|
||||
C) validate row counts and data after the migration
|
||||
D) restore last night's backup before doing anything else`
|
||||
|
||||
// scenBackupRestoreOrderTest: order the steps for a safe data-changing
|
||||
// migration, guaranteeing recoverability.
|
||||
//
|
||||
// ground truth: to make a destructive migration recoverable you must
|
||||
// snapshot the CURRENT state first (A - a fresh pg_dump), then apply the
|
||||
// migration (B), then validate the result (C). Restoring last night's
|
||||
// backup (D) first is wrong: it is a stale snapshot that would throw
|
||||
// away today's writes before any change has even been made, and it cannot
|
||||
// be the safety net for the migration.
|
||||
func scenBackupRestoreOrderTest() testkit.Test {
|
||||
prompt := `You are about to apply a destructive migration to a
|
||||
production Postgres database and must guarantee you can get back to the
|
||||
current state. The available steps are:
|
||||
|
||||
` + scenBackupRestoreOptions + `
|
||||
|
||||
Respond with only a JSON array of the step letters, in the order from
|
||||
first to last, that makes this migration recoverable - include exactly
|
||||
the steps you would run, in order.`
|
||||
|
||||
return testkit.Test{
|
||||
ID: "scen-backup-restore-order",
|
||||
Category: "operations",
|
||||
Subcategory: "scenario",
|
||||
Description: "Order the steps (fresh backup, then migrate, then validate) that make a destructive migration recoverable.",
|
||||
Prompt: prompt,
|
||||
Eval: eval.JSONStringArrayEquals([]string{"A", "B", "C"}),
|
||||
}
|
||||
}
|
||||
|
||||
// scenCertExpirySymptom is the inline symptom for
|
||||
// scenCertexpiryTriageTest.
|
||||
const scenCertExpirySymptom = `$ date
|
||||
Sat Aug 29 09:14:02 UTC 2026
|
||||
|
||||
$ curl -v https://gateway.internal/
|
||||
...
|
||||
* SSL connection using TLSv1.3
|
||||
* SSL certificate verify result: certificate has expired
|
||||
|
||||
$ openssl x509 -in /etc/ssl/svc.crt -noout -dates
|
||||
notBefore=Aug 1 12:00:00 2025 GMT
|
||||
notAfter=Aug 1 12:00:00 2026 GMT`
|
||||
|
||||
// scenCertexpiryTriageTest: triage a TLS failure whose cause is an
|
||||
// expired certificate.
|
||||
//
|
||||
// ground truth: `date` anchors wall time at Aug 29 2026; the openssl
|
||||
// -dates readout shows the served cert valid until Aug 1 2026 - the
|
||||
// current date is past notAfter, so the certificate expired and the
|
||||
// cause category is cert-expired. Clock-skew would need the host clock
|
||||
// to disagree with reality, but the cert's own window explains the
|
||||
// failure exactly, and cipher-mismatch would show a handshake/algo error,
|
||||
// not a verify result. The standard command to inspect a certificate's
|
||||
// validity end date directly is openssl x509 -noout -enddate (the
|
||||
// readout above came from the -dates variant of the same inspection).
|
||||
func scenCertexpiryTriageTest() testkit.Test {
|
||||
prompt := `Clients now fail to connect to gateway.internal with a TLS
|
||||
verification error. Here is the diagnostic session:
|
||||
|
||||
` + scenCertExpirySymptom + `
|
||||
|
||||
What is the cause, and which standard command inspects a certificate's
|
||||
validity directly? Respond with only a JSON object:
|
||||
{"cause":"<one of: cert-expired, clock-skew, cipher-mismatch>",
|
||||
"check_command":"<one of: openssl-x509-enddate, date, openssl-cipher>"}`
|
||||
|
||||
evaluator := eval.Mean(
|
||||
eval.JSONField("cause", "cert-expired"),
|
||||
eval.JSONField("check_command", "openssl-x509-enddate"),
|
||||
)
|
||||
|
||||
return testkit.Test{
|
||||
ID: "scen-certexpiry-triage",
|
||||
Category: "operations",
|
||||
Subcategory: "scenario",
|
||||
Description: "Identify an expired certificate as the TLS failure cause and the openssl command that inspects validity.",
|
||||
Prompt: prompt,
|
||||
Eval: evaluator,
|
||||
}
|
||||
}
|
||||
|
||||
// scenSystemdStatus is the inline systemctl status excerpt for
|
||||
// scenSystemdFailreasonTest.
|
||||
const scenSystemdStatus = `$ systemctl status worker-queue
|
||||
● worker-queue.service - Queue worker
|
||||
Loaded: loaded (/etc/systemd/system/worker-queue.service; enabled)
|
||||
Active: failed (Result: exit-code) since Tue 2026-08-25 09:12:03 UTC; 12s ago
|
||||
Main PID: 4821 (code=exited, status=1/FAILURE)
|
||||
CPU: 120ms
|
||||
|
||||
$ journalctl -u worker-queue -n 6
|
||||
Aug 25 09:12:03 host worker-queue[4821]: config/openapi.yaml: open: no such file or directory
|
||||
Aug 25 09:12:03 host worker-queue[4821]: fatal: failed to load configuration
|
||||
Aug 25 09:12:03 host worker-queue[4821]: exiting after startup failure`
|
||||
|
||||
// scenSystemdFailreasonTest: reason about why a systemd unit failed from
|
||||
// an inline status + journal excerpt.
|
||||
//
|
||||
// ground truth: the journal line "config/openapi.yaml: open: no such
|
||||
// file or directory" - the binary exits because its configuration file
|
||||
// cannot be found at the expected path. The category is missing-config,
|
||||
// not permission (no "Permission denied"), not missing-binary (the
|
||||
// binary ran and printed a config-diagnostic trace), and not port-in-use
|
||||
// (no bind error). The fix is to provide or point the unit at the
|
||||
// correct config path, not to reinstall the binary or chase a port.
|
||||
func scenSystemdFailreasonTest() testkit.Test {
|
||||
prompt := `A systemd unit is failing. Here is the status and journal:
|
||||
|
||||
` + scenSystemdStatus + `
|
||||
|
||||
What category of failure is this? Respond with only a JSON object:
|
||||
{"category":"<one of: missing-config, permission, missing-binary, port-in-use>"}
|
||||
|
||||
Choose exactly one category from the enumerated list.`
|
||||
|
||||
return testkit.Test{
|
||||
ID: "scen-systemd-failreason",
|
||||
Category: "operations",
|
||||
Subcategory: "scenario",
|
||||
Description: "Classify a systemd unit failure (missing config file) from inline status and journal output.",
|
||||
Prompt: prompt,
|
||||
Eval: eval.JSONField("category", "missing-config"),
|
||||
}
|
||||
}
|
||||
|
||||
// scenPortbindError is the inline startup error for scenPortbindFailTest.
|
||||
const scenPortbindError = `$ service listener start
|
||||
(1) listen tcp :8080: bind: address already in use
|
||||
listener: could not bind to 0.0.0.0:8080, exiting
|
||||
|
||||
$ ss -ltnp
|
||||
State Recv-Q Send-Q Local Address:Port Peer Address:Port Process
|
||||
LISTEN 0 128 0.0.0.0:8080 0.0.0.0:* users:(("listener",pid=1044,fd=3))`
|
||||
|
||||
// scenPortbindFailTest: diagnose an "address already in use" bind
|
||||
// failure and the command that identifies the occupying process.
|
||||
//
|
||||
// ground truth: the startup error "bind: address already in use" for
|
||||
// port 8080, and the `ss -ltnp` output shows an existing listener (pid
|
||||
// 1044) already bound to 0.0.0.0:8080 - a duplicate/lingering instance
|
||||
// still holds the port. The cause category is port-in-use, and the
|
||||
// standard command that lists listening sockets with the owning process
|
||||
// is `ss -ltnp` (ss with listener, numerical, port and process flags);
|
||||
// netstat is the older equivalent but ss is the modern standard on this
|
||||
// home-cluster stack.
|
||||
func scenPortbindFailTest() testkit.Test {
|
||||
prompt := `A service fails to start. Here is the output:
|
||||
|
||||
` + scenPortbindError + `
|
||||
|
||||
What is the cause, and which command identified the process holding the
|
||||
port? Respond with only a JSON object:
|
||||
{"cause":"<one of: port-in-use, missing-binary, config-error>",
|
||||
"command":"<one of: ss -ltnp, netstat -r, lsblk>"}`
|
||||
|
||||
evaluator := eval.Mean(
|
||||
eval.JSONField("cause", "port-in-use"),
|
||||
eval.JSONField("command", "ss -ltnp"),
|
||||
)
|
||||
|
||||
return testkit.Test{
|
||||
ID: "scen-portbind-fail",
|
||||
Category: "operations",
|
||||
Subcategory: "scenario",
|
||||
Description: "Diagnose an address-in-use bind failure and the ss -ltnp command that reveals the holding process.",
|
||||
Prompt: prompt,
|
||||
Eval: evaluator,
|
||||
}
|
||||
}
|
||||
|
||||
// scenLvmSteps lists the LVM extend steps for scenLvmResizeOrderTest.
|
||||
const scenLvmSteps = `A) lvextend -L +20G /dev/vg0/data (grow the logical volume)
|
||||
B) pvs (check free space in the volume group)
|
||||
C) resize2fs /dev/vg0/data (grow the filesystem to fill the LV)
|
||||
D) umount /dev/vg0/data (unmount the volume)`
|
||||
|
||||
// scenLvmResizeOrderTest: order the steps of a safe LVM logical-volume
|
||||
// resize.
|
||||
//
|
||||
// ground truth: to extend a filesystem on LVM you first confirm there is
|
||||
// free space in the volume group (B - `pvs`), then extend the logical
|
||||
// volume (A - `lvextend`), then grow the filesystem to fill the newly
|
||||
// added space (C - `resize2fs`). Doing C before A, or A before
|
||||
// confirming free space in B, fails (nothing to grow into); D (umount)
|
||||
// is not required for an online extend of a mounted ext4/xfs volume.
|
||||
func scenLvmResizeOrderTest() testkit.Test {
|
||||
prompt := `You must safely grow the filesystem on LVM logical volume
|
||||
/dev/vg0/data while it stays online. The available steps are:
|
||||
|
||||
` + scenLvmSteps + `
|
||||
|
||||
Respond with only a JSON array of the step letters, in the order from
|
||||
first to last, that correctly extends this volume - include exactly the
|
||||
steps you would run, in order.`
|
||||
|
||||
return testkit.Test{
|
||||
ID: "scen-lvm-resize-order",
|
||||
Category: "operations",
|
||||
Subcategory: "scenario",
|
||||
Description: "Order the LVM resize steps (check VG space, extend LV, then grow the filesystem) for an online extend.",
|
||||
Prompt: prompt,
|
||||
Eval: eval.JSONStringArrayEquals([]string{"B", "A", "C"}),
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,357 @@
|
||||
package tests
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"github.com/lukaszraczylo/llm-testbench/internal/testkit"
|
||||
)
|
||||
|
||||
// TestRegisterScenarioTests_Wiring calls registerScenarioTests into a fresh
|
||||
// registry and checks it contributes a full operations/scenario layer with
|
||||
// scen-* IDs, mirroring the wiring-test pattern used for round-2
|
||||
// categories. It also keeps registerScenarioTests referenced so it is not
|
||||
// reported as dead code while the parent still wires catalog integration.
|
||||
func TestRegisterScenarioTests_Wiring(t *testing.T) {
|
||||
r := testkit.NewRegistry()
|
||||
registerScenarioTests(r)
|
||||
|
||||
if n := r.Len(); n < 10 || n > 12 {
|
||||
t.Fatalf("registerScenarioTests registered %d tests, want 10-12", n)
|
||||
}
|
||||
for _, tc := range r.All() {
|
||||
if tc.Category != "operations" {
|
||||
t.Errorf("test %s has Category %q, want operations", tc.ID, tc.Category)
|
||||
}
|
||||
if tc.Subcategory != "scenario" {
|
||||
t.Errorf("test %s has Subcategory %q, want scenario", tc.ID, tc.Subcategory)
|
||||
}
|
||||
if len(tc.ID) < 5 || tc.ID[:5] != "scen-" {
|
||||
t.Errorf("test %s has an ID not prefixed scen-", tc.ID)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestScenDiskpressureFirstcmdTest_Eval(t *testing.T) {
|
||||
tc := scenDiskpressureFirstcmdTest()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{
|
||||
name: "correct du|sort one-liner",
|
||||
response: `{"first_command":"du -xh --max-depth=1 / 2>/dev/null | sort -rh | head -20"}`,
|
||||
want: 1,
|
||||
},
|
||||
{
|
||||
name: "correct fenced with prose",
|
||||
response: "Here is the command:\n```json\n{\"first_command\":\"du -x --max-depth=1 / 2>/dev/null | sort -h\"}\n```",
|
||||
want: 1,
|
||||
},
|
||||
{
|
||||
name: "wrong: ls does not measure recursive usage",
|
||||
response: `{"first_command":"ls -lah /"}`,
|
||||
want: 0,
|
||||
},
|
||||
{
|
||||
name: "wrong: destructive rm as first step",
|
||||
response: `{"first_command":"rm -rf /var/log && rm -rf /tmp"}`,
|
||||
want: 0,
|
||||
},
|
||||
{
|
||||
name: "wrong: du but no sort",
|
||||
response: `{"first_command":"du -sh /var"}`,
|
||||
want: 0,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestScenInodeExhaustionTest_Eval(t *testing.T) {
|
||||
tc := scenInodeExhaustionTest()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{"correct", `{"cause":"inodes-exhausted","command":"df -hi"}`, 1},
|
||||
{"correct fenced", "```json\n{\"cause\":\"inodes-exhausted\",\"command\":\"df -hi\"}\n```", 1},
|
||||
{"wrong cause", `{"cause":"disk-full","command":"df -hi"}`, 0.5},
|
||||
{"wrong command", `{"cause":"inodes-exhausted","command":"df -h"}`, 0.5},
|
||||
{"both wrong", `{"cause":"quota-exceeded","command":"stat -f"}`, 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestScenCrashloopOrderTest_Eval(t *testing.T) {
|
||||
tc := scenCrashloopOrderTest()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{"both steps", `["B","A"]`, 1},
|
||||
{"both steps fenced", "```json\n[\"B\",\"A\"]\n```", 1},
|
||||
{
|
||||
// The correct answer discusses the rejected options (C as an
|
||||
// unhelpful action, E as premature) at length; the JSON
|
||||
// array still governs, so the prose cannot change the score.
|
||||
name: "both steps with long discussion of rejected options",
|
||||
response: "```json\n[\"B\",\"A\"]\n```\n\n`kubectl describe pod` (B) shows the last exit reason and `kubectl logs --previous` (A) shows the actual crash output; either may come first since both are read-only. " +
|
||||
"A blind `kubectl rollout restart` (C) would not reveal the cause and `kubectl scale --replicas=0` (D) would only stop the workload; " +
|
||||
"ssh'ing to a node for journalctl (E) is node-level tooling that is premature while the container logs are local.",
|
||||
want: 1,
|
||||
},
|
||||
{"order is free", `["A","B"]`, 1},
|
||||
{"only one diagnostic step", `["A"]`, 0.5},
|
||||
// An action step in the answer costs credit: set overlap 2/3.
|
||||
{"correct plus an action step", `["B","A","D"]`, 2.0 / 3.0},
|
||||
{"all wrong", `["C","D","E"]`, 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestScenNetLayerIsolateTest_Eval(t *testing.T) {
|
||||
tc := scenNetLayerIsolateTest()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{"correct", `{"layer":"dns"}`, 1},
|
||||
{"correct fenced", "```json\n{\"layer\":\"dns\"}\n```", 1},
|
||||
{
|
||||
name: "correct, discussing other layers at length",
|
||||
response: `{"layer":"dns"}
|
||||
|
||||
This is not tls or http: curl never reached the handshake because it could not resolve the name at all; the local ping to 10.0.0.7 rules out ip-routing on the segment.`,
|
||||
want: 1,
|
||||
},
|
||||
{"wrong layer", `{"layer":"tls"}`, 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestScenServiceDegradeTest_Eval(t *testing.T) {
|
||||
tc := scenServiceDegradeTest()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{"correct", `{"action":"rollback-config"}`, 1},
|
||||
{"correct fenced", "```json\n{\"action\":\"rollback-config\"}\n```", 1},
|
||||
{
|
||||
name: "correct, discussing rejected actions at length",
|
||||
response: `{"action":"rollback-config"}
|
||||
|
||||
Scaling out is wrong because CPU and memory are both low - there is no resource pressure to scale for; a restart would not revert the config file; and a backup restore is wrong because there is no data loss to recover from. The regression tracks the 14:00 canary pool-size change, so reverting the config is the fix.`,
|
||||
want: 1,
|
||||
},
|
||||
{"wrong action", `{"action":"restart"}`, 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestScenLogRootcauseMCQTest_Eval(t *testing.T) {
|
||||
tc := scenLogRootcauseMCQTest()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{"correct", `{"answer":"A"}`, 1},
|
||||
{"correct fenced", "```json\n{\"answer\":\"A\"}\n```", 1},
|
||||
{
|
||||
name: "correct, discussing every distractor at length",
|
||||
response: `{"answer":"A"}
|
||||
|
||||
This is not disk full (there is no out-of-space message in the log), not a slow query (no long-running query is shown), and not an authentication failure (no password-authentication-failed line) - it is the classic max_connections exhaustion signal.`,
|
||||
want: 1,
|
||||
},
|
||||
{"wrong letter", `{"answer":"C"}`, 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestScenBackupRestoreOrderTest_Eval(t *testing.T) {
|
||||
tc := scenBackupRestoreOrderTest()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{"correct order", `["A","B","C"]`, 1},
|
||||
{"correct order fenced", "```json\n[\"A\",\"B\",\"C\"]\n```", 1},
|
||||
{
|
||||
// The correct answer is allowed to explain at length why
|
||||
// restoring last night's backup (D) first is wrong; the JSON
|
||||
// array is what is scored.
|
||||
name: "correct order, discussing rejected restore-first at length",
|
||||
response: `["A","B","C"]
|
||||
|
||||
Restoring last night's backup first (D) would throw away today's writes before any change is made - it is a stale snapshot, not a safety net for the migration, so I take a fresh pg_dump first.`,
|
||||
want: 1,
|
||||
},
|
||||
{"wrong: restore first", `["D","A"]`, 0},
|
||||
{"wrong: reordered", `["A","C","B"]`, 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestScenCertexpiryTriageTest_Eval(t *testing.T) {
|
||||
tc := scenCertexpiryTriageTest()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{"correct", `{"cause":"cert-expired","check_command":"openssl-x509-enddate"}`, 1},
|
||||
{"correct fenced", "```json\n{\"cause\":\"cert-expired\",\"check_command\":\"openssl-x509-enddate\"}\n```", 1},
|
||||
{"wrong cause", `{"cause":"clock-skew","check_command":"openssl-x509-enddate"}`, 0.5},
|
||||
{"wrong command", `{"cause":"cert-expired","check_command":"date"}`, 0.5},
|
||||
{"both wrong", `{"cause":"cipher-mismatch","check_command":"openssl-cipher"}`, 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestScenSystemdFailreasonTest_Eval(t *testing.T) {
|
||||
tc := scenSystemdFailreasonTest()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{"correct", `{"category":"missing-config"}`, 1},
|
||||
{"correct fenced", "```json\n{\"category\":\"missing-config\"}\n```", 1},
|
||||
{
|
||||
name: "correct, discussing rejected categories at length",
|
||||
response: `{"category":"missing-config"}
|
||||
|
||||
This is not a permission problem (no Permission denied in the journal), not a missing binary (the worker ran and printed a config-diagnostic trace), and not a port in use (no bind error) - the unit fails because config/openapi.yaml cannot be opened.`,
|
||||
want: 1,
|
||||
},
|
||||
{"wrong category", `{"category":"permission"}`, 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestScenPortbindFailTest_Eval(t *testing.T) {
|
||||
tc := scenPortbindFailTest()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{"correct", `{"cause":"port-in-use","command":"ss -ltnp"}`, 1},
|
||||
{"correct fenced", "```json\n{\"cause\":\"port-in-use\",\"command\":\"ss -ltnp\"}\n```", 1},
|
||||
{"wrong cause", `{"cause":"config-error","command":"ss -ltnp"}`, 0.5},
|
||||
{"wrong command", `{"cause":"port-in-use","command":"lsblk"}`, 0.5},
|
||||
{"both wrong", `{"cause":"missing-binary","command":"netstat -r"}`, 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestScenLvmResizeOrderTest_Eval(t *testing.T) {
|
||||
tc := scenLvmResizeOrderTest()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{"correct order", `["B","A","C"]`, 1},
|
||||
{"correct order fenced", "```json\n[\"B\",\"A\",\"C\"]\n```", 1},
|
||||
{"wrong: grows fs before extending LV", `["A","C","B"]`, 0},
|
||||
{"wrong: includes umount", `["B","A","C","D"]`, 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,469 @@
|
||||
package tests
|
||||
|
||||
import (
|
||||
"github.com/lukaszraczylo/llm-testbench/internal/eval"
|
||||
"github.com/lukaszraczylo/llm-testbench/internal/testkit"
|
||||
)
|
||||
|
||||
// registerHardTests registers every programming/hard test. This subcategory
|
||||
// stresses harder daily-programming accuracy - concurrency/async reasoning,
|
||||
// subtle language traps, algorithm output tracing, and API-contract edge
|
||||
// cases - each forced to a short, machine-checkable answer.
|
||||
func registerHardTests(r *testkit.Registry) {
|
||||
r.Register(hardFloatBinaddTest())
|
||||
r.Register(hardNullishOverOrTest())
|
||||
r.Register(hardGoroutineRaceTest())
|
||||
r.Register(hardDeferNamedReturnTest())
|
||||
r.Register(hardIntDivisionTest())
|
||||
r.Register(hardRegexGreedyTest())
|
||||
r.Register(hardPromiseAllSettledTest())
|
||||
r.Register(hardContextTimeoutTest())
|
||||
r.Register(hardJSONUnmarshalNullTest())
|
||||
r.Register(hardVarShadowTest())
|
||||
r.Register(hardPyVersionSortTest())
|
||||
r.Register(hardGoParenBalanceTest())
|
||||
}
|
||||
|
||||
// hardFloatBinaddTest: judge whether 0.1 + 0.2 === 0.3 holds in
|
||||
// JavaScript / IEEE-754 doubles.
|
||||
//
|
||||
// ground truth: neither 0.1 nor 0.2 has an exact binary representation,
|
||||
// so 0.1 + 0.2 computes to the nearest double to the mathematical sum,
|
||||
// which is 0.30000000000000004, not the double for the literal 0.3.
|
||||
// Therefore the strict equality comparison evaluates to false. This is a
|
||||
// canonical, deterministic fact of IEEE-754 double arithmetic, independent
|
||||
// of any runtime; verified with node during authoring.
|
||||
func hardFloatBinaddTest() testkit.Test {
|
||||
prompt := `In JavaScript, evaluate this comparison:
|
||||
|
||||
0.1 + 0.2 === 0.3
|
||||
|
||||
Does the strict-equality comparison evaluate to true or to false?
|
||||
Respond with only a JSON object:
|
||||
{"equal": true}
|
||||
or
|
||||
{"equal": false}`
|
||||
|
||||
return testkit.Test{
|
||||
ID: "hard-float-binadd",
|
||||
Category: "programming",
|
||||
Subcategory: "hard",
|
||||
Description: "Judge IEEE-754 double equality for 0.1 + 0.2 === 0.3 in JavaScript (false).",
|
||||
Prompt: prompt,
|
||||
Eval: eval.JSONField("equal", false),
|
||||
}
|
||||
}
|
||||
|
||||
// hardNullishOverOrTest: trace the value of a nullish-coalescing
|
||||
// expression over a falsy (but non-null) operand.
|
||||
//
|
||||
// ground truth: ?? returns the right-hand operand only when the left-hand
|
||||
// operand is null or undefined. 0 is neither null nor undefined, so
|
||||
// n ?? 42 evaluates to 0 - unlike ||, which would have returned 42 for
|
||||
// the falsy 0. This is the core distinction between nullish coalescing
|
||||
// and logical OR. Verified with node during authoring.
|
||||
func hardNullishOverOrTest() testkit.Test {
|
||||
prompt := `Consider this TypeScript code:
|
||||
|
||||
const n = 0;
|
||||
const v = n ?? 42;
|
||||
|
||||
What is the value of v?
|
||||
Respond with only a JSON object:
|
||||
{"value": ...}
|
||||
|
||||
Give the numeric value only.`
|
||||
|
||||
return testkit.Test{
|
||||
ID: "hard-nullish-over-or",
|
||||
Category: "programming",
|
||||
Subcategory: "hard",
|
||||
Description: "Trace nullish-coalescing over a falsy-but-non-null operand: 0 ?? 42 is 0, not 42.",
|
||||
Prompt: prompt,
|
||||
Eval: eval.JSONField("value", 0),
|
||||
}
|
||||
}
|
||||
|
||||
// hardGoroutineRaceTest: judge whether an unsynchronized shared counter
|
||||
// is guaranteed to reach N*M exactly.
|
||||
//
|
||||
// ground truth: incrementing a shared int from multiple goroutines with no
|
||||
// synchronization is a data race (Go's race detector would flag it). A
|
||||
// read-modify-write of a shared variable is not atomic; two goroutines can
|
||||
// read the same value before either writes back, so increments are lost.
|
||||
// The final value is some value strictly between M (at best serialized,
|
||||
// one goroutine's work) and N*M, but it is NOT guaranteed to equal
|
||||
// N*M. This claim follows from the Go memory model / data-race semantics
|
||||
// and cannot be recomputed by running (the outcome is nondeterministic).
|
||||
func hardGoroutineRaceTest() testkit.Test {
|
||||
prompt := `A Go program starts 8 goroutines, each of which performs:
|
||||
counter++
|
||||
1000 times, where counter is a shared int with NO synchronization
|
||||
(no mutex, no atomic) between the goroutines. The program then waits for
|
||||
all 8 goroutines and prints counter.
|
||||
|
||||
Is counter's final value guaranteed to be exactly 8000?
|
||||
Respond with only a JSON object:
|
||||
{"guaranteed": true}
|
||||
or
|
||||
{"guaranteed": false}`
|
||||
|
||||
return testkit.Test{
|
||||
ID: "hard-goroutine-race",
|
||||
Category: "programming",
|
||||
Subcategory: "hard",
|
||||
Description: "Judge whether an unsynchronized shared counter is guaranteed to reach N*M (false: data race).",
|
||||
Prompt: prompt,
|
||||
Eval: eval.JSONField("guaranteed", false),
|
||||
}
|
||||
}
|
||||
|
||||
// hardDeferNamedReturnTest: trace the return value of a function whose
|
||||
// deferred closure writes its named return during a normal (non-panic)
|
||||
// return.
|
||||
//
|
||||
// ground truth: return 7 first assigns the named result r = 7; then,
|
||||
// still before the function returns to its caller, the deferred closure
|
||||
// runs and overwrites r = 99. The caller observes the named return after
|
||||
// all defers have run, so f() returns 99 - not 7 and not 5. This is the
|
||||
// classic named-return-plus-defer trap. Verified with go run during
|
||||
// authoring.
|
||||
func hardDeferNamedReturnTest() testkit.Test {
|
||||
prompt := `Consider this Go code:
|
||||
|
||||
func f() (r int) {
|
||||
defer func() { r = 99 }()
|
||||
r = 5
|
||||
return 7
|
||||
}
|
||||
|
||||
What value does f() return?
|
||||
Respond with only the number, no explanation.`
|
||||
|
||||
return testkit.Test{
|
||||
ID: "hard-defer-named-return",
|
||||
Category: "programming",
|
||||
Subcategory: "hard",
|
||||
Description: "Trace a deferred closure overwriting a named return value during a normal return (99).",
|
||||
Prompt: prompt,
|
||||
Eval: eval.Numeric(eval.ExtractLastNumber[int], hardDeferNamedReturnWant, 0),
|
||||
}
|
||||
}
|
||||
|
||||
// hardDeferNamedReturnWant is f()'s return value.
|
||||
//
|
||||
// ground truth: the deferred closure runs during the function's epilogue,
|
||||
// after the explicit `return 7` has set the named result r = 7, and
|
||||
// overwrites it with 99; the caller therefore receives 99 (see the full
|
||||
// derivation on hardDeferNamedReturnTest). Recomputed by running the
|
||||
// identical snippet with go run during authoring.
|
||||
const hardDeferNamedReturnWant = 99
|
||||
|
||||
// hardIntDivisionTest: judge the result of integer division being
|
||||
// assigned to a floating-point variable.
|
||||
//
|
||||
// ground truth: a / b with two int operands performs integer division,
|
||||
// which truncates toward zero: 7 / 2 = 3 (not 3.5). That integral
|
||||
// result is then implicitly converted to double, so d = 3.0. The common
|
||||
// error is assuming floating division because the target is a double.
|
||||
// Verified with go run during authoring.
|
||||
func hardIntDivisionTest() testkit.Test {
|
||||
prompt := `Consider this C code:
|
||||
|
||||
int a = 7;
|
||||
int b = 2;
|
||||
double d = a / b;
|
||||
|
||||
What is the value of d?
|
||||
Respond with only the number, no explanation.`
|
||||
|
||||
return testkit.Test{
|
||||
ID: "hard-int-division",
|
||||
Category: "programming",
|
||||
Subcategory: "hard",
|
||||
Description: "Trace integer-division-then-widen: 7 / 2 is 3 (stored as 3.0), not 3.5.",
|
||||
Prompt: prompt,
|
||||
Eval: eval.Numeric(eval.ExtractLastNumber[float64], hardIntDivisionWant, 0),
|
||||
}
|
||||
}
|
||||
|
||||
// hardIntDivisionWant is d's value.
|
||||
//
|
||||
// ground truth: integers divide as integers (truncating), giving 7 / 2 =
|
||||
// 3, converted to double 3.0; the float value is 3, so Numeric want is
|
||||
// 3.0. Recomputed by running the identical arithmetic during authoring.
|
||||
const hardIntDivisionWant = 3.0
|
||||
|
||||
// hardRegexGreedyTest: trace which substrings two greedy capture groups
|
||||
// of an anchored pattern match.
|
||||
//
|
||||
// ground truth: quantifiers are greedy by default - each takes as much as
|
||||
// it can while still allowing the rest of the pattern to match. In
|
||||
// /^(\d+)(\d+)$/, the first \d+ grabs the longest prefix that leaves at
|
||||
// least one digit for the second \d+ (which needs at least one match
|
||||
// because + is one-or-more). That longest prefix of "12345" is "1234",
|
||||
// leaving "5" for group 2. A naive symmetric split like ["12","345"]
|
||||
// fails because group 1 would then still be able to grow. Verified with
|
||||
// node during authoring.
|
||||
func hardRegexGreedyTest() testkit.Test {
|
||||
prompt := `Consider this regular expression applied to the string "12345":
|
||||
|
||||
^(\d+)(\d+)$
|
||||
|
||||
(^ and $ anchor to the whole string, and \d+ means one or more digits.)
|
||||
|
||||
What are the two captured groups? Respond with only a JSON array of the
|
||||
two captured strings, in group order, for example ["a","b"]`
|
||||
|
||||
return testkit.Test{
|
||||
ID: "hard-regex-greedy",
|
||||
Category: "programming",
|
||||
Subcategory: "hard",
|
||||
Description: "Trace greedy capture-group backtracking: ^(one-or-more digits)(one-or-more digits)$ on 12345 yields 1234 and 5.",
|
||||
Prompt: prompt,
|
||||
Eval: eval.JSONStringArrayEquals([]string{"1234", "5"}),
|
||||
}
|
||||
}
|
||||
|
||||
// hardPromiseAllSettledTest: trace the per-result status values of
|
||||
// Promise.allSettled over a mixed resolve/reject input.
|
||||
//
|
||||
// ground truth: Promise.allSettled always resolves, and each element of
|
||||
// its result array is an object whose status is the literal "fulfilled"
|
||||
// for a resolved input and "rejected" for a rejected input, preserving
|
||||
// input order. Promise.resolve(42) yields {status:"fulfilled", value:42};
|
||||
// Promise.reject(new Error("boom")) yields {status:"rejected",
|
||||
// reason:...}. So the two status values, in order, are "fulfilled" then
|
||||
// "rejected". Verified with node during authoring.
|
||||
func hardPromiseAllSettledTest() testkit.Test {
|
||||
prompt := `Consider this TypeScript code:
|
||||
|
||||
const results = await Promise.allSettled([
|
||||
Promise.resolve(42),
|
||||
Promise.reject(new Error("boom")),
|
||||
]);
|
||||
|
||||
Each element of results has a "status" field.
|
||||
|
||||
List the two status values, in array order. Respond with only a JSON
|
||||
array of the two status strings, for example ["a","b"]`
|
||||
|
||||
return testkit.Test{
|
||||
ID: "hard-promise-allsettled",
|
||||
Category: "programming",
|
||||
Subcategory: "hard",
|
||||
Description: "Trace Promise.allSettled result statuses over a fulfilled+rejected input (fulfilled, rejected).",
|
||||
Prompt: prompt,
|
||||
Eval: eval.JSONStringArrayEquals([]string{"fulfilled", "rejected"}),
|
||||
}
|
||||
}
|
||||
|
||||
// hardContextTimeoutTest: identify the exact error string a timed-out
|
||||
// context's Err() returns.
|
||||
//
|
||||
// ground truth: when a context created by context.WithTimeout fires, its
|
||||
// Done channel closes and Err() returns the sentinel
|
||||
// context.DeadlineExceeded, whose Error() string (per the context
|
||||
// package) is exactly "context deadline exceeded" - which is distinct
|
||||
// from "context canceled", the string of context.Canceled returned when a
|
||||
// cancel function is called. Verified with go run during authoring.
|
||||
func hardContextTimeoutTest() testkit.Test {
|
||||
prompt := `In Go, a context created with context.WithTimeout fires after its
|
||||
deadline. At that point:
|
||||
|
||||
ctx.Err()
|
||||
|
||||
returns a non-nil error. What is the exact string of that error?
|
||||
Respond with only a JSON object:
|
||||
{"err": "..."}
|
||||
|
||||
with the exact string value.`
|
||||
|
||||
return testkit.Test{
|
||||
ID: "hard-context-timeout",
|
||||
Category: "programming",
|
||||
Subcategory: "hard",
|
||||
Description: "Identify ctx.Err() after a context.WithTimeout deadline fires (context deadline exceeded).",
|
||||
Prompt: prompt,
|
||||
Eval: eval.JSONField("err", "context deadline exceeded"),
|
||||
}
|
||||
}
|
||||
|
||||
// hardJSONUnmarshalNullTest: judge the behavior of json.Unmarshal on a
|
||||
// literal JSON null into an already-populated struct.
|
||||
//
|
||||
// ground truth: per encoding/json, a JSON null into a non-nil pointer
|
||||
// target is a no-op: Unmarshal returns nil (no error) and leaves the
|
||||
// target's existing value unchanged. It does not zero the struct and does
|
||||
// not error. Verified with go run during authoring (a struct holding
|
||||
// Items: [1] stays [1] with nil error).
|
||||
func hardJSONUnmarshalNullTest() testkit.Test {
|
||||
prompt := `In Go:
|
||||
|
||||
var s Struct // s is a populated struct value (some fields set)
|
||||
err := json.Unmarshal([]byte("null"), &s)
|
||||
|
||||
Does Unmarshal return an error, and is s left unchanged (no fields
|
||||
cleared)?
|
||||
Respond with only a JSON object:
|
||||
{"returns_error": true}
|
||||
or
|
||||
{"returns_error": false}`
|
||||
|
||||
return testkit.Test{
|
||||
ID: "hard-json-unmarshal-null",
|
||||
Category: "programming",
|
||||
Subcategory: "hard",
|
||||
Description: "Judge json.Unmarshal of a literal null into a populated struct (no error, unchanged).",
|
||||
Prompt: prompt,
|
||||
Eval: eval.JSONField("returns_error", false),
|
||||
}
|
||||
}
|
||||
|
||||
// hardVarShadowTest: judge whether an inner := declaration shadows an
|
||||
// outer variable without changing it.
|
||||
//
|
||||
// ground truth: `count := 5` inside the if block declares a NEW, inner
|
||||
// variable that shadows the outer count for the duration of that block
|
||||
// (Go's := always creates a new variable when at least one variable on
|
||||
// the left is new in the current scope). Incrementing the inner count
|
||||
// leaves the outer count untouched, so after the block outer count is
|
||||
// still 0. Verified with go run during authoring.
|
||||
func hardVarShadowTest() testkit.Test {
|
||||
prompt := `Consider this Go code:
|
||||
|
||||
count := 0
|
||||
if true {
|
||||
count := 5
|
||||
count++
|
||||
}
|
||||
// count here?
|
||||
|
||||
Does the outer count remain unchanged (equal to 0) after the if block?
|
||||
Respond with only a JSON object:
|
||||
{"outer_unchanged": true}
|
||||
or
|
||||
{"outer_unchanged": false}`
|
||||
|
||||
return testkit.Test{
|
||||
ID: "hard-var-shadow",
|
||||
Category: "programming",
|
||||
Subcategory: "hard",
|
||||
Description: "Trace Go variable shadowing: an inner := leaves the outer variable unchanged (true).",
|
||||
Prompt: prompt,
|
||||
Eval: eval.JSONField("outer_unchanged", true),
|
||||
}
|
||||
}
|
||||
|
||||
// hardPyVersionVersions is the inline version list for
|
||||
// hardPyVersionSortTest.
|
||||
const hardPyVersionVersions = `versions = [
|
||||
"1.9.0",
|
||||
"1.10.0",
|
||||
"1.2.0",
|
||||
"2.0.0",
|
||||
"0.9.9",
|
||||
]`
|
||||
|
||||
// hardPyVersionSortWant is the expected stdout of a correct script: the
|
||||
// second-highest version under numeric component-by-component ordering.
|
||||
//
|
||||
// ground truth: ordering versions by numeric components puts 0.9.9 <
|
||||
// 1.2.0 < 1.9.0 < 1.10.0 < 2.0.0 (1.10.0 is newer than 1.9.0 because
|
||||
// its second component 10 > 9), so the second-highest is 1.10.0. A
|
||||
// plain lexicographic string sort (the trap) would instead report 1.9.0,
|
||||
// which is exactly the failure the test discriminates. Recomputed by
|
||||
// running the correct script with python3 during authoring.
|
||||
const hardPyVersionSortWant = "1.10.0"
|
||||
|
||||
// hardPyVersionSortTest: write a python3 script that prints the
|
||||
// second-highest version under numeric component ordering.
|
||||
//
|
||||
// ground truth: see hardPyVersionSortWant; correctness is determined by
|
||||
// running the script (version ordering is behavior, not a text match).
|
||||
func hardPyVersionSortTest() testkit.Test {
|
||||
prompt := `Here is an inline list of software version strings:
|
||||
|
||||
` + "```python" + `
|
||||
` + hardPyVersionVersions + `
|
||||
` + "```" + `
|
||||
|
||||
Write a python3 program that sorts the versions numerically, component by
|
||||
component (natural version ordering: "1.10.0" is newer than "1.9.0"
|
||||
because the second component 10 > 9), and prints only the second-highest
|
||||
version.`
|
||||
|
||||
return testkit.Test{
|
||||
ID: "hard-py-version-sort",
|
||||
Category: "programming",
|
||||
Subcategory: "hard",
|
||||
Description: "Write a python3 script printing the second-highest version by numeric component ordering.",
|
||||
System: terseCodeOnly,
|
||||
Prompt: prompt,
|
||||
Eval: eval.PyRun(eval.PassthroughHarness, hardPyVersionSortWant),
|
||||
}
|
||||
}
|
||||
|
||||
// hardGoParenBalanceHarness is a complete, independent driver file (its
|
||||
// own package clause and imports) that eval.GoRun writes to harness.go
|
||||
// alongside the model's IsBalanced in solution.go. It drives IsBalanced
|
||||
// across balanced, nested, mismatched, unclosed, and wrongly-ordered
|
||||
// cases, printing PASS only if every case matches its expected result.
|
||||
const hardGoParenBalanceHarness = `package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
)
|
||||
|
||||
func main() {
|
||||
cases := []struct {
|
||||
in string
|
||||
want bool
|
||||
}{
|
||||
{"", true},
|
||||
{"()", true},
|
||||
{"()[]{}", true},
|
||||
{"([{}])", true},
|
||||
{"(", false},
|
||||
{"(]", false},
|
||||
{"([)]", false},
|
||||
{"((", false},
|
||||
{")(", false},
|
||||
}
|
||||
for _, c := range cases {
|
||||
if got := IsBalanced(c.in); got != c.want {
|
||||
fmt.Printf("FAIL %q: got %v want %v\n", c.in, got, c.want)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
fmt.Println("PASS")
|
||||
}`
|
||||
|
||||
// hardGoParenBalanceTest: implement a stack-based balanced-parentheses
|
||||
// checker.
|
||||
//
|
||||
// ground truth: the harness above is the oracle - it exercises empty,
|
||||
// balanced, nested, mismatched-closer, trailing-opener, and wrong-order
|
||||
// inputs, so a correct stack-based implementation prints PASS. There is no
|
||||
// separate hardcoded constant beyond the PASS sentinel.
|
||||
func hardGoParenBalanceTest() testkit.Test {
|
||||
prompt := `Implement a Go function with exactly this signature:
|
||||
|
||||
func IsBalanced(s string) bool
|
||||
|
||||
It must report whether s's parentheses are balanced and correctly nested:
|
||||
'(' '[' '{' are openers, ')' ']' '}' are the matching closers. An empty
|
||||
string is balanced (return true). Return false for any unclosed opener,
|
||||
any closer with no matching open, or any wrong-nesting (e.g. "([)]").`
|
||||
|
||||
return testkit.Test{
|
||||
ID: "hard-go-paren-balance",
|
||||
Category: "programming",
|
||||
Subcategory: "hard",
|
||||
Description: "Implement a stack-based balanced-parentheses checker for (), [], and {}.",
|
||||
System: terseCodeOnly,
|
||||
Prompt: prompt,
|
||||
Eval: eval.GoRun(hardGoParenBalanceHarness, "PASS"),
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,360 @@
|
||||
package tests
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestHardFloatBinaddTest_Eval(t *testing.T) {
|
||||
tc := hardFloatBinaddTest()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{"correct", `{"equal":false}`, 1},
|
||||
{"correct fenced with prose", "The answer is:\n```json\n{\"equal\":false}\n```", 1},
|
||||
{"wrong: claims equal", `{"equal":true}`, 0},
|
||||
{"wrong: non-boolean", `{"equal":"yes"}`, 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestHardNullishOverOrTest_Eval(t *testing.T) {
|
||||
tc := hardNullishOverOrTest()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{"correct", `{"value":0}`, 1},
|
||||
{"correct as string", `{"value":"0"}`, 1},
|
||||
{"wrong: OR semantics", `{"value":42}`, 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestHardGoroutineRaceTest_Eval(t *testing.T) {
|
||||
tc := hardGoroutineRaceTest()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{"correct", `{"guaranteed":false}`, 1},
|
||||
{"correct fenced", "```json\n{\"guaranteed\":false}\n```", 1},
|
||||
{"wrong: claims guaranteed", `{"guaranteed":true}`, 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestHardDeferNamedReturnTest_Eval(t *testing.T) {
|
||||
tc := hardDeferNamedReturnTest()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{"correct", "99", 1},
|
||||
{"correct prose-wrapped", "The function returns 99.", 1},
|
||||
{"wrong: returns statement value", "7", 0},
|
||||
{"wrong: pre-defer value", "5", 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestHardIntDivisionTest_Eval(t *testing.T) {
|
||||
tc := hardIntDivisionTest()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{"correct", "3", 1},
|
||||
{"correct as float", "3.0", 1},
|
||||
{"correct prose-wrapped", "It is 3.", 1},
|
||||
{"wrong: assumes float division", "3.5", 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestHardRegexGreedyTest_Eval(t *testing.T) {
|
||||
tc := hardRegexGreedyTest()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{"correct", `["1234","5"]`, 1},
|
||||
{"correct fenced", "```json\n[\"1234\",\"5\"]\n```", 1},
|
||||
{"wrong: symmetric split", `["12","345"]`, 0},
|
||||
{"wrong: reversed", `["5","1234"]`, 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestHardPromiseAllSettledTest_Eval(t *testing.T) {
|
||||
tc := hardPromiseAllSettledTest()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{"correct", `["fulfilled","rejected"]`, 1},
|
||||
{"correct fenced", "```json\n[\"fulfilled\",\"rejected\"]\n```", 1},
|
||||
{"wrong: reordered", `["rejected","fulfilled"]`, 0},
|
||||
{"wrong: both fulfilled", `["fulfilled","fulfilled"]`, 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestHardContextTimeoutTest_Eval(t *testing.T) {
|
||||
tc := hardContextTimeoutTest()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{"correct", `{"err":"context deadline exceeded"}`, 1},
|
||||
{"correct fenced", "```json\n{\"err\":\"context deadline exceeded\"}\n```", 1},
|
||||
{"wrong: canceled sentinel", `{"err":"context canceled"}`, 0},
|
||||
{"wrong: vague", `{"err":"some error"}`, 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestHardJSONUnmarshalNullTest_Eval(t *testing.T) {
|
||||
tc := hardJSONUnmarshalNullTest()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{"correct", `{"returns_error":false}`, 1},
|
||||
{"correct fenced", "```json\n{\"returns_error\":false}\n```", 1},
|
||||
{"wrong: claims error", `{"returns_error":true}`, 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestHardVarShadowTest_Eval(t *testing.T) {
|
||||
tc := hardVarShadowTest()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{"correct", `{"outer_unchanged":true}`, 1},
|
||||
{"correct fenced", "```json\n{\"outer_unchanged\":true}\n```", 1},
|
||||
{"wrong: claims outer changed", `{"outer_unchanged":false}`, 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestHardPyVersionSortTest_Eval exercises the PyRun evaluator with a
|
||||
// hand-written correct script and a lexicographic (wrong) script.
|
||||
func TestHardPyVersionSortTest_Eval(t *testing.T) {
|
||||
tc := hardPyVersionSortTest()
|
||||
|
||||
correctScript := "```python\n" + `versions = [
|
||||
"1.9.0",
|
||||
"1.10.0",
|
||||
"1.2.0",
|
||||
"2.0.0",
|
||||
"0.9.9",
|
||||
]
|
||||
|
||||
def key(v):
|
||||
return tuple(int(p) for p in v.split("."))
|
||||
|
||||
sorted_versions = sorted(versions, key=key)
|
||||
print(sorted_versions[-2])
|
||||
` + "```"
|
||||
|
||||
wrongLexicographic := "```python\n" + `versions = [
|
||||
"1.9.0",
|
||||
"1.10.0",
|
||||
"1.2.0",
|
||||
"2.0.0",
|
||||
"0.9.9",
|
||||
]
|
||||
print(sorted(versions)[-2])
|
||||
` + "```"
|
||||
|
||||
wrongHardcoded := "```python\nprint('2.0.0')\n```"
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{"correct numeric sort", correctScript, 1},
|
||||
{"wrong: lexicographic sort", wrongLexicographic, 0},
|
||||
{"wrong: hardcoded highest", wrongHardcoded, 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Skipped {
|
||||
t.Skip("python3 not available, skipping exec-evaluator assertion")
|
||||
}
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestHardGoParenBalanceTest_Eval exercises the GoRun evaluator with a
|
||||
// hand-written correct function and a wrong one.
|
||||
func TestHardGoParenBalanceTest_Eval(t *testing.T) {
|
||||
tc := hardGoParenBalanceTest()
|
||||
|
||||
correctFunc := "```go\n" + `func IsBalanced(s string) bool {
|
||||
pairs := map[byte]byte{')': '(', ']': '[', '}': '{'}
|
||||
stack := make([]byte, 0, len(s))
|
||||
for i := 0; i < len(s); i++ {
|
||||
switch s[i] {
|
||||
case '(', '[', '{':
|
||||
stack = append(stack, s[i])
|
||||
case ')', ']', '}':
|
||||
if len(stack) == 0 || stack[len(stack)-1] != pairs[s[i]] {
|
||||
return false
|
||||
}
|
||||
stack = stack[:len(stack)-1]
|
||||
}
|
||||
}
|
||||
return len(stack) == 0
|
||||
}
|
||||
` + "```"
|
||||
|
||||
wrongFunc := "```go\n" + `func IsBalanced(s string) bool {
|
||||
return true
|
||||
}
|
||||
` + "```"
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{"correct stack implementation", correctFunc, 1},
|
||||
{"wrong: always true", wrongFunc, 0},
|
||||
{"wrong: no code at all", "I would use a stack here.", 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Skipped {
|
||||
t.Skip("go toolchain not available, skipping exec-evaluator assertion")
|
||||
}
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestHardDeferNamedReturnWant_GroundTruth recomputes f()'s return value
|
||||
// by running an equivalent function in-process, independent of the
|
||||
// hardDeferNamedReturnWant constant.
|
||||
func TestHardDeferNamedReturnWant_GroundTruth(t *testing.T) {
|
||||
f := func() (r int) {
|
||||
defer func() { r = 99 }()
|
||||
r = 5
|
||||
return 7
|
||||
}
|
||||
if got := f(); got != hardDeferNamedReturnWant {
|
||||
t.Errorf("equivalent deferred-named-return function returned %d, want %d", got, hardDeferNamedReturnWant)
|
||||
}
|
||||
}
|
||||
|
||||
// TestHardIntDivisionWant_GroundTruth recomputes for a double result of
|
||||
// 7 / 2 by hand, independent of the hardIntDivisionWant constant.
|
||||
func TestHardIntDivisionWant_GroundTruth(t *testing.T) {
|
||||
a, b := 7, 2
|
||||
want := float64(a / b) // integer division truncates before widening
|
||||
if want != hardIntDivisionWant {
|
||||
t.Errorf("recomputed integer division = %v, want %v", want, hardIntDivisionWant)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,534 @@
|
||||
// Package-level hardening tests for the five subcategories that measured
|
||||
// 100% perfect-score across all three baseline models (see the health
|
||||
// audit over the 2026-08-18 3-model run): agents/delegation,
|
||||
// databases/postgres, research/web, security/appsec, security/secrets.
|
||||
// Every test there was saturated - zero discrimination signal. These ten
|
||||
// tests are trap-shaped on purpose: each embeds a near-miss answer that
|
||||
// plausible-but-wrong reasoning produces, and every expected value carries
|
||||
// a derivation comment.
|
||||
package tests
|
||||
|
||||
import (
|
||||
"github.com/lukaszraczylo/llm-testbench/internal/eval"
|
||||
"github.com/lukaszraczylo/llm-testbench/internal/testkit"
|
||||
)
|
||||
|
||||
// registerSaturationTests adds the hardening tests to the registry. They
|
||||
// land in existing subcategories, so the per-category register functions
|
||||
// stay untouched; the subcategory's test count only grows.
|
||||
func registerSaturationTests(r *testkit.Registry) {
|
||||
r.Register(satDelegCapacityTrapTest())
|
||||
r.Register(satDelegNearMissRosterTest())
|
||||
r.Register(satPGCountStarTrapTest())
|
||||
r.Register(satPGTimeoutTrapTest())
|
||||
r.Register(satWebRobotsUAScopeTest())
|
||||
r.Register(satWebHreflangSelfTest())
|
||||
r.Register(satAppsecORMTrapTest())
|
||||
r.Register(satAppsecPOSTTrapTest())
|
||||
r.Register(satSecretsRotationMathTest())
|
||||
r.Register(satSecretsErrorPathTest())
|
||||
}
|
||||
|
||||
// satDelegCapacityTrapTest: the roster's specialist for the job exists but
|
||||
// is unavailable. Tests whether the model force-fits the task to a
|
||||
// mismatched agent, over-coordinates with the orchestrator, or correctly
|
||||
// keeps it in the main thread.
|
||||
func satDelegCapacityTrapTest() testkit.Test {
|
||||
prompt := `You are coordinating agents. Available specialist roster:
|
||||
|
||||
` + delegRoster + `
|
||||
|
||||
Situation: the CI pipeline for the main branch has been failing for the
|
||||
past hour on a Docker build step. It needs diagnosing now. The roster's
|
||||
infra/CI specialist is already working on an unrelated production incident
|
||||
and will not free up for several hours. There is exactly one task.
|
||||
|
||||
Respond with only a JSON object, no other text, of the form
|
||||
{"best_next_step": "...", "specialist_this_task_belongs_to": "..."} where:
|
||||
- best_next_step is exactly one of: "dispatch_code_writer",
|
||||
"dispatch_orchestrator", "handle_in_main_thread", "wait_for_specialist"
|
||||
- specialist_this_task_belongs_to is the roster agent name whose
|
||||
description covers this task, regardless of their availability.`
|
||||
return testkit.Test{
|
||||
ID: "agent-deleg-capacity-trap",
|
||||
Category: "agents",
|
||||
Subcategory: "delegation",
|
||||
Description: "Route a task whose specialist is busy: no force-fit to a mismatched agent, no orchestrator for a single task, and still name the correct owning specialist",
|
||||
Prompt: prompt,
|
||||
// ground truth: the task is diagnosing a failing CI build, which
|
||||
// the roster assigns to devops-debugger ("Diagnoses failing
|
||||
// deployments, CI pipelines, and infrastructure incidents"). It is
|
||||
// busy, so the remaining options are judged against the roster:
|
||||
// code-writer "Implements new features or bug fixes by writing and
|
||||
// editing source code directly" - a generalist forced onto infra
|
||||
// diagnosis; orchestrator "Plans and sequences a multi-agent
|
||||
// workflow" - there is one task, nothing to sequence; waiting on a
|
||||
// specialist that is hours away while CI is down blocks delivery
|
||||
// for no gain. A single task with its specialist unavailable is
|
||||
// handled directly (handle_in_main_thread). The owning specialist
|
||||
// on the roster is unambiguously devops-debugger.
|
||||
Eval: eval.Mean(
|
||||
eval.JSONField[string]("best_next_step", "handle_in_main_thread"),
|
||||
eval.JSONField[string]("specialist_this_task_belongs_to", "devops-debugger"),
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
// satDelegNearMissRosterTest: 4-task mapping where every task has a
|
||||
// same-vocabulary distractor (docs vs web lookup, release notes vs prose
|
||||
// docs, diff review vs security review).
|
||||
func satDelegNearMissRosterTest() testkit.Test {
|
||||
prompt := `You are coordinating agents. Specialist roster:
|
||||
|
||||
` + delegRoster + `
|
||||
|
||||
Map each task to exactly one roster agent - the one whose roster
|
||||
description covers it. Tasks:
|
||||
- task1: "Update the API guide in docs/ so it matches the endpoints as
|
||||
they exist in the repository today."
|
||||
- task2: "Determine from the vendor's public pricing page which of their
|
||||
plans includes SSO."
|
||||
- task3: "Produce the v2.3 release notes by generating the changelog
|
||||
section and tagging the release."
|
||||
- task4: "Audit the auth handler for vulnerabilities before it merges."
|
||||
|
||||
Respond with only a JSON object of the form
|
||||
{"task1": "...", "task2": "...", "task3": "...", "task4": "..."}
|
||||
where each value is exactly one roster agent name.`
|
||||
return testkit.Test{
|
||||
ID: "agent-deleg-near-miss-roster",
|
||||
Category: "agents",
|
||||
Subcategory: "delegation",
|
||||
Description: "Route 4 tasks whose phrasing overlaps two roster descriptions each (repo docs vs web lookup, changelog vs prose docs, diff review vs security audit)",
|
||||
Prompt: prompt,
|
||||
// ground truth, per the delegRoster descriptions:
|
||||
// task1 -> docs-writer: it writes/updates prose documentation
|
||||
// ("READMEs, guides, design docs") from repository content.
|
||||
// web-researcher is wrong: "Does not read or write anything in the
|
||||
// repository" and no external search is needed.
|
||||
// task2 -> web-researcher: the answer lives on the vendor's public
|
||||
// page, "Searches the public web and summarizes external
|
||||
// information"; docs-writer is the distractor ("page" sounds like
|
||||
// documentation) but the task reads the web, not the repo.
|
||||
// task3 -> release-manager: "Manages version tagging, changelog
|
||||
// generation, and release publishing" covers exactly the phrasing
|
||||
// (generate changelog + tag). docs-writer is the distractor:
|
||||
// release notes here are the release process, not a guide.
|
||||
// task4 -> security-reviewer: "Audits code or config for security
|
||||
// vulnerabilities"; code-reviewer is the distractor ("before it
|
||||
// merges" sounds like PR review) but code-reviewer reviews
|
||||
// "correctness, style, and risk", while the stated goal is
|
||||
// vulnerabilities.
|
||||
Eval: eval.Mean(
|
||||
eval.JSONField[string]("task1", "docs-writer"),
|
||||
eval.JSONField[string]("task2", "web-researcher"),
|
||||
eval.JSONField[string]("task3", "release-manager"),
|
||||
eval.JSONField[string]("task4", "security-reviewer"),
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
// satPGCountStarFixture is the orders/line-items data for the join
|
||||
// fan-out trap.
|
||||
const satPGCountStarFixture = `CREATE TABLE orders (id bigint PRIMARY KEY, region text NOT NULL);
|
||||
CREATE TABLE order_items (id bigint PRIMARY KEY,
|
||||
order_id bigint NOT NULL REFERENCES orders(id));
|
||||
|
||||
INSERT INTO orders (id, region) VALUES
|
||||
(1001, 'eu'), (1002, 'eu'), (1003, 'us'),
|
||||
(1004, 'eu'), (1005, 'ap'), (1006, 'eu');
|
||||
|
||||
INSERT INTO order_items (id, order_id) VALUES
|
||||
(9001, 1001), (9002, 1001), (9003, 1001),
|
||||
(9004, 1002),
|
||||
(9005, 1003), (9006, 1003), (9007, 1003), (9008, 1003), (9009, 1003),
|
||||
(9010, 1004), (9011, 1004),
|
||||
(9012, 1005);`
|
||||
|
||||
// satPGCountStarTrapTest: COUNT(*) over an INNER JOIN counts line items,
|
||||
// not orders, and the join silently drops the order with zero items. Every
|
||||
// number is derivable from the fixture, so the evaluator never needs to
|
||||
// re-run SQL.
|
||||
func satPGCountStarTrapTest() testkit.Test {
|
||||
prompt := `Here is a PostgreSQL schema with data:
|
||||
|
||||
` + "```sql\n" + satPGCountStarFixture + "\n```" + `
|
||||
|
||||
The business question is: how many ORDERS are in region 'eu'?
|
||||
A developer writes:
|
||||
|
||||
` + "```sql\nSELECT count(*) FROM orders o JOIN order_items oi ON oi.order_id = o.id WHERE o.region = 'eu';\n```" + `
|
||||
|
||||
Respond with only a JSON object, no other text:
|
||||
{"orders_in_eu": <number>, "rows_the_join_query_returns": <number>,
|
||||
"count_star_answers_the_question": true or false}`
|
||||
return testkit.Test{
|
||||
ID: "pg-count-star-vs-join",
|
||||
Category: "databases",
|
||||
Subcategory: "postgres",
|
||||
Description: "Reason about COUNT(*) over an INNER JOIN against a fixed fixture: join fan-out inflates the count and the join drops the childless order",
|
||||
Prompt: prompt,
|
||||
// ground truth: orders with region 'eu' are 1001, 1002, 1004, 1006
|
||||
// (INSERTs above) = 4. The join emits one row per order_item:
|
||||
// 1001 -> 3 items (9001-9003), 1002 -> 1 (9004), 1004 -> 2 (9010,
|
||||
// 9011), and 1006 -> 0 items, so the INNER JOIN drops it entirely.
|
||||
// count(*) therefore returns 3+1+2 = 6. 6 != 4, so the count(*)
|
||||
// query does not answer the question (EXISTS or COUNT(DISTINCT
|
||||
// o.id) would return 4).
|
||||
Eval: eval.Mean(
|
||||
eval.JSONField[int]("orders_in_eu", 4),
|
||||
eval.JSONField[int]("rows_the_join_query_returns", 6),
|
||||
eval.JSONField[bool]("count_star_answers_the_question", false),
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
// satPGTimeoutTrapTest: two sessions, two timeouts, one rule each. The
|
||||
// near-miss is applying idle_in_transaction_session_timeout to a busy
|
||||
// session (or statement_timeout to an idle one), and starting the idle
|
||||
// clock at BEGIN instead of at the last statement's completion.
|
||||
func satPGTimeoutTrapTest() testkit.Test {
|
||||
prompt := `A PostgreSQL server is configured with:
|
||||
|
||||
` + "```\nstatement_timeout = '30s'\nidle_in_transaction_session_timeout = '5s'\n```" + `
|
||||
|
||||
Two sessions on this server:
|
||||
- Session A: BEGIN at 11:00:00, runs statements until 11:00:40 (the last
|
||||
one finishes then), and has been waiting for client input ever since.
|
||||
The transaction is still open.
|
||||
- Session B: BEGIN at 11:00:53, starts one long-running query at 11:00:55
|
||||
which is still executing at 11:01:00.
|
||||
|
||||
Assume no client input arrives and no query finishes after 11:01:00.
|
||||
Respond with only a JSON object, no other text:
|
||||
{"session_a_killed_at": "HH:MM:SS", "session_a_rule": "...",
|
||||
"session_b_killed_at": "HH:MM:SS", "session_b_rule": "..."}
|
||||
where each time is in UTC and each rule is exactly one of:
|
||||
"statement_timeout", "idle_in_transaction_timeout", "not_killed".`
|
||||
return testkit.Test{
|
||||
ID: "pg-timeout-trap",
|
||||
Category: "databases",
|
||||
Subcategory: "postgres",
|
||||
Description: "Apply statement_timeout and idle_in_transaction_session_timeout to the right sessions: idle clock starts at the last statement's completion, not BEGIN, and never applies to a session executing a statement",
|
||||
Prompt: prompt,
|
||||
// ground truth: PostgreSQL's idle_in_transaction_session_timeout
|
||||
// terminates a session "idle ... within an open transaction" -
|
||||
// the clock starts when the session goes idle in transaction,
|
||||
// which for A is 11:00:40 (its last statement finished then), so
|
||||
// A is killed at 11:00:45 - not 11:00:05, since A was executing
|
||||
// statements from 11:00:00 to 11:00:40. A is not subject to
|
||||
// statement_timeout: no statement is running. B is idle in
|
||||
// transaction only from 11:00:53 to 11:00:55 (2s, under the 5s
|
||||
// cap) and then EXECUTES a statement, so the idle timeout no
|
||||
// longer applies; statement_timeout bounds a single running
|
||||
// statement: 11:00:55 + 30s = 11:01:25.
|
||||
Eval: eval.Mean(
|
||||
eval.JSONField[string]("session_a_killed_at", "11:00:45"),
|
||||
eval.JSONField[string]("session_a_rule", "idle_in_transaction_timeout"),
|
||||
eval.JSONField[string]("session_b_killed_at", "11:01:25"),
|
||||
eval.JSONField[string]("session_b_rule", "statement_timeout"),
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
// satWebRobotsFixture carries two rule groups and a longest-match pair.
|
||||
const satWebRobotsFixture = `User-agent: *
|
||||
Disallow: /cdn-cgi/
|
||||
Allow: /cdn-cgi/style.css
|
||||
Sitemap: https://example.com/sitemap.xml
|
||||
|
||||
User-agent: GoogleOther
|
||||
Disallow: /`
|
||||
|
||||
// satWebRobotsUAScopeTest: GoogleOther's rules do not apply to Googlebot,
|
||||
// and within Googlebot's group the longest path match decides. The
|
||||
// near-miss is answering from the GoogleOther block alone, and answering
|
||||
// the Allow/Disallow pair by document order instead of path length.
|
||||
func satWebRobotsUAScopeTest() testkit.Test {
|
||||
prompt := `A site serves this robots.txt at https://example.com/robots.txt:
|
||||
|
||||
` + "```\n" + satWebRobotsFixture + "\n```" + `
|
||||
|
||||
Consider a GET request for https://example.com/cdn-cgi/style.css.
|
||||
Respond with only a JSON object, no other text:
|
||||
{"googlebot_allowed": true or false, "googleother_allowed": true or false,
|
||||
"deciding_rule_for_googlebot": "allow" or "disallow" or "no_rule"}`
|
||||
return testkit.Test{
|
||||
ID: "web-robots-ua-scope-trap",
|
||||
Category: "research",
|
||||
Subcategory: "web",
|
||||
Description: "robots.txt group matching across two User-agent blocks plus longest-path-match precedence - a separate group's Disallow does not apply to the other crawler",
|
||||
Prompt: prompt,
|
||||
// ground truth: per RFC 9309, a crawler only applies the rule
|
||||
// group(s) whose User-agent matches it, and within a group the
|
||||
// path pattern with the LONGEST match wins. Googlebot does not
|
||||
// match the GoogleOther group, so only the '*' group applies:
|
||||
// the URL matches Disallow /cdn-cgi/ (9 chars) and Allow
|
||||
// /cdn-cgi/style.css (18 chars); 18 > 9, so Allow wins ->
|
||||
// googlebot_allowed true, deciding_rule_for_googlebot "allow".
|
||||
// GoogleOther matches its own group, where "Disallow: /" covers
|
||||
// every path -> googleother_allowed false.
|
||||
Eval: eval.Mean(
|
||||
eval.JSONField[bool]("googlebot_allowed", true),
|
||||
eval.JSONField[bool]("googleother_allowed", false),
|
||||
eval.JSONField[string]("deciding_rule_for_googlebot", "allow"),
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
// satWebHreflangSelfTest: an hreflang set missing its self-reference and
|
||||
// x-default's sentinel status. The near-miss is counting the canonical
|
||||
// link as the hreflang self-reference, and treating x-default as a
|
||||
// language.
|
||||
func satWebHreflangSelfTest() testkit.Test {
|
||||
prompt := `The page https://example.com/en-us/ serves this head section:
|
||||
|
||||
` + "```html\n" + `<html lang="en-US">
|
||||
<head>
|
||||
<link rel="alternate" hreflang="en-AU" href="https://example.com/en-au/">
|
||||
<link rel="alternate" hreflang="en-GB" href="https://example.com/en-gb/">
|
||||
<link rel="alternate" hreflang="x-default" href="https://example.com/">
|
||||
<link rel="canonical" href="https://example.com/en-us/">
|
||||
</head>` + "\n```" + `
|
||||
|
||||
Google's hreflang rule: an hreflang group must include a self-reference -
|
||||
an hreflang annotation whose href is the page itself - and every
|
||||
annotation it lists must point back reciprocally.
|
||||
Respond with only a JSON object, no other text:
|
||||
{"missing_self_reference_for": "the BCP 47 language tag of the page that
|
||||
has no hreflang annotation pointing at it", "self_reference_present":
|
||||
true or false, "canonical_counts_as_hreflang_self_reference": true or
|
||||
false, "x_default_is_a_language_tag": true or false}`
|
||||
return testkit.Test{
|
||||
ID: "web-hreflang-self-reference-trap",
|
||||
Category: "research",
|
||||
Subcategory: "web",
|
||||
Description: "hreflang self-reference validation: the page's own language has no annotation, rel=canonical is a different mechanism, and x-default is a sentinel, not a BCP 47 tag",
|
||||
Prompt: prompt,
|
||||
// ground truth: the page is en-US (lang attribute and canonical
|
||||
// URL agree). The hreflang annotations cover en-AU, en-GB,
|
||||
// x-default only - no annotation has href
|
||||
// https://example.com/en-us/, so the missing self-reference tag
|
||||
// is "en-US" and self_reference_present is false.
|
||||
// canonical_counts_as_hreflang_self_reference is false: rel
|
||||
// -canonical deduplicates URLs for indexing; hreflang self-
|
||||
// reference is a separate signal on rel=alternate link elements
|
||||
// - the canonical matching the page URL does not satisfy it.
|
||||
// x_default_is_a_language_tag is false: x-default is Google's
|
||||
// sentinel for "none of the specific languages", not a BCP 47
|
||||
// tag (it carries no language subtag; the other two do).
|
||||
Eval: eval.Mean(
|
||||
eval.JSONField[string]("missing_self_reference_for", "en-US"),
|
||||
eval.JSONField[bool]("self_reference_present", false),
|
||||
eval.JSONField[bool]("canonical_counts_as_hreflang_self_reference", false),
|
||||
eval.JSONField[bool]("x_default_is_a_language_tag", false),
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
// satAppsecORMFixture shows three SQLAlchemy call sites; only one builds
|
||||
// SQL by concatenation before wrapping it in text().
|
||||
const satAppsecORMFixture = `from sqlalchemy import text
|
||||
|
||||
def get_user(conn, name, team):
|
||||
sql = text("SELECT * FROM users WHERE name = :name AND team = :team")
|
||||
return conn.execute(sql, {"name": name, "team": team}).fetchone()
|
||||
|
||||
def search_users(conn, pattern):
|
||||
# callers pass the raw 'q' query-parameter straight in
|
||||
sql = text("SELECT * FROM users WHERE bio LIKE '%" + pattern + "%'")
|
||||
return conn.execute(sql).fetchall()
|
||||
|
||||
def list_orders(conn, status):
|
||||
return conn.execute(
|
||||
text("SELECT * FROM orders WHERE status = :s"), {"s": status}
|
||||
).fetchall()`
|
||||
|
||||
// satAppsecORMTrapTest: SQLAlchemy text() is a wrapper, not a sanitizer -
|
||||
// the concatenation already fused attacker input into the SQL string
|
||||
// before text() ever sees it. The near-miss is "ORM == safe".
|
||||
func satAppsecORMTrapTest() testkit.Test {
|
||||
prompt := `Flask application (the 'q' request parameter is attacker-controlled):
|
||||
|
||||
` + "```python\n" + satAppsecORMFixture + "\n```" + `
|
||||
|
||||
Respond with only a JSON object, no other text:
|
||||
{"injectable_function": "one of get_user | search_users | list_orders",
|
||||
"text_wrapper_neutralizes_concatenation": true or false}`
|
||||
return testkit.Test{
|
||||
ID: "sec-orm-interpolation-trap",
|
||||
Category: "security",
|
||||
Subcategory: "appsec",
|
||||
Description: "Spot SQL injection inside a SQLAlchemy text() call built by string concatenation - the ORM wrapper neutralizes nothing once the string is already fused",
|
||||
Prompt: prompt,
|
||||
// ground truth: get_user and list_orders pass user values as
|
||||
// bound parameters (:name/:team, :s) - the driver sends values
|
||||
// out-of-band, so no injection. search_users builds the SQL
|
||||
// string with + pattern + BEFORE text() wraps it: by the time
|
||||
// text() sees it, the attacker's input is part of the statement
|
||||
// (pattern = "x%' OR 1=1 -- " breaks out of the LIKE literal).
|
||||
// text() marks a string as raw SQL for the ORM; it never
|
||||
// parameterizes or escapes - text_wrapper_neutralizes_
|
||||
// concatenation is false.
|
||||
Eval: eval.Mean(
|
||||
eval.JSONField[string]("injectable_function", "search_users"),
|
||||
eval.JSONField[bool]("text_wrapper_neutralizes_concatenation", false),
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
// satAppsecPOSTFixture shows a logout control where the visible anchor is
|
||||
// a decoy - the actual request is the auto-submitted POST form.
|
||||
const satAppsecPOSTFixture = `<a href="#" onclick="document.getElementById('f').submit(); return false;">
|
||||
Log out
|
||||
</a>
|
||||
<form id="f" action="/api/account/logout" method="post">
|
||||
<input type="hidden" name="confirm" value="true">
|
||||
</form>`
|
||||
|
||||
// satAppsecPOSTTrapTest: the app switched a token-less state-changing
|
||||
// endpoint from GET to POST as its whole CSRF mitigation. Tests whether
|
||||
// the model knows POST alone is not a defense (cross-site <form
|
||||
// method=post> auto-submits with cookies), even when the page only ever
|
||||
// submits it via JavaScript.
|
||||
func satAppsecPOSTTrapTest() testkit.Test {
|
||||
prompt := `A session-authenticated web app renders this logout control. The session
|
||||
cookie is SameSite=None; Secure (sent with every cross-site request), and
|
||||
there is no CSRF token anywhere in the request.
|
||||
|
||||
` + "```\n" + satAppsecPOSTFixture + "\n```" + `
|
||||
|
||||
The developer's note: "I moved logout from GET to POST, so CSRF no longer
|
||||
applies - only JavaScript on THIS page submits the form."
|
||||
|
||||
Assume an attacker page on evil.example hosts its own form with
|
||||
action="https://app.example/api/account/logout", method="post". The victim
|
||||
visits evil.example but clicks nothing and types nothing there.
|
||||
|
||||
Respond with only a JSON object, no other text:
|
||||
{"attacker_form_reaches_logout": true or false,
|
||||
"post_method_alone_prevents_csrf": true or false,
|
||||
"attacker_needs_javascript": true or false}`
|
||||
return testkit.Test{
|
||||
ID: "sec-csrf-post-trap",
|
||||
Category: "security",
|
||||
Subcategory: "appsec",
|
||||
Description: "CSRF after a GET-to-POST switch without a token: cross-site form auto-submit carries the cookie, so the method is not the defense; the attacker needs one line of JS, not knowledge of the page's DOM",
|
||||
Prompt: prompt,
|
||||
// ground truth: SameSite=None means the browser attaches the
|
||||
// session cookie to the attacker's cross-site POST, and there is
|
||||
// no token, so the request is accepted as the victim ->
|
||||
// attacker_form_reaches_logout true; the server authenticates by
|
||||
// cookie, not by which page submitted the form.
|
||||
// post_method_alone_prevents_csrf false: method only restricts
|
||||
// bookmark/simple-link attacks; a hosted <form method=post>
|
||||
// replays the state change. (With SameSite=Lax a cross-site POST
|
||||
// would instead drop the cookie - the prompt pins SameSite=None
|
||||
// so the answer is forced.)
|
||||
// attacker_needs_javascript true: the prompt pins "clicks nothing",
|
||||
// and without JavaScript a hosted form only submits when the victim
|
||||
// presses its own submit button - a cross-site <img>/<a> cannot POST
|
||||
// with the cookie context this request needs, and <meta
|
||||
// http-equiv="refresh"> cannot carry a POST body. The silent attack
|
||||
// requires form.submit().
|
||||
Eval: eval.Mean(
|
||||
eval.JSONField[bool]("attacker_form_reaches_logout", true),
|
||||
eval.JSONField[bool]("post_method_alone_prevents_csrf", false),
|
||||
eval.JSONField[bool]("attacker_needs_javascript", true),
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
// satRotationFixture gives the three durations for the rotation
|
||||
// deadline math.
|
||||
const satRotationFixture = `Internal CA certificate expires: 2026-09-30 18:00 UTC
|
||||
Worst realistic time to mint a replacement cert once rotation starts: 10 h
|
||||
After the new cert is deployed, the reverse proxy's graceful-reload
|
||||
window (old workers serve the old cert for up to this long after the
|
||||
reload is issued): 6 h`
|
||||
|
||||
// satSecretsRotationMathTest: latest safe start time = expiry minus the
|
||||
// sum of both sequential windows. The near-miss is subtracting only the
|
||||
// minting time and forgetting the propagation tail.
|
||||
func satSecretsRotationMathTest() testkit.Test {
|
||||
prompt := `Certificate rotation facts for an internal service:
|
||||
|
||||
` + "```\n" + satRotationFixture + "\n```" + `
|
||||
|
||||
Respond with only a JSON object, no other text:
|
||||
{"total_lead_time_hours": <number>, "latest_start": "YYYY-MM-DD HH:MM",
|
||||
"safe_if_started_2026_09_30_08_00": true or false}
|
||||
(Use UTC throughout. latest_start is the latest time rotation can START
|
||||
and still have the old certificate fully replaced everywhere before it
|
||||
expires.)`
|
||||
return testkit.Test{
|
||||
ID: "sec-rotation-window-math",
|
||||
Category: "security",
|
||||
Subcategory: "secrets",
|
||||
Description: "Certificate rotation deadline math: lead time is minting plus propagation in series, and starting late leaves the old cert serving past expiry",
|
||||
Prompt: prompt,
|
||||
// ground truth: the two windows are sequential - a replacement
|
||||
// can only start propagating once it exists: 10 h + 6 h = 16 h
|
||||
// worst-case lead time. 2026-09-30 18:00 minus 16 h =
|
||||
// 2026-09-30 02:00 latest start. Starting at 08:00 finishes at
|
||||
// the worst at 08:00 + 16 h = 2026-10-01 00:00, six hours after
|
||||
// the old cert expired -> not safe.
|
||||
Eval: eval.Mean(
|
||||
eval.JSONField[int]("total_lead_time_hours", 16),
|
||||
eval.JSONField[string]("latest_start", "2026-09-30 02:00"),
|
||||
eval.JSONField[bool]("safe_if_started_2026_09_30_08_00", false),
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
// satErrorPathFixture traces a password hash from a raise through an
|
||||
// error handler into a log line, while the client only sees a generic
|
||||
// response body.
|
||||
const satErrorPathFixture = `1| def login(user, password):
|
||||
2| row = db.get_user(user)
|
||||
3| if not verify(password, row.hash):
|
||||
4| raise AuthError(f"invalid credentials for user={user} "
|
||||
5| f"hash={row.hash}")
|
||||
6| return token_for(user)
|
||||
7|
|
||||
8| @app.errorhandler(AuthError)
|
||||
9| def on_auth_error(exc):
|
||||
10| app.logger.error(str(exc))
|
||||
11| return "login failed", 401`
|
||||
|
||||
// satSecretsErrorPathTest: the secret (password hash) is interpolated at
|
||||
// the raise, survives the handler's str(exc), and lands in the log even
|
||||
// though the HTTP response is sanitized. The near-miss is "the client
|
||||
// never sees it, so it's fine", and the reverse trap of flagging the
|
||||
// attacker-supplied username as the secret.
|
||||
func satSecretsErrorPathTest() testkit.Test {
|
||||
prompt := `Flask login flow (numbers are source lines):
|
||||
|
||||
` + "```\n" + satErrorPathFixture + "\n```" + `
|
||||
|
||||
row.hash is the stored password hash (sensitive). user comes straight
|
||||
from the login request body.
|
||||
Respond with only a JSON object, no other text:
|
||||
{"hash_reaches_server_logs": true or false, "hash_visible_in_http_response":
|
||||
true or false, "sensitive_value_in_message": "one of user | hash | none"}`
|
||||
return testkit.Test{
|
||||
ID: "sec-error-path-leak-trap",
|
||||
Category: "security",
|
||||
Subcategory: "secrets",
|
||||
Description: "Trace a password hash through raise -> errorhandler -> logger.error: the client response is sanitized but the log line is not, and the echoed user field is attacker-known",
|
||||
Prompt: prompt,
|
||||
// ground truth: the raise on lines 4-5 interpolates row.hash into
|
||||
// the exception message; the handler on line 10 logs str(exc) -
|
||||
// the full message - so the hash reaches the server logs: true.
|
||||
// The handler returns only the literal "login failed" (line 11),
|
||||
// never str(exc), so the hash is not visible in the HTTP
|
||||
// response: false. Between user (attacker-supplied in the request
|
||||
// body, already known to its owner) and hash (sensitive), the
|
||||
// sensitive interpolated value is the hash.
|
||||
Eval: eval.Mean(
|
||||
eval.JSONField[bool]("hash_reaches_server_logs", true),
|
||||
eval.JSONField[bool]("hash_visible_in_http_response", false),
|
||||
eval.JSONField[string]("sensitive_value_in_message", "hash"),
|
||||
),
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,400 @@
|
||||
package tests
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// Each saturation test gets a table pinning: the full-credit answer, the
|
||||
// trap answer (the near-miss the test exists to catch), and a garbage
|
||||
// answer. Expected partial scores follow from eval.Mean's per-field mean.
|
||||
|
||||
func TestSatDelegCapacityTrapTest_Eval(t *testing.T) {
|
||||
tc := satDelegCapacityTrapTest()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{
|
||||
name: "correct: main thread, names busy specialist",
|
||||
response: `{"best_next_step":"handle_in_main_thread","specialist_this_task_belongs_to":"devops-debugger"}`,
|
||||
want: 1,
|
||||
},
|
||||
{
|
||||
name: "trap: force-fits the generalist builder",
|
||||
response: `{"best_next_step":"dispatch_code_writer","specialist_this_task_belongs_to":"devops-debugger"}`,
|
||||
want: 0.5,
|
||||
},
|
||||
{
|
||||
name: "trap: orchestrator for a single task",
|
||||
response: `{"best_next_step":"dispatch_orchestrator","specialist_this_task_belongs_to":"orchestrator"}`,
|
||||
want: 0,
|
||||
},
|
||||
{
|
||||
name: "correct step but wrong owning specialist",
|
||||
response: `{"best_next_step":"handle_in_main_thread","specialist_this_task_belongs_to":"code-writer"}`,
|
||||
want: 0.5,
|
||||
},
|
||||
{
|
||||
name: "wait-it-out answer",
|
||||
response: `{"best_next_step":"wait_for_specialist","specialist_this_task_belongs_to":"devops-debugger"}`,
|
||||
want: 0.5,
|
||||
},
|
||||
{name: "garbage", response: `not json`, want: 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestSatDelegNearMissRosterTest_Eval(t *testing.T) {
|
||||
tc := satDelegNearMissRosterTest()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{
|
||||
name: "all correct",
|
||||
response: `{"task1":"docs-writer","task2":"web-researcher","task3":"release-manager","task4":"security-reviewer"}`,
|
||||
want: 1,
|
||||
},
|
||||
{
|
||||
name: "trap: web-researcher for the repo guide, docs-writer for the pricing page",
|
||||
response: `{"task1":"web-researcher","task2":"docs-writer","task3":"release-manager","task4":"security-reviewer"}`,
|
||||
want: 0.5,
|
||||
},
|
||||
{
|
||||
name: "trap: docs-writer for release notes, code-reviewer for the audit",
|
||||
response: `{"task1":"docs-writer","task2":"web-researcher","task3":"docs-writer","task4":"code-reviewer"}`,
|
||||
want: 0.5,
|
||||
},
|
||||
{
|
||||
name: "all swapped to the distractors",
|
||||
response: `{"task1":"web-researcher","task2":"docs-writer","task3":"docs-writer","task4":"code-reviewer"}`,
|
||||
want: 0,
|
||||
},
|
||||
{
|
||||
name: "case-insensitive correct",
|
||||
response: `{"task1":"Docs-Writer","task2":"Web-Researcher","task3":"Release-Manager","task4":"Security-Reviewer"}`,
|
||||
want: 1,
|
||||
},
|
||||
{name: "garbage", response: `["docs-writer"]`, want: 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestSatPGCountStarTrapTest_Eval(t *testing.T) {
|
||||
tc := satPGCountStarTrapTest()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{
|
||||
name: "correct",
|
||||
response: `{"orders_in_eu": 4, "rows_the_join_query_returns": 6, "count_star_answers_the_question": false}`,
|
||||
want: 1,
|
||||
},
|
||||
{
|
||||
name: "numeric strings accepted",
|
||||
response: `{"orders_in_eu": "4", "rows_the_join_query_returns": "6", "count_star_answers_the_question": "false"}`,
|
||||
want: 1,
|
||||
},
|
||||
{
|
||||
name: "trap: counts joined rows as orders",
|
||||
response: `{"orders_in_eu": 6, "rows_the_join_query_returns": 6, "count_star_answers_the_question": true}`,
|
||||
want: 0.3333333333333333,
|
||||
},
|
||||
{
|
||||
name: "trap: forgets the childless order drops out of the inner join",
|
||||
response: `{"orders_in_eu": 4, "rows_the_join_query_returns": 7, "count_star_answers_the_question": false}`,
|
||||
want: 0.6666666666666666,
|
||||
},
|
||||
{name: "garbage", response: `4`, want: 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestSatPGTimeoutTrapTest_Eval(t *testing.T) {
|
||||
tc := satPGTimeoutTrapTest()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{
|
||||
name: "correct",
|
||||
response: `{"session_a_killed_at":"11:00:45","session_a_rule":"idle_in_transaction_timeout","session_b_killed_at":"11:01:25","session_b_rule":"statement_timeout"}`,
|
||||
want: 1,
|
||||
},
|
||||
{
|
||||
name: "trap: idle clock started at BEGIN",
|
||||
response: `{"session_a_killed_at":"11:00:05","session_a_rule":"idle_in_transaction_timeout","session_b_killed_at":"11:01:25","session_b_rule":"statement_timeout"}`,
|
||||
want: 0.75,
|
||||
},
|
||||
{
|
||||
name: "trap: timeouts swapped between sessions",
|
||||
response: `{"session_a_killed_at":"11:00:30","session_a_rule":"statement_timeout","session_b_killed_at":"11:00:58","session_b_rule":"idle_in_transaction_timeout"}`,
|
||||
want: 0,
|
||||
},
|
||||
{
|
||||
name: "two fields right",
|
||||
response: `{"session_a_killed_at":"11:00:45","session_a_rule":"idle_in_transaction_timeout","session_b_killed_at":"11:01:00","session_b_rule":"not_killed"}`,
|
||||
want: 0.5,
|
||||
},
|
||||
{name: "garbage", response: `no json here`, want: 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestSatWebRobotsUAScopeTest_Eval(t *testing.T) {
|
||||
tc := satWebRobotsUAScopeTest()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{
|
||||
name: "correct: longest match wins for googlebot, group isolation for GoogleOther",
|
||||
response: `{"googlebot_allowed": true, "googleother_allowed": false, "deciding_rule_for_googlebot": "allow"}`,
|
||||
want: 1,
|
||||
},
|
||||
{
|
||||
name: "trap: GoogleOther's disallow applied to googlebot too",
|
||||
response: `{"googlebot_allowed": false, "googleother_allowed": false, "deciding_rule_for_googlebot": "disallow"}`,
|
||||
want: 0.3333333333333333,
|
||||
},
|
||||
{
|
||||
name: "trap: wildcard group treated as absent",
|
||||
response: `{"googlebot_allowed": true, "googleother_allowed": true, "deciding_rule_for_googlebot": "allow"}`,
|
||||
want: 0.6666666666666666,
|
||||
},
|
||||
{name: "garbage", response: `allow`, want: 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestSatWebHreflangSelfTest_Eval(t *testing.T) {
|
||||
tc := satWebHreflangSelfTest()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{
|
||||
name: "correct",
|
||||
response: `{"missing_self_reference_for":"en-US","self_reference_present":false,"canonical_counts_as_hreflang_self_reference":false,"x_default_is_a_language_tag":false}`,
|
||||
want: 1,
|
||||
},
|
||||
{
|
||||
name: "tag casing normalized",
|
||||
response: `{"missing_self_reference_for":"en-us","self_reference_present":false,"canonical_counts_as_hreflang_self_reference":false,"x_default_is_a_language_tag":false}`,
|
||||
want: 1,
|
||||
},
|
||||
{
|
||||
name: "trap: canonical counted as the self-reference",
|
||||
response: `{"missing_self_reference_for":"none","self_reference_present":true,"canonical_counts_as_hreflang_self_reference":true,"x_default_is_a_language_tag":false}`,
|
||||
want: 0.25,
|
||||
},
|
||||
{
|
||||
name: "trap: x-default treated as a language",
|
||||
response: `{"missing_self_reference_for":"en-US","self_reference_present":false,"canonical_counts_as_hreflang_self_reference":false,"x_default_is_a_language_tag":true}`,
|
||||
want: 0.75,
|
||||
},
|
||||
{name: "garbage", response: `en-US`, want: 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestSatAppsecORMTrapTest_Eval(t *testing.T) {
|
||||
tc := satAppsecORMTrapTest()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{
|
||||
name: "correct",
|
||||
response: `{"injectable_function": "search_users", "text_wrapper_neutralizes_concatenation": false}`,
|
||||
want: 1,
|
||||
},
|
||||
{
|
||||
name: "trap: ORM assumed safe everywhere",
|
||||
response: `{"injectable_function": "none", "text_wrapper_neutralizes_concatenation": true}`,
|
||||
want: 0,
|
||||
},
|
||||
{
|
||||
name: "right function, wrong mechanism",
|
||||
response: `{"injectable_function": "search_users", "text_wrapper_neutralizes_concatenation": true}`,
|
||||
want: 0.5,
|
||||
},
|
||||
{name: "garbage", response: `search_users`, want: 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestSatAppsecPOSTTrapTest_Eval(t *testing.T) {
|
||||
tc := satAppsecPOSTTrapTest()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{
|
||||
name: "correct",
|
||||
response: `{"attacker_form_reaches_logout": true, "post_method_alone_prevents_csrf": false, "attacker_needs_javascript": true}`,
|
||||
want: 1,
|
||||
},
|
||||
{
|
||||
name: "trap: POST assumed to prevent CSRF",
|
||||
response: `{"attacker_form_reaches_logout": false, "post_method_alone_prevents_csrf": true, "attacker_needs_javascript": true}`,
|
||||
want: 0.3333333333333333,
|
||||
},
|
||||
{
|
||||
name: "trap: attacker blocked for lacking the page's JS",
|
||||
response: `{"attacker_form_reaches_logout": false, "post_method_alone_prevents_csrf": false, "attacker_needs_javascript": false}`,
|
||||
want: 0.3333333333333333,
|
||||
},
|
||||
{name: "garbage", response: `{"a":1}`, want: 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestSatSecretsRotationMathTest_Eval(t *testing.T) {
|
||||
tc := satSecretsRotationMathTest()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{
|
||||
name: "correct",
|
||||
response: `{"total_lead_time_hours": 16, "latest_start": "2026-09-30 02:00", "safe_if_started_2026_09_30_08_00": false}`,
|
||||
want: 1,
|
||||
},
|
||||
{
|
||||
name: "trap: minting time only, propagation forgotten",
|
||||
response: `{"total_lead_time_hours": 10, "latest_start": "2026-09-30 08:00", "safe_if_started_2026_09_30_08_00": true}`,
|
||||
want: 0,
|
||||
},
|
||||
{
|
||||
name: "sum right, start-time arithmetic wrong",
|
||||
response: `{"total_lead_time_hours": 16, "latest_start": "2026-09-30 02:00", "safe_if_started_2026_09_30_08_00": true}`,
|
||||
want: 0.6666666666666666,
|
||||
},
|
||||
{name: "garbage", response: `16 hours`, want: 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestSatSecretsErrorPathTest_Eval(t *testing.T) {
|
||||
tc := satSecretsErrorPathTest()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{
|
||||
name: "correct",
|
||||
response: `{"hash_reaches_server_logs": true, "hash_visible_in_http_response": false, "sensitive_value_in_message": "hash"}`,
|
||||
want: 1,
|
||||
},
|
||||
{
|
||||
name: "trap: sanitized response assumed to mean no leak",
|
||||
response: `{"hash_reaches_server_logs": false, "hash_visible_in_http_response": false, "sensitive_value_in_message": "hash"}`,
|
||||
want: 0.6666666666666666,
|
||||
},
|
||||
{
|
||||
name: "trap: attacker-known user flagged instead of the hash",
|
||||
response: `{"hash_reaches_server_logs": true, "hash_visible_in_http_response": false, "sensitive_value_in_message": "user"}`,
|
||||
want: 0.6666666666666666,
|
||||
},
|
||||
{
|
||||
name: "trap: leak judged through the log AND the response",
|
||||
response: `{"hash_reaches_server_logs": true, "hash_visible_in_http_response": true, "sensitive_value_in_message": "hash"}`,
|
||||
want: 0.6666666666666666,
|
||||
},
|
||||
{name: "garbage", response: `hash`, want: 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tc.Eval.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -23,29 +23,49 @@ func registerSecurityCryptoTests(r *testkit.Registry) {
|
||||
r.Register(secHashVsEncryptPIITest())
|
||||
}
|
||||
|
||||
// secWeakPasswordHashPattern matches a mention of a fast, general-purpose
|
||||
// hash (MD5, plain SHA-1/256/512) - the ones that must never be used
|
||||
// directly for password storage because they lack a tunable work factor.
|
||||
var secWeakPasswordHashPattern = regexp.MustCompile(`(?i)\b(md5|sha-?1|sha-?256|sha-?512)\b`)
|
||||
// secWeakHashEndorsementPattern matches a weak-hash mention in an
|
||||
// endorsement-shaped position — a verb recommending it ("hash with MD5",
|
||||
// "just hash the password with SHA-256"), or a positive predicate about it
|
||||
// ("SHA-256 is a fine choice") — rather than every mention. The guard then
|
||||
// still requires a negation cue near such a mention (see
|
||||
// secNoUnnegatedMention).
|
||||
//
|
||||
// Triggering on any mention is too strict: a fully correct live answer
|
||||
// (probe, 2026-08-29) lists the discouraged hashes as bare bullets under a
|
||||
// negated heading ("Do **not** use fast general-purpose ... including:") —
|
||||
// the cue sits on the heading, but the negation window stops at the line
|
||||
// break and the heading is more than a window-width away, so each bullet
|
||||
// read as an unnegated endorsement and cost the model half the score.
|
||||
// A bare bullet names, it does not recommend — the predicate branch still
|
||||
// catches an endorsed bullet ("- **MD5** is fine for passwords").
|
||||
var secWeakHashEndorsementPattern = regexp.MustCompile("(?i)(?:" +
|
||||
// a verb recommending it: "hash with MD5", "store passwords hashed with
|
||||
// MD5", "just hash the password with SHA-256" — the filler loop only
|
||||
// bridges recommendation boilerplate, never another algorithm name, so
|
||||
// "prefer argon2 over md5" (a correct answer) is not an endorsement.
|
||||
"(?:hash(?:es|ed|ing)?|store|stores|stored|crypt|encrypt|use|using|used|recommend|recommends|prefer|prefers|should(?:\\s+(?:also|just))?|want)" +
|
||||
"\\s+(?:(?:it|its|the|a|an|password|passwords|user|users|hash|hashed|with|using|as|via|by|plain|raw|fast|for|directly|simply|just|them|to)[ \\t]*\\n?[ \\t]*(?:-[ \\t]*)?){0,6}(?:md5|sha-?1|sha-?256|sha-?512)" +
|
||||
`|(?:^|[^A-Za-z0-9_-])(?:md5|sha-?1|sha-?256|sha-?512)[^.;\n]{0,25}?\b(?:is|are|was|were|seems?|looks?|works?|would\s+be|should\s+be)\s+(?:a\s+|just\s+|still\s+|totally\s+|actually\s+)?(?:fine|good|great|safe|ok|okay|acceptable|perfect|sufficient|enough|best|better|the\s+(?:right|best)\s+choice)` +
|
||||
")")
|
||||
|
||||
// secWeakHashNamedAndNegated scores full credit only when the response BOTH
|
||||
// safely negates every mention of a weak/fast general-purpose hash (or
|
||||
// never mentions one) AND actually names at least one of them by name
|
||||
// (md5/sha-1/sha-256/sha-512) - C11's fix folded into a single AND rather
|
||||
// than an independent extra ContainsAny term, so a wrong answer that
|
||||
// avoids endorsing any weak/fast general-purpose hash with no nearby
|
||||
// negation (or never mentions one) AND actually names at least one of them
|
||||
// by name (md5/sha-1/sha-256/sha-512) - C11's fix folded into a single AND
|
||||
// rather than an independent extra ContainsAny term, so a wrong answer that
|
||||
// recommends SHA-256 outright cannot earn partial credit merely for having
|
||||
// typed "SHA-256" while doing so.
|
||||
func secWeakHashNamedAndNegated() eval.Evaluator {
|
||||
named := eval.ContainsAny("md5", "sha-1", "sha1", "sha-256", "sha256")
|
||||
negated := secNoUnnegatedMention(secWeakPasswordHashPattern)
|
||||
negated := secNoUnnegatedMention(secWeakHashEndorsementPattern)
|
||||
return eval.EvaluatorFunc(func(ctx context.Context, response string) eval.Score {
|
||||
if n := negated.Evaluate(ctx, response); n.Value != 1 {
|
||||
return eval.Score{Value: 0, Detail: "unnegated mention of a discouraged hash: " + n.Detail}
|
||||
return eval.Score{Value: 0, Detail: "endorses a discouraged hash without negating it: " + n.Detail}
|
||||
}
|
||||
if m := named.Evaluate(ctx, response); m.Value != 1 {
|
||||
return eval.Score{Value: 0, Detail: "never names a weak hash by name"}
|
||||
}
|
||||
return eval.Score{Value: 1, Detail: "a weak hash is named and every mention is safely negated"}
|
||||
return eval.Score{Value: 1, Detail: "a weak hash is named and none is endorsed without negation"}
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@@ -28,6 +28,20 @@ func TestSecPasswordHashChoiceTest_Eval(t *testing.T) {
|
||||
response: "bcrypt is fine. Don't use SHA-256 alone, it lacks a work factor.",
|
||||
want: 1,
|
||||
},
|
||||
{
|
||||
// Regression from the live probe (2026-08-29): a fully correct
|
||||
// answer names the discouraged hashes as bullets under a negated
|
||||
// heading ("Do **not** use ... including:"). The heading ends in
|
||||
// ":" - not a clause boundary - so its cue must reach every bullet.
|
||||
name: "correct probe answer: discouraged hashes as bullets under a negated heading",
|
||||
response: "Use Argon2id (or bcrypt with cost >= 12) to store passwords.\n\n" +
|
||||
"Do **not** use fast general-purpose cryptographic hashes for storing passwords, including:\n" +
|
||||
"- **MD5**\n" +
|
||||
"- **SHA-1**\n" +
|
||||
"- **SHA-256**\n" +
|
||||
"- **SHA-512**",
|
||||
want: 1,
|
||||
},
|
||||
{
|
||||
name: "wrong: recommends SHA-256 unnegated",
|
||||
response: "Just hash the password with SHA-256 before storing it.",
|
||||
|
||||
Reference in New Issue
Block a user