From 708622e2a8e18cc9be750e72cf478139c698a757 Mon Sep 17 00:00:00 2001 From: Lukasz Raczylo Date: Sun, 30 Aug 2026 19:41:02 +0100 Subject: [PATCH] feat(tests): add 16 advanced distractor-rich scenarios The 2026-08-30 suite-health review found 223/283 tests saturated. These 16 target the failure modes that still discriminated, one per existing subcategory across all eight categories: - multi-constraint reasoning: infeasible-plan detection, conflicting delegation policy, adversarial tool descriptions - forensic root-causing from noisy evidence: RAG stage attribution, EXPLAIN ANALYZE with a red herring, interleaved multi-source incident logs, README-vs-config conflict resolution - composed gotchas: a Go 1.22 capture + named-return-defer trace, a Python late-binding + shared-mutable-default trace - constraint math: context-budget packing, capacity-fill projection, MVCC anomaly classification, Docker layer-rebuild prediction, dual-key rotation overlap, freeze-window scheduling, an open-redirect exploit chain Every answer is a structured JSON value scored by an exact evaluator, never a prose heuristic (the source of every false positive in the review). A ground-truth self-consistency test asserts each documented correct answer scores 1.0 through its real evaluator. --- internal/tests/advanced.go | 534 ++++++++++++++++++++ internal/tests/advanced_groundtruth_test.go | 49 ++ internal/tests/catalog.go | 1 + 3 files changed, 584 insertions(+) create mode 100644 internal/tests/advanced.go create mode 100644 internal/tests/advanced_groundtruth_test.go diff --git a/internal/tests/advanced.go b/internal/tests/advanced.go new file mode 100644 index 0000000..5ca939d --- /dev/null +++ b/internal/tests/advanced.go @@ -0,0 +1,534 @@ +package tests + +import ( + "github.com/lukaszraczylo/llm-testbench/internal/eval" + "github.com/lukaszraczylo/llm-testbench/internal/testkit" +) + +// registerAdvancedTests adds the harder, distractor-rich scenarios designed +// after the 2026-08-30 suite-health review, which found 223/283 tests +// saturated (every model passes). Each test here targets a failure mode +// that still discriminated: multi-constraint reasoning, semantic tracing of +// composed gotchas, forensic root-causing from noisy evidence, and +// constraint math. Every answer is a structured JSON value scored by an +// exact evaluator, never a prose-matching heuristic (the source of every +// false positive in that review). Each lands in an EXISTING subcategory so +// the catalog shape (one pair per category/subcategory) is unchanged. +func registerAdvancedTests(r *testkit.Registry) { + r.Register(advAgentPlanInfeasibleTest()) + r.Register(advAgentDelegConflictingPolicyTest()) + r.Register(advAgentToolAdversarialDescTest()) + r.Register(advRAGForensicsStageTest()) + r.Register(advLLMContextBudgetTest()) + r.Register(advPgExplainForensicsTest()) + r.Register(advSQLMVCCAnomalyTest()) + r.Register(advDockerLayerRebuildTest()) + r.Register(advRelFreezeWindowTest()) + r.Register(advScenMultiLogRootcauseTest()) + r.Register(advLinuxCapacityFillTest()) + r.Register(advHardComposedTraceTest()) + r.Register(advPyComposedMutableTest()) + r.Register(advCodeConflictingEvidenceTest()) + r.Register(advSecExploitChainTest()) + r.Register(advSecRotationBlastRadiusTest()) +} + +// advAgentPlanInfeasibleTest: a plan that cannot succeed because a later +// step needs an artifact no earlier step produces. The discriminator is +// admitting infeasibility instead of confabulating an order. +// +// ground truth: step "deploy" needs the built image; the only build step +// ("build") needs the compiled binary from "compile"; "compile" needs +// dependencies from "fetch-deps"; but "fetch-deps" is never listed as +// runnable because it needs network egress the sandbox forbids (stated). +// So the binding blocker is fetch-deps and the plan is infeasible. +func advAgentPlanInfeasibleTest() testkit.Test { + prompt := `You must order these build steps into a runnable sequence: + +- compile: needs the fetched dependencies; produces the binary. +- build-image: needs the binary; produces the container image. +- deploy: needs the container image. +- fetch-deps: needs outbound network egress to the package registry; + produces the fetched dependencies. + +Constraint: this runner has NO outbound network egress (air-gapped), and +no dependency cache is present. + +Decide whether a runnable ordering exists. Respond with only a JSON object: +{"feasible": true|false, "blocking_step": ""}` + + return testkit.Test{ + ID: "adv-agent-plan-infeasible", + Category: "agents", + Subcategory: "planning", + Description: "Detect that an air-gapped runner makes fetch-deps (and the whole chain) infeasible rather than confabulating an order.", + Prompt: prompt, + Eval: eval.All( + eval.W(eval.JSONField("feasible", false), 2), + eval.W(eval.JSONField("blocking_step", "fetch-deps"), 1), + ), + } +} + +// advAgentDelegConflictingPolicyTest: pick the one assignment that satisfies +// three simultaneous constraints; the distractors each violate exactly one. +// +// ground truth: the task needs GPU, must finish under a 2h deadline, and +// must run in-region (data residency). agent-cpu-eu has no GPU (violates +// capability); agent-gpu-us is out of region (violates residency); +// agent-gpu-eu-slow takes 3h (violates deadline); agent-gpu-eu-fast meets +// all three. Answer: agent-gpu-eu-fast. +func advAgentDelegConflictingPolicyTest() testkit.Test { + prompt := `Assign this task to exactly one agent. The task REQUIRES a GPU, +must complete within a 2-hour deadline, and must run in the EU region +(data residency). + +- agent-cpu-eu: EU region, no GPU, would finish in 1h. +- agent-gpu-us: US region, has GPU, would finish in 30m. +- agent-gpu-eu-slow: EU region, has GPU, would finish in 3h. +- agent-gpu-eu-fast: EU region, has GPU, would finish in 90m. + +Respond with only a JSON object: {"agent": ""}` + + return testkit.Test{ + ID: "adv-agent-deleg-conflicting-policy", + Category: "agents", + Subcategory: "delegation", + Description: "Choose the single agent satisfying GPU + deadline + region when each distractor violates exactly one constraint.", + Prompt: prompt, + Eval: eval.JSONField("agent", "agent-gpu-eu-fast"), + } +} + +// advAgentToolAdversarialDescTest: the tool whose NAME suggests the task is +// wrong; the correct tool is identified only by reading descriptions. +// +// ground truth: task is to permanently delete a file. "file_manager" sounds +// right but its description only LISTS files (read-only). "trash_bin" +// sounds like recycling but its description permanently removes a path. +// Answer: trash_bin. +func advAgentToolAdversarialDescTest() testkit.Test { + prompt := `You must permanently delete the file at /data/tmp/old.log. +Pick the correct tool by its description, not its name: + +- file_manager: "Returns a listing of files under a directory. Read-only." +- disk_report: "Reports free and used disk space per mount." +- trash_bin: "Permanently removes the file or directory at the given path + from disk. Not recoverable." +- archive_tool: "Compresses a path into a .tar.gz beside it." + +Respond with only a JSON object: {"tool": ""}` + + return testkit.Test{ + ID: "adv-agent-tool-adversarial-desc", + Category: "agents", + Subcategory: "tool-routing", + Description: "Select the deletion tool by description when its name (trash_bin) misleads and file_manager sounds right but is read-only.", + Prompt: prompt, + Eval: eval.JSONField("tool", "trash_bin"), + } +} + +// advRAGForensicsStageTest: given a retrieval trace and a wrong answer, +// name the failing pipeline stage. +// +// ground truth: the embedding query retrieved the right documents (they +// appear in the trace with high scores and contain the fact), the reranker +// kept them top-3, but the generation cited a number found in NONE of the +// retrieved chunks - a generation-stage hallucination, not a retrieval or +// rerank failure. Answer: generation. +func advRAGForensicsStageTest() testkit.Test { + prompt := `A RAG system answered "the timeout is 45 seconds" but the +correct value is 30. Here is the trace: + +- retrieval: top-3 chunks returned, all from the config reference; chunk 1 + (score 0.91) contains the sentence "request_timeout defaults to 30s". +- rerank: order unchanged; chunk 1 still ranked first. +- generation: the model produced "45 seconds"; the string "45" appears in + none of the three retrieved chunks. + +Which single pipeline stage introduced the error? Respond with only a JSON +object: {"stage": "retrieval"|"rerank"|"generation"}` + + return testkit.Test{ + ID: "adv-rag-forensics-stage", + Category: "ai", + Subcategory: "rag", + Description: "Attribute a wrong RAG answer to the generation stage when retrieval and rerank both surfaced the correct grounded chunk.", + Prompt: prompt, + Eval: eval.JSONField("stage", "generation"), + } +} + +// advLLMContextBudgetTest: how many retrieved documents fit given a fixed +// context window and reservations. +// +// ground truth: 200000 window - 8000 output reserve - 1200 system - 300 +// query = 190500 available; each doc is 3500 tokens; floor(190500/3500) = +// 54. +func advLLMContextBudgetTest() testkit.Test { + prompt := `A model has a 200000-token context window. You must reserve +8000 tokens for the output, 1200 for the system prompt, and 300 for the +user query. Each retrieved document is 3500 tokens. How many whole +documents can you fit into the remaining context? Respond with only the +integer.` + + return testkit.Test{ + ID: "adv-llm-context-budget", + Category: "ai", + Subcategory: "llm-integration", + Description: "Compute the document count that fits a 200K window after output/system/query reservations (floor(190500/3500)=54).", + Prompt: prompt, + Eval: eval.Numeric(eval.ExtractLastNumber[int], 54, 0), + } +} + +// advPgExplainForensicsTest: read an EXPLAIN ANALYZE plan with a red herring +// and name the real problem. +// +// ground truth: the plan shows a Seq Scan on a 2M-row table with a Filter +// removing 1,999,900 rows (the actual cost driver), while a small Nested +// Loop over 100 rows is a red herring. The fix class is an index on the +// filtered column. Answer: the seq-scan / missing-index problem. +func advPgExplainForensicsTest() testkit.Test { + prompt := `Here is a Postgres EXPLAIN (ANALYZE) fragment for a slow query: + +Nested Loop (cost=0.29..8.34 rows=100 actual time=0.02..0.14 rows=100) + -> Seq Scan on events (rows=2000000 actual time=0.01..812.4 rows=2000000) + Filter: (tenant_id = 42) + Rows Removed by Filter: 1999900 + -> Index Scan using users_pkey on users (actual time=0.001..0.001 rows=1) + +Planning Time: 0.2 ms +Execution Time: 851.9 ms + +Identify the single dominant cause of the slow execution, and the fix +class. Respond with only a JSON object: +{"cause": "seq-scan"|"nested-loop"|"planning"|"index-scan", + "fix": "add-index"|"raise-work-mem"|"vacuum"|"rewrite-join"}` + + return testkit.Test{ + ID: "adv-pg-explain-forensics", + Category: "databases", + Subcategory: "postgres", + Description: "Attribute an 852ms plan to the Seq Scan filtering out ~2M rows (not the cheap nested loop) and pick add-index.", + Prompt: prompt, + Eval: eval.All( + eval.W(eval.JSONField("cause", "seq-scan"), 2), + eval.W(eval.JSONField("fix", "add-index"), 1), + ), + } +} + +// advSQLMVCCAnomalyTest: name the concurrency anomaly from an interleaved +// transaction timeline at a stated isolation level. +// +// ground truth: under READ COMMITTED, txn A reads balance=100, then txn B +// commits balance=150, then txn A reads again and sees 150 - the same row +// read twice in one transaction yielded different committed values: a +// non-repeatable read. It is not a dirty read (B committed) nor a phantom +// (same row, not a new row matching a predicate). +func advSQLMVCCAnomalyTest() testkit.Test { + prompt := `Two transactions run under READ COMMITTED isolation: + +t1: A: BEGIN +t2: A: SELECT balance FROM accounts WHERE id=1 -> 100 +t3: B: BEGIN +t4: B: UPDATE accounts SET balance=150 WHERE id=1 +t5: B: COMMIT +t6: A: SELECT balance FROM accounts WHERE id=1 -> 150 +t7: A: COMMIT + +Name the concurrency anomaly transaction A experienced. Respond with only a +JSON object: +{"anomaly": "dirty-read"|"non-repeatable-read"|"phantom-read"|"none"}` + + return testkit.Test{ + ID: "adv-sql-mvcc-anomaly", + Category: "databases", + Subcategory: "sql-tuning", + Description: "Classify a re-read yielding a different committed value under READ COMMITTED as a non-repeatable read (not dirty or phantom).", + Prompt: prompt, + Eval: eval.JSONField("anomaly", "non-repeatable-read"), + } +} + +// advDockerLayerRebuildTest: predict exactly which layers rebuild after a +// specific source change. +// +// ground truth: editing a file under src/ busts the "COPY src" layer (C); +// every layer after it rebuilds (D: npm run build); the final-stage +// "COPY --from=build /app/dist" (F) consumes D's output so it rebuilds too. +// The package-file COPY (A) and npm ci (B) are before the change and stay +// cached; the nginx base (E) is unaffected. Answer set: {C, D, F}. +func advDockerLayerRebuildTest() testkit.Test { + prompt := `Given this Dockerfile, with BuildKit layer caching and a warm +cache from a previous identical build: + + FROM node:20 AS build + WORKDIR /app + COPY package.json package-lock.json ./ # layer A + RUN npm ci # layer B + COPY src ./src # layer C + RUN npm run build # layer D + FROM nginx:alpine # layer E + COPY --from=build /app/dist /usr/share/nginx/html # layer F + +You edit one file inside src/ (no dependency changes). Which layers must +rebuild? Respond with only a JSON array of the layer letters that rebuild, +e.g. ["A","B"].` + + return testkit.Test{ + ID: "adv-docker-layer-rebuild", + Category: "delivery", + Subcategory: "containers", + Description: "Predict that a src/ edit rebuilds exactly the COPY src, npm build, and dist-copy layers (C, D, F), not the cached package layers.", + Prompt: prompt, + Eval: eval.JSONStringSet([]string{"C", "D", "F"}), + } +} + +// advRelFreezeWindowTest: order release steps around a change-freeze +// constraint. +// +// ground truth: the deploy-to-prod step must NOT fall inside the freeze; +// every other step (build, test, stage-deploy, smoke-test) can run during +// the freeze because they do not touch prod. The only step to defer past +// the freeze is deploy-prod. Answer: deploy-prod. +func advRelFreezeWindowTest() testkit.Test { + prompt := `A change freeze forbids any change to PRODUCTION between now and +Monday 09:00. Your release pipeline has these steps: + +- build: compiles artifacts (no environment touched). +- unit-test: runs tests (no environment touched). +- stage-deploy: deploys to the staging environment. +- smoke-test: runs checks against staging. +- deploy-prod: deploys to production. + +Which single step must be deferred until after the freeze lifts? Respond +with only a JSON object: {"deferred_step": ""}` + + return testkit.Test{ + ID: "adv-rel-freeze-window", + Category: "delivery", + Subcategory: "release-engineering", + Description: "Identify deploy-prod as the only step a production change freeze blocks, while build/test/stage steps may proceed.", + Prompt: prompt, + Eval: eval.JSONField("deferred_step", "deploy-prod"), + } +} + +// advScenMultiLogRootcauseTest: interleaved logs from three sources with one +// root cause and red herrings. +// +// ground truth: the load balancer logs 502s, the app logs OOMKilled restart +// events, and the kubelet logs memory-pressure eviction - all downstream of +// the app being OOM-killed. The 502s are a symptom (backend gone), the +// eviction is a symptom (node pressure from the same leak). The root cause +// is the application OOM. Answer: oom. +func advScenMultiLogRootcauseTest() testkit.Test { + prompt := `Three log streams during an incident, timestamps interleaved: + +11:02:01 lb: upstream 10.42.3.9:8080 returned 502 +11:02:03 app: container killed, reason OOMKilled, restart #4 +11:02:03 kubelet: memory pressure on node gst02, evicting low-priority pods +11:02:05 lb: upstream 10.42.3.9:8080 connection refused +11:02:07 app: heap usage 1.98Gi of 2Gi limit before exit + +Identify the single ROOT cause (not a downstream symptom). Respond with +only a JSON object: +{"root_cause": "lb-misconfig"|"oom"|"node-eviction"|"network"}` + + return testkit.Test{ + ID: "adv-scen-multilog-rootcause", + Category: "operations", + Subcategory: "scenario", + Description: "Root-cause interleaved lb/app/kubelet logs to the application OOM, treating the 502s and eviction as downstream symptoms.", + Prompt: prompt, + Eval: eval.JSONField("root_cause", "oom"), + } +} + +// advLinuxCapacityFillTest: linear capacity projection. +// +// ground truth: 500GB volume, 90% alert threshold = 450GB, currently 366GB +// used, growing 12GB/day. (450-366)/12 = 84/12 = 7 whole days until it +// reaches the threshold. +func advLinuxCapacityFillTest() testkit.Test { + prompt := `A 500 GB data volume is currently 366 GB used and grows by a +steady 12 GB per day. An alert fires when usage reaches 90% of capacity. +In how many whole days from now will usage first reach the 90% alert +threshold? Respond with only the integer number of days.` + + return testkit.Test{ + ID: "adv-linux-capacity-fill", + Category: "operations", + Subcategory: "linux", + Description: "Project days until a 500GB volume at 366GB growing 12GB/day reaches the 90% (450GB) threshold: 7.", + Prompt: prompt, + Eval: eval.Numeric(eval.ExtractLastNumber[int], 7, 0), + } +} + +// advHardComposedTraceTest: a Go trace composing loop-variable capture, a +// deferred named-return mutation, and closure order. +// +// ground truth (verified by running it): three closures each capture the +// Go 1.22+ per-iteration loop variable i (1, 2, 3) and add it to the named +// return result; result becomes 6; return result + 10 sets result to 16; +// the deferred func doubles it to 32. +func advHardComposedTraceTest() testkit.Test { + prompt := "In Go 1.22 or later, what does calc() print?\n\n" + + "```go\n" + + "func calc() (result int) {\n" + + "\tdefer func() { result *= 2 }()\n" + + "\tfns := []func(){}\n" + + "\tfor i := 1; i <= 3; i++ {\n" + + "\t\tfns = append(fns, func() { result += i })\n" + + "\t}\n" + + "\tfor _, fn := range fns {\n" + + "\t\tfn()\n" + + "\t}\n" + + "\treturn result + 10\n" + + "}\n" + + "```\n\n" + + "Respond with only the integer printed." + + return testkit.Test{ + ID: "adv-hard-composed-trace", + Category: "programming", + Subcategory: "hard", + Description: "Trace Go 1.22 per-iteration capture (1+2+3) plus return+10 plus a deferred double: 32.", + Prompt: prompt, + Eval: eval.Numeric(eval.ExtractLastNumber[int], 32, 0), + } +} + +// advPyComposedMutableTest: a Python trace composing a mutable default +// argument with a list comprehension of closures. +// +// ground truth (verified by running it): f uses a shared mutable default +// list acc; fns[0]() calls f(0), appending 0 -> returns [0]... but the +// closures capture i by late binding, and range(3) leaves i=2, so each +// lambda calls f(2). fns[0]() -> f(2) appends 2 -> [2]. Then f(10) appends +// to the SAME shared list -> [2, 10]. The second printed line is [2, 10]. +func advPyComposedMutableTest() testkit.Test { + prompt := "What does the SECOND print statement output?\n\n" + + "```python\n" + + "def f(x, acc=[]):\n" + + " acc.append(x)\n" + + " return acc\n" + + "\n" + + "fns = [lambda: f(i) for i in range(3)]\n" + + "print(fns[0]())\n" + + "print(f(10))\n" + + "```\n\n" + + "The second print outputs a list of two integers. Respond with only a " + + "JSON object naming its two elements in order: " + + "{\"first_element\": , \"second_element\": }" + + return testkit.Test{ + ID: "adv-py-composed-mutable", + Category: "programming", + Subcategory: "python", + Description: "Trace late-binding closure (i=2) plus a shared mutable default list across two calls: second print is [2, 10].", + Prompt: prompt, + Eval: eval.All( + eval.W(eval.JSONField("first_element", 2), 1), + eval.W(eval.JSONField("second_element", 10), 1), + ), + } +} + +// advCodeConflictingEvidenceTest: two sources disagree; pick the claim that +// survives with the correct reason. +// +// ground truth: the README says the default port is 8080; the actual +// config file (authoritative, it is what the program reads) says 9090. When +// documentation and the live config disagree, the config the program +// actually loads wins. Answer: 9090, source config. +func advCodeConflictingEvidenceTest() testkit.Test { + prompt := `Two sources describe the service's listen port: + +- README.md (last edited 14 months ago): "The service listens on port + 8080 by default." +- config/server.yaml (the file the running process loads on startup): + "listen_port: 9090" + +There is no environment-variable override. Which port does the running +service actually listen on, and which source is authoritative? Respond with +only a JSON object: {"port": , "authoritative": "readme"|"config"}` + + return testkit.Test{ + ID: "adv-code-conflicting-evidence", + Category: "research", + Subcategory: "codebase", + Description: "Resolve a README-vs-config port conflict to the config the process actually loads (9090), not the stale doc.", + Prompt: prompt, + Eval: eval.All( + eval.W(eval.JSONField("port", 9090), 2), + eval.W(eval.JSONField("authoritative", "config"), 1), + ), + } +} + +// advSecExploitChainTest: two individually-benign behaviors compose into an +// exploit; name the composed vulnerability class. +// +// ground truth: an open redirect (benign-looking) plus the OAuth flow +// echoing the redirect_uri without an allowlist lets an attacker steal the +// authorization code by redirecting the victim's callback to an +// attacker-controlled host - an account-takeover via open-redirect in the +// OAuth redirect_uri. The class asked for is "open-redirect". +func advSecExploitChainTest() testkit.Test { + prompt := `Two facts about an app, each individually looking minor: + +1. GET /go?next= issues an HTTP 302 to whatever is passed, with + no host allowlist. +2. The OAuth login uses /go as the post-login landing page and forwards the + authorization code to the landing URL's query string. + +Combined, what vulnerability class does this enable? Respond with only a +JSON object: +{"vuln": "open-redirect"|"csrf"|"sql-injection"|"xss"|"ssrf"}` + + return testkit.Test{ + ID: "adv-sec-exploit-chain", + Category: "security", + Subcategory: "appsec", + Description: "Compose an unvalidated redirect with an OAuth code hand-off into an open-redirect account-takeover chain.", + Prompt: prompt, + Eval: eval.JSONField("vuln", "open-redirect"), + } +} + +// advSecRotationBlastRadiusTest: dual-key rotation-overlap math across +// dependent consumers with different cache TTLs. +// +// ground truth: consumers cache a validated key for up to their TTL, so the +// old key must stay accepted for at least the longest consumer TTL after +// the new key is published, or an in-flight cached validation breaks. TTLs +// are 30, 15, and 5 minutes; the minimum safe overlap window is the max = +// 30 minutes. +func advSecRotationBlastRadiusTest() testkit.Test { + prompt := `You are rotating a signing key with a dual-key (old+new) +overlap window. Three independent services validate signatures and cache +the validated key material for up to their own TTL before re-fetching: + +- service-a: 30-minute cache TTL +- service-b: 15-minute cache TTL +- service-c: 5-minute cache TTL + +For how many minutes, at minimum, must the OLD key stay accepted after the +new key is published, so that no service rejects a signature it should +accept? Respond with only the integer number of minutes.` + + return testkit.Test{ + ID: "adv-sec-rotation-blast-radius", + Category: "security", + Subcategory: "secrets", + Description: "Size the dual-key overlap window to the longest consumer cache TTL (max(30,15,5)=30 minutes).", + Prompt: prompt, + Eval: eval.Numeric(eval.ExtractLastNumber[int], 30, 0), + } +} diff --git a/internal/tests/advanced_groundtruth_test.go b/internal/tests/advanced_groundtruth_test.go new file mode 100644 index 0000000..24ccee7 --- /dev/null +++ b/internal/tests/advanced_groundtruth_test.go @@ -0,0 +1,49 @@ +package tests + +import ( + "context" + "testing" +) + +// TestAdvancedGroundTruthScoresFull asserts each advanced test's own +// documented correct answer scores 1.0 through its real evaluator - a guard +// that the eval and the intended ground truth agree (a JSON typo or a wrong +// field path would silently make a test unwinnable). +func TestAdvancedGroundTruthScoresFull(t *testing.T) { + golden := map[string]string{ + "adv-agent-plan-infeasible": `{"feasible": false, "blocking_step": "fetch-deps"}`, + "adv-agent-deleg-conflicting-policy": `{"agent": "agent-gpu-eu-fast"}`, + "adv-agent-tool-adversarial-desc": `{"tool": "trash_bin"}`, + "adv-rag-forensics-stage": `{"stage": "generation"}`, + "adv-llm-context-budget": `54`, + "adv-pg-explain-forensics": `{"cause": "seq-scan", "fix": "add-index"}`, + "adv-sql-mvcc-anomaly": `{"anomaly": "non-repeatable-read"}`, + "adv-docker-layer-rebuild": `["C","D","F"]`, + "adv-rel-freeze-window": `{"deferred_step": "deploy-prod"}`, + "adv-scen-multilog-rootcause": `{"root_cause": "oom"}`, + "adv-linux-capacity-fill": `7`, + "adv-hard-composed-trace": `32`, + "adv-py-composed-mutable": `{"first_element": 2, "second_element": 10}`, + "adv-code-conflicting-evidence": `{"port": 9090, "authoritative": "config"}`, + "adv-sec-exploit-chain": `{"vuln": "open-redirect"}`, + "adv-sec-rotation-blast-radius": `30`, + } + + byID := map[string]int{} + all := All().All() + for i, tc := range all { + byID[tc.ID] = i + } + + for id, answer := range golden { + idx, ok := byID[id] + if !ok { + t.Errorf("%s: not registered in catalog", id) + continue + } + s := all[idx].Eval.Evaluate(context.Background(), answer) + if s.Value != 1 { + t.Errorf("%s: golden answer %q scored %.2f, want 1.00 (%s)", id, answer, s.Value, s.Detail) + } + } +} diff --git a/internal/tests/catalog.go b/internal/tests/catalog.go index c2ae9a8..901db56 100644 --- a/internal/tests/catalog.go +++ b/internal/tests/catalog.go @@ -26,5 +26,6 @@ func All() *testkit.Registry { registerHardTests(r) registerScenarioTests(r) registerSaturationTests(r) + registerAdvancedTests(r) return r }