Update

.github/workflows/release.yml

@@ -6,22 +6,29 @@ on:
   workflow_dispatch:
 
 permissions:
-  contents: write
+  contents: read
-  packages: write
+
+concurrency:
+  group: release-${{ github.ref }}
+  cancel-in-progress: false
 
 jobs:
   release:
     name: Version, tag, and release
     runs-on: ubuntu-latest
+    timeout-minutes: 30
+    permissions:
+      contents: write
+      packages: write
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v4
+        uses: actions/checkout@v7
         with:
           fetch-depth: 0
 
       - name: Calculate semantic version
         id: semver
-        uses: lukaszraczylo/semver-generator@v1
+        uses: lukaszraczylo/semver-generator@b5eb8ae15a62d76e81dc4661a49209a8502cf1d3 # v1.17.18
         with:
           config_file: semver.yaml
           repository_local: true
@@ -45,7 +52,7 @@ jobs:
 
       - name: Set up Go
         if: steps.tag.outputs.new == 'true'
-        uses: actions/setup-go@v5
+        uses: actions/setup-go@v7
         with:
           go-version-file: go.mod
 
@@ -55,15 +62,15 @@ jobs:
 
       - name: Set up QEMU
         if: steps.tag.outputs.new == 'true'
-        uses: docker/setup-qemu-action@v3
+        uses: docker/setup-qemu-action@96fe6ef7f33517b61c61be40b68a1882f3264fb8 # v4.2.0
 
       - name: Set up Docker Buildx
         if: steps.tag.outputs.new == 'true'
-        uses: docker/setup-buildx-action@v3
+        uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0
 
       - name: Log in to GitHub Container Registry
         if: steps.tag.outputs.new == 'true'
-        uses: docker/login-action@v3
+        uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0
         with:
           registry: ghcr.io
           username: ${{ github.actor }}
@@ -71,7 +78,7 @@ jobs:
 
       - name: Run goreleaser
         if: steps.tag.outputs.new == 'true'
-        uses: goreleaser/goreleaser-action@v6
+        uses: goreleaser/goreleaser-action@f06c13b6b1a9625abc9e6e439d9c05a8f2190e94 # v7.2.3
         with:
           distribution: goreleaser
           version: "~> v2"

README.md

@@ -64,9 +64,17 @@ Both `list` and `run` accept:
 
 - `--models` - comma-separated model list, overriding the config file.
 - `--format` - `table`, `markdown`, or `json`. Default: `table`.
+- `--tests` - comma-separated exact test IDs (no globs), overriding
+  `--category`/`--subcategory` - for probing individual tests cheaply.
+- `--repeat N` - sample each (model, test) pair N times. Sampling
+  temperature is pinned to 0, so disagreement between repeats
+  exposes response instability (server-side batching, MoE routing) rather
+  than decoding randomness. Default: 1.
 - `--concurrency` - override the config file's `concurrency`.
 - `--timeout` - override the config file's `request_timeout` (for example
   `60s`).
+- `--out FILE` - write the run as a JSON artifact (raw per-attempt results
+  plus the per-test discrimination rollup) for later `llmtest compare`.
 - `--quiet` - suppress progress output on stderr. Progress is on by
   default: a run can issue hundreds of requests at 30-50s each, and a
   silent CLI reads as a hang.
@@ -78,6 +86,54 @@ with a `TRUNCATED` marker when a response was cut off by the token
 budget). stdout carries only the table/markdown/json report, so `llmtest
 run --format json > results.json` stays pipeable regardless of `--quiet`.
 
+With `--repeat N>1`, each (model, test) cell reports the mean over
+attempts; when attempts disagreed, the cell appends the `[min-max]` range
+and the discrimination rollup flags the test as unstable.
+
+### Comparing two runs
+
+`compare` diffs two saved artifacts (from `--out`) per (model, test) pair.
+It re-aggregates each side with the same truncation policy the report
+uses, so a comparison stays honest even when only some attempts were cut
+off. A mean drop of >= 0.01 is a regression, an equal-size rise an
+improvement; tests present on only one side are listed as
+`baseline-only`/`current-only`. The exit code reflects only diff
+failures (unreadable artifacts), not regressions:
+
+```sh
+go run ./cmd/llmtest run --out baseline.json
+# ... change config, upgrade the endpoint, swap model ...
+go run ./cmd/llmtest run --out current.json
+go run ./cmd/llmtest compare baseline.json current.json
+```
+
+### Auditing suite health
+
+`health` reads one or more saved artifacts and rolls them up per
+subcategory: what share of (model, test) cells score perfect, how many
+tests are saturated (perfect in >= 90% of their cells, so they cannot
+separate models), and which tests still carry signal. Cells pool across
+artifacts taking each (model, test) pair's WORST artifact, so a lucky
+re-run cannot hide a failure; subcategories at 100% perfect across every
+model contribute zero ranking information and need harder tests:
+
+```sh
+go run ./cmd/llmtest run --out run1.json
+go run ./cmd/llmtest health run1.json              # one artifact
+go run ./cmd/llmtest health run1.json run2.json    # pooled audit
+```
+
+### Reading the discrimination rollup
+
+`table` and `markdown` output end with a `Discrimination` section. Its
+headline counts the tests whose cross-model score spread reaches
+`DiscriminationSpread` (0.05) - the tests that actually separate models -
+plus the tests whose repeated attempts disagreed. A suite where nearly
+every test scores 1.00 for every model measures nothing about relative
+capability, and this section makes that visible per test instead of
+averaging it away; with a single model there is no spread, so the section
+reports instability only.
+
 If `config.yaml` omits `concurrency`, `request_timeout`, or
 `max_tokens_default`, `internal/config` fills in `8`, `300s`, and `12000`.
 The effective per-call token budget is `max(test.MaxTokens,
@@ -108,16 +164,22 @@ needs a local toolchain that is not installed (for example, a Python
 exec-evaluator when `python3` is missing). Skipped tests are excluded from
 every mean.
 
-A cell carries a trailing `!` when the model's response was cut off by the
+A cell carries a trailing `!` when at least one of its attempts was cut
-token budget (`finish_reason=length`) - for example `0.00 (6000tok)!`. A
+off by the token budget (`finish_reason=length`) - for example
-truncated response's score is still whatever the evaluator computed on the
+`1.00 (420tok)!`. Truncation policy: a truncated attempt was scored on
-partial text, so a `!` cell needs a closer look: raise
+partial text, so its score measures the token budget as much as the
-`max_tokens_default` and re-run. `table` and `markdown` print a one-line
+model. Attempts that finished normally are therefore the only ones
-legend explaining `!` whenever at least one cell has it. `json` carries the
+averaged into the cell's mean and its `[min-max]` range; the truncated
-same signal per result as `"truncated": true`, plus `"finish_reason"` and
+sample is discarded and marked by the `!`. Only when every scored attempt
-`"response_text"` (the exact normalized text that was scored) so a
+was truncated does the mean fall back to the partial-text scores - then
-truncated or unexpectedly low-scoring result can be debugged without
+the `!` means the whole cell is unreliable and `max_tokens_default` needs
-re-running the model.
+rising before the number can be read as capability. `compare` reuses this
+policy, so a truncated sample in one artifact cannot fake a regression.
+`table` and `markdown` print a one-line legend explaining `!` whenever at
+least one cell has it. `json` carries the signal per attempt as
+`"truncated": true`, plus `"finish_reason"` and `"response_text"` (the
+exact normalized text that was scored) so a truncated or unexpectedly
+low-scoring result can be debugged without re-running the model.
 
 ## Test catalog
 
@@ -129,20 +191,117 @@ is cheap, the test file's `_test.go` recomputes it independently (for
 example, `internal/tests/python_test.go` recomputes the cosine similarity
 with `math.Sqrt` rather than trusting the constant).
 
+1063 tests across 105 subcategories (each holds at least 10, enforced by
+`internal/tests/catalog_test.go`). Subcategory minimums and the exact ID
+list are the source of truth; `llmtest list` prints them live.
+
 | Category / subcategory | Tests |
 | --- | --- |
-| programming / golang | `go-struct-align`, `go-worker-pool`, `go-semver-classify` |
+| programming / golang (10) | `go-bounded-fanout`, `go-channel-deadlock`, `go-context-cancellation-trace`, `go-defer-recover-trace`, `go-generics-constraint-fix`, `go-method-set-interface`, `go-semver-classify`, `go-slice-append-aliasing`, `go-struct-align`, `go-worker-pool` |
-| programming / python | `py-log-triage`, `py-cosine` |
+| programming / python (10) | `py-asyncio-gather-trace`, `py-cosine`, `py-dict-comprehension-trace`, `py-generator-exhaustion-trace`, `py-json-transform`, `py-log-triage`, `py-mutable-default-arg`, `py-pathlib-rewrite`, `py-regex-log-extraction`, `py-softmax` |
-| programming / typescript | `ts-debounce-composable` |
+| programming / typescript (10) | `ts-array-method-chain`, `ts-computed-vs-watch`, `ts-debounce-composable`, `ts-discriminated-union-narrowing`, `ts-eslint-flat-config`, `ts-generic-utility-type`, `ts-optional-chaining-nullish`, `ts-promise-allsettled-trace`, `ts-tsconfig-strict-flag`, `ts-vue-ref-reactive-unwrap` |
-| programming / c | `c-struct-size` |
+| programming / c (10) | `c-array-decay-sizeof`, `c-bitmask-ops`, `c-integer-promotion-overflow`, `c-macro-expansion-pitfall`, `c-pointer-arithmetic`, `c-string-function-output`, `c-struct-bitfield-packing`, `c-struct-size`, `c-undefined-behavior-spot`, `c-union-size-lp64` |
-| operations / macos | `macos-timeout-portability`, `macos-launchd-cron` |
+| programming / hard (12) | `hard-context-timeout`, `hard-defer-named-return`, `hard-float-binadd`, `hard-go-paren-balance`, `hard-goroutine-race`, `hard-int-division`, `hard-json-unmarshal-null`, `hard-nullish-over-or`, `hard-promise-allsettled`, `hard-py-version-sort`, `hard-regex-greedy`, `hard-var-shadow` |
-| operations / linux | `linux-pct-exec`, `linux-systemd-oneshot` |
+| operations / macos (10) | `macos-apfs-snapshot`, `macos-caffeinate`, `macos-launchctl-bootstrap`, `macos-launchd-cron`, `macos-mdfind-spotlight`, `macos-plutil-plist-read`, `macos-sed-inplace`, `macos-ssh-identity-agent`, `macos-stat-bsd-gnu`, `macos-timeout-portability` |
-| operations / kubernetes | `k8s-crashloop-gitops` |
+| operations / linux (10) | `linux-cgroup-oom-diagnosis`, `linux-cron-expression`, `linux-journalctl-filter`, `linux-lvm-extend-order`, `linux-nftables-dnat`, `linux-pct-exec`, `linux-proc-meminfo-available`, `linux-ssh-port-forward`, `linux-systemd-oneshot`, `linux-systemd-timer` |
-| research / web | `web-robots-ai-crawlers` |
+| operations / kubernetes (10) | `k8s-argocd-outofsync`, `k8s-cnpg-pdb`, `k8s-crashloop-gitops`, `k8s-imagepullbackoff`, `k8s-networkpolicy-dns-block`, `k8s-pending-taints`, `k8s-pvc-storageclass-mismatch`, `k8s-qos-class`, `k8s-service-selector-mismatch`, `k8s-traefik-ingressroute-host` |
-| research / whitepapers | `paper-hnsw-params` |
+| operations / scenario (11) | `scen-backup-restore-order`, `scen-certexpiry-triage`, `scen-crashloop-order`, `scen-diskpressure-firstcmd`, `scen-inode-exhaustion`, `scen-log-rootcause-mcq`, `scen-lvm-resize-order`, `scen-net-layer-isolate`, `scen-portbind-fail`, `scen-service-degrade`, `scen-systemd-failreason` |
-| research / codebase | `code-trace-go` |
+| research / web (12) | `web-canonical-vs-redirect`, `web-cors-missing-header`, `web-dns-mx-record`, `web-hreflang-reciprocal`, `web-hreflang-self-reference-trap`, `web-html-product-extract`, `web-http-status-scenarios`, `web-robots-ai-crawlers`, `web-robots-ua-scope-trap`, `web-security-headers-audit`, `web-sitemap-max-urls`, `web-url-parse-components` |
-| agents / tool-routing | `agent-tool-routing` |
+| research / whitepapers (10) | `paper-attention-scale-factor`, `paper-bloom-fp-rate`, `paper-btree-height-levels`, `paper-cap-availability-choice`, `paper-hnsw-params`, `paper-lsm-write-amplification`, `paper-pq-compression-ratio`, `paper-raft-quorum-failures`, `paper-recall-at-k`, `paper-tfidf-term-score` |
-| agents / planning | `agent-plan-ordering` |
+| research / codebase (10) | `code-bigo-dedup`, `code-bug-line-diff`, `code-commit-bisect`, `code-dead-functions`, `code-generic-return-type`, `code-import-cycle`, `code-race-variable`, `code-trace-go`, `code-trace-python`, `code-trace-ts` |
+| agents / tool-routing (10) | `agent-tool-routing`, `agent-tool-routing-ambiguous`, `agent-tool-routing-chaining`, `agent-tool-routing-cheapest`, `agent-tool-routing-distractors`, `agent-tool-routing-missing-param`, `agent-tool-routing-multistep`, `agent-tool-routing-no-tool-needed`, `agent-tool-routing-parallel`, `agent-tool-routing-safety` |
+| agents / planning (10) | `agent-plan-critical-path`, `agent-plan-cycle-detect`, `agent-plan-idempotency`, `agent-plan-milestone-date`, `agent-plan-minimal-replan`, `agent-plan-ordering`, `agent-plan-parallel-steps`, `agent-plan-repair`, `agent-plan-resource-constrained`, `agent-plan-rollback-trigger` |
+| agents / delegation (12) | `agent-deleg-batch-vs-separate`, `agent-deleg-build-then-verify`, `agent-deleg-capacity-trap`, `agent-deleg-escalation-to-human`, `agent-deleg-handoff-context`, `agent-deleg-main-thread-vs-delegate`, `agent-deleg-minimal-privilege`, `agent-deleg-near-miss-roster`, `agent-deleg-parallel-dispatch-safety`, `agent-deleg-reviewer-independence`, `agent-deleg-task-to-specialist`, `agent-deleg-verify-vs-trust` |
+| databases / postgres (12) | `pg-bloat-vacuum`, `pg-count-star-vs-join`, `pg-deadlock-lock-order`, `pg-explain-seq-vs-index`, `pg-index-choice`, `pg-isolation-anomaly`, `pg-listen-notify`, `pg-n-plus-one`, `pg-partial-index`, `pg-pool-sizing`, `pg-replica-failover`, `pg-timeout-trap` |
+| databases / redis (10) | `redis-cache-stampede`, `redis-incr-atomicity`, `redis-keyspace-anti-pattern`, `redis-lua-atomicity`, `redis-memory-estimate`, `redis-multi-vs-pipeline`, `redis-noeviction-write`, `redis-pubsub-delivery`, `redis-scan-vs-keys`, `redis-structure-choice` |
+| databases / sql-tuning (10) | `sql-aggregate-sum`, `sql-composite-index-skip-column`, `sql-covering-index`, `sql-equivalent-rewrite`, `sql-groupby-having`, `sql-join-counts`, `sql-keyset-pagination`, `sql-null-in-where`, `sql-orderby-index-serving`, `sql-window-rank` |
+| security / appsec (12) | `sec-csrf-post-trap`, `sec-csrf-requirement`, `sec-idor-spot`, `sec-input-validation-boundary`, `sec-open-redirect-spot`, `sec-orm-interpolation-trap`, `sec-path-traversal-spot`, `sec-rate-limit-placement`, `sec-secret-log-spot`, `sec-sqli-spot`, `sec-ssrf-spot`, `sec-xss-sink-spot` |
+| security / crypto (10) | `sec-aes-gcm-nonce-reuse`, `sec-cert-chain-validation`, `sec-constant-time-compare`, `sec-hash-vs-encrypt-pii`, `sec-hmac-vs-signature`, `sec-jwt-alg-none`, `sec-password-hash-choice`, `sec-rand-source-choice`, `sec-rotate-order`, `sec-tls-floor-version` |
+| security / secrets (12) | `sec-diff-secret-spot`, `sec-error-path-leak-trap`, `sec-fork-pr-ci-exposure`, `sec-gitignore-history-nuance`, `sec-hardcoded-secret-spot`, `sec-k8s-secret-base64`, `sec-least-privilege-scope`, `sec-remediation-order`, `sec-rotation-no-downtime`, `sec-rotation-window-math`, `sec-ssh-agent-identity`, `sec-vault-tradeoff` |
+| delivery / git (10) | `git-bisect-steps`, `git-cherry-pick-vs-revert`, `git-conventional-commit-classify`, `git-detached-head-recovery`, `git-force-with-lease-vs-force`, `git-gitignore-negation-trace`, `git-hook-choice`, `git-rebase-vs-merge`, `git-stash-pop-conflict`, `git-worktree-use-case` |
+| delivery / containers (10) | `docker-build-arg-vs-env`, `docker-copy-vs-add`, `docker-dockerignore-effect`, `docker-entrypoint-cmd-trace`, `docker-healthcheck-semantics`, `docker-image-size-math`, `docker-layer-cache-bust`, `docker-layer-count`, `docker-multistage-size-benefit`, `docker-nonroot-user-caps` |
+| delivery / release-engineering (10) | `rel-canary-vs-bluegreen`, `rel-checksums-signing-purpose`, `rel-clean-tree-requirement`, `rel-commit-changelog-mapping`, `rel-hotfix-flow-ordering`, `rel-pin-vs-range-policy`, `rel-pipeline-stage-ordering`, `rel-rollback-ordering`, `rel-semver-bump-changelog`, `rel-tag-to-release-sequence` |
+| ai / vector-search (10) | `vec-cosine-vs-dot`, `vec-distance-to-similarity`, `vec-embedding-dimension-tradeoff`, `vec-hnsw-efsearch-tradeoff`, `vec-index-build-query-tradeoff`, `vec-near-duplicate-threshold`, `vec-pq-memory-math`, `vec-pre-vs-post-filtering`, `vec-recall-at-k`, `vec-rrf-fusion` |
+| ai / llm-integration (10) | `llm-429-retry-backoff`, `llm-context-overflow-strategy`, `llm-embedding-batch-math`, `llm-field-semantics`, `llm-role-placement`, `llm-sse-stream-done`, `llm-stop-sequence-trace`, `llm-temperature-zero-caveat`, `llm-token-budget-reasoning`, `llm-tool-call-handling` |
+| ai / rag (10) | `rag-chunk-size-tradeoff`, `rag-citation-grounding`, `rag-context-selection-vs-stuffing`, `rag-eval-metric-choice`, `rag-hallucination-mitigation-ordering`, `rag-index-staleness`, `rag-multihop-decomposition`, `rag-preassembly-dedup`, `rag-reranker-placement`, `rag-retrieval-failure-mode` |
+| daily / text-processing (10) | `dtext-cron-dst-skip`, `dtext-csv-revenue`, `dtext-duration-sum`, `dtext-invoice-vat-rounding`, `dtext-json-merge-patch`, `dtext-meeting-zones`, `dtext-percent-symmetry-trap`, `dtext-sla-date-math`, `dtext-url-components`, `dtext-utf8-bytes-vs-chars` |
+| daily / terminal (10) | `dterm-cp-n-versus-mv`, `dterm-du-total-max`, `dterm-find-mtime-plus7`, `dterm-grep-lines-vs-matches`, `dterm-log-top-2xx-path`, `dterm-pipefail-exit-code`, `dterm-ps-memory-column`, `dterm-single-quote-expansion`, `dterm-strip-port-uniq-count`, `dterm-tail-grep-buffering` |
+| daily / office-math (10) | `doffice-discount-tax-order`, `doffice-file-quota-gib`, `doffice-fx-wire-fee`, `doffice-mileage-threshold`, `doffice-overtime-pay`, `doffice-payroll-loaded`, `doffice-prorated-fee`, `doffice-reams-ceiling`, `doffice-shift-roster`, `doffice-uptime-budget` |
+| daily / policy-rules (10) | `dpol-approval-boundaries`, `dpol-cert-expiry-window`, `dpol-expense-per-night-cap`, `dpol-holiday-premium`, `dpol-marginal-brackets`, `dpol-oncall-rotation`, `dpol-retention-window`, `dpol-shipping-business-days`, `dpol-sla-tiered-credit`, `dpol-vacation-accrual-cap` |
+| daily / scheduling (10) | `dsched-billing-anchor-clamp`, `dsched-contract-notice-window`, `dsched-deadline-countdown`, `dsched-dst-backup-hours`, `dsched-dst-meeting-overlap`, `dsched-flight-cross-tz`, `dsched-payroll-period-walk`, `dsched-rota-mod7`, `dsched-utc-cross-date`, `dsched-weekend-shift-deadline` |
+| daily / shopping (10) | `dshop-bill-split-penny`, `dshop-buy-one-free`, `dshop-coupon-threshold`, `dshop-free-shipping-threshold`, `dshop-fuel-range-check`, `dshop-fuel-trip-cost`, `dshop-pack-mix-optimum`, `dshop-stacked-discount`, `dshop-unit-price-per-litre`, `dshop-vat-cashback` |
+| daily / finance (10) | `dfin-commute-season-ticket`, `dfin-compound-vs-simple`, `dfin-deposit-cap-weekly`, `dfin-fx-round-trip`, `dfin-income-tax-single-band`, `dfin-installment-vs-lump`, `dfin-min-payment-interest`, `dfin-prorata-rent`, `dfin-raise-compounding`, `dfin-simple-interest-18mo` |
+| daily / travel (10) | `dtrv-baggage-started-kg`, `dtrv-car-hire-80h`, `dtrv-hotel-3-nights`, `dtrv-layover-share`, `dtrv-mileage-tiers`, `dtrv-taxi-split-surge`, `dtrv-three-city-meeting`, `dtrv-toll-vs-detour`, `dtrv-train-connection`, `dtrv-visa-window` |
+| daily / cooking (10) | `dck-bakers-percentages`, `dck-coffee-unit-price`, `dck-cordial-dilution`, `dck-dinner-parallel-starts`, `dck-granola-label-math`, `dck-pan-area-scaling`, `dck-roast-start-timing`, `dck-scale-recipe-4-to-10`, `dck-starter-feed-ratio`, `dck-temp-convert-probe` |
+| daily / fitness (10) | `dft-acsm-incline-walk`, `dft-cadence-step-goal`, `dft-energy-balance-weeks`, `dft-ftp-interval-work`, `dft-karvonen-zones`, `dft-lean-mass-target`, `dft-pace-from-5k`, `dft-plank-compound-weeks`, `dft-swim-set-totals`, `dft-tenk-decimal-pace` |
+| daily / gardening (10) | `dgd-bed-plant-spacing`, `dgd-compost-cn-alligation`, `dgd-fertiliser-season-bags`, `dgd-hedge-compound-prune`, `dgd-irrigation-zone-cycle`, `dgd-mower-throughput-overlap`, `dgd-mulch-volume-bags`, `dgd-rain-equivalent-watering`, `dgd-raised-bed-soil-bags`, `dgd-sowing-transplant-dates` |
+| daily / home-repair (10) | `drp-bath-mix-fill-time`, `drp-concrete-post-hole-bags`, `drp-door-refix-timeline`, `drp-ladder-pythagoras-rule`, `drp-laminate-planks-with-waste`, `drp-led-swap-energy-savings`, `drp-paint-two-coat-cans`, `drp-shelf-bracket-fencepost`, `drp-thermostat-setback-savings`, `drp-tile-count-breakage` |
+| daily / car (10) | `dcar-average-speed-restart`, `dcar-detailing-overlap`, `dcar-dilution-ratio`, `dcar-hire-mileage-charge`, `dcar-mpg-to-litres`, `dcar-parking-blocks`, `dcar-service-whichever-first`, `dcar-tank-range-reserve`, `dcar-tread-life`, `dcar-trip-fuel-cost` |
+| daily / events (10) | `dev-catering-crate-wastage`, `dev-coaches-per-started`, `dev-deposit-after-discount`, `dev-flower-breakage-budget`, `dev-keg-pint-count`, `dev-place-card-sheets`, `dev-setup-working-days`, `dev-shift-block-overshoot`, `dev-table-grid-vs-area`, `dev-wedding-timeline` |
+| daily / medication (10) | `dmed-antibiotic-course-volume`, `dmed-bioavailability-tablets`, `dmed-combo-analgesic-cap`, `dmed-insulin-daily-total`, `dmed-iv-drip-rate`, `dmed-pack-refill-count`, `dmed-suspension-dose-volume`, `dmed-tablet-cost-per-pack`, `dmed-unit-conversion-excretion`, `dmed-weight-based-dose` |
+| daily / hiking (10) | `dhike-average-speed-halves`, `dhike-descent-time-allowance`, `dhike-drive-ferry-arrival`, `dhike-food-resupply-days`, `dhike-heat-water-fuel`, `dhike-map-scale-ground-time`, `dhike-naismith-summit-time`, `dhike-required-split-pace`, `dhike-tent-count-weight`, `dhike-water-gap-refill` |
+| daily / energy (10) | `den-battery-arbitrage-losses`, `den-carbon-tier-bands`, `den-draught-payback-share`, `den-ev-overnight-charge`, `den-heat-pump-vs-gas-cost`, `den-load-profile-weighted-mean`, `den-oil-litres-efficiency`, `den-solar-self-consumption`, `den-standing-charge-days`, `den-tou-bill-supply-charge` |
+| daily / childcare (10) | `dch-age-in-days`, `dch-daycare-funded-hours`, `dch-formula-annual-cost`, `dch-formula-can-count`, `dch-milk-bank-net-days`, `dch-night-sleep-minutes`, `dch-paracetamol-suspension-dose`, `dch-twin-feed-total`, `dch-volume-per-kg`, `dch-wake-window-clock` |
+| daily / insurance (10) | `dins-annual-premium-discount`, `dins-contents-depreciation`, `dins-deductible-coinsurance`, `dins-excess-per-claim`, `dins-family-deductible-met`, `dins-income-protection-gap`, `dins-out-of-pocket-cap`, `dins-pet-annual-limit`, `dins-rental-damage-waiver`, `dins-waiting-period-date` |
+| daily / nutrition (10) | `dnut-alcohol-units-abv`, `dnut-bmi-weight-band`, `dnut-caffeine-daily-cap`, `dnut-drink-sugar-averages`, `dnut-hydration-bottles`, `dnut-macro-energy-split`, `dnut-percent-dv-two-servings`, `dnut-protein-scoop-rounding`, `dnut-serving-scale-pack`, `dnut-tdee-deficit-target` |
+| daily / pets (10) | `dpets-cat-calories`, `dpets-dog-food-percent`, `dpets-dog-walk-km`, `dpets-dog-water`, `dpets-flea-boxes`, `dpets-grooming-cycle`, `dpets-med-mg-kg`, `dpets-puppy-vaccines`, `dpets-rabbit-hay`, `dpets-tank-volume` |
+| daily / commuting (10) | `dcomm-bus-interval`, `dcomm-carpool-rebate`, `dcomm-cycle-arrival`, `dcomm-fuel-share`, `dcomm-metro-platform`, `dcomm-office-cost`, `dcomm-parking-blocks`, `dcomm-rail-payg-vs-pass`, `dcomm-railcard-breakeven`, `dcomm-toll-vs-free` |
+| daily / laundry (10) | `dlau-concentrate-dose`, `dlau-damp-dry-clock`, `dlau-delicates-loads`, `dlau-detergent-pricing`, `dlau-fleece-blanket`, `dlau-home-vs-laundrette`, `dlau-mixed-load-cycles`, `dlau-pod-schedule`, `dlau-sanitiser-dilution`, `dlau-washer-energy` |
+| daily / woodwork (10) | `dwood-decking-boards`, `dwood-mitre-posts`, `dwood-mortar-waste`, `dwood-offcut-kerf`, `dwood-oil-coats`, `dwood-post-holes`, `dwood-price-break`, `dwood-screw-boxes`, `dwood-timber-invoice`, `dwood-varnish-tins` |
+| daily / grow (10) | `dgrow-fert-tank`, `dgrow-germination`, `dgrow-heat-pump`, `dgrow-light-bill`, `dgrow-mix-ratio`, `dgrow-nutrient-dose`, `dgrow-reservoir-bottles`, `dgrow-seed-cost`, `dgrow-seed-trays`, `dgrow-watering-cans` |
+| daily / home-maintenance (10) | `dhr-caulking-tubes`, `dhr-dryer-annual`, `dhr-filter-packs`, `dhr-insulation-payback`, `dhr-mower-runtime`, `dhr-pool-shock`, `dhr-render-time`, `dhr-service-plan`, `dhr-tank-ibc`, `dhr-tank-top-up` |
+| daily / bathroom-plumbing (10) | `dbp-anti-slip-mats`, `dbp-bath-fill`, `dbp-descaling`, `dbp-drip-line`, `dbp-gutter-run`, `dbp-heater-energy`, `dbp-leak-waste`, `dbp-shower-annual`, `dbp-sump-pump`, `dbp-wall-tiles` |
+| daily / cleaning-supplies (10) | `dcs-bin-liners`, `dcs-bleach-sprayers`, `dcs-cart-discount`, `dcs-cleaner-hours`, `dcs-detergent-packs`, `dcs-floor-polish`, `dcs-mop-buckets`, `dcs-tablets-vs-powder`, `dcs-vacuum-annual`, `dcs-window-sprayer` |
+| daily / painting-decorating (10) | `dpaint-booth-energy`, `dpaint-crew-hours`, `dpaint-gloss-doors`, `dpaint-pot-sizes`, `dpaint-scaffold`, `dpaint-skirting`, `dpaint-sprayer-amort`, `dpaint-tint`, `dpaint-two-coats`, `dpaint-wallpaper` |
+| daily / moving-house (10) | `dmov-box-packs`, `dmov-crate-trips`, `dmov-deposit-order`, `dmov-lift-tonnage`, `dmov-piano-surcharge`, `dmov-route-fuel`, `dmov-transit-insurance`, `dmov-two-quotes`, `dmov-utility-final`, `dmov-van-trips` |
+| daily / diy-electrical (10) | `delec-breaker`, `delec-cable`, `delec-downlights`, `delec-duty-cycle`, `delec-festoon`, `delec-heat-mat`, `delec-kettle`, `delec-router-sla`, `delec-sleeves`, `delec-surge` |
+| daily / waste-recycling (10) | `dwaste-bin-days`, `dwaste-bottles`, `dwaste-bulbs`, `dwaste-compost`, `dwaste-ewaste`, `dwaste-green-bags`, `dwaste-landfill`, `dwaste-oil`, `dwaste-shredder`, `dwaste-skips` |
+| daily / diy-pest-control (10) | `dpest-ant-bait-mix`, `dpest-bait-stations`, `dpest-flea-fogger`, `dpest-mosquito-traps`, `dpest-pest-contract`, `dpest-rodent-blocks`, `dpest-snake-fence`, `dpest-spray-perimeter`, `dpest-termite-warranty`, `dpest-wasp-plan` |
+| daily / home-safety (10) | `dsafe-cctv-archive`, `dsafe-extinguisher-service`, `dsafe-first-aid-stock`, `dsafe-handrail-fixings`, `dsafe-inspection-penalty`, `dsafe-ladder-hire`, `dsafe-monitoring-exit`, `dsafe-nightlight-annual`, `dsafe-smoke-batteries`, `dsafe-vent-fan` |
+| daily / water-usage (10) | `dwat-bath-heat`, `dwat-filter-vs-jug`, `dwat-hose-vs-bucket`, `dwat-leak-trace`, `dwat-meter-bill`, `dwat-pool-fill`, `dwat-shower-vs-bath`, `dwat-softener-salt`, `dwat-sprinkler`, `dwat-tank-payback` |
+| daily / home-security (10) | `dsec-doorbell-battery`, `dsec-dummy-camera`, `dsec-fake-presence`, `dsec-grille-bars`, `dsec-hasp-screws`, `dsec-monitoring-plan`, `dsec-motion-sensors`, `dsec-photocell`, `dsec-siren-cells`, `dsec-window-film` |
+| daily / cooling-load (10) | `dcool-ac-size`, `dcool-car-ac`, `dcool-dehumidifier`, `dcool-evap-vs-ac`, `dcool-fan-energy`, `dcool-fan-vs-ac`, `dcool-filter-cleans`, `dcool-ice-blocks`, `dcool-thermostat`, `dcool-window-shading` |
+| daily / telecom-plans (10) | `dtel-broadband-speed`, `dtel-contract-uplift`, `dtel-data-overage`, `dtel-family-plan`, `dtel-modem-power`, `dtel-payg-vs-bundle`, `dtel-prepaid-card`, `dtel-roaming-cap`, `dtel-sim-plan`, `dtel-tether-data` |
+| daily / heating-load (10) | `dheat-boiler-kwh`, `dheat-cylinder`, `dheat-duty-cycle`, `dheat-heat-pump`, `dheat-insulation`, `dheat-oil-tank`, `dheat-radiator`, `dheat-tariff`, `dheat-thermostat`, `dheat-zoning` |
+| daily / timesheet-pay (10) | `dpay-bonus-tax`, `dpay-clock-rounding`, `dpay-contractor-day`, `dpay-leave-accrual`, `dpay-mileage`, `dpay-overtime`, `dpay-parttime-prorata`, `dpay-salary-raise`, `dpay-shift-differential`, `dpay-tax-band` |
+| daily / fuel-trip (10) | `dfuel-boat-range`, `dfuel-economy-mixed`, `dfuel-ev-charge`, `dfuel-flight-uplift`, `dfuel-generator-kwh`, `dfuel-hedge-mix`, `dfuel-jerrycan-trip`, `dfuel-lpg-payback`, `dfuel-moto-range`, `dfuel-truck-legs` |
+| daily / pool-chemicals (10) | `dpool-algae-shock`, `dpool-backwash`, `dpool-floc`, `dpool-heat-pump`, `dpool-ph-adjust`, `dpool-ppm-dose`, `dpool-salt-cell`, `dpool-turnover`, `dpool-uv`, `dpool-volume-m3` |
+| daily / garden-irrigation (10) | `dirr-drip-emitters`, `dirr-drip-flow`, `dirr-fert-spread`, `dirr-greenhouse-mist`, `dirr-hose-fill`, `dirr-pond-topup`, `dirr-soil-compost`, `dirr-sprinkler`, `dirr-timer-days`, `dirr-water-tank` |
+| daily / aquarium-water (10) | `daqu-chiller`, `daqu-co2`, `daqu-filter-flow`, `daqu-lighting`, `daqu-medication`, `daqu-nitrification`, `daqu-potash`, `daqu-salt-mix`, `daqu-tank-litres`, `daqu-waterchange` |
+| daily / ev-charging (10) | `devch-ac-session`, `devch-battery-value`, `devch-cable-loss`, `devch-dc-fast`, `devch-degradation`, `devch-fleet-balancer`, `devch-range-added`, `devch-solar-surplus`, `devch-toll-pass`, `devch-tou-window` |
+| daily / tile-flooring (10) | `dtile-adhesive-coverage`, `dtile-border-run`, `dtile-budget`, `dtile-cut-waste`, `dtile-drain-slope`, `dtile-grid-count`, `dtile-grout-volume`, `dtile-levelling`, `dtile-room-area`, `dtile-skirting-cost` |
+| daily / decking-fencing (10) | `ddeck-board-area`, `ddeck-concrete-footings`, `ddeck-fence-panel-run`, `ddeck-gate-hinges`, `ddeck-joist-spacing`, `ddeck-oil-restoration`, `ddeck-railing-load`, `ddeck-slope-steps`, `ddeck-stair-stringer`, `ddeck-subframe-clamps` |
+| daily / loft-insulation (10) | `dloft-blown-depth`, `dloft-cistern-insul`, `dloft-downlight-covers`, `dloft-hatch-draft`, `dloft-heat-loss`, `dloft-knee-wall`, `dloft-pipe-jacket`, `dloft-spray-foam`, `dloft-target-u`, `dloft-vapour-barrier` |
+| daily / coffee-brewing (10) | `dbrew-caffeine-budget`, `dbrew-cold-brew-yield`, `dbrew-cup-cost`, `dbrew-descale-tablets`, `dbrew-espresso-shots`, `dbrew-extraction-yield`, `dbrew-french-press`, `dbrew-grind-throughput`, `dbrew-mineral-dose`, `dbrew-syrup-ratio` |
+| daily / laundry-care (10) | `dlc-bleach-dilution`, `dlc-detergent-cost`, `dlc-drum-load`, `dlc-dryclean-vs-home`, `dlc-dryer-energy`, `dlc-load-split`, `dlc-softener-vs-vinegar`, `dlc-spin-extraction`, `dlc-stain-soak-supply`, `dlc-water-use` |
+| daily / bike-care (10) | `dbike-battery-range`, `dbike-brake-pad-cost`, `dbike-chain-lifetime`, `dbike-chain-wear`, `dbike-energy-gels`, `dbike-gear-development`, `dbike-hire-mileage`, `dbike-patch-box`, `dbike-tuneup-vs-parts`, `dbike-tyre-tread` |
+| daily / houseplant-care (10) | `dhpl-cutting-trays`, `dhpl-fertilizer-bottle`, `dhpl-foot-candle-convert`, `dhpl-grow-light-bill`, `dhpl-neem-spray-rounds`, `dhpl-repot-soil-volume`, `dhpl-reservoir-top-up`, `dhpl-shelf-bundle-deal`, `dhpl-soil-mix-split`, `dhpl-watering-schedule` |
+| daily / lawn-care (10) | `dlawn-clippings`, `dlawn-fence-posts`, `dlawn-fertilizer-season`, `dlawn-irrigation`, `dlawn-mowing-time`, `dlawn-reseed-bags`, `dlawn-robot-wire`, `dlawn-spreader-fills`, `dlawn-topdress`, `dlawn-two-stroke` |
+| daily / data-backup (10) | `dbak-321-plan`, `dbak-archive-drives`, `dbak-cloud-bill`, `dbak-lto-tapes`, `dbak-raid-usable`, `dbak-ransomware-anomaly`, `dbak-restore-time`, `dbak-snapshot-retention`, `dbak-upload-throughput`, `dbak-upload-window` |
+| daily / pet-nutrition (10) | `dpn-bag-days`, `dpn-cat-water`, `dpn-joint-jar`, `dpn-kibble-cups`, `dpn-multi-cat-bags`, `dpn-puppy-meals`, `dpn-raw-mix`, `dpn-split-wet`, `dpn-treat-budget`, `dpn-wet-can-boxes` |
+| daily / bread-baking (10) | `dbake-annual-flour`, `dbake-hydration`, `dbake-pan-volume`, `dbake-proof-rate`, `dbake-roll-division`, `dbake-scale`, `dbake-schedule`, `dbake-starter-hydration`, `dbake-water-temp`, `dbake-yeast-convert` |
+| daily / sewing-tailoring (10) | `dsew-batting`, `dsew-bias-binding`, `dsew-button-spacing`, `dsew-elastic`, `dsew-fusible`, `dsew-layout`, `dsew-seam-allowance`, `dsew-shrinkage`, `dsew-thread-cone`, `dsew-zip-roll` |
+| daily / bbq-grilling (10) | `dbbq-brine`, `dbbq-burger-packs`, `dbbq-coal-indirect`, `dbbq-low-slow`, `dbbq-propane`, `dbbq-rub-batch`, `dbbq-sauce-bottles`, `dbbq-service-queue`, `dbbq-thaw-schedule`, `dbbq-veg-trays` |
+| daily / candle-making (10) | `dcand-batch-cost`, `dcand-blend-ratio`, `dcand-double-pour`, `dcand-jar-fill`, `dcand-market-margin`, `dcand-pool-ring`, `dcand-shared-batch`, `dcand-timeline`, `dcand-topup-bags`, `dcand-wick-spools` |
+| daily / freshwater-fishkeeping (10) | `dfish-boat-current`, `dfish-fillet-yield`, `dfish-fry-hatch`, `dfish-grow-out`, `dfish-koi-ration`, `dfish-net-coverage`, `dfish-pond-liner`, `dfish-pond-volume`, `dfish-transport-crates`, `dfish-worm-harvest` |
+| daily / home-canning (10) | `dcann-batch-cost`, `dcann-blanch-altitude`, `dcann-hard-boil`, `dcann-jam-pectin`, `dcann-jam-yield`, `dcann-kraut-salt`, `dcann-lid-packs`, `dcann-pickle-brine`, `dcann-pressure-session`, `dcann-sugar-per-jar` |
+| daily / beekeeping (10) | `dbee-extraction-yield`, `dbee-fence-perimeter`, `dbee-hive-gain`, `dbee-mite-wash`, `dbee-nuc-split`, `dbee-queen-graft`, `dbee-season-revenue`, `dbee-super-fill`, `dbee-syrup-mix`, `dbee-winter-feed` |
+| daily / knitting (10) | `dknit-blanket-squares`, `dknit-dye-wool`, `dknit-gauge-cast-on`, `dknit-hat-decrease`, `dknit-mitten-commission`, `dknit-rib-cuff`, `dknit-scarf-time`, `dknit-sock-pairs`, `dknit-sweater-yardage`, `dknit-yarn-skeins` |
+| daily / soap-making (10) | `dsoap-curing-sales`, `dsoap-fragrance-load`, `dsoap-loaf-cut`, `dsoap-lye-bags`, `dsoap-lye-solution`, `dsoap-lye-strength`, `dsoap-market-profit`, `dsoap-mold-loaves`, `dsoap-oil-blend`, `dsoap-superfat` |
+| daily / pottery-ceramics (10) | dpot-clay-bags`, `dpot-commission-discount`, `dpot-course-profit`, `dpot-drying-rate`, `dpot-glaze-batch`, `dpot-grog-share`, `dpot-kiln-energy`, `dpot-mug-margin`, `dpot-shrinkage`, `dpot-throwing-shift |
+| daily / cheese-making (10) | dcheese-affinage-profit`, `dcheese-brine-saturation`, `dcheese-calcium-dose`, `dcheese-drying-loft`, `dcheese-milk-yield`, `dcheese-mother-stir`, `dcheese-mozzarella-sales`, `dcheese-press-force`, `dcheese-press-schedule`, `dcheese-rennet-dose |
+| daily / home-brewing (10) | `dbeer-abv`, `dbeer-boil-energy`, `dbeer-chiller-time`, `dbeer-efficiency`, `dbeer-growler-margin`, `dbeer-hops-ibu`, `dbeer-mash-sparge`, `dbeer-priming-sugar`, `dbeer-strike-temp`, `dbeer-yeast-pitch` |
+| daily / embroidery (10) | `demb-class-net`, `demb-floss-length`, `demb-hoop-area`, `demb-kit-profit`, `demb-pattern-scale`, `demb-quilt-edge`, `demb-ribbon-round`, `demb-runner-fabric`, `demb-stitch-time`, `demb-tassel-cut` |
+| daily / jewelry-making (10) | `djew-bead-strand`, `djew-casting-sprue`, `djew-chain-price`, `djew-clay-shrink`, `djew-earring-wire`, `djew-jump-ring`, `djew-pendant-weight`, `djew-ring-blank`, `djew-wire-gauge`, `djew-wire-loop` |
+| daily / snow-removal (10) | `dsnow-brine-mix`, `dsnow-ice-melt`, `dsnow-job-price`, `dsnow-plow-shifts`, `dsnow-roof-load`, `dsnow-salt-driveway`, `dsnow-salting-rounds`, `dsnow-sand-salt-stock`, `dsnow-shovel-rate`, `dsnow-snow-haul` |
+| daily / camping (10) | `dcamp-tent-capacity`, `dcamp-pack-weight`, `dcamp-water-tablets`, `dcamp-gas-canister`, `dcamp-cooler-ice`, `dcamp-clothesline`, `dcamp-rope-load`, `dcamp-headlamp-battery`, `dcamp-firewood-split`, `dcamp-fee-per-night` |
+| daily / photography (10) | `dphot-exposure-stops`, `dphot-hyperfocal`, `dphot-memory-card`, `dphot-golden-hour`, `dphot-print-size`, `dphot-film-contact`, `dphot-sandbag-packs`, `dphot-stock-royalty`, `dphot-lens-rental`, `dphot-timelapse-clock` |
+| daily / records (10) | `dvrc-shipping-boxes`, `dvrc-sleeve-packs`, `dvrc-bulk-price`, `dvrc-play-hours`, `dvrc-flip-clock`, `dvrc-crate-load`, `dvrc-clean-fluid`, `dvrc-grading-score`, `dvrc-belt-life`, `dvrc-shelf-space` |
+| daily / skiing (10) | `dski-lift-pass`, `dski-slope-grade`, `dski-wax-season`, `dski-lift-queue`, `dski-snowmaking`, `dski-boot-shell`, `dski-storage-fee`, `dski-kids-camp`, `dski-gear-value`, `dski-club-minibus` |
+| daily / vanlife (10) | `dvlf-solar-array`, `dvlf-water-tank`, `dvlf-fuel-budget`, `dvlf-propane`, `dvlf-mileage`, `dvlf-plywood-cuts`, `dvlf-grey-water`, `dvlf-fridge-amps`, `dvlf-drive-time`, `dvlf-campsite-bill` |
+| daily / arcade (10) | `darc-token-packs`, `darc-claw-net`, `darc-hoops-tickets`, `darc-dance-session`, `darc-hockey-league`, `darc-redemption-hours`, `darc-pinball-games`, `darc-ticket-dispenser`, `darc-photo-booth`, `darc-prize-wall` |
+| daily / 3d-printing (10) | `d3dp-cost-per-print`, `d3dp-print-time`, `d3dp-bed-layout`, `d3dp-resin-batch`, `d3dp-spool-swaps`, `d3dp-support-share`, `d3dp-post-cure`, `d3dp-flow-limit`, `d3dp-power-bill`, `d3dp-stock-order` |
+| daily / car-wash (10) | `dcw-foam-dilution`, `dcw-bay-throughput`, `dcw-water-reclaim`, `dcw-package-margin`, `dcw-clay-bars`, `dcw-vac-bags`, `dcw-tyre-dressing`, `dcw-queue-length`, `dcw-wash-plan`, `dcw-tip-pool` |
 
 ## Adding a test
 
@@ -159,8 +318,9 @@ with `math.Sqrt` rather than trusting the constant).
 4. Add table-driven `_test.go` cases covering a correct answer, a wrong
    answer, and at least one edge case (a common trap, a partial match, or a
    malformed response).
-5. Add the test's ID to the `wantCatalog` map in
+5. If you introduced a new subcategory, add it to `wantSubcats` in
-   `internal/tests/catalog_test.go`.
+   `internal/tests/catalog_test.go`; every subcategory needs at least ten
+   tests, so a one-test subcategory will not pass the catalog gate.
 
 ## Adding an evaluator
 
@@ -194,4 +354,4 @@ golangci-lint run       # if the binary is installed
 Every unit test runs offline against a mock `llm.Client` or a local
 `httptest.Server`; none of them call the live endpoint. The exec-evaluator
 tests (`GoRun`, `PyRun`, `CRun`) detect a missing `go`, `python3`, or `cc`
-toolchain and skip rather than fail.
+toolchain and skip rather than fail.

RESULTS-2026-08-17.md

@@ -1,277 +0,0 @@
-## Per-test scores
-
-| Test | Category | Subcategory | uni/deepseek-v4-flash-0731 | uni/qwen3.6-27b | uni/btl-4 |
-| --- | --- | --- | --- | --- | --- |
-| agent-deleg-batch-vs-separate | agents | delegation | 1.00 (161tok) | 1.00 (367tok) | 1.00 (894tok) |
-| agent-deleg-build-then-verify | agents | delegation | 1.00 (382tok) | 1.00 (494tok) | 1.00 (455tok) |
-| agent-deleg-escalation-to-human | agents | delegation | 1.00 (157tok) | 1.00 (202tok) | 1.00 (683tok) |
-| agent-deleg-handoff-context | agents | delegation | 1.00 (198tok) | 1.00 (633tok) | 1.00 (1037tok) |
-| agent-deleg-main-thread-vs-delegate | agents | delegation | 1.00 (154tok) | 1.00 (200tok) | 1.00 (456tok) |
-| agent-deleg-minimal-privilege | agents | delegation | 1.00 (471tok) | 1.00 (579tok) | 1.00 (632tok) |
-| agent-deleg-parallel-dispatch-safety | agents | delegation | 1.00 (231tok) | 1.00 (284tok) | 1.00 (901tok) |
-| agent-deleg-reviewer-independence | agents | delegation | 1.00 (679tok) | 1.00 (836tok) | 1.00 (1355tok) |
-| agent-deleg-task-to-specialist | agents | delegation | 1.00 (628tok) | 1.00 (654tok) | 1.00 (1050tok) |
-| agent-deleg-verify-vs-trust | agents | delegation | 1.00 (155tok) | 1.00 (269tok) | 1.00 (726tok) |
-| agent-plan-critical-path | agents | planning | 1.00 (567tok) | 1.00 (470tok) | 1.00 (1079tok) |
-| agent-plan-cycle-detect | agents | planning | 1.00 (306tok) | 1.00 (315tok) | 1.00 (1195tok) |
-| agent-plan-idempotency | agents | planning | 1.00 (499tok) | 1.00 (553tok) | 1.00 (1335tok) |
-| agent-plan-milestone-date | agents | planning | 1.00 (794tok) | 1.00 (236tok) | 1.00 (892tok) |
-| agent-plan-minimal-replan | agents | planning | 1.00 (406tok) | 1.00 (451tok) | 1.00 (865tok) |
-| agent-plan-ordering | agents | planning | 1.00 (415tok) | 1.00 (379tok) | 1.00 (905tok) |
-| agent-plan-parallel-steps | agents | planning | 1.00 (385tok) | 1.00 (322tok) | 1.00 (1120tok) |
-| agent-plan-repair | agents | planning | 1.00 (346tok) | 1.00 (312tok) | 1.00 (907tok) |
-| agent-plan-resource-constrained | agents | planning | 1.00 (1216tok) | 1.00 (577tok) | 1.00 (1633tok) |
-| agent-plan-rollback-trigger | agents | planning | 1.00 (370tok) | 1.00 (442tok) | 1.00 (1142tok) |
-| agent-tool-routing | agents | tool-routing | 1.00 (323tok) | 1.00 (434tok) | 1.00 (461tok) |
-| agent-tool-routing-ambiguous | agents | tool-routing | 1.00 (277tok) | 1.00 (336tok) | 1.00 (526tok) |
-| agent-tool-routing-chaining | agents | tool-routing | 1.00 (218tok) | 1.00 (302tok) | 1.00 (301tok) |
-| agent-tool-routing-cheapest | agents | tool-routing | 1.00 (260tok) | 1.00 (331tok) | 1.00 (421tok) |
-| agent-tool-routing-distractors | agents | tool-routing | 1.00 (555tok) | 1.00 (543tok) | 1.00 (602tok) |
-| agent-tool-routing-missing-param | agents | tool-routing | 1.00 (201tok) | 1.00 (478tok) | 0.50 (502tok) |
-| agent-tool-routing-multistep | agents | tool-routing | 1.00 (247tok) | 1.00 (326tok) | 1.00 (358tok) |
-| agent-tool-routing-no-tool-needed | agents | tool-routing | 1.00 (291tok) | 1.00 (362tok) | 1.00 (669tok) |
-| agent-tool-routing-parallel | agents | tool-routing | 1.00 (319tok) | 1.00 (380tok) | 1.00 (391tok) |
-| agent-tool-routing-safety | agents | tool-routing | 1.00 (194tok) | 1.00 (253tok) | 1.00 (232tok) |
-| llm-429-retry-backoff | ai | llm-integration | 1.00 (188tok) | 1.00 (278tok) | 1.00 (886tok) |
-| llm-context-overflow-strategy | ai | llm-integration | 1.00 (340tok) | 1.00 (739tok) | 1.00 (1373tok) |
-| llm-embedding-batch-math | ai | llm-integration | 1.00 (164tok) | 1.00 (252tok) | 1.00 (619tok) |
-| llm-field-semantics | ai | llm-integration | 1.00 (373tok) | 1.00 (401tok) | 1.00 (983tok) |
-| llm-role-placement | ai | llm-integration | 1.00 (181tok) | 1.00 (206tok) | 1.00 (418tok) |
-| llm-sse-stream-done | ai | llm-integration | 1.00 (297tok) | 1.00 (352tok) | 1.00 (792tok) |
-| llm-stop-sequence-trace | ai | llm-integration | 1.00 (450tok) | 1.00 (587tok) | 1.00 (1297tok) |
-| llm-temperature-zero-caveat | ai | llm-integration | 1.00 (974tok) | 1.00 (1957tok) | 1.00 (2400tok) |
-| llm-token-budget-reasoning | ai | llm-integration | 1.00 (185tok) | 1.00 (237tok) | 1.00 (297tok) |
-| llm-tool-call-handling | ai | llm-integration | 1.00 (195tok) | 1.00 (318tok) | 1.00 (806tok) |
-| rag-chunk-size-tradeoff | ai | rag | 1.00 (222tok) | 1.00 (312tok) | 1.00 (788tok) |
-| rag-citation-grounding | ai | rag | 1.00 (712tok) | 1.00 (236tok) | 1.00 (842tok) |
-| rag-context-selection-vs-stuffing | ai | rag | 1.00 (204tok) | 1.00 (316tok) | 1.00 (602tok) |
-| rag-eval-metric-choice | ai | rag | 1.00 (330tok) | 1.00 (395tok) | 1.00 (924tok) |
-| rag-hallucination-mitigation-ordering | ai | rag | 1.00 (331tok) | 1.00 (401tok) | 1.00 (1374tok) |
-| rag-index-staleness | ai | rag | 1.00 (291tok) | 1.00 (317tok) | 1.00 (895tok) |
-| rag-multihop-decomposition | ai | rag | 1.00 (358tok) | 1.00 (442tok) | 1.00 (1185tok) |
-| rag-preassembly-dedup | ai | rag | 1.00 (1395tok) | 1.00 (1194tok) | 1.00 (1304tok) |
-| rag-reranker-placement | ai | rag | 1.00 (182tok) | 1.00 (256tok) | 1.00 (306tok) |
-| rag-retrieval-failure-mode | ai | rag | 1.00 (150tok) | 1.00 (335tok) | 1.00 (480tok) |
-| vec-cosine-vs-dot | ai | vector-search | 1.00 (217tok) | 1.00 (298tok) | 1.00 (653tok) |
-| vec-distance-to-similarity | ai | vector-search | 1.00 (238tok) | 1.00 (306tok) | 1.00 (882tok) |
-| vec-embedding-dimension-tradeoff | ai | vector-search | 1.00 (244tok) | 1.00 (425tok) | 1.00 (524tok) |
-| vec-hnsw-efsearch-tradeoff | ai | vector-search | 1.00 (298tok) | 1.00 (261tok) | 1.00 (722tok) |
-| vec-index-build-query-tradeoff | ai | vector-search | 1.00 (319tok) | 1.00 (361tok) | 1.00 (521tok) |
-| vec-near-duplicate-threshold | ai | vector-search | 1.00 (349tok) | 1.00 (408tok) | 1.00 (1202tok) |
-| vec-pq-memory-math | ai | vector-search | 1.00 (238tok) | 1.00 (338tok) | 1.00 (865tok) |
-| vec-pre-vs-post-filtering | ai | vector-search | 1.00 (195tok) | 1.00 (455tok) | 1.00 (1043tok) |
-| vec-recall-at-k | ai | vector-search | 1.00 (258tok) | 1.00 (313tok) | 1.00 (811tok) |
-| vec-rrf-fusion | ai | vector-search | 1.00 (690tok) | 1.00 (618tok) | 1.00 (1466tok) |
-| pg-bloat-vacuum | databases | postgres | 1.00 (211tok) | 1.00 (288tok) | 1.00 (453tok) |
-| pg-deadlock-lock-order | databases | postgres | 1.00 (327tok) | 1.00 (391tok) | 1.00 (1138tok) |
-| pg-explain-seq-vs-index | databases | postgres | 1.00 (347tok) | 1.00 (622tok) | 1.00 (1166tok) |
-| pg-index-choice | databases | postgres | 1.00 (287tok) | 1.00 (433tok) | 1.00 (612tok) |
-| pg-isolation-anomaly | databases | postgres | 1.00 (240tok) | 1.00 (377tok) | 1.00 (813tok) |
-| pg-listen-notify | databases | postgres | 1.00 (603tok) | 1.00 (254tok) | 1.00 (1204tok) |
-| pg-n-plus-one | databases | postgres | 1.00 (213tok) | 1.00 (253tok) | 1.00 (566tok) |
-| pg-partial-index | databases | postgres | 1.00 (356tok) | 1.00 (433tok) | 1.00 (988tok) |
-| pg-pool-sizing | databases | postgres | 1.00 (160tok) | 1.00 (155tok) | 1.00 (265tok) |
-| pg-replica-failover | databases | postgres | 1.00 (254tok) | 1.00 (331tok) | 1.00 (1404tok) |
-| redis-cache-stampede | databases | redis | 1.00 (534tok) | 1.00 (477tok) | 1.00 (1341tok) |
-| redis-incr-atomicity | databases | redis | 1.00 (345tok) | 1.00 (593tok) | 1.00 (1025tok) |
-| redis-keyspace-anti-pattern | databases | redis | 1.00 (275tok) | 1.00 (378tok) | 1.00 (673tok) |
-| redis-lua-atomicity | databases | redis | 1.00 (135tok) | 1.00 (324tok) | 1.00 (646tok) |
-| redis-memory-estimate | databases | redis | 1.00 (184tok) | 1.00 (228tok) | 1.00 (992tok) |
-| redis-multi-vs-pipeline | databases | redis | 1.00 (275tok) | 1.00 (1734tok) | 0.00 (594tok) |
-| redis-noeviction-write | databases | redis | 1.00 (163tok) | 1.00 (182tok) | 1.00 (490tok) |
-| redis-pubsub-delivery | databases | redis | 1.00 (240tok) | 1.00 (588tok) | 1.00 (1009tok) |
-| redis-scan-vs-keys | databases | redis | 0.50 (774tok) | 0.50 (1063tok) | 0.50 (1968tok) |
-| redis-structure-choice | databases | redis | 1.00 (417tok) | 1.00 (417tok) | 1.00 (1616tok) |
-| sql-aggregate-sum | databases | sql-tuning | 1.00 (280tok) | 1.00 (380tok) | 1.00 (1101tok) |
-| sql-composite-index-skip-column | databases | sql-tuning | 1.00 (746tok) | 1.00 (1933tok) | 1.00 (1082tok) |
-| sql-covering-index | databases | sql-tuning | 1.00 (160tok) | 1.00 (529tok) | 1.00 (676tok) |
-| sql-equivalent-rewrite | databases | sql-tuning | 1.00 (330tok) | 1.00 (509tok) | 1.00 (1043tok) |
-| sql-groupby-having | databases | sql-tuning | 1.00 (610tok) | 1.00 (660tok) | 1.00 (1376tok) |
-| sql-join-counts | databases | sql-tuning | 1.00 (462tok) | 1.00 (662tok) | 1.00 (1644tok) |
-| sql-keyset-pagination | databases | sql-tuning | 1.00 (205tok) | 1.00 (256tok) | 1.00 (827tok) |
-| sql-null-in-where | databases | sql-tuning | 1.00 (331tok) | 1.00 (421tok) | 1.00 (917tok) |
-| sql-orderby-index-serving | databases | sql-tuning | 1.00 (249tok) | 1.00 (383tok) | 1.00 (680tok) |
-| sql-window-rank | databases | sql-tuning | 1.00 (286tok) | 1.00 (382tok) | 1.00 (414tok) |
-| docker-build-arg-vs-env | delivery | containers | 1.00 (243tok) | 1.00 (408tok) | 1.00 (978tok) |
-| docker-copy-vs-add | delivery | containers | 1.00 (195tok) | 1.00 (276tok) | 1.00 (856tok) |
-| docker-dockerignore-effect | delivery | containers | 1.00 (648tok) | 1.00 (389tok) | 1.00 (400tok) |
-| docker-entrypoint-cmd-trace | delivery | containers | 1.00 (197tok) | 1.00 (313tok) | 1.00 (1055tok) |
-| docker-healthcheck-semantics | delivery | containers | 1.00 (143tok) | 1.00 (472tok) | 1.00 (810tok) |
-| docker-image-size-math | delivery | containers | 1.00 (230tok) | 1.00 (284tok) | 1.00 (526tok) |
-| docker-layer-cache-bust | delivery | containers | 1.00 (337tok) | 1.00 (568tok) | 1.00 (550tok) |
-| docker-layer-count | delivery | containers | 1.00 (665tok) | 1.00 (423tok) | 1.00 (1668tok) |
-| docker-multistage-size-benefit | delivery | containers | 1.00 (1683tok) | 1.00 (1174tok) | 1.00 (1560tok) |
-| docker-nonroot-user-caps | delivery | containers | 1.00 (554tok) | 1.00 (5843tok) | 1.00 (1866tok) |
-| git-bisect-steps | delivery | git | 1.00 (4242tok) | 1.00 (4859tok) | 1.00 (2517tok) |
-| git-cherry-pick-vs-revert | delivery | git | 1.00 (204tok) | 1.00 (240tok) | 1.00 (432tok) |
-| git-conventional-commit-classify | delivery | git | 1.00 (429tok) | 1.00 (453tok) | 1.00 (1451tok) |
-| git-detached-head-recovery | delivery | git | 1.00 (1389tok) | 0.00 (12166tok)! | 0.00 (2045tok) |
-| git-force-with-lease-vs-force | delivery | git | 0.50 (1465tok) | 0.50 (2207tok) | 1.00 (1206tok) |
-| git-gitignore-negation-trace | delivery | git | 0.75 (834tok) | 1.00 (955tok) | 0.75 (1210tok) |
-| git-hook-choice | delivery | git | 1.00 (279tok) | 1.00 (255tok) | 1.00 (756tok) |
-| git-rebase-vs-merge | delivery | git | 1.00 (236tok) | 1.00 (239tok) | 1.00 (948tok) |
-| git-stash-pop-conflict | delivery | git | 1.00 (328tok) | 1.00 (436tok) | 1.00 (1168tok) |
-| git-worktree-use-case | delivery | git | 1.00 (502tok) | 1.00 (2334tok) | 1.00 (1292tok) |
-| rel-canary-vs-bluegreen | delivery | release-engineering | 1.00 (261tok) | 1.00 (318tok) | 1.00 (678tok) |
-| rel-checksums-signing-purpose | delivery | release-engineering | 1.00 (697tok) | 1.00 (286tok) | 0.50 (563tok) |
-| rel-clean-tree-requirement | delivery | release-engineering | 1.00 (902tok) | 0.50 (360tok) | 0.50 (732tok) |
-| rel-commit-changelog-mapping | delivery | release-engineering | 1.00 (378tok) | 1.00 (378tok) | 1.00 (989tok) |
-| rel-hotfix-flow-ordering | delivery | release-engineering | 0.00 (838tok) | 0.00 (1591tok) | 1.00 (2009tok) |
-| rel-pin-vs-range-policy | delivery | release-engineering | 1.00 (228tok) | 1.00 (307tok) | 1.00 (1018tok) |
-| rel-pipeline-stage-ordering | delivery | release-engineering | 0.00 (989tok) | 0.00 (340tok) | 0.00 (1054tok) |
-| rel-rollback-ordering | delivery | release-engineering | 1.00 (265tok) | 1.00 (976tok) | 1.00 (1610tok) |
-| rel-semver-bump-changelog | delivery | release-engineering | 1.00 (275tok) | 1.00 (345tok) | 1.00 (939tok) |
-| rel-tag-to-release-sequence | delivery | release-engineering | 1.00 (333tok) | 1.00 (464tok) | 1.00 (1374tok) |
-| k8s-argocd-outofsync | operations | kubernetes | 1.00 (415tok) | 1.00 (529tok) | 1.00 (1304tok) |
-| k8s-cnpg-pdb | operations | kubernetes | 1.00 (625tok) | 1.00 (4965tok) | 1.00 (1733tok) |
-| k8s-crashloop-gitops | operations | kubernetes | 0.75 (782tok) | 0.75 (2285tok) | 1.00 (2392tok) |
-| k8s-imagepullbackoff | operations | kubernetes | 1.00 (305tok) | 1.00 (378tok) | 1.00 (724tok) |
-| k8s-networkpolicy-dns-block | operations | kubernetes | 1.00 (964tok) | 1.00 (2399tok) | 1.00 (2102tok) |
-| k8s-pending-taints | operations | kubernetes | 1.00 (268tok) | 1.00 (398tok) | 1.00 (1052tok) |
-| k8s-pvc-storageclass-mismatch | operations | kubernetes | 1.00 (308tok) | 1.00 (440tok) | 1.00 (1030tok) |
-| k8s-qos-class | operations | kubernetes | 1.00 (299tok) | 1.00 (319tok) | 1.00 (871tok) |
-| k8s-service-selector-mismatch | operations | kubernetes | 1.00 (296tok) | 1.00 (360tok) | 1.00 (639tok) |
-| k8s-traefik-ingressroute-host | operations | kubernetes | 1.00 (324tok) | 1.00 (344tok) | 1.00 (517tok) |
-| linux-cgroup-oom-diagnosis | operations | linux | 1.00 (200tok) | 1.00 (450tok) | 1.00 (1058tok) |
-| linux-cron-expression | operations | linux | 1.00 (222tok) | 1.00 (231tok) | 1.00 (1033tok) |
-| linux-journalctl-filter | operations | linux | 1.00 (240tok) | 1.00 (271tok) | 1.00 (758tok) |
-| linux-lvm-extend-order | operations | linux | 1.00 (426tok) | 1.00 (517tok) | 1.00 (1248tok) |
-| linux-nftables-dnat | operations | linux | 1.00 (401tok) | 1.00 (740tok) | 1.00 (1146tok) |
-| linux-pct-exec | operations | linux | 1.00 (593tok) | 1.00 (6904tok) | 1.00 (968tok) |
-| linux-proc-meminfo-available | operations | linux | 1.00 (272tok) | 1.00 (511tok) | 1.00 (1087tok) |
-| linux-ssh-port-forward | operations | linux | 1.00 (431tok) | 0.50 (1566tok) | 1.00 (837tok) |
-| linux-systemd-oneshot | operations | linux | 1.00 (715tok) | 0.75 (3848tok) | 1.00 (580tok) |
-| linux-systemd-timer | operations | linux | 1.00 (781tok) | 0.00 (12134tok)! | 0.67 (741tok) |
-| macos-apfs-snapshot | operations | macos | 1.00 (827tok) | 0.00 (12121tok)! | 1.00 (1083tok) |
-| macos-caffeinate | operations | macos | 1.00 (777tok) | 1.00 (1285tok) | 1.00 (1223tok) |
-| macos-launchctl-bootstrap | operations | macos | 1.00 (1416tok) | 1.00 (2506tok) | 1.00 (1744tok) |
-| macos-launchd-cron | operations | macos | 1.00 (1535tok) | 1.00 (4637tok) | 1.00 (2191tok) |
-| macos-mdfind-spotlight | operations | macos | 1.00 (604tok) | 1.00 (4675tok) | 1.00 (1221tok) |
-| macos-plutil-plist-read | operations | macos | 1.00 (448tok) | 1.00 (741tok) | 1.00 (1164tok) |
-| macos-sed-inplace | operations | macos | 1.00 (483tok) | 1.00 (492tok) | 1.00 (819tok) |
-| macos-ssh-identity-agent | operations | macos | 1.00 (1021tok) | 1.00 (3259tok) | 1.00 (1655tok) |
-| macos-stat-bsd-gnu | operations | macos | 1.00 (444tok) | 1.00 (468tok) | 1.00 (1257tok) |
-| macos-timeout-portability | operations | macos | 1.00 (1417tok) | 0.00 (12140tok)! | 0.67 (3692tok) |
-| c-array-decay-sizeof | programming | c | 1.00 (181tok) | 1.00 (354tok) | 1.00 (842tok) |
-| c-bitmask-ops | programming | c | 1.00 (173tok) | 1.00 (260tok) | 1.00 (942tok) |
-| c-integer-promotion-overflow | programming | c | 1.00 (159tok) | 1.00 (316tok) | 1.00 (967tok) |
-| c-macro-expansion-pitfall | programming | c | 1.00 (253tok) | 1.00 (668tok) | 1.00 (760tok) |
-| c-pointer-arithmetic | programming | c | 1.00 (289tok) | 1.00 (327tok) | 1.00 (1179tok) |
-| c-string-function-output | programming | c | 1.00 (698tok) | 1.00 (880tok) | 1.00 (780tok) |
-| c-struct-bitfield-packing | programming | c | 1.00 (486tok) | 1.00 (657tok) | 1.00 (1188tok) |
-| c-struct-size | programming | c | 1.00 (312tok) | 1.00 (365tok) | 1.00 (1135tok) |
-| c-undefined-behavior-spot | programming | c | 1.00 (260tok) | 1.00 (331tok) | 1.00 (1651tok) |
-| c-union-size-lp64 | programming | c | 1.00 (307tok) | 1.00 (408tok) | 1.00 (1342tok) |
-| go-bounded-fanout | programming | golang | 1.00 (1628tok) | 0.00 (2714tok) | 1.00 (2483tok) |
-| go-channel-deadlock | programming | golang | 1.00 (277tok) | 1.00 (410tok) | 1.00 (561tok) |
-| go-context-cancellation-trace | programming | golang | 1.00 (283tok) | 1.00 (1093tok) | 1.00 (927tok) |
-| go-defer-recover-trace | programming | golang | 1.00 (392tok) | 1.00 (415tok) | 1.00 (1090tok) |
-| go-generics-constraint-fix | programming | golang | 1.00 (709tok) | 1.00 (6720tok) | 1.00 (1776tok) |
-| go-method-set-interface | programming | golang | 1.00 (256tok) | 1.00 (1828tok) | 1.00 (1278tok) |
-| go-semver-classify | programming | golang | 1.00 (616tok) | 1.00 (450tok) | 1.00 (1211tok) |
-| go-slice-append-aliasing | programming | golang | 1.00 (433tok) | 1.00 (365tok) | 1.00 (1087tok) |
-| go-struct-align | programming | golang | 1.00 (517tok) | 1.00 (1682tok) | 1.00 (2084tok) |
-| go-worker-pool | programming | golang | 1.00 (972tok) | 0.00 (12193tok)! | 0.00 (894tok) |
-| py-asyncio-gather-trace | programming | python | 1.00 (249tok) | 1.00 (342tok) | 1.00 (1352tok) |
-| py-cosine | programming | python | 1.00 (4455tok) | 1.00 (3928tok) | 1.00 (3882tok) |
-| py-dict-comprehension-trace | programming | python | 1.00 (346tok) | 1.00 (337tok) | 0.00 (12090tok)! |
-| py-generator-exhaustion-trace | programming | python | 1.00 (207tok) | 1.00 (266tok) | 1.00 (1208tok) |
-| py-json-transform | programming | python | 1.00 (434tok) | 1.00 (648tok) | 1.00 (1079tok) |
-| py-log-triage | programming | python | 1.00 (1727tok) | 1.00 (2375tok) | 1.00 (1645tok) |
-| py-mutable-default-arg | programming | python | 1.00 (299tok) | 1.00 (269tok) | 1.00 (950tok) |
-| py-pathlib-rewrite | programming | python | 1.00 (657tok) | 1.00 (1705tok) | 1.00 (659tok) |
-| py-regex-log-extraction | programming | python | 0.00 (956tok) | 1.00 (1616tok) | 1.00 (1249tok) |
-| py-softmax | programming | python | 1.00 (2134tok) | 1.00 (2861tok) | 1.00 (1834tok) |
-| ts-array-method-chain | programming | typescript | 1.00 (197tok) | 1.00 (263tok) | 1.00 (1146tok) |
-| ts-computed-vs-watch | programming | typescript | 1.00 (226tok) | 1.00 (300tok) | 1.00 (823tok) |
-| ts-debounce-composable | programming | typescript | 1.00 (687tok) | 1.00 (8871tok) | 1.00 (1521tok) |
-| ts-discriminated-union-narrowing | programming | typescript | 1.00 (328tok) | 1.00 (436tok) | 1.00 (1624tok) |
-| ts-eslint-flat-config | programming | typescript | 1.00 (903tok) | 1.00 (3197tok) | 1.00 (4007tok) |
-| ts-generic-utility-type | programming | typescript | 1.00 (2547tok) | 0.25 (12152tok)! | 1.00 (1759tok) |
-| ts-optional-chaining-nullish | programming | typescript | 1.00 (323tok) | 1.00 (392tok) | 1.00 (1513tok) |
-| ts-promise-allsettled-trace | programming | typescript | 1.00 (186tok) | 1.00 (282tok) | 1.00 (287tok) |
-| ts-tsconfig-strict-flag | programming | typescript | 1.00 (210tok) | 1.00 (665tok) | 1.00 (529tok) |
-| ts-vue-ref-reactive-unwrap | programming | typescript | 1.00 (247tok) | 0.00 (12087tok) | 0.50 (629tok) |
-| code-bigo-dedup | research | codebase | 1.00 (278tok) | 1.00 (313tok) | 1.00 (920tok) |
-| code-bug-line-diff | research | codebase | 1.00 (344tok) | 1.00 (607tok) | 1.00 (892tok) |
-| code-commit-bisect | research | codebase | 1.00 (474tok) | 1.00 (585tok) | 1.00 (996tok) |
-| code-dead-functions | research | codebase | 1.00 (225tok) | 1.00 (312tok) | 1.00 (404tok) |
-| code-generic-return-type | research | codebase | 1.00 (171tok) | 1.00 (248tok) | 1.00 (599tok) |
-| code-import-cycle | research | codebase | 1.00 (150tok) | 1.00 (230tok) | 1.00 (973tok) |
-| code-race-variable | research | codebase | 1.00 (188tok) | 1.00 (409tok) | 1.00 (643tok) |
-| code-trace-go | research | codebase | 1.00 (766tok) | 1.00 (690tok) | 1.00 (1514tok) |
-| code-trace-python | research | codebase | 1.00 (378tok) | 1.00 (345tok) | 1.00 (981tok) |
-| code-trace-ts | research | codebase | 1.00 (447tok) | 1.00 (356tok) | 1.00 (958tok) |
-| web-canonical-vs-redirect | research | web | 1.00 (249tok) | 1.00 (400tok) | 1.00 (767tok) |
-| web-cors-missing-header | research | web | 1.00 (221tok) | 1.00 (369tok) | 1.00 (999tok) |
-| web-dns-mx-record | research | web | 1.00 (186tok) | 1.00 (204tok) | 1.00 (529tok) |
-| web-hreflang-reciprocal | research | web | 1.00 (496tok) | 1.00 (540tok) | 1.00 (1319tok) |
-| web-html-product-extract | research | web | 1.00 (203tok) | 1.00 (315tok) | 1.00 (956tok) |
-| web-http-status-scenarios | research | web | 1.00 (425tok) | 1.00 (516tok) | 1.00 (1719tok) |
-| web-robots-ai-crawlers | research | web | 1.00 (517tok) | 1.00 (591tok) | 1.00 (1469tok) |
-| web-security-headers-audit | research | web | 1.00 (327tok) | 1.00 (409tok) | 1.00 (1250tok) |
-| web-sitemap-max-urls | research | web | 1.00 (127tok) | 1.00 (272tok) | 1.00 (958tok) |
-| web-url-parse-components | research | web | 1.00 (367tok) | 1.00 (316tok) | 1.00 (987tok) |
-| paper-attention-scale-factor | research | whitepapers | 1.00 (300tok) | 1.00 (378tok) | 1.00 (608tok) |
-| paper-bloom-fp-rate | research | whitepapers | 1.00 (5487tok) | 1.00 (5125tok) | 1.00 (2928tok) |
-| paper-btree-height-levels | research | whitepapers | 1.00 (501tok) | 1.00 (541tok) | 1.00 (1499tok) |
-| paper-cap-availability-choice | research | whitepapers | 1.00 (269tok) | 1.00 (366tok) | 1.00 (879tok) |
-| paper-hnsw-params | research | whitepapers | 1.00 (346tok) | 1.00 (541tok) | 1.00 (1337tok) |
-| paper-lsm-write-amplification | research | whitepapers | 1.00 (416tok) | 1.00 (529tok) | 1.00 (1351tok) |
-| paper-pq-compression-ratio | research | whitepapers | 1.00 (387tok) | 1.00 (468tok) | 1.00 (1112tok) |
-| paper-raft-quorum-failures | research | whitepapers | 1.00 (392tok) | 1.00 (418tok) | 1.00 (1350tok) |
-| paper-recall-at-k | research | whitepapers | 1.00 (430tok) | 1.00 (492tok) | 1.00 (1158tok) |
-| paper-tfidf-term-score | research | whitepapers | 1.00 (527tok) | 1.00 (604tok) | 1.00 (1595tok) |
-| sec-csrf-requirement | security | appsec | 1.00 (187tok) | 1.00 (297tok) | 1.00 (726tok) |
-| sec-idor-spot | security | appsec | 1.00 (282tok) | 1.00 (424tok) | 1.00 (658tok) |
-| sec-input-validation-boundary | security | appsec | 0.00 (12089tok)! | 1.00 (2320tok) | 1.00 (1382tok) |
-| sec-open-redirect-spot | security | appsec | 1.00 (179tok) | 1.00 (241tok) | 1.00 (502tok) |
-| sec-path-traversal-spot | security | appsec | 1.00 (277tok) | 1.00 (331tok) | 1.00 (628tok) |
-| sec-rate-limit-placement | security | appsec | 1.00 (184tok) | 1.00 (397tok) | 1.00 (1185tok) |
-| sec-secret-log-spot | security | appsec | 1.00 (305tok) | 1.00 (292tok) | 1.00 (471tok) |
-| sec-sqli-spot | security | appsec | 1.00 (229tok) | 1.00 (311tok) | 1.00 (504tok) |
-| sec-ssrf-spot | security | appsec | 1.00 (285tok) | 1.00 (485tok) | 1.00 (628tok) |
-| sec-xss-sink-spot | security | appsec | 1.00 (225tok) | 1.00 (312tok) | 1.00 (606tok) |
-| sec-aes-gcm-nonce-reuse | security | crypto | 1.00 (2811tok) | 1.00 (10585tok) | 1.00 (3603tok) |
-| sec-cert-chain-validation | security | crypto | 1.00 (244tok) | 1.00 (326tok) | 1.00 (1189tok) |
-| sec-constant-time-compare | security | crypto | 1.00 (468tok) | 1.00 (462tok) | 1.00 (874tok) |
-| sec-hash-vs-encrypt-pii | security | crypto | 1.00 (201tok) | 1.00 (250tok) | 1.00 (543tok) |
-| sec-hmac-vs-signature | security | crypto | 1.00 (488tok) | 1.00 (375tok) | 1.00 (577tok) |
-| sec-jwt-alg-none | security | crypto | 1.00 (210tok) | 1.00 (337tok) | 1.00 (1050tok) |
-| sec-password-hash-choice | security | crypto | 0.50 (1597tok) | 0.50 (638tok) | 0.50 (1936tok) |
-| sec-rand-source-choice | security | crypto | 0.50 (435tok) | 1.00 (839tok) | 0.50 (1270tok) |
-| sec-rotate-order | security | crypto | 1.00 (354tok) | 1.00 (665tok) | 1.00 (6504tok) |
-| sec-tls-floor-version | security | crypto | 1.00 (155tok) | 1.00 (311tok) | 1.00 (959tok) |
-| sec-diff-secret-spot | security | secrets | 1.00 (250tok) | 1.00 (394tok) | 1.00 (1159tok) |
-| sec-fork-pr-ci-exposure | security | secrets | 1.00 (344tok) | 1.00 (289tok) | 1.00 (376tok) |
-| sec-gitignore-history-nuance | security | secrets | 1.00 (376tok) | 1.00 (311tok) | 1.00 (819tok) |
-| sec-hardcoded-secret-spot | security | secrets | 1.00 (170tok) | 1.00 (202tok) | 1.00 (243tok) |
-| sec-k8s-secret-base64 | security | secrets | 1.00 (148tok) | 1.00 (199tok) | 1.00 (328tok) |
-| sec-least-privilege-scope | security | secrets | 1.00 (177tok) | 1.00 (230tok) | 1.00 (770tok) |
-| sec-remediation-order | security | secrets | 1.00 (1843tok) | 1.00 (506tok) | 0.00 (12098tok)! |
-| sec-rotation-no-downtime | security | secrets | 1.00 (451tok) | 1.00 (501tok) | 1.00 (465tok) |
-| sec-ssh-agent-identity | security | secrets | 1.00 (1689tok) | 1.00 (2416tok) | 1.00 (2843tok) |
-| sec-vault-tradeoff | security | secrets | 1.00 (305tok) | 1.00 (407tok) | 1.00 (740tok) |
-
-! = response truncated by the token budget (finish_reason=length); increase max_tokens_default to avoid
-
-## Category rollup (mean score)
-
-| Category | uni/deepseek-v4-flash-0731 | uni/qwen3.6-27b | uni/btl-4 |
-| --- | --- | --- | --- |
-| agents | 1.00 | 1.00 | 0.98 |
-| ai | 1.00 | 1.00 | 1.00 |
-| databases | 0.98 | 0.98 | 0.95 |
-| delivery | 0.91 | 0.87 | 0.89 |
-| operations | 0.99 | 0.87 | 0.98 |
-| programming | 0.97 | 0.91 | 0.94 |
-| research | 1.00 | 1.00 | 1.00 |
-| security | 0.93 | 0.98 | 0.93 |
-
-## Model summary
-
-| Model | Mean | Passed | Partial | Failed | Errors | Mean latency | Total tokens | Tok/s |
-| --- | --- | --- | --- | --- | --- | --- | --- | --- |
-| uni/deepseek-v4-flash-0731 | 0.97 | 240 | 6 | 4 | 0 | 5.685104556s | 138851 | 71.6 |
-| uni/qwen3.6-27b | 0.95 | 233 | 8 | 9 | 0 | 23.689755872s | 291412 | 40.7 |
-| uni/btl-4 | 0.96 | 234 | 10 | 6 | 0 | 13.135007924s | 295086 | 77.7 |

cmd/llmtest/main.go

@@ -14,6 +14,7 @@ import (
 	"github.com/lukaszraczylo/llm-testbench/internal/llm"
 	"github.com/lukaszraczylo/llm-testbench/internal/report"
 	"github.com/lukaszraczylo/llm-testbench/internal/runner"
+	"github.com/lukaszraczylo/llm-testbench/internal/testkit"
 	"github.com/lukaszraczylo/llm-testbench/internal/tests"
 )
 
@@ -37,9 +38,12 @@ func main() {
 		err = runCommand(os.Args[2:])
 	case "list":
 		err = listCommand(os.Args[2:])
+	case "compare":
+		err = compareCommand(os.Args[2:])
+	case "health":
+		err = healthCommand(os.Args[2:])
 	case "version", "-v", "--version":
 		fmt.Println("llmtest " + version)
-		return
 	case "-h", "--help", "help":
 		usage()
 		return
@@ -59,9 +63,16 @@ func usage() {
 	fmt.Fprint(os.Stderr, `llmtest - LLM accuracy testing framework
 
 Usage:
-  llmtest run  [flags]   Run the test catalog against configured models.
+  llmtest run     [flags]   Run the test catalog against configured models.
-  llmtest list [flags]   List the test catalog.
+  llmtest list    [flags]   List the test catalog.
-  llmtest version        Print the build version.
+  llmtest compare baseline.json current.json
+                            Diff two saved run artifacts (--out files).
+  llmtest health  file.json [more.json ...]
+                            Audit suite health: which subcategories are
+                            saturated (every model passes everything), and
+                            which tests still carry signal. Pools multiple
+                            artifacts.
+  llmtest version           Print the build version.
 
 Run "llmtest run -h" or "llmtest list -h" for flag details.
 `)
@@ -103,9 +114,12 @@ func runCommand(args []string) error {
 	fs := flag.NewFlagSet("run", flag.ExitOnError)
 	shared := bindSharedFlags(fs)
 	modelsCSV := fs.String("models", "", "comma-separated model override (defaults to config's models)")
+	testsCSV := fs.String("tests", "", "comma-separated exact test IDs (overrides --category/--subcategory)")
 	format := fs.String("format", "table", "output format: table|markdown|json")
 	concurrency := fs.Int("concurrency", 0, "override config's concurrency (0 = use config)")
 	timeout := fs.Duration("timeout", 0, "override config's request_timeout (0 = use config)")
+	repeat := fs.Int("repeat", 1, "samples per (model, test); >1 exposes response instability at temperature 0")
+	out := fs.String("out", "", "write the run as a JSON artifact to this file (for llmtest compare)")
 	quiet := fs.Bool("quiet", false, "suppress progress output on stderr")
 	if err := fs.Parse(args); err != nil {
 		return err
@@ -134,9 +148,9 @@ func runCommand(args []string) error {
 	client := llm.NewOpenAIClient(cfg.Endpoint, cfg.APIKey, reqTimeout)
 
 	registry := tests.All()
-	selected := registry.Filter(shared.category, shared.subcategory)
+	selected, err := selectTests(registry, *testsCSV, shared.category, shared.subcategory)
-	if len(selected) == 0 {
+	if err != nil {
-		return fmt.Errorf("no tests matched category=%q subcategory=%q", shared.category, shared.subcategory)
+		return err
 	}
 
 	var reporter runner.ProgressReporter = runner.NoopProgressReporter{}
@@ -149,6 +163,7 @@ func runCommand(args []string) error {
 		Temperature:      requestTemperature,
 		MaxTokensDefault: cfg.MaxTokensDefault,
 		Reporter:         reporter,
+		Repeat:           *repeat,
 	})
 
 	ctx := context.Background()
@@ -159,9 +174,90 @@ func runCommand(args []string) error {
 		return err
 	}
 
+	if *out != "" {
+		if err := report.WriteArtifact(*out, selected, models, results); err != nil {
+			return fmt.Errorf("write --out: %w", err)
+		}
+	}
+
 	return report.Render(os.Stdout, f, selected, models, results)
 }
 
+// selectTests resolves the test subset: --tests (exact IDs, error on any
+// unknown ID) overrides --category/--subcategory filtering.
+func selectTests(registry *testkit.Registry, testsCSV, category, subcategory string) ([]testkit.Test, error) {
+	if testsCSV == "" {
+		selected := registry.Filter(category, subcategory)
+		if len(selected) == 0 {
+			return nil, fmt.Errorf("no tests matched category=%q subcategory=%q", category, subcategory)
+		}
+		return selected, nil
+	}
+
+	ids := splitCSV(testsCSV)
+	out := make([]testkit.Test, 0, len(ids))
+	seen := make(map[string]bool, len(ids))
+	for _, id := range ids {
+		if seen[id] {
+			continue
+		}
+		seen[id] = true
+		t, ok := registry.Get(id)
+		if !ok {
+			return nil, fmt.Errorf("unknown test id %q (see llmtest list)", id)
+		}
+		out = append(out, t)
+	}
+	if len(out) == 0 {
+		return nil, fmt.Errorf("--tests parsed to zero ids")
+	}
+	return out, nil
+}
+
+// compareCommand diffs two saved artifacts from `llmtest run --out`.
+func compareCommand(args []string) error {
+	fs := flag.NewFlagSet("compare", flag.ExitOnError)
+	if err := fs.Parse(args); err != nil {
+		return err
+	}
+	if fs.NArg() != 2 {
+		return fmt.Errorf("usage: llmtest compare baseline.json current.json")
+	}
+
+	baseline, err := report.LoadArtifact(fs.Arg(0))
+	if err != nil {
+		return err
+	}
+
+	current, err := report.LoadArtifact(fs.Arg(1))
+	if err != nil {
+		return err
+	}
+
+	return report.RenderCompare(os.Stdout, report.CompareArtifacts(baseline, current))
+}
+
+// healthCommand audits one or more saved artifacts for suite health:
+// per-subcategory saturation and the tests that still discriminate.
+func healthCommand(args []string) error {
+	fs := flag.NewFlagSet("health", flag.ExitOnError)
+	if err := fs.Parse(args); err != nil {
+		return err
+	}
+	if fs.NArg() < 1 {
+		return fmt.Errorf("usage: llmtest health artifact.json [more.json ...]")
+	}
+	artifacts := make([]report.Artifact, 0, fs.NArg())
+	for _, p := range fs.Args() {
+		a, err := report.LoadArtifact(p)
+		if err != nil {
+			return fmt.Errorf("load %s: %w", p, err)
+		}
+		artifacts = append(artifacts, a)
+	}
+	return report.RenderHealth(os.Stdout, report.AuditHealth(artifacts))
+}
+
 // validateFormat parses and validates the --format flag value, pulled out
 // of runCommand as its own function so it is unit-testable without going
 // through flag parsing / a live run (N8).

cmd/llmtest/main_test.go

@@ -1,9 +1,12 @@
 package main
 
 import (
+	"strings"
 	"testing"
 
+	"github.com/lukaszraczylo/llm-testbench/internal/eval"
 	"github.com/lukaszraczylo/llm-testbench/internal/report"
+	"github.com/lukaszraczylo/llm-testbench/internal/testkit"
 )
 
 func TestSplitCSV(t *testing.T) {
@@ -61,3 +64,40 @@ func TestValidateFormat(t *testing.T) {
 		})
 	}
 }
+
+func TestSelectTests(t *testing.T) {
+	reg := testkit.NewRegistry()
+	reg.Register(testkit.Test{ID: "a", Category: "x", Subcategory: "s", Eval: eval.ContainsAny("a")})
+	reg.Register(testkit.Test{ID: "b", Category: "y", Subcategory: "s", Eval: eval.ContainsAny("b")})
+
+	got, err := selectTests(reg, "b,a", "", "")
+	if err != nil {
+		t.Fatalf("selectTests() error = %v", err)
+	}
+	if len(got) != 2 || got[0].ID != "b" || got[1].ID != "a" {
+		t.Errorf("selection = %+v, want [b a] in flag order", got)
+	}
+
+	if _, err := selectTests(reg, "a,nope", "", ""); err == nil || !strings.Contains(err.Error(), "nope") {
+		t.Errorf("unknown id error = %v, want mention of %q", err, "nope")
+	}
+
+	got, err = selectTests(reg, "", "y", "")
+	if err != nil || len(got) != 1 || got[0].ID != "b" {
+		t.Errorf("category filter = %+v, %v; want [b], nil", got, err)
+	}
+
+	if _, err := selectTests(reg, "", "zzz", ""); err == nil {
+		t.Error("empty category filter: error = nil, want no-match error")
+	}
+}
+
+func TestHealthCommand_RequiresArtifacts(t *testing.T) {
+	if err := healthCommand(nil); err == nil || !strings.Contains(err.Error(), "usage:") {
+		t.Errorf("healthCommand(nil) error = %v, want usage error", err)
+	}
+	if err := healthCommand([]string{"/nonexistent-artifact.json"}); err == nil ||
+		!strings.Contains(err.Error(), "load /nonexistent-artifact.json") {
+		t.Errorf("healthCommand(missing file) error = %v, want load error naming the file", err)
+	}
+}

internal/report/compare.go

@@ -0,0 +1,221 @@
+package report
+
+import (
+	"fmt"
+	"io"
+	"math"
+	"sort"
+	"strings"
+)
+
+// Comparison is the per-(model, test) diff between two saved artifacts.
+// Scores are means over attempts, so a test that flapped between runs
+// shows up as a regression on one attempt basis only if the mean moved.
+type Comparison struct {
+	Model        string
+	TestID       string
+	Category     string
+	Subcategory  string
+	Status       string
+	BaselineMean float64
+	CurrentMean  float64
+	Delta        float64
+}
+
+// RegressionDelta is the mean-score drop at which a test counts as a
+// regression. Below this, run-to-run sampling noise dominates.
+const RegressionDelta = 0.01
+
+type attemptKey struct{ model, testID string }
+
+// meanAttempts collapses saved attempts into per-(model,test) means and
+// carries category metadata from the first attempt that has it. Truncated
+// attempts are excluded from the mean when any attempt finished normally,
+// mirroring the truncation policy in indexResults; when every scored
+// attempt was truncated, the partial-text scores are used rather than
+// reporting no data.
+func meanAttempts(results []AttemptResult) map[attemptKey]Comparison {
+	type agg struct {
+		cat      string
+		sb       string
+		sum      float64
+		n        int
+		cleanSum float64
+		cleanN   int
+		errs     int
+	}
+	by := make(map[attemptKey]*agg)
+	order := make([]attemptKey, 0, len(results))
+	for _, r := range results {
+		k := attemptKey{r.Model, r.TestID}
+		a, seen := by[k]
+		if !seen {
+			a = &agg{}
+			by[k] = a
+			order = append(order, k)
+		}
+		if a.cat == "" {
+			a.cat, a.sb = r.Category, r.Subcategory
+		}
+		if r.Error != "" {
+			a.errs++
+			continue
+		}
+		if r.Skipped {
+			continue
+		}
+		a.sum += r.Score
+		a.n++
+		if !r.Truncated {
+			a.cleanSum += r.Score
+			a.cleanN++
+		}
+	}
+
+	out := make(map[attemptKey]Comparison, len(by))
+	for _, k := range order {
+		a := by[k]
+		c := Comparison{Model: k.model, TestID: k.testID, Category: a.cat, Subcategory: a.sb}
+		switch {
+		case a.n == 0:
+			c.Status = "error"
+		case a.cleanN > 0:
+			c.BaselineMean = a.cleanSum / float64(a.cleanN)
+		default:
+			c.BaselineMean = a.sum / float64(a.n)
+		}
+		out[k] = c
+	}
+	return out
+}
+
+// CompareArtifacts diffs two artifacts per (model, test), sorted by
+// delta ascending (worst regressions first), then test ID.
+func CompareArtifacts(baseline, current Artifact) []Comparison {
+	base := meanAttempts(baseline.Results)
+	cur := meanAttempts(current.Results)
+
+	keys := make(map[attemptKey]bool, len(base)+len(cur))
+	for k := range base {
+		keys[k] = true
+	}
+	for k := range cur {
+		keys[k] = true
+	}
+
+	out := make([]Comparison, 0, len(keys))
+	for k := range keys {
+		b, hasB := base[k]
+		c, hasC := cur[k]
+		switch {
+		case hasB && !hasC:
+			b.CurrentMean = math.NaN()
+			b.Delta = math.NaN()
+			b.Status = "baseline-only"
+			out = append(out, b)
+		case !hasB && hasC:
+			c.BaselineMean = math.NaN()
+			c.Delta = math.NaN()
+			c.Status = "current-only"
+			out = append(out, c)
+		default:
+			m := Comparison{
+				Model:        k.model,
+				TestID:       k.testID,
+				Category:     c.Category,
+				Subcategory:  c.Subcategory,
+				BaselineMean: b.BaselineMean,
+				CurrentMean:  c.BaselineMean,
+			}
+			switch {
+			case b.Status == "error" || c.Status == "error":
+				m.Status = "error"
+				m.Delta = math.NaN()
+			default:
+				m.Delta = m.CurrentMean - m.BaselineMean
+				switch {
+				case m.Delta <= -RegressionDelta:
+					m.Status = "regression"
+				case m.Delta >= RegressionDelta:
+					m.Status = "improvement"
+				default:
+					m.Status = "stable"
+				}
+			}
+			out = append(out, m)
+		}
+	}
+
+	sort.Slice(out, func(i, j int) bool {
+		di, dj := out[i].Delta, out[j].Delta
+		if math.IsNaN(di) != math.IsNaN(dj) {
+			return math.IsNaN(di)
+		}
+		if math.IsNaN(di) && math.IsNaN(dj) {
+			if out[i].Status != out[j].Status {
+				return out[i].Status < out[j].Status
+			}
+			return out[i].TestID < out[j].TestID
+		}
+		if di != dj {
+			return di < dj
+		}
+		if out[i].TestID != out[j].TestID {
+			return out[i].TestID < out[j].TestID
+		}
+		return out[i].Model < out[j].Model
+	})
+	return out
+}
+
+// RenderCompare writes a comparison report: headline counts, then only
+// changed/missing rows (stable rows are summarized, not listed).
+func RenderCompare(w io.Writer, comparisons []Comparison) error {
+	ew := &errWriter{w: w}
+
+	var reg, imp, stable, added, removed, errored int
+	for _, c := range comparisons {
+		switch c.Status {
+		case "regression":
+			reg++
+		case "improvement":
+			imp++
+		case "baseline-only":
+			removed++
+		case "current-only":
+			added++
+		case "error":
+			errored++
+		default:
+			stable++
+		}
+	}
+
+	ew.println(fmt.Sprintf("== Compare: %d regressions, %d improvements, %d stable, %d errors, %d baseline-only, %d current-only ==",
+		reg, imp, stable, errored, removed, added))
+	ew.println()
+	ew.println("MODEL\tTEST\tCATEGORY\tSUBCATEGORY\tBASELINE\tCURRENT\tDELTA\tSTATUS")
+	shown := 0
+	for _, c := range comparisons {
+		if c.Status == "stable" {
+			continue
+		}
+		ew.println(strings.Join([]string{
+			c.Model, c.TestID, c.Category, c.Subcategory,
+			meanCellText(c.BaselineMean), meanCellText(c.CurrentMean),
+			deltaText(c.Delta), c.Status,
+		}, "\t"))
+		shown++
+	}
+	if shown == 0 {
+		ew.println("(no changed, missing, or erroring tests)")
+	}
+	return ew.err
+}
+
+func deltaText(v float64) string {
+	if math.IsNaN(v) {
+		return "N/A"
+	}
+	return fmt.Sprintf("%+.2f", v)
+}

internal/report/compare_test.go

@@ -0,0 +1,114 @@
+package report
+
+import (
+	"bytes"
+	"math"
+	"strings"
+	"testing"
+)
+
+func attempt(model, testID string, score float64) AttemptResult {
+	return AttemptResult{Model: model, TestID: testID, Category: "c", Subcategory: "s", Score: score}
+}
+
+func TestCompareArtifacts_ClassifiesRegressionImprovementStable(t *testing.T) {
+	baseline := Artifact{Results: []AttemptResult{
+		attempt("m1", "reg", 1.0),
+		attempt("m1", "imp", 0.2),
+		attempt("m1", "stable", 1.0),
+	}}
+	current := Artifact{Results: []AttemptResult{
+		attempt("m1", "reg", 0.4),
+		attempt("m1", "imp", 1.0),
+		attempt("m1", "stable", 0.995),
+	}}
+
+	got := CompareArtifacts(baseline, current)
+	byID := map[string]Comparison{}
+	for _, c := range got {
+		byID[c.TestID] = c
+	}
+	if byID["reg"].Status != "regression" || byID["reg"].Delta > -0.5 {
+		t.Errorf("reg = %+v, want regression delta ~ -0.6", byID["reg"])
+	}
+	if byID["imp"].Status != "improvement" {
+		t.Errorf("imp = %+v, want improvement", byID["imp"])
+	}
+	if byID["stable"].Status != "stable" {
+		t.Errorf("stable = %+v, want stable", byID["stable"])
+	}
+	// Worst delta first.
+	if got[0].TestID != "reg" {
+		t.Errorf("first row = %q, want worst regression %q", got[0].TestID, "reg")
+	}
+}
+
+func TestCompareArtifacts_AveragesAttemptsAndFlagsMembership(t *testing.T) {
+	baseline := Artifact{Results: []AttemptResult{
+		attempt("m1", "flaky", 1.0), attempt("m1", "flaky", 0.0), // mean 0.5
+		attempt("m1", "gone", 1.0),
+	}}
+	current := Artifact{Results: []AttemptResult{
+		attempt("m1", "flaky", 1.0), attempt("m1", "flaky", 1.0), // mean 1.0 => improvement
+		attempt("m1", "new", 0.5),
+	}}
+
+	got := CompareArtifacts(baseline, current)
+	byID := map[string]Comparison{}
+	for _, c := range got {
+		byID[c.TestID] = c
+	}
+	if byID["flaky"].Status != "improvement" || math.Abs(byID["flaky"].BaselineMean-0.5) > 1e-9 {
+		t.Errorf("flaky = %+v, want improvement from mean 0.5", byID["flaky"])
+	}
+	if byID["gone"].Status != "baseline-only" {
+		t.Errorf("gone = %+v, want baseline-only", byID["gone"])
+	}
+	if byID["new"].Status != "current-only" {
+		t.Errorf("new = %+v, want current-only", byID["new"])
+	}
+}
+
+func TestCompareArtifacts_TruncatedAttemptExcludedFromMean(t *testing.T) {
+	clean := func(score float64) AttemptResult {
+		return AttemptResult{Model: "m1", TestID: "t", Category: "c", Subcategory: "s", Score: score}
+	}
+	trunc := func(score float64) AttemptResult {
+		return AttemptResult{Model: "m1", TestID: "t", Category: "c", Subcategory: "s", Score: score, Truncated: true}
+	}
+	baseline := Artifact{Results: []AttemptResult{clean(1.0), trunc(0.0)}} // clean mean 1.0
+	current := Artifact{Results: []AttemptResult{trunc(0.4), trunc(0.6)}}  // all truncated => fallback 0.5
+
+	got := CompareArtifacts(baseline, current)
+	if len(got) != 1 {
+		t.Fatalf("comparisons = %d, want 1", len(got))
+	}
+	if math.Abs(got[0].BaselineMean-1.0) > 1e-9 {
+		t.Errorf("baseline mean = %v, want 1.0 (truncated attempt excluded)", got[0].BaselineMean)
+	}
+	if math.Abs(got[0].CurrentMean-0.5) > 1e-9 {
+		t.Errorf("current mean = %v, want 0.5 (all-truncated fallback)", got[0].CurrentMean)
+	}
+	if got[0].Status != "regression" {
+		t.Errorf("status = %q, want regression", got[0].Status)
+	}
+}
+
+func TestRenderCompare_HeadlineAndRows(t *testing.T) {
+	baseline := Artifact{Results: []AttemptResult{attempt("m1", "a", 1.0), attempt("m1", "b", 1.0)}}
+	current := Artifact{Results: []AttemptResult{attempt("m1", "a", 0.0), attempt("m1", "b", 1.0)}}
+
+	var buf bytes.Buffer
+	if err := RenderCompare(&buf, CompareArtifacts(baseline, current)); err != nil {
+		t.Fatalf("RenderCompare() error = %v", err)
+	}
+	out := buf.String()
+	for _, want := range []string{"1 regressions", "0 improvements", "1 stable", "a", "-1.00"} {
+		if !strings.Contains(out, want) {
+			t.Errorf("output missing %q:\n%s", want, out)
+		}
+	}
+	if strings.Contains(out, "\nb\t") {
+		t.Errorf("stable row b should be summarized, not listed:\n%s", out)
+	}
+}

internal/report/discrimination.go

@@ -0,0 +1,114 @@
+package report
+
+import (
+	"math"
+	"sort"
+
+	"github.com/lukaszraczylo/llm-testbench/internal/testkit"
+)
+
+// DiscriminationSpread is the cross-model score gap at which a test starts
+// to separate models. The last full run showed a median spread of 0.0
+// across 250 tests: nearly every test scored >=0.99 for every model, so
+// the suite measured nothing about relative capability. This number makes
+// that visible per test instead of averaged away.
+const DiscriminationSpread = 0.05
+
+// testStat is the per-test measurement rollup: cross-model discrimination
+// (spread, only meaningful with 2+ models), attempt stability (did
+// --repeat samples disagree), and aggregate score. Computed over the
+// attempt-aggregated cells so a flaky test shows as unstable rather than
+// as a random single score.
+type testStat struct {
+	testID      string
+	category    string
+	subcategory string
+	mean        float64 // mean over model cell means (scored cells only)
+	spread      float64 // max-min over model cell means; NaN with <2 scored models
+	unstable    bool    // some model's attempts disagreed beyond epsilon
+	truncated   bool    // some cell was token-budget truncated
+	scored      bool    // at least one scored cell
+}
+
+// unstableEpsilon absorbs float noise in averaged scores; a real
+// disagreement between attempts is orders of magnitude larger.
+const unstableEpsilon = 1e-9
+
+// testStats computes the per-test rollup for every test in tests, ordered
+// by category, subcategory, ID (same order as the score table).
+func testStats(tests []testkit.Test, models []string, idx cellIndex) []testStat {
+	out := make([]testStat, 0, len(tests))
+	for _, t := range sortedTests(tests) {
+		st := testStat{testID: t.ID, category: t.Category, subcategory: t.Subcategory, spread: math.NaN()}
+		var sum float64
+		var n int
+		var lo, hi float64
+		for _, model := range models {
+			c, ok := idx[t.ID][model]
+			if !ok || c.errorAll || c.skippedAll || c.scored == 0 {
+				continue
+			}
+			v := c.result.Score.Value
+			if n == 0 {
+				lo, hi = v, v
+			} else {
+				lo = math.Min(lo, v)
+				hi = math.Max(hi, v)
+			}
+			sum += v
+			n++
+			st.scored = true
+			if c.maxScore-c.minScore > unstableEpsilon {
+				st.unstable = true
+			}
+			st.truncated = st.truncated || c.result.Truncated()
+		}
+		if n > 1 {
+			st.spread = hi - lo
+			st.mean = sum / float64(n)
+		} else if n == 1 {
+			st.mean = sum
+		}
+		out = append(out, st)
+	}
+	return out
+}
+
+// interestingStats narrows testStats to rows worth reading: tests that
+// discriminate between models, tests whose repeated attempts disagreed,
+// and (when no cross-model data exists) tests that any model failed to
+// pass outright. Sorted by spread descending, unstable first within equal
+// spread.
+func interestingStats(stats []testStat, multiModel bool) []testStat {
+	var out []testStat
+	for _, st := range stats {
+		if !st.scored {
+			continue
+		}
+		switch {
+		case st.unstable:
+		case multiModel && !math.IsNaN(st.spread) && st.spread >= DiscriminationSpread:
+		case !multiModel && st.mean < passThreshold:
+		default:
+			continue
+		}
+		out = append(out, st)
+	}
+	sort.SliceStable(out, func(i, j int) bool {
+		if out[i].unstable != out[j].unstable {
+			return out[i].unstable
+		}
+		si, sj := out[i].spread, out[j].spread
+		if math.IsNaN(si) {
+			si = -1
+		}
+		if math.IsNaN(sj) {
+			sj = -1
+		}
+		if si != sj {
+			return si > sj
+		}
+		return out[i].testID < out[j].testID
+	})
+	return out
+}

internal/report/discrimination_test.go

@@ -0,0 +1,86 @@
+package report
+
+import (
+	"bytes"
+	"encoding/json"
+	"strings"
+	"testing"
+
+	"github.com/lukaszraczylo/llm-testbench/internal/eval"
+	"github.com/lukaszraczylo/llm-testbench/internal/runner"
+	"github.com/lukaszraczylo/llm-testbench/internal/testkit"
+)
+
+func TestDiscrimination_TableShowsSpreadAndInstability(t *testing.T) {
+	tests := []testkit.Test{
+		{ID: "sep", Category: "c", Subcategory: "s"},
+		{ID: "same", Category: "c", Subcategory: "s"},
+	}
+	results := []runner.Result{
+		{Model: "m1", TestID: "sep", Score: eval.Score{Value: 1}},
+		{Model: "m2", TestID: "sep", Score: eval.Score{Value: 0.2}},
+		{Model: "m1", TestID: "same", Score: eval.Score{Value: 1}},
+		{Model: "m2", TestID: "same", Score: eval.Score{Value: 1}},
+		// Flaky pair for m1/sep: a second attempt disagrees with the first.
+		{Model: "m1", TestID: "sep", Attempt: 1, Score: eval.Score{Value: 0.4}},
+	}
+
+	var buf bytes.Buffer
+	if err := Render(&buf, FormatTable, tests, []string{"m1", "m2"}, results); err != nil {
+		t.Fatalf("Render() error = %v", err)
+	}
+	out := buf.String()
+
+	for _, want := range []string{
+		"== Discrimination: 1/2 scored tests separate models by >= 0.05; 1 unstable under repeat ==",
+		"sep",
+		"[0.40-1.00]", // unstable range in the score table cell
+	} {
+		if !strings.Contains(out, want) {
+			t.Errorf("output missing %q:\n%s", want, out)
+		}
+	}
+	if strings.Contains(out, "\nsame\t") {
+		// "same" appears in the per-test table; ensure it is NOT in the
+		// discrimination rows by checking the section body only.
+		section := out[strings.Index(out, "== Discrimination"):]
+		if strings.Contains(section, "same") {
+			t.Errorf("non-discriminating test listed in discrimination section:\n%s", section)
+		}
+	}
+}
+
+func TestDiscrimination_JSONStatsAndAttemptRoundTrip(t *testing.T) {
+	tests := []testkit.Test{{ID: "t1", Category: "c", Subcategory: "s"}}
+	results := []runner.Result{
+		{Model: "m1", TestID: "t1", Score: eval.Score{Value: 1}},
+		{Model: "m2", TestID: "t1", Score: eval.Score{Value: 0.5}},
+		{Model: "m1", TestID: "t1", Attempt: 1, Score: eval.Score{Value: 0.5}},
+	}
+
+	var buf bytes.Buffer
+	if err := Render(&buf, FormatJSON, tests, []string{"m1", "m2"}, results); err != nil {
+		t.Fatalf("Render() error = %v", err)
+	}
+	var doc jsonArtifact
+	if err := json.Unmarshal(buf.Bytes(), &doc); err != nil {
+		t.Fatalf("unmarshal: %v", err)
+	}
+	if len(doc.Stats) != 1 {
+		t.Fatalf("len(stats) = %d, want 1", len(doc.Stats))
+	}
+	st := doc.Stats[0]
+	if !st.Unstable {
+		t.Error("Unstable = false, want true (attempts 1.0 vs 0.5)")
+	}
+	if st.Spread < 0.24 || st.Spread > 0.26 {
+		t.Errorf("Spread = %v, want ~0.25 (m1 mean 0.75 vs m2 0.5)", st.Spread)
+	}
+
+	// Attempt indices survive the round trip for compare.
+	var attempts []AttemptResult
+	attempts = append(attempts, doc.Results...)
+	if attempts[2].Attempt != 1 {
+		t.Errorf("attempt = %d, want 1", attempts[2].Attempt)
+	}
+}

internal/report/health.go

@@ -0,0 +1,222 @@
+package report
+
+import (
+	"fmt"
+	"io"
+	"sort"
+)
+
+// PerfectScore is the score at which a (model, test) cell counts as a
+// perfect pass. Partial-credit evaluators score between 0 and 1; anything
+// below this is not a perfect pass.
+const PerfectScore = 0.999
+
+// SaturationRate is the share of a test's scored cells that must be
+// perfect for the test to count as saturated - passed by every model it
+// was run against, every attempt. Saturated tests carry no signal about
+// relative capability; they only detect outright regressions.
+const SaturationRate = 0.9
+
+// SubHealth is the suite-health rollup for one category/subcategory pair,
+// pooled across one or more artifacts (so single-model probe runs can be
+// combined with a full multi-model run). Cells are attempt-aggregated
+// (model, test) means: a pair counts once, taking its WORST artifact -
+// health asks "can any observed run fail this test?", so pooling over
+// re-runs must not let a lucky second attempt hide a real failure.
+type SubHealth struct {
+	Category    string
+	Subcategory string
+	WeakTests   []string // non-saturated test IDs, sorted - the signal-carriers
+	Tests       int      // distinct tests with at least one scored cell
+	Cells       int      // scored (model, test) cells
+	Perfect     int      // cells at PerfectScore or above
+	Saturated   int      // tests perfect in >= SaturationRate of their cells
+	Unstable    int      // tests whose repeated attempts disagreed
+	Truncated   int      // tests with a truncated cell
+}
+
+// PerfectRate is the share of scored cells that are perfect.
+func (h SubHealth) PerfectRate() float64 {
+	if h.Cells == 0 {
+		return 0
+	}
+	return float64(h.Perfect) / float64(h.Cells)
+}
+
+// HealthReport pools every subcategory present in the artifacts.
+type HealthReport struct {
+	Subcategories []SubHealth
+	Models        []string
+	Tests         int // distinct scored tests across all subcategories
+	Cells         int
+	Perfect       int
+	Saturated     int
+	Artifacts     int
+}
+
+// healthTestID scopes a test ID by model for cell pooling.
+type healthCellID struct{ model, testID string }
+
+// healthTest accumulates one test's cells across artifacts.
+type healthTest struct {
+	cells                 map[healthCellID]float64
+	category, subcategory string
+	unstable, truncated   bool
+}
+
+// AuditHealth pools artifacts into a per-subcategory health view.
+// Errored and skipped attempts never form cells (meanAttempts already
+// separates them by Status); truncated attempts follow the same
+// exclusion policy as compare. Per-test unstable/truncated flags are
+// OR-ed across artifacts: once flaky, always flagged.
+func AuditHealth(artifacts []Artifact) HealthReport {
+	agg := map[string]*healthTest{} // key: testID
+	modelSet := map[string]bool{}
+
+	for _, a := range artifacts {
+		for _, c := range meanAttempts(a.Results) {
+			if c.Status != "" && c.Status != "scored" {
+				continue
+			}
+			modelSet[c.Model] = true
+			ht := agg[c.TestID]
+			if ht == nil {
+				ht = &healthTest{category: c.Category, subcategory: c.Subcategory, cells: map[healthCellID]float64{}}
+				if ht.category == "" {
+					ht.category = "unknown"
+				}
+				if ht.subcategory == "" {
+					ht.subcategory = "unknown"
+				}
+				agg[c.TestID] = ht
+			}
+			id := healthCellID{c.Model, c.TestID}
+			if prev, ok := ht.cells[id]; !ok || c.BaselineMean < prev {
+				// Pool the worst observed mean for this cell. On the
+				// first artifact BaselineMean is the current run's mean.
+				ht.cells[id] = c.BaselineMean
+			}
+		}
+		for _, s := range a.Stats {
+			if ht, ok := agg[s.TestID]; ok {
+				ht.unstable = ht.unstable || s.Unstable
+				ht.truncated = ht.truncated || s.Truncated
+			}
+		}
+	}
+
+	models := make([]string, 0, len(modelSet))
+	for m := range modelSet {
+		models = append(models, m)
+	}
+	sort.Strings(models)
+
+	bySub := map[string]*SubHealth{}
+	var tot, totCells, totPerfect, totSat int
+	for testID, ht := range agg {
+		sk := ht.category + "/" + ht.subcategory
+		sh := bySub[sk]
+		if sh == nil {
+			sh = &SubHealth{Category: ht.category, Subcategory: ht.subcategory}
+			bySub[sk] = sh
+		}
+		sh.Tests++
+		tot++
+		var perfect int
+		for _, s := range ht.cells {
+			sh.Cells++
+			totCells++
+			if s >= PerfectScore {
+				perfect++
+				sh.Perfect++
+				totPerfect++
+			}
+		}
+		if sh.Cells > 0 && ht.unstable {
+			sh.Unstable++
+		}
+		if ht.truncated {
+			sh.Truncated++
+		}
+		if float64(perfect)/float64(len(ht.cells)) >= SaturationRate {
+			sh.Saturated++
+			totSat++
+		} else {
+			sh.WeakTests = append(sh.WeakTests, testID)
+		}
+	}
+
+	report := HealthReport{Artifacts: len(artifacts), Models: models, Tests: tot, Cells: totCells, Perfect: totPerfect, Saturated: totSat}
+	for _, sh := range bySub {
+		sort.Strings(sh.WeakTests)
+		report.Subcategories = append(report.Subcategories, *sh)
+	}
+	sort.Slice(report.Subcategories, func(i, j int) bool {
+		a, b := report.Subcategories[i], report.Subcategories[j]
+		if a.PerfectRate() != b.PerfectRate() {
+			return a.PerfectRate() > b.PerfectRate()
+		}
+		if a.Category != b.Category {
+			return a.Category < b.Category
+		}
+		return a.Subcategory < b.Subcategory
+	})
+	return report
+}
+
+// weakCap bounds listed weak tests per subcategory in the text report;
+// the counts stay accurate regardless.
+const weakCap = 8
+
+// RenderHealth writes the audit: headline numbers, then one line per
+// subcategory sorted by perfect-cell rate descending (most saturated -
+// least informative - first), listing its non-saturated tests.
+func RenderHealth(w io.Writer, r HealthReport) error {
+	ew := &errWriter{w: w}
+	perfectRate := 0.0
+	if r.Cells > 0 {
+		perfectRate = float64(r.Perfect) / float64(r.Cells)
+	}
+	ew.println(fmt.Sprintf("== Suite health: %d tests, %d scored (model,test) cells over %d model(s), %d artifact(s) ==",
+		r.Tests, r.Cells, len(r.Models), r.Artifacts))
+	ew.println(fmt.Sprintf("perfect cells: %d/%d (%.1f%%) | saturated tests (no signal): %d/%d\n",
+		r.Perfect, r.Cells, perfectRate*100, r.Saturated, r.Tests))
+	if r.Saturated == r.Tests && r.Tests > 0 {
+		ew.println("every scored test is saturated: the suite cannot rank these models at all.")
+	}
+	ew.println(fmt.Sprintf("%-30s %5s %6s %8s %9s %8s %8s  WEAK TESTS",
+		"SUBCATEGORY", "TESTS", "CELLS", "PERFECT%", "SATURATED", "UNSTBL", "TRUNC"))
+	for _, sh := range r.Subcategories {
+		name := sh.Category + "/" + sh.Subcategory
+		weak := ""
+		if len(sh.WeakTests) > 0 {
+			weak = fmt.Sprintf("%d", len(sh.WeakTests))
+			if len(sh.WeakTests) > weakCap {
+				weak += ": " + joinN(sh.WeakTests, weakCap) + ", ..."
+			} else {
+				weak += ": " + joinAll(sh.WeakTests)
+			}
+		}
+		ew.println(fmt.Sprintf("%-30s %5d %6d %8.1f %9d %8d %8d  %s",
+			name, sh.Tests, sh.Cells, sh.PerfectRate()*100, sh.Saturated, sh.Unstable, sh.Truncated, weak))
+	}
+	return ew.err
+}
+
+func joinN(xs []string, n int) string {
+	if len(xs) <= n {
+		return joinAll(xs)
+	}
+	return joinAll(xs[:n])
+}
+
+func joinAll(xs []string) string {
+	out := ""
+	for i, x := range xs {
+		if i > 0 {
+			out += ", "
+		}
+		out += x
+	}
+	return out
+}

internal/report/health_test.go

@@ -0,0 +1,200 @@
+package report
+
+import (
+	"bytes"
+	"strings"
+	"testing"
+)
+
+// scoredAttempt builds a scored attempt in subcategory (cat, sub).
+func scoredAttempt(model, testID, cat, sub string, score float64) AttemptResult {
+	return AttemptResult{Model: model, TestID: testID, Category: cat, Subcategory: sub, Score: score}
+}
+
+func subHealth(t *testing.T, r HealthReport, cat, sub string) SubHealth {
+	t.Helper()
+	for _, sh := range r.Subcategories {
+		if sh.Category == cat && sh.Subcategory == sub {
+			return sh
+		}
+	}
+	t.Fatalf("subcategory %s/%s missing from report", cat, sub)
+	return SubHealth{}
+}
+
+func TestAuditHealth_SaturatedVersusWeakTests(t *testing.T) {
+	// a: perfect for both models => saturated, no signal.
+	// b: model m2 fails => weak, carries signal.
+	art := Artifact{Results: []AttemptResult{
+		scoredAttempt("m1", "a", "c", "s", 1.0),
+		scoredAttempt("m2", "a", "c", "s", 1.0),
+		scoredAttempt("m1", "b", "c", "s", 1.0),
+		scoredAttempt("m2", "b", "c", "s", 0.0),
+	}}
+	r := AuditHealth([]Artifact{art})
+
+	sh := subHealth(t, r, "c", "s")
+	if sh.Tests != 2 || sh.Cells != 4 {
+		t.Fatalf("tests/cells = %d/%d, want 2/4", sh.Tests, sh.Cells)
+	}
+	if sh.Perfect != 3 {
+		t.Errorf("perfect = %d, want 3", sh.Perfect)
+	}
+	if sh.Saturated != 1 {
+		t.Errorf("saturated = %d, want 1 (only test a)", sh.Saturated)
+	}
+	if len(sh.WeakTests) != 1 || sh.WeakTests[0] != "b" {
+		t.Errorf("weak = %v, want [b]", sh.WeakTests)
+	}
+	if r.Saturated != 1 || r.Tests != 2 {
+		t.Errorf("report totals = %d/%d, want 1/2", r.Saturated, r.Tests)
+	}
+	if len(r.Models) != 2 || r.Models[0] != "m1" {
+		t.Errorf("models = %v, want [m1 m2]", r.Models)
+	}
+}
+
+func TestAuditHealth_PoolsWorstCellAcrossArtifacts(t *testing.T) {
+	// A flaky test that passes on a second artifact must still count as
+	// weak: health asks whether any observed run failed it.
+	a1 := Artifact{Results: []AttemptResult{
+		scoredAttempt("m1", "flaky", "c", "s", 0.0),
+	}}
+	a2 := Artifact{Results: []AttemptResult{
+		scoredAttempt("m1", "flaky", "c", "s", 1.0),
+	}}
+	r := AuditHealth([]Artifact{a1, a2})
+
+	sh := subHealth(t, r, "c", "s")
+	if sh.Saturated != 0 || len(sh.WeakTests) != 1 {
+		t.Errorf("saturated=%d weak=%v, want 0/[flaky] (worst cell pooled)", sh.Saturated, sh.WeakTests)
+	}
+	if sh.Cells != 1 {
+		t.Errorf("cells = %d, want 1 (same model,test pooled)", sh.Cells)
+	}
+}
+
+func TestAuditHealth_AveragesRepeatedAttemptsPerCell(t *testing.T) {
+	// Two attempts of one cell: mean 0.5 < PerfectScore => not perfect.
+	art := Artifact{Results: []AttemptResult{
+		scoredAttempt("m1", "t", "c", "s", 1.0),
+		scoredAttempt("m1", "t", "c", "s", 0.0),
+	}}
+	r := AuditHealth([]Artifact{art})
+
+	sh := subHealth(t, r, "c", "s")
+	if sh.Cells != 1 || sh.Perfect != 0 {
+		t.Errorf("cells/perfect = %d/%d, want 1/0 (attempt mean 0.5)", sh.Cells, sh.Perfect)
+	}
+}
+
+func TestAuditHealth_ErrorsAndSkipsExcluded(t *testing.T) {
+	art := Artifact{Results: []AttemptResult{
+		{Model: "m1", TestID: "boom", Category: "c", Subcategory: "s", Error: "404"},
+		{Model: "m1", TestID: "skip", Category: "c", Subcategory: "s", Skipped: true},
+		scoredAttempt("m1", "ok", "c", "s", 1.0),
+	}}
+	r := AuditHealth([]Artifact{art})
+
+	sh := subHealth(t, r, "c", "s")
+	if sh.Tests != 1 || sh.Cells != 1 || sh.Saturated != 1 {
+		t.Errorf("tests/cells/saturated = %d/%d/%d, want 1/1/1 (error+skip excluded)",
+			sh.Tests, sh.Cells, sh.Saturated)
+	}
+}
+
+func TestAuditHealth_FlagsUnstableAndTruncated(t *testing.T) {
+	art := Artifact{
+		Results: []AttemptResult{
+			scoredAttempt("m1", "u", "c", "s", 1.0),
+			scoredAttempt("m1", "t", "c", "s", 0.5),
+		},
+		Stats: []jsonStatRow{
+			{TestID: "u", Unstable: true},
+			{TestID: "t", Truncated: true},
+		},
+	}
+	r := AuditHealth([]Artifact{art})
+
+	sh := subHealth(t, r, "c", "s")
+	if sh.Unstable != 1 || sh.Truncated != 1 {
+		t.Errorf("unstable/truncated = %d/%d, want 1/1", sh.Unstable, sh.Truncated)
+	}
+}
+
+func TestAuditHealth_SaturationBoundary(t *testing.T) {
+	// 9 perfect cells + 1 failed cell = 0.9 rate = at SaturationRate =>
+	// saturated. 8/9 = 0.888 < rate => weak.
+	perfect9 := Artifact{}
+	for i := range 9 {
+		perfect9.Results = append(perfect9.Results, scoredAttempt("m"+string(rune('a'+i)), "t1", "c", "s", 1.0))
+	}
+	perfect9.Results = append(perfect9.Results, scoredAttempt("mj", "t1", "c", "s", 0.0))
+
+	if r := AuditHealth([]Artifact{perfect9}); r.Saturated != 1 {
+		t.Errorf("9/10 perfect: saturated = %d, want 1 (rate == SaturationRate)", r.Saturated)
+	}
+
+	eight := Artifact{}
+	for i := range 8 {
+		eight.Results = append(eight.Results, scoredAttempt("m"+string(rune('a'+i)), "t1", "c", "s", 1.0))
+	}
+	eight.Results = append(eight.Results,
+		scoredAttempt("mi", "t1", "c", "s", 0.0),
+		scoredAttempt("mj", "t1", "c", "s", 0.0))
+
+	if r := AuditHealth([]Artifact{eight}); r.Saturated != 0 {
+		t.Errorf("8/10 perfect: saturated = %d, want 0", r.Saturated)
+	}
+}
+
+func TestAuditHealth_SortsMostSaturatedFirst(t *testing.T) {
+	art := Artifact{Results: []AttemptResult{
+		scoredAttempt("m1", "hard", "c", "low", 0.0),
+		scoredAttempt("m1", "easy1", "c", "high", 1.0),
+		scoredAttempt("m1", "easy2", "c", "high", 1.0),
+	}}
+	r := AuditHealth([]Artifact{art})
+
+	if len(r.Subcategories) != 2 {
+		t.Fatalf("subcategories = %d, want 2", len(r.Subcategories))
+	}
+	if got := r.Subcategories[0].Subcategory; got != "high" {
+		t.Errorf("first row = %q, want most saturated %q", got, "high")
+	}
+}
+
+func TestRenderHealth_HeadlineAndWeakList(t *testing.T) {
+	art := Artifact{Results: []AttemptResult{
+		scoredAttempt("m1", "a", "c", "s", 1.0),
+		scoredAttempt("m1", "b", "c", "s", 0.0),
+	}}
+	var buf bytes.Buffer
+	if err := RenderHealth(&buf, AuditHealth([]Artifact{art})); err != nil {
+		t.Fatalf("RenderHealth() error = %v", err)
+	}
+	out := buf.String()
+	for _, want := range []string{"2 tests", "2 scored", "c/s", "50.0", "1: b"} {
+		if !strings.Contains(out, want) {
+			t.Errorf("output missing %q:\n%s", want, out)
+		}
+	}
+	// "a" is saturated: it must not appear anywhere in the weak list.
+	if strings.Contains(out, ": a") || strings.Contains(out, ", a,") || strings.HasSuffix(strings.TrimSpace(out), " a") {
+		t.Errorf("saturated test a should not be listed weak:\n%s", out)
+	}
+}
+
+func TestRenderHealth_AllSaturatedVerdict(t *testing.T) {
+	art := Artifact{Results: []AttemptResult{
+		scoredAttempt("m1", "a", "c", "s", 1.0),
+		scoredAttempt("m2", "a", "c", "s", 1.0),
+	}}
+	var buf bytes.Buffer
+	if err := RenderHealth(&buf, AuditHealth([]Artifact{art})); err != nil {
+		t.Fatalf("RenderHealth() error = %v", err)
+	}
+	if !strings.Contains(buf.String(), "cannot rank these models") {
+		t.Errorf("missing all-saturated verdict:\n%s", buf.String())
+	}
+}

internal/report/json.go

@@ -2,14 +2,17 @@ package report
 
 import (
 	"encoding/json"
+	"fmt"
 	"io"
+	"math"
+	"os"
 
 	"github.com/lukaszraczylo/llm-testbench/internal/runner"
 	"github.com/lukaszraczylo/llm-testbench/internal/testkit"
 )
 
-// jsonResult is the JSON-serializable projection of a runner.Result, joined
+// jsonResult is the JSON-serializable projection of one runner.Result
-// with its test's category/subcategory for self-contained output.
+// (one attempt), joined with its test's category/subcategory.
 type jsonResult struct {
 	Model            string  `json:"model"`
 	TestID           string  `json:"test_id"`
@@ -26,16 +29,43 @@ type jsonResult struct {
 	Tokens           int     `json:"tokens"`
 	Skipped          bool    `json:"skipped"`
 	Truncated        bool    `json:"truncated"`
+	Attempt          int     `json:"attempt,omitempty"`
 }
 
-// renderJSON dumps raw results as a JSON array, joined with test metadata.
+// jsonArtifact is the --out / --format json document: every raw attempt
-func renderJSON(w io.Writer, tests []testkit.Test, results []runner.Result) error {
+// plus the per-test discrimination rollup, so downstream tooling
+// (compare) can re-derive stability without re-running the suite.
+type jsonArtifact struct {
+	Results []jsonResult  `json:"results"`
+	Stats   []jsonStatRow `json:"stats,omitempty"`
+}
+
+// jsonStatRow is the JSON projection of testStat.
+type jsonStatRow struct {
+	TestID      string  `json:"test_id"`
+	Category    string  `json:"category"`
+	Subcategory string  `json:"subcategory"`
+	Mean        float64 `json:"mean"`
+	Spread      float64 `json:"spread,omitempty"`
+	Unstable    bool    `json:"unstable,omitempty"`
+	Truncated   bool    `json:"truncated,omitempty"`
+}
+
+// renderJSON dumps the run as a JSON artifact document.
+func renderJSON(w io.Writer, tests []testkit.Test, models []string, results []runner.Result) error {
+	artifact := buildArtifact(tests, models, results)
+	enc := json.NewEncoder(w)
+	enc.SetIndent("", "  ")
+	return enc.Encode(artifact)
+}
+
+func buildArtifact(tests []testkit.Test, models []string, results []runner.Result) jsonArtifact {
 	testByID := make(map[string]testkit.Test, len(tests))
 	for _, t := range tests {
 		testByID[t.ID] = t
 	}
 
-	out := make([]jsonResult, 0, len(results))
+	out := jsonArtifact{Results: make([]jsonResult, 0, len(results))}
 	for _, r := range results {
 		t := testByID[r.TestID]
 		jr := jsonResult{
@@ -53,14 +83,79 @@ func renderJSON(w io.Writer, tests []testkit.Test, results []runner.Result) erro
 			PromptTokens:     r.PromptTokens,
 			CompletionTokens: r.CompletionTokens,
 			Tokens:           r.TotalTokens(),
+			Attempt:          r.Attempt,
 		}
 		if r.Err != nil {
 			jr.Error = r.Err.Error()
 		}
-		out = append(out, jr)
+		out.Results = append(out.Results, jr)
 	}
 
-	enc := json.NewEncoder(w)
+	idx := indexResults(results)
-	enc.SetIndent("", "  ")
+	for _, st := range testStats(tests, models, idx) {
-	return enc.Encode(out)
+		if !st.scored {
+			continue
+		}
+		row := jsonStatRow{
+			TestID:      st.testID,
+			Category:    st.category,
+			Subcategory: st.subcategory,
+			Mean:        round6(st.mean),
+			Unstable:    st.unstable,
+			Truncated:   st.truncated,
+		}
+		if !math.IsNaN(st.spread) {
+			row.Spread = round6(st.spread)
+		}
+		out.Stats = append(out.Stats, row)
+	}
+	return out
+}
+
+func round6(v float64) float64 {
+	return math.Round(v*1e6) / 1e6
+}
+
+// Artifact is a saved run document: every raw attempt plus the per-test
+// discrimination rollup. Exported so the compare command can consume it.
+type Artifact = jsonArtifact
+
+// AttemptResult is one saved attempt. Exported alias of jsonResult.
+type AttemptResult = jsonResult
+
+// LoadArtifact reads a --out JSON artifact written by a previous run.
+// Accepts both the current document shape and the legacy bare array of
+// results, so artifacts from earlier releases remain comparable.
+func LoadArtifact(path string) (jsonArtifact, error) {
+	b, err := os.ReadFile(path) //nolint:gosec // path is the operator's own `llmtest compare` argument, not external input
+	if err != nil {
+		return jsonArtifact{}, err
+	}
+
+	var probe struct {
+		Results []jsonResult `json:"results"`
+	}
+	if err := json.Unmarshal(b, &probe); err == nil && probe.Results != nil {
+		var a jsonArtifact
+		if err := json.Unmarshal(b, &a); err != nil {
+			return jsonArtifact{}, fmt.Errorf("%s: %w", path, err)
+		}
+		return a, nil
+	}
+
+	var legacy []jsonResult
+	if err := json.Unmarshal(b, &legacy); err != nil {
+		return jsonArtifact{}, fmt.Errorf("%s: not a llmtest json artifact: %w", path, err)
+	}
+	return jsonArtifact{Results: legacy}, nil
+}
+
+// WriteArtifact saves the run as a JSON artifact at path, for later
+// `llmtest compare` runs, regardless of the stdout report format.
+func WriteArtifact(path string, tests []testkit.Test, models []string, results []runner.Result) error {
+	b, err := json.MarshalIndent(buildArtifact(tests, models, results), "", "  ")
+	if err != nil {
+		return err
+	}
+	return os.WriteFile(path, append(b, '\n'), 0o644) //nolint:gosec // 0o644: the artifact is a report meant to be shared/committed, not a secret file
 }

internal/report/markdown.go

@@ -10,8 +10,8 @@ import (
 	"github.com/lukaszraczylo/llm-testbench/internal/testkit"
 )
 
-// renderMarkdown writes the same three tables as renderTable, formatted as
+// renderMarkdown writes the same four sections as renderTable, formatted
-// GitHub-flavored markdown tables.
+// as GitHub-flavored markdown tables.
 func renderMarkdown(w io.Writer, tests []testkit.Test, models []string, results []runner.Result) error {
 	idx := indexResults(results)
 	ew := &errWriter{w: w}
@@ -24,9 +24,9 @@ func renderMarkdown(w io.Writer, tests []testkit.Test, models []string, results
 	for _, t := range sortedTests(tests) {
 		row := []string{t.ID, t.Category, t.Subcategory}
 		for _, model := range models {
-			r, ok := idx[t.ID][model]
+			c := idx[t.ID][model]
-			row = append(row, cellText(r, ok))
+			row = append(row, cellText(c))
-			anyTruncated = anyTruncated || r.Truncated()
+			anyTruncated = anyTruncated || c.result.Truncated()
 		}
 		writeMarkdownRow(ew, row)
 	}
@@ -48,11 +48,16 @@ func renderMarkdown(w io.Writer, tests []testkit.Test, models []string, results
 		writeMarkdownRow(ew, row)
 	}
 
+	ew.println()
+	ew.println("## Discrimination")
+	ew.println()
+	writeDiscriminationRows(markdownPrinter{ew: ew}, testStats(tests, models, idx), len(models) > 1)
+
 	ew.println()
 	ew.println("## Model summary")
 	ew.println()
 	writeMarkdownHeader(ew, []string{"Model", "Mean", "Passed", "Partial", "Failed", "Errors", "Mean latency", "Total tokens", "Tok/s"})
-	for _, ms := range summarize(models, results) {
+	for _, ms := range summarize(models, idx) {
 		writeMarkdownRow(ew, []string{
 			ms.model,
 			meanCellText(ms.overallMean),
@@ -69,6 +74,18 @@ func renderMarkdown(w io.Writer, tests []testkit.Test, models []string, results
 	return ew.err
 }
 
+// markdownPrinter adapts writeDiscriminationRows to pipe-delimited rows.
+// GFM needs a blank line between the note paragraph and the table, so
+// header() emits one first.
+type markdownPrinter struct{ ew *errWriter }
+
+func (mp markdownPrinter) header(cells ...string) {
+	mp.ew.println()
+	writeMarkdownHeader(mp.ew, cells)
+}
+func (mp markdownPrinter) row(cells ...string) { writeMarkdownRow(mp.ew, cells) }
+func (mp markdownPrinter) note(text string)    { mp.ew.println(text) }
+
 func writeMarkdownHeader(ew *errWriter, cols []string) {
 	ew.println("| " + strings.Join(cols, " | ") + " |")
 	sep := make([]string, len(cols))

internal/report/report.go

@@ -9,6 +9,8 @@ import (
 	"sort"
 	"time"
 
+	"github.com/lukaszraczylo/llm-testbench/internal/eval"
+	"github.com/lukaszraczylo/llm-testbench/internal/llm"
 	"github.com/lukaszraczylo/llm-testbench/internal/runner"
 	"github.com/lukaszraczylo/llm-testbench/internal/testkit"
 )
@@ -35,22 +37,125 @@ func Render(w io.Writer, format Format, tests []testkit.Test, models []string, r
 	case FormatMarkdown:
 		return renderMarkdown(w, tests, models, results)
 	case FormatJSON:
-		return renderJSON(w, tests, results)
+		return renderJSON(w, tests, models, results)
 	default:
 		return fmt.Errorf("report: unknown format %q", format)
 	}
 }
 
-// resultIndex looks up a Result by testID then model.
+// modelCell is the aggregate view of every attempt recorded for one
-type resultIndex map[string]map[string]runner.Result
+// (testID, model) pair: with Config.Repeat = 1 this is the single result;
+// with repeats the Score is the mean over scored attempts, and minScore/
+// maxScore expose the within-model spread (instability) that repeats are
+// measured to find.
+type modelCell struct {
+	result     runner.Result // representative: last scored attempt, mean score patched in
+	minScore   float64
+	maxScore   float64
+	meanTokens float64
+	attempts   int
+	scored     int
+	skippedAll bool
+	errorAll   bool
+	ok         bool
+}
 
-func indexResults(results []runner.Result) resultIndex {
+// cellIndex looks up the aggregate cell for a testID then model.
-	idx := make(resultIndex)
+type cellIndex map[string]map[string]modelCell
+
+func indexResults(results []runner.Result) cellIndex {
+	type key struct{ testID, model string }
+	grouped := make(map[key][]runner.Result)
+	order := make([]key, 0, len(results))
 	for _, r := range results {
-		if idx[r.TestID] == nil {
+		k := key{r.TestID, r.Model}
-			idx[r.TestID] = make(map[string]runner.Result)
+		if _, seen := grouped[k]; !seen {
+			order = append(order, k)
 		}
-		idx[r.TestID][r.Model] = r
+		grouped[k] = append(grouped[k], r)
+	}
+
+	idx := make(cellIndex)
+	for _, k := range order {
+		attempts := grouped[k]
+		sort.Slice(attempts, func(i, j int) bool { return attempts[i].Attempt < attempts[j].Attempt })
+
+		c := modelCell{ok: true, attempts: len(attempts)}
+		// Truncation policy: a truncated attempt was scored on partial
+		// text, so its score measures the token budget as much as the
+		// model. Attempts that finished normally are therefore preferred
+		// for the cell mean (and for the min-max instability range); only
+		// when EVERY scored attempt was cut off does the mean fall back to
+		// the truncated scores. A cell is flagged "!" whenever any attempt
+		// was truncated, clean mean or not.
+		var sum, toks, cleanSum, cleanToks float64
+		var cleanN, scoredN int
+		var cleanMin, cleanMax float64
+		var anyTrunc bool
+		var last, lastClean runner.Result
+		for _, a := range attempts {
+			last = a
+			switch {
+			case a.Err != nil:
+				continue
+			case a.Score.Skipped:
+				continue
+			}
+			v := a.Score.Value
+			sum += v
+			toks += float64(a.TotalTokens())
+			scoredN++
+			if a.Truncated() {
+				anyTrunc = true
+				continue
+			}
+			if cleanN == 0 {
+				cleanMin, cleanMax = v, v
+			} else {
+				cleanMin = math.Min(cleanMin, v)
+				cleanMax = math.Max(cleanMax, v)
+			}
+			cleanSum += v
+			cleanToks += float64(a.TotalTokens())
+			cleanN++
+			lastClean = a
+		}
+		c.scored = scoredN
+
+		rep := last
+		if cleanN > 0 {
+			rep = lastClean
+		}
+		switch {
+		case scoredN > 0 && cleanN > 0:
+			rep.Score = eval.Score{
+				Value:  cleanSum / float64(cleanN),
+				Detail: lastClean.Score.Detail,
+			}
+			c.minScore, c.maxScore = cleanMin, cleanMax
+			c.meanTokens = cleanToks / float64(cleanN)
+			if anyTrunc {
+				rep.FinishReason = llm.FinishReasonLength
+			}
+		case scoredN > 0:
+			rep.Score = eval.Score{
+				Value:  sum / float64(scoredN),
+				Detail: last.Score.Detail,
+			}
+			c.minScore, c.maxScore = rep.Score.Value, rep.Score.Value
+			c.meanTokens = toks / float64(scoredN)
+			rep.FinishReason = llm.FinishReasonLength
+		case c.attempts > 0 && last.Score.Skipped:
+			c.skippedAll = true
+		case c.attempts > 0 && last.Err != nil:
+			c.errorAll = true
+		}
+
+		c.result = rep
+		if idx[k.testID] == nil {
+			idx[k.testID] = make(map[string]modelCell)
+		}
+		idx[k.testID][k.model] = c
 	}
 	return idx
 }
@@ -72,34 +177,41 @@ func sortedTests(tests []testkit.Test) []testkit.Test {
 	return out
 }
 
-// truncatedSuffix marks a cell whose underlying response was cut off by
+// truncatedSuffix marks a cell where any attempt was cut off by the token
-// the token budget (finish_reason=length): the score may not reflect what
+// budget (finish_reason=length). Per the truncation policy in
-// the model would have answered with a larger budget. truncatedLegend is
+// indexResults, the cell mean then comes from the attempts that finished
-// the one-line explanation printed once per table/markdown report.
+// normally; "!" warns that at least one sample was discarded (or, when no
+// attempt finished, that the whole mean is partial-text based).
+// truncatedLegend is the one-line explanation printed once per
+// table/markdown report.
 const (
 	truncatedSuffix = "!"
-	truncatedLegend = "! = response truncated by the token budget (finish_reason=length); increase max_tokens_default to avoid"
+	truncatedLegend = "! = some attempt was truncated by the token budget (finish_reason=length); the mean uses only attempts that finished, increase max_tokens_default if every attempt is flagged"
 )
 
 // cellText renders one (test, model) cell for the score table: "score
 // (Ntok)" for a scored result, or ERR/skip/N/A otherwise, with a
-// truncatedSuffix appended when the response was cut off by the token
+// truncatedSuffix appended when any attempt was cut off by the token
-// budget.
+// budget. With repeat attempts, a "[min-max]" range is appended when the
-func cellText(r runner.Result, found bool) string {
+// attempts disagreed, marking the cell unstable.
-	if !found {
+func cellText(c modelCell) string {
+	if !c.ok {
+		return "N/A"
+	}
+	switch {
+	case c.errorAll:
+		return "ERR"
+	case c.skippedAll:
+		return "skip"
+	case c.scored == 0:
 		return "N/A"
 	}
 
-	var text string
+	text := fmt.Sprintf("%.2f (%dtok)", c.result.Score.Value, int(math.Round(c.meanTokens)))
-	switch {
+	if c.attempts > 1 && c.maxScore > c.minScore {
-	case r.Err != nil:
+		text += fmt.Sprintf(" [%0.2f-%0.2f]", c.minScore, c.maxScore)
-		text = "ERR"
-	case r.Score.Skipped:
-		text = "skip"
-	default:
-		text = fmt.Sprintf("%.2f (%dtok)", r.Score.Value, r.TotalTokens())
 	}
-	if r.Truncated() {
+	if c.result.Truncated() {
 		text += truncatedSuffix
 	}
 	return text
@@ -112,7 +224,7 @@ type categoryMean struct {
 	category string
 }
 
-func categoryRollup(tests []testkit.Test, models []string, idx resultIndex) []categoryMean {
+func categoryRollup(tests []testkit.Test, models []string, idx cellIndex) []categoryMean {
 	order := make([]string, 0)
 	byCategory := make(map[string][]testkit.Test)
 	for _, t := range sortedTests(tests) {
@@ -129,11 +241,11 @@ func categoryRollup(tests []testkit.Test, models []string, idx resultIndex) []ca
 			var sum float64
 			var n int
 			for _, t := range byCategory[cat] {
-				r, ok := idx[t.ID][model]
+				c, ok := idx[t.ID][model]
-				if !ok || r.Err != nil || r.Score.Skipped {
+				if !ok || c.errorAll || c.skippedAll || c.scored == 0 {
 					continue
 				}
-				sum += r.Score.Value
+				sum += c.result.Score.Value
 				n++
 			}
 			if n > 0 {
@@ -191,49 +303,49 @@ type modelSummary struct {
 	avgTokPerSec float64 // math.NaN() when no timed, non-skipped data
 }
 
-func summarize(models []string, results []runner.Result) []modelSummary {
+// summarize aggregates one model's results per test (attempts already
-	byModel := make(map[string][]runner.Result, len(models))
+// merged by indexResults), so pass/partial/fail counts stay per-test even
-	for _, r := range results {
+// with Config.Repeat > 1.
-		byModel[r.Model] = append(byModel[r.Model], r)
+func summarize(models []string, idx cellIndex) []modelSummary {
-	}
-
 	out := make([]modelSummary, 0, len(models))
 	for _, model := range models {
 		ms := modelSummary{model: model}
-		var scoreSum float64
+		var scoreSum, latencySecondsSum, meanTokenSum float64
-		var scoredCount int
+		var scoredCount, latencyCount int
 		var latencySum time.Duration
-		var latencyCount int
-		var completionTokenSum int
-		var latencySecondsSum float64
 
-		for _, r := range byModel[model] {
+		for _, cells := range idx {
-			ms.totalTokens += r.TotalTokens()
+			c, ok := cells[model]
-			switch {
+			if !ok {
-			case r.Err != nil:
-				ms.errors++
 				continue
-			case r.Score.Skipped:
+			}
+			meanTokenSum += c.meanTokens * float64(max(c.scored, 1))
+			if c.errorAll {
+				ms.errors += c.attempts
+				continue
+			}
+			if c.skippedAll || c.scored == 0 {
 				continue
 			}
 
-			scoreSum += r.Score.Value
+			v := c.result.Score.Value
+			scoreSum += v
 			scoredCount++
-			latencySum += r.Latency
+			latencySum += c.result.Latency
 			latencyCount++
-			completionTokenSum += r.CompletionTokens
+			latencySecondsSum += c.result.Latency.Seconds()
-			latencySecondsSum += r.Latency.Seconds()
 
 			switch {
-			case r.Score.Value >= passThreshold:
+			case v >= passThreshold:
 				ms.passed++
-			case r.Score.Value <= 0:
+			case v <= 0:
 				ms.failed++
 			default:
 				ms.partial++
 			}
 		}
 
+		ms.totalTokens = int(math.Round(meanTokenSum))
 		if scoredCount > 0 {
 			ms.overallMean = scoreSum / float64(scoredCount)
 			ms.meanLatency = latencySum / time.Duration(latencyCount)
@@ -241,11 +353,11 @@ func summarize(models []string, results []runner.Result) []modelSummary {
 			ms.overallMean = math.NaN()
 		}
 
-		// Aggregate ratio (sum of completion tokens over sum of elapsed
+		// Aggregate ratio (completion tokens over elapsed seconds for the
-		// seconds), not a mean of per-test ratios: robust to a handful of
+		// representative attempt), not a mean of per-test ratios: robust to
-		// very short responses skewing a per-test-averaged rate.
+		// a handful of very short responses skewing a per-test-averaged rate.
 		if latencySecondsSum > 0 {
-			ms.avgTokPerSec = float64(completionTokenSum) / latencySecondsSum
+			ms.avgTokPerSec = float64(ms.totalTokens) / latencySecondsSum
 		} else {
 			ms.avgTokPerSec = math.NaN()
 		}

internal/report/report_test.go

@@ -10,6 +10,7 @@ import (
 	"time"
 
 	"github.com/lukaszraczylo/llm-testbench/internal/eval"
+	"github.com/lukaszraczylo/llm-testbench/internal/llm"
 	"github.com/lukaszraczylo/llm-testbench/internal/runner"
 	"github.com/lukaszraczylo/llm-testbench/internal/testkit"
 )
@@ -33,6 +34,47 @@ func sampleResults() []runner.Result {
 	}
 }
 
+// Truncation policy: attempts cut off by the token budget are excluded
+// from a cell's mean when any attempt finished normally; the cell is still
+// flagged "!" so the discarded sample is visible. When every scored
+// attempt was truncated, the mean falls back to the partial-text scores.
+func TestIndexResults_TruncatedAttemptExcludedFromMean(t *testing.T) {
+	idx := indexResults([]runner.Result{
+		{Model: "m1", TestID: "t", Attempt: 0, Score: eval.Score{Value: 1}, PromptTokens: 4},
+		{Model: "m1", TestID: "t", Attempt: 1, Score: eval.Score{Value: 0}, FinishReason: llm.FinishReasonLength, PromptTokens: 100},
+	})
+	c := idx["t"]["m1"]
+	if got := c.result.Score.Value; got != 1 {
+		t.Errorf("mean = %v, want 1 (clean attempts only)", got)
+	}
+	if !c.result.Truncated() {
+		t.Error("cell should be flagged truncated")
+	}
+	if c.minScore != 1 || c.maxScore != 1 {
+		t.Errorf("min/max = %v/%v, want 1/1 (truncated attempt excluded from instability range)", c.minScore, c.maxScore)
+	}
+	if got := int(c.meanTokens); got != 4 {
+		t.Errorf("meanTokens = %d, want 4 (clean attempts only)", got)
+	}
+}
+
+func TestIndexResults_AllAttemptsTruncatedFallBackToPartialScores(t *testing.T) {
+	idx := indexResults([]runner.Result{
+		{Model: "m1", TestID: "t", Attempt: 0, Score: eval.Score{Value: 0.4}, FinishReason: llm.FinishReasonLength, PromptTokens: 10},
+		{Model: "m1", TestID: "t", Attempt: 1, Score: eval.Score{Value: 0.6}, FinishReason: llm.FinishReasonLength, PromptTokens: 20},
+	})
+	c := idx["t"]["m1"]
+	if got := c.result.Score.Value; math.Abs(got-0.5) > 1e-9 {
+		t.Errorf("mean = %v, want 0.5 (fallback over all-truncated attempts)", got)
+	}
+	if !c.result.Truncated() {
+		t.Error("cell should be flagged truncated")
+	}
+	if c.scored != 2 {
+		t.Errorf("scored = %d, want 2", c.scored)
+	}
+}
+
 func TestRenderTable_ContainsExpectedCells(t *testing.T) {
 	var buf bytes.Buffer
 	if err := Render(&buf, FormatTable, sampleTests(), []string{"m1", "m2"}, sampleResults()); err != nil {
@@ -85,10 +127,11 @@ func TestRenderJSON_DumpsRawResults(t *testing.T) {
 		t.Fatalf("Render() error = %v", err)
 	}
 
-	var out []jsonResult
+	var doc jsonArtifact
-	if err := json.Unmarshal(buf.Bytes(), &out); err != nil {
+	if err := json.Unmarshal(buf.Bytes(), &doc); err != nil {
 		t.Fatalf("json.Unmarshal() error = %v; output:\n%s", err, buf.String())
 	}
+	out := doc.Results
 	if len(out) != len(sampleResults()) {
 		t.Fatalf("len(out) = %d, want %d", len(out), len(sampleResults()))
 	}
@@ -173,14 +216,14 @@ func TestRenderJSON_IncludesResponseTextFinishReasonAndTruncated(t *testing.T) {
 		t.Fatalf("Render() error = %v", err)
 	}
 
-	var out []jsonResult
+	var doc jsonArtifact
-	if err := json.Unmarshal(buf.Bytes(), &out); err != nil {
+	if err := json.Unmarshal(buf.Bytes(), &doc); err != nil {
 		t.Fatalf("json.Unmarshal() error = %v; output:\n%s", err, buf.String())
 	}
-	if len(out) != 1 {
+	if len(doc.Results) != 1 {
-		t.Fatalf("len(out) = %d, want 1", len(out))
+		t.Fatalf("len(results) = %d, want 1", len(doc.Results))
 	}
-	got := out[0]
+	got := doc.Results[0]
 	if got.ResponseText != "partial answer" {
 		t.Errorf("ResponseText = %q, want %q", got.ResponseText, "partial answer")
 	}
@@ -211,7 +254,7 @@ func TestSummarize_Buckets(t *testing.T) {
 		{Model: "m1", TestID: "d", Err: errors.New("fail")},
 		{Model: "m1", TestID: "e", Score: eval.Score{Skipped: true}},
 	}
-	summaries := summarize([]string{"m1"}, results)
+	summaries := summarize([]string{"m1"}, indexResults(results))
 	if len(summaries) != 1 {
 		t.Fatalf("summarize() len = %d, want 1", len(summaries))
 	}
@@ -236,7 +279,7 @@ func TestSummarize_AvgTokPerSec(t *testing.T) {
 		{Model: "m1", TestID: "err", Err: errors.New("fail"), Latency: time.Second, CompletionTokens: 999},
 		{Model: "m1", TestID: "skipped", Score: eval.Score{Skipped: true}, Latency: time.Second, CompletionTokens: 999},
 	}
-	summaries := summarize([]string{"m1"}, results)
+	summaries := summarize([]string{"m1"}, indexResults(results))
 	ms := summaries[0]
 
 	want := 110.0 / 2.1
@@ -250,7 +293,7 @@ func TestSummarize_AvgTokPerSec_NoDataIsNaN(t *testing.T) {
 		{Model: "m1", TestID: "err", Err: errors.New("fail")},
 		{Model: "m1", TestID: "skipped", Score: eval.Score{Skipped: true}},
 	}
-	summaries := summarize([]string{"m1"}, results)
+	summaries := summarize([]string{"m1"}, indexResults(results))
 	if !math.IsNaN(summaries[0].avgTokPerSec) {
 		t.Errorf("avgTokPerSec = %v, want NaN when there is no timed data", summaries[0].avgTokPerSec)
 	}

internal/report/table.go

@@ -26,8 +26,8 @@ func (ew *errWriter) println(args ...any) {
 	_, ew.err = fmt.Fprintln(ew.w, args...)
 }
 
-// renderTable writes three plain-text tables: per-test scores, category
+// renderTable writes four plain-text tables: per-test scores, category
-// rollup, and model summary.
+// rollup, discrimination/stability rollup, and model summary.
 func renderTable(w io.Writer, tests []testkit.Test, models []string, results []runner.Result) error {
 	idx := indexResults(results)
 	tw := tabwriter.NewWriter(w, 0, 4, 2, ' ', 0)
@@ -37,7 +37,9 @@ func renderTable(w io.Writer, tests []testkit.Test, models []string, results []r
 	ew.println()
 	writeCategoryTable(ew, tests, models, idx)
 	ew.println()
-	writeSummaryTable(ew, models, results)
+	writeDiscriminationTable(ew, testStats(tests, models, idx), len(models) > 1)
+	ew.println()
+	writeSummaryTable(ew, models, idx)
 	if ew.err != nil {
 		return ew.err
 	}
@@ -45,7 +47,7 @@ func renderTable(w io.Writer, tests []testkit.Test, models []string, results []r
 	return tw.Flush()
 }
 
-func writeScoreTable(ew *errWriter, tests []testkit.Test, models []string, idx resultIndex) {
+func writeScoreTable(ew *errWriter, tests []testkit.Test, models []string, idx cellIndex) {
 	ew.println("== Per-test scores ==")
 	header := []string{"TEST", "CATEGORY", "SUBCATEGORY"}
 	header = append(header, models...)
@@ -55,9 +57,9 @@ func writeScoreTable(ew *errWriter, tests []testkit.Test, models []string, idx r
 	for _, t := range sortedTests(tests) {
 		row := []string{t.ID, t.Category, t.Subcategory}
 		for _, model := range models {
-			r, ok := idx[t.ID][model]
+			c := idx[t.ID][model]
-			row = append(row, cellText(r, ok))
+			row = append(row, cellText(c))
-			anyTruncated = anyTruncated || r.Truncated()
+			anyTruncated = anyTruncated || c.result.Truncated()
 		}
 		ew.println(strings.Join(row, "\t"))
 	}
@@ -66,7 +68,7 @@ func writeScoreTable(ew *errWriter, tests []testkit.Test, models []string, idx r
 	}
 }
 
-func writeCategoryTable(ew *errWriter, tests []testkit.Test, models []string, idx resultIndex) {
+func writeCategoryTable(ew *errWriter, tests []testkit.Test, models []string, idx cellIndex) {
 	ew.println("== Category rollup (mean score) ==")
 	header := append([]string{"CATEGORY"}, models...)
 	ew.println(strings.Join(header, "\t"))
@@ -80,11 +82,82 @@ func writeCategoryTable(ew *errWriter, tests []testkit.Test, models []string, id
 	}
 }
 
-func writeSummaryTable(ew *errWriter, models []string, results []runner.Result) {
+// writeDiscriminationTable lists the tests that actually measure
+// something: cross-model spread >= DiscriminationSpread, or attempts that
+// disagreed under --repeat. With a single model there is no spread to
+// speak of, so the non-passing tests are listed instead.
+func writeDiscriminationTable(ew *errWriter, stats []testStat, multiModel bool) {
+	writeDiscriminationRows(tablePrinter{ew: ew}, stats, multiModel)
+}
+
+// discriminationSink abstracts the tab (table) vs pipe (markdown) output
+// so the discrimination section is written once.
+type discriminationSink interface {
+	header(cells ...string)
+	row(cells ...string)
+	note(text string)
+}
+
+type tablePrinter struct{ ew *errWriter }
+
+func (tp tablePrinter) header(cells ...string) { tp.ew.println(strings.Join(cells, "\t")) }
+func (tp tablePrinter) row(cells ...string)    { tp.ew.println(strings.Join(cells, "\t")) }
+func (tp tablePrinter) note(text string)       { tp.ew.println(text) }
+
+func writeDiscriminationRows(sink discriminationSink, stats []testStat, multiModel bool) {
+	discriminating, unstable, scored := 0, 0, 0
+	for _, st := range stats {
+		if !st.scored {
+			continue
+		}
+		scored++
+		if st.unstable {
+			unstable++
+		}
+		if multiModel && !math.IsNaN(st.spread) && st.spread >= DiscriminationSpread {
+			discriminating++
+		}
+	}
+
+	if multiModel {
+		sink.note(fmt.Sprintf("== Discrimination: %d/%d scored tests separate models by >= %.2f; %d unstable under repeat ==",
+			discriminating, scored, DiscriminationSpread, unstable))
+	} else {
+		sink.note(fmt.Sprintf("== Discrimination: single model (no cross-model spread); %d unstable under repeat ==", unstable))
+	}
+
+	rows := interestingStats(stats, multiModel)
+	if len(rows) == 0 {
+		sink.note("no discriminating or unstable tests: every scored test passed identically for every model")
+		return
+	}
+
+	sink.header("TEST", "CATEGORY", "SUBCATEGORY", "MEAN", "SPREAD", "UNSTABLE", "TRUNC")
+	for _, st := range rows {
+		sink.row(
+			st.testID,
+			st.category,
+			st.subcategory,
+			meanCellText(st.mean),
+			meanCellText(st.spread),
+			yesNo(st.unstable),
+			yesNo(st.truncated),
+		)
+	}
+}
+
+func yesNo(b bool) string {
+	if b {
+		return "yes"
+	}
+	return "-"
+}
+
+func writeSummaryTable(ew *errWriter, models []string, idx cellIndex) {
 	ew.println("== Model summary ==")
 	ew.println(strings.Join([]string{"MODEL", "MEAN", "PASSED", "PARTIAL", "FAILED", "ERRORS", "MEAN_LATENCY", "TOTAL_TOKENS", "TOK/S"}, "\t"))
 
-	for _, ms := range summarize(models, results) {
+	for _, ms := range summarize(models, idx) {
 		row := []string{
 			ms.model,
 			meanCellText(ms.overallMean),

internal/runner/runner.go

@@ -20,7 +20,9 @@ import (
 // referencing FinishReason separately.
 const truncatedDetailPrefix = "TRUNCATED: "
 
-// Result is the outcome of running one test against one model.
+// Result is the outcome of running one test against one model. Attempt is
+// the zero-based index of this sample when Config.Repeat > 1; it is 0 for
+// single-sample runs.
 type Result struct {
 	Err              error
 	Model            string
@@ -31,6 +33,7 @@ type Result struct {
 	Latency          time.Duration
 	PromptTokens     int
 	CompletionTokens int
+	Attempt          int
 }
 
 // Truncated reports whether the model's response was cut off by the token
@@ -54,6 +57,12 @@ type Config struct {
 	Concurrency      int
 	Temperature      float64
 	MaxTokensDefault int
+	// Repeat runs each (model, test) combination this many times (minimum
+	// 1). Even at temperature 0, reasoning models sample differently per
+	// call; repeats expose that instability so a score can be trusted
+	// (stable across attempts) or flagged (flaky) instead of silently
+	// treated as a capability measurement.
+	Repeat int
 }
 
 // Runner executes a set of tests against a set of models.
@@ -71,18 +80,24 @@ func New(client llm.Client, cfg Config) *Runner {
 }
 
 // Run executes every combination of models x tests, bounded by
-// cfg.Concurrency in-flight calls, and returns one Result per combination.
+// cfg.Concurrency in-flight calls, and returns one Result per combination
-// A per-call failure is captured in Result.Err rather than aborting the run.
+// (times Config.Repeat samples each). A per-call failure is captured in
+// Result.Err rather than aborting the run.
 func (r *Runner) Run(ctx context.Context, models []string, tests []testkit.Test) []Result {
 	type job struct {
-		model string
+		model   string
-		test  testkit.Test
+		test    testkit.Test
+		attempt int
 	}
 
-	jobs := make([]job, 0, len(models)*len(tests))
+	repeat := max(r.cfg.Repeat, 1)
+
+	jobs := make([]job, 0, len(models)*len(tests)*repeat)
 	for _, m := range models {
 		for _, t := range tests {
-			jobs = append(jobs, job{model: m, test: t})
+			for a := range repeat {
+				jobs = append(jobs, job{model: m, test: t, attempt: a})
+			}
 		}
 	}
 
@@ -103,6 +118,7 @@ func (r *Runner) Run(ctx context.Context, models []string, tests []testkit.Test)
 	for i, j := range jobs {
 		g.Go(func() error {
 			res := r.runOne(gctx, j.model, j.test)
+			res.Attempt = j.attempt
 			results[i] = res
 			// atomic.AddInt32 makes the completion counter safe under
 			// Config.Concurrency-many concurrent goroutines; ReportDone

internal/runner/runner_test.go

@@ -377,3 +377,36 @@ func TestRunner_Run_TracksLatencyAndTokens(t *testing.T) {
 		t.Errorf("Latency = %v, want 42ms", results[0].Latency)
 	}
 }
+
+func TestRunner_Run_RepeatSamplesEachCombination(t *testing.T) {
+	client := &mockClient{}
+	r := New(client, Config{Concurrency: 2, MaxTokensDefault: 100, Repeat: 3})
+
+	tests := []testkit.Test{{ID: "t1", Category: "c", Prompt: "p1", Eval: echoEval()}}
+	models := []string{"model-a", "model-b"}
+
+	results := r.Run(context.Background(), models, tests)
+
+	if len(results) != 6 {
+		t.Fatalf("len(results) = %d, want 6 (2 models x 1 test x 3 repeats)", len(results))
+	}
+	seen := make(map[[2]string][]int)
+	for _, res := range results {
+		k := [2]string{res.Model, res.TestID}
+		seen[k] = append(seen[k], res.Attempt)
+	}
+	for k, attempts := range seen {
+		if len(attempts) != 3 {
+			t.Errorf("%v: %d attempts, want 3", k, len(attempts))
+		}
+		got := map[int]bool{}
+		for _, a := range attempts {
+			got[a] = true
+		}
+		for a := range 3 {
+			if !got[a] {
+				t.Errorf("%v: missing attempt %d (have %v)", k, a, attempts)
+			}
+		}
+	}
+}

internal/tests/catalog.go

@@ -23,5 +23,8 @@ func All() *testkit.Registry {
 	registerSecurityTests(r)
 	registerDeliveryTests(r)
 	registerAITests(r)
+	registerHardTests(r)
+	registerScenarioTests(r)
+	registerSaturationTests(r)
 	return r
 }

internal/tests/catalog_test.go

@@ -64,8 +64,8 @@ func TestAll_SubcategoryMinimums(t *testing.T) {
 	const minTestsPerSubcategory = 10
 
 	wantSubcats := map[string][]string{
-		"programming": {"golang", "python", "typescript", "c"},
+		"programming": {"golang", "python", "typescript", "c", "hard"},
-		"operations":  {"macos", "linux", "kubernetes"},
+		"operations":  {"macos", "linux", "kubernetes", "scenario"},
 		"research":    {"web", "whitepapers", "codebase"},
 		"agents":      {"tool-routing", "planning", "delegation"},
 		"databases":   {"postgres", "redis", "sql-tuning"},

internal/tests/databases_redis.go

@@ -159,27 +159,58 @@ pipelined commands? Respond with only "yes" or "no".`
 	}
 }
 
-// dbKeysCommandPattern matches an explicit mention of the Redis KEYS
+// dbKeysEndorsementPattern matches a KEYS mention in an endorsement-shaped
-// command actually being invoked - "KEYS *" (its most common prod-danger
+// position — a verb recommending it ("use `KEYS`", "just run KEYS *"), a
-// form) or a backtick-quoted `KEYS` - not a generic use of the English
+// positive predicate ("KEYS * is fine/simplest"), or a bare imperative line
-// word "keys" elsewhere in prose ("the keys to fixing this...").
+// ("run this:\nKEYS *") — rather than every mention. The guard then still
-var dbKeysCommandPattern = regexp.MustCompile("(?:`KEYS`|\\bKEYS\\s*\\*)")
+// requires a negation cue near such a mention (see dbNoBareKeysInProd).
+//
+// Triggering on any mention is too strict: a fully correct live answer
+// (probe, 2026-08-29) opens with "`KEYS *` is dangerous..." and later says
+// "so `KEYS *` can block the server while it walks through millions of
+// keys" — harm descriptions whose words ("can block") sit outside the
+// negation cue list, yet they are exactly the explanation the prompt asks
+// for, not an endorsement.
+//
+// The predicate branch's leading (?:^|[^...]) guard stops a hyphen-joined
+// identifier from matching mid-word and keeps the match start at the term
+// area rather than arbitrary prose.
+var dbKeysEndorsementPattern = regexp.MustCompile("(?i)(?:" +
+
+	// a verb recommending it: "use `KEYS`", "just run KEYS *", "stick with KEYS *"
+
+	"(?:use|using|used|run|running|runs|execute|executing|call|calling|should(?:\\s+(?:also|just))?|better\\s+to|go\\s+with|opt\\s+for|stick\\s+with|switch(?:ing)?\\s+(?:back\\s+)?to)\\s+(?:the\\s+)?(?:redis\\s+)?(?:command\\s+)?(?:`KEYS`|\\bKEYS\\s*\\*)" +
+
+	// a positive predicate about it: "KEYS * is fine", "`KEYS` is simplest"
+
+	"|(?:^|[^A-Za-z0-9_-])(?:`KEYS`|\\bKEYS\\s*\\*)[^.;\\n]{0,30}?\\b(?:is|are|was|were|seems?|looks?|works?|stays?|remains?|would\\s+be|should\\s+be)\\s+(?:a\\s+|just\\s+|totally\\s+|still\\s+|actually\\s+)?(?:fine|good|great|safe|ok|okay|acceptable|simplest|easiest|better|fast\\s+enough)" +
+
+	// a bare command line after a colon or full stop: "run this:\\nKEYS *" -
+
+	// requiring the break keeps a line-initial mention that merely
+
+	// continues a wrapped sentence from triggering.
+
+	"|(?:[.:]|^)[ \\t]*(?:\\r?\\n)[ \\t*`>-]*(?:`KEYS`|\\bKEYS\\s*\\*)" +
+
+	")")
 
 // dbNegationWindow is how many characters around a KEYS-command mention
 // are searched for a negation cue.
 const dbNegationWindow = 60
 
-// dbNoBareKeysInProd scores full credit unless the response instructs
+// dbNoBareKeysInProd scores full credit unless the response endorses
-// running "KEYS *" against a production instance without any negating
+// running "KEYS *" against a production instance (see
-// context (a warning not to, or an instruction to use SCAN instead is
+// dbKeysEndorsementPattern) with no negating context nearby — a warning
-// fine). This is deliberately not eval.NotContains, which would also zero
+// not to, or an instruction to use SCAN instead, is fine. This is
-// out the best possible answer: one that correctly explains why NOT to run
+// deliberately not eval.NotContains, which would also zero out the best
-// KEYS in production.
+// possible answer: one that correctly explains why NOT to run KEYS in
+// production.
 //
 // Delegates to eval.NoUnnegatedMention (D5), the primitive shared with
 // kubernetes.go, security.go, and delivery_git.go's equivalent guards.
 func dbNoBareKeysInProd() eval.Evaluator {
-	return eval.NoUnnegatedMention(dbKeysCommandPattern, dbNegationWindow, nil)
+	return eval.NoUnnegatedMention(dbKeysEndorsementPattern, dbNegationWindow, nil)
 }
 
 // dbRedisScanVsKeysTest: require SCAN over KEYS for a hot-path production

internal/tests/databases_redis_test.go

@@ -147,10 +147,33 @@ func TestDBNoBareKeysInProd(t *testing.T) {
 			want:     0,
 		},
 		{
-			name:     "negation cue too far away to count",
+			// The guard now triggers on an endorsement-shaped mention only,
+			// and the negation window is measured from the endorsement
+			// phrase ("...run KEYS *"), not from the sentence start — so a
+			// hedging "never mind" that is still within 60 chars counts.
+			name:     "negation cue close to the endorsement phrase",
 			response: "Never mind the migration for now, but you could casually run KEYS * to check this quickly.",
+			want:     1,
+		},
+		{
+			// ground truth: same endorsement, but the cue is now >60 chars
+			// from "run KEYS *" (73), outside the window - still an
+			// unnegated recommendation.
+			name:     "negation cue too far away to count",
+			response: "Never mind everything else that was said in the earlier review about migrations, but you could casually run KEYS * to check this quickly.",
 			want:     0,
 		},
+		{
+			// Regression from the live probe (2026-08-29): a fully correct
+			// answer describes the harm ("can block", no cue word in the
+			// list) and only negates one mention ("avoid `KEYS *` unless").
+			// None of these is an endorsement, so the guard must stay at 1.
+			name: "correct probe answer: harm description without cue words",
+			response: "`KEYS *` is dangerous in production because it scans the entire keyspace synchronously.\n" +
+				"Redis is single-threaded for command execution, so `KEYS *` can block the server while it walks through millions of keys.\n" +
+				"For production code, use `SCAN` with a reasonable `COUNT`, and avoid `KEYS *` unless you are in development.",
+			want: 1,
+		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {

internal/tests/operations_scenario.go

@@ -0,0 +1,538 @@
+package tests
+
+import (
+	"context"
+	"encoding/json"
+	"strings"
+
+	"github.com/lukaszraczylo/llm-testbench/internal/eval"
+	"github.com/lukaszraczylo/llm-testbench/internal/testkit"
+)
+
+func registerScenarioTests(r *testkit.Registry) {
+	r.Register(scenDiskpressureFirstcmdTest())
+	r.Register(scenInodeExhaustionTest())
+	r.Register(scenCrashloopOrderTest())
+	r.Register(scenNetLayerIsolateTest())
+	r.Register(scenServiceDegradeTest())
+	r.Register(scenLogRootcauseMCQTest())
+	r.Register(scenBackupRestoreOrderTest())
+	r.Register(scenCertexpiryTriageTest())
+	r.Register(scenSystemdFailreasonTest())
+	r.Register(scenPortbindFailTest())
+	r.Register(scenLvmResizeOrderTest())
+}
+
+// scenDiskpressureDf is the inline df output for scenDiskpressureFirstcmdTest.
+const scenDiskpressureDf = `Filesystem      Size  Used Avail Use% Mounted on
+/dev/sda1       100G  100G    0G 100% /
+tmpfs           16G   1.1G   15G   8% /run
+/dev/sdb1       500G   30G  470G   6% /data`
+
+// scenFirstDiskCommand scores a response whose {"first_command"} field is a
+// safe, size-sorted du scan of the root filesystem. It demands the two
+// essential verb flags (du to measure recursive usage, sort to order the
+// result) and rejects a destructive first step (rm). A naive model that
+// jumps straight to deleting files, or that lists with ls (which does not
+// measure recursive usage) or find -size (which only matches known sizes),
+// scores zero. This is deliberately not a bare ContainsAny on the whole
+// response: the prompt forces a JSON object, and matching inside the
+// command field keeps the check grounded in an actual command rather than
+// prose that merely names du.
+func scenFirstDiskCommand() eval.Evaluator {
+	return eval.EvaluatorFunc(func(_ context.Context, response string) eval.Score {
+		raw, err := eval.ExtractJSON(response)
+		if err != nil {
+			return eval.Score{Value: 0, Detail: "no JSON object found"}
+		}
+		var obj map[string]string
+		if err := json.Unmarshal([]byte(raw), &obj); err != nil {
+			return eval.Score{Value: 0, Detail: "JSON not usable as a string map"}
+		}
+		cmd := strings.ToLower(strings.TrimSpace(obj["first_command"]))
+		if cmd == "" {
+			return eval.Score{Value: 0, Detail: "first_command empty"}
+		}
+		for _, req := range []string{"du", "sort"} {
+			if !strings.Contains(cmd, req) {
+				return eval.Score{Value: 0, Detail: "first_command missing " + req}
+			}
+		}
+		if strings.HasPrefix(cmd, "rm") || strings.Contains(cmd, " rm ") || strings.Contains(cmd, "rm -") {
+			return eval.Score{Value: 0, Detail: "first_command starts by deleting files, not measuring"}
+		}
+		return eval.Score{Value: 1, Detail: "first_command is a safe du|sort scan of the filesystem"}
+	})
+}
+
+// scenDiskpressureFirstcmdTest: give the exact one-line first command to
+// find what is filling a 100%-full root filesystem.
+//
+// ground truth: / is 100% full on /dev/sda1 while /data (/dev/sdb1) has
+// 470G free, so the scan must stay within the / filesystem and rank top-
+// level directories by size. The canonical, standard-OSS one-liner is
+// `du -xh --max-depth=1 / 2>/dev/null | sort -rh | head -20` - du with
+// -x to not descend into the nearly-empty /data mount, a depth of 1 to get
+// per-top-level-directory usage, sort -rh descending by human size, and
+// head to the top consumers. ls does not measure recursive usage and find
+// -size only matches known file sizes, so neither finds the fillers.
+func scenDiskpressureFirstcmdTest() testkit.Test {
+	prompt := `This Linux host's root filesystem /dev/sda1 is 100%
+full; /data on /dev/sdb1 has plenty of free space. You must find which
+paths under / are consuming the space, so you can free it. Here is the
+relevant df output:
+
+` + scenDiskpressureDf + `
+
+Give the exact one-line shell command you would run FIRST to identify the
+largest consumers under /, staying on the / filesystem and producing a
+size-sorted list. Respond with only a JSON object:
+{"first_command":"<the exact one-line command>"}`
+
+	return testkit.Test{
+		ID:          "scen-diskpressure-firstcmd",
+		Category:    "operations",
+		Subcategory: "scenario",
+		Description: "Give the exact first shell command (du|sort) to find what filled a 100%-full root filesystem.",
+		Prompt:      prompt,
+		Eval:        scenFirstDiskCommand(),
+	}
+}
+
+// scenInodeExhaustionDfHi is the inline df -hi output for
+// scenInodeExhaustionTest.
+const scenInodeExhaustionDfHi = `$ df -h /
+Filesystem      Size  Used Avail Use% Mounted on
+/dev/sda1       100G   35G   65G  35% /
+
+$ df -hi /
+Filesystem     Inodes IUsed IFree IUse% Mounted on
+/dev/sda1       1.0M  1.0M    0  100% /`
+
+// scenInodeExhaustionTest: diagnose "No space left on device" that is
+// actually inode exhaustion (blocks free, inodes exhausted).
+//
+// ground truth: writes fail with "No space left on device" even though
+// df -h shows 65G free, because df -hi shows the filesystem's inode
+// table is 100% used (1.0M of 1.0M). Inode exhaustion, not block
+// exhaustion, is the cause; the command that reveals it is `df -hi`
+// (df with the inode flag). The fix is deleting cheap small files
+// (often an unbounded cache dir filling the inode table with tiny files),
+// not growing the disk (which adds blocks, not inodes) or swapping mounts.
+func scenInodeExhaustionTest() testkit.Test {
+	prompt := `Deleting nothing and with the disk apparently not full,
+this host now fails writes with "No space left on device". Here are two
+diagnostic outputs:
+
+` + scenInodeExhaustionDfHi + `
+
+What is the cause, and which exact command produced the revealing
+output? Respond with only a JSON object:
+{"cause":"<one of: inodes-exhausted, disk-full, quota-exceeded>",
+"command":"<one of: df -hi, df -h, stat -f> "}`
+
+	evaluator := eval.Mean(
+		eval.JSONField("cause", "inodes-exhausted"),
+		eval.JSONField("command", "df -hi"),
+	)
+
+	return testkit.Test{
+		ID:          "scen-inode-exhaustion",
+		Category:    "operations",
+		Subcategory: "scenario",
+		Description: "Recognize ENOSPC from inode exhaustion (not disk space) and the df -hi command that reveals it.",
+		Prompt:      prompt,
+		Eval:        evaluator,
+	}
+}
+
+// scenCrashloopOrderTest: an incident where the operator must pick the
+// diagnostic steps to find why a CrashLoopBackOff container crashes,
+// before taking any action.
+//
+// ground truth: the only observation is the CrashLoopBackOff status and
+// the restart count - nothing about *why* the container dies. Both
+// read-only diagnostics are needed: B (`kubectl describe pod`, recent
+// events and the last exit reason - rules out OOM-kill, image problems,
+// probe failures) and A (`kubectl logs <pod> --previous`, the actual
+// error output of the crashed run - names the failing line). Neither
+// alone roots an app crash: describe shows the exit code but not the
+// error message; logs --previous show the error but not the node-level
+// events. Whether describe or logs comes first is a matter of taste, so
+// the evaluator scores the SET {A, B}. What the test discriminates is
+// diagnose-before-acting: C (`rollout restart`) and D (`scale
+// --replicas=0`) act without diagnosing, and E (ssh to a node for
+// journalctl) reaches for node-level tooling prematurely when the
+// container's own state and logs are one kubectl call away.
+func scenCrashloopOrderTest() testkit.Test {
+	prompt := `A pod named ingestion-5d7c9b6f8b-p2k1q in namespace data
+keeps crashing and restarting: kubectl get pods shows STATUS
+CrashLoopBackOff, RESTARTS 14. You know nothing else about why it dies,
+and you must find the root cause before changing anything. These are the
+available steps:
+
+A) kubectl logs <pod> --previous   (read the last crashed run's output)
+B) kubectl describe pod <pod>      (inspect events and last exit reason)
+C) kubectl rollout restart deployment ingestion
+D) kubectl scale deployment ingestion --replicas=0
+E) ssh to a node and run journalctl -u kubelet
+
+Respond with only a JSON array of the step letters that make up your
+diagnosis, containing exactly the diagnostic steps you would take to find
+the root cause (any order) - do not include steps that only act without
+diagnosing.`
+
+	return testkit.Test{
+		ID:          "scen-crashloop-order",
+		Category:    "operations",
+		Subcategory: "scenario",
+		Description: "Pick the diagnostic steps (describe + logs --previous) that root-cause a CrashLoopBackOff before acting.",
+		Prompt:      prompt,
+		Eval:        eval.JSONStringSet([]string{"A", "B"}),
+	}
+}
+
+// scenNetLayerSymptom is the inline diagnostic session for
+// scenNetLayerIsolateTest.
+const scenNetLayerSymptom = `$ curl -sS https://api.example.org/data
+curl: (6) Could not resolve host: api.example.org
+
+$ ping -c1 10.0.0.7
+64 bytes from 10.0.0.7: icmp_seq=1 ttl=64 time=0.8 ms
+(10.0.0.7 is a working internal host on the same subnet)
+
+$ cat /etc/resolv.conf
+nameserver 10.0.0.53`
+
+// scenNetLayerIsolateTest: isolate which network layer is broken from
+// inline symptoms and pick the strict layer among an enumerated set.
+//
+// ground truth: curl fails with "Could not resolve host" - DNS
+// resolution, not a routing or TCP problem (an ICMP packet to an internal
+// host on the same subnet succeeds, so IP reachability on the local
+// segment is fine, and curl never even reached TCP/TLS because it could
+// not translate the name). The nameserver 10.0.0.53 is configured but
+// the name does not resolve, so the failing layer is strictly dns; it is
+// not ip-routing, tcp-connect, tls, or http, none of which are reached
+// when resolution itself fails.
+func scenNetLayerIsolateTest() testkit.Test {
+	prompt := `A service on this host cannot reach https://api.example.org.
+Here is the diagnostic session:
+
+` + scenNetLayerSymptom + `
+
+At which network layer is the failure occurring? Respond with only a
+JSON object:
+{"layer":"<one of: dns, ip-routing, tcp-connect, tls, http>"}
+
+Choose exactly one layer from the enumerated list.`
+
+	return testkit.Test{
+		ID:          "scen-net-layer-isolate",
+		Category:    "operations",
+		Subcategory: "scenario",
+		Description: "Isolate a network failure to the DNS layer from inline curl/ping/resolv.conf symptoms.",
+		Prompt:      prompt,
+		Eval:        eval.JSONField("layer", "dns"),
+	}
+}
+
+// scenServiceDegradeContext is the inline telemetry + deploy timeline for
+// scenServiceDegradeTest.
+const scenServiceDegradeContext = `Canary deployed to 100% at 14:00: config file
+change lowering the app's connection-pool size (pool: 200 -> 50) and
+raising the worker count.
+
+Latency (p99) of the checkout service API:
+  13:30  210ms
+  14:05  1400ms
+  14:30  1650ms
+  15:00  1700ms
+
+Resource metrics for the checkout pods: CPU at 12%, memory at 40% -
+both low. No crash, no 5xx errors, no data loss.`
+
+// scenServiceDegradeTest: a service-degradation decision tree where the
+// regression is tracked to a config change with healthy resources.
+//
+// ground truth: the p99 latency jumped from ~210ms to ~1400ms exactly at
+// the 14:00 canary rollout, whose only meaningful change is the
+// connection-pool size drop (200 -> 50) - the obvious pattern match is the
+// config change, and CPU/memory are low so the pool is not saturated by
+// resource pressure. The correct action is to roll back / revert that
+// config change (rollback-config). scale-out is wrong (no CPU/memory
+// pressure to scale for), restart alone is wrong (it does not revert the
+// config), and restore-backup is wrong (no data loss to recover from).
+func scenServiceDegradeTest() testkit.Test {
+	prompt := `The checkout service API latency spiked and stayed high.
+Here is the context:
+
+` + scenServiceDegradeContext + `
+
+Which single action best restores service? Respond with only a JSON
+object:
+{"action":"<one of: rollback-config, scale-out, restore-backup, restart>"}`
+
+	return testkit.Test{
+		ID:          "scen-service-degrade",
+		Category:    "operations",
+		Subcategory: "scenario",
+		Description: "Pick the correct remediation (config rollback) for a canary-introduced service degradation with healthy resources.",
+		Prompt:      prompt,
+		Eval:        eval.JSONField("action", "rollback-config"),
+	}
+}
+
+// scenPgConnectionLog is the inline log excerpt for
+// scenLogRootcauseMCQTest.
+const scenPgConnectionLog = `2026-08-29 14:22:11 UTC FATAL:  remaining connection slots are reserved for non-replication superuser connections
+2026-08-29 14:22:11 UTC DETAIL:  1000 concurrent connections, 100 remaining connection slots available
+2026-08-29 14:22:12 UTC ERROR:  connection to server at "db.internal" (10.0.1.25), port 5432 failed: FATAL:  remaining connection slots are reserved for non-replication superuser connections`
+
+// scenLogRootcauseMCQTest: map a log pattern to its root cause from
+// lettered options.
+//
+// ground truth: "remaining connection slots are reserved" with a detail
+// line counting 1000 concurrent connections is the textbook PostgreSQL
+// signal that the connection slot limit (max_connections) has been
+// reached - applications are holding more connections than the server is
+// configured to allow. It is not a disk-full condition (no out-of-space
+// message), not a slow-query problem (no long query shown), and not an
+// authentication failure (no `password authentication failed` line).
+func scenLogRootcauseMCQTest() testkit.Test {
+	prompt := `Applications calling the Postgres database below now fail to
+connect. Here is a log excerpt:
+
+` + scenPgConnectionLog + `
+
+What is the root cause of these connection failures?
+
+A) connection pool exhausted (max_connections reached)
+B) disk full
+C) a single slow query blocking others
+D) authentication failure
+
+Respond with only a JSON object: {"answer":"<the single letter A-D>"}`
+
+	return testkit.Test{
+		ID:          "scen-log-rootcause-mcq",
+		Category:    "operations",
+		Subcategory: "scenario",
+		Description: "Map a PostgreSQL max_connections log pattern to its root cause via a lettered MCQ.",
+		Prompt:      prompt,
+		Eval:        eval.JSONField("answer", "A"),
+	}
+}
+
+// scenBackupRestoreOptions lists the available steps for
+// scenBackupRestoreOrderTest.
+const scenBackupRestoreOptions = `A) take a fresh logical backup (pg_dump) of the current data
+B) apply the destructive migration
+C) validate row counts and data after the migration
+D) restore last night's backup before doing anything else`
+
+// scenBackupRestoreOrderTest: order the steps for a safe data-changing
+// migration, guaranteeing recoverability.
+//
+// ground truth: to make a destructive migration recoverable you must
+// snapshot the CURRENT state first (A - a fresh pg_dump), then apply the
+// migration (B), then validate the result (C). Restoring last night's
+// backup (D) first is wrong: it is a stale snapshot that would throw
+// away today's writes before any change has even been made, and it cannot
+// be the safety net for the migration.
+func scenBackupRestoreOrderTest() testkit.Test {
+	prompt := `You are about to apply a destructive migration to a
+production Postgres database and must guarantee you can get back to the
+current state. The available steps are:
+
+` + scenBackupRestoreOptions + `
+
+Respond with only a JSON array of the step letters, in the order from
+first to last, that makes this migration recoverable - include exactly
+the steps you would run, in order.`
+
+	return testkit.Test{
+		ID:          "scen-backup-restore-order",
+		Category:    "operations",
+		Subcategory: "scenario",
+		Description: "Order the steps (fresh backup, then migrate, then validate) that make a destructive migration recoverable.",
+		Prompt:      prompt,
+		Eval:        eval.JSONStringArrayEquals([]string{"A", "B", "C"}),
+	}
+}
+
+// scenCertExpirySymptom is the inline symptom for
+// scenCertexpiryTriageTest.
+const scenCertExpirySymptom = `$ date
+Sat Aug 29 09:14:02 UTC 2026
+
+$ curl -v https://gateway.internal/
+...
+* SSL connection using TLSv1.3
+* SSL certificate verify result: certificate has expired
+
+$ openssl x509 -in /etc/ssl/svc.crt -noout -dates
+notBefore=Aug  1 12:00:00 2025 GMT
+notAfter=Aug  1 12:00:00 2026 GMT`
+
+// scenCertexpiryTriageTest: triage a TLS failure whose cause is an
+// expired certificate.
+//
+// ground truth: `date` anchors wall time at Aug 29 2026; the openssl
+// -dates readout shows the served cert valid until Aug 1 2026 - the
+// current date is past notAfter, so the certificate expired and the
+// cause category is cert-expired. Clock-skew would need the host clock
+// to disagree with reality, but the cert's own window explains the
+// failure exactly, and cipher-mismatch would show a handshake/algo error,
+// not a verify result. The standard command to inspect a certificate's
+// validity end date directly is openssl x509 -noout -enddate (the
+// readout above came from the -dates variant of the same inspection).
+func scenCertexpiryTriageTest() testkit.Test {
+	prompt := `Clients now fail to connect to gateway.internal with a TLS
+verification error. Here is the diagnostic session:
+
+` + scenCertExpirySymptom + `
+
+What is the cause, and which standard command inspects a certificate's
+validity directly? Respond with only a JSON object:
+{"cause":"<one of: cert-expired, clock-skew, cipher-mismatch>",
+"check_command":"<one of: openssl-x509-enddate, date, openssl-cipher>"}`
+
+	evaluator := eval.Mean(
+		eval.JSONField("cause", "cert-expired"),
+		eval.JSONField("check_command", "openssl-x509-enddate"),
+	)
+
+	return testkit.Test{
+		ID:          "scen-certexpiry-triage",
+		Category:    "operations",
+		Subcategory: "scenario",
+		Description: "Identify an expired certificate as the TLS failure cause and the openssl command that inspects validity.",
+		Prompt:      prompt,
+		Eval:        evaluator,
+	}
+}
+
+// scenSystemdStatus is the inline systemctl status excerpt for
+// scenSystemdFailreasonTest.
+const scenSystemdStatus = `$ systemctl status worker-queue
+● worker-queue.service - Queue worker
+     Loaded: loaded (/etc/systemd/system/worker-queue.service; enabled)
+     Active: failed (Result: exit-code) since Tue 2026-08-25 09:12:03 UTC; 12s ago
+   Main PID: 4821 (code=exited, status=1/FAILURE)
+        CPU: 120ms
+
+$ journalctl -u worker-queue -n 6
+Aug 25 09:12:03 host worker-queue[4821]: config/openapi.yaml: open: no such file or directory
+Aug 25 09:12:03 host worker-queue[4821]: fatal: failed to load configuration
+Aug 25 09:12:03 host worker-queue[4821]: exiting after startup failure`
+
+// scenSystemdFailreasonTest: reason about why a systemd unit failed from
+// an inline status + journal excerpt.
+//
+// ground truth: the journal line "config/openapi.yaml: open: no such
+// file or directory" - the binary exits because its configuration file
+// cannot be found at the expected path. The category is missing-config,
+// not permission (no "Permission denied"), not missing-binary (the
+// binary ran and printed a config-diagnostic trace), and not port-in-use
+// (no bind error). The fix is to provide or point the unit at the
+// correct config path, not to reinstall the binary or chase a port.
+func scenSystemdFailreasonTest() testkit.Test {
+	prompt := `A systemd unit is failing. Here is the status and journal:
+
+` + scenSystemdStatus + `
+
+What category of failure is this? Respond with only a JSON object:
+{"category":"<one of: missing-config, permission, missing-binary, port-in-use>"}
+
+Choose exactly one category from the enumerated list.`
+
+	return testkit.Test{
+		ID:          "scen-systemd-failreason",
+		Category:    "operations",
+		Subcategory: "scenario",
+		Description: "Classify a systemd unit failure (missing config file) from inline status and journal output.",
+		Prompt:      prompt,
+		Eval:        eval.JSONField("category", "missing-config"),
+	}
+}
+
+// scenPortbindError is the inline startup error for scenPortbindFailTest.
+const scenPortbindError = `$ service listener start
+(1) listen tcp :8080: bind: address already in use
+listener: could not bind to 0.0.0.0:8080, exiting
+
+$ ss -ltnp
+State    Recv-Q   Send-Q   Local Address:Port   Peer Address:Port  Process
+LISTEN   0      128    0.0.0.0:8080          0.0.0.0:*          users:(("listener",pid=1044,fd=3))`
+
+// scenPortbindFailTest: diagnose an "address already in use" bind
+// failure and the command that identifies the occupying process.
+//
+// ground truth: the startup error "bind: address already in use" for
+// port 8080, and the `ss -ltnp` output shows an existing listener (pid
+// 1044) already bound to 0.0.0.0:8080 - a duplicate/lingering instance
+// still holds the port. The cause category is port-in-use, and the
+// standard command that lists listening sockets with the owning process
+// is `ss -ltnp` (ss with listener, numerical, port and process flags);
+// netstat is the older equivalent but ss is the modern standard on this
+// home-cluster stack.
+func scenPortbindFailTest() testkit.Test {
+	prompt := `A service fails to start. Here is the output:
+
+` + scenPortbindError + `
+
+What is the cause, and which command identified the process holding the
+port? Respond with only a JSON object:
+{"cause":"<one of: port-in-use, missing-binary, config-error>",
+"command":"<one of: ss -ltnp, netstat -r, lsblk>"}`
+
+	evaluator := eval.Mean(
+		eval.JSONField("cause", "port-in-use"),
+		eval.JSONField("command", "ss -ltnp"),
+	)
+
+	return testkit.Test{
+		ID:          "scen-portbind-fail",
+		Category:    "operations",
+		Subcategory: "scenario",
+		Description: "Diagnose an address-in-use bind failure and the ss -ltnp command that reveals the holding process.",
+		Prompt:      prompt,
+		Eval:        evaluator,
+	}
+}
+
+// scenLvmSteps lists the LVM extend steps for scenLvmResizeOrderTest.
+const scenLvmSteps = `A) lvextend -L +20G /dev/vg0/data        (grow the logical volume)
+B) pvs                                     (check free space in the volume group)
+C) resize2fs /dev/vg0/data                 (grow the filesystem to fill the LV)
+D) umount /dev/vg0/data                    (unmount the volume)`
+
+// scenLvmResizeOrderTest: order the steps of a safe LVM logical-volume
+// resize.
+//
+// ground truth: to extend a filesystem on LVM you first confirm there is
+// free space in the volume group (B - `pvs`), then extend the logical
+// volume (A - `lvextend`), then grow the filesystem to fill the newly
+// added space (C - `resize2fs`). Doing C before A, or A before
+// confirming free space in B, fails (nothing to grow into); D (umount)
+// is not required for an online extend of a mounted ext4/xfs volume.
+func scenLvmResizeOrderTest() testkit.Test {
+	prompt := `You must safely grow the filesystem on LVM logical volume
+/dev/vg0/data while it stays online. The available steps are:
+
+` + scenLvmSteps + `
+
+Respond with only a JSON array of the step letters, in the order from
+first to last, that correctly extends this volume - include exactly the
+steps you would run, in order.`
+
+	return testkit.Test{
+		ID:          "scen-lvm-resize-order",
+		Category:    "operations",
+		Subcategory: "scenario",
+		Description: "Order the LVM resize steps (check VG space, extend LV, then grow the filesystem) for an online extend.",
+		Prompt:      prompt,
+		Eval:        eval.JSONStringArrayEquals([]string{"B", "A", "C"}),
+	}
+}

internal/tests/operations_scenario_test.go

@@ -0,0 +1,357 @@
+package tests
+
+import (
+	"context"
+	"testing"
+
+	"github.com/lukaszraczylo/llm-testbench/internal/testkit"
+)
+
+// TestRegisterScenarioTests_Wiring calls registerScenarioTests into a fresh
+// registry and checks it contributes a full operations/scenario layer with
+// scen-* IDs, mirroring the wiring-test pattern used for round-2
+// categories. It also keeps registerScenarioTests referenced so it is not
+// reported as dead code while the parent still wires catalog integration.
+func TestRegisterScenarioTests_Wiring(t *testing.T) {
+	r := testkit.NewRegistry()
+	registerScenarioTests(r)
+
+	if n := r.Len(); n < 10 || n > 12 {
+		t.Fatalf("registerScenarioTests registered %d tests, want 10-12", n)
+	}
+	for _, tc := range r.All() {
+		if tc.Category != "operations" {
+			t.Errorf("test %s has Category %q, want operations", tc.ID, tc.Category)
+		}
+		if tc.Subcategory != "scenario" {
+			t.Errorf("test %s has Subcategory %q, want scenario", tc.ID, tc.Subcategory)
+		}
+		if len(tc.ID) < 5 || tc.ID[:5] != "scen-" {
+			t.Errorf("test %s has an ID not prefixed scen-", tc.ID)
+		}
+	}
+}
+
+func TestScenDiskpressureFirstcmdTest_Eval(t *testing.T) {
+	tc := scenDiskpressureFirstcmdTest()
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{
+			name:     "correct du|sort one-liner",
+			response: `{"first_command":"du -xh --max-depth=1 / 2>/dev/null | sort -rh | head -20"}`,
+			want:     1,
+		},
+		{
+			name:     "correct fenced with prose",
+			response: "Here is the command:\n```json\n{\"first_command\":\"du -x --max-depth=1 / 2>/dev/null | sort -h\"}\n```",
+			want:     1,
+		},
+		{
+			name:     "wrong: ls does not measure recursive usage",
+			response: `{"first_command":"ls -lah /"}`,
+			want:     0,
+		},
+		{
+			name:     "wrong: destructive rm as first step",
+			response: `{"first_command":"rm -rf /var/log && rm -rf /tmp"}`,
+			want:     0,
+		},
+		{
+			name:     "wrong: du but no sort",
+			response: `{"first_command":"du -sh /var"}`,
+			want:     0,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}
+
+func TestScenInodeExhaustionTest_Eval(t *testing.T) {
+	tc := scenInodeExhaustionTest()
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{"correct", `{"cause":"inodes-exhausted","command":"df -hi"}`, 1},
+		{"correct fenced", "```json\n{\"cause\":\"inodes-exhausted\",\"command\":\"df -hi\"}\n```", 1},
+		{"wrong cause", `{"cause":"disk-full","command":"df -hi"}`, 0.5},
+		{"wrong command", `{"cause":"inodes-exhausted","command":"df -h"}`, 0.5},
+		{"both wrong", `{"cause":"quota-exceeded","command":"stat -f"}`, 0},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}
+
+func TestScenCrashloopOrderTest_Eval(t *testing.T) {
+	tc := scenCrashloopOrderTest()
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{"both steps", `["B","A"]`, 1},
+		{"both steps fenced", "```json\n[\"B\",\"A\"]\n```", 1},
+		{
+			// The correct answer discusses the rejected options (C as an
+			// unhelpful action, E as premature) at length; the JSON
+			// array still governs, so the prose cannot change the score.
+			name: "both steps with long discussion of rejected options",
+			response: "```json\n[\"B\",\"A\"]\n```\n\n`kubectl describe pod` (B) shows the last exit reason and `kubectl logs --previous` (A) shows the actual crash output; either may come first since both are read-only. " +
+				"A blind `kubectl rollout restart` (C) would not reveal the cause and `kubectl scale --replicas=0` (D) would only stop the workload; " +
+				"ssh'ing to a node for journalctl (E) is node-level tooling that is premature while the container logs are local.",
+			want: 1,
+		},
+		{"order is free", `["A","B"]`, 1},
+		{"only one diagnostic step", `["A"]`, 0.5},
+		// An action step in the answer costs credit: set overlap 2/3.
+		{"correct plus an action step", `["B","A","D"]`, 2.0 / 3.0},
+		{"all wrong", `["C","D","E"]`, 0},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}
+
+func TestScenNetLayerIsolateTest_Eval(t *testing.T) {
+	tc := scenNetLayerIsolateTest()
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{"correct", `{"layer":"dns"}`, 1},
+		{"correct fenced", "```json\n{\"layer\":\"dns\"}\n```", 1},
+		{
+			name: "correct, discussing other layers at length",
+			response: `{"layer":"dns"}
+
+This is not tls or http: curl never reached the handshake because it could not resolve the name at all; the local ping to 10.0.0.7 rules out ip-routing on the segment.`,
+			want: 1,
+		},
+		{"wrong layer", `{"layer":"tls"}`, 0},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}
+
+func TestScenServiceDegradeTest_Eval(t *testing.T) {
+	tc := scenServiceDegradeTest()
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{"correct", `{"action":"rollback-config"}`, 1},
+		{"correct fenced", "```json\n{\"action\":\"rollback-config\"}\n```", 1},
+		{
+			name: "correct, discussing rejected actions at length",
+			response: `{"action":"rollback-config"}
+
+Scaling out is wrong because CPU and memory are both low - there is no resource pressure to scale for; a restart would not revert the config file; and a backup restore is wrong because there is no data loss to recover from. The regression tracks the 14:00 canary pool-size change, so reverting the config is the fix.`,
+			want: 1,
+		},
+		{"wrong action", `{"action":"restart"}`, 0},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}
+
+func TestScenLogRootcauseMCQTest_Eval(t *testing.T) {
+	tc := scenLogRootcauseMCQTest()
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{"correct", `{"answer":"A"}`, 1},
+		{"correct fenced", "```json\n{\"answer\":\"A\"}\n```", 1},
+		{
+			name: "correct, discussing every distractor at length",
+			response: `{"answer":"A"}
+
+This is not disk full (there is no out-of-space message in the log), not a slow query (no long-running query is shown), and not an authentication failure (no password-authentication-failed line) - it is the classic max_connections exhaustion signal.`,
+			want: 1,
+		},
+		{"wrong letter", `{"answer":"C"}`, 0},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}
+
+func TestScenBackupRestoreOrderTest_Eval(t *testing.T) {
+	tc := scenBackupRestoreOrderTest()
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{"correct order", `["A","B","C"]`, 1},
+		{"correct order fenced", "```json\n[\"A\",\"B\",\"C\"]\n```", 1},
+		{
+			// The correct answer is allowed to explain at length why
+			// restoring last night's backup (D) first is wrong; the JSON
+			// array is what is scored.
+			name: "correct order, discussing rejected restore-first at length",
+			response: `["A","B","C"]
+
+Restoring last night's backup first (D) would throw away today's writes before any change is made - it is a stale snapshot, not a safety net for the migration, so I take a fresh pg_dump first.`,
+			want: 1,
+		},
+		{"wrong: restore first", `["D","A"]`, 0},
+		{"wrong: reordered", `["A","C","B"]`, 0},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}
+
+func TestScenCertexpiryTriageTest_Eval(t *testing.T) {
+	tc := scenCertexpiryTriageTest()
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{"correct", `{"cause":"cert-expired","check_command":"openssl-x509-enddate"}`, 1},
+		{"correct fenced", "```json\n{\"cause\":\"cert-expired\",\"check_command\":\"openssl-x509-enddate\"}\n```", 1},
+		{"wrong cause", `{"cause":"clock-skew","check_command":"openssl-x509-enddate"}`, 0.5},
+		{"wrong command", `{"cause":"cert-expired","check_command":"date"}`, 0.5},
+		{"both wrong", `{"cause":"cipher-mismatch","check_command":"openssl-cipher"}`, 0},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}
+
+func TestScenSystemdFailreasonTest_Eval(t *testing.T) {
+	tc := scenSystemdFailreasonTest()
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{"correct", `{"category":"missing-config"}`, 1},
+		{"correct fenced", "```json\n{\"category\":\"missing-config\"}\n```", 1},
+		{
+			name: "correct, discussing rejected categories at length",
+			response: `{"category":"missing-config"}
+
+This is not a permission problem (no Permission denied in the journal), not a missing binary (the worker ran and printed a config-diagnostic trace), and not a port in use (no bind error) - the unit fails because config/openapi.yaml cannot be opened.`,
+			want: 1,
+		},
+		{"wrong category", `{"category":"permission"}`, 0},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}
+
+func TestScenPortbindFailTest_Eval(t *testing.T) {
+	tc := scenPortbindFailTest()
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{"correct", `{"cause":"port-in-use","command":"ss -ltnp"}`, 1},
+		{"correct fenced", "```json\n{\"cause\":\"port-in-use\",\"command\":\"ss -ltnp\"}\n```", 1},
+		{"wrong cause", `{"cause":"config-error","command":"ss -ltnp"}`, 0.5},
+		{"wrong command", `{"cause":"port-in-use","command":"lsblk"}`, 0.5},
+		{"both wrong", `{"cause":"missing-binary","command":"netstat -r"}`, 0},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}
+
+func TestScenLvmResizeOrderTest_Eval(t *testing.T) {
+	tc := scenLvmResizeOrderTest()
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{"correct order", `["B","A","C"]`, 1},
+		{"correct order fenced", "```json\n[\"B\",\"A\",\"C\"]\n```", 1},
+		{"wrong: grows fs before extending LV", `["A","C","B"]`, 0},
+		{"wrong: includes umount", `["B","A","C","D"]`, 0},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}

internal/tests/programming_hard.go

@@ -0,0 +1,469 @@
+package tests
+
+import (
+	"github.com/lukaszraczylo/llm-testbench/internal/eval"
+	"github.com/lukaszraczylo/llm-testbench/internal/testkit"
+)
+
+// registerHardTests registers every programming/hard test. This subcategory
+// stresses harder daily-programming accuracy - concurrency/async reasoning,
+// subtle language traps, algorithm output tracing, and API-contract edge
+// cases - each forced to a short, machine-checkable answer.
+func registerHardTests(r *testkit.Registry) {
+	r.Register(hardFloatBinaddTest())
+	r.Register(hardNullishOverOrTest())
+	r.Register(hardGoroutineRaceTest())
+	r.Register(hardDeferNamedReturnTest())
+	r.Register(hardIntDivisionTest())
+	r.Register(hardRegexGreedyTest())
+	r.Register(hardPromiseAllSettledTest())
+	r.Register(hardContextTimeoutTest())
+	r.Register(hardJSONUnmarshalNullTest())
+	r.Register(hardVarShadowTest())
+	r.Register(hardPyVersionSortTest())
+	r.Register(hardGoParenBalanceTest())
+}
+
+// hardFloatBinaddTest: judge whether 0.1 + 0.2 === 0.3 holds in
+// JavaScript / IEEE-754 doubles.
+//
+// ground truth: neither 0.1 nor 0.2 has an exact binary representation,
+// so 0.1 + 0.2 computes to the nearest double to the mathematical sum,
+// which is 0.30000000000000004, not the double for the literal 0.3.
+// Therefore the strict equality comparison evaluates to false. This is a
+// canonical, deterministic fact of IEEE-754 double arithmetic, independent
+// of any runtime; verified with node during authoring.
+func hardFloatBinaddTest() testkit.Test {
+	prompt := `In JavaScript, evaluate this comparison:
+
+0.1 + 0.2 === 0.3
+
+Does the strict-equality comparison evaluate to true or to false?
+Respond with only a JSON object:
+{"equal": true}
+or
+{"equal": false}`
+
+	return testkit.Test{
+		ID:          "hard-float-binadd",
+		Category:    "programming",
+		Subcategory: "hard",
+		Description: "Judge IEEE-754 double equality for 0.1 + 0.2 === 0.3 in JavaScript (false).",
+		Prompt:      prompt,
+		Eval:        eval.JSONField("equal", false),
+	}
+}
+
+// hardNullishOverOrTest: trace the value of a nullish-coalescing
+// expression over a falsy (but non-null) operand.
+//
+// ground truth: ?? returns the right-hand operand only when the left-hand
+// operand is null or undefined. 0 is neither null nor undefined, so
+// n ?? 42 evaluates to 0 - unlike ||, which would have returned 42 for
+// the falsy 0. This is the core distinction between nullish coalescing
+// and logical OR. Verified with node during authoring.
+func hardNullishOverOrTest() testkit.Test {
+	prompt := `Consider this TypeScript code:
+
+const n = 0;
+const v = n ?? 42;
+
+What is the value of v?
+Respond with only a JSON object:
+{"value": ...}
+
+Give the numeric value only.`
+
+	return testkit.Test{
+		ID:          "hard-nullish-over-or",
+		Category:    "programming",
+		Subcategory: "hard",
+		Description: "Trace nullish-coalescing over a falsy-but-non-null operand: 0 ?? 42 is 0, not 42.",
+		Prompt:      prompt,
+		Eval:        eval.JSONField("value", 0),
+	}
+}
+
+// hardGoroutineRaceTest: judge whether an unsynchronized shared counter
+// is guaranteed to reach N*M exactly.
+//
+// ground truth: incrementing a shared int from multiple goroutines with no
+// synchronization is a data race (Go's race detector would flag it). A
+// read-modify-write of a shared variable is not atomic; two goroutines can
+// read the same value before either writes back, so increments are lost.
+// The final value is some value strictly between M (at best serialized,
+// one goroutine's work) and N*M, but it is NOT guaranteed to equal
+// N*M. This claim follows from the Go memory model / data-race semantics
+// and cannot be recomputed by running (the outcome is nondeterministic).
+func hardGoroutineRaceTest() testkit.Test {
+	prompt := `A Go program starts 8 goroutines, each of which performs:
+counter++
+1000 times, where counter is a shared int with NO synchronization
+(no mutex, no atomic) between the goroutines. The program then waits for
+all 8 goroutines and prints counter.
+
+Is counter's final value guaranteed to be exactly 8000?
+Respond with only a JSON object:
+{"guaranteed": true}
+or
+{"guaranteed": false}`
+
+	return testkit.Test{
+		ID:          "hard-goroutine-race",
+		Category:    "programming",
+		Subcategory: "hard",
+		Description: "Judge whether an unsynchronized shared counter is guaranteed to reach N*M (false: data race).",
+		Prompt:      prompt,
+		Eval:        eval.JSONField("guaranteed", false),
+	}
+}
+
+// hardDeferNamedReturnTest: trace the return value of a function whose
+// deferred closure writes its named return during a normal (non-panic)
+// return.
+//
+// ground truth: return 7 first assigns the named result r = 7; then,
+// still before the function returns to its caller, the deferred closure
+// runs and overwrites r = 99. The caller observes the named return after
+// all defers have run, so f() returns 99 - not 7 and not 5. This is the
+// classic named-return-plus-defer trap. Verified with go run during
+// authoring.
+func hardDeferNamedReturnTest() testkit.Test {
+	prompt := `Consider this Go code:
+
+func f() (r int) {
+	defer func() { r = 99 }()
+	r = 5
+	return 7
+}
+
+What value does f() return?
+Respond with only the number, no explanation.`
+
+	return testkit.Test{
+		ID:          "hard-defer-named-return",
+		Category:    "programming",
+		Subcategory: "hard",
+		Description: "Trace a deferred closure overwriting a named return value during a normal return (99).",
+		Prompt:      prompt,
+		Eval:        eval.Numeric(eval.ExtractLastNumber[int], hardDeferNamedReturnWant, 0),
+	}
+}
+
+// hardDeferNamedReturnWant is f()'s return value.
+//
+// ground truth: the deferred closure runs during the function's epilogue,
+// after the explicit `return 7` has set the named result r = 7, and
+// overwrites it with 99; the caller therefore receives 99 (see the full
+// derivation on hardDeferNamedReturnTest). Recomputed by running the
+// identical snippet with go run during authoring.
+const hardDeferNamedReturnWant = 99
+
+// hardIntDivisionTest: judge the result of integer division being
+// assigned to a floating-point variable.
+//
+// ground truth: a / b with two int operands performs integer division,
+// which truncates toward zero: 7 / 2 = 3 (not 3.5). That integral
+// result is then implicitly converted to double, so d = 3.0. The common
+// error is assuming floating division because the target is a double.
+// Verified with go run during authoring.
+func hardIntDivisionTest() testkit.Test {
+	prompt := `Consider this C code:
+
+int a = 7;
+int b = 2;
+double d = a / b;
+
+What is the value of d?
+Respond with only the number, no explanation.`
+
+	return testkit.Test{
+		ID:          "hard-int-division",
+		Category:    "programming",
+		Subcategory: "hard",
+		Description: "Trace integer-division-then-widen: 7 / 2 is 3 (stored as 3.0), not 3.5.",
+		Prompt:      prompt,
+		Eval:        eval.Numeric(eval.ExtractLastNumber[float64], hardIntDivisionWant, 0),
+	}
+}
+
+// hardIntDivisionWant is d's value.
+//
+// ground truth: integers divide as integers (truncating), giving 7 / 2 =
+// 3, converted to double 3.0; the float value is 3, so Numeric want is
+// 3.0. Recomputed by running the identical arithmetic during authoring.
+const hardIntDivisionWant = 3.0
+
+// hardRegexGreedyTest: trace which substrings two greedy capture groups
+// of an anchored pattern match.
+//
+// ground truth: quantifiers are greedy by default - each takes as much as
+// it can while still allowing the rest of the pattern to match. In
+// /^(\d+)(\d+)$/, the first \d+ grabs the longest prefix that leaves at
+// least one digit for the second \d+ (which needs at least one match
+// because + is one-or-more). That longest prefix of "12345" is "1234",
+// leaving "5" for group 2. A naive symmetric split like ["12","345"]
+// fails because group 1 would then still be able to grow. Verified with
+// node during authoring.
+func hardRegexGreedyTest() testkit.Test {
+	prompt := `Consider this regular expression applied to the string "12345":
+
+^(\d+)(\d+)$
+
+(^ and $ anchor to the whole string, and \d+ means one or more digits.)
+
+What are the two captured groups? Respond with only a JSON array of the
+two captured strings, in group order, for example ["a","b"]`
+
+	return testkit.Test{
+		ID:          "hard-regex-greedy",
+		Category:    "programming",
+		Subcategory: "hard",
+		Description: "Trace greedy capture-group backtracking: ^(one-or-more digits)(one-or-more digits)$ on 12345 yields 1234 and 5.",
+		Prompt:      prompt,
+		Eval:        eval.JSONStringArrayEquals([]string{"1234", "5"}),
+	}
+}
+
+// hardPromiseAllSettledTest: trace the per-result status values of
+// Promise.allSettled over a mixed resolve/reject input.
+//
+// ground truth: Promise.allSettled always resolves, and each element of
+// its result array is an object whose status is the literal "fulfilled"
+// for a resolved input and "rejected" for a rejected input, preserving
+// input order. Promise.resolve(42) yields {status:"fulfilled", value:42};
+// Promise.reject(new Error("boom")) yields {status:"rejected",
+// reason:...}. So the two status values, in order, are "fulfilled" then
+// "rejected". Verified with node during authoring.
+func hardPromiseAllSettledTest() testkit.Test {
+	prompt := `Consider this TypeScript code:
+
+const results = await Promise.allSettled([
+  Promise.resolve(42),
+  Promise.reject(new Error("boom")),
+]);
+
+Each element of results has a "status" field.
+
+List the two status values, in array order. Respond with only a JSON
+array of the two status strings, for example ["a","b"]`
+
+	return testkit.Test{
+		ID:          "hard-promise-allsettled",
+		Category:    "programming",
+		Subcategory: "hard",
+		Description: "Trace Promise.allSettled result statuses over a fulfilled+rejected input (fulfilled, rejected).",
+		Prompt:      prompt,
+		Eval:        eval.JSONStringArrayEquals([]string{"fulfilled", "rejected"}),
+	}
+}
+
+// hardContextTimeoutTest: identify the exact error string a timed-out
+// context's Err() returns.
+//
+// ground truth: when a context created by context.WithTimeout fires, its
+// Done channel closes and Err() returns the sentinel
+// context.DeadlineExceeded, whose Error() string (per the context
+// package) is exactly "context deadline exceeded" - which is distinct
+// from "context canceled", the string of context.Canceled returned when a
+// cancel function is called. Verified with go run during authoring.
+func hardContextTimeoutTest() testkit.Test {
+	prompt := `In Go, a context created with context.WithTimeout fires after its
+deadline. At that point:
+
+ctx.Err()
+
+returns a non-nil error. What is the exact string of that error?
+Respond with only a JSON object:
+{"err": "..."}
+
+with the exact string value.`
+
+	return testkit.Test{
+		ID:          "hard-context-timeout",
+		Category:    "programming",
+		Subcategory: "hard",
+		Description: "Identify ctx.Err() after a context.WithTimeout deadline fires (context deadline exceeded).",
+		Prompt:      prompt,
+		Eval:        eval.JSONField("err", "context deadline exceeded"),
+	}
+}
+
+// hardJSONUnmarshalNullTest: judge the behavior of json.Unmarshal on a
+// literal JSON null into an already-populated struct.
+//
+// ground truth: per encoding/json, a JSON null into a non-nil pointer
+// target is a no-op: Unmarshal returns nil (no error) and leaves the
+// target's existing value unchanged. It does not zero the struct and does
+// not error. Verified with go run during authoring (a struct holding
+// Items: [1] stays [1] with nil error).
+func hardJSONUnmarshalNullTest() testkit.Test {
+	prompt := `In Go:
+
+var s Struct  // s is a populated struct value (some fields set)
+err := json.Unmarshal([]byte("null"), &s)
+
+Does Unmarshal return an error, and is s left unchanged (no fields
+cleared)?
+Respond with only a JSON object:
+{"returns_error": true}
+or
+{"returns_error": false}`
+
+	return testkit.Test{
+		ID:          "hard-json-unmarshal-null",
+		Category:    "programming",
+		Subcategory: "hard",
+		Description: "Judge json.Unmarshal of a literal null into a populated struct (no error, unchanged).",
+		Prompt:      prompt,
+		Eval:        eval.JSONField("returns_error", false),
+	}
+}
+
+// hardVarShadowTest: judge whether an inner := declaration shadows an
+// outer variable without changing it.
+//
+// ground truth: `count := 5` inside the if block declares a NEW, inner
+// variable that shadows the outer count for the duration of that block
+// (Go's := always creates a new variable when at least one variable on
+// the left is new in the current scope). Incrementing the inner count
+// leaves the outer count untouched, so after the block outer count is
+// still 0. Verified with go run during authoring.
+func hardVarShadowTest() testkit.Test {
+	prompt := `Consider this Go code:
+
+count := 0
+if true {
+	count := 5
+	count++
+}
+// count here?
+
+Does the outer count remain unchanged (equal to 0) after the if block?
+Respond with only a JSON object:
+{"outer_unchanged": true}
+or
+{"outer_unchanged": false}`
+
+	return testkit.Test{
+		ID:          "hard-var-shadow",
+		Category:    "programming",
+		Subcategory: "hard",
+		Description: "Trace Go variable shadowing: an inner := leaves the outer variable unchanged (true).",
+		Prompt:      prompt,
+		Eval:        eval.JSONField("outer_unchanged", true),
+	}
+}
+
+// hardPyVersionVersions is the inline version list for
+// hardPyVersionSortTest.
+const hardPyVersionVersions = `versions = [
+    "1.9.0",
+    "1.10.0",
+    "1.2.0",
+    "2.0.0",
+    "0.9.9",
+]`
+
+// hardPyVersionSortWant is the expected stdout of a correct script: the
+// second-highest version under numeric component-by-component ordering.
+//
+// ground truth: ordering versions by numeric components puts 0.9.9 <
+// 1.2.0 < 1.9.0 < 1.10.0 < 2.0.0 (1.10.0 is newer than 1.9.0 because
+// its second component 10 > 9), so the second-highest is 1.10.0. A
+// plain lexicographic string sort (the trap) would instead report 1.9.0,
+// which is exactly the failure the test discriminates. Recomputed by
+// running the correct script with python3 during authoring.
+const hardPyVersionSortWant = "1.10.0"
+
+// hardPyVersionSortTest: write a python3 script that prints the
+// second-highest version under numeric component ordering.
+//
+// ground truth: see hardPyVersionSortWant; correctness is determined by
+// running the script (version ordering is behavior, not a text match).
+func hardPyVersionSortTest() testkit.Test {
+	prompt := `Here is an inline list of software version strings:
+
+` + "```python" + `
+` + hardPyVersionVersions + `
+` + "```" + `
+
+Write a python3 program that sorts the versions numerically, component by
+component (natural version ordering: "1.10.0" is newer than "1.9.0"
+because the second component 10 > 9), and prints only the second-highest
+version.`
+
+	return testkit.Test{
+		ID:          "hard-py-version-sort",
+		Category:    "programming",
+		Subcategory: "hard",
+		Description: "Write a python3 script printing the second-highest version by numeric component ordering.",
+		System:      terseCodeOnly,
+		Prompt:      prompt,
+		Eval:        eval.PyRun(eval.PassthroughHarness, hardPyVersionSortWant),
+	}
+}
+
+// hardGoParenBalanceHarness is a complete, independent driver file (its
+// own package clause and imports) that eval.GoRun writes to harness.go
+// alongside the model's IsBalanced in solution.go. It drives IsBalanced
+// across balanced, nested, mismatched, unclosed, and wrongly-ordered
+// cases, printing PASS only if every case matches its expected result.
+const hardGoParenBalanceHarness = `package main
+
+import (
+	"fmt"
+	"os"
+)
+
+func main() {
+	cases := []struct {
+		in   string
+		want bool
+	}{
+		{"", true},
+		{"()", true},
+		{"()[]{}", true},
+		{"([{}])", true},
+		{"(", false},
+		{"(]", false},
+		{"([)]", false},
+		{"((", false},
+		{")(", false},
+	}
+	for _, c := range cases {
+		if got := IsBalanced(c.in); got != c.want {
+			fmt.Printf("FAIL %q: got %v want %v\n", c.in, got, c.want)
+			os.Exit(1)
+		}
+	}
+	fmt.Println("PASS")
+}`
+
+// hardGoParenBalanceTest: implement a stack-based balanced-parentheses
+// checker.
+//
+// ground truth: the harness above is the oracle - it exercises empty,
+// balanced, nested, mismatched-closer, trailing-opener, and wrong-order
+// inputs, so a correct stack-based implementation prints PASS. There is no
+// separate hardcoded constant beyond the PASS sentinel.
+func hardGoParenBalanceTest() testkit.Test {
+	prompt := `Implement a Go function with exactly this signature:
+
+func IsBalanced(s string) bool
+
+It must report whether s's parentheses are balanced and correctly nested:
+'(' '[' '{' are openers, ')' ']' '}' are the matching closers. An empty
+string is balanced (return true). Return false for any unclosed opener,
+any closer with no matching open, or any wrong-nesting (e.g. "([)]").`
+
+	return testkit.Test{
+		ID:          "hard-go-paren-balance",
+		Category:    "programming",
+		Subcategory: "hard",
+		Description: "Implement a stack-based balanced-parentheses checker for (), [], and {}.",
+		System:      terseCodeOnly,
+		Prompt:      prompt,
+		Eval:        eval.GoRun(hardGoParenBalanceHarness, "PASS"),
+	}
+}

internal/tests/programming_hard_test.go

@@ -0,0 +1,360 @@
+package tests
+
+import (
+	"context"
+	"testing"
+)
+
+func TestHardFloatBinaddTest_Eval(t *testing.T) {
+	tc := hardFloatBinaddTest()
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{"correct", `{"equal":false}`, 1},
+		{"correct fenced with prose", "The answer is:\n```json\n{\"equal\":false}\n```", 1},
+		{"wrong: claims equal", `{"equal":true}`, 0},
+		{"wrong: non-boolean", `{"equal":"yes"}`, 0},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}
+
+func TestHardNullishOverOrTest_Eval(t *testing.T) {
+	tc := hardNullishOverOrTest()
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{"correct", `{"value":0}`, 1},
+		{"correct as string", `{"value":"0"}`, 1},
+		{"wrong: OR semantics", `{"value":42}`, 0},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}
+
+func TestHardGoroutineRaceTest_Eval(t *testing.T) {
+	tc := hardGoroutineRaceTest()
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{"correct", `{"guaranteed":false}`, 1},
+		{"correct fenced", "```json\n{\"guaranteed\":false}\n```", 1},
+		{"wrong: claims guaranteed", `{"guaranteed":true}`, 0},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}
+
+func TestHardDeferNamedReturnTest_Eval(t *testing.T) {
+	tc := hardDeferNamedReturnTest()
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{"correct", "99", 1},
+		{"correct prose-wrapped", "The function returns 99.", 1},
+		{"wrong: returns statement value", "7", 0},
+		{"wrong: pre-defer value", "5", 0},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}
+
+func TestHardIntDivisionTest_Eval(t *testing.T) {
+	tc := hardIntDivisionTest()
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{"correct", "3", 1},
+		{"correct as float", "3.0", 1},
+		{"correct prose-wrapped", "It is 3.", 1},
+		{"wrong: assumes float division", "3.5", 0},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}
+
+func TestHardRegexGreedyTest_Eval(t *testing.T) {
+	tc := hardRegexGreedyTest()
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{"correct", `["1234","5"]`, 1},
+		{"correct fenced", "```json\n[\"1234\",\"5\"]\n```", 1},
+		{"wrong: symmetric split", `["12","345"]`, 0},
+		{"wrong: reversed", `["5","1234"]`, 0},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}
+
+func TestHardPromiseAllSettledTest_Eval(t *testing.T) {
+	tc := hardPromiseAllSettledTest()
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{"correct", `["fulfilled","rejected"]`, 1},
+		{"correct fenced", "```json\n[\"fulfilled\",\"rejected\"]\n```", 1},
+		{"wrong: reordered", `["rejected","fulfilled"]`, 0},
+		{"wrong: both fulfilled", `["fulfilled","fulfilled"]`, 0},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}
+
+func TestHardContextTimeoutTest_Eval(t *testing.T) {
+	tc := hardContextTimeoutTest()
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{"correct", `{"err":"context deadline exceeded"}`, 1},
+		{"correct fenced", "```json\n{\"err\":\"context deadline exceeded\"}\n```", 1},
+		{"wrong: canceled sentinel", `{"err":"context canceled"}`, 0},
+		{"wrong: vague", `{"err":"some error"}`, 0},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}
+
+func TestHardJSONUnmarshalNullTest_Eval(t *testing.T) {
+	tc := hardJSONUnmarshalNullTest()
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{"correct", `{"returns_error":false}`, 1},
+		{"correct fenced", "```json\n{\"returns_error\":false}\n```", 1},
+		{"wrong: claims error", `{"returns_error":true}`, 0},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}
+
+func TestHardVarShadowTest_Eval(t *testing.T) {
+	tc := hardVarShadowTest()
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{"correct", `{"outer_unchanged":true}`, 1},
+		{"correct fenced", "```json\n{\"outer_unchanged\":true}\n```", 1},
+		{"wrong: claims outer changed", `{"outer_unchanged":false}`, 0},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}
+
+// TestHardPyVersionSortTest_Eval exercises the PyRun evaluator with a
+// hand-written correct script and a lexicographic (wrong) script.
+func TestHardPyVersionSortTest_Eval(t *testing.T) {
+	tc := hardPyVersionSortTest()
+
+	correctScript := "```python\n" + `versions = [
+    "1.9.0",
+    "1.10.0",
+    "1.2.0",
+    "2.0.0",
+    "0.9.9",
+]
+
+def key(v):
+    return tuple(int(p) for p in v.split("."))
+
+sorted_versions = sorted(versions, key=key)
+print(sorted_versions[-2])
+` + "```"
+
+	wrongLexicographic := "```python\n" + `versions = [
+    "1.9.0",
+    "1.10.0",
+    "1.2.0",
+    "2.0.0",
+    "0.9.9",
+]
+print(sorted(versions)[-2])
+` + "```"
+
+	wrongHardcoded := "```python\nprint('2.0.0')\n```"
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{"correct numeric sort", correctScript, 1},
+		{"wrong: lexicographic sort", wrongLexicographic, 0},
+		{"wrong: hardcoded highest", wrongHardcoded, 0},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Skipped {
+				t.Skip("python3 not available, skipping exec-evaluator assertion")
+			}
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}
+
+// TestHardGoParenBalanceTest_Eval exercises the GoRun evaluator with a
+// hand-written correct function and a wrong one.
+func TestHardGoParenBalanceTest_Eval(t *testing.T) {
+	tc := hardGoParenBalanceTest()
+
+	correctFunc := "```go\n" + `func IsBalanced(s string) bool {
+	pairs := map[byte]byte{')': '(', ']': '[', '}': '{'}
+	stack := make([]byte, 0, len(s))
+	for i := 0; i < len(s); i++ {
+		switch s[i] {
+		case '(', '[', '{':
+			stack = append(stack, s[i])
+		case ')', ']', '}':
+			if len(stack) == 0 || stack[len(stack)-1] != pairs[s[i]] {
+				return false
+			}
+			stack = stack[:len(stack)-1]
+		}
+	}
+	return len(stack) == 0
+}
+` + "```"
+
+	wrongFunc := "```go\n" + `func IsBalanced(s string) bool {
+	return true
+}
+` + "```"
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{"correct stack implementation", correctFunc, 1},
+		{"wrong: always true", wrongFunc, 0},
+		{"wrong: no code at all", "I would use a stack here.", 0},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Skipped {
+				t.Skip("go toolchain not available, skipping exec-evaluator assertion")
+			}
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}
+
+// TestHardDeferNamedReturnWant_GroundTruth recomputes f()'s return value
+// by running an equivalent function in-process, independent of the
+// hardDeferNamedReturnWant constant.
+func TestHardDeferNamedReturnWant_GroundTruth(t *testing.T) {
+	f := func() (r int) {
+		defer func() { r = 99 }()
+		r = 5
+		return 7
+	}
+	if got := f(); got != hardDeferNamedReturnWant {
+		t.Errorf("equivalent deferred-named-return function returned %d, want %d", got, hardDeferNamedReturnWant)
+	}
+}
+
+// TestHardIntDivisionWant_GroundTruth recomputes for a double result of
+// 7 / 2 by hand, independent of the hardIntDivisionWant constant.
+func TestHardIntDivisionWant_GroundTruth(t *testing.T) {
+	a, b := 7, 2
+	want := float64(a / b) // integer division truncates before widening
+	if want != hardIntDivisionWant {
+		t.Errorf("recomputed integer division = %v, want %v", want, hardIntDivisionWant)
+	}
+}

internal/tests/saturation.go

@@ -0,0 +1,534 @@
+// Package-level hardening tests for the five subcategories that measured
+// 100% perfect-score across all three baseline models (see the health
+// audit over the 2026-08-18 3-model run): agents/delegation,
+// databases/postgres, research/web, security/appsec, security/secrets.
+// Every test there was saturated - zero discrimination signal. These ten
+// tests are trap-shaped on purpose: each embeds a near-miss answer that
+// plausible-but-wrong reasoning produces, and every expected value carries
+// a derivation comment.
+package tests
+
+import (
+	"github.com/lukaszraczylo/llm-testbench/internal/eval"
+	"github.com/lukaszraczylo/llm-testbench/internal/testkit"
+)
+
+// registerSaturationTests adds the hardening tests to the registry. They
+// land in existing subcategories, so the per-category register functions
+// stay untouched; the subcategory's test count only grows.
+func registerSaturationTests(r *testkit.Registry) {
+	r.Register(satDelegCapacityTrapTest())
+	r.Register(satDelegNearMissRosterTest())
+	r.Register(satPGCountStarTrapTest())
+	r.Register(satPGTimeoutTrapTest())
+	r.Register(satWebRobotsUAScopeTest())
+	r.Register(satWebHreflangSelfTest())
+	r.Register(satAppsecORMTrapTest())
+	r.Register(satAppsecPOSTTrapTest())
+	r.Register(satSecretsRotationMathTest())
+	r.Register(satSecretsErrorPathTest())
+}
+
+// satDelegCapacityTrapTest: the roster's specialist for the job exists but
+// is unavailable. Tests whether the model force-fits the task to a
+// mismatched agent, over-coordinates with the orchestrator, or correctly
+// keeps it in the main thread.
+func satDelegCapacityTrapTest() testkit.Test {
+	prompt := `You are coordinating agents. Available specialist roster:
+
+` + delegRoster + `
+
+Situation: the CI pipeline for the main branch has been failing for the
+past hour on a Docker build step. It needs diagnosing now. The roster's
+infra/CI specialist is already working on an unrelated production incident
+and will not free up for several hours. There is exactly one task.
+
+Respond with only a JSON object, no other text, of the form
+{"best_next_step": "...", "specialist_this_task_belongs_to": "..."} where:
+- best_next_step is exactly one of: "dispatch_code_writer",
+  "dispatch_orchestrator", "handle_in_main_thread", "wait_for_specialist"
+- specialist_this_task_belongs_to is the roster agent name whose
+  description covers this task, regardless of their availability.`
+	return testkit.Test{
+		ID:          "agent-deleg-capacity-trap",
+		Category:    "agents",
+		Subcategory: "delegation",
+		Description: "Route a task whose specialist is busy: no force-fit to a mismatched agent, no orchestrator for a single task, and still name the correct owning specialist",
+		Prompt:      prompt,
+		// ground truth: the task is diagnosing a failing CI build, which
+		// the roster assigns to devops-debugger ("Diagnoses failing
+		// deployments, CI pipelines, and infrastructure incidents"). It is
+		// busy, so the remaining options are judged against the roster:
+		// code-writer "Implements new features or bug fixes by writing and
+		// editing source code directly" - a generalist forced onto infra
+		// diagnosis; orchestrator "Plans and sequences a multi-agent
+		// workflow" - there is one task, nothing to sequence; waiting on a
+		// specialist that is hours away while CI is down blocks delivery
+		// for no gain. A single task with its specialist unavailable is
+		// handled directly (handle_in_main_thread). The owning specialist
+		// on the roster is unambiguously devops-debugger.
+		Eval: eval.Mean(
+			eval.JSONField[string]("best_next_step", "handle_in_main_thread"),
+			eval.JSONField[string]("specialist_this_task_belongs_to", "devops-debugger"),
+		),
+	}
+}
+
+// satDelegNearMissRosterTest: 4-task mapping where every task has a
+// same-vocabulary distractor (docs vs web lookup, release notes vs prose
+// docs, diff review vs security review).
+func satDelegNearMissRosterTest() testkit.Test {
+	prompt := `You are coordinating agents. Specialist roster:
+
+` + delegRoster + `
+
+Map each task to exactly one roster agent - the one whose roster
+description covers it. Tasks:
+- task1: "Update the API guide in docs/ so it matches the endpoints as
+  they exist in the repository today."
+- task2: "Determine from the vendor's public pricing page which of their
+  plans includes SSO."
+- task3: "Produce the v2.3 release notes by generating the changelog
+  section and tagging the release."
+- task4: "Audit the auth handler for vulnerabilities before it merges."
+
+Respond with only a JSON object of the form
+{"task1": "...", "task2": "...", "task3": "...", "task4": "..."}
+where each value is exactly one roster agent name.`
+	return testkit.Test{
+		ID:          "agent-deleg-near-miss-roster",
+		Category:    "agents",
+		Subcategory: "delegation",
+		Description: "Route 4 tasks whose phrasing overlaps two roster descriptions each (repo docs vs web lookup, changelog vs prose docs, diff review vs security audit)",
+		Prompt:      prompt,
+		// ground truth, per the delegRoster descriptions:
+		// task1 -> docs-writer: it writes/updates prose documentation
+		// ("READMEs, guides, design docs") from repository content.
+		// web-researcher is wrong: "Does not read or write anything in the
+		// repository" and no external search is needed.
+		// task2 -> web-researcher: the answer lives on the vendor's public
+		// page, "Searches the public web and summarizes external
+		// information"; docs-writer is the distractor ("page" sounds like
+		// documentation) but the task reads the web, not the repo.
+		// task3 -> release-manager: "Manages version tagging, changelog
+		// generation, and release publishing" covers exactly the phrasing
+		// (generate changelog + tag). docs-writer is the distractor:
+		// release notes here are the release process, not a guide.
+		// task4 -> security-reviewer: "Audits code or config for security
+		// vulnerabilities"; code-reviewer is the distractor ("before it
+		// merges" sounds like PR review) but code-reviewer reviews
+		// "correctness, style, and risk", while the stated goal is
+		// vulnerabilities.
+		Eval: eval.Mean(
+			eval.JSONField[string]("task1", "docs-writer"),
+			eval.JSONField[string]("task2", "web-researcher"),
+			eval.JSONField[string]("task3", "release-manager"),
+			eval.JSONField[string]("task4", "security-reviewer"),
+		),
+	}
+}
+
+// satPGCountStarFixture is the orders/line-items data for the join
+// fan-out trap.
+const satPGCountStarFixture = `CREATE TABLE orders (id bigint PRIMARY KEY, region text NOT NULL);
+CREATE TABLE order_items (id bigint PRIMARY KEY,
+                          order_id bigint NOT NULL REFERENCES orders(id));
+
+INSERT INTO orders (id, region) VALUES
+  (1001, 'eu'), (1002, 'eu'), (1003, 'us'),
+  (1004, 'eu'), (1005, 'ap'), (1006, 'eu');
+
+INSERT INTO order_items (id, order_id) VALUES
+  (9001, 1001), (9002, 1001), (9003, 1001),
+  (9004, 1002),
+  (9005, 1003), (9006, 1003), (9007, 1003), (9008, 1003), (9009, 1003),
+  (9010, 1004), (9011, 1004),
+  (9012, 1005);`
+
+// satPGCountStarTrapTest: COUNT(*) over an INNER JOIN counts line items,
+// not orders, and the join silently drops the order with zero items. Every
+// number is derivable from the fixture, so the evaluator never needs to
+// re-run SQL.
+func satPGCountStarTrapTest() testkit.Test {
+	prompt := `Here is a PostgreSQL schema with data:
+
+` + "```sql\n" + satPGCountStarFixture + "\n```" + `
+
+The business question is: how many ORDERS are in region 'eu'?
+A developer writes:
+
+` + "```sql\nSELECT count(*) FROM orders o JOIN order_items oi ON oi.order_id = o.id WHERE o.region = 'eu';\n```" + `
+
+Respond with only a JSON object, no other text:
+{"orders_in_eu": <number>, "rows_the_join_query_returns": <number>,
+ "count_star_answers_the_question": true or false}`
+	return testkit.Test{
+		ID:          "pg-count-star-vs-join",
+		Category:    "databases",
+		Subcategory: "postgres",
+		Description: "Reason about COUNT(*) over an INNER JOIN against a fixed fixture: join fan-out inflates the count and the join drops the childless order",
+		Prompt:      prompt,
+		// ground truth: orders with region 'eu' are 1001, 1002, 1004, 1006
+		// (INSERTs above) = 4. The join emits one row per order_item:
+		// 1001 -> 3 items (9001-9003), 1002 -> 1 (9004), 1004 -> 2 (9010,
+		// 9011), and 1006 -> 0 items, so the INNER JOIN drops it entirely.
+		// count(*) therefore returns 3+1+2 = 6. 6 != 4, so the count(*)
+		// query does not answer the question (EXISTS or COUNT(DISTINCT
+		// o.id) would return 4).
+		Eval: eval.Mean(
+			eval.JSONField[int]("orders_in_eu", 4),
+			eval.JSONField[int]("rows_the_join_query_returns", 6),
+			eval.JSONField[bool]("count_star_answers_the_question", false),
+		),
+	}
+}
+
+// satPGTimeoutTrapTest: two sessions, two timeouts, one rule each. The
+// near-miss is applying idle_in_transaction_session_timeout to a busy
+// session (or statement_timeout to an idle one), and starting the idle
+// clock at BEGIN instead of at the last statement's completion.
+func satPGTimeoutTrapTest() testkit.Test {
+	prompt := `A PostgreSQL server is configured with:
+
+` + "```\nstatement_timeout = '30s'\nidle_in_transaction_session_timeout = '5s'\n```" + `
+
+Two sessions on this server:
+- Session A: BEGIN at 11:00:00, runs statements until 11:00:40 (the last
+  one finishes then), and has been waiting for client input ever since.
+  The transaction is still open.
+- Session B: BEGIN at 11:00:53, starts one long-running query at 11:00:55
+  which is still executing at 11:01:00.
+
+Assume no client input arrives and no query finishes after 11:01:00.
+Respond with only a JSON object, no other text:
+{"session_a_killed_at": "HH:MM:SS", "session_a_rule": "...",
+ "session_b_killed_at": "HH:MM:SS", "session_b_rule": "..."}
+where each time is in UTC and each rule is exactly one of:
+"statement_timeout", "idle_in_transaction_timeout", "not_killed".`
+	return testkit.Test{
+		ID:          "pg-timeout-trap",
+		Category:    "databases",
+		Subcategory: "postgres",
+		Description: "Apply statement_timeout and idle_in_transaction_session_timeout to the right sessions: idle clock starts at the last statement's completion, not BEGIN, and never applies to a session executing a statement",
+		Prompt:      prompt,
+		// ground truth: PostgreSQL's idle_in_transaction_session_timeout
+		// terminates a session "idle ... within an open transaction" -
+		// the clock starts when the session goes idle in transaction,
+		// which for A is 11:00:40 (its last statement finished then), so
+		// A is killed at 11:00:45 - not 11:00:05, since A was executing
+		// statements from 11:00:00 to 11:00:40. A is not subject to
+		// statement_timeout: no statement is running. B is idle in
+		// transaction only from 11:00:53 to 11:00:55 (2s, under the 5s
+		// cap) and then EXECUTES a statement, so the idle timeout no
+		// longer applies; statement_timeout bounds a single running
+		// statement: 11:00:55 + 30s = 11:01:25.
+		Eval: eval.Mean(
+			eval.JSONField[string]("session_a_killed_at", "11:00:45"),
+			eval.JSONField[string]("session_a_rule", "idle_in_transaction_timeout"),
+			eval.JSONField[string]("session_b_killed_at", "11:01:25"),
+			eval.JSONField[string]("session_b_rule", "statement_timeout"),
+		),
+	}
+}
+
+// satWebRobotsFixture carries two rule groups and a longest-match pair.
+const satWebRobotsFixture = `User-agent: *
+Disallow: /cdn-cgi/
+Allow: /cdn-cgi/style.css
+Sitemap: https://example.com/sitemap.xml
+
+User-agent: GoogleOther
+Disallow: /`
+
+// satWebRobotsUAScopeTest: GoogleOther's rules do not apply to Googlebot,
+// and within Googlebot's group the longest path match decides. The
+// near-miss is answering from the GoogleOther block alone, and answering
+// the Allow/Disallow pair by document order instead of path length.
+func satWebRobotsUAScopeTest() testkit.Test {
+	prompt := `A site serves this robots.txt at https://example.com/robots.txt:
+
+` + "```\n" + satWebRobotsFixture + "\n```" + `
+
+Consider a GET request for https://example.com/cdn-cgi/style.css.
+Respond with only a JSON object, no other text:
+{"googlebot_allowed": true or false, "googleother_allowed": true or false,
+ "deciding_rule_for_googlebot": "allow" or "disallow" or "no_rule"}`
+	return testkit.Test{
+		ID:          "web-robots-ua-scope-trap",
+		Category:    "research",
+		Subcategory: "web",
+		Description: "robots.txt group matching across two User-agent blocks plus longest-path-match precedence - a separate group's Disallow does not apply to the other crawler",
+		Prompt:      prompt,
+		// ground truth: per RFC 9309, a crawler only applies the rule
+		// group(s) whose User-agent matches it, and within a group the
+		// path pattern with the LONGEST match wins. Googlebot does not
+		// match the GoogleOther group, so only the '*' group applies:
+		// the URL matches Disallow /cdn-cgi/ (9 chars) and Allow
+		// /cdn-cgi/style.css (18 chars); 18 > 9, so Allow wins ->
+		// googlebot_allowed true, deciding_rule_for_googlebot "allow".
+		// GoogleOther matches its own group, where "Disallow: /" covers
+		// every path -> googleother_allowed false.
+		Eval: eval.Mean(
+			eval.JSONField[bool]("googlebot_allowed", true),
+			eval.JSONField[bool]("googleother_allowed", false),
+			eval.JSONField[string]("deciding_rule_for_googlebot", "allow"),
+		),
+	}
+}
+
+// satWebHreflangSelfTest: an hreflang set missing its self-reference and
+// x-default's sentinel status. The near-miss is counting the canonical
+// link as the hreflang self-reference, and treating x-default as a
+// language.
+func satWebHreflangSelfTest() testkit.Test {
+	prompt := `The page https://example.com/en-us/ serves this head section:
+
+` + "```html\n" + `<html lang="en-US">
+<head>
+  <link rel="alternate" hreflang="en-AU" href="https://example.com/en-au/">
+  <link rel="alternate" hreflang="en-GB" href="https://example.com/en-gb/">
+  <link rel="alternate" hreflang="x-default" href="https://example.com/">
+  <link rel="canonical" href="https://example.com/en-us/">
+</head>` + "\n```" + `
+
+Google's hreflang rule: an hreflang group must include a self-reference -
+an hreflang annotation whose href is the page itself - and every
+annotation it lists must point back reciprocally.
+Respond with only a JSON object, no other text:
+{"missing_self_reference_for": "the BCP 47 language tag of the page that
+has no hreflang annotation pointing at it", "self_reference_present":
+true or false, "canonical_counts_as_hreflang_self_reference": true or
+false, "x_default_is_a_language_tag": true or false}`
+	return testkit.Test{
+		ID:          "web-hreflang-self-reference-trap",
+		Category:    "research",
+		Subcategory: "web",
+		Description: "hreflang self-reference validation: the page's own language has no annotation, rel=canonical is a different mechanism, and x-default is a sentinel, not a BCP 47 tag",
+		Prompt:      prompt,
+		// ground truth: the page is en-US (lang attribute and canonical
+		// URL agree). The hreflang annotations cover en-AU, en-GB,
+		// x-default only - no annotation has href
+		// https://example.com/en-us/, so the missing self-reference tag
+		// is "en-US" and self_reference_present is false.
+		// canonical_counts_as_hreflang_self_reference is false: rel
+		// -canonical deduplicates URLs for indexing; hreflang self-
+		// reference is a separate signal on rel=alternate link elements
+		// - the canonical matching the page URL does not satisfy it.
+		// x_default_is_a_language_tag is false: x-default is Google's
+		// sentinel for "none of the specific languages", not a BCP 47
+		// tag (it carries no language subtag; the other two do).
+		Eval: eval.Mean(
+			eval.JSONField[string]("missing_self_reference_for", "en-US"),
+			eval.JSONField[bool]("self_reference_present", false),
+			eval.JSONField[bool]("canonical_counts_as_hreflang_self_reference", false),
+			eval.JSONField[bool]("x_default_is_a_language_tag", false),
+		),
+	}
+}
+
+// satAppsecORMFixture shows three SQLAlchemy call sites; only one builds
+// SQL by concatenation before wrapping it in text().
+const satAppsecORMFixture = `from sqlalchemy import text
+
+def get_user(conn, name, team):
+    sql = text("SELECT * FROM users WHERE name = :name AND team = :team")
+    return conn.execute(sql, {"name": name, "team": team}).fetchone()
+
+def search_users(conn, pattern):
+    # callers pass the raw 'q' query-parameter straight in
+    sql = text("SELECT * FROM users WHERE bio LIKE '%" + pattern + "%'")
+    return conn.execute(sql).fetchall()
+
+def list_orders(conn, status):
+    return conn.execute(
+        text("SELECT * FROM orders WHERE status = :s"), {"s": status}
+    ).fetchall()`
+
+// satAppsecORMTrapTest: SQLAlchemy text() is a wrapper, not a sanitizer -
+// the concatenation already fused attacker input into the SQL string
+// before text() ever sees it. The near-miss is "ORM == safe".
+func satAppsecORMTrapTest() testkit.Test {
+	prompt := `Flask application (the 'q' request parameter is attacker-controlled):
+
+` + "```python\n" + satAppsecORMFixture + "\n```" + `
+
+Respond with only a JSON object, no other text:
+{"injectable_function": "one of get_user | search_users | list_orders",
+ "text_wrapper_neutralizes_concatenation": true or false}`
+	return testkit.Test{
+		ID:          "sec-orm-interpolation-trap",
+		Category:    "security",
+		Subcategory: "appsec",
+		Description: "Spot SQL injection inside a SQLAlchemy text() call built by string concatenation - the ORM wrapper neutralizes nothing once the string is already fused",
+		Prompt:      prompt,
+		// ground truth: get_user and list_orders pass user values as
+		// bound parameters (:name/:team, :s) - the driver sends values
+		// out-of-band, so no injection. search_users builds the SQL
+		// string with + pattern + BEFORE text() wraps it: by the time
+		// text() sees it, the attacker's input is part of the statement
+		// (pattern = "x%' OR 1=1 -- " breaks out of the LIKE literal).
+		// text() marks a string as raw SQL for the ORM; it never
+		// parameterizes or escapes - text_wrapper_neutralizes_
+		// concatenation is false.
+		Eval: eval.Mean(
+			eval.JSONField[string]("injectable_function", "search_users"),
+			eval.JSONField[bool]("text_wrapper_neutralizes_concatenation", false),
+		),
+	}
+}
+
+// satAppsecPOSTFixture shows a logout control where the visible anchor is
+// a decoy - the actual request is the auto-submitted POST form.
+const satAppsecPOSTFixture = `<a href="#" onclick="document.getElementById('f').submit(); return false;">
+  Log out
+</a>
+<form id="f" action="/api/account/logout" method="post">
+  <input type="hidden" name="confirm" value="true">
+</form>`
+
+// satAppsecPOSTTrapTest: the app switched a token-less state-changing
+// endpoint from GET to POST as its whole CSRF mitigation. Tests whether
+// the model knows POST alone is not a defense (cross-site <form
+// method=post> auto-submits with cookies), even when the page only ever
+// submits it via JavaScript.
+func satAppsecPOSTTrapTest() testkit.Test {
+	prompt := `A session-authenticated web app renders this logout control. The session
+cookie is SameSite=None; Secure (sent with every cross-site request), and
+there is no CSRF token anywhere in the request.
+
+` + "```\n" + satAppsecPOSTFixture + "\n```" + `
+
+The developer's note: "I moved logout from GET to POST, so CSRF no longer
+applies - only JavaScript on THIS page submits the form."
+
+Assume an attacker page on evil.example hosts its own form with
+action="https://app.example/api/account/logout", method="post". The victim
+visits evil.example but clicks nothing and types nothing there.
+
+Respond with only a JSON object, no other text:
+{"attacker_form_reaches_logout": true or false,
+ "post_method_alone_prevents_csrf": true or false,
+ "attacker_needs_javascript": true or false}`
+	return testkit.Test{
+		ID:          "sec-csrf-post-trap",
+		Category:    "security",
+		Subcategory: "appsec",
+		Description: "CSRF after a GET-to-POST switch without a token: cross-site form auto-submit carries the cookie, so the method is not the defense; the attacker needs one line of JS, not knowledge of the page's DOM",
+		Prompt:      prompt,
+		// ground truth: SameSite=None means the browser attaches the
+		// session cookie to the attacker's cross-site POST, and there is
+		// no token, so the request is accepted as the victim ->
+		// attacker_form_reaches_logout true; the server authenticates by
+		// cookie, not by which page submitted the form.
+		// post_method_alone_prevents_csrf false: method only restricts
+		// bookmark/simple-link attacks; a hosted <form method=post>
+		// replays the state change. (With SameSite=Lax a cross-site POST
+		// would instead drop the cookie - the prompt pins SameSite=None
+		// so the answer is forced.)
+		// attacker_needs_javascript true: the prompt pins "clicks nothing",
+		// and without JavaScript a hosted form only submits when the victim
+		// presses its own submit button - a cross-site <img>/<a> cannot POST
+		// with the cookie context this request needs, and <meta
+		// http-equiv="refresh"> cannot carry a POST body. The silent attack
+		// requires form.submit().
+		Eval: eval.Mean(
+			eval.JSONField[bool]("attacker_form_reaches_logout", true),
+			eval.JSONField[bool]("post_method_alone_prevents_csrf", false),
+			eval.JSONField[bool]("attacker_needs_javascript", true),
+		),
+	}
+}
+
+// satRotationFixture gives the three durations for the rotation
+// deadline math.
+const satRotationFixture = `Internal CA certificate expires:   2026-09-30 18:00 UTC
+Worst realistic time to mint a replacement cert once rotation starts: 10 h
+After the new cert is deployed, the reverse proxy's graceful-reload
+window (old workers serve the old cert for up to this long after the
+reload is issued):                6 h`
+
+// satSecretsRotationMathTest: latest safe start time = expiry minus the
+// sum of both sequential windows. The near-miss is subtracting only the
+// minting time and forgetting the propagation tail.
+func satSecretsRotationMathTest() testkit.Test {
+	prompt := `Certificate rotation facts for an internal service:
+
+` + "```\n" + satRotationFixture + "\n```" + `
+
+Respond with only a JSON object, no other text:
+{"total_lead_time_hours": <number>, "latest_start": "YYYY-MM-DD HH:MM",
+ "safe_if_started_2026_09_30_08_00": true or false}
+(Use UTC throughout. latest_start is the latest time rotation can START
+and still have the old certificate fully replaced everywhere before it
+expires.)`
+	return testkit.Test{
+		ID:          "sec-rotation-window-math",
+		Category:    "security",
+		Subcategory: "secrets",
+		Description: "Certificate rotation deadline math: lead time is minting plus propagation in series, and starting late leaves the old cert serving past expiry",
+		Prompt:      prompt,
+		// ground truth: the two windows are sequential - a replacement
+		// can only start propagating once it exists: 10 h + 6 h = 16 h
+		// worst-case lead time. 2026-09-30 18:00 minus 16 h =
+		// 2026-09-30 02:00 latest start. Starting at 08:00 finishes at
+		// the worst at 08:00 + 16 h = 2026-10-01 00:00, six hours after
+		// the old cert expired -> not safe.
+		Eval: eval.Mean(
+			eval.JSONField[int]("total_lead_time_hours", 16),
+			eval.JSONField[string]("latest_start", "2026-09-30 02:00"),
+			eval.JSONField[bool]("safe_if_started_2026_09_30_08_00", false),
+		),
+	}
+}
+
+// satErrorPathFixture traces a password hash from a raise through an
+// error handler into a log line, while the client only sees a generic
+// response body.
+const satErrorPathFixture = `1| def login(user, password):
+2|     row = db.get_user(user)
+3|     if not verify(password, row.hash):
+4|         raise AuthError(f"invalid credentials for user={user} "
+5|                         f"hash={row.hash}")
+6|     return token_for(user)
+7|
+8| @app.errorhandler(AuthError)
+9| def on_auth_error(exc):
+10|     app.logger.error(str(exc))
+11|     return "login failed", 401`
+
+// satSecretsErrorPathTest: the secret (password hash) is interpolated at
+// the raise, survives the handler's str(exc), and lands in the log even
+// though the HTTP response is sanitized. The near-miss is "the client
+// never sees it, so it's fine", and the reverse trap of flagging the
+// attacker-supplied username as the secret.
+func satSecretsErrorPathTest() testkit.Test {
+	prompt := `Flask login flow (numbers are source lines):
+
+` + "```\n" + satErrorPathFixture + "\n```" + `
+
+row.hash is the stored password hash (sensitive). user comes straight
+from the login request body.
+Respond with only a JSON object, no other text:
+{"hash_reaches_server_logs": true or false, "hash_visible_in_http_response":
+true or false, "sensitive_value_in_message": "one of user | hash | none"}`
+	return testkit.Test{
+		ID:          "sec-error-path-leak-trap",
+		Category:    "security",
+		Subcategory: "secrets",
+		Description: "Trace a password hash through raise -> errorhandler -> logger.error: the client response is sanitized but the log line is not, and the echoed user field is attacker-known",
+		Prompt:      prompt,
+		// ground truth: the raise on lines 4-5 interpolates row.hash into
+		// the exception message; the handler on line 10 logs str(exc) -
+		// the full message - so the hash reaches the server logs: true.
+		// The handler returns only the literal "login failed" (line 11),
+		// never str(exc), so the hash is not visible in the HTTP
+		// response: false. Between user (attacker-supplied in the request
+		// body, already known to its owner) and hash (sensitive), the
+		// sensitive interpolated value is the hash.
+		Eval: eval.Mean(
+			eval.JSONField[bool]("hash_reaches_server_logs", true),
+			eval.JSONField[bool]("hash_visible_in_http_response", false),
+			eval.JSONField[string]("sensitive_value_in_message", "hash"),
+		),
+	}
+}

internal/tests/saturation_test.go

@@ -0,0 +1,400 @@
+package tests
+
+import (
+	"context"
+	"testing"
+)
+
+// Each saturation test gets a table pinning: the full-credit answer, the
+// trap answer (the near-miss the test exists to catch), and a garbage
+// answer. Expected partial scores follow from eval.Mean's per-field mean.
+
+func TestSatDelegCapacityTrapTest_Eval(t *testing.T) {
+	tc := satDelegCapacityTrapTest()
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{
+			name:     "correct: main thread, names busy specialist",
+			response: `{"best_next_step":"handle_in_main_thread","specialist_this_task_belongs_to":"devops-debugger"}`,
+			want:     1,
+		},
+		{
+			name:     "trap: force-fits the generalist builder",
+			response: `{"best_next_step":"dispatch_code_writer","specialist_this_task_belongs_to":"devops-debugger"}`,
+			want:     0.5,
+		},
+		{
+			name:     "trap: orchestrator for a single task",
+			response: `{"best_next_step":"dispatch_orchestrator","specialist_this_task_belongs_to":"orchestrator"}`,
+			want:     0,
+		},
+		{
+			name:     "correct step but wrong owning specialist",
+			response: `{"best_next_step":"handle_in_main_thread","specialist_this_task_belongs_to":"code-writer"}`,
+			want:     0.5,
+		},
+		{
+			name:     "wait-it-out answer",
+			response: `{"best_next_step":"wait_for_specialist","specialist_this_task_belongs_to":"devops-debugger"}`,
+			want:     0.5,
+		},
+		{name: "garbage", response: `not json`, want: 0},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}
+
+func TestSatDelegNearMissRosterTest_Eval(t *testing.T) {
+	tc := satDelegNearMissRosterTest()
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{
+			name:     "all correct",
+			response: `{"task1":"docs-writer","task2":"web-researcher","task3":"release-manager","task4":"security-reviewer"}`,
+			want:     1,
+		},
+		{
+			name:     "trap: web-researcher for the repo guide, docs-writer for the pricing page",
+			response: `{"task1":"web-researcher","task2":"docs-writer","task3":"release-manager","task4":"security-reviewer"}`,
+			want:     0.5,
+		},
+		{
+			name:     "trap: docs-writer for release notes, code-reviewer for the audit",
+			response: `{"task1":"docs-writer","task2":"web-researcher","task3":"docs-writer","task4":"code-reviewer"}`,
+			want:     0.5,
+		},
+		{
+			name:     "all swapped to the distractors",
+			response: `{"task1":"web-researcher","task2":"docs-writer","task3":"docs-writer","task4":"code-reviewer"}`,
+			want:     0,
+		},
+		{
+			name:     "case-insensitive correct",
+			response: `{"task1":"Docs-Writer","task2":"Web-Researcher","task3":"Release-Manager","task4":"Security-Reviewer"}`,
+			want:     1,
+		},
+		{name: "garbage", response: `["docs-writer"]`, want: 0},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}
+
+func TestSatPGCountStarTrapTest_Eval(t *testing.T) {
+	tc := satPGCountStarTrapTest()
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{
+			name:     "correct",
+			response: `{"orders_in_eu": 4, "rows_the_join_query_returns": 6, "count_star_answers_the_question": false}`,
+			want:     1,
+		},
+		{
+			name:     "numeric strings accepted",
+			response: `{"orders_in_eu": "4", "rows_the_join_query_returns": "6", "count_star_answers_the_question": "false"}`,
+			want:     1,
+		},
+		{
+			name:     "trap: counts joined rows as orders",
+			response: `{"orders_in_eu": 6, "rows_the_join_query_returns": 6, "count_star_answers_the_question": true}`,
+			want:     0.3333333333333333,
+		},
+		{
+			name:     "trap: forgets the childless order drops out of the inner join",
+			response: `{"orders_in_eu": 4, "rows_the_join_query_returns": 7, "count_star_answers_the_question": false}`,
+			want:     0.6666666666666666,
+		},
+		{name: "garbage", response: `4`, want: 0},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}
+
+func TestSatPGTimeoutTrapTest_Eval(t *testing.T) {
+	tc := satPGTimeoutTrapTest()
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{
+			name:     "correct",
+			response: `{"session_a_killed_at":"11:00:45","session_a_rule":"idle_in_transaction_timeout","session_b_killed_at":"11:01:25","session_b_rule":"statement_timeout"}`,
+			want:     1,
+		},
+		{
+			name:     "trap: idle clock started at BEGIN",
+			response: `{"session_a_killed_at":"11:00:05","session_a_rule":"idle_in_transaction_timeout","session_b_killed_at":"11:01:25","session_b_rule":"statement_timeout"}`,
+			want:     0.75,
+		},
+		{
+			name:     "trap: timeouts swapped between sessions",
+			response: `{"session_a_killed_at":"11:00:30","session_a_rule":"statement_timeout","session_b_killed_at":"11:00:58","session_b_rule":"idle_in_transaction_timeout"}`,
+			want:     0,
+		},
+		{
+			name:     "two fields right",
+			response: `{"session_a_killed_at":"11:00:45","session_a_rule":"idle_in_transaction_timeout","session_b_killed_at":"11:01:00","session_b_rule":"not_killed"}`,
+			want:     0.5,
+		},
+		{name: "garbage", response: `no json here`, want: 0},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}
+
+func TestSatWebRobotsUAScopeTest_Eval(t *testing.T) {
+	tc := satWebRobotsUAScopeTest()
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{
+			name:     "correct: longest match wins for googlebot, group isolation for GoogleOther",
+			response: `{"googlebot_allowed": true, "googleother_allowed": false, "deciding_rule_for_googlebot": "allow"}`,
+			want:     1,
+		},
+		{
+			name:     "trap: GoogleOther's disallow applied to googlebot too",
+			response: `{"googlebot_allowed": false, "googleother_allowed": false, "deciding_rule_for_googlebot": "disallow"}`,
+			want:     0.3333333333333333,
+		},
+		{
+			name:     "trap: wildcard group treated as absent",
+			response: `{"googlebot_allowed": true, "googleother_allowed": true, "deciding_rule_for_googlebot": "allow"}`,
+			want:     0.6666666666666666,
+		},
+		{name: "garbage", response: `allow`, want: 0},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}
+
+func TestSatWebHreflangSelfTest_Eval(t *testing.T) {
+	tc := satWebHreflangSelfTest()
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{
+			name:     "correct",
+			response: `{"missing_self_reference_for":"en-US","self_reference_present":false,"canonical_counts_as_hreflang_self_reference":false,"x_default_is_a_language_tag":false}`,
+			want:     1,
+		},
+		{
+			name:     "tag casing normalized",
+			response: `{"missing_self_reference_for":"en-us","self_reference_present":false,"canonical_counts_as_hreflang_self_reference":false,"x_default_is_a_language_tag":false}`,
+			want:     1,
+		},
+		{
+			name:     "trap: canonical counted as the self-reference",
+			response: `{"missing_self_reference_for":"none","self_reference_present":true,"canonical_counts_as_hreflang_self_reference":true,"x_default_is_a_language_tag":false}`,
+			want:     0.25,
+		},
+		{
+			name:     "trap: x-default treated as a language",
+			response: `{"missing_self_reference_for":"en-US","self_reference_present":false,"canonical_counts_as_hreflang_self_reference":false,"x_default_is_a_language_tag":true}`,
+			want:     0.75,
+		},
+		{name: "garbage", response: `en-US`, want: 0},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}
+
+func TestSatAppsecORMTrapTest_Eval(t *testing.T) {
+	tc := satAppsecORMTrapTest()
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{
+			name:     "correct",
+			response: `{"injectable_function": "search_users", "text_wrapper_neutralizes_concatenation": false}`,
+			want:     1,
+		},
+		{
+			name:     "trap: ORM assumed safe everywhere",
+			response: `{"injectable_function": "none", "text_wrapper_neutralizes_concatenation": true}`,
+			want:     0,
+		},
+		{
+			name:     "right function, wrong mechanism",
+			response: `{"injectable_function": "search_users", "text_wrapper_neutralizes_concatenation": true}`,
+			want:     0.5,
+		},
+		{name: "garbage", response: `search_users`, want: 0},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}
+
+func TestSatAppsecPOSTTrapTest_Eval(t *testing.T) {
+	tc := satAppsecPOSTTrapTest()
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{
+			name:     "correct",
+			response: `{"attacker_form_reaches_logout": true, "post_method_alone_prevents_csrf": false, "attacker_needs_javascript": true}`,
+			want:     1,
+		},
+		{
+			name:     "trap: POST assumed to prevent CSRF",
+			response: `{"attacker_form_reaches_logout": false, "post_method_alone_prevents_csrf": true, "attacker_needs_javascript": true}`,
+			want:     0.3333333333333333,
+		},
+		{
+			name:     "trap: attacker blocked for lacking the page's JS",
+			response: `{"attacker_form_reaches_logout": false, "post_method_alone_prevents_csrf": false, "attacker_needs_javascript": false}`,
+			want:     0.3333333333333333,
+		},
+		{name: "garbage", response: `{"a":1}`, want: 0},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}
+
+func TestSatSecretsRotationMathTest_Eval(t *testing.T) {
+	tc := satSecretsRotationMathTest()
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{
+			name:     "correct",
+			response: `{"total_lead_time_hours": 16, "latest_start": "2026-09-30 02:00", "safe_if_started_2026_09_30_08_00": false}`,
+			want:     1,
+		},
+		{
+			name:     "trap: minting time only, propagation forgotten",
+			response: `{"total_lead_time_hours": 10, "latest_start": "2026-09-30 08:00", "safe_if_started_2026_09_30_08_00": true}`,
+			want:     0,
+		},
+		{
+			name:     "sum right, start-time arithmetic wrong",
+			response: `{"total_lead_time_hours": 16, "latest_start": "2026-09-30 02:00", "safe_if_started_2026_09_30_08_00": true}`,
+			want:     0.6666666666666666,
+		},
+		{name: "garbage", response: `16 hours`, want: 0},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}
+
+func TestSatSecretsErrorPathTest_Eval(t *testing.T) {
+	tc := satSecretsErrorPathTest()
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{
+			name:     "correct",
+			response: `{"hash_reaches_server_logs": true, "hash_visible_in_http_response": false, "sensitive_value_in_message": "hash"}`,
+			want:     1,
+		},
+		{
+			name:     "trap: sanitized response assumed to mean no leak",
+			response: `{"hash_reaches_server_logs": false, "hash_visible_in_http_response": false, "sensitive_value_in_message": "hash"}`,
+			want:     0.6666666666666666,
+		},
+		{
+			name:     "trap: attacker-known user flagged instead of the hash",
+			response: `{"hash_reaches_server_logs": true, "hash_visible_in_http_response": false, "sensitive_value_in_message": "user"}`,
+			want:     0.6666666666666666,
+		},
+		{
+			name:     "trap: leak judged through the log AND the response",
+			response: `{"hash_reaches_server_logs": true, "hash_visible_in_http_response": true, "sensitive_value_in_message": "hash"}`,
+			want:     0.6666666666666666,
+		},
+		{name: "garbage", response: `hash`, want: 0},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}

internal/tests/security_crypto.go

@@ -23,29 +23,49 @@ func registerSecurityCryptoTests(r *testkit.Registry) {
 	r.Register(secHashVsEncryptPIITest())
 }
 
-// secWeakPasswordHashPattern matches a mention of a fast, general-purpose
+// secWeakHashEndorsementPattern matches a weak-hash mention in an
-// hash (MD5, plain SHA-1/256/512) - the ones that must never be used
+// endorsement-shaped position — a verb recommending it ("hash with MD5",
-// directly for password storage because they lack a tunable work factor.
+// "just hash the password with SHA-256"), or a positive predicate about it
-var secWeakPasswordHashPattern = regexp.MustCompile(`(?i)\b(md5|sha-?1|sha-?256|sha-?512)\b`)
+// ("SHA-256 is a fine choice") — rather than every mention. The guard then
+// still requires a negation cue near such a mention (see
+// secNoUnnegatedMention).
+//
+// Triggering on any mention is too strict: a fully correct live answer
+// (probe, 2026-08-29) lists the discouraged hashes as bare bullets under a
+// negated heading ("Do **not** use fast general-purpose ... including:") —
+// the cue sits on the heading, but the negation window stops at the line
+// break and the heading is more than a window-width away, so each bullet
+// read as an unnegated endorsement and cost the model half the score.
+// A bare bullet names, it does not recommend — the predicate branch still
+// catches an endorsed bullet ("- **MD5** is fine for passwords").
+var secWeakHashEndorsementPattern = regexp.MustCompile("(?i)(?:" +
+	// a verb recommending it: "hash with MD5", "store passwords hashed with
+	// MD5", "just hash the password with SHA-256" — the filler loop only
+	// bridges recommendation boilerplate, never another algorithm name, so
+	// "prefer argon2 over md5" (a correct answer) is not an endorsement.
+	"(?:hash(?:es|ed|ing)?|store|stores|stored|crypt|encrypt|use|using|used|recommend|recommends|prefer|prefers|should(?:\\s+(?:also|just))?|want)" +
+	"\\s+(?:(?:it|its|the|a|an|password|passwords|user|users|hash|hashed|with|using|as|via|by|plain|raw|fast|for|directly|simply|just|them|to)[ \\t]*\\n?[ \\t]*(?:-[ \\t]*)?){0,6}(?:md5|sha-?1|sha-?256|sha-?512)" +
+	`|(?:^|[^A-Za-z0-9_-])(?:md5|sha-?1|sha-?256|sha-?512)[^.;\n]{0,25}?\b(?:is|are|was|were|seems?|looks?|works?|would\s+be|should\s+be)\s+(?:a\s+|just\s+|still\s+|totally\s+|actually\s+)?(?:fine|good|great|safe|ok|okay|acceptable|perfect|sufficient|enough|best|better|the\s+(?:right|best)\s+choice)` +
+	")")
 
 // secWeakHashNamedAndNegated scores full credit only when the response BOTH
-// safely negates every mention of a weak/fast general-purpose hash (or
+// avoids endorsing any weak/fast general-purpose hash with no nearby
-// never mentions one) AND actually names at least one of them by name
+// negation (or never mentions one) AND actually names at least one of them
-// (md5/sha-1/sha-256/sha-512) - C11's fix folded into a single AND rather
+// by name (md5/sha-1/sha-256/sha-512) - C11's fix folded into a single AND
-// than an independent extra ContainsAny term, so a wrong answer that
+// rather than an independent extra ContainsAny term, so a wrong answer that
 // recommends SHA-256 outright cannot earn partial credit merely for having
 // typed "SHA-256" while doing so.
 func secWeakHashNamedAndNegated() eval.Evaluator {
 	named := eval.ContainsAny("md5", "sha-1", "sha1", "sha-256", "sha256")
-	negated := secNoUnnegatedMention(secWeakPasswordHashPattern)
+	negated := secNoUnnegatedMention(secWeakHashEndorsementPattern)
 	return eval.EvaluatorFunc(func(ctx context.Context, response string) eval.Score {
 		if n := negated.Evaluate(ctx, response); n.Value != 1 {
-			return eval.Score{Value: 0, Detail: "unnegated mention of a discouraged hash: " + n.Detail}
+			return eval.Score{Value: 0, Detail: "endorses a discouraged hash without negating it: " + n.Detail}
 		}
 		if m := named.Evaluate(ctx, response); m.Value != 1 {
 			return eval.Score{Value: 0, Detail: "never names a weak hash by name"}
 		}
-		return eval.Score{Value: 1, Detail: "a weak hash is named and every mention is safely negated"}
+		return eval.Score{Value: 1, Detail: "a weak hash is named and none is endorsed without negation"}
 	})
 }
 

internal/tests/security_crypto_test.go

@@ -28,6 +28,20 @@ func TestSecPasswordHashChoiceTest_Eval(t *testing.T) {
 			response: "bcrypt is fine. Don't use SHA-256 alone, it lacks a work factor.",
 			want:     1,
 		},
+		{
+			// Regression from the live probe (2026-08-29): a fully correct
+			// answer names the discouraged hashes as bullets under a negated
+			// heading ("Do **not** use ... including:"). The heading ends in
+			// ":" - not a clause boundary - so its cue must reach every bullet.
+			name: "correct probe answer: discouraged hashes as bullets under a negated heading",
+			response: "Use Argon2id (or bcrypt with cost >= 12) to store passwords.\n\n" +
+				"Do **not** use fast general-purpose cryptographic hashes for storing passwords, including:\n" +
+				"- **MD5**\n" +
+				"- **SHA-1**\n" +
+				"- **SHA-256**\n" +
+				"- **SHA-512**",
+			want: 1,
+		},
 		{
 			name:     "wrong: recommends SHA-256 unnegated",
 			response: "Just hash the password with SHA-256 before storing it.",