From cde7c7ce14a2cf3a39cf6801f83e03d4f66a3482 Mon Sep 17 00:00:00 2001 From: Lukasz Raczylo Date: Sun, 30 Aug 2026 12:37:38 +0100 Subject: [PATCH] Update --- README.md | 210 +++++++- RESULTS-2026-08-17.md | 277 ----------- cmd/llmtest/main.go | 110 ++++- cmd/llmtest/main_test.go | 40 ++ internal/report/compare.go | 221 +++++++++ internal/report/compare_test.go | 114 +++++ internal/report/discrimination.go | 114 +++++ internal/report/discrimination_test.go | 86 ++++ internal/report/health.go | 222 +++++++++ internal/report/health_test.go | 200 ++++++++ internal/report/json.go | 113 ++++- internal/report/markdown.go | 29 +- internal/report/report.go | 226 ++++++--- internal/report/report_test.go | 63 ++- internal/report/table.go | 93 +++- internal/runner/runner.go | 30 +- internal/runner/runner_test.go | 33 ++ internal/tests/catalog.go | 3 + internal/tests/catalog_test.go | 4 +- internal/tests/databases_redis.go | 55 ++- internal/tests/databases_redis_test.go | 25 +- internal/tests/operations_scenario.go | 538 +++++++++++++++++++++ internal/tests/operations_scenario_test.go | 357 ++++++++++++++ internal/tests/programming_hard.go | 469 ++++++++++++++++++ internal/tests/programming_hard_test.go | 360 ++++++++++++++ internal/tests/saturation.go | 534 ++++++++++++++++++++ internal/tests/saturation_test.go | 400 +++++++++++++++ internal/tests/security_crypto.go | 42 +- internal/tests/security_crypto_test.go | 14 + 29 files changed, 4548 insertions(+), 434 deletions(-) delete mode 100644 RESULTS-2026-08-17.md create mode 100644 internal/report/compare.go create mode 100644 internal/report/compare_test.go create mode 100644 internal/report/discrimination.go create mode 100644 internal/report/discrimination_test.go create mode 100644 internal/report/health.go create mode 100644 internal/report/health_test.go create mode 100644 internal/tests/operations_scenario.go create mode 100644 internal/tests/operations_scenario_test.go create mode 100644 internal/tests/programming_hard.go create mode 100644 internal/tests/programming_hard_test.go create mode 100644 internal/tests/saturation.go create mode 100644 internal/tests/saturation_test.go diff --git a/README.md b/README.md index 702414a..e79f40d 100644 --- a/README.md +++ b/README.md @@ -64,9 +64,17 @@ Both `list` and `run` accept: - `--models` - comma-separated model list, overriding the config file. - `--format` - `table`, `markdown`, or `json`. Default: `table`. +- `--tests` - comma-separated exact test IDs (no globs), overriding + `--category`/`--subcategory` - for probing individual tests cheaply. +- `--repeat N` - sample each (model, test) pair N times. Sampling + temperature is pinned to 0, so disagreement between repeats + exposes response instability (server-side batching, MoE routing) rather + than decoding randomness. Default: 1. - `--concurrency` - override the config file's `concurrency`. - `--timeout` - override the config file's `request_timeout` (for example `60s`). +- `--out FILE` - write the run as a JSON artifact (raw per-attempt results + plus the per-test discrimination rollup) for later `llmtest compare`. - `--quiet` - suppress progress output on stderr. Progress is on by default: a run can issue hundreds of requests at 30-50s each, and a silent CLI reads as a hang. @@ -78,6 +86,54 @@ with a `TRUNCATED` marker when a response was cut off by the token budget). stdout carries only the table/markdown/json report, so `llmtest run --format json > results.json` stays pipeable regardless of `--quiet`. +With `--repeat N>1`, each (model, test) cell reports the mean over +attempts; when attempts disagreed, the cell appends the `[min-max]` range +and the discrimination rollup flags the test as unstable. + +### Comparing two runs + +`compare` diffs two saved artifacts (from `--out`) per (model, test) pair. +It re-aggregates each side with the same truncation policy the report +uses, so a comparison stays honest even when only some attempts were cut +off. A mean drop of >= 0.01 is a regression, an equal-size rise an +improvement; tests present on only one side are listed as +`baseline-only`/`current-only`. The exit code reflects only diff +failures (unreadable artifacts), not regressions: + +```sh +go run ./cmd/llmtest run --out baseline.json +# ... change config, upgrade the endpoint, swap model ... +go run ./cmd/llmtest run --out current.json +go run ./cmd/llmtest compare baseline.json current.json +``` + +### Auditing suite health + +`health` reads one or more saved artifacts and rolls them up per +subcategory: what share of (model, test) cells score perfect, how many +tests are saturated (perfect in >= 90% of their cells, so they cannot +separate models), and which tests still carry signal. Cells pool across +artifacts taking each (model, test) pair's WORST artifact, so a lucky +re-run cannot hide a failure; subcategories at 100% perfect across every +model contribute zero ranking information and need harder tests: + +```sh +go run ./cmd/llmtest run --out run1.json +go run ./cmd/llmtest health run1.json # one artifact +go run ./cmd/llmtest health run1.json run2.json # pooled audit +``` + +### Reading the discrimination rollup + +`table` and `markdown` output end with a `Discrimination` section. Its +headline counts the tests whose cross-model score spread reaches +`DiscriminationSpread` (0.05) - the tests that actually separate models - +plus the tests whose repeated attempts disagreed. A suite where nearly +every test scores 1.00 for every model measures nothing about relative +capability, and this section makes that visible per test instead of +averaging it away; with a single model there is no spread, so the section +reports instability only. + If `config.yaml` omits `concurrency`, `request_timeout`, or `max_tokens_default`, `internal/config` fills in `8`, `300s`, and `12000`. The effective per-call token budget is `max(test.MaxTokens, @@ -108,16 +164,22 @@ needs a local toolchain that is not installed (for example, a Python exec-evaluator when `python3` is missing). Skipped tests are excluded from every mean. -A cell carries a trailing `!` when the model's response was cut off by the -token budget (`finish_reason=length`) - for example `0.00 (6000tok)!`. A -truncated response's score is still whatever the evaluator computed on the -partial text, so a `!` cell needs a closer look: raise -`max_tokens_default` and re-run. `table` and `markdown` print a one-line -legend explaining `!` whenever at least one cell has it. `json` carries the -same signal per result as `"truncated": true`, plus `"finish_reason"` and -`"response_text"` (the exact normalized text that was scored) so a -truncated or unexpectedly low-scoring result can be debugged without -re-running the model. +A cell carries a trailing `!` when at least one of its attempts was cut +off by the token budget (`finish_reason=length`) - for example +`1.00 (420tok)!`. Truncation policy: a truncated attempt was scored on +partial text, so its score measures the token budget as much as the +model. Attempts that finished normally are therefore the only ones +averaged into the cell's mean and its `[min-max]` range; the truncated +sample is discarded and marked by the `!`. Only when every scored attempt +was truncated does the mean fall back to the partial-text scores - then +the `!` means the whole cell is unreliable and `max_tokens_default` needs +rising before the number can be read as capability. `compare` reuses this +policy, so a truncated sample in one artifact cannot fake a regression. +`table` and `markdown` print a one-line legend explaining `!` whenever at +least one cell has it. `json` carries the signal per attempt as +`"truncated": true`, plus `"finish_reason"` and `"response_text"` (the +exact normalized text that was scored) so a truncated or unexpectedly +low-scoring result can be debugged without re-running the model. ## Test catalog @@ -129,20 +191,117 @@ is cheap, the test file's `_test.go` recomputes it independently (for example, `internal/tests/python_test.go` recomputes the cosine similarity with `math.Sqrt` rather than trusting the constant). +1063 tests across 105 subcategories (each holds at least 10, enforced by +`internal/tests/catalog_test.go`). Subcategory minimums and the exact ID +list are the source of truth; `llmtest list` prints them live. + | Category / subcategory | Tests | | --- | --- | -| programming / golang | `go-struct-align`, `go-worker-pool`, `go-semver-classify` | -| programming / python | `py-log-triage`, `py-cosine` | -| programming / typescript | `ts-debounce-composable` | -| programming / c | `c-struct-size` | -| operations / macos | `macos-timeout-portability`, `macos-launchd-cron` | -| operations / linux | `linux-pct-exec`, `linux-systemd-oneshot` | -| operations / kubernetes | `k8s-crashloop-gitops` | -| research / web | `web-robots-ai-crawlers` | -| research / whitepapers | `paper-hnsw-params` | -| research / codebase | `code-trace-go` | -| agents / tool-routing | `agent-tool-routing` | -| agents / planning | `agent-plan-ordering` | +| programming / golang (10) | `go-bounded-fanout`, `go-channel-deadlock`, `go-context-cancellation-trace`, `go-defer-recover-trace`, `go-generics-constraint-fix`, `go-method-set-interface`, `go-semver-classify`, `go-slice-append-aliasing`, `go-struct-align`, `go-worker-pool` | +| programming / python (10) | `py-asyncio-gather-trace`, `py-cosine`, `py-dict-comprehension-trace`, `py-generator-exhaustion-trace`, `py-json-transform`, `py-log-triage`, `py-mutable-default-arg`, `py-pathlib-rewrite`, `py-regex-log-extraction`, `py-softmax` | +| programming / typescript (10) | `ts-array-method-chain`, `ts-computed-vs-watch`, `ts-debounce-composable`, `ts-discriminated-union-narrowing`, `ts-eslint-flat-config`, `ts-generic-utility-type`, `ts-optional-chaining-nullish`, `ts-promise-allsettled-trace`, `ts-tsconfig-strict-flag`, `ts-vue-ref-reactive-unwrap` | +| programming / c (10) | `c-array-decay-sizeof`, `c-bitmask-ops`, `c-integer-promotion-overflow`, `c-macro-expansion-pitfall`, `c-pointer-arithmetic`, `c-string-function-output`, `c-struct-bitfield-packing`, `c-struct-size`, `c-undefined-behavior-spot`, `c-union-size-lp64` | +| programming / hard (12) | `hard-context-timeout`, `hard-defer-named-return`, `hard-float-binadd`, `hard-go-paren-balance`, `hard-goroutine-race`, `hard-int-division`, `hard-json-unmarshal-null`, `hard-nullish-over-or`, `hard-promise-allsettled`, `hard-py-version-sort`, `hard-regex-greedy`, `hard-var-shadow` | +| operations / macos (10) | `macos-apfs-snapshot`, `macos-caffeinate`, `macos-launchctl-bootstrap`, `macos-launchd-cron`, `macos-mdfind-spotlight`, `macos-plutil-plist-read`, `macos-sed-inplace`, `macos-ssh-identity-agent`, `macos-stat-bsd-gnu`, `macos-timeout-portability` | +| operations / linux (10) | `linux-cgroup-oom-diagnosis`, `linux-cron-expression`, `linux-journalctl-filter`, `linux-lvm-extend-order`, `linux-nftables-dnat`, `linux-pct-exec`, `linux-proc-meminfo-available`, `linux-ssh-port-forward`, `linux-systemd-oneshot`, `linux-systemd-timer` | +| operations / kubernetes (10) | `k8s-argocd-outofsync`, `k8s-cnpg-pdb`, `k8s-crashloop-gitops`, `k8s-imagepullbackoff`, `k8s-networkpolicy-dns-block`, `k8s-pending-taints`, `k8s-pvc-storageclass-mismatch`, `k8s-qos-class`, `k8s-service-selector-mismatch`, `k8s-traefik-ingressroute-host` | +| operations / scenario (11) | `scen-backup-restore-order`, `scen-certexpiry-triage`, `scen-crashloop-order`, `scen-diskpressure-firstcmd`, `scen-inode-exhaustion`, `scen-log-rootcause-mcq`, `scen-lvm-resize-order`, `scen-net-layer-isolate`, `scen-portbind-fail`, `scen-service-degrade`, `scen-systemd-failreason` | +| research / web (12) | `web-canonical-vs-redirect`, `web-cors-missing-header`, `web-dns-mx-record`, `web-hreflang-reciprocal`, `web-hreflang-self-reference-trap`, `web-html-product-extract`, `web-http-status-scenarios`, `web-robots-ai-crawlers`, `web-robots-ua-scope-trap`, `web-security-headers-audit`, `web-sitemap-max-urls`, `web-url-parse-components` | +| research / whitepapers (10) | `paper-attention-scale-factor`, `paper-bloom-fp-rate`, `paper-btree-height-levels`, `paper-cap-availability-choice`, `paper-hnsw-params`, `paper-lsm-write-amplification`, `paper-pq-compression-ratio`, `paper-raft-quorum-failures`, `paper-recall-at-k`, `paper-tfidf-term-score` | +| research / codebase (10) | `code-bigo-dedup`, `code-bug-line-diff`, `code-commit-bisect`, `code-dead-functions`, `code-generic-return-type`, `code-import-cycle`, `code-race-variable`, `code-trace-go`, `code-trace-python`, `code-trace-ts` | +| agents / tool-routing (10) | `agent-tool-routing`, `agent-tool-routing-ambiguous`, `agent-tool-routing-chaining`, `agent-tool-routing-cheapest`, `agent-tool-routing-distractors`, `agent-tool-routing-missing-param`, `agent-tool-routing-multistep`, `agent-tool-routing-no-tool-needed`, `agent-tool-routing-parallel`, `agent-tool-routing-safety` | +| agents / planning (10) | `agent-plan-critical-path`, `agent-plan-cycle-detect`, `agent-plan-idempotency`, `agent-plan-milestone-date`, `agent-plan-minimal-replan`, `agent-plan-ordering`, `agent-plan-parallel-steps`, `agent-plan-repair`, `agent-plan-resource-constrained`, `agent-plan-rollback-trigger` | +| agents / delegation (12) | `agent-deleg-batch-vs-separate`, `agent-deleg-build-then-verify`, `agent-deleg-capacity-trap`, `agent-deleg-escalation-to-human`, `agent-deleg-handoff-context`, `agent-deleg-main-thread-vs-delegate`, `agent-deleg-minimal-privilege`, `agent-deleg-near-miss-roster`, `agent-deleg-parallel-dispatch-safety`, `agent-deleg-reviewer-independence`, `agent-deleg-task-to-specialist`, `agent-deleg-verify-vs-trust` | +| databases / postgres (12) | `pg-bloat-vacuum`, `pg-count-star-vs-join`, `pg-deadlock-lock-order`, `pg-explain-seq-vs-index`, `pg-index-choice`, `pg-isolation-anomaly`, `pg-listen-notify`, `pg-n-plus-one`, `pg-partial-index`, `pg-pool-sizing`, `pg-replica-failover`, `pg-timeout-trap` | +| databases / redis (10) | `redis-cache-stampede`, `redis-incr-atomicity`, `redis-keyspace-anti-pattern`, `redis-lua-atomicity`, `redis-memory-estimate`, `redis-multi-vs-pipeline`, `redis-noeviction-write`, `redis-pubsub-delivery`, `redis-scan-vs-keys`, `redis-structure-choice` | +| databases / sql-tuning (10) | `sql-aggregate-sum`, `sql-composite-index-skip-column`, `sql-covering-index`, `sql-equivalent-rewrite`, `sql-groupby-having`, `sql-join-counts`, `sql-keyset-pagination`, `sql-null-in-where`, `sql-orderby-index-serving`, `sql-window-rank` | +| security / appsec (12) | `sec-csrf-post-trap`, `sec-csrf-requirement`, `sec-idor-spot`, `sec-input-validation-boundary`, `sec-open-redirect-spot`, `sec-orm-interpolation-trap`, `sec-path-traversal-spot`, `sec-rate-limit-placement`, `sec-secret-log-spot`, `sec-sqli-spot`, `sec-ssrf-spot`, `sec-xss-sink-spot` | +| security / crypto (10) | `sec-aes-gcm-nonce-reuse`, `sec-cert-chain-validation`, `sec-constant-time-compare`, `sec-hash-vs-encrypt-pii`, `sec-hmac-vs-signature`, `sec-jwt-alg-none`, `sec-password-hash-choice`, `sec-rand-source-choice`, `sec-rotate-order`, `sec-tls-floor-version` | +| security / secrets (12) | `sec-diff-secret-spot`, `sec-error-path-leak-trap`, `sec-fork-pr-ci-exposure`, `sec-gitignore-history-nuance`, `sec-hardcoded-secret-spot`, `sec-k8s-secret-base64`, `sec-least-privilege-scope`, `sec-remediation-order`, `sec-rotation-no-downtime`, `sec-rotation-window-math`, `sec-ssh-agent-identity`, `sec-vault-tradeoff` | +| delivery / git (10) | `git-bisect-steps`, `git-cherry-pick-vs-revert`, `git-conventional-commit-classify`, `git-detached-head-recovery`, `git-force-with-lease-vs-force`, `git-gitignore-negation-trace`, `git-hook-choice`, `git-rebase-vs-merge`, `git-stash-pop-conflict`, `git-worktree-use-case` | +| delivery / containers (10) | `docker-build-arg-vs-env`, `docker-copy-vs-add`, `docker-dockerignore-effect`, `docker-entrypoint-cmd-trace`, `docker-healthcheck-semantics`, `docker-image-size-math`, `docker-layer-cache-bust`, `docker-layer-count`, `docker-multistage-size-benefit`, `docker-nonroot-user-caps` | +| delivery / release-engineering (10) | `rel-canary-vs-bluegreen`, `rel-checksums-signing-purpose`, `rel-clean-tree-requirement`, `rel-commit-changelog-mapping`, `rel-hotfix-flow-ordering`, `rel-pin-vs-range-policy`, `rel-pipeline-stage-ordering`, `rel-rollback-ordering`, `rel-semver-bump-changelog`, `rel-tag-to-release-sequence` | +| ai / vector-search (10) | `vec-cosine-vs-dot`, `vec-distance-to-similarity`, `vec-embedding-dimension-tradeoff`, `vec-hnsw-efsearch-tradeoff`, `vec-index-build-query-tradeoff`, `vec-near-duplicate-threshold`, `vec-pq-memory-math`, `vec-pre-vs-post-filtering`, `vec-recall-at-k`, `vec-rrf-fusion` | +| ai / llm-integration (10) | `llm-429-retry-backoff`, `llm-context-overflow-strategy`, `llm-embedding-batch-math`, `llm-field-semantics`, `llm-role-placement`, `llm-sse-stream-done`, `llm-stop-sequence-trace`, `llm-temperature-zero-caveat`, `llm-token-budget-reasoning`, `llm-tool-call-handling` | +| ai / rag (10) | `rag-chunk-size-tradeoff`, `rag-citation-grounding`, `rag-context-selection-vs-stuffing`, `rag-eval-metric-choice`, `rag-hallucination-mitigation-ordering`, `rag-index-staleness`, `rag-multihop-decomposition`, `rag-preassembly-dedup`, `rag-reranker-placement`, `rag-retrieval-failure-mode` | +| daily / text-processing (10) | `dtext-cron-dst-skip`, `dtext-csv-revenue`, `dtext-duration-sum`, `dtext-invoice-vat-rounding`, `dtext-json-merge-patch`, `dtext-meeting-zones`, `dtext-percent-symmetry-trap`, `dtext-sla-date-math`, `dtext-url-components`, `dtext-utf8-bytes-vs-chars` | +| daily / terminal (10) | `dterm-cp-n-versus-mv`, `dterm-du-total-max`, `dterm-find-mtime-plus7`, `dterm-grep-lines-vs-matches`, `dterm-log-top-2xx-path`, `dterm-pipefail-exit-code`, `dterm-ps-memory-column`, `dterm-single-quote-expansion`, `dterm-strip-port-uniq-count`, `dterm-tail-grep-buffering` | +| daily / office-math (10) | `doffice-discount-tax-order`, `doffice-file-quota-gib`, `doffice-fx-wire-fee`, `doffice-mileage-threshold`, `doffice-overtime-pay`, `doffice-payroll-loaded`, `doffice-prorated-fee`, `doffice-reams-ceiling`, `doffice-shift-roster`, `doffice-uptime-budget` | +| daily / policy-rules (10) | `dpol-approval-boundaries`, `dpol-cert-expiry-window`, `dpol-expense-per-night-cap`, `dpol-holiday-premium`, `dpol-marginal-brackets`, `dpol-oncall-rotation`, `dpol-retention-window`, `dpol-shipping-business-days`, `dpol-sla-tiered-credit`, `dpol-vacation-accrual-cap` | +| daily / scheduling (10) | `dsched-billing-anchor-clamp`, `dsched-contract-notice-window`, `dsched-deadline-countdown`, `dsched-dst-backup-hours`, `dsched-dst-meeting-overlap`, `dsched-flight-cross-tz`, `dsched-payroll-period-walk`, `dsched-rota-mod7`, `dsched-utc-cross-date`, `dsched-weekend-shift-deadline` | +| daily / shopping (10) | `dshop-bill-split-penny`, `dshop-buy-one-free`, `dshop-coupon-threshold`, `dshop-free-shipping-threshold`, `dshop-fuel-range-check`, `dshop-fuel-trip-cost`, `dshop-pack-mix-optimum`, `dshop-stacked-discount`, `dshop-unit-price-per-litre`, `dshop-vat-cashback` | +| daily / finance (10) | `dfin-commute-season-ticket`, `dfin-compound-vs-simple`, `dfin-deposit-cap-weekly`, `dfin-fx-round-trip`, `dfin-income-tax-single-band`, `dfin-installment-vs-lump`, `dfin-min-payment-interest`, `dfin-prorata-rent`, `dfin-raise-compounding`, `dfin-simple-interest-18mo` | +| daily / travel (10) | `dtrv-baggage-started-kg`, `dtrv-car-hire-80h`, `dtrv-hotel-3-nights`, `dtrv-layover-share`, `dtrv-mileage-tiers`, `dtrv-taxi-split-surge`, `dtrv-three-city-meeting`, `dtrv-toll-vs-detour`, `dtrv-train-connection`, `dtrv-visa-window` | +| daily / cooking (10) | `dck-bakers-percentages`, `dck-coffee-unit-price`, `dck-cordial-dilution`, `dck-dinner-parallel-starts`, `dck-granola-label-math`, `dck-pan-area-scaling`, `dck-roast-start-timing`, `dck-scale-recipe-4-to-10`, `dck-starter-feed-ratio`, `dck-temp-convert-probe` | +| daily / fitness (10) | `dft-acsm-incline-walk`, `dft-cadence-step-goal`, `dft-energy-balance-weeks`, `dft-ftp-interval-work`, `dft-karvonen-zones`, `dft-lean-mass-target`, `dft-pace-from-5k`, `dft-plank-compound-weeks`, `dft-swim-set-totals`, `dft-tenk-decimal-pace` | +| daily / gardening (10) | `dgd-bed-plant-spacing`, `dgd-compost-cn-alligation`, `dgd-fertiliser-season-bags`, `dgd-hedge-compound-prune`, `dgd-irrigation-zone-cycle`, `dgd-mower-throughput-overlap`, `dgd-mulch-volume-bags`, `dgd-rain-equivalent-watering`, `dgd-raised-bed-soil-bags`, `dgd-sowing-transplant-dates` | +| daily / home-repair (10) | `drp-bath-mix-fill-time`, `drp-concrete-post-hole-bags`, `drp-door-refix-timeline`, `drp-ladder-pythagoras-rule`, `drp-laminate-planks-with-waste`, `drp-led-swap-energy-savings`, `drp-paint-two-coat-cans`, `drp-shelf-bracket-fencepost`, `drp-thermostat-setback-savings`, `drp-tile-count-breakage` | +| daily / car (10) | `dcar-average-speed-restart`, `dcar-detailing-overlap`, `dcar-dilution-ratio`, `dcar-hire-mileage-charge`, `dcar-mpg-to-litres`, `dcar-parking-blocks`, `dcar-service-whichever-first`, `dcar-tank-range-reserve`, `dcar-tread-life`, `dcar-trip-fuel-cost` | +| daily / events (10) | `dev-catering-crate-wastage`, `dev-coaches-per-started`, `dev-deposit-after-discount`, `dev-flower-breakage-budget`, `dev-keg-pint-count`, `dev-place-card-sheets`, `dev-setup-working-days`, `dev-shift-block-overshoot`, `dev-table-grid-vs-area`, `dev-wedding-timeline` | +| daily / medication (10) | `dmed-antibiotic-course-volume`, `dmed-bioavailability-tablets`, `dmed-combo-analgesic-cap`, `dmed-insulin-daily-total`, `dmed-iv-drip-rate`, `dmed-pack-refill-count`, `dmed-suspension-dose-volume`, `dmed-tablet-cost-per-pack`, `dmed-unit-conversion-excretion`, `dmed-weight-based-dose` | +| daily / hiking (10) | `dhike-average-speed-halves`, `dhike-descent-time-allowance`, `dhike-drive-ferry-arrival`, `dhike-food-resupply-days`, `dhike-heat-water-fuel`, `dhike-map-scale-ground-time`, `dhike-naismith-summit-time`, `dhike-required-split-pace`, `dhike-tent-count-weight`, `dhike-water-gap-refill` | +| daily / energy (10) | `den-battery-arbitrage-losses`, `den-carbon-tier-bands`, `den-draught-payback-share`, `den-ev-overnight-charge`, `den-heat-pump-vs-gas-cost`, `den-load-profile-weighted-mean`, `den-oil-litres-efficiency`, `den-solar-self-consumption`, `den-standing-charge-days`, `den-tou-bill-supply-charge` | +| daily / childcare (10) | `dch-age-in-days`, `dch-daycare-funded-hours`, `dch-formula-annual-cost`, `dch-formula-can-count`, `dch-milk-bank-net-days`, `dch-night-sleep-minutes`, `dch-paracetamol-suspension-dose`, `dch-twin-feed-total`, `dch-volume-per-kg`, `dch-wake-window-clock` | +| daily / insurance (10) | `dins-annual-premium-discount`, `dins-contents-depreciation`, `dins-deductible-coinsurance`, `dins-excess-per-claim`, `dins-family-deductible-met`, `dins-income-protection-gap`, `dins-out-of-pocket-cap`, `dins-pet-annual-limit`, `dins-rental-damage-waiver`, `dins-waiting-period-date` | +| daily / nutrition (10) | `dnut-alcohol-units-abv`, `dnut-bmi-weight-band`, `dnut-caffeine-daily-cap`, `dnut-drink-sugar-averages`, `dnut-hydration-bottles`, `dnut-macro-energy-split`, `dnut-percent-dv-two-servings`, `dnut-protein-scoop-rounding`, `dnut-serving-scale-pack`, `dnut-tdee-deficit-target` | +| daily / pets (10) | `dpets-cat-calories`, `dpets-dog-food-percent`, `dpets-dog-walk-km`, `dpets-dog-water`, `dpets-flea-boxes`, `dpets-grooming-cycle`, `dpets-med-mg-kg`, `dpets-puppy-vaccines`, `dpets-rabbit-hay`, `dpets-tank-volume` | +| daily / commuting (10) | `dcomm-bus-interval`, `dcomm-carpool-rebate`, `dcomm-cycle-arrival`, `dcomm-fuel-share`, `dcomm-metro-platform`, `dcomm-office-cost`, `dcomm-parking-blocks`, `dcomm-rail-payg-vs-pass`, `dcomm-railcard-breakeven`, `dcomm-toll-vs-free` | +| daily / laundry (10) | `dlau-concentrate-dose`, `dlau-damp-dry-clock`, `dlau-delicates-loads`, `dlau-detergent-pricing`, `dlau-fleece-blanket`, `dlau-home-vs-laundrette`, `dlau-mixed-load-cycles`, `dlau-pod-schedule`, `dlau-sanitiser-dilution`, `dlau-washer-energy` | +| daily / woodwork (10) | `dwood-decking-boards`, `dwood-mitre-posts`, `dwood-mortar-waste`, `dwood-offcut-kerf`, `dwood-oil-coats`, `dwood-post-holes`, `dwood-price-break`, `dwood-screw-boxes`, `dwood-timber-invoice`, `dwood-varnish-tins` | +| daily / grow (10) | `dgrow-fert-tank`, `dgrow-germination`, `dgrow-heat-pump`, `dgrow-light-bill`, `dgrow-mix-ratio`, `dgrow-nutrient-dose`, `dgrow-reservoir-bottles`, `dgrow-seed-cost`, `dgrow-seed-trays`, `dgrow-watering-cans` | +| daily / home-maintenance (10) | `dhr-caulking-tubes`, `dhr-dryer-annual`, `dhr-filter-packs`, `dhr-insulation-payback`, `dhr-mower-runtime`, `dhr-pool-shock`, `dhr-render-time`, `dhr-service-plan`, `dhr-tank-ibc`, `dhr-tank-top-up` | +| daily / bathroom-plumbing (10) | `dbp-anti-slip-mats`, `dbp-bath-fill`, `dbp-descaling`, `dbp-drip-line`, `dbp-gutter-run`, `dbp-heater-energy`, `dbp-leak-waste`, `dbp-shower-annual`, `dbp-sump-pump`, `dbp-wall-tiles` | +| daily / cleaning-supplies (10) | `dcs-bin-liners`, `dcs-bleach-sprayers`, `dcs-cart-discount`, `dcs-cleaner-hours`, `dcs-detergent-packs`, `dcs-floor-polish`, `dcs-mop-buckets`, `dcs-tablets-vs-powder`, `dcs-vacuum-annual`, `dcs-window-sprayer` | +| daily / painting-decorating (10) | `dpaint-booth-energy`, `dpaint-crew-hours`, `dpaint-gloss-doors`, `dpaint-pot-sizes`, `dpaint-scaffold`, `dpaint-skirting`, `dpaint-sprayer-amort`, `dpaint-tint`, `dpaint-two-coats`, `dpaint-wallpaper` | +| daily / moving-house (10) | `dmov-box-packs`, `dmov-crate-trips`, `dmov-deposit-order`, `dmov-lift-tonnage`, `dmov-piano-surcharge`, `dmov-route-fuel`, `dmov-transit-insurance`, `dmov-two-quotes`, `dmov-utility-final`, `dmov-van-trips` | +| daily / diy-electrical (10) | `delec-breaker`, `delec-cable`, `delec-downlights`, `delec-duty-cycle`, `delec-festoon`, `delec-heat-mat`, `delec-kettle`, `delec-router-sla`, `delec-sleeves`, `delec-surge` | +| daily / waste-recycling (10) | `dwaste-bin-days`, `dwaste-bottles`, `dwaste-bulbs`, `dwaste-compost`, `dwaste-ewaste`, `dwaste-green-bags`, `dwaste-landfill`, `dwaste-oil`, `dwaste-shredder`, `dwaste-skips` | +| daily / diy-pest-control (10) | `dpest-ant-bait-mix`, `dpest-bait-stations`, `dpest-flea-fogger`, `dpest-mosquito-traps`, `dpest-pest-contract`, `dpest-rodent-blocks`, `dpest-snake-fence`, `dpest-spray-perimeter`, `dpest-termite-warranty`, `dpest-wasp-plan` | +| daily / home-safety (10) | `dsafe-cctv-archive`, `dsafe-extinguisher-service`, `dsafe-first-aid-stock`, `dsafe-handrail-fixings`, `dsafe-inspection-penalty`, `dsafe-ladder-hire`, `dsafe-monitoring-exit`, `dsafe-nightlight-annual`, `dsafe-smoke-batteries`, `dsafe-vent-fan` | +| daily / water-usage (10) | `dwat-bath-heat`, `dwat-filter-vs-jug`, `dwat-hose-vs-bucket`, `dwat-leak-trace`, `dwat-meter-bill`, `dwat-pool-fill`, `dwat-shower-vs-bath`, `dwat-softener-salt`, `dwat-sprinkler`, `dwat-tank-payback` | +| daily / home-security (10) | `dsec-doorbell-battery`, `dsec-dummy-camera`, `dsec-fake-presence`, `dsec-grille-bars`, `dsec-hasp-screws`, `dsec-monitoring-plan`, `dsec-motion-sensors`, `dsec-photocell`, `dsec-siren-cells`, `dsec-window-film` | +| daily / cooling-load (10) | `dcool-ac-size`, `dcool-car-ac`, `dcool-dehumidifier`, `dcool-evap-vs-ac`, `dcool-fan-energy`, `dcool-fan-vs-ac`, `dcool-filter-cleans`, `dcool-ice-blocks`, `dcool-thermostat`, `dcool-window-shading` | +| daily / telecom-plans (10) | `dtel-broadband-speed`, `dtel-contract-uplift`, `dtel-data-overage`, `dtel-family-plan`, `dtel-modem-power`, `dtel-payg-vs-bundle`, `dtel-prepaid-card`, `dtel-roaming-cap`, `dtel-sim-plan`, `dtel-tether-data` | +| daily / heating-load (10) | `dheat-boiler-kwh`, `dheat-cylinder`, `dheat-duty-cycle`, `dheat-heat-pump`, `dheat-insulation`, `dheat-oil-tank`, `dheat-radiator`, `dheat-tariff`, `dheat-thermostat`, `dheat-zoning` | +| daily / timesheet-pay (10) | `dpay-bonus-tax`, `dpay-clock-rounding`, `dpay-contractor-day`, `dpay-leave-accrual`, `dpay-mileage`, `dpay-overtime`, `dpay-parttime-prorata`, `dpay-salary-raise`, `dpay-shift-differential`, `dpay-tax-band` | +| daily / fuel-trip (10) | `dfuel-boat-range`, `dfuel-economy-mixed`, `dfuel-ev-charge`, `dfuel-flight-uplift`, `dfuel-generator-kwh`, `dfuel-hedge-mix`, `dfuel-jerrycan-trip`, `dfuel-lpg-payback`, `dfuel-moto-range`, `dfuel-truck-legs` | +| daily / pool-chemicals (10) | `dpool-algae-shock`, `dpool-backwash`, `dpool-floc`, `dpool-heat-pump`, `dpool-ph-adjust`, `dpool-ppm-dose`, `dpool-salt-cell`, `dpool-turnover`, `dpool-uv`, `dpool-volume-m3` | +| daily / garden-irrigation (10) | `dirr-drip-emitters`, `dirr-drip-flow`, `dirr-fert-spread`, `dirr-greenhouse-mist`, `dirr-hose-fill`, `dirr-pond-topup`, `dirr-soil-compost`, `dirr-sprinkler`, `dirr-timer-days`, `dirr-water-tank` | +| daily / aquarium-water (10) | `daqu-chiller`, `daqu-co2`, `daqu-filter-flow`, `daqu-lighting`, `daqu-medication`, `daqu-nitrification`, `daqu-potash`, `daqu-salt-mix`, `daqu-tank-litres`, `daqu-waterchange` | +| daily / ev-charging (10) | `devch-ac-session`, `devch-battery-value`, `devch-cable-loss`, `devch-dc-fast`, `devch-degradation`, `devch-fleet-balancer`, `devch-range-added`, `devch-solar-surplus`, `devch-toll-pass`, `devch-tou-window` | +| daily / tile-flooring (10) | `dtile-adhesive-coverage`, `dtile-border-run`, `dtile-budget`, `dtile-cut-waste`, `dtile-drain-slope`, `dtile-grid-count`, `dtile-grout-volume`, `dtile-levelling`, `dtile-room-area`, `dtile-skirting-cost` | +| daily / decking-fencing (10) | `ddeck-board-area`, `ddeck-concrete-footings`, `ddeck-fence-panel-run`, `ddeck-gate-hinges`, `ddeck-joist-spacing`, `ddeck-oil-restoration`, `ddeck-railing-load`, `ddeck-slope-steps`, `ddeck-stair-stringer`, `ddeck-subframe-clamps` | +| daily / loft-insulation (10) | `dloft-blown-depth`, `dloft-cistern-insul`, `dloft-downlight-covers`, `dloft-hatch-draft`, `dloft-heat-loss`, `dloft-knee-wall`, `dloft-pipe-jacket`, `dloft-spray-foam`, `dloft-target-u`, `dloft-vapour-barrier` | +| daily / coffee-brewing (10) | `dbrew-caffeine-budget`, `dbrew-cold-brew-yield`, `dbrew-cup-cost`, `dbrew-descale-tablets`, `dbrew-espresso-shots`, `dbrew-extraction-yield`, `dbrew-french-press`, `dbrew-grind-throughput`, `dbrew-mineral-dose`, `dbrew-syrup-ratio` | +| daily / laundry-care (10) | `dlc-bleach-dilution`, `dlc-detergent-cost`, `dlc-drum-load`, `dlc-dryclean-vs-home`, `dlc-dryer-energy`, `dlc-load-split`, `dlc-softener-vs-vinegar`, `dlc-spin-extraction`, `dlc-stain-soak-supply`, `dlc-water-use` | +| daily / bike-care (10) | `dbike-battery-range`, `dbike-brake-pad-cost`, `dbike-chain-lifetime`, `dbike-chain-wear`, `dbike-energy-gels`, `dbike-gear-development`, `dbike-hire-mileage`, `dbike-patch-box`, `dbike-tuneup-vs-parts`, `dbike-tyre-tread` | +| daily / houseplant-care (10) | `dhpl-cutting-trays`, `dhpl-fertilizer-bottle`, `dhpl-foot-candle-convert`, `dhpl-grow-light-bill`, `dhpl-neem-spray-rounds`, `dhpl-repot-soil-volume`, `dhpl-reservoir-top-up`, `dhpl-shelf-bundle-deal`, `dhpl-soil-mix-split`, `dhpl-watering-schedule` | +| daily / lawn-care (10) | `dlawn-clippings`, `dlawn-fence-posts`, `dlawn-fertilizer-season`, `dlawn-irrigation`, `dlawn-mowing-time`, `dlawn-reseed-bags`, `dlawn-robot-wire`, `dlawn-spreader-fills`, `dlawn-topdress`, `dlawn-two-stroke` | +| daily / data-backup (10) | `dbak-321-plan`, `dbak-archive-drives`, `dbak-cloud-bill`, `dbak-lto-tapes`, `dbak-raid-usable`, `dbak-ransomware-anomaly`, `dbak-restore-time`, `dbak-snapshot-retention`, `dbak-upload-throughput`, `dbak-upload-window` | +| daily / pet-nutrition (10) | `dpn-bag-days`, `dpn-cat-water`, `dpn-joint-jar`, `dpn-kibble-cups`, `dpn-multi-cat-bags`, `dpn-puppy-meals`, `dpn-raw-mix`, `dpn-split-wet`, `dpn-treat-budget`, `dpn-wet-can-boxes` | +| daily / bread-baking (10) | `dbake-annual-flour`, `dbake-hydration`, `dbake-pan-volume`, `dbake-proof-rate`, `dbake-roll-division`, `dbake-scale`, `dbake-schedule`, `dbake-starter-hydration`, `dbake-water-temp`, `dbake-yeast-convert` | +| daily / sewing-tailoring (10) | `dsew-batting`, `dsew-bias-binding`, `dsew-button-spacing`, `dsew-elastic`, `dsew-fusible`, `dsew-layout`, `dsew-seam-allowance`, `dsew-shrinkage`, `dsew-thread-cone`, `dsew-zip-roll` | +| daily / bbq-grilling (10) | `dbbq-brine`, `dbbq-burger-packs`, `dbbq-coal-indirect`, `dbbq-low-slow`, `dbbq-propane`, `dbbq-rub-batch`, `dbbq-sauce-bottles`, `dbbq-service-queue`, `dbbq-thaw-schedule`, `dbbq-veg-trays` | +| daily / candle-making (10) | `dcand-batch-cost`, `dcand-blend-ratio`, `dcand-double-pour`, `dcand-jar-fill`, `dcand-market-margin`, `dcand-pool-ring`, `dcand-shared-batch`, `dcand-timeline`, `dcand-topup-bags`, `dcand-wick-spools` | +| daily / freshwater-fishkeeping (10) | `dfish-boat-current`, `dfish-fillet-yield`, `dfish-fry-hatch`, `dfish-grow-out`, `dfish-koi-ration`, `dfish-net-coverage`, `dfish-pond-liner`, `dfish-pond-volume`, `dfish-transport-crates`, `dfish-worm-harvest` | +| daily / home-canning (10) | `dcann-batch-cost`, `dcann-blanch-altitude`, `dcann-hard-boil`, `dcann-jam-pectin`, `dcann-jam-yield`, `dcann-kraut-salt`, `dcann-lid-packs`, `dcann-pickle-brine`, `dcann-pressure-session`, `dcann-sugar-per-jar` | +| daily / beekeeping (10) | `dbee-extraction-yield`, `dbee-fence-perimeter`, `dbee-hive-gain`, `dbee-mite-wash`, `dbee-nuc-split`, `dbee-queen-graft`, `dbee-season-revenue`, `dbee-super-fill`, `dbee-syrup-mix`, `dbee-winter-feed` | +| daily / knitting (10) | `dknit-blanket-squares`, `dknit-dye-wool`, `dknit-gauge-cast-on`, `dknit-hat-decrease`, `dknit-mitten-commission`, `dknit-rib-cuff`, `dknit-scarf-time`, `dknit-sock-pairs`, `dknit-sweater-yardage`, `dknit-yarn-skeins` | +| daily / soap-making (10) | `dsoap-curing-sales`, `dsoap-fragrance-load`, `dsoap-loaf-cut`, `dsoap-lye-bags`, `dsoap-lye-solution`, `dsoap-lye-strength`, `dsoap-market-profit`, `dsoap-mold-loaves`, `dsoap-oil-blend`, `dsoap-superfat` | +| daily / pottery-ceramics (10) | dpot-clay-bags`, `dpot-commission-discount`, `dpot-course-profit`, `dpot-drying-rate`, `dpot-glaze-batch`, `dpot-grog-share`, `dpot-kiln-energy`, `dpot-mug-margin`, `dpot-shrinkage`, `dpot-throwing-shift | +| daily / cheese-making (10) | dcheese-affinage-profit`, `dcheese-brine-saturation`, `dcheese-calcium-dose`, `dcheese-drying-loft`, `dcheese-milk-yield`, `dcheese-mother-stir`, `dcheese-mozzarella-sales`, `dcheese-press-force`, `dcheese-press-schedule`, `dcheese-rennet-dose | +| daily / home-brewing (10) | `dbeer-abv`, `dbeer-boil-energy`, `dbeer-chiller-time`, `dbeer-efficiency`, `dbeer-growler-margin`, `dbeer-hops-ibu`, `dbeer-mash-sparge`, `dbeer-priming-sugar`, `dbeer-strike-temp`, `dbeer-yeast-pitch` | +| daily / embroidery (10) | `demb-class-net`, `demb-floss-length`, `demb-hoop-area`, `demb-kit-profit`, `demb-pattern-scale`, `demb-quilt-edge`, `demb-ribbon-round`, `demb-runner-fabric`, `demb-stitch-time`, `demb-tassel-cut` | +| daily / jewelry-making (10) | `djew-bead-strand`, `djew-casting-sprue`, `djew-chain-price`, `djew-clay-shrink`, `djew-earring-wire`, `djew-jump-ring`, `djew-pendant-weight`, `djew-ring-blank`, `djew-wire-gauge`, `djew-wire-loop` | +| daily / snow-removal (10) | `dsnow-brine-mix`, `dsnow-ice-melt`, `dsnow-job-price`, `dsnow-plow-shifts`, `dsnow-roof-load`, `dsnow-salt-driveway`, `dsnow-salting-rounds`, `dsnow-sand-salt-stock`, `dsnow-shovel-rate`, `dsnow-snow-haul` | +| daily / camping (10) | `dcamp-tent-capacity`, `dcamp-pack-weight`, `dcamp-water-tablets`, `dcamp-gas-canister`, `dcamp-cooler-ice`, `dcamp-clothesline`, `dcamp-rope-load`, `dcamp-headlamp-battery`, `dcamp-firewood-split`, `dcamp-fee-per-night` | +| daily / photography (10) | `dphot-exposure-stops`, `dphot-hyperfocal`, `dphot-memory-card`, `dphot-golden-hour`, `dphot-print-size`, `dphot-film-contact`, `dphot-sandbag-packs`, `dphot-stock-royalty`, `dphot-lens-rental`, `dphot-timelapse-clock` | +| daily / records (10) | `dvrc-shipping-boxes`, `dvrc-sleeve-packs`, `dvrc-bulk-price`, `dvrc-play-hours`, `dvrc-flip-clock`, `dvrc-crate-load`, `dvrc-clean-fluid`, `dvrc-grading-score`, `dvrc-belt-life`, `dvrc-shelf-space` | +| daily / skiing (10) | `dski-lift-pass`, `dski-slope-grade`, `dski-wax-season`, `dski-lift-queue`, `dski-snowmaking`, `dski-boot-shell`, `dski-storage-fee`, `dski-kids-camp`, `dski-gear-value`, `dski-club-minibus` | +| daily / vanlife (10) | `dvlf-solar-array`, `dvlf-water-tank`, `dvlf-fuel-budget`, `dvlf-propane`, `dvlf-mileage`, `dvlf-plywood-cuts`, `dvlf-grey-water`, `dvlf-fridge-amps`, `dvlf-drive-time`, `dvlf-campsite-bill` | +| daily / arcade (10) | `darc-token-packs`, `darc-claw-net`, `darc-hoops-tickets`, `darc-dance-session`, `darc-hockey-league`, `darc-redemption-hours`, `darc-pinball-games`, `darc-ticket-dispenser`, `darc-photo-booth`, `darc-prize-wall` | +| daily / 3d-printing (10) | `d3dp-cost-per-print`, `d3dp-print-time`, `d3dp-bed-layout`, `d3dp-resin-batch`, `d3dp-spool-swaps`, `d3dp-support-share`, `d3dp-post-cure`, `d3dp-flow-limit`, `d3dp-power-bill`, `d3dp-stock-order` | +| daily / car-wash (10) | `dcw-foam-dilution`, `dcw-bay-throughput`, `dcw-water-reclaim`, `dcw-package-margin`, `dcw-clay-bars`, `dcw-vac-bags`, `dcw-tyre-dressing`, `dcw-queue-length`, `dcw-wash-plan`, `dcw-tip-pool` | ## Adding a test @@ -159,8 +318,9 @@ with `math.Sqrt` rather than trusting the constant). 4. Add table-driven `_test.go` cases covering a correct answer, a wrong answer, and at least one edge case (a common trap, a partial match, or a malformed response). -5. Add the test's ID to the `wantCatalog` map in - `internal/tests/catalog_test.go`. +5. If you introduced a new subcategory, add it to `wantSubcats` in + `internal/tests/catalog_test.go`; every subcategory needs at least ten + tests, so a one-test subcategory will not pass the catalog gate. ## Adding an evaluator @@ -194,4 +354,4 @@ golangci-lint run # if the binary is installed Every unit test runs offline against a mock `llm.Client` or a local `httptest.Server`; none of them call the live endpoint. The exec-evaluator tests (`GoRun`, `PyRun`, `CRun`) detect a missing `go`, `python3`, or `cc` -toolchain and skip rather than fail. +toolchain and skip rather than fail. \ No newline at end of file diff --git a/RESULTS-2026-08-17.md b/RESULTS-2026-08-17.md deleted file mode 100644 index 965871d..0000000 --- a/RESULTS-2026-08-17.md +++ /dev/null @@ -1,277 +0,0 @@ -## Per-test scores - -| Test | Category | Subcategory | uni/deepseek-v4-flash-0731 | uni/qwen3.6-27b | uni/btl-4 | -| --- | --- | --- | --- | --- | --- | -| agent-deleg-batch-vs-separate | agents | delegation | 1.00 (161tok) | 1.00 (367tok) | 1.00 (894tok) | -| agent-deleg-build-then-verify | agents | delegation | 1.00 (382tok) | 1.00 (494tok) | 1.00 (455tok) | -| agent-deleg-escalation-to-human | agents | delegation | 1.00 (157tok) | 1.00 (202tok) | 1.00 (683tok) | -| agent-deleg-handoff-context | agents | delegation | 1.00 (198tok) | 1.00 (633tok) | 1.00 (1037tok) | -| agent-deleg-main-thread-vs-delegate | agents | delegation | 1.00 (154tok) | 1.00 (200tok) | 1.00 (456tok) | -| agent-deleg-minimal-privilege | agents | delegation | 1.00 (471tok) | 1.00 (579tok) | 1.00 (632tok) | -| agent-deleg-parallel-dispatch-safety | agents | delegation | 1.00 (231tok) | 1.00 (284tok) | 1.00 (901tok) | -| agent-deleg-reviewer-independence | agents | delegation | 1.00 (679tok) | 1.00 (836tok) | 1.00 (1355tok) | -| agent-deleg-task-to-specialist | agents | delegation | 1.00 (628tok) | 1.00 (654tok) | 1.00 (1050tok) | -| agent-deleg-verify-vs-trust | agents | delegation | 1.00 (155tok) | 1.00 (269tok) | 1.00 (726tok) | -| agent-plan-critical-path | agents | planning | 1.00 (567tok) | 1.00 (470tok) | 1.00 (1079tok) | -| agent-plan-cycle-detect | agents | planning | 1.00 (306tok) | 1.00 (315tok) | 1.00 (1195tok) | -| agent-plan-idempotency | agents | planning | 1.00 (499tok) | 1.00 (553tok) | 1.00 (1335tok) | -| agent-plan-milestone-date | agents | planning | 1.00 (794tok) | 1.00 (236tok) | 1.00 (892tok) | -| agent-plan-minimal-replan | agents | planning | 1.00 (406tok) | 1.00 (451tok) | 1.00 (865tok) | -| agent-plan-ordering | agents | planning | 1.00 (415tok) | 1.00 (379tok) | 1.00 (905tok) | -| agent-plan-parallel-steps | agents | planning | 1.00 (385tok) | 1.00 (322tok) | 1.00 (1120tok) | -| agent-plan-repair | agents | planning | 1.00 (346tok) | 1.00 (312tok) | 1.00 (907tok) | -| agent-plan-resource-constrained | agents | planning | 1.00 (1216tok) | 1.00 (577tok) | 1.00 (1633tok) | -| agent-plan-rollback-trigger | agents | planning | 1.00 (370tok) | 1.00 (442tok) | 1.00 (1142tok) | -| agent-tool-routing | agents | tool-routing | 1.00 (323tok) | 1.00 (434tok) | 1.00 (461tok) | -| agent-tool-routing-ambiguous | agents | tool-routing | 1.00 (277tok) | 1.00 (336tok) | 1.00 (526tok) | -| agent-tool-routing-chaining | agents | tool-routing | 1.00 (218tok) | 1.00 (302tok) | 1.00 (301tok) | -| agent-tool-routing-cheapest | agents | tool-routing | 1.00 (260tok) | 1.00 (331tok) | 1.00 (421tok) | -| agent-tool-routing-distractors | agents | tool-routing | 1.00 (555tok) | 1.00 (543tok) | 1.00 (602tok) | -| agent-tool-routing-missing-param | agents | tool-routing | 1.00 (201tok) | 1.00 (478tok) | 0.50 (502tok) | -| agent-tool-routing-multistep | agents | tool-routing | 1.00 (247tok) | 1.00 (326tok) | 1.00 (358tok) | -| agent-tool-routing-no-tool-needed | agents | tool-routing | 1.00 (291tok) | 1.00 (362tok) | 1.00 (669tok) | -| agent-tool-routing-parallel | agents | tool-routing | 1.00 (319tok) | 1.00 (380tok) | 1.00 (391tok) | -| agent-tool-routing-safety | agents | tool-routing | 1.00 (194tok) | 1.00 (253tok) | 1.00 (232tok) | -| llm-429-retry-backoff | ai | llm-integration | 1.00 (188tok) | 1.00 (278tok) | 1.00 (886tok) | -| llm-context-overflow-strategy | ai | llm-integration | 1.00 (340tok) | 1.00 (739tok) | 1.00 (1373tok) | -| llm-embedding-batch-math | ai | llm-integration | 1.00 (164tok) | 1.00 (252tok) | 1.00 (619tok) | -| llm-field-semantics | ai | llm-integration | 1.00 (373tok) | 1.00 (401tok) | 1.00 (983tok) | -| llm-role-placement | ai | llm-integration | 1.00 (181tok) | 1.00 (206tok) | 1.00 (418tok) | -| llm-sse-stream-done | ai | llm-integration | 1.00 (297tok) | 1.00 (352tok) | 1.00 (792tok) | -| llm-stop-sequence-trace | ai | llm-integration | 1.00 (450tok) | 1.00 (587tok) | 1.00 (1297tok) | -| llm-temperature-zero-caveat | ai | llm-integration | 1.00 (974tok) | 1.00 (1957tok) | 1.00 (2400tok) | -| llm-token-budget-reasoning | ai | llm-integration | 1.00 (185tok) | 1.00 (237tok) | 1.00 (297tok) | -| llm-tool-call-handling | ai | llm-integration | 1.00 (195tok) | 1.00 (318tok) | 1.00 (806tok) | -| rag-chunk-size-tradeoff | ai | rag | 1.00 (222tok) | 1.00 (312tok) | 1.00 (788tok) | -| rag-citation-grounding | ai | rag | 1.00 (712tok) | 1.00 (236tok) | 1.00 (842tok) | -| rag-context-selection-vs-stuffing | ai | rag | 1.00 (204tok) | 1.00 (316tok) | 1.00 (602tok) | -| rag-eval-metric-choice | ai | rag | 1.00 (330tok) | 1.00 (395tok) | 1.00 (924tok) | -| rag-hallucination-mitigation-ordering | ai | rag | 1.00 (331tok) | 1.00 (401tok) | 1.00 (1374tok) | -| rag-index-staleness | ai | rag | 1.00 (291tok) | 1.00 (317tok) | 1.00 (895tok) | -| rag-multihop-decomposition | ai | rag | 1.00 (358tok) | 1.00 (442tok) | 1.00 (1185tok) | -| rag-preassembly-dedup | ai | rag | 1.00 (1395tok) | 1.00 (1194tok) | 1.00 (1304tok) | -| rag-reranker-placement | ai | rag | 1.00 (182tok) | 1.00 (256tok) | 1.00 (306tok) | -| rag-retrieval-failure-mode | ai | rag | 1.00 (150tok) | 1.00 (335tok) | 1.00 (480tok) | -| vec-cosine-vs-dot | ai | vector-search | 1.00 (217tok) | 1.00 (298tok) | 1.00 (653tok) | -| vec-distance-to-similarity | ai | vector-search | 1.00 (238tok) | 1.00 (306tok) | 1.00 (882tok) | -| vec-embedding-dimension-tradeoff | ai | vector-search | 1.00 (244tok) | 1.00 (425tok) | 1.00 (524tok) | -| vec-hnsw-efsearch-tradeoff | ai | vector-search | 1.00 (298tok) | 1.00 (261tok) | 1.00 (722tok) | -| vec-index-build-query-tradeoff | ai | vector-search | 1.00 (319tok) | 1.00 (361tok) | 1.00 (521tok) | -| vec-near-duplicate-threshold | ai | vector-search | 1.00 (349tok) | 1.00 (408tok) | 1.00 (1202tok) | -| vec-pq-memory-math | ai | vector-search | 1.00 (238tok) | 1.00 (338tok) | 1.00 (865tok) | -| vec-pre-vs-post-filtering | ai | vector-search | 1.00 (195tok) | 1.00 (455tok) | 1.00 (1043tok) | -| vec-recall-at-k | ai | vector-search | 1.00 (258tok) | 1.00 (313tok) | 1.00 (811tok) | -| vec-rrf-fusion | ai | vector-search | 1.00 (690tok) | 1.00 (618tok) | 1.00 (1466tok) | -| pg-bloat-vacuum | databases | postgres | 1.00 (211tok) | 1.00 (288tok) | 1.00 (453tok) | -| pg-deadlock-lock-order | databases | postgres | 1.00 (327tok) | 1.00 (391tok) | 1.00 (1138tok) | -| pg-explain-seq-vs-index | databases | postgres | 1.00 (347tok) | 1.00 (622tok) | 1.00 (1166tok) | -| pg-index-choice | databases | postgres | 1.00 (287tok) | 1.00 (433tok) | 1.00 (612tok) | -| pg-isolation-anomaly | databases | postgres | 1.00 (240tok) | 1.00 (377tok) | 1.00 (813tok) | -| pg-listen-notify | databases | postgres | 1.00 (603tok) | 1.00 (254tok) | 1.00 (1204tok) | -| pg-n-plus-one | databases | postgres | 1.00 (213tok) | 1.00 (253tok) | 1.00 (566tok) | -| pg-partial-index | databases | postgres | 1.00 (356tok) | 1.00 (433tok) | 1.00 (988tok) | -| pg-pool-sizing | databases | postgres | 1.00 (160tok) | 1.00 (155tok) | 1.00 (265tok) | -| pg-replica-failover | databases | postgres | 1.00 (254tok) | 1.00 (331tok) | 1.00 (1404tok) | -| redis-cache-stampede | databases | redis | 1.00 (534tok) | 1.00 (477tok) | 1.00 (1341tok) | -| redis-incr-atomicity | databases | redis | 1.00 (345tok) | 1.00 (593tok) | 1.00 (1025tok) | -| redis-keyspace-anti-pattern | databases | redis | 1.00 (275tok) | 1.00 (378tok) | 1.00 (673tok) | -| redis-lua-atomicity | databases | redis | 1.00 (135tok) | 1.00 (324tok) | 1.00 (646tok) | -| redis-memory-estimate | databases | redis | 1.00 (184tok) | 1.00 (228tok) | 1.00 (992tok) | -| redis-multi-vs-pipeline | databases | redis | 1.00 (275tok) | 1.00 (1734tok) | 0.00 (594tok) | -| redis-noeviction-write | databases | redis | 1.00 (163tok) | 1.00 (182tok) | 1.00 (490tok) | -| redis-pubsub-delivery | databases | redis | 1.00 (240tok) | 1.00 (588tok) | 1.00 (1009tok) | -| redis-scan-vs-keys | databases | redis | 0.50 (774tok) | 0.50 (1063tok) | 0.50 (1968tok) | -| redis-structure-choice | databases | redis | 1.00 (417tok) | 1.00 (417tok) | 1.00 (1616tok) | -| sql-aggregate-sum | databases | sql-tuning | 1.00 (280tok) | 1.00 (380tok) | 1.00 (1101tok) | -| sql-composite-index-skip-column | databases | sql-tuning | 1.00 (746tok) | 1.00 (1933tok) | 1.00 (1082tok) | -| sql-covering-index | databases | sql-tuning | 1.00 (160tok) | 1.00 (529tok) | 1.00 (676tok) | -| sql-equivalent-rewrite | databases | sql-tuning | 1.00 (330tok) | 1.00 (509tok) | 1.00 (1043tok) | -| sql-groupby-having | databases | sql-tuning | 1.00 (610tok) | 1.00 (660tok) | 1.00 (1376tok) | -| sql-join-counts | databases | sql-tuning | 1.00 (462tok) | 1.00 (662tok) | 1.00 (1644tok) | -| sql-keyset-pagination | databases | sql-tuning | 1.00 (205tok) | 1.00 (256tok) | 1.00 (827tok) | -| sql-null-in-where | databases | sql-tuning | 1.00 (331tok) | 1.00 (421tok) | 1.00 (917tok) | -| sql-orderby-index-serving | databases | sql-tuning | 1.00 (249tok) | 1.00 (383tok) | 1.00 (680tok) | -| sql-window-rank | databases | sql-tuning | 1.00 (286tok) | 1.00 (382tok) | 1.00 (414tok) | -| docker-build-arg-vs-env | delivery | containers | 1.00 (243tok) | 1.00 (408tok) | 1.00 (978tok) | -| docker-copy-vs-add | delivery | containers | 1.00 (195tok) | 1.00 (276tok) | 1.00 (856tok) | -| docker-dockerignore-effect | delivery | containers | 1.00 (648tok) | 1.00 (389tok) | 1.00 (400tok) | -| docker-entrypoint-cmd-trace | delivery | containers | 1.00 (197tok) | 1.00 (313tok) | 1.00 (1055tok) | -| docker-healthcheck-semantics | delivery | containers | 1.00 (143tok) | 1.00 (472tok) | 1.00 (810tok) | -| docker-image-size-math | delivery | containers | 1.00 (230tok) | 1.00 (284tok) | 1.00 (526tok) | -| docker-layer-cache-bust | delivery | containers | 1.00 (337tok) | 1.00 (568tok) | 1.00 (550tok) | -| docker-layer-count | delivery | containers | 1.00 (665tok) | 1.00 (423tok) | 1.00 (1668tok) | -| docker-multistage-size-benefit | delivery | containers | 1.00 (1683tok) | 1.00 (1174tok) | 1.00 (1560tok) | -| docker-nonroot-user-caps | delivery | containers | 1.00 (554tok) | 1.00 (5843tok) | 1.00 (1866tok) | -| git-bisect-steps | delivery | git | 1.00 (4242tok) | 1.00 (4859tok) | 1.00 (2517tok) | -| git-cherry-pick-vs-revert | delivery | git | 1.00 (204tok) | 1.00 (240tok) | 1.00 (432tok) | -| git-conventional-commit-classify | delivery | git | 1.00 (429tok) | 1.00 (453tok) | 1.00 (1451tok) | -| git-detached-head-recovery | delivery | git | 1.00 (1389tok) | 0.00 (12166tok)! | 0.00 (2045tok) | -| git-force-with-lease-vs-force | delivery | git | 0.50 (1465tok) | 0.50 (2207tok) | 1.00 (1206tok) | -| git-gitignore-negation-trace | delivery | git | 0.75 (834tok) | 1.00 (955tok) | 0.75 (1210tok) | -| git-hook-choice | delivery | git | 1.00 (279tok) | 1.00 (255tok) | 1.00 (756tok) | -| git-rebase-vs-merge | delivery | git | 1.00 (236tok) | 1.00 (239tok) | 1.00 (948tok) | -| git-stash-pop-conflict | delivery | git | 1.00 (328tok) | 1.00 (436tok) | 1.00 (1168tok) | -| git-worktree-use-case | delivery | git | 1.00 (502tok) | 1.00 (2334tok) | 1.00 (1292tok) | -| rel-canary-vs-bluegreen | delivery | release-engineering | 1.00 (261tok) | 1.00 (318tok) | 1.00 (678tok) | -| rel-checksums-signing-purpose | delivery | release-engineering | 1.00 (697tok) | 1.00 (286tok) | 0.50 (563tok) | -| rel-clean-tree-requirement | delivery | release-engineering | 1.00 (902tok) | 0.50 (360tok) | 0.50 (732tok) | -| rel-commit-changelog-mapping | delivery | release-engineering | 1.00 (378tok) | 1.00 (378tok) | 1.00 (989tok) | -| rel-hotfix-flow-ordering | delivery | release-engineering | 0.00 (838tok) | 0.00 (1591tok) | 1.00 (2009tok) | -| rel-pin-vs-range-policy | delivery | release-engineering | 1.00 (228tok) | 1.00 (307tok) | 1.00 (1018tok) | -| rel-pipeline-stage-ordering | delivery | release-engineering | 0.00 (989tok) | 0.00 (340tok) | 0.00 (1054tok) | -| rel-rollback-ordering | delivery | release-engineering | 1.00 (265tok) | 1.00 (976tok) | 1.00 (1610tok) | -| rel-semver-bump-changelog | delivery | release-engineering | 1.00 (275tok) | 1.00 (345tok) | 1.00 (939tok) | -| rel-tag-to-release-sequence | delivery | release-engineering | 1.00 (333tok) | 1.00 (464tok) | 1.00 (1374tok) | -| k8s-argocd-outofsync | operations | kubernetes | 1.00 (415tok) | 1.00 (529tok) | 1.00 (1304tok) | -| k8s-cnpg-pdb | operations | kubernetes | 1.00 (625tok) | 1.00 (4965tok) | 1.00 (1733tok) | -| k8s-crashloop-gitops | operations | kubernetes | 0.75 (782tok) | 0.75 (2285tok) | 1.00 (2392tok) | -| k8s-imagepullbackoff | operations | kubernetes | 1.00 (305tok) | 1.00 (378tok) | 1.00 (724tok) | -| k8s-networkpolicy-dns-block | operations | kubernetes | 1.00 (964tok) | 1.00 (2399tok) | 1.00 (2102tok) | -| k8s-pending-taints | operations | kubernetes | 1.00 (268tok) | 1.00 (398tok) | 1.00 (1052tok) | -| k8s-pvc-storageclass-mismatch | operations | kubernetes | 1.00 (308tok) | 1.00 (440tok) | 1.00 (1030tok) | -| k8s-qos-class | operations | kubernetes | 1.00 (299tok) | 1.00 (319tok) | 1.00 (871tok) | -| k8s-service-selector-mismatch | operations | kubernetes | 1.00 (296tok) | 1.00 (360tok) | 1.00 (639tok) | -| k8s-traefik-ingressroute-host | operations | kubernetes | 1.00 (324tok) | 1.00 (344tok) | 1.00 (517tok) | -| linux-cgroup-oom-diagnosis | operations | linux | 1.00 (200tok) | 1.00 (450tok) | 1.00 (1058tok) | -| linux-cron-expression | operations | linux | 1.00 (222tok) | 1.00 (231tok) | 1.00 (1033tok) | -| linux-journalctl-filter | operations | linux | 1.00 (240tok) | 1.00 (271tok) | 1.00 (758tok) | -| linux-lvm-extend-order | operations | linux | 1.00 (426tok) | 1.00 (517tok) | 1.00 (1248tok) | -| linux-nftables-dnat | operations | linux | 1.00 (401tok) | 1.00 (740tok) | 1.00 (1146tok) | -| linux-pct-exec | operations | linux | 1.00 (593tok) | 1.00 (6904tok) | 1.00 (968tok) | -| linux-proc-meminfo-available | operations | linux | 1.00 (272tok) | 1.00 (511tok) | 1.00 (1087tok) | -| linux-ssh-port-forward | operations | linux | 1.00 (431tok) | 0.50 (1566tok) | 1.00 (837tok) | -| linux-systemd-oneshot | operations | linux | 1.00 (715tok) | 0.75 (3848tok) | 1.00 (580tok) | -| linux-systemd-timer | operations | linux | 1.00 (781tok) | 0.00 (12134tok)! | 0.67 (741tok) | -| macos-apfs-snapshot | operations | macos | 1.00 (827tok) | 0.00 (12121tok)! | 1.00 (1083tok) | -| macos-caffeinate | operations | macos | 1.00 (777tok) | 1.00 (1285tok) | 1.00 (1223tok) | -| macos-launchctl-bootstrap | operations | macos | 1.00 (1416tok) | 1.00 (2506tok) | 1.00 (1744tok) | -| macos-launchd-cron | operations | macos | 1.00 (1535tok) | 1.00 (4637tok) | 1.00 (2191tok) | -| macos-mdfind-spotlight | operations | macos | 1.00 (604tok) | 1.00 (4675tok) | 1.00 (1221tok) | -| macos-plutil-plist-read | operations | macos | 1.00 (448tok) | 1.00 (741tok) | 1.00 (1164tok) | -| macos-sed-inplace | operations | macos | 1.00 (483tok) | 1.00 (492tok) | 1.00 (819tok) | -| macos-ssh-identity-agent | operations | macos | 1.00 (1021tok) | 1.00 (3259tok) | 1.00 (1655tok) | -| macos-stat-bsd-gnu | operations | macos | 1.00 (444tok) | 1.00 (468tok) | 1.00 (1257tok) | -| macos-timeout-portability | operations | macos | 1.00 (1417tok) | 0.00 (12140tok)! | 0.67 (3692tok) | -| c-array-decay-sizeof | programming | c | 1.00 (181tok) | 1.00 (354tok) | 1.00 (842tok) | -| c-bitmask-ops | programming | c | 1.00 (173tok) | 1.00 (260tok) | 1.00 (942tok) | -| c-integer-promotion-overflow | programming | c | 1.00 (159tok) | 1.00 (316tok) | 1.00 (967tok) | -| c-macro-expansion-pitfall | programming | c | 1.00 (253tok) | 1.00 (668tok) | 1.00 (760tok) | -| c-pointer-arithmetic | programming | c | 1.00 (289tok) | 1.00 (327tok) | 1.00 (1179tok) | -| c-string-function-output | programming | c | 1.00 (698tok) | 1.00 (880tok) | 1.00 (780tok) | -| c-struct-bitfield-packing | programming | c | 1.00 (486tok) | 1.00 (657tok) | 1.00 (1188tok) | -| c-struct-size | programming | c | 1.00 (312tok) | 1.00 (365tok) | 1.00 (1135tok) | -| c-undefined-behavior-spot | programming | c | 1.00 (260tok) | 1.00 (331tok) | 1.00 (1651tok) | -| c-union-size-lp64 | programming | c | 1.00 (307tok) | 1.00 (408tok) | 1.00 (1342tok) | -| go-bounded-fanout | programming | golang | 1.00 (1628tok) | 0.00 (2714tok) | 1.00 (2483tok) | -| go-channel-deadlock | programming | golang | 1.00 (277tok) | 1.00 (410tok) | 1.00 (561tok) | -| go-context-cancellation-trace | programming | golang | 1.00 (283tok) | 1.00 (1093tok) | 1.00 (927tok) | -| go-defer-recover-trace | programming | golang | 1.00 (392tok) | 1.00 (415tok) | 1.00 (1090tok) | -| go-generics-constraint-fix | programming | golang | 1.00 (709tok) | 1.00 (6720tok) | 1.00 (1776tok) | -| go-method-set-interface | programming | golang | 1.00 (256tok) | 1.00 (1828tok) | 1.00 (1278tok) | -| go-semver-classify | programming | golang | 1.00 (616tok) | 1.00 (450tok) | 1.00 (1211tok) | -| go-slice-append-aliasing | programming | golang | 1.00 (433tok) | 1.00 (365tok) | 1.00 (1087tok) | -| go-struct-align | programming | golang | 1.00 (517tok) | 1.00 (1682tok) | 1.00 (2084tok) | -| go-worker-pool | programming | golang | 1.00 (972tok) | 0.00 (12193tok)! | 0.00 (894tok) | -| py-asyncio-gather-trace | programming | python | 1.00 (249tok) | 1.00 (342tok) | 1.00 (1352tok) | -| py-cosine | programming | python | 1.00 (4455tok) | 1.00 (3928tok) | 1.00 (3882tok) | -| py-dict-comprehension-trace | programming | python | 1.00 (346tok) | 1.00 (337tok) | 0.00 (12090tok)! | -| py-generator-exhaustion-trace | programming | python | 1.00 (207tok) | 1.00 (266tok) | 1.00 (1208tok) | -| py-json-transform | programming | python | 1.00 (434tok) | 1.00 (648tok) | 1.00 (1079tok) | -| py-log-triage | programming | python | 1.00 (1727tok) | 1.00 (2375tok) | 1.00 (1645tok) | -| py-mutable-default-arg | programming | python | 1.00 (299tok) | 1.00 (269tok) | 1.00 (950tok) | -| py-pathlib-rewrite | programming | python | 1.00 (657tok) | 1.00 (1705tok) | 1.00 (659tok) | -| py-regex-log-extraction | programming | python | 0.00 (956tok) | 1.00 (1616tok) | 1.00 (1249tok) | -| py-softmax | programming | python | 1.00 (2134tok) | 1.00 (2861tok) | 1.00 (1834tok) | -| ts-array-method-chain | programming | typescript | 1.00 (197tok) | 1.00 (263tok) | 1.00 (1146tok) | -| ts-computed-vs-watch | programming | typescript | 1.00 (226tok) | 1.00 (300tok) | 1.00 (823tok) | -| ts-debounce-composable | programming | typescript | 1.00 (687tok) | 1.00 (8871tok) | 1.00 (1521tok) | -| ts-discriminated-union-narrowing | programming | typescript | 1.00 (328tok) | 1.00 (436tok) | 1.00 (1624tok) | -| ts-eslint-flat-config | programming | typescript | 1.00 (903tok) | 1.00 (3197tok) | 1.00 (4007tok) | -| ts-generic-utility-type | programming | typescript | 1.00 (2547tok) | 0.25 (12152tok)! | 1.00 (1759tok) | -| ts-optional-chaining-nullish | programming | typescript | 1.00 (323tok) | 1.00 (392tok) | 1.00 (1513tok) | -| ts-promise-allsettled-trace | programming | typescript | 1.00 (186tok) | 1.00 (282tok) | 1.00 (287tok) | -| ts-tsconfig-strict-flag | programming | typescript | 1.00 (210tok) | 1.00 (665tok) | 1.00 (529tok) | -| ts-vue-ref-reactive-unwrap | programming | typescript | 1.00 (247tok) | 0.00 (12087tok) | 0.50 (629tok) | -| code-bigo-dedup | research | codebase | 1.00 (278tok) | 1.00 (313tok) | 1.00 (920tok) | -| code-bug-line-diff | research | codebase | 1.00 (344tok) | 1.00 (607tok) | 1.00 (892tok) | -| code-commit-bisect | research | codebase | 1.00 (474tok) | 1.00 (585tok) | 1.00 (996tok) | -| code-dead-functions | research | codebase | 1.00 (225tok) | 1.00 (312tok) | 1.00 (404tok) | -| code-generic-return-type | research | codebase | 1.00 (171tok) | 1.00 (248tok) | 1.00 (599tok) | -| code-import-cycle | research | codebase | 1.00 (150tok) | 1.00 (230tok) | 1.00 (973tok) | -| code-race-variable | research | codebase | 1.00 (188tok) | 1.00 (409tok) | 1.00 (643tok) | -| code-trace-go | research | codebase | 1.00 (766tok) | 1.00 (690tok) | 1.00 (1514tok) | -| code-trace-python | research | codebase | 1.00 (378tok) | 1.00 (345tok) | 1.00 (981tok) | -| code-trace-ts | research | codebase | 1.00 (447tok) | 1.00 (356tok) | 1.00 (958tok) | -| web-canonical-vs-redirect | research | web | 1.00 (249tok) | 1.00 (400tok) | 1.00 (767tok) | -| web-cors-missing-header | research | web | 1.00 (221tok) | 1.00 (369tok) | 1.00 (999tok) | -| web-dns-mx-record | research | web | 1.00 (186tok) | 1.00 (204tok) | 1.00 (529tok) | -| web-hreflang-reciprocal | research | web | 1.00 (496tok) | 1.00 (540tok) | 1.00 (1319tok) | -| web-html-product-extract | research | web | 1.00 (203tok) | 1.00 (315tok) | 1.00 (956tok) | -| web-http-status-scenarios | research | web | 1.00 (425tok) | 1.00 (516tok) | 1.00 (1719tok) | -| web-robots-ai-crawlers | research | web | 1.00 (517tok) | 1.00 (591tok) | 1.00 (1469tok) | -| web-security-headers-audit | research | web | 1.00 (327tok) | 1.00 (409tok) | 1.00 (1250tok) | -| web-sitemap-max-urls | research | web | 1.00 (127tok) | 1.00 (272tok) | 1.00 (958tok) | -| web-url-parse-components | research | web | 1.00 (367tok) | 1.00 (316tok) | 1.00 (987tok) | -| paper-attention-scale-factor | research | whitepapers | 1.00 (300tok) | 1.00 (378tok) | 1.00 (608tok) | -| paper-bloom-fp-rate | research | whitepapers | 1.00 (5487tok) | 1.00 (5125tok) | 1.00 (2928tok) | -| paper-btree-height-levels | research | whitepapers | 1.00 (501tok) | 1.00 (541tok) | 1.00 (1499tok) | -| paper-cap-availability-choice | research | whitepapers | 1.00 (269tok) | 1.00 (366tok) | 1.00 (879tok) | -| paper-hnsw-params | research | whitepapers | 1.00 (346tok) | 1.00 (541tok) | 1.00 (1337tok) | -| paper-lsm-write-amplification | research | whitepapers | 1.00 (416tok) | 1.00 (529tok) | 1.00 (1351tok) | -| paper-pq-compression-ratio | research | whitepapers | 1.00 (387tok) | 1.00 (468tok) | 1.00 (1112tok) | -| paper-raft-quorum-failures | research | whitepapers | 1.00 (392tok) | 1.00 (418tok) | 1.00 (1350tok) | -| paper-recall-at-k | research | whitepapers | 1.00 (430tok) | 1.00 (492tok) | 1.00 (1158tok) | -| paper-tfidf-term-score | research | whitepapers | 1.00 (527tok) | 1.00 (604tok) | 1.00 (1595tok) | -| sec-csrf-requirement | security | appsec | 1.00 (187tok) | 1.00 (297tok) | 1.00 (726tok) | -| sec-idor-spot | security | appsec | 1.00 (282tok) | 1.00 (424tok) | 1.00 (658tok) | -| sec-input-validation-boundary | security | appsec | 0.00 (12089tok)! | 1.00 (2320tok) | 1.00 (1382tok) | -| sec-open-redirect-spot | security | appsec | 1.00 (179tok) | 1.00 (241tok) | 1.00 (502tok) | -| sec-path-traversal-spot | security | appsec | 1.00 (277tok) | 1.00 (331tok) | 1.00 (628tok) | -| sec-rate-limit-placement | security | appsec | 1.00 (184tok) | 1.00 (397tok) | 1.00 (1185tok) | -| sec-secret-log-spot | security | appsec | 1.00 (305tok) | 1.00 (292tok) | 1.00 (471tok) | -| sec-sqli-spot | security | appsec | 1.00 (229tok) | 1.00 (311tok) | 1.00 (504tok) | -| sec-ssrf-spot | security | appsec | 1.00 (285tok) | 1.00 (485tok) | 1.00 (628tok) | -| sec-xss-sink-spot | security | appsec | 1.00 (225tok) | 1.00 (312tok) | 1.00 (606tok) | -| sec-aes-gcm-nonce-reuse | security | crypto | 1.00 (2811tok) | 1.00 (10585tok) | 1.00 (3603tok) | -| sec-cert-chain-validation | security | crypto | 1.00 (244tok) | 1.00 (326tok) | 1.00 (1189tok) | -| sec-constant-time-compare | security | crypto | 1.00 (468tok) | 1.00 (462tok) | 1.00 (874tok) | -| sec-hash-vs-encrypt-pii | security | crypto | 1.00 (201tok) | 1.00 (250tok) | 1.00 (543tok) | -| sec-hmac-vs-signature | security | crypto | 1.00 (488tok) | 1.00 (375tok) | 1.00 (577tok) | -| sec-jwt-alg-none | security | crypto | 1.00 (210tok) | 1.00 (337tok) | 1.00 (1050tok) | -| sec-password-hash-choice | security | crypto | 0.50 (1597tok) | 0.50 (638tok) | 0.50 (1936tok) | -| sec-rand-source-choice | security | crypto | 0.50 (435tok) | 1.00 (839tok) | 0.50 (1270tok) | -| sec-rotate-order | security | crypto | 1.00 (354tok) | 1.00 (665tok) | 1.00 (6504tok) | -| sec-tls-floor-version | security | crypto | 1.00 (155tok) | 1.00 (311tok) | 1.00 (959tok) | -| sec-diff-secret-spot | security | secrets | 1.00 (250tok) | 1.00 (394tok) | 1.00 (1159tok) | -| sec-fork-pr-ci-exposure | security | secrets | 1.00 (344tok) | 1.00 (289tok) | 1.00 (376tok) | -| sec-gitignore-history-nuance | security | secrets | 1.00 (376tok) | 1.00 (311tok) | 1.00 (819tok) | -| sec-hardcoded-secret-spot | security | secrets | 1.00 (170tok) | 1.00 (202tok) | 1.00 (243tok) | -| sec-k8s-secret-base64 | security | secrets | 1.00 (148tok) | 1.00 (199tok) | 1.00 (328tok) | -| sec-least-privilege-scope | security | secrets | 1.00 (177tok) | 1.00 (230tok) | 1.00 (770tok) | -| sec-remediation-order | security | secrets | 1.00 (1843tok) | 1.00 (506tok) | 0.00 (12098tok)! | -| sec-rotation-no-downtime | security | secrets | 1.00 (451tok) | 1.00 (501tok) | 1.00 (465tok) | -| sec-ssh-agent-identity | security | secrets | 1.00 (1689tok) | 1.00 (2416tok) | 1.00 (2843tok) | -| sec-vault-tradeoff | security | secrets | 1.00 (305tok) | 1.00 (407tok) | 1.00 (740tok) | - -! = response truncated by the token budget (finish_reason=length); increase max_tokens_default to avoid - -## Category rollup (mean score) - -| Category | uni/deepseek-v4-flash-0731 | uni/qwen3.6-27b | uni/btl-4 | -| --- | --- | --- | --- | -| agents | 1.00 | 1.00 | 0.98 | -| ai | 1.00 | 1.00 | 1.00 | -| databases | 0.98 | 0.98 | 0.95 | -| delivery | 0.91 | 0.87 | 0.89 | -| operations | 0.99 | 0.87 | 0.98 | -| programming | 0.97 | 0.91 | 0.94 | -| research | 1.00 | 1.00 | 1.00 | -| security | 0.93 | 0.98 | 0.93 | - -## Model summary - -| Model | Mean | Passed | Partial | Failed | Errors | Mean latency | Total tokens | Tok/s | -| --- | --- | --- | --- | --- | --- | --- | --- | --- | -| uni/deepseek-v4-flash-0731 | 0.97 | 240 | 6 | 4 | 0 | 5.685104556s | 138851 | 71.6 | -| uni/qwen3.6-27b | 0.95 | 233 | 8 | 9 | 0 | 23.689755872s | 291412 | 40.7 | -| uni/btl-4 | 0.96 | 234 | 10 | 6 | 0 | 13.135007924s | 295086 | 77.7 | diff --git a/cmd/llmtest/main.go b/cmd/llmtest/main.go index 4970dd0..a92d4d7 100644 --- a/cmd/llmtest/main.go +++ b/cmd/llmtest/main.go @@ -14,6 +14,7 @@ import ( "github.com/lukaszraczylo/llm-testbench/internal/llm" "github.com/lukaszraczylo/llm-testbench/internal/report" "github.com/lukaszraczylo/llm-testbench/internal/runner" + "github.com/lukaszraczylo/llm-testbench/internal/testkit" "github.com/lukaszraczylo/llm-testbench/internal/tests" ) @@ -37,9 +38,12 @@ func main() { err = runCommand(os.Args[2:]) case "list": err = listCommand(os.Args[2:]) + case "compare": + err = compareCommand(os.Args[2:]) + case "health": + err = healthCommand(os.Args[2:]) case "version", "-v", "--version": fmt.Println("llmtest " + version) - return case "-h", "--help", "help": usage() return @@ -59,9 +63,16 @@ func usage() { fmt.Fprint(os.Stderr, `llmtest - LLM accuracy testing framework Usage: - llmtest run [flags] Run the test catalog against configured models. - llmtest list [flags] List the test catalog. - llmtest version Print the build version. + llmtest run [flags] Run the test catalog against configured models. + llmtest list [flags] List the test catalog. + llmtest compare baseline.json current.json + Diff two saved run artifacts (--out files). + llmtest health file.json [more.json ...] + Audit suite health: which subcategories are + saturated (every model passes everything), and + which tests still carry signal. Pools multiple + artifacts. + llmtest version Print the build version. Run "llmtest run -h" or "llmtest list -h" for flag details. `) @@ -103,9 +114,12 @@ func runCommand(args []string) error { fs := flag.NewFlagSet("run", flag.ExitOnError) shared := bindSharedFlags(fs) modelsCSV := fs.String("models", "", "comma-separated model override (defaults to config's models)") + testsCSV := fs.String("tests", "", "comma-separated exact test IDs (overrides --category/--subcategory)") format := fs.String("format", "table", "output format: table|markdown|json") concurrency := fs.Int("concurrency", 0, "override config's concurrency (0 = use config)") timeout := fs.Duration("timeout", 0, "override config's request_timeout (0 = use config)") + repeat := fs.Int("repeat", 1, "samples per (model, test); >1 exposes response instability at temperature 0") + out := fs.String("out", "", "write the run as a JSON artifact to this file (for llmtest compare)") quiet := fs.Bool("quiet", false, "suppress progress output on stderr") if err := fs.Parse(args); err != nil { return err @@ -134,9 +148,9 @@ func runCommand(args []string) error { client := llm.NewOpenAIClient(cfg.Endpoint, cfg.APIKey, reqTimeout) registry := tests.All() - selected := registry.Filter(shared.category, shared.subcategory) - if len(selected) == 0 { - return fmt.Errorf("no tests matched category=%q subcategory=%q", shared.category, shared.subcategory) + selected, err := selectTests(registry, *testsCSV, shared.category, shared.subcategory) + if err != nil { + return err } var reporter runner.ProgressReporter = runner.NoopProgressReporter{} @@ -149,6 +163,7 @@ func runCommand(args []string) error { Temperature: requestTemperature, MaxTokensDefault: cfg.MaxTokensDefault, Reporter: reporter, + Repeat: *repeat, }) ctx := context.Background() @@ -159,9 +174,90 @@ func runCommand(args []string) error { return err } + if *out != "" { + if err := report.WriteArtifact(*out, selected, models, results); err != nil { + return fmt.Errorf("write --out: %w", err) + } + } + return report.Render(os.Stdout, f, selected, models, results) } +// selectTests resolves the test subset: --tests (exact IDs, error on any +// unknown ID) overrides --category/--subcategory filtering. +func selectTests(registry *testkit.Registry, testsCSV, category, subcategory string) ([]testkit.Test, error) { + if testsCSV == "" { + selected := registry.Filter(category, subcategory) + if len(selected) == 0 { + return nil, fmt.Errorf("no tests matched category=%q subcategory=%q", category, subcategory) + } + return selected, nil + } + + ids := splitCSV(testsCSV) + out := make([]testkit.Test, 0, len(ids)) + seen := make(map[string]bool, len(ids)) + for _, id := range ids { + if seen[id] { + continue + } + seen[id] = true + t, ok := registry.Get(id) + if !ok { + return nil, fmt.Errorf("unknown test id %q (see llmtest list)", id) + } + out = append(out, t) + } + if len(out) == 0 { + return nil, fmt.Errorf("--tests parsed to zero ids") + } + return out, nil +} + +// compareCommand diffs two saved artifacts from `llmtest run --out`. +func compareCommand(args []string) error { + fs := flag.NewFlagSet("compare", flag.ExitOnError) + if err := fs.Parse(args); err != nil { + return err + } + if fs.NArg() != 2 { + return fmt.Errorf("usage: llmtest compare baseline.json current.json") + } + + baseline, err := report.LoadArtifact(fs.Arg(0)) + if err != nil { + return err + } + + current, err := report.LoadArtifact(fs.Arg(1)) + if err != nil { + return err + } + + return report.RenderCompare(os.Stdout, report.CompareArtifacts(baseline, current)) +} + +// healthCommand audits one or more saved artifacts for suite health: +// per-subcategory saturation and the tests that still discriminate. +func healthCommand(args []string) error { + fs := flag.NewFlagSet("health", flag.ExitOnError) + if err := fs.Parse(args); err != nil { + return err + } + if fs.NArg() < 1 { + return fmt.Errorf("usage: llmtest health artifact.json [more.json ...]") + } + artifacts := make([]report.Artifact, 0, fs.NArg()) + for _, p := range fs.Args() { + a, err := report.LoadArtifact(p) + if err != nil { + return fmt.Errorf("load %s: %w", p, err) + } + artifacts = append(artifacts, a) + } + return report.RenderHealth(os.Stdout, report.AuditHealth(artifacts)) +} + // validateFormat parses and validates the --format flag value, pulled out // of runCommand as its own function so it is unit-testable without going // through flag parsing / a live run (N8). diff --git a/cmd/llmtest/main_test.go b/cmd/llmtest/main_test.go index 68d225d..50c66e5 100644 --- a/cmd/llmtest/main_test.go +++ b/cmd/llmtest/main_test.go @@ -1,9 +1,12 @@ package main import ( + "strings" "testing" + "github.com/lukaszraczylo/llm-testbench/internal/eval" "github.com/lukaszraczylo/llm-testbench/internal/report" + "github.com/lukaszraczylo/llm-testbench/internal/testkit" ) func TestSplitCSV(t *testing.T) { @@ -61,3 +64,40 @@ func TestValidateFormat(t *testing.T) { }) } } + +func TestSelectTests(t *testing.T) { + reg := testkit.NewRegistry() + reg.Register(testkit.Test{ID: "a", Category: "x", Subcategory: "s", Eval: eval.ContainsAny("a")}) + reg.Register(testkit.Test{ID: "b", Category: "y", Subcategory: "s", Eval: eval.ContainsAny("b")}) + + got, err := selectTests(reg, "b,a", "", "") + if err != nil { + t.Fatalf("selectTests() error = %v", err) + } + if len(got) != 2 || got[0].ID != "b" || got[1].ID != "a" { + t.Errorf("selection = %+v, want [b a] in flag order", got) + } + + if _, err := selectTests(reg, "a,nope", "", ""); err == nil || !strings.Contains(err.Error(), "nope") { + t.Errorf("unknown id error = %v, want mention of %q", err, "nope") + } + + got, err = selectTests(reg, "", "y", "") + if err != nil || len(got) != 1 || got[0].ID != "b" { + t.Errorf("category filter = %+v, %v; want [b], nil", got, err) + } + + if _, err := selectTests(reg, "", "zzz", ""); err == nil { + t.Error("empty category filter: error = nil, want no-match error") + } +} + +func TestHealthCommand_RequiresArtifacts(t *testing.T) { + if err := healthCommand(nil); err == nil || !strings.Contains(err.Error(), "usage:") { + t.Errorf("healthCommand(nil) error = %v, want usage error", err) + } + if err := healthCommand([]string{"/nonexistent-artifact.json"}); err == nil || + !strings.Contains(err.Error(), "load /nonexistent-artifact.json") { + t.Errorf("healthCommand(missing file) error = %v, want load error naming the file", err) + } +} diff --git a/internal/report/compare.go b/internal/report/compare.go new file mode 100644 index 0000000..4a5a830 --- /dev/null +++ b/internal/report/compare.go @@ -0,0 +1,221 @@ +package report + +import ( + "fmt" + "io" + "math" + "sort" + "strings" +) + +// Comparison is the per-(model, test) diff between two saved artifacts. +// Scores are means over attempts, so a test that flapped between runs +// shows up as a regression on one attempt basis only if the mean moved. +type Comparison struct { + Model string + TestID string + Category string + Subcategory string + Status string + BaselineMean float64 + CurrentMean float64 + Delta float64 +} + +// RegressionDelta is the mean-score drop at which a test counts as a +// regression. Below this, run-to-run sampling noise dominates. +const RegressionDelta = 0.01 + +type attemptKey struct{ model, testID string } + +// meanAttempts collapses saved attempts into per-(model,test) means and +// carries category metadata from the first attempt that has it. Truncated +// attempts are excluded from the mean when any attempt finished normally, +// mirroring the truncation policy in indexResults; when every scored +// attempt was truncated, the partial-text scores are used rather than +// reporting no data. +func meanAttempts(results []AttemptResult) map[attemptKey]Comparison { + type agg struct { + cat string + sb string + sum float64 + n int + cleanSum float64 + cleanN int + errs int + } + by := make(map[attemptKey]*agg) + order := make([]attemptKey, 0, len(results)) + for _, r := range results { + k := attemptKey{r.Model, r.TestID} + a, seen := by[k] + if !seen { + a = &agg{} + by[k] = a + order = append(order, k) + } + if a.cat == "" { + a.cat, a.sb = r.Category, r.Subcategory + } + if r.Error != "" { + a.errs++ + continue + } + if r.Skipped { + continue + } + a.sum += r.Score + a.n++ + if !r.Truncated { + a.cleanSum += r.Score + a.cleanN++ + } + } + + out := make(map[attemptKey]Comparison, len(by)) + for _, k := range order { + a := by[k] + c := Comparison{Model: k.model, TestID: k.testID, Category: a.cat, Subcategory: a.sb} + switch { + case a.n == 0: + c.Status = "error" + case a.cleanN > 0: + c.BaselineMean = a.cleanSum / float64(a.cleanN) + default: + c.BaselineMean = a.sum / float64(a.n) + } + out[k] = c + } + return out +} + +// CompareArtifacts diffs two artifacts per (model, test), sorted by +// delta ascending (worst regressions first), then test ID. +func CompareArtifacts(baseline, current Artifact) []Comparison { + base := meanAttempts(baseline.Results) + cur := meanAttempts(current.Results) + + keys := make(map[attemptKey]bool, len(base)+len(cur)) + for k := range base { + keys[k] = true + } + for k := range cur { + keys[k] = true + } + + out := make([]Comparison, 0, len(keys)) + for k := range keys { + b, hasB := base[k] + c, hasC := cur[k] + switch { + case hasB && !hasC: + b.CurrentMean = math.NaN() + b.Delta = math.NaN() + b.Status = "baseline-only" + out = append(out, b) + case !hasB && hasC: + c.BaselineMean = math.NaN() + c.Delta = math.NaN() + c.Status = "current-only" + out = append(out, c) + default: + m := Comparison{ + Model: k.model, + TestID: k.testID, + Category: c.Category, + Subcategory: c.Subcategory, + BaselineMean: b.BaselineMean, + CurrentMean: c.BaselineMean, + } + switch { + case b.Status == "error" || c.Status == "error": + m.Status = "error" + m.Delta = math.NaN() + default: + m.Delta = m.CurrentMean - m.BaselineMean + switch { + case m.Delta <= -RegressionDelta: + m.Status = "regression" + case m.Delta >= RegressionDelta: + m.Status = "improvement" + default: + m.Status = "stable" + } + } + out = append(out, m) + } + } + + sort.Slice(out, func(i, j int) bool { + di, dj := out[i].Delta, out[j].Delta + if math.IsNaN(di) != math.IsNaN(dj) { + return math.IsNaN(di) + } + if math.IsNaN(di) && math.IsNaN(dj) { + if out[i].Status != out[j].Status { + return out[i].Status < out[j].Status + } + return out[i].TestID < out[j].TestID + } + if di != dj { + return di < dj + } + if out[i].TestID != out[j].TestID { + return out[i].TestID < out[j].TestID + } + return out[i].Model < out[j].Model + }) + return out +} + +// RenderCompare writes a comparison report: headline counts, then only +// changed/missing rows (stable rows are summarized, not listed). +func RenderCompare(w io.Writer, comparisons []Comparison) error { + ew := &errWriter{w: w} + + var reg, imp, stable, added, removed, errored int + for _, c := range comparisons { + switch c.Status { + case "regression": + reg++ + case "improvement": + imp++ + case "baseline-only": + removed++ + case "current-only": + added++ + case "error": + errored++ + default: + stable++ + } + } + + ew.println(fmt.Sprintf("== Compare: %d regressions, %d improvements, %d stable, %d errors, %d baseline-only, %d current-only ==", + reg, imp, stable, errored, removed, added)) + ew.println() + ew.println("MODEL\tTEST\tCATEGORY\tSUBCATEGORY\tBASELINE\tCURRENT\tDELTA\tSTATUS") + shown := 0 + for _, c := range comparisons { + if c.Status == "stable" { + continue + } + ew.println(strings.Join([]string{ + c.Model, c.TestID, c.Category, c.Subcategory, + meanCellText(c.BaselineMean), meanCellText(c.CurrentMean), + deltaText(c.Delta), c.Status, + }, "\t")) + shown++ + } + if shown == 0 { + ew.println("(no changed, missing, or erroring tests)") + } + return ew.err +} + +func deltaText(v float64) string { + if math.IsNaN(v) { + return "N/A" + } + return fmt.Sprintf("%+.2f", v) +} diff --git a/internal/report/compare_test.go b/internal/report/compare_test.go new file mode 100644 index 0000000..d6d1176 --- /dev/null +++ b/internal/report/compare_test.go @@ -0,0 +1,114 @@ +package report + +import ( + "bytes" + "math" + "strings" + "testing" +) + +func attempt(model, testID string, score float64) AttemptResult { + return AttemptResult{Model: model, TestID: testID, Category: "c", Subcategory: "s", Score: score} +} + +func TestCompareArtifacts_ClassifiesRegressionImprovementStable(t *testing.T) { + baseline := Artifact{Results: []AttemptResult{ + attempt("m1", "reg", 1.0), + attempt("m1", "imp", 0.2), + attempt("m1", "stable", 1.0), + }} + current := Artifact{Results: []AttemptResult{ + attempt("m1", "reg", 0.4), + attempt("m1", "imp", 1.0), + attempt("m1", "stable", 0.995), + }} + + got := CompareArtifacts(baseline, current) + byID := map[string]Comparison{} + for _, c := range got { + byID[c.TestID] = c + } + if byID["reg"].Status != "regression" || byID["reg"].Delta > -0.5 { + t.Errorf("reg = %+v, want regression delta ~ -0.6", byID["reg"]) + } + if byID["imp"].Status != "improvement" { + t.Errorf("imp = %+v, want improvement", byID["imp"]) + } + if byID["stable"].Status != "stable" { + t.Errorf("stable = %+v, want stable", byID["stable"]) + } + // Worst delta first. + if got[0].TestID != "reg" { + t.Errorf("first row = %q, want worst regression %q", got[0].TestID, "reg") + } +} + +func TestCompareArtifacts_AveragesAttemptsAndFlagsMembership(t *testing.T) { + baseline := Artifact{Results: []AttemptResult{ + attempt("m1", "flaky", 1.0), attempt("m1", "flaky", 0.0), // mean 0.5 + attempt("m1", "gone", 1.0), + }} + current := Artifact{Results: []AttemptResult{ + attempt("m1", "flaky", 1.0), attempt("m1", "flaky", 1.0), // mean 1.0 => improvement + attempt("m1", "new", 0.5), + }} + + got := CompareArtifacts(baseline, current) + byID := map[string]Comparison{} + for _, c := range got { + byID[c.TestID] = c + } + if byID["flaky"].Status != "improvement" || math.Abs(byID["flaky"].BaselineMean-0.5) > 1e-9 { + t.Errorf("flaky = %+v, want improvement from mean 0.5", byID["flaky"]) + } + if byID["gone"].Status != "baseline-only" { + t.Errorf("gone = %+v, want baseline-only", byID["gone"]) + } + if byID["new"].Status != "current-only" { + t.Errorf("new = %+v, want current-only", byID["new"]) + } +} + +func TestCompareArtifacts_TruncatedAttemptExcludedFromMean(t *testing.T) { + clean := func(score float64) AttemptResult { + return AttemptResult{Model: "m1", TestID: "t", Category: "c", Subcategory: "s", Score: score} + } + trunc := func(score float64) AttemptResult { + return AttemptResult{Model: "m1", TestID: "t", Category: "c", Subcategory: "s", Score: score, Truncated: true} + } + baseline := Artifact{Results: []AttemptResult{clean(1.0), trunc(0.0)}} // clean mean 1.0 + current := Artifact{Results: []AttemptResult{trunc(0.4), trunc(0.6)}} // all truncated => fallback 0.5 + + got := CompareArtifacts(baseline, current) + if len(got) != 1 { + t.Fatalf("comparisons = %d, want 1", len(got)) + } + if math.Abs(got[0].BaselineMean-1.0) > 1e-9 { + t.Errorf("baseline mean = %v, want 1.0 (truncated attempt excluded)", got[0].BaselineMean) + } + if math.Abs(got[0].CurrentMean-0.5) > 1e-9 { + t.Errorf("current mean = %v, want 0.5 (all-truncated fallback)", got[0].CurrentMean) + } + if got[0].Status != "regression" { + t.Errorf("status = %q, want regression", got[0].Status) + } +} + +func TestRenderCompare_HeadlineAndRows(t *testing.T) { + baseline := Artifact{Results: []AttemptResult{attempt("m1", "a", 1.0), attempt("m1", "b", 1.0)}} + current := Artifact{Results: []AttemptResult{attempt("m1", "a", 0.0), attempt("m1", "b", 1.0)}} + + var buf bytes.Buffer + if err := RenderCompare(&buf, CompareArtifacts(baseline, current)); err != nil { + t.Fatalf("RenderCompare() error = %v", err) + } + out := buf.String() + for _, want := range []string{"1 regressions", "0 improvements", "1 stable", "a", "-1.00"} { + if !strings.Contains(out, want) { + t.Errorf("output missing %q:\n%s", want, out) + } + } + if strings.Contains(out, "\nb\t") { + t.Errorf("stable row b should be summarized, not listed:\n%s", out) + } +} diff --git a/internal/report/discrimination.go b/internal/report/discrimination.go new file mode 100644 index 0000000..8860529 --- /dev/null +++ b/internal/report/discrimination.go @@ -0,0 +1,114 @@ +package report + +import ( + "math" + "sort" + + "github.com/lukaszraczylo/llm-testbench/internal/testkit" +) + +// DiscriminationSpread is the cross-model score gap at which a test starts +// to separate models. The last full run showed a median spread of 0.0 +// across 250 tests: nearly every test scored >=0.99 for every model, so +// the suite measured nothing about relative capability. This number makes +// that visible per test instead of averaged away. +const DiscriminationSpread = 0.05 + +// testStat is the per-test measurement rollup: cross-model discrimination +// (spread, only meaningful with 2+ models), attempt stability (did +// --repeat samples disagree), and aggregate score. Computed over the +// attempt-aggregated cells so a flaky test shows as unstable rather than +// as a random single score. +type testStat struct { + testID string + category string + subcategory string + mean float64 // mean over model cell means (scored cells only) + spread float64 // max-min over model cell means; NaN with <2 scored models + unstable bool // some model's attempts disagreed beyond epsilon + truncated bool // some cell was token-budget truncated + scored bool // at least one scored cell +} + +// unstableEpsilon absorbs float noise in averaged scores; a real +// disagreement between attempts is orders of magnitude larger. +const unstableEpsilon = 1e-9 + +// testStats computes the per-test rollup for every test in tests, ordered +// by category, subcategory, ID (same order as the score table). +func testStats(tests []testkit.Test, models []string, idx cellIndex) []testStat { + out := make([]testStat, 0, len(tests)) + for _, t := range sortedTests(tests) { + st := testStat{testID: t.ID, category: t.Category, subcategory: t.Subcategory, spread: math.NaN()} + var sum float64 + var n int + var lo, hi float64 + for _, model := range models { + c, ok := idx[t.ID][model] + if !ok || c.errorAll || c.skippedAll || c.scored == 0 { + continue + } + v := c.result.Score.Value + if n == 0 { + lo, hi = v, v + } else { + lo = math.Min(lo, v) + hi = math.Max(hi, v) + } + sum += v + n++ + st.scored = true + if c.maxScore-c.minScore > unstableEpsilon { + st.unstable = true + } + st.truncated = st.truncated || c.result.Truncated() + } + if n > 1 { + st.spread = hi - lo + st.mean = sum / float64(n) + } else if n == 1 { + st.mean = sum + } + out = append(out, st) + } + return out +} + +// interestingStats narrows testStats to rows worth reading: tests that +// discriminate between models, tests whose repeated attempts disagreed, +// and (when no cross-model data exists) tests that any model failed to +// pass outright. Sorted by spread descending, unstable first within equal +// spread. +func interestingStats(stats []testStat, multiModel bool) []testStat { + var out []testStat + for _, st := range stats { + if !st.scored { + continue + } + switch { + case st.unstable: + case multiModel && !math.IsNaN(st.spread) && st.spread >= DiscriminationSpread: + case !multiModel && st.mean < passThreshold: + default: + continue + } + out = append(out, st) + } + sort.SliceStable(out, func(i, j int) bool { + if out[i].unstable != out[j].unstable { + return out[i].unstable + } + si, sj := out[i].spread, out[j].spread + if math.IsNaN(si) { + si = -1 + } + if math.IsNaN(sj) { + sj = -1 + } + if si != sj { + return si > sj + } + return out[i].testID < out[j].testID + }) + return out +} diff --git a/internal/report/discrimination_test.go b/internal/report/discrimination_test.go new file mode 100644 index 0000000..8d30d02 --- /dev/null +++ b/internal/report/discrimination_test.go @@ -0,0 +1,86 @@ +package report + +import ( + "bytes" + "encoding/json" + "strings" + "testing" + + "github.com/lukaszraczylo/llm-testbench/internal/eval" + "github.com/lukaszraczylo/llm-testbench/internal/runner" + "github.com/lukaszraczylo/llm-testbench/internal/testkit" +) + +func TestDiscrimination_TableShowsSpreadAndInstability(t *testing.T) { + tests := []testkit.Test{ + {ID: "sep", Category: "c", Subcategory: "s"}, + {ID: "same", Category: "c", Subcategory: "s"}, + } + results := []runner.Result{ + {Model: "m1", TestID: "sep", Score: eval.Score{Value: 1}}, + {Model: "m2", TestID: "sep", Score: eval.Score{Value: 0.2}}, + {Model: "m1", TestID: "same", Score: eval.Score{Value: 1}}, + {Model: "m2", TestID: "same", Score: eval.Score{Value: 1}}, + // Flaky pair for m1/sep: a second attempt disagrees with the first. + {Model: "m1", TestID: "sep", Attempt: 1, Score: eval.Score{Value: 0.4}}, + } + + var buf bytes.Buffer + if err := Render(&buf, FormatTable, tests, []string{"m1", "m2"}, results); err != nil { + t.Fatalf("Render() error = %v", err) + } + out := buf.String() + + for _, want := range []string{ + "== Discrimination: 1/2 scored tests separate models by >= 0.05; 1 unstable under repeat ==", + "sep", + "[0.40-1.00]", // unstable range in the score table cell + } { + if !strings.Contains(out, want) { + t.Errorf("output missing %q:\n%s", want, out) + } + } + if strings.Contains(out, "\nsame\t") { + // "same" appears in the per-test table; ensure it is NOT in the + // discrimination rows by checking the section body only. + section := out[strings.Index(out, "== Discrimination"):] + if strings.Contains(section, "same") { + t.Errorf("non-discriminating test listed in discrimination section:\n%s", section) + } + } +} + +func TestDiscrimination_JSONStatsAndAttemptRoundTrip(t *testing.T) { + tests := []testkit.Test{{ID: "t1", Category: "c", Subcategory: "s"}} + results := []runner.Result{ + {Model: "m1", TestID: "t1", Score: eval.Score{Value: 1}}, + {Model: "m2", TestID: "t1", Score: eval.Score{Value: 0.5}}, + {Model: "m1", TestID: "t1", Attempt: 1, Score: eval.Score{Value: 0.5}}, + } + + var buf bytes.Buffer + if err := Render(&buf, FormatJSON, tests, []string{"m1", "m2"}, results); err != nil { + t.Fatalf("Render() error = %v", err) + } + var doc jsonArtifact + if err := json.Unmarshal(buf.Bytes(), &doc); err != nil { + t.Fatalf("unmarshal: %v", err) + } + if len(doc.Stats) != 1 { + t.Fatalf("len(stats) = %d, want 1", len(doc.Stats)) + } + st := doc.Stats[0] + if !st.Unstable { + t.Error("Unstable = false, want true (attempts 1.0 vs 0.5)") + } + if st.Spread < 0.24 || st.Spread > 0.26 { + t.Errorf("Spread = %v, want ~0.25 (m1 mean 0.75 vs m2 0.5)", st.Spread) + } + + // Attempt indices survive the round trip for compare. + var attempts []AttemptResult + attempts = append(attempts, doc.Results...) + if attempts[2].Attempt != 1 { + t.Errorf("attempt = %d, want 1", attempts[2].Attempt) + } +} diff --git a/internal/report/health.go b/internal/report/health.go new file mode 100644 index 0000000..78126de --- /dev/null +++ b/internal/report/health.go @@ -0,0 +1,222 @@ +package report + +import ( + "fmt" + "io" + "sort" +) + +// PerfectScore is the score at which a (model, test) cell counts as a +// perfect pass. Partial-credit evaluators score between 0 and 1; anything +// below this is not a perfect pass. +const PerfectScore = 0.999 + +// SaturationRate is the share of a test's scored cells that must be +// perfect for the test to count as saturated - passed by every model it +// was run against, every attempt. Saturated tests carry no signal about +// relative capability; they only detect outright regressions. +const SaturationRate = 0.9 + +// SubHealth is the suite-health rollup for one category/subcategory pair, +// pooled across one or more artifacts (so single-model probe runs can be +// combined with a full multi-model run). Cells are attempt-aggregated +// (model, test) means: a pair counts once, taking its WORST artifact - +// health asks "can any observed run fail this test?", so pooling over +// re-runs must not let a lucky second attempt hide a real failure. +type SubHealth struct { + Category string + Subcategory string + WeakTests []string // non-saturated test IDs, sorted - the signal-carriers + Tests int // distinct tests with at least one scored cell + Cells int // scored (model, test) cells + Perfect int // cells at PerfectScore or above + Saturated int // tests perfect in >= SaturationRate of their cells + Unstable int // tests whose repeated attempts disagreed + Truncated int // tests with a truncated cell +} + +// PerfectRate is the share of scored cells that are perfect. +func (h SubHealth) PerfectRate() float64 { + if h.Cells == 0 { + return 0 + } + return float64(h.Perfect) / float64(h.Cells) +} + +// HealthReport pools every subcategory present in the artifacts. +type HealthReport struct { + Subcategories []SubHealth + Models []string + Tests int // distinct scored tests across all subcategories + Cells int + Perfect int + Saturated int + Artifacts int +} + +// healthTestID scopes a test ID by model for cell pooling. +type healthCellID struct{ model, testID string } + +// healthTest accumulates one test's cells across artifacts. +type healthTest struct { + cells map[healthCellID]float64 + category, subcategory string + unstable, truncated bool +} + +// AuditHealth pools artifacts into a per-subcategory health view. +// Errored and skipped attempts never form cells (meanAttempts already +// separates them by Status); truncated attempts follow the same +// exclusion policy as compare. Per-test unstable/truncated flags are +// OR-ed across artifacts: once flaky, always flagged. +func AuditHealth(artifacts []Artifact) HealthReport { + agg := map[string]*healthTest{} // key: testID + modelSet := map[string]bool{} + + for _, a := range artifacts { + for _, c := range meanAttempts(a.Results) { + if c.Status != "" && c.Status != "scored" { + continue + } + modelSet[c.Model] = true + ht := agg[c.TestID] + if ht == nil { + ht = &healthTest{category: c.Category, subcategory: c.Subcategory, cells: map[healthCellID]float64{}} + if ht.category == "" { + ht.category = "unknown" + } + if ht.subcategory == "" { + ht.subcategory = "unknown" + } + agg[c.TestID] = ht + } + id := healthCellID{c.Model, c.TestID} + if prev, ok := ht.cells[id]; !ok || c.BaselineMean < prev { + // Pool the worst observed mean for this cell. On the + // first artifact BaselineMean is the current run's mean. + ht.cells[id] = c.BaselineMean + } + } + for _, s := range a.Stats { + if ht, ok := agg[s.TestID]; ok { + ht.unstable = ht.unstable || s.Unstable + ht.truncated = ht.truncated || s.Truncated + } + } + } + + models := make([]string, 0, len(modelSet)) + for m := range modelSet { + models = append(models, m) + } + sort.Strings(models) + + bySub := map[string]*SubHealth{} + var tot, totCells, totPerfect, totSat int + for testID, ht := range agg { + sk := ht.category + "/" + ht.subcategory + sh := bySub[sk] + if sh == nil { + sh = &SubHealth{Category: ht.category, Subcategory: ht.subcategory} + bySub[sk] = sh + } + sh.Tests++ + tot++ + var perfect int + for _, s := range ht.cells { + sh.Cells++ + totCells++ + if s >= PerfectScore { + perfect++ + sh.Perfect++ + totPerfect++ + } + } + if sh.Cells > 0 && ht.unstable { + sh.Unstable++ + } + if ht.truncated { + sh.Truncated++ + } + if float64(perfect)/float64(len(ht.cells)) >= SaturationRate { + sh.Saturated++ + totSat++ + } else { + sh.WeakTests = append(sh.WeakTests, testID) + } + } + + report := HealthReport{Artifacts: len(artifacts), Models: models, Tests: tot, Cells: totCells, Perfect: totPerfect, Saturated: totSat} + for _, sh := range bySub { + sort.Strings(sh.WeakTests) + report.Subcategories = append(report.Subcategories, *sh) + } + sort.Slice(report.Subcategories, func(i, j int) bool { + a, b := report.Subcategories[i], report.Subcategories[j] + if a.PerfectRate() != b.PerfectRate() { + return a.PerfectRate() > b.PerfectRate() + } + if a.Category != b.Category { + return a.Category < b.Category + } + return a.Subcategory < b.Subcategory + }) + return report +} + +// weakCap bounds listed weak tests per subcategory in the text report; +// the counts stay accurate regardless. +const weakCap = 8 + +// RenderHealth writes the audit: headline numbers, then one line per +// subcategory sorted by perfect-cell rate descending (most saturated - +// least informative - first), listing its non-saturated tests. +func RenderHealth(w io.Writer, r HealthReport) error { + ew := &errWriter{w: w} + perfectRate := 0.0 + if r.Cells > 0 { + perfectRate = float64(r.Perfect) / float64(r.Cells) + } + ew.println(fmt.Sprintf("== Suite health: %d tests, %d scored (model,test) cells over %d model(s), %d artifact(s) ==", + r.Tests, r.Cells, len(r.Models), r.Artifacts)) + ew.println(fmt.Sprintf("perfect cells: %d/%d (%.1f%%) | saturated tests (no signal): %d/%d\n", + r.Perfect, r.Cells, perfectRate*100, r.Saturated, r.Tests)) + if r.Saturated == r.Tests && r.Tests > 0 { + ew.println("every scored test is saturated: the suite cannot rank these models at all.") + } + ew.println(fmt.Sprintf("%-30s %5s %6s %8s %9s %8s %8s WEAK TESTS", + "SUBCATEGORY", "TESTS", "CELLS", "PERFECT%", "SATURATED", "UNSTBL", "TRUNC")) + for _, sh := range r.Subcategories { + name := sh.Category + "/" + sh.Subcategory + weak := "" + if len(sh.WeakTests) > 0 { + weak = fmt.Sprintf("%d", len(sh.WeakTests)) + if len(sh.WeakTests) > weakCap { + weak += ": " + joinN(sh.WeakTests, weakCap) + ", ..." + } else { + weak += ": " + joinAll(sh.WeakTests) + } + } + ew.println(fmt.Sprintf("%-30s %5d %6d %8.1f %9d %8d %8d %s", + name, sh.Tests, sh.Cells, sh.PerfectRate()*100, sh.Saturated, sh.Unstable, sh.Truncated, weak)) + } + return ew.err +} + +func joinN(xs []string, n int) string { + if len(xs) <= n { + return joinAll(xs) + } + return joinAll(xs[:n]) +} + +func joinAll(xs []string) string { + out := "" + for i, x := range xs { + if i > 0 { + out += ", " + } + out += x + } + return out +} diff --git a/internal/report/health_test.go b/internal/report/health_test.go new file mode 100644 index 0000000..763cc3c --- /dev/null +++ b/internal/report/health_test.go @@ -0,0 +1,200 @@ +package report + +import ( + "bytes" + "strings" + "testing" +) + +// scoredAttempt builds a scored attempt in subcategory (cat, sub). +func scoredAttempt(model, testID, cat, sub string, score float64) AttemptResult { + return AttemptResult{Model: model, TestID: testID, Category: cat, Subcategory: sub, Score: score} +} + +func subHealth(t *testing.T, r HealthReport, cat, sub string) SubHealth { + t.Helper() + for _, sh := range r.Subcategories { + if sh.Category == cat && sh.Subcategory == sub { + return sh + } + } + t.Fatalf("subcategory %s/%s missing from report", cat, sub) + return SubHealth{} +} + +func TestAuditHealth_SaturatedVersusWeakTests(t *testing.T) { + // a: perfect for both models => saturated, no signal. + // b: model m2 fails => weak, carries signal. + art := Artifact{Results: []AttemptResult{ + scoredAttempt("m1", "a", "c", "s", 1.0), + scoredAttempt("m2", "a", "c", "s", 1.0), + scoredAttempt("m1", "b", "c", "s", 1.0), + scoredAttempt("m2", "b", "c", "s", 0.0), + }} + r := AuditHealth([]Artifact{art}) + + sh := subHealth(t, r, "c", "s") + if sh.Tests != 2 || sh.Cells != 4 { + t.Fatalf("tests/cells = %d/%d, want 2/4", sh.Tests, sh.Cells) + } + if sh.Perfect != 3 { + t.Errorf("perfect = %d, want 3", sh.Perfect) + } + if sh.Saturated != 1 { + t.Errorf("saturated = %d, want 1 (only test a)", sh.Saturated) + } + if len(sh.WeakTests) != 1 || sh.WeakTests[0] != "b" { + t.Errorf("weak = %v, want [b]", sh.WeakTests) + } + if r.Saturated != 1 || r.Tests != 2 { + t.Errorf("report totals = %d/%d, want 1/2", r.Saturated, r.Tests) + } + if len(r.Models) != 2 || r.Models[0] != "m1" { + t.Errorf("models = %v, want [m1 m2]", r.Models) + } +} + +func TestAuditHealth_PoolsWorstCellAcrossArtifacts(t *testing.T) { + // A flaky test that passes on a second artifact must still count as + // weak: health asks whether any observed run failed it. + a1 := Artifact{Results: []AttemptResult{ + scoredAttempt("m1", "flaky", "c", "s", 0.0), + }} + a2 := Artifact{Results: []AttemptResult{ + scoredAttempt("m1", "flaky", "c", "s", 1.0), + }} + r := AuditHealth([]Artifact{a1, a2}) + + sh := subHealth(t, r, "c", "s") + if sh.Saturated != 0 || len(sh.WeakTests) != 1 { + t.Errorf("saturated=%d weak=%v, want 0/[flaky] (worst cell pooled)", sh.Saturated, sh.WeakTests) + } + if sh.Cells != 1 { + t.Errorf("cells = %d, want 1 (same model,test pooled)", sh.Cells) + } +} + +func TestAuditHealth_AveragesRepeatedAttemptsPerCell(t *testing.T) { + // Two attempts of one cell: mean 0.5 < PerfectScore => not perfect. + art := Artifact{Results: []AttemptResult{ + scoredAttempt("m1", "t", "c", "s", 1.0), + scoredAttempt("m1", "t", "c", "s", 0.0), + }} + r := AuditHealth([]Artifact{art}) + + sh := subHealth(t, r, "c", "s") + if sh.Cells != 1 || sh.Perfect != 0 { + t.Errorf("cells/perfect = %d/%d, want 1/0 (attempt mean 0.5)", sh.Cells, sh.Perfect) + } +} + +func TestAuditHealth_ErrorsAndSkipsExcluded(t *testing.T) { + art := Artifact{Results: []AttemptResult{ + {Model: "m1", TestID: "boom", Category: "c", Subcategory: "s", Error: "404"}, + {Model: "m1", TestID: "skip", Category: "c", Subcategory: "s", Skipped: true}, + scoredAttempt("m1", "ok", "c", "s", 1.0), + }} + r := AuditHealth([]Artifact{art}) + + sh := subHealth(t, r, "c", "s") + if sh.Tests != 1 || sh.Cells != 1 || sh.Saturated != 1 { + t.Errorf("tests/cells/saturated = %d/%d/%d, want 1/1/1 (error+skip excluded)", + sh.Tests, sh.Cells, sh.Saturated) + } +} + +func TestAuditHealth_FlagsUnstableAndTruncated(t *testing.T) { + art := Artifact{ + Results: []AttemptResult{ + scoredAttempt("m1", "u", "c", "s", 1.0), + scoredAttempt("m1", "t", "c", "s", 0.5), + }, + Stats: []jsonStatRow{ + {TestID: "u", Unstable: true}, + {TestID: "t", Truncated: true}, + }, + } + r := AuditHealth([]Artifact{art}) + + sh := subHealth(t, r, "c", "s") + if sh.Unstable != 1 || sh.Truncated != 1 { + t.Errorf("unstable/truncated = %d/%d, want 1/1", sh.Unstable, sh.Truncated) + } +} + +func TestAuditHealth_SaturationBoundary(t *testing.T) { + // 9 perfect cells + 1 failed cell = 0.9 rate = at SaturationRate => + // saturated. 8/9 = 0.888 < rate => weak. + perfect9 := Artifact{} + for i := range 9 { + perfect9.Results = append(perfect9.Results, scoredAttempt("m"+string(rune('a'+i)), "t1", "c", "s", 1.0)) + } + perfect9.Results = append(perfect9.Results, scoredAttempt("mj", "t1", "c", "s", 0.0)) + + if r := AuditHealth([]Artifact{perfect9}); r.Saturated != 1 { + t.Errorf("9/10 perfect: saturated = %d, want 1 (rate == SaturationRate)", r.Saturated) + } + + eight := Artifact{} + for i := range 8 { + eight.Results = append(eight.Results, scoredAttempt("m"+string(rune('a'+i)), "t1", "c", "s", 1.0)) + } + eight.Results = append(eight.Results, + scoredAttempt("mi", "t1", "c", "s", 0.0), + scoredAttempt("mj", "t1", "c", "s", 0.0)) + + if r := AuditHealth([]Artifact{eight}); r.Saturated != 0 { + t.Errorf("8/10 perfect: saturated = %d, want 0", r.Saturated) + } +} + +func TestAuditHealth_SortsMostSaturatedFirst(t *testing.T) { + art := Artifact{Results: []AttemptResult{ + scoredAttempt("m1", "hard", "c", "low", 0.0), + scoredAttempt("m1", "easy1", "c", "high", 1.0), + scoredAttempt("m1", "easy2", "c", "high", 1.0), + }} + r := AuditHealth([]Artifact{art}) + + if len(r.Subcategories) != 2 { + t.Fatalf("subcategories = %d, want 2", len(r.Subcategories)) + } + if got := r.Subcategories[0].Subcategory; got != "high" { + t.Errorf("first row = %q, want most saturated %q", got, "high") + } +} + +func TestRenderHealth_HeadlineAndWeakList(t *testing.T) { + art := Artifact{Results: []AttemptResult{ + scoredAttempt("m1", "a", "c", "s", 1.0), + scoredAttempt("m1", "b", "c", "s", 0.0), + }} + var buf bytes.Buffer + if err := RenderHealth(&buf, AuditHealth([]Artifact{art})); err != nil { + t.Fatalf("RenderHealth() error = %v", err) + } + out := buf.String() + for _, want := range []string{"2 tests", "2 scored", "c/s", "50.0", "1: b"} { + if !strings.Contains(out, want) { + t.Errorf("output missing %q:\n%s", want, out) + } + } + // "a" is saturated: it must not appear anywhere in the weak list. + if strings.Contains(out, ": a") || strings.Contains(out, ", a,") || strings.HasSuffix(strings.TrimSpace(out), " a") { + t.Errorf("saturated test a should not be listed weak:\n%s", out) + } +} + +func TestRenderHealth_AllSaturatedVerdict(t *testing.T) { + art := Artifact{Results: []AttemptResult{ + scoredAttempt("m1", "a", "c", "s", 1.0), + scoredAttempt("m2", "a", "c", "s", 1.0), + }} + var buf bytes.Buffer + if err := RenderHealth(&buf, AuditHealth([]Artifact{art})); err != nil { + t.Fatalf("RenderHealth() error = %v", err) + } + if !strings.Contains(buf.String(), "cannot rank these models") { + t.Errorf("missing all-saturated verdict:\n%s", buf.String()) + } +} diff --git a/internal/report/json.go b/internal/report/json.go index ba593da..a64e5d7 100644 --- a/internal/report/json.go +++ b/internal/report/json.go @@ -2,14 +2,17 @@ package report import ( "encoding/json" + "fmt" "io" + "math" + "os" "github.com/lukaszraczylo/llm-testbench/internal/runner" "github.com/lukaszraczylo/llm-testbench/internal/testkit" ) -// jsonResult is the JSON-serializable projection of a runner.Result, joined -// with its test's category/subcategory for self-contained output. +// jsonResult is the JSON-serializable projection of one runner.Result +// (one attempt), joined with its test's category/subcategory. type jsonResult struct { Model string `json:"model"` TestID string `json:"test_id"` @@ -26,16 +29,43 @@ type jsonResult struct { Tokens int `json:"tokens"` Skipped bool `json:"skipped"` Truncated bool `json:"truncated"` + Attempt int `json:"attempt,omitempty"` } -// renderJSON dumps raw results as a JSON array, joined with test metadata. -func renderJSON(w io.Writer, tests []testkit.Test, results []runner.Result) error { +// jsonArtifact is the --out / --format json document: every raw attempt +// plus the per-test discrimination rollup, so downstream tooling +// (compare) can re-derive stability without re-running the suite. +type jsonArtifact struct { + Results []jsonResult `json:"results"` + Stats []jsonStatRow `json:"stats,omitempty"` +} + +// jsonStatRow is the JSON projection of testStat. +type jsonStatRow struct { + TestID string `json:"test_id"` + Category string `json:"category"` + Subcategory string `json:"subcategory"` + Mean float64 `json:"mean"` + Spread float64 `json:"spread,omitempty"` + Unstable bool `json:"unstable,omitempty"` + Truncated bool `json:"truncated,omitempty"` +} + +// renderJSON dumps the run as a JSON artifact document. +func renderJSON(w io.Writer, tests []testkit.Test, models []string, results []runner.Result) error { + artifact := buildArtifact(tests, models, results) + enc := json.NewEncoder(w) + enc.SetIndent("", " ") + return enc.Encode(artifact) +} + +func buildArtifact(tests []testkit.Test, models []string, results []runner.Result) jsonArtifact { testByID := make(map[string]testkit.Test, len(tests)) for _, t := range tests { testByID[t.ID] = t } - out := make([]jsonResult, 0, len(results)) + out := jsonArtifact{Results: make([]jsonResult, 0, len(results))} for _, r := range results { t := testByID[r.TestID] jr := jsonResult{ @@ -53,14 +83,79 @@ func renderJSON(w io.Writer, tests []testkit.Test, results []runner.Result) erro PromptTokens: r.PromptTokens, CompletionTokens: r.CompletionTokens, Tokens: r.TotalTokens(), + Attempt: r.Attempt, } if r.Err != nil { jr.Error = r.Err.Error() } - out = append(out, jr) + out.Results = append(out.Results, jr) } - enc := json.NewEncoder(w) - enc.SetIndent("", " ") - return enc.Encode(out) + idx := indexResults(results) + for _, st := range testStats(tests, models, idx) { + if !st.scored { + continue + } + row := jsonStatRow{ + TestID: st.testID, + Category: st.category, + Subcategory: st.subcategory, + Mean: round6(st.mean), + Unstable: st.unstable, + Truncated: st.truncated, + } + if !math.IsNaN(st.spread) { + row.Spread = round6(st.spread) + } + out.Stats = append(out.Stats, row) + } + return out +} + +func round6(v float64) float64 { + return math.Round(v*1e6) / 1e6 +} + +// Artifact is a saved run document: every raw attempt plus the per-test +// discrimination rollup. Exported so the compare command can consume it. +type Artifact = jsonArtifact + +// AttemptResult is one saved attempt. Exported alias of jsonResult. +type AttemptResult = jsonResult + +// LoadArtifact reads a --out JSON artifact written by a previous run. +// Accepts both the current document shape and the legacy bare array of +// results, so artifacts from earlier releases remain comparable. +func LoadArtifact(path string) (jsonArtifact, error) { + b, err := os.ReadFile(path) //nolint:gosec // path is the operator's own `llmtest compare` argument, not external input + if err != nil { + return jsonArtifact{}, err + } + + var probe struct { + Results []jsonResult `json:"results"` + } + if err := json.Unmarshal(b, &probe); err == nil && probe.Results != nil { + var a jsonArtifact + if err := json.Unmarshal(b, &a); err != nil { + return jsonArtifact{}, fmt.Errorf("%s: %w", path, err) + } + return a, nil + } + + var legacy []jsonResult + if err := json.Unmarshal(b, &legacy); err != nil { + return jsonArtifact{}, fmt.Errorf("%s: not a llmtest json artifact: %w", path, err) + } + return jsonArtifact{Results: legacy}, nil +} + +// WriteArtifact saves the run as a JSON artifact at path, for later +// `llmtest compare` runs, regardless of the stdout report format. +func WriteArtifact(path string, tests []testkit.Test, models []string, results []runner.Result) error { + b, err := json.MarshalIndent(buildArtifact(tests, models, results), "", " ") + if err != nil { + return err + } + return os.WriteFile(path, append(b, '\n'), 0o644) //nolint:gosec // 0o644: the artifact is a report meant to be shared/committed, not a secret file } diff --git a/internal/report/markdown.go b/internal/report/markdown.go index ad229f3..c389d6e 100644 --- a/internal/report/markdown.go +++ b/internal/report/markdown.go @@ -10,8 +10,8 @@ import ( "github.com/lukaszraczylo/llm-testbench/internal/testkit" ) -// renderMarkdown writes the same three tables as renderTable, formatted as -// GitHub-flavored markdown tables. +// renderMarkdown writes the same four sections as renderTable, formatted +// as GitHub-flavored markdown tables. func renderMarkdown(w io.Writer, tests []testkit.Test, models []string, results []runner.Result) error { idx := indexResults(results) ew := &errWriter{w: w} @@ -24,9 +24,9 @@ func renderMarkdown(w io.Writer, tests []testkit.Test, models []string, results for _, t := range sortedTests(tests) { row := []string{t.ID, t.Category, t.Subcategory} for _, model := range models { - r, ok := idx[t.ID][model] - row = append(row, cellText(r, ok)) - anyTruncated = anyTruncated || r.Truncated() + c := idx[t.ID][model] + row = append(row, cellText(c)) + anyTruncated = anyTruncated || c.result.Truncated() } writeMarkdownRow(ew, row) } @@ -48,11 +48,16 @@ func renderMarkdown(w io.Writer, tests []testkit.Test, models []string, results writeMarkdownRow(ew, row) } + ew.println() + ew.println("## Discrimination") + ew.println() + writeDiscriminationRows(markdownPrinter{ew: ew}, testStats(tests, models, idx), len(models) > 1) + ew.println() ew.println("## Model summary") ew.println() writeMarkdownHeader(ew, []string{"Model", "Mean", "Passed", "Partial", "Failed", "Errors", "Mean latency", "Total tokens", "Tok/s"}) - for _, ms := range summarize(models, results) { + for _, ms := range summarize(models, idx) { writeMarkdownRow(ew, []string{ ms.model, meanCellText(ms.overallMean), @@ -69,6 +74,18 @@ func renderMarkdown(w io.Writer, tests []testkit.Test, models []string, results return ew.err } +// markdownPrinter adapts writeDiscriminationRows to pipe-delimited rows. +// GFM needs a blank line between the note paragraph and the table, so +// header() emits one first. +type markdownPrinter struct{ ew *errWriter } + +func (mp markdownPrinter) header(cells ...string) { + mp.ew.println() + writeMarkdownHeader(mp.ew, cells) +} +func (mp markdownPrinter) row(cells ...string) { writeMarkdownRow(mp.ew, cells) } +func (mp markdownPrinter) note(text string) { mp.ew.println(text) } + func writeMarkdownHeader(ew *errWriter, cols []string) { ew.println("| " + strings.Join(cols, " | ") + " |") sep := make([]string, len(cols)) diff --git a/internal/report/report.go b/internal/report/report.go index 876cae0..c8fe84a 100644 --- a/internal/report/report.go +++ b/internal/report/report.go @@ -9,6 +9,8 @@ import ( "sort" "time" + "github.com/lukaszraczylo/llm-testbench/internal/eval" + "github.com/lukaszraczylo/llm-testbench/internal/llm" "github.com/lukaszraczylo/llm-testbench/internal/runner" "github.com/lukaszraczylo/llm-testbench/internal/testkit" ) @@ -35,22 +37,125 @@ func Render(w io.Writer, format Format, tests []testkit.Test, models []string, r case FormatMarkdown: return renderMarkdown(w, tests, models, results) case FormatJSON: - return renderJSON(w, tests, results) + return renderJSON(w, tests, models, results) default: return fmt.Errorf("report: unknown format %q", format) } } -// resultIndex looks up a Result by testID then model. -type resultIndex map[string]map[string]runner.Result +// modelCell is the aggregate view of every attempt recorded for one +// (testID, model) pair: with Config.Repeat = 1 this is the single result; +// with repeats the Score is the mean over scored attempts, and minScore/ +// maxScore expose the within-model spread (instability) that repeats are +// measured to find. +type modelCell struct { + result runner.Result // representative: last scored attempt, mean score patched in + minScore float64 + maxScore float64 + meanTokens float64 + attempts int + scored int + skippedAll bool + errorAll bool + ok bool +} -func indexResults(results []runner.Result) resultIndex { - idx := make(resultIndex) +// cellIndex looks up the aggregate cell for a testID then model. +type cellIndex map[string]map[string]modelCell + +func indexResults(results []runner.Result) cellIndex { + type key struct{ testID, model string } + grouped := make(map[key][]runner.Result) + order := make([]key, 0, len(results)) for _, r := range results { - if idx[r.TestID] == nil { - idx[r.TestID] = make(map[string]runner.Result) + k := key{r.TestID, r.Model} + if _, seen := grouped[k]; !seen { + order = append(order, k) } - idx[r.TestID][r.Model] = r + grouped[k] = append(grouped[k], r) + } + + idx := make(cellIndex) + for _, k := range order { + attempts := grouped[k] + sort.Slice(attempts, func(i, j int) bool { return attempts[i].Attempt < attempts[j].Attempt }) + + c := modelCell{ok: true, attempts: len(attempts)} + // Truncation policy: a truncated attempt was scored on partial + // text, so its score measures the token budget as much as the + // model. Attempts that finished normally are therefore preferred + // for the cell mean (and for the min-max instability range); only + // when EVERY scored attempt was cut off does the mean fall back to + // the truncated scores. A cell is flagged "!" whenever any attempt + // was truncated, clean mean or not. + var sum, toks, cleanSum, cleanToks float64 + var cleanN, scoredN int + var cleanMin, cleanMax float64 + var anyTrunc bool + var last, lastClean runner.Result + for _, a := range attempts { + last = a + switch { + case a.Err != nil: + continue + case a.Score.Skipped: + continue + } + v := a.Score.Value + sum += v + toks += float64(a.TotalTokens()) + scoredN++ + if a.Truncated() { + anyTrunc = true + continue + } + if cleanN == 0 { + cleanMin, cleanMax = v, v + } else { + cleanMin = math.Min(cleanMin, v) + cleanMax = math.Max(cleanMax, v) + } + cleanSum += v + cleanToks += float64(a.TotalTokens()) + cleanN++ + lastClean = a + } + c.scored = scoredN + + rep := last + if cleanN > 0 { + rep = lastClean + } + switch { + case scoredN > 0 && cleanN > 0: + rep.Score = eval.Score{ + Value: cleanSum / float64(cleanN), + Detail: lastClean.Score.Detail, + } + c.minScore, c.maxScore = cleanMin, cleanMax + c.meanTokens = cleanToks / float64(cleanN) + if anyTrunc { + rep.FinishReason = llm.FinishReasonLength + } + case scoredN > 0: + rep.Score = eval.Score{ + Value: sum / float64(scoredN), + Detail: last.Score.Detail, + } + c.minScore, c.maxScore = rep.Score.Value, rep.Score.Value + c.meanTokens = toks / float64(scoredN) + rep.FinishReason = llm.FinishReasonLength + case c.attempts > 0 && last.Score.Skipped: + c.skippedAll = true + case c.attempts > 0 && last.Err != nil: + c.errorAll = true + } + + c.result = rep + if idx[k.testID] == nil { + idx[k.testID] = make(map[string]modelCell) + } + idx[k.testID][k.model] = c } return idx } @@ -72,34 +177,41 @@ func sortedTests(tests []testkit.Test) []testkit.Test { return out } -// truncatedSuffix marks a cell whose underlying response was cut off by -// the token budget (finish_reason=length): the score may not reflect what -// the model would have answered with a larger budget. truncatedLegend is -// the one-line explanation printed once per table/markdown report. +// truncatedSuffix marks a cell where any attempt was cut off by the token +// budget (finish_reason=length). Per the truncation policy in +// indexResults, the cell mean then comes from the attempts that finished +// normally; "!" warns that at least one sample was discarded (or, when no +// attempt finished, that the whole mean is partial-text based). +// truncatedLegend is the one-line explanation printed once per +// table/markdown report. const ( truncatedSuffix = "!" - truncatedLegend = "! = response truncated by the token budget (finish_reason=length); increase max_tokens_default to avoid" + truncatedLegend = "! = some attempt was truncated by the token budget (finish_reason=length); the mean uses only attempts that finished, increase max_tokens_default if every attempt is flagged" ) // cellText renders one (test, model) cell for the score table: "score // (Ntok)" for a scored result, or ERR/skip/N/A otherwise, with a -// truncatedSuffix appended when the response was cut off by the token -// budget. -func cellText(r runner.Result, found bool) string { - if !found { +// truncatedSuffix appended when any attempt was cut off by the token +// budget. With repeat attempts, a "[min-max]" range is appended when the +// attempts disagreed, marking the cell unstable. +func cellText(c modelCell) string { + if !c.ok { + return "N/A" + } + switch { + case c.errorAll: + return "ERR" + case c.skippedAll: + return "skip" + case c.scored == 0: return "N/A" } - var text string - switch { - case r.Err != nil: - text = "ERR" - case r.Score.Skipped: - text = "skip" - default: - text = fmt.Sprintf("%.2f (%dtok)", r.Score.Value, r.TotalTokens()) + text := fmt.Sprintf("%.2f (%dtok)", c.result.Score.Value, int(math.Round(c.meanTokens))) + if c.attempts > 1 && c.maxScore > c.minScore { + text += fmt.Sprintf(" [%0.2f-%0.2f]", c.minScore, c.maxScore) } - if r.Truncated() { + if c.result.Truncated() { text += truncatedSuffix } return text @@ -112,7 +224,7 @@ type categoryMean struct { category string } -func categoryRollup(tests []testkit.Test, models []string, idx resultIndex) []categoryMean { +func categoryRollup(tests []testkit.Test, models []string, idx cellIndex) []categoryMean { order := make([]string, 0) byCategory := make(map[string][]testkit.Test) for _, t := range sortedTests(tests) { @@ -129,11 +241,11 @@ func categoryRollup(tests []testkit.Test, models []string, idx resultIndex) []ca var sum float64 var n int for _, t := range byCategory[cat] { - r, ok := idx[t.ID][model] - if !ok || r.Err != nil || r.Score.Skipped { + c, ok := idx[t.ID][model] + if !ok || c.errorAll || c.skippedAll || c.scored == 0 { continue } - sum += r.Score.Value + sum += c.result.Score.Value n++ } if n > 0 { @@ -191,49 +303,49 @@ type modelSummary struct { avgTokPerSec float64 // math.NaN() when no timed, non-skipped data } -func summarize(models []string, results []runner.Result) []modelSummary { - byModel := make(map[string][]runner.Result, len(models)) - for _, r := range results { - byModel[r.Model] = append(byModel[r.Model], r) - } - +// summarize aggregates one model's results per test (attempts already +// merged by indexResults), so pass/partial/fail counts stay per-test even +// with Config.Repeat > 1. +func summarize(models []string, idx cellIndex) []modelSummary { out := make([]modelSummary, 0, len(models)) for _, model := range models { ms := modelSummary{model: model} - var scoreSum float64 - var scoredCount int + var scoreSum, latencySecondsSum, meanTokenSum float64 + var scoredCount, latencyCount int var latencySum time.Duration - var latencyCount int - var completionTokenSum int - var latencySecondsSum float64 - for _, r := range byModel[model] { - ms.totalTokens += r.TotalTokens() - switch { - case r.Err != nil: - ms.errors++ + for _, cells := range idx { + c, ok := cells[model] + if !ok { continue - case r.Score.Skipped: + } + meanTokenSum += c.meanTokens * float64(max(c.scored, 1)) + if c.errorAll { + ms.errors += c.attempts + continue + } + if c.skippedAll || c.scored == 0 { continue } - scoreSum += r.Score.Value + v := c.result.Score.Value + scoreSum += v scoredCount++ - latencySum += r.Latency + latencySum += c.result.Latency latencyCount++ - completionTokenSum += r.CompletionTokens - latencySecondsSum += r.Latency.Seconds() + latencySecondsSum += c.result.Latency.Seconds() switch { - case r.Score.Value >= passThreshold: + case v >= passThreshold: ms.passed++ - case r.Score.Value <= 0: + case v <= 0: ms.failed++ default: ms.partial++ } } + ms.totalTokens = int(math.Round(meanTokenSum)) if scoredCount > 0 { ms.overallMean = scoreSum / float64(scoredCount) ms.meanLatency = latencySum / time.Duration(latencyCount) @@ -241,11 +353,11 @@ func summarize(models []string, results []runner.Result) []modelSummary { ms.overallMean = math.NaN() } - // Aggregate ratio (sum of completion tokens over sum of elapsed - // seconds), not a mean of per-test ratios: robust to a handful of - // very short responses skewing a per-test-averaged rate. + // Aggregate ratio (completion tokens over elapsed seconds for the + // representative attempt), not a mean of per-test ratios: robust to + // a handful of very short responses skewing a per-test-averaged rate. if latencySecondsSum > 0 { - ms.avgTokPerSec = float64(completionTokenSum) / latencySecondsSum + ms.avgTokPerSec = float64(ms.totalTokens) / latencySecondsSum } else { ms.avgTokPerSec = math.NaN() } diff --git a/internal/report/report_test.go b/internal/report/report_test.go index f275046..2b619ed 100644 --- a/internal/report/report_test.go +++ b/internal/report/report_test.go @@ -10,6 +10,7 @@ import ( "time" "github.com/lukaszraczylo/llm-testbench/internal/eval" + "github.com/lukaszraczylo/llm-testbench/internal/llm" "github.com/lukaszraczylo/llm-testbench/internal/runner" "github.com/lukaszraczylo/llm-testbench/internal/testkit" ) @@ -33,6 +34,47 @@ func sampleResults() []runner.Result { } } +// Truncation policy: attempts cut off by the token budget are excluded +// from a cell's mean when any attempt finished normally; the cell is still +// flagged "!" so the discarded sample is visible. When every scored +// attempt was truncated, the mean falls back to the partial-text scores. +func TestIndexResults_TruncatedAttemptExcludedFromMean(t *testing.T) { + idx := indexResults([]runner.Result{ + {Model: "m1", TestID: "t", Attempt: 0, Score: eval.Score{Value: 1}, PromptTokens: 4}, + {Model: "m1", TestID: "t", Attempt: 1, Score: eval.Score{Value: 0}, FinishReason: llm.FinishReasonLength, PromptTokens: 100}, + }) + c := idx["t"]["m1"] + if got := c.result.Score.Value; got != 1 { + t.Errorf("mean = %v, want 1 (clean attempts only)", got) + } + if !c.result.Truncated() { + t.Error("cell should be flagged truncated") + } + if c.minScore != 1 || c.maxScore != 1 { + t.Errorf("min/max = %v/%v, want 1/1 (truncated attempt excluded from instability range)", c.minScore, c.maxScore) + } + if got := int(c.meanTokens); got != 4 { + t.Errorf("meanTokens = %d, want 4 (clean attempts only)", got) + } +} + +func TestIndexResults_AllAttemptsTruncatedFallBackToPartialScores(t *testing.T) { + idx := indexResults([]runner.Result{ + {Model: "m1", TestID: "t", Attempt: 0, Score: eval.Score{Value: 0.4}, FinishReason: llm.FinishReasonLength, PromptTokens: 10}, + {Model: "m1", TestID: "t", Attempt: 1, Score: eval.Score{Value: 0.6}, FinishReason: llm.FinishReasonLength, PromptTokens: 20}, + }) + c := idx["t"]["m1"] + if got := c.result.Score.Value; math.Abs(got-0.5) > 1e-9 { + t.Errorf("mean = %v, want 0.5 (fallback over all-truncated attempts)", got) + } + if !c.result.Truncated() { + t.Error("cell should be flagged truncated") + } + if c.scored != 2 { + t.Errorf("scored = %d, want 2", c.scored) + } +} + func TestRenderTable_ContainsExpectedCells(t *testing.T) { var buf bytes.Buffer if err := Render(&buf, FormatTable, sampleTests(), []string{"m1", "m2"}, sampleResults()); err != nil { @@ -85,10 +127,11 @@ func TestRenderJSON_DumpsRawResults(t *testing.T) { t.Fatalf("Render() error = %v", err) } - var out []jsonResult - if err := json.Unmarshal(buf.Bytes(), &out); err != nil { + var doc jsonArtifact + if err := json.Unmarshal(buf.Bytes(), &doc); err != nil { t.Fatalf("json.Unmarshal() error = %v; output:\n%s", err, buf.String()) } + out := doc.Results if len(out) != len(sampleResults()) { t.Fatalf("len(out) = %d, want %d", len(out), len(sampleResults())) } @@ -173,14 +216,14 @@ func TestRenderJSON_IncludesResponseTextFinishReasonAndTruncated(t *testing.T) { t.Fatalf("Render() error = %v", err) } - var out []jsonResult - if err := json.Unmarshal(buf.Bytes(), &out); err != nil { + var doc jsonArtifact + if err := json.Unmarshal(buf.Bytes(), &doc); err != nil { t.Fatalf("json.Unmarshal() error = %v; output:\n%s", err, buf.String()) } - if len(out) != 1 { - t.Fatalf("len(out) = %d, want 1", len(out)) + if len(doc.Results) != 1 { + t.Fatalf("len(results) = %d, want 1", len(doc.Results)) } - got := out[0] + got := doc.Results[0] if got.ResponseText != "partial answer" { t.Errorf("ResponseText = %q, want %q", got.ResponseText, "partial answer") } @@ -211,7 +254,7 @@ func TestSummarize_Buckets(t *testing.T) { {Model: "m1", TestID: "d", Err: errors.New("fail")}, {Model: "m1", TestID: "e", Score: eval.Score{Skipped: true}}, } - summaries := summarize([]string{"m1"}, results) + summaries := summarize([]string{"m1"}, indexResults(results)) if len(summaries) != 1 { t.Fatalf("summarize() len = %d, want 1", len(summaries)) } @@ -236,7 +279,7 @@ func TestSummarize_AvgTokPerSec(t *testing.T) { {Model: "m1", TestID: "err", Err: errors.New("fail"), Latency: time.Second, CompletionTokens: 999}, {Model: "m1", TestID: "skipped", Score: eval.Score{Skipped: true}, Latency: time.Second, CompletionTokens: 999}, } - summaries := summarize([]string{"m1"}, results) + summaries := summarize([]string{"m1"}, indexResults(results)) ms := summaries[0] want := 110.0 / 2.1 @@ -250,7 +293,7 @@ func TestSummarize_AvgTokPerSec_NoDataIsNaN(t *testing.T) { {Model: "m1", TestID: "err", Err: errors.New("fail")}, {Model: "m1", TestID: "skipped", Score: eval.Score{Skipped: true}}, } - summaries := summarize([]string{"m1"}, results) + summaries := summarize([]string{"m1"}, indexResults(results)) if !math.IsNaN(summaries[0].avgTokPerSec) { t.Errorf("avgTokPerSec = %v, want NaN when there is no timed data", summaries[0].avgTokPerSec) } diff --git a/internal/report/table.go b/internal/report/table.go index b7335a1..84fe25b 100644 --- a/internal/report/table.go +++ b/internal/report/table.go @@ -26,8 +26,8 @@ func (ew *errWriter) println(args ...any) { _, ew.err = fmt.Fprintln(ew.w, args...) } -// renderTable writes three plain-text tables: per-test scores, category -// rollup, and model summary. +// renderTable writes four plain-text tables: per-test scores, category +// rollup, discrimination/stability rollup, and model summary. func renderTable(w io.Writer, tests []testkit.Test, models []string, results []runner.Result) error { idx := indexResults(results) tw := tabwriter.NewWriter(w, 0, 4, 2, ' ', 0) @@ -37,7 +37,9 @@ func renderTable(w io.Writer, tests []testkit.Test, models []string, results []r ew.println() writeCategoryTable(ew, tests, models, idx) ew.println() - writeSummaryTable(ew, models, results) + writeDiscriminationTable(ew, testStats(tests, models, idx), len(models) > 1) + ew.println() + writeSummaryTable(ew, models, idx) if ew.err != nil { return ew.err } @@ -45,7 +47,7 @@ func renderTable(w io.Writer, tests []testkit.Test, models []string, results []r return tw.Flush() } -func writeScoreTable(ew *errWriter, tests []testkit.Test, models []string, idx resultIndex) { +func writeScoreTable(ew *errWriter, tests []testkit.Test, models []string, idx cellIndex) { ew.println("== Per-test scores ==") header := []string{"TEST", "CATEGORY", "SUBCATEGORY"} header = append(header, models...) @@ -55,9 +57,9 @@ func writeScoreTable(ew *errWriter, tests []testkit.Test, models []string, idx r for _, t := range sortedTests(tests) { row := []string{t.ID, t.Category, t.Subcategory} for _, model := range models { - r, ok := idx[t.ID][model] - row = append(row, cellText(r, ok)) - anyTruncated = anyTruncated || r.Truncated() + c := idx[t.ID][model] + row = append(row, cellText(c)) + anyTruncated = anyTruncated || c.result.Truncated() } ew.println(strings.Join(row, "\t")) } @@ -66,7 +68,7 @@ func writeScoreTable(ew *errWriter, tests []testkit.Test, models []string, idx r } } -func writeCategoryTable(ew *errWriter, tests []testkit.Test, models []string, idx resultIndex) { +func writeCategoryTable(ew *errWriter, tests []testkit.Test, models []string, idx cellIndex) { ew.println("== Category rollup (mean score) ==") header := append([]string{"CATEGORY"}, models...) ew.println(strings.Join(header, "\t")) @@ -80,11 +82,82 @@ func writeCategoryTable(ew *errWriter, tests []testkit.Test, models []string, id } } -func writeSummaryTable(ew *errWriter, models []string, results []runner.Result) { +// writeDiscriminationTable lists the tests that actually measure +// something: cross-model spread >= DiscriminationSpread, or attempts that +// disagreed under --repeat. With a single model there is no spread to +// speak of, so the non-passing tests are listed instead. +func writeDiscriminationTable(ew *errWriter, stats []testStat, multiModel bool) { + writeDiscriminationRows(tablePrinter{ew: ew}, stats, multiModel) +} + +// discriminationSink abstracts the tab (table) vs pipe (markdown) output +// so the discrimination section is written once. +type discriminationSink interface { + header(cells ...string) + row(cells ...string) + note(text string) +} + +type tablePrinter struct{ ew *errWriter } + +func (tp tablePrinter) header(cells ...string) { tp.ew.println(strings.Join(cells, "\t")) } +func (tp tablePrinter) row(cells ...string) { tp.ew.println(strings.Join(cells, "\t")) } +func (tp tablePrinter) note(text string) { tp.ew.println(text) } + +func writeDiscriminationRows(sink discriminationSink, stats []testStat, multiModel bool) { + discriminating, unstable, scored := 0, 0, 0 + for _, st := range stats { + if !st.scored { + continue + } + scored++ + if st.unstable { + unstable++ + } + if multiModel && !math.IsNaN(st.spread) && st.spread >= DiscriminationSpread { + discriminating++ + } + } + + if multiModel { + sink.note(fmt.Sprintf("== Discrimination: %d/%d scored tests separate models by >= %.2f; %d unstable under repeat ==", + discriminating, scored, DiscriminationSpread, unstable)) + } else { + sink.note(fmt.Sprintf("== Discrimination: single model (no cross-model spread); %d unstable under repeat ==", unstable)) + } + + rows := interestingStats(stats, multiModel) + if len(rows) == 0 { + sink.note("no discriminating or unstable tests: every scored test passed identically for every model") + return + } + + sink.header("TEST", "CATEGORY", "SUBCATEGORY", "MEAN", "SPREAD", "UNSTABLE", "TRUNC") + for _, st := range rows { + sink.row( + st.testID, + st.category, + st.subcategory, + meanCellText(st.mean), + meanCellText(st.spread), + yesNo(st.unstable), + yesNo(st.truncated), + ) + } +} + +func yesNo(b bool) string { + if b { + return "yes" + } + return "-" +} + +func writeSummaryTable(ew *errWriter, models []string, idx cellIndex) { ew.println("== Model summary ==") ew.println(strings.Join([]string{"MODEL", "MEAN", "PASSED", "PARTIAL", "FAILED", "ERRORS", "MEAN_LATENCY", "TOTAL_TOKENS", "TOK/S"}, "\t")) - for _, ms := range summarize(models, results) { + for _, ms := range summarize(models, idx) { row := []string{ ms.model, meanCellText(ms.overallMean), diff --git a/internal/runner/runner.go b/internal/runner/runner.go index 1b1bf27..dec116f 100644 --- a/internal/runner/runner.go +++ b/internal/runner/runner.go @@ -20,7 +20,9 @@ import ( // referencing FinishReason separately. const truncatedDetailPrefix = "TRUNCATED: " -// Result is the outcome of running one test against one model. +// Result is the outcome of running one test against one model. Attempt is +// the zero-based index of this sample when Config.Repeat > 1; it is 0 for +// single-sample runs. type Result struct { Err error Model string @@ -31,6 +33,7 @@ type Result struct { Latency time.Duration PromptTokens int CompletionTokens int + Attempt int } // Truncated reports whether the model's response was cut off by the token @@ -54,6 +57,12 @@ type Config struct { Concurrency int Temperature float64 MaxTokensDefault int + // Repeat runs each (model, test) combination this many times (minimum + // 1). Even at temperature 0, reasoning models sample differently per + // call; repeats expose that instability so a score can be trusted + // (stable across attempts) or flagged (flaky) instead of silently + // treated as a capability measurement. + Repeat int } // Runner executes a set of tests against a set of models. @@ -71,18 +80,24 @@ func New(client llm.Client, cfg Config) *Runner { } // Run executes every combination of models x tests, bounded by -// cfg.Concurrency in-flight calls, and returns one Result per combination. -// A per-call failure is captured in Result.Err rather than aborting the run. +// cfg.Concurrency in-flight calls, and returns one Result per combination +// (times Config.Repeat samples each). A per-call failure is captured in +// Result.Err rather than aborting the run. func (r *Runner) Run(ctx context.Context, models []string, tests []testkit.Test) []Result { type job struct { - model string - test testkit.Test + model string + test testkit.Test + attempt int } - jobs := make([]job, 0, len(models)*len(tests)) + repeat := max(r.cfg.Repeat, 1) + + jobs := make([]job, 0, len(models)*len(tests)*repeat) for _, m := range models { for _, t := range tests { - jobs = append(jobs, job{model: m, test: t}) + for a := range repeat { + jobs = append(jobs, job{model: m, test: t, attempt: a}) + } } } @@ -103,6 +118,7 @@ func (r *Runner) Run(ctx context.Context, models []string, tests []testkit.Test) for i, j := range jobs { g.Go(func() error { res := r.runOne(gctx, j.model, j.test) + res.Attempt = j.attempt results[i] = res // atomic.AddInt32 makes the completion counter safe under // Config.Concurrency-many concurrent goroutines; ReportDone diff --git a/internal/runner/runner_test.go b/internal/runner/runner_test.go index 48ceef6..49743f2 100644 --- a/internal/runner/runner_test.go +++ b/internal/runner/runner_test.go @@ -377,3 +377,36 @@ func TestRunner_Run_TracksLatencyAndTokens(t *testing.T) { t.Errorf("Latency = %v, want 42ms", results[0].Latency) } } + +func TestRunner_Run_RepeatSamplesEachCombination(t *testing.T) { + client := &mockClient{} + r := New(client, Config{Concurrency: 2, MaxTokensDefault: 100, Repeat: 3}) + + tests := []testkit.Test{{ID: "t1", Category: "c", Prompt: "p1", Eval: echoEval()}} + models := []string{"model-a", "model-b"} + + results := r.Run(context.Background(), models, tests) + + if len(results) != 6 { + t.Fatalf("len(results) = %d, want 6 (2 models x 1 test x 3 repeats)", len(results)) + } + seen := make(map[[2]string][]int) + for _, res := range results { + k := [2]string{res.Model, res.TestID} + seen[k] = append(seen[k], res.Attempt) + } + for k, attempts := range seen { + if len(attempts) != 3 { + t.Errorf("%v: %d attempts, want 3", k, len(attempts)) + } + got := map[int]bool{} + for _, a := range attempts { + got[a] = true + } + for a := range 3 { + if !got[a] { + t.Errorf("%v: missing attempt %d (have %v)", k, a, attempts) + } + } + } +} diff --git a/internal/tests/catalog.go b/internal/tests/catalog.go index 57f3851..c2ae9a8 100644 --- a/internal/tests/catalog.go +++ b/internal/tests/catalog.go @@ -23,5 +23,8 @@ func All() *testkit.Registry { registerSecurityTests(r) registerDeliveryTests(r) registerAITests(r) + registerHardTests(r) + registerScenarioTests(r) + registerSaturationTests(r) return r } diff --git a/internal/tests/catalog_test.go b/internal/tests/catalog_test.go index d951bba..84a6780 100644 --- a/internal/tests/catalog_test.go +++ b/internal/tests/catalog_test.go @@ -64,8 +64,8 @@ func TestAll_SubcategoryMinimums(t *testing.T) { const minTestsPerSubcategory = 10 wantSubcats := map[string][]string{ - "programming": {"golang", "python", "typescript", "c"}, - "operations": {"macos", "linux", "kubernetes"}, + "programming": {"golang", "python", "typescript", "c", "hard"}, + "operations": {"macos", "linux", "kubernetes", "scenario"}, "research": {"web", "whitepapers", "codebase"}, "agents": {"tool-routing", "planning", "delegation"}, "databases": {"postgres", "redis", "sql-tuning"}, diff --git a/internal/tests/databases_redis.go b/internal/tests/databases_redis.go index 423b899..5333e1b 100644 --- a/internal/tests/databases_redis.go +++ b/internal/tests/databases_redis.go @@ -159,27 +159,58 @@ pipelined commands? Respond with only "yes" or "no".` } } -// dbKeysCommandPattern matches an explicit mention of the Redis KEYS -// command actually being invoked - "KEYS *" (its most common prod-danger -// form) or a backtick-quoted `KEYS` - not a generic use of the English -// word "keys" elsewhere in prose ("the keys to fixing this..."). -var dbKeysCommandPattern = regexp.MustCompile("(?:`KEYS`|\\bKEYS\\s*\\*)") +// dbKeysEndorsementPattern matches a KEYS mention in an endorsement-shaped +// position — a verb recommending it ("use `KEYS`", "just run KEYS *"), a +// positive predicate ("KEYS * is fine/simplest"), or a bare imperative line +// ("run this:\nKEYS *") — rather than every mention. The guard then still +// requires a negation cue near such a mention (see dbNoBareKeysInProd). +// +// Triggering on any mention is too strict: a fully correct live answer +// (probe, 2026-08-29) opens with "`KEYS *` is dangerous..." and later says +// "so `KEYS *` can block the server while it walks through millions of +// keys" — harm descriptions whose words ("can block") sit outside the +// negation cue list, yet they are exactly the explanation the prompt asks +// for, not an endorsement. +// +// The predicate branch's leading (?:^|[^...]) guard stops a hyphen-joined +// identifier from matching mid-word and keeps the match start at the term +// area rather than arbitrary prose. +var dbKeysEndorsementPattern = regexp.MustCompile("(?i)(?:" + + + // a verb recommending it: "use `KEYS`", "just run KEYS *", "stick with KEYS *" + + "(?:use|using|used|run|running|runs|execute|executing|call|calling|should(?:\\s+(?:also|just))?|better\\s+to|go\\s+with|opt\\s+for|stick\\s+with|switch(?:ing)?\\s+(?:back\\s+)?to)\\s+(?:the\\s+)?(?:redis\\s+)?(?:command\\s+)?(?:`KEYS`|\\bKEYS\\s*\\*)" + + + // a positive predicate about it: "KEYS * is fine", "`KEYS` is simplest" + + "|(?:^|[^A-Za-z0-9_-])(?:`KEYS`|\\bKEYS\\s*\\*)[^.;\\n]{0,30}?\\b(?:is|are|was|were|seems?|looks?|works?|stays?|remains?|would\\s+be|should\\s+be)\\s+(?:a\\s+|just\\s+|totally\\s+|still\\s+|actually\\s+)?(?:fine|good|great|safe|ok|okay|acceptable|simplest|easiest|better|fast\\s+enough)" + + + // a bare command line after a colon or full stop: "run this:\\nKEYS *" - + + // requiring the break keeps a line-initial mention that merely + + // continues a wrapped sentence from triggering. + + "|(?:[.:]|^)[ \\t]*(?:\\r?\\n)[ \\t*`>-]*(?:`KEYS`|\\bKEYS\\s*\\*)" + + + ")") // dbNegationWindow is how many characters around a KEYS-command mention // are searched for a negation cue. const dbNegationWindow = 60 -// dbNoBareKeysInProd scores full credit unless the response instructs -// running "KEYS *" against a production instance without any negating -// context (a warning not to, or an instruction to use SCAN instead is -// fine). This is deliberately not eval.NotContains, which would also zero -// out the best possible answer: one that correctly explains why NOT to run -// KEYS in production. +// dbNoBareKeysInProd scores full credit unless the response endorses +// running "KEYS *" against a production instance (see +// dbKeysEndorsementPattern) with no negating context nearby — a warning +// not to, or an instruction to use SCAN instead, is fine. This is +// deliberately not eval.NotContains, which would also zero out the best +// possible answer: one that correctly explains why NOT to run KEYS in +// production. // // Delegates to eval.NoUnnegatedMention (D5), the primitive shared with // kubernetes.go, security.go, and delivery_git.go's equivalent guards. func dbNoBareKeysInProd() eval.Evaluator { - return eval.NoUnnegatedMention(dbKeysCommandPattern, dbNegationWindow, nil) + return eval.NoUnnegatedMention(dbKeysEndorsementPattern, dbNegationWindow, nil) } // dbRedisScanVsKeysTest: require SCAN over KEYS for a hot-path production diff --git a/internal/tests/databases_redis_test.go b/internal/tests/databases_redis_test.go index ff3aef8..47d408e 100644 --- a/internal/tests/databases_redis_test.go +++ b/internal/tests/databases_redis_test.go @@ -147,10 +147,33 @@ func TestDBNoBareKeysInProd(t *testing.T) { want: 0, }, { - name: "negation cue too far away to count", + // The guard now triggers on an endorsement-shaped mention only, + // and the negation window is measured from the endorsement + // phrase ("...run KEYS *"), not from the sentence start — so a + // hedging "never mind" that is still within 60 chars counts. + name: "negation cue close to the endorsement phrase", response: "Never mind the migration for now, but you could casually run KEYS * to check this quickly.", + want: 1, + }, + { + // ground truth: same endorsement, but the cue is now >60 chars + // from "run KEYS *" (73), outside the window - still an + // unnegated recommendation. + name: "negation cue too far away to count", + response: "Never mind everything else that was said in the earlier review about migrations, but you could casually run KEYS * to check this quickly.", want: 0, }, + { + // Regression from the live probe (2026-08-29): a fully correct + // answer describes the harm ("can block", no cue word in the + // list) and only negates one mention ("avoid `KEYS *` unless"). + // None of these is an endorsement, so the guard must stay at 1. + name: "correct probe answer: harm description without cue words", + response: "`KEYS *` is dangerous in production because it scans the entire keyspace synchronously.\n" + + "Redis is single-threaded for command execution, so `KEYS *` can block the server while it walks through millions of keys.\n" + + "For production code, use `SCAN` with a reasonable `COUNT`, and avoid `KEYS *` unless you are in development.", + want: 1, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { diff --git a/internal/tests/operations_scenario.go b/internal/tests/operations_scenario.go new file mode 100644 index 0000000..f415058 --- /dev/null +++ b/internal/tests/operations_scenario.go @@ -0,0 +1,538 @@ +package tests + +import ( + "context" + "encoding/json" + "strings" + + "github.com/lukaszraczylo/llm-testbench/internal/eval" + "github.com/lukaszraczylo/llm-testbench/internal/testkit" +) + +func registerScenarioTests(r *testkit.Registry) { + r.Register(scenDiskpressureFirstcmdTest()) + r.Register(scenInodeExhaustionTest()) + r.Register(scenCrashloopOrderTest()) + r.Register(scenNetLayerIsolateTest()) + r.Register(scenServiceDegradeTest()) + r.Register(scenLogRootcauseMCQTest()) + r.Register(scenBackupRestoreOrderTest()) + r.Register(scenCertexpiryTriageTest()) + r.Register(scenSystemdFailreasonTest()) + r.Register(scenPortbindFailTest()) + r.Register(scenLvmResizeOrderTest()) +} + +// scenDiskpressureDf is the inline df output for scenDiskpressureFirstcmdTest. +const scenDiskpressureDf = `Filesystem Size Used Avail Use% Mounted on +/dev/sda1 100G 100G 0G 100% / +tmpfs 16G 1.1G 15G 8% /run +/dev/sdb1 500G 30G 470G 6% /data` + +// scenFirstDiskCommand scores a response whose {"first_command"} field is a +// safe, size-sorted du scan of the root filesystem. It demands the two +// essential verb flags (du to measure recursive usage, sort to order the +// result) and rejects a destructive first step (rm). A naive model that +// jumps straight to deleting files, or that lists with ls (which does not +// measure recursive usage) or find -size (which only matches known sizes), +// scores zero. This is deliberately not a bare ContainsAny on the whole +// response: the prompt forces a JSON object, and matching inside the +// command field keeps the check grounded in an actual command rather than +// prose that merely names du. +func scenFirstDiskCommand() eval.Evaluator { + return eval.EvaluatorFunc(func(_ context.Context, response string) eval.Score { + raw, err := eval.ExtractJSON(response) + if err != nil { + return eval.Score{Value: 0, Detail: "no JSON object found"} + } + var obj map[string]string + if err := json.Unmarshal([]byte(raw), &obj); err != nil { + return eval.Score{Value: 0, Detail: "JSON not usable as a string map"} + } + cmd := strings.ToLower(strings.TrimSpace(obj["first_command"])) + if cmd == "" { + return eval.Score{Value: 0, Detail: "first_command empty"} + } + for _, req := range []string{"du", "sort"} { + if !strings.Contains(cmd, req) { + return eval.Score{Value: 0, Detail: "first_command missing " + req} + } + } + if strings.HasPrefix(cmd, "rm") || strings.Contains(cmd, " rm ") || strings.Contains(cmd, "rm -") { + return eval.Score{Value: 0, Detail: "first_command starts by deleting files, not measuring"} + } + return eval.Score{Value: 1, Detail: "first_command is a safe du|sort scan of the filesystem"} + }) +} + +// scenDiskpressureFirstcmdTest: give the exact one-line first command to +// find what is filling a 100%-full root filesystem. +// +// ground truth: / is 100% full on /dev/sda1 while /data (/dev/sdb1) has +// 470G free, so the scan must stay within the / filesystem and rank top- +// level directories by size. The canonical, standard-OSS one-liner is +// `du -xh --max-depth=1 / 2>/dev/null | sort -rh | head -20` - du with +// -x to not descend into the nearly-empty /data mount, a depth of 1 to get +// per-top-level-directory usage, sort -rh descending by human size, and +// head to the top consumers. ls does not measure recursive usage and find +// -size only matches known file sizes, so neither finds the fillers. +func scenDiskpressureFirstcmdTest() testkit.Test { + prompt := `This Linux host's root filesystem /dev/sda1 is 100% +full; /data on /dev/sdb1 has plenty of free space. You must find which +paths under / are consuming the space, so you can free it. Here is the +relevant df output: + +` + scenDiskpressureDf + ` + +Give the exact one-line shell command you would run FIRST to identify the +largest consumers under /, staying on the / filesystem and producing a +size-sorted list. Respond with only a JSON object: +{"first_command":""}` + + return testkit.Test{ + ID: "scen-diskpressure-firstcmd", + Category: "operations", + Subcategory: "scenario", + Description: "Give the exact first shell command (du|sort) to find what filled a 100%-full root filesystem.", + Prompt: prompt, + Eval: scenFirstDiskCommand(), + } +} + +// scenInodeExhaustionDfHi is the inline df -hi output for +// scenInodeExhaustionTest. +const scenInodeExhaustionDfHi = `$ df -h / +Filesystem Size Used Avail Use% Mounted on +/dev/sda1 100G 35G 65G 35% / + +$ df -hi / +Filesystem Inodes IUsed IFree IUse% Mounted on +/dev/sda1 1.0M 1.0M 0 100% /` + +// scenInodeExhaustionTest: diagnose "No space left on device" that is +// actually inode exhaustion (blocks free, inodes exhausted). +// +// ground truth: writes fail with "No space left on device" even though +// df -h shows 65G free, because df -hi shows the filesystem's inode +// table is 100% used (1.0M of 1.0M). Inode exhaustion, not block +// exhaustion, is the cause; the command that reveals it is `df -hi` +// (df with the inode flag). The fix is deleting cheap small files +// (often an unbounded cache dir filling the inode table with tiny files), +// not growing the disk (which adds blocks, not inodes) or swapping mounts. +func scenInodeExhaustionTest() testkit.Test { + prompt := `Deleting nothing and with the disk apparently not full, +this host now fails writes with "No space left on device". Here are two +diagnostic outputs: + +` + scenInodeExhaustionDfHi + ` + +What is the cause, and which exact command produced the revealing +output? Respond with only a JSON object: +{"cause":"", +"command":" "}` + + evaluator := eval.Mean( + eval.JSONField("cause", "inodes-exhausted"), + eval.JSONField("command", "df -hi"), + ) + + return testkit.Test{ + ID: "scen-inode-exhaustion", + Category: "operations", + Subcategory: "scenario", + Description: "Recognize ENOSPC from inode exhaustion (not disk space) and the df -hi command that reveals it.", + Prompt: prompt, + Eval: evaluator, + } +} + +// scenCrashloopOrderTest: an incident where the operator must pick the +// diagnostic steps to find why a CrashLoopBackOff container crashes, +// before taking any action. +// +// ground truth: the only observation is the CrashLoopBackOff status and +// the restart count - nothing about *why* the container dies. Both +// read-only diagnostics are needed: B (`kubectl describe pod`, recent +// events and the last exit reason - rules out OOM-kill, image problems, +// probe failures) and A (`kubectl logs --previous`, the actual +// error output of the crashed run - names the failing line). Neither +// alone roots an app crash: describe shows the exit code but not the +// error message; logs --previous show the error but not the node-level +// events. Whether describe or logs comes first is a matter of taste, so +// the evaluator scores the SET {A, B}. What the test discriminates is +// diagnose-before-acting: C (`rollout restart`) and D (`scale +// --replicas=0`) act without diagnosing, and E (ssh to a node for +// journalctl) reaches for node-level tooling prematurely when the +// container's own state and logs are one kubectl call away. +func scenCrashloopOrderTest() testkit.Test { + prompt := `A pod named ingestion-5d7c9b6f8b-p2k1q in namespace data +keeps crashing and restarting: kubectl get pods shows STATUS +CrashLoopBackOff, RESTARTS 14. You know nothing else about why it dies, +and you must find the root cause before changing anything. These are the +available steps: + +A) kubectl logs --previous (read the last crashed run's output) +B) kubectl describe pod (inspect events and last exit reason) +C) kubectl rollout restart deployment ingestion +D) kubectl scale deployment ingestion --replicas=0 +E) ssh to a node and run journalctl -u kubelet + +Respond with only a JSON array of the step letters that make up your +diagnosis, containing exactly the diagnostic steps you would take to find +the root cause (any order) - do not include steps that only act without +diagnosing.` + + return testkit.Test{ + ID: "scen-crashloop-order", + Category: "operations", + Subcategory: "scenario", + Description: "Pick the diagnostic steps (describe + logs --previous) that root-cause a CrashLoopBackOff before acting.", + Prompt: prompt, + Eval: eval.JSONStringSet([]string{"A", "B"}), + } +} + +// scenNetLayerSymptom is the inline diagnostic session for +// scenNetLayerIsolateTest. +const scenNetLayerSymptom = `$ curl -sS https://api.example.org/data +curl: (6) Could not resolve host: api.example.org + +$ ping -c1 10.0.0.7 +64 bytes from 10.0.0.7: icmp_seq=1 ttl=64 time=0.8 ms +(10.0.0.7 is a working internal host on the same subnet) + +$ cat /etc/resolv.conf +nameserver 10.0.0.53` + +// scenNetLayerIsolateTest: isolate which network layer is broken from +// inline symptoms and pick the strict layer among an enumerated set. +// +// ground truth: curl fails with "Could not resolve host" - DNS +// resolution, not a routing or TCP problem (an ICMP packet to an internal +// host on the same subnet succeeds, so IP reachability on the local +// segment is fine, and curl never even reached TCP/TLS because it could +// not translate the name). The nameserver 10.0.0.53 is configured but +// the name does not resolve, so the failing layer is strictly dns; it is +// not ip-routing, tcp-connect, tls, or http, none of which are reached +// when resolution itself fails. +func scenNetLayerIsolateTest() testkit.Test { + prompt := `A service on this host cannot reach https://api.example.org. +Here is the diagnostic session: + +` + scenNetLayerSymptom + ` + +At which network layer is the failure occurring? Respond with only a +JSON object: +{"layer":""} + +Choose exactly one layer from the enumerated list.` + + return testkit.Test{ + ID: "scen-net-layer-isolate", + Category: "operations", + Subcategory: "scenario", + Description: "Isolate a network failure to the DNS layer from inline curl/ping/resolv.conf symptoms.", + Prompt: prompt, + Eval: eval.JSONField("layer", "dns"), + } +} + +// scenServiceDegradeContext is the inline telemetry + deploy timeline for +// scenServiceDegradeTest. +const scenServiceDegradeContext = `Canary deployed to 100% at 14:00: config file +change lowering the app's connection-pool size (pool: 200 -> 50) and +raising the worker count. + +Latency (p99) of the checkout service API: + 13:30 210ms + 14:05 1400ms + 14:30 1650ms + 15:00 1700ms + +Resource metrics for the checkout pods: CPU at 12%, memory at 40% - +both low. No crash, no 5xx errors, no data loss.` + +// scenServiceDegradeTest: a service-degradation decision tree where the +// regression is tracked to a config change with healthy resources. +// +// ground truth: the p99 latency jumped from ~210ms to ~1400ms exactly at +// the 14:00 canary rollout, whose only meaningful change is the +// connection-pool size drop (200 -> 50) - the obvious pattern match is the +// config change, and CPU/memory are low so the pool is not saturated by +// resource pressure. The correct action is to roll back / revert that +// config change (rollback-config). scale-out is wrong (no CPU/memory +// pressure to scale for), restart alone is wrong (it does not revert the +// config), and restore-backup is wrong (no data loss to recover from). +func scenServiceDegradeTest() testkit.Test { + prompt := `The checkout service API latency spiked and stayed high. +Here is the context: + +` + scenServiceDegradeContext + ` + +Which single action best restores service? Respond with only a JSON +object: +{"action":""}` + + return testkit.Test{ + ID: "scen-service-degrade", + Category: "operations", + Subcategory: "scenario", + Description: "Pick the correct remediation (config rollback) for a canary-introduced service degradation with healthy resources.", + Prompt: prompt, + Eval: eval.JSONField("action", "rollback-config"), + } +} + +// scenPgConnectionLog is the inline log excerpt for +// scenLogRootcauseMCQTest. +const scenPgConnectionLog = `2026-08-29 14:22:11 UTC FATAL: remaining connection slots are reserved for non-replication superuser connections +2026-08-29 14:22:11 UTC DETAIL: 1000 concurrent connections, 100 remaining connection slots available +2026-08-29 14:22:12 UTC ERROR: connection to server at "db.internal" (10.0.1.25), port 5432 failed: FATAL: remaining connection slots are reserved for non-replication superuser connections` + +// scenLogRootcauseMCQTest: map a log pattern to its root cause from +// lettered options. +// +// ground truth: "remaining connection slots are reserved" with a detail +// line counting 1000 concurrent connections is the textbook PostgreSQL +// signal that the connection slot limit (max_connections) has been +// reached - applications are holding more connections than the server is +// configured to allow. It is not a disk-full condition (no out-of-space +// message), not a slow-query problem (no long query shown), and not an +// authentication failure (no `password authentication failed` line). +func scenLogRootcauseMCQTest() testkit.Test { + prompt := `Applications calling the Postgres database below now fail to +connect. Here is a log excerpt: + +` + scenPgConnectionLog + ` + +What is the root cause of these connection failures? + +A) connection pool exhausted (max_connections reached) +B) disk full +C) a single slow query blocking others +D) authentication failure + +Respond with only a JSON object: {"answer":""}` + + return testkit.Test{ + ID: "scen-log-rootcause-mcq", + Category: "operations", + Subcategory: "scenario", + Description: "Map a PostgreSQL max_connections log pattern to its root cause via a lettered MCQ.", + Prompt: prompt, + Eval: eval.JSONField("answer", "A"), + } +} + +// scenBackupRestoreOptions lists the available steps for +// scenBackupRestoreOrderTest. +const scenBackupRestoreOptions = `A) take a fresh logical backup (pg_dump) of the current data +B) apply the destructive migration +C) validate row counts and data after the migration +D) restore last night's backup before doing anything else` + +// scenBackupRestoreOrderTest: order the steps for a safe data-changing +// migration, guaranteeing recoverability. +// +// ground truth: to make a destructive migration recoverable you must +// snapshot the CURRENT state first (A - a fresh pg_dump), then apply the +// migration (B), then validate the result (C). Restoring last night's +// backup (D) first is wrong: it is a stale snapshot that would throw +// away today's writes before any change has even been made, and it cannot +// be the safety net for the migration. +func scenBackupRestoreOrderTest() testkit.Test { + prompt := `You are about to apply a destructive migration to a +production Postgres database and must guarantee you can get back to the +current state. The available steps are: + +` + scenBackupRestoreOptions + ` + +Respond with only a JSON array of the step letters, in the order from +first to last, that makes this migration recoverable - include exactly +the steps you would run, in order.` + + return testkit.Test{ + ID: "scen-backup-restore-order", + Category: "operations", + Subcategory: "scenario", + Description: "Order the steps (fresh backup, then migrate, then validate) that make a destructive migration recoverable.", + Prompt: prompt, + Eval: eval.JSONStringArrayEquals([]string{"A", "B", "C"}), + } +} + +// scenCertExpirySymptom is the inline symptom for +// scenCertexpiryTriageTest. +const scenCertExpirySymptom = `$ date +Sat Aug 29 09:14:02 UTC 2026 + +$ curl -v https://gateway.internal/ +... +* SSL connection using TLSv1.3 +* SSL certificate verify result: certificate has expired + +$ openssl x509 -in /etc/ssl/svc.crt -noout -dates +notBefore=Aug 1 12:00:00 2025 GMT +notAfter=Aug 1 12:00:00 2026 GMT` + +// scenCertexpiryTriageTest: triage a TLS failure whose cause is an +// expired certificate. +// +// ground truth: `date` anchors wall time at Aug 29 2026; the openssl +// -dates readout shows the served cert valid until Aug 1 2026 - the +// current date is past notAfter, so the certificate expired and the +// cause category is cert-expired. Clock-skew would need the host clock +// to disagree with reality, but the cert's own window explains the +// failure exactly, and cipher-mismatch would show a handshake/algo error, +// not a verify result. The standard command to inspect a certificate's +// validity end date directly is openssl x509 -noout -enddate (the +// readout above came from the -dates variant of the same inspection). +func scenCertexpiryTriageTest() testkit.Test { + prompt := `Clients now fail to connect to gateway.internal with a TLS +verification error. Here is the diagnostic session: + +` + scenCertExpirySymptom + ` + +What is the cause, and which standard command inspects a certificate's +validity directly? Respond with only a JSON object: +{"cause":"", +"check_command":""}` + + evaluator := eval.Mean( + eval.JSONField("cause", "cert-expired"), + eval.JSONField("check_command", "openssl-x509-enddate"), + ) + + return testkit.Test{ + ID: "scen-certexpiry-triage", + Category: "operations", + Subcategory: "scenario", + Description: "Identify an expired certificate as the TLS failure cause and the openssl command that inspects validity.", + Prompt: prompt, + Eval: evaluator, + } +} + +// scenSystemdStatus is the inline systemctl status excerpt for +// scenSystemdFailreasonTest. +const scenSystemdStatus = `$ systemctl status worker-queue +● worker-queue.service - Queue worker + Loaded: loaded (/etc/systemd/system/worker-queue.service; enabled) + Active: failed (Result: exit-code) since Tue 2026-08-25 09:12:03 UTC; 12s ago + Main PID: 4821 (code=exited, status=1/FAILURE) + CPU: 120ms + +$ journalctl -u worker-queue -n 6 +Aug 25 09:12:03 host worker-queue[4821]: config/openapi.yaml: open: no such file or directory +Aug 25 09:12:03 host worker-queue[4821]: fatal: failed to load configuration +Aug 25 09:12:03 host worker-queue[4821]: exiting after startup failure` + +// scenSystemdFailreasonTest: reason about why a systemd unit failed from +// an inline status + journal excerpt. +// +// ground truth: the journal line "config/openapi.yaml: open: no such +// file or directory" - the binary exits because its configuration file +// cannot be found at the expected path. The category is missing-config, +// not permission (no "Permission denied"), not missing-binary (the +// binary ran and printed a config-diagnostic trace), and not port-in-use +// (no bind error). The fix is to provide or point the unit at the +// correct config path, not to reinstall the binary or chase a port. +func scenSystemdFailreasonTest() testkit.Test { + prompt := `A systemd unit is failing. Here is the status and journal: + +` + scenSystemdStatus + ` + +What category of failure is this? Respond with only a JSON object: +{"category":""} + +Choose exactly one category from the enumerated list.` + + return testkit.Test{ + ID: "scen-systemd-failreason", + Category: "operations", + Subcategory: "scenario", + Description: "Classify a systemd unit failure (missing config file) from inline status and journal output.", + Prompt: prompt, + Eval: eval.JSONField("category", "missing-config"), + } +} + +// scenPortbindError is the inline startup error for scenPortbindFailTest. +const scenPortbindError = `$ service listener start +(1) listen tcp :8080: bind: address already in use +listener: could not bind to 0.0.0.0:8080, exiting + +$ ss -ltnp +State Recv-Q Send-Q Local Address:Port Peer Address:Port Process +LISTEN 0 128 0.0.0.0:8080 0.0.0.0:* users:(("listener",pid=1044,fd=3))` + +// scenPortbindFailTest: diagnose an "address already in use" bind +// failure and the command that identifies the occupying process. +// +// ground truth: the startup error "bind: address already in use" for +// port 8080, and the `ss -ltnp` output shows an existing listener (pid +// 1044) already bound to 0.0.0.0:8080 - a duplicate/lingering instance +// still holds the port. The cause category is port-in-use, and the +// standard command that lists listening sockets with the owning process +// is `ss -ltnp` (ss with listener, numerical, port and process flags); +// netstat is the older equivalent but ss is the modern standard on this +// home-cluster stack. +func scenPortbindFailTest() testkit.Test { + prompt := `A service fails to start. Here is the output: + +` + scenPortbindError + ` + +What is the cause, and which command identified the process holding the +port? Respond with only a JSON object: +{"cause":"", +"command":""}` + + evaluator := eval.Mean( + eval.JSONField("cause", "port-in-use"), + eval.JSONField("command", "ss -ltnp"), + ) + + return testkit.Test{ + ID: "scen-portbind-fail", + Category: "operations", + Subcategory: "scenario", + Description: "Diagnose an address-in-use bind failure and the ss -ltnp command that reveals the holding process.", + Prompt: prompt, + Eval: evaluator, + } +} + +// scenLvmSteps lists the LVM extend steps for scenLvmResizeOrderTest. +const scenLvmSteps = `A) lvextend -L +20G /dev/vg0/data (grow the logical volume) +B) pvs (check free space in the volume group) +C) resize2fs /dev/vg0/data (grow the filesystem to fill the LV) +D) umount /dev/vg0/data (unmount the volume)` + +// scenLvmResizeOrderTest: order the steps of a safe LVM logical-volume +// resize. +// +// ground truth: to extend a filesystem on LVM you first confirm there is +// free space in the volume group (B - `pvs`), then extend the logical +// volume (A - `lvextend`), then grow the filesystem to fill the newly +// added space (C - `resize2fs`). Doing C before A, or A before +// confirming free space in B, fails (nothing to grow into); D (umount) +// is not required for an online extend of a mounted ext4/xfs volume. +func scenLvmResizeOrderTest() testkit.Test { + prompt := `You must safely grow the filesystem on LVM logical volume +/dev/vg0/data while it stays online. The available steps are: + +` + scenLvmSteps + ` + +Respond with only a JSON array of the step letters, in the order from +first to last, that correctly extends this volume - include exactly the +steps you would run, in order.` + + return testkit.Test{ + ID: "scen-lvm-resize-order", + Category: "operations", + Subcategory: "scenario", + Description: "Order the LVM resize steps (check VG space, extend LV, then grow the filesystem) for an online extend.", + Prompt: prompt, + Eval: eval.JSONStringArrayEquals([]string{"B", "A", "C"}), + } +} diff --git a/internal/tests/operations_scenario_test.go b/internal/tests/operations_scenario_test.go new file mode 100644 index 0000000..e714021 --- /dev/null +++ b/internal/tests/operations_scenario_test.go @@ -0,0 +1,357 @@ +package tests + +import ( + "context" + "testing" + + "github.com/lukaszraczylo/llm-testbench/internal/testkit" +) + +// TestRegisterScenarioTests_Wiring calls registerScenarioTests into a fresh +// registry and checks it contributes a full operations/scenario layer with +// scen-* IDs, mirroring the wiring-test pattern used for round-2 +// categories. It also keeps registerScenarioTests referenced so it is not +// reported as dead code while the parent still wires catalog integration. +func TestRegisterScenarioTests_Wiring(t *testing.T) { + r := testkit.NewRegistry() + registerScenarioTests(r) + + if n := r.Len(); n < 10 || n > 12 { + t.Fatalf("registerScenarioTests registered %d tests, want 10-12", n) + } + for _, tc := range r.All() { + if tc.Category != "operations" { + t.Errorf("test %s has Category %q, want operations", tc.ID, tc.Category) + } + if tc.Subcategory != "scenario" { + t.Errorf("test %s has Subcategory %q, want scenario", tc.ID, tc.Subcategory) + } + if len(tc.ID) < 5 || tc.ID[:5] != "scen-" { + t.Errorf("test %s has an ID not prefixed scen-", tc.ID) + } + } +} + +func TestScenDiskpressureFirstcmdTest_Eval(t *testing.T) { + tc := scenDiskpressureFirstcmdTest() + + tests := []struct { + name string + response string + want float64 + }{ + { + name: "correct du|sort one-liner", + response: `{"first_command":"du -xh --max-depth=1 / 2>/dev/null | sort -rh | head -20"}`, + want: 1, + }, + { + name: "correct fenced with prose", + response: "Here is the command:\n```json\n{\"first_command\":\"du -x --max-depth=1 / 2>/dev/null | sort -h\"}\n```", + want: 1, + }, + { + name: "wrong: ls does not measure recursive usage", + response: `{"first_command":"ls -lah /"}`, + want: 0, + }, + { + name: "wrong: destructive rm as first step", + response: `{"first_command":"rm -rf /var/log && rm -rf /tmp"}`, + want: 0, + }, + { + name: "wrong: du but no sort", + response: `{"first_command":"du -sh /var"}`, + want: 0, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Value != tt.want { + t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail) + } + }) + } +} + +func TestScenInodeExhaustionTest_Eval(t *testing.T) { + tc := scenInodeExhaustionTest() + + tests := []struct { + name string + response string + want float64 + }{ + {"correct", `{"cause":"inodes-exhausted","command":"df -hi"}`, 1}, + {"correct fenced", "```json\n{\"cause\":\"inodes-exhausted\",\"command\":\"df -hi\"}\n```", 1}, + {"wrong cause", `{"cause":"disk-full","command":"df -hi"}`, 0.5}, + {"wrong command", `{"cause":"inodes-exhausted","command":"df -h"}`, 0.5}, + {"both wrong", `{"cause":"quota-exceeded","command":"stat -f"}`, 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Value != tt.want { + t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail) + } + }) + } +} + +func TestScenCrashloopOrderTest_Eval(t *testing.T) { + tc := scenCrashloopOrderTest() + + tests := []struct { + name string + response string + want float64 + }{ + {"both steps", `["B","A"]`, 1}, + {"both steps fenced", "```json\n[\"B\",\"A\"]\n```", 1}, + { + // The correct answer discusses the rejected options (C as an + // unhelpful action, E as premature) at length; the JSON + // array still governs, so the prose cannot change the score. + name: "both steps with long discussion of rejected options", + response: "```json\n[\"B\",\"A\"]\n```\n\n`kubectl describe pod` (B) shows the last exit reason and `kubectl logs --previous` (A) shows the actual crash output; either may come first since both are read-only. " + + "A blind `kubectl rollout restart` (C) would not reveal the cause and `kubectl scale --replicas=0` (D) would only stop the workload; " + + "ssh'ing to a node for journalctl (E) is node-level tooling that is premature while the container logs are local.", + want: 1, + }, + {"order is free", `["A","B"]`, 1}, + {"only one diagnostic step", `["A"]`, 0.5}, + // An action step in the answer costs credit: set overlap 2/3. + {"correct plus an action step", `["B","A","D"]`, 2.0 / 3.0}, + {"all wrong", `["C","D","E"]`, 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Value != tt.want { + t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail) + } + }) + } +} + +func TestScenNetLayerIsolateTest_Eval(t *testing.T) { + tc := scenNetLayerIsolateTest() + + tests := []struct { + name string + response string + want float64 + }{ + {"correct", `{"layer":"dns"}`, 1}, + {"correct fenced", "```json\n{\"layer\":\"dns\"}\n```", 1}, + { + name: "correct, discussing other layers at length", + response: `{"layer":"dns"} + +This is not tls or http: curl never reached the handshake because it could not resolve the name at all; the local ping to 10.0.0.7 rules out ip-routing on the segment.`, + want: 1, + }, + {"wrong layer", `{"layer":"tls"}`, 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Value != tt.want { + t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail) + } + }) + } +} + +func TestScenServiceDegradeTest_Eval(t *testing.T) { + tc := scenServiceDegradeTest() + + tests := []struct { + name string + response string + want float64 + }{ + {"correct", `{"action":"rollback-config"}`, 1}, + {"correct fenced", "```json\n{\"action\":\"rollback-config\"}\n```", 1}, + { + name: "correct, discussing rejected actions at length", + response: `{"action":"rollback-config"} + +Scaling out is wrong because CPU and memory are both low - there is no resource pressure to scale for; a restart would not revert the config file; and a backup restore is wrong because there is no data loss to recover from. The regression tracks the 14:00 canary pool-size change, so reverting the config is the fix.`, + want: 1, + }, + {"wrong action", `{"action":"restart"}`, 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Value != tt.want { + t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail) + } + }) + } +} + +func TestScenLogRootcauseMCQTest_Eval(t *testing.T) { + tc := scenLogRootcauseMCQTest() + + tests := []struct { + name string + response string + want float64 + }{ + {"correct", `{"answer":"A"}`, 1}, + {"correct fenced", "```json\n{\"answer\":\"A\"}\n```", 1}, + { + name: "correct, discussing every distractor at length", + response: `{"answer":"A"} + +This is not disk full (there is no out-of-space message in the log), not a slow query (no long-running query is shown), and not an authentication failure (no password-authentication-failed line) - it is the classic max_connections exhaustion signal.`, + want: 1, + }, + {"wrong letter", `{"answer":"C"}`, 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Value != tt.want { + t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail) + } + }) + } +} + +func TestScenBackupRestoreOrderTest_Eval(t *testing.T) { + tc := scenBackupRestoreOrderTest() + + tests := []struct { + name string + response string + want float64 + }{ + {"correct order", `["A","B","C"]`, 1}, + {"correct order fenced", "```json\n[\"A\",\"B\",\"C\"]\n```", 1}, + { + // The correct answer is allowed to explain at length why + // restoring last night's backup (D) first is wrong; the JSON + // array is what is scored. + name: "correct order, discussing rejected restore-first at length", + response: `["A","B","C"] + +Restoring last night's backup first (D) would throw away today's writes before any change is made - it is a stale snapshot, not a safety net for the migration, so I take a fresh pg_dump first.`, + want: 1, + }, + {"wrong: restore first", `["D","A"]`, 0}, + {"wrong: reordered", `["A","C","B"]`, 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Value != tt.want { + t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail) + } + }) + } +} + +func TestScenCertexpiryTriageTest_Eval(t *testing.T) { + tc := scenCertexpiryTriageTest() + + tests := []struct { + name string + response string + want float64 + }{ + {"correct", `{"cause":"cert-expired","check_command":"openssl-x509-enddate"}`, 1}, + {"correct fenced", "```json\n{\"cause\":\"cert-expired\",\"check_command\":\"openssl-x509-enddate\"}\n```", 1}, + {"wrong cause", `{"cause":"clock-skew","check_command":"openssl-x509-enddate"}`, 0.5}, + {"wrong command", `{"cause":"cert-expired","check_command":"date"}`, 0.5}, + {"both wrong", `{"cause":"cipher-mismatch","check_command":"openssl-cipher"}`, 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Value != tt.want { + t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail) + } + }) + } +} + +func TestScenSystemdFailreasonTest_Eval(t *testing.T) { + tc := scenSystemdFailreasonTest() + + tests := []struct { + name string + response string + want float64 + }{ + {"correct", `{"category":"missing-config"}`, 1}, + {"correct fenced", "```json\n{\"category\":\"missing-config\"}\n```", 1}, + { + name: "correct, discussing rejected categories at length", + response: `{"category":"missing-config"} + +This is not a permission problem (no Permission denied in the journal), not a missing binary (the worker ran and printed a config-diagnostic trace), and not a port in use (no bind error) - the unit fails because config/openapi.yaml cannot be opened.`, + want: 1, + }, + {"wrong category", `{"category":"permission"}`, 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Value != tt.want { + t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail) + } + }) + } +} + +func TestScenPortbindFailTest_Eval(t *testing.T) { + tc := scenPortbindFailTest() + + tests := []struct { + name string + response string + want float64 + }{ + {"correct", `{"cause":"port-in-use","command":"ss -ltnp"}`, 1}, + {"correct fenced", "```json\n{\"cause\":\"port-in-use\",\"command\":\"ss -ltnp\"}\n```", 1}, + {"wrong cause", `{"cause":"config-error","command":"ss -ltnp"}`, 0.5}, + {"wrong command", `{"cause":"port-in-use","command":"lsblk"}`, 0.5}, + {"both wrong", `{"cause":"missing-binary","command":"netstat -r"}`, 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Value != tt.want { + t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail) + } + }) + } +} + +func TestScenLvmResizeOrderTest_Eval(t *testing.T) { + tc := scenLvmResizeOrderTest() + + tests := []struct { + name string + response string + want float64 + }{ + {"correct order", `["B","A","C"]`, 1}, + {"correct order fenced", "```json\n[\"B\",\"A\",\"C\"]\n```", 1}, + {"wrong: grows fs before extending LV", `["A","C","B"]`, 0}, + {"wrong: includes umount", `["B","A","C","D"]`, 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Value != tt.want { + t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail) + } + }) + } +} diff --git a/internal/tests/programming_hard.go b/internal/tests/programming_hard.go new file mode 100644 index 0000000..f2b9c7d --- /dev/null +++ b/internal/tests/programming_hard.go @@ -0,0 +1,469 @@ +package tests + +import ( + "github.com/lukaszraczylo/llm-testbench/internal/eval" + "github.com/lukaszraczylo/llm-testbench/internal/testkit" +) + +// registerHardTests registers every programming/hard test. This subcategory +// stresses harder daily-programming accuracy - concurrency/async reasoning, +// subtle language traps, algorithm output tracing, and API-contract edge +// cases - each forced to a short, machine-checkable answer. +func registerHardTests(r *testkit.Registry) { + r.Register(hardFloatBinaddTest()) + r.Register(hardNullishOverOrTest()) + r.Register(hardGoroutineRaceTest()) + r.Register(hardDeferNamedReturnTest()) + r.Register(hardIntDivisionTest()) + r.Register(hardRegexGreedyTest()) + r.Register(hardPromiseAllSettledTest()) + r.Register(hardContextTimeoutTest()) + r.Register(hardJSONUnmarshalNullTest()) + r.Register(hardVarShadowTest()) + r.Register(hardPyVersionSortTest()) + r.Register(hardGoParenBalanceTest()) +} + +// hardFloatBinaddTest: judge whether 0.1 + 0.2 === 0.3 holds in +// JavaScript / IEEE-754 doubles. +// +// ground truth: neither 0.1 nor 0.2 has an exact binary representation, +// so 0.1 + 0.2 computes to the nearest double to the mathematical sum, +// which is 0.30000000000000004, not the double for the literal 0.3. +// Therefore the strict equality comparison evaluates to false. This is a +// canonical, deterministic fact of IEEE-754 double arithmetic, independent +// of any runtime; verified with node during authoring. +func hardFloatBinaddTest() testkit.Test { + prompt := `In JavaScript, evaluate this comparison: + +0.1 + 0.2 === 0.3 + +Does the strict-equality comparison evaluate to true or to false? +Respond with only a JSON object: +{"equal": true} +or +{"equal": false}` + + return testkit.Test{ + ID: "hard-float-binadd", + Category: "programming", + Subcategory: "hard", + Description: "Judge IEEE-754 double equality for 0.1 + 0.2 === 0.3 in JavaScript (false).", + Prompt: prompt, + Eval: eval.JSONField("equal", false), + } +} + +// hardNullishOverOrTest: trace the value of a nullish-coalescing +// expression over a falsy (but non-null) operand. +// +// ground truth: ?? returns the right-hand operand only when the left-hand +// operand is null or undefined. 0 is neither null nor undefined, so +// n ?? 42 evaluates to 0 - unlike ||, which would have returned 42 for +// the falsy 0. This is the core distinction between nullish coalescing +// and logical OR. Verified with node during authoring. +func hardNullishOverOrTest() testkit.Test { + prompt := `Consider this TypeScript code: + +const n = 0; +const v = n ?? 42; + +What is the value of v? +Respond with only a JSON object: +{"value": ...} + +Give the numeric value only.` + + return testkit.Test{ + ID: "hard-nullish-over-or", + Category: "programming", + Subcategory: "hard", + Description: "Trace nullish-coalescing over a falsy-but-non-null operand: 0 ?? 42 is 0, not 42.", + Prompt: prompt, + Eval: eval.JSONField("value", 0), + } +} + +// hardGoroutineRaceTest: judge whether an unsynchronized shared counter +// is guaranteed to reach N*M exactly. +// +// ground truth: incrementing a shared int from multiple goroutines with no +// synchronization is a data race (Go's race detector would flag it). A +// read-modify-write of a shared variable is not atomic; two goroutines can +// read the same value before either writes back, so increments are lost. +// The final value is some value strictly between M (at best serialized, +// one goroutine's work) and N*M, but it is NOT guaranteed to equal +// N*M. This claim follows from the Go memory model / data-race semantics +// and cannot be recomputed by running (the outcome is nondeterministic). +func hardGoroutineRaceTest() testkit.Test { + prompt := `A Go program starts 8 goroutines, each of which performs: +counter++ +1000 times, where counter is a shared int with NO synchronization +(no mutex, no atomic) between the goroutines. The program then waits for +all 8 goroutines and prints counter. + +Is counter's final value guaranteed to be exactly 8000? +Respond with only a JSON object: +{"guaranteed": true} +or +{"guaranteed": false}` + + return testkit.Test{ + ID: "hard-goroutine-race", + Category: "programming", + Subcategory: "hard", + Description: "Judge whether an unsynchronized shared counter is guaranteed to reach N*M (false: data race).", + Prompt: prompt, + Eval: eval.JSONField("guaranteed", false), + } +} + +// hardDeferNamedReturnTest: trace the return value of a function whose +// deferred closure writes its named return during a normal (non-panic) +// return. +// +// ground truth: return 7 first assigns the named result r = 7; then, +// still before the function returns to its caller, the deferred closure +// runs and overwrites r = 99. The caller observes the named return after +// all defers have run, so f() returns 99 - not 7 and not 5. This is the +// classic named-return-plus-defer trap. Verified with go run during +// authoring. +func hardDeferNamedReturnTest() testkit.Test { + prompt := `Consider this Go code: + +func f() (r int) { + defer func() { r = 99 }() + r = 5 + return 7 +} + +What value does f() return? +Respond with only the number, no explanation.` + + return testkit.Test{ + ID: "hard-defer-named-return", + Category: "programming", + Subcategory: "hard", + Description: "Trace a deferred closure overwriting a named return value during a normal return (99).", + Prompt: prompt, + Eval: eval.Numeric(eval.ExtractLastNumber[int], hardDeferNamedReturnWant, 0), + } +} + +// hardDeferNamedReturnWant is f()'s return value. +// +// ground truth: the deferred closure runs during the function's epilogue, +// after the explicit `return 7` has set the named result r = 7, and +// overwrites it with 99; the caller therefore receives 99 (see the full +// derivation on hardDeferNamedReturnTest). Recomputed by running the +// identical snippet with go run during authoring. +const hardDeferNamedReturnWant = 99 + +// hardIntDivisionTest: judge the result of integer division being +// assigned to a floating-point variable. +// +// ground truth: a / b with two int operands performs integer division, +// which truncates toward zero: 7 / 2 = 3 (not 3.5). That integral +// result is then implicitly converted to double, so d = 3.0. The common +// error is assuming floating division because the target is a double. +// Verified with go run during authoring. +func hardIntDivisionTest() testkit.Test { + prompt := `Consider this C code: + +int a = 7; +int b = 2; +double d = a / b; + +What is the value of d? +Respond with only the number, no explanation.` + + return testkit.Test{ + ID: "hard-int-division", + Category: "programming", + Subcategory: "hard", + Description: "Trace integer-division-then-widen: 7 / 2 is 3 (stored as 3.0), not 3.5.", + Prompt: prompt, + Eval: eval.Numeric(eval.ExtractLastNumber[float64], hardIntDivisionWant, 0), + } +} + +// hardIntDivisionWant is d's value. +// +// ground truth: integers divide as integers (truncating), giving 7 / 2 = +// 3, converted to double 3.0; the float value is 3, so Numeric want is +// 3.0. Recomputed by running the identical arithmetic during authoring. +const hardIntDivisionWant = 3.0 + +// hardRegexGreedyTest: trace which substrings two greedy capture groups +// of an anchored pattern match. +// +// ground truth: quantifiers are greedy by default - each takes as much as +// it can while still allowing the rest of the pattern to match. In +// /^(\d+)(\d+)$/, the first \d+ grabs the longest prefix that leaves at +// least one digit for the second \d+ (which needs at least one match +// because + is one-or-more). That longest prefix of "12345" is "1234", +// leaving "5" for group 2. A naive symmetric split like ["12","345"] +// fails because group 1 would then still be able to grow. Verified with +// node during authoring. +func hardRegexGreedyTest() testkit.Test { + prompt := `Consider this regular expression applied to the string "12345": + +^(\d+)(\d+)$ + +(^ and $ anchor to the whole string, and \d+ means one or more digits.) + +What are the two captured groups? Respond with only a JSON array of the +two captured strings, in group order, for example ["a","b"]` + + return testkit.Test{ + ID: "hard-regex-greedy", + Category: "programming", + Subcategory: "hard", + Description: "Trace greedy capture-group backtracking: ^(one-or-more digits)(one-or-more digits)$ on 12345 yields 1234 and 5.", + Prompt: prompt, + Eval: eval.JSONStringArrayEquals([]string{"1234", "5"}), + } +} + +// hardPromiseAllSettledTest: trace the per-result status values of +// Promise.allSettled over a mixed resolve/reject input. +// +// ground truth: Promise.allSettled always resolves, and each element of +// its result array is an object whose status is the literal "fulfilled" +// for a resolved input and "rejected" for a rejected input, preserving +// input order. Promise.resolve(42) yields {status:"fulfilled", value:42}; +// Promise.reject(new Error("boom")) yields {status:"rejected", +// reason:...}. So the two status values, in order, are "fulfilled" then +// "rejected". Verified with node during authoring. +func hardPromiseAllSettledTest() testkit.Test { + prompt := `Consider this TypeScript code: + +const results = await Promise.allSettled([ + Promise.resolve(42), + Promise.reject(new Error("boom")), +]); + +Each element of results has a "status" field. + +List the two status values, in array order. Respond with only a JSON +array of the two status strings, for example ["a","b"]` + + return testkit.Test{ + ID: "hard-promise-allsettled", + Category: "programming", + Subcategory: "hard", + Description: "Trace Promise.allSettled result statuses over a fulfilled+rejected input (fulfilled, rejected).", + Prompt: prompt, + Eval: eval.JSONStringArrayEquals([]string{"fulfilled", "rejected"}), + } +} + +// hardContextTimeoutTest: identify the exact error string a timed-out +// context's Err() returns. +// +// ground truth: when a context created by context.WithTimeout fires, its +// Done channel closes and Err() returns the sentinel +// context.DeadlineExceeded, whose Error() string (per the context +// package) is exactly "context deadline exceeded" - which is distinct +// from "context canceled", the string of context.Canceled returned when a +// cancel function is called. Verified with go run during authoring. +func hardContextTimeoutTest() testkit.Test { + prompt := `In Go, a context created with context.WithTimeout fires after its +deadline. At that point: + +ctx.Err() + +returns a non-nil error. What is the exact string of that error? +Respond with only a JSON object: +{"err": "..."} + +with the exact string value.` + + return testkit.Test{ + ID: "hard-context-timeout", + Category: "programming", + Subcategory: "hard", + Description: "Identify ctx.Err() after a context.WithTimeout deadline fires (context deadline exceeded).", + Prompt: prompt, + Eval: eval.JSONField("err", "context deadline exceeded"), + } +} + +// hardJSONUnmarshalNullTest: judge the behavior of json.Unmarshal on a +// literal JSON null into an already-populated struct. +// +// ground truth: per encoding/json, a JSON null into a non-nil pointer +// target is a no-op: Unmarshal returns nil (no error) and leaves the +// target's existing value unchanged. It does not zero the struct and does +// not error. Verified with go run during authoring (a struct holding +// Items: [1] stays [1] with nil error). +func hardJSONUnmarshalNullTest() testkit.Test { + prompt := `In Go: + +var s Struct // s is a populated struct value (some fields set) +err := json.Unmarshal([]byte("null"), &s) + +Does Unmarshal return an error, and is s left unchanged (no fields +cleared)? +Respond with only a JSON object: +{"returns_error": true} +or +{"returns_error": false}` + + return testkit.Test{ + ID: "hard-json-unmarshal-null", + Category: "programming", + Subcategory: "hard", + Description: "Judge json.Unmarshal of a literal null into a populated struct (no error, unchanged).", + Prompt: prompt, + Eval: eval.JSONField("returns_error", false), + } +} + +// hardVarShadowTest: judge whether an inner := declaration shadows an +// outer variable without changing it. +// +// ground truth: `count := 5` inside the if block declares a NEW, inner +// variable that shadows the outer count for the duration of that block +// (Go's := always creates a new variable when at least one variable on +// the left is new in the current scope). Incrementing the inner count +// leaves the outer count untouched, so after the block outer count is +// still 0. Verified with go run during authoring. +func hardVarShadowTest() testkit.Test { + prompt := `Consider this Go code: + +count := 0 +if true { + count := 5 + count++ +} +// count here? + +Does the outer count remain unchanged (equal to 0) after the if block? +Respond with only a JSON object: +{"outer_unchanged": true} +or +{"outer_unchanged": false}` + + return testkit.Test{ + ID: "hard-var-shadow", + Category: "programming", + Subcategory: "hard", + Description: "Trace Go variable shadowing: an inner := leaves the outer variable unchanged (true).", + Prompt: prompt, + Eval: eval.JSONField("outer_unchanged", true), + } +} + +// hardPyVersionVersions is the inline version list for +// hardPyVersionSortTest. +const hardPyVersionVersions = `versions = [ + "1.9.0", + "1.10.0", + "1.2.0", + "2.0.0", + "0.9.9", +]` + +// hardPyVersionSortWant is the expected stdout of a correct script: the +// second-highest version under numeric component-by-component ordering. +// +// ground truth: ordering versions by numeric components puts 0.9.9 < +// 1.2.0 < 1.9.0 < 1.10.0 < 2.0.0 (1.10.0 is newer than 1.9.0 because +// its second component 10 > 9), so the second-highest is 1.10.0. A +// plain lexicographic string sort (the trap) would instead report 1.9.0, +// which is exactly the failure the test discriminates. Recomputed by +// running the correct script with python3 during authoring. +const hardPyVersionSortWant = "1.10.0" + +// hardPyVersionSortTest: write a python3 script that prints the +// second-highest version under numeric component ordering. +// +// ground truth: see hardPyVersionSortWant; correctness is determined by +// running the script (version ordering is behavior, not a text match). +func hardPyVersionSortTest() testkit.Test { + prompt := `Here is an inline list of software version strings: + +` + "```python" + ` +` + hardPyVersionVersions + ` +` + "```" + ` + +Write a python3 program that sorts the versions numerically, component by +component (natural version ordering: "1.10.0" is newer than "1.9.0" +because the second component 10 > 9), and prints only the second-highest +version.` + + return testkit.Test{ + ID: "hard-py-version-sort", + Category: "programming", + Subcategory: "hard", + Description: "Write a python3 script printing the second-highest version by numeric component ordering.", + System: terseCodeOnly, + Prompt: prompt, + Eval: eval.PyRun(eval.PassthroughHarness, hardPyVersionSortWant), + } +} + +// hardGoParenBalanceHarness is a complete, independent driver file (its +// own package clause and imports) that eval.GoRun writes to harness.go +// alongside the model's IsBalanced in solution.go. It drives IsBalanced +// across balanced, nested, mismatched, unclosed, and wrongly-ordered +// cases, printing PASS only if every case matches its expected result. +const hardGoParenBalanceHarness = `package main + +import ( + "fmt" + "os" +) + +func main() { + cases := []struct { + in string + want bool + }{ + {"", true}, + {"()", true}, + {"()[]{}", true}, + {"([{}])", true}, + {"(", false}, + {"(]", false}, + {"([)]", false}, + {"((", false}, + {")(", false}, + } + for _, c := range cases { + if got := IsBalanced(c.in); got != c.want { + fmt.Printf("FAIL %q: got %v want %v\n", c.in, got, c.want) + os.Exit(1) + } + } + fmt.Println("PASS") +}` + +// hardGoParenBalanceTest: implement a stack-based balanced-parentheses +// checker. +// +// ground truth: the harness above is the oracle - it exercises empty, +// balanced, nested, mismatched-closer, trailing-opener, and wrong-order +// inputs, so a correct stack-based implementation prints PASS. There is no +// separate hardcoded constant beyond the PASS sentinel. +func hardGoParenBalanceTest() testkit.Test { + prompt := `Implement a Go function with exactly this signature: + +func IsBalanced(s string) bool + +It must report whether s's parentheses are balanced and correctly nested: +'(' '[' '{' are openers, ')' ']' '}' are the matching closers. An empty +string is balanced (return true). Return false for any unclosed opener, +any closer with no matching open, or any wrong-nesting (e.g. "([)]").` + + return testkit.Test{ + ID: "hard-go-paren-balance", + Category: "programming", + Subcategory: "hard", + Description: "Implement a stack-based balanced-parentheses checker for (), [], and {}.", + System: terseCodeOnly, + Prompt: prompt, + Eval: eval.GoRun(hardGoParenBalanceHarness, "PASS"), + } +} diff --git a/internal/tests/programming_hard_test.go b/internal/tests/programming_hard_test.go new file mode 100644 index 0000000..2121331 --- /dev/null +++ b/internal/tests/programming_hard_test.go @@ -0,0 +1,360 @@ +package tests + +import ( + "context" + "testing" +) + +func TestHardFloatBinaddTest_Eval(t *testing.T) { + tc := hardFloatBinaddTest() + + tests := []struct { + name string + response string + want float64 + }{ + {"correct", `{"equal":false}`, 1}, + {"correct fenced with prose", "The answer is:\n```json\n{\"equal\":false}\n```", 1}, + {"wrong: claims equal", `{"equal":true}`, 0}, + {"wrong: non-boolean", `{"equal":"yes"}`, 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Value != tt.want { + t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail) + } + }) + } +} + +func TestHardNullishOverOrTest_Eval(t *testing.T) { + tc := hardNullishOverOrTest() + + tests := []struct { + name string + response string + want float64 + }{ + {"correct", `{"value":0}`, 1}, + {"correct as string", `{"value":"0"}`, 1}, + {"wrong: OR semantics", `{"value":42}`, 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Value != tt.want { + t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail) + } + }) + } +} + +func TestHardGoroutineRaceTest_Eval(t *testing.T) { + tc := hardGoroutineRaceTest() + + tests := []struct { + name string + response string + want float64 + }{ + {"correct", `{"guaranteed":false}`, 1}, + {"correct fenced", "```json\n{\"guaranteed\":false}\n```", 1}, + {"wrong: claims guaranteed", `{"guaranteed":true}`, 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Value != tt.want { + t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail) + } + }) + } +} + +func TestHardDeferNamedReturnTest_Eval(t *testing.T) { + tc := hardDeferNamedReturnTest() + + tests := []struct { + name string + response string + want float64 + }{ + {"correct", "99", 1}, + {"correct prose-wrapped", "The function returns 99.", 1}, + {"wrong: returns statement value", "7", 0}, + {"wrong: pre-defer value", "5", 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Value != tt.want { + t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail) + } + }) + } +} + +func TestHardIntDivisionTest_Eval(t *testing.T) { + tc := hardIntDivisionTest() + + tests := []struct { + name string + response string + want float64 + }{ + {"correct", "3", 1}, + {"correct as float", "3.0", 1}, + {"correct prose-wrapped", "It is 3.", 1}, + {"wrong: assumes float division", "3.5", 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Value != tt.want { + t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail) + } + }) + } +} + +func TestHardRegexGreedyTest_Eval(t *testing.T) { + tc := hardRegexGreedyTest() + + tests := []struct { + name string + response string + want float64 + }{ + {"correct", `["1234","5"]`, 1}, + {"correct fenced", "```json\n[\"1234\",\"5\"]\n```", 1}, + {"wrong: symmetric split", `["12","345"]`, 0}, + {"wrong: reversed", `["5","1234"]`, 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Value != tt.want { + t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail) + } + }) + } +} + +func TestHardPromiseAllSettledTest_Eval(t *testing.T) { + tc := hardPromiseAllSettledTest() + + tests := []struct { + name string + response string + want float64 + }{ + {"correct", `["fulfilled","rejected"]`, 1}, + {"correct fenced", "```json\n[\"fulfilled\",\"rejected\"]\n```", 1}, + {"wrong: reordered", `["rejected","fulfilled"]`, 0}, + {"wrong: both fulfilled", `["fulfilled","fulfilled"]`, 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Value != tt.want { + t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail) + } + }) + } +} + +func TestHardContextTimeoutTest_Eval(t *testing.T) { + tc := hardContextTimeoutTest() + + tests := []struct { + name string + response string + want float64 + }{ + {"correct", `{"err":"context deadline exceeded"}`, 1}, + {"correct fenced", "```json\n{\"err\":\"context deadline exceeded\"}\n```", 1}, + {"wrong: canceled sentinel", `{"err":"context canceled"}`, 0}, + {"wrong: vague", `{"err":"some error"}`, 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Value != tt.want { + t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail) + } + }) + } +} + +func TestHardJSONUnmarshalNullTest_Eval(t *testing.T) { + tc := hardJSONUnmarshalNullTest() + + tests := []struct { + name string + response string + want float64 + }{ + {"correct", `{"returns_error":false}`, 1}, + {"correct fenced", "```json\n{\"returns_error\":false}\n```", 1}, + {"wrong: claims error", `{"returns_error":true}`, 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Value != tt.want { + t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail) + } + }) + } +} + +func TestHardVarShadowTest_Eval(t *testing.T) { + tc := hardVarShadowTest() + + tests := []struct { + name string + response string + want float64 + }{ + {"correct", `{"outer_unchanged":true}`, 1}, + {"correct fenced", "```json\n{\"outer_unchanged\":true}\n```", 1}, + {"wrong: claims outer changed", `{"outer_unchanged":false}`, 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Value != tt.want { + t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail) + } + }) + } +} + +// TestHardPyVersionSortTest_Eval exercises the PyRun evaluator with a +// hand-written correct script and a lexicographic (wrong) script. +func TestHardPyVersionSortTest_Eval(t *testing.T) { + tc := hardPyVersionSortTest() + + correctScript := "```python\n" + `versions = [ + "1.9.0", + "1.10.0", + "1.2.0", + "2.0.0", + "0.9.9", +] + +def key(v): + return tuple(int(p) for p in v.split(".")) + +sorted_versions = sorted(versions, key=key) +print(sorted_versions[-2]) +` + "```" + + wrongLexicographic := "```python\n" + `versions = [ + "1.9.0", + "1.10.0", + "1.2.0", + "2.0.0", + "0.9.9", +] +print(sorted(versions)[-2]) +` + "```" + + wrongHardcoded := "```python\nprint('2.0.0')\n```" + + tests := []struct { + name string + response string + want float64 + }{ + {"correct numeric sort", correctScript, 1}, + {"wrong: lexicographic sort", wrongLexicographic, 0}, + {"wrong: hardcoded highest", wrongHardcoded, 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Skipped { + t.Skip("python3 not available, skipping exec-evaluator assertion") + } + if got.Value != tt.want { + t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail) + } + }) + } +} + +// TestHardGoParenBalanceTest_Eval exercises the GoRun evaluator with a +// hand-written correct function and a wrong one. +func TestHardGoParenBalanceTest_Eval(t *testing.T) { + tc := hardGoParenBalanceTest() + + correctFunc := "```go\n" + `func IsBalanced(s string) bool { + pairs := map[byte]byte{')': '(', ']': '[', '}': '{'} + stack := make([]byte, 0, len(s)) + for i := 0; i < len(s); i++ { + switch s[i] { + case '(', '[', '{': + stack = append(stack, s[i]) + case ')', ']', '}': + if len(stack) == 0 || stack[len(stack)-1] != pairs[s[i]] { + return false + } + stack = stack[:len(stack)-1] + } + } + return len(stack) == 0 +} +` + "```" + + wrongFunc := "```go\n" + `func IsBalanced(s string) bool { + return true +} +` + "```" + + tests := []struct { + name string + response string + want float64 + }{ + {"correct stack implementation", correctFunc, 1}, + {"wrong: always true", wrongFunc, 0}, + {"wrong: no code at all", "I would use a stack here.", 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Skipped { + t.Skip("go toolchain not available, skipping exec-evaluator assertion") + } + if got.Value != tt.want { + t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail) + } + }) + } +} + +// TestHardDeferNamedReturnWant_GroundTruth recomputes f()'s return value +// by running an equivalent function in-process, independent of the +// hardDeferNamedReturnWant constant. +func TestHardDeferNamedReturnWant_GroundTruth(t *testing.T) { + f := func() (r int) { + defer func() { r = 99 }() + r = 5 + return 7 + } + if got := f(); got != hardDeferNamedReturnWant { + t.Errorf("equivalent deferred-named-return function returned %d, want %d", got, hardDeferNamedReturnWant) + } +} + +// TestHardIntDivisionWant_GroundTruth recomputes for a double result of +// 7 / 2 by hand, independent of the hardIntDivisionWant constant. +func TestHardIntDivisionWant_GroundTruth(t *testing.T) { + a, b := 7, 2 + want := float64(a / b) // integer division truncates before widening + if want != hardIntDivisionWant { + t.Errorf("recomputed integer division = %v, want %v", want, hardIntDivisionWant) + } +} diff --git a/internal/tests/saturation.go b/internal/tests/saturation.go new file mode 100644 index 0000000..01e120d --- /dev/null +++ b/internal/tests/saturation.go @@ -0,0 +1,534 @@ +// Package-level hardening tests for the five subcategories that measured +// 100% perfect-score across all three baseline models (see the health +// audit over the 2026-08-18 3-model run): agents/delegation, +// databases/postgres, research/web, security/appsec, security/secrets. +// Every test there was saturated - zero discrimination signal. These ten +// tests are trap-shaped on purpose: each embeds a near-miss answer that +// plausible-but-wrong reasoning produces, and every expected value carries +// a derivation comment. +package tests + +import ( + "github.com/lukaszraczylo/llm-testbench/internal/eval" + "github.com/lukaszraczylo/llm-testbench/internal/testkit" +) + +// registerSaturationTests adds the hardening tests to the registry. They +// land in existing subcategories, so the per-category register functions +// stay untouched; the subcategory's test count only grows. +func registerSaturationTests(r *testkit.Registry) { + r.Register(satDelegCapacityTrapTest()) + r.Register(satDelegNearMissRosterTest()) + r.Register(satPGCountStarTrapTest()) + r.Register(satPGTimeoutTrapTest()) + r.Register(satWebRobotsUAScopeTest()) + r.Register(satWebHreflangSelfTest()) + r.Register(satAppsecORMTrapTest()) + r.Register(satAppsecPOSTTrapTest()) + r.Register(satSecretsRotationMathTest()) + r.Register(satSecretsErrorPathTest()) +} + +// satDelegCapacityTrapTest: the roster's specialist for the job exists but +// is unavailable. Tests whether the model force-fits the task to a +// mismatched agent, over-coordinates with the orchestrator, or correctly +// keeps it in the main thread. +func satDelegCapacityTrapTest() testkit.Test { + prompt := `You are coordinating agents. Available specialist roster: + +` + delegRoster + ` + +Situation: the CI pipeline for the main branch has been failing for the +past hour on a Docker build step. It needs diagnosing now. The roster's +infra/CI specialist is already working on an unrelated production incident +and will not free up for several hours. There is exactly one task. + +Respond with only a JSON object, no other text, of the form +{"best_next_step": "...", "specialist_this_task_belongs_to": "..."} where: +- best_next_step is exactly one of: "dispatch_code_writer", + "dispatch_orchestrator", "handle_in_main_thread", "wait_for_specialist" +- specialist_this_task_belongs_to is the roster agent name whose + description covers this task, regardless of their availability.` + return testkit.Test{ + ID: "agent-deleg-capacity-trap", + Category: "agents", + Subcategory: "delegation", + Description: "Route a task whose specialist is busy: no force-fit to a mismatched agent, no orchestrator for a single task, and still name the correct owning specialist", + Prompt: prompt, + // ground truth: the task is diagnosing a failing CI build, which + // the roster assigns to devops-debugger ("Diagnoses failing + // deployments, CI pipelines, and infrastructure incidents"). It is + // busy, so the remaining options are judged against the roster: + // code-writer "Implements new features or bug fixes by writing and + // editing source code directly" - a generalist forced onto infra + // diagnosis; orchestrator "Plans and sequences a multi-agent + // workflow" - there is one task, nothing to sequence; waiting on a + // specialist that is hours away while CI is down blocks delivery + // for no gain. A single task with its specialist unavailable is + // handled directly (handle_in_main_thread). The owning specialist + // on the roster is unambiguously devops-debugger. + Eval: eval.Mean( + eval.JSONField[string]("best_next_step", "handle_in_main_thread"), + eval.JSONField[string]("specialist_this_task_belongs_to", "devops-debugger"), + ), + } +} + +// satDelegNearMissRosterTest: 4-task mapping where every task has a +// same-vocabulary distractor (docs vs web lookup, release notes vs prose +// docs, diff review vs security review). +func satDelegNearMissRosterTest() testkit.Test { + prompt := `You are coordinating agents. Specialist roster: + +` + delegRoster + ` + +Map each task to exactly one roster agent - the one whose roster +description covers it. Tasks: +- task1: "Update the API guide in docs/ so it matches the endpoints as + they exist in the repository today." +- task2: "Determine from the vendor's public pricing page which of their + plans includes SSO." +- task3: "Produce the v2.3 release notes by generating the changelog + section and tagging the release." +- task4: "Audit the auth handler for vulnerabilities before it merges." + +Respond with only a JSON object of the form +{"task1": "...", "task2": "...", "task3": "...", "task4": "..."} +where each value is exactly one roster agent name.` + return testkit.Test{ + ID: "agent-deleg-near-miss-roster", + Category: "agents", + Subcategory: "delegation", + Description: "Route 4 tasks whose phrasing overlaps two roster descriptions each (repo docs vs web lookup, changelog vs prose docs, diff review vs security audit)", + Prompt: prompt, + // ground truth, per the delegRoster descriptions: + // task1 -> docs-writer: it writes/updates prose documentation + // ("READMEs, guides, design docs") from repository content. + // web-researcher is wrong: "Does not read or write anything in the + // repository" and no external search is needed. + // task2 -> web-researcher: the answer lives on the vendor's public + // page, "Searches the public web and summarizes external + // information"; docs-writer is the distractor ("page" sounds like + // documentation) but the task reads the web, not the repo. + // task3 -> release-manager: "Manages version tagging, changelog + // generation, and release publishing" covers exactly the phrasing + // (generate changelog + tag). docs-writer is the distractor: + // release notes here are the release process, not a guide. + // task4 -> security-reviewer: "Audits code or config for security + // vulnerabilities"; code-reviewer is the distractor ("before it + // merges" sounds like PR review) but code-reviewer reviews + // "correctness, style, and risk", while the stated goal is + // vulnerabilities. + Eval: eval.Mean( + eval.JSONField[string]("task1", "docs-writer"), + eval.JSONField[string]("task2", "web-researcher"), + eval.JSONField[string]("task3", "release-manager"), + eval.JSONField[string]("task4", "security-reviewer"), + ), + } +} + +// satPGCountStarFixture is the orders/line-items data for the join +// fan-out trap. +const satPGCountStarFixture = `CREATE TABLE orders (id bigint PRIMARY KEY, region text NOT NULL); +CREATE TABLE order_items (id bigint PRIMARY KEY, + order_id bigint NOT NULL REFERENCES orders(id)); + +INSERT INTO orders (id, region) VALUES + (1001, 'eu'), (1002, 'eu'), (1003, 'us'), + (1004, 'eu'), (1005, 'ap'), (1006, 'eu'); + +INSERT INTO order_items (id, order_id) VALUES + (9001, 1001), (9002, 1001), (9003, 1001), + (9004, 1002), + (9005, 1003), (9006, 1003), (9007, 1003), (9008, 1003), (9009, 1003), + (9010, 1004), (9011, 1004), + (9012, 1005);` + +// satPGCountStarTrapTest: COUNT(*) over an INNER JOIN counts line items, +// not orders, and the join silently drops the order with zero items. Every +// number is derivable from the fixture, so the evaluator never needs to +// re-run SQL. +func satPGCountStarTrapTest() testkit.Test { + prompt := `Here is a PostgreSQL schema with data: + +` + "```sql\n" + satPGCountStarFixture + "\n```" + ` + +The business question is: how many ORDERS are in region 'eu'? +A developer writes: + +` + "```sql\nSELECT count(*) FROM orders o JOIN order_items oi ON oi.order_id = o.id WHERE o.region = 'eu';\n```" + ` + +Respond with only a JSON object, no other text: +{"orders_in_eu": , "rows_the_join_query_returns": , + "count_star_answers_the_question": true or false}` + return testkit.Test{ + ID: "pg-count-star-vs-join", + Category: "databases", + Subcategory: "postgres", + Description: "Reason about COUNT(*) over an INNER JOIN against a fixed fixture: join fan-out inflates the count and the join drops the childless order", + Prompt: prompt, + // ground truth: orders with region 'eu' are 1001, 1002, 1004, 1006 + // (INSERTs above) = 4. The join emits one row per order_item: + // 1001 -> 3 items (9001-9003), 1002 -> 1 (9004), 1004 -> 2 (9010, + // 9011), and 1006 -> 0 items, so the INNER JOIN drops it entirely. + // count(*) therefore returns 3+1+2 = 6. 6 != 4, so the count(*) + // query does not answer the question (EXISTS or COUNT(DISTINCT + // o.id) would return 4). + Eval: eval.Mean( + eval.JSONField[int]("orders_in_eu", 4), + eval.JSONField[int]("rows_the_join_query_returns", 6), + eval.JSONField[bool]("count_star_answers_the_question", false), + ), + } +} + +// satPGTimeoutTrapTest: two sessions, two timeouts, one rule each. The +// near-miss is applying idle_in_transaction_session_timeout to a busy +// session (or statement_timeout to an idle one), and starting the idle +// clock at BEGIN instead of at the last statement's completion. +func satPGTimeoutTrapTest() testkit.Test { + prompt := `A PostgreSQL server is configured with: + +` + "```\nstatement_timeout = '30s'\nidle_in_transaction_session_timeout = '5s'\n```" + ` + +Two sessions on this server: +- Session A: BEGIN at 11:00:00, runs statements until 11:00:40 (the last + one finishes then), and has been waiting for client input ever since. + The transaction is still open. +- Session B: BEGIN at 11:00:53, starts one long-running query at 11:00:55 + which is still executing at 11:01:00. + +Assume no client input arrives and no query finishes after 11:01:00. +Respond with only a JSON object, no other text: +{"session_a_killed_at": "HH:MM:SS", "session_a_rule": "...", + "session_b_killed_at": "HH:MM:SS", "session_b_rule": "..."} +where each time is in UTC and each rule is exactly one of: +"statement_timeout", "idle_in_transaction_timeout", "not_killed".` + return testkit.Test{ + ID: "pg-timeout-trap", + Category: "databases", + Subcategory: "postgres", + Description: "Apply statement_timeout and idle_in_transaction_session_timeout to the right sessions: idle clock starts at the last statement's completion, not BEGIN, and never applies to a session executing a statement", + Prompt: prompt, + // ground truth: PostgreSQL's idle_in_transaction_session_timeout + // terminates a session "idle ... within an open transaction" - + // the clock starts when the session goes idle in transaction, + // which for A is 11:00:40 (its last statement finished then), so + // A is killed at 11:00:45 - not 11:00:05, since A was executing + // statements from 11:00:00 to 11:00:40. A is not subject to + // statement_timeout: no statement is running. B is idle in + // transaction only from 11:00:53 to 11:00:55 (2s, under the 5s + // cap) and then EXECUTES a statement, so the idle timeout no + // longer applies; statement_timeout bounds a single running + // statement: 11:00:55 + 30s = 11:01:25. + Eval: eval.Mean( + eval.JSONField[string]("session_a_killed_at", "11:00:45"), + eval.JSONField[string]("session_a_rule", "idle_in_transaction_timeout"), + eval.JSONField[string]("session_b_killed_at", "11:01:25"), + eval.JSONField[string]("session_b_rule", "statement_timeout"), + ), + } +} + +// satWebRobotsFixture carries two rule groups and a longest-match pair. +const satWebRobotsFixture = `User-agent: * +Disallow: /cdn-cgi/ +Allow: /cdn-cgi/style.css +Sitemap: https://example.com/sitemap.xml + +User-agent: GoogleOther +Disallow: /` + +// satWebRobotsUAScopeTest: GoogleOther's rules do not apply to Googlebot, +// and within Googlebot's group the longest path match decides. The +// near-miss is answering from the GoogleOther block alone, and answering +// the Allow/Disallow pair by document order instead of path length. +func satWebRobotsUAScopeTest() testkit.Test { + prompt := `A site serves this robots.txt at https://example.com/robots.txt: + +` + "```\n" + satWebRobotsFixture + "\n```" + ` + +Consider a GET request for https://example.com/cdn-cgi/style.css. +Respond with only a JSON object, no other text: +{"googlebot_allowed": true or false, "googleother_allowed": true or false, + "deciding_rule_for_googlebot": "allow" or "disallow" or "no_rule"}` + return testkit.Test{ + ID: "web-robots-ua-scope-trap", + Category: "research", + Subcategory: "web", + Description: "robots.txt group matching across two User-agent blocks plus longest-path-match precedence - a separate group's Disallow does not apply to the other crawler", + Prompt: prompt, + // ground truth: per RFC 9309, a crawler only applies the rule + // group(s) whose User-agent matches it, and within a group the + // path pattern with the LONGEST match wins. Googlebot does not + // match the GoogleOther group, so only the '*' group applies: + // the URL matches Disallow /cdn-cgi/ (9 chars) and Allow + // /cdn-cgi/style.css (18 chars); 18 > 9, so Allow wins -> + // googlebot_allowed true, deciding_rule_for_googlebot "allow". + // GoogleOther matches its own group, where "Disallow: /" covers + // every path -> googleother_allowed false. + Eval: eval.Mean( + eval.JSONField[bool]("googlebot_allowed", true), + eval.JSONField[bool]("googleother_allowed", false), + eval.JSONField[string]("deciding_rule_for_googlebot", "allow"), + ), + } +} + +// satWebHreflangSelfTest: an hreflang set missing its self-reference and +// x-default's sentinel status. The near-miss is counting the canonical +// link as the hreflang self-reference, and treating x-default as a +// language. +func satWebHreflangSelfTest() testkit.Test { + prompt := `The page https://example.com/en-us/ serves this head section: + +` + "```html\n" + ` + + + + + +` + "\n```" + ` + +Google's hreflang rule: an hreflang group must include a self-reference - +an hreflang annotation whose href is the page itself - and every +annotation it lists must point back reciprocally. +Respond with only a JSON object, no other text: +{"missing_self_reference_for": "the BCP 47 language tag of the page that +has no hreflang annotation pointing at it", "self_reference_present": +true or false, "canonical_counts_as_hreflang_self_reference": true or +false, "x_default_is_a_language_tag": true or false}` + return testkit.Test{ + ID: "web-hreflang-self-reference-trap", + Category: "research", + Subcategory: "web", + Description: "hreflang self-reference validation: the page's own language has no annotation, rel=canonical is a different mechanism, and x-default is a sentinel, not a BCP 47 tag", + Prompt: prompt, + // ground truth: the page is en-US (lang attribute and canonical + // URL agree). The hreflang annotations cover en-AU, en-GB, + // x-default only - no annotation has href + // https://example.com/en-us/, so the missing self-reference tag + // is "en-US" and self_reference_present is false. + // canonical_counts_as_hreflang_self_reference is false: rel + // -canonical deduplicates URLs for indexing; hreflang self- + // reference is a separate signal on rel=alternate link elements + // - the canonical matching the page URL does not satisfy it. + // x_default_is_a_language_tag is false: x-default is Google's + // sentinel for "none of the specific languages", not a BCP 47 + // tag (it carries no language subtag; the other two do). + Eval: eval.Mean( + eval.JSONField[string]("missing_self_reference_for", "en-US"), + eval.JSONField[bool]("self_reference_present", false), + eval.JSONField[bool]("canonical_counts_as_hreflang_self_reference", false), + eval.JSONField[bool]("x_default_is_a_language_tag", false), + ), + } +} + +// satAppsecORMFixture shows three SQLAlchemy call sites; only one builds +// SQL by concatenation before wrapping it in text(). +const satAppsecORMFixture = `from sqlalchemy import text + +def get_user(conn, name, team): + sql = text("SELECT * FROM users WHERE name = :name AND team = :team") + return conn.execute(sql, {"name": name, "team": team}).fetchone() + +def search_users(conn, pattern): + # callers pass the raw 'q' query-parameter straight in + sql = text("SELECT * FROM users WHERE bio LIKE '%" + pattern + "%'") + return conn.execute(sql).fetchall() + +def list_orders(conn, status): + return conn.execute( + text("SELECT * FROM orders WHERE status = :s"), {"s": status} + ).fetchall()` + +// satAppsecORMTrapTest: SQLAlchemy text() is a wrapper, not a sanitizer - +// the concatenation already fused attacker input into the SQL string +// before text() ever sees it. The near-miss is "ORM == safe". +func satAppsecORMTrapTest() testkit.Test { + prompt := `Flask application (the 'q' request parameter is attacker-controlled): + +` + "```python\n" + satAppsecORMFixture + "\n```" + ` + +Respond with only a JSON object, no other text: +{"injectable_function": "one of get_user | search_users | list_orders", + "text_wrapper_neutralizes_concatenation": true or false}` + return testkit.Test{ + ID: "sec-orm-interpolation-trap", + Category: "security", + Subcategory: "appsec", + Description: "Spot SQL injection inside a SQLAlchemy text() call built by string concatenation - the ORM wrapper neutralizes nothing once the string is already fused", + Prompt: prompt, + // ground truth: get_user and list_orders pass user values as + // bound parameters (:name/:team, :s) - the driver sends values + // out-of-band, so no injection. search_users builds the SQL + // string with + pattern + BEFORE text() wraps it: by the time + // text() sees it, the attacker's input is part of the statement + // (pattern = "x%' OR 1=1 -- " breaks out of the LIKE literal). + // text() marks a string as raw SQL for the ORM; it never + // parameterizes or escapes - text_wrapper_neutralizes_ + // concatenation is false. + Eval: eval.Mean( + eval.JSONField[string]("injectable_function", "search_users"), + eval.JSONField[bool]("text_wrapper_neutralizes_concatenation", false), + ), + } +} + +// satAppsecPOSTFixture shows a logout control where the visible anchor is +// a decoy - the actual request is the auto-submitted POST form. +const satAppsecPOSTFixture = ` + Log out + +
+ +
` + +// satAppsecPOSTTrapTest: the app switched a token-less state-changing +// endpoint from GET to POST as its whole CSRF mitigation. Tests whether +// the model knows POST alone is not a defense (cross-site
auto-submits with cookies), even when the page only ever +// submits it via JavaScript. +func satAppsecPOSTTrapTest() testkit.Test { + prompt := `A session-authenticated web app renders this logout control. The session +cookie is SameSite=None; Secure (sent with every cross-site request), and +there is no CSRF token anywhere in the request. + +` + "```\n" + satAppsecPOSTFixture + "\n```" + ` + +The developer's note: "I moved logout from GET to POST, so CSRF no longer +applies - only JavaScript on THIS page submits the form." + +Assume an attacker page on evil.example hosts its own form with +action="https://app.example/api/account/logout", method="post". The victim +visits evil.example but clicks nothing and types nothing there. + +Respond with only a JSON object, no other text: +{"attacker_form_reaches_logout": true or false, + "post_method_alone_prevents_csrf": true or false, + "attacker_needs_javascript": true or false}` + return testkit.Test{ + ID: "sec-csrf-post-trap", + Category: "security", + Subcategory: "appsec", + Description: "CSRF after a GET-to-POST switch without a token: cross-site form auto-submit carries the cookie, so the method is not the defense; the attacker needs one line of JS, not knowledge of the page's DOM", + Prompt: prompt, + // ground truth: SameSite=None means the browser attaches the + // session cookie to the attacker's cross-site POST, and there is + // no token, so the request is accepted as the victim -> + // attacker_form_reaches_logout true; the server authenticates by + // cookie, not by which page submitted the form. + // post_method_alone_prevents_csrf false: method only restricts + // bookmark/simple-link attacks; a hosted + // replays the state change. (With SameSite=Lax a cross-site POST + // would instead drop the cookie - the prompt pins SameSite=None + // so the answer is forced.) + // attacker_needs_javascript true: the prompt pins "clicks nothing", + // and without JavaScript a hosted form only submits when the victim + // presses its own submit button - a cross-site / cannot POST + // with the cookie context this request needs, and cannot carry a POST body. The silent attack + // requires form.submit(). + Eval: eval.Mean( + eval.JSONField[bool]("attacker_form_reaches_logout", true), + eval.JSONField[bool]("post_method_alone_prevents_csrf", false), + eval.JSONField[bool]("attacker_needs_javascript", true), + ), + } +} + +// satRotationFixture gives the three durations for the rotation +// deadline math. +const satRotationFixture = `Internal CA certificate expires: 2026-09-30 18:00 UTC +Worst realistic time to mint a replacement cert once rotation starts: 10 h +After the new cert is deployed, the reverse proxy's graceful-reload +window (old workers serve the old cert for up to this long after the +reload is issued): 6 h` + +// satSecretsRotationMathTest: latest safe start time = expiry minus the +// sum of both sequential windows. The near-miss is subtracting only the +// minting time and forgetting the propagation tail. +func satSecretsRotationMathTest() testkit.Test { + prompt := `Certificate rotation facts for an internal service: + +` + "```\n" + satRotationFixture + "\n```" + ` + +Respond with only a JSON object, no other text: +{"total_lead_time_hours": , "latest_start": "YYYY-MM-DD HH:MM", + "safe_if_started_2026_09_30_08_00": true or false} +(Use UTC throughout. latest_start is the latest time rotation can START +and still have the old certificate fully replaced everywhere before it +expires.)` + return testkit.Test{ + ID: "sec-rotation-window-math", + Category: "security", + Subcategory: "secrets", + Description: "Certificate rotation deadline math: lead time is minting plus propagation in series, and starting late leaves the old cert serving past expiry", + Prompt: prompt, + // ground truth: the two windows are sequential - a replacement + // can only start propagating once it exists: 10 h + 6 h = 16 h + // worst-case lead time. 2026-09-30 18:00 minus 16 h = + // 2026-09-30 02:00 latest start. Starting at 08:00 finishes at + // the worst at 08:00 + 16 h = 2026-10-01 00:00, six hours after + // the old cert expired -> not safe. + Eval: eval.Mean( + eval.JSONField[int]("total_lead_time_hours", 16), + eval.JSONField[string]("latest_start", "2026-09-30 02:00"), + eval.JSONField[bool]("safe_if_started_2026_09_30_08_00", false), + ), + } +} + +// satErrorPathFixture traces a password hash from a raise through an +// error handler into a log line, while the client only sees a generic +// response body. +const satErrorPathFixture = `1| def login(user, password): +2| row = db.get_user(user) +3| if not verify(password, row.hash): +4| raise AuthError(f"invalid credentials for user={user} " +5| f"hash={row.hash}") +6| return token_for(user) +7| +8| @app.errorhandler(AuthError) +9| def on_auth_error(exc): +10| app.logger.error(str(exc)) +11| return "login failed", 401` + +// satSecretsErrorPathTest: the secret (password hash) is interpolated at +// the raise, survives the handler's str(exc), and lands in the log even +// though the HTTP response is sanitized. The near-miss is "the client +// never sees it, so it's fine", and the reverse trap of flagging the +// attacker-supplied username as the secret. +func satSecretsErrorPathTest() testkit.Test { + prompt := `Flask login flow (numbers are source lines): + +` + "```\n" + satErrorPathFixture + "\n```" + ` + +row.hash is the stored password hash (sensitive). user comes straight +from the login request body. +Respond with only a JSON object, no other text: +{"hash_reaches_server_logs": true or false, "hash_visible_in_http_response": +true or false, "sensitive_value_in_message": "one of user | hash | none"}` + return testkit.Test{ + ID: "sec-error-path-leak-trap", + Category: "security", + Subcategory: "secrets", + Description: "Trace a password hash through raise -> errorhandler -> logger.error: the client response is sanitized but the log line is not, and the echoed user field is attacker-known", + Prompt: prompt, + // ground truth: the raise on lines 4-5 interpolates row.hash into + // the exception message; the handler on line 10 logs str(exc) - + // the full message - so the hash reaches the server logs: true. + // The handler returns only the literal "login failed" (line 11), + // never str(exc), so the hash is not visible in the HTTP + // response: false. Between user (attacker-supplied in the request + // body, already known to its owner) and hash (sensitive), the + // sensitive interpolated value is the hash. + Eval: eval.Mean( + eval.JSONField[bool]("hash_reaches_server_logs", true), + eval.JSONField[bool]("hash_visible_in_http_response", false), + eval.JSONField[string]("sensitive_value_in_message", "hash"), + ), + } +} diff --git a/internal/tests/saturation_test.go b/internal/tests/saturation_test.go new file mode 100644 index 0000000..a1a744f --- /dev/null +++ b/internal/tests/saturation_test.go @@ -0,0 +1,400 @@ +package tests + +import ( + "context" + "testing" +) + +// Each saturation test gets a table pinning: the full-credit answer, the +// trap answer (the near-miss the test exists to catch), and a garbage +// answer. Expected partial scores follow from eval.Mean's per-field mean. + +func TestSatDelegCapacityTrapTest_Eval(t *testing.T) { + tc := satDelegCapacityTrapTest() + + tests := []struct { + name string + response string + want float64 + }{ + { + name: "correct: main thread, names busy specialist", + response: `{"best_next_step":"handle_in_main_thread","specialist_this_task_belongs_to":"devops-debugger"}`, + want: 1, + }, + { + name: "trap: force-fits the generalist builder", + response: `{"best_next_step":"dispatch_code_writer","specialist_this_task_belongs_to":"devops-debugger"}`, + want: 0.5, + }, + { + name: "trap: orchestrator for a single task", + response: `{"best_next_step":"dispatch_orchestrator","specialist_this_task_belongs_to":"orchestrator"}`, + want: 0, + }, + { + name: "correct step but wrong owning specialist", + response: `{"best_next_step":"handle_in_main_thread","specialist_this_task_belongs_to":"code-writer"}`, + want: 0.5, + }, + { + name: "wait-it-out answer", + response: `{"best_next_step":"wait_for_specialist","specialist_this_task_belongs_to":"devops-debugger"}`, + want: 0.5, + }, + {name: "garbage", response: `not json`, want: 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Value != tt.want { + t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail) + } + }) + } +} + +func TestSatDelegNearMissRosterTest_Eval(t *testing.T) { + tc := satDelegNearMissRosterTest() + + tests := []struct { + name string + response string + want float64 + }{ + { + name: "all correct", + response: `{"task1":"docs-writer","task2":"web-researcher","task3":"release-manager","task4":"security-reviewer"}`, + want: 1, + }, + { + name: "trap: web-researcher for the repo guide, docs-writer for the pricing page", + response: `{"task1":"web-researcher","task2":"docs-writer","task3":"release-manager","task4":"security-reviewer"}`, + want: 0.5, + }, + { + name: "trap: docs-writer for release notes, code-reviewer for the audit", + response: `{"task1":"docs-writer","task2":"web-researcher","task3":"docs-writer","task4":"code-reviewer"}`, + want: 0.5, + }, + { + name: "all swapped to the distractors", + response: `{"task1":"web-researcher","task2":"docs-writer","task3":"docs-writer","task4":"code-reviewer"}`, + want: 0, + }, + { + name: "case-insensitive correct", + response: `{"task1":"Docs-Writer","task2":"Web-Researcher","task3":"Release-Manager","task4":"Security-Reviewer"}`, + want: 1, + }, + {name: "garbage", response: `["docs-writer"]`, want: 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Value != tt.want { + t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail) + } + }) + } +} + +func TestSatPGCountStarTrapTest_Eval(t *testing.T) { + tc := satPGCountStarTrapTest() + + tests := []struct { + name string + response string + want float64 + }{ + { + name: "correct", + response: `{"orders_in_eu": 4, "rows_the_join_query_returns": 6, "count_star_answers_the_question": false}`, + want: 1, + }, + { + name: "numeric strings accepted", + response: `{"orders_in_eu": "4", "rows_the_join_query_returns": "6", "count_star_answers_the_question": "false"}`, + want: 1, + }, + { + name: "trap: counts joined rows as orders", + response: `{"orders_in_eu": 6, "rows_the_join_query_returns": 6, "count_star_answers_the_question": true}`, + want: 0.3333333333333333, + }, + { + name: "trap: forgets the childless order drops out of the inner join", + response: `{"orders_in_eu": 4, "rows_the_join_query_returns": 7, "count_star_answers_the_question": false}`, + want: 0.6666666666666666, + }, + {name: "garbage", response: `4`, want: 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Value != tt.want { + t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail) + } + }) + } +} + +func TestSatPGTimeoutTrapTest_Eval(t *testing.T) { + tc := satPGTimeoutTrapTest() + + tests := []struct { + name string + response string + want float64 + }{ + { + name: "correct", + response: `{"session_a_killed_at":"11:00:45","session_a_rule":"idle_in_transaction_timeout","session_b_killed_at":"11:01:25","session_b_rule":"statement_timeout"}`, + want: 1, + }, + { + name: "trap: idle clock started at BEGIN", + response: `{"session_a_killed_at":"11:00:05","session_a_rule":"idle_in_transaction_timeout","session_b_killed_at":"11:01:25","session_b_rule":"statement_timeout"}`, + want: 0.75, + }, + { + name: "trap: timeouts swapped between sessions", + response: `{"session_a_killed_at":"11:00:30","session_a_rule":"statement_timeout","session_b_killed_at":"11:00:58","session_b_rule":"idle_in_transaction_timeout"}`, + want: 0, + }, + { + name: "two fields right", + response: `{"session_a_killed_at":"11:00:45","session_a_rule":"idle_in_transaction_timeout","session_b_killed_at":"11:01:00","session_b_rule":"not_killed"}`, + want: 0.5, + }, + {name: "garbage", response: `no json here`, want: 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Value != tt.want { + t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail) + } + }) + } +} + +func TestSatWebRobotsUAScopeTest_Eval(t *testing.T) { + tc := satWebRobotsUAScopeTest() + + tests := []struct { + name string + response string + want float64 + }{ + { + name: "correct: longest match wins for googlebot, group isolation for GoogleOther", + response: `{"googlebot_allowed": true, "googleother_allowed": false, "deciding_rule_for_googlebot": "allow"}`, + want: 1, + }, + { + name: "trap: GoogleOther's disallow applied to googlebot too", + response: `{"googlebot_allowed": false, "googleother_allowed": false, "deciding_rule_for_googlebot": "disallow"}`, + want: 0.3333333333333333, + }, + { + name: "trap: wildcard group treated as absent", + response: `{"googlebot_allowed": true, "googleother_allowed": true, "deciding_rule_for_googlebot": "allow"}`, + want: 0.6666666666666666, + }, + {name: "garbage", response: `allow`, want: 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Value != tt.want { + t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail) + } + }) + } +} + +func TestSatWebHreflangSelfTest_Eval(t *testing.T) { + tc := satWebHreflangSelfTest() + + tests := []struct { + name string + response string + want float64 + }{ + { + name: "correct", + response: `{"missing_self_reference_for":"en-US","self_reference_present":false,"canonical_counts_as_hreflang_self_reference":false,"x_default_is_a_language_tag":false}`, + want: 1, + }, + { + name: "tag casing normalized", + response: `{"missing_self_reference_for":"en-us","self_reference_present":false,"canonical_counts_as_hreflang_self_reference":false,"x_default_is_a_language_tag":false}`, + want: 1, + }, + { + name: "trap: canonical counted as the self-reference", + response: `{"missing_self_reference_for":"none","self_reference_present":true,"canonical_counts_as_hreflang_self_reference":true,"x_default_is_a_language_tag":false}`, + want: 0.25, + }, + { + name: "trap: x-default treated as a language", + response: `{"missing_self_reference_for":"en-US","self_reference_present":false,"canonical_counts_as_hreflang_self_reference":false,"x_default_is_a_language_tag":true}`, + want: 0.75, + }, + {name: "garbage", response: `en-US`, want: 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Value != tt.want { + t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail) + } + }) + } +} + +func TestSatAppsecORMTrapTest_Eval(t *testing.T) { + tc := satAppsecORMTrapTest() + + tests := []struct { + name string + response string + want float64 + }{ + { + name: "correct", + response: `{"injectable_function": "search_users", "text_wrapper_neutralizes_concatenation": false}`, + want: 1, + }, + { + name: "trap: ORM assumed safe everywhere", + response: `{"injectable_function": "none", "text_wrapper_neutralizes_concatenation": true}`, + want: 0, + }, + { + name: "right function, wrong mechanism", + response: `{"injectable_function": "search_users", "text_wrapper_neutralizes_concatenation": true}`, + want: 0.5, + }, + {name: "garbage", response: `search_users`, want: 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Value != tt.want { + t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail) + } + }) + } +} + +func TestSatAppsecPOSTTrapTest_Eval(t *testing.T) { + tc := satAppsecPOSTTrapTest() + + tests := []struct { + name string + response string + want float64 + }{ + { + name: "correct", + response: `{"attacker_form_reaches_logout": true, "post_method_alone_prevents_csrf": false, "attacker_needs_javascript": true}`, + want: 1, + }, + { + name: "trap: POST assumed to prevent CSRF", + response: `{"attacker_form_reaches_logout": false, "post_method_alone_prevents_csrf": true, "attacker_needs_javascript": true}`, + want: 0.3333333333333333, + }, + { + name: "trap: attacker blocked for lacking the page's JS", + response: `{"attacker_form_reaches_logout": false, "post_method_alone_prevents_csrf": false, "attacker_needs_javascript": false}`, + want: 0.3333333333333333, + }, + {name: "garbage", response: `{"a":1}`, want: 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Value != tt.want { + t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail) + } + }) + } +} + +func TestSatSecretsRotationMathTest_Eval(t *testing.T) { + tc := satSecretsRotationMathTest() + + tests := []struct { + name string + response string + want float64 + }{ + { + name: "correct", + response: `{"total_lead_time_hours": 16, "latest_start": "2026-09-30 02:00", "safe_if_started_2026_09_30_08_00": false}`, + want: 1, + }, + { + name: "trap: minting time only, propagation forgotten", + response: `{"total_lead_time_hours": 10, "latest_start": "2026-09-30 08:00", "safe_if_started_2026_09_30_08_00": true}`, + want: 0, + }, + { + name: "sum right, start-time arithmetic wrong", + response: `{"total_lead_time_hours": 16, "latest_start": "2026-09-30 02:00", "safe_if_started_2026_09_30_08_00": true}`, + want: 0.6666666666666666, + }, + {name: "garbage", response: `16 hours`, want: 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Value != tt.want { + t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail) + } + }) + } +} + +func TestSatSecretsErrorPathTest_Eval(t *testing.T) { + tc := satSecretsErrorPathTest() + + tests := []struct { + name string + response string + want float64 + }{ + { + name: "correct", + response: `{"hash_reaches_server_logs": true, "hash_visible_in_http_response": false, "sensitive_value_in_message": "hash"}`, + want: 1, + }, + { + name: "trap: sanitized response assumed to mean no leak", + response: `{"hash_reaches_server_logs": false, "hash_visible_in_http_response": false, "sensitive_value_in_message": "hash"}`, + want: 0.6666666666666666, + }, + { + name: "trap: attacker-known user flagged instead of the hash", + response: `{"hash_reaches_server_logs": true, "hash_visible_in_http_response": false, "sensitive_value_in_message": "user"}`, + want: 0.6666666666666666, + }, + { + name: "trap: leak judged through the log AND the response", + response: `{"hash_reaches_server_logs": true, "hash_visible_in_http_response": true, "sensitive_value_in_message": "hash"}`, + want: 0.6666666666666666, + }, + {name: "garbage", response: `hash`, want: 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tc.Eval.Evaluate(context.Background(), tt.response) + if got.Value != tt.want { + t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail) + } + }) + } +} diff --git a/internal/tests/security_crypto.go b/internal/tests/security_crypto.go index a325074..93d2a15 100644 --- a/internal/tests/security_crypto.go +++ b/internal/tests/security_crypto.go @@ -23,29 +23,49 @@ func registerSecurityCryptoTests(r *testkit.Registry) { r.Register(secHashVsEncryptPIITest()) } -// secWeakPasswordHashPattern matches a mention of a fast, general-purpose -// hash (MD5, plain SHA-1/256/512) - the ones that must never be used -// directly for password storage because they lack a tunable work factor. -var secWeakPasswordHashPattern = regexp.MustCompile(`(?i)\b(md5|sha-?1|sha-?256|sha-?512)\b`) +// secWeakHashEndorsementPattern matches a weak-hash mention in an +// endorsement-shaped position — a verb recommending it ("hash with MD5", +// "just hash the password with SHA-256"), or a positive predicate about it +// ("SHA-256 is a fine choice") — rather than every mention. The guard then +// still requires a negation cue near such a mention (see +// secNoUnnegatedMention). +// +// Triggering on any mention is too strict: a fully correct live answer +// (probe, 2026-08-29) lists the discouraged hashes as bare bullets under a +// negated heading ("Do **not** use fast general-purpose ... including:") — +// the cue sits on the heading, but the negation window stops at the line +// break and the heading is more than a window-width away, so each bullet +// read as an unnegated endorsement and cost the model half the score. +// A bare bullet names, it does not recommend — the predicate branch still +// catches an endorsed bullet ("- **MD5** is fine for passwords"). +var secWeakHashEndorsementPattern = regexp.MustCompile("(?i)(?:" + + // a verb recommending it: "hash with MD5", "store passwords hashed with + // MD5", "just hash the password with SHA-256" — the filler loop only + // bridges recommendation boilerplate, never another algorithm name, so + // "prefer argon2 over md5" (a correct answer) is not an endorsement. + "(?:hash(?:es|ed|ing)?|store|stores|stored|crypt|encrypt|use|using|used|recommend|recommends|prefer|prefers|should(?:\\s+(?:also|just))?|want)" + + "\\s+(?:(?:it|its|the|a|an|password|passwords|user|users|hash|hashed|with|using|as|via|by|plain|raw|fast|for|directly|simply|just|them|to)[ \\t]*\\n?[ \\t]*(?:-[ \\t]*)?){0,6}(?:md5|sha-?1|sha-?256|sha-?512)" + + `|(?:^|[^A-Za-z0-9_-])(?:md5|sha-?1|sha-?256|sha-?512)[^.;\n]{0,25}?\b(?:is|are|was|were|seems?|looks?|works?|would\s+be|should\s+be)\s+(?:a\s+|just\s+|still\s+|totally\s+|actually\s+)?(?:fine|good|great|safe|ok|okay|acceptable|perfect|sufficient|enough|best|better|the\s+(?:right|best)\s+choice)` + + ")") // secWeakHashNamedAndNegated scores full credit only when the response BOTH -// safely negates every mention of a weak/fast general-purpose hash (or -// never mentions one) AND actually names at least one of them by name -// (md5/sha-1/sha-256/sha-512) - C11's fix folded into a single AND rather -// than an independent extra ContainsAny term, so a wrong answer that +// avoids endorsing any weak/fast general-purpose hash with no nearby +// negation (or never mentions one) AND actually names at least one of them +// by name (md5/sha-1/sha-256/sha-512) - C11's fix folded into a single AND +// rather than an independent extra ContainsAny term, so a wrong answer that // recommends SHA-256 outright cannot earn partial credit merely for having // typed "SHA-256" while doing so. func secWeakHashNamedAndNegated() eval.Evaluator { named := eval.ContainsAny("md5", "sha-1", "sha1", "sha-256", "sha256") - negated := secNoUnnegatedMention(secWeakPasswordHashPattern) + negated := secNoUnnegatedMention(secWeakHashEndorsementPattern) return eval.EvaluatorFunc(func(ctx context.Context, response string) eval.Score { if n := negated.Evaluate(ctx, response); n.Value != 1 { - return eval.Score{Value: 0, Detail: "unnegated mention of a discouraged hash: " + n.Detail} + return eval.Score{Value: 0, Detail: "endorses a discouraged hash without negating it: " + n.Detail} } if m := named.Evaluate(ctx, response); m.Value != 1 { return eval.Score{Value: 0, Detail: "never names a weak hash by name"} } - return eval.Score{Value: 1, Detail: "a weak hash is named and every mention is safely negated"} + return eval.Score{Value: 1, Detail: "a weak hash is named and none is endorsed without negation"} }) } diff --git a/internal/tests/security_crypto_test.go b/internal/tests/security_crypto_test.go index eb01531..cd54bc1 100644 --- a/internal/tests/security_crypto_test.go +++ b/internal/tests/security_crypto_test.go @@ -28,6 +28,20 @@ func TestSecPasswordHashChoiceTest_Eval(t *testing.T) { response: "bcrypt is fine. Don't use SHA-256 alone, it lacks a work factor.", want: 1, }, + { + // Regression from the live probe (2026-08-29): a fully correct + // answer names the discouraged hashes as bullets under a negated + // heading ("Do **not** use ... including:"). The heading ends in + // ":" - not a clause boundary - so its cue must reach every bullet. + name: "correct probe answer: discouraged hashes as bullets under a negated heading", + response: "Use Argon2id (or bcrypt with cost >= 12) to store passwords.\n\n" + + "Do **not** use fast general-purpose cryptographic hashes for storing passwords, including:\n" + + "- **MD5**\n" + + "- **SHA-1**\n" + + "- **SHA-256**\n" + + "- **SHA-512**", + want: 1, + }, { name: "wrong: recommends SHA-256 unnegated", response: "Just hash the password with SHA-256 before storing it.",