mirror of
https://github.com/lukaszraczylo/llm-testbench.git
synced 2026-10-19 01:27:32 +00:00
feat: LLM accuracy testing framework with 17-test catalog
Framework runs a category/subcategory test catalog against multiple LLMs through an OpenAI-compatible endpoint. Deterministic evaluators score each response. Reports render as table, markdown, or JSON.
This commit is contained in:
@@ -0,0 +1,25 @@
|
||||
version: "2"
|
||||
|
||||
linters:
|
||||
default: standard
|
||||
enable:
|
||||
- gosec
|
||||
settings:
|
||||
govet:
|
||||
enable:
|
||||
- fieldalignment
|
||||
gosec:
|
||||
# Keep gosec's default rule set; false positives are annotated inline
|
||||
# with "#nosec Gxxx -- reason" at the flagged line (see config.go,
|
||||
# internal/eval/exec.go, internal/tests/c_test.go) rather than
|
||||
# globally excluded here.
|
||||
excludes: []
|
||||
|
||||
issues:
|
||||
# Exec evaluators (internal/eval/exec.go) legitimately build and run
|
||||
# go/python3/cc commands against model-generated source under a locked-down
|
||||
# temp dir + minimal env allowlist (see minimalExecEnv); that is flagged by
|
||||
# gosec G204 by design and is annotated at each call site, not excluded
|
||||
# here.
|
||||
max-issues-per-linter: 0
|
||||
max-same-issues: 0
|
||||
Reference in New Issue
Block a user