mirror of
https://github.com/lukaszraczylo/llm-testbench.git
synced 2026-10-15 01:07:26 +00:00
feat: LLM accuracy testing framework with 17-test catalog
Framework runs a category/subcategory test catalog against multiple LLMs through an OpenAI-compatible endpoint. Deterministic evaluators score each response. Reports render as table, markdown, or JSON.
This commit is contained in:
@@ -0,0 +1,29 @@
|
||||
package eval
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"regexp"
|
||||
)
|
||||
|
||||
// regexEval scores full credit when pattern matches the response.
|
||||
type regexEval struct {
|
||||
pattern *regexp.Regexp
|
||||
raw string
|
||||
}
|
||||
|
||||
// Regex returns an Evaluator awarding full credit when pattern matches
|
||||
// anywhere in the response, zero otherwise. pattern must be a valid RE2
|
||||
// expression; it panics on an invalid pattern since evaluators are built
|
||||
// once at catalog registration time, not per-request.
|
||||
func Regex(pattern string) Evaluator {
|
||||
re := regexp.MustCompile(pattern)
|
||||
return regexEval{pattern: re, raw: pattern}
|
||||
}
|
||||
|
||||
func (r regexEval) Evaluate(_ context.Context, response string) Score {
|
||||
if r.pattern.MatchString(response) {
|
||||
return Score{Value: 1, Detail: fmt.Sprintf("matched /%s/", r.raw)}
|
||||
}
|
||||
return Score{Value: 0, Detail: fmt.Sprintf("no match for /%s/", r.raw)}
|
||||
}
|
||||
Reference in New Issue
Block a user