mirror of
https://github.com/lukaszraczylo/llm-testbench.git
synced 2026-10-14 00:57:45 +00:00
feat: LLM accuracy testing framework with 17-test catalog
Framework runs a category/subcategory test catalog against multiple LLMs through an OpenAI-compatible endpoint. Deterministic evaluators score each response. Reports render as table, markdown, or JSON.
This commit is contained in:
@@ -0,0 +1,27 @@
|
||||
package eval
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// equalsEval scores full credit on a trimmed, case-insensitive exact match.
|
||||
type equalsEval struct {
|
||||
want string
|
||||
}
|
||||
|
||||
// Equals returns an Evaluator awarding full credit when the response,
|
||||
// trimmed and lower-cased, equals want (also trimmed and lower-cased).
|
||||
func Equals(want string) Evaluator {
|
||||
return equalsEval{want: want}
|
||||
}
|
||||
|
||||
func (e equalsEval) Evaluate(_ context.Context, response string) Score {
|
||||
got := strings.ToLower(strings.TrimSpace(response))
|
||||
want := strings.ToLower(strings.TrimSpace(e.want))
|
||||
if got == want {
|
||||
return Score{Value: 1, Detail: fmt.Sprintf("equals %q", e.want)}
|
||||
}
|
||||
return Score{Value: 0, Detail: fmt.Sprintf("got %q, want %q", strings.TrimSpace(response), e.want)}
|
||||
}
|
||||
Reference in New Issue
Block a user