mirror of
https://github.com/lukaszraczylo/llm-testbench.git
synced 2026-10-12 00:45:19 +00:00
lukaszraczylo
1d51d4cbbd
Framework runs a category/subcategory test catalog against multiple LLMs through an OpenAI-compatible endpoint. Deterministic evaluators score each response. Reports render as table, markdown, or JSON.
86 lines
2.6 KiB
Go
86 lines
2.6 KiB
Go
package eval
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"strings"
|
|
)
|
|
|
|
// containsAll scores the fraction of substrings present in the response,
|
|
// case-insensitive. Full credit only when every substring matches.
|
|
type containsAll struct {
|
|
substrings []string
|
|
}
|
|
|
|
// ContainsAll returns an Evaluator that awards matched/total credit for how
|
|
// many of substrings appear (case-insensitive) in the response.
|
|
func ContainsAll(substrings ...string) Evaluator {
|
|
return containsAll{substrings: substrings}
|
|
}
|
|
|
|
func (c containsAll) Evaluate(_ context.Context, response string) Score {
|
|
if len(c.substrings) == 0 {
|
|
return Score{Value: 1, Detail: "no substrings required"}
|
|
}
|
|
lower := strings.ToLower(response)
|
|
matched := 0
|
|
var missing []string
|
|
for _, s := range c.substrings {
|
|
if strings.Contains(lower, strings.ToLower(s)) {
|
|
matched++
|
|
} else {
|
|
missing = append(missing, s)
|
|
}
|
|
}
|
|
value := float64(matched) / float64(len(c.substrings))
|
|
detail := fmt.Sprintf("matched %d/%d", matched, len(c.substrings))
|
|
if len(missing) > 0 {
|
|
detail += fmt.Sprintf("; missing: %s", strings.Join(missing, ", "))
|
|
}
|
|
return Score{Value: value, Detail: detail}
|
|
}
|
|
|
|
// containsAny awards full credit if at least one alternative matches.
|
|
type containsAny struct {
|
|
substrings []string
|
|
}
|
|
|
|
// ContainsAny returns an Evaluator that awards full credit if any substring
|
|
// (case-insensitive) appears in the response, zero otherwise.
|
|
func ContainsAny(substrings ...string) Evaluator {
|
|
return containsAny{substrings: substrings}
|
|
}
|
|
|
|
func (c containsAny) Evaluate(_ context.Context, response string) Score {
|
|
lower := strings.ToLower(response)
|
|
for _, s := range c.substrings {
|
|
if strings.Contains(lower, strings.ToLower(s)) {
|
|
return Score{Value: 1, Detail: fmt.Sprintf("matched %q", s)}
|
|
}
|
|
}
|
|
return Score{Value: 0, Detail: fmt.Sprintf("none of %v matched", c.substrings)}
|
|
}
|
|
|
|
// notContains is a guard: full credit unless a forbidden substring appears.
|
|
type notContains struct {
|
|
substrings []string
|
|
}
|
|
|
|
// NotContains returns a composable guard Evaluator: full credit unless the
|
|
// response contains any of substrings (case-insensitive), in which case it
|
|
// scores zero. Useful to penalize unsafe/undesired suggestions (e.g.
|
|
// "kubectl edit").
|
|
func NotContains(substrings ...string) Evaluator {
|
|
return notContains{substrings: substrings}
|
|
}
|
|
|
|
func (c notContains) Evaluate(_ context.Context, response string) Score {
|
|
lower := strings.ToLower(response)
|
|
for _, s := range c.substrings {
|
|
if strings.Contains(lower, strings.ToLower(s)) {
|
|
return Score{Value: 0, Detail: fmt.Sprintf("forbidden %q present", s)}
|
|
}
|
|
}
|
|
return Score{Value: 1, Detail: "no forbidden substrings present"}
|
|
}
|