mirror of
https://github.com/lukaszraczylo/llm-testbench.git
synced 2026-10-12 00:45:19 +00:00
lukaszraczylo
ace50757e4
Reasoning models burn thousands of completion tokens before the answer. Small per-test caps caused silent truncation and zero scores. The runner now floors max_tokens at the config default (12000). The client parses finish_reason. Reports mark truncated cells and the JSON format carries response text for debugging. A stderr progress reporter prints one line per completed request; --quiet disables it.
112 lines
3.3 KiB
Go
112 lines
3.3 KiB
Go
package eval
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"strings"
|
|
)
|
|
|
|
// decodeStringArray extracts response's JSON value via ExtractJSON (the
|
|
// first ```json fenced block if present, otherwise the last balanced JSON
|
|
// value in the text) and decodes it as a []string. Elements that are not
|
|
// strings cause an error.
|
|
func decodeStringArray(response string) ([]string, error) {
|
|
raw, err := ExtractJSON(response)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
var arr []string
|
|
if err := json.Unmarshal([]byte(raw), &arr); err != nil {
|
|
return nil, fmt.Errorf("invalid JSON string array: %w", err)
|
|
}
|
|
return arr, nil
|
|
}
|
|
|
|
// jsonStringSetEval scores a JSON array of strings as an unordered set,
|
|
// case-insensitive and whitespace-trimmed.
|
|
type jsonStringSetEval struct {
|
|
want []string
|
|
}
|
|
|
|
// JSONStringSet returns an Evaluator that parses response's JSON array (see
|
|
// decodeStringArray/ExtractJSON) as a set of strings (case-insensitive,
|
|
// trimmed) and compares it to want, order-independent. Score is the
|
|
// Jaccard-style overlap
|
|
// |intersection| / max(len(want), len(got)), so both missing and extraneous
|
|
// entries cost credit; full credit requires an exact set match.
|
|
func JSONStringSet(want []string) Evaluator {
|
|
return jsonStringSetEval{want: want}
|
|
}
|
|
|
|
func normalizeSet(items []string) map[string]struct{} {
|
|
set := make(map[string]struct{}, len(items))
|
|
for _, s := range items {
|
|
set[strings.ToLower(strings.TrimSpace(s))] = struct{}{}
|
|
}
|
|
return set
|
|
}
|
|
|
|
func (j jsonStringSetEval) Evaluate(_ context.Context, response string) Score {
|
|
got, err := decodeStringArray(response)
|
|
if err != nil {
|
|
return Score{Value: 0, Detail: err.Error()}
|
|
}
|
|
|
|
wantSet := normalizeSet(j.want)
|
|
gotSet := normalizeSet(got)
|
|
|
|
matched := 0
|
|
for k := range gotSet {
|
|
if _, ok := wantSet[k]; ok {
|
|
matched++
|
|
}
|
|
}
|
|
|
|
denom := len(wantSet)
|
|
if len(gotSet) > denom {
|
|
denom = len(gotSet)
|
|
}
|
|
if denom == 0 {
|
|
return Score{Value: 1, Detail: "both sets empty"}
|
|
}
|
|
|
|
value := float64(matched) / float64(denom)
|
|
return Score{
|
|
Value: value,
|
|
Detail: fmt.Sprintf("matched %d/%d (got %v, want %v)", matched, denom, got, j.want),
|
|
}
|
|
}
|
|
|
|
// jsonStringArrayEqualsEval scores full credit only on an exact, in-order
|
|
// match of a JSON string array.
|
|
type jsonStringArrayEqualsEval struct {
|
|
want []string
|
|
}
|
|
|
|
// JSONStringArrayEquals returns an Evaluator that parses response's JSON
|
|
// array (see decodeStringArray/ExtractJSON) as []string and awards full
|
|
// credit only when it exactly equals want element-for-element, in order
|
|
// (case-insensitive, trimmed). Any mismatch, including reordering, scores
|
|
// zero.
|
|
func JSONStringArrayEquals(want []string) Evaluator {
|
|
return jsonStringArrayEqualsEval{want: want}
|
|
}
|
|
|
|
func (j jsonStringArrayEqualsEval) Evaluate(_ context.Context, response string) Score {
|
|
got, err := decodeStringArray(response)
|
|
if err != nil {
|
|
return Score{Value: 0, Detail: err.Error()}
|
|
}
|
|
|
|
if len(got) != len(j.want) {
|
|
return Score{Value: 0, Detail: fmt.Sprintf("length %d, want %d (got %v, want %v)", len(got), len(j.want), got, j.want)}
|
|
}
|
|
for i := range got {
|
|
if !strings.EqualFold(strings.TrimSpace(got[i]), strings.TrimSpace(j.want[i])) {
|
|
return Score{Value: 0, Detail: fmt.Sprintf("index %d: got %q, want %q (got %v, want %v)", i, got[i], j.want[i], got, j.want)}
|
|
}
|
|
}
|
|
return Score{Value: 1, Detail: fmt.Sprintf("exact match: %v", got)}
|
|
}
|