mirror of
https://github.com/lukaszraczylo/llm-testbench.git
synced 2026-10-14 00:57:45 +00:00
feat: LLM accuracy testing framework with 17-test catalog
Framework runs a category/subcategory test catalog against multiple LLMs through an OpenAI-compatible endpoint. Deterministic evaluators score each response. Reports render as table, markdown, or JSON.
This commit is contained in:
@@ -0,0 +1,60 @@
|
||||
package report
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"io"
|
||||
|
||||
"github.com/lukaszraczylo/llm-testbench/internal/runner"
|
||||
"github.com/lukaszraczylo/llm-testbench/internal/testkit"
|
||||
)
|
||||
|
||||
// jsonResult is the JSON-serializable projection of a runner.Result, joined
|
||||
// with its test's category/subcategory for self-contained output.
|
||||
type jsonResult struct {
|
||||
Model string `json:"model"`
|
||||
TestID string `json:"test_id"`
|
||||
Category string `json:"category,omitempty"`
|
||||
Subcategory string `json:"subcategory,omitempty"`
|
||||
Detail string `json:"detail,omitempty"`
|
||||
Error string `json:"error,omitempty"`
|
||||
Score float64 `json:"score"`
|
||||
LatencyMS int64 `json:"latency_ms"`
|
||||
PromptTokens int `json:"prompt_tokens"`
|
||||
CompletionTokens int `json:"completion_tokens"`
|
||||
Tokens int `json:"tokens"`
|
||||
Skipped bool `json:"skipped"`
|
||||
}
|
||||
|
||||
// renderJSON dumps raw results as a JSON array, joined with test metadata.
|
||||
func renderJSON(w io.Writer, tests []testkit.Test, results []runner.Result) error {
|
||||
testByID := make(map[string]testkit.Test, len(tests))
|
||||
for _, t := range tests {
|
||||
testByID[t.ID] = t
|
||||
}
|
||||
|
||||
out := make([]jsonResult, 0, len(results))
|
||||
for _, r := range results {
|
||||
t := testByID[r.TestID]
|
||||
jr := jsonResult{
|
||||
Model: r.Model,
|
||||
TestID: r.TestID,
|
||||
Category: t.Category,
|
||||
Subcategory: t.Subcategory,
|
||||
Score: r.Score.Value,
|
||||
Detail: r.Score.Detail,
|
||||
Skipped: r.Score.Skipped,
|
||||
LatencyMS: r.Latency.Milliseconds(),
|
||||
PromptTokens: r.PromptTokens,
|
||||
CompletionTokens: r.CompletionTokens,
|
||||
Tokens: r.TotalTokens(),
|
||||
}
|
||||
if r.Err != nil {
|
||||
jr.Error = r.Err.Error()
|
||||
}
|
||||
out = append(out, jr)
|
||||
}
|
||||
|
||||
enc := json.NewEncoder(w)
|
||||
enc.SetIndent("", " ")
|
||||
return enc.Encode(out)
|
||||
}
|
||||
Reference in New Issue
Block a user