mirror of
https://github.com/lukaszraczylo/llm-testbench.git
synced 2026-10-13 00:48:29 +00:00
fix(catalog): unify negation guard and harden round-2 evaluators
A shared NoUnnegatedMention primitive replaces four per-category implementations, with a bidirectional clause-scoped window and an adjectival cue set. The docker layer fixture drops WORKDIR, which creates a real layer and made the pinned answer wrong. The replica failover test now forces a JSON verdict so a negated answer cannot score. Remaining exact-answer tests migrate to ExactToken. Ordering prompts shuffle their candidate lists so echoing scores zero.
This commit is contained in:
@@ -89,9 +89,9 @@ func jsonValueAs[T comparable](v any) (T, error) {
|
||||
}
|
||||
return any(s).(T), nil
|
||||
case bool:
|
||||
b, ok := v.(bool)
|
||||
if !ok {
|
||||
return zero, fmt.Errorf("expected bool, got %T (%v)", v, v)
|
||||
b, err := asBool(v)
|
||||
if err != nil {
|
||||
return zero, err
|
||||
}
|
||||
return any(b).(T), nil
|
||||
case float64:
|
||||
@@ -134,6 +134,30 @@ func asFloat64(v any) (float64, error) {
|
||||
}
|
||||
}
|
||||
|
||||
// asBool coerces a decoded JSON value to bool. A string is accepted too
|
||||
// (DC2, mirroring asFloat64's B9 numeric-string coercion), case-insensitive
|
||||
// and trimmed: a model that answers {"guaranteed_visible":"false"} instead
|
||||
// of {"guaranteed_visible":false} - both are the same value, and the
|
||||
// prompt asked for a JSON field, not a JSON type - must not score 0 for a
|
||||
// mechanical typing difference. Any other string fails.
|
||||
func asBool(v any) (bool, error) {
|
||||
switch b := v.(type) {
|
||||
case bool:
|
||||
return b, nil
|
||||
case string:
|
||||
switch strings.ToLower(strings.TrimSpace(b)) {
|
||||
case "true":
|
||||
return true, nil
|
||||
case "false":
|
||||
return false, nil
|
||||
default:
|
||||
return false, fmt.Errorf("expected bool, got non-boolean string %q", b)
|
||||
}
|
||||
default:
|
||||
return false, fmt.Errorf("expected bool, got %T (%v)", v, v)
|
||||
}
|
||||
}
|
||||
|
||||
// jsonFieldEval extracts one field by path from the response's JSON value
|
||||
// (see ExtractJSON: the first ```json fenced block if present, otherwise
|
||||
// the last balanced JSON value in the response) and compares it to want.
|
||||
@@ -193,7 +217,17 @@ func (j jsonFieldEval[T]) Evaluate(_ context.Context, response string) Score {
|
||||
func equalJSONValue[T comparable](got, want T) bool {
|
||||
if gs, ok := any(got).(string); ok {
|
||||
ws := any(want).(string)
|
||||
return strings.EqualFold(strings.TrimSpace(gs), strings.TrimSpace(ws))
|
||||
return strings.EqualFold(enumNormalize(gs), enumNormalize(ws))
|
||||
}
|
||||
return got == want
|
||||
}
|
||||
|
||||
// enumNormalize folds hyphens to spaces and collapses repeated whitespace
|
||||
// (DC4), so a closed-vocabulary enum answer like "non-repeatable-read" and
|
||||
// "non repeatable read" compare equal: a model choosing its own spelling
|
||||
// of a multi-word enum token should not be marked wrong for a formatting
|
||||
// difference the prompt's closed vocabulary did not unambiguously specify.
|
||||
func enumNormalize(s string) string {
|
||||
s = strings.ReplaceAll(s, "-", " ")
|
||||
return strings.Join(strings.Fields(s), " ")
|
||||
}
|
||||
|
||||
@@ -127,6 +127,58 @@ func TestJSONField_Float_NumericStringCoercion(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestJSONField_Bool_StringCoercion is the DC2 regression, mirroring B9's
|
||||
// numeric-string coercion: a model answering {"guaranteed_visible":"false"}
|
||||
// instead of {"guaranteed_visible":false} must still score 1.
|
||||
func TestJSONField_Bool_StringCoercion(t *testing.T) {
|
||||
e := JSONField("guaranteed_visible", false)
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{"bool string coerces (lowercase)", `{"guaranteed_visible":"false"}`, 1},
|
||||
{"bool string coerces with whitespace", `{"guaranteed_visible":" false "}`, 1},
|
||||
{"bool string coerces, different case", `{"guaranteed_visible":"False"}`, 1},
|
||||
{"wrong bool string", `{"guaranteed_visible":"true"}`, 0},
|
||||
{"non-boolean string does not coerce", `{"guaranteed_visible":"nope"}`, 0},
|
||||
{"native bool still works", `{"guaranteed_visible":false}`, 1},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := e.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestJSONField_EnumHyphenSpaceFold is the DC4 regression: a multi-word
|
||||
// enum answer spelled with spaces instead of hyphens (or vice versa) must
|
||||
// still score 1.
|
||||
func TestJSONField_EnumHyphenSpaceFold(t *testing.T) {
|
||||
e := JSONField("anomaly", "non-repeatable-read")
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{"exact hyphenated form", `{"anomaly":"non-repeatable-read"}`, 1},
|
||||
{"space-separated form folds equal (DC4 bug probe)", `{"anomaly":"non repeatable read"}`, 1},
|
||||
{"mixed hyphen/space form folds equal (DC4 bug probe)", `{"anomaly":"non repeatable-read"}`, 1},
|
||||
{"a genuinely different enum value still fails", `{"anomaly":"dirty-read"}`, 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := e.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestJSONField_NestedPath(t *testing.T) {
|
||||
e := JSONField("task1", "search_web")
|
||||
got := e.Evaluate(context.Background(), `{"task1":"search_web","task2":"read_file"}`)
|
||||
|
||||
@@ -0,0 +1,145 @@
|
||||
package eval
|
||||
|
||||
import (
|
||||
"context"
|
||||
"regexp"
|
||||
)
|
||||
|
||||
// negationCueAlternation is the shared union of cue words/phrases that turn
|
||||
// a mention of a discouraged term into a warning against it rather than an
|
||||
// endorsement of it. It is the union of four independently-accumulated
|
||||
// per-category cue lists (kubernetes.go's noLiveKubectlMutation,
|
||||
// security.go's secNoUnnegatedMention, databases_redis.go's
|
||||
// dbNoBareKeysInProd, delivery_git.go's delNoBareForcePush), extended with
|
||||
// adjectival negation forms a good answer commonly uses instead of an
|
||||
// imperative "don't/never" cue - e.g. "MD5 is unsuitable for password
|
||||
// storage" or "SHA-256 is too fast to be safe here" carry no "don't"/"never"
|
||||
// at all, only an adjective describing why the term is discouraged (D5, C2).
|
||||
const negationCueAlternation = `don'?t|do not|never|avoid|not|cannot|can'?t|without|no need|` +
|
||||
`shouldn'?t|should not|must not|isn'?t|wasn'?t|danger|dangerous|` +
|
||||
`unsuitable|inappropriate|unsafe|insecure|weak|broken|deprecated|obsolete|` +
|
||||
`too fast|predictable|unacceptable|blocks|wrong|problem`
|
||||
|
||||
// negationCuePattern matches any cue word/phrase in negationCueAlternation,
|
||||
// case-insensitively, as a whole word/phrase (not a substring of an
|
||||
// unrelated longer word).
|
||||
var negationCuePattern = regexp.MustCompile(`(?i)\b(?:` + negationCueAlternation + `)\b`)
|
||||
|
||||
// negationBareNoColonPattern matches a bare "no:" cue (e.g. "no: MD5 lacks a
|
||||
// work factor"), kept as its own pattern since a trailing \b after the
|
||||
// punctuation mark ":" does not reliably assert what a word-boundary
|
||||
// alternation needs.
|
||||
var negationBareNoColonPattern = regexp.MustCompile(`(?i)\bno\s*:`)
|
||||
|
||||
// negationDirectionalCueAlternation lists cues that are only a genuine
|
||||
// negation of a term mentioned AFTER them, never of a term mentioned
|
||||
// BEFORE them: in "use bcrypt instead of MD5", "instead of" negates MD5
|
||||
// (which follows it), but in "use MD5 instead of proper hashing", the same
|
||||
// cue phrase follows MD5 while MD5 is still being recommended - checking
|
||||
// "instead of"/"rather than" on both sides of a match would wrongly treat
|
||||
// that second sentence as a safe, negated mention too (C2, D5).
|
||||
const negationDirectionalCueAlternation = `instead of|rather than`
|
||||
|
||||
// negationDirectionalCuePattern matches negationDirectionalCueAlternation.
|
||||
// Checked only in the window immediately BEFORE a match, never after.
|
||||
var negationDirectionalCuePattern = regexp.MustCompile(`(?i)\b(?:` + negationDirectionalCueAlternation + `)\b`)
|
||||
|
||||
// clauseWindowBounds computes the [lo,hi) bidirectional search window
|
||||
// around a match [start,end) in response: outward up to maxChars
|
||||
// characters in each direction, but stopped early at the nearest clause
|
||||
// boundary - a '.', ';', or a blank-line paragraph gap - whichever comes
|
||||
// first, so a negation cue from an unrelated adjacent clause or sentence
|
||||
// can never count as negating a genuinely unqualified endorsement
|
||||
// elsewhere in a longer response (D5, C2). This is a known, accepted
|
||||
// limitation for sentence-final abbreviations (e.g. "e.g.", "etc.") that
|
||||
// happen to sit inside the window - none of this framework's fixtures
|
||||
// place a negation cue immediately after one.
|
||||
func clauseWindowBounds(response string, start, end, maxChars int) (lo, hi int) {
|
||||
lo = start - maxChars
|
||||
if lo < 0 {
|
||||
lo = 0
|
||||
}
|
||||
for i := start - 1; i >= lo; i-- {
|
||||
c := response[i]
|
||||
if c == '.' || c == ';' {
|
||||
lo = i + 1
|
||||
break
|
||||
}
|
||||
if c == '\n' && i > 0 && response[i-1] == '\n' {
|
||||
lo = i + 1
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
hi = end + maxChars
|
||||
if hi > len(response) {
|
||||
hi = len(response)
|
||||
}
|
||||
for i := end; i < hi; i++ {
|
||||
c := response[i]
|
||||
if c == '.' || c == ';' {
|
||||
hi = i
|
||||
break
|
||||
}
|
||||
if c == '\n' && i+1 < len(response) && response[i+1] == '\n' {
|
||||
hi = i
|
||||
break
|
||||
}
|
||||
}
|
||||
return lo, hi
|
||||
}
|
||||
|
||||
// negationCueNearby reports whether a negation cue (bidirectional,
|
||||
// clause-scoped) negates the match [start,end) in response. Directional
|
||||
// cues (instead of/rather than) are checked only in the PRE-window.
|
||||
func negationCueNearby(response string, start, end, maxChars int) bool {
|
||||
lo, hi := clauseWindowBounds(response, start, end, maxChars)
|
||||
pre, post := response[lo:start], response[end:hi]
|
||||
|
||||
if negationCuePattern.MatchString(pre) || negationCuePattern.MatchString(post) {
|
||||
return true
|
||||
}
|
||||
if negationBareNoColonPattern.MatchString(pre) || negationBareNoColonPattern.MatchString(post) {
|
||||
return true
|
||||
}
|
||||
return negationDirectionalCuePattern.MatchString(pre)
|
||||
}
|
||||
|
||||
// NoUnnegatedMention returns an Evaluator scoring full credit unless
|
||||
// badPattern matches somewhere in response with no negation cue nearby -
|
||||
// i.e. it penalizes an unqualified endorsement of a discouraged
|
||||
// term/command/phrase but not a warning against it. This is deliberately
|
||||
// not NotContains, which would also zero out the best possible answer (one
|
||||
// that correctly names the discouraged thing while warning against it).
|
||||
//
|
||||
// The search window around each match is bidirectional (checked both
|
||||
// immediately before AND after the match, since a good answer negates a
|
||||
// discouraged term with the cue on either side - "never use MD5" as much as
|
||||
// "MD5 is unsuitable here") and clause-scoped (see clauseWindowBounds), so a
|
||||
// cue from an unrelated adjacent sentence never falsely negates a
|
||||
// genuinely unqualified endorsement elsewhere in a longer response.
|
||||
// maxChars additionally caps the window width in each direction.
|
||||
//
|
||||
// exempt, if non-nil, is matched against the text immediately following
|
||||
// each match (response[end:]); a match whose trailing text satisfies
|
||||
// exempt is treated as inherently safe and skips the negation check
|
||||
// entirely (e.g. "--force-with-lease" is the safe form of "--force" and
|
||||
// never needs a negation cue to be a correct recommendation).
|
||||
func NoUnnegatedMention(badPattern *regexp.Regexp, maxChars int, exempt *regexp.Regexp) Evaluator {
|
||||
return EvaluatorFunc(func(_ context.Context, response string) Score {
|
||||
matches := badPattern.FindAllStringIndex(response, -1)
|
||||
if len(matches) == 0 {
|
||||
return Score{Value: 1, Detail: "no mention of a discouraged term"}
|
||||
}
|
||||
for _, loc := range matches {
|
||||
start, end := loc[0], loc[1]
|
||||
if exempt != nil && exempt.MatchString(response[end:]) {
|
||||
continue
|
||||
}
|
||||
if !negationCueNearby(response, start, end, maxChars) {
|
||||
return Score{Value: 0, Detail: "unnegated mention of a discouraged term"}
|
||||
}
|
||||
}
|
||||
return Score{Value: 1, Detail: "every mention of a discouraged term is negated"}
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,111 @@
|
||||
package eval
|
||||
|
||||
import (
|
||||
"context"
|
||||
"regexp"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestNoUnnegatedMention(t *testing.T) {
|
||||
badPattern := regexp.MustCompile(`(?i)\bMD5\b`)
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{"no mention at all", "Use bcrypt to hash the password.", 1},
|
||||
{"unnegated bare endorsement", "Just hash the password with MD5.", 0},
|
||||
{"pre-window imperative cue", "Never use MD5 for password storage.", 1},
|
||||
{"post-window adjectival cue", "MD5 is unsuitable for password storage.", 1},
|
||||
{"post-window 'too fast' adjectival cue", "MD5 is too fast to be safe for passwords.", 1},
|
||||
{"post-window 'insecure' adjectival cue", "MD5 is insecure for this purpose.", 1},
|
||||
{"post-window 'predictable' adjectival cue", "MD5 output is predictable given no work factor.", 1},
|
||||
{"bare 'no:' cue", "no: MD5 lacks a tunable work factor.", 1},
|
||||
{
|
||||
name: "clause-scoped: cue in an unrelated PRECEDING sentence does not count",
|
||||
response: "Never mind the deployment steps. Just hash the password with MD5.",
|
||||
want: 0,
|
||||
},
|
||||
{
|
||||
name: "clause-scoped: cue in an unrelated FOLLOWING sentence does not count",
|
||||
response: "Just hash the password with MD5. Never mind, that was a joke about something else.",
|
||||
want: 0,
|
||||
},
|
||||
{
|
||||
name: "directional cue 'instead of' correctly negates the term it precedes",
|
||||
response: "Use bcrypt instead of MD5 for password storage.",
|
||||
want: 1,
|
||||
},
|
||||
{
|
||||
name: "directional cue 'instead of' AFTER the term does not negate it (recommends MD5 first)",
|
||||
response: "Use MD5 instead of a slower alternative for password storage.",
|
||||
want: 0,
|
||||
},
|
||||
{
|
||||
name: "directional cue 'rather than' correctly negates the term it precedes",
|
||||
response: "Use argon2 rather than MD5.",
|
||||
want: 1,
|
||||
},
|
||||
{
|
||||
name: "multiple mentions: all negated scores full credit",
|
||||
response: "Never use MD5. Avoid MD5 for anything security-sensitive.",
|
||||
want: 1,
|
||||
},
|
||||
{
|
||||
name: "multiple mentions: one unnegated mention scores zero",
|
||||
response: "Never use MD5 for tokens. For passwords, just hash with MD5 directly.",
|
||||
want: 0,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
evaluator := NoUnnegatedMention(badPattern, 60, nil)
|
||||
got := evaluator.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("NoUnnegatedMention().Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestNoUnnegatedMention_Exempt(t *testing.T) {
|
||||
badPattern := regexp.MustCompile(`(?i)--force\b`)
|
||||
exempt := regexp.MustCompile(`(?i)^-with-lease`)
|
||||
evaluator := NoUnnegatedMention(badPattern, 60, exempt)
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{"exempted safe form scores full credit unnegated", "Use --force-with-lease to push safely.", 1},
|
||||
{"bare --force unnegated scores zero", "Just run git push --force.", 0},
|
||||
{"bare --force negated scores full credit", "Never run git push --force on a shared branch.", 1},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := evaluator.Evaluate(context.Background(), tt.response)
|
||||
if got.Value != tt.want {
|
||||
t.Errorf("Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestClauseWindowBounds(t *testing.T) {
|
||||
response := "First sentence here. Second sentence has TERM in it. Third sentence follows."
|
||||
start := len("First sentence here. Second sentence has ")
|
||||
end := start + len("TERM")
|
||||
|
||||
lo, hi := clauseWindowBounds(response, start, end, 60)
|
||||
pre := response[lo:start]
|
||||
post := response[end:hi]
|
||||
|
||||
if want := " Second sentence has "; pre != want {
|
||||
t.Errorf("pre = %q, want %q", pre, want)
|
||||
}
|
||||
if want := " in it"; post != want {
|
||||
t.Errorf("post = %q, want %q", post, want)
|
||||
}
|
||||
}
|
||||
@@ -97,7 +97,18 @@ func classifyMatch(text string, start, end int) (numberCandidate, bool) {
|
||||
// ("float64x2") keep a word byte after the x and stay rejected.
|
||||
isMultiplier := (text[end] == 'x' || text[end] == 'X') &&
|
||||
(end+1 >= len(text) || !isWordByte(text[end+1]))
|
||||
if !isMultiplier {
|
||||
// Permit a glued byte-size unit suffix ("30MB" = 30, no space)
|
||||
// (DC1): case-insensitive kb/mb/gb/tb, with nothing word-like
|
||||
// after it - "300MBs" (plural) keeps a word byte after the unit
|
||||
// and stays rejected, the same strictness as the x/X case above.
|
||||
isUnitSuffix := false
|
||||
if end+2 <= len(text) {
|
||||
switch strings.ToLower(text[end : end+2]) {
|
||||
case "kb", "mb", "gb", "tb":
|
||||
isUnitSuffix = end+2 >= len(text) || !isWordByte(text[end+2])
|
||||
}
|
||||
}
|
||||
if !isMultiplier && !isUnitSuffix {
|
||||
return numberCandidate{}, false // e.g. "24bytes" with no separator
|
||||
}
|
||||
}
|
||||
|
||||
@@ -29,6 +29,14 @@ func TestExtractLastNumber(t *testing.T) {
|
||||
{"multiplier suffix accepted", "The compression ratio is 64x.", 64, false},
|
||||
{"multiplier suffix at end of text", "speedup: 8x", 8, false},
|
||||
{"hex literal never a multiplier", "the flag is 0x2A so the answer is 42", 42, false},
|
||||
// DC1: glued byte-size unit suffixes (kb/mb/gb/tb) extract the
|
||||
// same as a multiplier suffix.
|
||||
{"glued MB unit suffix", "The final image is 30MB.", 30, false},
|
||||
{"glued lowercase mb unit suffix", "total: 30mb", 30, false},
|
||||
{"glued GB unit suffix", "Budget: 60GB", 60, false},
|
||||
{"glued KB unit suffix", "Size: 512KB", 512, false},
|
||||
{"glued TB unit suffix", "Capacity: 4TB", 4, false},
|
||||
{"plural unit suffix still rejected as an identifier", "roughly 300MBs of headroom, so the answer is 300", 300, false},
|
||||
{"hyphen-compound number accepted as a last resort", "It is a 24-byte struct.", 24, false},
|
||||
// Documented limitation, not a regression: the heuristic has no
|
||||
// notion of "the total" vs. "a breakdown component" and returns
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
package tests
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/lukaszraczylo/llm-testbench/internal/eval"
|
||||
"github.com/lukaszraczylo/llm-testbench/internal/testkit"
|
||||
)
|
||||
@@ -81,6 +83,15 @@ id: {"temperature":"...","max_tokens":"...","stop":"..."}`
|
||||
}
|
||||
}
|
||||
|
||||
// llmTokenBudgetMaxTokens and llmTokenBudgetThinkTokens are the parameters
|
||||
// for llmTokenBudgetReasoningTest, interpolated into the prompt (C14) so
|
||||
// the displayed values can never drift from what llmTokenBudgetReasoningWant
|
||||
// is actually computed from.
|
||||
const (
|
||||
llmTokenBudgetMaxTokens = 500
|
||||
llmTokenBudgetThinkTokens = 437
|
||||
)
|
||||
|
||||
// llmTokenBudgetReasoningWant is derived by plain subtraction, matching the
|
||||
// arithmetic ai_llmintegration_test.go independently recomputes.
|
||||
//
|
||||
@@ -92,21 +103,22 @@ id: {"temperature":"...","max_tokens":"...","stop":"..."}`
|
||||
// than a small fixed cap. Here: max_tokens = 500, and the model's <think>
|
||||
// block consumes 437 completion tokens before the visible answer begins.
|
||||
// Tokens remaining for the visible answer = 500 - 437 = 63.
|
||||
const llmTokenBudgetReasoningWant = 500 - 437
|
||||
const llmTokenBudgetReasoningWant = llmTokenBudgetMaxTokens - llmTokenBudgetThinkTokens
|
||||
|
||||
func llmTokenBudgetReasoningTest() testkit.Test {
|
||||
prompt := `An OpenAI-compatible chat completion request sets
|
||||
max_tokens = 500. This caps the total number of completion tokens the
|
||||
prompt := fmt.Sprintf(`An OpenAI-compatible chat completion request sets
|
||||
max_tokens = %d. This caps the total number of completion tokens the
|
||||
response may contain, counting every token the model generates: a
|
||||
reasoning model's hidden <think>...</think> content counts against this
|
||||
same budget before its visible answer, since both are completion tokens.
|
||||
|
||||
For this request, the model's <think>...</think> block consumes exactly
|
||||
437 completion tokens before it begins writing its visible answer.
|
||||
%d completion tokens before it begins writing its visible answer.
|
||||
|
||||
How many completion tokens remain in the max_tokens budget for the visible
|
||||
answer, once the <think> block has been generated? Respond with only the
|
||||
number.`
|
||||
number, with no commas, units, or other text (for example: 63).`,
|
||||
llmTokenBudgetMaxTokens, llmTokenBudgetThinkTokens)
|
||||
|
||||
return testkit.Test{
|
||||
ID: "llm-token-budget-reasoning",
|
||||
@@ -251,7 +263,7 @@ with only one word.`
|
||||
Subcategory: "llm-integration",
|
||||
Description: "Identify system (not user) as the role that carries a fixed, non-negotiable operating instruction.",
|
||||
Prompt: prompt,
|
||||
Eval: eval.Equals("system"),
|
||||
Eval: eval.ExactToken("system"),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -309,26 +321,37 @@ final event? Respond with only that exact text, nothing else.`
|
||||
Subcategory: "llm-integration",
|
||||
Description: "Recall the exact [DONE] sentinel text of the final SSE event in an OpenAI-compatible stream.",
|
||||
Prompt: prompt,
|
||||
Eval: eval.Equals("[DONE]"),
|
||||
Eval: eval.ExactToken("[DONE]"),
|
||||
}
|
||||
}
|
||||
|
||||
// llmEmbeddingBatchDocCount and llmEmbeddingBatchSize are the parameters
|
||||
// for llmEmbeddingBatchMathTest, interpolated into the prompt (C14) so the
|
||||
// displayed values can never drift from what llmEmbeddingBatchMathWant is
|
||||
// actually computed from.
|
||||
const (
|
||||
llmEmbeddingBatchDocCount = 10000
|
||||
llmEmbeddingBatchSize = 96
|
||||
)
|
||||
|
||||
// llmEmbeddingBatchMathWant is derived by calling aiCeilDiv, not
|
||||
// hardcoded.
|
||||
//
|
||||
// ground truth: ceil(10000/96) - 96*104 = 9984, leaving 16 documents that
|
||||
// still need one more (partial) request, so 104 full batches plus 1
|
||||
// partial batch = 105 requests total.
|
||||
var llmEmbeddingBatchMathWant = aiCeilDiv(10000, 96)
|
||||
var llmEmbeddingBatchMathWant = aiCeilDiv(llmEmbeddingBatchDocCount, llmEmbeddingBatchSize)
|
||||
|
||||
func llmEmbeddingBatchMathTest() testkit.Test {
|
||||
prompt := `You must generate embeddings for 10,000 documents. The
|
||||
embeddings API accepts a maximum of 96 items per request, and every
|
||||
prompt := fmt.Sprintf(`You must generate embeddings for %d documents. The
|
||||
embeddings API accepts a maximum of %d items per request, and every
|
||||
request you send should be as full as possible (maximally batched) to
|
||||
minimize the number of requests.
|
||||
|
||||
How many API requests are needed in total to embed all 10,000 documents?
|
||||
Respond with only the number.`
|
||||
How many API requests are needed in total to embed all %d documents?
|
||||
Respond with only the number, with no commas, units, or other text (for
|
||||
example: 105).`,
|
||||
llmEmbeddingBatchDocCount, llmEmbeddingBatchSize, llmEmbeddingBatchDocCount)
|
||||
|
||||
return testkit.Test{
|
||||
ID: "llm-embedding-batch-math",
|
||||
@@ -364,6 +387,6 @@ client? Respond with only that exact returned text, nothing else.`
|
||||
Subcategory: "llm-integration",
|
||||
Description: "Trace the exact text an API returns when generation is cut by a stop=[\"\\n\\n\"] sequence, excluding the sequence itself.",
|
||||
Prompt: prompt,
|
||||
Eval: eval.Equals("Answer: 42"),
|
||||
Eval: eval.ExactToken("Answer: 42"),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -22,6 +22,11 @@ func TestLlmFieldSemanticsTest_Eval(t *testing.T) {
|
||||
response: "Here is my answer:\n```json\n{\"temperature\":\"sampling_randomness\",\"max_tokens\":\"completion_token_cap\",\"stop\":\"generation_end_strings\"}\n```",
|
||||
want: 1,
|
||||
},
|
||||
{
|
||||
name: "all correct, different spacing",
|
||||
response: `{ "temperature": "sampling_randomness", "max_tokens": "completion_token_cap", "stop": "generation_end_strings" }`,
|
||||
want: 1,
|
||||
},
|
||||
{
|
||||
name: "one wrong",
|
||||
response: `{"temperature":"output_format","max_tokens":"completion_token_cap","stop":"generation_end_strings"}`,
|
||||
@@ -65,6 +70,7 @@ func TestLlmTokenBudgetReasoningTest_Eval(t *testing.T) {
|
||||
}{
|
||||
{"exact", "63", 1},
|
||||
{"prose wrapped", "63 completion tokens remain.", 1},
|
||||
{"fenced", "```\n63\n```", 1},
|
||||
{"wrong: forgot to subtract", "500", 0},
|
||||
{"wrong: reported the reasoning tokens used, not remaining", "437", 0},
|
||||
}
|
||||
@@ -87,6 +93,7 @@ func TestLlm429RetryBackoffTest_Eval(t *testing.T) {
|
||||
}{
|
||||
{"all correct", `{"should_retry":"yes","wait_seconds":2}`, 1},
|
||||
{"all correct fenced", "```json\n{\"should_retry\":\"yes\",\"wait_seconds\":2}\n```", 1},
|
||||
{"all correct, different spacing", `{ "should_retry": "yes", "wait_seconds": 2 }`, 1},
|
||||
{"wrong: should not retry", `{"should_retry":"no","wait_seconds":2}`, 0.5},
|
||||
{"wrong: ignored Retry-After value", `{"should_retry":"yes","wait_seconds":30}`, 0.5},
|
||||
{"both wrong", `{"should_retry":"no","wait_seconds":30}`, 0},
|
||||
@@ -110,6 +117,7 @@ func TestLlmContextOverflowStrategyTest_Eval(t *testing.T) {
|
||||
}{
|
||||
{"correct: drop-oldest-turns", `{"strategy":"drop-oldest-turns"}`, 1},
|
||||
{"correct, fenced with prose", "Given the no-altering constraint:\n```json\n{\"strategy\":\"drop-oldest-turns\"}\n```", 1},
|
||||
{"correct, different spacing", `{ "strategy": "drop-oldest-turns" }`, 1},
|
||||
{"wrong: summarize-oldest-turns", `{"strategy":"summarize-oldest-turns"}`, 0},
|
||||
{"wrong: truncate-system-prompt", `{"strategy":"truncate-system-prompt"}`, 0},
|
||||
}
|
||||
@@ -132,6 +140,7 @@ func TestLlmToolCallHandlingTest_Eval(t *testing.T) {
|
||||
}{
|
||||
{"all correct", `{"next_role":"tool","id_field":"tool_call_id"}`, 1},
|
||||
{"all correct fenced", "```json\n{\"next_role\":\"tool\",\"id_field\":\"tool_call_id\"}\n```", 1},
|
||||
{"all correct, different spacing", `{ "next_role": "tool", "id_field": "tool_call_id" }`, 1},
|
||||
{"wrong role", `{"next_role":"assistant","id_field":"tool_call_id"}`, 0.5},
|
||||
{"wrong field", `{"next_role":"tool","id_field":"id"}`, 0.5},
|
||||
{"both wrong", `{"next_role":"user","id_field":"call_id"}`, 0},
|
||||
@@ -155,7 +164,9 @@ func TestLlmRolePlacementTest_Eval(t *testing.T) {
|
||||
}{
|
||||
{"correct: system", "system", 1},
|
||||
{"correct, different case", "System", 1},
|
||||
{"correct, quoted", `"system"`, 1},
|
||||
{"wrong: user", "user", 0},
|
||||
{"wrong: assistant", "assistant", 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
@@ -179,6 +190,16 @@ func TestLlmTemperatureZeroCaveatTest_Eval(t *testing.T) {
|
||||
response: "Floating-point arithmetic is not strictly associative, and provider-side batching or GPU kernel routing changes summation order, which can flip the top token.",
|
||||
want: 1,
|
||||
},
|
||||
{
|
||||
name: "both reasons present, alternate phrasing",
|
||||
response: "Numerical precision in floating-point math is not associative, and the hardware kernel path a batched request is routed through can differ between calls.",
|
||||
want: 1,
|
||||
},
|
||||
{
|
||||
name: "both reasons present, terse phrasing",
|
||||
response: "Floating-point precision issues plus parallel execution batching on the GPU.",
|
||||
want: 1,
|
||||
},
|
||||
{
|
||||
name: "only the numerical reason",
|
||||
response: "It is because of floating point rounding differences.",
|
||||
@@ -194,6 +215,11 @@ func TestLlmTemperatureZeroCaveatTest_Eval(t *testing.T) {
|
||||
response: "The model just has some inherent randomness left even at temperature zero.",
|
||||
want: 0,
|
||||
},
|
||||
{
|
||||
name: "neither reason, wrong explanation entirely",
|
||||
response: "It happens because the random seed is not fixed across requests.",
|
||||
want: 0,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
@@ -255,6 +281,7 @@ func TestLlmEmbeddingBatchMathTest_Eval(t *testing.T) {
|
||||
}{
|
||||
{"exact", "105", 1},
|
||||
{"prose wrapped", "You need 105 requests.", 1},
|
||||
{"fenced", "```\n105\n```", 1},
|
||||
{"wrong: dropped the partial batch", "104", 0},
|
||||
{"wrong: off by a lot", "96", 0},
|
||||
}
|
||||
@@ -277,6 +304,7 @@ func TestLlmStopSequenceTraceTest_Eval(t *testing.T) {
|
||||
}{
|
||||
{"correct", "Answer: 42", 1},
|
||||
{"correct with whitespace", " Answer: 42 ", 1},
|
||||
{"correct, quoted", `"Answer: 42"`, 1},
|
||||
{"wrong: included text past the stop sequence", "Answer: 42\n\nExplanation: 6 times 7 is 42.", 0},
|
||||
{"wrong: paraphrased instead of quoting exactly", "The answer is 42", 0},
|
||||
}
|
||||
|
||||
+39
-16
@@ -40,9 +40,9 @@ that fact.
|
||||
Three chunk-size options are available: 128 tokens, 512 tokens, and 2048
|
||||
tokens.
|
||||
|
||||
Which of these is the smallest option that reliably holds a full 450-token
|
||||
fact without ever splitting it across two chunks? Respond with only a
|
||||
JSON object: {"chunk_size_tokens":128|512|2048}`
|
||||
Which of these is the smallest option large enough to contain a full
|
||||
450-token fact? Respond with only a JSON object:
|
||||
{"chunk_size_tokens":128|512|2048}`
|
||||
|
||||
return testkit.Test{
|
||||
ID: "rag-chunk-size-tradeoff",
|
||||
@@ -62,14 +62,17 @@ JSON object: {"chunk_size_tokens":128|512|2048}`
|
||||
// candidates that retrieval already surfaced, so it must run second;
|
||||
// generation consumes the reranker's narrowed, reordered candidates last.
|
||||
func ragRerankerPlacementTest() testkit.Test {
|
||||
// C4: the stages below are hand-shuffled to a different order than the
|
||||
// correct pipeline order, so an echo of the presentation order does not
|
||||
// score 1.0.
|
||||
prompt := `A RAG pipeline has these three stages, listed here in no
|
||||
particular order:
|
||||
- generate: have the LLM produce the final answer using the selected
|
||||
chunks as context.
|
||||
- retrieve: run a broad search (vector and/or keyword) over the index to
|
||||
produce an initial candidate set of chunks.
|
||||
- rerank: use a more expensive, more accurate model (e.g. a cross-encoder)
|
||||
to reorder a candidate set by relevance and narrow it to the best few.
|
||||
- generate: have the LLM produce the final answer using the selected
|
||||
chunks as context.
|
||||
|
||||
Give the correct pipeline order as a JSON array of the three stage names,
|
||||
e.g. ["a","b","c"]. Respond with only the JSON array.`
|
||||
@@ -122,7 +125,7 @@ based)? Respond with only one word: keyword or semantic.`
|
||||
Subcategory: "rag",
|
||||
Description: "Identify keyword (exact-term) search as the method that fails on a synonym-only query/document pair.",
|
||||
Prompt: prompt,
|
||||
Eval: eval.Equals("keyword"),
|
||||
Eval: eval.ExactToken("keyword"),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -137,8 +140,12 @@ the answer be traceable back to, and what must the system do instead of
|
||||
guessing when the retrieved context does not actually contain the
|
||||
information needed to answer the question?`
|
||||
|
||||
// C10: widen the traceability group beyond "cite"/"citation"/"source" -
|
||||
// a response can correctly state the grounding requirement using
|
||||
// "retrieved context"/"provided context"/"grounded"/"supported by"/
|
||||
// "traceable" without ever using the word "cite" itself.
|
||||
evaluator := eval.Mean(
|
||||
eval.ContainsAny("cite", "citation", "source", "quote", "quoted"),
|
||||
eval.ContainsAny("cite", "citation", "source", "quote", "quoted", "retrieved context", "provided context", "grounded", "supported by", "traceable"),
|
||||
eval.ContainsAny("insufficient", "does not contain", "doesn't contain", "cannot answer", "can't answer", "no relevant", "unable to find", "not found in the context", "not present in the context"),
|
||||
)
|
||||
|
||||
@@ -185,7 +192,7 @@ unused ("selective")? Respond with only one word: stuffed or selective.`
|
||||
Subcategory: "rag",
|
||||
Description: "Choose relevance-filtered ('selective') context assembly over budget-filling ('stuffed') given low-relevance chunks degrade quality.",
|
||||
Prompt: prompt,
|
||||
Eval: eval.Equals("selective"),
|
||||
Eval: eval.ExactToken("selective"),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -284,15 +291,23 @@ specific problem in each scenario? Respond with only a JSON object:
|
||||
// unsupported claims can be flagged/removed, since flagging depends on
|
||||
// verification's result.
|
||||
func ragHallucinationMitigationOrderingTest() testkit.Test {
|
||||
// C4: hand-shuffled to a different order than the correct answer, so an
|
||||
// echo of the presentation order does not score 1.0. C8: step_instruct
|
||||
// is reworded to make explicit that it embeds the ALREADY-RETRIEVED
|
||||
// context into the assembled prompt, which forces instruct to run
|
||||
// after retrieve (the original "instruct the model... to answer using
|
||||
// only the retrieved context" wording was ambiguous about whether this
|
||||
// was a static, retrieval-independent system instruction).
|
||||
prompt := `A RAG pipeline's hallucination-mitigation steps are listed
|
||||
here, identified by id, in no particular order:
|
||||
- step_retrieve: retrieve grounding context relevant to the query.
|
||||
- step_instruct: instruct the model, in the prompt, to answer using only
|
||||
the retrieved context and to cite it.
|
||||
- step_flag: flag or remove any claim that verification could not support.
|
||||
- step_generate: generate the answer.
|
||||
- step_retrieve: retrieve grounding context relevant to the query.
|
||||
- step_verify: verify each claim in the generated answer against the
|
||||
retrieved context.
|
||||
- step_flag: flag or remove any claim that verification could not support.
|
||||
- step_instruct: assemble the final prompt, embedding the retrieved
|
||||
context together with the instruction to answer only from it and cite
|
||||
it.
|
||||
|
||||
Give the correct order these steps must run in, as a JSON array of their
|
||||
ids, e.g. ["step_a","step_b"]. Respond with only the JSON array.`
|
||||
@@ -323,18 +338,22 @@ ids, e.g. ["step_a","step_b"]. Respond with only the JSON array.`
|
||||
// answer). frag_x (team headcount) answers a question never asked and
|
||||
// must be excluded.
|
||||
func ragMultihopDecompositionTest() testkit.Test {
|
||||
// C4: hand-shuffled to a different order than the correct answer (the
|
||||
// roster previously happened to already list the non-distractor
|
||||
// fragments in the correct dependency order), so echoing the
|
||||
// presentation order (minus frag_x) does not score 1.0.
|
||||
prompt := `Multi-hop question: "What is the annual maintenance cost of
|
||||
the storage tier used by the database that backs the service which
|
||||
handles user authentication?"
|
||||
|
||||
Candidate sub-question fragments, identified by id, in no particular
|
||||
order:
|
||||
- frag_q: "Which service handles user authentication?"
|
||||
- frag_p: "Which database backs that service?"
|
||||
- frag_s: "Which storage tier does that database use?"
|
||||
- frag_r: "What is the annual maintenance cost of that storage tier?"
|
||||
- frag_x: "What is the total headcount of the team that owns that
|
||||
service?"
|
||||
- frag_r: "What is the annual maintenance cost of that storage tier?"
|
||||
- frag_q: "Which service handles user authentication?"
|
||||
- frag_p: "Which database backs that service?"
|
||||
|
||||
Exactly one of these fragments (frag_x) does not help answer the
|
||||
multi-hop question and must be excluded. Give the remaining fragments, in
|
||||
@@ -363,8 +382,12 @@ the final context (rather than passing all 3 copies straight through to
|
||||
the LLM) matters: name what assembling all 3 copies wastes, and what
|
||||
problem it risks introducing into the generated answer.`
|
||||
|
||||
// CC4: widen the waste group with "tokens" (bare plural, not just
|
||||
// "token budget"), "over-weight", and "dominat[e/ing/ion]" - a response
|
||||
// can correctly describe the wasted-capacity consequence without using
|
||||
// the word "budget"/"waste" specifically.
|
||||
evaluator := eval.Mean(
|
||||
eval.ContainsAny("token budget", "context window", "context budget", "waste", "wastes", "wasted"),
|
||||
eval.ContainsAny("token budget", "context window", "context budget", "waste", "wastes", "wasted", "tokens", "over-weight", "dominat"),
|
||||
eval.ContainsAny("redundant", "redundancy", "duplicate", "duplication", "repetition", "repetitive", "diversity", "biased", "bias", "skew"),
|
||||
)
|
||||
|
||||
|
||||
@@ -14,6 +14,7 @@ func TestRagChunkSizeTradeoffTest_Eval(t *testing.T) {
|
||||
}{
|
||||
{"correct: 512", `{"chunk_size_tokens":512}`, 1},
|
||||
{"correct, fenced with prose", "The smallest viable option is:\n```json\n{\"chunk_size_tokens\":512}\n```", 1},
|
||||
{"correct, different spacing", `{ "chunk_size_tokens": 512 }`, 1},
|
||||
{"wrong: too small, would split a fact", `{"chunk_size_tokens":128}`, 0},
|
||||
{"wrong: larger than needed, more dilution", `{"chunk_size_tokens":2048}`, 0},
|
||||
}
|
||||
@@ -39,6 +40,7 @@ func TestRagRerankerPlacementTest_Eval(t *testing.T) {
|
||||
{"correct order different case", `["Retrieve","Rerank","Generate"]`, 1},
|
||||
{"reranker before retrieval", `["rerank","retrieve","generate"]`, 0},
|
||||
{"generate before rerank", `["retrieve","generate","rerank"]`, 0},
|
||||
{"wrong: echoes the prompt's shuffled presentation order verbatim (C4 bug probe)", `["generate","retrieve","rerank"]`, 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
@@ -59,7 +61,9 @@ func TestRagRetrievalFailureModeTest_Eval(t *testing.T) {
|
||||
}{
|
||||
{"correct: keyword", "keyword", 1},
|
||||
{"correct, different case", "Keyword", 1},
|
||||
{"correct, quoted", `"keyword"`, 1},
|
||||
{"wrong: semantic", "semantic", 0},
|
||||
{"wrong: both", "both", 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
@@ -83,14 +87,28 @@ func TestRagCitationGroundingTest_Eval(t *testing.T) {
|
||||
response: "Every claim must be traceable to a cited source in the retrieved context; if the context does not contain the answer, the system must say it cannot answer rather than guess.",
|
||||
want: 1,
|
||||
},
|
||||
{
|
||||
name: "both requirements present, alternate phrasing",
|
||||
response: "Answers must be grounded in and supported by the provided context, with every claim traceable back to it; when the context lacks the needed information, the system must say it is unable to find it instead of guessing.",
|
||||
want: 1,
|
||||
},
|
||||
{
|
||||
name: "both requirements present, terse phrasing",
|
||||
response: "Cite the source for every claim; if no relevant context exists, decline to answer.",
|
||||
want: 1,
|
||||
},
|
||||
{
|
||||
name: "only the citation requirement",
|
||||
response: "Every claim in the answer must include a citation back to the retrieved context.",
|
||||
want: 0.5,
|
||||
},
|
||||
{
|
||||
// C10 note: rephrased to avoid "retrieved context" (now one of
|
||||
// the widened traceability-group cues), so this case still
|
||||
// cleanly isolates the decline requirement from the citation
|
||||
// requirement.
|
||||
name: "only the decline requirement",
|
||||
response: "If the retrieved context does not contain the answer, the system must say it is unable to find it.",
|
||||
response: "If the answer isn't there, the system must say it cannot answer.",
|
||||
want: 0.5,
|
||||
},
|
||||
{
|
||||
@@ -98,6 +116,11 @@ func TestRagCitationGroundingTest_Eval(t *testing.T) {
|
||||
response: "The answer should just sound confident and helpful.",
|
||||
want: 0,
|
||||
},
|
||||
{
|
||||
name: "neither requirement, wrong advice entirely",
|
||||
response: "Just make the answer as detailed as possible.",
|
||||
want: 0,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
@@ -118,7 +141,9 @@ func TestRagContextSelectionVsStuffingTest_Eval(t *testing.T) {
|
||||
}{
|
||||
{"correct: selective", "selective", 1},
|
||||
{"correct, different case", "Selective", 1},
|
||||
{"correct, quoted", `"selective"`, 1},
|
||||
{"wrong: stuffed", "stuffed", 0},
|
||||
{"wrong: uses all the budget", "all of it", 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
@@ -147,6 +172,11 @@ func TestRagIndexStalenessTest_Eval(t *testing.T) {
|
||||
response: "```json\n{\"problem\":\"stale-chunks-coexist\",\"fix\":\"delete-old-chunks-before-reinsert\"}\n```",
|
||||
want: 1,
|
||||
},
|
||||
{
|
||||
name: "all correct, different spacing",
|
||||
response: `{ "problem": "stale-chunks-coexist", "fix": "delete-old-chunks-before-reinsert" }`,
|
||||
want: 1,
|
||||
},
|
||||
{
|
||||
name: "wrong problem",
|
||||
response: `{"problem":"embedding-model-drift","fix":"delete-old-chunks-before-reinsert"}`,
|
||||
@@ -190,6 +220,11 @@ func TestRagEvalMetricChoiceTest_Eval(t *testing.T) {
|
||||
response: "```json\n{\"scenario_a\":\"context_precision\",\"scenario_b\":\"faithfulness\"}\n```",
|
||||
want: 1,
|
||||
},
|
||||
{
|
||||
name: "all correct, different spacing",
|
||||
response: `{ "scenario_a": "context_precision", "scenario_b": "faithfulness" }`,
|
||||
want: 1,
|
||||
},
|
||||
{
|
||||
name: "scenario_a wrong",
|
||||
response: `{"scenario_a":"context_recall","scenario_b":"faithfulness"}`,
|
||||
@@ -233,6 +268,11 @@ func TestRagHallucinationMitigationOrderingTest_Eval(t *testing.T) {
|
||||
response: "```json\n[\"step_retrieve\",\"step_instruct\",\"step_generate\",\"step_verify\",\"step_flag\"]\n```",
|
||||
want: 1,
|
||||
},
|
||||
{
|
||||
name: "correct order, different case",
|
||||
response: `["Step_Retrieve","Step_Instruct","Step_Generate","Step_Verify","Step_Flag"]`,
|
||||
want: 1,
|
||||
},
|
||||
{
|
||||
name: "verify before generate",
|
||||
response: `["step_retrieve","step_instruct","step_verify","step_generate","step_flag"]`,
|
||||
@@ -243,6 +283,11 @@ func TestRagHallucinationMitigationOrderingTest_Eval(t *testing.T) {
|
||||
response: `["step_retrieve","step_instruct","step_generate","step_flag","step_verify"]`,
|
||||
want: 0,
|
||||
},
|
||||
{
|
||||
name: "wrong: echoes the prompt's shuffled presentation order verbatim (C4 bug probe)",
|
||||
response: `["step_flag","step_generate","step_retrieve","step_verify","step_instruct"]`,
|
||||
want: 0,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
@@ -271,6 +316,11 @@ func TestRagMultihopDecompositionTest_Eval(t *testing.T) {
|
||||
response: "```json\n[\"frag_q\",\"frag_p\",\"frag_s\",\"frag_r\"]\n```",
|
||||
want: 1,
|
||||
},
|
||||
{
|
||||
name: "correct order, different case",
|
||||
response: `["Frag_Q","Frag_P","Frag_S","Frag_R"]`,
|
||||
want: 1,
|
||||
},
|
||||
{
|
||||
name: "distractor incorrectly included",
|
||||
response: `["frag_q","frag_p","frag_s","frag_r","frag_x"]`,
|
||||
@@ -281,6 +331,11 @@ func TestRagMultihopDecompositionTest_Eval(t *testing.T) {
|
||||
response: `["frag_p","frag_q","frag_s","frag_r"]`,
|
||||
want: 0,
|
||||
},
|
||||
{
|
||||
name: "wrong: echoes the prompt's shuffled roster order verbatim, minus the distractor (C4 bug probe)",
|
||||
response: `["frag_s","frag_r","frag_q","frag_p"]`,
|
||||
want: 0,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
@@ -304,6 +359,16 @@ func TestRagPreAssemblyDedupTest_Eval(t *testing.T) {
|
||||
response: "Passing all 3 copies wastes context window budget, and the redundant repetition risks biasing the answer toward that one duplicated point.",
|
||||
want: 1,
|
||||
},
|
||||
{
|
||||
name: "both reasons present, alternate phrasing",
|
||||
response: "Sending all 3 duplicates dominates the token budget with the same content, and the repetition biases the model toward over-weighting that one point.",
|
||||
want: 1,
|
||||
},
|
||||
{
|
||||
name: "both reasons present, terse phrasing",
|
||||
response: "It wastes tokens and skews the answer toward the duplicated content.",
|
||||
want: 1,
|
||||
},
|
||||
{
|
||||
name: "only the waste reason",
|
||||
response: "It wastes token budget that could hold other useful chunks.",
|
||||
@@ -319,6 +384,11 @@ func TestRagPreAssemblyDedupTest_Eval(t *testing.T) {
|
||||
response: "It just looks messy to a human reviewer.",
|
||||
want: 0,
|
||||
},
|
||||
{
|
||||
name: "neither reason, wrong justification entirely",
|
||||
response: "It makes the prompt harder for a human to read.",
|
||||
want: 0,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package tests
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/lukaszraczylo/llm-testbench/internal/eval"
|
||||
@@ -65,7 +66,7 @@ division needed? Respond with only one word: yes or no.`
|
||||
Subcategory: "vector-search",
|
||||
Description: "Confirm that for two unit-normalized vectors, the plain dot product equals the cosine similarity.",
|
||||
Prompt: prompt,
|
||||
Eval: eval.Equals("yes"),
|
||||
Eval: eval.ExactToken("yes"),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -91,18 +92,23 @@ var vecRecallAtKWant = wpRecallAtK(vecRecallAtKRelevant, vecRecallAtKRetrieved)
|
||||
// ranked top-k retrieval list, distinct from the whitepapers recall@k
|
||||
// fixture.
|
||||
func vecRecallAtKTest() testkit.Test {
|
||||
// C14: the relevant/retrieved lists are interpolated from
|
||||
// vecRecallAtKRelevant/vecRecallAtKRetrieved (the vec-rrf-fusion
|
||||
// pattern) rather than hand-typed, so the displayed lists can never
|
||||
// drift from what vecRecallAtKWant is actually computed from.
|
||||
prompt := `Recall@k is a retrieval-quality metric: for one query, it is
|
||||
the fraction of that query's truly relevant items that appear anywhere
|
||||
within the system's top k retrieved results, regardless of exact rank.
|
||||
recall@k = |relevant intersect top-k| / |relevant|.
|
||||
|
||||
A query's truly relevant vectors are: V3, V8, V11, V14, V20.
|
||||
A query's truly relevant vectors are: ` + strings.Join(vecRecallAtKRelevant, ", ") + `.
|
||||
|
||||
A vector search system's ranked top-5 results for that query are, in rank
|
||||
order: V8, V1, V14, V6, V9.
|
||||
order: ` + strings.Join(vecRecallAtKRetrieved, ", ") + `.
|
||||
|
||||
Compute recall@5 for this query. Respond with only the number, as a
|
||||
decimal (e.g. 0.6), rounded to 4 decimal places.`
|
||||
Compute recall@5 for this query. Respond with only the number, rounded to
|
||||
4 decimal places, with no commas, units, or other text (for example:
|
||||
0.4000).`
|
||||
|
||||
return testkit.Test{
|
||||
ID: "vec-recall-at-k",
|
||||
@@ -138,7 +144,7 @@ go up, or both go down? Respond with only one word: up or down.`
|
||||
Subcategory: "vector-search",
|
||||
Description: "State that increasing HNSW's efSearch moves recall and query latency in the same direction (up).",
|
||||
Prompt: prompt,
|
||||
Eval: eval.Equals("up"),
|
||||
Eval: eval.ExactToken("up"),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -147,26 +153,36 @@ go up, or both go down? Respond with only one word: up or down.`
|
||||
// (different m, nbits, dimension, and a total-bytes framing rather than a
|
||||
// ratio).
|
||||
//
|
||||
// vecPQMemoryMathM, vecPQMemoryMathNBits, and vecPQMemoryMathVectorCount are
|
||||
// the PQ parameters for vecPQMemoryMathTest, interpolated into the prompt
|
||||
// (C14) so the displayed parameters can never drift from what
|
||||
// vecPQMemoryMathWant is actually computed from.
|
||||
const (
|
||||
vecPQMemoryMathM = 16
|
||||
vecPQMemoryMathNBits = 4
|
||||
vecPQMemoryMathVectorCount = 1000000
|
||||
)
|
||||
|
||||
// ground truth: codeBytes = m*nbits/8 = 16*4/8 = 8 bytes per compressed
|
||||
// vector (via wpQuantizedCodeBytes, already defined in
|
||||
// whitepapers_compute.go). Total for 1,000,000 vectors = 8 * 1,000,000 =
|
||||
// 8,000,000 bytes. ai_vectorsearch_test.go independently recomputes this
|
||||
// with plain arithmetic.
|
||||
var vecPQMemoryMathWant = wpQuantizedCodeBytes(16, 4) * 1000000
|
||||
var vecPQMemoryMathWant = wpQuantizedCodeBytes(vecPQMemoryMathM, vecPQMemoryMathNBits) * vecPQMemoryMathVectorCount
|
||||
|
||||
func vecPQMemoryMathTest() testkit.Test {
|
||||
prompt := `Product quantization (PQ) compresses each vector by splitting
|
||||
prompt := fmt.Sprintf(`Product quantization (PQ) compresses each vector by splitting
|
||||
it into m equal-length subvectors and quantizing each one independently
|
||||
against its own codebook, storing only each subvector's nbits-bit centroid
|
||||
index. Concatenating the m per-subvector codes gives the full compressed
|
||||
representation, with no wasted bits (assume m*nbits is a multiple of 8).
|
||||
|
||||
An index uses m = 16 subvectors and nbits = 4 bits per subvector.
|
||||
An index uses m = %d subvectors and nbits = %d bits per subvector.
|
||||
|
||||
What is the total memory, in bytes, needed to store the compressed
|
||||
representations of 1,000,000 vectors indexed with these PQ parameters?
|
||||
representations of %d vectors indexed with these PQ parameters?
|
||||
Respond with only the integer number of bytes, with no commas, units, or
|
||||
other text (for example: 12345678).`
|
||||
other text (for example: 12345678).`, vecPQMemoryMathM, vecPQMemoryMathNBits, vecPQMemoryMathVectorCount)
|
||||
|
||||
return testkit.Test{
|
||||
ID: "vec-pq-memory-math",
|
||||
@@ -207,7 +223,7 @@ only one word: pre or post.`
|
||||
Subcategory: "vector-search",
|
||||
Description: "Choose pre-filtering over post-filtering ANN search when the metadata filter is highly selective (0.5%).",
|
||||
Prompt: prompt,
|
||||
Eval: eval.Equals("pre"),
|
||||
Eval: eval.ExactToken("pre"),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -280,6 +296,12 @@ Respond with only the number, rounded to 4 decimal places.`
|
||||
}
|
||||
}
|
||||
|
||||
// vecDistanceToSimilarityD is the Euclidean distance for
|
||||
// vecDistanceToSimilarityTest, interpolated into the prompt (C14) so the
|
||||
// displayed value can never drift from what vecDistanceToSimilarityWant is
|
||||
// actually computed from.
|
||||
const vecDistanceToSimilarityD = 0.6
|
||||
|
||||
// vecDistanceToSimilarityWant is derived by calling
|
||||
// aiUnitDistanceToCosineSimilarity, not hardcoded.
|
||||
//
|
||||
@@ -287,18 +309,19 @@ Respond with only the number, rounded to 4 decimal places.`
|
||||
// cos_sim = 1 - d^2/2. With d = 0.6, d^2 = 0.36, cos_sim = 1 - 0.18 = 0.82.
|
||||
// ai_vectorsearch_test.go independently recomputes this with plain
|
||||
// arithmetic.
|
||||
var vecDistanceToSimilarityWant = round4dp(aiUnitDistanceToCosineSimilarity(0.6))
|
||||
var vecDistanceToSimilarityWant = round4dp(aiUnitDistanceToCosineSimilarity(vecDistanceToSimilarityD))
|
||||
|
||||
func vecDistanceToSimilarityTest() testkit.Test {
|
||||
prompt := `Two vectors are both unit-length (L2-normalized to magnitude
|
||||
prompt := fmt.Sprintf(`Two vectors are both unit-length (L2-normalized to magnitude
|
||||
1). For unit-length vectors, squared Euclidean distance relates to cosine
|
||||
similarity by: ||a-b||^2 = |a|^2 + |b|^2 - 2*cos_sim(a,b) = 2 - 2*cos_sim(a,b),
|
||||
since |a|=|b|=1. Rearranging: cos_sim(a,b) = 1 - (||a-b||^2) / 2.
|
||||
|
||||
The Euclidean distance between this pair is 0.6.
|
||||
The Euclidean distance between this pair is %v.
|
||||
|
||||
Using the formula above, compute cos_sim(a,b). Respond with only the
|
||||
number, rounded to 4 decimal places.`
|
||||
number, rounded to 4 decimal places, with no commas, units, or other text
|
||||
(for example: 0.8200).`, vecDistanceToSimilarityD)
|
||||
|
||||
return testkit.Test{
|
||||
ID: "vec-distance-to-similarity",
|
||||
@@ -331,16 +354,31 @@ const vecNearDuplicateThreshold = 0.97
|
||||
// this with plain arithmetic.
|
||||
var vecNearDuplicateThresholdWant = round4dp(cosineSimilarity(vecNearDuplicateA, vecNearDuplicateB) - vecNearDuplicateThreshold)
|
||||
|
||||
// aiFormatIntVector renders a []float64 of whole numbers as "[1, 2, 2, 1]"
|
||||
// prompt text, used to interpolate a fixture vector directly from the Go
|
||||
// slice it is graded against (C14) rather than a hand-typed duplicate.
|
||||
func aiFormatIntVector(v []float64) string {
|
||||
parts := make([]string, len(v))
|
||||
for i, x := range v {
|
||||
parts[i] = fmt.Sprintf("%d", int(x))
|
||||
}
|
||||
return "[" + strings.Join(parts, ", ") + "]"
|
||||
}
|
||||
|
||||
func vecNearDuplicateThresholdTest() testkit.Test {
|
||||
prompt := `A vector index deduplicates near-identical items: a candidate
|
||||
prompt := fmt.Sprintf(`A vector index deduplicates near-identical items: a candidate
|
||||
pair is treated as a near-duplicate only when its cosine similarity is at
|
||||
or above 0.97.
|
||||
or above %v.
|
||||
|
||||
Candidate pair: a = [1, 2, 2, 1], b = [2, 2, 1, 1].
|
||||
Candidate pair: a = %s, b = %s.
|
||||
|
||||
Compute the cosine similarity of a and b, then subtract the 0.97 threshold
|
||||
Compute the cosine similarity of a and b, then subtract the %v threshold
|
||||
from it (similarity minus threshold). Respond with only the resulting
|
||||
number, rounded to 4 decimal places (it may be negative).`
|
||||
number, rounded to 4 decimal places, with no commas, units, or other text
|
||||
(it may be negative; for example: -0.0700).`,
|
||||
vecNearDuplicateThreshold,
|
||||
aiFormatIntVector(vecNearDuplicateA), aiFormatIntVector(vecNearDuplicateB),
|
||||
vecNearDuplicateThreshold)
|
||||
|
||||
return testkit.Test{
|
||||
ID: "vec-near-duplicate-threshold",
|
||||
@@ -414,7 +452,7 @@ Two embedding models are candidates:
|
||||
- option 768: produces 768-dimensional embeddings.
|
||||
|
||||
Which option's total uncompressed storage fits within the 60 GB budget?
|
||||
Respond with only a JSON object: {"choice":"128"|"768"}`
|
||||
Respond with only a JSON object: {"choice":128|768}`
|
||||
|
||||
return testkit.Test{
|
||||
ID: "vec-embedding-dimension-tradeoff",
|
||||
@@ -422,6 +460,9 @@ Respond with only a JSON object: {"choice":"128"|"768"}`
|
||||
Subcategory: "vector-search",
|
||||
Description: "Pick the embedding dimension (128 vs 768) whose uncompressed float32 storage fits a fixed 60 GB budget for 100M vectors.",
|
||||
Prompt: prompt,
|
||||
Eval: eval.JSONField("choice", "128"),
|
||||
// C7: choice is a number (mirrors rag-chunk-size-tradeoff's
|
||||
// chunk_size_tokens), not a quoted string - the prompt's own
|
||||
// template shows it unquoted.
|
||||
Eval: eval.JSONField("choice", 128),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -23,7 +23,16 @@ func TestVecCosineVsDotTest_Eval(t *testing.T) {
|
||||
want float64
|
||||
}{
|
||||
{"correct: yes", "yes", 1},
|
||||
{"correct: yes, prose", "Yes, since both magnitudes are 1.", 0},
|
||||
{"correct: different case", "Yes", 1},
|
||||
{"correct: quoted", `"yes"`, 1},
|
||||
// C3: this row's own label was self-contradictory - "correct"
|
||||
// paired with want:0. The prompt asks for "only one word: yes or
|
||||
// no"; a full sentence is not a bare/quoted/fenced token even
|
||||
// though its content is substantively correct, so want:0 was
|
||||
// already the right assertion - only the misleading label needed
|
||||
// fixing, matching how eval.ExactToken treats sentence-wrapping
|
||||
// everywhere else in this codebase (it is not a summarizer).
|
||||
{"wrong: substantively correct but sentence-wrapped, not a bare token", "Yes, since both magnitudes are 1.", 0},
|
||||
{"wrong: no", "no", 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
@@ -66,6 +75,7 @@ func TestVecRecallAtKTest_Eval(t *testing.T) {
|
||||
}{
|
||||
{"exact", "0.4", 1},
|
||||
{"prose wrapped", "recall@5 is 0.4.", 1},
|
||||
{"trailing period", "0.4.", 1},
|
||||
{"wrong: counted an extra hit", "0.6", 0},
|
||||
{"wrong: missed a hit", "0.2", 0},
|
||||
}
|
||||
@@ -87,7 +97,10 @@ func TestVecHNSWEfSearchTradeoffTest_Eval(t *testing.T) {
|
||||
want float64
|
||||
}{
|
||||
{"correct: up", "up", 1},
|
||||
{"correct: uppercase", "UP", 1},
|
||||
{"correct: trailing period", "up.", 1},
|
||||
{"wrong: down", "down", 0},
|
||||
{"wrong: unchanged", "unchanged", 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
@@ -123,7 +136,9 @@ func TestVecPQMemoryMathTest_Eval(t *testing.T) {
|
||||
}{
|
||||
{"exact", "8000000", 1},
|
||||
{"prose wrapped", "The total is 8000000 bytes.", 1},
|
||||
{"fenced", "```\n8000000\n```", 1},
|
||||
{"wrong", "512", 0},
|
||||
{"wrong: bit/byte confusion (8x too large)", "64000000", 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
@@ -143,7 +158,10 @@ func TestVecPreVsPostFilteringTest_Eval(t *testing.T) {
|
||||
want float64
|
||||
}{
|
||||
{"correct: pre", "pre", 1},
|
||||
{"correct: uppercase", "PRE", 1},
|
||||
{"correct: trailing period", "pre.", 1},
|
||||
{"wrong: post", "post", 0},
|
||||
{"wrong: both", "both", 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
@@ -180,6 +198,7 @@ func TestVecRRFFusionTest_Eval(t *testing.T) {
|
||||
{"within tolerance", "0.0307", 1},
|
||||
{"prose wrapped", "The RRF score for D5 is 0.0308.", 1},
|
||||
{"wrong: only counted one list", "0.0159", 0},
|
||||
{"wrong: outside tolerance", "0.0328", 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
@@ -214,7 +233,9 @@ func TestVecDistanceToSimilarityTest_Eval(t *testing.T) {
|
||||
}{
|
||||
{"exact", "0.82", 1},
|
||||
{"prose wrapped", "cos_sim(a,b) = 0.82", 1},
|
||||
{"fenced", "```\n0.82\n```", 1},
|
||||
{"wrong: used d not d^2", "0.7", 0},
|
||||
{"wrong: forgot to subtract from 1", "0.18", 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
@@ -252,6 +273,7 @@ func TestVecNearDuplicateThresholdTest_Eval(t *testing.T) {
|
||||
}{
|
||||
{"exact", "-0.07", 1},
|
||||
{"prose wrapped", "similarity minus threshold is -0.07.", 1},
|
||||
{"fenced", "```\n-0.07\n```", 1},
|
||||
{"wrong: forgot the sign", "0.07", 0},
|
||||
{"wrong: reported similarity, not the difference", "0.9", 0},
|
||||
}
|
||||
@@ -274,6 +296,7 @@ func TestVecIndexBuildQueryTradeoffTest_Eval(t *testing.T) {
|
||||
}{
|
||||
{"all correct", `{"scenario_a":"ivf-flat","scenario_b":"hnsw"}`, 1},
|
||||
{"all correct fenced", "```json\n{\"scenario_a\":\"ivf-flat\",\"scenario_b\":\"hnsw\"}\n```", 1},
|
||||
{"all correct, different case", `{"scenario_a":"IVF-FLAT","scenario_b":"HNSW"}`, 1},
|
||||
{"scenario_a wrong", `{"scenario_a":"hnsw","scenario_b":"hnsw"}`, 0.5},
|
||||
{"scenario_b wrong", `{"scenario_a":"ivf-flat","scenario_b":"ivf-flat"}`, 0.5},
|
||||
{"both swapped", `{"scenario_a":"hnsw","scenario_b":"ivf-flat"}`, 0},
|
||||
@@ -309,6 +332,15 @@ func TestVecEmbeddingDimensionTradeoffTest_Eval(t *testing.T) {
|
||||
{"correct: 128", `{"choice":"128"}`, 1},
|
||||
{"correct, fenced with prose", "The 128-dim option fits:\n```json\n{\"choice\":\"128\"}\n```", 1},
|
||||
{"wrong: 768", `{"choice":"768"}`, 0},
|
||||
{"wrong: neither valid option", `{"choice":"512"}`, 0},
|
||||
{
|
||||
// C7 bug probe: the prompt's own template shows choice
|
||||
// unquoted ({"choice":128|768}), so a genuinely numeric
|
||||
// (unquoted) response must also score full credit.
|
||||
name: "correct: unquoted numeric form matching the prompt's own template (C7 bug probe)",
|
||||
response: `{"choice":128}`,
|
||||
want: 1,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
|
||||
@@ -4,7 +4,6 @@ import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/lukaszraczylo/llm-testbench/internal/eval"
|
||||
"github.com/lukaszraczylo/llm-testbench/internal/testkit"
|
||||
@@ -23,28 +22,6 @@ func registerDatabasesTests(r *testkit.Registry) {
|
||||
registerDBSQLTuningTests(r)
|
||||
}
|
||||
|
||||
// dbExactAnswer returns an Evaluator awarding full credit when the
|
||||
// response, trimmed of whitespace, at most one layer of surrounding quote
|
||||
// characters (' , ", or `), and a single trailing sentence-ending period,
|
||||
// equals want case-insensitively. Used across the databases category for
|
||||
// prompts that force a single short forced-vocabulary answer (e.g. "yes"
|
||||
// or "no", "keyset" or "offset"): it accepts every materially-correct form
|
||||
// of that answer (bare, quoted, differently-cased, or with trailing
|
||||
// punctuation) without loosening the match to accept a wrong answer.
|
||||
func dbExactAnswer(want string) eval.Evaluator {
|
||||
return eval.EvaluatorFunc(func(_ context.Context, response string) eval.Score {
|
||||
got := strings.TrimSpace(response)
|
||||
got = strings.Trim(got, "\"'`")
|
||||
got = strings.TrimSuffix(strings.TrimSpace(got), ".")
|
||||
got = strings.TrimSpace(got)
|
||||
wantTrimmed := strings.TrimSpace(want)
|
||||
if strings.EqualFold(got, wantTrimmed) {
|
||||
return eval.Score{Value: 1, Detail: fmt.Sprintf("equals %q", wantTrimmed)}
|
||||
}
|
||||
return eval.Score{Value: 0, Detail: fmt.Sprintf("got %q, want %q", got, wantTrimmed)}
|
||||
})
|
||||
}
|
||||
|
||||
// dbJSONArrayLength returns an Evaluator awarding full credit when the
|
||||
// response's extracted JSON array (see eval.ExtractJSON) has exactly want
|
||||
// elements, zero otherwise. Paired with per-index eval.JSONField checks so
|
||||
|
||||
@@ -219,10 +219,19 @@ acquisition order (by id, ascending) would both transactions need to follow
|
||||
to prevent this deadlock? Respond with only a JSON object:
|
||||
{"deadlock":true|false,"safe_order_ids":[<first id to lock>,<second id to lock>]}`
|
||||
|
||||
evaluator := eval.Mean(
|
||||
eval.JSONField("deadlock", true),
|
||||
eval.JSONField("safe_order_ids[0]", 1),
|
||||
eval.JSONField("safe_order_ids[1]", 2),
|
||||
// D9: weight the deadlock-diagnosis boolean 3x the safe-order answer,
|
||||
// so getting the core yes/no diagnosis right dominates the score more
|
||||
// than the specific ordering of an already-correct diagnosis. The two
|
||||
// safe_order_ids indices are grouped into one combined "order" share
|
||||
// (an equal-weighted Mean of the two), not two separate top-level
|
||||
// terms, so the 3:1 ratio holds between "diagnosis" and "order" as
|
||||
// conceptual units, not among three arbitrary equal terms.
|
||||
evaluator := eval.All(
|
||||
eval.W(eval.JSONField("deadlock", true), 3),
|
||||
eval.W(eval.Mean(
|
||||
eval.JSONField("safe_order_ids[0]", 1),
|
||||
eval.JSONField("safe_order_ids[1]", 2),
|
||||
), 1),
|
||||
)
|
||||
|
||||
return testkit.Test{
|
||||
@@ -235,9 +244,9 @@ to prevent this deadlock? Respond with only a JSON object:
|
||||
}
|
||||
}
|
||||
|
||||
// dbPGReplicaFailoverTest: explain that an asynchronously-replicated commit
|
||||
// can be lost when the primary crashes before the replica catches up and
|
||||
// gets promoted.
|
||||
// dbPGReplicaFailoverTest: identify that an asynchronously-replicated
|
||||
// commit is NOT guaranteed visible when the primary crashes before the
|
||||
// replica catches up and gets promoted.
|
||||
//
|
||||
// ground truth: with asynchronous streaming replication (no
|
||||
// synchronous_commit), a transaction's COMMIT returns to the client as
|
||||
@@ -247,6 +256,14 @@ to prevent this deadlock? Respond with only a JSON object:
|
||||
// without ever having applied that transaction's WAL - so the committed
|
||||
// transaction is not visible on it and is effectively lost, unless it can
|
||||
// be recovered from the old primary's WAL after the fact.
|
||||
//
|
||||
// D2: redesigned from a free-text ContainsAny("lost", ...) check to a
|
||||
// forced-vocabulary JSON boolean + reason, since "lost"/"data loss" as a
|
||||
// bare substring search matched its own exact negation - a response
|
||||
// stating "nothing is lost" or "no data loss occurs" contains the literal
|
||||
// substring "lost"/"data loss" and was wrongly scored 1.0 for asserting
|
||||
// the opposite of the correct answer. A forced true/false field cannot be
|
||||
// polarity-confused this way.
|
||||
func dbPGReplicaFailoverTest() testkit.Test {
|
||||
prompt := `A Postgres primary has one streaming replica, using
|
||||
asynchronous replication (synchronous_commit is not enabled). A client's
|
||||
@@ -255,19 +272,19 @@ commit reaches the replica, the primary host crashes, and the replica is
|
||||
promoted to become the new primary.
|
||||
|
||||
Is that committed transaction guaranteed to be visible on the newly
|
||||
promoted primary? Explain what happens to it and why, in one or two
|
||||
sentences.`
|
||||
promoted primary? Respond with only a JSON object:
|
||||
{"guaranteed_visible":true|false,"reason":"<one of: async-wal-not-replicated, sync-replication-guarantees-durability, replica-storage-corrupted>"}`
|
||||
|
||||
evaluator := eval.All(
|
||||
eval.W(eval.ContainsAny("lost", "not visible", "not guaranteed", "may be lost", "can be lost", "data loss"), 2),
|
||||
eval.W(eval.ContainsAny("async", "asynchronous"), 1),
|
||||
eval.W(eval.JSONField("guaranteed_visible", false), 3),
|
||||
eval.W(eval.JSONField("reason", "async-wal-not-replicated"), 2),
|
||||
)
|
||||
|
||||
return testkit.Test{
|
||||
ID: "pg-replica-failover",
|
||||
Category: "databases",
|
||||
Subcategory: "postgres",
|
||||
Description: "Explain that an asynchronously-replicated commit can be lost on failover if the primary crashes before the replica catches up.",
|
||||
Description: "Identify that an asynchronously-replicated commit is not guaranteed visible on failover if the primary crashes before the replica catches up.",
|
||||
Prompt: prompt,
|
||||
Eval: evaluator,
|
||||
}
|
||||
|
||||
@@ -111,9 +111,20 @@ func TestDBPGDeadlockLockOrderTest_Eval(t *testing.T) {
|
||||
}{
|
||||
{"correct", `{"deadlock":true,"safe_order_ids":[1,2]}`, 1},
|
||||
{"correct fenced with prose", "Yes, they deadlock.\n```json\n{\"deadlock\":true,\"safe_order_ids\":[1,2]}\n```", 1},
|
||||
{"says no deadlock", `{"deadlock":false,"safe_order_ids":[1,2]}`, 2.0 / 3.0},
|
||||
{"reversed safe order", `{"deadlock":true,"safe_order_ids":[2,1]}`, 1.0 / 3.0},
|
||||
// D9: deadlock (weight 3) now dominates the score over the safe
|
||||
// order (weight 1, combined as one unit): getting the core
|
||||
// diagnosis wrong costs far more than getting the order wrong.
|
||||
{"says no deadlock", `{"deadlock":false,"safe_order_ids":[1,2]}`, 0.25},
|
||||
{"reversed safe order", `{"deadlock":true,"safe_order_ids":[2,1]}`, 0.75},
|
||||
{"everything wrong", `{"deadlock":false,"safe_order_ids":[2,1]}`, 0},
|
||||
{
|
||||
// D9 bug probe: the diagnosis matters far more than the order -
|
||||
// a correct diagnosis with a fully-wrong order still beats a
|
||||
// wrong diagnosis with a fully-correct order.
|
||||
name: "correct diagnosis with wrong order still outscores wrong diagnosis with correct order (D9 bug probe)",
|
||||
response: `{"deadlock":true,"safe_order_ids":[2,1]}`,
|
||||
want: 0.75,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
@@ -134,24 +145,35 @@ func TestDBPGReplicaFailoverTest_Eval(t *testing.T) {
|
||||
want float64
|
||||
}{
|
||||
{
|
||||
name: "correct: names data loss and async cause",
|
||||
response: "Because replication is asynchronous, that commit may be lost - the replica never received its WAL before promotion.",
|
||||
name: "correct object",
|
||||
response: `{"guaranteed_visible":false,"reason":"async-wal-not-replicated"}`,
|
||||
want: 1,
|
||||
},
|
||||
{
|
||||
name: "correct: alternate phrasing",
|
||||
response: "No, it is not guaranteed to be visible; async replication means the promoted replica can be missing it.",
|
||||
name: "correct fenced with prose",
|
||||
response: "```json\n{\"guaranteed_visible\":false,\"reason\":\"async-wal-not-replicated\"}\n```",
|
||||
want: 1,
|
||||
},
|
||||
{
|
||||
name: "wrong: claims it is guaranteed durable",
|
||||
response: "Yes, Postgres guarantees every committed transaction is always visible after failover.",
|
||||
want: 0,
|
||||
response: `{"guaranteed_visible":true,"reason":"async-wal-not-replicated"}`,
|
||||
want: 0.4,
|
||||
},
|
||||
{
|
||||
name: "partial: mentions loss but not the async cause",
|
||||
response: "That transaction can be lost after the failover.",
|
||||
want: 2.0 / 3.0,
|
||||
name: "right verdict, wrong reason",
|
||||
response: `{"guaranteed_visible":false,"reason":"replica-storage-corrupted"}`,
|
||||
want: 0.6,
|
||||
},
|
||||
{
|
||||
// D2 bug probe: the exact negation of the correct answer -
|
||||
// stating no data is lost - used to score 1.0 under the old
|
||||
// ContainsAny("lost", "data loss", ...) substring search,
|
||||
// since "nothing is lost" and "no data loss" both contain
|
||||
// those literal substrings. The forced boolean field cannot be
|
||||
// polarity-confused this way.
|
||||
name: "wrong: exact negation of the correct answer scores near-zero, not 1.0 (D2 bug probe)",
|
||||
response: `{"guaranteed_visible":true,"reason":"sync-replication-guarantees-durability"}`,
|
||||
want: 0,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
|
||||
@@ -1,9 +1,7 @@
|
||||
package tests
|
||||
|
||||
import (
|
||||
"context"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/lukaszraczylo/llm-testbench/internal/eval"
|
||||
"github.com/lukaszraczylo/llm-testbench/internal/testkit"
|
||||
@@ -12,7 +10,7 @@ import (
|
||||
// registerDBRedisTests registers every databases/redis test.
|
||||
func registerDBRedisTests(r *testkit.Registry) {
|
||||
r.Register(dbRedisStructureChoiceTest())
|
||||
r.Register(dbRedisTTLEvictionTest())
|
||||
r.Register(dbRedisNoevictionWriteTest())
|
||||
r.Register(dbRedisIncrAtomicityTest())
|
||||
r.Register(dbRedisMultiVsPipelineTest())
|
||||
r.Register(dbRedisScanVsKeysTest())
|
||||
@@ -74,14 +72,14 @@ Respond with only a JSON object mapping each letter to one structure name:
|
||||
}
|
||||
}
|
||||
|
||||
// dbRedisTTLEvictionTest: determine what happens to a write under
|
||||
// dbRedisNoevictionWriteTest: determine what happens to a write under
|
||||
// maxmemory-policy=noeviction when Redis is already at maxmemory.
|
||||
//
|
||||
// ground truth: noeviction means Redis will never evict a key to make
|
||||
// room. A write that would grow memory usage while already at maxmemory
|
||||
// is instead rejected outright with an OOM error - the write does not
|
||||
// silently succeed and no key is evicted to make space.
|
||||
func dbRedisTTLEvictionTest() testkit.Test {
|
||||
func dbRedisNoevictionWriteTest() testkit.Test {
|
||||
prompt := `A Redis instance has maxmemory-policy set to "noeviction" and
|
||||
is currently at maxmemory. A client sends a write command that would
|
||||
increase memory usage further.
|
||||
@@ -89,12 +87,12 @@ increase memory usage further.
|
||||
What happens to that write? Respond with only one word: reject or evict.`
|
||||
|
||||
return testkit.Test{
|
||||
ID: "redis-ttl-eviction",
|
||||
ID: "redis-noeviction-write",
|
||||
Category: "databases",
|
||||
Subcategory: "redis",
|
||||
Description: "Determine that a write is rejected (not silently allowed, and nothing evicted) under maxmemory-policy=noeviction at maxmemory.",
|
||||
Prompt: prompt,
|
||||
Eval: dbExactAnswer("reject"),
|
||||
Eval: eval.ExactToken("reject"),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -157,7 +155,7 @@ pipelined commands? Respond with only "yes" or "no".`
|
||||
Subcategory: "redis",
|
||||
Description: "Confirm plain pipelining (unlike MULTI/EXEC) gives no guarantee against another client's commands interleaving.",
|
||||
Prompt: prompt,
|
||||
Eval: dbExactAnswer("no"),
|
||||
Eval: eval.ExactToken("no"),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -167,50 +165,34 @@ pipelined commands? Respond with only "yes" or "no".`
|
||||
// word "keys" elsewhere in prose ("the keys to fixing this...").
|
||||
var dbKeysCommandPattern = regexp.MustCompile("(?:`KEYS`|\\bKEYS\\s*\\*)")
|
||||
|
||||
// dbNegationCuePattern matches a word that turns a mention of the KEYS
|
||||
// command into a warning against running it in production, rather than an
|
||||
// instruction to run it.
|
||||
var dbNegationCuePattern = regexp.MustCompile(`(?i)\b(don'?t|do not|never|avoid|instead of|not|shouldn'?t|should not|rather than|without|no need|danger|dangerous)\b`)
|
||||
|
||||
// dbNegationWindow is how many characters before a KEYS-command mention
|
||||
// dbNegationWindow is how many characters around a KEYS-command mention
|
||||
// are searched for a negation cue.
|
||||
const dbNegationWindow = 60
|
||||
|
||||
// dbNegationWindowStart returns the earliest byte offset in response to
|
||||
// search for a negation cue before a KEYS-command mention starting at
|
||||
// start: the current line, extended back at most dbNegationWindow
|
||||
// characters.
|
||||
func dbNegationWindowStart(response string, start int) int {
|
||||
lineStart := strings.LastIndexByte(response[:start], '\n') + 1
|
||||
return max(start-dbNegationWindow, lineStart)
|
||||
}
|
||||
|
||||
// dbNoBareKeysInProd scores full credit unless the response instructs
|
||||
// running "KEYS *" against a production instance without any negating
|
||||
// context (a warning not to, or an instruction to use SCAN instead is
|
||||
// fine). This is deliberately not eval.NotContains, which would also zero
|
||||
// out the best possible answer: one that correctly explains why NOT to run
|
||||
// KEYS in production.
|
||||
//
|
||||
// Delegates to eval.NoUnnegatedMention (D5), the primitive shared with
|
||||
// kubernetes.go, security.go, and delivery_git.go's equivalent guards.
|
||||
func dbNoBareKeysInProd() eval.Evaluator {
|
||||
return eval.EvaluatorFunc(func(_ context.Context, response string) eval.Score {
|
||||
matches := dbKeysCommandPattern.FindAllStringIndex(response, -1)
|
||||
if len(matches) == 0 {
|
||||
return eval.Score{Value: 1, Detail: "no mention of running the KEYS command"}
|
||||
}
|
||||
for _, loc := range matches {
|
||||
start := loc[0]
|
||||
windowStart := dbNegationWindowStart(response, start)
|
||||
if !dbNegationCuePattern.MatchString(response[windowStart:start]) {
|
||||
return eval.Score{Value: 0, Detail: "unnegated mention of running the KEYS command"}
|
||||
}
|
||||
}
|
||||
return eval.Score{Value: 1, Detail: "every mention of KEYS is negated"}
|
||||
})
|
||||
return eval.NoUnnegatedMention(dbKeysCommandPattern, dbNegationWindow, nil)
|
||||
}
|
||||
|
||||
// dbRedisScanVsKeysTest: require SCAN over KEYS for a hot-path production
|
||||
// lookup, using a negation-aware guard so an answer that correctly warns
|
||||
// against KEYS is not penalized for mentioning it.
|
||||
//
|
||||
// ground truth (DC9): KEYS * is an O(n) full-keyspace scan that blocks
|
||||
// Redis's single command-processing thread for its entire duration - on a
|
||||
// production instance holding millions of keys, called thousands of times
|
||||
// per second, this stalls every other client for the scan's whole
|
||||
// duration. SCAN instead walks the keyspace incrementally via a cursor,
|
||||
// returning a small batch per call and never blocking the server for more
|
||||
// than that batch, making it the safe hot-path replacement.
|
||||
func dbRedisScanVsKeysTest() testkit.Test {
|
||||
prompt := `A request handler that runs thousands of times per second
|
||||
against a production Redis instance holding millions of keys currently
|
||||
@@ -254,7 +236,7 @@ specific message it missed while disconnected? Respond with only "yes" or
|
||||
Subcategory: "redis",
|
||||
Description: "Confirm Redis pub/sub gives no delivery guarantee (no backlog/replay) to a subscriber that was disconnected when a message was published.",
|
||||
Prompt: prompt,
|
||||
Eval: dbExactAnswer("no"),
|
||||
Eval: eval.ExactToken("no"),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -266,9 +248,16 @@ specific message it missed while disconnected? Respond with only "yes" or
|
||||
// the server does not process any other client's command, so a script is
|
||||
// guaranteed to run start-to-finish with nothing interleaved.
|
||||
func dbRedisLuaAtomicityTest() testkit.Test {
|
||||
prompt := `While one client's Lua script is executing via EVAL, can any
|
||||
other client's command run on the Redis server before that script
|
||||
finishes? Respond with only "yes" or "no".`
|
||||
// DC6: narrowed to a data command (SET) specifically - "any other
|
||||
// command whatsoever" is a broader claim than Redis's single-threaded
|
||||
// script guarantee actually covers, since some non-data/administrative
|
||||
// commands can be processed in certain configurations while a script
|
||||
// runs. A data command like SET (one that reads/writes the keyspace)
|
||||
// is unambiguously blocked, matching the ground truth exactly.
|
||||
prompt := `While one client's Lua script is executing via EVAL, can
|
||||
another client's SET command (a data command touching the keyspace) run
|
||||
on the Redis server before that script finishes? Respond with only "yes"
|
||||
or "no".`
|
||||
|
||||
return testkit.Test{
|
||||
ID: "redis-lua-atomicity",
|
||||
@@ -276,7 +265,7 @@ finishes? Respond with only "yes" or "no".`
|
||||
Subcategory: "redis",
|
||||
Description: "Confirm a Lua script run via EVAL executes atomically, with no other client command able to interleave.",
|
||||
Prompt: prompt,
|
||||
Eval: dbExactAnswer("no"),
|
||||
Eval: eval.ExactToken("no"),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -44,8 +44,8 @@ func TestDBRedisStructureChoiceTest_Eval(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestDBRedisTTLEvictionTest_Eval(t *testing.T) {
|
||||
tc := dbRedisTTLEvictionTest()
|
||||
func TestDBRedisNoevictionWriteTest_Eval(t *testing.T) {
|
||||
tc := dbRedisNoevictionWriteTest()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
|
||||
@@ -14,7 +14,7 @@ func registerDBSQLTuningTests(r *testkit.Registry) {
|
||||
r.Register(dbSQLNullInWhereTest())
|
||||
r.Register(dbSQLGroupByHavingTest())
|
||||
r.Register(dbSQLWindowRankTest())
|
||||
r.Register(dbSQLCompositeIndexOrderTest())
|
||||
r.Register(dbSQLCompositeIndexSkipColumnTest())
|
||||
r.Register(dbSQLKeysetPaginationTest())
|
||||
r.Register(dbSQLCoveringIndexTest())
|
||||
r.Register(dbSQLEquivalentRewriteTest())
|
||||
@@ -336,12 +336,18 @@ ORDER BY region;
|
||||
Respond with only a JSON array of objects, ordered exactly as the query's
|
||||
ORDER BY would return them: [{"region":"...","total":<number>}, ...]`
|
||||
|
||||
evaluator := eval.Mean(
|
||||
dbJSONArrayLength(len(dbSQLGroupByHavingWant)),
|
||||
eval.JSONField("[0].region", dbSQLGroupByHavingWant[0].Region),
|
||||
eval.JSONField("[0].total", dbSQLGroupByHavingWant[0].Total),
|
||||
eval.JSONField("[1].region", dbSQLGroupByHavingWant[1].Region),
|
||||
eval.JSONField("[1].total", dbSQLGroupByHavingWant[1].Total),
|
||||
// D9: weight the array-length check 2x each individual field check, so
|
||||
// a response that leaks an extra group through (e.g. forgetting to
|
||||
// apply the HAVING filter) cannot stay close to full credit just by
|
||||
// getting the two checked elements right - the structural correctness
|
||||
// of "exactly the right number of groups" matters more than any one
|
||||
// field.
|
||||
evaluator := eval.All(
|
||||
eval.W(dbJSONArrayLength(len(dbSQLGroupByHavingWant)), 2),
|
||||
eval.W(eval.JSONField("[0].region", dbSQLGroupByHavingWant[0].Region), 1),
|
||||
eval.W(eval.JSONField("[0].total", dbSQLGroupByHavingWant[0].Total), 1),
|
||||
eval.W(eval.JSONField("[1].region", dbSQLGroupByHavingWant[1].Region), 1),
|
||||
eval.W(eval.JSONField("[1].total", dbSQLGroupByHavingWant[1].Total), 1),
|
||||
)
|
||||
|
||||
return testkit.Test{
|
||||
@@ -440,44 +446,74 @@ in each department? Respond with only a JSON object:
|
||||
}
|
||||
}
|
||||
|
||||
// dbSQLCompositeIndexSchema is the inline schema + query for
|
||||
// dbSQLCompositeIndexOrderTest.
|
||||
const dbSQLCompositeIndexSchema = `Table: sessions(id bigserial, user_id bigint, created_at timestamptz, ip inet)
|
||||
// dbSQLCompositeIndexSkipSchema is the inline schema + index + query for
|
||||
// dbSQLCompositeIndexSkipColumnTest.
|
||||
//
|
||||
// D10: this test replaces the former sql-composite-index-order, which
|
||||
// duplicated pg-index-choice (databases_postgres.go) almost exactly -
|
||||
// same "pick equality-then-range column order" question shape, just a
|
||||
// different table/column naming. This test instead covers a genuinely
|
||||
// distinct btree composite-index rule: a query that skips the middle
|
||||
// column of a 3-column index entirely.
|
||||
const dbSQLCompositeIndexSkipSchema = `Table: events(id bigserial, tenant_id bigint, event_type text, created_at timestamptz)
|
||||
|
||||
Index: CREATE INDEX idx_events_tenant_type_created ON events(tenant_id, event_type, created_at);
|
||||
|
||||
Query:
|
||||
SELECT * FROM sessions
|
||||
WHERE user_id = $1 AND created_at > $2
|
||||
ORDER BY created_at DESC
|
||||
LIMIT 50;`
|
||||
SELECT * FROM events
|
||||
WHERE tenant_id = 42 AND created_at > '2026-01-01';`
|
||||
|
||||
// dbSQLCompositeIndexOrderTest: pick the composite index column order
|
||||
// (equality column first, then the range/sort column) that best serves an
|
||||
// inline query.
|
||||
// dbSQLCompositeIndexSkipColumnTest: identify which column of a 3-column
|
||||
// composite index actually serves as an index seek (Index Cond) versus
|
||||
// which column can only be applied as a post-index Filter, when the
|
||||
// query's WHERE clause skips the index's middle column entirely.
|
||||
//
|
||||
// ground truth: user_id is filtered by exact equality; created_at is
|
||||
// filtered by a range AND drives ORDER BY. Leading a composite index with
|
||||
// the equality column narrows to one user's rows first; following with
|
||||
// created_at then serves both the range filter and the sort from the
|
||||
// index directly. Reversing the order (created_at, user_id) would force
|
||||
// scanning every session in the time range across all users before
|
||||
// filtering by user_id.
|
||||
func dbSQLCompositeIndexOrderTest() testkit.Test {
|
||||
prompt := `Here is a table schema and a query that runs frequently:
|
||||
// ground truth: the composite btree index (tenant_id, event_type,
|
||||
// created_at) is physically sorted first by tenant_id, then by event_type
|
||||
// WITHIN each tenant_id, then by created_at WITHIN each (tenant_id,
|
||||
// event_type) pair. The query constrains tenant_id (equality) and
|
||||
// created_at (range) but never constrains event_type - the index's middle
|
||||
// column - at all. Because created_at is only sorted within each
|
||||
// event_type sub-group (not globally within a tenant_id), Postgres cannot
|
||||
// use created_at as part of the index seek when event_type is
|
||||
// unconstrained: it can only narrow the seek to the tenant_id=42 range
|
||||
// (Index Cond: tenant_id = 42) and must apply created_at > '2026-01-01' as
|
||||
// a Filter evaluated against every row in that range, not as part of the
|
||||
// seek itself. This is the standard btree composite-index rule that only
|
||||
// a contiguous, leading, query-constrained prefix of index columns can be
|
||||
// used for seeking - skipping an unconstrained column strands every
|
||||
// column after it as filter-only.
|
||||
func dbSQLCompositeIndexSkipColumnTest() testkit.Test {
|
||||
prompt := `Here is a table schema, a composite index, and a query that
|
||||
runs frequently:
|
||||
|
||||
` + "```\n" + dbSQLCompositeIndexSchema + "\n```" + `
|
||||
` + "```\n" + dbSQLCompositeIndexSkipSchema + "\n```" + `
|
||||
|
||||
Which two columns, in which order, should a single composite index be
|
||||
defined on to best serve this exact query (equality filters should lead a
|
||||
composite index, ahead of range/sort columns)? Respond with only a JSON
|
||||
array of the two column names in index-definition order, e.g. ["a","b"].`
|
||||
Note the query does NOT filter on event_type, the index's middle column.
|
||||
A btree index can only be used as a seek (an Index Cond) for a contiguous
|
||||
prefix of its columns that the query actually constrains, starting from
|
||||
the leading column; it cannot skip an unconstrained column to seek on a
|
||||
later one, because index entries are only sorted by that later column
|
||||
WITHIN each value of the skipped column.
|
||||
|
||||
Which single column does this index serve as an efficient Index Cond
|
||||
(seek)? Which single column can only be applied as a Filter after the
|
||||
index narrows to the matching range (not as part of the index seek
|
||||
itself)? Respond with only a JSON object:
|
||||
{"index_cond_column":"<column>","filter_only_column":"<column>"}`
|
||||
|
||||
evaluator := eval.Mean(
|
||||
eval.JSONField("index_cond_column", "tenant_id"),
|
||||
eval.JSONField("filter_only_column", "created_at"),
|
||||
)
|
||||
|
||||
return testkit.Test{
|
||||
ID: "sql-composite-index-order",
|
||||
ID: "sql-composite-index-skip-column",
|
||||
Category: "databases",
|
||||
Subcategory: "sql-tuning",
|
||||
Description: "Pick composite index column order (user_id, created_at) for an inline equality+range+sort query.",
|
||||
Description: "Identify that skipping a 3-column composite index's middle column strands the query's range filter as a post-index Filter, not a seek.",
|
||||
Prompt: prompt,
|
||||
Eval: eval.JSONStringArrayEquals([]string{"user_id", "created_at"}),
|
||||
Eval: evaluator,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -507,7 +543,7 @@ word: offset or keyset.`
|
||||
Subcategory: "sql-tuning",
|
||||
Description: "Pick keyset/seek pagination over OFFSET pagination for constant latency 20M rows deep into a 50M-row table.",
|
||||
Prompt: prompt,
|
||||
Eval: dbExactAnswer("keyset"),
|
||||
Eval: eval.ExactToken("keyset"),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -535,7 +571,7 @@ only "yes" or "no".`
|
||||
Subcategory: "sql-tuning",
|
||||
Description: "Confirm an index containing every column a query needs (customer_id, status) enables an index-only scan with no heap visit.",
|
||||
Prompt: prompt,
|
||||
Eval: dbExactAnswer("yes"),
|
||||
Eval: eval.ExactToken("yes"),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -202,10 +202,22 @@ func TestDBSQLGroupByHavingTest_Eval(t *testing.T) {
|
||||
}{
|
||||
{"correct", `[{"region":"APAC","total":400},{"region":"EU","total":450}]`, 1},
|
||||
{"correct fenced with prose", "Result:\n```json\n[{\"region\":\"APAC\",\"total\":400},{\"region\":\"EU\",\"total\":450}]\n```", 1},
|
||||
{"wrong order", `[{"region":"EU","total":450},{"region":"APAC","total":400}]`, 0.2},
|
||||
{"missing a group", `[{"region":"APAC","total":400}]`, 0.4},
|
||||
{"wrong total for one group", `[{"region":"APAC","total":401},{"region":"EU","total":450}]`, 0.8},
|
||||
{"forgot HAVING filter, includes US", `[{"region":"APAC","total":400},{"region":"EU","total":450},{"region":"US","total":210}]`, 0.8},
|
||||
// D9: dbJSONArrayLength is now weighted 2x each individual field
|
||||
// check (total weight 6, not 5), so these expected values shift
|
||||
// from the old equal-weight Mean.
|
||||
{"wrong order", `[{"region":"EU","total":450},{"region":"APAC","total":400}]`, 2.0 / 6.0},
|
||||
{"missing a group", `[{"region":"APAC","total":400}]`, 2.0 / 6.0},
|
||||
{"wrong total for one group", `[{"region":"APAC","total":401},{"region":"EU","total":450}]`, 5.0 / 6.0},
|
||||
{
|
||||
// D9 bug probe: this exact case is D9's motivating scenario -
|
||||
// leaking an extra group through by forgetting the HAVING
|
||||
// filter now scores lower (4/6) than it did under the old
|
||||
// equal-weight design (4/5=0.8), since the structural
|
||||
// array-length mismatch now costs more.
|
||||
name: "forgot HAVING filter, includes US (D9 bug probe)",
|
||||
response: `[{"region":"APAC","total":400},{"region":"EU","total":450},{"region":"US","total":210}]`,
|
||||
want: 4.0 / 6.0,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
@@ -273,19 +285,34 @@ func TestDBSQLWindowRankTest_Eval(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestDBSQLCompositeIndexOrderTest_Eval(t *testing.T) {
|
||||
tc := dbSQLCompositeIndexOrderTest()
|
||||
func TestDBSQLCompositeIndexSkipColumnTest_Eval(t *testing.T) {
|
||||
tc := dbSQLCompositeIndexSkipColumnTest()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
want float64
|
||||
}{
|
||||
{"correct", `["user_id", "created_at"]`, 1},
|
||||
{"correct fenced", "```json\n[\"user_id\", \"created_at\"]\n```", 1},
|
||||
{"reversed order", `["created_at", "user_id"]`, 0},
|
||||
{"missing a column", `["user_id"]`, 0},
|
||||
{"wrong column entirely", `["user_id", "ip"]`, 0},
|
||||
{"correct", `{"index_cond_column":"tenant_id","filter_only_column":"created_at"}`, 1},
|
||||
{"correct fenced with prose", "```json\n{\"index_cond_column\":\"tenant_id\",\"filter_only_column\":\"created_at\"}\n```", 1},
|
||||
{
|
||||
name: "swapped: both wrong",
|
||||
response: `{"index_cond_column":"created_at","filter_only_column":"tenant_id"}`,
|
||||
want: 0,
|
||||
},
|
||||
{
|
||||
// D10 bug probe: the natural wrong answer is assuming every
|
||||
// column present in the index (including the skipped middle
|
||||
// column, event_type) serves as an index seek.
|
||||
name: "wrong: assumes the skipped middle column also seeks (D10 bug probe)",
|
||||
response: `{"index_cond_column":"event_type","filter_only_column":"created_at"}`,
|
||||
want: 0.5,
|
||||
},
|
||||
{
|
||||
name: "wrong: filter_only_column names the skipped middle column instead",
|
||||
response: `{"index_cond_column":"tenant_id","filter_only_column":"event_type"}`,
|
||||
want: 0.5,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
|
||||
@@ -85,12 +85,21 @@ with only the exact command line, nothing else.`
|
||||
Subcategory: "containers",
|
||||
Description: "Trace ENTRYPOINT+CMD exec-form interaction to the exact resulting startup command line.",
|
||||
Prompt: prompt,
|
||||
Eval: eval.Equals("python3 app.py --port 8080"),
|
||||
Eval: eval.ExactToken("python3 app.py --port 8080"),
|
||||
}
|
||||
}
|
||||
|
||||
// delDockerMultistageSizeBenefitTest: explain how a multi-stage build
|
||||
// shrinks the final image and name a minimal final-stage base.
|
||||
//
|
||||
// ground truth (DC9): a multi-stage build compiles in one stage (here the
|
||||
// full "golang:1.25" toolchain image) and copies only the compiled binary
|
||||
// artifact into a separate final stage, whose base image never needs the
|
||||
// compiler, package caches, or any other build-time tooling. Discarding
|
||||
// the build stage's layers from the shipped image (by starting the final
|
||||
// stage from a minimal base - scratch, distroless, or alpine - rather than
|
||||
// continuing to build image) is what drops the image from over 900MB to a
|
||||
// small final size.
|
||||
func delDockerMultistageSizeBenefitTest() testkit.Test {
|
||||
prompt := `A Go service's Dockerfile currently uses a single
|
||||
"golang:1.25" base image (full Go toolchain, apt package caches, and build
|
||||
@@ -99,8 +108,13 @@ over 900MB. Explain how switching to a multi-stage build reduces the final
|
||||
image size, and name the kind of minimal base image you would copy the
|
||||
compiled binary into for the final stage.`
|
||||
|
||||
// D8: require the actual multi-stage build artifact (COPY --from, or
|
||||
// naming the build/final stage split), not just the bare word
|
||||
// "multi-stage" - a response could say "multi-stage" in passing
|
||||
// without demonstrating it understands the mechanism that actually
|
||||
// drops the toolchain layer.
|
||||
evaluator := eval.All(
|
||||
eval.W(eval.ContainsAll("multi-stage"), 2),
|
||||
eval.W(eval.ContainsAny("multi-stage", "COPY --from", "final stage", "build stage"), 2),
|
||||
eval.W(eval.ContainsAny("scratch", "distroless", "alpine"), 2),
|
||||
)
|
||||
|
||||
@@ -136,7 +150,8 @@ Final stage (this is what actually ships): base image alpine at 8MB, plus a
|
||||
COPY of the compiled binary from the build stage adds 22MB.
|
||||
|
||||
What is the total size, in MB, of the final shipped image? Respond with
|
||||
only the number.`
|
||||
only the number, digits only, with no commas, units, or other text (for
|
||||
example: 30).`
|
||||
|
||||
return testkit.Test{
|
||||
ID: "docker-image-size-math",
|
||||
@@ -165,8 +180,12 @@ instruction that makes the container run as a non-root user, and name the
|
||||
Linux capability set it should end up running with at runtime, given it
|
||||
needs no special privileges.`
|
||||
|
||||
// D8: bare "USER" is a common English word (the response could say
|
||||
// "the user experience" and still match) - require the actual USER
|
||||
// instruction artifact (a concrete non-root UID/username, or the
|
||||
// adduser/useradd command that creates one) instead.
|
||||
evaluator := eval.All(
|
||||
eval.W(eval.ContainsAll("USER"), 2),
|
||||
eval.W(eval.ContainsAny("USER 1000", "USER app", "USER nonroot", "adduser", "useradd"), 2),
|
||||
eval.W(eval.ContainsAny("cap-drop=all", "cap-drop all", "drop all capabilities", "capdrop=all", "drop all"), 2),
|
||||
)
|
||||
|
||||
@@ -188,6 +207,15 @@ var delDockerHealthcheckRetriesWant = 3
|
||||
|
||||
// delDockerHealthcheckSemanticsTest: read the HEALTHCHECK retries value
|
||||
// correctly rather than the interval or timeout values.
|
||||
//
|
||||
// ground truth (DC9): Docker's HEALTHCHECK instruction's three timing
|
||||
// flags each control a different thing - --interval is the time between
|
||||
// probe runs, --timeout is how long a single probe may take before it
|
||||
// counts as failed, and --retries is the number of CONSECUTIVE failed
|
||||
// probes required before Docker flips the container's health status to
|
||||
// "unhealthy". The instruction here sets --retries=3, so exactly 3
|
||||
// consecutive failures (not --interval's 30 or --timeout's 3-with-a-unit)
|
||||
// is the answer.
|
||||
func delDockerHealthcheckSemanticsTest() testkit.Test {
|
||||
prompt := `Given this Dockerfile instruction:
|
||||
|
||||
@@ -330,22 +358,35 @@ Respond with only a JSON object:
|
||||
|
||||
// delDockerLayerCountDockerfile is the inline numbered Dockerfile for
|
||||
// delDockerLayerCountTest.
|
||||
//
|
||||
// D1: this fixture previously included a "WORKDIR /app" line and claimed
|
||||
// WORKDIR was metadata-only, matching a widely-repeated but factually
|
||||
// wrong claim. Verified empirically on this machine (both BuildKit and,
|
||||
// separately, `docker inspect` layer counts) that WORKDIR DOES create a
|
||||
// real new filesystem layer: building the equivalent Dockerfile WITH a
|
||||
// WORKDIR line yields 3 new layers over the alpine:3.20 base, not 2.
|
||||
// WORKDIR is dropped from this fixture entirely (rather than trying to
|
||||
// special-case it) so the only layer-creating instructions present are the
|
||||
// uncontested ones (RUN/COPY), keeping want=2 unambiguous.
|
||||
// delivery_containers_test.go's TestDelDockerLayerCountWant_GroundTruth
|
||||
// verifies this exact fixture by actually building it with docker,
|
||||
// skipping if docker is unavailable.
|
||||
const delDockerLayerCountDockerfile = `1: FROM alpine:3.20
|
||||
2: WORKDIR /app
|
||||
3: COPY app /app/app
|
||||
4: RUN chmod +x /app/app
|
||||
5: ENV PORT=8080
|
||||
6: EXPOSE 8080
|
||||
7: USER 1000
|
||||
8: ENTRYPOINT ["/app/app"]`
|
||||
2: COPY app /app/app
|
||||
3: RUN chmod +x /app/app
|
||||
4: ENV PORT=8080
|
||||
5: EXPOSE 8080
|
||||
6: USER 1000
|
||||
7: ENTRYPOINT ["/app/app"]`
|
||||
|
||||
// delDockerLayerCountWant is the number of new filesystem layers this
|
||||
// Dockerfile adds on top of its base image.
|
||||
//
|
||||
// ground truth: only RUN, COPY, and ADD instructions create a new
|
||||
// filesystem layer; WORKDIR, ENV, EXPOSE, USER, ENTRYPOINT, CMD, and LABEL
|
||||
// only update image metadata/config and add no layer. This Dockerfile has
|
||||
// exactly one COPY (line 3) and one RUN (line 4): 2 new layers.
|
||||
// filesystem layer; ENV, EXPOSE, USER, ENTRYPOINT, CMD, and LABEL only
|
||||
// update image metadata/config and add no layer (WORKDIR is NOT in this
|
||||
// metadata-only list - see the fixture comment above). This Dockerfile has
|
||||
// exactly one COPY (line 2) and one RUN (line 3): 2 new layers.
|
||||
var delDockerLayerCountWant = 2
|
||||
|
||||
// delDockerLayerCountTest: count only the new filesystem layers a
|
||||
|
||||
@@ -2,6 +2,11 @@ package tests
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
@@ -86,6 +91,15 @@ func TestDelDockerMultistageSizeBenefitTest_Eval(t *testing.T) {
|
||||
response: "Switch the final image to alpine instead of golang.",
|
||||
want: 0.5,
|
||||
},
|
||||
{
|
||||
// D8 bug probe: a terse, technical-syntax answer that never
|
||||
// says the literal word "multi-stage" but genuinely
|
||||
// demonstrates the mechanism (COPY --from a named build
|
||||
// stage) must still score full credit.
|
||||
name: "correct: technical syntax only, never says 'multi-stage' literally (D8 bug probe)",
|
||||
response: "In the final stage: COPY --from=builder /app /app, using scratch as the base.",
|
||||
want: 1,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
@@ -110,6 +124,13 @@ func TestDelDockerImageSizeMathTest_Eval(t *testing.T) {
|
||||
{name: "with units spelled out", response: "8 + 22 = 30 megabytes.", want: 1},
|
||||
{name: "wrong: sums every layer including the discarded build stage", response: "534", want: 0},
|
||||
{name: "wrong: only the alpine base, forgets the binary layer", response: "8", want: 0},
|
||||
{
|
||||
// DC1 bug probe: a glued unit suffix ("30MB", no space) must
|
||||
// extract the same as the spaced form.
|
||||
name: "glued unit suffix, no space (DC1 bug probe)",
|
||||
response: "The final image is 30MB.",
|
||||
want: 1,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
@@ -150,8 +171,18 @@ func TestDelDockerNonrootUserCapsTest_Eval(t *testing.T) {
|
||||
want: 0,
|
||||
},
|
||||
{
|
||||
name: "wrong: only addresses the user, not capabilities",
|
||||
// D8: a generic "add a USER instruction" mention with no
|
||||
// concrete UID/username/adduser artifact is exactly the
|
||||
// vacuous echo of the prompt's own wording ("this Dockerfile
|
||||
// currently has no USER instruction") that this fix targets -
|
||||
// it no longer earns partial credit on its own.
|
||||
name: "wrong: generic USER mention with no concrete artifact, still misses capabilities",
|
||||
response: "Add a USER instruction pointing at a non-root UID.",
|
||||
want: 0,
|
||||
},
|
||||
{
|
||||
name: "partial: concrete USER artifact but capabilities still missing",
|
||||
response: "Add `USER 1000` to the Dockerfile.",
|
||||
want: 0.5,
|
||||
},
|
||||
}
|
||||
@@ -280,8 +311,17 @@ func TestDelDockerLayerCountTest_Eval(t *testing.T) {
|
||||
{name: "bare number", response: "2", want: 1},
|
||||
{name: "sentence form", response: "This Dockerfile adds 2 new layers.", want: 1},
|
||||
{name: "explains then answers", response: "Only COPY and RUN create layers here, so the answer is 2.", want: 1},
|
||||
{name: "wrong: counts every instruction", response: "8", want: 0},
|
||||
{name: "wrong: counts every instruction", response: "7", want: 0},
|
||||
{name: "wrong: forgets the RUN layer", response: "1", want: 0},
|
||||
{
|
||||
// D1 bug probe: WORKDIR is not in this fixture at all, but a
|
||||
// wrong answer that (incorrectly) assumes it's present AND
|
||||
// counts it as a layer-creating instruction must still score 0
|
||||
// (want stays 2 either way, since WORKDIR is genuinely absent).
|
||||
name: "wrong: assumes a WORKDIR layer that is not in this fixture (D1 bug probe)",
|
||||
response: "3",
|
||||
want: 0,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
@@ -292,3 +332,77 @@ func TestDelDockerLayerCountTest_Eval(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// delDockerStripLineNumbers removes the "N: " prompt-display prefix from
|
||||
// each line of a numbered Dockerfile fixture, producing real Dockerfile
|
||||
// syntax that can actually be built.
|
||||
func delDockerStripLineNumbers(text string) string {
|
||||
lines := strings.Split(text, "\n")
|
||||
for i, line := range lines {
|
||||
if idx := strings.Index(line, ": "); idx >= 0 {
|
||||
lines[i] = line[idx+2:]
|
||||
}
|
||||
}
|
||||
return strings.Join(lines, "\n")
|
||||
}
|
||||
|
||||
// delDockerInspectLayerCount runs `docker inspect <ref> --format
|
||||
// '{{len .RootFS.Layers}}'` and returns the parsed layer count.
|
||||
func delDockerInspectLayerCount(t *testing.T, ref string) int {
|
||||
t.Helper()
|
||||
// #nosec G204 -- ref is always a fixed literal image tag/reference
|
||||
// built earlier in this same test, never external/user-controlled
|
||||
// input.
|
||||
out, err := exec.Command("docker", "inspect", ref, "--format", "{{len .RootFS.Layers}}").Output() //nolint:gosec // see comment above
|
||||
if err != nil {
|
||||
t.Fatalf("docker inspect %s: %v", ref, err)
|
||||
}
|
||||
n, err := strconv.Atoi(strings.TrimSpace(string(out)))
|
||||
if err != nil {
|
||||
t.Fatalf("docker inspect %s: unparseable layer count %q: %v", ref, string(out), err)
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
// TestDelDockerLayerCountWant_GroundTruth verifies delDockerLayerCountWant
|
||||
// against a real Docker build (D1) rather than trusting the derivation
|
||||
// comment alone: it builds delDockerLayerCountDockerfile (line numbers
|
||||
// stripped) with a dummy "app" binary, then diffs the built image's
|
||||
// RootFS.Layers count against the alpine:3.20 base image's own layer
|
||||
// count. Skips if docker is not installed or its daemon is unreachable.
|
||||
func TestDelDockerLayerCountWant_GroundTruth(t *testing.T) {
|
||||
if _, err := exec.LookPath("docker"); err != nil {
|
||||
t.Skip("docker not installed")
|
||||
}
|
||||
if err := exec.Command("docker", "info").Run(); err != nil {
|
||||
t.Skip("docker daemon not reachable")
|
||||
}
|
||||
|
||||
dir := t.TempDir()
|
||||
dockerfile := delDockerStripLineNumbers(delDockerLayerCountDockerfile)
|
||||
if err := os.WriteFile(filepath.Join(dir, "Dockerfile"), []byte(dockerfile), 0o600); err != nil {
|
||||
t.Fatalf("write Dockerfile: %v", err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(dir, "app"), []byte("#!/bin/sh\necho hi\n"), 0o600); err != nil {
|
||||
t.Fatalf("write app: %v", err)
|
||||
}
|
||||
|
||||
const tag = "llmtestbench-layercount-groundtruth:test"
|
||||
// #nosec G204 -- tag is a fixed literal and dir is a t.TempDir() path,
|
||||
// never external/user-controlled input.
|
||||
build := exec.Command("docker", "build", "-t", tag, dir) //nolint:gosec // see comment above
|
||||
if out, err := build.CombinedOutput(); err != nil {
|
||||
t.Fatalf("docker build: %v\n%s", err, out)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
_ = exec.Command("docker", "rmi", tag).Run()
|
||||
})
|
||||
|
||||
totalLayers := delDockerInspectLayerCount(t, tag)
|
||||
baseLayers := delDockerInspectLayerCount(t, "alpine:3.20")
|
||||
newLayers := totalLayers - baseLayers
|
||||
|
||||
if newLayers != delDockerLayerCountWant {
|
||||
t.Fatalf("docker-verified new layer count = %d (total %d, base %d), want %d", newLayers, totalLayers, baseLayers, delDockerLayerCountWant)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2,8 +2,8 @@ package tests
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/lukaszraczylo/llm-testbench/internal/eval"
|
||||
"github.com/lukaszraczylo/llm-testbench/internal/testkit"
|
||||
@@ -23,66 +23,22 @@ func registerDeliveryGitTests(r *testkit.Registry) {
|
||||
}
|
||||
|
||||
// --- negation-aware helpers, shared across delivery_git.go and
|
||||
// delivery_release.go (same package). Duplicated in spirit from
|
||||
// operations/kubernetes's noLiveKubectlMutation/negationCuePattern rather
|
||||
// than imported from kubernetes.go, so this category's files stay
|
||||
// self-contained: round-2 authors work in isolated worktrees on
|
||||
// databases/security/delivery/ai in parallel, and depending on another
|
||||
// category's unexported symbols would create a merge-time coupling this
|
||||
// worktree cannot see or coordinate on. ---
|
||||
// delivery_release.go (same package), delegating to eval.NoUnnegatedMention
|
||||
// (D5) - the primitive unifying this file's, kubernetes.go's, security.go's,
|
||||
// and databases_redis.go's four independently-written equivalents. ---
|
||||
|
||||
// delNegationCuePattern matches a word that turns a mention of a forbidden
|
||||
// phrase into a warning against doing it, rather than an instruction to do
|
||||
// it (e.g. "never use bare --force", "must not silently modify").
|
||||
var delNegationCuePattern = regexp.MustCompile(`(?i)\b(don'?t|do not|never|avoid|instead of|not|cannot|can'?t|rather than|without|no need|must not|should not|shouldn'?t)\b`)
|
||||
|
||||
// delNegationWindow is how many characters before a forbidden-phrase match
|
||||
// delNegationWindow is how many characters around a forbidden-phrase match
|
||||
// are searched for a negation cue.
|
||||
const delNegationWindow = 60
|
||||
|
||||
// delNegationWindowStart returns the earliest byte offset in response to
|
||||
// search for a negation cue before a forbidden-phrase match starting at
|
||||
// start. The window is the current line, extended back to the start of the
|
||||
// immediately preceding line when that line is non-empty, so a hard-wrapped
|
||||
// sentence whose negation cue landed on the line above still counts. Either
|
||||
// way the window never reaches more than delNegationWindow characters
|
||||
// before start.
|
||||
func delNegationWindowStart(response string, start int) int {
|
||||
curLineStart := strings.LastIndexByte(response[:start], '\n') + 1
|
||||
|
||||
windowFloor := curLineStart
|
||||
if curLineStart > 0 {
|
||||
prevLineEnd := curLineStart - 1
|
||||
prevLineStart := strings.LastIndexByte(response[:prevLineEnd], '\n') + 1
|
||||
if strings.TrimSpace(response[prevLineStart:prevLineEnd]) != "" {
|
||||
windowFloor = prevLineStart
|
||||
}
|
||||
}
|
||||
|
||||
return max(start-delNegationWindow, windowFloor)
|
||||
}
|
||||
|
||||
// delNoUnnegatedMention returns an Evaluator scoring full credit unless
|
||||
// forbidden matches response at some position with no negation cue in the
|
||||
// preceding window - i.e. the response recommends the forbidden thing
|
||||
// outright rather than warning against it. This is deliberately not
|
||||
// eval.NotContains, which would also zero out the best possible answer (one
|
||||
// that correctly explains why NOT to do the forbidden thing).
|
||||
func delNoUnnegatedMention(forbidden *regexp.Regexp, safeDetail string) eval.Evaluator {
|
||||
return eval.EvaluatorFunc(func(_ context.Context, response string) eval.Score {
|
||||
matches := forbidden.FindAllStringIndex(response, -1)
|
||||
if len(matches) == 0 {
|
||||
return eval.Score{Value: 1, Detail: "no forbidden mention: " + safeDetail}
|
||||
}
|
||||
for _, loc := range matches {
|
||||
start := loc[0]
|
||||
windowStart := delNegationWindowStart(response, start)
|
||||
if !delNegationCuePattern.MatchString(response[windowStart:start]) {
|
||||
return eval.Score{Value: 0, Detail: "unnegated forbidden mention"}
|
||||
}
|
||||
}
|
||||
return eval.Score{Value: 1, Detail: "every forbidden mention is negated"}
|
||||
})
|
||||
// forbidden matches response at some position with no negation cue nearby -
|
||||
// i.e. the response recommends the forbidden thing outright rather than
|
||||
// warning against it. This is deliberately not eval.NotContains, which
|
||||
// would also zero out the best possible answer (one that correctly
|
||||
// explains why NOT to do the forbidden thing).
|
||||
func delNoUnnegatedMention(forbidden *regexp.Regexp) eval.Evaluator {
|
||||
return eval.NoUnnegatedMention(forbidden, delNegationWindow, nil)
|
||||
}
|
||||
|
||||
// delForceFlagPattern matches every "--force" mention. delForceWithLeaseSuffixPattern
|
||||
@@ -99,23 +55,7 @@ var delForceWithLeaseSuffixPattern = regexp.MustCompile(`(?i)^-with-lease`)
|
||||
// it always scores full credit; a response that recommends bare --force
|
||||
// outright scores zero.
|
||||
func delNoBareForcePush() eval.Evaluator {
|
||||
return eval.EvaluatorFunc(func(_ context.Context, response string) eval.Score {
|
||||
matches := delForceFlagPattern.FindAllStringIndex(response, -1)
|
||||
if len(matches) == 0 {
|
||||
return eval.Score{Value: 1, Detail: "no --force mention"}
|
||||
}
|
||||
for _, loc := range matches {
|
||||
start, end := loc[0], loc[1]
|
||||
if delForceWithLeaseSuffixPattern.MatchString(response[end:]) {
|
||||
continue // "--force-with-lease": the safe form, not bare force
|
||||
}
|
||||
windowStart := delNegationWindowStart(response, start)
|
||||
if !delNegationCuePattern.MatchString(response[windowStart:start]) {
|
||||
return eval.Score{Value: 0, Detail: "unnegated bare --force mention"}
|
||||
}
|
||||
}
|
||||
return eval.Score{Value: 1, Detail: "every bare --force mention is negated or absent"}
|
||||
})
|
||||
return eval.NoUnnegatedMention(delForceFlagPattern, delNegationWindow, delForceWithLeaseSuffixPattern)
|
||||
}
|
||||
|
||||
// delGitBisectCommitCount is the number of candidate commits between the
|
||||
@@ -132,6 +72,13 @@ const delGitBisectCommitCount = 137
|
||||
// ceil(log2(N)): 2^7 = 128 < 137 <= 256 = 2^8, so ceil(log2(137)) = 8.
|
||||
// delivery_git_test.go recomputes this with math.Ceil(math.Log2(137))
|
||||
// rather than trusting the hardcoded literal alone.
|
||||
//
|
||||
// DC7: git's own interactive CLI, when you run `git bisect start`, prints
|
||||
// a "roughly N steps" hint computed differently (closer to floor(log2(N)),
|
||||
// giving "roughly 7 steps" for 137 candidates here) - a typical-case
|
||||
// estimate, not a guaranteed upper bound. The prompt deliberately asks for
|
||||
// the "maximum number of steps needed" to disambiguate that it wants the
|
||||
// true worst case (8), not git's own displayed estimate (7).
|
||||
var delGitBisectStepsWant = 8
|
||||
|
||||
// delGitBisectStepsTest: derive the worst-case number of `git bisect`
|
||||
@@ -209,6 +156,13 @@ commit6: "test(exec): add toolchain-missing skip cases for GoRun/PyRun/CRun"`
|
||||
|
||||
// delGitConventionalCommitClassifyTest: classify 6 commit messages, one per
|
||||
// standard conventional-commit type, by their type prefix.
|
||||
//
|
||||
// ground truth (DC9): each commit message's own prefix names its
|
||||
// Conventional Commits type directly - commit1="feat(...)" -> feat,
|
||||
// commit2="fix(...)" -> fix, commit3="chore(...)" -> chore,
|
||||
// commit4="docs: ..." -> docs, commit5="refactor(...)" -> refactor,
|
||||
// commit6="test(...)" -> test. No inference beyond reading the stated
|
||||
// prefix is required.
|
||||
func delGitConventionalCommitClassifyTest() testkit.Test {
|
||||
prompt := `Here are 6 commit messages from a project using Conventional
|
||||
Commits:
|
||||
@@ -319,7 +273,7 @@ with only the hook name.`
|
||||
Subcategory: "git",
|
||||
Description: "Pick commit-msg as the hook that enforces a commit message format at commit time.",
|
||||
Prompt: prompt,
|
||||
Eval: eval.Equals("commit-msg"),
|
||||
Eval: eval.ExactToken("commit-msg"),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -333,6 +287,34 @@ with only the hook name.`
|
||||
// Reversing the order (checkout first) would move HEAD away from the
|
||||
// detached commits before a branch ref exists to save them, losing the
|
||||
// work to garbage collection.
|
||||
// delGitDetachedHeadRecoveryWant lists every accepted git-subcommand-array
|
||||
// answer for delGitDetachedHeadRecoveryTest (D6): the two-step "branch" then
|
||||
// "checkout" sequence from the ground truth above, or either of the two
|
||||
// single, atomic commands that create the branch and switch onto it in one
|
||||
// step - "git checkout -b <name>" (named here by its subcommand alone,
|
||||
// "checkout") or the newer "git switch -c <name>" ("switch"). Both
|
||||
// single-command forms are equally safe: HEAD never sits in an
|
||||
// unprotected, still-detached state between two separate commands.
|
||||
var delGitDetachedHeadRecoveryWant = [][]string{
|
||||
{"branch", "checkout"},
|
||||
{"checkout"},
|
||||
{"switch"},
|
||||
}
|
||||
|
||||
// delGitDetachedHeadRecoveryEval scores full credit when the response's
|
||||
// JSON string array exactly matches any one of
|
||||
// delGitDetachedHeadRecoveryWant's accepted sequences (D6).
|
||||
func delGitDetachedHeadRecoveryEval() eval.Evaluator {
|
||||
return eval.EvaluatorFunc(func(ctx context.Context, response string) eval.Score {
|
||||
for _, want := range delGitDetachedHeadRecoveryWant {
|
||||
if eval.JSONStringArrayEquals(want).Evaluate(ctx, response).Value == 1 {
|
||||
return eval.Score{Value: 1, Detail: fmt.Sprintf("matches accepted sequence %v", want)}
|
||||
}
|
||||
}
|
||||
return eval.Score{Value: 0, Detail: fmt.Sprintf("does not match any accepted sequence %v", delGitDetachedHeadRecoveryWant)}
|
||||
})
|
||||
}
|
||||
|
||||
func delGitDetachedHeadRecoveryTest() testkit.Test {
|
||||
prompt := `You ran "git checkout <old-commit-sha>" directly (not a
|
||||
branch name), then made 3 new commits while HEAD was detached. You only now
|
||||
@@ -346,9 +328,9 @@ losing them. Respond with only a JSON array of subcommand names, e.g.
|
||||
ID: "git-detached-head-recovery",
|
||||
Category: "delivery",
|
||||
Subcategory: "git",
|
||||
Description: "Order git branch then git checkout to save 3 detached-HEAD commits onto a new branch.",
|
||||
Description: "Order git branch then git checkout to save 3 detached-HEAD commits onto a new branch (or the single-command checkout -b / switch -c equivalent).",
|
||||
Prompt: prompt,
|
||||
Eval: eval.JSONStringArrayEquals([]string{"branch", "checkout"}),
|
||||
Eval: delGitDetachedHeadRecoveryEval(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -374,7 +356,7 @@ Respond with only one word: cherry-pick or revert.`
|
||||
Subcategory: "git",
|
||||
Description: "Pick git revert (not cherry-pick) to undo an already-shared commit without rewriting history.",
|
||||
Prompt: prompt,
|
||||
Eval: eval.Equals("revert"),
|
||||
Eval: eval.ExactToken("revert"),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -471,6 +453,6 @@ dropped or kept.`
|
||||
Subcategory: "git",
|
||||
Description: "State that a conflicting git stash pop keeps the stash entry rather than dropping it.",
|
||||
Prompt: prompt,
|
||||
Eval: eval.Equals("kept"),
|
||||
Eval: eval.ExactToken("kept"),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -235,6 +235,9 @@ func TestDelGitDetachedHeadRecoveryTest_Eval(t *testing.T) {
|
||||
{name: "correct order different case", response: `["Branch","Checkout"]`, want: 1},
|
||||
{name: "reversed order", response: `["checkout","branch"]`, want: 0},
|
||||
{name: "wrong second subcommand", response: `["branch","switch"]`, want: 0},
|
||||
{name: "single-command checkout -b is equally correct (D6)", response: `["checkout"]`, want: 1},
|
||||
{name: "single-command switch -c is equally correct (D6)", response: `["switch"]`, want: 1},
|
||||
{name: "bare branch alone leaves HEAD detached, not accepted (D6)", response: `["branch"]`, want: 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
|
||||
@@ -1,7 +1,11 @@
|
||||
package tests
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/lukaszraczylo/llm-testbench/internal/eval"
|
||||
"github.com/lukaszraczylo/llm-testbench/internal/testkit"
|
||||
@@ -29,6 +33,36 @@ func registerDeliveryReleaseTests(r *testkit.Registry) {
|
||||
// parameter - a minor-level change under Semantic Versioning. The highest
|
||||
// severity present (feat) determines the bump: v2.3.1 with a minor bump
|
||||
// becomes 2.4.0 (MINOR increments, PATCH resets to 0).
|
||||
// delRelVersionWant is the correct next-version string for
|
||||
// delRelSemverBumpChangelogTest.
|
||||
const delRelVersionWant = "2.4.0"
|
||||
|
||||
// delRelVersionEval scores full credit when the response's JSON "version"
|
||||
// field equals delRelVersionWant, tolerant of an optional leading "v"/"V"
|
||||
// prefix (DC3): the prompt's own last-release version is v-prefixed
|
||||
// ("v2.3.1"), so a model that mirrors that same convention in its answer
|
||||
// ("v2.4.0") is not wrong, just differently formatted.
|
||||
func delRelVersionEval() eval.Evaluator {
|
||||
return eval.EvaluatorFunc(func(_ context.Context, response string) eval.Score {
|
||||
raw, err := eval.ExtractJSON(response)
|
||||
if err != nil {
|
||||
return eval.Score{Value: 0, Detail: err.Error()}
|
||||
}
|
||||
var parsed struct {
|
||||
Version string `json:"version"`
|
||||
}
|
||||
if err := json.Unmarshal([]byte(raw), &parsed); err != nil {
|
||||
return eval.Score{Value: 0, Detail: fmt.Sprintf("invalid JSON: %v", err)}
|
||||
}
|
||||
got := strings.TrimSpace(parsed.Version)
|
||||
got = strings.TrimPrefix(strings.TrimPrefix(got, "v"), "V")
|
||||
if strings.EqualFold(got, delRelVersionWant) {
|
||||
return eval.Score{Value: 1, Detail: fmt.Sprintf("version = %s", got)}
|
||||
}
|
||||
return eval.Score{Value: 0, Detail: fmt.Sprintf("version: got %q, want %q (optional leading v)", parsed.Version, delRelVersionWant)}
|
||||
})
|
||||
}
|
||||
|
||||
func delRelSemverBumpChangelogTest() testkit.Test {
|
||||
prompt := `Since the last release v2.3.1, these changes have merged:
|
||||
|
||||
@@ -47,7 +81,7 @@ Respond with only a JSON object: {"version":"X.Y.Z"}`
|
||||
Subcategory: "release-engineering",
|
||||
Description: "Derive the next semver version (minor bump to 2.4.0) from a merged-change list dominated by one backward-compatible feat.",
|
||||
Prompt: prompt,
|
||||
Eval: eval.JSONField("version", "2.4.0"),
|
||||
Eval: delRelVersionEval(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -81,7 +115,7 @@ silently modify tracked files this way.`
|
||||
|
||||
evaluator := eval.All(
|
||||
eval.W(eval.ContainsAny("clean tree", "clean working tree", "clean git tree", "uncommitted", "dirty"), 2),
|
||||
eval.W(delNoUnnegatedMention(delRelSilentlyModifyPattern, "no unsafe recommendation to silently modify tracked files"), 2),
|
||||
eval.W(delNoUnnegatedMention(delRelSilentlyModifyPattern), 2),
|
||||
)
|
||||
|
||||
return testkit.Test{
|
||||
@@ -115,6 +149,10 @@ var delRelRollbackOrderingWant = []string{
|
||||
// delRelRollbackOrderingTest: order the steps of a production rollback
|
||||
// after a failed deploy.
|
||||
func delRelRollbackOrderingTest() testkit.Test {
|
||||
// D7: the added clause rules out a reading where stopping new traffic
|
||||
// (or any other mitigating action) could come before paging on-call -
|
||||
// the response must be coordinated from the start, not a solo action
|
||||
// taken before anyone else is even aware of the incident.
|
||||
prompt := `Release v3.1.0 was just deployed and is now failing health
|
||||
checks in production. The previous release v3.0.4 was healthy. Give the
|
||||
ordered list of steps to safely roll back to the last known-good release,
|
||||
@@ -122,6 +160,9 @@ choosing from and ordering exactly these labels:
|
||||
["redeploy-previous-image", "page-oncall", "confirm-health-checks-pass",
|
||||
"stop-new-traffic-if-needed", "post-incident-notes"]
|
||||
|
||||
The on-call rotation must be engaged before any mitigating action is
|
||||
taken, so the response stays coordinated from the start.
|
||||
|
||||
Respond with only a JSON array of the labels, in the order you would
|
||||
perform them.`
|
||||
|
||||
@@ -197,6 +238,9 @@ var delRelTagToReleaseWant = []string{
|
||||
// delRelTagToReleaseSequenceTest: order the steps from a verified main
|
||||
// branch to a published, announced GitHub release.
|
||||
func delRelTagToReleaseSequenceTest() testkit.Test {
|
||||
// D7: the added clause disambiguates that the changelog is generated
|
||||
// FROM the pushed tag (the goreleaser/gh-release flow this test
|
||||
// models), ruling out generating it before the tag is pushed.
|
||||
prompt := `Starting from an already-merged, untested-since-merge main
|
||||
branch, give the ordered sequence of steps to reach a published,
|
||||
announced GitHub release with generated release notes, choosing from and
|
||||
@@ -204,6 +248,10 @@ ordering exactly these labels:
|
||||
["run-tests", "create-git-tag", "push-tag", "generate-changelog",
|
||||
"create-github-release", "announce"]
|
||||
|
||||
The changelog is generated from the pushed tag by the release tool (it
|
||||
reads the git log since the previous tag, so the tag must already be
|
||||
pushed before it can run).
|
||||
|
||||
Respond with only a JSON array of the labels, in the order you would
|
||||
perform them.`
|
||||
|
||||
@@ -273,9 +321,12 @@ cosign key. In 1-2 sentences, explain what problem each of these two things
|
||||
solves for someone downloading the release: the checksums file, and the
|
||||
signature over it.`
|
||||
|
||||
// D8: "identity" is too generic (could describe many unrelated
|
||||
// things) - replaced with the more specific authentic/provenance/
|
||||
// signed-by/publisher terms, keeping the other pre-existing phrasings.
|
||||
evaluator := eval.All(
|
||||
eval.W(eval.ContainsAny("integrity", "corrupt", "tamper", "not been altered", "not been modified"), 2),
|
||||
eval.W(eval.ContainsAny("authentic", "provenance", "came from", "verify the publisher", "signed by", "identity"), 2),
|
||||
eval.W(eval.ContainsAny("authentic", "provenance", "came from", "verify the publisher", "signed by", "publisher"), 2),
|
||||
)
|
||||
|
||||
return testkit.Test{
|
||||
|
||||
@@ -18,6 +18,19 @@ func TestDelRelSemverBumpChangelogTest_Eval(t *testing.T) {
|
||||
{name: "extra whitespace inside the object", response: `{ "version" : "2.4.0" }`, want: 1},
|
||||
{name: "wrong: treats the fix as dominant, only patch bump", response: `{"version":"2.3.2"}`, want: 0},
|
||||
{name: "wrong: major bump for no breaking change", response: `{"version":"3.0.0"}`, want: 0},
|
||||
{
|
||||
// DC3 bug probe: the prompt's own last-release version is
|
||||
// v-prefixed ("v2.3.1"), so a model mirroring that convention
|
||||
// must not be marked wrong for the same leading "v".
|
||||
name: "correct: leading v prefix, mirroring the prompt's own convention (DC3 bug probe)",
|
||||
response: `{"version":"v2.4.0"}`,
|
||||
want: 1,
|
||||
},
|
||||
{
|
||||
name: "correct: uppercase V prefix (DC3 bug probe)",
|
||||
response: `{"version":"V2.4.0"}`,
|
||||
want: 1,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
@@ -239,6 +252,14 @@ func TestDelRelChecksumsSigningPurposeTest_Eval(t *testing.T) {
|
||||
response: "Both files just make the release page look more professional.",
|
||||
want: 0,
|
||||
},
|
||||
{
|
||||
// D8 bug probe: "publisher" alone (no "identity") is a
|
||||
// sufficient authenticity term now that "identity" was
|
||||
// replaced with more specific alternatives.
|
||||
name: "correct: bare 'publisher' term (D8 bug probe)",
|
||||
response: "The checksum catches corruption in transit; the signature ties the file back to the publisher.",
|
||||
want: 1,
|
||||
},
|
||||
{
|
||||
name: "wrong: only explains the checksum, not the signature",
|
||||
response: "The checksum lets you confirm the binary was not corrupted in transit.",
|
||||
|
||||
@@ -11,10 +11,10 @@ import (
|
||||
// 10 tests each, all categorized "delivery", and no duplicate IDs across the
|
||||
// three sibling files (testkit.Registry.Register panics on a duplicate ID,
|
||||
// which would fail this test outright). Mirrors
|
||||
// TestRegisterAgentsTests_Wiring in agents_test.go; catalog.go and
|
||||
// catalog_test.go are not touched by this worktree, so this is the only
|
||||
// place registerDeliveryTests is exercised until the orchestrator wires it
|
||||
// into All() at integration.
|
||||
// TestRegisterAgentsTests_Wiring in agents_test.go; catalog.go's All() now
|
||||
// also wires registerDeliveryTests directly (DC8), but this test still
|
||||
// exercises the registration function in isolation from the other
|
||||
// categories.
|
||||
func TestRegisterDeliveryTests_Wiring(t *testing.T) {
|
||||
r := testkit.NewRegistry()
|
||||
registerDeliveryTests(r)
|
||||
@@ -38,8 +38,8 @@ func TestRegisterDeliveryTests_Wiring(t *testing.T) {
|
||||
if tc.Description == "" {
|
||||
t.Errorf("test %q: Description is empty", tc.ID)
|
||||
}
|
||||
if tc.MaxTokens < 0 {
|
||||
t.Errorf("test %q: MaxTokens = %d, want >= 0", tc.ID, tc.MaxTokens)
|
||||
if tc.MaxTokens != 0 {
|
||||
t.Errorf("test %q: MaxTokens = %d, want 0 (omitted, per round-2 rule 7)", tc.ID, tc.MaxTokens)
|
||||
}
|
||||
counts[tc.Subcategory]++
|
||||
}
|
||||
|
||||
@@ -3,7 +3,6 @@ package tests
|
||||
import (
|
||||
"context"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/lukaszraczylo/llm-testbench/internal/eval"
|
||||
"github.com/lukaszraczylo/llm-testbench/internal/testkit"
|
||||
@@ -69,37 +68,9 @@ func mentionsLimitIncrease() eval.Evaluator {
|
||||
// start, or as a bulleted list item.
|
||||
var kubectlLiveMutationPattern = regexp.MustCompile(`(?i)kubectl\s+(edit|patch)\b`)
|
||||
|
||||
// negationCuePattern matches a word that turns a mention of the forbidden
|
||||
// command into a warning against running it, rather than an instruction to
|
||||
// run it (e.g. "do not run kubectl patch...").
|
||||
var negationCuePattern = regexp.MustCompile(`(?i)\b(don'?t|do not|never|avoid|instead of|not|cannot|can'?t|rather than|without|no need)\b`)
|
||||
|
||||
// negationWindow is how many characters before a "kubectl edit|patch"
|
||||
// mention are searched for a negation cue.
|
||||
const negationWindow = 60
|
||||
|
||||
// negationWindowStart returns the earliest byte offset in response to
|
||||
// search for a negation cue before a "kubectl edit|patch" match starting
|
||||
// at start. The window is the current line, extended back to the start of
|
||||
// the immediately preceding line when that line is non-empty: a
|
||||
// hard-wrapped sentence whose negation cue landed on the line above (e.g.
|
||||
// "...do not run\nkubectl edit...") would otherwise score 0 even though it
|
||||
// is the correct, negated answer. Either way, the window never reaches
|
||||
// more than negationWindow characters before start.
|
||||
func negationWindowStart(response string, start int) int {
|
||||
curLineStart := strings.LastIndexByte(response[:start], '\n') + 1
|
||||
|
||||
windowFloor := curLineStart
|
||||
if curLineStart > 0 {
|
||||
prevLineEnd := curLineStart - 1 // the newline separating the two lines
|
||||
prevLineStart := strings.LastIndexByte(response[:prevLineEnd], '\n') + 1
|
||||
if strings.TrimSpace(response[prevLineStart:prevLineEnd]) != "" {
|
||||
windowFloor = prevLineStart
|
||||
}
|
||||
}
|
||||
|
||||
return max(start-negationWindow, windowFloor)
|
||||
}
|
||||
// noLiveKubectlMutationWindow is how many characters around a "kubectl
|
||||
// edit|patch" mention are searched for a negation cue.
|
||||
const noLiveKubectlMutationWindow = 60
|
||||
|
||||
// noLiveKubectlMutation scores full credit unless the response instructs
|
||||
// running "kubectl edit"/"kubectl patch" against the live cluster. A
|
||||
@@ -109,29 +80,13 @@ func negationWindowStart(response string, start int) int {
|
||||
// would also zero out the best possible answer (one that correctly
|
||||
// explains why NOT to run kubectl edit/patch).
|
||||
//
|
||||
// Every occurrence goes through the same negation-window check, including
|
||||
// one at the start of a line (a bulleted "don't do this" list item, or a
|
||||
// hard-wrapped sentence whose negation cue landed on the line above): a
|
||||
// response is not penalized just because word-wrap or list formatting put
|
||||
// the command text at a line start. A genuinely bare imperative still
|
||||
// fails, because negationWindowStart never finds a negation cue in its
|
||||
// (possibly line-extended) window.
|
||||
// Delegates to eval.NoUnnegatedMention (D5), the primitive shared with
|
||||
// security.go, databases_redis.go, and delivery_git.go's equivalent
|
||||
// guards: its bidirectional, clause-scoped window finds a negation cue
|
||||
// before OR after the match, and correctly ignores a cue that belongs to
|
||||
// an unrelated adjacent sentence.
|
||||
func noLiveKubectlMutation() eval.Evaluator {
|
||||
return eval.EvaluatorFunc(func(_ context.Context, response string) eval.Score {
|
||||
matches := kubectlLiveMutationPattern.FindAllStringIndex(response, -1)
|
||||
if len(matches) == 0 {
|
||||
return eval.Score{Value: 1, Detail: "no mention of kubectl edit/patch"}
|
||||
}
|
||||
|
||||
for _, loc := range matches {
|
||||
start := loc[0]
|
||||
windowStart := negationWindowStart(response, start)
|
||||
if !negationCuePattern.MatchString(response[windowStart:start]) {
|
||||
return eval.Score{Value: 0, Detail: "unnegated mention of kubectl edit/patch"}
|
||||
}
|
||||
}
|
||||
return eval.Score{Value: 1, Detail: "every mention of kubectl edit/patch is negated"}
|
||||
})
|
||||
return eval.NoUnnegatedMention(kubectlLiveMutationPattern, noLiveKubectlMutationWindow, nil)
|
||||
}
|
||||
|
||||
// k8sCrashloopGitopsTest: diagnose an OOMKilled CrashLoopBackOff and
|
||||
|
||||
@@ -1,9 +1,7 @@
|
||||
package tests
|
||||
|
||||
import (
|
||||
"context"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/lukaszraczylo/llm-testbench/internal/eval"
|
||||
"github.com/lukaszraczylo/llm-testbench/internal/testkit"
|
||||
@@ -19,80 +17,22 @@ func registerSecurityTests(r *testkit.Registry) {
|
||||
registerSecuritySecretsTests(r)
|
||||
}
|
||||
|
||||
// secNegationCuePattern matches a word that turns a mention of a
|
||||
// discouraged term (a weak algorithm, an insecure package) into a warning
|
||||
// against it rather than an endorsement of it. Adapted from kubernetes.go's
|
||||
// negationCuePattern, with "must not"/"isn't"/"wasn't"/"shouldn't"/"should
|
||||
// not" added: security answers in this category more often phrase the
|
||||
// warning as "X is not suitable" or "X must not be used" than as
|
||||
// kubernetes.go's "don't run X" imperative.
|
||||
var secNegationCuePattern = regexp.MustCompile(`(?i)\b(don'?t|do not|never|avoid|instead of|not|cannot|can'?t|rather than|without|no need|shouldn'?t|should not|must not|isn'?t|wasn'?t)\b`)
|
||||
|
||||
// secNegationWindow is how many characters around a discouraged-term match
|
||||
// are searched for a negation cue.
|
||||
const secNegationWindow = 60
|
||||
|
||||
// secNegationNearby reports whether a negation cue appears within
|
||||
// secNegationWindow characters immediately before OR after the match
|
||||
// [start,end) in response. This is deliberately bidirectional, unlike
|
||||
// kubernetes.go's negationWindowStart (which only looks before the match):
|
||||
// kubernetes.go's forbidden phrase is an imperative ("kubectl edit"), so the
|
||||
// negation cue naturally precedes it ("don't run kubectl edit"). This
|
||||
// category's discouraged terms are usually algorithm/package names, and a
|
||||
// good answer just as often negates them with the cue AFTER the term
|
||||
// ("SHA-256 is not appropriate for passwords", "math/rand must never be
|
||||
// used here") as before it ("never use MD5", "avoid math/rand").
|
||||
func secNegationNearby(response string, start, end int) bool {
|
||||
lo := start - secNegationWindow
|
||||
if lo < 0 {
|
||||
lo = 0
|
||||
}
|
||||
hi := end + secNegationWindow
|
||||
if hi > len(response) {
|
||||
hi = len(response)
|
||||
}
|
||||
return secNegationCuePattern.MatchString(response[lo:start]) || secNegationCuePattern.MatchString(response[end:hi])
|
||||
}
|
||||
|
||||
// secNoUnnegatedMention returns an Evaluator scoring full credit unless
|
||||
// badPattern matches somewhere in the response with no negation cue nearby -
|
||||
// i.e. it penalizes an unqualified endorsement of a discouraged
|
||||
// algorithm/package but not a warning against using it. This is deliberately
|
||||
// not eval.NotContains, which would also zero out the best possible answer
|
||||
// (one that correctly names the discouraged term while warning against it).
|
||||
//
|
||||
// Delegates to eval.NoUnnegatedMention (D5), the primitive shared with
|
||||
// kubernetes.go, databases_redis.go, and delivery_git.go's equivalent
|
||||
// guards: its bidirectional, clause-scoped window catches a negation cue
|
||||
// whether it precedes the discouraged term ("never use MD5") or follows it
|
||||
// ("MD5 is unsuitable for password storage") (C2).
|
||||
func secNoUnnegatedMention(badPattern *regexp.Regexp) eval.Evaluator {
|
||||
return eval.EvaluatorFunc(func(_ context.Context, response string) eval.Score {
|
||||
matches := badPattern.FindAllStringIndex(response, -1)
|
||||
if len(matches) == 0 {
|
||||
return eval.Score{Value: 1, Detail: "no mention of a discouraged term"}
|
||||
}
|
||||
for _, loc := range matches {
|
||||
if !secNegationNearby(response, loc[0], loc[1]) {
|
||||
return eval.Score{Value: 0, Detail: "unnegated mention of a discouraged term"}
|
||||
}
|
||||
}
|
||||
return eval.Score{Value: 1, Detail: "every mention of a discouraged term is negated"}
|
||||
})
|
||||
}
|
||||
|
||||
// secExactAnswer returns an Evaluator awarding full credit when the
|
||||
// response, trimmed of whitespace, at most one layer of surrounding quote
|
||||
// characters ('", or `), and a single trailing sentence-ending period,
|
||||
// equals want case-insensitively. Mirrors codebase_analysis.go's
|
||||
// codeExactAnswer: it accepts every materially-correct form of a forced
|
||||
// single-token/short-phrase answer (bare, quoted, differently-cased, or with
|
||||
// trailing punctuation) without loosening the match to accept a substring of
|
||||
// a longer, wrong answer.
|
||||
func secExactAnswer(want string) eval.Evaluator {
|
||||
return eval.EvaluatorFunc(func(_ context.Context, response string) eval.Score {
|
||||
got := strings.TrimSpace(response)
|
||||
got = strings.Trim(got, "\"'`")
|
||||
got = strings.TrimSuffix(strings.TrimSpace(got), ".")
|
||||
got = strings.TrimSpace(got)
|
||||
wantTrimmed := strings.TrimSpace(want)
|
||||
if strings.EqualFold(got, wantTrimmed) {
|
||||
return eval.Score{Value: 1, Detail: "equals " + wantTrimmed}
|
||||
}
|
||||
return eval.Score{Value: 0, Detail: "got " + got + ", want " + wantTrimmed}
|
||||
})
|
||||
return eval.NoUnnegatedMention(badPattern, secNegationWindow, nil)
|
||||
}
|
||||
|
||||
@@ -1,6 +1,9 @@
|
||||
package tests
|
||||
|
||||
import (
|
||||
"context"
|
||||
"regexp"
|
||||
|
||||
"github.com/lukaszraczylo/llm-testbench/internal/eval"
|
||||
"github.com/lukaszraczylo/llm-testbench/internal/testkit"
|
||||
)
|
||||
@@ -137,8 +140,10 @@ its 1-based position in this listing:
|
||||
|
||||
` + secPathTraversalFixture + `
|
||||
|
||||
Which line number introduces a path-traversal vulnerability, and what
|
||||
category of fix addresses it? Respond with only a JSON object:
|
||||
Which line number builds the unsanitized filesystem path (concatenating a
|
||||
caller-supplied value onto a base directory with no cleaning of ".."
|
||||
segments), and what category of fix addresses it? Respond with only a JSON
|
||||
object:
|
||||
{"line":<number>,"fix":"<one of: sanitize-path, escape-output, rate-limit>"}`
|
||||
|
||||
evaluator := eval.Mean(
|
||||
@@ -313,7 +318,7 @@ Respond with only one word: yes or no.`
|
||||
Subcategory: "appsec",
|
||||
Description: "Decide whether a header-bearer-token-only endpoint (no cookies) needs CSRF protection.",
|
||||
Prompt: prompt,
|
||||
Eval: secExactAnswer("no"),
|
||||
Eval: eval.ExactToken("no"),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -352,6 +357,28 @@ a JSON object:
|
||||
}
|
||||
}
|
||||
|
||||
// secQuantityUpperBoundConcretePattern matches a concrete numeric
|
||||
// upper-bound expression: an explicit min/max pair, or a numeric range like
|
||||
// "1..10000" (CC3).
|
||||
var secQuantityUpperBoundConcretePattern = regexp.MustCompile(`(?i)\d+\s*\.\.\s*\d+|\bmin(?:imum)?\b[^.\n]{0,20}\bmax(?:imum)?\b|\bmax(?:imum)?\b[^.\n]{0,20}\bmin(?:imum)?\b`)
|
||||
|
||||
// secQuantityUpperBoundEval scores full credit when the response either
|
||||
// uses one of the generic upper-bound words ("maximum", "upper bound",
|
||||
// "cap", "overflow", "too large", "excessively large", "unbounded") or
|
||||
// states a concrete numeric bound expression instead (CC3).
|
||||
func secQuantityUpperBoundEval() eval.Evaluator {
|
||||
words := eval.ContainsAny("maximum", "upper bound", "cap", "overflow", "too large", "excessively large", "unbounded")
|
||||
return eval.EvaluatorFunc(func(ctx context.Context, response string) eval.Score {
|
||||
if w := words.Evaluate(ctx, response); w.Value == 1 {
|
||||
return w
|
||||
}
|
||||
if secQuantityUpperBoundConcretePattern.MatchString(response) {
|
||||
return eval.Score{Value: 1, Detail: "matches a concrete numeric upper-bound expression"}
|
||||
}
|
||||
return eval.Score{Value: 0, Detail: "no upper-bound cue (generic word or concrete numeric bound) found"}
|
||||
})
|
||||
}
|
||||
|
||||
// secInputValidationBoundaryTest: name the two numeric boundary checks
|
||||
// missing from an unvalidated order-quantity field.
|
||||
//
|
||||
@@ -371,9 +398,15 @@ validation runs on it.
|
||||
Name the two numeric boundary checks that must be added to quantity before
|
||||
it is used, and why each one matters.`
|
||||
|
||||
// CC3: the upper-bound group's plain words ("cap", "overflow") are
|
||||
// generic enough to plausibly appear in unrelated prose. A response
|
||||
// that instead states a concrete numeric bound (a min/max pair, or an
|
||||
// explicit range like "1..10000") is an equally valid, even more
|
||||
// specific answer, so it is accepted as an alternative alongside the
|
||||
// existing word list rather than only via those words.
|
||||
evaluator := eval.Mean(
|
||||
eval.ContainsAny("negative", "<= 0", "zero or negative", "greater than zero", "greater than 0", "positive"),
|
||||
eval.ContainsAny("maximum", "upper bound", "cap", "overflow", "too large", "excessively large", "unbounded"),
|
||||
secQuantityUpperBoundEval(),
|
||||
)
|
||||
|
||||
return testkit.Test{
|
||||
|
||||
@@ -230,6 +230,14 @@ func TestSecInputValidationBoundaryTest_Eval(t *testing.T) {
|
||||
response: "Reject negative quantities.",
|
||||
want: 0.5,
|
||||
},
|
||||
{
|
||||
// CC3 bug probe: a concrete numeric range is an equally valid,
|
||||
// even more specific upper-bound answer, even without any of
|
||||
// the generic upper-bound words.
|
||||
name: "correct: concrete numeric range instead of generic upper-bound words (CC3 bug probe)",
|
||||
response: "Reject quantity <= 0, and only accept a range of 1..10000.",
|
||||
want: 1,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
|
||||
@@ -1,7 +1,10 @@
|
||||
package tests
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/lukaszraczylo/llm-testbench/internal/eval"
|
||||
"github.com/lukaszraczylo/llm-testbench/internal/testkit"
|
||||
@@ -25,6 +28,27 @@ func registerSecurityCryptoTests(r *testkit.Registry) {
|
||||
// directly for password storage because they lack a tunable work factor.
|
||||
var secWeakPasswordHashPattern = regexp.MustCompile(`(?i)\b(md5|sha-?1|sha-?256|sha-?512)\b`)
|
||||
|
||||
// secWeakHashNamedAndNegated scores full credit only when the response BOTH
|
||||
// safely negates every mention of a weak/fast general-purpose hash (or
|
||||
// never mentions one) AND actually names at least one of them by name
|
||||
// (md5/sha-1/sha-256/sha-512) - C11's fix folded into a single AND rather
|
||||
// than an independent extra ContainsAny term, so a wrong answer that
|
||||
// recommends SHA-256 outright cannot earn partial credit merely for having
|
||||
// typed "SHA-256" while doing so.
|
||||
func secWeakHashNamedAndNegated() eval.Evaluator {
|
||||
named := eval.ContainsAny("md5", "sha-1", "sha1", "sha-256", "sha256")
|
||||
negated := secNoUnnegatedMention(secWeakPasswordHashPattern)
|
||||
return eval.EvaluatorFunc(func(ctx context.Context, response string) eval.Score {
|
||||
if n := negated.Evaluate(ctx, response); n.Value != 1 {
|
||||
return eval.Score{Value: 0, Detail: "unnegated mention of a discouraged hash: " + n.Detail}
|
||||
}
|
||||
if m := named.Evaluate(ctx, response); m.Value != 1 {
|
||||
return eval.Score{Value: 0, Detail: "never names a weak hash by name"}
|
||||
}
|
||||
return eval.Score{Value: 1, Detail: "a weak hash is named and every mention is safely negated"}
|
||||
})
|
||||
}
|
||||
|
||||
// secPasswordHashChoiceTest: recommend bcrypt/argon2 for password storage
|
||||
// while not being penalized for correctly warning against SHA/MD5.
|
||||
//
|
||||
@@ -40,9 +64,19 @@ func secPasswordHashChoiceTest() testkit.Test {
|
||||
hashing approach should be used to store user passwords at rest, and which
|
||||
common hash functions must never be used for this specific purpose?`
|
||||
|
||||
// C11: secNoUnnegatedMention alone scores full credit for a response
|
||||
// that never names any weak hash at all ("no mention" is vacuously
|
||||
// negation-safe), which rewards skipping the prompt's second half
|
||||
// entirely ("which common hash functions must never be used"). A
|
||||
// combined (not independent) second term additionally requires the
|
||||
// response to actually name at least one weak hash BY NAME while ALSO
|
||||
// safely negating it - folded into one AND, not a bare extra
|
||||
// ContainsAny term, since an independent term would give partial credit
|
||||
// to a wrong answer merely for mentioning "SHA-256" while recommending
|
||||
// it (as opposed to warning against it).
|
||||
evaluator := eval.Mean(
|
||||
eval.ContainsAny("bcrypt", "argon2"),
|
||||
secNoUnnegatedMention(secWeakPasswordHashPattern),
|
||||
secWeakHashNamedAndNegated(),
|
||||
)
|
||||
|
||||
return testkit.Test{
|
||||
@@ -73,9 +107,14 @@ using Go's == operator on the two byte slices.
|
||||
Name the security defect this introduces, and the correct Go API to use
|
||||
instead of ==.`
|
||||
|
||||
// CC1: "constant-time"/"constant time" describes the FIX (a
|
||||
// constant-time comparison), not the defect itself, so it belongs in
|
||||
// the fix group rather than the defect-naming group - a response that
|
||||
// only ever says "constant-time" never actually names the timing-attack
|
||||
// defect the prompt asks for.
|
||||
evaluator := eval.Mean(
|
||||
eval.ContainsAny("timing attack", "timing side channel", "time-based", "constant-time", "constant time"),
|
||||
eval.ContainsAny("hmac.equal", "subtle.constanttimecompare", "constanttimecompare"),
|
||||
eval.ContainsAny("timing attack", "timing side channel", "time-based"),
|
||||
eval.ContainsAny("hmac.equal", "subtle.constanttimecompare", "constanttimecompare", "constant-time", "constant time"),
|
||||
)
|
||||
|
||||
return testkit.Test{
|
||||
@@ -122,9 +161,35 @@ Respond with only a JSON object:
|
||||
}
|
||||
}
|
||||
|
||||
// secTLSFloorPattern requires the digits 1.2 as a standalone version
|
||||
// number, tolerant of a leading "TLS"/"TLSv" and surrounding punctuation.
|
||||
const secTLSFloorPattern = `\b1\.2\b`
|
||||
// secTLSFloorPrefixPattern strips an optional "TLS"/"TLSv" prefix (with or
|
||||
// without a separating space) from the front of the normalized response
|
||||
// before comparing what remains to the exact literal "1.2" (C1).
|
||||
//
|
||||
// A plain `\b1\.2\b` substring search had two independent failure modes:
|
||||
// "TLSv1.2" (no space) has no word boundary between "v" and "1" - both are
|
||||
// \w characters - so it never matched an otherwise-correct compact answer;
|
||||
// and a longer response like "TLS 1.3 only; 1.2 is legacy" contains "1.2"
|
||||
// as a substring (inside the "legacy" clause) and was wrongly accepted even
|
||||
// though 1.3, not 1.2, was the actual chosen floor. Stripping the prefix
|
||||
// and then requiring the ENTIRE normalized response to equal "1.2" fixes
|
||||
// both: "TLSv1.2" reduces to "1.2" and matches, while the longer sentence
|
||||
// never reduces to the bare literal and correctly fails.
|
||||
var secTLSFloorPrefixPattern = regexp.MustCompile(`(?i)^tlsv?\s*`)
|
||||
|
||||
// secTLSFloorVersionEval normalizes the response (fences/quotes/asterisks/
|
||||
// trailing period stripped via eval.NormalizeExactToken), strips a leading
|
||||
// TLS/TLSv prefix, and requires the remaining text to equal exactly "1.2".
|
||||
func secTLSFloorVersionEval() eval.Evaluator {
|
||||
return eval.EvaluatorFunc(func(_ context.Context, response string) eval.Score {
|
||||
got := eval.NormalizeExactToken(response)
|
||||
got = secTLSFloorPrefixPattern.ReplaceAllString(got, "")
|
||||
got = strings.TrimSpace(got)
|
||||
if got == "1.2" {
|
||||
return eval.Score{Value: 1, Detail: "equals 1.2"}
|
||||
}
|
||||
return eval.Score{Value: 0, Detail: fmt.Sprintf("got %q, want \"1.2\"", got)}
|
||||
})
|
||||
}
|
||||
|
||||
// secTLSFloorVersionTest: name the minimum TLS protocol version that
|
||||
// excludes SSLv3/TLS 1.0/1.1 while remaining broadly compatible.
|
||||
@@ -148,7 +213,7 @@ to accept? Respond with only the version number, e.g. 1.2.`
|
||||
Subcategory: "crypto",
|
||||
Description: "Name TLS 1.2 as the minimum floor that excludes SSLv3/TLS 1.0/1.1 while staying broadly compatible.",
|
||||
Prompt: prompt,
|
||||
Eval: eval.Regex(secTLSFloorPattern),
|
||||
Eval: secTLSFloorVersionEval(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -170,8 +235,13 @@ the API.
|
||||
What is the concrete security consequence of reusing a GCM nonce with the
|
||||
same key, and what is the fix?`
|
||||
|
||||
// CC2: bare "authentication" is too generic - a response could mention
|
||||
// authentication in some unrelated sense and get credit without
|
||||
// actually describing the nonce-reuse consequence. Tightened to the
|
||||
// specific consequence terms: forge/forgery, keystream, or the GCM
|
||||
// "auth tag" reuse mechanism itself.
|
||||
evaluator := eval.Mean(
|
||||
eval.ContainsAny("forge", "forgery", "authentication", "keystream", "xor", "plaintext recovery", "break confidentiality"),
|
||||
eval.ContainsAny("forge", "forgery", "auth tag", "authentication tag", "keystream", "xor", "plaintext recovery", "break confidentiality"),
|
||||
eval.ContainsAny("random nonce", "unique nonce", "fresh nonce", "never reuse", "crypto/rand"),
|
||||
)
|
||||
|
||||
@@ -204,9 +274,8 @@ math/rand to produce 32 random bytes, then hex-encodes them for the reset
|
||||
link.
|
||||
|
||||
Which Go standard-library package should be used instead of math/rand for
|
||||
this purpose, and confirm: math/rand must never be used to generate a
|
||||
security-sensitive token like this one. Explain the one concrete risk of
|
||||
the current approach.`
|
||||
this purpose? Explain the one concrete risk of using math/rand for a
|
||||
security-sensitive token like this one.`
|
||||
|
||||
evaluator := eval.Mean(
|
||||
eval.ContainsAny("crypto/rand"),
|
||||
@@ -249,8 +318,8 @@ issued under the old key and is still within its validity window, and it
|
||||
must never allow a window where a still-valid token fails verification.
|
||||
|
||||
Order these 4 steps correctly:
|
||||
["add-new-key-to-verify-set", "start-signing-with-new-key",
|
||||
"wait-for-old-tokens-to-expire", "remove-old-key-from-verify-set"]
|
||||
["remove-old-key-from-verify-set", "add-new-key-to-verify-set",
|
||||
"wait-for-old-tokens-to-expire", "start-signing-with-new-key"]
|
||||
|
||||
Respond with only a JSON array containing all 4 step ids in the correct
|
||||
order.`
|
||||
@@ -290,13 +359,18 @@ Should they use a shared-secret HMAC or an asymmetric signature scheme
|
||||
Subcategory: "crypto",
|
||||
Description: "Choose asymmetric signing over shared-secret HMAC for cross-organization webhook verification.",
|
||||
Prompt: prompt,
|
||||
Eval: secExactAnswer("asymmetric"),
|
||||
Eval: eval.ExactToken("asymmetric"),
|
||||
}
|
||||
}
|
||||
|
||||
// secManInTheMiddlePattern accepts the standard term for the attack with
|
||||
// or without hyphens/spaces between the words.
|
||||
const secManInTheMiddlePattern = `(?i)man[\s-]?in[\s-]?the[\s-]?middle`
|
||||
// secManInTheMiddlePattern accepts every standard name for this attack
|
||||
// class (C6): the classic "man-in-the-middle" phrase (hyphenated, spaced,
|
||||
// or run together), its "MITM" abbreviation, the newer gender-neutral
|
||||
// "adversary-in-the-middle"/"machine-in-the-middle"/"person-in-the-middle"
|
||||
// phrasings and their "AiTM" abbreviation, and "on-path" (the more recent
|
||||
// vendor-neutral term some standards bodies now prefer for the same
|
||||
// network position).
|
||||
const secManInTheMiddlePattern = `(?i)\bmitm\b|\baitm\b|on[\s-]?path\b|(?:man|machine|adversary|person)[\s-]?in[\s-]?the[\s-]?middle`
|
||||
|
||||
// secCertChainValidationTest: name man-in-the-middle as the attack exposed
|
||||
// by skipping certificate-chain verification.
|
||||
|
||||
@@ -38,6 +38,21 @@ func TestSecPasswordHashChoiceTest_Eval(t *testing.T) {
|
||||
response: "Store passwords hashed with MD5 for speed.",
|
||||
want: 0,
|
||||
},
|
||||
{
|
||||
// C11 bug probe: naming a weak hash while recommending it
|
||||
// (not warning against it) must not earn any credit for
|
||||
// "naming" the discouraged term - the naming credit is folded
|
||||
// into the same AND as the negation check, not an independent
|
||||
// term.
|
||||
name: "wrong: names SHA-256 by name but still recommends it (C11 bug probe)",
|
||||
response: "SHA-256 is a fine choice for hashing user passwords.",
|
||||
want: 0,
|
||||
},
|
||||
{
|
||||
name: "wrong: warns against a hash never actually named by name (C11 bug probe)",
|
||||
response: "Use bcrypt, and avoid weak, fast, general-purpose hashing algorithms for passwords.",
|
||||
want: 0.5,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
@@ -68,8 +83,18 @@ func TestSecConstantTimeCompareTest_Eval(t *testing.T) {
|
||||
want: 1,
|
||||
},
|
||||
{
|
||||
name: "correct: constant-time phrasing, ConstantTimeCompare",
|
||||
// CC1: "constant-time" describes the FIX, not the defect - a
|
||||
// response that only ever says "constant-time" (never "timing
|
||||
// attack"/"timing side channel"/"time-based") does not
|
||||
// independently name the defect class the prompt asks for, so
|
||||
// this now earns only the fix-group's credit, not both.
|
||||
name: "partial: only fix-side 'constant-time' phrasing, never names the defect class (CC1)",
|
||||
response: "Comparing with == leaks timing information (constant-time is required). Use crypto/subtle's ConstantTimeCompare function.",
|
||||
want: 0.5,
|
||||
},
|
||||
{
|
||||
name: "correct: names the defect AND uses 'constant-time' for the fix (CC1 bug probe)",
|
||||
response: "This is a timing attack. Fix it by doing a constant-time comparison via crypto/subtle.",
|
||||
want: 1,
|
||||
},
|
||||
{
|
||||
@@ -127,7 +152,36 @@ func TestSecTLSFloorVersionTest_Eval(t *testing.T) {
|
||||
}{
|
||||
{name: "correct bare", response: "1.2", want: 1},
|
||||
{name: "correct with TLS prefix", response: "TLS 1.2", want: 1},
|
||||
{name: "correct in a sentence", response: "The minimum should be 1.2.", want: 1},
|
||||
{name: "correct with trailing period", response: "1.2.", want: 1},
|
||||
{
|
||||
// C1: "TLSv1.2" (no space) has no word boundary between "v"
|
||||
// and "1" - a plain \b1\.2\b regex rejected it. The
|
||||
// normalize-then-strip-prefix design correctly accepts it.
|
||||
name: "correct: TLSv prefix with no space (C1 bug probe)",
|
||||
response: "TLSv1.2",
|
||||
want: 1,
|
||||
},
|
||||
{
|
||||
// This response is no longer accepted (C1): the prompt asks to
|
||||
// "respond with only the version number", and a full
|
||||
// sentence-wrapped answer is the same category of over-permissive
|
||||
// substring match that let "TLS 1.3 only; 1.2 is legacy" (below)
|
||||
// wrongly score 1 under the old \b1\.2\b substring search -
|
||||
// consistent with how every other eval.ExactToken-style forced
|
||||
// single-token test in this codebase treats full-sentence
|
||||
// wrapping.
|
||||
name: "wrong: sentence-wrapped, not a bare/quoted/fenced token",
|
||||
response: "The minimum should be 1.2.",
|
||||
want: 0,
|
||||
},
|
||||
{
|
||||
// C1 bug probe: the actual chosen floor is 1.3, but the old
|
||||
// substring search matched "1.2" anyway because it appears
|
||||
// later in the same response, in an unrelated aside.
|
||||
name: "wrong: chooses 1.3 but mentions 1.2 in an aside (C1 bug probe)",
|
||||
response: "TLS 1.3 only; 1.2 is legacy.",
|
||||
want: 0,
|
||||
},
|
||||
{name: "wrong: TLS 1.3", response: "1.3", want: 0},
|
||||
{name: "wrong: TLS 1.0", response: "1.0", want: 0},
|
||||
}
|
||||
@@ -174,6 +228,19 @@ func TestSecAESGCMNonceReuseTest_Eval(t *testing.T) {
|
||||
response: "Just use a bigger nonce size.",
|
||||
want: 0,
|
||||
},
|
||||
{
|
||||
// CC2 bug probe: bare "authentication" (with no forge/forgery/
|
||||
// keystream/auth-tag consequence term) no longer triggers the
|
||||
// defect-naming group on its own.
|
||||
name: "wrong: bare 'authentication' mention with no actual consequence term (CC2 bug probe)",
|
||||
response: "This affects authentication somehow. Use a fresh random nonce per encryption.",
|
||||
want: 0.5,
|
||||
},
|
||||
{
|
||||
name: "correct: specific 'auth tag' consequence term (CC2 bug probe)",
|
||||
response: "The auth tag can be forged if the nonce repeats. Use a fresh random nonce per encryption.",
|
||||
want: 1,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
@@ -262,6 +329,14 @@ func TestSecRotateOrderTest_Eval(t *testing.T) {
|
||||
response: `["add-new-key-to-verify-set","remove-old-key-from-verify-set","start-signing-with-new-key","wait-for-old-tokens-to-expire"]`,
|
||||
want: 0,
|
||||
},
|
||||
{
|
||||
// C4 bug probe: the prompt's presented list is now
|
||||
// hand-shuffled to a different order than the correct answer,
|
||||
// so an echo of the prompt's own list order must score 0, not 1.
|
||||
name: "wrong: echoes the prompt's shuffled presentation order verbatim (C4 bug probe)",
|
||||
response: `["remove-old-key-from-verify-set","add-new-key-to-verify-set","wait-for-old-tokens-to-expire","start-signing-with-new-key"]`,
|
||||
want: 0,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
@@ -308,6 +383,11 @@ func TestSecCertChainValidationTest_Eval(t *testing.T) {
|
||||
{name: "correct hyphenated", response: "man-in-the-middle", want: 1},
|
||||
{name: "correct spaced in a sentence", response: "This exposes the client to a man in the middle attack.", want: 1},
|
||||
{name: "correct capitalized", response: "MITM stands for Man-In-The-Middle, which is the attack here.", want: 1},
|
||||
{name: "correct: bare MITM abbreviation (C6 bug probe)", response: "MITM", want: 1},
|
||||
{name: "correct: AiTM abbreviation (C6 bug probe)", response: "AiTM (adversary-in-the-middle)", want: 1},
|
||||
{name: "correct: on-path attacker (C6 bug probe)", response: "This is an on-path attacker.", want: 1},
|
||||
{name: "correct: adversary-in-the-middle phrasing (C6 bug probe)", response: "adversary-in-the-middle", want: 1},
|
||||
{name: "correct: machine-in-the-middle phrasing (C6 bug probe)", response: "machine-in-the-middle", want: 1},
|
||||
{name: "wrong: replay attack", response: "replay attack", want: 0},
|
||||
{name: "wrong: denial of service", response: "denial of service", want: 0},
|
||||
}
|
||||
|
||||
@@ -81,8 +81,8 @@ func secRemediationOrderTest() testkit.Test {
|
||||
to a shared remote that multiple collaborators have cloned.
|
||||
|
||||
Order these 4 remediation steps correctly:
|
||||
["rotate-the-leaked-credential", "remove-secret-from-current-code",
|
||||
"rewrite-git-history-to-purge-it", "force-push-and-notify-collaborators"]
|
||||
["force-push-and-notify-collaborators", "remove-secret-from-current-code",
|
||||
"rotate-the-leaked-credential", "rewrite-git-history-to-purge-it"]
|
||||
|
||||
Respond with only a JSON array containing all 4 step ids in the correct
|
||||
order.`
|
||||
@@ -158,7 +158,7 @@ with only one word: yes or no.`
|
||||
Subcategory: "secrets",
|
||||
Description: "Confirm that Kubernetes Secret base64 encoding is not encryption and provides no confidentiality by itself.",
|
||||
Prompt: prompt,
|
||||
Eval: secExactAnswer("no"),
|
||||
Eval: eval.ExactToken("no"),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -189,9 +189,9 @@ replicas must be rotated without causing any request to fail
|
||||
authentication during the rollout.
|
||||
|
||||
Order these 4 steps correctly:
|
||||
["create-new-credential-alongside-old",
|
||||
"deploy-config-with-new-credential-to-all-replicas",
|
||||
"verify-all-replicas-using-new-credential", "revoke-old-credential"]
|
||||
["verify-all-replicas-using-new-credential", "revoke-old-credential",
|
||||
"create-new-credential-alongside-old",
|
||||
"deploy-config-with-new-credential-to-all-replicas"]
|
||||
|
||||
Respond with only a JSON array containing all 4 step ids in the correct
|
||||
order.`
|
||||
@@ -300,7 +300,7 @@ request from a fork? Respond with only one word: yes or no.`
|
||||
Subcategory: "secrets",
|
||||
Description: "Confirm repository secrets must not be exposed to CI workflow runs triggered by pull requests from forks.",
|
||||
Prompt: prompt,
|
||||
Eval: secExactAnswer("no"),
|
||||
Eval: eval.ExactToken("no"),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -364,6 +364,6 @@ current working tree? Respond with only one word: yes or no.`
|
||||
Subcategory: "secrets",
|
||||
Description: "Confirm a later .gitignore entry and working-tree deletion do not remove a secret already present in earlier git history.",
|
||||
Prompt: prompt,
|
||||
Eval: secExactAnswer("yes"),
|
||||
Eval: eval.ExactToken("yes"),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -62,6 +62,14 @@ func TestSecRemediationOrderTest_Eval(t *testing.T) {
|
||||
response: `["rewrite-git-history-to-purge-it","rotate-the-leaked-credential","remove-secret-from-current-code","force-push-and-notify-collaborators"]`,
|
||||
want: 0,
|
||||
},
|
||||
{
|
||||
// C4 bug probe: the prompt's presented list is hand-shuffled to
|
||||
// a different order than the correct answer, so echoing the
|
||||
// prompt's own list order must score 0.
|
||||
name: "wrong: echoes the prompt's shuffled presentation order verbatim (C4 bug probe)",
|
||||
response: `["force-push-and-notify-collaborators","remove-secret-from-current-code","rotate-the-leaked-credential","rewrite-git-history-to-purge-it"]`,
|
||||
want: 0,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
@@ -154,6 +162,14 @@ func TestSecRotationNoDowntimeTest_Eval(t *testing.T) {
|
||||
response: `["create-new-credential-alongside-old","revoke-old-credential","deploy-config-with-new-credential-to-all-replicas","verify-all-replicas-using-new-credential"]`,
|
||||
want: 0,
|
||||
},
|
||||
{
|
||||
// C4 bug probe: the prompt's presented list is hand-shuffled to
|
||||
// a different order than the correct answer, so echoing the
|
||||
// prompt's own list order must score 0.
|
||||
name: "wrong: echoes the prompt's shuffled presentation order verbatim (C4 bug probe)",
|
||||
response: `["verify-all-replicas-using-new-credential","revoke-old-credential","create-new-credential-alongside-old","deploy-config-with-new-credential-to-all-replicas"]`,
|
||||
want: 0,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
|
||||
@@ -34,8 +34,8 @@ func TestRegisterSecurityTests_Wiring(t *testing.T) {
|
||||
if tc.Description == "" {
|
||||
t.Errorf("test %q: Description is empty", tc.ID)
|
||||
}
|
||||
if tc.MaxTokens < 0 {
|
||||
t.Errorf("test %q: MaxTokens = %d, want >= 0", tc.ID, tc.MaxTokens)
|
||||
if tc.MaxTokens != 0 {
|
||||
t.Errorf("test %q: MaxTokens = %d, want 0 (omitted, per round-2 rule 7)", tc.ID, tc.MaxTokens)
|
||||
}
|
||||
counts[tc.Subcategory]++
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user