diff --git a/internal/eval/exacttoken.go b/internal/eval/exacttoken.go new file mode 100644 index 0000000..62ff410 --- /dev/null +++ b/internal/eval/exacttoken.go @@ -0,0 +1,91 @@ +package eval + +import ( + "context" + "fmt" + "strings" +) + +// NormalizeExactToken strips formatting noise from s before an exact-token +// comparison: it extracts a fenced code block if s has one (see +// ExtractCodeBlock), trims surrounding whitespace, strips one trailing +// sentence-ending period, strips one layer of surrounding quote/backtick +// decoration, strips a *paired* leading/trailing run of asterisks +// (**bolded**/*italic* markdown - never a single, unpaired asterisk, which +// can be meaningful content rather than decoration: "*string" is a Go +// pointer type, not "string" wrapped in emphasis), and trims again. The +// quote/backtick/asterisk/period stripping repeats to a fixed point so +// nested decoration ("**\"GET\"**.") fully resolves regardless of which +// layer is outermost. It is exported so a test file that needs to compare +// against more than one literal variant (e.g. "O(n^2)" vs "O(n²)" vs +// "O(n**2)") can normalize once and compare the result to each variant, +// rather than calling ExactToken repeatedly. +func NormalizeExactToken(s string) string { + got := strings.TrimSpace(ExtractCodeBlock(s, "")) + + periodStripped := false + for { + before := got + if !periodStripped && strings.HasSuffix(got, ".") { + got = strings.TrimSpace(strings.TrimSuffix(got, ".")) + periodStripped = true + } + got = strings.Trim(got, "\"'`") + got = stripPairedAsterisks(got) + got = strings.TrimSpace(got) + if got == before { + break + } + } + return got +} + +// stripPairedAsterisks removes a leading run of '*' and a trailing run of +// '*' from s, but only when both runs have the same, non-zero length: a +// balanced **bold** or *italic* wrapper. An unbalanced run (a single +// leading '*' with no trailing one, as in the Go pointer type "*string") +// is left untouched, since it is markdown noise for one and meaningful +// content for the other, and this function cannot tell them apart from +// the character alone. +func stripPairedAsterisks(s string) string { + lead := 0 + for lead < len(s) && s[lead] == '*' { + lead++ + } + trail := 0 + for trail < len(s)-lead && s[len(s)-1-trail] == '*' { + trail++ + } + if lead > 0 && lead == trail { + return s[lead : len(s)-trail] + } + return s +} + +// exactTokenEval scores full credit on a normalized, case-insensitive +// whole-answer match. +type exactTokenEval struct { + want string +} + +// ExactToken returns an Evaluator awarding full credit when the response, +// after NormalizeExactToken, equals want (also normalized) case- +// insensitively. It exists for prompts that force a single, literal +// answer token or short phrase - one the prompt itself may quote, bold, or +// fence - where a substring-containment evaluator (ContainsAny etc.) would +// wrongly score a response that merely restates the prompt's own option +// list, or a negated answer that happens to contain the right substring +// ("not O(n^2), it's O(n)"). ExactToken anchors to the whole normalized +// answer, so neither false-positive scores. +func ExactToken(want string) Evaluator { + return exactTokenEval{want: want} +} + +func (e exactTokenEval) Evaluate(_ context.Context, response string) Score { + got := NormalizeExactToken(response) + want := NormalizeExactToken(e.want) + if strings.EqualFold(got, want) { + return Score{Value: 1, Detail: fmt.Sprintf("equals %q", want)} + } + return Score{Value: 0, Detail: fmt.Sprintf("got %q, want %q", got, want)} +} diff --git a/internal/eval/exacttoken_test.go b/internal/eval/exacttoken_test.go new file mode 100644 index 0000000..a977876 --- /dev/null +++ b/internal/eval/exacttoken_test.go @@ -0,0 +1,71 @@ +package eval + +import ( + "context" + "testing" +) + +func TestNormalizeExactToken(t *testing.T) { + tests := []struct { + name string + in string + want string + }{ + {"bare", "GET", "GET"}, + {"surrounding whitespace", " GET \n", "GET"}, + {"single-quoted", "'GET'", "GET"}, + {"double-quoted", "\"GET\"", "GET"}, + {"backtick-quoted", "`GET`", "GET"}, + {"bolded", "**GET**", "GET"}, + {"trailing period", "GET.", "GET"}, + {"quoted with trailing period", "\"GET\".", "GET"}, + {"fenced, untagged", "```\nGET\n```", "GET"}, + {"fenced, tagged", "```text\nGET\n```", "GET"}, + {"fenced with surrounding prose", "The answer is:\n```\nGET\n```", "GET"}, + {"multi-word phrase", "Access-Control-Allow-Headers", "Access-Control-Allow-Headers"}, + {"multi-word phrase bolded with period", "**Access-Control-Allow-Headers**.", "Access-Control-Allow-Headers"}, + // Regression: an unpaired leading asterisk is meaningful content + // (a Go pointer type), not markdown decoration, and must survive. + {"unpaired leading asterisk is content, not decoration", "*string", "*string"}, + {"unpaired trailing asterisk is content, not decoration", "string*", "string*"}, + {"nested quote-then-bold with trailing period", "**\"GET\"**.", "GET"}, + {"italic single-asterisk wrap", "*GET*", "GET"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := NormalizeExactToken(tt.in) + if got != tt.want { + t.Errorf("NormalizeExactToken(%q) = %q, want %q", tt.in, got, tt.want) + } + }) + } +} + +func TestExactToken(t *testing.T) { + tests := []struct { + name string + want string + response string + wantVal float64 + }{ + {"bare exact match", "MX", "MX", 1}, + {"case insensitive", "MX", "mx", 1}, + {"quoted", "MX", "'MX'", 1}, + {"bolded", "MX", "**MX**", 1}, + {"trailing period", "MX", "MX.", 1}, + {"fenced", "MX", "```\nMX\n```", 1}, + {"restates a longer option list scores 0", "MX", "A, AAAA, CNAME, TXT, NS, MX", 0}, + {"wrong answer", "MX", "TXT", 0}, + {"empty response", "MX", "", 0}, + {"phrase exact match", "Access-Control-Allow-Headers", "Access-Control-Allow-Headers", 1}, + {"phrase restated with prose scores 0", "Access-Control-Allow-Headers", "The missing header is Access-Control-Allow-Headers.", 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := ExactToken(tt.want).Evaluate(context.Background(), tt.response) + if got.Value != tt.wantVal { + t.Errorf("ExactToken(%q).Evaluate(%q) = %v, want %v (detail: %s)", tt.want, tt.response, got.Value, tt.wantVal, got.Detail) + } + }) + } +} diff --git a/internal/eval/jsonfield.go b/internal/eval/jsonfield.go index 4b36fb4..8cc9896 100644 --- a/internal/eval/jsonfield.go +++ b/internal/eval/jsonfield.go @@ -111,12 +111,24 @@ func jsonValueAs[T comparable](v any) (T, error) { } } +// asFloat64 coerces a decoded JSON value to float64. A string is accepted +// too (B9), coerced only when it parses cleanly as a number end to end +// (strconv.ParseFloat on the trimmed string): a model that answers +// {"line":"8"} instead of {"line":8} - both are the same value, and the +// prompt asked for a JSON field, not a JSON type - must not score 0 for a +// mechanical typing difference. A non-numeric string ("eight") still fails. func asFloat64(v any) (float64, error) { switch n := v.(type) { case float64: return n, nil case json.Number: return n.Float64() + case string: + f, err := strconv.ParseFloat(strings.TrimSpace(n), 64) + if err != nil { + return 0, fmt.Errorf("expected number, got non-numeric string %q", n) + } + return f, nil default: return 0, fmt.Errorf("expected number, got %T (%v)", v, v) } diff --git a/internal/eval/jsonfield_test.go b/internal/eval/jsonfield_test.go index 644d48c..f885434 100644 --- a/internal/eval/jsonfield_test.go +++ b/internal/eval/jsonfield_test.go @@ -90,6 +90,43 @@ func TestJSONField_Int(t *testing.T) { } } +// TestJSONField_Int_NumericStringCoercion is the B9 regression: a model +// answering {"line":"8"} instead of {"line":8} must still score 1 - the +// prompt asked for a JSON field with the right value, not a specific JSON +// type. +func TestJSONField_Int_NumericStringCoercion(t *testing.T) { + e := JSONField("line", 8) + tests := []struct { + name string + response string + want float64 + }{ + {"numeric string coerces", `{"line":"8"}`, 1}, + {"numeric string with whitespace coerces", `{"line":" 8 "}`, 1}, + {"wrong numeric string", `{"line":"9"}`, 0}, + {"non-numeric string does not coerce", `{"line":"eight"}`, 0}, + {"native number still works", `{"line":8}`, 1}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := e.Evaluate(context.Background(), tt.response) + if got.Value != tt.want { + t.Errorf("Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail) + } + }) + } +} + +// TestJSONField_Float_NumericStringCoercion mirrors the int case for a +// float64-typed field. +func TestJSONField_Float_NumericStringCoercion(t *testing.T) { + e := JSONField("port", float64(8080)) + got := e.Evaluate(context.Background(), `{"port":"8080"}`) + if got.Value != 1 { + t.Errorf("Evaluate() = %v, want 1 (detail: %s)", got.Value, got.Detail) + } +} + func TestJSONField_NestedPath(t *testing.T) { e := JSONField("task1", "search_web") got := e.Evaluate(context.Background(), `{"task1":"search_web","task2":"read_file"}`) diff --git a/internal/eval/numeric.go b/internal/eval/numeric.go index 4af0bef..40d9524 100644 --- a/internal/eval/numeric.go +++ b/internal/eval/numeric.go @@ -18,12 +18,17 @@ type Number interface { } // digitPattern matches unsigned integers and decimals, including a -// leading-dot form with no integer part (".9896"). Longer alternatives are -// listed first so the match is not truncated to just the integer part of -// "0.9896" or just the dot-less ".9896". It deliberately excludes the sign: -// classifyMatch decides, per match, whether a leading '-' in the source -// text is a real minus sign or a compound-word hyphen ("x86-64"). -var digitPattern = regexp.MustCompile(`\d+\.\d+|\.\d+|\d+`) +// comma-grouped thousands form ("50,000", "1,234.5") and a leading-dot form +// with no integer part (".9896"). Longer/more specific alternatives are +// listed first so the match is not truncated to just "50" out of "50,000", +// or just the integer part of "0.9896", or just the dot-less ".9896". The +// comma-group alternative requires groups of exactly 3 digits after the +// first comma (`\d{1,3}(?:,\d{3})+`), so it does not misfire on an +// unrelated comma followed by a differently-sized number ("in 2024, 50 +// states"). It deliberately excludes the sign: classifyMatch decides, per +// match, whether a leading '-' in the source text is a real minus sign or +// a compound-word hyphen ("x86-64"). +var digitPattern = regexp.MustCompile(`\d{1,3}(?:,\d{3})+(?:\.\d+)?|\d+\.\d+|\.\d+|\d+`) // isWordByte reports whether b is a letter, digit, or underscore - the // standard regex \w character class - used to detect a digit run glued @@ -47,6 +52,11 @@ type numberCandidate struct { compoundHyphen bool } +// isDigitByte reports whether b is an ASCII digit. +func isDigitByte(b byte) bool { + return b >= '0' && b <= '9' +} + // classifyMatch inspects the characters immediately surrounding the // unsigned digit run text[start:end] and classifies it: // @@ -56,9 +66,14 @@ type numberCandidate struct { // - a '-' immediately before the run is a real minus sign only when the // character before THAT is not alphanumeric, so "x86-64" does not read // as "x86" followed by the negative number -64; +// - a digit run immediately preceded by ':' whose own left neighbour is +// a digit is the second term of a ratio ("64:1"'s "1") and is never a +// real candidate, at any tier, so extraction lands on the first term; // - a number joined by a hyphen to an adjacent word on either side // ("64-bit", "8-byte", the "64" in "x86-64" once its bogus sign is -// stripped) is flagged compoundHyphen rather than rejected outright. +// stripped) is flagged compoundHyphen rather than rejected outright; +// - a comma-grouped literal ("50,000", "1,234.5") has its commas +// stripped so the returned literal parses with strconv.ParseFloat. func classifyMatch(text string, start, end int) (numberCandidate, bool) { literal := text[start:end] @@ -73,6 +88,9 @@ func classifyMatch(text string, start, end int) (numberCandidate, bool) { if start > 0 && isWordByte(text[start-1]) { return numberCandidate{}, false // e.g. the "64" in "LP64" } + if start > 0 && text[start-1] == ':' && start-2 >= 0 && isDigitByte(text[start-2]) { + return numberCandidate{}, false // e.g. the "1" in "64:1" + } if end < len(text) && isWordByte(text[end]) { // Permit a multiplier suffix ("64x" = 64 times): a single trailing // x/X with nothing word-like after it. Hex ("0x2A") and identifiers @@ -90,6 +108,7 @@ func classifyMatch(text string, start, end int) (numberCandidate, bool) { compound = true // e.g. "64-bit", "24-byte" } + literal = strings.ReplaceAll(literal, ",", "") return numberCandidate{literal: literal, compoundHyphen: compound}, true } diff --git a/internal/eval/numeric_test.go b/internal/eval/numeric_test.go index b225a5c..7f306b4 100644 --- a/internal/eval/numeric_test.go +++ b/internal/eval/numeric_test.go @@ -35,6 +35,19 @@ func TestExtractLastNumber(t *testing.T) { // the last standalone number, which here is not the answer a // human would pick. Prompts should ask for the number alone. {"ambiguous parenthetical breakdown returns the wrong number by design", "Total: 24 bytes (22 bytes of members plus 2 tail padding)", 2, false}, + // B1: comma-grouped thousands separators. + {"comma-grouped thousands", "50,000", 50000, false}, + {"comma-grouped thousands with unit", "50,000 URLs", 50000, false}, + {"comma-grouped thousands in a sentence", "The limit is 50,000.", 50000, false}, + {"comma-grouped with a decimal tail", "1,234.5", 1234.5, false}, + {"comma-grouped, multiple groups", "12,345,678", 12345678, false}, + {"unrelated comma is not a thousands group", "in 2024, 50 states voted", 50, false}, + // B6: ratio form "N:M" must extract the first term, not the second. + {"ratio form extracts the first term", "64:1", 64, false}, + // Documented limitation, not a regression (B6): the word-form ratio + // is not cheaply distinguishable from two unrelated numbers, so + // this pins the current (non-ideal) behaviour deliberately. + {"word-form ratio is a documented limitation, not fixed", "The ratio is 64 to 1.", 1, false}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { diff --git a/internal/tests/agents_planning.go b/internal/tests/agents_planning.go index e50e576..42f752f 100644 --- a/internal/tests/agents_planning.go +++ b/internal/tests/agents_planning.go @@ -477,21 +477,25 @@ Respond with only a JSON object: {"next_action":"..."}` // // ground truth: prep_env finishes on day 0+3=3, prep_data finishes on day // 0+5=5. Milestone M cannot start until BOTH finish, so M starts on -// max(3,5)=5, and finishes 2 days later, on day 5+2=7. +// max(3,5)=5, and finishes 2 days later, on day 5+2=7. B7: the prompt now +// states the t+d finish-day rule explicitly rather than relying on a +// "day 0" framing a reader could also read as 1-indexed (which would make +// 6 a defensible, if unintended, answer); 7 remains the only correct one. var planMilestoneWant = max(3, 5) + 2 func planMilestoneDateTest() testkit.Test { - prompt := `Two prerequisite work items both start on day 0 and run in -parallel: + prompt := `Two prerequisite work items run in parallel, starting at day 0: - prep_env takes 3 days. - prep_data takes 5 days. Milestone M can only start once BOTH prep_env and prep_data have finished, and M itself takes 2 days once it starts. -On what day number does milestone M finish (counting the day work starts -as day 0, and each duration in whole days with no gaps)? Respond with only -the number.` +Use this rule: a task starting at day t with duration d finishes at day +t+d. Work starts at day 0. + +On what day number does milestone M finish? Respond with only the +number.` return testkit.Test{ ID: "agent-plan-milestone-date", diff --git a/internal/tests/catalog.go b/internal/tests/catalog.go index 8685cba..d9f86f6 100644 --- a/internal/tests/catalog.go +++ b/internal/tests/catalog.go @@ -2,11 +2,10 @@ package tests import "github.com/lukaszraczylo/llm-testbench/internal/testkit" -// All builds a Registry containing the full 17-test catalog: 3 -// programming/golang, 2 programming/python, 1 programming/typescript, 1 -// programming/c, 2 operations/macos, 2 operations/linux, 1 -// operations/kubernetes, 1 research/web, 1 research/whitepapers, 1 -// research/codebase, and 2 agents (tool-routing, planning). +// All builds a Registry containing the full test catalog, spanning the +// programming (golang, python, typescript, c), operations (macos, linux, +// kubernetes), research (web, whitepapers, codebase), and agents +// (tool-routing, planning) categories. func All() *testkit.Registry { r := testkit.NewRegistry() registerGolangTests(r) diff --git a/internal/tests/codebase_analysis.go b/internal/tests/codebase_analysis.go index 4fb3e5d..e0ca84b 100644 --- a/internal/tests/codebase_analysis.go +++ b/internal/tests/codebase_analysis.go @@ -1,6 +1,10 @@ package tests import ( + "context" + "fmt" + "regexp" + "github.com/lukaszraczylo/llm-testbench/internal/eval" "github.com/lukaszraczylo/llm-testbench/internal/testkit" ) @@ -157,8 +161,26 @@ const codeDedupSnippet = `func Dedup(a []int) []int { // codeBigODedupPattern accepts every common notation for O(n^2): a caret, // a double-star, the unicode superscript two, or n*n, each optionally -// wrapped as O(...) with flexible internal spacing. -const codeBigODedupPattern = `(?i)o\(\s*n\s*(\^\s*2|\*\*\s*2|²|\*\s*n)\s*\)` +// wrapped as O(...) with flexible internal spacing. Anchored to the whole +// string (codeBigODedupEval matches it against the whole normalized +// response, never a substring), so "not O(n^2), it's O(n)" - which +// contains the literal substring "O(n^2)" - correctly fails (B5) instead +// of matching the way an unanchored eval.Regex would. +var codeBigODedupPattern = regexp.MustCompile(`(?i)^o\(\s*n\s*(\^\s*2|\*\*\s*2|²|\*\s*n)\s*\)$`) + +// codeBigODedupEval anchors eval.NormalizeExactToken's normalization (fence +// extraction, quote/asterisk/period stripping) to codeBigODedupPattern, +// combining "accept several equivalent notations" with "match the whole +// answer, not a substring anywhere in it." +func codeBigODedupEval() eval.Evaluator { + return eval.EvaluatorFunc(func(_ context.Context, response string) eval.Score { + normalized := eval.NormalizeExactToken(response) + if codeBigODedupPattern.MatchString(normalized) { + return eval.Score{Value: 1, Detail: fmt.Sprintf("matches O(n^2) notation: %q", normalized)} + } + return eval.Score{Value: 0, Detail: fmt.Sprintf("does not match O(n^2) notation: %q", normalized)} + }) +} // codeBigODedupTest: state the worst-case time complexity of an inline // nested-loop deduplication function. @@ -181,7 +203,7 @@ terms of n = len(a)? Respond with only the Big-O expression, e.g. O(n).` Subcategory: "codebase", Description: "State the worst-case Big-O time complexity of an inline nested-loop deduplication function.", Prompt: prompt, - Eval: eval.Regex(codeBigODedupPattern), + Eval: codeBigODedupEval(), } } diff --git a/internal/tests/codebase_analysis_test.go b/internal/tests/codebase_analysis_test.go index bb4640d..9435fc9 100644 --- a/internal/tests/codebase_analysis_test.go +++ b/internal/tests/codebase_analysis_test.go @@ -14,6 +14,10 @@ func TestCodeBugLineDiffTest_Eval(t *testing.T) { want float64 }{ {"correct: root-cause line", `{"line":8}`, 1}, + // B9: a numeric-string JSON value must coerce the same as a + // native number - the prompt asked for the right line, not a + // specific JSON type. + {"correct as a numeric string", `{"line":"8"}`, 1}, {"wrong: points at the panic site instead of the root cause", `{"line":10}`, 0}, {"wrong line entirely", `{"line":6}`, 0}, } @@ -87,8 +91,14 @@ func TestCodeBigODedupTest_Eval(t *testing.T) { {"unicode superscript form", "O(n²)", 1}, {"n*n form", "O(n*n)", 1}, {"lowercase o", "o(n^2)", 1}, + {"quoted", `"O(n^2)"`, 1}, + {"trailing period", "O(n^2).", 1}, {"wrong: linear", "O(n)", 0}, {"wrong: linearithmic", "O(n log n)", 0}, + // B5: the correct notation appears as a substring of a negated + // sentence; the whole-response-anchored evaluator must not match + // on that substring. + {"negated: contains the substring O(n^2) but names O(n) as the answer", "not O(n^2), it's O(n) with a hash set", 0}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { diff --git a/internal/tests/codebase_trace.go b/internal/tests/codebase_trace.go index 48c4411..4478ab1 100644 --- a/internal/tests/codebase_trace.go +++ b/internal/tests/codebase_trace.go @@ -1,34 +1,25 @@ package tests import ( - "context" - "fmt" "strconv" - "strings" "github.com/lukaszraczylo/llm-testbench/internal/eval" "github.com/lukaszraczylo/llm-testbench/internal/testkit" ) // codeExactAnswer returns an Evaluator awarding full credit when the -// response, trimmed of whitespace, at most one layer of surrounding quote -// characters (' , ", or `), and a single trailing sentence-ending period, -// equals want case-insensitively. This accepts every materially-correct -// form of a forced single-token answer (bare, quoted, differently-cased, -// or with trailing punctuation) without loosening the match to accept a -// substring of a longer, wrong answer. +// response equals want under eval.ExactToken's normalization: a fenced +// code block extracted if present, whitespace trimmed, one layer of +// surrounding quote/backtick/asterisk decoration stripped, one trailing +// sentence-ending period stripped, compared case-insensitively. This +// accepts every materially-correct form of a forced single-token answer +// (bare, quoted, bolded, fenced, differently-cased, or with trailing +// punctuation) without loosening the match to accept a substring of a +// longer, wrong answer. Kept as a thin wrapper (rather than calling +// eval.ExactToken directly at each call site) so this file's intent - +// "the traced program's exact output token" - stays named and greppable. func codeExactAnswer(want string) eval.Evaluator { - return eval.EvaluatorFunc(func(_ context.Context, response string) eval.Score { - got := strings.TrimSpace(response) - got = strings.Trim(got, "\"'`") - got = strings.TrimSuffix(strings.TrimSpace(got), ".") - got = strings.TrimSpace(got) - wantTrimmed := strings.TrimSpace(want) - if strings.EqualFold(got, wantTrimmed) { - return eval.Score{Value: 1, Detail: fmt.Sprintf("equals %q", wantTrimmed)} - } - return eval.Score{Value: 0, Detail: fmt.Sprintf("got %q, want %q", got, wantTrimmed)} - }) + return eval.ExactToken(want) } // codeTracePythonSource is the inline Python function traced by diff --git a/internal/tests/codebase_trace_test.go b/internal/tests/codebase_trace_test.go index 26c8b0b..dabf96a 100644 --- a/internal/tests/codebase_trace_test.go +++ b/internal/tests/codebase_trace_test.go @@ -120,6 +120,39 @@ func TestCodeTraceTSWant_GroundTruth(t *testing.T) { if got != codeTraceTSWant { t.Errorf("codeTSPackTrace(%v) = %q, want %q", items, got, codeTraceTSWant) } + + // BN4: cross-check against a real JS engine too, not just the Go + // replica above. node cannot execute codeTraceTSSource's TypeScript + // type annotations directly, so this is the same logic with the two + // annotations (": number[]", ": string") stripped - everything else, + // including the reduce/ternary/toString(2) logic under test, is + // identical source. + if _, err := exec.LookPath("node"); err != nil { + t.Skip("node not found on PATH, skipping exec ground-truth cross-check") + } + + const jsSource = `function pack(items) { + return items.reduce((acc, x, i) => { + if (i % 2 === 0) return acc + x; + return acc + x * 2; + }, 0).toString(2); +} +console.log(pack([3, 4, 5, 6, 7])); +` + // #nosec G204 -- jsSource is a fixed, hardcoded string (the TS source + // embedded in the prompt with its two type annotations stripped), not + // external or response-controlled input. + cmd := exec.Command("node", "-e", jsSource) //nolint:gosec // see comment above + var stdout bytes.Buffer + cmd.Stdout = &stdout + cmd.Stderr = os.Stderr + if err := cmd.Run(); err != nil { + t.Fatalf("running node ground-truth script failed: %v", err) + } + nodeGot := strings.TrimSpace(stdout.String()) + if nodeGot != codeTraceTSWant { + t.Errorf("node-computed pack([3,4,5,6,7]) = %q, want %q", nodeGot, codeTraceTSWant) + } } func TestCodeTraceTSTest_Eval(t *testing.T) { diff --git a/internal/tests/golang.go b/internal/tests/golang.go index 23af0f4..986f907 100644 --- a/internal/tests/golang.go +++ b/internal/tests/golang.go @@ -492,7 +492,9 @@ printed, nothing else.` Subcategory: "golang", Description: "Trace the exact output of a context cancellation propagating synchronously from a parent to a derived context.", Prompt: prompt, - Eval: eval.Equals("context canceled"), + // A7: eval.ExactToken so a fenced or quoted rendering of the exact + // printed text still scores full credit. + Eval: eval.ExactToken("context canceled"), } } diff --git a/internal/tests/golang_test.go b/internal/tests/golang_test.go index 73b6c97..59dba35 100644 --- a/internal/tests/golang_test.go +++ b/internal/tests/golang_test.go @@ -479,6 +479,10 @@ func TestGoContextCancellationTraceTest_Eval(t *testing.T) { {"exact correct", "context canceled", 1}, {"correct case-insensitive", "CONTEXT CANCELED", 1}, {"correct with surrounding whitespace", " context canceled ", 1}, + // A7 regression: fenced/quoted decoration on the correct answer + // must still score full credit. + {"fenced", "```\ncontext canceled\n```", 1}, + {"quoted with trailing period", `"context canceled".`, 1}, {"wrong: assumes select falls to default", "not done", 0}, {"wrong: wrong sentinel error", "context deadline exceeded", 0}, } diff --git a/internal/tests/kubernetes.go b/internal/tests/kubernetes.go index 92465a8..08e5a67 100644 --- a/internal/tests/kubernetes.go +++ b/internal/tests/kubernetes.go @@ -349,7 +349,10 @@ spec: // app=inventory-api-svc - a value that matches no pod's labels, so the // Service has zero Endpoints. The single wrong field is the Service's // spec.selector.app value; it must be changed to "inventory-api" to match -// the pods. +// the pods. The prompt's JSON template pre-fills "field":"selector.app" +// as part of the requested response shape, so a model that never +// diagnosed the field at all could still copy that value verbatim; only +// "value" is actually evidence the model found the fix (AN2). func k8sServiceSelectorMismatchTest() testkit.Test { prompt := `Requests to this Service time out with no available endpoints. Here is the Deployment's pod template and the Service: @@ -360,10 +363,7 @@ Which single field in the Service YAML is wrong, and what value should it be changed to so the Service's Endpoints include these pods? Respond with only a JSON object: {"field":"selector.app","value":""}` - evaluator := eval.Mean( - eval.JSONField("field", "selector.app"), - eval.JSONField("value", "inventory-api"), - ) + evaluator := eval.JSONField("value", "inventory-api") return testkit.Test{ ID: "k8s-service-selector-mismatch", @@ -510,8 +510,11 @@ const k8sTraefikIngressRouteYAML = "apiVersion: traefik.io/v1alpha1\n" + " port: 80" // k8sTraefikIngressRoutePattern requires the corrected match expression, -// with the backtick-quoted domain exactly matching app.raczylo.com. -const k8sTraefikIngressRoutePattern = "(?i)host\\(`app\\.raczylo\\.com`\\)" +// with the domain app.raczylo.com quoted with either backticks (Traefik's +// own matcher syntax, as shown in the prompt's example) or double quotes +// (a common, equally unambiguous way a model renders the same literal) +// (AN4). +const k8sTraefikIngressRoutePattern = "(?i)host\\([`\"]app\\.raczylo\\.com[`\"]\\)" // k8sTraefikIngressRouteHostTest: correct a Traefik IngressRoute's Host() // match rule to the actual domain the service must answer for. @@ -581,7 +584,10 @@ BestEffort.` Subcategory: "kubernetes", Description: "Derive a pod's QoS class (Burstable) from requests that are set but not equal to limits.", Prompt: prompt, - Eval: eval.Equals("Burstable"), + // A7: eval.ExactToken (not eval.Equals) so a fenced, quoted, or + // bolded "Burstable" - a common way a model formats a forced + // one-word answer - still scores full credit. + Eval: eval.ExactToken("Burstable"), } } @@ -600,10 +606,15 @@ with what minimum value, to enforce "at least 2 available"?` // bounds voluntary disruptions (like a node drain evicting pods) for a // selected set of pods. Setting its minAvailable field to 2 enforces // that at least 2 of the 3 Postgres pods must remain available before - // the eviction API will permit another one to be drained. + // the eviction API will permit another one to be drained. The + // minAvailable check is a single regex tolerant of any short + // same-line phrasing between the key and the value (A10) - e.g. + // "minAvailable is set to 2" or "minAvailable: \"2\"" - rather than an + // enumerated list of exact phrasings that misses anything not on the + // list. evaluator := eval.Mean( eval.ContainsAll("PodDisruptionBudget"), - eval.ContainsAny("minAvailable: 2", "minAvailable=2", "minAvailable 2", "minAvailable of 2"), + eval.Regex(`(?i)minAvailable\b[^\n]{0,20}\b2\b`), ) return testkit.Test{ diff --git a/internal/tests/kubernetes_test.go b/internal/tests/kubernetes_test.go index 84be335..bd47d56 100644 --- a/internal/tests/kubernetes_test.go +++ b/internal/tests/kubernetes_test.go @@ -238,7 +238,12 @@ func TestK8sServiceSelectorMismatchTest_Eval(t *testing.T) { {"prose then JSON", "The selector doesn't match the pod labels.\n" + `{"field":"selector.app","value":"inventory-api"}`, 1}, {"fenced JSON", "```json\n" + `{"field":"selector.app","value":"inventory-api"}` + "\n```", 1}, {"points at the wrong field (targetPort instead of selector)", `{"field":"targetPort","value":"8080"}`, 0}, - {"right field, wrong replacement value", `{"field":"selector.app","value":"inventory-api-svc"}`, 0.5}, + // AN2: only "value" is graded, since the prompt's own JSON template + // pre-fills "field":"selector.app" - copying that back verbatim is + // not evidence of diagnosis, so it no longer earns partial credit + // on its own (this case used to score 0.5). + {"right field, wrong replacement value", `{"field":"selector.app","value":"inventory-api-svc"}`, 0}, + {"field name wrong but value correct still scores full credit (AN2 bug probe)", `{"field":"wrongfield","value":"inventory-api"}`, 1}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -327,6 +332,7 @@ func TestK8sTraefikIngressRouteHostTest_Eval(t *testing.T) { {"uppercase Host keyword variant", "MATCH: HOST(`app.raczylo.com`)", 1}, {"still the old domain", "match: Host(`app.example.com`)", 0}, {"right function, wrong domain typo", "match: Host(`app.raczlo.com`)", 0}, + {"double-quoted domain (AN4 bug probe)", `match: Host("app.raczylo.com")`, 1}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -349,6 +355,11 @@ func TestK8sQoSClassTest_Eval(t *testing.T) { {"exact word", "Burstable", 1}, {"lowercase", "burstable", 1}, {"trailing whitespace", "Burstable\n", 1}, + // A7 regression: fenced/quoted/bolded/trailing-period decoration + // on the correct answer must still score full credit. + {"fenced", "```\nBurstable\n```", 1}, + {"quoted", `"Burstable"`, 1}, + {"bolded with period", "**Burstable**.", 1}, {"wrong class, mistakes it for Guaranteed", "Guaranteed", 0}, {"wrong class, mistakes it for BestEffort", "BestEffort", 0}, } @@ -375,6 +386,8 @@ func TestK8sCNPGPDBTest_Eval(t *testing.T) { {"english phrasing of the value", "Create a PodDisruptionBudget with a minAvailable of 2.", 1}, {"wrong resource entirely (a NetworkPolicy cannot do this)", "Use a NetworkPolicy with minAvailable: 2.", 0.5}, {"right resource, wrong/missing value", "Use a PodDisruptionBudget, but leave the fields at their defaults.", 0.5}, + {"'set to' phrasing, not covered by the old enumerated substring list (A10 bug probe)", "Use a PodDisruptionBudget with minAvailable set to 2.", 1}, + {"quoted numeric value in YAML (A10 bug probe)", `Use a PodDisruptionBudget with minAvailable: "2".`, 1}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { diff --git a/internal/tests/linux.go b/internal/tests/linux.go index e3b63b9..d756c92 100644 --- a/internal/tests/linux.go +++ b/internal/tests/linux.go @@ -1,6 +1,9 @@ package tests import ( + "context" + "regexp" + "github.com/lukaszraczylo/llm-testbench/internal/eval" "github.com/lukaszraczylo/llm-testbench/internal/testkit" ) @@ -74,11 +77,19 @@ contents only.` } } -// linuxJournalctlFilterPattern requires journalctl filtered to unit -// nginx.service and a since-window of 2 hours, in the canonical -// "-u UNIT --since WINDOW" order (the overwhelmingly common idiom), on a -// single line so a scattered mention across paragraphs does not count. -const linuxJournalctlFilterPattern = `(?i)\bjournalctl\b[^\n]*?-u\s+nginx\.service\b[^\n]*?--since\s+["']?-?2\s*h(?:ours?)?(?:\s+ago)?["']?` +// linuxJournalctlUnitPattern matches the unit filter in either short (-u) +// or long (--unit) form, tolerant of "=" or whitespace between the option +// and its value (journalctl, via getopt_long, accepts both). It is checked +// independently of linuxJournalctlSincePattern below (A3), so either +// option can appear on the line before the other - the original single +// regex hardcoded "-u ... --since ..." order and rejected the equally +// valid "--unit"/"-S" long/short forms and "--since=" spelling. +const linuxJournalctlUnitPattern = `(?i)\bjournalctl\b[^\n]*(?:-u|--unit)[=\s]+nginx\.service\b` + +// linuxJournalctlSincePattern matches the since-window filter in either +// short (-S) or long (--since) form, independent of +// linuxJournalctlUnitPattern (A3). +const linuxJournalctlSincePattern = `(?i)\bjournalctl\b[^\n]*(?:-S|--since)[=\s]+["']?-?2\s*h(?:ours?)?(?:\s+ago)?["']?` // linuxJournalctlFilterTest: one-line journalctl command filtered to one // unit over a relative time window. @@ -86,13 +97,18 @@ func linuxJournalctlFilterTest() testkit.Test { prompt := `Give a single, one-line journalctl command that shows only log entries for the systemd unit "nginx.service" from the last 2 hours.` + evaluator := eval.All( + eval.W(eval.Regex(linuxJournalctlUnitPattern), 1), + eval.W(eval.Regex(linuxJournalctlSincePattern), 1), + ) + return testkit.Test{ ID: "linux-journalctl-filter", Category: "operations", Subcategory: "linux", Description: "One-line journalctl command filtered by unit and a relative 2-hour time window.", Prompt: prompt, - Eval: eval.Regex(linuxJournalctlFilterPattern), + Eval: evaluator, } } @@ -138,11 +154,16 @@ JSON object: {"oom_killed": true or false, "oom_kill_count": }.` // linuxNftablesDNATPattern requires a DNAT rule (iptables or nft form) on // the nat table's prerouting chain, matching TCP port 8080 and forwarding -// to 10.0.5.20:80, on a single line. iptables and nft flags/keywords follow -// a fixed canonical order (table/chain, then match, then target), so this -// uses the same non-greedy same-line anchoring as the pct-exec lesson -// rather than a permutation of independently-orderable flags. -const linuxNftablesDNATPattern = `(?i)(?:iptables|nft)\b[^\n]*?nat[^\n]*?prerouting[^\n]*?(?:--dport|dport)\s*8080[^\n]*?dnat[^\n]*?10\.0\.5\.20:80` +// to 10.0.5.20:80, on a single line. The "nat" table and "prerouting" +// chain tokens are matched order-free relative to each other (A12): a +// syntactically valid iptables invocation can give -A PREROUTING before +// -t nat (e.g. "iptables -A PREROUTING -t nat -p tcp --dport 8080 -j DNAT +// --to-destination 10.0.5.20:80"), and the original pattern's fixed +// "nat...prerouting" order rejected that ordering. The rest of the match +// (dport, dnat, destination) keeps its original same-line, in-order +// anchoring, since iptables and nft do put the match/target keywords after +// the table/chain in both valid orderings. +const linuxNftablesDNATPattern = `(?i)(?:iptables|nft)\b[^\n]*?(?:nat[^\n]*?prerouting|prerouting[^\n]*?nat)[^\n]*?(?:--dport|dport)\s*8080[^\n]*?dnat[^\n]*?10\.0\.5\.20:80` // linuxNftablesDNATTest: one-line DNAT rule forwarding an external port to // an internal host:port. @@ -194,9 +215,17 @@ data. Respond with only a JSON array of the step ids, e.g. } } -// linuxSSHPortForwardPattern requires an ssh local port-forward (-L) from -// local port 15432 to the remote Postgres, connecting to the correct host. -const linuxSSHPortForwardPattern = `(?i)\bssh\b[^\n]*?-L\s*15432:(?:127\.0\.0\.1|localhost):5432[^\n]*?ops@db1\.internal` +// linuxSSHPortForwardSpecPattern matches the -L local-forward flag and its +// argument, independent of where the SSH destination appears on the line - +// -L can come before or after the destination in a valid ssh invocation +// (A6). +const linuxSSHPortForwardSpecPattern = `(?i)\bssh\b[^\n]*-L\s*15432:(?:127\.0\.0\.1|localhost):5432\b` + +// linuxSSHPortForwardDestPattern matches the SSH destination in either its +// "user@host" form or the equivalent "-l user ... host" form (A6): both +// name the same destination, and the original pattern only accepted the +// former. +const linuxSSHPortForwardDestPattern = `(?i)\bssh\b[^\n]*(?:ops@db1\.internal|-l\s+ops\b[^\n]*\bdb1\.internal\b)` // linuxSSHPortForwardTest: ssh -L local port-forward command derived from a // scenario. @@ -207,13 +236,18 @@ single, one-line ssh command, run from your laptop, that forwards your laptop's local port 15432 to that remote Postgres instance, so a local client can connect to localhost:15432.` + evaluator := eval.Mean( + eval.Regex(linuxSSHPortForwardSpecPattern), + eval.Regex(linuxSSHPortForwardDestPattern), + ) + return testkit.Test{ ID: "linux-ssh-port-forward", Category: "operations", Subcategory: "linux", Description: "One-line ssh -L local port-forward command reaching a Postgres instance that only listens on remote loopback.", Prompt: prompt, - Eval: eval.Regex(linuxSSHPortForwardPattern), + Eval: evaluator, } } @@ -221,7 +255,26 @@ client can connect to localhost:15432.` // "03:30 every Monday", tolerant only of whitespace width between fields - // unlike eval.Equals, this stays correct even if the model adds a leading // or trailing space, since the fields themselves have no valid rephrasing. -const linuxCronExpressionPattern = `^\s*30\s+3\s+\*\s+\*\s+1\s*$` +// (?m) makes ^/$ match per line rather than only at the very start/end of +// the whole string, so the expression still matches as one line within a +// longer, fenced, or multi-line response (A5). +const linuxCronExpressionPattern = `(?m)^\s*30\s+3\s+\*\s+\*\s+1\s*$` + +// linuxCronExpressionEval strips a markdown code fence (if any) before +// applying linuxCronExpressionPattern. Without this, a response that +// wraps the correct expression in a ` + "```" + ` fence scored 0, because the fence +// markers and surrounding whitespace kept the line-anchored regex from +// finding a clean match (A5). +func linuxCronExpressionEval() eval.Evaluator { + re := regexp.MustCompile(linuxCronExpressionPattern) + return eval.EvaluatorFunc(func(_ context.Context, response string) eval.Score { + stripped := eval.ExtractCodeBlock(response, "") + if re.MatchString(stripped) { + return eval.Score{Value: 1, Detail: "matches the 5-field cron expression"} + } + return eval.Score{Value: 0, Detail: "does not match the 5-field cron expression"} + }) +} // linuxCronExpressionTest: produce the 5-field cron expression matching a // stated schedule. @@ -241,7 +294,7 @@ with only the expression, nothing else.` Subcategory: "linux", Description: "Produce the 5-field cron expression for a 03:30-every-Monday schedule.", Prompt: prompt, - Eval: eval.Regex(linuxCronExpressionPattern), + Eval: linuxCronExpressionEval(), } } @@ -258,11 +311,15 @@ unit's WantedBy needs.` // paired with a same-named .service unit (the work). The .timer's // OnCalendar= key sets the schedule (e.g. "*-*-* 02:00:00" for daily at // 02:00), and its [Install] section needs WantedBy=timers.target for - // systemctl enable to activate it. + // systemctl enable to activate it. The WantedBy check accepts "=", + // ":", or "(is) set to" between the key and its value (A15), not just a + // bare "WantedBy=timers.target" substring - a response that phrases + // this in prose ("WantedBy is set to timers.target") states exactly the + // same fact. evaluator := eval.Mean( eval.ContainsAll(".service", ".timer"), eval.ContainsAny("OnCalendar"), - eval.ContainsAll("WantedBy=timers.target"), + eval.Regex(`(?i)WantedBy\s*(?:=|:|\b(?:is\s+)?set to\b)\s*timers\.target`), ) return testkit.Test{ @@ -284,7 +341,12 @@ Buffers: 512000 kB Cached: 3200000 kB` // linuxProcMeminfoAvailableWant is MemAvailable (9876000 kB) converted to -// whole megabytes, floored: 9876000 / 1024 = 9644.53125 -> 9644. +// whole megabytes, floored: 9876000 / 1024 = 9644.53125 -> 9644. The +// prompt asks for the value rounded down, so 9644 is the single exact +// correct answer - tolerance 0 below, not 1 (A17): a tolerance of 1 was +// letting off-by-one wrong answers (9643 or 9645) score full credit for a +// deterministic arithmetic conversion with no room for a legitimate +// rephrasing. var linuxProcMeminfoAvailableWant = 9876000 / 1024 // linuxProcMeminfoAvailableTest: interpret /proc/meminfo's MemAvailable @@ -312,6 +374,6 @@ so convert by dividing by 1024, not 1000. Respond with only the number.` Subcategory: "linux", Description: "Convert /proc/meminfo's MemAvailable field (not MemFree) to whole megabytes.", Prompt: prompt, - Eval: eval.Numeric(eval.ExtractLastNumber[int], linuxProcMeminfoAvailableWant, 1), + Eval: eval.Numeric(eval.ExtractLastNumber[int], linuxProcMeminfoAvailableWant, 0), } } diff --git a/internal/tests/linux_test.go b/internal/tests/linux_test.go index b335483..5cb48ef 100644 --- a/internal/tests/linux_test.go +++ b/internal/tests/linux_test.go @@ -138,8 +138,16 @@ func TestLinuxJournalctlFilterTest_Eval(t *testing.T) { {"quoted negative window", `journalctl -u nginx.service --since "-2 hours"`, 1}, {"english phrasing window", `journalctl -u nginx.service --since "2 hours ago"`, 1}, {"compact -2h form, no quotes", `Run: journalctl -u nginx.service --since -2h`, 1}, - {"wrong unit", `journalctl -u apache.service --since "-2 hours"`, 0}, - {"missing unit filter entirely", `journalctl --since "-2 hours"`, 0}, + // The unit and since checks are independent (A3): a response that + // nails one but not the other now earns that check's half-credit + // rather than scoring 0 outright, the same partial-credit + // convention eval.All already uses elsewhere in this file. + {"wrong unit still credits the correct since-window", `journalctl -u apache.service --since "-2 hours"`, 0.5}, + {"missing unit filter still credits the correct since-window", `journalctl --since "-2 hours"`, 0.5}, + {"missing both filters entirely scores 0", `journalctl -f`, 0}, + {"long-form --unit= (A3 bug probe)", `journalctl --unit=nginx.service --since=-2h`, 1}, + {"short-form -S (A3 bug probe)", `journalctl -u nginx.service -S -2h`, 1}, + {"since option before unit option, order-free (A3 bug probe)", `journalctl --since "-2 hours" -u nginx.service`, 1}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -208,6 +216,11 @@ func TestLinuxNftablesDNATTest_Eval(t *testing.T) { response: `iptables -t nat -A OUTPUT -p tcp --dport 8080 -j DNAT --to-destination 10.0.5.20:80`, want: 0, }, + { + name: "-A PREROUTING given before -t nat is equally valid iptables syntax (A12 bug probe)", + response: `iptables -A PREROUTING -t nat -p tcp --dport 8080 -j DNAT --to-destination 10.0.5.20:80`, + want: 1, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -254,8 +267,13 @@ func TestLinuxSSHPortForwardTest_Eval(t *testing.T) { {"loopback IP form", `ssh -L 15432:127.0.0.1:5432 ops@db1.internal`, 1}, {"localhost form", `ssh -L 15432:localhost:5432 ops@db1.internal`, 1}, {"extra flags before -L", `ssh -N -L 15432:127.0.0.1:5432 ops@db1.internal`, 1}, - {"wrong local port", `ssh -L 5432:127.0.0.1:5432 ops@db1.internal`, 0}, - {"wrong host", `ssh -L 15432:127.0.0.1:5432 ops@db2.internal`, 0}, + // The forward-spec and destination checks are independent (A6): + // getting one right and the other wrong now earns that check's + // half-credit instead of scoring 0 outright. + {"wrong local port still credits the correct destination", `ssh -L 5432:127.0.0.1:5432 ops@db1.internal`, 0.5}, + {"wrong host still credits the correct forward spec", `ssh -L 15432:127.0.0.1:5432 ops@db2.internal`, 0.5}, + {"destination given before -L, options-after-destination (A6 bug probe)", `ssh ops@db1.internal -L 15432:127.0.0.1:5432`, 1}, + {"-l user form for the destination (A6 bug probe)", `ssh -L 15432:127.0.0.1:5432 -l ops db1.internal`, 1}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -280,6 +298,8 @@ func TestLinuxCronExpressionTest_Eval(t *testing.T) { {"tab-separated fields", "30\t3\t*\t*\t1", 1}, {"wrong day of week", "30 3 * * 2", 0}, {"wrong time", "45 3 * * 1", 0}, + {"fenced code block, no language tag (A5 bug probe)", "```\n30 3 * * 1\n```", 1}, + {"fenced code block with a language tag (A5 bug probe)", "```cron\n30 3 * * 1\n```", 1}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -326,6 +346,16 @@ WantedBy=timers.target.`, response: `Just add a backup.service and backup.timer, and enable it with WantedBy=multi-user.target.`, want: 1.0 / 3.0, }, + { + name: "WantedBy with a colon separator (A15 bug probe)", + response: `backup.service + backup.timer, OnCalendar=*-*-* 02:00:00, WantedBy: timers.target`, + want: 1, + }, + { + name: "WantedBy phrased as 'set to' in prose (A15 bug probe)", + response: `Use backup.service and backup.timer. OnCalendar=02:00 daily. WantedBy is set to timers.target under [Install].`, + want: 1, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -350,6 +380,8 @@ func TestLinuxProcMeminfoAvailableTest_Eval(t *testing.T) { {"with unit suffix", "9644 MB", 1}, {"divided by 1000 instead of 1024", "9876", 0}, {"used MemFree instead of MemAvailable", "1000", 0}, + {"off-by-one must not pass now that tolerance is 0, not 1 (A17 bug probe)", "9645", 0}, + {"off-by-one the other direction must not pass (A17 bug probe)", "9643", 0}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { diff --git a/internal/tests/macos.go b/internal/tests/macos.go index 61984a7..ca6168d 100644 --- a/internal/tests/macos.go +++ b/internal/tests/macos.go @@ -1,6 +1,10 @@ package tests import ( + "context" + "regexp" + "strings" + "github.com/lukaszraczylo/llm-testbench/internal/eval" "github.com/lukaszraczylo/llm-testbench/internal/testkit" ) @@ -95,7 +99,12 @@ file belongs in for a per-machine (not per-user) job.` // single-quoted or double-quoted extension argument, invoking a foo->bar // substitution on file.txt, on a single line (no DOTALL) so a scattered, // multi-paragraph mention of the same tokens does not count as a real -// one-line command. +// one-line command. The substitution's delimiter is accepted as any of the +// common sed delimiter characters (/, |, comma, #, colon, semicolon, @), +// not just "/" - "s/foo/bar/" is the conventional example, but any +// consistently-used delimiter is equally correct BSD sed syntax (AN3). Go's +// RE2 engine has no backreferences, so each delimiter choice is spelled out +// as its own alternative rather than captured-and-reused. // // ground truth: BSD sed's -i flag takes a mandatory in-place-suffix // argument (used for backup files); passing an empty single-quoted or @@ -104,7 +113,7 @@ file belongs in for a per-machine (not per-user) job.` // "sed -i 's/foo/bar/' file.txt" (no argument to -i) works on Linux but // errors on macOS, where it is parsed as "-i 's/foo/bar/'" (using the sed // script itself as the backup suffix) followed by a missing script operand. -const macosSedPattern = `(?i)\bsed\b[^\n]*-i\s*(''|"")[^\n]*s/foo/bar/g?[^\n]*file\.txt` +const macosSedPattern = `(?i)\bsed\b[^\n]*-i\s*(''|"")[^\n]*(?:s/foo/bar/|s\|foo\|bar\||s,foo,bar,|s#foo#bar#|s:foo:bar:|s;foo;bar;|s@foo@bar@)g?[^\n]*file\.txt` // macosSedInplaceTest: give the portable BSD sed one-liner for in-place // substitution with no backup file, contrasted with the GNU-only form that @@ -158,7 +167,7 @@ explicit domain targeting).` evaluator := eval.Mean( eval.ContainsAny("bootstrap"), eval.ContainsAny("bootstrap system", "system domain", "domain system", "target system", "system target"), - eval.ContainsAny("deprecated", "avoid launchctl load", "no longer recommended", "outdated"), + eval.ContainsAny("deprecated", "avoid launchctl load", "no longer recommended", "outdated", "legacy", "superseded", "obsolete", "replaced"), ) return testkit.Test{ @@ -171,9 +180,39 @@ explicit domain targeting).` } } -// macosPlutilPattern requires the built-in plutil -p pretty-printer -// invoked directly on the plist path, on one line. -const macosPlutilPattern = `(?i)\bplutil\b[^\n]*-p[^\n]*/Library/Preferences/com\.example\.app\.plist` +// macosPlutilFlagPattern requires a properly bounded "-p" flag token +// (immediately preceded by whitespace or the start of the response, and +// immediately followed by whitespace or the end of the response). An +// unbounded substring search for "-p" also matches inside the word +// "pretty-print" (its internal hyphen followed by "p"), which would let a +// response that only talks *about* pretty-printing - without actually +// invoking "plutil -p" - score a false positive (A2). +var macosPlutilFlagPattern = regexp.MustCompile(`(?i)(?:^|\s)-p(?:\s|$)`) + +// macosPlutilPlistReadEval accepts three materially correct ways to read a +// plist file's contents in human-readable form using only macOS-bundled +// tools: "plutil -p " (with -p bounded as an actual flag token, not +// matched as a substring of "pretty-print"), "plutil -convert xml1 -o - +// " (convert to XML and write to stdout instead of editing in +// place), and "defaults read " (A2). +func macosPlutilPlistReadEval() eval.Evaluator { + return eval.EvaluatorFunc(func(_ context.Context, response string) eval.Score { + if !strings.Contains(response, "com.example.app") { + return eval.Score{Value: 0, Detail: "does not name the target plist/domain"} + } + lower := strings.ToLower(response) + switch { + case strings.Contains(lower, "plutil") && macosPlutilFlagPattern.MatchString(response): + return eval.Score{Value: 1, Detail: "plutil -p (a properly bounded flag token)"} + case strings.Contains(lower, "plutil") && strings.Contains(lower, "-convert") && strings.Contains(lower, "xml1"): + return eval.Score{Value: 1, Detail: "plutil -convert xml1"} + case strings.Contains(lower, "defaults read"): + return eval.Score{Value: 1, Detail: "defaults read"} + default: + return eval.Score{Value: 0, Detail: "no recognized plist-reading command"} + } + }) +} // macosPlutilPlistReadTest: pretty-print a plist file's contents using a // tool that ships with macOS (no Homebrew installs). @@ -190,17 +229,39 @@ that prints the plist's contents to your terminal in a readable // ground truth: plutil ships with macOS and its -p flag pretty-prints a // plist (binary or XML) to stdout in a human-readable form, taking the // file path directly - no domain-name lookup or Homebrew install - // needed, unlike "defaults read ". + // needed, unlike "defaults read ". "plutil -convert xml1 -o -" + // (convert to XML, write to stdout) and "defaults read" are also + // accepted as materially correct alternatives (A2). return testkit.Test{ ID: "macos-plutil-plist-read", Category: "operations", Subcategory: "macos", Description: "Pretty-print a plist file's contents using the built-in plutil -p command.", Prompt: prompt, - Eval: eval.Regex(macosPlutilPattern), + Eval: macosPlutilPlistReadEval(), } } +// macosMdfindSpotlightEval gates the whole score on mdfind actually being +// used: the prompt itself supplies both a plausible flag ("-name" is a real +// "find" flag too) and the search term ("invoice"), so a bare filesystem +// walk with "find" that happens to echo those prompt-supplied tokens must +// not be able to reach a passing score. Only once mdfind is confirmed +// present does the response get scored on the remaining, more permissive +// criteria (A13). +func macosMdfindSpotlightEval() eval.Evaluator { + sub := eval.Mean( + eval.ContainsAny("-onlyin", "-name"), + eval.ContainsAny("invoice"), + ) + return eval.EvaluatorFunc(func(ctx context.Context, response string) eval.Score { + if !strings.Contains(strings.ToLower(response), "mdfind") { + return eval.Score{Value: 0, Detail: "mdfind not used; a filesystem walk is not what was asked for"} + } + return sub.Evaluate(ctx, response) + }) +} + // macosMdfindSpotlightTest: use mdfind (Spotlight's metadata index) instead // of a full filesystem walk to find files by name. func macosMdfindSpotlightTest() testkit.Test { @@ -211,20 +272,16 @@ whose name contains "invoice", scoped only to your home directory (~).` // ground truth: mdfind queries the already-built Spotlight index rather // than walking the filesystem; -onlyin scopes the search to a // directory, and either -name or a plain query string matches on file - // name. - evaluator := eval.Mean( - eval.ContainsAll("mdfind"), - eval.ContainsAny("-onlyin", "-name"), - eval.ContainsAny("invoice"), - ) - + // name. mdfind's presence is a hard gate (A13): "-name" is also a real + // "find" flag, and "invoice" is echoed straight from the prompt, so + // neither can carry the score for a wrong, find-based answer. return testkit.Test{ ID: "macos-mdfind-spotlight", Category: "operations", Subcategory: "macos", Description: "Use mdfind against Spotlight's index, scoped to a directory, instead of a full filesystem walk.", Prompt: prompt, - Eval: evaluator, + Eval: macosMdfindSpotlightEval(), } } @@ -238,11 +295,14 @@ the command to create a local snapshot now, and the separate command to list the local snapshots that currently exist.` // ground truth: tmutil manages local APFS snapshots independently of - // having a Time Machine backup destination attached. "tmutil snapshot" - // creates one immediately; "tmutil listlocalsnapshots /" lists the - // local snapshots that exist for the root volume. + // having a Time Machine backup destination attached. "tmutil + // localsnapshot" is the current, documented verb that creates one + // immediately ("tmutil snapshot" does not appear in tmutil's own usage + // listing on current macOS and is accepted only as a legacy/undocumented + // alias - verified on this machine, A1); "tmutil listlocalsnapshots /" + // lists the local snapshots that exist for the root volume. evaluator := eval.Mean( - eval.ContainsAny("tmutil snapshot"), + eval.ContainsAny("tmutil localsnapshot", "tmutil snapshot"), eval.ContainsAny("tmutil listlocalsnapshots", "listlocalsnapshots"), ) @@ -300,11 +360,13 @@ tool built into macOS, that does this.` // ground truth: caffeinate, bundled with macOS, takes a power-management // assertion for as long as a given utility runs when you pass that // utility as its argument, and releases the assertion automatically - // when the utility exits - no manual cleanup needed. - evaluator := eval.Mean( - eval.ContainsAll("caffeinate"), - eval.ContainsAny("migrate.sh"), - ) + // when the utility exits - no manual cleanup needed. migrate.sh must be + // caffeinate's own argument on the same command line (e.g. + // "caffeinate ./migrate.sh"), not merely mentioned somewhere in the + // response: a backgrounded "caffeinate & ./migrate.sh" variant runs the + // two independently, so caffeinate's assertion does not track the + // script's lifetime and would not auto-release when it exits (A11). + evaluator := eval.Regex(`(?i)\bcaffeinate\b[^\n&;|]*\.?/?migrate\.sh`) return testkit.Test{ ID: "macos-caffeinate", @@ -317,8 +379,11 @@ tool built into macOS, that does this.` } // macosStatPattern requires the BSD stat -f format-string form printing -// only the byte size (%z) of file.txt. -const macosStatPattern = `(?i)\bstat\b[^\n]*-f\s*"?%z"?[^\n]*file\.txt` +// only the byte size (%z) of file.txt. The %z format specifier is accepted +// bare or wrapped in either single or double quotes (both are valid shell +// quoting for a format string with no special characters in it) - not just +// double quotes (A4). +const macosStatPattern = `(?i)\bstat\b[^\n]*-f\s*['"]?%z['"]?[^\n]*file\.txt` // macosStatBSDGNUTest: give the BSD stat command (macOS's built-in stat) // for a file's size in bytes, and name the differing GNU stat flag. diff --git a/internal/tests/macos_test.go b/internal/tests/macos_test.go index 3aaa953..8826d4a 100644 --- a/internal/tests/macos_test.go +++ b/internal/tests/macos_test.go @@ -92,6 +92,9 @@ func TestMacosSedInplaceTest_Eval(t *testing.T) { {"global replace flag still matches", `sed -i '' 's/foo/bar/g' file.txt`, 1}, {"GNU-style with no extension argument is wrong on macOS", `sed -i 's/foo/bar/' file.txt`, 0}, {"backup extension creates a file, contradicting the requirement", `sed -i.bak 's/foo/bar/' file.txt`, 0}, + {"pipe delimiter is equally valid sed syntax (AN3)", `sed -i '' 's|foo|bar|' file.txt`, 1}, + {"comma delimiter (AN3)", `sed -i '' 's,foo,bar,g' file.txt`, 1}, + {"hash delimiter (AN3)", `sed -i "" 's#foo#bar#' file.txt`, 1}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -128,6 +131,10 @@ Apple deprecated for lacking an explicit domain target.` {"good answer, alternate phrasing", goodPhrasing, 1}, {"wrong subcommand (deprecated load only) scores 0", badWrongSubcommand, 0}, {"missing domain target and deprecation note loses two thirds", badMissingDomain, 1.0 / 3.0}, + {"'legacy' deprecation cue (A14)", `Use launchctl bootstrap system /Library/LaunchDaemons/com.example.myjob.plist; launchctl load is the legacy subcommand.`, 1}, + {"'superseded' deprecation cue (A14)", `launchctl bootstrap, system domain target, superseded launchctl load.`, 1}, + {"'obsolete' deprecation cue (A14)", `bootstrap system domain target; launchctl load is obsolete.`, 1}, + {"'replaced' deprecation cue (A14)", `launchctl bootstrap system domain target replaced launchctl load.`, 1}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -152,6 +159,9 @@ func TestMacosPlutilPlistReadTest_Eval(t *testing.T) { {"quoted path", `plutil -p "/Library/Preferences/com.example.app.plist"`, 1}, {"wrong tool entirely (cat prints binary garbage)", `cat /Library/Preferences/com.example.app.plist`, 0}, {"plutil without -p just validates, does not print", `plutil /Library/Preferences/com.example.app.plist`, 0}, + {"plutil -convert xml1 to stdout is a materially correct alternative (A2)", `plutil -convert xml1 -o - /Library/Preferences/com.example.app.plist`, 1}, + {"defaults read is a materially correct alternative (A2)", `defaults read com.example.app`, 1}, + {"mentions pretty-print in prose without invoking -p as a bounded flag (A2 bug probe)", `plutil can pretty-print the plist at /Library/Preferences/com.example.app.plist, though I don't recall the exact flag.`, 0}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -174,8 +184,9 @@ func TestMacosMdfindSpotlightTest_Eval(t *testing.T) { {"onlyin with query", `mdfind -onlyin ~ "invoice"`, 1}, {"name flag", `mdfind -onlyin ~ -name invoice`, 1}, {"prose wrapped", `Run mdfind -onlyin ~ invoice to search the Spotlight index.`, 1}, - {"uses find instead of mdfind, walks the filesystem", `find ~ -iname "*invoice*"`, 1.0 / 3.0}, - {"mdfind used but searching for the wrong term entirely", `mdfind -onlyin ~ "receipt"`, 2.0 / 3.0}, + {"uses find instead of mdfind, walks the filesystem, gated to 0 (A13)", `find ~ -iname "*invoice*"`, 0}, + {"mdfind used but searching for the wrong term entirely", `mdfind -onlyin ~ "receipt"`, 0.5}, + {"find command echoing both prompt-supplied tokens must not pass the gate (A13 bug probe)", `find ~ -name "*invoice*"`, 0}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -200,6 +211,8 @@ func TestMacosAPFSSnapshotTest_Eval(t *testing.T) { {"listlocalsnapshots without explicit tmutil prefix restated", "First tmutil snapshot, then check with listlocalsnapshots /.", 1}, {"only creates, never lists", "Just run tmutil snapshot.", 0.5}, {"wrong tool (diskutil) mentioned instead of tmutil", "Use diskutil apfs snapshot / to create one.", 0}, + {"current documented localsnapshot verb (A1)", "Create: tmutil localsnapshot\nList: tmutil listlocalsnapshots /", 1}, + {"localsnapshot only, never lists (A1)", "Just run tmutil localsnapshot.", 0.5}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -266,8 +279,9 @@ func TestMacosCaffeinateTest_Eval(t *testing.T) { {"bare invocation", `caffeinate ./migrate.sh`, 1}, {"with system-sleep flag", `caffeinate -s ./migrate.sh`, 1}, {"prose wrapped", "Run `caffeinate -i ./migrate.sh` and it stops holding the assertion once migrate.sh exits.", 1}, - {"wrong script named, not the target", `caffeinate ./other-script.sh`, 0.5}, + {"wrong script named, not the target, scores 0 now that migrate.sh must be on the same line (A11)", `caffeinate ./other-script.sh`, 0}, {"unrelated tool, does not auto-release on exit", `Just run pmset noidle before starting your script.`, 0}, + {"backgrounded caffeinate is a broken answer and must not pass (A11 bug probe)", `caffeinate & ./migrate.sh`, 0}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -292,6 +306,7 @@ func TestMacosStatBSDGNUTest_Eval(t *testing.T) { {"prose wrapped", "Run stat -f%z file.txt on macOS; the GNU equivalent flag is -c.", 1}, {"correct BSD command but never names the GNU flag", `stat -f%z file.txt`, 0.5}, {"GNU -c%s syntax used on macOS scores half: -c token matches but -f BSD form is missing", `stat -c%s file.txt`, 0.5}, + {"single-quoted format string (A4)", `stat -f '%z' file.txt, and on Linux GNU stat uses -c.`, 1}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { diff --git a/internal/tests/python.go b/internal/tests/python.go index b9e448c..3f6b5a7 100644 --- a/internal/tests/python.go +++ b/internal/tests/python.go @@ -234,7 +234,9 @@ printed, nothing else.` Subcategory: "python", Description: "Trace the classic mutable-default-argument bug across three calls sharing one list.", Prompt: prompt, - Eval: eval.Equals("[1, 2, 3]"), + // A7: eval.ExactToken so a fenced or quoted rendering of the exact + // printed text still scores full credit. + Eval: eval.ExactToken("[1, 2, 3]"), } } @@ -270,7 +272,9 @@ printed, nothing else.` Subcategory: "python", Description: "Trace the exact output of a length-filtered dict comprehension whose items are sorted and printed.", Prompt: prompt, - Eval: eval.Equals("[('apple', 5), ('banana', 6), ('cherry', 6), ('date', 4)]"), + // A7: eval.ExactToken so a fenced or quoted rendering of the exact + // printed text still scores full credit. + Eval: eval.ExactToken("[('apple', 5), ('banana', 6), ('cherry', 6), ('date', 4)]"), } } @@ -313,7 +317,9 @@ printed, nothing else.` Subcategory: "python", Description: "Trace the exact output of iterating a generator a second time after it is already exhausted.", Prompt: prompt, - Eval: eval.Equals("[0, 1, 2] []"), + // A7: eval.ExactToken so a fenced or quoted rendering of the exact + // printed text still scores full credit. + Eval: eval.ExactToken("[0, 1, 2] []"), } } @@ -345,9 +351,15 @@ keeping the same behavior: recursively find every ".yaml" file under base_dir and return their full paths. Respond with only the rewritten function.` + // A16: Path.rglob("*.yaml") and Path.glob("**/*.yaml") are behaviorally + // equivalent recursive searches ("**/" is glob's own recursive + // wildcard), so the recursive-method check accepts either instead of + // only the literal substring "rglob"; the "*.yaml" pattern is checked + // separately from which method carries it. evaluator := eval.All( eval.W(eval.ContainsAny("from pathlib import Path", "import pathlib"), 1), - eval.W(eval.ContainsAll("rglob", "*.yaml"), 2), + eval.W(eval.ContainsAny("rglob", `glob("**/`, "glob('**/"), 1), + eval.W(eval.ContainsAll("*.yaml"), 1), ) return testkit.Test{ diff --git a/internal/tests/python_test.go b/internal/tests/python_test.go index 01fb2fc..742c737 100644 --- a/internal/tests/python_test.go +++ b/internal/tests/python_test.go @@ -254,6 +254,10 @@ func TestPyMutableDefaultArgTest_Eval(t *testing.T) { {"exact correct", "[1, 2, 3]", 1}, {"correct with surrounding whitespace", " [1, 2, 3] ", 1}, {"correct with trailing newline", "[1, 2, 3]\n", 1}, + // A7 regression: fenced/quoted decoration on the correct answer + // must still score full credit. + {"fenced", "```\n[1, 2, 3]\n```", 1}, + {"quoted", `"[1, 2, 3]"`, 1}, {"wrong: assumes a fresh list per call", "[3]", 0}, {"wrong: only counts explicit-arg calls", "[1, 2]", 0}, } @@ -293,6 +297,9 @@ func TestPyDictComprehensionTraceTest_Eval(t *testing.T) { {"exact correct", "[('apple', 5), ('banana', 6), ('cherry', 6), ('date', 4)]", 1}, {"correct with surrounding whitespace", " [('apple', 5), ('banana', 6), ('cherry', 6), ('date', 4)] ", 1}, {"correct uppercase-folded", "[('APPLE', 5), ('BANANA', 6), ('CHERRY', 6), ('DATE', 4)]", 1}, + // A7 regression: fenced decoration on the correct answer must + // still score full credit. + {"fenced", "```\n[('apple', 5), ('banana', 6), ('cherry', 6), ('date', 4)]\n```", 1}, {"wrong: includes fig, forgets the length filter", "[('apple', 5), ('banana', 6), ('cherry', 6), ('date', 4), ('fig', 3)]", 0}, {"wrong: unsorted, dict insertion order", "[('apple', 5), ('banana', 6), ('cherry', 6), ('fig', 3), ('date', 4)]", 0}, } @@ -331,6 +338,10 @@ func TestPyGeneratorExhaustionTraceTest_Eval(t *testing.T) { {"exact correct", "[0, 1, 2] []", 1}, {"correct with surrounding whitespace", " [0, 1, 2] [] ", 1}, {"correct with trailing newline", "[0, 1, 2] []\n", 1}, + // A7 regression: fenced/quoted decoration on the correct answer + // must still score full credit. + {"fenced", "```\n[0, 1, 2] []\n```", 1}, + {"quoted", `"[0, 1, 2] []"`, 1}, {"wrong: assumes the generator restarts", "[0, 1, 2] [0, 1, 2]", 0}, {"wrong: assumes an error is raised and printed", "StopIteration", 0}, } @@ -370,6 +381,13 @@ def find_configs(base_dir): return [str(p) for p in Path(base_dir).glob("*.yaml")] ` + "```" + goodRewriteGlobRecursiveWildcard := "```python\n" + `from pathlib import Path + + +def find_configs(base_dir): + return [str(p) for p in Path(base_dir).glob("**/*.yaml")] +` + "```" + tests := []struct { name string response string @@ -378,7 +396,8 @@ def find_configs(base_dir): {"good rewrite: from pathlib import Path", goodRewrite, 1}, {"good rewrite: import pathlib", goodRewriteAltImport, 1}, {"wrong: unchanged os.path/os.walk code", stillUsesOsPath, 0}, - {"partial: uses pathlib but non-recursive glob, not rglob", missingRglob, 2.0 / 3.0}, // (1*1 + 2*0.5)/3: ContainsAny hits, ContainsAll("rglob","*.yaml") matches only "*.yaml" + {"partial: uses pathlib but non-recursive glob, not rglob", missingRglob, 2.0 / 3.0}, // (1*1 + 1*0 + 1*1)/3: pathlib import ok, neither rglob nor glob("**/ present, *.yaml ok + {"good rewrite: glob(\"**/*.yaml\") is behaviorally equivalent to rglob (A16 bug probe)", goodRewriteGlobRecursiveWildcard, 1}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { diff --git a/internal/tests/typescript.go b/internal/tests/typescript.go index f1bf338..2218962 100644 --- a/internal/tests/typescript.go +++ b/internal/tests/typescript.go @@ -142,9 +142,19 @@ given an interface with a nested object property, that nested object's own properties are also optional). Do not use the "any" type anywhere. Respond with only the type definition.` + // A9: the mapped-type variable name is the author's choice, not + // mandated syntax - "K" is one common convention but "P" (as used in + // TypeScript's own built-in Partial) is equally correct, so the + // recursive-reference check accepts any identifier there rather than + // only the literal "DeepPartial" substring. The "keyof T" check + // is similarly relaxed to tolerate any whitespace width between the two + // words, not just a single space. evaluator := eval.All( - eval.W(eval.ContainsAll("DeepPartial", "keyof T"), 2), - eval.W(eval.ContainsAny("DeepPartial"), 1), + eval.W(eval.Mean( + eval.ContainsAll("DeepPartial"), + eval.Regex(`(?i)keyof\s+T\b`), + ), 2), + eval.W(eval.Regex(`DeepPartial<\s*T\[\s*\w+\s*\]\s*>`), 1), eval.W(noAnyEscapeHatch(), 1), ) @@ -316,8 +326,13 @@ configuration as an eslint.config.js flat config, preserving the same rules (semi: error, prefer-const: error) and the ecmaVersion/sourceType settings via languageOptions. Respond with only the eslint.config.js file content.` + // A flat eslint.config.js can equally validly be authored as an ES + // module (export default [...]) or as CommonJS (module.exports = [...]), + // depending on the project's own module system; requiring "export + // default" specifically rejected an otherwise-correct CommonJS answer + // (A8). evaluator := eval.All( - eval.W(eval.ContainsAll("export default"), 2), + eval.W(eval.ContainsAny("export default", "module.exports"), 2), eval.W(eval.ContainsAny("defineConfig(", "languageOptions"), 1), ) @@ -447,6 +462,8 @@ printed, nothing else.` Subcategory: "typescript", Description: "Trace the exact output of optional chaining combined with nullish coalescing on a falsy-but-present value.", Prompt: prompt, - Eval: eval.Equals("localhost:0"), + // A7: eval.ExactToken so a fenced or quoted rendering of the exact + // printed text still scores full credit. + Eval: eval.ExactToken("localhost:0"), } } diff --git a/internal/tests/typescript_test.go b/internal/tests/typescript_test.go index e58a17d..3936be9 100644 --- a/internal/tests/typescript_test.go +++ b/internal/tests/typescript_test.go @@ -163,6 +163,28 @@ func TestTsGenericUtilityTypeTest_Eval(t *testing.T) { {"correct recursive DeepPartial", goodDeepPartial, 1}, {"wrong: shallow only, no recursion into nested objects", missingRecursion, 0.75}, // (2*1 + 1*0 + 1*1)/4: mapped type ok, no DeepPartial reference, no any {"wrong: uses any, defeating the point", usesAny, 0.5}, // (2*1 + 1*0 + 1*0)/4: mapped type ok, no recursion, uses any + { + // A9: "P" (TypeScript's own Partial convention) is an + // equally valid mapped-type variable name to "K" - the + // recursive-reference check must not hardcode the literal + // identifier. + name: "correct recursive DeepPartial using P instead of K as the mapped-type variable (A9 bug probe)", + response: "```ts\n" + `type DeepPartial = { + [P in keyof T]?: T[P] extends object ? DeepPartial : T[P]; +}; +` + "```", + want: 1, + }, + { + // A9: "keyof T" with extra internal whitespace is the same + // requirement, just formatted differently. + name: "correct recursive DeepPartial with extra whitespace around keyof T (A9 bug probe)", + response: "```ts\n" + `type DeepPartial = { + [K in keyof T]?: T[K] extends object ? DeepPartial : T[K]; +}; +` + "```", + want: 1, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -285,6 +307,14 @@ export default defineConfig([ rules: { semi: "error", "prefer-const": "error" }, }, ]); +` + "```" + + goodFlatConfigCommonJS := "```js\n" + `module.exports = [ + { + languageOptions: { ecmaVersion: "latest", sourceType: "module" }, + rules: { semi: "error", "prefer-const": "error" }, + }, +]; ` + "```" stillLegacyFormat := "```json\n" + tsOldEslintrcConfig + "\n```" @@ -296,6 +326,7 @@ export default defineConfig([ }{ {"good: plain array export default", goodFlatConfig, 1}, {"good: defineConfig helper", goodFlatConfigWithHelper, 1}, + {"good: CommonJS module.exports form is equally valid flat config (A8 bug probe)", goodFlatConfigCommonJS, 1}, {"wrong: left as legacy .eslintrc JSON", stillLegacyFormat, 0}, } for _, tt := range tests { @@ -400,6 +431,10 @@ func TestTsOptionalChainingNullishTraceTest_Eval(t *testing.T) { {"exact correct", "localhost:0", 1}, {"correct with surrounding whitespace", " localhost:0 ", 1}, {"correct case-insensitive", "LOCALHOST:0", 1}, + // A7 regression: fenced/quoted decoration on the correct answer + // must still score full credit. + {"fenced", "```\nlocalhost:0\n```", 1}, + {"quoted with trailing period", `"localhost:0".`, 1}, {"wrong: treats ?? like || and replaces the falsy 0", "localhost:8080", 0}, {"wrong: assumes host is present", "example.com:0", 0}, } diff --git a/internal/tests/web_protocol.go b/internal/tests/web_protocol.go index b00f02a..958fcdd 100644 --- a/internal/tests/web_protocol.go +++ b/internal/tests/web_protocol.go @@ -1,6 +1,9 @@ package tests import ( + "context" + "regexp" + "github.com/lukaszraczylo/llm-testbench/internal/eval" "github.com/lukaszraczylo/llm-testbench/internal/testkit" ) @@ -34,8 +37,9 @@ scenario2: "The client did not send any authentication credentials, and the requested resource requires a logged-in user." scenario3: "The client is authenticated, but their account does not have permission to access the requested resource." -scenario4: "The resource has permanently moved to a new URL; the server -wants clients (and search engines) to use the new URL from now on."` +scenario4: "The resource has permanently moved to a new URL; the original +request was a GET, and the server wants clients (and search engines) to +use the new URL from now on."` // webHTTPStatusScenariosTest: map 4 inline HTTP scenarios to their correct // status codes. @@ -43,7 +47,11 @@ wants clients (and search engines) to use the new URL from now on."` // ground truth: malformed request body -> 400 Bad Request; missing // credentials on a resource that requires them -> 401 Unauthorized; // authenticated but lacking permission -> 403 Forbidden; permanent -// relocation -> 301 Moved Permanently. +// relocation of a GET request -> 301 Moved Permanently (B8: the scenario +// states the original request is a GET, so 308 Permanent Redirect - which +// exists specifically to guarantee an unchanged method and body on +// replay, relevant for POST/PUT - is not equally defensible here; 301 is +// the unique correct answer). func webHTTPStatusScenariosTest() testkit.Test { prompt := `For each of these 4 request-handling scenarios, give the single most correct HTTP status code: @@ -70,24 +78,39 @@ code as a number: {"scenario1":,"scenario2":,"scenario3":,"max_tolerable_failures // description of a worst-case (minimum-fanout) B-tree capacity model // written for this test, pinning order m and a target key count. const paperBTreeExcerpt = `A B-tree of order m allows every internal node between ceil(m/2) and m -children, keeping the tree shallow and well balanced even as it grows, +children, except the root, which may have as few as two children, keeping +the tree shallow and well balanced even as it grows, which bounds the number of disk reads a lookup needs to at most one per level. For capacity planning this report models the worst case: a tree where every internal node has the minimum allowed fanout, f equals @@ -177,12 +178,19 @@ partition between two data centers, both sides keep accepting writes independently rather than refusing requests, accepting that the two sides may briefly disagree and need reconciliation once the partition heals.` -// paperCAPAvailabilityPattern anchors on the trimmed response being -// exactly the single word "availability" (case-insensitive, with an -// optional trailing period), accepting every materially-correct form of -// the forced one-word answer without also matching a response that names -// the wrong guarantee. -const paperCAPAvailabilityPattern = `(?i)^\s*availability\.?\s*$` +// paperCAPAvailabilityWant is the single word this test's prompt forces as +// the answer. +// +// ground truth: the excerpt states both data centers "keep accepting +// writes independently rather than refusing requests" during the +// partition. Per the CAP theorem as the excerpt defines it, continuing to +// answer every request rather than refusing (erroring) is the definition +// of availability, so the system prioritizes availability over +// consistency during the partition. B2: scored with eval.ExactToken so a +// quoted or bolded "Availability" - both forms the prompt's own phrasing +// invites - still matches; the prior anchored regex required the bare +// word and scored those correct answers 0. +const paperCAPAvailabilityWant = "Availability" // paperCAPAvailabilityChoiceTest: identify, from an inline CAP-theorem // excerpt and a described design decision, which guarantee (consistency or @@ -204,7 +212,7 @@ the CAP theorem? Respond with only one word: "Consistency" or Subcategory: "whitepapers", Description: "Identify which CAP-theorem guarantee a described partition-handling design decision prioritizes.", Prompt: prompt, - Eval: eval.Regex(paperCAPAvailabilityPattern), + Eval: eval.ExactToken(paperCAPAvailabilityWant), } } diff --git a/internal/tests/whitepapers_systems_test.go b/internal/tests/whitepapers_systems_test.go index eac2c40..a773e09 100644 --- a/internal/tests/whitepapers_systems_test.go +++ b/internal/tests/whitepapers_systems_test.go @@ -164,6 +164,11 @@ func TestPaperCAPAvailabilityChoiceTest_Eval(t *testing.T) { {"correct with trailing period", "Availability.", 1}, {"wrong: consistency", "Consistency", 0}, {"wrong: extra words break the forced one-word format", "The answer is availability", 0}, + // B2: the prompt's own phrasing quotes the two options, so a + // quoted or bolded answer must still score full credit. + {"correct, quoted (prompt's own phrasing)", `"Availability"`, 1}, + {"correct, bolded", "**Availability**", 1}, + {"correct, quoted with trailing period", `"Availability".`, 1}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { diff --git a/internal/tests/whitepapers_vector.go b/internal/tests/whitepapers_vector.go index c5e58e3..97d1789 100644 --- a/internal/tests/whitepapers_vector.go +++ b/internal/tests/whitepapers_vector.go @@ -114,8 +114,8 @@ total number of documents in the corpus and df(t) is the number of documents that contain t at least once. The TF-IDF weight of a term in a document is the product tf(t,d) times idf(t): a term that appears often in one document but rarely elsewhere scores highest, while a term appearing in -every document scores near zero regardless of how often it repeats, since -its idf approaches the logarithm of one.` +every document scores zero regardless of how often it repeats, since its +idf is ln(1) = 0 exactly.` // paperTFIDFDoc1, paperTFIDFDoc2, and paperTFIDFDoc3 are the pre-tokenized // (whitespace-split, lowercase) mini-corpus for paperTFIDFTermScoreTest. @@ -185,6 +185,12 @@ surfaced, never recover ones it dropped.` // paperRecallRelevant and paperRecallRetrievedTop6 are the inline // relevant-set and top-6 retrieved-ranking fixtures for // paperRecallAtKTest. +// +// ground truth: relevant = {D2,D5,D7,D9} (size 4); of those, D2, D5, and +// D7 appear somewhere in the top-6 retrieved list, but D9 does not, so the +// intersection has size 3. recall@6 = |intersection| / |relevant| = 3/4 = +// 0.75. whitepapers_vector_test.go independently recomputes this by +// counting the intersection directly, not via wpRecallAtK. var ( paperRecallRelevant = []string{"D2", "D5", "D7", "D9"} paperRecallRetrievedTop6 = []string{"D1", "D5", "D3", "D7", "D6", "D2"} diff --git a/internal/tests/whitepapers_vector_test.go b/internal/tests/whitepapers_vector_test.go index a31384c..489773e 100644 --- a/internal/tests/whitepapers_vector_test.go +++ b/internal/tests/whitepapers_vector_test.go @@ -32,6 +32,9 @@ func TestPaperPQCompressionRatioTest_Eval(t *testing.T) { }{ {"correct", "64", 1}, {"prose wrapped", "The compression ratio is 64x.", 1}, + // B6: ratio notation must extract the first term (64), not the + // second (the "1" of "64:1"). + {"ratio notation", "64:1", 1}, {"wrong", "8", 0}, } for _, tt := range tests {