mirror of
https://github.com/lukaszraczylo/llm-testbench.git
synced 2026-10-12 00:45:19 +00:00
fix(catalog): harden evaluators against correct-answer phrasing variance
Both reviewers found evaluators that scored correct answers as zero: hardcoded flag order, quote-style rigidity, missing multiline anchors, comma-grouped numbers, ratio notation, and numeric strings in JSON. A shared ExactToken evaluator now normalizes fences, quotes, and trailing periods for exact-answer tests. The obsolete tmutil snapshot reference now accepts the current localsnapshot verb.
This commit is contained in:
@@ -477,21 +477,25 @@ Respond with only a JSON object: {"next_action":"..."}`
|
||||
//
|
||||
// ground truth: prep_env finishes on day 0+3=3, prep_data finishes on day
|
||||
// 0+5=5. Milestone M cannot start until BOTH finish, so M starts on
|
||||
// max(3,5)=5, and finishes 2 days later, on day 5+2=7.
|
||||
// max(3,5)=5, and finishes 2 days later, on day 5+2=7. B7: the prompt now
|
||||
// states the t+d finish-day rule explicitly rather than relying on a
|
||||
// "day 0" framing a reader could also read as 1-indexed (which would make
|
||||
// 6 a defensible, if unintended, answer); 7 remains the only correct one.
|
||||
var planMilestoneWant = max(3, 5) + 2
|
||||
|
||||
func planMilestoneDateTest() testkit.Test {
|
||||
prompt := `Two prerequisite work items both start on day 0 and run in
|
||||
parallel:
|
||||
prompt := `Two prerequisite work items run in parallel, starting at day 0:
|
||||
- prep_env takes 3 days.
|
||||
- prep_data takes 5 days.
|
||||
|
||||
Milestone M can only start once BOTH prep_env and prep_data have finished,
|
||||
and M itself takes 2 days once it starts.
|
||||
|
||||
On what day number does milestone M finish (counting the day work starts
|
||||
as day 0, and each duration in whole days with no gaps)? Respond with only
|
||||
the number.`
|
||||
Use this rule: a task starting at day t with duration d finishes at day
|
||||
t+d. Work starts at day 0.
|
||||
|
||||
On what day number does milestone M finish? Respond with only the
|
||||
number.`
|
||||
|
||||
return testkit.Test{
|
||||
ID: "agent-plan-milestone-date",
|
||||
|
||||
@@ -2,11 +2,10 @@ package tests
|
||||
|
||||
import "github.com/lukaszraczylo/llm-testbench/internal/testkit"
|
||||
|
||||
// All builds a Registry containing the full 17-test catalog: 3
|
||||
// programming/golang, 2 programming/python, 1 programming/typescript, 1
|
||||
// programming/c, 2 operations/macos, 2 operations/linux, 1
|
||||
// operations/kubernetes, 1 research/web, 1 research/whitepapers, 1
|
||||
// research/codebase, and 2 agents (tool-routing, planning).
|
||||
// All builds a Registry containing the full test catalog, spanning the
|
||||
// programming (golang, python, typescript, c), operations (macos, linux,
|
||||
// kubernetes), research (web, whitepapers, codebase), and agents
|
||||
// (tool-routing, planning) categories.
|
||||
func All() *testkit.Registry {
|
||||
r := testkit.NewRegistry()
|
||||
registerGolangTests(r)
|
||||
|
||||
@@ -1,6 +1,10 @@
|
||||
package tests
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"regexp"
|
||||
|
||||
"github.com/lukaszraczylo/llm-testbench/internal/eval"
|
||||
"github.com/lukaszraczylo/llm-testbench/internal/testkit"
|
||||
)
|
||||
@@ -157,8 +161,26 @@ const codeDedupSnippet = `func Dedup(a []int) []int {
|
||||
|
||||
// codeBigODedupPattern accepts every common notation for O(n^2): a caret,
|
||||
// a double-star, the unicode superscript two, or n*n, each optionally
|
||||
// wrapped as O(...) with flexible internal spacing.
|
||||
const codeBigODedupPattern = `(?i)o\(\s*n\s*(\^\s*2|\*\*\s*2|²|\*\s*n)\s*\)`
|
||||
// wrapped as O(...) with flexible internal spacing. Anchored to the whole
|
||||
// string (codeBigODedupEval matches it against the whole normalized
|
||||
// response, never a substring), so "not O(n^2), it's O(n)" - which
|
||||
// contains the literal substring "O(n^2)" - correctly fails (B5) instead
|
||||
// of matching the way an unanchored eval.Regex would.
|
||||
var codeBigODedupPattern = regexp.MustCompile(`(?i)^o\(\s*n\s*(\^\s*2|\*\*\s*2|²|\*\s*n)\s*\)$`)
|
||||
|
||||
// codeBigODedupEval anchors eval.NormalizeExactToken's normalization (fence
|
||||
// extraction, quote/asterisk/period stripping) to codeBigODedupPattern,
|
||||
// combining "accept several equivalent notations" with "match the whole
|
||||
// answer, not a substring anywhere in it."
|
||||
func codeBigODedupEval() eval.Evaluator {
|
||||
return eval.EvaluatorFunc(func(_ context.Context, response string) eval.Score {
|
||||
normalized := eval.NormalizeExactToken(response)
|
||||
if codeBigODedupPattern.MatchString(normalized) {
|
||||
return eval.Score{Value: 1, Detail: fmt.Sprintf("matches O(n^2) notation: %q", normalized)}
|
||||
}
|
||||
return eval.Score{Value: 0, Detail: fmt.Sprintf("does not match O(n^2) notation: %q", normalized)}
|
||||
})
|
||||
}
|
||||
|
||||
// codeBigODedupTest: state the worst-case time complexity of an inline
|
||||
// nested-loop deduplication function.
|
||||
@@ -181,7 +203,7 @@ terms of n = len(a)? Respond with only the Big-O expression, e.g. O(n).`
|
||||
Subcategory: "codebase",
|
||||
Description: "State the worst-case Big-O time complexity of an inline nested-loop deduplication function.",
|
||||
Prompt: prompt,
|
||||
Eval: eval.Regex(codeBigODedupPattern),
|
||||
Eval: codeBigODedupEval(),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -14,6 +14,10 @@ func TestCodeBugLineDiffTest_Eval(t *testing.T) {
|
||||
want float64
|
||||
}{
|
||||
{"correct: root-cause line", `{"line":8}`, 1},
|
||||
// B9: a numeric-string JSON value must coerce the same as a
|
||||
// native number - the prompt asked for the right line, not a
|
||||
// specific JSON type.
|
||||
{"correct as a numeric string", `{"line":"8"}`, 1},
|
||||
{"wrong: points at the panic site instead of the root cause", `{"line":10}`, 0},
|
||||
{"wrong line entirely", `{"line":6}`, 0},
|
||||
}
|
||||
@@ -87,8 +91,14 @@ func TestCodeBigODedupTest_Eval(t *testing.T) {
|
||||
{"unicode superscript form", "O(n²)", 1},
|
||||
{"n*n form", "O(n*n)", 1},
|
||||
{"lowercase o", "o(n^2)", 1},
|
||||
{"quoted", `"O(n^2)"`, 1},
|
||||
{"trailing period", "O(n^2).", 1},
|
||||
{"wrong: linear", "O(n)", 0},
|
||||
{"wrong: linearithmic", "O(n log n)", 0},
|
||||
// B5: the correct notation appears as a substring of a negated
|
||||
// sentence; the whole-response-anchored evaluator must not match
|
||||
// on that substring.
|
||||
{"negated: contains the substring O(n^2) but names O(n) as the answer", "not O(n^2), it's O(n) with a hash set", 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
|
||||
@@ -1,34 +1,25 @@
|
||||
package tests
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/lukaszraczylo/llm-testbench/internal/eval"
|
||||
"github.com/lukaszraczylo/llm-testbench/internal/testkit"
|
||||
)
|
||||
|
||||
// codeExactAnswer returns an Evaluator awarding full credit when the
|
||||
// response, trimmed of whitespace, at most one layer of surrounding quote
|
||||
// characters (' , ", or `), and a single trailing sentence-ending period,
|
||||
// equals want case-insensitively. This accepts every materially-correct
|
||||
// form of a forced single-token answer (bare, quoted, differently-cased,
|
||||
// or with trailing punctuation) without loosening the match to accept a
|
||||
// substring of a longer, wrong answer.
|
||||
// response equals want under eval.ExactToken's normalization: a fenced
|
||||
// code block extracted if present, whitespace trimmed, one layer of
|
||||
// surrounding quote/backtick/asterisk decoration stripped, one trailing
|
||||
// sentence-ending period stripped, compared case-insensitively. This
|
||||
// accepts every materially-correct form of a forced single-token answer
|
||||
// (bare, quoted, bolded, fenced, differently-cased, or with trailing
|
||||
// punctuation) without loosening the match to accept a substring of a
|
||||
// longer, wrong answer. Kept as a thin wrapper (rather than calling
|
||||
// eval.ExactToken directly at each call site) so this file's intent -
|
||||
// "the traced program's exact output token" - stays named and greppable.
|
||||
func codeExactAnswer(want string) eval.Evaluator {
|
||||
return eval.EvaluatorFunc(func(_ context.Context, response string) eval.Score {
|
||||
got := strings.TrimSpace(response)
|
||||
got = strings.Trim(got, "\"'`")
|
||||
got = strings.TrimSuffix(strings.TrimSpace(got), ".")
|
||||
got = strings.TrimSpace(got)
|
||||
wantTrimmed := strings.TrimSpace(want)
|
||||
if strings.EqualFold(got, wantTrimmed) {
|
||||
return eval.Score{Value: 1, Detail: fmt.Sprintf("equals %q", wantTrimmed)}
|
||||
}
|
||||
return eval.Score{Value: 0, Detail: fmt.Sprintf("got %q, want %q", got, wantTrimmed)}
|
||||
})
|
||||
return eval.ExactToken(want)
|
||||
}
|
||||
|
||||
// codeTracePythonSource is the inline Python function traced by
|
||||
|
||||
@@ -120,6 +120,39 @@ func TestCodeTraceTSWant_GroundTruth(t *testing.T) {
|
||||
if got != codeTraceTSWant {
|
||||
t.Errorf("codeTSPackTrace(%v) = %q, want %q", items, got, codeTraceTSWant)
|
||||
}
|
||||
|
||||
// BN4: cross-check against a real JS engine too, not just the Go
|
||||
// replica above. node cannot execute codeTraceTSSource's TypeScript
|
||||
// type annotations directly, so this is the same logic with the two
|
||||
// annotations (": number[]", ": string") stripped - everything else,
|
||||
// including the reduce/ternary/toString(2) logic under test, is
|
||||
// identical source.
|
||||
if _, err := exec.LookPath("node"); err != nil {
|
||||
t.Skip("node not found on PATH, skipping exec ground-truth cross-check")
|
||||
}
|
||||
|
||||
const jsSource = `function pack(items) {
|
||||
return items.reduce((acc, x, i) => {
|
||||
if (i % 2 === 0) return acc + x;
|
||||
return acc + x * 2;
|
||||
}, 0).toString(2);
|
||||
}
|
||||
console.log(pack([3, 4, 5, 6, 7]));
|
||||
`
|
||||
// #nosec G204 -- jsSource is a fixed, hardcoded string (the TS source
|
||||
// embedded in the prompt with its two type annotations stripped), not
|
||||
// external or response-controlled input.
|
||||
cmd := exec.Command("node", "-e", jsSource) //nolint:gosec // see comment above
|
||||
var stdout bytes.Buffer
|
||||
cmd.Stdout = &stdout
|
||||
cmd.Stderr = os.Stderr
|
||||
if err := cmd.Run(); err != nil {
|
||||
t.Fatalf("running node ground-truth script failed: %v", err)
|
||||
}
|
||||
nodeGot := strings.TrimSpace(stdout.String())
|
||||
if nodeGot != codeTraceTSWant {
|
||||
t.Errorf("node-computed pack([3,4,5,6,7]) = %q, want %q", nodeGot, codeTraceTSWant)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCodeTraceTSTest_Eval(t *testing.T) {
|
||||
|
||||
@@ -492,7 +492,9 @@ printed, nothing else.`
|
||||
Subcategory: "golang",
|
||||
Description: "Trace the exact output of a context cancellation propagating synchronously from a parent to a derived context.",
|
||||
Prompt: prompt,
|
||||
Eval: eval.Equals("context canceled"),
|
||||
// A7: eval.ExactToken so a fenced or quoted rendering of the exact
|
||||
// printed text still scores full credit.
|
||||
Eval: eval.ExactToken("context canceled"),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -479,6 +479,10 @@ func TestGoContextCancellationTraceTest_Eval(t *testing.T) {
|
||||
{"exact correct", "context canceled", 1},
|
||||
{"correct case-insensitive", "CONTEXT CANCELED", 1},
|
||||
{"correct with surrounding whitespace", " context canceled ", 1},
|
||||
// A7 regression: fenced/quoted decoration on the correct answer
|
||||
// must still score full credit.
|
||||
{"fenced", "```\ncontext canceled\n```", 1},
|
||||
{"quoted with trailing period", `"context canceled".`, 1},
|
||||
{"wrong: assumes select falls to default", "not done", 0},
|
||||
{"wrong: wrong sentinel error", "context deadline exceeded", 0},
|
||||
}
|
||||
|
||||
@@ -349,7 +349,10 @@ spec:
|
||||
// app=inventory-api-svc - a value that matches no pod's labels, so the
|
||||
// Service has zero Endpoints. The single wrong field is the Service's
|
||||
// spec.selector.app value; it must be changed to "inventory-api" to match
|
||||
// the pods.
|
||||
// the pods. The prompt's JSON template pre-fills "field":"selector.app"
|
||||
// as part of the requested response shape, so a model that never
|
||||
// diagnosed the field at all could still copy that value verbatim; only
|
||||
// "value" is actually evidence the model found the fix (AN2).
|
||||
func k8sServiceSelectorMismatchTest() testkit.Test {
|
||||
prompt := `Requests to this Service time out with no available
|
||||
endpoints. Here is the Deployment's pod template and the Service:
|
||||
@@ -360,10 +363,7 @@ Which single field in the Service YAML is wrong, and what value should it
|
||||
be changed to so the Service's Endpoints include these pods? Respond with
|
||||
only a JSON object: {"field":"selector.app","value":"<correct value>"}`
|
||||
|
||||
evaluator := eval.Mean(
|
||||
eval.JSONField("field", "selector.app"),
|
||||
eval.JSONField("value", "inventory-api"),
|
||||
)
|
||||
evaluator := eval.JSONField("value", "inventory-api")
|
||||
|
||||
return testkit.Test{
|
||||
ID: "k8s-service-selector-mismatch",
|
||||
@@ -510,8 +510,11 @@ const k8sTraefikIngressRouteYAML = "apiVersion: traefik.io/v1alpha1\n" +
|
||||
" port: 80"
|
||||
|
||||
// k8sTraefikIngressRoutePattern requires the corrected match expression,
|
||||
// with the backtick-quoted domain exactly matching app.raczylo.com.
|
||||
const k8sTraefikIngressRoutePattern = "(?i)host\\(`app\\.raczylo\\.com`\\)"
|
||||
// with the domain app.raczylo.com quoted with either backticks (Traefik's
|
||||
// own matcher syntax, as shown in the prompt's example) or double quotes
|
||||
// (a common, equally unambiguous way a model renders the same literal)
|
||||
// (AN4).
|
||||
const k8sTraefikIngressRoutePattern = "(?i)host\\([`\"]app\\.raczylo\\.com[`\"]\\)"
|
||||
|
||||
// k8sTraefikIngressRouteHostTest: correct a Traefik IngressRoute's Host()
|
||||
// match rule to the actual domain the service must answer for.
|
||||
@@ -581,7 +584,10 @@ BestEffort.`
|
||||
Subcategory: "kubernetes",
|
||||
Description: "Derive a pod's QoS class (Burstable) from requests that are set but not equal to limits.",
|
||||
Prompt: prompt,
|
||||
Eval: eval.Equals("Burstable"),
|
||||
// A7: eval.ExactToken (not eval.Equals) so a fenced, quoted, or
|
||||
// bolded "Burstable" - a common way a model formats a forced
|
||||
// one-word answer - still scores full credit.
|
||||
Eval: eval.ExactToken("Burstable"),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -600,10 +606,15 @@ with what minimum value, to enforce "at least 2 available"?`
|
||||
// bounds voluntary disruptions (like a node drain evicting pods) for a
|
||||
// selected set of pods. Setting its minAvailable field to 2 enforces
|
||||
// that at least 2 of the 3 Postgres pods must remain available before
|
||||
// the eviction API will permit another one to be drained.
|
||||
// the eviction API will permit another one to be drained. The
|
||||
// minAvailable check is a single regex tolerant of any short
|
||||
// same-line phrasing between the key and the value (A10) - e.g.
|
||||
// "minAvailable is set to 2" or "minAvailable: \"2\"" - rather than an
|
||||
// enumerated list of exact phrasings that misses anything not on the
|
||||
// list.
|
||||
evaluator := eval.Mean(
|
||||
eval.ContainsAll("PodDisruptionBudget"),
|
||||
eval.ContainsAny("minAvailable: 2", "minAvailable=2", "minAvailable 2", "minAvailable of 2"),
|
||||
eval.Regex(`(?i)minAvailable\b[^\n]{0,20}\b2\b`),
|
||||
)
|
||||
|
||||
return testkit.Test{
|
||||
|
||||
@@ -238,7 +238,12 @@ func TestK8sServiceSelectorMismatchTest_Eval(t *testing.T) {
|
||||
{"prose then JSON", "The selector doesn't match the pod labels.\n" + `{"field":"selector.app","value":"inventory-api"}`, 1},
|
||||
{"fenced JSON", "```json\n" + `{"field":"selector.app","value":"inventory-api"}` + "\n```", 1},
|
||||
{"points at the wrong field (targetPort instead of selector)", `{"field":"targetPort","value":"8080"}`, 0},
|
||||
{"right field, wrong replacement value", `{"field":"selector.app","value":"inventory-api-svc"}`, 0.5},
|
||||
// AN2: only "value" is graded, since the prompt's own JSON template
|
||||
// pre-fills "field":"selector.app" - copying that back verbatim is
|
||||
// not evidence of diagnosis, so it no longer earns partial credit
|
||||
// on its own (this case used to score 0.5).
|
||||
{"right field, wrong replacement value", `{"field":"selector.app","value":"inventory-api-svc"}`, 0},
|
||||
{"field name wrong but value correct still scores full credit (AN2 bug probe)", `{"field":"wrongfield","value":"inventory-api"}`, 1},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
@@ -327,6 +332,7 @@ func TestK8sTraefikIngressRouteHostTest_Eval(t *testing.T) {
|
||||
{"uppercase Host keyword variant", "MATCH: HOST(`app.raczylo.com`)", 1},
|
||||
{"still the old domain", "match: Host(`app.example.com`)", 0},
|
||||
{"right function, wrong domain typo", "match: Host(`app.raczlo.com`)", 0},
|
||||
{"double-quoted domain (AN4 bug probe)", `match: Host("app.raczylo.com")`, 1},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
@@ -349,6 +355,11 @@ func TestK8sQoSClassTest_Eval(t *testing.T) {
|
||||
{"exact word", "Burstable", 1},
|
||||
{"lowercase", "burstable", 1},
|
||||
{"trailing whitespace", "Burstable\n", 1},
|
||||
// A7 regression: fenced/quoted/bolded/trailing-period decoration
|
||||
// on the correct answer must still score full credit.
|
||||
{"fenced", "```\nBurstable\n```", 1},
|
||||
{"quoted", `"Burstable"`, 1},
|
||||
{"bolded with period", "**Burstable**.", 1},
|
||||
{"wrong class, mistakes it for Guaranteed", "Guaranteed", 0},
|
||||
{"wrong class, mistakes it for BestEffort", "BestEffort", 0},
|
||||
}
|
||||
@@ -375,6 +386,8 @@ func TestK8sCNPGPDBTest_Eval(t *testing.T) {
|
||||
{"english phrasing of the value", "Create a PodDisruptionBudget with a minAvailable of 2.", 1},
|
||||
{"wrong resource entirely (a NetworkPolicy cannot do this)", "Use a NetworkPolicy with minAvailable: 2.", 0.5},
|
||||
{"right resource, wrong/missing value", "Use a PodDisruptionBudget, but leave the fields at their defaults.", 0.5},
|
||||
{"'set to' phrasing, not covered by the old enumerated substring list (A10 bug probe)", "Use a PodDisruptionBudget with minAvailable set to 2.", 1},
|
||||
{"quoted numeric value in YAML (A10 bug probe)", `Use a PodDisruptionBudget with minAvailable: "2".`, 1},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
|
||||
+83
-21
@@ -1,6 +1,9 @@
|
||||
package tests
|
||||
|
||||
import (
|
||||
"context"
|
||||
"regexp"
|
||||
|
||||
"github.com/lukaszraczylo/llm-testbench/internal/eval"
|
||||
"github.com/lukaszraczylo/llm-testbench/internal/testkit"
|
||||
)
|
||||
@@ -74,11 +77,19 @@ contents only.`
|
||||
}
|
||||
}
|
||||
|
||||
// linuxJournalctlFilterPattern requires journalctl filtered to unit
|
||||
// nginx.service and a since-window of 2 hours, in the canonical
|
||||
// "-u UNIT --since WINDOW" order (the overwhelmingly common idiom), on a
|
||||
// single line so a scattered mention across paragraphs does not count.
|
||||
const linuxJournalctlFilterPattern = `(?i)\bjournalctl\b[^\n]*?-u\s+nginx\.service\b[^\n]*?--since\s+["']?-?2\s*h(?:ours?)?(?:\s+ago)?["']?`
|
||||
// linuxJournalctlUnitPattern matches the unit filter in either short (-u)
|
||||
// or long (--unit) form, tolerant of "=" or whitespace between the option
|
||||
// and its value (journalctl, via getopt_long, accepts both). It is checked
|
||||
// independently of linuxJournalctlSincePattern below (A3), so either
|
||||
// option can appear on the line before the other - the original single
|
||||
// regex hardcoded "-u ... --since ..." order and rejected the equally
|
||||
// valid "--unit"/"-S" long/short forms and "--since=" spelling.
|
||||
const linuxJournalctlUnitPattern = `(?i)\bjournalctl\b[^\n]*(?:-u|--unit)[=\s]+nginx\.service\b`
|
||||
|
||||
// linuxJournalctlSincePattern matches the since-window filter in either
|
||||
// short (-S) or long (--since) form, independent of
|
||||
// linuxJournalctlUnitPattern (A3).
|
||||
const linuxJournalctlSincePattern = `(?i)\bjournalctl\b[^\n]*(?:-S|--since)[=\s]+["']?-?2\s*h(?:ours?)?(?:\s+ago)?["']?`
|
||||
|
||||
// linuxJournalctlFilterTest: one-line journalctl command filtered to one
|
||||
// unit over a relative time window.
|
||||
@@ -86,13 +97,18 @@ func linuxJournalctlFilterTest() testkit.Test {
|
||||
prompt := `Give a single, one-line journalctl command that shows only
|
||||
log entries for the systemd unit "nginx.service" from the last 2 hours.`
|
||||
|
||||
evaluator := eval.All(
|
||||
eval.W(eval.Regex(linuxJournalctlUnitPattern), 1),
|
||||
eval.W(eval.Regex(linuxJournalctlSincePattern), 1),
|
||||
)
|
||||
|
||||
return testkit.Test{
|
||||
ID: "linux-journalctl-filter",
|
||||
Category: "operations",
|
||||
Subcategory: "linux",
|
||||
Description: "One-line journalctl command filtered by unit and a relative 2-hour time window.",
|
||||
Prompt: prompt,
|
||||
Eval: eval.Regex(linuxJournalctlFilterPattern),
|
||||
Eval: evaluator,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -138,11 +154,16 @@ JSON object: {"oom_killed": true or false, "oom_kill_count": <number>}.`
|
||||
|
||||
// linuxNftablesDNATPattern requires a DNAT rule (iptables or nft form) on
|
||||
// the nat table's prerouting chain, matching TCP port 8080 and forwarding
|
||||
// to 10.0.5.20:80, on a single line. iptables and nft flags/keywords follow
|
||||
// a fixed canonical order (table/chain, then match, then target), so this
|
||||
// uses the same non-greedy same-line anchoring as the pct-exec lesson
|
||||
// rather than a permutation of independently-orderable flags.
|
||||
const linuxNftablesDNATPattern = `(?i)(?:iptables|nft)\b[^\n]*?nat[^\n]*?prerouting[^\n]*?(?:--dport|dport)\s*8080[^\n]*?dnat[^\n]*?10\.0\.5\.20:80`
|
||||
// to 10.0.5.20:80, on a single line. The "nat" table and "prerouting"
|
||||
// chain tokens are matched order-free relative to each other (A12): a
|
||||
// syntactically valid iptables invocation can give -A PREROUTING before
|
||||
// -t nat (e.g. "iptables -A PREROUTING -t nat -p tcp --dport 8080 -j DNAT
|
||||
// --to-destination 10.0.5.20:80"), and the original pattern's fixed
|
||||
// "nat...prerouting" order rejected that ordering. The rest of the match
|
||||
// (dport, dnat, destination) keeps its original same-line, in-order
|
||||
// anchoring, since iptables and nft do put the match/target keywords after
|
||||
// the table/chain in both valid orderings.
|
||||
const linuxNftablesDNATPattern = `(?i)(?:iptables|nft)\b[^\n]*?(?:nat[^\n]*?prerouting|prerouting[^\n]*?nat)[^\n]*?(?:--dport|dport)\s*8080[^\n]*?dnat[^\n]*?10\.0\.5\.20:80`
|
||||
|
||||
// linuxNftablesDNATTest: one-line DNAT rule forwarding an external port to
|
||||
// an internal host:port.
|
||||
@@ -194,9 +215,17 @@ data. Respond with only a JSON array of the step ids, e.g.
|
||||
}
|
||||
}
|
||||
|
||||
// linuxSSHPortForwardPattern requires an ssh local port-forward (-L) from
|
||||
// local port 15432 to the remote Postgres, connecting to the correct host.
|
||||
const linuxSSHPortForwardPattern = `(?i)\bssh\b[^\n]*?-L\s*15432:(?:127\.0\.0\.1|localhost):5432[^\n]*?ops@db1\.internal`
|
||||
// linuxSSHPortForwardSpecPattern matches the -L local-forward flag and its
|
||||
// argument, independent of where the SSH destination appears on the line -
|
||||
// -L can come before or after the destination in a valid ssh invocation
|
||||
// (A6).
|
||||
const linuxSSHPortForwardSpecPattern = `(?i)\bssh\b[^\n]*-L\s*15432:(?:127\.0\.0\.1|localhost):5432\b`
|
||||
|
||||
// linuxSSHPortForwardDestPattern matches the SSH destination in either its
|
||||
// "user@host" form or the equivalent "-l user ... host" form (A6): both
|
||||
// name the same destination, and the original pattern only accepted the
|
||||
// former.
|
||||
const linuxSSHPortForwardDestPattern = `(?i)\bssh\b[^\n]*(?:ops@db1\.internal|-l\s+ops\b[^\n]*\bdb1\.internal\b)`
|
||||
|
||||
// linuxSSHPortForwardTest: ssh -L local port-forward command derived from a
|
||||
// scenario.
|
||||
@@ -207,13 +236,18 @@ single, one-line ssh command, run from your laptop, that forwards your
|
||||
laptop's local port 15432 to that remote Postgres instance, so a local
|
||||
client can connect to localhost:15432.`
|
||||
|
||||
evaluator := eval.Mean(
|
||||
eval.Regex(linuxSSHPortForwardSpecPattern),
|
||||
eval.Regex(linuxSSHPortForwardDestPattern),
|
||||
)
|
||||
|
||||
return testkit.Test{
|
||||
ID: "linux-ssh-port-forward",
|
||||
Category: "operations",
|
||||
Subcategory: "linux",
|
||||
Description: "One-line ssh -L local port-forward command reaching a Postgres instance that only listens on remote loopback.",
|
||||
Prompt: prompt,
|
||||
Eval: eval.Regex(linuxSSHPortForwardPattern),
|
||||
Eval: evaluator,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -221,7 +255,26 @@ client can connect to localhost:15432.`
|
||||
// "03:30 every Monday", tolerant only of whitespace width between fields -
|
||||
// unlike eval.Equals, this stays correct even if the model adds a leading
|
||||
// or trailing space, since the fields themselves have no valid rephrasing.
|
||||
const linuxCronExpressionPattern = `^\s*30\s+3\s+\*\s+\*\s+1\s*$`
|
||||
// (?m) makes ^/$ match per line rather than only at the very start/end of
|
||||
// the whole string, so the expression still matches as one line within a
|
||||
// longer, fenced, or multi-line response (A5).
|
||||
const linuxCronExpressionPattern = `(?m)^\s*30\s+3\s+\*\s+\*\s+1\s*$`
|
||||
|
||||
// linuxCronExpressionEval strips a markdown code fence (if any) before
|
||||
// applying linuxCronExpressionPattern. Without this, a response that
|
||||
// wraps the correct expression in a ` + "```" + ` fence scored 0, because the fence
|
||||
// markers and surrounding whitespace kept the line-anchored regex from
|
||||
// finding a clean match (A5).
|
||||
func linuxCronExpressionEval() eval.Evaluator {
|
||||
re := regexp.MustCompile(linuxCronExpressionPattern)
|
||||
return eval.EvaluatorFunc(func(_ context.Context, response string) eval.Score {
|
||||
stripped := eval.ExtractCodeBlock(response, "")
|
||||
if re.MatchString(stripped) {
|
||||
return eval.Score{Value: 1, Detail: "matches the 5-field cron expression"}
|
||||
}
|
||||
return eval.Score{Value: 0, Detail: "does not match the 5-field cron expression"}
|
||||
})
|
||||
}
|
||||
|
||||
// linuxCronExpressionTest: produce the 5-field cron expression matching a
|
||||
// stated schedule.
|
||||
@@ -241,7 +294,7 @@ with only the expression, nothing else.`
|
||||
Subcategory: "linux",
|
||||
Description: "Produce the 5-field cron expression for a 03:30-every-Monday schedule.",
|
||||
Prompt: prompt,
|
||||
Eval: eval.Regex(linuxCronExpressionPattern),
|
||||
Eval: linuxCronExpressionEval(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -258,11 +311,15 @@ unit's WantedBy needs.`
|
||||
// paired with a same-named .service unit (the work). The .timer's
|
||||
// OnCalendar= key sets the schedule (e.g. "*-*-* 02:00:00" for daily at
|
||||
// 02:00), and its [Install] section needs WantedBy=timers.target for
|
||||
// systemctl enable to activate it.
|
||||
// systemctl enable to activate it. The WantedBy check accepts "=",
|
||||
// ":", or "(is) set to" between the key and its value (A15), not just a
|
||||
// bare "WantedBy=timers.target" substring - a response that phrases
|
||||
// this in prose ("WantedBy is set to timers.target") states exactly the
|
||||
// same fact.
|
||||
evaluator := eval.Mean(
|
||||
eval.ContainsAll(".service", ".timer"),
|
||||
eval.ContainsAny("OnCalendar"),
|
||||
eval.ContainsAll("WantedBy=timers.target"),
|
||||
eval.Regex(`(?i)WantedBy\s*(?:=|:|\b(?:is\s+)?set to\b)\s*timers\.target`),
|
||||
)
|
||||
|
||||
return testkit.Test{
|
||||
@@ -284,7 +341,12 @@ Buffers: 512000 kB
|
||||
Cached: 3200000 kB`
|
||||
|
||||
// linuxProcMeminfoAvailableWant is MemAvailable (9876000 kB) converted to
|
||||
// whole megabytes, floored: 9876000 / 1024 = 9644.53125 -> 9644.
|
||||
// whole megabytes, floored: 9876000 / 1024 = 9644.53125 -> 9644. The
|
||||
// prompt asks for the value rounded down, so 9644 is the single exact
|
||||
// correct answer - tolerance 0 below, not 1 (A17): a tolerance of 1 was
|
||||
// letting off-by-one wrong answers (9643 or 9645) score full credit for a
|
||||
// deterministic arithmetic conversion with no room for a legitimate
|
||||
// rephrasing.
|
||||
var linuxProcMeminfoAvailableWant = 9876000 / 1024
|
||||
|
||||
// linuxProcMeminfoAvailableTest: interpret /proc/meminfo's MemAvailable
|
||||
@@ -312,6 +374,6 @@ so convert by dividing by 1024, not 1000. Respond with only the number.`
|
||||
Subcategory: "linux",
|
||||
Description: "Convert /proc/meminfo's MemAvailable field (not MemFree) to whole megabytes.",
|
||||
Prompt: prompt,
|
||||
Eval: eval.Numeric(eval.ExtractLastNumber[int], linuxProcMeminfoAvailableWant, 1),
|
||||
Eval: eval.Numeric(eval.ExtractLastNumber[int], linuxProcMeminfoAvailableWant, 0),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -138,8 +138,16 @@ func TestLinuxJournalctlFilterTest_Eval(t *testing.T) {
|
||||
{"quoted negative window", `journalctl -u nginx.service --since "-2 hours"`, 1},
|
||||
{"english phrasing window", `journalctl -u nginx.service --since "2 hours ago"`, 1},
|
||||
{"compact -2h form, no quotes", `Run: journalctl -u nginx.service --since -2h`, 1},
|
||||
{"wrong unit", `journalctl -u apache.service --since "-2 hours"`, 0},
|
||||
{"missing unit filter entirely", `journalctl --since "-2 hours"`, 0},
|
||||
// The unit and since checks are independent (A3): a response that
|
||||
// nails one but not the other now earns that check's half-credit
|
||||
// rather than scoring 0 outright, the same partial-credit
|
||||
// convention eval.All already uses elsewhere in this file.
|
||||
{"wrong unit still credits the correct since-window", `journalctl -u apache.service --since "-2 hours"`, 0.5},
|
||||
{"missing unit filter still credits the correct since-window", `journalctl --since "-2 hours"`, 0.5},
|
||||
{"missing both filters entirely scores 0", `journalctl -f`, 0},
|
||||
{"long-form --unit= (A3 bug probe)", `journalctl --unit=nginx.service --since=-2h`, 1},
|
||||
{"short-form -S (A3 bug probe)", `journalctl -u nginx.service -S -2h`, 1},
|
||||
{"since option before unit option, order-free (A3 bug probe)", `journalctl --since "-2 hours" -u nginx.service`, 1},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
@@ -208,6 +216,11 @@ func TestLinuxNftablesDNATTest_Eval(t *testing.T) {
|
||||
response: `iptables -t nat -A OUTPUT -p tcp --dport 8080 -j DNAT --to-destination 10.0.5.20:80`,
|
||||
want: 0,
|
||||
},
|
||||
{
|
||||
name: "-A PREROUTING given before -t nat is equally valid iptables syntax (A12 bug probe)",
|
||||
response: `iptables -A PREROUTING -t nat -p tcp --dport 8080 -j DNAT --to-destination 10.0.5.20:80`,
|
||||
want: 1,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
@@ -254,8 +267,13 @@ func TestLinuxSSHPortForwardTest_Eval(t *testing.T) {
|
||||
{"loopback IP form", `ssh -L 15432:127.0.0.1:5432 [email protected]`, 1},
|
||||
{"localhost form", `ssh -L 15432:localhost:5432 [email protected]`, 1},
|
||||
{"extra flags before -L", `ssh -N -L 15432:127.0.0.1:5432 [email protected]`, 1},
|
||||
{"wrong local port", `ssh -L 5432:127.0.0.1:5432 [email protected]`, 0},
|
||||
{"wrong host", `ssh -L 15432:127.0.0.1:5432 [email protected]`, 0},
|
||||
// The forward-spec and destination checks are independent (A6):
|
||||
// getting one right and the other wrong now earns that check's
|
||||
// half-credit instead of scoring 0 outright.
|
||||
{"wrong local port still credits the correct destination", `ssh -L 5432:127.0.0.1:5432 [email protected]`, 0.5},
|
||||
{"wrong host still credits the correct forward spec", `ssh -L 15432:127.0.0.1:5432 [email protected]`, 0.5},
|
||||
{"destination given before -L, options-after-destination (A6 bug probe)", `ssh [email protected] -L 15432:127.0.0.1:5432`, 1},
|
||||
{"-l user form for the destination (A6 bug probe)", `ssh -L 15432:127.0.0.1:5432 -l ops db1.internal`, 1},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
@@ -280,6 +298,8 @@ func TestLinuxCronExpressionTest_Eval(t *testing.T) {
|
||||
{"tab-separated fields", "30\t3\t*\t*\t1", 1},
|
||||
{"wrong day of week", "30 3 * * 2", 0},
|
||||
{"wrong time", "45 3 * * 1", 0},
|
||||
{"fenced code block, no language tag (A5 bug probe)", "```\n30 3 * * 1\n```", 1},
|
||||
{"fenced code block with a language tag (A5 bug probe)", "```cron\n30 3 * * 1\n```", 1},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
@@ -326,6 +346,16 @@ WantedBy=timers.target.`,
|
||||
response: `Just add a backup.service and backup.timer, and enable it with WantedBy=multi-user.target.`,
|
||||
want: 1.0 / 3.0,
|
||||
},
|
||||
{
|
||||
name: "WantedBy with a colon separator (A15 bug probe)",
|
||||
response: `backup.service + backup.timer, OnCalendar=*-*-* 02:00:00, WantedBy: timers.target`,
|
||||
want: 1,
|
||||
},
|
||||
{
|
||||
name: "WantedBy phrased as 'set to' in prose (A15 bug probe)",
|
||||
response: `Use backup.service and backup.timer. OnCalendar=02:00 daily. WantedBy is set to timers.target under [Install].`,
|
||||
want: 1,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
@@ -350,6 +380,8 @@ func TestLinuxProcMeminfoAvailableTest_Eval(t *testing.T) {
|
||||
{"with unit suffix", "9644 MB", 1},
|
||||
{"divided by 1000 instead of 1024", "9876", 0},
|
||||
{"used MemFree instead of MemAvailable", "1000", 0},
|
||||
{"off-by-one must not pass now that tolerance is 0, not 1 (A17 bug probe)", "9645", 0},
|
||||
{"off-by-one the other direction must not pass (A17 bug probe)", "9643", 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
|
||||
+92
-27
@@ -1,6 +1,10 @@
|
||||
package tests
|
||||
|
||||
import (
|
||||
"context"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/lukaszraczylo/llm-testbench/internal/eval"
|
||||
"github.com/lukaszraczylo/llm-testbench/internal/testkit"
|
||||
)
|
||||
@@ -95,7 +99,12 @@ file belongs in for a per-machine (not per-user) job.`
|
||||
// single-quoted or double-quoted extension argument, invoking a foo->bar
|
||||
// substitution on file.txt, on a single line (no DOTALL) so a scattered,
|
||||
// multi-paragraph mention of the same tokens does not count as a real
|
||||
// one-line command.
|
||||
// one-line command. The substitution's delimiter is accepted as any of the
|
||||
// common sed delimiter characters (/, |, comma, #, colon, semicolon, @),
|
||||
// not just "/" - "s/foo/bar/" is the conventional example, but any
|
||||
// consistently-used delimiter is equally correct BSD sed syntax (AN3). Go's
|
||||
// RE2 engine has no backreferences, so each delimiter choice is spelled out
|
||||
// as its own alternative rather than captured-and-reused.
|
||||
//
|
||||
// ground truth: BSD sed's -i flag takes a mandatory in-place-suffix
|
||||
// argument (used for backup files); passing an empty single-quoted or
|
||||
@@ -104,7 +113,7 @@ file belongs in for a per-machine (not per-user) job.`
|
||||
// "sed -i 's/foo/bar/' file.txt" (no argument to -i) works on Linux but
|
||||
// errors on macOS, where it is parsed as "-i 's/foo/bar/'" (using the sed
|
||||
// script itself as the backup suffix) followed by a missing script operand.
|
||||
const macosSedPattern = `(?i)\bsed\b[^\n]*-i\s*(''|"")[^\n]*s/foo/bar/g?[^\n]*file\.txt`
|
||||
const macosSedPattern = `(?i)\bsed\b[^\n]*-i\s*(''|"")[^\n]*(?:s/foo/bar/|s\|foo\|bar\||s,foo,bar,|s#foo#bar#|s:foo:bar:|s;foo;bar;|s@foo@bar@)g?[^\n]*file\.txt`
|
||||
|
||||
// macosSedInplaceTest: give the portable BSD sed one-liner for in-place
|
||||
// substitution with no backup file, contrasted with the GNU-only form that
|
||||
@@ -158,7 +167,7 @@ explicit domain targeting).`
|
||||
evaluator := eval.Mean(
|
||||
eval.ContainsAny("bootstrap"),
|
||||
eval.ContainsAny("bootstrap system", "system domain", "domain system", "target system", "system target"),
|
||||
eval.ContainsAny("deprecated", "avoid launchctl load", "no longer recommended", "outdated"),
|
||||
eval.ContainsAny("deprecated", "avoid launchctl load", "no longer recommended", "outdated", "legacy", "superseded", "obsolete", "replaced"),
|
||||
)
|
||||
|
||||
return testkit.Test{
|
||||
@@ -171,9 +180,39 @@ explicit domain targeting).`
|
||||
}
|
||||
}
|
||||
|
||||
// macosPlutilPattern requires the built-in plutil -p pretty-printer
|
||||
// invoked directly on the plist path, on one line.
|
||||
const macosPlutilPattern = `(?i)\bplutil\b[^\n]*-p[^\n]*/Library/Preferences/com\.example\.app\.plist`
|
||||
// macosPlutilFlagPattern requires a properly bounded "-p" flag token
|
||||
// (immediately preceded by whitespace or the start of the response, and
|
||||
// immediately followed by whitespace or the end of the response). An
|
||||
// unbounded substring search for "-p" also matches inside the word
|
||||
// "pretty-print" (its internal hyphen followed by "p"), which would let a
|
||||
// response that only talks *about* pretty-printing - without actually
|
||||
// invoking "plutil -p" - score a false positive (A2).
|
||||
var macosPlutilFlagPattern = regexp.MustCompile(`(?i)(?:^|\s)-p(?:\s|$)`)
|
||||
|
||||
// macosPlutilPlistReadEval accepts three materially correct ways to read a
|
||||
// plist file's contents in human-readable form using only macOS-bundled
|
||||
// tools: "plutil -p <path>" (with -p bounded as an actual flag token, not
|
||||
// matched as a substring of "pretty-print"), "plutil -convert xml1 -o -
|
||||
// <path>" (convert to XML and write to stdout instead of editing in
|
||||
// place), and "defaults read <domain-or-path>" (A2).
|
||||
func macosPlutilPlistReadEval() eval.Evaluator {
|
||||
return eval.EvaluatorFunc(func(_ context.Context, response string) eval.Score {
|
||||
if !strings.Contains(response, "com.example.app") {
|
||||
return eval.Score{Value: 0, Detail: "does not name the target plist/domain"}
|
||||
}
|
||||
lower := strings.ToLower(response)
|
||||
switch {
|
||||
case strings.Contains(lower, "plutil") && macosPlutilFlagPattern.MatchString(response):
|
||||
return eval.Score{Value: 1, Detail: "plutil -p (a properly bounded flag token)"}
|
||||
case strings.Contains(lower, "plutil") && strings.Contains(lower, "-convert") && strings.Contains(lower, "xml1"):
|
||||
return eval.Score{Value: 1, Detail: "plutil -convert xml1"}
|
||||
case strings.Contains(lower, "defaults read"):
|
||||
return eval.Score{Value: 1, Detail: "defaults read"}
|
||||
default:
|
||||
return eval.Score{Value: 0, Detail: "no recognized plist-reading command"}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// macosPlutilPlistReadTest: pretty-print a plist file's contents using a
|
||||
// tool that ships with macOS (no Homebrew installs).
|
||||
@@ -190,17 +229,39 @@ that prints the plist's contents to your terminal in a readable
|
||||
// ground truth: plutil ships with macOS and its -p flag pretty-prints a
|
||||
// plist (binary or XML) to stdout in a human-readable form, taking the
|
||||
// file path directly - no domain-name lookup or Homebrew install
|
||||
// needed, unlike "defaults read <domain>".
|
||||
// needed, unlike "defaults read <domain>". "plutil -convert xml1 -o -"
|
||||
// (convert to XML, write to stdout) and "defaults read" are also
|
||||
// accepted as materially correct alternatives (A2).
|
||||
return testkit.Test{
|
||||
ID: "macos-plutil-plist-read",
|
||||
Category: "operations",
|
||||
Subcategory: "macos",
|
||||
Description: "Pretty-print a plist file's contents using the built-in plutil -p command.",
|
||||
Prompt: prompt,
|
||||
Eval: eval.Regex(macosPlutilPattern),
|
||||
Eval: macosPlutilPlistReadEval(),
|
||||
}
|
||||
}
|
||||
|
||||
// macosMdfindSpotlightEval gates the whole score on mdfind actually being
|
||||
// used: the prompt itself supplies both a plausible flag ("-name" is a real
|
||||
// "find" flag too) and the search term ("invoice"), so a bare filesystem
|
||||
// walk with "find" that happens to echo those prompt-supplied tokens must
|
||||
// not be able to reach a passing score. Only once mdfind is confirmed
|
||||
// present does the response get scored on the remaining, more permissive
|
||||
// criteria (A13).
|
||||
func macosMdfindSpotlightEval() eval.Evaluator {
|
||||
sub := eval.Mean(
|
||||
eval.ContainsAny("-onlyin", "-name"),
|
||||
eval.ContainsAny("invoice"),
|
||||
)
|
||||
return eval.EvaluatorFunc(func(ctx context.Context, response string) eval.Score {
|
||||
if !strings.Contains(strings.ToLower(response), "mdfind") {
|
||||
return eval.Score{Value: 0, Detail: "mdfind not used; a filesystem walk is not what was asked for"}
|
||||
}
|
||||
return sub.Evaluate(ctx, response)
|
||||
})
|
||||
}
|
||||
|
||||
// macosMdfindSpotlightTest: use mdfind (Spotlight's metadata index) instead
|
||||
// of a full filesystem walk to find files by name.
|
||||
func macosMdfindSpotlightTest() testkit.Test {
|
||||
@@ -211,20 +272,16 @@ whose name contains "invoice", scoped only to your home directory (~).`
|
||||
// ground truth: mdfind queries the already-built Spotlight index rather
|
||||
// than walking the filesystem; -onlyin scopes the search to a
|
||||
// directory, and either -name or a plain query string matches on file
|
||||
// name.
|
||||
evaluator := eval.Mean(
|
||||
eval.ContainsAll("mdfind"),
|
||||
eval.ContainsAny("-onlyin", "-name"),
|
||||
eval.ContainsAny("invoice"),
|
||||
)
|
||||
|
||||
// name. mdfind's presence is a hard gate (A13): "-name" is also a real
|
||||
// "find" flag, and "invoice" is echoed straight from the prompt, so
|
||||
// neither can carry the score for a wrong, find-based answer.
|
||||
return testkit.Test{
|
||||
ID: "macos-mdfind-spotlight",
|
||||
Category: "operations",
|
||||
Subcategory: "macos",
|
||||
Description: "Use mdfind against Spotlight's index, scoped to a directory, instead of a full filesystem walk.",
|
||||
Prompt: prompt,
|
||||
Eval: evaluator,
|
||||
Eval: macosMdfindSpotlightEval(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -238,11 +295,14 @@ the command to create a local snapshot now, and the separate command to
|
||||
list the local snapshots that currently exist.`
|
||||
|
||||
// ground truth: tmutil manages local APFS snapshots independently of
|
||||
// having a Time Machine backup destination attached. "tmutil snapshot"
|
||||
// creates one immediately; "tmutil listlocalsnapshots /" lists the
|
||||
// local snapshots that exist for the root volume.
|
||||
// having a Time Machine backup destination attached. "tmutil
|
||||
// localsnapshot" is the current, documented verb that creates one
|
||||
// immediately ("tmutil snapshot" does not appear in tmutil's own usage
|
||||
// listing on current macOS and is accepted only as a legacy/undocumented
|
||||
// alias - verified on this machine, A1); "tmutil listlocalsnapshots /"
|
||||
// lists the local snapshots that exist for the root volume.
|
||||
evaluator := eval.Mean(
|
||||
eval.ContainsAny("tmutil snapshot"),
|
||||
eval.ContainsAny("tmutil localsnapshot", "tmutil snapshot"),
|
||||
eval.ContainsAny("tmutil listlocalsnapshots", "listlocalsnapshots"),
|
||||
)
|
||||
|
||||
@@ -300,11 +360,13 @@ tool built into macOS, that does this.`
|
||||
// ground truth: caffeinate, bundled with macOS, takes a power-management
|
||||
// assertion for as long as a given utility runs when you pass that
|
||||
// utility as its argument, and releases the assertion automatically
|
||||
// when the utility exits - no manual cleanup needed.
|
||||
evaluator := eval.Mean(
|
||||
eval.ContainsAll("caffeinate"),
|
||||
eval.ContainsAny("migrate.sh"),
|
||||
)
|
||||
// when the utility exits - no manual cleanup needed. migrate.sh must be
|
||||
// caffeinate's own argument on the same command line (e.g.
|
||||
// "caffeinate ./migrate.sh"), not merely mentioned somewhere in the
|
||||
// response: a backgrounded "caffeinate & ./migrate.sh" variant runs the
|
||||
// two independently, so caffeinate's assertion does not track the
|
||||
// script's lifetime and would not auto-release when it exits (A11).
|
||||
evaluator := eval.Regex(`(?i)\bcaffeinate\b[^\n&;|]*\.?/?migrate\.sh`)
|
||||
|
||||
return testkit.Test{
|
||||
ID: "macos-caffeinate",
|
||||
@@ -317,8 +379,11 @@ tool built into macOS, that does this.`
|
||||
}
|
||||
|
||||
// macosStatPattern requires the BSD stat -f format-string form printing
|
||||
// only the byte size (%z) of file.txt.
|
||||
const macosStatPattern = `(?i)\bstat\b[^\n]*-f\s*"?%z"?[^\n]*file\.txt`
|
||||
// only the byte size (%z) of file.txt. The %z format specifier is accepted
|
||||
// bare or wrapped in either single or double quotes (both are valid shell
|
||||
// quoting for a format string with no special characters in it) - not just
|
||||
// double quotes (A4).
|
||||
const macosStatPattern = `(?i)\bstat\b[^\n]*-f\s*['"]?%z['"]?[^\n]*file\.txt`
|
||||
|
||||
// macosStatBSDGNUTest: give the BSD stat command (macOS's built-in stat)
|
||||
// for a file's size in bytes, and name the differing GNU stat flag.
|
||||
|
||||
@@ -92,6 +92,9 @@ func TestMacosSedInplaceTest_Eval(t *testing.T) {
|
||||
{"global replace flag still matches", `sed -i '' 's/foo/bar/g' file.txt`, 1},
|
||||
{"GNU-style with no extension argument is wrong on macOS", `sed -i 's/foo/bar/' file.txt`, 0},
|
||||
{"backup extension creates a file, contradicting the requirement", `sed -i.bak 's/foo/bar/' file.txt`, 0},
|
||||
{"pipe delimiter is equally valid sed syntax (AN3)", `sed -i '' 's|foo|bar|' file.txt`, 1},
|
||||
{"comma delimiter (AN3)", `sed -i '' 's,foo,bar,g' file.txt`, 1},
|
||||
{"hash delimiter (AN3)", `sed -i "" 's#foo#bar#' file.txt`, 1},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
@@ -128,6 +131,10 @@ Apple deprecated for lacking an explicit domain target.`
|
||||
{"good answer, alternate phrasing", goodPhrasing, 1},
|
||||
{"wrong subcommand (deprecated load only) scores 0", badWrongSubcommand, 0},
|
||||
{"missing domain target and deprecation note loses two thirds", badMissingDomain, 1.0 / 3.0},
|
||||
{"'legacy' deprecation cue (A14)", `Use launchctl bootstrap system /Library/LaunchDaemons/com.example.myjob.plist; launchctl load is the legacy subcommand.`, 1},
|
||||
{"'superseded' deprecation cue (A14)", `launchctl bootstrap, system domain target, superseded launchctl load.`, 1},
|
||||
{"'obsolete' deprecation cue (A14)", `bootstrap system domain target; launchctl load is obsolete.`, 1},
|
||||
{"'replaced' deprecation cue (A14)", `launchctl bootstrap system domain target replaced launchctl load.`, 1},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
@@ -152,6 +159,9 @@ func TestMacosPlutilPlistReadTest_Eval(t *testing.T) {
|
||||
{"quoted path", `plutil -p "/Library/Preferences/com.example.app.plist"`, 1},
|
||||
{"wrong tool entirely (cat prints binary garbage)", `cat /Library/Preferences/com.example.app.plist`, 0},
|
||||
{"plutil without -p just validates, does not print", `plutil /Library/Preferences/com.example.app.plist`, 0},
|
||||
{"plutil -convert xml1 to stdout is a materially correct alternative (A2)", `plutil -convert xml1 -o - /Library/Preferences/com.example.app.plist`, 1},
|
||||
{"defaults read is a materially correct alternative (A2)", `defaults read com.example.app`, 1},
|
||||
{"mentions pretty-print in prose without invoking -p as a bounded flag (A2 bug probe)", `plutil can pretty-print the plist at /Library/Preferences/com.example.app.plist, though I don't recall the exact flag.`, 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
@@ -174,8 +184,9 @@ func TestMacosMdfindSpotlightTest_Eval(t *testing.T) {
|
||||
{"onlyin with query", `mdfind -onlyin ~ "invoice"`, 1},
|
||||
{"name flag", `mdfind -onlyin ~ -name invoice`, 1},
|
||||
{"prose wrapped", `Run mdfind -onlyin ~ invoice to search the Spotlight index.`, 1},
|
||||
{"uses find instead of mdfind, walks the filesystem", `find ~ -iname "*invoice*"`, 1.0 / 3.0},
|
||||
{"mdfind used but searching for the wrong term entirely", `mdfind -onlyin ~ "receipt"`, 2.0 / 3.0},
|
||||
{"uses find instead of mdfind, walks the filesystem, gated to 0 (A13)", `find ~ -iname "*invoice*"`, 0},
|
||||
{"mdfind used but searching for the wrong term entirely", `mdfind -onlyin ~ "receipt"`, 0.5},
|
||||
{"find command echoing both prompt-supplied tokens must not pass the gate (A13 bug probe)", `find ~ -name "*invoice*"`, 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
@@ -200,6 +211,8 @@ func TestMacosAPFSSnapshotTest_Eval(t *testing.T) {
|
||||
{"listlocalsnapshots without explicit tmutil prefix restated", "First tmutil snapshot, then check with listlocalsnapshots /.", 1},
|
||||
{"only creates, never lists", "Just run tmutil snapshot.", 0.5},
|
||||
{"wrong tool (diskutil) mentioned instead of tmutil", "Use diskutil apfs snapshot / to create one.", 0},
|
||||
{"current documented localsnapshot verb (A1)", "Create: tmutil localsnapshot\nList: tmutil listlocalsnapshots /", 1},
|
||||
{"localsnapshot only, never lists (A1)", "Just run tmutil localsnapshot.", 0.5},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
@@ -266,8 +279,9 @@ func TestMacosCaffeinateTest_Eval(t *testing.T) {
|
||||
{"bare invocation", `caffeinate ./migrate.sh`, 1},
|
||||
{"with system-sleep flag", `caffeinate -s ./migrate.sh`, 1},
|
||||
{"prose wrapped", "Run `caffeinate -i ./migrate.sh` and it stops holding the assertion once migrate.sh exits.", 1},
|
||||
{"wrong script named, not the target", `caffeinate ./other-script.sh`, 0.5},
|
||||
{"wrong script named, not the target, scores 0 now that migrate.sh must be on the same line (A11)", `caffeinate ./other-script.sh`, 0},
|
||||
{"unrelated tool, does not auto-release on exit", `Just run pmset noidle before starting your script.`, 0},
|
||||
{"backgrounded caffeinate is a broken answer and must not pass (A11 bug probe)", `caffeinate & ./migrate.sh`, 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
@@ -292,6 +306,7 @@ func TestMacosStatBSDGNUTest_Eval(t *testing.T) {
|
||||
{"prose wrapped", "Run stat -f%z file.txt on macOS; the GNU equivalent flag is -c.", 1},
|
||||
{"correct BSD command but never names the GNU flag", `stat -f%z file.txt`, 0.5},
|
||||
{"GNU -c%s syntax used on macOS scores half: -c token matches but -f BSD form is missing", `stat -c%s file.txt`, 0.5},
|
||||
{"single-quoted format string (A4)", `stat -f '%z' file.txt, and on Linux GNU stat uses -c.`, 1},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
|
||||
@@ -234,7 +234,9 @@ printed, nothing else.`
|
||||
Subcategory: "python",
|
||||
Description: "Trace the classic mutable-default-argument bug across three calls sharing one list.",
|
||||
Prompt: prompt,
|
||||
Eval: eval.Equals("[1, 2, 3]"),
|
||||
// A7: eval.ExactToken so a fenced or quoted rendering of the exact
|
||||
// printed text still scores full credit.
|
||||
Eval: eval.ExactToken("[1, 2, 3]"),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -270,7 +272,9 @@ printed, nothing else.`
|
||||
Subcategory: "python",
|
||||
Description: "Trace the exact output of a length-filtered dict comprehension whose items are sorted and printed.",
|
||||
Prompt: prompt,
|
||||
Eval: eval.Equals("[('apple', 5), ('banana', 6), ('cherry', 6), ('date', 4)]"),
|
||||
// A7: eval.ExactToken so a fenced or quoted rendering of the exact
|
||||
// printed text still scores full credit.
|
||||
Eval: eval.ExactToken("[('apple', 5), ('banana', 6), ('cherry', 6), ('date', 4)]"),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -313,7 +317,9 @@ printed, nothing else.`
|
||||
Subcategory: "python",
|
||||
Description: "Trace the exact output of iterating a generator a second time after it is already exhausted.",
|
||||
Prompt: prompt,
|
||||
Eval: eval.Equals("[0, 1, 2] []"),
|
||||
// A7: eval.ExactToken so a fenced or quoted rendering of the exact
|
||||
// printed text still scores full credit.
|
||||
Eval: eval.ExactToken("[0, 1, 2] []"),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -345,9 +351,15 @@ keeping the same behavior: recursively find every ".yaml" file under
|
||||
base_dir and return their full paths. Respond with only the rewritten
|
||||
function.`
|
||||
|
||||
// A16: Path.rglob("*.yaml") and Path.glob("**/*.yaml") are behaviorally
|
||||
// equivalent recursive searches ("**/" is glob's own recursive
|
||||
// wildcard), so the recursive-method check accepts either instead of
|
||||
// only the literal substring "rglob"; the "*.yaml" pattern is checked
|
||||
// separately from which method carries it.
|
||||
evaluator := eval.All(
|
||||
eval.W(eval.ContainsAny("from pathlib import Path", "import pathlib"), 1),
|
||||
eval.W(eval.ContainsAll("rglob", "*.yaml"), 2),
|
||||
eval.W(eval.ContainsAny("rglob", `glob("**/`, "glob('**/"), 1),
|
||||
eval.W(eval.ContainsAll("*.yaml"), 1),
|
||||
)
|
||||
|
||||
return testkit.Test{
|
||||
|
||||
@@ -254,6 +254,10 @@ func TestPyMutableDefaultArgTest_Eval(t *testing.T) {
|
||||
{"exact correct", "[1, 2, 3]", 1},
|
||||
{"correct with surrounding whitespace", " [1, 2, 3] ", 1},
|
||||
{"correct with trailing newline", "[1, 2, 3]\n", 1},
|
||||
// A7 regression: fenced/quoted decoration on the correct answer
|
||||
// must still score full credit.
|
||||
{"fenced", "```\n[1, 2, 3]\n```", 1},
|
||||
{"quoted", `"[1, 2, 3]"`, 1},
|
||||
{"wrong: assumes a fresh list per call", "[3]", 0},
|
||||
{"wrong: only counts explicit-arg calls", "[1, 2]", 0},
|
||||
}
|
||||
@@ -293,6 +297,9 @@ func TestPyDictComprehensionTraceTest_Eval(t *testing.T) {
|
||||
{"exact correct", "[('apple', 5), ('banana', 6), ('cherry', 6), ('date', 4)]", 1},
|
||||
{"correct with surrounding whitespace", " [('apple', 5), ('banana', 6), ('cherry', 6), ('date', 4)] ", 1},
|
||||
{"correct uppercase-folded", "[('APPLE', 5), ('BANANA', 6), ('CHERRY', 6), ('DATE', 4)]", 1},
|
||||
// A7 regression: fenced decoration on the correct answer must
|
||||
// still score full credit.
|
||||
{"fenced", "```\n[('apple', 5), ('banana', 6), ('cherry', 6), ('date', 4)]\n```", 1},
|
||||
{"wrong: includes fig, forgets the length filter", "[('apple', 5), ('banana', 6), ('cherry', 6), ('date', 4), ('fig', 3)]", 0},
|
||||
{"wrong: unsorted, dict insertion order", "[('apple', 5), ('banana', 6), ('cherry', 6), ('fig', 3), ('date', 4)]", 0},
|
||||
}
|
||||
@@ -331,6 +338,10 @@ func TestPyGeneratorExhaustionTraceTest_Eval(t *testing.T) {
|
||||
{"exact correct", "[0, 1, 2] []", 1},
|
||||
{"correct with surrounding whitespace", " [0, 1, 2] [] ", 1},
|
||||
{"correct with trailing newline", "[0, 1, 2] []\n", 1},
|
||||
// A7 regression: fenced/quoted decoration on the correct answer
|
||||
// must still score full credit.
|
||||
{"fenced", "```\n[0, 1, 2] []\n```", 1},
|
||||
{"quoted", `"[0, 1, 2] []"`, 1},
|
||||
{"wrong: assumes the generator restarts", "[0, 1, 2] [0, 1, 2]", 0},
|
||||
{"wrong: assumes an error is raised and printed", "StopIteration", 0},
|
||||
}
|
||||
@@ -370,6 +381,13 @@ def find_configs(base_dir):
|
||||
return [str(p) for p in Path(base_dir).glob("*.yaml")]
|
||||
` + "```"
|
||||
|
||||
goodRewriteGlobRecursiveWildcard := "```python\n" + `from pathlib import Path
|
||||
|
||||
|
||||
def find_configs(base_dir):
|
||||
return [str(p) for p in Path(base_dir).glob("**/*.yaml")]
|
||||
` + "```"
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
response string
|
||||
@@ -378,7 +396,8 @@ def find_configs(base_dir):
|
||||
{"good rewrite: from pathlib import Path", goodRewrite, 1},
|
||||
{"good rewrite: import pathlib", goodRewriteAltImport, 1},
|
||||
{"wrong: unchanged os.path/os.walk code", stillUsesOsPath, 0},
|
||||
{"partial: uses pathlib but non-recursive glob, not rglob", missingRglob, 2.0 / 3.0}, // (1*1 + 2*0.5)/3: ContainsAny hits, ContainsAll("rglob","*.yaml") matches only "*.yaml"
|
||||
{"partial: uses pathlib but non-recursive glob, not rglob", missingRglob, 2.0 / 3.0}, // (1*1 + 1*0 + 1*1)/3: pathlib import ok, neither rglob nor glob("**/ present, *.yaml ok
|
||||
{"good rewrite: glob(\"**/*.yaml\") is behaviorally equivalent to rglob (A16 bug probe)", goodRewriteGlobRecursiveWildcard, 1},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
|
||||
@@ -142,9 +142,19 @@ given an interface with a nested object property, that nested object's own
|
||||
properties are also optional). Do not use the "any" type anywhere. Respond
|
||||
with only the type definition.`
|
||||
|
||||
// A9: the mapped-type variable name is the author's choice, not
|
||||
// mandated syntax - "K" is one common convention but "P" (as used in
|
||||
// TypeScript's own built-in Partial<T>) is equally correct, so the
|
||||
// recursive-reference check accepts any identifier there rather than
|
||||
// only the literal "DeepPartial<T[K]>" substring. The "keyof T" check
|
||||
// is similarly relaxed to tolerate any whitespace width between the two
|
||||
// words, not just a single space.
|
||||
evaluator := eval.All(
|
||||
eval.W(eval.ContainsAll("DeepPartial<T>", "keyof T"), 2),
|
||||
eval.W(eval.ContainsAny("DeepPartial<T[K]>"), 1),
|
||||
eval.W(eval.Mean(
|
||||
eval.ContainsAll("DeepPartial<T>"),
|
||||
eval.Regex(`(?i)keyof\s+T\b`),
|
||||
), 2),
|
||||
eval.W(eval.Regex(`DeepPartial<\s*T\[\s*\w+\s*\]\s*>`), 1),
|
||||
eval.W(noAnyEscapeHatch(), 1),
|
||||
)
|
||||
|
||||
@@ -316,8 +326,13 @@ configuration as an eslint.config.js flat config, preserving the same rules
|
||||
(semi: error, prefer-const: error) and the ecmaVersion/sourceType settings
|
||||
via languageOptions. Respond with only the eslint.config.js file content.`
|
||||
|
||||
// A flat eslint.config.js can equally validly be authored as an ES
|
||||
// module (export default [...]) or as CommonJS (module.exports = [...]),
|
||||
// depending on the project's own module system; requiring "export
|
||||
// default" specifically rejected an otherwise-correct CommonJS answer
|
||||
// (A8).
|
||||
evaluator := eval.All(
|
||||
eval.W(eval.ContainsAll("export default"), 2),
|
||||
eval.W(eval.ContainsAny("export default", "module.exports"), 2),
|
||||
eval.W(eval.ContainsAny("defineConfig(", "languageOptions"), 1),
|
||||
)
|
||||
|
||||
@@ -447,6 +462,8 @@ printed, nothing else.`
|
||||
Subcategory: "typescript",
|
||||
Description: "Trace the exact output of optional chaining combined with nullish coalescing on a falsy-but-present value.",
|
||||
Prompt: prompt,
|
||||
Eval: eval.Equals("localhost:0"),
|
||||
// A7: eval.ExactToken so a fenced or quoted rendering of the exact
|
||||
// printed text still scores full credit.
|
||||
Eval: eval.ExactToken("localhost:0"),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -163,6 +163,28 @@ func TestTsGenericUtilityTypeTest_Eval(t *testing.T) {
|
||||
{"correct recursive DeepPartial", goodDeepPartial, 1},
|
||||
{"wrong: shallow only, no recursion into nested objects", missingRecursion, 0.75}, // (2*1 + 1*0 + 1*1)/4: mapped type ok, no DeepPartial<T[K]> reference, no any
|
||||
{"wrong: uses any, defeating the point", usesAny, 0.5}, // (2*1 + 1*0 + 1*0)/4: mapped type ok, no recursion, uses any
|
||||
{
|
||||
// A9: "P" (TypeScript's own Partial<T> convention) is an
|
||||
// equally valid mapped-type variable name to "K" - the
|
||||
// recursive-reference check must not hardcode the literal
|
||||
// identifier.
|
||||
name: "correct recursive DeepPartial using P instead of K as the mapped-type variable (A9 bug probe)",
|
||||
response: "```ts\n" + `type DeepPartial<T> = {
|
||||
[P in keyof T]?: T[P] extends object ? DeepPartial<T[P]> : T[P];
|
||||
};
|
||||
` + "```",
|
||||
want: 1,
|
||||
},
|
||||
{
|
||||
// A9: "keyof T" with extra internal whitespace is the same
|
||||
// requirement, just formatted differently.
|
||||
name: "correct recursive DeepPartial with extra whitespace around keyof T (A9 bug probe)",
|
||||
response: "```ts\n" + `type DeepPartial<T> = {
|
||||
[K in keyof T]?: T[K] extends object ? DeepPartial<T[K]> : T[K];
|
||||
};
|
||||
` + "```",
|
||||
want: 1,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
@@ -285,6 +307,14 @@ export default defineConfig([
|
||||
rules: { semi: "error", "prefer-const": "error" },
|
||||
},
|
||||
]);
|
||||
` + "```"
|
||||
|
||||
goodFlatConfigCommonJS := "```js\n" + `module.exports = [
|
||||
{
|
||||
languageOptions: { ecmaVersion: "latest", sourceType: "module" },
|
||||
rules: { semi: "error", "prefer-const": "error" },
|
||||
},
|
||||
];
|
||||
` + "```"
|
||||
|
||||
stillLegacyFormat := "```json\n" + tsOldEslintrcConfig + "\n```"
|
||||
@@ -296,6 +326,7 @@ export default defineConfig([
|
||||
}{
|
||||
{"good: plain array export default", goodFlatConfig, 1},
|
||||
{"good: defineConfig helper", goodFlatConfigWithHelper, 1},
|
||||
{"good: CommonJS module.exports form is equally valid flat config (A8 bug probe)", goodFlatConfigCommonJS, 1},
|
||||
{"wrong: left as legacy .eslintrc JSON", stillLegacyFormat, 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
@@ -400,6 +431,10 @@ func TestTsOptionalChainingNullishTraceTest_Eval(t *testing.T) {
|
||||
{"exact correct", "localhost:0", 1},
|
||||
{"correct with surrounding whitespace", " localhost:0 ", 1},
|
||||
{"correct case-insensitive", "LOCALHOST:0", 1},
|
||||
// A7 regression: fenced/quoted decoration on the correct answer
|
||||
// must still score full credit.
|
||||
{"fenced", "```\nlocalhost:0\n```", 1},
|
||||
{"quoted with trailing period", `"localhost:0".`, 1},
|
||||
{"wrong: treats ?? like || and replaces the falsy 0", "localhost:8080", 0},
|
||||
{"wrong: assumes host is present", "example.com:0", 0},
|
||||
}
|
||||
|
||||
@@ -1,6 +1,9 @@
|
||||
package tests
|
||||
|
||||
import (
|
||||
"context"
|
||||
"regexp"
|
||||
|
||||
"github.com/lukaszraczylo/llm-testbench/internal/eval"
|
||||
"github.com/lukaszraczylo/llm-testbench/internal/testkit"
|
||||
)
|
||||
@@ -34,8 +37,9 @@ scenario2: "The client did not send any authentication credentials, and
|
||||
the requested resource requires a logged-in user."
|
||||
scenario3: "The client is authenticated, but their account does not have
|
||||
permission to access the requested resource."
|
||||
scenario4: "The resource has permanently moved to a new URL; the server
|
||||
wants clients (and search engines) to use the new URL from now on."`
|
||||
scenario4: "The resource has permanently moved to a new URL; the original
|
||||
request was a GET, and the server wants clients (and search engines) to
|
||||
use the new URL from now on."`
|
||||
|
||||
// webHTTPStatusScenariosTest: map 4 inline HTTP scenarios to their correct
|
||||
// status codes.
|
||||
@@ -43,7 +47,11 @@ wants clients (and search engines) to use the new URL from now on."`
|
||||
// ground truth: malformed request body -> 400 Bad Request; missing
|
||||
// credentials on a resource that requires them -> 401 Unauthorized;
|
||||
// authenticated but lacking permission -> 403 Forbidden; permanent
|
||||
// relocation -> 301 Moved Permanently.
|
||||
// relocation of a GET request -> 301 Moved Permanently (B8: the scenario
|
||||
// states the original request is a GET, so 308 Permanent Redirect - which
|
||||
// exists specifically to guarantee an unchanged method and body on
|
||||
// replay, relevant for POST/PUT - is not equally defensible here; 301 is
|
||||
// the unique correct answer).
|
||||
func webHTTPStatusScenariosTest() testkit.Test {
|
||||
prompt := `For each of these 4 request-handling scenarios, give the single most
|
||||
correct HTTP status code:
|
||||
@@ -70,24 +78,39 @@ code as a number: {"scenario1":<number>,"scenario2":<number>,"scenario3":<number
|
||||
}
|
||||
}
|
||||
|
||||
// webDNSMXRecordPattern requires the literal token "MX" (word-bounded,
|
||||
// case-insensitive) anywhere in the response, materially correct whether
|
||||
// or not the response also says "record" or names the other record types
|
||||
// it is not.
|
||||
const webDNSMXRecordPattern = `(?i)\bMX\b`
|
||||
// webDNSMXRecordTrailingWordPattern strips an optional trailing
|
||||
// "record"/"records" word (with or without a final period) that the
|
||||
// prompt's own "the record type" phrasing invites a model to echo back,
|
||||
// e.g. "MX record" or "MX records.", before the exact-token comparison.
|
||||
var webDNSMXRecordTrailingWordPattern = regexp.MustCompile(`(?i)\s+records?\.?\s*$`)
|
||||
|
||||
// webDNSMXRecordEval wraps eval.ExactToken(want) to also accept a response
|
||||
// carrying that optional trailing "record"/"records" word (B4).
|
||||
func webDNSMXRecordEval(want string) eval.Evaluator {
|
||||
exact := eval.ExactToken(want)
|
||||
return eval.EvaluatorFunc(func(ctx context.Context, response string) eval.Score {
|
||||
stripped := webDNSMXRecordTrailingWordPattern.ReplaceAllString(response, "")
|
||||
return exact.Evaluate(ctx, stripped)
|
||||
})
|
||||
}
|
||||
|
||||
// webDNSMXRecordTest: name the DNS record type used to publish a domain's
|
||||
// prioritized mail servers.
|
||||
//
|
||||
// ground truth: mail-routing priority is published exclusively via MX
|
||||
// (Mail Exchange) records; no other standard record type carries a
|
||||
// priority value for mail delivery.
|
||||
// priority value for mail delivery. B4: the prompt's own example list used
|
||||
// to include "MX" itself, which the prior substring evaluator would then
|
||||
// score correct even for a response that only restated the option list
|
||||
// without ever answering the question; MX is no longer in the example
|
||||
// list, and the evaluator now anchors to the whole answer instead of
|
||||
// matching a substring anywhere in the response.
|
||||
func webDNSMXRecordTest() testkit.Test {
|
||||
prompt := `example.com wants mail delivered to mailA.example.com as its primary
|
||||
mail server (priority 10), falling back to mailB.example.com (priority
|
||||
20) only if the primary is unreachable. Which single DNS record type
|
||||
publishes this prioritized mail-routing configuration? Respond with only
|
||||
the record type (e.g. A, AAAA, CNAME, MX, TXT, NS).`
|
||||
the record type (e.g. A, AAAA, CNAME, TXT, NS).`
|
||||
|
||||
return testkit.Test{
|
||||
ID: "web-dns-mx-record",
|
||||
@@ -95,7 +118,7 @@ the record type (e.g. A, AAAA, CNAME, MX, TXT, NS).`
|
||||
Subcategory: "web",
|
||||
Description: "Name the DNS record type used to publish a domain's prioritized mail servers.",
|
||||
Prompt: prompt,
|
||||
Eval: eval.Regex(webDNSMXRecordPattern),
|
||||
Eval: webDNSMXRecordEval("MX"),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -106,7 +129,10 @@ the record type (e.g. A, AAAA, CNAME, MX, TXT, NS).`
|
||||
// ground truth: a 301 redirect would break the tracking links (they would
|
||||
// stop resolving to their own URL), so the correct fix is a canonical
|
||||
// tag: it consolidates ranking signals onto the preferred URL while
|
||||
// leaving every tracked variant independently reachable.
|
||||
// leaving every tracked variant independently reachable. B3: the prompt
|
||||
// itself quotes both options ("canonical" or "redirect"), so scored with
|
||||
// eval.ExactToken - the prior anchored regex required the bare word and
|
||||
// scored a quoted or bolded correct answer 0.
|
||||
func webCanonicalVsRedirectTest() testkit.Test {
|
||||
prompt := `An e-commerce site serves identical product content at these three
|
||||
URLs, which must all keep working since they are used as tracking links
|
||||
@@ -129,6 +155,6 @@ you use a canonical tag or a 301 redirect? Respond with only one word:
|
||||
Subcategory: "web",
|
||||
Description: "Choose canonical tag over 301 redirect for a duplicate-content scenario requiring all URL variants to stay reachable.",
|
||||
Prompt: prompt,
|
||||
Eval: eval.Regex(`(?i)^\s*canonical\.?\s*$`),
|
||||
Eval: eval.ExactToken("canonical"),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -15,6 +15,10 @@ func TestWebSitemapMaxURLsTest_Eval(t *testing.T) {
|
||||
}{
|
||||
{"correct", "50000", 1},
|
||||
{"prose wrapped", "The limit is 50000 URLs per sitemap file.", 1},
|
||||
// B1: the prompt itself primes comma grouping with "52,000", so a
|
||||
// comma-grouped correct answer must not score 0.
|
||||
{"comma-grouped", "50,000", 1},
|
||||
{"comma-grouped in a sentence", "The limit is 50,000 URLs.", 1},
|
||||
{"wrong: confused with the uncompressed size limit in MB", "50", 0},
|
||||
{"wrong entirely", "10000", 0},
|
||||
}
|
||||
@@ -61,8 +65,14 @@ func TestWebDNSMXRecordTest_Eval(t *testing.T) {
|
||||
{"bare", "MX", 1},
|
||||
{"lowercase", "mx", 1},
|
||||
{"with the word record", "MX record", 1},
|
||||
{"with the word records and a period", "MX records.", 1},
|
||||
{"quoted", `"MX"`, 1},
|
||||
{"wrong: CNAME", "CNAME", 0},
|
||||
{"wrong: A record", "A", 0},
|
||||
// B4: restating the prompt's example list (which no longer
|
||||
// includes MX at all) must not score 1 the way the old
|
||||
// substring-anywhere-in-the-response evaluator would have.
|
||||
{"restating the full example list scores 0", "A, AAAA, CNAME, TXT, NS", 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
@@ -86,6 +96,9 @@ func TestWebCanonicalVsRedirectTest_Eval(t *testing.T) {
|
||||
{"correct capitalized with period", "Canonical.", 1},
|
||||
{"wrong: redirect", "redirect", 0},
|
||||
{"wrong: extra words break the forced one-word format", "Use a canonical tag", 0},
|
||||
// B3: the prompt's own phrasing quotes both options.
|
||||
{"correct, quoted (prompt's own phrasing)", `"canonical"`, 1},
|
||||
{"correct, bolded", "**canonical**", 1},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
|
||||
@@ -19,20 +19,24 @@ HTTP/1.1 204 No Content
|
||||
Access-Control-Allow-Origin: https://app.example.com
|
||||
Access-Control-Allow-Methods: GET, POST, PUT`
|
||||
|
||||
// webCORSMissingHeaderPattern requires the literal token
|
||||
// "Access-Control-Allow-Headers" (case-insensitive) anywhere in the
|
||||
// response, accepting it with or without a trailing colon or surrounding
|
||||
// prose.
|
||||
const webCORSMissingHeaderPattern = `(?i)access-control-allow-headers`
|
||||
|
||||
// webCORSMissingHeaderTest: identify the single missing CORS response
|
||||
// header that will cause the browser to block the request.
|
||||
// webCORSMissingHeaderWant is the single header name this test's prompt
|
||||
// forces as the answer.
|
||||
//
|
||||
// ground truth: the browser sent Access-Control-Request-Headers:
|
||||
// X-Custom-Auth in the preflight, but the server's response never echoes
|
||||
// that header name back in an Access-Control-Allow-Headers response
|
||||
// header, so the browser blocks the actual PUT request even though the
|
||||
// origin and method are both explicitly allowed.
|
||||
// origin and method are both explicitly allowed. B5: scored with
|
||||
// eval.ExactToken, anchored to the whole answer, rather than a substring
|
||||
// match anywhere in the response - a substring match would also score a
|
||||
// response that talks about the header without ever naming it as the
|
||||
// answer (e.g. explaining Access-Control-Allow-Methods at length while
|
||||
// mentioning "Access-Control-Allow-Headers" only as one of several names
|
||||
// it considered and rejected).
|
||||
const webCORSMissingHeaderWant = "Access-Control-Allow-Headers"
|
||||
|
||||
// webCORSMissingHeaderTest: identify the single missing CORS response
|
||||
// header that will cause the browser to block the request.
|
||||
func webCORSMissingHeaderTest() testkit.Test {
|
||||
prompt := `Here is a CORS preflight exchange:
|
||||
|
||||
@@ -49,7 +53,7 @@ nothing else.`
|
||||
Subcategory: "web",
|
||||
Description: "Identify the single missing CORS response header that causes a browser to block a preflighted request.",
|
||||
Prompt: prompt,
|
||||
Eval: eval.Regex(webCORSMissingHeaderPattern),
|
||||
Eval: eval.ExactToken(webCORSMissingHeaderWant),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -14,8 +14,17 @@ func TestWebCORSMissingHeaderTest_Eval(t *testing.T) {
|
||||
want float64
|
||||
}{
|
||||
{"bare header name", "Access-Control-Allow-Headers", 1},
|
||||
{"with a colon", "Access-Control-Allow-Headers:", 1},
|
||||
{"in a sentence", "The missing header is Access-Control-Allow-Headers.", 1},
|
||||
{"quoted", `"Access-Control-Allow-Headers"`, 1},
|
||||
{"bolded", "**Access-Control-Allow-Headers**", 1},
|
||||
{"trailing period", "Access-Control-Allow-Headers.", 1},
|
||||
// B5: ExactToken anchors to the whole normalized answer, per the
|
||||
// prompt's own "Respond with only the header name, nothing else" -
|
||||
// a colon is extra content beyond the header name, and a
|
||||
// sentence wrapping the name is not the forced bare-answer format
|
||||
// either; both now correctly score 0 rather than matching as a
|
||||
// substring anywhere in the response.
|
||||
{"with a colon: extra content, not the bare name", "Access-Control-Allow-Headers:", 0},
|
||||
{"in a sentence: extra words break the forced format", "The missing header is Access-Control-Allow-Headers.", 0},
|
||||
{"wrong: names an already-present header", "Access-Control-Allow-Origin", 0},
|
||||
{"wrong: names an already-present header", "Access-Control-Allow-Methods", 0},
|
||||
}
|
||||
|
||||
@@ -117,7 +117,8 @@ Respond with only a JSON object: {"quorum_size":<number>,"max_tolerable_failures
|
||||
// description of a worst-case (minimum-fanout) B-tree capacity model
|
||||
// written for this test, pinning order m and a target key count.
|
||||
const paperBTreeExcerpt = `A B-tree of order m allows every internal node between ceil(m/2) and m
|
||||
children, keeping the tree shallow and well balanced even as it grows,
|
||||
children, except the root, which may have as few as two children, keeping
|
||||
the tree shallow and well balanced even as it grows,
|
||||
which bounds the number of disk reads a lookup needs to at most one per
|
||||
level. For capacity planning this report models the worst case: a tree
|
||||
where every internal node has the minimum allowed fanout, f equals
|
||||
@@ -177,12 +178,19 @@ partition between two data centers, both sides keep accepting writes
|
||||
independently rather than refusing requests, accepting that the two sides
|
||||
may briefly disagree and need reconciliation once the partition heals.`
|
||||
|
||||
// paperCAPAvailabilityPattern anchors on the trimmed response being
|
||||
// exactly the single word "availability" (case-insensitive, with an
|
||||
// optional trailing period), accepting every materially-correct form of
|
||||
// the forced one-word answer without also matching a response that names
|
||||
// the wrong guarantee.
|
||||
const paperCAPAvailabilityPattern = `(?i)^\s*availability\.?\s*$`
|
||||
// paperCAPAvailabilityWant is the single word this test's prompt forces as
|
||||
// the answer.
|
||||
//
|
||||
// ground truth: the excerpt states both data centers "keep accepting
|
||||
// writes independently rather than refusing requests" during the
|
||||
// partition. Per the CAP theorem as the excerpt defines it, continuing to
|
||||
// answer every request rather than refusing (erroring) is the definition
|
||||
// of availability, so the system prioritizes availability over
|
||||
// consistency during the partition. B2: scored with eval.ExactToken so a
|
||||
// quoted or bolded "Availability" - both forms the prompt's own phrasing
|
||||
// invites - still matches; the prior anchored regex required the bare
|
||||
// word and scored those correct answers 0.
|
||||
const paperCAPAvailabilityWant = "Availability"
|
||||
|
||||
// paperCAPAvailabilityChoiceTest: identify, from an inline CAP-theorem
|
||||
// excerpt and a described design decision, which guarantee (consistency or
|
||||
@@ -204,7 +212,7 @@ the CAP theorem? Respond with only one word: "Consistency" or
|
||||
Subcategory: "whitepapers",
|
||||
Description: "Identify which CAP-theorem guarantee a described partition-handling design decision prioritizes.",
|
||||
Prompt: prompt,
|
||||
Eval: eval.Regex(paperCAPAvailabilityPattern),
|
||||
Eval: eval.ExactToken(paperCAPAvailabilityWant),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -164,6 +164,11 @@ func TestPaperCAPAvailabilityChoiceTest_Eval(t *testing.T) {
|
||||
{"correct with trailing period", "Availability.", 1},
|
||||
{"wrong: consistency", "Consistency", 0},
|
||||
{"wrong: extra words break the forced one-word format", "The answer is availability", 0},
|
||||
// B2: the prompt's own phrasing quotes the two options, so a
|
||||
// quoted or bolded answer must still score full credit.
|
||||
{"correct, quoted (prompt's own phrasing)", `"Availability"`, 1},
|
||||
{"correct, bolded", "**Availability**", 1},
|
||||
{"correct, quoted with trailing period", `"Availability".`, 1},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
|
||||
@@ -114,8 +114,8 @@ total number of documents in the corpus and df(t) is the number of
|
||||
documents that contain t at least once. The TF-IDF weight of a term in a
|
||||
document is the product tf(t,d) times idf(t): a term that appears often in
|
||||
one document but rarely elsewhere scores highest, while a term appearing in
|
||||
every document scores near zero regardless of how often it repeats, since
|
||||
its idf approaches the logarithm of one.`
|
||||
every document scores zero regardless of how often it repeats, since its
|
||||
idf is ln(1) = 0 exactly.`
|
||||
|
||||
// paperTFIDFDoc1, paperTFIDFDoc2, and paperTFIDFDoc3 are the pre-tokenized
|
||||
// (whitespace-split, lowercase) mini-corpus for paperTFIDFTermScoreTest.
|
||||
@@ -185,6 +185,12 @@ surfaced, never recover ones it dropped.`
|
||||
// paperRecallRelevant and paperRecallRetrievedTop6 are the inline
|
||||
// relevant-set and top-6 retrieved-ranking fixtures for
|
||||
// paperRecallAtKTest.
|
||||
//
|
||||
// ground truth: relevant = {D2,D5,D7,D9} (size 4); of those, D2, D5, and
|
||||
// D7 appear somewhere in the top-6 retrieved list, but D9 does not, so the
|
||||
// intersection has size 3. recall@6 = |intersection| / |relevant| = 3/4 =
|
||||
// 0.75. whitepapers_vector_test.go independently recomputes this by
|
||||
// counting the intersection directly, not via wpRecallAtK.
|
||||
var (
|
||||
paperRecallRelevant = []string{"D2", "D5", "D7", "D9"}
|
||||
paperRecallRetrievedTop6 = []string{"D1", "D5", "D3", "D7", "D6", "D2"}
|
||||
|
||||
@@ -32,6 +32,9 @@ func TestPaperPQCompressionRatioTest_Eval(t *testing.T) {
|
||||
}{
|
||||
{"correct", "64", 1},
|
||||
{"prose wrapped", "The compression ratio is 64x.", 1},
|
||||
// B6: ratio notation must extract the first term (64), not the
|
||||
// second (the "1" of "64:1").
|
||||
{"ratio notation", "64:1", 1},
|
||||
{"wrong", "8", 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
|
||||
Reference in New Issue
Block a user