fix(catalog): harden evaluators against correct-answer phrasing variance

Both reviewers found evaluators that scored correct answers as zero:
hardcoded flag order, quote-style rigidity, missing multiline anchors,
comma-grouped numbers, ratio notation, and numeric strings in JSON.
A shared ExactToken evaluator now normalizes fences, quotes, and
trailing periods for exact-answer tests. The obsolete tmutil snapshot
reference now accepts the current localsnapshot verb.
This commit is contained in:
2026-08-17 02:48:05 +01:00
parent 8ab83a57ca
commit 7264fe81b7
32 changed files with 815 additions and 152 deletions
+33
View File
@@ -120,6 +120,39 @@ func TestCodeTraceTSWant_GroundTruth(t *testing.T) {
if got != codeTraceTSWant {
t.Errorf("codeTSPackTrace(%v) = %q, want %q", items, got, codeTraceTSWant)
}
// BN4: cross-check against a real JS engine too, not just the Go
// replica above. node cannot execute codeTraceTSSource's TypeScript
// type annotations directly, so this is the same logic with the two
// annotations (": number[]", ": string") stripped - everything else,
// including the reduce/ternary/toString(2) logic under test, is
// identical source.
if _, err := exec.LookPath("node"); err != nil {
t.Skip("node not found on PATH, skipping exec ground-truth cross-check")
}
const jsSource = `function pack(items) {
return items.reduce((acc, x, i) => {
if (i % 2 === 0) return acc + x;
return acc + x * 2;
}, 0).toString(2);
}
console.log(pack([3, 4, 5, 6, 7]));
`
// #nosec G204 -- jsSource is a fixed, hardcoded string (the TS source
// embedded in the prompt with its two type annotations stripped), not
// external or response-controlled input.
cmd := exec.Command("node", "-e", jsSource) //nolint:gosec // see comment above
var stdout bytes.Buffer
cmd.Stdout = &stdout
cmd.Stderr = os.Stderr
if err := cmd.Run(); err != nil {
t.Fatalf("running node ground-truth script failed: %v", err)
}
nodeGot := strings.TrimSpace(stdout.String())
if nodeGot != codeTraceTSWant {
t.Errorf("node-computed pack([3,4,5,6,7]) = %q, want %q", nodeGot, codeTraceTSWant)
}
}
func TestCodeTraceTSTest_Eval(t *testing.T) {