From 499956f8ccd7967881800d91ed44bf30c709b9fc Mon Sep 17 00:00:00 2001 From: Lukasz Raczylo Date: Sun, 30 Aug 2026 14:19:08 +0100 Subject: [PATCH] fix(tests): disambiguate two prompts, exempt --force-if-includes git-detached-head-recovery: all three benchmarked models answered ["branch"], which the prompt's literal ask (save the commits) makes correct while the evaluator requires ending attached. The prompt now demands ending checked out on the new branch and names the bare-branch gap explicitly. scen-crashloop-order: the instruction excluded only acting steps, so including the read-only journalctl step E followed the letter of the prompt. The instruction now asks for the smallest sufficient set and excludes redundant diagnostics. git-force-with-lease-vs-force: exempt --force-if-includes, the stricter safe variant, from the bare --force pattern. --- internal/tests/delivery_git.go | 8 +++++--- internal/tests/operations_scenario.go | 7 ++++--- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/internal/tests/delivery_git.go b/internal/tests/delivery_git.go index 84887c4..dc187f9 100644 --- a/internal/tests/delivery_git.go +++ b/internal/tests/delivery_git.go @@ -46,7 +46,7 @@ func delNoUnnegatedMention(forbidden *regexp.Regexp) eval.Evaluator { // "-with-lease" - the safe form, exempt from the bare-force check entirely // rather than needing a negation cue. var delForceFlagPattern = regexp.MustCompile(`(?i)--force\b`) -var delForceWithLeaseSuffixPattern = regexp.MustCompile(`(?i)^-with-lease`) +var delForceWithLeaseSuffixPattern = regexp.MustCompile(`(?i)^-(with-lease|if-includes)`) // delNoBareForcePush scores full credit unless the response recommends a // bare "--force" push (not "--force-with-lease") with no negation cue @@ -320,8 +320,10 @@ func delGitDetachedHeadRecoveryTest() testkit.Test { branch name), then made 3 new commits while HEAD was detached. You only now realize you need to keep that work, and HEAD is still detached. Give the ordered list of git subcommands (just the subcommand names, e.g. "branch") -you must run, in order, to save the 3 commits onto a new branch without -losing them. Respond with only a JSON array of subcommand names, e.g. +you must run, in order, to save the 3 commits onto a new branch AND end up +checked out on that branch, with HEAD attached to it (a bare "git branch" +alone saves the commits but leaves HEAD detached, which is not enough +here). Respond with only a JSON array of subcommand names, e.g. ["a","b"].` return testkit.Test{ diff --git a/internal/tests/operations_scenario.go b/internal/tests/operations_scenario.go index f415058..60cca4c 100644 --- a/internal/tests/operations_scenario.go +++ b/internal/tests/operations_scenario.go @@ -178,9 +178,10 @@ D) kubectl scale deployment ingestion --replicas=0 E) ssh to a node and run journalctl -u kubelet Respond with only a JSON array of the step letters that make up your -diagnosis, containing exactly the diagnostic steps you would take to find -the root cause (any order) - do not include steps that only act without -diagnosing.` +diagnosis, containing exactly the SMALLEST set of steps sufficient to find +the root cause (any order). Do not include steps that only act without +diagnosing, and do not include diagnostics that are redundant when a +simpler kubectl call already surfaces the same information.` return testkit.Test{ ID: "scen-crashloop-order",