diff --git a/internal/eval/negation.go b/internal/eval/negation.go index 5eed94d..f605351 100644 --- a/internal/eval/negation.go +++ b/internal/eval/negation.go @@ -3,6 +3,7 @@ package eval import ( "context" "regexp" + "strings" ) // negationCueAlternation is the shared union of cue words/phrases that turn @@ -18,7 +19,20 @@ import ( const negationCueAlternation = `don'?t|do not|never|avoid|not|cannot|can'?t|without|no need|` + `shouldn'?t|should not|must not|isn'?t|wasn'?t|danger|dangerous|` + `unsuitable|inappropriate|unsafe|insecure|weak|broken|deprecated|obsolete|` + - `too fast|predictable|unacceptable|blocks|wrong|problem` + `too fast|predictable|unacceptable|blocks|wrong|problem|` + + // Consequence-description cues: a good answer often names the discouraged + // term while describing the harm it causes, with no imperative or + // adjectival cue at all - "a bare --force will silently overwrite their + // work", "math/rand is a deterministic PRNG that relies on a seed" + // (observed as all-model false positives in the 2026-08-30 3-model run). + `overwrit\w*|clobber\w*|silently|discard\w*|deterministic|seeded|guessable|reproducible|` + + `lose|loses|losing|lost|` + + // Modal harm phrases: "a bare --force will replace the remote branch", + // "would replace ... your rewritten history". The modal keeps these + // narrow - a bare "replaces"/"rewritten" also appears in genuine + // endorsements ("make sure your rewritten history replaces theirs") and + // must not count as a cue. + `will replace|would replace|a bare` // negationCuePattern matches any cue word/phrase in negationCueAlternation, // case-insensitively, as a whole word/phrase (not a substring of an @@ -44,24 +58,51 @@ const negationDirectionalCueAlternation = `instead of|rather than` // Checked only in the window immediately BEFORE a match, never after. var negationDirectionalCuePattern = regexp.MustCompile(`(?i)\b(?:` + negationDirectionalCueAlternation + `)\b`) +// isSentenceDot reports whether response[i] is a sentence-ending '.': a dot +// followed by whitespace or end-of-string that does not terminate a common +// abbreviation ("e.g.", "i.e.", "etc.", "vs."). A dot glued to a following +// non-space character ("e.g.,", "release/2.4") is part of a token, never a +// clause boundary. +func isSentenceDot(response string, i int) bool { + if response[i] != '.' { + return false + } + if i+1 < len(response) && response[i+1] != ' ' && response[i+1] != '\n' && response[i+1] != '\t' { + return false + } + for _, abbr := range []string{"e.g", "i.e", "etc", "vs"} { + if i >= len(abbr) && strings.EqualFold(response[i-len(abbr):i], abbr) { + return false + } + } + return true +} + // clauseWindowBounds computes the [lo,hi) bidirectional search window // around a match [start,end) in response: outward up to maxChars // characters in each direction, but stopped early at the nearest clause -// boundary - a '.', ';', or a blank-line paragraph gap - whichever comes -// first, so a negation cue from an unrelated adjacent clause or sentence -// can never count as negating a genuinely unqualified endorsement -// elsewhere in a longer response (D5, C2). This is a known, accepted -// limitation for sentence-final abbreviations (e.g. "e.g.", "etc.") that -// happen to sit inside the window - none of this framework's fixtures -// place a negation cue immediately after one. +// boundary - a sentence-ending '.', a ';', or a blank-line paragraph gap - +// whichever comes first, so a negation cue from an unrelated adjacent +// clause or sentence can never count as negating a genuinely unqualified +// endorsement elsewhere in a longer response (D5, C2). +// +// A '.' counts as a clause boundary only when followed by whitespace or +// end-of-string. A dot glued to a following non-space character is part of +// a token, not a sentence end: the dots inside "e.g.," and a version +// number like "release/2.4" were observed cutting the window between a +// match and its genuine negation cue ("Do not edit ... (e.g., `kubectl +// edit`)" and "git push --force origin release/2.4\n``` should never be +// used") in the 2026-08-30 3-model run, zeroing correct answers from all +// three models. func clauseWindowBounds(response string, start, end, maxChars int) (lo, hi int) { + lo = start - maxChars if lo < 0 { lo = 0 } for i := start - 1; i >= lo; i-- { c := response[i] - if c == '.' || c == ';' { + if isSentenceDot(response, i) || c == ';' { lo = i + 1 break } @@ -77,7 +118,7 @@ func clauseWindowBounds(response string, start, end, maxChars int) (lo, hi int) } for i := end; i < hi; i++ { c := response[i] - if c == '.' || c == ';' { + if isSentenceDot(response, i) || c == ';' { hi = i break } @@ -89,6 +130,24 @@ func clauseWindowBounds(response string, start, end, maxChars int) (lo, hi int) return lo, hi } +// sameClause reports whether response[from:to] contains no clause boundary +// (sentence-ending dot, ';', or blank-line paragraph gap), i.e. the two +// offsets sit inside one clause. Used for enumeration inheritance in +// NoUnnegatedMention; deliberately has no maxChars cap - a clause is a +// clause however long its comma list runs. +func sameClause(response string, from, to int) bool { + for i := from; i < to; i++ { + c := response[i] + if isSentenceDot(response, i) || c == ';' { + return false + } + if c == '\n' && i+1 < to && response[i+1] == '\n' { + return false + } + } + return true +} + // negationCueNearby reports whether a negation cue (bidirectional, // clause-scoped) negates the match [start,end) in response. Directional // cues (instead of/rather than) are checked only in the PRE-window. @@ -125,20 +184,64 @@ func negationCueNearby(response string, start, end, maxChars int) bool { // exempt is treated as inherently safe and skips the negation check // entirely (e.g. "--force-with-lease" is the safe form of "--force" and // never needs a negation cue to be a correct recommendation). +// stripMarkdownEmphasis removes backticks and double-asterisk bold markers +// before negation scanning. Both are semantically empty but inflate the +// character distance between a cue and its match past the window cap: +// "you should **not** fix this ... with `kubectl edit`, `kubectl patch`" +// zeroed correct answers from all three models in the 2026-08-30 run. +// Single asterisks are kept - they can be literal command text ("KEYS *"). +func stripMarkdownEmphasis(s string) string { + // Blank lines hugging a code fence are display layout, not a paragraph + // break: "A bare:\n\n```bash\ngit push --force ...\n```\n\nshould never + // be used" is one sentence split around a display block, and the gap + // must not cut the clause window between the fenced example and the + // prose negating it (observed false positive, 2026-08-30 run). + s = fenceLeadingGapPattern.ReplaceAllString(s, "\n```") + s = fenceTrailingGapPattern.ReplaceAllString(s, "```$1\n") + // A code-fence line ("```" or "```bash") must not become an empty line + // either - same fake-paragraph-gap failure mode. + s = strings.ReplaceAll(s, "```", " ") + s = strings.ReplaceAll(s, "**", "") + return strings.ReplaceAll(s, "`", "") +} + +// fenceLeadingGapPattern and fenceTrailingGapPattern match the blank lines +// immediately before an opening code fence and after a closing one. +var ( + fenceLeadingGapPattern = regexp.MustCompile(`\n[ \t]*\n+(?:[ \t]*\n)*` + "```") + fenceTrailingGapPattern = regexp.MustCompile("```" + `([a-zA-Z0-9]*)\n[ \t]*\n+(?:[ \t]*\n)*`) +) + func NoUnnegatedMention(badPattern *regexp.Regexp, maxChars int, exempt *regexp.Regexp) Evaluator { return EvaluatorFunc(func(_ context.Context, response string) Score { + response = stripMarkdownEmphasis(response) matches := badPattern.FindAllStringIndex(response, -1) if len(matches) == 0 { return Score{Value: 1, Detail: "no mention of a discouraged term"} } + // prevNegatedEnd is the end offset of the most recent match that was + // found negated; -1 when there is none. A later match with no cue of + // its own inherits that negation when no sentence boundary separates + // the two - a single cue distributes over a comma enumeration of the + // same discouraged family ("not ... with kubectl edit, kubectl + // patch, or kubectl scale"), where each successive item drifts + // further from the shared cue than maxChars allows (observed as + // all-model false positives in the 2026-08-30 3-model run). + prevNegatedEnd := -1 for _, loc := range matches { start, end := loc[0], loc[1] if exempt != nil && exempt.MatchString(response[end:]) { continue } - if !negationCueNearby(response, start, end, maxChars) { - return Score{Value: 0, Detail: "unnegated mention of a discouraged term"} + if negationCueNearby(response, start, end, maxChars) { + prevNegatedEnd = end + continue } + if prevNegatedEnd >= 0 && sameClause(response, prevNegatedEnd, start) { + prevNegatedEnd = end + continue + } + return Score{Value: 0, Detail: "unnegated mention of a discouraged term"} } return Score{Value: 1, Detail: "every mention of a discouraged term is negated"} }) diff --git a/internal/eval/negation_test.go b/internal/eval/negation_test.go index 26ad9d7..8652f19 100644 --- a/internal/eval/negation_test.go +++ b/internal/eval/negation_test.go @@ -22,6 +22,58 @@ func TestNoUnnegatedMention(t *testing.T) { {"post-window 'insecure' adjectival cue", "MD5 is insecure for this purpose.", 1}, {"post-window 'predictable' adjectival cue", "MD5 output is predictable given no work factor.", 1}, {"bare 'no:' cue", "no: MD5 lacks a tunable work factor.", 1}, + // Regression fixtures from the 2026-08-30 3-model run: all four shapes + // below zeroed correct answers from every model tested. + { + name: "abbreviation dot does not cut the clause window ('e.g.,')", + response: "Do not edit the live values directly (e.g., MD5 configuration) since drift follows.", + want: 1, + }, + { + name: "abbreviation dot with space does not cut the clause window ('e.g. ')", + response: "Do not reach for weak digests, e.g. MD5, in any password path.", + want: 1, + }, + { + name: "mid-token version dot does not cut the clause window", + response: "Using v2.4 of the library with MD5 should never be done for passwords.", + want: 1, + }, + { + name: "markdown bold and backticks do not inflate the cue distance", + response: "You should **not** store passwords hashed by any weak digest like `MD5` here.", + want: 1, + }, + { + name: "consequence-description cue 'deterministic' negates", + response: "MD5 here acts as a deterministic mapping an attacker can precompute.", + want: 1, + }, + { + name: "consequence-description cue 'silently overwriting' negates", + response: "Choosing MD5 means silently overwriting your security margin.", + want: 1, + }, + { + name: "sentence-ending dot still cuts the window", + response: "Never trust the old design. Hash the password with MD5 today.", + want: 0, + }, + { + name: "blank lines around a fenced display block do not cut the window", + response: "A digest like:\n\n```text\nMD5\n```\n\nshould never touch a password path.", + want: 1, + }, + { + name: "one cue distributes over a comma enumeration in the same clause", + response: "Do not protect anything using outdated digests such as MD5, or an unsalted MD5, or a doubled MD5 for that matter.", + want: 1, + }, + { + name: "enumeration inheritance stops at a sentence boundary", + response: "Never rely on MD5 for new designs. MD5 is what this legacy tool emits, so wire MD5 into the new verifier as-is.", + want: 0, + }, { name: "clause-scoped: cue in an unrelated PRECEDING sentence does not count", response: "Never mind the deployment steps. Just hash the password with MD5.",