feat(v0.6.0): review hardening — live active_skills clustering, computable fingerprints

Full codebase review (multi-agent, adversarially verified) surfaced several
documented-but-dead mechanisms and doc/code drift. Fixes:

- adam-observe: struggle signals now emit `active_skills`, so silent_drift's
  primary cluster key AND §5b skill-attribution sub-clustering (+1 rubric
  bonus) actually fire — both were silently dead (no struggle signal carried
  the field).
- adam-cooldown: new `--compute` CLI deterministically derives
  proposal_fingerprint. The exported computeProposalFingerprint() was never
  called and the analyst was told to hand-compute a djb2 hash it cannot
  reproduce. Spec now mandates a *stable* cluster id so fingerprints reproduce
  across /reflect runs. Removed one dead normalization line.
- spec: reinforcement proposals excluded from A/B tracking — agents/adam.md
  contradicted itself (:376 included, :476 excluded); SKILL.md aligned.
- adam-nudge: PENDING_CHECK_PATHS now mirrors the full install set
  (adam-utils / adam-batch / adam-rollback were missing).
- adam-explain: synthesized clustering summary carries `regressions: 0`
  (structural consistency with parsed summaries).
- docs: test-count drift (87/94 -> 126) and "350-line hook" (-> ~600) fixed;
  adam-score header documents severity_sum/severity_by_type; adam-batch §4
  reference corrected.

Tests: +12 assertions (114 -> 126), all green. New regression tests cover the
active_skills fix and --compute, plus boundary gaps the review flagged:
retry_loop/weak_agent thresholds, A/B exact +/-25% deltas, cooldown 30d
blacklist edge.
This commit is contained in:
2026-05-29 01:57:44 +01:00
parent 2d9257922f
commit 4b36d6c09e
10 changed files with 219 additions and 20 deletions
+1 -1
View File
@@ -4,7 +4,7 @@
// automatically curated batch of production-failure evidence."
//
// Each batch groups entries by (signal_type, cluster_key) where cluster_key
// follows the same clustering rules as agents/adam.md §4:
// follows the same clustering rules as agents/adam.md ## Signal types / ## Process step 4:
// correction → tokenized phrase (cross-cwd)
// retry_loop → tool
// weak_agent → subagent_type
+34 -4
View File
@@ -4,8 +4,12 @@
//
// CLI:
// adam-cooldown.mjs --skill <slug> --fingerprint <hash> [--home <path>]
// adam-cooldown.mjs --compute --skill <slug> --cluster <id> [--diff-file <path>]
// → prints {"fingerprint":"<djb2_base36>"}; diff body read from --diff-file
// or stdin. This is how proposal_fingerprint is populated (the analyst
// runs it via Bash after drafting a proposal).
//
// Output: JSON one-liner with shape
// Output (gate mode): JSON one-liner with shape
// { "status": "cool"|"cooldown"|"blacklisted",
// "reason": "<human-readable reason>",
// "blocked_by": { "file": "<basename>", "days_remaining": <int> } | null }
@@ -33,12 +37,15 @@ const DAY_MS = 86400000;
export const LEGACY_FINGERPRINT = "legacy";
function parseArgs(argv) {
const args = { home: null, skill: null, fingerprint: null, help: false };
const args = { home: null, skill: null, fingerprint: null, compute: false, cluster: null, diffFile: null, help: false };
for (let i = 0; i < argv.length; i++) {
const a = argv[i];
if (a === "--home" && i + 1 < argv.length) args.home = argv[++i];
else if (a === "--skill" && i + 1 < argv.length) args.skill = argv[++i];
else if (a === "--fingerprint" && i + 1 < argv.length) args.fingerprint = argv[++i];
else if (a === "--cluster" && i + 1 < argv.length) args.cluster = argv[++i];
else if (a === "--diff-file" && i + 1 < argv.length) args.diffFile = argv[++i];
else if (a === "--compute") args.compute = true;
else if (a === "--help" || a === "-h") args.help = true;
}
return args;
@@ -158,9 +165,11 @@ export function computeProposalFingerprint(proposal) {
if (!proposal || typeof proposal !== "object") return LEGACY_FINGERPRINT;
const skill = proposal.skill_slug || proposal.target_skill || proposal.skill || "";
const cluster = proposal.signal_cluster_id || proposal.cluster_id || "";
// normalized_diff_body: whitespace (incl. newlines) collapsed to single
// spaces, then trimmed. Matches agents/adam.md §"Per-(skill, fingerprint)
// cooldown". (No trailing-newline strip needed — \s+ already absorbed them.)
const diff = String(proposal.diff_body || proposal.proposed_change || "")
.replace(/\s+/g, " ")
.replace(/\n+$/g, "")
.trim();
return djb2(`${skill}\n${cluster}\n${diff}`);
}
@@ -168,7 +177,28 @@ export function computeProposalFingerprint(proposal) {
function main() {
const args = parseArgs(process.argv.slice(2));
if (args.help) {
process.stdout.write("usage: adam-cooldown.mjs --skill <slug> --fingerprint <hash> [--home <path>]\n");
process.stdout.write(
"usage: adam-cooldown.mjs --skill <slug> --fingerprint <hash> [--home <path>]\n" +
" adam-cooldown.mjs --compute --skill <slug> --cluster <id> [--diff-file <path>]\n"
);
process.exit(0);
}
// --compute: deterministically derive a proposal_fingerprint. The analyst
// invokes this (it has Bash) after drafting a proposal, then writes the
// result into proposal frontmatter so the cooldown gate keys on it.
if (args.compute) {
let diff = "";
if (args.diffFile) {
try { diff = readFileSync(args.diffFile, "utf8"); } catch { /* empty → still deterministic */ }
} else {
try { diff = readFileSync(0, "utf8"); } catch { /* no stdin */ }
}
const fp = computeProposalFingerprint({
skill_slug: args.skill || "",
signal_cluster_id: args.cluster || "",
diff_body: diff,
});
process.stdout.write(JSON.stringify({ fingerprint: fp }) + "\n");
process.exit(0);
}
if (!args.skill || !args.fingerprint) {
+1
View File
@@ -135,6 +135,7 @@ export function parseTrace(text) {
considered: clusters.length,
emitted,
skipped: clusters.length - emitted,
regressions: 0,
reasons,
};
}
+2 -1
View File
@@ -23,7 +23,8 @@
// Output: JSON object
// {
// "sessions": [
// {"session_id": "...", "negative_count": N, "task_completed_count": M, "dampener": 1.0}
// {"session_id": "...", "negative_count": N, "task_completed_count": M,
// "severity_sum": S, "severity_by_type": {"<type>": N, ...}, "dampener": 1.0}
// ],
// "reinforcement_candidates": [
// {"skill_slug": "tdd-loop", "count": 3, "recent_ts": "..."}
+151
View File
@@ -71,6 +71,17 @@ assert_grep() {
fi
}
assert_no_grep() {
local file="$1" pattern="$2" name="$3"
if grep -qE "$pattern" "$file" 2>/dev/null; then
echo " FAIL: $name (pattern $pattern unexpectedly present in $file)"
FAIL=$((FAIL+1))
else
echo " PASS: $name"
PASS=$((PASS+1))
fi
}
# --- Test 1: correction signal ---
echo "Test 1: user correction"
reset_state
@@ -1839,6 +1850,146 @@ else
echo " FAIL: expected 8 context_window entries (got $cw_len)"; FAIL=$((FAIL+1))
fi
# --- Test 103: silent_drift carries active_skills (its primary cluster key) ---
echo "Test 103: silent_drift emits active_skills (§5b skill-attribution)"
reset_state
echo '{"hook_event_name":"PreToolUse","tool_name":"Skill","tool_input":{"skill":"tdd"},"session_id":"sSK","cwd":"/tmp/x"}' \
| HOOK_RUN >/dev/null 2>&1 || true
for i in 1 2 3 4 5; do
echo "{\"hook_event_name\":\"PostToolUse\",\"tool_name\":\"Read\",\"tool_input\":{\"file_path\":\"/tmp/sk-$i\"},\"tool_response\":{\"content\":\"ok\"},\"session_id\":\"sSK\",\"cwd\":\"/tmp/x\"}" \
| HOOK_RUN >/dev/null 2>&1 || true
done
assert_grep "$ROOT/journal.jsonl" '"type":"silent_drift"' "silent_drift emitted after 5 reads with skill active"
assert_grep "$ROOT/journal.jsonl" '"active_skills":\["tdd"\]' "silent_drift carries active_skills cluster key"
# --- Test 104: retry_loop fires at threshold 3, not below ---
echo "Test 104: retry_loop boundary (2x no fire, 3x fires)"
reset_state
for i in 1 2; do
echo '{"hook_event_name":"PostToolUse","tool_name":"Bash","tool_input":{"command":"make"},"session_id":"sRT","cwd":"/tmp/x"}' \
| HOOK_RUN >/dev/null 2>&1 || true
done
assert_no_grep "$ROOT/journal.jsonl" '"type":"retry_loop"' "2x same args does NOT emit retry_loop"
echo '{"hook_event_name":"PostToolUse","tool_name":"Bash","tool_input":{"command":"make"},"session_id":"sRT","cwd":"/tmp/x"}' \
| HOOK_RUN >/dev/null 2>&1 || true
assert_grep "$ROOT/journal.jsonl" '"type":"retry_loop"' "3x same args emits retry_loop"
# --- Test 105: weak_agent fires at 2 dispatches, not at 1 ---
echo "Test 105: weak_agent boundary (1x no fire, 2x fires)"
reset_state
echo '{"hook_event_name":"PostToolUse","tool_name":"Agent","tool_input":{"subagent_type":"explorer"},"session_id":"sWA","cwd":"/tmp/x"}' \
| HOOK_RUN >/dev/null 2>&1 || true
assert_no_grep "$ROOT/journal.jsonl" '"type":"weak_agent"' "1x agent dispatch does NOT emit weak_agent"
echo '{"hook_event_name":"PostToolUse","tool_name":"Agent","tool_input":{"subagent_type":"explorer"},"session_id":"sWA","cwd":"/tmp/x"}' \
| HOOK_RUN >/dev/null 2>&1 || true
assert_grep "$ROOT/journal.jsonl" '"type":"weak_agent"' "2x same agent in window emits weak_agent"
# --- Test 106: adam-cooldown --compute deterministic + input-sensitive ---
echo "Test 106: adam-cooldown --compute fingerprint"
fp1=$(printf 'add section X' | COOLDOWN_RUN --compute --skill foo --cluster k1 2>/dev/null)
fp2=$(printf 'add section X' | COOLDOWN_RUN --compute --skill foo --cluster k1 2>/dev/null)
fp3=$(printf 'add section X' | COOLDOWN_RUN --compute --skill foo --cluster k2 2>/dev/null)
if [ -n "$fp1" ] && [ "$fp1" = "$fp2" ] && echo "$fp1" | grep -q '"fingerprint":'; then
echo " PASS: --compute deterministic for identical inputs"; PASS=$((PASS+1))
else
echo " FAIL: --compute not deterministic (got '$fp1' vs '$fp2')"; FAIL=$((FAIL+1))
fi
if [ "$fp1" != "$fp3" ]; then
echo " PASS: --compute sensitive to cluster id"; PASS=$((PASS+1))
else
echo " FAIL: --compute ignored cluster id (both '$fp1')"; FAIL=$((FAIL+1))
fi
# --- Test 107: A/B boundary — exactly -25% delta → improved ---
echo "Test 107: A/B exact -25% boundary (4 pre / 3 post → improved)"
reset_state
applied_at_ms=$(node -e 'console.log(Date.now() - 14*86400000)')
cat > "$ROOT/ab-tracking.jsonl" <<EOF
{"applied_at":$applied_at_ms,"proposal_id":"ab-b25-001","proposal_type":"memory","target_skill":"b1","proposal_fingerprint":"fpB1","originating_signals":[{"type":"correction","count":4,"session_ids":["sB1"]}],"pre_window_days":7}
EOF
> "$ROOT/journal.jsonl"
for i in 1 2 3 4; do
pre_ts=$(node -e "console.log(new Date(Date.now() - (15 + $i*0.3) * 86400000).toISOString())")
echo "{\"ts\":\"$pre_ts\",\"session\":\"sB1\",\"type\":\"correction\",\"phrase\":\"x\"}" >> "$ROOT/journal.jsonl"
done
for i in 1 2 3; do
post_ts=$(node -e "console.log(new Date(Date.now() - (8 + $i*0.3) * 86400000).toISOString())")
echo "{\"ts\":\"$post_ts\",\"session\":\"sB1\",\"type\":\"correction\",\"phrase\":\"y\"}" >> "$ROOT/journal.jsonl"
done
out=$(ABMEASURE_RUN --format json 2>/dev/null)
if echo "$out" | node -e 'let b="";process.stdin.on("data",d=>b+=d).on("end",()=>{const a=JSON.parse(b);const e=a.find(x=>x.proposal_id==="ab-b25-001");process.exit(e&&e.pre_count===4&&e.post_count===3&&e.delta_pct===-25&&e.status==="improved"?0:1)})'; then
echo " PASS: -25% boundary classified improved"; PASS=$((PASS+1))
else
echo " FAIL: -25% boundary misclassified (got: $out)"; FAIL=$((FAIL+1))
fi
rm -f "$ROOT/ab-tracking.jsonl"
# --- Test 108: A/B boundary — exactly +25% delta → regressed ---
echo "Test 108: A/B exact +25% boundary (4 pre / 5 post → regressed)"
reset_state
applied_at_ms=$(node -e 'console.log(Date.now() - 14*86400000)')
cat > "$ROOT/ab-tracking.jsonl" <<EOF
{"applied_at":$applied_at_ms,"proposal_id":"ab-b25-002","proposal_type":"memory","target_skill":"b2","proposal_fingerprint":"fpB2","originating_signals":[{"type":"correction","count":4,"session_ids":["sB2"]}],"pre_window_days":7}
EOF
> "$ROOT/journal.jsonl"
for i in 1 2 3 4; do
pre_ts=$(node -e "console.log(new Date(Date.now() - (15 + $i*0.3) * 86400000).toISOString())")
echo "{\"ts\":\"$pre_ts\",\"session\":\"sB2\",\"type\":\"correction\",\"phrase\":\"x\"}" >> "$ROOT/journal.jsonl"
done
for i in 1 2 3 4 5; do
post_ts=$(node -e "console.log(new Date(Date.now() - (8 + $i*0.3) * 86400000).toISOString())")
echo "{\"ts\":\"$post_ts\",\"session\":\"sB2\",\"type\":\"correction\",\"phrase\":\"y\"}" >> "$ROOT/journal.jsonl"
done
out=$(ABMEASURE_RUN --format json 2>/dev/null)
if echo "$out" | node -e 'let b="";process.stdin.on("data",d=>b+=d).on("end",()=>{const a=JSON.parse(b);const e=a.find(x=>x.proposal_id==="ab-b25-002");process.exit(e&&e.pre_count===4&&e.post_count===5&&e.delta_pct===25&&e.status==="regressed"?0:1)})'; then
echo " PASS: +25% boundary classified regressed"; PASS=$((PASS+1))
else
echo " FAIL: +25% boundary misclassified (got: $out)"; FAIL=$((FAIL+1))
fi
rm -f "$ROOT/ab-tracking.jsonl"
# --- Test 109: cooldown blacklist 30d boundary (day 29 active, day 31 expired) ---
echo "Test 109: blacklist 30d boundary"
reset_state
ts29=$(node -e 'console.log(Date.now() - 29*86400000)')
cat > "$ROOT/rejected/2026-blk-29.md" <<EOF
---
id: blk-29
type: skill_edit
target_skill: blkskill
proposal_fingerprint: fpZ
auto_apply_blacklist: true
applied_at: $ts29
---
body
EOF
out29=$(COOLDOWN_RUN --skill blkskill --fingerprint fpZ 2>/dev/null)
if echo "$out29" | grep -q '"status":"blacklisted"'; then
echo " PASS: day-29 blacklist still active"; PASS=$((PASS+1))
else
echo " FAIL: day-29 should be blacklisted (got: $out29)"; FAIL=$((FAIL+1))
fi
rm -f "$ROOT/rejected/2026-blk-29.md"
ts31=$(node -e 'console.log(Date.now() - 31*86400000)')
cat > "$ROOT/rejected/2026-blk-31.md" <<EOF
---
id: blk-31
type: skill_edit
target_skill: blkskill
proposal_fingerprint: fpZ
auto_apply_blacklist: true
applied_at: $ts31
---
body
EOF
out31=$(COOLDOWN_RUN --skill blkskill --fingerprint fpZ 2>/dev/null)
if echo "$out31" | grep -q '"status":"cool"'; then
echo " PASS: day-31 blacklist expired → cool"; PASS=$((PASS+1))
else
echo " FAIL: day-31 should be cool (got: $out31)"; FAIL=$((FAIL+1))
fi
rm -f "$ROOT/rejected/2026-blk-31.md"
echo
echo "Results: $PASS passed, $FAIL failed"
[ "$FAIL" = "0" ]