mirror of
https://github.com/lukaszraczylo/claude-adam.git
synced 2026-07-07 03:44:26 +00:00
feat(v0.6.5): execution-grounded skill-utility report (adam-skill-utility)
Ranks skills by good:bad outcome co-occurrence (Wilson LB + lift vs baseline) over the journal's active_skills payloads — the SkillsInjector (arXiv 2605.29794) execution-grounded utility signal Δ(s), computed from data already collected, no training. - reuses adam-score NEGATIVE_SIGNAL_TYPES + entrySeverity (single source of truth) - registered in install.sh helper-script copy loop - /reflect pre-step surfaces worst below-baseline skills to the USER as advisory (co-occurrence != causation; not fed to the analyst's proposal machinery) - Test 119 added; full suite 141/141 green
This commit is contained in:
@@ -18,6 +18,7 @@ APPLYREIN="$REAL_HOME/.claude/adam/scripts/adam-apply-reinforcement.mjs"
|
||||
UPGRADE="$REAL_HOME/.claude/adam/scripts/adam-upgrade.mjs"
|
||||
BATCH="$REAL_HOME/.claude/adam/scripts/adam-batch.mjs"
|
||||
ROLLBACK="$REAL_HOME/.claude/adam/scripts/adam-rollback.mjs"
|
||||
SKILLUTIL="$REAL_HOME/.claude/adam/scripts/adam-skill-utility.mjs"
|
||||
|
||||
TMP_HOME="$(mktemp -d -t adam-test.XXXXXX)"
|
||||
trap 'rm -rf "$TMP_HOME"' EXIT INT TERM
|
||||
@@ -37,6 +38,7 @@ APPLYREIN_RUN(){ HOME="$TMP_HOME" node "$APPLYREIN" "$@" --home "$TMP_HOME/.clau
|
||||
UPGRADE_RUN() { HOME="$TMP_HOME" node "$UPGRADE" "$@"; }
|
||||
BATCH_RUN() { HOME="$TMP_HOME" node "$BATCH" "$@"; }
|
||||
ROLLBACK_RUN(){ HOME="$TMP_HOME" node "$ROLLBACK" "$@"; }
|
||||
SKILLUTIL_RUN(){ HOME="$TMP_HOME" node "$SKILLUTIL" "$@"; }
|
||||
|
||||
PASS=0
|
||||
FAIL=0
|
||||
@@ -2148,6 +2150,32 @@ else
|
||||
fi
|
||||
rm -f "$ROOT/proposals/"*rb-ab-001* "$ROOT/applied/"*rb-ab-001* "$ROOT/ab-tracking.jsonl" "$ROOT/active-nudges.json"
|
||||
|
||||
# --- Test 119: adam-skill-utility ranks friction-correlated skills below baseline ---
|
||||
echo "Test 119: adam-skill-utility computes per-skill good:bad utility (execution-grounded Δ)"
|
||||
reset_state
|
||||
SU_INPUT="$TMP_HOME/su-input.jsonl"
|
||||
{
|
||||
for i in 1 2 3 4 5; do echo "{\"ts\":\"2026-05-20T0$i:00:00Z\",\"session\":\"sSU\",\"type\":\"task_completed\",\"active_skills\":[\"goodskill\"]}"; done
|
||||
for i in 1 2 3 4 5; do echo "{\"ts\":\"2026-05-20T1$i:00:00Z\",\"session\":\"sSU\",\"type\":\"dead_end\",\"count\":8,\"active_skills\":[\"badskill\"]}"; done
|
||||
} > "$SU_INPUT"
|
||||
su_out=$(SKILLUTIL_RUN --input "$SU_INPUT" --json --min 3 2>/dev/null)
|
||||
su_check=$(echo "$su_out" | node -e '
|
||||
let buf=""; process.stdin.on("data",d=>buf+=d).on("end",()=>{
|
||||
try {
|
||||
const p=JSON.parse(buf);
|
||||
const bad=p.skills.find(s=>s.skill==="badskill");
|
||||
const good=p.skills.find(s=>s.skill==="goodskill");
|
||||
const ok = bad && good && bad.lift<0 && good.lift>0 && p.skills[0].skill==="badskill" && bad.neg===5 && good.pos===5;
|
||||
console.log(ok?"ok":"bad:"+JSON.stringify({bad,good,first:p.skills[0]&&p.skills[0].skill}));
|
||||
} catch(e){ console.log("parse-error:"+e.message); }
|
||||
});')
|
||||
if [ "$su_check" = "ok" ]; then
|
||||
echo " PASS: badskill below baseline + ranked worst-first, goodskill above"; PASS=$((PASS+1))
|
||||
else
|
||||
echo " FAIL: skill-utility ranking wrong ($su_check)"; FAIL=$((FAIL+1))
|
||||
fi
|
||||
rm -f "$SU_INPUT"
|
||||
|
||||
echo
|
||||
echo "Results: $PASS passed, $FAIL failed"
|
||||
[ "$FAIL" = "0" ]
|
||||
|
||||
Reference in New Issue
Block a user