mirror of
https://github.com/lukaszraczylo/claude-adam.git
synced 2026-07-11 04:01:46 +00:00
fix(v0.6.2): A/B volume normalization + memory frontmatter schema
Two issues surfaced by running ADAM's /reflect loop on a large real journal
(4015 entries, 119 sessions) — both caused false/broken auto-apply behavior.
1. A/B over-reported regressions (adam-ab-measure.mjs).
Regressions were measured on RAW originating-signal counts pre vs post. On a
busy, growing journal almost every signal count rises post-apply regardless
of whether the proposal helped — so the loop flagged 9 false "regressions"
(and would auto-roll-back good proposals). Now the delta is computed on the
signal's SHARE of total activity (rate = count / window-total). Falls back to
the raw-count delta when the signal is the only activity in the window
(preserves prior behavior + all existing A/B tests). Output adds
raw_delta_pct, pre_total, post_total, normalized for transparency.
2. Memory frontmatter drift (agents/adam.md, SKILL.md).
The drafting protocol emitted flat `type:`/`originSessionId:` with a prose
`name`, but the live auto-memory store uses `name` = slug plus a
`metadata: {node_type, type, originSessionId}` block. Auto-applied memories
could fail to load/categorize. Protocol + apply-time validation now require
the live metadata.* schema and cross-checking against an existing file.
Tests: 132 -> 134. New: volume growth (raw +200%) with flat activity-share
classifies neutral, not regressed; a genuine share increase still classifies
regressed.
This commit is contained in:
@@ -2014,6 +2014,50 @@ done
|
||||
assert_grep "$ROOT/journal.jsonl" '"type":"retry_loop"' "3x byte-identical reads emit retry_loop"
|
||||
assert_no_grep "$ROOT/journal.jsonl" '"type":"file_reread"' "byte-identical reads NOT double-counted as file_reread (sameToolArgs>=RETRY guard)"
|
||||
|
||||
# --- Test 112: A/B volume normalization — busier journal does NOT fake a regression ---
|
||||
echo "Test 112: A/B volume-normalized (raw +200% but flat share → neutral)"
|
||||
reset_state
|
||||
applied_at_ms=$(node -e 'console.log(Date.now() - 14*86400000)')
|
||||
cat > "$ROOT/ab-tracking.jsonl" <<EOF
|
||||
{"applied_at":$applied_at_ms,"proposal_id":"ab-vol-001","proposal_type":"memory","target_skill":"vol","proposal_fingerprint":"fpV","originating_signals":[{"type":"correction","count":2,"session_ids":["sV"]}],"pre_window_days":7}
|
||||
EOF
|
||||
> "$ROOT/journal.jsonl"
|
||||
# pre window: 2 correction + 8 dead_end (rate 0.2)
|
||||
for i in 1 2; do ts=$(node -e "console.log(new Date(Date.now()-(15+$i*0.2)*86400000).toISOString())"); echo "{\"ts\":\"$ts\",\"session\":\"sV\",\"type\":\"correction\"}" >> "$ROOT/journal.jsonl"; done
|
||||
for i in 1 2 3 4 5 6 7 8; do ts=$(node -e "console.log(new Date(Date.now()-(15+$i*0.1)*86400000).toISOString())"); echo "{\"ts\":\"$ts\",\"session\":\"sV\",\"type\":\"dead_end\"}" >> "$ROOT/journal.jsonl"; done
|
||||
# post window: 6 correction + 24 dead_end (rate 0.2 — share unchanged, raw count +200%)
|
||||
for i in $(seq 1 6); do ts=$(node -e "console.log(new Date(Date.now()-(8+$i*0.1)*86400000).toISOString())"); echo "{\"ts\":\"$ts\",\"session\":\"sV\",\"type\":\"correction\"}" >> "$ROOT/journal.jsonl"; done
|
||||
for i in $(seq 1 24); do ts=$(node -e "console.log(new Date(Date.now()-(8+$i*0.05)*86400000).toISOString())"); echo "{\"ts\":\"$ts\",\"session\":\"sV\",\"type\":\"dead_end\"}" >> "$ROOT/journal.jsonl"; done
|
||||
out=$(ABMEASURE_RUN --format json 2>/dev/null)
|
||||
if echo "$out" | node -e 'let b="";process.stdin.on("data",d=>b+=d).on("end",()=>{const a=JSON.parse(b);const e=a.find(x=>x.proposal_id==="ab-vol-001");process.exit(e&&e.normalized===true&&e.raw_delta_pct===200&&e.status==="neutral"?0:1)})'; then
|
||||
echo " PASS: volume growth normalized → neutral (raw +200%)"; PASS=$((PASS+1))
|
||||
else
|
||||
echo " FAIL: volume normalization wrong (got: $out)"; FAIL=$((FAIL+1))
|
||||
fi
|
||||
rm -f "$ROOT/ab-tracking.jsonl"
|
||||
|
||||
# --- Test 113: A/B genuine rate regression still flagged ---
|
||||
echo "Test 113: A/B genuine share increase → regressed"
|
||||
reset_state
|
||||
applied_at_ms=$(node -e 'console.log(Date.now() - 14*86400000)')
|
||||
cat > "$ROOT/ab-tracking.jsonl" <<EOF
|
||||
{"applied_at":$applied_at_ms,"proposal_id":"ab-vol-002","proposal_type":"memory","target_skill":"vol2","proposal_fingerprint":"fpV2","originating_signals":[{"type":"correction","count":2,"session_ids":["sV2"]}],"pre_window_days":7}
|
||||
EOF
|
||||
> "$ROOT/journal.jsonl"
|
||||
# pre: 2 correction + 8 dead_end (rate 0.2)
|
||||
for i in 1 2; do ts=$(node -e "console.log(new Date(Date.now()-(15+$i*0.2)*86400000).toISOString())"); echo "{\"ts\":\"$ts\",\"session\":\"sV2\",\"type\":\"correction\"}" >> "$ROOT/journal.jsonl"; done
|
||||
for i in 1 2 3 4 5 6 7 8; do ts=$(node -e "console.log(new Date(Date.now()-(15+$i*0.1)*86400000).toISOString())"); echo "{\"ts\":\"$ts\",\"session\":\"sV2\",\"type\":\"dead_end\"}" >> "$ROOT/journal.jsonl"; done
|
||||
# post: 6 correction + 6 dead_end (rate 0.5 — share up → genuine regression)
|
||||
for i in $(seq 1 6); do ts=$(node -e "console.log(new Date(Date.now()-(8+$i*0.1)*86400000).toISOString())"); echo "{\"ts\":\"$ts\",\"session\":\"sV2\",\"type\":\"correction\"}" >> "$ROOT/journal.jsonl"; done
|
||||
for i in 1 2 3 4 5 6; do ts=$(node -e "console.log(new Date(Date.now()-(8+$i*0.07)*86400000).toISOString())"); echo "{\"ts\":\"$ts\",\"session\":\"sV2\",\"type\":\"dead_end\"}" >> "$ROOT/journal.jsonl"; done
|
||||
out=$(ABMEASURE_RUN --format json 2>/dev/null)
|
||||
if echo "$out" | node -e 'let b="";process.stdin.on("data",d=>b+=d).on("end",()=>{const a=JSON.parse(b);const e=a.find(x=>x.proposal_id==="ab-vol-002");process.exit(e&&e.normalized===true&&e.status==="regressed"?0:1)})'; then
|
||||
echo " PASS: genuine share increase → regressed"; PASS=$((PASS+1))
|
||||
else
|
||||
echo " FAIL: genuine regression missed (got: $out)"; FAIL=$((FAIL+1))
|
||||
fi
|
||||
rm -f "$ROOT/ab-tracking.jsonl"
|
||||
|
||||
echo
|
||||
echo "Results: $PASS passed, $FAIL failed"
|
||||
[ "$FAIL" = "0" ]
|
||||
|
||||
Reference in New Issue
Block a user