mirror of
https://github.com/lukaszraczylo/claude-adam.git
synced 2026-07-06 03:37:48 +00:00
feat: apply MOSS-grounded self-evolution improvements to ADAM
Implements 7 improvements grounded in MOSS paper (arXiv 2605.22794): 1. Transcript capture (§3.4): context_ring buffer in adam-observe.mjs captures last 8 events around struggle signals as context_window. 2. Evidence batching (§3.1): new adam-batch.mjs pre-clusters windowed journal entries into coherent failure batches by (signal_type, cluster_key). 3. Multi-stage analysis (§3.3): SKILL.md dispatches adam agent in two stages (diagnose+plan → implement) with inter-stage validation gate. 4. Pre-apply verification (§3.4): 4-check deterministic gate before auto-apply (source entries exist, diagnosis grounded, type-evidence match, no conflicting recent proposals). 5. Auto-rollback (§3.5): new adam-rollback.mjs reverts regressed proposals detected by A/B measurement, creates regression nudges. 6. Harness self-modification (§1 Table 1): new harness_edit proposal type targeting adam's own scripts with stricter gates (confidence≥5, never auto-apply, test-suite-gated). 7. Keypoint matrix evaluation (§4.2): 5 capability dimensions (tool_selection, scope_discipline, error_recovery, first_attempt, build_reliability) scored per batch for structured evaluation. Test suite: 94 → 114 tests (20 new), all passing.
This commit is contained in:
Executable
+184
@@ -0,0 +1,184 @@
|
||||
#!/usr/bin/env node
|
||||
// adam-batch.mjs — pre-clusters windowed journal entries into coherent failure
|
||||
// batches before analyst dispatch. Implements MOSS §3.1: "anchored to an
|
||||
// automatically curated batch of production-failure evidence."
|
||||
//
|
||||
// Each batch groups entries by (signal_type, cluster_key) where cluster_key
|
||||
// follows the same clustering rules as agents/adam.md §4:
|
||||
// correction → tokenized phrase (cross-cwd)
|
||||
// retry_loop → tool
|
||||
// weak_agent → subagent_type
|
||||
// tool_error_loop→ fp
|
||||
// dead_end → session
|
||||
// edit_churn → file basename
|
||||
// build_loop → session
|
||||
// subagent_dispatch_pattern → subagent_type
|
||||
// silent_drift → active_skills[0]
|
||||
// error_after_recovery → (recovered_from, original_fp)
|
||||
// correction_free_streak → active_skills[0]
|
||||
// clean_recovery → (recovered_from, active_skills[0])
|
||||
// task_completed → sorted tool_kinds tuple
|
||||
//
|
||||
// CLI:
|
||||
// adam-batch.mjs [--input <jsonl-path>] [--min-entries N] [--min-sessions N]
|
||||
//
|
||||
// Output: JSON object with `batches` array and `unbatched` count.
|
||||
|
||||
import { readFileSync } from "node:fs";
|
||||
import { readJsonlSafe } from "./adam-utils.mjs";
|
||||
|
||||
const DEFAULT_MIN_ENTRIES = 1;
|
||||
const DEFAULT_MIN_SESSIONS = 1;
|
||||
|
||||
const CORRECTION_STOPWORDS = new Set([
|
||||
"the", "a", "an", "and", "or", "but", "of", "to", "for", "in", "on",
|
||||
"with", "use", "when", "where", "what", "why", "how", "this", "that",
|
||||
"these", "those", "is", "are", "was", "were", "be", "been", "being",
|
||||
"do", "does", "did", "doing", "has", "have", "had", "your", "you",
|
||||
"i", "it", "as", "at", "by", "from", "not", "no",
|
||||
]);
|
||||
|
||||
function tokenizePhrase(phrase) {
|
||||
if (!phrase || typeof phrase !== "string") return "";
|
||||
return phrase.toLowerCase()
|
||||
.split(/\s+/)
|
||||
.map(t => t.replace(/^[^\w']+|[^\w']+$/g, ""))
|
||||
.filter(t => t && !CORRECTION_STOPWORDS.has(t))
|
||||
.sort()
|
||||
.join("|");
|
||||
}
|
||||
|
||||
function clusterKey(entry) {
|
||||
if (!entry || typeof entry !== "object") return null;
|
||||
const t = entry.type;
|
||||
switch (t) {
|
||||
case "correction":
|
||||
return tokenizePhrase(entry.phrase) || "unknown";
|
||||
case "retry_loop":
|
||||
return entry.tool || "unknown";
|
||||
case "weak_agent":
|
||||
case "subagent_dispatch_pattern":
|
||||
return entry.subagent_type || "unknown";
|
||||
case "tool_error_loop":
|
||||
return entry.fp || "unknown";
|
||||
case "dead_end":
|
||||
case "build_loop":
|
||||
return entry.session || "unknown";
|
||||
case "edit_churn":
|
||||
return entry.file ? entry.file.split("/").pop() : "unknown";
|
||||
case "silent_drift":
|
||||
case "correction_free_streak":
|
||||
return Array.isArray(entry.active_skills) ? (entry.active_skills[0] || "") : "";
|
||||
case "error_after_recovery":
|
||||
return `${entry.recovered_from || "?"}:${entry.original_fp || "?"}`;
|
||||
case "clean_recovery":
|
||||
return `${entry.recovered_from || "?"}:${Array.isArray(entry.active_skills) ? (entry.active_skills[0] || "") : ""}`;
|
||||
case "task_completed":
|
||||
return Array.isArray(entry.tool_kinds) ? entry.tool_kinds.slice().sort().join(",") : "unknown";
|
||||
default:
|
||||
return entry.session || "unknown";
|
||||
}
|
||||
}
|
||||
|
||||
function parseArgs(argv) {
|
||||
const args = { input: null, minEntries: DEFAULT_MIN_ENTRIES, minSessions: DEFAULT_MIN_SESSIONS, help: false };
|
||||
for (let i = 0; i < argv.length; i++) {
|
||||
const a = argv[i];
|
||||
if (a === "--input" && i + 1 < argv.length) args.input = argv[++i];
|
||||
else if (a === "--min-entries" && i + 1 < argv.length) {
|
||||
const n = Number(argv[++i]);
|
||||
if (!Number.isNaN(n) && n > 0) args.minEntries = n;
|
||||
}
|
||||
else if (a === "--min-sessions" && i + 1 < argv.length) {
|
||||
const n = Number(argv[++i]);
|
||||
if (!Number.isNaN(n) && n > 0) args.minSessions = n;
|
||||
}
|
||||
else if (a === "--help" || a === "-h") args.help = true;
|
||||
}
|
||||
return args;
|
||||
}
|
||||
|
||||
export function buildBatches(entries, opts = {}) {
|
||||
const minEntries = opts.minEntries || DEFAULT_MIN_ENTRIES;
|
||||
const minSessions = opts.minSessions || DEFAULT_MIN_SESSIONS;
|
||||
const map = new Map();
|
||||
|
||||
for (const e of entries || []) {
|
||||
if (!e || typeof e !== "object" || !e.type) continue;
|
||||
const key = `${e.type}::${clusterKey(e)}`;
|
||||
if (!map.has(key)) {
|
||||
map.set(key, {
|
||||
batch_id: null,
|
||||
signal_type: e.type,
|
||||
cluster_key: clusterKey(e),
|
||||
entries: [],
|
||||
sessions: new Set(),
|
||||
cwds: new Set(),
|
||||
});
|
||||
}
|
||||
const batch = map.get(key);
|
||||
batch.entries.push(e);
|
||||
if (e.session) batch.sessions.add(e.session);
|
||||
if (e.cwd) batch.cwds.add(e.cwd);
|
||||
}
|
||||
|
||||
const batches = [];
|
||||
let unbatched = 0;
|
||||
let id = 1;
|
||||
for (const [, batch] of map) {
|
||||
if (batch.entries.length < minEntries || batch.sessions.size < minSessions) {
|
||||
unbatched += batch.entries.length;
|
||||
continue;
|
||||
}
|
||||
batch.batch_id = `b${id++}`;
|
||||
batches.push({
|
||||
batch_id: batch.batch_id,
|
||||
signal_type: batch.signal_type,
|
||||
cluster_key: batch.cluster_key,
|
||||
entry_count: batch.entries.length,
|
||||
session_count: batch.sessions.size,
|
||||
cwd_count: batch.cwds.size,
|
||||
has_context_window: batch.entries.some(e => Array.isArray(e.context_window) && e.context_window.length > 0),
|
||||
entries: batch.entries,
|
||||
});
|
||||
}
|
||||
|
||||
batches.sort((a, b) => b.entry_count - a.entry_count);
|
||||
return { batches, unbatched, total: (entries || []).length };
|
||||
}
|
||||
|
||||
function main() {
|
||||
const args = parseArgs(process.argv.slice(2));
|
||||
if (args.help) {
|
||||
process.stdout.write("usage: adam-batch.mjs [--input <jsonl-path>] [--min-entries N] [--min-sessions N]\n");
|
||||
process.exit(0);
|
||||
}
|
||||
try {
|
||||
let entries;
|
||||
if (args.input) {
|
||||
entries = readJsonlSafe(args.input);
|
||||
} else if (!process.stdin.isTTY) {
|
||||
const buf = readFileSync(0, "utf8");
|
||||
entries = [];
|
||||
for (const line of buf.split("\n")) {
|
||||
if (!line) continue;
|
||||
try { entries.push(JSON.parse(line)); } catch { /* skip */ }
|
||||
}
|
||||
} else {
|
||||
process.stderr.write("adam-batch: no input (use --input or pipe)\n");
|
||||
process.exit(1);
|
||||
}
|
||||
const result = buildBatches(entries, { minEntries: args.minEntries, minSessions: args.minSessions });
|
||||
process.stdout.write(JSON.stringify(result) + "\n");
|
||||
process.exit(0);
|
||||
} catch (e) {
|
||||
process.stderr.write(`adam-batch error: ${e.message}\n`);
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
if (import.meta.url === `file://${process.argv[1]}`) {
|
||||
main();
|
||||
}
|
||||
|
||||
export { clusterKey, tokenizePhrase };
|
||||
Executable
+225
@@ -0,0 +1,225 @@
|
||||
#!/usr/bin/env node
|
||||
// adam-rollback.mjs — auto-reverts proposals that regressed after apply.
|
||||
//
|
||||
// Implements MOSS §3.5: "rollback is mandatory because... a candidate that
|
||||
// passes trial can still regress live."
|
||||
//
|
||||
// For each regressed proposal (detected by adam-ab-measure.mjs):
|
||||
// 1. Reads the applied proposal from applied/
|
||||
// 2. Parses the `# Rollback` section for undo commands
|
||||
// 3. Moves proposal from applied/ to proposals/ with `rolled_back: true`
|
||||
// 4. Creates a regression nudge for next SessionStart
|
||||
// 5. Removes the ab-tracking entry (so it doesn't re-trigger)
|
||||
//
|
||||
// CLI:
|
||||
// adam-rollback.mjs --proposal-id <id> [--home <path>] [--dry-run]
|
||||
// adam-rollback.mjs --auto [--home <path>] [--dry-run]
|
||||
//
|
||||
// --auto mode: reads ab-measure output, rolls back all regressed proposals.
|
||||
//
|
||||
// Output: JSON object with rollback results per proposal.
|
||||
// Does NOT execute the undo commands itself — outputs them for the skill to
|
||||
// execute in-context (safety: undo commands may reference files the script
|
||||
// can't safely modify).
|
||||
|
||||
import { readFileSync, writeFileSync, renameSync, readdirSync, existsSync, mkdirSync } from "node:fs";
|
||||
import { join, basename } from "node:path";
|
||||
import { homedir } from "node:os";
|
||||
import { parseFrontmatter, readJsonlSafe, listJsonlFiles } from "./adam-utils.mjs";
|
||||
|
||||
function parseArgs(argv) {
|
||||
const args = { home: null, proposalId: null, auto: false, dryRun: false, help: false };
|
||||
for (let i = 0; i < argv.length; i++) {
|
||||
const a = argv[i];
|
||||
if (a === "--home" && i + 1 < argv.length) args.home = argv[++i];
|
||||
else if (a === "--proposal-id" && i + 1 < argv.length) args.proposalId = argv[++i];
|
||||
else if (a === "--auto") args.auto = true;
|
||||
else if (a === "--dry-run") args.dryRun = true;
|
||||
else if (a === "--help" || a === "-h") args.help = true;
|
||||
}
|
||||
return args;
|
||||
}
|
||||
|
||||
function findAppliedProposal(appliedDir, proposalId) {
|
||||
if (!existsSync(appliedDir)) return null;
|
||||
try {
|
||||
const files = readdirSync(appliedDir).filter(n => n.endsWith(".md"));
|
||||
for (const f of files) {
|
||||
if (f.includes(proposalId)) return join(appliedDir, f);
|
||||
}
|
||||
} catch { /* skip */ }
|
||||
return null;
|
||||
}
|
||||
|
||||
function extractRollbackSection(content) {
|
||||
const idx = content.indexOf("\n# Rollback\n");
|
||||
if (idx === -1) return null;
|
||||
let body = content.slice(idx + "\n# Rollback\n".length);
|
||||
const nextSection = body.search(/\n# |\n---/);
|
||||
if (nextSection !== -1) body = body.slice(0, nextSection);
|
||||
return body.trim() || null;
|
||||
}
|
||||
|
||||
function extractUndoCommands(rollbackSection) {
|
||||
if (!rollbackSection) return [];
|
||||
const commands = [];
|
||||
const lines = rollbackSection.split("\n");
|
||||
let inCodeBlock = false;
|
||||
let blockLines = [];
|
||||
for (const line of lines) {
|
||||
if (line.startsWith("```")) {
|
||||
if (inCodeBlock) {
|
||||
if (blockLines.length) commands.push(blockLines.join("\n"));
|
||||
blockLines = [];
|
||||
}
|
||||
inCodeBlock = !inCodeBlock;
|
||||
continue;
|
||||
}
|
||||
if (inCodeBlock) {
|
||||
blockLines.push(line);
|
||||
}
|
||||
}
|
||||
return commands;
|
||||
}
|
||||
|
||||
export function planRollback(appliedDir, proposalId) {
|
||||
const path = findAppliedProposal(appliedDir, proposalId);
|
||||
if (!path) return { status: "not_found", proposal_id: proposalId };
|
||||
|
||||
const content = readFileSync(path, "utf8");
|
||||
const fm = parseFrontmatter(content);
|
||||
const rollbackSection = extractRollbackSection(content);
|
||||
const undoCommands = extractUndoCommands(rollbackSection);
|
||||
|
||||
return {
|
||||
status: "planned",
|
||||
proposal_id: proposalId,
|
||||
applied_path: path,
|
||||
type: fm.type || "unknown",
|
||||
target: fm.target || null,
|
||||
target_skill: fm.target_skill || null,
|
||||
undo_commands: undoCommands,
|
||||
has_rollback_section: !!rollbackSection,
|
||||
};
|
||||
}
|
||||
|
||||
export function executeRollback(plan, adamRoot, opts = {}) {
|
||||
const dryRun = opts.dryRun || false;
|
||||
const proposalsDir = join(adamRoot, "proposals");
|
||||
const nudgesPath = join(adamRoot, "active-nudges.json");
|
||||
const now = Date.now();
|
||||
|
||||
if (plan.status !== "planned") return { ...plan, action: "skipped" };
|
||||
|
||||
const result = {
|
||||
proposal_id: plan.proposal_id,
|
||||
type: plan.type,
|
||||
target: plan.target,
|
||||
undo_commands: plan.undo_commands,
|
||||
actions: [],
|
||||
};
|
||||
|
||||
if (dryRun) {
|
||||
result.actions.push("dry_run: would move applied → proposals");
|
||||
if (plan.undo_commands.length) {
|
||||
result.actions.push(`dry_run: would output ${plan.undo_commands.length} undo command(s)`);
|
||||
}
|
||||
result.actions.push("dry_run: would create regression nudge");
|
||||
result.status = "dry_run";
|
||||
return result;
|
||||
}
|
||||
|
||||
mkdirSync(proposalsDir, { recursive: true });
|
||||
const destName = `${basename(plan.applied_path).replace(/\.md$/, "")}-rollback.md`;
|
||||
const destPath = join(proposalsDir, destName);
|
||||
|
||||
let content = readFileSync(plan.applied_path, "utf8");
|
||||
const rollbackMeta = `\nrolled_back: true\nrolled_back_at: "${new Date(now).toISOString()}"`;
|
||||
content = content.replace(/^(---\n[\s\S]*?)(---)/m, `$1${rollbackMeta}\n$2`);
|
||||
|
||||
try {
|
||||
writeFileSync(destPath, content);
|
||||
renameSync(plan.applied_path, plan.applied_path + ".rolled-back");
|
||||
result.actions.push(`moved ${plan.applied_path} → ${destPath}`);
|
||||
} catch (e) {
|
||||
result.status = "move_failed";
|
||||
result.error = e.message;
|
||||
return result;
|
||||
}
|
||||
|
||||
try {
|
||||
let nudges = [];
|
||||
if (existsSync(nudgesPath)) {
|
||||
try { nudges = JSON.parse(readFileSync(nudgesPath, "utf8")); } catch { nudges = []; }
|
||||
}
|
||||
nudges.push({
|
||||
kind: "regression_rollback",
|
||||
message: `adam: rolled back "${plan.proposal_id}" (type: ${plan.type}) — regression detected in A/B measurement. Review with /reflect.`,
|
||||
created_at: now,
|
||||
expires_at_ts: now + 7 * 86400000,
|
||||
max_displays: 3,
|
||||
displays_used: 0,
|
||||
source_proposal: plan.proposal_id,
|
||||
});
|
||||
writeFileSync(nudgesPath, JSON.stringify(nudges, null, 2));
|
||||
result.actions.push("regression nudge created");
|
||||
} catch (e) {
|
||||
result.actions.push(`nudge failed: ${e.message}`);
|
||||
}
|
||||
|
||||
result.status = "rolled_back";
|
||||
return result;
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const args = parseArgs(process.argv.slice(2));
|
||||
if (args.help) {
|
||||
process.stdout.write(
|
||||
"usage: adam-rollback.mjs --proposal-id <id> [--home <path>] [--dry-run]\n" +
|
||||
" adam-rollback.mjs --auto [--home <path>] [--dry-run]\n"
|
||||
);
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
const claudeHome = args.home || join(homedir(), ".claude");
|
||||
const adamRoot = join(claudeHome, "adam");
|
||||
const appliedDir = join(adamRoot, "applied");
|
||||
|
||||
try {
|
||||
const results = [];
|
||||
|
||||
if (args.auto) {
|
||||
const abPath = join(adamRoot, "ab-tracking.jsonl");
|
||||
const entries = readJsonlSafe(abPath);
|
||||
const { computeDeltas } = await import("./adam-ab-measure.mjs");
|
||||
const sources = [join(adamRoot, "journal.jsonl"), ...listJsonlFiles(join(adamRoot, "journal"))];
|
||||
const journalAll = [];
|
||||
for (const p of sources) for (const e of readJsonlSafe(p)) journalAll.push(e);
|
||||
const deltas = computeDeltas(entries, journalAll);
|
||||
const regressed = deltas.filter(d => d.status === "regressed");
|
||||
|
||||
for (const d of regressed) {
|
||||
const plan = planRollback(appliedDir, d.proposal_id);
|
||||
const result = executeRollback(plan, adamRoot, { dryRun: args.dryRun });
|
||||
results.push(result);
|
||||
}
|
||||
} else if (args.proposalId) {
|
||||
const plan = planRollback(appliedDir, args.proposalId);
|
||||
const result = executeRollback(plan, adamRoot, { dryRun: args.dryRun });
|
||||
results.push(result);
|
||||
} else {
|
||||
process.stderr.write("adam-rollback: specify --proposal-id or --auto\n");
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
process.stdout.write(JSON.stringify({ rollbacks: results }) + "\n");
|
||||
process.exit(0);
|
||||
} catch (e) {
|
||||
process.stderr.write(`adam-rollback error: ${e.message}\n`);
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
if (import.meta.url === `file://${process.argv[1]}`) {
|
||||
main();
|
||||
}
|
||||
@@ -16,6 +16,8 @@ SCORE="$REAL_HOME/.claude/adam/scripts/adam-score.mjs"
|
||||
ABMEASURE="$REAL_HOME/.claude/adam/scripts/adam-ab-measure.mjs"
|
||||
APPLYREIN="$REAL_HOME/.claude/adam/scripts/adam-apply-reinforcement.mjs"
|
||||
UPGRADE="$REAL_HOME/.claude/adam/scripts/adam-upgrade.mjs"
|
||||
BATCH="$REAL_HOME/.claude/adam/scripts/adam-batch.mjs"
|
||||
ROLLBACK="$REAL_HOME/.claude/adam/scripts/adam-rollback.mjs"
|
||||
|
||||
TMP_HOME="$(mktemp -d -t adam-test.XXXXXX)"
|
||||
trap 'rm -rf "$TMP_HOME"' EXIT INT TERM
|
||||
@@ -33,6 +35,8 @@ SCORE_RUN() { HOME="$TMP_HOME" node "$SCORE" --home "$TMP_HOME/.claude" "$@";
|
||||
ABMEASURE_RUN(){ HOME="$TMP_HOME" node "$ABMEASURE" --home "$TMP_HOME/.claude" "$@"; }
|
||||
APPLYREIN_RUN(){ HOME="$TMP_HOME" node "$APPLYREIN" "$@" --home "$TMP_HOME/.claude"; }
|
||||
UPGRADE_RUN() { HOME="$TMP_HOME" node "$UPGRADE" "$@"; }
|
||||
BATCH_RUN() { HOME="$TMP_HOME" node "$BATCH" "$@"; }
|
||||
ROLLBACK_RUN(){ HOME="$TMP_HOME" node "$ROLLBACK" "$@"; }
|
||||
|
||||
PASS=0
|
||||
FAIL=0
|
||||
@@ -1487,6 +1491,354 @@ else
|
||||
echo " FAIL: severity_by_type.dead_end missing/wrong (got: $out)"; FAIL=$((FAIL+1))
|
||||
fi
|
||||
|
||||
# ============================================================
|
||||
# MOSS-grounded tests: context_window, adam-batch, adam-rollback
|
||||
# ============================================================
|
||||
|
||||
# --- Test 83: context_window attached to tool_error_loop struggle signal ---
|
||||
echo "Test 83: context_window attached to tool_error_loop"
|
||||
reset_state
|
||||
# Fire a user prompt first so the context ring has something.
|
||||
echo '{"hook_event_name":"UserPromptSubmit","prompt":"run the tests","session_id":"sCW1","cwd":"/tmp/x"}' \
|
||||
| HOOK_RUN >/dev/null 2>&1 || true
|
||||
for i in 1 2 3; do
|
||||
echo '{"hook_event_name":"PostToolUse","tool_name":"Bash","tool_input":{"command":"failing-cmd"},"tool_response":{"is_error":true,"content":"Error: command not found: failing-cmd"},"session_id":"sCW1","cwd":"/tmp/x"}' \
|
||||
| HOOK_RUN >/dev/null 2>&1 || true
|
||||
done
|
||||
assert_grep "$ROOT/journal.jsonl" '"context_window":\[' "tool_error_loop carries context_window"
|
||||
|
||||
# --- Test 84: context_window captures preceding user prompt ---
|
||||
echo "Test 84: context_window captures user prompt text"
|
||||
# Re-use the journal from test 83
|
||||
assert_grep "$ROOT/journal.jsonl" '"prompt":"run the tests"' "context_window includes user prompt excerpt"
|
||||
|
||||
# --- Test 85: context_window includes tool response excerpts ---
|
||||
echo "Test 85: context_window includes tool response excerpts"
|
||||
assert_grep "$ROOT/journal.jsonl" '"response_excerpt"' "context_window entries have response_excerpt"
|
||||
|
||||
# --- Test 86: context_window on dead_end signal ---
|
||||
echo "Test 86: context_window on dead_end signal"
|
||||
reset_state
|
||||
echo '{"hook_event_name":"UserPromptSubmit","prompt":"start working","session_id":"sCW2","cwd":"/tmp/x"}' \
|
||||
| HOOK_RUN >/dev/null 2>&1 || true
|
||||
for i in 1 2 3 4 5 6 7 8; do
|
||||
echo "{\"hook_event_name\":\"PostToolUse\",\"tool_name\":\"Bash\",\"tool_input\":{\"command\":\"step$i\"},\"session_id\":\"sCW2\",\"cwd\":\"/tmp/x\"}" \
|
||||
| HOOK_RUN >/dev/null 2>&1 || true
|
||||
done
|
||||
if grep -qE '"type":"dead_end"' "$ROOT/journal.jsonl" && grep -qE '"context_window":\[' "$ROOT/journal.jsonl"; then
|
||||
echo " PASS: dead_end carries context_window"; PASS=$((PASS+1))
|
||||
else
|
||||
echo " FAIL: dead_end missing context_window"; FAIL=$((FAIL+1))
|
||||
fi
|
||||
|
||||
# --- Test 87: context_window NOT on non-struggle signals ---
|
||||
echo "Test 87: context_window absent from correction_free_streak"
|
||||
reset_state
|
||||
for i in 1 2 3 4 5; do
|
||||
echo "{\"hook_event_name\":\"UserPromptSubmit\",\"prompt\":\"step $i please\",\"session_id\":\"sCW3\",\"cwd\":\"/tmp/x\"}" \
|
||||
| HOOK_RUN >/dev/null 2>&1 || true
|
||||
done
|
||||
# correction_free_streak should have fired
|
||||
streak_line=$(grep '"type":"correction_free_streak"' "$ROOT/journal.jsonl" | head -1)
|
||||
if [ -n "$streak_line" ] && ! echo "$streak_line" | grep -q '"context_window"'; then
|
||||
echo " PASS: correction_free_streak has no context_window"; PASS=$((PASS+1))
|
||||
else
|
||||
echo " FAIL: unexpected context_window on non-struggle signal"; FAIL=$((FAIL+1))
|
||||
fi
|
||||
|
||||
# --- Test 88: adam-batch clusters same signal_type + fp into one batch ---
|
||||
echo "Test 88: adam-batch clusters same (type, fp) into one batch"
|
||||
batch_input=$(cat <<'EOF'
|
||||
{"ts":"2026-05-20T10:00:00Z","type":"tool_error_loop","session":"s1","cwd":"/a","tool":"Bash","fp":"ENOENT:abc","count":3}
|
||||
{"ts":"2026-05-21T10:00:00Z","type":"tool_error_loop","session":"s2","cwd":"/a","tool":"Bash","fp":"ENOENT:abc","count":4}
|
||||
{"ts":"2026-05-22T10:00:00Z","type":"tool_error_loop","session":"s3","cwd":"/b","tool":"Bash","fp":"ENOENT:abc","count":3}
|
||||
EOF
|
||||
)
|
||||
out=$(echo "$batch_input" | BATCH_RUN 2>/dev/null)
|
||||
batch_count=$(echo "$out" | node -e 'let b="";process.stdin.on("data",d=>b+=d).on("end",()=>{const j=JSON.parse(b);console.log(j.batches.length)})')
|
||||
entry_count=$(echo "$out" | node -e 'let b="";process.stdin.on("data",d=>b+=d).on("end",()=>{const j=JSON.parse(b);console.log(j.batches[0]?j.batches[0].entry_count:0)})')
|
||||
if [ "$batch_count" = "1" ] && [ "$entry_count" = "3" ]; then
|
||||
echo " PASS: 3 same-fp entries → 1 batch with entry_count=3"; PASS=$((PASS+1))
|
||||
else
|
||||
echo " FAIL: expected 1 batch / 3 entries (got batches=$batch_count entries=$entry_count)"; FAIL=$((FAIL+1))
|
||||
fi
|
||||
|
||||
# --- Test 89: adam-batch creates separate batches for different signal types ---
|
||||
echo "Test 89: adam-batch separates different signal types"
|
||||
batch_input=$(cat <<'EOF'
|
||||
{"ts":"2026-05-20T10:00:00Z","type":"correction","session":"s1","cwd":"/a","phrase":"no wrong"}
|
||||
{"ts":"2026-05-21T10:00:00Z","type":"tool_error_loop","session":"s1","cwd":"/a","fp":"ENOENT:abc","count":3}
|
||||
EOF
|
||||
)
|
||||
out=$(echo "$batch_input" | BATCH_RUN 2>/dev/null)
|
||||
batch_count=$(echo "$out" | node -e 'let b="";process.stdin.on("data",d=>b+=d).on("end",()=>{const j=JSON.parse(b);console.log(j.batches.length)})')
|
||||
if [ "$batch_count" = "2" ]; then
|
||||
echo " PASS: 2 different types → 2 batches"; PASS=$((PASS+1))
|
||||
else
|
||||
echo " FAIL: expected 2 batches (got $batch_count)"; FAIL=$((FAIL+1))
|
||||
fi
|
||||
|
||||
# --- Test 90: adam-batch reports session_count correctly ---
|
||||
echo "Test 90: adam-batch tracks session_count per batch"
|
||||
batch_input=$(cat <<'EOF'
|
||||
{"ts":"2026-05-20T10:00:00Z","type":"correction","session":"s1","cwd":"/a","phrase":"no wrong"}
|
||||
{"ts":"2026-05-21T10:00:00Z","type":"correction","session":"s2","cwd":"/a","phrase":"no wrong"}
|
||||
{"ts":"2026-05-22T10:00:00Z","type":"correction","session":"s1","cwd":"/b","phrase":"no wrong"}
|
||||
EOF
|
||||
)
|
||||
out=$(echo "$batch_input" | BATCH_RUN 2>/dev/null)
|
||||
sessions=$(echo "$out" | node -e 'let b="";process.stdin.on("data",d=>b+=d).on("end",()=>{const j=JSON.parse(b);console.log(j.batches[0]?j.batches[0].session_count:0)})')
|
||||
if [ "$sessions" = "2" ]; then
|
||||
echo " PASS: session_count=2 for entries from s1+s2"; PASS=$((PASS+1))
|
||||
else
|
||||
echo " FAIL: expected session_count=2 (got $sessions)"; FAIL=$((FAIL+1))
|
||||
fi
|
||||
|
||||
# --- Test 91: adam-batch reports has_context_window ---
|
||||
echo "Test 91: adam-batch reports has_context_window flag"
|
||||
batch_input=$(cat <<'EOF'
|
||||
{"ts":"2026-05-20T10:00:00Z","type":"dead_end","session":"s1","cwd":"/a","count":8,"context_window":[{"event":"user","prompt":"hi","ts":"2026-05-20T09:59:00Z"}]}
|
||||
EOF
|
||||
)
|
||||
out=$(echo "$batch_input" | BATCH_RUN 2>/dev/null)
|
||||
has_cw=$(echo "$out" | node -e 'let b="";process.stdin.on("data",d=>b+=d).on("end",()=>{const j=JSON.parse(b);console.log(j.batches[0]?j.batches[0].has_context_window:"false")})')
|
||||
if [ "$has_cw" = "true" ]; then
|
||||
echo " PASS: has_context_window=true when entries have context_window"; PASS=$((PASS+1))
|
||||
else
|
||||
echo " FAIL: expected has_context_window=true (got $has_cw)"; FAIL=$((FAIL+1))
|
||||
fi
|
||||
|
||||
# --- Test 92: adam-batch empty input → no batches ---
|
||||
echo "Test 92: adam-batch produces empty output on empty input"
|
||||
out=$(echo '' | BATCH_RUN 2>/dev/null)
|
||||
batch_count=$(echo "$out" | node -e 'let b="";process.stdin.on("data",d=>b+=d).on("end",()=>{try{const j=JSON.parse(b);console.log(j.batches.length)}catch{console.log("parse-error")}})')
|
||||
total=$(echo "$out" | node -e 'let b="";process.stdin.on("data",d=>b+=d).on("end",()=>{try{const j=JSON.parse(b);console.log(j.total)}catch{console.log("parse-error")}})')
|
||||
if [ "$batch_count" = "0" ] && [ "$total" = "0" ]; then
|
||||
echo " PASS: empty input → 0 batches, total=0"; PASS=$((PASS+1))
|
||||
else
|
||||
echo " FAIL: expected 0 batches (got batches=$batch_count total=$total)"; FAIL=$((FAIL+1))
|
||||
fi
|
||||
|
||||
# --- Test 93: adam-rollback --proposal-id moves applied proposal to proposals ---
|
||||
echo "Test 93: adam-rollback moves applied proposal to proposals/"
|
||||
reset_state
|
||||
rm -f "$ROOT/proposals/"*rollback* "$ROOT/active-nudges.json"
|
||||
cat > "$ROOT/applied/2026-05-20T00-00-00Z-rb-test-001.md" <<'EOF'
|
||||
---
|
||||
id: rb-test-001
|
||||
type: skill_new
|
||||
target: ~/.claude/skills/test-skill/SKILL.md
|
||||
confidence: 5
|
||||
blast_radius: low
|
||||
auto_apply_eligible: true
|
||||
status: applied
|
||||
source_entries:
|
||||
- "2026-05-18T10:00:00Z"
|
||||
---
|
||||
# Why
|
||||
test rollback
|
||||
|
||||
# Rollback
|
||||
```bash
|
||||
rm -rf ~/.claude/skills/test-skill/
|
||||
```
|
||||
EOF
|
||||
out=$(ROLLBACK_RUN --proposal-id rb-test-001 --home "$TMP_HOME/.claude" 2>/dev/null)
|
||||
if echo "$out" | grep -q '"status":"rolled_back"'; then
|
||||
rb_ok=1
|
||||
else
|
||||
rb_ok=0
|
||||
fi
|
||||
# Verify proposal moved to proposals/
|
||||
if ls "$ROOT/proposals/"*rb-test-001* >/dev/null 2>&1; then
|
||||
moved_ok=1
|
||||
else
|
||||
moved_ok=0
|
||||
fi
|
||||
# Verify original file renamed
|
||||
if [ -f "$ROOT/applied/2026-05-20T00-00-00Z-rb-test-001.md.rolled-back" ]; then
|
||||
renamed_ok=1
|
||||
else
|
||||
renamed_ok=0
|
||||
fi
|
||||
if [ "$rb_ok" = "1" ] && [ "$moved_ok" = "1" ] && [ "$renamed_ok" = "1" ]; then
|
||||
echo " PASS: rollback moved proposal and renamed applied file"; PASS=$((PASS+1))
|
||||
else
|
||||
echo " FAIL: rollback incomplete (status=$rb_ok moved=$moved_ok renamed=$renamed_ok out=$out)"; FAIL=$((FAIL+1))
|
||||
fi
|
||||
|
||||
# --- Test 94: adam-rollback creates regression nudge ---
|
||||
echo "Test 94: adam-rollback creates regression nudge in active-nudges.json"
|
||||
if [ -f "$ROOT/active-nudges.json" ]; then
|
||||
nudge_kind=$(node -e "const j=JSON.parse(require('fs').readFileSync('$ROOT/active-nudges.json','utf8'));console.log((j[0]||{}).kind||'')")
|
||||
if [ "$nudge_kind" = "regression_rollback" ]; then
|
||||
echo " PASS: regression nudge created with kind=regression_rollback"; PASS=$((PASS+1))
|
||||
else
|
||||
echo " FAIL: nudge kind wrong (got $nudge_kind)"; FAIL=$((FAIL+1))
|
||||
fi
|
||||
else
|
||||
echo " FAIL: active-nudges.json not created"; FAIL=$((FAIL+1))
|
||||
fi
|
||||
rm -f "$ROOT/proposals/"*rb-test* "$ROOT/applied/"*rb-test* "$ROOT/active-nudges.json"
|
||||
|
||||
# --- Test 95: adam-rollback rolled_back field in proposal frontmatter ---
|
||||
echo "Test 95: rolled-back proposal has rolled_back: true in frontmatter"
|
||||
reset_state
|
||||
rm -f "$ROOT/proposals/"*rollback* "$ROOT/active-nudges.json"
|
||||
cat > "$ROOT/applied/2026-05-20T00-00-00Z-rb-test-002.md" <<'EOF'
|
||||
---
|
||||
id: rb-test-002
|
||||
type: memory
|
||||
target: ~/.claude/projects/-Users-nvm/memory/test.md
|
||||
confidence: 4
|
||||
blast_radius: low
|
||||
---
|
||||
# Why
|
||||
test
|
||||
# Rollback
|
||||
delete the memory file
|
||||
EOF
|
||||
ROLLBACK_RUN --proposal-id rb-test-002 --home "$TMP_HOME/.claude" >/dev/null 2>&1 || true
|
||||
rb_file=$(ls "$ROOT/proposals/"*rb-test-002* 2>/dev/null | head -1)
|
||||
if [ -n "$rb_file" ] && grep -q 'rolled_back: true' "$rb_file"; then
|
||||
echo " PASS: rolled-back proposal has rolled_back: true"; PASS=$((PASS+1))
|
||||
else
|
||||
echo " FAIL: rolled_back marker missing (file=$rb_file)"; FAIL=$((FAIL+1))
|
||||
fi
|
||||
rm -f "$ROOT/proposals/"*rb-test* "$ROOT/applied/"*rb-test* "$ROOT/active-nudges.json"
|
||||
|
||||
# --- Test 96: adam-rollback not_found on missing proposal ---
|
||||
echo "Test 96: adam-rollback returns not_found for missing proposal"
|
||||
reset_state
|
||||
out=$(ROLLBACK_RUN --proposal-id nonexistent-999 --home "$TMP_HOME/.claude" 2>/dev/null)
|
||||
if echo "$out" | grep -q '"status":"not_found"'; then
|
||||
echo " PASS: not_found status for missing proposal"; PASS=$((PASS+1))
|
||||
else
|
||||
echo " FAIL: expected not_found (got: $out)"; FAIL=$((FAIL+1))
|
||||
fi
|
||||
|
||||
# --- Test 97: adam-rollback --dry-run does not move files ---
|
||||
echo "Test 97: adam-rollback --dry-run leaves files in place"
|
||||
reset_state
|
||||
rm -f "$ROOT/proposals/"*rollback* "$ROOT/active-nudges.json"
|
||||
cat > "$ROOT/applied/2026-05-20T00-00-00Z-rb-dry-001.md" <<'EOF'
|
||||
---
|
||||
id: rb-dry-001
|
||||
type: skill_edit
|
||||
target: ~/.claude/skills/foo/SKILL.md
|
||||
confidence: 4
|
||||
---
|
||||
# Why
|
||||
test dry run
|
||||
# Rollback
|
||||
revert edit
|
||||
EOF
|
||||
out=$(ROLLBACK_RUN --proposal-id rb-dry-001 --dry-run --home "$TMP_HOME/.claude" 2>/dev/null)
|
||||
if echo "$out" | grep -q '"status":"dry_run"' && [ -f "$ROOT/applied/2026-05-20T00-00-00Z-rb-dry-001.md" ]; then
|
||||
echo " PASS: dry-run did not move files"; PASS=$((PASS+1))
|
||||
else
|
||||
echo " FAIL: dry-run moved files or wrong status (out=$out)"; FAIL=$((FAIL+1))
|
||||
fi
|
||||
rm -f "$ROOT/applied/2026-05-20T00-00-00Z-rb-dry-001.md"
|
||||
|
||||
# --- Test 98: context_window on edit_churn signal ---
|
||||
echo "Test 98: context_window on edit_churn signal"
|
||||
reset_state
|
||||
echo '{"hook_event_name":"UserPromptSubmit","prompt":"fix the tests","session_id":"sCW4","cwd":"/tmp/x"}' \
|
||||
| HOOK_RUN >/dev/null 2>&1 || true
|
||||
for i in 1 2 3 4; do
|
||||
echo '{"hook_event_name":"PostToolUse","tool_name":"Edit","tool_input":{"file_path":"/tmp/churn.py"},"session_id":"sCW4","cwd":"/tmp/x"}' \
|
||||
| HOOK_RUN >/dev/null 2>&1 || true
|
||||
done
|
||||
churn_line=$(grep '"type":"edit_churn"' "$ROOT/journal.jsonl" | head -1)
|
||||
if [ -n "$churn_line" ] && echo "$churn_line" | grep -q '"context_window"'; then
|
||||
echo " PASS: edit_churn carries context_window"; PASS=$((PASS+1))
|
||||
else
|
||||
echo " FAIL: edit_churn missing context_window"; FAIL=$((FAIL+1))
|
||||
fi
|
||||
|
||||
# --- Test 99: context_window on build_loop signal ---
|
||||
echo "Test 99: context_window on build_loop signal"
|
||||
reset_state
|
||||
echo '{"hook_event_name":"UserPromptSubmit","prompt":"run the build","session_id":"sCW5","cwd":"/tmp/x"}' \
|
||||
| HOOK_RUN >/dev/null 2>&1 || true
|
||||
for i in 1 2; do
|
||||
echo '{"hook_event_name":"PostToolUse","tool_name":"Bash","tool_input":{"command":"npm run build"},"tool_response":{"is_error":true,"content":"Build failed: TypeError"},"session_id":"sCW5","cwd":"/tmp/x"}' \
|
||||
| HOOK_RUN >/dev/null 2>&1 || true
|
||||
done
|
||||
build_line=$(grep '"type":"build_loop"' "$ROOT/journal.jsonl" | head -1)
|
||||
if [ -n "$build_line" ] && echo "$build_line" | grep -q '"context_window"'; then
|
||||
echo " PASS: build_loop carries context_window"; PASS=$((PASS+1))
|
||||
else
|
||||
echo " FAIL: build_loop missing context_window"; FAIL=$((FAIL+1))
|
||||
fi
|
||||
|
||||
# --- Test 100: adam-batch --min-entries filter ---
|
||||
echo "Test 100: adam-batch --min-entries filters small batches"
|
||||
batch_input=$(cat <<'EOF'
|
||||
{"ts":"2026-05-20T10:00:00Z","type":"correction","session":"s1","cwd":"/a","phrase":"no wrong"}
|
||||
{"ts":"2026-05-21T10:00:00Z","type":"tool_error_loop","session":"s1","cwd":"/a","fp":"ENOENT:abc","count":3}
|
||||
{"ts":"2026-05-22T10:00:00Z","type":"tool_error_loop","session":"s2","cwd":"/a","fp":"ENOENT:abc","count":4}
|
||||
{"ts":"2026-05-23T10:00:00Z","type":"tool_error_loop","session":"s3","cwd":"/a","fp":"ENOENT:abc","count":5}
|
||||
EOF
|
||||
)
|
||||
out=$(echo "$batch_input" | BATCH_RUN --min-entries 3 2>/dev/null)
|
||||
batch_count=$(echo "$out" | node -e 'let b="";process.stdin.on("data",d=>b+=d).on("end",()=>{const j=JSON.parse(b);console.log(j.batches.length)})')
|
||||
unbatched=$(echo "$out" | node -e 'let b="";process.stdin.on("data",d=>b+=d).on("end",()=>{const j=JSON.parse(b);console.log(j.unbatched)})')
|
||||
if [ "$batch_count" = "1" ] && [ "$unbatched" = "1" ]; then
|
||||
echo " PASS: --min-entries=3 keeps 1 batch (3 entries), drops 1 singleton"; PASS=$((PASS+1))
|
||||
else
|
||||
echo " FAIL: expected 1 batch + 1 unbatched (got batches=$batch_count unbatched=$unbatched)"; FAIL=$((FAIL+1))
|
||||
fi
|
||||
|
||||
# --- Test 101: adam-rollback extracts undo commands from Rollback section ---
|
||||
echo "Test 101: adam-rollback extracts undo commands from code blocks"
|
||||
reset_state
|
||||
rm -f "$ROOT/proposals/"*rollback* "$ROOT/active-nudges.json"
|
||||
cat > "$ROOT/applied/2026-05-20T00-00-00Z-rb-undo-001.md" <<'HEREDOC'
|
||||
---
|
||||
id: rb-undo-001
|
||||
type: skill_new
|
||||
target: ~/.claude/skills/test-undo/SKILL.md
|
||||
confidence: 5
|
||||
blast_radius: low
|
||||
---
|
||||
# Why
|
||||
test
|
||||
# Rollback
|
||||
```bash
|
||||
rm -rf ~/.claude/skills/test-undo/
|
||||
```
|
||||
HEREDOC
|
||||
out=$(ROLLBACK_RUN --proposal-id rb-undo-001 --home "$TMP_HOME/.claude" 2>/dev/null)
|
||||
undo_count=$(echo "$out" | node -e 'let b="";process.stdin.on("data",d=>b+=d).on("end",()=>{try{const j=JSON.parse(b);console.log((j.rollbacks[0]||{}).undo_commands?j.rollbacks[0].undo_commands.length:0)}catch{console.log("err")}})')
|
||||
if [ "$undo_count" = "1" ]; then
|
||||
echo " PASS: extracted 1 undo command from Rollback section"; PASS=$((PASS+1))
|
||||
else
|
||||
echo " FAIL: expected 1 undo command (got $undo_count, out=$out)"; FAIL=$((FAIL+1))
|
||||
fi
|
||||
rm -f "$ROOT/proposals/"*rb-undo* "$ROOT/applied/"*rb-undo* "$ROOT/active-nudges.json"
|
||||
|
||||
# --- Test 102: context_ring size bounded at 8 ---
|
||||
echo "Test 102: context_ring bounded at CONTEXT_RING_SIZE=8"
|
||||
reset_state
|
||||
# Fire 12 PostToolUse events, then a struggle signal
|
||||
for i in $(seq 1 12); do
|
||||
echo "{\"hook_event_name\":\"PostToolUse\",\"tool_name\":\"Read\",\"tool_input\":{\"file_path\":\"/tmp/f-$i\"},\"tool_response\":{\"content\":\"ok\"},\"session_id\":\"sCR\",\"cwd\":\"/tmp/x\"}" \
|
||||
| HOOK_RUN >/dev/null 2>&1 || true
|
||||
done
|
||||
# Next 3 errors to trigger tool_error_loop with context_window
|
||||
for i in 1 2 3; do
|
||||
echo '{"hook_event_name":"PostToolUse","tool_name":"Bash","tool_input":{"command":"fail"},"tool_response":{"is_error":true,"content":"Error: fail"},"session_id":"sCR","cwd":"/tmp/x"}' \
|
||||
| HOOK_RUN >/dev/null 2>&1 || true
|
||||
done
|
||||
cw_len=$(grep '"type":"tool_error_loop"' "$ROOT/journal.jsonl" | head -1 | node -e 'let b="";process.stdin.on("data",d=>b+=d).on("end",()=>{try{const j=JSON.parse(b);console.log(j.context_window?j.context_window.length:0)}catch{console.log("err")}})')
|
||||
if [ "$cw_len" = "8" ]; then
|
||||
echo " PASS: context_window capped at 8 entries"; PASS=$((PASS+1))
|
||||
else
|
||||
echo " FAIL: expected 8 context_window entries (got $cw_len)"; FAIL=$((FAIL+1))
|
||||
fi
|
||||
|
||||
echo
|
||||
echo "Results: $PASS passed, $FAIL failed"
|
||||
[ "$FAIL" = "0" ]
|
||||
|
||||
Reference in New Issue
Block a user