diff --git a/cmd/hooks/user-prompt/main.go b/cmd/hooks/user-prompt/main.go index 201441c..2211e84 100644 --- a/cmd/hooks/user-prompt/main.go +++ b/cmd/hooks/user-prompt/main.go @@ -10,6 +10,7 @@ import ( "time" "github.com/lukaszraczylo/claude-mnemonic/pkg/hooks" + "github.com/lukaszraczylo/claude-mnemonic/pkg/sanitize" ) // Input is the hook input from Claude Code. @@ -148,14 +149,14 @@ func handleUserPrompt(ctx *hooks.HookContext, input *Input) (string, error) { obsText += "Key facts:\n" for _, fact := range facts { if factStr, ok := fact.(string); ok { - obsText += fmt.Sprintf("- %s\n", factStr) + obsText += fmt.Sprintf("- %s\n", sanitize.StripSystemXML(factStr)) } } obsText += "\n" } if narrative, ok := obsMap["narrative"].(string); ok && narrative != "" { - obsText += narrative + "\n\n" + obsText += sanitize.StripSystemXML(narrative) + "\n\n" } obsTokens := estimateTokens(obsText) diff --git a/go.mod b/go.mod index bfd1432..3070165 100644 --- a/go.mod +++ b/go.mod @@ -15,7 +15,7 @@ require ( github.com/smacker/go-tree-sitter v0.0.0-20240827094217-dd81d9e9be82 github.com/stretchr/testify v1.11.1 github.com/sugarme/tokenizer v0.3.0 - github.com/yalue/onnxruntime_go v1.25.0 + github.com/yalue/onnxruntime_go v1.27.0 golang.org/x/sync v0.19.0 gorm.io/driver/sqlite v1.6.0 gorm.io/gorm v1.31.1 diff --git a/go.sum b/go.sum index cb544af..e890be8 100644 --- a/go.sum +++ b/go.sum @@ -54,6 +54,8 @@ github.com/sugarme/regexpset v0.0.0-20200920021344-4d4ec8eaf93c h1:pwb4kNSHb4K89 github.com/sugarme/regexpset v0.0.0-20200920021344-4d4ec8eaf93c/go.mod h1:2gwkXLWbDGUQWeL3RtpCmcY4mzCtU13kb9UsAg9xMaw= github.com/yalue/onnxruntime_go v1.25.0 h1:nlhVau1BpLZ/BYr+WpPZCJRD/WES0qo6dK7aKyyAs3g= github.com/yalue/onnxruntime_go v1.25.0/go.mod h1:b4X26A8pekNb1ACJ58wAXgNKeUCGEAQ9dmACut9Sm/4= +github.com/yalue/onnxruntime_go v1.27.0 h1:c1YSgDNtpf0WGtxj3YeRIb8VC5LmM1J+Ve3uHdteC1U= +github.com/yalue/onnxruntime_go v1.27.0/go.mod h1:b4X26A8pekNb1ACJ58wAXgNKeUCGEAQ9dmACut9Sm/4= golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4= golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= diff --git a/internal/worker/sdk/prompts.go b/internal/worker/sdk/prompts.go index af5e865..6fbf215 100644 --- a/internal/worker/sdk/prompts.go +++ b/internal/worker/sdk/prompts.go @@ -49,6 +49,10 @@ func BuildObservationPrompt(exec ToolExecution) string { inputJSON, _ := json.MarshalIndent(toolInput, " ", " ") outputJSON, _ := json.MarshalIndent(toolOutput, " ", " ") + // Strip system XML artifacts from tool data before building prompt + cleanInput := StripSystemXML(string(inputJSON)) + cleanOutput := StripSystemXML(string(outputJSON)) + timestamp := time.UnixMilli(exec.CreatedAtEpoch).Format(time.RFC3339) var sb strings.Builder @@ -58,8 +62,8 @@ func BuildObservationPrompt(exec ToolExecution) string { if exec.CWD != "" { sb.WriteString(fmt.Sprintf(" %s\n", exec.CWD)) } - sb.WriteString(fmt.Sprintf(" %s\n", truncate(string(inputJSON), 3000))) - sb.WriteString(fmt.Sprintf(" %s\n", truncate(string(outputJSON), 5000))) + sb.WriteString(fmt.Sprintf(" %s\n", truncate(cleanInput, 3000))) + sb.WriteString(fmt.Sprintf(" %s\n", truncate(cleanOutput, 5000))) sb.WriteString("") return sb.String() @@ -84,8 +88,10 @@ func BuildSummaryPrompt(req SummaryRequest) string { sb.WriteString("Write progress notes of what was done, what was learned, and what's next. This is a checkpoint to capture progress so far. The session is ongoing - you may receive more requests and tool executions after this summary. Write \"next_steps\" as the current trajectory of work (what's actively being worked on or coming up next), not as post-session future work. Always write at least a minimal summary explaining current progress, even if work is still in early stages, so that users see a summary output tied to each request.\n\n") if req.LastAssistantMessage != "" { + // Strip system XML artifacts from captured transcript content + cleanMsg := StripSystemXML(req.LastAssistantMessage) sb.WriteString("Claude's Full Response to User:\n") - sb.WriteString(truncate(req.LastAssistantMessage, 4000)) + sb.WriteString(truncate(cleanMsg, 4000)) sb.WriteString("\n\n") } diff --git a/internal/worker/sdk/sanitize.go b/internal/worker/sdk/sanitize.go new file mode 100644 index 0000000..9f65580 --- /dev/null +++ b/internal/worker/sdk/sanitize.go @@ -0,0 +1,8 @@ +// Package sdk provides SDK agent integration for claude-mnemonic. +package sdk + +import "github.com/lukaszraczylo/claude-mnemonic/pkg/sanitize" + +// StripSystemXML removes known Claude Code internal XML blocks from text. +// Delegates to the shared sanitize package. +var StripSystemXML = sanitize.StripSystemXML diff --git a/pkg/sanitize/sanitize.go b/pkg/sanitize/sanitize.go new file mode 100644 index 0000000..b269960 --- /dev/null +++ b/pkg/sanitize/sanitize.go @@ -0,0 +1,63 @@ +// Package sanitize provides content cleaning utilities for stripping +// Claude Code internal XML artifacts from captured text. +package sanitize + +import ( + "regexp" + "strings" +) + +// systemXMLTags lists Claude Code internal XML tags that should be stripped +// from captured content before processing. These are system-level artifacts +// that pollute observations and summaries when stored. +var systemXMLTags = []string{ + // Claude Code task/agent system + "task-notification", + // System reminders injected by Claude Code + "system-reminder", + // Claude-mnemonic's own context injection + "relevant-memory", + // Hook output wrappers + "user-prompt-submit-hook", + // Large output persistence + "persisted-output", + // Tool loading system + "available-deferred-tools", + // Fast mode info + "fast_mode_info", + // Anthropic internal + "antml_thinking", + "antml_function_calls", +} + +// systemXMLRegexps are compiled regexps for each tag, built once at init. +var systemXMLRegexps []*regexp.Regexp + +func init() { + systemXMLRegexps = make([]*regexp.Regexp, len(systemXMLTags)) + for i, tag := range systemXMLTags { + // Match opening tag (with optional attributes), content (including newlines), and closing tag + systemXMLRegexps[i] = regexp.MustCompile(`(?s)<` + regexp.QuoteMeta(tag) + `[^>]*>.*?`) + } +} + +// StripSystemXML removes known Claude Code internal XML blocks from text. +// This prevents system artifacts like , , +// and from being stored in observations and summaries. +func StripSystemXML(s string) string { + // Quick check: if no angle brackets, nothing to strip + if !strings.Contains(s, "<") { + return s + } + + for _, re := range systemXMLRegexps { + s = re.ReplaceAllString(s, "") + } + + // Clean up resulting double-blank-lines from removed blocks + for strings.Contains(s, "\n\n\n") { + s = strings.ReplaceAll(s, "\n\n\n", "\n\n") + } + + return strings.TrimSpace(s) +} diff --git a/pkg/sanitize/sanitize_test.go b/pkg/sanitize/sanitize_test.go new file mode 100644 index 0000000..f5a828e --- /dev/null +++ b/pkg/sanitize/sanitize_test.go @@ -0,0 +1,113 @@ +package sanitize + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestStripSystemXML(t *testing.T) { + tests := []struct { + name string + input string + expected string + }{ + { + name: "no XML tags", + input: "Hello, this is plain text with no XML.", + expected: "Hello, this is plain text with no XML.", + }, + { + name: "empty string", + input: "", + expected: "", + }, + { + name: "task-notification block", + input: "Before\n\nabc123\ncompleted\nAgent completed\nSome result text\n\nAfter", + expected: "Before\n\nAfter", + }, + { + name: "system-reminder block", + input: "Content before\n\nSome system reminder text\n\nContent after", + expected: "Content before\n\nContent after", + }, + { + name: "relevant-memory block", + input: "\n# Relevant Knowledge\n## 1. Something\n\nActual content", + expected: "Actual content", + }, + { + name: "system-reminder with attributes", + input: "Before reminder text After", + expected: "Before After", + }, + { + name: "multiple different tags", + input: "Start\ndone\nMiddle\nreminder\nEnd", + expected: "Start\n\nMiddle\n\nEnd", + }, + { + name: "persisted-output block", + input: "Result: \nLarge output stored at: /tmp/foo\n done", + expected: "Result: done", + }, + { + name: "available-deferred-tools block", + input: "\nTool1\nTool2\n\nUser message", + expected: "User message", + }, + { + name: "fast_mode_info block", + input: "Text \nFast mode uses same model\n more text", + expected: "Text more text", + }, + { + name: "preserves non-system XML", + input: "bugfixFixed thing", + expected: "bugfixFixed thing", + }, + { + name: "no angle brackets fast path", + input: "Just plain text without any special characters", + expected: "Just plain text without any special characters", + }, + { + name: "collapses triple newlines", + input: "Before\nx\n\n\nAfter", + expected: "Before\n\nAfter", + }, + { + name: "real-world task notification", + input: "Here is the analysis:\n\na077fd04aed547ce1\ntoolu_01Kw2GwsArQQR1F9aV3VfzhP\ncompleted\nAgent \"Analyze agency-agents repo structure\" completed\nNow I have all the information I need. Let me compile a comprehensive report.\n\n## Agency-Agents Directory Exploration Report\n\n### 1. Agent File Count and Organization\n\n**Total Agent Files (excluding strategy docs):** 61 agents across 9 categories\n\nThe repo has 61 agents.", + expected: "Here is the analysis:\n\nThe repo has 61 agents.", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := StripSystemXML(tt.input) + assert.Equal(t, tt.expected, result) + }) + } +} + +func BenchmarkStripSystemXML(b *testing.B) { + // Text with no XML (fast path) + b.Run("no_xml", func(b *testing.B) { + text := "This is plain text without any XML tags at all." + b.ResetTimer() + for i := 0; i < b.N; i++ { + StripSystemXML(text) + } + }) + + // Text with system XML to strip + b.Run("with_xml", func(b *testing.B) { + text := "Before\n\nabc\ndone\n\nreminder text here\nAfter" + b.ResetTimer() + for i := 0; i < b.N; i++ { + StripSystemXML(text) + } + }) +} diff --git a/ui/package-lock.json b/ui/package-lock.json index 19a6d07..ffc3e03 100644 --- a/ui/package-lock.json +++ b/ui/package-lock.json @@ -1,12 +1,12 @@ { "name": "claude-mnemonic-dashboard", - "version": "v0.10.5-15-g385d05a-dirty", + "version": "v0.11.43-2-g11fd196-dirty", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "claude-mnemonic-dashboard", - "version": "v0.10.5-15-g385d05a-dirty", + "version": "v0.11.43-2-g11fd196-dirty", "dependencies": { "vis-data": "^7.1.9", "vis-network": "^9.1.9", diff --git a/ui/package.json b/ui/package.json index ff8d721..67f33a6 100644 --- a/ui/package.json +++ b/ui/package.json @@ -1,6 +1,6 @@ { "name": "claude-mnemonic-dashboard", - "version": "v0.10.5-15-g385d05a-dirty", + "version": "v0.11.43-2-g11fd196-dirty", "private": true, "type": "module", "scripts": {