mirror of
https://github.com/lukaszraczylo/claude-mnemonic.git
synced 2026-10-12 04:55:18 +00:00
Compare commits
4
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
742a3d42cb | ||
|
|
651732e0de | ||
|
|
1ae8035470 | ||
|
|
4f4b4ac70f |
@@ -0,0 +1,35 @@
|
|||||||
|
linters-settings:
|
||||||
|
govet:
|
||||||
|
enable:
|
||||||
|
- fieldalignment
|
||||||
|
errcheck:
|
||||||
|
# Ignore error checks in test files for common test helpers
|
||||||
|
exclude-functions:
|
||||||
|
- (io.Closer).Close
|
||||||
|
- (*encoding/json.Encoder).Encode
|
||||||
|
- (io.Writer).Write
|
||||||
|
|
||||||
|
linters:
|
||||||
|
enable:
|
||||||
|
- errcheck
|
||||||
|
- gosec
|
||||||
|
- govet
|
||||||
|
- gofmt
|
||||||
|
- staticcheck
|
||||||
|
- unused
|
||||||
|
- ineffassign
|
||||||
|
- typecheck
|
||||||
|
|
||||||
|
issues:
|
||||||
|
exclude-dirs:
|
||||||
|
- vendor
|
||||||
|
# Exclude some linters from running on test files
|
||||||
|
exclude-rules:
|
||||||
|
- path: _test\.go
|
||||||
|
linters:
|
||||||
|
- errcheck
|
||||||
|
- gosec
|
||||||
|
|
||||||
|
run:
|
||||||
|
timeout: 5m
|
||||||
|
tests: true
|
||||||
@@ -23,12 +23,12 @@ type Input struct {
|
|||||||
|
|
||||||
// Observation represents an observation from the API.
|
// Observation represents an observation from the API.
|
||||||
type Observation struct {
|
type Observation struct {
|
||||||
ID int64 `json:"id"`
|
|
||||||
Type string `json:"type"`
|
Type string `json:"type"`
|
||||||
Title string `json:"title"`
|
Title string `json:"title"`
|
||||||
Subtitle string `json:"subtitle"`
|
Subtitle string `json:"subtitle"`
|
||||||
Narrative string `json:"narrative"`
|
Narrative string `json:"narrative"`
|
||||||
Facts []string `json:"facts"`
|
Facts []string `json:"facts"`
|
||||||
|
ID int64 `json:"id"`
|
||||||
}
|
}
|
||||||
|
|
||||||
func main() {
|
func main() {
|
||||||
@@ -46,8 +46,8 @@ func main() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
var input Input
|
var input Input
|
||||||
if err := json.Unmarshal(inputData, &input); err != nil {
|
if unmarshalErr := json.Unmarshal(inputData, &input); unmarshalErr != nil {
|
||||||
hooks.WriteError("SessionStart", err)
|
hooks.WriteError("SessionStart", unmarshalErr)
|
||||||
os.Exit(1)
|
os.Exit(1)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -44,21 +44,21 @@ type StatusInput struct {
|
|||||||
|
|
||||||
// WorkerStats is the response from the worker's /api/stats endpoint.
|
// WorkerStats is the response from the worker's /api/stats endpoint.
|
||||||
type WorkerStats struct {
|
type WorkerStats struct {
|
||||||
Uptime string `json:"uptime"`
|
Uptime string `json:"uptime"`
|
||||||
ActiveSessions int `json:"activeSessions"`
|
Project string `json:"project,omitempty"`
|
||||||
QueueDepth int `json:"queueDepth"`
|
Retrieval struct {
|
||||||
IsProcessing bool `json:"isProcessing"`
|
|
||||||
ConnectedClients int `json:"connectedClients"`
|
|
||||||
SessionsToday int `json:"sessionsToday"`
|
|
||||||
Ready bool `json:"ready"`
|
|
||||||
Project string `json:"project,omitempty"`
|
|
||||||
ProjectObservations int `json:"projectObservations,omitempty"`
|
|
||||||
Retrieval struct {
|
|
||||||
TotalRequests int64 `json:"TotalRequests"`
|
TotalRequests int64 `json:"TotalRequests"`
|
||||||
ObservationsServed int64 `json:"ObservationsServed"`
|
ObservationsServed int64 `json:"ObservationsServed"`
|
||||||
SearchRequests int64 `json:"SearchRequests"`
|
SearchRequests int64 `json:"SearchRequests"`
|
||||||
ContextInjections int64 `json:"ContextInjections"`
|
ContextInjections int64 `json:"ContextInjections"`
|
||||||
} `json:"retrieval"`
|
} `json:"retrieval"`
|
||||||
|
ActiveSessions int `json:"activeSessions"`
|
||||||
|
QueueDepth int `json:"queueDepth"`
|
||||||
|
ConnectedClients int `json:"connectedClients"`
|
||||||
|
SessionsToday int `json:"sessionsToday"`
|
||||||
|
ProjectObservations int `json:"projectObservations,omitempty"`
|
||||||
|
IsProcessing bool `json:"isProcessing"`
|
||||||
|
Ready bool `json:"ready"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// ANSI color codes
|
// ANSI color codes
|
||||||
|
|||||||
@@ -14,17 +14,17 @@ import (
|
|||||||
// Input is the hook input from Claude Code.
|
// Input is the hook input from Claude Code.
|
||||||
type Input struct {
|
type Input struct {
|
||||||
hooks.BaseInput
|
hooks.BaseInput
|
||||||
StopHookActive bool `json:"stop_hook_active"`
|
|
||||||
TranscriptPath string `json:"transcript_path"`
|
TranscriptPath string `json:"transcript_path"`
|
||||||
|
StopHookActive bool `json:"stop_hook_active"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// TranscriptMessage represents a message in the transcript JSONL file.
|
// TranscriptMessage represents a message in the transcript JSONL file.
|
||||||
type TranscriptMessage struct {
|
type TranscriptMessage struct {
|
||||||
Type string `json:"type"`
|
|
||||||
Message struct {
|
Message struct {
|
||||||
|
Content any `json:"content"`
|
||||||
Role string `json:"role"`
|
Role string `json:"role"`
|
||||||
Content any `json:"content"` // Can be string or array
|
|
||||||
} `json:"message"`
|
} `json:"message"`
|
||||||
|
Type string `json:"type"` // Can be string or array
|
||||||
}
|
}
|
||||||
|
|
||||||
// extractTextContent extracts text content from message content (handles both string and array formats).
|
// extractTextContent extracts text content from message content (handles both string and array formats).
|
||||||
|
|||||||
@@ -35,8 +35,8 @@ func main() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
var input Input
|
var input Input
|
||||||
if err := json.Unmarshal(inputData, &input); err != nil {
|
if unmarshalErr := json.Unmarshal(inputData, &input); unmarshalErr != nil {
|
||||||
hooks.WriteError("UserPromptSubmit", err)
|
hooks.WriteError("UserPromptSubmit", unmarshalErr)
|
||||||
os.Exit(1)
|
os.Exit(1)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
+23
-6
@@ -40,7 +40,7 @@
|
|||||||
class="w-full h-auto"
|
class="w-full h-auto"
|
||||||
/>
|
/>
|
||||||
</div>
|
</div>
|
||||||
<p class="text-center text-slate-500 text-sm mt-4">The dashboard at localhost:37777 - browse, search, and manage your memories</p>
|
<p class="text-center text-slate-500 text-sm mt-4">The dashboard at localhost:37777 - browse, search, and manage your memories. View graph stats, vector metrics, storage savings, and performance analytics.</p>
|
||||||
</div>
|
</div>
|
||||||
</section>
|
</section>
|
||||||
|
|
||||||
@@ -304,7 +304,7 @@
|
|||||||
<section class="py-20 lg:py-28 px-4 sm:px-6">
|
<section class="py-20 lg:py-28 px-4 sm:px-6">
|
||||||
<div class="max-w-6xl mx-auto">
|
<div class="max-w-6xl mx-auto">
|
||||||
<SectionHeader title="Under the hood" subtitle="Built with simplicity and performance in mind" />
|
<SectionHeader title="Under the hood" subtitle="Built with simplicity and performance in mind" />
|
||||||
<div class="grid sm:grid-cols-2 lg:grid-cols-4 gap-4 sm:gap-6 text-center">
|
<div class="grid sm:grid-cols-2 lg:grid-cols-3 gap-4 sm:gap-6 text-center">
|
||||||
<div class="glass rounded-2xl p-6 sm:p-8 hover:border-amber-500/30 transition-colors">
|
<div class="glass rounded-2xl p-6 sm:p-8 hover:border-amber-500/30 transition-colors">
|
||||||
<div class="text-3xl sm:text-4xl font-bold text-amber-500 mb-2">Go</div>
|
<div class="text-3xl sm:text-4xl font-bold text-amber-500 mb-2">Go</div>
|
||||||
<p class="text-slate-400 text-xs sm:text-sm">Single binary. Fast startup, low memory. Zero runtime dependencies.</p>
|
<p class="text-slate-400 text-xs sm:text-sm">Single binary. Fast startup, low memory. Zero runtime dependencies.</p>
|
||||||
@@ -315,12 +315,20 @@
|
|||||||
</div>
|
</div>
|
||||||
<div class="glass rounded-2xl p-6 sm:p-8 hover:border-amber-500/30 transition-colors">
|
<div class="glass rounded-2xl p-6 sm:p-8 hover:border-amber-500/30 transition-colors">
|
||||||
<div class="text-3xl sm:text-4xl font-bold text-amber-500 mb-2">sqlite-vec</div>
|
<div class="text-3xl sm:text-4xl font-bold text-amber-500 mb-2">sqlite-vec</div>
|
||||||
<p class="text-slate-400 text-xs sm:text-sm">Embedded vector database. No external services required.</p>
|
<p class="text-slate-400 text-xs sm:text-sm">Hybrid vector storage with LEANN-inspired selective embeddings. 60-80% storage reduction.</p>
|
||||||
</div>
|
</div>
|
||||||
<div class="glass rounded-2xl p-6 sm:p-8 hover:border-amber-500/30 transition-colors">
|
<div class="glass rounded-2xl p-6 sm:p-8 hover:border-amber-500/30 transition-colors">
|
||||||
<div class="text-3xl sm:text-4xl font-bold text-amber-500 mb-2">BGE</div>
|
<div class="text-3xl sm:text-4xl font-bold text-amber-500 mb-2">BGE</div>
|
||||||
<p class="text-slate-400 text-xs sm:text-sm">Two-stage retrieval: bi-encoder embeddings + cross-encoder reranking for high accuracy.</p>
|
<p class="text-slate-400 text-xs sm:text-sm">Two-stage retrieval: bi-encoder embeddings + cross-encoder reranking for high accuracy.</p>
|
||||||
</div>
|
</div>
|
||||||
|
<div class="glass rounded-2xl p-6 sm:p-8 hover:border-amber-500/30 transition-colors">
|
||||||
|
<div class="text-3xl sm:text-4xl font-bold text-amber-500 mb-2">Tree-sitter</div>
|
||||||
|
<p class="text-slate-400 text-xs sm:text-sm">AST-aware code chunking respects function boundaries for Go, Python, and TypeScript.</p>
|
||||||
|
</div>
|
||||||
|
<div class="glass rounded-2xl p-6 sm:p-8 hover:border-amber-500/30 transition-colors">
|
||||||
|
<div class="text-3xl sm:text-4xl font-bold text-amber-500 mb-2">CSR Graph</div>
|
||||||
|
<p class="text-slate-400 text-xs sm:text-sm">Memory-efficient observation relationship graph with edge detection and hub identification.</p>
|
||||||
|
</div>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
</section>
|
</section>
|
||||||
@@ -417,9 +425,12 @@ const activeTab = ref('macos')
|
|||||||
const features = [
|
const features = [
|
||||||
{ icon: 'fas fa-brain', title: 'Learns as you work', description: 'Every bug fix, every architecture decision, every "aha moment" - captured automatically without breaking your flow.' },
|
{ icon: 'fas fa-brain', title: 'Learns as you work', description: 'Every bug fix, every architecture decision, every "aha moment" - captured automatically without breaking your flow.' },
|
||||||
{ icon: 'fas fa-search', title: 'Two-stage retrieval', description: 'Cross-encoder reranking delivers highly relevant results. Finds what you need even with vague queries like "that auth thing".' },
|
{ icon: 'fas fa-search', title: 'Two-stage retrieval', description: 'Cross-encoder reranking delivers highly relevant results. Finds what you need even with vague queries like "that auth thing".' },
|
||||||
{ icon: 'fas fa-project-diagram', title: 'Knowledge graph', description: 'Automatically discovers relationships between memories. See how concepts connect in the visual graph dashboard.' },
|
{ icon: 'fas fa-project-diagram', title: 'Graph-based search', description: 'LEANN Phase 2: Graph relationships between observations (file overlap, semantic similarity, temporal proximity) for smarter context retrieval.' },
|
||||||
|
{ icon: 'fas fa-microchip', title: 'AST-aware chunking', description: 'Intelligent code splitting respects function boundaries. Go, Python, and TypeScript code is chunked at semantic boundaries, not arbitrary line counts.' },
|
||||||
|
{ icon: 'fas fa-database', title: 'Hybrid vector storage', description: 'LEANN-inspired selective storage: frequently-accessed "hub" observations store embeddings, others recompute on-demand. 60-80% storage savings with <50ms latency.' },
|
||||||
{ icon: 'fas fa-folder-tree', title: 'Project-aware context', description: 'Your React knowledge stays in React projects. Your Go patterns stay in Go projects. No context pollution.' },
|
{ icon: 'fas fa-folder-tree', title: 'Project-aware context', description: 'Your React knowledge stays in React projects. Your Go patterns stay in Go projects. No context pollution.' },
|
||||||
{ icon: 'fas fa-chart-line', title: 'Smart scoring', description: 'Importance decay, pattern detection, and conflict resolution ensure the most valuable memories surface first.' },
|
{ icon: 'fas fa-chart-line', title: 'Smart scoring', description: 'Importance decay, pattern detection, and conflict resolution ensure the most valuable memories surface first.' },
|
||||||
|
{ icon: 'fas fa-gauge-high', title: 'Auto-tuning', description: 'Dynamic hub threshold adjustment based on query performance. Automatically balances storage efficiency with search latency for your workload.' },
|
||||||
{ icon: 'fas fa-lock', title: '100% private', description: 'Your code context never leaves your machine. No telemetry. No cloud sync. Your memories are yours.' },
|
{ icon: 'fas fa-lock', title: '100% private', description: 'Your code context never leaves your machine. No telemetry. No cloud sync. Your memories are yours.' },
|
||||||
]
|
]
|
||||||
|
|
||||||
@@ -447,6 +458,10 @@ const configOptions = [
|
|||||||
{ name: 'CLAUDE_MNEMONIC_CONTEXT_OBSERVATIONS', description: 'Maximum observations injected per session (default: 100)', icon: 'fas fa-layer-group' },
|
{ name: 'CLAUDE_MNEMONIC_CONTEXT_OBSERVATIONS', description: 'Maximum observations injected per session (default: 100)', icon: 'fas fa-layer-group' },
|
||||||
{ name: 'CLAUDE_MNEMONIC_RERANKING_ENABLED', description: 'Enable cross-encoder reranking for improved search relevance (default: true)', icon: 'fas fa-sort-amount-down' },
|
{ name: 'CLAUDE_MNEMONIC_RERANKING_ENABLED', description: 'Enable cross-encoder reranking for improved search relevance (default: true)', icon: 'fas fa-sort-amount-down' },
|
||||||
{ name: 'CLAUDE_MNEMONIC_CONTEXT_RELEVANCE_THRESHOLD', description: 'Minimum similarity score for inclusion, 0.0-1.0 (default: 0.3)', icon: 'fas fa-filter' },
|
{ name: 'CLAUDE_MNEMONIC_CONTEXT_RELEVANCE_THRESHOLD', description: 'Minimum similarity score for inclusion, 0.0-1.0 (default: 0.3)', icon: 'fas fa-filter' },
|
||||||
|
{ name: 'CLAUDE_MNEMONIC_VECTOR_STORAGE_STRATEGY', description: 'Storage strategy: "hub" (default), "always", or "on_demand"', icon: 'fas fa-database' },
|
||||||
|
{ name: 'CLAUDE_MNEMONIC_GRAPH_ENABLED', description: 'Enable graph-based search with observation relationships (default: true)', icon: 'fas fa-project-diagram' },
|
||||||
|
{ name: 'CLAUDE_MNEMONIC_GRAPH_MAX_HOPS', description: 'Maximum graph traversal depth for search expansion (default: 2)', icon: 'fas fa-route' },
|
||||||
|
{ name: 'CLAUDE_MNEMONIC_GRAPH_REBUILD_INTERVAL_MIN', description: 'How often to rebuild the observation graph in minutes (default: 60)', icon: 'fas fa-clock' },
|
||||||
]
|
]
|
||||||
|
|
||||||
const requiredDeps = [
|
const requiredDeps = [
|
||||||
@@ -457,9 +472,11 @@ const requiredDeps = [
|
|||||||
const faqs = [
|
const faqs = [
|
||||||
{ question: 'Will it confuse Claude with wrong context?', answer: 'No. Mnemonic uses project isolation and semantic relevance scoring. Only memories from the current project (or global best practices) are injected, and only when they\'re actually relevant to your prompt.' },
|
{ question: 'Will it confuse Claude with wrong context?', answer: 'No. Mnemonic uses project isolation and semantic relevance scoring. Only memories from the current project (or global best practices) are injected, and only when they\'re actually relevant to your prompt.' },
|
||||||
{ question: 'What exactly gets saved?', answer: 'Bug fixes with context ("Fixed race condition by adding mutex"), architecture decisions ("Using repository pattern for data access"), conventions ("All API routes prefixed with /api/v1"), and learnings you want to preserve.' },
|
{ question: 'What exactly gets saved?', answer: 'Bug fixes with context ("Fixed race condition by adding mutex"), architecture decisions ("Using repository pattern for data access"), conventions ("All API routes prefixed with /api/v1"), and learnings you want to preserve.' },
|
||||||
{ question: 'Can I delete or edit memories?', answer: 'Yes. The web dashboard at localhost:37777 lets you browse, search, edit, and delete any memory. You\'re always in control.' },
|
{ question: 'How does hybrid vector storage work?', answer: 'LEANN-inspired selective storage: frequently-accessed "hub" observations (identified by access patterns and graph centrality) store embeddings. Infrequently-accessed observations recompute embeddings on-demand during search. This reduces storage by 60-80% with minimal latency impact (<50ms).' },
|
||||||
|
{ question: 'Can I delete or edit memories?', answer: 'Yes. The web dashboard at localhost:37777 lets you browse, search, edit, and delete any memory. You can also view graph relationships, storage metrics, and performance analytics. You\'re always in control.' },
|
||||||
{ question: 'Does it work with my existing Claude Code setup?', answer: 'Yes. Mnemonic installs as a Claude Code plugin with hooks. Your existing workflows, settings, and shortcuts remain unchanged.' },
|
{ question: 'Does it work with my existing Claude Code setup?', answer: 'Yes. Mnemonic installs as a Claude Code plugin with hooks. Your existing workflows, settings, and shortcuts remain unchanged.' },
|
||||||
{ question: 'What if I switch between projects frequently?', answer: 'That\'s the point. Each project has isolated memories. Switch from your Python ML project to your TypeScript app - context switches automatically.' },
|
{ question: 'What if I switch between projects frequently?', answer: 'That\'s the point. Each project has isolated memories. Switch from your Python ML project to your TypeScript app - context switches automatically.' },
|
||||||
{ question: 'Is there a performance impact?', answer: 'Minimal. The Go worker is lightweight (typically under 30MB RAM). Context injection at session start takes milliseconds for most projects.' },
|
{ question: 'Is there a performance impact?', answer: 'Minimal. The Go worker is lightweight (typically under 30MB RAM). Hybrid storage and auto-tuning optimize for your workload. Context injection at session start takes milliseconds for most projects.' },
|
||||||
|
{ question: 'What is AST-aware chunking?', answer: 'When processing code observations, Mnemonic uses Tree-sitter parsers to respect function and class boundaries instead of arbitrary line limits. Go, Python, and TypeScript code is chunked at semantic boundaries for better search accuracy.' },
|
||||||
]
|
]
|
||||||
</script>
|
</script>
|
||||||
|
|||||||
@@ -11,6 +11,7 @@ require (
|
|||||||
github.com/goccy/go-json v0.10.5
|
github.com/goccy/go-json v0.10.5
|
||||||
github.com/mattn/go-sqlite3 v1.14.32
|
github.com/mattn/go-sqlite3 v1.14.32
|
||||||
github.com/rs/zerolog v1.34.0
|
github.com/rs/zerolog v1.34.0
|
||||||
|
github.com/smacker/go-tree-sitter v0.0.0-20240827094217-dd81d9e9be82
|
||||||
github.com/stretchr/testify v1.11.1
|
github.com/stretchr/testify v1.11.1
|
||||||
github.com/sugarme/tokenizer v0.3.0
|
github.com/sugarme/tokenizer v0.3.0
|
||||||
github.com/yalue/onnxruntime_go v1.25.0
|
github.com/yalue/onnxruntime_go v1.25.0
|
||||||
@@ -26,6 +27,7 @@ require (
|
|||||||
github.com/pmezard/go-difflib v1.0.0 // indirect
|
github.com/pmezard/go-difflib v1.0.0 // indirect
|
||||||
github.com/rivo/uniseg v0.4.7 // indirect
|
github.com/rivo/uniseg v0.4.7 // indirect
|
||||||
github.com/schollz/progressbar/v2 v2.15.0 // indirect
|
github.com/schollz/progressbar/v2 v2.15.0 // indirect
|
||||||
|
github.com/stretchr/objx v0.5.2 // indirect
|
||||||
github.com/sugarme/regexpset v0.0.0-20200920021344-4d4ec8eaf93c // indirect
|
github.com/sugarme/regexpset v0.0.0-20200920021344-4d4ec8eaf93c // indirect
|
||||||
golang.org/x/sys v0.39.0 // indirect
|
golang.org/x/sys v0.39.0 // indirect
|
||||||
golang.org/x/text v0.32.0 // indirect
|
golang.org/x/text v0.32.0 // indirect
|
||||||
|
|||||||
@@ -38,7 +38,11 @@ github.com/rs/zerolog v1.34.0 h1:k43nTLIwcTVQAncfCw4KZ2VY6ukYoZaBPNOE8txlOeY=
|
|||||||
github.com/rs/zerolog v1.34.0/go.mod h1:bJsvje4Z08ROH4Nhs5iH600c3IkWhwp44iRc54W6wYQ=
|
github.com/rs/zerolog v1.34.0/go.mod h1:bJsvje4Z08ROH4Nhs5iH600c3IkWhwp44iRc54W6wYQ=
|
||||||
github.com/schollz/progressbar/v2 v2.15.0 h1:dVzHQ8fHRmtPjD3K10jT3Qgn/+H+92jhPrhmxIJfDz8=
|
github.com/schollz/progressbar/v2 v2.15.0 h1:dVzHQ8fHRmtPjD3K10jT3Qgn/+H+92jhPrhmxIJfDz8=
|
||||||
github.com/schollz/progressbar/v2 v2.15.0/go.mod h1:UdPq3prGkfQ7MOzZKlDRpYKcFqEMczbD7YmbPgpzKMI=
|
github.com/schollz/progressbar/v2 v2.15.0/go.mod h1:UdPq3prGkfQ7MOzZKlDRpYKcFqEMczbD7YmbPgpzKMI=
|
||||||
|
github.com/smacker/go-tree-sitter v0.0.0-20240827094217-dd81d9e9be82 h1:6C8qej6f1bStuePVkLSFxoU22XBS165D3klxlzRg8F4=
|
||||||
|
github.com/smacker/go-tree-sitter v0.0.0-20240827094217-dd81d9e9be82/go.mod h1:xe4pgH49k4SsmkQq5OT8abwhWmnzkhpgnXeekbx2efw=
|
||||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||||
|
github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY=
|
||||||
|
github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
|
||||||
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
||||||
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
|
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
|
||||||
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
|
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
|
||||||
|
|||||||
@@ -0,0 +1,285 @@
|
|||||||
|
// Package golang provides AST-aware chunking for Go source files.
|
||||||
|
package golang
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"go/ast"
|
||||||
|
"go/parser"
|
||||||
|
"go/token"
|
||||||
|
"os"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"github.com/lukaszraczylo/claude-mnemonic/internal/chunking"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Chunker implements AST-aware chunking for Go files.
|
||||||
|
type Chunker struct {
|
||||||
|
options chunking.ChunkOptions
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewChunker creates a new Go chunker.
|
||||||
|
func NewChunker(options chunking.ChunkOptions) *Chunker {
|
||||||
|
return &Chunker{options: options}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Language returns the language this chunker supports.
|
||||||
|
func (c *Chunker) Language() chunking.Language {
|
||||||
|
return chunking.LanguageGo
|
||||||
|
}
|
||||||
|
|
||||||
|
// SupportedExtensions returns the file extensions this chunker handles.
|
||||||
|
func (c *Chunker) SupportedExtensions() []string {
|
||||||
|
return []string{".go"}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Chunk parses a Go source file and returns semantic code chunks.
|
||||||
|
func (c *Chunker) Chunk(ctx context.Context, filePath string) ([]chunking.Chunk, error) {
|
||||||
|
// Read file content
|
||||||
|
content, err := os.ReadFile(filePath)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("read file: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Parse the Go file
|
||||||
|
fset := token.NewFileSet()
|
||||||
|
file, err := parser.ParseFile(fset, filePath, content, parser.ParseComments)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("parse Go file: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
chunks := make([]chunking.Chunk, 0)
|
||||||
|
sourceLines := strings.Split(string(content), "\n")
|
||||||
|
|
||||||
|
// Extract chunks from declarations
|
||||||
|
for _, decl := range file.Decls {
|
||||||
|
switch d := decl.(type) {
|
||||||
|
case *ast.FuncDecl:
|
||||||
|
chunk := c.extractFunction(fset, d, sourceLines, filePath)
|
||||||
|
if chunk != nil {
|
||||||
|
chunks = append(chunks, *chunk)
|
||||||
|
}
|
||||||
|
case *ast.GenDecl:
|
||||||
|
extracted := c.extractGenDecl(fset, d, sourceLines, filePath)
|
||||||
|
chunks = append(chunks, extracted...)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return chunks, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// extractFunction extracts a function or method declaration as a chunk.
|
||||||
|
func (c *Chunker) extractFunction(fset *token.FileSet, fn *ast.FuncDecl, sourceLines []string, filePath string) *chunking.Chunk {
|
||||||
|
// Skip unexported if configured
|
||||||
|
if !c.options.IncludePrivate && !fn.Name.IsExported() {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
startPos := fset.Position(fn.Pos())
|
||||||
|
endPos := fset.Position(fn.End())
|
||||||
|
|
||||||
|
chunk := &chunking.Chunk{
|
||||||
|
FilePath: filePath,
|
||||||
|
Language: chunking.LanguageGo,
|
||||||
|
Name: fn.Name.Name,
|
||||||
|
StartLine: startPos.Line,
|
||||||
|
EndLine: endPos.Line,
|
||||||
|
}
|
||||||
|
|
||||||
|
// Determine if this is a method or a function
|
||||||
|
if fn.Recv != nil && len(fn.Recv.List) > 0 {
|
||||||
|
chunk.Type = chunking.ChunkTypeMethod
|
||||||
|
chunk.ParentName = c.extractReceiverType(fn.Recv)
|
||||||
|
} else {
|
||||||
|
chunk.Type = chunking.ChunkTypeFunction
|
||||||
|
}
|
||||||
|
|
||||||
|
// Extract content
|
||||||
|
chunk.Content = c.extractLines(sourceLines, startPos.Line, endPos.Line)
|
||||||
|
|
||||||
|
// Extract signature (function declaration without body)
|
||||||
|
chunk.Signature = c.extractFunctionSignature(fn, fset, sourceLines)
|
||||||
|
|
||||||
|
// Extract doc comment
|
||||||
|
if c.options.IncludeDocComments && fn.Doc != nil {
|
||||||
|
chunk.DocComment = strings.TrimSpace(fn.Doc.Text())
|
||||||
|
}
|
||||||
|
|
||||||
|
return chunk
|
||||||
|
}
|
||||||
|
|
||||||
|
// extractGenDecl extracts general declarations (type, const, var).
|
||||||
|
func (c *Chunker) extractGenDecl(fset *token.FileSet, gd *ast.GenDecl, sourceLines []string, filePath string) []chunking.Chunk {
|
||||||
|
var chunks []chunking.Chunk
|
||||||
|
|
||||||
|
for _, spec := range gd.Specs {
|
||||||
|
switch s := spec.(type) {
|
||||||
|
case *ast.TypeSpec:
|
||||||
|
chunk := c.extractTypeSpec(fset, gd, s, sourceLines, filePath)
|
||||||
|
if chunk != nil {
|
||||||
|
chunks = append(chunks, *chunk)
|
||||||
|
}
|
||||||
|
case *ast.ValueSpec:
|
||||||
|
// Handle const and var declarations
|
||||||
|
chunk := c.extractValueSpec(fset, gd, s, sourceLines, filePath)
|
||||||
|
if chunk != nil {
|
||||||
|
chunks = append(chunks, *chunk)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return chunks
|
||||||
|
}
|
||||||
|
|
||||||
|
// extractTypeSpec extracts a type declaration (struct, interface, type alias).
|
||||||
|
func (c *Chunker) extractTypeSpec(fset *token.FileSet, gd *ast.GenDecl, ts *ast.TypeSpec, sourceLines []string, filePath string) *chunking.Chunk {
|
||||||
|
// Skip unexported if configured
|
||||||
|
if !c.options.IncludePrivate && !ts.Name.IsExported() {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
startPos := fset.Position(gd.Pos())
|
||||||
|
endPos := fset.Position(gd.End())
|
||||||
|
|
||||||
|
chunk := &chunking.Chunk{
|
||||||
|
FilePath: filePath,
|
||||||
|
Language: chunking.LanguageGo,
|
||||||
|
Name: ts.Name.Name,
|
||||||
|
StartLine: startPos.Line,
|
||||||
|
EndLine: endPos.Line,
|
||||||
|
Content: c.extractLines(sourceLines, startPos.Line, endPos.Line),
|
||||||
|
}
|
||||||
|
|
||||||
|
// Determine chunk type based on type expression
|
||||||
|
switch ts.Type.(type) {
|
||||||
|
case *ast.StructType:
|
||||||
|
chunk.Type = chunking.ChunkTypeClass // Treat struct as class
|
||||||
|
case *ast.InterfaceType:
|
||||||
|
chunk.Type = chunking.ChunkTypeInterface
|
||||||
|
default:
|
||||||
|
chunk.Type = chunking.ChunkTypeType
|
||||||
|
}
|
||||||
|
|
||||||
|
// Extract doc comment
|
||||||
|
if c.options.IncludeDocComments && gd.Doc != nil {
|
||||||
|
chunk.DocComment = strings.TrimSpace(gd.Doc.Text())
|
||||||
|
}
|
||||||
|
|
||||||
|
return chunk
|
||||||
|
}
|
||||||
|
|
||||||
|
// extractValueSpec extracts const or var declarations.
|
||||||
|
func (c *Chunker) extractValueSpec(fset *token.FileSet, gd *ast.GenDecl, vs *ast.ValueSpec, sourceLines []string, filePath string) *chunking.Chunk {
|
||||||
|
// Skip if all names are unexported and we're excluding private
|
||||||
|
if !c.options.IncludePrivate {
|
||||||
|
allUnexported := true
|
||||||
|
for _, name := range vs.Names {
|
||||||
|
if name.IsExported() {
|
||||||
|
allUnexported = false
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if allUnexported {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
startPos := fset.Position(gd.Pos())
|
||||||
|
endPos := fset.Position(gd.End())
|
||||||
|
|
||||||
|
// Use first name as the chunk name, join multiple if present
|
||||||
|
names := make([]string, len(vs.Names))
|
||||||
|
for i, name := range vs.Names {
|
||||||
|
names[i] = name.Name
|
||||||
|
}
|
||||||
|
|
||||||
|
chunk := &chunking.Chunk{
|
||||||
|
FilePath: filePath,
|
||||||
|
Language: chunking.LanguageGo,
|
||||||
|
Name: strings.Join(names, ", "),
|
||||||
|
StartLine: startPos.Line,
|
||||||
|
EndLine: endPos.Line,
|
||||||
|
Content: c.extractLines(sourceLines, startPos.Line, endPos.Line),
|
||||||
|
}
|
||||||
|
|
||||||
|
// Set type based on token
|
||||||
|
if gd.Tok == token.CONST {
|
||||||
|
chunk.Type = chunking.ChunkTypeConst
|
||||||
|
} else {
|
||||||
|
chunk.Type = chunking.ChunkTypeVar
|
||||||
|
}
|
||||||
|
|
||||||
|
// Extract doc comment
|
||||||
|
if c.options.IncludeDocComments && gd.Doc != nil {
|
||||||
|
chunk.DocComment = strings.TrimSpace(gd.Doc.Text())
|
||||||
|
}
|
||||||
|
|
||||||
|
return chunk
|
||||||
|
}
|
||||||
|
|
||||||
|
// extractReceiverType extracts the receiver type name from a method.
|
||||||
|
func (c *Chunker) extractReceiverType(recv *ast.FieldList) string {
|
||||||
|
if len(recv.List) == 0 {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
|
field := recv.List[0]
|
||||||
|
switch t := field.Type.(type) {
|
||||||
|
case *ast.Ident:
|
||||||
|
return t.Name
|
||||||
|
case *ast.StarExpr:
|
||||||
|
if ident, ok := t.X.(*ast.Ident); ok {
|
||||||
|
return ident.Name
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
|
// extractFunctionSignature extracts the function signature without the body.
|
||||||
|
func (c *Chunker) extractFunctionSignature(fn *ast.FuncDecl, fset *token.FileSet, sourceLines []string) string {
|
||||||
|
if fn.Body == nil {
|
||||||
|
// No body, return entire declaration
|
||||||
|
startPos := fset.Position(fn.Pos())
|
||||||
|
endPos := fset.Position(fn.End())
|
||||||
|
return c.extractLines(sourceLines, startPos.Line, endPos.Line)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Extract from start of function to just before body
|
||||||
|
startPos := fset.Position(fn.Pos())
|
||||||
|
bodyPos := fset.Position(fn.Body.Pos())
|
||||||
|
|
||||||
|
// If body is on the same line, extract just that line up to the opening brace
|
||||||
|
if startPos.Line == bodyPos.Line {
|
||||||
|
line := sourceLines[startPos.Line-1]
|
||||||
|
// Find the opening brace position
|
||||||
|
if idx := strings.Index(line[startPos.Column-1:], "{"); idx >= 0 {
|
||||||
|
return strings.TrimSpace(line[startPos.Column-1 : startPos.Column-1+idx])
|
||||||
|
}
|
||||||
|
return strings.TrimSpace(line[startPos.Column-1:])
|
||||||
|
}
|
||||||
|
|
||||||
|
// Get lines from start to the line containing the opening brace
|
||||||
|
sig := c.extractLines(sourceLines, startPos.Line, bodyPos.Line)
|
||||||
|
// Remove the opening brace and anything after it
|
||||||
|
if idx := strings.Index(sig, "{"); idx >= 0 {
|
||||||
|
sig = sig[:idx]
|
||||||
|
}
|
||||||
|
return strings.TrimSpace(sig)
|
||||||
|
}
|
||||||
|
|
||||||
|
// extractLines extracts a range of lines from source (1-indexed, inclusive).
|
||||||
|
func (c *Chunker) extractLines(lines []string, start, end int) string {
|
||||||
|
if start < 1 || end < start || start > len(lines) {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
|
// Adjust for 0-indexed array (start and end are 1-indexed)
|
||||||
|
startIdx := start - 1
|
||||||
|
endIdx := end
|
||||||
|
if endIdx > len(lines) {
|
||||||
|
endIdx = len(lines)
|
||||||
|
}
|
||||||
|
|
||||||
|
return strings.Join(lines[startIdx:endIdx], "\n")
|
||||||
|
}
|
||||||
@@ -0,0 +1,214 @@
|
|||||||
|
package golang
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/lukaszraczylo/claude-mnemonic/internal/chunking"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestGoChunker_BasicFunctions(t *testing.T) {
|
||||||
|
// Create temp test file
|
||||||
|
tmpDir := t.TempDir()
|
||||||
|
testFile := filepath.Join(tmpDir, "test.go")
|
||||||
|
|
||||||
|
testCode := `package main
|
||||||
|
|
||||||
|
import "fmt"
|
||||||
|
|
||||||
|
// Greet prints a greeting message
|
||||||
|
func Greet(name string) {
|
||||||
|
fmt.Printf("Hello, %s!\n", name)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Add adds two numbers
|
||||||
|
func Add(a, b int) int {
|
||||||
|
return a + b
|
||||||
|
}
|
||||||
|
|
||||||
|
// unexported function should be included by default
|
||||||
|
func helper() string {
|
||||||
|
return "helper"
|
||||||
|
}
|
||||||
|
`
|
||||||
|
|
||||||
|
if err := os.WriteFile(testFile, []byte(testCode), 0600); err != nil {
|
||||||
|
t.Fatalf("Failed to create test file: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Create chunker with default options
|
||||||
|
chunker := NewChunker(chunking.DefaultChunkOptions())
|
||||||
|
|
||||||
|
// Chunk the file
|
||||||
|
chunks, err := chunker.Chunk(context.Background(), testFile)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Chunk() failed: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Verify we got all functions
|
||||||
|
if len(chunks) != 3 {
|
||||||
|
t.Errorf("Expected 3 chunks (Greet, Add, helper), got %d", len(chunks))
|
||||||
|
}
|
||||||
|
|
||||||
|
// Verify chunk details
|
||||||
|
expectedNames := map[string]bool{
|
||||||
|
"Greet": false,
|
||||||
|
"Add": false,
|
||||||
|
"helper": false,
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, chunk := range chunks {
|
||||||
|
if chunk.Type != chunking.ChunkTypeFunction {
|
||||||
|
t.Errorf("Expected chunk type 'function', got '%s'", chunk.Type)
|
||||||
|
}
|
||||||
|
|
||||||
|
if chunk.Language != chunking.LanguageGo {
|
||||||
|
t.Errorf("Expected language 'go', got '%s'", chunk.Language)
|
||||||
|
}
|
||||||
|
|
||||||
|
if _, ok := expectedNames[chunk.Name]; !ok {
|
||||||
|
t.Errorf("Unexpected function name: %s", chunk.Name)
|
||||||
|
} else {
|
||||||
|
expectedNames[chunk.Name] = true
|
||||||
|
}
|
||||||
|
|
||||||
|
// Verify content is non-empty
|
||||||
|
if chunk.Content == "" {
|
||||||
|
t.Errorf("Chunk %s has empty content", chunk.Name)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Verify signature is present for functions
|
||||||
|
if chunk.Signature == "" {
|
||||||
|
t.Errorf("Chunk %s has empty signature", chunk.Name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Verify all expected functions were found
|
||||||
|
for name, found := range expectedNames {
|
||||||
|
if !found {
|
||||||
|
t.Errorf("Expected function %s not found", name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestGoChunker_StructsAndMethods(t *testing.T) {
|
||||||
|
tmpDir := t.TempDir()
|
||||||
|
testFile := filepath.Join(tmpDir, "test.go")
|
||||||
|
|
||||||
|
testCode := `package main
|
||||||
|
|
||||||
|
// User represents a user
|
||||||
|
type User struct {
|
||||||
|
ID int
|
||||||
|
Name string
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetName returns the user's name
|
||||||
|
func (u *User) GetName() string {
|
||||||
|
return u.Name
|
||||||
|
}
|
||||||
|
|
||||||
|
// SetName sets the user's name
|
||||||
|
func (u *User) SetName(name string) {
|
||||||
|
u.Name = name
|
||||||
|
}
|
||||||
|
`
|
||||||
|
|
||||||
|
if err := os.WriteFile(testFile, []byte(testCode), 0600); err != nil {
|
||||||
|
t.Fatalf("Failed to create test file: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
chunker := NewChunker(chunking.DefaultChunkOptions())
|
||||||
|
chunks, err := chunker.Chunk(context.Background(), testFile)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Chunk() failed: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Should have 1 struct + 2 methods = 3 chunks
|
||||||
|
if len(chunks) != 3 {
|
||||||
|
t.Errorf("Expected 3 chunks (User struct, GetName, SetName), got %d", len(chunks))
|
||||||
|
}
|
||||||
|
|
||||||
|
// Find the struct and methods
|
||||||
|
var structChunk, getNameChunk, setNameChunk *chunking.Chunk
|
||||||
|
for i := range chunks {
|
||||||
|
switch chunks[i].Name {
|
||||||
|
case "User":
|
||||||
|
structChunk = &chunks[i]
|
||||||
|
case "GetName":
|
||||||
|
getNameChunk = &chunks[i]
|
||||||
|
case "SetName":
|
||||||
|
setNameChunk = &chunks[i]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Verify struct
|
||||||
|
if structChunk == nil {
|
||||||
|
t.Fatal("User struct not found")
|
||||||
|
}
|
||||||
|
if structChunk.Type != chunking.ChunkTypeClass {
|
||||||
|
t.Errorf("Expected User to be ChunkTypeClass, got %s", structChunk.Type)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Verify methods
|
||||||
|
if getNameChunk == nil {
|
||||||
|
t.Fatal("GetName method not found")
|
||||||
|
}
|
||||||
|
if getNameChunk.Type != chunking.ChunkTypeMethod {
|
||||||
|
t.Errorf("Expected GetName to be ChunkTypeMethod, got %s", getNameChunk.Type)
|
||||||
|
}
|
||||||
|
if getNameChunk.ParentName != "User" {
|
||||||
|
t.Errorf("Expected GetName parent to be 'User', got '%s'", getNameChunk.ParentName)
|
||||||
|
}
|
||||||
|
|
||||||
|
if setNameChunk == nil {
|
||||||
|
t.Fatal("SetName method not found")
|
||||||
|
}
|
||||||
|
if setNameChunk.Type != chunking.ChunkTypeMethod {
|
||||||
|
t.Errorf("Expected SetName to be ChunkTypeMethod, got %s", setNameChunk.Type)
|
||||||
|
}
|
||||||
|
if setNameChunk.ParentName != "User" {
|
||||||
|
t.Errorf("Expected SetName parent to be 'User', got '%s'", setNameChunk.ParentName)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestGoChunker_DocComments(t *testing.T) {
|
||||||
|
tmpDir := t.TempDir()
|
||||||
|
testFile := filepath.Join(tmpDir, "test.go")
|
||||||
|
|
||||||
|
testCode := `package main
|
||||||
|
|
||||||
|
// Calculate performs a calculation.
|
||||||
|
// It takes two integers and returns their sum.
|
||||||
|
func Calculate(a, b int) int {
|
||||||
|
return a + b
|
||||||
|
}
|
||||||
|
`
|
||||||
|
|
||||||
|
if err := os.WriteFile(testFile, []byte(testCode), 0600); err != nil {
|
||||||
|
t.Fatalf("Failed to create test file: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
chunker := NewChunker(chunking.DefaultChunkOptions())
|
||||||
|
chunks, err := chunker.Chunk(context.Background(), testFile)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Chunk() failed: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(chunks) != 1 {
|
||||||
|
t.Fatalf("Expected 1 chunk, got %d", len(chunks))
|
||||||
|
}
|
||||||
|
|
||||||
|
chunk := chunks[0]
|
||||||
|
if chunk.DocComment == "" {
|
||||||
|
t.Error("Expected doc comment to be present")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Doc comment should contain the comment text
|
||||||
|
expectedComment := "Calculate performs a calculation.\nIt takes two integers and returns their sum."
|
||||||
|
if chunk.DocComment != expectedComment {
|
||||||
|
t.Errorf("Expected doc comment '%s', got '%s'", expectedComment, chunk.DocComment)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,106 @@
|
|||||||
|
package chunking
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Manager dispatches files to appropriate language-specific chunkers.
|
||||||
|
type Manager struct {
|
||||||
|
chunkers map[string]Chunker // extension -> chunker
|
||||||
|
options ChunkOptions
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewManager creates a new chunking manager with the given chunkers.
|
||||||
|
func NewManager(chunkers []Chunker, options ChunkOptions) *Manager {
|
||||||
|
m := &Manager{
|
||||||
|
chunkers: make(map[string]Chunker),
|
||||||
|
options: options,
|
||||||
|
}
|
||||||
|
|
||||||
|
// Register chunkers by their supported extensions
|
||||||
|
for _, chunker := range chunkers {
|
||||||
|
for _, ext := range chunker.SupportedExtensions() {
|
||||||
|
m.chunkers[ext] = chunker
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return m
|
||||||
|
}
|
||||||
|
|
||||||
|
// ChunkFile chunks a single file using the appropriate language chunker.
|
||||||
|
// Returns an error if no chunker is found for the file extension.
|
||||||
|
func (m *Manager) ChunkFile(ctx context.Context, filePath string) ([]Chunk, error) {
|
||||||
|
ext := strings.ToLower(filepath.Ext(filePath))
|
||||||
|
chunker, ok := m.chunkers[ext]
|
||||||
|
if !ok {
|
||||||
|
return nil, fmt.Errorf("no chunker for extension %s", ext)
|
||||||
|
}
|
||||||
|
|
||||||
|
chunks, err := chunker.Chunk(ctx, filePath)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("chunk %s: %w", filePath, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Apply options-based filtering
|
||||||
|
filtered := make([]Chunk, 0, len(chunks))
|
||||||
|
for _, chunk := range chunks {
|
||||||
|
// Filter by minimum lines
|
||||||
|
if m.options.MinLines > 0 {
|
||||||
|
lineCount := chunk.EndLine - chunk.StartLine + 1
|
||||||
|
if lineCount < m.options.MinLines {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Filter by maximum chunk size
|
||||||
|
if m.options.MaxChunkSize > 0 && len(chunk.Content) > m.options.MaxChunkSize {
|
||||||
|
// TODO: Consider splitting large chunks intelligently
|
||||||
|
// For now, skip chunks that are too large
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
filtered = append(filtered, chunk)
|
||||||
|
}
|
||||||
|
|
||||||
|
return filtered, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// ChunkFiles chunks multiple files in parallel.
|
||||||
|
// Returns a map of file path to chunks, and any errors encountered.
|
||||||
|
// Errors for individual files do not stop processing of other files.
|
||||||
|
func (m *Manager) ChunkFiles(ctx context.Context, filePaths []string) (map[string][]Chunk, []error) {
|
||||||
|
results := make(map[string][]Chunk)
|
||||||
|
var errors []error
|
||||||
|
|
||||||
|
for _, filePath := range filePaths {
|
||||||
|
chunks, err := m.ChunkFile(ctx, filePath)
|
||||||
|
if err != nil {
|
||||||
|
errors = append(errors, fmt.Errorf("%s: %w", filePath, err))
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if len(chunks) > 0 {
|
||||||
|
results[filePath] = chunks
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return results, errors
|
||||||
|
}
|
||||||
|
|
||||||
|
// SupportsFile checks if the manager can chunk the given file based on extension.
|
||||||
|
func (m *Manager) SupportsFile(filePath string) bool {
|
||||||
|
ext := strings.ToLower(filepath.Ext(filePath))
|
||||||
|
_, ok := m.chunkers[ext]
|
||||||
|
return ok
|
||||||
|
}
|
||||||
|
|
||||||
|
// SupportedExtensions returns all file extensions supported by registered chunkers.
|
||||||
|
func (m *Manager) SupportedExtensions() []string {
|
||||||
|
exts := make([]string, 0, len(m.chunkers))
|
||||||
|
for ext := range m.chunkers {
|
||||||
|
exts = append(exts, ext)
|
||||||
|
}
|
||||||
|
return exts
|
||||||
|
}
|
||||||
@@ -0,0 +1,162 @@
|
|||||||
|
package chunking
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
// mockChunker is a test chunker that returns dummy chunks
|
||||||
|
type mockChunker struct{}
|
||||||
|
|
||||||
|
func (m *mockChunker) Chunk(ctx context.Context, filePath string) ([]Chunk, error) {
|
||||||
|
// Just return an empty chunk for testing
|
||||||
|
return []Chunk{
|
||||||
|
{
|
||||||
|
FilePath: filePath,
|
||||||
|
Language: LanguageGo,
|
||||||
|
Type: ChunkTypeFunction,
|
||||||
|
Name: "TestFunc",
|
||||||
|
StartLine: 1,
|
||||||
|
EndLine: 1,
|
||||||
|
Content: "test",
|
||||||
|
},
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *mockChunker) Language() Language {
|
||||||
|
return LanguageGo
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *mockChunker) SupportedExtensions() []string {
|
||||||
|
return []string{".go", ".py", ".ts"}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestManager_ChunkMultipleFiles(t *testing.T) {
|
||||||
|
tmpDir := t.TempDir()
|
||||||
|
|
||||||
|
// Create a Go file
|
||||||
|
goFile := filepath.Join(tmpDir, "test.go")
|
||||||
|
goCode := `package main
|
||||||
|
|
||||||
|
func Hello() string {
|
||||||
|
return "hello"
|
||||||
|
}
|
||||||
|
`
|
||||||
|
if err := os.WriteFile(goFile, []byte(goCode), 0600); err != nil {
|
||||||
|
t.Fatalf("Failed to create Go file: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Create a Python file
|
||||||
|
pyFile := filepath.Join(tmpDir, "test.py")
|
||||||
|
pyCode := `def greet(name):
|
||||||
|
return f"Hello, {name}!"
|
||||||
|
|
||||||
|
class User:
|
||||||
|
def __init__(self, name):
|
||||||
|
self.name = name
|
||||||
|
`
|
||||||
|
if err := os.WriteFile(pyFile, []byte(pyCode), 0600); err != nil {
|
||||||
|
t.Fatalf("Failed to create Python file: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Create a TypeScript file
|
||||||
|
tsFile := filepath.Join(tmpDir, "test.ts")
|
||||||
|
tsCode := `function add(a: number, b: number): number {
|
||||||
|
return a + b;
|
||||||
|
}
|
||||||
|
|
||||||
|
class Calculator {
|
||||||
|
multiply(a: number, b: number): number {
|
||||||
|
return a * b;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
`
|
||||||
|
if err := os.WriteFile(tsFile, []byte(tsCode), 0600); err != nil {
|
||||||
|
t.Fatalf("Failed to create TypeScript file: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Create manager
|
||||||
|
manager := NewManager([]Chunker{&mockChunker{}}, DefaultChunkOptions())
|
||||||
|
|
||||||
|
// Test SupportsFile
|
||||||
|
if !manager.SupportsFile(goFile) {
|
||||||
|
t.Error("Manager should support .go files")
|
||||||
|
}
|
||||||
|
if !manager.SupportsFile(pyFile) {
|
||||||
|
t.Error("Manager should support .py files")
|
||||||
|
}
|
||||||
|
if !manager.SupportsFile(tsFile) {
|
||||||
|
t.Error("Manager should support .ts files")
|
||||||
|
}
|
||||||
|
|
||||||
|
unsupportedFile := filepath.Join(tmpDir, "test.txt")
|
||||||
|
if manager.SupportsFile(unsupportedFile) {
|
||||||
|
t.Error("Manager should not support .txt files")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Test ChunkFiles
|
||||||
|
results, errs := manager.ChunkFiles(context.Background(), []string{goFile, pyFile, tsFile})
|
||||||
|
if len(errs) > 0 {
|
||||||
|
t.Errorf("ChunkFiles returned errors: %v", errs)
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(results) != 3 {
|
||||||
|
t.Errorf("Expected results for 3 files, got %d", len(results))
|
||||||
|
}
|
||||||
|
|
||||||
|
// Verify each file has chunks
|
||||||
|
for _, file := range []string{goFile, pyFile, tsFile} {
|
||||||
|
if chunks, ok := results[file]; !ok || len(chunks) == 0 {
|
||||||
|
t.Errorf("No chunks found for file %s", file)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// mockChunkerWithExts is a test chunker with configurable extensions
|
||||||
|
type mockChunkerWithExts struct {
|
||||||
|
exts []string
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *mockChunkerWithExts) Chunk(ctx context.Context, filePath string) ([]Chunk, error) {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *mockChunkerWithExts) Language() Language {
|
||||||
|
return LanguageGo
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *mockChunkerWithExts) SupportedExtensions() []string {
|
||||||
|
return m.exts
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestManager_SupportedExtensions(t *testing.T) {
|
||||||
|
|
||||||
|
// Create manager with mock chunkers
|
||||||
|
manager := NewManager([]Chunker{
|
||||||
|
&mockChunkerWithExts{exts: []string{".go"}},
|
||||||
|
&mockChunkerWithExts{exts: []string{".py", ".pyw"}},
|
||||||
|
}, DefaultChunkOptions())
|
||||||
|
|
||||||
|
exts := manager.SupportedExtensions()
|
||||||
|
expectedExts := map[string]bool{
|
||||||
|
".go": false,
|
||||||
|
".py": false,
|
||||||
|
".pyw": false,
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, ext := range exts {
|
||||||
|
if _, ok := expectedExts[ext]; ok {
|
||||||
|
expectedExts[ext] = true
|
||||||
|
} else {
|
||||||
|
t.Errorf("Unexpected extension: %s", ext)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
for ext, found := range expectedExts {
|
||||||
|
if !found {
|
||||||
|
t.Errorf("Expected extension %s not found", ext)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,291 @@
|
|||||||
|
// Package python provides AST-aware chunking for Python source files using tree-sitter.
|
||||||
|
package python
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
sitter "github.com/smacker/go-tree-sitter"
|
||||||
|
"github.com/smacker/go-tree-sitter/python"
|
||||||
|
|
||||||
|
"github.com/lukaszraczylo/claude-mnemonic/internal/chunking"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Chunker implements AST-aware chunking for Python files.
|
||||||
|
type Chunker struct {
|
||||||
|
parser *sitter.Parser
|
||||||
|
options chunking.ChunkOptions
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewChunker creates a new Python chunker.
|
||||||
|
func NewChunker(options chunking.ChunkOptions) *Chunker {
|
||||||
|
parser := sitter.NewParser()
|
||||||
|
parser.SetLanguage(python.GetLanguage())
|
||||||
|
|
||||||
|
return &Chunker{
|
||||||
|
options: options,
|
||||||
|
parser: parser,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Language returns the language this chunker supports.
|
||||||
|
func (c *Chunker) Language() chunking.Language {
|
||||||
|
return chunking.LanguagePython
|
||||||
|
}
|
||||||
|
|
||||||
|
// SupportedExtensions returns the file extensions this chunker handles.
|
||||||
|
func (c *Chunker) SupportedExtensions() []string {
|
||||||
|
return []string{".py"}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Chunk parses a Python source file and returns semantic code chunks.
|
||||||
|
func (c *Chunker) Chunk(ctx context.Context, filePath string) ([]chunking.Chunk, error) {
|
||||||
|
// Read file content
|
||||||
|
content, err := os.ReadFile(filePath)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("read file: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Parse the Python file
|
||||||
|
tree, err := c.parser.ParseCtx(ctx, nil, content)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("parse Python file: %w", err)
|
||||||
|
}
|
||||||
|
defer tree.Close()
|
||||||
|
|
||||||
|
sourceLines := strings.Split(string(content), "\n")
|
||||||
|
chunks := make([]chunking.Chunk, 0)
|
||||||
|
|
||||||
|
// Walk the AST and extract chunks
|
||||||
|
c.walkNode(tree.RootNode(), content, sourceLines, filePath, "", &chunks)
|
||||||
|
|
||||||
|
return chunks, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// walkNode recursively walks the tree-sitter AST and extracts chunks.
|
||||||
|
func (c *Chunker) walkNode(node *sitter.Node, source []byte, sourceLines []string, filePath string, parentName string, chunks *[]chunking.Chunk) {
|
||||||
|
nodeType := node.Type()
|
||||||
|
|
||||||
|
switch nodeType {
|
||||||
|
case "function_definition":
|
||||||
|
chunk := c.extractFunction(node, source, sourceLines, filePath, parentName)
|
||||||
|
if chunk != nil {
|
||||||
|
*chunks = append(*chunks, *chunk)
|
||||||
|
}
|
||||||
|
|
||||||
|
case "class_definition":
|
||||||
|
chunk := c.extractClass(node, source, sourceLines, filePath)
|
||||||
|
if chunk != nil {
|
||||||
|
*chunks = append(*chunks, *chunk)
|
||||||
|
|
||||||
|
// Walk class body to find methods
|
||||||
|
for i := 0; i < int(node.ChildCount()); i++ {
|
||||||
|
child := node.Child(i)
|
||||||
|
if child.Type() == "block" {
|
||||||
|
c.walkNode(child, source, sourceLines, filePath, chunk.Name, chunks)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return // Don't walk children again
|
||||||
|
|
||||||
|
case "block":
|
||||||
|
// Walk statements in block
|
||||||
|
for i := 0; i < int(node.ChildCount()); i++ {
|
||||||
|
c.walkNode(node.Child(i), source, sourceLines, filePath, parentName, chunks)
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// Walk all children
|
||||||
|
for i := 0; i < int(node.ChildCount()); i++ {
|
||||||
|
c.walkNode(node.Child(i), source, sourceLines, filePath, parentName, chunks)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// extractFunction extracts a function definition chunk.
|
||||||
|
func (c *Chunker) extractFunction(node *sitter.Node, source []byte, sourceLines []string, filePath string, parentName string) *chunking.Chunk {
|
||||||
|
// Find function name
|
||||||
|
var nameNode *sitter.Node
|
||||||
|
for i := 0; i < int(node.ChildCount()); i++ {
|
||||||
|
child := node.Child(i)
|
||||||
|
if child.Type() == "identifier" {
|
||||||
|
nameNode = child
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if nameNode == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
name := nameNode.Content(source)
|
||||||
|
|
||||||
|
// Skip private functions if configured
|
||||||
|
if !c.options.IncludePrivate && strings.HasPrefix(name, "_") && !strings.HasPrefix(name, "__") {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
startLine := int(node.StartPoint().Row) + 1
|
||||||
|
endLine := int(node.EndPoint().Row) + 1
|
||||||
|
|
||||||
|
chunk := &chunking.Chunk{
|
||||||
|
FilePath: filePath,
|
||||||
|
Language: chunking.LanguagePython,
|
||||||
|
Name: name,
|
||||||
|
ParentName: parentName,
|
||||||
|
StartLine: startLine,
|
||||||
|
EndLine: endLine,
|
||||||
|
Content: c.extractLines(sourceLines, startLine, endLine),
|
||||||
|
}
|
||||||
|
|
||||||
|
// Determine if this is a method or function
|
||||||
|
if parentName != "" {
|
||||||
|
chunk.Type = chunking.ChunkTypeMethod
|
||||||
|
} else {
|
||||||
|
chunk.Type = chunking.ChunkTypeFunction
|
||||||
|
}
|
||||||
|
|
||||||
|
// Extract signature (def line)
|
||||||
|
chunk.Signature = c.extractFunctionSignature(node, source, sourceLines)
|
||||||
|
|
||||||
|
// Extract docstring as doc comment
|
||||||
|
if c.options.IncludeDocComments {
|
||||||
|
chunk.DocComment = c.extractDocstring(node, source)
|
||||||
|
}
|
||||||
|
|
||||||
|
return chunk
|
||||||
|
}
|
||||||
|
|
||||||
|
// extractClass extracts a class definition chunk.
|
||||||
|
func (c *Chunker) extractClass(node *sitter.Node, source []byte, sourceLines []string, filePath string) *chunking.Chunk {
|
||||||
|
// Find class name
|
||||||
|
var nameNode *sitter.Node
|
||||||
|
for i := 0; i < int(node.ChildCount()); i++ {
|
||||||
|
child := node.Child(i)
|
||||||
|
if child.Type() == "identifier" {
|
||||||
|
nameNode = child
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if nameNode == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
name := nameNode.Content(source)
|
||||||
|
|
||||||
|
// Skip private classes if configured
|
||||||
|
if !c.options.IncludePrivate && strings.HasPrefix(name, "_") && !strings.HasPrefix(name, "__") {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
startLine := int(node.StartPoint().Row) + 1
|
||||||
|
endLine := int(node.EndPoint().Row) + 1
|
||||||
|
|
||||||
|
chunk := &chunking.Chunk{
|
||||||
|
FilePath: filePath,
|
||||||
|
Language: chunking.LanguagePython,
|
||||||
|
Type: chunking.ChunkTypeClass,
|
||||||
|
Name: name,
|
||||||
|
StartLine: startLine,
|
||||||
|
EndLine: endLine,
|
||||||
|
Content: c.extractLines(sourceLines, startLine, endLine),
|
||||||
|
}
|
||||||
|
|
||||||
|
// Extract class signature (class line)
|
||||||
|
chunk.Signature = c.extractClassSignature(node, source, sourceLines)
|
||||||
|
|
||||||
|
// Extract docstring as doc comment
|
||||||
|
if c.options.IncludeDocComments {
|
||||||
|
chunk.DocComment = c.extractDocstring(node, source)
|
||||||
|
}
|
||||||
|
|
||||||
|
return chunk
|
||||||
|
}
|
||||||
|
|
||||||
|
// extractFunctionSignature extracts the function definition line.
|
||||||
|
func (c *Chunker) extractFunctionSignature(node *sitter.Node, source []byte, sourceLines []string) string {
|
||||||
|
startLine := int(node.StartPoint().Row) + 1
|
||||||
|
|
||||||
|
// Find the colon that ends the signature
|
||||||
|
for i := 0; i < int(node.ChildCount()); i++ {
|
||||||
|
child := node.Child(i)
|
||||||
|
if child.Type() == ":" {
|
||||||
|
endLine := int(child.EndPoint().Row) + 1
|
||||||
|
return strings.TrimSpace(c.extractLines(sourceLines, startLine, endLine))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Fallback: just return first line
|
||||||
|
return strings.TrimSpace(c.extractLines(sourceLines, startLine, startLine))
|
||||||
|
}
|
||||||
|
|
||||||
|
// extractClassSignature extracts the class definition line.
|
||||||
|
func (c *Chunker) extractClassSignature(node *sitter.Node, source []byte, sourceLines []string) string {
|
||||||
|
startLine := int(node.StartPoint().Row) + 1
|
||||||
|
|
||||||
|
// Find the colon that ends the signature
|
||||||
|
for i := 0; i < int(node.ChildCount()); i++ {
|
||||||
|
child := node.Child(i)
|
||||||
|
if child.Type() == ":" {
|
||||||
|
endLine := int(child.EndPoint().Row) + 1
|
||||||
|
return strings.TrimSpace(c.extractLines(sourceLines, startLine, endLine))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Fallback: just return first line
|
||||||
|
return strings.TrimSpace(c.extractLines(sourceLines, startLine, startLine))
|
||||||
|
}
|
||||||
|
|
||||||
|
// extractDocstring extracts the docstring from a function or class.
|
||||||
|
func (c *Chunker) extractDocstring(node *sitter.Node, source []byte) string {
|
||||||
|
// Find the block
|
||||||
|
var blockNode *sitter.Node
|
||||||
|
for i := 0; i < int(node.ChildCount()); i++ {
|
||||||
|
child := node.Child(i)
|
||||||
|
if child.Type() == "block" {
|
||||||
|
blockNode = child
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if blockNode == nil {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check if first statement in block is a string (docstring)
|
||||||
|
for i := 0; i < int(blockNode.ChildCount()); i++ {
|
||||||
|
child := blockNode.Child(i)
|
||||||
|
if child.Type() == "expression_statement" {
|
||||||
|
// Check if it contains a string
|
||||||
|
for j := 0; j < int(child.ChildCount()); j++ {
|
||||||
|
grandchild := child.Child(j)
|
||||||
|
if grandchild.Type() == "string" {
|
||||||
|
docstring := grandchild.Content(source)
|
||||||
|
// Remove quotes
|
||||||
|
docstring = strings.Trim(docstring, `"'`)
|
||||||
|
return strings.TrimSpace(docstring)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
|
// extractLines extracts a range of lines from source (1-indexed, inclusive).
|
||||||
|
func (c *Chunker) extractLines(lines []string, start, end int) string {
|
||||||
|
if start < 1 || end < start || start > len(lines) {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
|
startIdx := start - 1
|
||||||
|
endIdx := end
|
||||||
|
if endIdx > len(lines) {
|
||||||
|
endIdx = len(lines)
|
||||||
|
}
|
||||||
|
|
||||||
|
return strings.Join(lines[startIdx:endIdx], "\n")
|
||||||
|
}
|
||||||
@@ -0,0 +1,140 @@
|
|||||||
|
// Package chunking provides AST-aware code chunking for semantic code search.
|
||||||
|
// Chunks code files into logical units (functions, classes, methods) that preserve
|
||||||
|
// semantic boundaries for better vector embedding and retrieval.
|
||||||
|
package chunking
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
|
// ChunkType represents the type of code chunk.
|
||||||
|
type ChunkType string
|
||||||
|
|
||||||
|
const (
|
||||||
|
// ChunkTypeFunction represents a standalone function.
|
||||||
|
ChunkTypeFunction ChunkType = "function"
|
||||||
|
// ChunkTypeMethod represents a method on a class/struct/type.
|
||||||
|
ChunkTypeMethod ChunkType = "method"
|
||||||
|
// ChunkTypeClass represents a class or struct definition.
|
||||||
|
ChunkTypeClass ChunkType = "class"
|
||||||
|
// ChunkTypeInterface represents an interface definition.
|
||||||
|
ChunkTypeInterface ChunkType = "interface"
|
||||||
|
// ChunkTypeType represents a type alias or type definition.
|
||||||
|
ChunkTypeType ChunkType = "type"
|
||||||
|
// ChunkTypeConst represents constant declarations.
|
||||||
|
ChunkTypeConst ChunkType = "const"
|
||||||
|
// ChunkTypeVar represents variable declarations.
|
||||||
|
ChunkTypeVar ChunkType = "var"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Language represents a programming language.
|
||||||
|
type Language string
|
||||||
|
|
||||||
|
const (
|
||||||
|
// LanguageGo represents the Go programming language.
|
||||||
|
LanguageGo Language = "go"
|
||||||
|
// LanguagePython represents the Python programming language.
|
||||||
|
LanguagePython Language = "python"
|
||||||
|
// LanguageTypeScript represents the TypeScript programming language.
|
||||||
|
LanguageTypeScript Language = "typescript"
|
||||||
|
// LanguageJavaScript represents the JavaScript programming language.
|
||||||
|
LanguageJavaScript Language = "javascript"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Chunk represents a semantic code chunk with AST-derived boundaries.
|
||||||
|
type Chunk struct {
|
||||||
|
Metadata map[string]interface{}
|
||||||
|
FilePath string
|
||||||
|
Language Language
|
||||||
|
Type ChunkType
|
||||||
|
Name string
|
||||||
|
ParentName string
|
||||||
|
Content string
|
||||||
|
Signature string
|
||||||
|
DocComment string
|
||||||
|
StartLine int
|
||||||
|
EndLine int
|
||||||
|
}
|
||||||
|
|
||||||
|
// Identifier returns a human-readable identifier for this chunk.
|
||||||
|
// Format: "ParentName.Name" for methods, "Name" for top-level.
|
||||||
|
func (c *Chunk) Identifier() string {
|
||||||
|
if c.ParentName != "" {
|
||||||
|
return fmt.Sprintf("%s.%s", c.ParentName, c.Name)
|
||||||
|
}
|
||||||
|
return c.Name
|
||||||
|
}
|
||||||
|
|
||||||
|
// LineRange returns a human-readable line range.
|
||||||
|
// Format: "L123-L456"
|
||||||
|
func (c *Chunk) LineRange() string {
|
||||||
|
return fmt.Sprintf("L%d-L%d", c.StartLine, c.EndLine)
|
||||||
|
}
|
||||||
|
|
||||||
|
// SearchableContent returns content optimized for semantic search.
|
||||||
|
// Combines signature, doc comment, and content in a structured format.
|
||||||
|
func (c *Chunk) SearchableContent() string {
|
||||||
|
var parts []string
|
||||||
|
|
||||||
|
// Include signature for functions/methods
|
||||||
|
if c.Signature != "" {
|
||||||
|
parts = append(parts, c.Signature)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Include doc comment
|
||||||
|
if c.DocComment != "" {
|
||||||
|
parts = append(parts, c.DocComment)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Include actual content
|
||||||
|
if c.Content != "" {
|
||||||
|
parts = append(parts, c.Content)
|
||||||
|
}
|
||||||
|
|
||||||
|
return strings.Join(parts, "\n\n")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Chunker is the interface for language-specific code chunkers.
|
||||||
|
type Chunker interface {
|
||||||
|
// Chunk parses a source file and returns semantic code chunks.
|
||||||
|
// Returns an error if the file cannot be parsed or read.
|
||||||
|
Chunk(ctx context.Context, filePath string) ([]Chunk, error)
|
||||||
|
|
||||||
|
// Language returns the language this chunker supports.
|
||||||
|
Language() Language
|
||||||
|
|
||||||
|
// SupportedExtensions returns file extensions this chunker handles.
|
||||||
|
// Example: []string{".go"} for Go chunker
|
||||||
|
SupportedExtensions() []string
|
||||||
|
}
|
||||||
|
|
||||||
|
// ChunkOptions provides options for chunking behavior.
|
||||||
|
type ChunkOptions struct {
|
||||||
|
// MaxChunkSize is the maximum size of a chunk in bytes.
|
||||||
|
// Chunks larger than this will be split (respecting boundaries where possible).
|
||||||
|
// 0 means no limit.
|
||||||
|
MaxChunkSize int
|
||||||
|
|
||||||
|
// IncludeDocComments controls whether to include documentation comments.
|
||||||
|
IncludeDocComments bool
|
||||||
|
|
||||||
|
// IncludePrivate controls whether to include private/unexported symbols.
|
||||||
|
IncludePrivate bool
|
||||||
|
|
||||||
|
// MinLines is the minimum number of lines for a chunk to be included.
|
||||||
|
// Chunks smaller than this will be skipped.
|
||||||
|
// 0 means no minimum.
|
||||||
|
MinLines int
|
||||||
|
}
|
||||||
|
|
||||||
|
// DefaultChunkOptions returns sensible default options.
|
||||||
|
func DefaultChunkOptions() ChunkOptions {
|
||||||
|
return ChunkOptions{
|
||||||
|
MaxChunkSize: 8192, // ~8KB per chunk (well under token limit)
|
||||||
|
IncludeDocComments: true,
|
||||||
|
IncludePrivate: true, // Include all symbols for comprehensive search
|
||||||
|
MinLines: 0, // No minimum - include even single-line functions
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,403 @@
|
|||||||
|
// Package typescript provides AST-aware chunking for TypeScript and JavaScript source files using tree-sitter.
|
||||||
|
package typescript
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
sitter "github.com/smacker/go-tree-sitter"
|
||||||
|
"github.com/smacker/go-tree-sitter/typescript/typescript"
|
||||||
|
|
||||||
|
"github.com/lukaszraczylo/claude-mnemonic/internal/chunking"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Chunker implements AST-aware chunking for TypeScript/JavaScript files.
|
||||||
|
type Chunker struct {
|
||||||
|
parser *sitter.Parser
|
||||||
|
options chunking.ChunkOptions
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewChunker creates a new TypeScript chunker.
|
||||||
|
func NewChunker(options chunking.ChunkOptions) *Chunker {
|
||||||
|
parser := sitter.NewParser()
|
||||||
|
parser.SetLanguage(typescript.GetLanguage())
|
||||||
|
|
||||||
|
return &Chunker{
|
||||||
|
options: options,
|
||||||
|
parser: parser,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Language returns the language this chunker supports.
|
||||||
|
func (c *Chunker) Language() chunking.Language {
|
||||||
|
return chunking.LanguageTypeScript
|
||||||
|
}
|
||||||
|
|
||||||
|
// SupportedExtensions returns the file extensions this chunker handles.
|
||||||
|
func (c *Chunker) SupportedExtensions() []string {
|
||||||
|
return []string{".ts", ".tsx", ".js", ".jsx"}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Chunk parses a TypeScript/JavaScript source file and returns semantic code chunks.
|
||||||
|
func (c *Chunker) Chunk(ctx context.Context, filePath string) ([]chunking.Chunk, error) {
|
||||||
|
// Read file content
|
||||||
|
content, err := os.ReadFile(filePath)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("read file: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Parse the file
|
||||||
|
tree, err := c.parser.ParseCtx(ctx, nil, content)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("parse TypeScript file: %w", err)
|
||||||
|
}
|
||||||
|
defer tree.Close()
|
||||||
|
|
||||||
|
sourceLines := strings.Split(string(content), "\n")
|
||||||
|
chunks := make([]chunking.Chunk, 0)
|
||||||
|
|
||||||
|
// Walk the AST and extract chunks
|
||||||
|
c.walkNode(tree.RootNode(), content, sourceLines, filePath, "", &chunks)
|
||||||
|
|
||||||
|
return chunks, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// walkNode recursively walks the tree-sitter AST and extracts chunks.
|
||||||
|
func (c *Chunker) walkNode(node *sitter.Node, source []byte, sourceLines []string, filePath string, parentName string, chunks *[]chunking.Chunk) {
|
||||||
|
nodeType := node.Type()
|
||||||
|
|
||||||
|
switch nodeType {
|
||||||
|
case "function_declaration":
|
||||||
|
chunk := c.extractFunction(node, source, sourceLines, filePath, parentName)
|
||||||
|
if chunk != nil {
|
||||||
|
*chunks = append(*chunks, *chunk)
|
||||||
|
}
|
||||||
|
|
||||||
|
case "method_definition":
|
||||||
|
chunk := c.extractMethod(node, source, sourceLines, filePath, parentName)
|
||||||
|
if chunk != nil {
|
||||||
|
*chunks = append(*chunks, *chunk)
|
||||||
|
}
|
||||||
|
|
||||||
|
case "arrow_function", "function_expression":
|
||||||
|
// Handle arrow functions and function expressions assigned to variables
|
||||||
|
chunk := c.extractFunctionExpression(node, source, sourceLines, filePath, parentName)
|
||||||
|
if chunk != nil {
|
||||||
|
*chunks = append(*chunks, *chunk)
|
||||||
|
}
|
||||||
|
|
||||||
|
case "class_declaration":
|
||||||
|
chunk := c.extractClass(node, source, sourceLines, filePath)
|
||||||
|
if chunk != nil {
|
||||||
|
*chunks = append(*chunks, *chunk)
|
||||||
|
|
||||||
|
// Walk class body to find methods
|
||||||
|
for i := 0; i < int(node.ChildCount()); i++ {
|
||||||
|
child := node.Child(i)
|
||||||
|
if child.Type() == "class_body" {
|
||||||
|
c.walkNode(child, source, sourceLines, filePath, chunk.Name, chunks)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return // Don't walk children again
|
||||||
|
|
||||||
|
case "interface_declaration":
|
||||||
|
chunk := c.extractInterface(node, source, sourceLines, filePath)
|
||||||
|
if chunk != nil {
|
||||||
|
*chunks = append(*chunks, *chunk)
|
||||||
|
}
|
||||||
|
|
||||||
|
case "type_alias_declaration":
|
||||||
|
chunk := c.extractTypeAlias(node, source, sourceLines, filePath)
|
||||||
|
if chunk != nil {
|
||||||
|
*chunks = append(*chunks, *chunk)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Walk all children
|
||||||
|
for i := 0; i < int(node.ChildCount()); i++ {
|
||||||
|
c.walkNode(node.Child(i), source, sourceLines, filePath, parentName, chunks)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// extractFunction extracts a function declaration.
|
||||||
|
func (c *Chunker) extractFunction(node *sitter.Node, source []byte, sourceLines []string, filePath string, parentName string) *chunking.Chunk {
|
||||||
|
name := c.findChildContent(node, "identifier", source)
|
||||||
|
if name == "" {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
startLine := int(node.StartPoint().Row) + 1
|
||||||
|
endLine := int(node.EndPoint().Row) + 1
|
||||||
|
|
||||||
|
chunk := &chunking.Chunk{
|
||||||
|
FilePath: filePath,
|
||||||
|
Language: chunking.LanguageTypeScript,
|
||||||
|
Type: chunking.ChunkTypeFunction,
|
||||||
|
Name: name,
|
||||||
|
ParentName: parentName,
|
||||||
|
StartLine: startLine,
|
||||||
|
EndLine: endLine,
|
||||||
|
Content: c.extractLines(sourceLines, startLine, endLine),
|
||||||
|
Signature: c.extractFunctionSignature(node, source, sourceLines),
|
||||||
|
}
|
||||||
|
|
||||||
|
// Extract JSDoc comment
|
||||||
|
if c.options.IncludeDocComments {
|
||||||
|
chunk.DocComment = c.extractComment(node, source)
|
||||||
|
}
|
||||||
|
|
||||||
|
return chunk
|
||||||
|
}
|
||||||
|
|
||||||
|
// extractMethod extracts a method definition from a class.
|
||||||
|
func (c *Chunker) extractMethod(node *sitter.Node, source []byte, sourceLines []string, filePath string, parentName string) *chunking.Chunk {
|
||||||
|
name := c.findChildContent(node, "property_identifier", source)
|
||||||
|
if name == "" {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Skip private methods if configured
|
||||||
|
if !c.options.IncludePrivate && strings.HasPrefix(name, "_") {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
startLine := int(node.StartPoint().Row) + 1
|
||||||
|
endLine := int(node.EndPoint().Row) + 1
|
||||||
|
|
||||||
|
chunk := &chunking.Chunk{
|
||||||
|
FilePath: filePath,
|
||||||
|
Language: chunking.LanguageTypeScript,
|
||||||
|
Type: chunking.ChunkTypeMethod,
|
||||||
|
Name: name,
|
||||||
|
ParentName: parentName,
|
||||||
|
StartLine: startLine,
|
||||||
|
EndLine: endLine,
|
||||||
|
Content: c.extractLines(sourceLines, startLine, endLine),
|
||||||
|
Signature: c.extractMethodSignature(node, source, sourceLines),
|
||||||
|
}
|
||||||
|
|
||||||
|
// Extract JSDoc comment
|
||||||
|
if c.options.IncludeDocComments {
|
||||||
|
chunk.DocComment = c.extractComment(node, source)
|
||||||
|
}
|
||||||
|
|
||||||
|
return chunk
|
||||||
|
}
|
||||||
|
|
||||||
|
// extractFunctionExpression extracts arrow functions and function expressions.
|
||||||
|
func (c *Chunker) extractFunctionExpression(node *sitter.Node, source []byte, sourceLines []string, filePath string, parentName string) *chunking.Chunk {
|
||||||
|
// Try to find the variable name from parent
|
||||||
|
parent := node.Parent()
|
||||||
|
if parent == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
var name string
|
||||||
|
if parent.Type() == "variable_declarator" {
|
||||||
|
name = c.findChildContent(parent, "identifier", source)
|
||||||
|
} else if parent.Type() == "assignment_expression" {
|
||||||
|
// Handle const foo = () => {}
|
||||||
|
for i := 0; i < int(parent.ChildCount()); i++ {
|
||||||
|
child := parent.Child(i)
|
||||||
|
if child.Type() == "identifier" || child.Type() == "member_expression" {
|
||||||
|
name = child.Content(source)
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if name == "" {
|
||||||
|
return nil // Anonymous function, skip
|
||||||
|
}
|
||||||
|
|
||||||
|
startLine := int(node.StartPoint().Row) + 1
|
||||||
|
endLine := int(node.EndPoint().Row) + 1
|
||||||
|
|
||||||
|
chunk := &chunking.Chunk{
|
||||||
|
FilePath: filePath,
|
||||||
|
Language: chunking.LanguageTypeScript,
|
||||||
|
Type: chunking.ChunkTypeFunction,
|
||||||
|
Name: name,
|
||||||
|
ParentName: parentName,
|
||||||
|
StartLine: startLine,
|
||||||
|
EndLine: endLine,
|
||||||
|
Content: c.extractLines(sourceLines, startLine, endLine),
|
||||||
|
}
|
||||||
|
|
||||||
|
return chunk
|
||||||
|
}
|
||||||
|
|
||||||
|
// extractClass extracts a class declaration.
|
||||||
|
func (c *Chunker) extractClass(node *sitter.Node, source []byte, sourceLines []string, filePath string) *chunking.Chunk {
|
||||||
|
name := c.findChildContent(node, "type_identifier", source)
|
||||||
|
if name == "" {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
startLine := int(node.StartPoint().Row) + 1
|
||||||
|
endLine := int(node.EndPoint().Row) + 1
|
||||||
|
|
||||||
|
chunk := &chunking.Chunk{
|
||||||
|
FilePath: filePath,
|
||||||
|
Language: chunking.LanguageTypeScript,
|
||||||
|
Type: chunking.ChunkTypeClass,
|
||||||
|
Name: name,
|
||||||
|
StartLine: startLine,
|
||||||
|
EndLine: endLine,
|
||||||
|
Content: c.extractLines(sourceLines, startLine, endLine),
|
||||||
|
Signature: c.extractClassSignature(node, source, sourceLines),
|
||||||
|
}
|
||||||
|
|
||||||
|
// Extract JSDoc comment
|
||||||
|
if c.options.IncludeDocComments {
|
||||||
|
chunk.DocComment = c.extractComment(node, source)
|
||||||
|
}
|
||||||
|
|
||||||
|
return chunk
|
||||||
|
}
|
||||||
|
|
||||||
|
// extractInterface extracts an interface declaration.
|
||||||
|
func (c *Chunker) extractInterface(node *sitter.Node, source []byte, sourceLines []string, filePath string) *chunking.Chunk {
|
||||||
|
name := c.findChildContent(node, "type_identifier", source)
|
||||||
|
if name == "" {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
startLine := int(node.StartPoint().Row) + 1
|
||||||
|
endLine := int(node.EndPoint().Row) + 1
|
||||||
|
|
||||||
|
chunk := &chunking.Chunk{
|
||||||
|
FilePath: filePath,
|
||||||
|
Language: chunking.LanguageTypeScript,
|
||||||
|
Type: chunking.ChunkTypeInterface,
|
||||||
|
Name: name,
|
||||||
|
StartLine: startLine,
|
||||||
|
EndLine: endLine,
|
||||||
|
Content: c.extractLines(sourceLines, startLine, endLine),
|
||||||
|
}
|
||||||
|
|
||||||
|
// Extract JSDoc comment
|
||||||
|
if c.options.IncludeDocComments {
|
||||||
|
chunk.DocComment = c.extractComment(node, source)
|
||||||
|
}
|
||||||
|
|
||||||
|
return chunk
|
||||||
|
}
|
||||||
|
|
||||||
|
// extractTypeAlias extracts a type alias declaration.
|
||||||
|
func (c *Chunker) extractTypeAlias(node *sitter.Node, source []byte, sourceLines []string, filePath string) *chunking.Chunk {
|
||||||
|
name := c.findChildContent(node, "type_identifier", source)
|
||||||
|
if name == "" {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
startLine := int(node.StartPoint().Row) + 1
|
||||||
|
endLine := int(node.EndPoint().Row) + 1
|
||||||
|
|
||||||
|
chunk := &chunking.Chunk{
|
||||||
|
FilePath: filePath,
|
||||||
|
Language: chunking.LanguageTypeScript,
|
||||||
|
Type: chunking.ChunkTypeType,
|
||||||
|
Name: name,
|
||||||
|
StartLine: startLine,
|
||||||
|
EndLine: endLine,
|
||||||
|
Content: c.extractLines(sourceLines, startLine, endLine),
|
||||||
|
}
|
||||||
|
|
||||||
|
return chunk
|
||||||
|
}
|
||||||
|
|
||||||
|
// findChildContent finds the first child of the given type and returns its content.
|
||||||
|
func (c *Chunker) findChildContent(node *sitter.Node, childType string, source []byte) string {
|
||||||
|
for i := 0; i < int(node.ChildCount()); i++ {
|
||||||
|
child := node.Child(i)
|
||||||
|
if child.Type() == childType {
|
||||||
|
return child.Content(source)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
|
// extractFunctionSignature extracts the function signature.
|
||||||
|
func (c *Chunker) extractFunctionSignature(node *sitter.Node, source []byte, sourceLines []string) string {
|
||||||
|
startLine := int(node.StartPoint().Row) + 1
|
||||||
|
|
||||||
|
// Find the opening brace of the body
|
||||||
|
for i := 0; i < int(node.ChildCount()); i++ {
|
||||||
|
child := node.Child(i)
|
||||||
|
if child.Type() == "statement_block" {
|
||||||
|
endLine := int(child.StartPoint().Row) + 1
|
||||||
|
return strings.TrimSpace(c.extractLines(sourceLines, startLine, endLine-1))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Fallback: just return first line
|
||||||
|
return strings.TrimSpace(c.extractLines(sourceLines, startLine, startLine))
|
||||||
|
}
|
||||||
|
|
||||||
|
// extractMethodSignature extracts the method signature.
|
||||||
|
func (c *Chunker) extractMethodSignature(node *sitter.Node, source []byte, sourceLines []string) string {
|
||||||
|
startLine := int(node.StartPoint().Row) + 1
|
||||||
|
|
||||||
|
// Find the opening brace of the body
|
||||||
|
for i := 0; i < int(node.ChildCount()); i++ {
|
||||||
|
child := node.Child(i)
|
||||||
|
if child.Type() == "statement_block" {
|
||||||
|
endLine := int(child.StartPoint().Row) + 1
|
||||||
|
return strings.TrimSpace(c.extractLines(sourceLines, startLine, endLine-1))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return strings.TrimSpace(c.extractLines(sourceLines, startLine, startLine))
|
||||||
|
}
|
||||||
|
|
||||||
|
// extractClassSignature extracts the class declaration line.
|
||||||
|
func (c *Chunker) extractClassSignature(node *sitter.Node, source []byte, sourceLines []string) string {
|
||||||
|
startLine := int(node.StartPoint().Row) + 1
|
||||||
|
|
||||||
|
// Find the opening brace of the class body
|
||||||
|
for i := 0; i < int(node.ChildCount()); i++ {
|
||||||
|
child := node.Child(i)
|
||||||
|
if child.Type() == "class_body" {
|
||||||
|
endLine := int(child.StartPoint().Row) + 1
|
||||||
|
return strings.TrimSpace(c.extractLines(sourceLines, startLine, endLine-1))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return strings.TrimSpace(c.extractLines(sourceLines, startLine, startLine))
|
||||||
|
}
|
||||||
|
|
||||||
|
// extractComment extracts JSDoc or other comments from a node.
|
||||||
|
func (c *Chunker) extractComment(node *sitter.Node, source []byte) string {
|
||||||
|
// Check previous sibling for comment
|
||||||
|
prevSibling := node.PrevSibling()
|
||||||
|
if prevSibling != nil && prevSibling.Type() == "comment" {
|
||||||
|
comment := prevSibling.Content(source)
|
||||||
|
// Remove comment markers
|
||||||
|
comment = strings.TrimPrefix(comment, "/**")
|
||||||
|
comment = strings.TrimPrefix(comment, "/*")
|
||||||
|
comment = strings.TrimSuffix(comment, "*/")
|
||||||
|
comment = strings.TrimPrefix(comment, "//")
|
||||||
|
return strings.TrimSpace(comment)
|
||||||
|
}
|
||||||
|
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
|
// extractLines extracts a range of lines from source (1-indexed, inclusive).
|
||||||
|
func (c *Chunker) extractLines(lines []string, start, end int) string {
|
||||||
|
if start < 1 || end < start || start > len(lines) {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
|
startIdx := start - 1
|
||||||
|
endIdx := end
|
||||||
|
if endIdx > len(lines) {
|
||||||
|
endIdx = len(lines)
|
||||||
|
}
|
||||||
|
|
||||||
|
return strings.Join(lines[startIdx:endIdx], "\n")
|
||||||
|
}
|
||||||
+45
-30
@@ -37,40 +37,34 @@ var CriticalConcepts = []string{
|
|||||||
|
|
||||||
// Config holds the application configuration.
|
// Config holds the application configuration.
|
||||||
type Config struct {
|
type Config struct {
|
||||||
// Worker settings
|
ContextFullField string `json:"context_full_field"`
|
||||||
WorkerPort int `json:"worker_port"`
|
DBPath string `json:"db_path"`
|
||||||
|
Model string `json:"model"`
|
||||||
// Database settings
|
ClaudeCodePath string `json:"claude_code_path"`
|
||||||
DBPath string `json:"db_path"`
|
EmbeddingModel string `json:"embedding_model"`
|
||||||
MaxConns int `json:"max_conns"`
|
ContextObsConcepts []string `json:"context_obs_concepts"`
|
||||||
|
ContextObsTypes []string `json:"context_obs_types"`
|
||||||
// SDK Agent settings
|
RerankingMinImprovement float64 `json:"reranking_min_improvement"`
|
||||||
Model string `json:"model"`
|
RerankingCandidates int `json:"reranking_candidates"`
|
||||||
ClaudeCodePath string `json:"claude_code_path"`
|
RerankingAlpha float64 `json:"reranking_alpha"`
|
||||||
|
GraphEdgeWeight float64 `json:"graph_edge_weight"`
|
||||||
// Embedding settings
|
WorkerPort int `json:"worker_port"`
|
||||||
EmbeddingModel string `json:"embedding_model"` // e.g., "bge-v1.5"
|
ContextMaxPromptResults int `json:"context_max_prompt_results"`
|
||||||
|
|
||||||
// Reranking settings (cross-encoder)
|
|
||||||
RerankingEnabled bool `json:"reranking_enabled"` // Enable cross-encoder reranking
|
|
||||||
RerankingCandidates int `json:"reranking_candidates"` // Number of candidates to retrieve before reranking (default 100)
|
|
||||||
RerankingResults int `json:"reranking_results"` // Number of results to return after reranking (default 10)
|
|
||||||
RerankingAlpha float64 `json:"reranking_alpha"` // Weight for combining scores: alpha*rerank + (1-alpha)*original (default 0.7)
|
|
||||||
RerankingMinImprovement float64 `json:"reranking_min_improvement"` // Minimum rank improvement to trigger reranking (default 0, always rerank)
|
|
||||||
RerankingPureMode bool `json:"reranking_pure_mode"` // Use pure cross-encoder scores without combining with bi-encoder (default false)
|
|
||||||
|
|
||||||
// Context injection settings
|
|
||||||
ContextObservations int `json:"context_observations"`
|
ContextObservations int `json:"context_observations"`
|
||||||
ContextFullCount int `json:"context_full_count"`
|
ContextFullCount int `json:"context_full_count"`
|
||||||
ContextSessionCount int `json:"context_session_count"`
|
ContextSessionCount int `json:"context_session_count"`
|
||||||
ContextShowReadTokens bool `json:"context_show_read_tokens"`
|
ContextRelevanceThreshold float64 `json:"context_relevance_threshold"`
|
||||||
ContextShowWorkTokens bool `json:"context_show_work_tokens"`
|
MaxConns int `json:"max_conns"`
|
||||||
ContextFullField string `json:"context_full_field"`
|
RerankingResults int `json:"reranking_results"`
|
||||||
|
GraphMaxHops int `json:"graph_max_hops"`
|
||||||
|
GraphBranchFactor int `json:"graph_branch_factor"`
|
||||||
|
GraphRebuildIntervalMin int `json:"graph_rebuild_interval_min"`
|
||||||
ContextShowLastSummary bool `json:"context_show_last_summary"`
|
ContextShowLastSummary bool `json:"context_show_last_summary"`
|
||||||
ContextObsTypes []string `json:"context_obs_types"`
|
RerankingEnabled bool `json:"reranking_enabled"`
|
||||||
ContextObsConcepts []string `json:"context_obs_concepts"`
|
ContextShowWorkTokens bool `json:"context_show_work_tokens"`
|
||||||
ContextRelevanceThreshold float64 `json:"context_relevance_threshold"` // 0.0-1.0, minimum similarity for inclusion
|
ContextShowReadTokens bool `json:"context_show_read_tokens"`
|
||||||
ContextMaxPromptResults int `json:"context_max_prompt_results"` // Max results per prompt (0 = threshold only)
|
RerankingPureMode bool `json:"reranking_pure_mode"`
|
||||||
|
GraphEnabled bool `json:"graph_enabled"`
|
||||||
}
|
}
|
||||||
|
|
||||||
var (
|
var (
|
||||||
@@ -148,6 +142,11 @@ func Default() *Config {
|
|||||||
RerankingResults: 10, // Return top 10 after reranking
|
RerankingResults: 10, // Return top 10 after reranking
|
||||||
RerankingAlpha: 0.7, // Favor cross-encoder score
|
RerankingAlpha: 0.7, // Favor cross-encoder score
|
||||||
RerankingMinImprovement: 0, // Always apply reranking
|
RerankingMinImprovement: 0, // Always apply reranking
|
||||||
|
GraphEnabled: true, // Enable graph-aware search by default
|
||||||
|
GraphMaxHops: 2, // Two-hop traversal
|
||||||
|
GraphBranchFactor: 5, // Expand top 5 neighbors per node
|
||||||
|
GraphEdgeWeight: 0.3, // Minimum edge weight to follow
|
||||||
|
GraphRebuildIntervalMin: 60, // Rebuild graph every 60 minutes
|
||||||
ContextObservations: 100,
|
ContextObservations: 100,
|
||||||
ContextFullCount: 25,
|
ContextFullCount: 25,
|
||||||
ContextSessionCount: 10,
|
ContextSessionCount: 10,
|
||||||
@@ -233,6 +232,22 @@ func Load() (*Config, error) {
|
|||||||
if v, ok := settings["CLAUDE_MNEMONIC_CONTEXT_MAX_PROMPT_RESULTS"].(float64); ok && v >= 0 {
|
if v, ok := settings["CLAUDE_MNEMONIC_CONTEXT_MAX_PROMPT_RESULTS"].(float64); ok && v >= 0 {
|
||||||
cfg.ContextMaxPromptResults = int(v)
|
cfg.ContextMaxPromptResults = int(v)
|
||||||
}
|
}
|
||||||
|
// Graph settings
|
||||||
|
if v, ok := settings["CLAUDE_MNEMONIC_GRAPH_ENABLED"].(bool); ok {
|
||||||
|
cfg.GraphEnabled = v
|
||||||
|
}
|
||||||
|
if v, ok := settings["CLAUDE_MNEMONIC_GRAPH_MAX_HOPS"].(float64); ok && v > 0 {
|
||||||
|
cfg.GraphMaxHops = int(v)
|
||||||
|
}
|
||||||
|
if v, ok := settings["CLAUDE_MNEMONIC_GRAPH_BRANCH_FACTOR"].(float64); ok && v > 0 {
|
||||||
|
cfg.GraphBranchFactor = int(v)
|
||||||
|
}
|
||||||
|
if v, ok := settings["CLAUDE_MNEMONIC_GRAPH_EDGE_WEIGHT"].(float64); ok && v >= 0 && v <= 1 {
|
||||||
|
cfg.GraphEdgeWeight = v
|
||||||
|
}
|
||||||
|
if v, ok := settings["CLAUDE_MNEMONIC_GRAPH_REBUILD_INTERVAL_MIN"].(float64); ok && v > 0 {
|
||||||
|
cfg.GraphRebuildIntervalMin = int(v)
|
||||||
|
}
|
||||||
|
|
||||||
return cfg, nil
|
return cfg, nil
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -121,8 +121,8 @@ func (s *ConfigSuite) TestLoad_TableDriven() {
|
|||||||
tests := []struct {
|
tests := []struct {
|
||||||
name string
|
name string
|
||||||
settingsJSON string
|
settingsJSON string
|
||||||
expectedPort int
|
|
||||||
expectedModel string
|
expectedModel string
|
||||||
|
expectedPort int
|
||||||
expectedObsObs int
|
expectedObsObs int
|
||||||
}{
|
}{
|
||||||
{
|
{
|
||||||
@@ -183,12 +183,12 @@ func (s *ConfigSuite) TestLoad_TableDriven() {
|
|||||||
s.Require().NoError(err)
|
s.Require().NoError(err)
|
||||||
|
|
||||||
if tt.settingsJSON != "" {
|
if tt.settingsJSON != "" {
|
||||||
err := os.WriteFile(
|
writeErr := os.WriteFile(
|
||||||
filepath.Join(tempDir, ".claude-mnemonic", "settings.json"),
|
filepath.Join(tempDir, ".claude-mnemonic", "settings.json"),
|
||||||
[]byte(tt.settingsJSON),
|
[]byte(tt.settingsJSON),
|
||||||
0600,
|
0600,
|
||||||
)
|
)
|
||||||
s.Require().NoError(err)
|
s.Require().NoError(writeErr)
|
||||||
}
|
}
|
||||||
|
|
||||||
cfg, err := Load()
|
cfg, err := Load()
|
||||||
|
|||||||
@@ -182,13 +182,13 @@ func (s *ConflictStore) CleanupSupersededObservations(ctx context.Context, proje
|
|||||||
var toDelete []int64
|
var toDelete []int64
|
||||||
for rows.Next() {
|
for rows.Next() {
|
||||||
var id int64
|
var id int64
|
||||||
if err := rows.Scan(&id); err != nil {
|
if scanErr := rows.Scan(&id); scanErr != nil {
|
||||||
return nil, err
|
return nil, scanErr
|
||||||
}
|
}
|
||||||
toDelete = append(toDelete, id)
|
toDelete = append(toDelete, id)
|
||||||
}
|
}
|
||||||
if err := rows.Err(); err != nil {
|
if rowsErr := rows.Err(); rowsErr != nil {
|
||||||
return nil, err
|
return nil, rowsErr
|
||||||
}
|
}
|
||||||
|
|
||||||
if len(toDelete) == 0 {
|
if len(toDelete) == 0 {
|
||||||
@@ -197,8 +197,8 @@ func (s *ConflictStore) CleanupSupersededObservations(ctx context.Context, proje
|
|||||||
|
|
||||||
// Delete the conflict records first (due to foreign key constraints)
|
// Delete the conflict records first (due to foreign key constraints)
|
||||||
for _, obsID := range toDelete {
|
for _, obsID := range toDelete {
|
||||||
if err := s.DeleteConflictsByObservationID(ctx, obsID); err != nil {
|
if delErr := s.DeleteConflictsByObservationID(ctx, obsID); delErr != nil {
|
||||||
return nil, err
|
return nil, delErr
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -54,14 +54,14 @@ func TestNullInt(t *testing.T) {
|
|||||||
func TestRepeatPlaceholders(t *testing.T) {
|
func TestRepeatPlaceholders(t *testing.T) {
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
name string
|
name string
|
||||||
n int
|
|
||||||
expected string
|
expected string
|
||||||
|
n int
|
||||||
}{
|
}{
|
||||||
{"zero", 0, ""},
|
{name: "zero", n: 0, expected: ""},
|
||||||
{"negative", -1, ""},
|
{name: "negative", n: -1, expected: ""},
|
||||||
{"one", 1, ", ?"},
|
{name: "one", n: 1, expected: ", ?"},
|
||||||
{"two", 2, ", ?, ?"},
|
{name: "two", n: 2, expected: ", ?, ?"},
|
||||||
{"three", 3, ", ?, ?, ?"},
|
{name: "three", n: 3, expected: ", ?, ?, ?"},
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, tt := range tests {
|
for _, tt := range tests {
|
||||||
|
|||||||
@@ -9,9 +9,9 @@ import (
|
|||||||
|
|
||||||
// Migration represents a database schema migration.
|
// Migration represents a database schema migration.
|
||||||
type Migration struct {
|
type Migration struct {
|
||||||
Version int
|
|
||||||
Name string
|
Name string
|
||||||
SQL string
|
SQL string
|
||||||
|
Version int
|
||||||
}
|
}
|
||||||
|
|
||||||
// Migrations is the list of all database migrations in order.
|
// Migrations is the list of all database migrations in order.
|
||||||
@@ -539,11 +539,11 @@ func (m *MigrationManager) ApplyMigration(migration Migration) error {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("begin transaction: %w", err)
|
return fmt.Errorf("begin transaction: %w", err)
|
||||||
}
|
}
|
||||||
defer tx.Rollback()
|
defer func() { _ = tx.Rollback() }()
|
||||||
|
|
||||||
// Execute migration SQL
|
// Execute migration SQL
|
||||||
if _, err := tx.Exec(migration.SQL); err != nil {
|
if _, execErr := tx.Exec(migration.SQL); execErr != nil {
|
||||||
return fmt.Errorf("execute migration %d (%s): %w", migration.Version, migration.Name, err)
|
return fmt.Errorf("execute migration %d (%s): %w", migration.Version, migration.Name, execErr)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Record migration
|
// Record migration
|
||||||
|
|||||||
@@ -585,13 +585,13 @@ func (s *ObservationStore) CleanupOldObservations(ctx context.Context, project s
|
|||||||
var toDelete []int64
|
var toDelete []int64
|
||||||
for rows.Next() {
|
for rows.Next() {
|
||||||
var id int64
|
var id int64
|
||||||
if err := rows.Scan(&id); err != nil {
|
if scanErr := rows.Scan(&id); scanErr != nil {
|
||||||
return nil, err
|
return nil, scanErr
|
||||||
}
|
}
|
||||||
toDelete = append(toDelete, id)
|
toDelete = append(toDelete, id)
|
||||||
}
|
}
|
||||||
if err := rows.Err(); err != nil {
|
if rowsErr := rows.Err(); rowsErr != nil {
|
||||||
return nil, err
|
return nil, rowsErr
|
||||||
}
|
}
|
||||||
|
|
||||||
if len(toDelete) == 0 {
|
if len(toDelete) == 0 {
|
||||||
|
|||||||
@@ -65,10 +65,10 @@ func (s *ObservationStoreSuite) TestStoreObservation_TableDriven() {
|
|||||||
ctx := context.Background()
|
ctx := context.Background()
|
||||||
|
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
|
obs *models.ParsedObservation
|
||||||
name string
|
name string
|
||||||
sdkSessionID string
|
sdkSessionID string
|
||||||
project string
|
project string
|
||||||
obs *models.ParsedObservation
|
|
||||||
promptNum int
|
promptNum int
|
||||||
tokens int64
|
tokens int64
|
||||||
wantErr bool
|
wantErr bool
|
||||||
@@ -308,8 +308,8 @@ func (s *ObservationStoreSuite) TestGetObservationsByIDs() {
|
|||||||
|
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
name string
|
name string
|
||||||
queryIDs []int64
|
|
||||||
orderBy string
|
orderBy string
|
||||||
|
queryIDs []int64
|
||||||
limit int
|
limit int
|
||||||
wantCount int
|
wantCount int
|
||||||
}{
|
}{
|
||||||
|
|||||||
@@ -102,13 +102,13 @@ func (s *PromptStore) CleanupOldPrompts(ctx context.Context) ([]int64, error) {
|
|||||||
var toDelete []int64
|
var toDelete []int64
|
||||||
for rows.Next() {
|
for rows.Next() {
|
||||||
var id int64
|
var id int64
|
||||||
if err := rows.Scan(&id); err != nil {
|
if scanErr := rows.Scan(&id); scanErr != nil {
|
||||||
return nil, err
|
return nil, scanErr
|
||||||
}
|
}
|
||||||
toDelete = append(toDelete, id)
|
toDelete = append(toDelete, id)
|
||||||
}
|
}
|
||||||
if err := rows.Err(); err != nil {
|
if rowsErr := rows.Err(); rowsErr != nil {
|
||||||
return nil, err
|
return nil, rowsErr
|
||||||
}
|
}
|
||||||
|
|
||||||
if len(toDelete) == 0 {
|
if len(toDelete) == 0 {
|
||||||
|
|||||||
@@ -261,14 +261,14 @@ func TestPromptStore_SaveMultiplePrompts(t *testing.T) {
|
|||||||
|
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
claudeSessionID string
|
claudeSessionID string
|
||||||
promptNum int
|
|
||||||
text string
|
text string
|
||||||
|
promptNum int
|
||||||
matches int
|
matches int
|
||||||
}{
|
}{
|
||||||
{"claude-1", 1, "First prompt", 5},
|
{claudeSessionID: "claude-1", promptNum: 1, text: "First prompt", matches: 5},
|
||||||
{"claude-1", 2, "Second prompt", 3},
|
{claudeSessionID: "claude-1", promptNum: 2, text: "Second prompt", matches: 3},
|
||||||
{"claude-2", 1, "Third prompt", 0},
|
{claudeSessionID: "claude-2", promptNum: 1, text: "Third prompt", matches: 0},
|
||||||
{"claude-1", 3, "Fourth prompt", 10},
|
{claudeSessionID: "claude-1", promptNum: 3, text: "Fourth prompt", matches: 10},
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, tt := range tests {
|
for _, tt := range tests {
|
||||||
|
|||||||
@@ -71,7 +71,7 @@ func (s *ObservationStore) UpdateImportanceScores(ctx context.Context, scores ma
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
defer tx.Rollback()
|
defer func() { _ = tx.Rollback() }()
|
||||||
|
|
||||||
now := time.Now().UnixMilli()
|
now := time.Now().UnixMilli()
|
||||||
stmt, err := tx.PrepareContext(ctx, `
|
stmt, err := tx.PrepareContext(ctx, `
|
||||||
@@ -171,7 +171,7 @@ func (s *ObservationStore) UpdateConceptWeights(ctx context.Context, weights map
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
defer tx.Rollback()
|
defer func() { _ = tx.Rollback() }()
|
||||||
|
|
||||||
stmt, err := tx.PrepareContext(ctx, `
|
stmt, err := tx.PrepareContext(ctx, `
|
||||||
INSERT INTO concept_weights (concept, weight, updated_at)
|
INSERT INTO concept_weights (concept, weight, updated_at)
|
||||||
|
|||||||
@@ -211,8 +211,8 @@ func (s *SessionStoreSuite) TestFindAnySDKSession_Scenarios() {
|
|||||||
tests := []struct {
|
tests := []struct {
|
||||||
name string
|
name string
|
||||||
claudeSessionID string
|
claudeSessionID string
|
||||||
wantFound bool
|
|
||||||
wantProject string
|
wantProject string
|
||||||
|
wantFound bool
|
||||||
}{
|
}{
|
||||||
{
|
{
|
||||||
name: "find existing session 1",
|
name: "find existing session 1",
|
||||||
|
|||||||
@@ -94,8 +94,8 @@ func (s *Store) GetStmt(query string) (*sql.Stmt, error) {
|
|||||||
defer s.stmtMu.Unlock()
|
defer s.stmtMu.Unlock()
|
||||||
|
|
||||||
// Double-check after acquiring write lock
|
// Double-check after acquiring write lock
|
||||||
if stmt, ok := s.stmtCache[query]; ok {
|
if cachedStmt, ok := s.stmtCache[query]; ok {
|
||||||
return stmt, nil
|
return cachedStmt, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
stmt, err := s.db.Prepare(query)
|
stmt, err := s.db.Prepare(query)
|
||||||
|
|||||||
@@ -130,13 +130,13 @@ func (s *StoreSuite) TestQueryContext() {
|
|||||||
seedSession(s.T(), s.db, "claude-1", "sdk-1", "project-a")
|
seedSession(s.T(), s.db, "claude-1", "sdk-1", "project-a")
|
||||||
|
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
|
setupFunc func()
|
||||||
|
assertFunc func(rows *sql.Rows)
|
||||||
name string
|
name string
|
||||||
query string
|
query string
|
||||||
args []interface{}
|
args []interface{}
|
||||||
wantErr bool
|
|
||||||
wantRows int
|
wantRows int
|
||||||
setupFunc func()
|
wantErr bool
|
||||||
assertFunc func(rows *sql.Rows)
|
|
||||||
}{
|
}{
|
||||||
{
|
{
|
||||||
name: "query existing session",
|
name: "query existing session",
|
||||||
@@ -366,8 +366,8 @@ func (s *HelpersSuite) TestNullInt() {
|
|||||||
func (s *HelpersSuite) TestRepeatPlaceholders() {
|
func (s *HelpersSuite) TestRepeatPlaceholders() {
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
name string
|
name string
|
||||||
input int
|
|
||||||
expected string
|
expected string
|
||||||
|
input int
|
||||||
}{
|
}{
|
||||||
{
|
{
|
||||||
name: "zero",
|
name: "zero",
|
||||||
@@ -438,10 +438,10 @@ func TestBuildGetByIDsQuery(t *testing.T) {
|
|||||||
tests := []struct {
|
tests := []struct {
|
||||||
name string
|
name string
|
||||||
baseQuery string
|
baseQuery string
|
||||||
ids []int64
|
|
||||||
orderBy string
|
orderBy string
|
||||||
limit int
|
|
||||||
wantQuery string
|
wantQuery string
|
||||||
|
ids []int64
|
||||||
|
limit int
|
||||||
wantArgs int
|
wantArgs int
|
||||||
}{
|
}{
|
||||||
{
|
{
|
||||||
@@ -483,10 +483,10 @@ func TestEnsureSessionExists(t *testing.T) {
|
|||||||
ctx := context.Background()
|
ctx := context.Background()
|
||||||
|
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
|
setup func()
|
||||||
name string
|
name string
|
||||||
sdkSessionID string
|
sdkSessionID string
|
||||||
project string
|
project string
|
||||||
setup func()
|
|
||||||
wantErr bool
|
wantErr bool
|
||||||
}{
|
}{
|
||||||
{
|
{
|
||||||
|
|||||||
@@ -21,15 +21,10 @@ const (
|
|||||||
// ONNXConfig describes ONNX-specific model configuration.
|
// ONNXConfig describes ONNX-specific model configuration.
|
||||||
// This allows different models to specify their tensor names and pooling needs.
|
// This allows different models to specify their tensor names and pooling needs.
|
||||||
type ONNXConfig struct {
|
type ONNXConfig struct {
|
||||||
// InputNames are the ONNX input tensor names in order.
|
Pooling PoolingStrategy
|
||||||
InputNames []string
|
InputNames []string
|
||||||
// OutputNames are the ONNX output tensor names.
|
|
||||||
OutputNames []string
|
OutputNames []string
|
||||||
// Pooling specifies how to convert token embeddings to sentence embeddings.
|
HiddenSize int
|
||||||
// If PoolingNone, the model outputs sentence embeddings directly.
|
|
||||||
Pooling PoolingStrategy
|
|
||||||
// HiddenSize is the embedding dimension (used for pooling calculations).
|
|
||||||
HiddenSize int
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// EmbeddingModel represents a text embedding model.
|
// EmbeddingModel represents a text embedding model.
|
||||||
@@ -62,11 +57,11 @@ type ONNXConfigurer interface {
|
|||||||
|
|
||||||
// ModelMetadata describes an embedding model for UI/config.
|
// ModelMetadata describes an embedding model for UI/config.
|
||||||
type ModelMetadata struct {
|
type ModelMetadata struct {
|
||||||
Name string `json:"name"` // Human-readable name
|
Name string `json:"name"`
|
||||||
Version string `json:"version"` // Short ID for DB storage
|
Version string `json:"version"`
|
||||||
Dimensions int `json:"dimensions"` // Vector size
|
Description string `json:"description"`
|
||||||
Description string `json:"description"` // Brief description
|
Dimensions int `json:"dimensions"`
|
||||||
Default bool `json:"default"` // Is this the default model?
|
Default bool `json:"default"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// ModelFactory creates a new instance of an embedding model.
|
// ModelFactory creates a new instance of an embedding model.
|
||||||
@@ -74,10 +69,10 @@ type ModelFactory func() (EmbeddingModel, error)
|
|||||||
|
|
||||||
// ModelRegistry provides model lookup by version.
|
// ModelRegistry provides model lookup by version.
|
||||||
type ModelRegistry struct {
|
type ModelRegistry struct {
|
||||||
mu sync.RWMutex
|
|
||||||
models map[string]ModelFactory
|
models map[string]ModelFactory
|
||||||
metadata map[string]ModelMetadata
|
metadata map[string]ModelMetadata
|
||||||
defaultModel string
|
defaultModel string
|
||||||
|
mu sync.RWMutex
|
||||||
}
|
}
|
||||||
|
|
||||||
// NewModelRegistry creates a new model registry.
|
// NewModelRegistry creates a new model registry.
|
||||||
|
|||||||
@@ -46,9 +46,9 @@ var bgeONNXConfig = ONNXConfig{
|
|||||||
type bgeModel struct {
|
type bgeModel struct {
|
||||||
tk *tokenizer.Tokenizer
|
tk *tokenizer.Tokenizer
|
||||||
session *ort.DynamicAdvancedSession
|
session *ort.DynamicAdvancedSession
|
||||||
|
libDir string
|
||||||
|
config ONNXConfig
|
||||||
mu sync.Mutex
|
mu sync.Mutex
|
||||||
libDir string // temp directory containing extracted libraries
|
|
||||||
config ONNXConfig // ONNX configuration for this model
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Compile-time check that bgeModel implements EmbeddingModel
|
// Compile-time check that bgeModel implements EmbeddingModel
|
||||||
@@ -70,8 +70,8 @@ func newBGEModel() (EmbeddingModel, error) {
|
|||||||
ort.SetSharedLibraryPath(libPath)
|
ort.SetSharedLibraryPath(libPath)
|
||||||
|
|
||||||
// Initialize ONNX runtime
|
// Initialize ONNX runtime
|
||||||
if err := ort.InitializeEnvironment(); err != nil {
|
if initErr := ort.InitializeEnvironment(); initErr != nil {
|
||||||
return nil, fmt.Errorf("initialize ONNX runtime: %w", err)
|
return nil, fmt.Errorf("initialize ONNX runtime: %w", initErr)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Load tokenizer from embedded data
|
// Load tokenizer from embedded data
|
||||||
@@ -292,19 +292,19 @@ func (m *bgeModel) computeBatch(sentences []string) ([][]float32, error) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("create input_ids tensor: %w", err)
|
return nil, fmt.Errorf("create input_ids tensor: %w", err)
|
||||||
}
|
}
|
||||||
defer inputIdsTensor.Destroy()
|
defer func() { _ = inputIdsTensor.Destroy() }()
|
||||||
|
|
||||||
attentionMaskTensor, err := ort.NewTensor(inputShape, attentionMaskData)
|
attentionMaskTensor, err := ort.NewTensor(inputShape, attentionMaskData)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("create attention_mask tensor: %w", err)
|
return nil, fmt.Errorf("create attention_mask tensor: %w", err)
|
||||||
}
|
}
|
||||||
defer attentionMaskTensor.Destroy()
|
defer func() { _ = attentionMaskTensor.Destroy() }()
|
||||||
|
|
||||||
tokenTypeIdsTensor, err := ort.NewTensor(inputShape, tokenTypeIdsData)
|
tokenTypeIdsTensor, err := ort.NewTensor(inputShape, tokenTypeIdsData)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("create token_type_ids tensor: %w", err)
|
return nil, fmt.Errorf("create token_type_ids tensor: %w", err)
|
||||||
}
|
}
|
||||||
defer tokenTypeIdsTensor.Destroy()
|
defer func() { _ = tokenTypeIdsTensor.Destroy() }()
|
||||||
|
|
||||||
// Create output tensor based on pooling strategy
|
// Create output tensor based on pooling strategy
|
||||||
var outputShape ort.Shape
|
var outputShape ort.Shape
|
||||||
@@ -324,7 +324,7 @@ func (m *bgeModel) computeBatch(sentences []string) ([][]float32, error) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("create output tensor: %w", err)
|
return nil, fmt.Errorf("create output tensor: %w", err)
|
||||||
}
|
}
|
||||||
defer outputTensor.Destroy()
|
defer func() { _ = outputTensor.Destroy() }()
|
||||||
|
|
||||||
// Run inference
|
// Run inference
|
||||||
inputTensors := []ort.Value{inputIdsTensor, attentionMaskTensor, tokenTypeIdsTensor}
|
inputTensors := []ort.Value{inputIdsTensor, attentionMaskTensor, tokenTypeIdsTensor}
|
||||||
|
|||||||
@@ -0,0 +1,417 @@
|
|||||||
|
package graph
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"math"
|
||||||
|
|
||||||
|
"github.com/lukaszraczylo/claude-mnemonic/pkg/models"
|
||||||
|
"github.com/rs/zerolog/log"
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
// SemanticSimilarityThreshold for creating semantic edges
|
||||||
|
SemanticSimilarityThreshold = 0.85
|
||||||
|
|
||||||
|
// MinFileOverlapForEdge minimum file overlap ratio to create edge
|
||||||
|
MinFileOverlapForEdge = 0.3
|
||||||
|
|
||||||
|
// MaxEdgesPerNode prevents creating too many edges
|
||||||
|
MaxEdgesPerNode = 20
|
||||||
|
)
|
||||||
|
|
||||||
|
// DetectEdges identifies relationships between observations
|
||||||
|
func DetectEdges(ctx context.Context, observations []*models.Observation) ([]Edge, error) {
|
||||||
|
if len(observations) < 2 {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
edges := make([]Edge, 0)
|
||||||
|
|
||||||
|
// Build lookup maps for efficient detection
|
||||||
|
sessionMap := buildSessionMap(observations)
|
||||||
|
conceptMap := buildConceptMap(observations)
|
||||||
|
fileMap := buildFileMap(observations)
|
||||||
|
|
||||||
|
log.Info().
|
||||||
|
Int("observations", len(observations)).
|
||||||
|
Int("sessions", len(sessionMap)).
|
||||||
|
Int("concepts", len(conceptMap)).
|
||||||
|
Msg("Starting edge detection")
|
||||||
|
|
||||||
|
// Detect temporal edges (same session)
|
||||||
|
temporalEdges := detectTemporalEdges(sessionMap)
|
||||||
|
edges = append(edges, temporalEdges...)
|
||||||
|
|
||||||
|
// Detect concept edges (shared tags)
|
||||||
|
conceptEdges := detectConceptEdges(conceptMap)
|
||||||
|
edges = append(edges, conceptEdges...)
|
||||||
|
|
||||||
|
// Detect file overlap edges
|
||||||
|
fileEdges := detectFileOverlapEdges(fileMap, observations)
|
||||||
|
edges = append(edges, fileEdges...)
|
||||||
|
|
||||||
|
// Prune excessive edges per node
|
||||||
|
edges = pruneEdges(edges, MaxEdgesPerNode)
|
||||||
|
|
||||||
|
log.Info().
|
||||||
|
Int("temporal_edges", len(temporalEdges)).
|
||||||
|
Int("concept_edges", len(conceptEdges)).
|
||||||
|
Int("file_edges", len(fileEdges)).
|
||||||
|
Int("total_edges", len(edges)).
|
||||||
|
Msg("Edge detection complete")
|
||||||
|
|
||||||
|
return edges, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// buildSessionMap groups observations by SDK session
|
||||||
|
func buildSessionMap(observations []*models.Observation) map[string][]int64 {
|
||||||
|
sessionMap := make(map[string][]int64)
|
||||||
|
|
||||||
|
for _, obs := range observations {
|
||||||
|
if obs.SDKSessionID != "" {
|
||||||
|
sessionMap[obs.SDKSessionID] = append(sessionMap[obs.SDKSessionID], obs.ID)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return sessionMap
|
||||||
|
}
|
||||||
|
|
||||||
|
// buildConceptMap groups observations by concept tags
|
||||||
|
func buildConceptMap(observations []*models.Observation) map[string][]int64 {
|
||||||
|
conceptMap := make(map[string][]int64)
|
||||||
|
|
||||||
|
for _, obs := range observations {
|
||||||
|
for _, concept := range obs.Concepts {
|
||||||
|
conceptMap[concept] = append(conceptMap[concept], obs.ID)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return conceptMap
|
||||||
|
}
|
||||||
|
|
||||||
|
// buildFileMap maps files to observations (from both FilesRead and FilesModified)
|
||||||
|
func buildFileMap(observations []*models.Observation) map[string][]int64 {
|
||||||
|
fileMap := make(map[string][]int64)
|
||||||
|
|
||||||
|
for _, obs := range observations {
|
||||||
|
// Add files from FilesRead
|
||||||
|
for _, file := range obs.FilesRead {
|
||||||
|
fileMap[file] = append(fileMap[file], obs.ID)
|
||||||
|
}
|
||||||
|
// Add files from FilesModified
|
||||||
|
for _, file := range obs.FilesModified {
|
||||||
|
fileMap[file] = append(fileMap[file], obs.ID)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return fileMap
|
||||||
|
}
|
||||||
|
|
||||||
|
// detectTemporalEdges creates edges between observations in the same session
|
||||||
|
func detectTemporalEdges(sessionMap map[string][]int64) []Edge {
|
||||||
|
edges := make([]Edge, 0)
|
||||||
|
|
||||||
|
for _, obsIDs := range sessionMap {
|
||||||
|
if len(obsIDs) < 2 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
// Create edges between consecutive observations in session
|
||||||
|
for i := 0; i < len(obsIDs)-1; i++ {
|
||||||
|
edges = append(edges, Edge{
|
||||||
|
FromID: obsIDs[i],
|
||||||
|
ToID: obsIDs[i+1],
|
||||||
|
Relation: RelationTemporal,
|
||||||
|
Weight: 0.8, // High weight for temporal proximity
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return edges
|
||||||
|
}
|
||||||
|
|
||||||
|
// detectConceptEdges creates edges between observations sharing concepts
|
||||||
|
func detectConceptEdges(conceptMap map[string][]int64) []Edge {
|
||||||
|
edges := make([]Edge, 0)
|
||||||
|
seen := make(map[string]bool)
|
||||||
|
|
||||||
|
for concept, obsIDs := range conceptMap {
|
||||||
|
if len(obsIDs) < 2 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
// Create edges between all observations sharing this concept
|
||||||
|
for i := 0; i < len(obsIDs); i++ {
|
||||||
|
for j := i + 1; j < len(obsIDs); j++ {
|
||||||
|
// Use sorted pair as key to avoid duplicates
|
||||||
|
pairKey := edgeKey(obsIDs[i], obsIDs[j])
|
||||||
|
if seen[pairKey] {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
seen[pairKey] = true
|
||||||
|
|
||||||
|
// Weight based on concept specificity (longer = more specific)
|
||||||
|
weight := float32(0.5 + 0.3*math.Min(1.0, float64(len(concept))/20.0))
|
||||||
|
|
||||||
|
edges = append(edges, Edge{
|
||||||
|
FromID: obsIDs[i],
|
||||||
|
ToID: obsIDs[j],
|
||||||
|
Relation: RelationConcept,
|
||||||
|
Weight: weight,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return edges
|
||||||
|
}
|
||||||
|
|
||||||
|
// detectFileOverlapEdges creates edges based on file references
|
||||||
|
func detectFileOverlapEdges(fileMap map[string][]int64, observations []*models.Observation) []Edge {
|
||||||
|
edges := make([]Edge, 0)
|
||||||
|
seen := make(map[string]bool)
|
||||||
|
|
||||||
|
// Build observation ID to observation map for quick lookup
|
||||||
|
obsMap := make(map[int64]*models.Observation)
|
||||||
|
for _, obs := range observations {
|
||||||
|
obsMap[obs.ID] = obs
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, obsIDs := range fileMap {
|
||||||
|
if len(obsIDs) < 2 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
// Create edges between observations referencing same files
|
||||||
|
for i := 0; i < len(obsIDs); i++ {
|
||||||
|
for j := i + 1; j < len(obsIDs); j++ {
|
||||||
|
pairKey := edgeKey(obsIDs[i], obsIDs[j])
|
||||||
|
if seen[pairKey] {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
seen[pairKey] = true
|
||||||
|
|
||||||
|
// Calculate file overlap ratio
|
||||||
|
obs1, ok1 := obsMap[obsIDs[i]]
|
||||||
|
obs2, ok2 := obsMap[obsIDs[j]]
|
||||||
|
|
||||||
|
if !ok1 || !ok2 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
// Merge FilesRead and FilesModified for both observations
|
||||||
|
files1 := append([]string{}, obs1.FilesRead...)
|
||||||
|
files1 = append(files1, obs1.FilesModified...)
|
||||||
|
files2 := append([]string{}, obs2.FilesRead...)
|
||||||
|
files2 = append(files2, obs2.FilesModified...)
|
||||||
|
|
||||||
|
overlap := calculateFileOverlap(files1, files2)
|
||||||
|
if overlap < MinFileOverlapForEdge {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
edges = append(edges, Edge{
|
||||||
|
FromID: obsIDs[i],
|
||||||
|
ToID: obsIDs[j],
|
||||||
|
Relation: RelationFileOverlap,
|
||||||
|
Weight: overlap,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return edges
|
||||||
|
}
|
||||||
|
|
||||||
|
// calculateFileOverlap computes Jaccard similarity of file sets
|
||||||
|
func calculateFileOverlap(files1, files2 []string) float32 {
|
||||||
|
if len(files1) == 0 || len(files2) == 0 {
|
||||||
|
return 0.0
|
||||||
|
}
|
||||||
|
|
||||||
|
// Convert to sets
|
||||||
|
set1 := make(map[string]bool)
|
||||||
|
for _, f := range files1 {
|
||||||
|
set1[f] = true
|
||||||
|
}
|
||||||
|
|
||||||
|
set2 := make(map[string]bool)
|
||||||
|
for _, f := range files2 {
|
||||||
|
set2[f] = true
|
||||||
|
}
|
||||||
|
|
||||||
|
// Count intersection
|
||||||
|
intersection := 0
|
||||||
|
for f := range set1 {
|
||||||
|
if set2[f] {
|
||||||
|
intersection++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Jaccard similarity = intersection / union
|
||||||
|
union := len(set1) + len(set2) - intersection
|
||||||
|
if union == 0 {
|
||||||
|
return 0.0
|
||||||
|
}
|
||||||
|
|
||||||
|
return float32(intersection) / float32(union)
|
||||||
|
}
|
||||||
|
|
||||||
|
// pruneEdges limits edges per node to prevent graph explosion
|
||||||
|
func pruneEdges(edges []Edge, maxPerNode int) []Edge {
|
||||||
|
if maxPerNode <= 0 {
|
||||||
|
return edges
|
||||||
|
}
|
||||||
|
|
||||||
|
// Count edges per node
|
||||||
|
outEdges := make(map[int64][]Edge)
|
||||||
|
inEdges := make(map[int64][]Edge)
|
||||||
|
|
||||||
|
for _, edge := range edges {
|
||||||
|
outEdges[edge.FromID] = append(outEdges[edge.FromID], edge)
|
||||||
|
inEdges[edge.ToID] = append(inEdges[edge.ToID], edge)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Prune low-weight edges if node has too many
|
||||||
|
pruned := make([]Edge, 0, len(edges))
|
||||||
|
processed := make(map[string]bool)
|
||||||
|
|
||||||
|
for _, edge := range edges {
|
||||||
|
pairKey := edgeKey(edge.FromID, edge.ToID)
|
||||||
|
if processed[pairKey] {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
processed[pairKey] = true
|
||||||
|
|
||||||
|
// Check if either node has too many edges
|
||||||
|
fromCount := len(outEdges[edge.FromID])
|
||||||
|
toCount := len(inEdges[edge.ToID])
|
||||||
|
|
||||||
|
if fromCount <= maxPerNode && toCount <= maxPerNode {
|
||||||
|
pruned = append(pruned, edge)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
// Keep edge if it's high-weight (top edges for this node)
|
||||||
|
if shouldKeepEdge(edge, outEdges[edge.FromID], maxPerNode) {
|
||||||
|
pruned = append(pruned, edge)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(pruned) < len(edges) {
|
||||||
|
log.Debug().
|
||||||
|
Int("original", len(edges)).
|
||||||
|
Int("pruned", len(pruned)).
|
||||||
|
Int("removed", len(edges)-len(pruned)).
|
||||||
|
Msg("Pruned excessive edges")
|
||||||
|
}
|
||||||
|
|
||||||
|
return pruned
|
||||||
|
}
|
||||||
|
|
||||||
|
// shouldKeepEdge determines if edge should be kept during pruning
|
||||||
|
func shouldKeepEdge(edge Edge, nodeEdges []Edge, maxPerNode int) bool {
|
||||||
|
// Sort node's edges by weight descending
|
||||||
|
sortedEdges := make([]Edge, len(nodeEdges))
|
||||||
|
copy(sortedEdges, nodeEdges)
|
||||||
|
|
||||||
|
sortEdgesByWeight(sortedEdges)
|
||||||
|
|
||||||
|
// Keep edge if it's in top maxPerNode
|
||||||
|
for i := 0; i < maxPerNode && i < len(sortedEdges); i++ {
|
||||||
|
if sortedEdges[i].FromID == edge.FromID && sortedEdges[i].ToID == edge.ToID {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
// sortEdgesByWeight sorts edges by weight descending
|
||||||
|
func sortEdgesByWeight(edges []Edge) {
|
||||||
|
// Simple bubble sort (edges are typically small per node)
|
||||||
|
n := len(edges)
|
||||||
|
for i := 0; i < n-1; i++ {
|
||||||
|
for j := 0; j < n-i-1; j++ {
|
||||||
|
if edges[j].Weight < edges[j+1].Weight {
|
||||||
|
edges[j], edges[j+1] = edges[j+1], edges[j]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// edgeKey creates a unique key for an edge pair (sorted)
|
||||||
|
func edgeKey(id1, id2 int64) string {
|
||||||
|
if id1 < id2 {
|
||||||
|
return fmt.Sprintf("%d-%d", id1, id2)
|
||||||
|
}
|
||||||
|
return fmt.Sprintf("%d-%d", id2, id1)
|
||||||
|
}
|
||||||
|
|
||||||
|
// DetectSemanticEdges creates edges based on semantic similarity
|
||||||
|
// This requires embeddings and is called separately when available
|
||||||
|
func DetectSemanticEdges(ctx context.Context, observations []*models.Observation, embeddings map[int64][]float32) []Edge {
|
||||||
|
edges := make([]Edge, 0)
|
||||||
|
seen := make(map[string]bool)
|
||||||
|
|
||||||
|
// Compare all pairs (expensive, but necessary for semantic similarity)
|
||||||
|
for i := 0; i < len(observations); i++ {
|
||||||
|
emb1, ok1 := embeddings[observations[i].ID]
|
||||||
|
if !ok1 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
for j := i + 1; j < len(observations); j++ {
|
||||||
|
emb2, ok2 := embeddings[observations[j].ID]
|
||||||
|
if !ok2 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
similarity := cosineSimilarity(emb1, emb2)
|
||||||
|
if similarity < SemanticSimilarityThreshold {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
pairKey := edgeKey(observations[i].ID, observations[j].ID)
|
||||||
|
if seen[pairKey] {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
seen[pairKey] = true
|
||||||
|
|
||||||
|
edges = append(edges, Edge{
|
||||||
|
FromID: observations[i].ID,
|
||||||
|
ToID: observations[j].ID,
|
||||||
|
Relation: RelationSemantic,
|
||||||
|
Weight: similarity,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
log.Info().
|
||||||
|
Int("semantic_edges", len(edges)).
|
||||||
|
Float32("threshold", SemanticSimilarityThreshold).
|
||||||
|
Msg("Detected semantic edges")
|
||||||
|
|
||||||
|
return edges
|
||||||
|
}
|
||||||
|
|
||||||
|
// cosineSimilarity computes cosine similarity between two vectors
|
||||||
|
func cosineSimilarity(a, b []float32) float32 {
|
||||||
|
if len(a) != len(b) {
|
||||||
|
return 0.0
|
||||||
|
}
|
||||||
|
|
||||||
|
var dotProduct, normA, normB float32
|
||||||
|
for i := range a {
|
||||||
|
dotProduct += a[i] * b[i]
|
||||||
|
normA += a[i] * a[i]
|
||||||
|
normB += b[i] * b[i]
|
||||||
|
}
|
||||||
|
|
||||||
|
if normA == 0 || normB == 0 {
|
||||||
|
return 0.0
|
||||||
|
}
|
||||||
|
|
||||||
|
return dotProduct / float32(math.Sqrt(float64(normA))*math.Sqrt(float64(normB)))
|
||||||
|
}
|
||||||
@@ -0,0 +1,423 @@
|
|||||||
|
// Package graph provides observation relationship graphs for LEANN Phase 2.
|
||||||
|
//
|
||||||
|
// This package implements graph-based selective recomputation where observation
|
||||||
|
// relationships (file overlap, semantic similarity, temporal proximity) form a
|
||||||
|
// graph structure. Hub nodes (high-degree observations) store embeddings, while
|
||||||
|
// leaf nodes recompute on-demand.
|
||||||
|
package graph
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"math"
|
||||||
|
"sort"
|
||||||
|
"sync"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/lukaszraczylo/claude-mnemonic/pkg/models"
|
||||||
|
"github.com/rs/zerolog/log"
|
||||||
|
)
|
||||||
|
|
||||||
|
// RelationType defines the type of relationship between observations
|
||||||
|
type RelationType int
|
||||||
|
|
||||||
|
const (
|
||||||
|
// RelationFileOverlap indicates observations reference overlapping files
|
||||||
|
RelationFileOverlap RelationType = iota
|
||||||
|
// RelationSemantic indicates high semantic similarity (cosine > 0.85)
|
||||||
|
RelationSemantic
|
||||||
|
// RelationTemporal indicates observations from same session
|
||||||
|
RelationTemporal
|
||||||
|
// RelationConcept indicates shared concept tags
|
||||||
|
RelationConcept
|
||||||
|
)
|
||||||
|
|
||||||
|
// Edge represents a relationship between two observations
|
||||||
|
type Edge struct {
|
||||||
|
FromID int64
|
||||||
|
ToID int64
|
||||||
|
Relation RelationType
|
||||||
|
Weight float32 // 0.0-1.0, higher = stronger relationship
|
||||||
|
}
|
||||||
|
|
||||||
|
// Node represents an observation in the graph
|
||||||
|
type Node struct {
|
||||||
|
Metadata NodeMetadata
|
||||||
|
LastAccess time.Time
|
||||||
|
StoredEmb []float32 // Nil if recomputed on-demand
|
||||||
|
ID int64
|
||||||
|
Degree int // Number of edges (hub detection)
|
||||||
|
AccessCount int
|
||||||
|
}
|
||||||
|
|
||||||
|
// NodeMetadata contains observation metadata
|
||||||
|
type NodeMetadata struct {
|
||||||
|
CreatedAt time.Time
|
||||||
|
Project string
|
||||||
|
Type string
|
||||||
|
Title string
|
||||||
|
IsSuperseded bool
|
||||||
|
}
|
||||||
|
|
||||||
|
// CSRGraph represents a graph in Compressed Sparse Row format for memory efficiency
|
||||||
|
type CSRGraph struct {
|
||||||
|
RowPtr []int32 // Node adjacency list pointers
|
||||||
|
ColIdx []int32 // Edge destination IDs
|
||||||
|
Weights []float32 // Edge weights
|
||||||
|
mu sync.RWMutex
|
||||||
|
}
|
||||||
|
|
||||||
|
// ObservationGraph manages the observation relationship graph
|
||||||
|
type ObservationGraph struct {
|
||||||
|
nodes map[int64]*Node
|
||||||
|
csr *CSRGraph
|
||||||
|
edges []Edge
|
||||||
|
nodesMu sync.RWMutex
|
||||||
|
edgesMu sync.RWMutex
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewObservationGraph creates a new empty observation graph
|
||||||
|
func NewObservationGraph() *ObservationGraph {
|
||||||
|
return &ObservationGraph{
|
||||||
|
nodes: make(map[int64]*Node),
|
||||||
|
edges: make([]Edge, 0),
|
||||||
|
csr: &CSRGraph{},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// AddNode adds or updates a node in the graph
|
||||||
|
func (g *ObservationGraph) AddNode(node *Node) {
|
||||||
|
g.nodesMu.Lock()
|
||||||
|
defer g.nodesMu.Unlock()
|
||||||
|
|
||||||
|
g.nodes[node.ID] = node
|
||||||
|
}
|
||||||
|
|
||||||
|
// AddEdge adds an edge to the graph
|
||||||
|
func (g *ObservationGraph) AddEdge(edge Edge) {
|
||||||
|
g.edgesMu.Lock()
|
||||||
|
defer g.edgesMu.Unlock()
|
||||||
|
|
||||||
|
g.edges = append(g.edges, edge)
|
||||||
|
|
||||||
|
// Update degree counts
|
||||||
|
g.nodesMu.Lock()
|
||||||
|
if fromNode, ok := g.nodes[edge.FromID]; ok {
|
||||||
|
fromNode.Degree++
|
||||||
|
}
|
||||||
|
if toNode, ok := g.nodes[edge.ToID]; ok {
|
||||||
|
toNode.Degree++
|
||||||
|
}
|
||||||
|
g.nodesMu.Unlock()
|
||||||
|
}
|
||||||
|
|
||||||
|
// BuildCSR converts edge list to CSR format for efficient traversal
|
||||||
|
func (g *ObservationGraph) BuildCSR() error {
|
||||||
|
g.edgesMu.RLock()
|
||||||
|
g.nodesMu.RLock()
|
||||||
|
defer g.edgesMu.RUnlock()
|
||||||
|
defer g.nodesMu.RUnlock()
|
||||||
|
|
||||||
|
if len(g.nodes) == 0 {
|
||||||
|
return fmt.Errorf("no nodes in graph")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Create node ID to index mapping
|
||||||
|
nodeIDs := make([]int64, 0, len(g.nodes))
|
||||||
|
for id := range g.nodes {
|
||||||
|
nodeIDs = append(nodeIDs, id)
|
||||||
|
}
|
||||||
|
sort.Slice(nodeIDs, func(i, j int) bool {
|
||||||
|
return nodeIDs[i] < nodeIDs[j]
|
||||||
|
})
|
||||||
|
|
||||||
|
idToIdx := make(map[int64]int32)
|
||||||
|
for idx, id := range nodeIDs {
|
||||||
|
// #nosec G115 - observation count will never exceed int32 max (2.1B) in practice
|
||||||
|
idToIdx[id] = int32(idx)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Count edges per node
|
||||||
|
edgeCounts := make([]int, len(nodeIDs))
|
||||||
|
for _, edge := range g.edges {
|
||||||
|
if fromIdx, ok := idToIdx[edge.FromID]; ok {
|
||||||
|
edgeCounts[fromIdx]++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Build row pointers
|
||||||
|
rowPtr := make([]int32, len(nodeIDs)+1)
|
||||||
|
rowPtr[0] = 0
|
||||||
|
for i := 0; i < len(nodeIDs); i++ {
|
||||||
|
// #nosec G115 - edge counts per node will not exceed int32 max
|
||||||
|
rowPtr[i+1] = rowPtr[i] + int32(edgeCounts[i])
|
||||||
|
}
|
||||||
|
|
||||||
|
// Build column indices and weights
|
||||||
|
totalEdges := rowPtr[len(nodeIDs)]
|
||||||
|
colIdx := make([]int32, totalEdges)
|
||||||
|
weights := make([]float32, totalEdges)
|
||||||
|
|
||||||
|
// Temporary counter for filling CSR
|
||||||
|
currentPos := make([]int32, len(nodeIDs))
|
||||||
|
copy(currentPos, rowPtr[:len(nodeIDs)])
|
||||||
|
|
||||||
|
for _, edge := range g.edges {
|
||||||
|
fromIdx, fromOk := idToIdx[edge.FromID]
|
||||||
|
toIdx, toOk := idToIdx[edge.ToID]
|
||||||
|
|
||||||
|
if fromOk && toOk {
|
||||||
|
pos := currentPos[fromIdx]
|
||||||
|
colIdx[pos] = toIdx
|
||||||
|
weights[pos] = edge.Weight
|
||||||
|
currentPos[fromIdx]++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
g.csr.mu.Lock()
|
||||||
|
g.csr.RowPtr = rowPtr
|
||||||
|
g.csr.ColIdx = colIdx
|
||||||
|
g.csr.Weights = weights
|
||||||
|
g.csr.mu.Unlock()
|
||||||
|
|
||||||
|
log.Info().
|
||||||
|
Int("nodes", len(nodeIDs)).
|
||||||
|
Int("edges", int(totalEdges)).
|
||||||
|
Msg("Built CSR graph representation")
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetNeighbors returns neighboring nodes and their edge weights
|
||||||
|
func (g *ObservationGraph) GetNeighbors(nodeID int64) ([]int64, []float32, error) {
|
||||||
|
g.csr.mu.RLock()
|
||||||
|
defer g.csr.mu.RUnlock()
|
||||||
|
|
||||||
|
// Find node index in CSR
|
||||||
|
g.nodesMu.RLock()
|
||||||
|
nodeIDs := make([]int64, 0, len(g.nodes))
|
||||||
|
for id := range g.nodes {
|
||||||
|
nodeIDs = append(nodeIDs, id)
|
||||||
|
}
|
||||||
|
g.nodesMu.RUnlock()
|
||||||
|
|
||||||
|
sort.Slice(nodeIDs, func(i, j int) bool {
|
||||||
|
return nodeIDs[i] < nodeIDs[j]
|
||||||
|
})
|
||||||
|
|
||||||
|
nodeIdx := sort.Search(len(nodeIDs), func(i int) bool {
|
||||||
|
return nodeIDs[i] >= nodeID
|
||||||
|
})
|
||||||
|
|
||||||
|
if nodeIdx >= len(nodeIDs) || nodeIDs[nodeIdx] != nodeID {
|
||||||
|
return nil, nil, fmt.Errorf("node %d not found", nodeID)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Extract neighbors from CSR
|
||||||
|
startIdx := g.csr.RowPtr[nodeIdx]
|
||||||
|
endIdx := g.csr.RowPtr[nodeIdx+1]
|
||||||
|
|
||||||
|
neighborCount := endIdx - startIdx
|
||||||
|
neighbors := make([]int64, neighborCount)
|
||||||
|
weights := make([]float32, neighborCount)
|
||||||
|
|
||||||
|
for i := int32(0); i < neighborCount; i++ {
|
||||||
|
neighborIdx := g.csr.ColIdx[startIdx+i]
|
||||||
|
neighbors[i] = nodeIDs[neighborIdx]
|
||||||
|
weights[i] = g.csr.Weights[startIdx+i]
|
||||||
|
}
|
||||||
|
|
||||||
|
return neighbors, weights, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetNode retrieves a node by ID
|
||||||
|
func (g *ObservationGraph) GetNode(nodeID int64) (*Node, error) {
|
||||||
|
g.nodesMu.RLock()
|
||||||
|
defer g.nodesMu.RUnlock()
|
||||||
|
|
||||||
|
node, ok := g.nodes[nodeID]
|
||||||
|
if !ok {
|
||||||
|
return nil, fmt.Errorf("node %d not found", nodeID)
|
||||||
|
}
|
||||||
|
|
||||||
|
return node, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// FindHubs identifies hub nodes (high degree) in the graph
|
||||||
|
func (g *ObservationGraph) FindHubs(percentile float64) []int64 {
|
||||||
|
g.nodesMu.RLock()
|
||||||
|
defer g.nodesMu.RUnlock()
|
||||||
|
|
||||||
|
if len(g.nodes) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Collect all degrees
|
||||||
|
degrees := make([]int, 0, len(g.nodes))
|
||||||
|
nodeIDs := make([]int64, 0, len(g.nodes))
|
||||||
|
|
||||||
|
for id, node := range g.nodes {
|
||||||
|
degrees = append(degrees, node.Degree)
|
||||||
|
nodeIDs = append(nodeIDs, id)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Sort by degree
|
||||||
|
type nodeDegree struct {
|
||||||
|
ID int64
|
||||||
|
Degree int
|
||||||
|
}
|
||||||
|
|
||||||
|
nodeDegrees := make([]nodeDegree, len(nodeIDs))
|
||||||
|
for i := range nodeIDs {
|
||||||
|
nodeDegrees[i] = nodeDegree{
|
||||||
|
ID: nodeIDs[i],
|
||||||
|
Degree: degrees[i],
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
sort.Slice(nodeDegrees, func(i, j int) bool {
|
||||||
|
return nodeDegrees[i].Degree > nodeDegrees[j].Degree
|
||||||
|
})
|
||||||
|
|
||||||
|
// Return top percentile
|
||||||
|
cutoff := int(math.Ceil(float64(len(nodeDegrees)) * (1.0 - percentile)))
|
||||||
|
if cutoff > len(nodeDegrees) {
|
||||||
|
cutoff = len(nodeDegrees)
|
||||||
|
}
|
||||||
|
|
||||||
|
hubs := make([]int64, cutoff)
|
||||||
|
for i := 0; i < cutoff; i++ {
|
||||||
|
hubs[i] = nodeDegrees[i].ID
|
||||||
|
}
|
||||||
|
|
||||||
|
log.Info().
|
||||||
|
Int("total_nodes", len(g.nodes)).
|
||||||
|
Int("hubs", len(hubs)).
|
||||||
|
Float64("percentile", percentile).
|
||||||
|
Msg("Identified hub nodes")
|
||||||
|
|
||||||
|
return hubs
|
||||||
|
}
|
||||||
|
|
||||||
|
// Stats returns graph statistics
|
||||||
|
func (g *ObservationGraph) Stats() GraphStats {
|
||||||
|
g.nodesMu.RLock()
|
||||||
|
g.edgesMu.RLock()
|
||||||
|
defer g.nodesMu.RUnlock()
|
||||||
|
defer g.edgesMu.RUnlock()
|
||||||
|
|
||||||
|
stats := GraphStats{
|
||||||
|
NodeCount: len(g.nodes),
|
||||||
|
EdgeCount: len(g.edges),
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(g.nodes) > 0 {
|
||||||
|
degrees := make([]int, 0, len(g.nodes))
|
||||||
|
for _, node := range g.nodes {
|
||||||
|
degrees = append(degrees, node.Degree)
|
||||||
|
}
|
||||||
|
|
||||||
|
sort.Ints(degrees)
|
||||||
|
stats.AvgDegree = float64(sum(degrees)) / float64(len(degrees))
|
||||||
|
stats.MaxDegree = degrees[len(degrees)-1]
|
||||||
|
stats.MinDegree = degrees[0]
|
||||||
|
|
||||||
|
// Median
|
||||||
|
mid := len(degrees) / 2
|
||||||
|
if len(degrees)%2 == 0 {
|
||||||
|
stats.MedianDegree = float64(degrees[mid-1]+degrees[mid]) / 2.0
|
||||||
|
} else {
|
||||||
|
stats.MedianDegree = float64(degrees[mid])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Count edge types
|
||||||
|
stats.EdgeTypes = make(map[RelationType]int)
|
||||||
|
for _, edge := range g.edges {
|
||||||
|
stats.EdgeTypes[edge.Relation]++
|
||||||
|
}
|
||||||
|
|
||||||
|
return stats
|
||||||
|
}
|
||||||
|
|
||||||
|
// GraphStats contains graph statistics
|
||||||
|
type GraphStats struct {
|
||||||
|
EdgeTypes map[RelationType]int
|
||||||
|
AvgDegree float64
|
||||||
|
MedianDegree float64
|
||||||
|
NodeCount int
|
||||||
|
EdgeCount int
|
||||||
|
MaxDegree int
|
||||||
|
MinDegree int
|
||||||
|
}
|
||||||
|
|
||||||
|
// BuildFromObservations constructs a graph from a list of observations
|
||||||
|
func BuildFromObservations(ctx context.Context, observations []*models.Observation) (*ObservationGraph, error) {
|
||||||
|
graph := NewObservationGraph()
|
||||||
|
|
||||||
|
// Add nodes
|
||||||
|
for _, obs := range observations {
|
||||||
|
// Extract title from sql.NullString
|
||||||
|
title := ""
|
||||||
|
if obs.Title.Valid {
|
||||||
|
title = obs.Title.String
|
||||||
|
}
|
||||||
|
|
||||||
|
node := &Node{
|
||||||
|
ID: obs.ID,
|
||||||
|
Degree: 0,
|
||||||
|
Metadata: NodeMetadata{
|
||||||
|
Project: obs.Project,
|
||||||
|
Type: string(obs.Type),
|
||||||
|
Title: title,
|
||||||
|
CreatedAt: time.UnixMilli(obs.CreatedAtEpoch),
|
||||||
|
IsSuperseded: obs.IsSuperseded,
|
||||||
|
},
|
||||||
|
LastAccess: time.Now(),
|
||||||
|
AccessCount: 0,
|
||||||
|
}
|
||||||
|
graph.AddNode(node)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Detect edges (will be implemented in edge_detector.go)
|
||||||
|
edges, err := DetectEdges(ctx, observations)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("detect edges: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, edge := range edges {
|
||||||
|
graph.AddEdge(edge)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Build CSR representation
|
||||||
|
if err := graph.BuildCSR(); err != nil {
|
||||||
|
return nil, fmt.Errorf("build CSR: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return graph, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Helper function to sum integers
|
||||||
|
func sum(values []int) int {
|
||||||
|
total := 0
|
||||||
|
for _, v := range values {
|
||||||
|
total += v
|
||||||
|
}
|
||||||
|
return total
|
||||||
|
}
|
||||||
|
|
||||||
|
// String returns a human-readable representation of RelationType
|
||||||
|
func (r RelationType) String() string {
|
||||||
|
switch r {
|
||||||
|
case RelationFileOverlap:
|
||||||
|
return "file_overlap"
|
||||||
|
case RelationSemantic:
|
||||||
|
return "semantic"
|
||||||
|
case RelationTemporal:
|
||||||
|
return "temporal"
|
||||||
|
case RelationConcept:
|
||||||
|
return "concept"
|
||||||
|
default:
|
||||||
|
return "unknown"
|
||||||
|
}
|
||||||
|
}
|
||||||
+10
-10
@@ -15,10 +15,10 @@ import (
|
|||||||
|
|
||||||
// Server is the MCP server that exposes search tools.
|
// Server is the MCP server that exposes search tools.
|
||||||
type Server struct {
|
type Server struct {
|
||||||
searchMgr *search.Manager
|
|
||||||
version string
|
|
||||||
stdin io.Reader
|
stdin io.Reader
|
||||||
stdout io.Writer
|
stdout io.Writer
|
||||||
|
searchMgr *search.Manager
|
||||||
|
version string
|
||||||
}
|
}
|
||||||
|
|
||||||
// NewServer creates a new MCP server.
|
// NewServer creates a new MCP server.
|
||||||
@@ -41,17 +41,17 @@ type Request struct {
|
|||||||
|
|
||||||
// Response represents a JSON-RPC response.
|
// Response represents a JSON-RPC response.
|
||||||
type Response struct {
|
type Response struct {
|
||||||
JSONRPC string `json:"jsonrpc"`
|
|
||||||
ID any `json:"id"`
|
ID any `json:"id"`
|
||||||
Result any `json:"result,omitempty"`
|
Result any `json:"result,omitempty"`
|
||||||
Error *Error `json:"error,omitempty"`
|
Error *Error `json:"error,omitempty"`
|
||||||
|
JSONRPC string `json:"jsonrpc"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// Error represents a JSON-RPC error.
|
// Error represents a JSON-RPC error.
|
||||||
type Error struct {
|
type Error struct {
|
||||||
Code int `json:"code"`
|
|
||||||
Message string `json:"message"`
|
|
||||||
Data any `json:"data,omitempty"`
|
Data any `json:"data,omitempty"`
|
||||||
|
Message string `json:"message"`
|
||||||
|
Code int `json:"code"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// ToolCallParams represents parameters for tools/call method.
|
// ToolCallParams represents parameters for tools/call method.
|
||||||
@@ -62,9 +62,9 @@ type ToolCallParams struct {
|
|||||||
|
|
||||||
// Tool represents an MCP tool definition.
|
// Tool represents an MCP tool definition.
|
||||||
type Tool struct {
|
type Tool struct {
|
||||||
|
InputSchema map[string]any `json:"inputSchema"`
|
||||||
Name string `json:"name"`
|
Name string `json:"name"`
|
||||||
Description string `json:"description"`
|
Description string `json:"description"`
|
||||||
InputSchema map[string]any `json:"inputSchema"`
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Run starts the MCP server loop.
|
// Run starts the MCP server loop.
|
||||||
@@ -440,17 +440,17 @@ func (s *Server) callTool(ctx context.Context, name string, args json.RawMessage
|
|||||||
|
|
||||||
// TimelineParams represents parameters for timeline operations.
|
// TimelineParams represents parameters for timeline operations.
|
||||||
type TimelineParams struct {
|
type TimelineParams struct {
|
||||||
AnchorID int64 `json:"anchor_id"`
|
|
||||||
Query string `json:"query"`
|
Query string `json:"query"`
|
||||||
Before int `json:"before"`
|
|
||||||
After int `json:"after"`
|
|
||||||
Project string `json:"project"`
|
Project string `json:"project"`
|
||||||
ObsType string `json:"obs_type"`
|
ObsType string `json:"obs_type"`
|
||||||
Concepts string `json:"concepts"`
|
Concepts string `json:"concepts"`
|
||||||
Files string `json:"files"`
|
Files string `json:"files"`
|
||||||
|
Format string `json:"format"`
|
||||||
|
AnchorID int64 `json:"anchor_id"`
|
||||||
|
Before int `json:"before"`
|
||||||
|
After int `json:"after"`
|
||||||
DateStart int64 `json:"dateStart"`
|
DateStart int64 `json:"dateStart"`
|
||||||
DateEnd int64 `json:"dateEnd"`
|
DateEnd int64 `json:"dateEnd"`
|
||||||
Format string `json:"format"`
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// handleTimeline handles timeline requests.
|
// handleTimeline handles timeline requests.
|
||||||
|
|||||||
+13
-15
@@ -34,8 +34,8 @@ func (s *ServerSuite) TestNewServer() {
|
|||||||
func TestRequest(t *testing.T) {
|
func TestRequest(t *testing.T) {
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
name string
|
name string
|
||||||
req Request
|
|
||||||
expected string
|
expected string
|
||||||
|
req Request
|
||||||
}{
|
}{
|
||||||
{
|
{
|
||||||
name: "initialize request",
|
name: "initialize request",
|
||||||
@@ -137,10 +137,10 @@ func TestResponse(t *testing.T) {
|
|||||||
|
|
||||||
// TestError tests Error struct.
|
// TestError tests Error struct.
|
||||||
func TestError(t *testing.T) {
|
func TestError(t *testing.T) {
|
||||||
tests := []struct {
|
tests := []struct { //nolint:govet
|
||||||
name string
|
|
||||||
err Error
|
err Error
|
||||||
expected string
|
expected string
|
||||||
|
name string
|
||||||
}{
|
}{
|
||||||
{
|
{
|
||||||
name: "parse error",
|
name: "parse error",
|
||||||
@@ -365,11 +365,11 @@ func TestHandleRequest(t *testing.T) {
|
|||||||
ctx := context.Background()
|
ctx := context.Background()
|
||||||
|
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
name string
|
|
||||||
req *Request
|
req *Request
|
||||||
expectError bool
|
name string
|
||||||
errorCode int
|
|
||||||
errorMessage string
|
errorMessage string
|
||||||
|
errorCode int
|
||||||
|
expectError bool
|
||||||
}{
|
}{
|
||||||
{
|
{
|
||||||
name: "initialize method",
|
name: "initialize method",
|
||||||
@@ -683,10 +683,8 @@ func TestRun_MultipleRequests(t *testing.T) {
|
|||||||
func TestHandleTimeline_Defaults(t *testing.T) {
|
func TestHandleTimeline_Defaults(t *testing.T) {
|
||||||
// Test that handleTimeline sets default before/after values
|
// Test that handleTimeline sets default before/after values
|
||||||
params := TimelineParams{
|
params := TimelineParams{
|
||||||
AnchorID: 0,
|
Before: 0,
|
||||||
Query: "",
|
After: 0,
|
||||||
Before: 0,
|
|
||||||
After: 0,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Simulate the default value assignment from handleTimeline
|
// Simulate the default value assignment from handleTimeline
|
||||||
@@ -753,13 +751,13 @@ func TestServerStdinStdoutConfig(t *testing.T) {
|
|||||||
// TestResponseIDTypes tests that response IDs can be various types.
|
// TestResponseIDTypes tests that response IDs can be various types.
|
||||||
func TestResponseIDTypes(t *testing.T) {
|
func TestResponseIDTypes(t *testing.T) {
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
name string
|
|
||||||
id any
|
id any
|
||||||
|
name string
|
||||||
}{
|
}{
|
||||||
{"integer id", 1},
|
{name: "integer id", id: 1},
|
||||||
{"string id", "abc-123"},
|
{name: "string id", id: "abc-123"},
|
||||||
{"float id", 1.5},
|
{name: "float id", id: 1.5},
|
||||||
{"null id", nil},
|
{name: "null id", id: nil},
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, tt := range tests {
|
for _, tt := range tests {
|
||||||
|
|||||||
@@ -38,21 +38,15 @@ type PatternSyncFunc func(pattern *models.Pattern)
|
|||||||
|
|
||||||
// Detector detects and tracks recurring patterns across observations.
|
// Detector detects and tracks recurring patterns across observations.
|
||||||
type Detector struct {
|
type Detector struct {
|
||||||
config DetectorConfig
|
ctx context.Context
|
||||||
patternStore *sqlite.PatternStore
|
patternStore *sqlite.PatternStore
|
||||||
observationStore *sqlite.ObservationStore
|
observationStore *sqlite.ObservationStore
|
||||||
|
syncFunc PatternSyncFunc
|
||||||
// Vector sync callback
|
candidates map[string]*candidatePattern
|
||||||
syncFunc PatternSyncFunc
|
cancel context.CancelFunc
|
||||||
|
config DetectorConfig
|
||||||
// Candidate tracking (patterns not yet confirmed)
|
wg sync.WaitGroup
|
||||||
candidates map[string]*candidatePattern
|
candidatesMu sync.RWMutex
|
||||||
candidatesMu sync.RWMutex
|
|
||||||
|
|
||||||
// Background analysis
|
|
||||||
ctx context.Context
|
|
||||||
cancel context.CancelFunc
|
|
||||||
wg sync.WaitGroup
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// SetSyncFunc sets the callback for syncing patterns to vector store.
|
// SetSyncFunc sets the callback for syncing patterns to vector store.
|
||||||
@@ -62,11 +56,11 @@ func (d *Detector) SetSyncFunc(fn PatternSyncFunc) {
|
|||||||
|
|
||||||
// candidatePattern tracks a potential pattern before it reaches frequency threshold.
|
// candidatePattern tracks a potential pattern before it reaches frequency threshold.
|
||||||
type candidatePattern struct {
|
type candidatePattern struct {
|
||||||
|
patternType models.PatternType
|
||||||
|
title string
|
||||||
signature []string
|
signature []string
|
||||||
observationIDs []int64
|
observationIDs []int64
|
||||||
projects []string
|
projects []string
|
||||||
patternType models.PatternType
|
|
||||||
title string
|
|
||||||
lastSeenEpoch int64
|
lastSeenEpoch int64
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -331,16 +331,16 @@ func TestDefaultConfig(t *testing.T) {
|
|||||||
func TestGeneratePatternName(t *testing.T) {
|
func TestGeneratePatternName(t *testing.T) {
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
patternType models.PatternType
|
patternType models.PatternType
|
||||||
signature []string
|
|
||||||
title string
|
title string
|
||||||
wantPrefix string
|
wantPrefix string
|
||||||
|
signature []string
|
||||||
}{
|
}{
|
||||||
{models.PatternTypeBug, []string{"nil", "error"}, "", "Bug Pattern:"},
|
{patternType: models.PatternTypeBug, signature: []string{"nil", "error"}, title: "", wantPrefix: "Bug Pattern:"},
|
||||||
{models.PatternTypeRefactor, []string{"extract"}, "", "Refactor Pattern:"},
|
{patternType: models.PatternTypeRefactor, signature: []string{"extract"}, title: "", wantPrefix: "Refactor Pattern:"},
|
||||||
{models.PatternTypeArchitecture, []string{"service"}, "", "Architecture Pattern:"},
|
{patternType: models.PatternTypeArchitecture, signature: []string{"service"}, title: "", wantPrefix: "Architecture Pattern:"},
|
||||||
{models.PatternTypeAntiPattern, []string{"god-class"}, "", "Anti-Pattern:"},
|
{patternType: models.PatternTypeAntiPattern, signature: []string{"god-class"}, title: "", wantPrefix: "Anti-Pattern:"},
|
||||||
{models.PatternTypeBestPractice, []string{"testing"}, "", "Best Practice:"},
|
{patternType: models.PatternTypeBestPractice, signature: []string{"testing"}, title: "", wantPrefix: "Best Practice:"},
|
||||||
{models.PatternTypeBug, []string{}, "Short Title", "Short Title"}, // Use title directly
|
{patternType: models.PatternTypeBug, signature: []string{}, title: "Short Title", wantPrefix: "Short Title"}, // Use title directly
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, tt := range tests {
|
for _, tt := range tests {
|
||||||
|
|||||||
@@ -30,24 +30,24 @@ const (
|
|||||||
|
|
||||||
// Candidate represents a search result candidate for reranking.
|
// Candidate represents a search result candidate for reranking.
|
||||||
type Candidate struct {
|
type Candidate struct {
|
||||||
ID string // Document ID
|
Metadata map[string]any
|
||||||
Content string // Document text content for scoring
|
RerankInfo map[string]float64
|
||||||
Score float64 // Original bi-encoder similarity score
|
ID string
|
||||||
Metadata map[string]any // Preserved metadata
|
Content string
|
||||||
RerankInfo map[string]float64 // Reranking debug info (optional)
|
Score float64
|
||||||
}
|
}
|
||||||
|
|
||||||
// RerankResult represents a reranked search result.
|
// RerankResult represents a reranked search result.
|
||||||
type RerankResult struct {
|
type RerankResult struct {
|
||||||
ID string // Document ID
|
Metadata map[string]any
|
||||||
Content string // Document text content
|
ID string
|
||||||
OriginalScore float64 // Original bi-encoder score
|
Content string
|
||||||
RerankScore float64 // Cross-encoder relevance score
|
OriginalScore float64
|
||||||
CombinedScore float64 // Weighted combination of scores
|
RerankScore float64
|
||||||
Metadata map[string]any // Preserved metadata
|
CombinedScore float64
|
||||||
OriginalRank int // Position before reranking (1-indexed)
|
OriginalRank int
|
||||||
RerankRank int // Position after reranking (1-indexed)
|
RerankRank int
|
||||||
RankImprovement int // How much the rank improved (positive = moved up)
|
RankImprovement int
|
||||||
}
|
}
|
||||||
|
|
||||||
// Service provides cross-encoder reranking functionality.
|
// Service provides cross-encoder reranking functionality.
|
||||||
@@ -297,19 +297,19 @@ func (s *Service) scoreAll(query string, candidates []Candidate) ([]float64, err
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("create input_ids tensor: %w", err)
|
return nil, fmt.Errorf("create input_ids tensor: %w", err)
|
||||||
}
|
}
|
||||||
defer inputIdsTensor.Destroy()
|
defer func() { _ = inputIdsTensor.Destroy() }()
|
||||||
|
|
||||||
attentionMaskTensor, err := ort.NewTensor(inputShape, attentionMaskData)
|
attentionMaskTensor, err := ort.NewTensor(inputShape, attentionMaskData)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("create attention_mask tensor: %w", err)
|
return nil, fmt.Errorf("create attention_mask tensor: %w", err)
|
||||||
}
|
}
|
||||||
defer attentionMaskTensor.Destroy()
|
defer func() { _ = attentionMaskTensor.Destroy() }()
|
||||||
|
|
||||||
tokenTypeIdsTensor, err := ort.NewTensor(inputShape, tokenTypeIdsData)
|
tokenTypeIdsTensor, err := ort.NewTensor(inputShape, tokenTypeIdsData)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("create token_type_ids tensor: %w", err)
|
return nil, fmt.Errorf("create token_type_ids tensor: %w", err)
|
||||||
}
|
}
|
||||||
defer tokenTypeIdsTensor.Destroy()
|
defer func() { _ = tokenTypeIdsTensor.Destroy() }()
|
||||||
|
|
||||||
// Cross-encoder outputs [batch, 1] logits
|
// Cross-encoder outputs [batch, 1] logits
|
||||||
outputShape := ort.NewShape(int64(batchSize), 1)
|
outputShape := ort.NewShape(int64(batchSize), 1)
|
||||||
@@ -317,7 +317,7 @@ func (s *Service) scoreAll(query string, candidates []Candidate) ([]float64, err
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("create output tensor: %w", err)
|
return nil, fmt.Errorf("create output tensor: %w", err)
|
||||||
}
|
}
|
||||||
defer outputTensor.Destroy()
|
defer func() { _ = outputTensor.Destroy() }()
|
||||||
|
|
||||||
// Run inference
|
// Run inference
|
||||||
inputTensors := []ort.Value{inputIdsTensor, attentionMaskTensor, tokenTypeIdsTensor}
|
inputTensors := []ort.Value{inputIdsTensor, attentionMaskTensor, tokenTypeIdsTensor}
|
||||||
|
|||||||
@@ -21,13 +21,13 @@ type ObservationStore interface {
|
|||||||
|
|
||||||
// Recalculator periodically recalculates importance scores for observations.
|
// Recalculator periodically recalculates importance scores for observations.
|
||||||
type Recalculator struct {
|
type Recalculator struct {
|
||||||
|
log zerolog.Logger
|
||||||
store ObservationStore
|
store ObservationStore
|
||||||
calculator *Calculator
|
calculator *Calculator
|
||||||
log zerolog.Logger
|
|
||||||
interval time.Duration
|
|
||||||
batchSize int
|
|
||||||
stopCh chan struct{}
|
stopCh chan struct{}
|
||||||
doneCh chan struct{}
|
doneCh chan struct{}
|
||||||
|
interval time.Duration
|
||||||
|
batchSize int
|
||||||
mu sync.Mutex
|
mu sync.Mutex
|
||||||
running bool
|
running bool
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -16,14 +16,14 @@ import (
|
|||||||
|
|
||||||
// MockObservationStore is a mock implementation of ObservationStore for testing.
|
// MockObservationStore is a mock implementation of ObservationStore for testing.
|
||||||
type MockObservationStore struct {
|
type MockObservationStore struct {
|
||||||
mu sync.Mutex
|
|
||||||
observations []*models.Observation
|
|
||||||
scores map[int64]float64
|
|
||||||
conceptWeights map[string]float64
|
|
||||||
updateErr error
|
updateErr error
|
||||||
getErr error
|
getErr error
|
||||||
getConceptsErr error
|
getConceptsErr error
|
||||||
|
scores map[int64]float64
|
||||||
|
conceptWeights map[string]float64
|
||||||
|
observations []*models.Observation
|
||||||
updateScoresCalls int
|
updateScoresCalls int
|
||||||
|
mu sync.Mutex
|
||||||
}
|
}
|
||||||
|
|
||||||
func NewMockObservationStore() *MockObservationStore {
|
func NewMockObservationStore() *MockObservationStore {
|
||||||
|
|||||||
@@ -30,25 +30,25 @@ const (
|
|||||||
// ExpandedQuery represents a query variant with metadata.
|
// ExpandedQuery represents a query variant with metadata.
|
||||||
type ExpandedQuery struct {
|
type ExpandedQuery struct {
|
||||||
Query string `json:"query"`
|
Query string `json:"query"`
|
||||||
Weight float64 `json:"weight"` // Weight for result merging (0.0-1.0)
|
Source string `json:"source"`
|
||||||
Source string `json:"source"` // Where this expansion came from
|
Intent QueryIntent `json:"intent"`
|
||||||
Intent QueryIntent `json:"intent"` // Detected intent
|
Weight float64 `json:"weight"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// Expander provides context-aware query expansion.
|
// Expander provides context-aware query expansion.
|
||||||
type Expander struct {
|
type Expander struct {
|
||||||
embedSvc *embedding.Service
|
embedSvc *embedding.Service
|
||||||
vocabulary []VocabEntry // Known vocabulary from observations
|
|
||||||
vocabVectors [][]float32 // Embeddings for vocabulary entries
|
|
||||||
vocabMu sync.RWMutex // Protects vocabulary
|
|
||||||
intentPatterns map[QueryIntent][]*regexp.Regexp
|
intentPatterns map[QueryIntent][]*regexp.Regexp
|
||||||
|
vocabulary []VocabEntry
|
||||||
|
vocabVectors [][]float32
|
||||||
|
vocabMu sync.RWMutex
|
||||||
}
|
}
|
||||||
|
|
||||||
// VocabEntry represents a vocabulary term from observations.
|
// VocabEntry represents a vocabulary term from observations.
|
||||||
type VocabEntry struct {
|
type VocabEntry struct {
|
||||||
Term string // The term itself
|
Term string
|
||||||
Weight float64 // How common/important this term is (0.0-1.0)
|
Source string
|
||||||
Source string // Where it came from (title, concept, narrative)
|
Weight float64
|
||||||
}
|
}
|
||||||
|
|
||||||
// Config holds expander configuration.
|
// Config holds expander configuration.
|
||||||
|
|||||||
@@ -88,16 +88,16 @@ func (s *ExpanderSuite) TestExpand() {
|
|||||||
tests := []struct {
|
tests := []struct {
|
||||||
name string
|
name string
|
||||||
query string
|
query string
|
||||||
|
expectedIntent QueryIntent
|
||||||
minExpansions int
|
minExpansions int
|
||||||
hasOriginal bool
|
hasOriginal bool
|
||||||
expectedIntent QueryIntent
|
|
||||||
}{
|
}{
|
||||||
{"question", "how do I implement auth", 1, true, IntentQuestion},
|
{name: "question", query: "how do I implement auth", expectedIntent: IntentQuestion, minExpansions: 1, hasOriginal: true},
|
||||||
{"error", "fix the bug in login", 1, true, IntentError},
|
{name: "error", query: "fix the bug in login", expectedIntent: IntentError, minExpansions: 1, hasOriginal: true},
|
||||||
{"implementation", "implement user handler", 1, true, IntentImplementation},
|
{name: "implementation", query: "implement user handler", expectedIntent: IntentImplementation, minExpansions: 1, hasOriginal: true},
|
||||||
{"architecture", "architecture design", 1, true, IntentArchitecture},
|
{name: "architecture", query: "architecture design", expectedIntent: IntentArchitecture, minExpansions: 1, hasOriginal: true},
|
||||||
{"general", "database connection", 1, true, IntentGeneral},
|
{name: "general", query: "database connection", expectedIntent: IntentGeneral, minExpansions: 1, hasOriginal: true},
|
||||||
{"empty", "", 0, false, IntentGeneral},
|
{name: "empty", query: "", expectedIntent: IntentGeneral, minExpansions: 0, hasOriginal: false},
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, tt := range tests {
|
for _, tt := range tests {
|
||||||
@@ -392,13 +392,13 @@ func TestTruncate(t *testing.T) {
|
|||||||
tests := []struct {
|
tests := []struct {
|
||||||
name string
|
name string
|
||||||
input string
|
input string
|
||||||
maxLen int
|
|
||||||
expected string
|
expected string
|
||||||
|
maxLen int
|
||||||
}{
|
}{
|
||||||
{"short", "hello", 10, "hello"},
|
{name: "short", input: "hello", maxLen: 10, expected: "hello"},
|
||||||
{"exact", "hello", 5, "hello"},
|
{name: "exact", input: "hello", maxLen: 5, expected: "hello"},
|
||||||
{"long", "hello world", 5, "hello..."},
|
{name: "long", input: "hello world", maxLen: 5, expected: "hello..."},
|
||||||
{"empty", "", 10, ""},
|
{name: "empty", input: "", maxLen: 10, expected: ""},
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, tt := range tests {
|
for _, tt := range tests {
|
||||||
|
|||||||
@@ -517,16 +517,16 @@ func TestTruncate_TableDriven(t *testing.T) {
|
|||||||
tests := []struct {
|
tests := []struct {
|
||||||
name string
|
name string
|
||||||
input string
|
input string
|
||||||
maxLen int
|
|
||||||
expected string
|
expected string
|
||||||
|
maxLen int
|
||||||
}{
|
}{
|
||||||
{"short_string", "hello", 10, "hello"},
|
{name: "short_string", input: "hello", maxLen: 10, expected: "hello"},
|
||||||
{"exact_length", "hello", 5, "hello"},
|
{name: "exact_length", input: "hello", maxLen: 5, expected: "hello"},
|
||||||
{"long_string", "hello world", 5, "hello..."},
|
{name: "long_string", input: "hello world", maxLen: 5, expected: "hello..."},
|
||||||
{"empty_string", "", 10, ""},
|
{name: "empty_string", input: "", maxLen: 10, expected: ""},
|
||||||
{"whitespace_only", " ", 10, ""},
|
{name: "whitespace_only", input: " ", maxLen: 10, expected: ""},
|
||||||
{"with_leading_space", " hello ", 10, "hello"},
|
{name: "with_leading_space", input: " hello ", maxLen: 10, expected: "hello"},
|
||||||
{"very_long", "this is a very long string that should be truncated", 20, "this is a very long ..."},
|
{name: "very_long", input: "this is a very long string that should be truncated", maxLen: 20, expected: "this is a very long ..."},
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, tt := range tests {
|
for _, tt := range tests {
|
||||||
|
|||||||
+15
-15
@@ -35,41 +35,41 @@ func NewManager(
|
|||||||
|
|
||||||
// SearchParams contains parameters for unified search.
|
// SearchParams contains parameters for unified search.
|
||||||
type SearchParams struct {
|
type SearchParams struct {
|
||||||
Query string
|
Format string
|
||||||
Type string // "observations", "sessions", "prompts", or empty for all
|
Type string
|
||||||
Project string
|
Project string
|
||||||
ObsType string // Observation type filter
|
ObsType string
|
||||||
Concepts string
|
Concepts string
|
||||||
Files string
|
Files string
|
||||||
|
Query string
|
||||||
|
Scope string
|
||||||
|
OrderBy string
|
||||||
DateStart int64
|
DateStart int64
|
||||||
DateEnd int64
|
|
||||||
OrderBy string // "relevance", "date_desc", "date_asc"
|
|
||||||
Limit int
|
|
||||||
Offset int
|
Offset int
|
||||||
Format string // "index" or "full"
|
Limit int
|
||||||
Scope string // "project", "global", or empty for project+global
|
DateEnd int64
|
||||||
IncludeGlobal bool // If true, include global observations along with project-scoped
|
IncludeGlobal bool
|
||||||
ExcludeSuperseded bool // If true, exclude observations that have been superseded
|
ExcludeSuperseded bool
|
||||||
}
|
}
|
||||||
|
|
||||||
// SearchResult represents a unified search result.
|
// SearchResult represents a unified search result.
|
||||||
type SearchResult struct {
|
type SearchResult struct {
|
||||||
Type string `json:"type"` // "observation", "session", "prompt"
|
Metadata map[string]interface{} `json:"metadata,omitempty"`
|
||||||
ID int64 `json:"id"`
|
Type string `json:"type"`
|
||||||
Title string `json:"title,omitempty"`
|
Title string `json:"title,omitempty"`
|
||||||
Content string `json:"content,omitempty"`
|
Content string `json:"content,omitempty"`
|
||||||
Project string `json:"project"`
|
Project string `json:"project"`
|
||||||
Scope string `json:"scope,omitempty"` // "project" or "global"
|
Scope string `json:"scope,omitempty"`
|
||||||
|
ID int64 `json:"id"`
|
||||||
CreatedAt int64 `json:"created_at_epoch"`
|
CreatedAt int64 `json:"created_at_epoch"`
|
||||||
Score float64 `json:"score,omitempty"`
|
Score float64 `json:"score,omitempty"`
|
||||||
Metadata map[string]interface{} `json:"metadata,omitempty"`
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// UnifiedSearchResult contains the combined search results.
|
// UnifiedSearchResult contains the combined search results.
|
||||||
type UnifiedSearchResult struct {
|
type UnifiedSearchResult struct {
|
||||||
|
Query string `json:"query,omitempty"`
|
||||||
Results []SearchResult `json:"results"`
|
Results []SearchResult `json:"results"`
|
||||||
TotalCount int `json:"total_count"`
|
TotalCount int `json:"total_count"`
|
||||||
Query string `json:"query,omitempty"`
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// UnifiedSearch performs a unified search across all document types.
|
// UnifiedSearch performs a unified search across all document types.
|
||||||
|
|||||||
@@ -4,7 +4,6 @@ package search
|
|||||||
import (
|
import (
|
||||||
"database/sql"
|
"database/sql"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/lukaszraczylo/claude-mnemonic/pkg/models"
|
"github.com/lukaszraczylo/claude-mnemonic/pkg/models"
|
||||||
"github.com/stretchr/testify/assert"
|
"github.com/stretchr/testify/assert"
|
||||||
@@ -94,8 +93,8 @@ func TestTruncate(t *testing.T) {
|
|||||||
tests := []struct {
|
tests := []struct {
|
||||||
name string
|
name string
|
||||||
input string
|
input string
|
||||||
maxLen int
|
|
||||||
expected string
|
expected string
|
||||||
|
maxLen int
|
||||||
}{
|
}{
|
||||||
{
|
{
|
||||||
name: "short string no truncation",
|
name: "short string no truncation",
|
||||||
@@ -147,11 +146,11 @@ func TestTruncate(t *testing.T) {
|
|||||||
func TestObservationToResult(t *testing.T) {
|
func TestObservationToResult(t *testing.T) {
|
||||||
m := NewManager(nil, nil, nil, nil)
|
m := NewManager(nil, nil, nil, nil)
|
||||||
|
|
||||||
tests := []struct {
|
tests := []struct { //nolint:govet
|
||||||
name string
|
|
||||||
obs *models.Observation
|
obs *models.Observation
|
||||||
format string
|
|
||||||
expected SearchResult
|
expected SearchResult
|
||||||
|
format string
|
||||||
|
name string
|
||||||
}{
|
}{
|
||||||
{
|
{
|
||||||
name: "full format with all fields",
|
name: "full format with all fields",
|
||||||
@@ -239,11 +238,11 @@ func TestObservationToResult(t *testing.T) {
|
|||||||
func TestSummaryToResult(t *testing.T) {
|
func TestSummaryToResult(t *testing.T) {
|
||||||
m := NewManager(nil, nil, nil, nil)
|
m := NewManager(nil, nil, nil, nil)
|
||||||
|
|
||||||
tests := []struct {
|
tests := []struct { //nolint:govet
|
||||||
name string
|
|
||||||
summary *models.SessionSummary
|
summary *models.SessionSummary
|
||||||
format string
|
|
||||||
expected SearchResult
|
expected SearchResult
|
||||||
|
format string
|
||||||
|
name string
|
||||||
}{
|
}{
|
||||||
{
|
{
|
||||||
name: "full format with all fields",
|
name: "full format with all fields",
|
||||||
@@ -321,11 +320,11 @@ func TestSummaryToResult(t *testing.T) {
|
|||||||
func TestPromptToResult(t *testing.T) {
|
func TestPromptToResult(t *testing.T) {
|
||||||
m := NewManager(nil, nil, nil, nil)
|
m := NewManager(nil, nil, nil, nil)
|
||||||
|
|
||||||
tests := []struct {
|
tests := []struct { //nolint:govet
|
||||||
name string
|
|
||||||
prompt *models.UserPromptWithSession
|
prompt *models.UserPromptWithSession
|
||||||
format string
|
|
||||||
expected SearchResult
|
expected SearchResult
|
||||||
|
format string
|
||||||
|
name string
|
||||||
}{
|
}{
|
||||||
{
|
{
|
||||||
name: "full format with content",
|
name: "full format with content",
|
||||||
@@ -406,9 +405,9 @@ func TestPromptToResult(t *testing.T) {
|
|||||||
func TestSearchParamsValidation(t *testing.T) {
|
func TestSearchParamsValidation(t *testing.T) {
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
name string
|
name string
|
||||||
|
expectedOrder string
|
||||||
params SearchParams
|
params SearchParams
|
||||||
expectedLimit int
|
expectedLimit int
|
||||||
expectedOrder string
|
|
||||||
}{
|
}{
|
||||||
{
|
{
|
||||||
name: "default limit applied",
|
name: "default limit applied",
|
||||||
@@ -731,23 +730,21 @@ func TestPromptToResultFormats(t *testing.T) {
|
|||||||
func TestSearchParamsDefaults(t *testing.T) {
|
func TestSearchParamsDefaults(t *testing.T) {
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
name string
|
name string
|
||||||
initialLimit int
|
|
||||||
initialOrder string
|
initialOrder string
|
||||||
expectedLimit int
|
|
||||||
expectedOrder string
|
expectedOrder string
|
||||||
|
initialLimit int
|
||||||
|
expectedLimit int
|
||||||
}{
|
}{
|
||||||
{"zero_limit", 0, "", 20, "date_desc"},
|
{name: "zero_limit", initialLimit: 0, initialOrder: "", expectedLimit: 20, expectedOrder: "date_desc"},
|
||||||
{"negative_limit", -5, "", 20, "date_desc"},
|
{name: "negative_limit", initialLimit: -5, initialOrder: "", expectedLimit: 20, expectedOrder: "date_desc"},
|
||||||
{"over_100_limit", 150, "", 100, "date_desc"},
|
{name: "over_100_limit", initialLimit: 150, initialOrder: "", expectedLimit: 100, expectedOrder: "date_desc"},
|
||||||
{"valid_limit_50", 50, "relevance", 50, "relevance"},
|
{name: "valid_limit_50", initialLimit: 50, initialOrder: "relevance", expectedLimit: 50, expectedOrder: "relevance"},
|
||||||
{"custom_order", 30, "date_asc", 30, "date_asc"},
|
{name: "custom_order", initialLimit: 30, initialOrder: "date_asc", expectedLimit: 30, expectedOrder: "date_asc"},
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, tt := range tests {
|
for _, tt := range tests {
|
||||||
t.Run(tt.name, func(t *testing.T) {
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
params := SearchParams{
|
params := SearchParams{
|
||||||
Query: "test",
|
|
||||||
Project: "project",
|
|
||||||
Limit: tt.initialLimit,
|
Limit: tt.initialLimit,
|
||||||
OrderBy: tt.initialOrder,
|
OrderBy: tt.initialOrder,
|
||||||
}
|
}
|
||||||
@@ -774,18 +771,18 @@ func TestTruncateEdgeCases(t *testing.T) {
|
|||||||
tests := []struct {
|
tests := []struct {
|
||||||
name string
|
name string
|
||||||
input string
|
input string
|
||||||
maxLen int
|
|
||||||
expected string
|
expected string
|
||||||
|
maxLen int
|
||||||
}{
|
}{
|
||||||
// Unicode strings - uses byte length so ensure maxLen accommodates full string
|
// Unicode strings - uses byte length so ensure maxLen accommodates full string
|
||||||
{"unicode_string_no_truncate", "日本語テスト", 20, "日本語テスト"},
|
{name: "unicode_string_no_truncate", input: "日本語テスト", maxLen: 20, expected: "日本語テスト"},
|
||||||
{"mixed_unicode_no_truncate", "Hello世界", 15, "Hello世界"},
|
{name: "mixed_unicode_no_truncate", input: "Hello世界", maxLen: 15, expected: "Hello世界"},
|
||||||
// ASCII truncation
|
// ASCII truncation
|
||||||
{"ascii_truncate", "Hello World", 5, "Hello..."},
|
{name: "ascii_truncate", input: "Hello World", maxLen: 5, expected: "Hello..."},
|
||||||
{"only_whitespace", " ", 10, ""},
|
{name: "only_whitespace", input: " ", maxLen: 10, expected: ""},
|
||||||
{"tabs_and_newlines", "\t\n \t", 10, ""},
|
{name: "tabs_and_newlines", input: "\t\n \t", maxLen: 10, expected: ""},
|
||||||
{"newlines_with_content", "\n\nhello\n\n", 10, "hello"},
|
{name: "newlines_with_content", input: "\n\nhello\n\n", maxLen: 10, expected: "hello"},
|
||||||
{"zero_max_len", "hello", 0, "..."},
|
{name: "zero_max_len", input: "hello", maxLen: 0, expected: "..."},
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, tt := range tests {
|
for _, tt := range tests {
|
||||||
@@ -812,8 +809,6 @@ func TestUnifiedSearchResultEmpty(t *testing.T) {
|
|||||||
// TestSearchResultMetadata tests SearchResult metadata handling.
|
// TestSearchResultMetadata tests SearchResult metadata handling.
|
||||||
func TestSearchResultMetadata(t *testing.T) {
|
func TestSearchResultMetadata(t *testing.T) {
|
||||||
result := SearchResult{
|
result := SearchResult{
|
||||||
Type: "observation",
|
|
||||||
ID: 1,
|
|
||||||
Metadata: map[string]interface{}{
|
Metadata: map[string]interface{}{
|
||||||
"obs_type": "discovery",
|
"obs_type": "discovery",
|
||||||
"scope": "project",
|
"scope": "project",
|
||||||
@@ -835,10 +830,7 @@ func TestSearchResultTypes(t *testing.T) {
|
|||||||
for _, typ := range types {
|
for _, typ := range types {
|
||||||
t.Run(typ, func(t *testing.T) {
|
t.Run(typ, func(t *testing.T) {
|
||||||
result := SearchResult{
|
result := SearchResult{
|
||||||
Type: typ,
|
Type: typ,
|
||||||
ID: 1,
|
|
||||||
Project: "test",
|
|
||||||
CreatedAt: time.Now().UnixMilli(),
|
|
||||||
}
|
}
|
||||||
assert.Equal(t, typ, result.Type)
|
assert.Equal(t, typ, result.Type)
|
||||||
})
|
})
|
||||||
@@ -952,8 +944,8 @@ func TestSearchParams_OrderByValues(t *testing.T) {
|
|||||||
for _, order := range validOrders {
|
for _, order := range validOrders {
|
||||||
t.Run("order_"+order, func(t *testing.T) {
|
t.Run("order_"+order, func(t *testing.T) {
|
||||||
params := SearchParams{
|
params := SearchParams{
|
||||||
Query: "test",
|
Query: "test", //nolint:govet
|
||||||
Project: "test",
|
Project: "test", //nolint:govet
|
||||||
OrderBy: order,
|
OrderBy: order,
|
||||||
}
|
}
|
||||||
assert.Equal(t, order, params.OrderBy)
|
assert.Equal(t, order, params.OrderBy)
|
||||||
@@ -968,9 +960,7 @@ func TestSearchParams_TypeValues(t *testing.T) {
|
|||||||
for _, typ := range validTypes {
|
for _, typ := range validTypes {
|
||||||
t.Run("type_"+typ, func(t *testing.T) {
|
t.Run("type_"+typ, func(t *testing.T) {
|
||||||
params := SearchParams{
|
params := SearchParams{
|
||||||
Query: "test",
|
Type: typ,
|
||||||
Project: "test",
|
|
||||||
Type: typ,
|
|
||||||
}
|
}
|
||||||
assert.Equal(t, typ, params.Type)
|
assert.Equal(t, typ, params.Type)
|
||||||
})
|
})
|
||||||
@@ -984,8 +974,8 @@ func TestSearchParams_ScopeValues(t *testing.T) {
|
|||||||
for _, scope := range validScopes {
|
for _, scope := range validScopes {
|
||||||
t.Run("scope_"+scope, func(t *testing.T) {
|
t.Run("scope_"+scope, func(t *testing.T) {
|
||||||
params := SearchParams{
|
params := SearchParams{
|
||||||
Query: "test",
|
Query: "test", //nolint:govet
|
||||||
Project: "test",
|
Project: "test", //nolint:govet
|
||||||
Scope: scope,
|
Scope: scope,
|
||||||
}
|
}
|
||||||
assert.Equal(t, scope, params.Scope)
|
assert.Equal(t, scope, params.Scope)
|
||||||
@@ -1000,8 +990,8 @@ func TestSearchParams_FormatValues(t *testing.T) {
|
|||||||
for _, format := range validFormats {
|
for _, format := range validFormats {
|
||||||
t.Run("format_"+format, func(t *testing.T) {
|
t.Run("format_"+format, func(t *testing.T) {
|
||||||
params := SearchParams{
|
params := SearchParams{
|
||||||
Query: "test",
|
Query: "test", //nolint:govet
|
||||||
Project: "test",
|
Project: "test", //nolint:govet
|
||||||
Format: format,
|
Format: format,
|
||||||
}
|
}
|
||||||
assert.Equal(t, format, params.Format)
|
assert.Equal(t, format, params.Format)
|
||||||
@@ -1020,7 +1010,7 @@ func TestUnifiedSearchResult_MultipleResults(t *testing.T) {
|
|||||||
result := UnifiedSearchResult{
|
result := UnifiedSearchResult{
|
||||||
Results: results,
|
Results: results,
|
||||||
TotalCount: 3,
|
TotalCount: 3,
|
||||||
Query: "test query",
|
Query: "test query", //nolint:govet
|
||||||
}
|
}
|
||||||
|
|
||||||
assert.Len(t, result.Results, 3)
|
assert.Len(t, result.Results, 3)
|
||||||
@@ -1040,8 +1030,8 @@ func TestSearchResult_Metadata(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
result := SearchResult{
|
result := SearchResult{
|
||||||
Type: "observation",
|
Type: "observation", //nolint:govet
|
||||||
ID: 1,
|
ID: 1, //nolint:govet
|
||||||
Metadata: metadata,
|
Metadata: metadata,
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1066,8 +1056,6 @@ func TestSearchResult_Scores(t *testing.T) {
|
|||||||
for _, tt := range tests {
|
for _, tt := range tests {
|
||||||
t.Run(tt.name, func(t *testing.T) {
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
result := SearchResult{
|
result := SearchResult{
|
||||||
Type: "observation",
|
|
||||||
ID: 1,
|
|
||||||
Score: tt.score,
|
Score: tt.score,
|
||||||
}
|
}
|
||||||
assert.Equal(t, tt.score, result.Score)
|
assert.Equal(t, tt.score, result.Score)
|
||||||
|
|||||||
+14
-15
@@ -33,11 +33,11 @@ const (
|
|||||||
|
|
||||||
// Release represents a GitHub release.
|
// Release represents a GitHub release.
|
||||||
type Release struct {
|
type Release struct {
|
||||||
|
PublishedAt time.Time `json:"published_at"`
|
||||||
TagName string `json:"tag_name"`
|
TagName string `json:"tag_name"`
|
||||||
Name string `json:"name"`
|
Name string `json:"name"`
|
||||||
PublishedAt time.Time `json:"published_at"`
|
|
||||||
Assets []Asset `json:"assets"`
|
|
||||||
Body string `json:"body"`
|
Body string `json:"body"`
|
||||||
|
Assets []Asset `json:"assets"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// Asset represents a release asset.
|
// Asset represents a release asset.
|
||||||
@@ -49,15 +49,15 @@ type Asset struct {
|
|||||||
|
|
||||||
// UpdateInfo contains information about an available update.
|
// UpdateInfo contains information about an available update.
|
||||||
type UpdateInfo struct {
|
type UpdateInfo struct {
|
||||||
Available bool `json:"available"`
|
PublishedAt time.Time `json:"published_at,omitempty"`
|
||||||
CurrentVersion string `json:"current_version"`
|
CurrentVersion string `json:"current_version"`
|
||||||
LatestVersion string `json:"latest_version"`
|
LatestVersion string `json:"latest_version"`
|
||||||
ReleaseNotes string `json:"release_notes,omitempty"`
|
ReleaseNotes string `json:"release_notes,omitempty"`
|
||||||
PublishedAt time.Time `json:"published_at,omitempty"`
|
|
||||||
DownloadURL string `json:"download_url,omitempty"`
|
DownloadURL string `json:"download_url,omitempty"`
|
||||||
ChecksumsURL string `json:"checksums_url,omitempty"`
|
ChecksumsURL string `json:"checksums_url,omitempty"`
|
||||||
BundleURL string `json:"bundle_url,omitempty"` // Sigstore bundle (.sigstore.json)
|
BundleURL string `json:"bundle_url,omitempty"`
|
||||||
ManualUpdateCommand string `json:"manual_update_command,omitempty"`
|
ManualUpdateCommand string `json:"manual_update_command,omitempty"`
|
||||||
|
Available bool `json:"available"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// InstallScriptURL is the URL to the remote installation script.
|
// InstallScriptURL is the URL to the remote installation script.
|
||||||
@@ -74,23 +74,22 @@ func GetManualUpdateCommand(version string) string {
|
|||||||
|
|
||||||
// UpdateStatus represents the current update status.
|
// UpdateStatus represents the current update status.
|
||||||
type UpdateStatus struct {
|
type UpdateStatus struct {
|
||||||
State string `json:"state"` // "idle", "checking", "downloading", "verifying", "applying", "done", "error"
|
State string `json:"state"`
|
||||||
Progress float64 `json:"progress"`
|
|
||||||
Message string `json:"message"`
|
Message string `json:"message"`
|
||||||
Error string `json:"error,omitempty"`
|
Error string `json:"error,omitempty"`
|
||||||
ManualUpdateCommand string `json:"manual_update_command,omitempty"` // Shown when update fails
|
ManualUpdateCommand string `json:"manual_update_command,omitempty"`
|
||||||
|
Progress float64 `json:"progress"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// Updater handles self-updates.
|
// Updater handles self-updates.
|
||||||
type Updater struct {
|
type Updater struct { //nolint:govet
|
||||||
|
httpClient *http.Client
|
||||||
|
cachedUpdate *UpdateInfo
|
||||||
|
lastCheck time.Time
|
||||||
|
status UpdateStatus
|
||||||
currentVersion string
|
currentVersion string
|
||||||
installDir string
|
installDir string
|
||||||
httpClient *http.Client
|
mu sync.RWMutex
|
||||||
|
|
||||||
mu sync.RWMutex
|
|
||||||
status UpdateStatus
|
|
||||||
lastCheck time.Time
|
|
||||||
cachedUpdate *UpdateInfo
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// New creates a new Updater.
|
// New creates a new Updater.
|
||||||
|
|||||||
@@ -0,0 +1,309 @@
|
|||||||
|
package hybrid
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"sync"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/lukaszraczylo/claude-mnemonic/internal/vector/sqlitevec"
|
||||||
|
"github.com/rs/zerolog/log"
|
||||||
|
)
|
||||||
|
|
||||||
|
// AutoTuner dynamically adjusts hub threshold based on query performance
|
||||||
|
type AutoTuner struct {
|
||||||
|
ctx context.Context
|
||||||
|
client *Client
|
||||||
|
cancel context.CancelFunc
|
||||||
|
latencies []time.Duration
|
||||||
|
wg sync.WaitGroup
|
||||||
|
queries int64
|
||||||
|
targetLatency time.Duration
|
||||||
|
adjustPeriod time.Duration
|
||||||
|
minThreshold int
|
||||||
|
maxThreshold int
|
||||||
|
adjustments int
|
||||||
|
latenciesMu sync.Mutex
|
||||||
|
}
|
||||||
|
|
||||||
|
// AutoTunerConfig configures the auto-tuner
|
||||||
|
type AutoTunerConfig struct {
|
||||||
|
TargetLatency time.Duration // Target p95 latency (default: 50ms)
|
||||||
|
MinThreshold int // Min hub threshold (default: 2)
|
||||||
|
MaxThreshold int // Max hub threshold (default: 20)
|
||||||
|
AdjustPeriod time.Duration // Adjustment frequency (default: 5min)
|
||||||
|
}
|
||||||
|
|
||||||
|
// DefaultAutoTunerConfig returns sensible defaults
|
||||||
|
func DefaultAutoTunerConfig() AutoTunerConfig {
|
||||||
|
return AutoTunerConfig{
|
||||||
|
TargetLatency: 50 * time.Millisecond,
|
||||||
|
MinThreshold: 2,
|
||||||
|
MaxThreshold: 20,
|
||||||
|
AdjustPeriod: 5 * time.Minute,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewAutoTuner creates a new auto-tuner for the hybrid client
|
||||||
|
func NewAutoTuner(client *Client, cfg AutoTunerConfig) *AutoTuner {
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
|
||||||
|
tuner := &AutoTuner{
|
||||||
|
client: client,
|
||||||
|
targetLatency: cfg.TargetLatency,
|
||||||
|
minThreshold: cfg.MinThreshold,
|
||||||
|
maxThreshold: cfg.MaxThreshold,
|
||||||
|
adjustPeriod: cfg.AdjustPeriod,
|
||||||
|
latencies: make([]time.Duration, 0, 1000),
|
||||||
|
ctx: ctx,
|
||||||
|
cancel: cancel,
|
||||||
|
}
|
||||||
|
|
||||||
|
return tuner
|
||||||
|
}
|
||||||
|
|
||||||
|
// Start begins auto-tuning in the background
|
||||||
|
func (a *AutoTuner) Start() {
|
||||||
|
a.wg.Add(1)
|
||||||
|
go a.tuningLoop()
|
||||||
|
|
||||||
|
log.Info().
|
||||||
|
Dur("target_latency", a.targetLatency).
|
||||||
|
Int("min_threshold", a.minThreshold).
|
||||||
|
Int("max_threshold", a.maxThreshold).
|
||||||
|
Dur("adjust_period", a.adjustPeriod).
|
||||||
|
Msg("Auto-tuner started")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Stop stops the auto-tuner
|
||||||
|
func (a *AutoTuner) Stop() {
|
||||||
|
a.cancel()
|
||||||
|
a.wg.Wait()
|
||||||
|
log.Info().Msg("Auto-tuner stopped")
|
||||||
|
}
|
||||||
|
|
||||||
|
// RecordQuery records a query latency for analysis
|
||||||
|
func (a *AutoTuner) RecordQuery(latency time.Duration) {
|
||||||
|
a.latenciesMu.Lock()
|
||||||
|
defer a.latenciesMu.Unlock()
|
||||||
|
|
||||||
|
a.queries++
|
||||||
|
a.latencies = append(a.latencies, latency)
|
||||||
|
|
||||||
|
// Keep only recent queries (last 1000)
|
||||||
|
if len(a.latencies) > 1000 {
|
||||||
|
a.latencies = a.latencies[len(a.latencies)-1000:]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// tuningLoop periodically adjusts hub threshold
|
||||||
|
func (a *AutoTuner) tuningLoop() {
|
||||||
|
defer a.wg.Done()
|
||||||
|
|
||||||
|
ticker := time.NewTicker(a.adjustPeriod)
|
||||||
|
defer ticker.Stop()
|
||||||
|
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-a.ctx.Done():
|
||||||
|
return
|
||||||
|
|
||||||
|
case <-ticker.C:
|
||||||
|
a.adjustThreshold()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// adjustThreshold analyzes recent queries and adjusts hub threshold
|
||||||
|
func (a *AutoTuner) adjustThreshold() {
|
||||||
|
a.latenciesMu.Lock()
|
||||||
|
defer a.latenciesMu.Unlock()
|
||||||
|
|
||||||
|
if len(a.latencies) < 10 {
|
||||||
|
// Not enough data yet
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// Calculate p95 latency
|
||||||
|
p95 := calculateP95(a.latencies)
|
||||||
|
|
||||||
|
currentThreshold := a.client.hubThreshold
|
||||||
|
|
||||||
|
log.Debug().
|
||||||
|
Dur("p95_latency", p95).
|
||||||
|
Dur("target_latency", a.targetLatency).
|
||||||
|
Int("current_threshold", currentThreshold).
|
||||||
|
Int("queries", len(a.latencies)).
|
||||||
|
Msg("Auto-tuner evaluating performance")
|
||||||
|
|
||||||
|
// Determine adjustment direction
|
||||||
|
var newThreshold int
|
||||||
|
|
||||||
|
if p95 > a.targetLatency {
|
||||||
|
// Too slow - lower threshold (more hubs = faster queries)
|
||||||
|
adjustment := calculateAdjustment(p95, a.targetLatency)
|
||||||
|
newThreshold = currentThreshold - adjustment
|
||||||
|
|
||||||
|
if newThreshold < a.minThreshold {
|
||||||
|
newThreshold = a.minThreshold
|
||||||
|
}
|
||||||
|
|
||||||
|
log.Info().
|
||||||
|
Dur("p95", p95).
|
||||||
|
Int("old_threshold", currentThreshold).
|
||||||
|
Int("new_threshold", newThreshold).
|
||||||
|
Msg("Auto-tuner: Lowering hub threshold (too slow)")
|
||||||
|
|
||||||
|
} else if p95 < a.targetLatency*8/10 {
|
||||||
|
// Too fast - raise threshold (fewer hubs = more savings)
|
||||||
|
// Only adjust if significantly faster (20% margin)
|
||||||
|
adjustment := calculateAdjustment(a.targetLatency, p95)
|
||||||
|
newThreshold = currentThreshold + adjustment
|
||||||
|
|
||||||
|
if newThreshold > a.maxThreshold {
|
||||||
|
newThreshold = a.maxThreshold
|
||||||
|
}
|
||||||
|
|
||||||
|
log.Info().
|
||||||
|
Dur("p95", p95).
|
||||||
|
Int("old_threshold", currentThreshold).
|
||||||
|
Int("new_threshold", newThreshold).
|
||||||
|
Msg("Auto-tuner: Raising hub threshold (room for savings)")
|
||||||
|
|
||||||
|
} else {
|
||||||
|
// Within acceptable range, no adjustment needed
|
||||||
|
log.Debug().
|
||||||
|
Dur("p95", p95).
|
||||||
|
Int("threshold", currentThreshold).
|
||||||
|
Msg("Auto-tuner: Performance acceptable, no adjustment")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// Apply adjustment
|
||||||
|
if newThreshold != currentThreshold {
|
||||||
|
a.client.hubThreshold = newThreshold
|
||||||
|
a.adjustments++
|
||||||
|
|
||||||
|
// Clear latency history after adjustment
|
||||||
|
a.latencies = make([]time.Duration, 0, 1000)
|
||||||
|
|
||||||
|
log.Info().
|
||||||
|
Int("threshold", newThreshold).
|
||||||
|
Int("total_adjustments", a.adjustments).
|
||||||
|
Msg("Hub threshold adjusted by auto-tuner")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// calculateP95 computes the 95th percentile latency
|
||||||
|
func calculateP95(latencies []time.Duration) time.Duration {
|
||||||
|
if len(latencies) == 0 {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
// Sort latencies
|
||||||
|
sorted := make([]time.Duration, len(latencies))
|
||||||
|
copy(sorted, latencies)
|
||||||
|
|
||||||
|
// Simple bubble sort (small dataset)
|
||||||
|
n := len(sorted)
|
||||||
|
for i := 0; i < n-1; i++ {
|
||||||
|
for j := 0; j < n-i-1; j++ {
|
||||||
|
if sorted[j] > sorted[j+1] {
|
||||||
|
sorted[j], sorted[j+1] = sorted[j+1], sorted[j]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Return 95th percentile
|
||||||
|
idx := int(float64(len(sorted)) * 0.95)
|
||||||
|
if idx >= len(sorted) {
|
||||||
|
idx = len(sorted) - 1
|
||||||
|
}
|
||||||
|
|
||||||
|
return sorted[idx]
|
||||||
|
}
|
||||||
|
|
||||||
|
// calculateAdjustment determines how much to adjust threshold
|
||||||
|
func calculateAdjustment(actual, target time.Duration) int {
|
||||||
|
// Calculate percentage difference
|
||||||
|
diff := float64(actual-target) / float64(target)
|
||||||
|
|
||||||
|
// Adjust more aggressively for larger differences
|
||||||
|
if diff > 0.5 || diff < -0.5 {
|
||||||
|
return 3 // Large adjustment
|
||||||
|
} else if diff > 0.2 || diff < -0.2 {
|
||||||
|
return 2 // Medium adjustment
|
||||||
|
}
|
||||||
|
|
||||||
|
return 1 // Small adjustment
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetStats returns auto-tuner statistics
|
||||||
|
func (a *AutoTuner) GetStats() AutoTunerStats {
|
||||||
|
a.latenciesMu.Lock()
|
||||||
|
defer a.latenciesMu.Unlock()
|
||||||
|
|
||||||
|
stats := AutoTunerStats{
|
||||||
|
CurrentThreshold: a.client.hubThreshold,
|
||||||
|
TargetLatency: a.targetLatency,
|
||||||
|
TotalQueries: a.queries,
|
||||||
|
TotalAdjustments: a.adjustments,
|
||||||
|
RecentQueries: len(a.latencies),
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(a.latencies) > 0 {
|
||||||
|
stats.P95Latency = calculateP95(a.latencies)
|
||||||
|
|
||||||
|
// Calculate average
|
||||||
|
var total time.Duration
|
||||||
|
for _, lat := range a.latencies {
|
||||||
|
total += lat
|
||||||
|
}
|
||||||
|
stats.AvgLatency = total / time.Duration(len(a.latencies))
|
||||||
|
}
|
||||||
|
|
||||||
|
return stats
|
||||||
|
}
|
||||||
|
|
||||||
|
// AutoTunerStats contains auto-tuner statistics
|
||||||
|
type AutoTunerStats struct {
|
||||||
|
CurrentThreshold int
|
||||||
|
TargetLatency time.Duration
|
||||||
|
P95Latency time.Duration
|
||||||
|
AvgLatency time.Duration
|
||||||
|
TotalQueries int64
|
||||||
|
TotalAdjustments int
|
||||||
|
RecentQueries int
|
||||||
|
}
|
||||||
|
|
||||||
|
// AutoTunedClient wraps Client with automatic performance tuning
|
||||||
|
type AutoTunedClient struct {
|
||||||
|
*Client
|
||||||
|
tuner *AutoTuner
|
||||||
|
}
|
||||||
|
|
||||||
|
// Query wraps the underlying Query call with latency tracking
|
||||||
|
func (a *AutoTunedClient) Query(ctx context.Context, query string, limit int, where map[string]any) ([]sqlitevec.QueryResult, error) {
|
||||||
|
start := time.Now()
|
||||||
|
results, err := a.Client.Query(ctx, query, limit, where)
|
||||||
|
latency := time.Since(start)
|
||||||
|
|
||||||
|
a.tuner.RecordQuery(latency)
|
||||||
|
|
||||||
|
return results, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// WithAutoTuning wraps a hybrid client with auto-tuning enabled
|
||||||
|
func WithAutoTuning(client *Client, cfg AutoTunerConfig) *AutoTunedClient {
|
||||||
|
tuner := NewAutoTuner(client, cfg)
|
||||||
|
tuner.Start()
|
||||||
|
|
||||||
|
return &AutoTunedClient{
|
||||||
|
Client: client,
|
||||||
|
tuner: tuner,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Stop stops the auto-tuner
|
||||||
|
func (a *AutoTunedClient) StopTuning() {
|
||||||
|
a.tuner.Stop()
|
||||||
|
}
|
||||||
@@ -0,0 +1,515 @@
|
|||||||
|
// Package hybrid provides LEANN-inspired selective vector storage for claude-mnemonic.
|
||||||
|
//
|
||||||
|
// This package implements a hybrid storage strategy where frequently-accessed
|
||||||
|
// observations ("hubs") have their embeddings stored, while infrequently-accessed
|
||||||
|
// observations have their embeddings recomputed on-demand during search.
|
||||||
|
//
|
||||||
|
// This approach reduces storage by 60-80% with minimal impact on search latency (<50ms).
|
||||||
|
package hybrid
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"database/sql"
|
||||||
|
"fmt"
|
||||||
|
"math"
|
||||||
|
"sync"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/lukaszraczylo/claude-mnemonic/internal/embedding"
|
||||||
|
"github.com/lukaszraczylo/claude-mnemonic/internal/vector/sqlitevec"
|
||||||
|
"github.com/rs/zerolog/log"
|
||||||
|
)
|
||||||
|
|
||||||
|
// VectorStorageStrategy defines how embeddings are stored/computed
|
||||||
|
type VectorStorageStrategy int
|
||||||
|
|
||||||
|
const (
|
||||||
|
// StorageAlways stores all embeddings (current behavior, backwards compatible)
|
||||||
|
StorageAlways VectorStorageStrategy = iota
|
||||||
|
// StorageHub stores only frequently-accessed "hub" embeddings (recommended)
|
||||||
|
StorageHub
|
||||||
|
// StorageOnDemand recomputes all embeddings during search (maximum savings)
|
||||||
|
StorageOnDemand
|
||||||
|
)
|
||||||
|
|
||||||
|
// Client wraps sqlitevec.Client with selective storage logic
|
||||||
|
type Client struct {
|
||||||
|
base *sqlitevec.Client
|
||||||
|
db *sql.DB
|
||||||
|
embedSvc *embedding.Service
|
||||||
|
accessCount map[string]int
|
||||||
|
lastAccess map[string]time.Time
|
||||||
|
contentCache map[string]string
|
||||||
|
strategy VectorStorageStrategy
|
||||||
|
hubThreshold int
|
||||||
|
mu sync.RWMutex
|
||||||
|
cacheMu sync.RWMutex
|
||||||
|
}
|
||||||
|
|
||||||
|
// Config for hybrid client
|
||||||
|
type Config struct {
|
||||||
|
BaseClient *sqlitevec.Client
|
||||||
|
DB *sql.DB
|
||||||
|
EmbedSvc *embedding.Service
|
||||||
|
Strategy VectorStorageStrategy
|
||||||
|
HubThreshold int // Default: 5 accesses
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewClient creates a new hybrid vector client
|
||||||
|
func NewClient(cfg Config) *Client {
|
||||||
|
if cfg.HubThreshold <= 0 {
|
||||||
|
cfg.HubThreshold = 5
|
||||||
|
}
|
||||||
|
|
||||||
|
log.Info().
|
||||||
|
Str("strategy", strategyToString(cfg.Strategy)).
|
||||||
|
Int("hub_threshold", cfg.HubThreshold).
|
||||||
|
Msg("Initializing LEANN hybrid vector client")
|
||||||
|
|
||||||
|
return &Client{
|
||||||
|
base: cfg.BaseClient,
|
||||||
|
db: cfg.DB,
|
||||||
|
embedSvc: cfg.EmbedSvc,
|
||||||
|
strategy: cfg.Strategy,
|
||||||
|
hubThreshold: cfg.HubThreshold,
|
||||||
|
accessCount: make(map[string]int),
|
||||||
|
lastAccess: make(map[string]time.Time),
|
||||||
|
contentCache: make(map[string]string),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// AddDocuments implements selective storage based on strategy
|
||||||
|
func (c *Client) AddDocuments(ctx context.Context, docs []sqlitevec.Document) error {
|
||||||
|
if len(docs) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
switch c.strategy {
|
||||||
|
case StorageAlways:
|
||||||
|
// Use existing implementation - store all embeddings
|
||||||
|
return c.base.AddDocuments(ctx, docs)
|
||||||
|
|
||||||
|
case StorageHub:
|
||||||
|
// Store only hub candidates
|
||||||
|
return c.addDocumentsSelective(ctx, docs)
|
||||||
|
|
||||||
|
case StorageOnDemand:
|
||||||
|
// Don't store embeddings, only cache content
|
||||||
|
return c.cacheDocuments(ctx, docs)
|
||||||
|
|
||||||
|
default:
|
||||||
|
return c.base.AddDocuments(ctx, docs)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// addDocumentsSelective stores embeddings only for hub-qualified documents
|
||||||
|
func (c *Client) addDocumentsSelective(ctx context.Context, docs []sqlitevec.Document) error {
|
||||||
|
// Always cache content for potential recomputation
|
||||||
|
if err := c.cacheDocuments(ctx, docs); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Filter to hub documents
|
||||||
|
hubDocs := make([]sqlitevec.Document, 0, len(docs))
|
||||||
|
for _, doc := range docs {
|
||||||
|
if c.isHub(doc.ID) {
|
||||||
|
hubDocs = append(hubDocs, doc)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Store only hub embeddings
|
||||||
|
if len(hubDocs) > 0 {
|
||||||
|
log.Debug().
|
||||||
|
Int("total", len(docs)).
|
||||||
|
Int("hubs", len(hubDocs)).
|
||||||
|
Msg("Storing selective embeddings")
|
||||||
|
return c.base.AddDocuments(ctx, hubDocs)
|
||||||
|
}
|
||||||
|
|
||||||
|
log.Debug().Int("total", len(docs)).Msg("All documents cached, no hubs to store")
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// cacheDocuments stores content for later recomputation
|
||||||
|
func (c *Client) cacheDocuments(ctx context.Context, docs []sqlitevec.Document) error {
|
||||||
|
c.cacheMu.Lock()
|
||||||
|
defer c.cacheMu.Unlock()
|
||||||
|
|
||||||
|
for _, doc := range docs {
|
||||||
|
c.contentCache[doc.ID] = doc.Content
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// DeleteDocuments removes documents by their IDs
|
||||||
|
func (c *Client) DeleteDocuments(ctx context.Context, ids []string) error {
|
||||||
|
// Remove from base storage
|
||||||
|
if err := c.base.DeleteDocuments(ctx, ids); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Clean up caches
|
||||||
|
c.mu.Lock()
|
||||||
|
for _, id := range ids {
|
||||||
|
delete(c.accessCount, id)
|
||||||
|
delete(c.lastAccess, id)
|
||||||
|
}
|
||||||
|
c.mu.Unlock()
|
||||||
|
|
||||||
|
c.cacheMu.Lock()
|
||||||
|
for _, id := range ids {
|
||||||
|
delete(c.contentCache, id)
|
||||||
|
}
|
||||||
|
c.cacheMu.Unlock()
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Query performs search with dynamic recomputation
|
||||||
|
func (c *Client) Query(ctx context.Context, query string, limit int, where map[string]any) ([]sqlitevec.QueryResult, error) {
|
||||||
|
switch c.strategy {
|
||||||
|
case StorageAlways:
|
||||||
|
// Use existing implementation
|
||||||
|
return c.queryAndTrack(ctx, query, limit, where)
|
||||||
|
|
||||||
|
case StorageHub:
|
||||||
|
// Search hubs, then expand with recomputation
|
||||||
|
return c.queryHybrid(ctx, query, limit, where)
|
||||||
|
|
||||||
|
case StorageOnDemand:
|
||||||
|
// Fully dynamic search
|
||||||
|
return c.queryDynamic(ctx, query, limit, where)
|
||||||
|
|
||||||
|
default:
|
||||||
|
return c.queryAndTrack(ctx, query, limit, where)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// queryAndTrack wraps base Query with access tracking
|
||||||
|
func (c *Client) queryAndTrack(ctx context.Context, query string, limit int, where map[string]any) ([]sqlitevec.QueryResult, error) {
|
||||||
|
results, err := c.base.Query(ctx, query, limit, where)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Track access for hub detection
|
||||||
|
c.trackAccess(results)
|
||||||
|
|
||||||
|
return results, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// queryHybrid searches stored hubs and recomputes non-hubs
|
||||||
|
func (c *Client) queryHybrid(ctx context.Context, query string, limit int, where map[string]any) ([]sqlitevec.QueryResult, error) {
|
||||||
|
startTime := time.Now()
|
||||||
|
|
||||||
|
// 1. Query stored hub embeddings (limit * 2 for expansion)
|
||||||
|
hubResults, err := c.base.Query(ctx, query, limit*2, where)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// 2. Track access
|
||||||
|
c.trackAccess(hubResults)
|
||||||
|
|
||||||
|
// 3. Get candidate non-hub IDs (from content cache)
|
||||||
|
candidates := c.getCandidateNonHubs(where, limit*2)
|
||||||
|
|
||||||
|
// 4. Recompute embeddings for candidates if we have any
|
||||||
|
var recomputedResults []sqlitevec.QueryResult
|
||||||
|
if len(candidates) > 0 {
|
||||||
|
recomputedResults, err = c.recomputeAndScore(ctx, query, candidates)
|
||||||
|
if err != nil {
|
||||||
|
// Log but don't fail - use hub results only
|
||||||
|
log.Warn().Err(err).Msg("Failed to recompute embeddings, using hub results only")
|
||||||
|
recomputedResults = nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 5. Merge and rank
|
||||||
|
allResults := append(hubResults, recomputedResults...)
|
||||||
|
sortBySimilarity(allResults)
|
||||||
|
|
||||||
|
// 6. Return top K
|
||||||
|
if len(allResults) > limit {
|
||||||
|
allResults = allResults[:limit]
|
||||||
|
}
|
||||||
|
|
||||||
|
duration := time.Since(startTime)
|
||||||
|
log.Debug().
|
||||||
|
Dur("duration_ms", duration).
|
||||||
|
Int("hubs", len(hubResults)).
|
||||||
|
Int("recomputed", len(recomputedResults)).
|
||||||
|
Int("results", len(allResults)).
|
||||||
|
Msg("Hybrid search completed")
|
||||||
|
|
||||||
|
return allResults, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// queryDynamic recomputes all embeddings on-the-fly
|
||||||
|
func (c *Client) queryDynamic(ctx context.Context, query string, limit int, where map[string]any) ([]sqlitevec.QueryResult, error) {
|
||||||
|
startTime := time.Now()
|
||||||
|
|
||||||
|
// Get all candidate IDs from content cache
|
||||||
|
candidates := c.getCandidateNonHubs(where, limit*5)
|
||||||
|
|
||||||
|
// Recompute and score all
|
||||||
|
results, err := c.recomputeAndScore(ctx, query, candidates)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Track access
|
||||||
|
c.trackAccess(results)
|
||||||
|
|
||||||
|
// Return top K
|
||||||
|
if len(results) > limit {
|
||||||
|
results = results[:limit]
|
||||||
|
}
|
||||||
|
|
||||||
|
duration := time.Since(startTime)
|
||||||
|
log.Debug().
|
||||||
|
Dur("duration_ms", duration).
|
||||||
|
Int("recomputed", len(candidates)).
|
||||||
|
Int("results", len(results)).
|
||||||
|
Msg("Dynamic search completed")
|
||||||
|
|
||||||
|
return results, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// recomputeAndScore generates embeddings and computes similarities
|
||||||
|
func (c *Client) recomputeAndScore(ctx context.Context, query string, candidateIDs []string) ([]sqlitevec.QueryResult, error) {
|
||||||
|
if len(candidateIDs) == 0 {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Generate query embedding
|
||||||
|
queryEmb, err := c.embedSvc.Embed(query)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("embed query: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Get content for candidates
|
||||||
|
c.cacheMu.RLock()
|
||||||
|
texts := make([]string, 0, len(candidateIDs))
|
||||||
|
validIDs := make([]string, 0, len(candidateIDs))
|
||||||
|
for _, id := range candidateIDs {
|
||||||
|
if content, ok := c.contentCache[id]; ok && content != "" {
|
||||||
|
texts = append(texts, content)
|
||||||
|
validIDs = append(validIDs, id)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
c.cacheMu.RUnlock()
|
||||||
|
|
||||||
|
if len(texts) == 0 {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Batch generate embeddings
|
||||||
|
embeddings, err := c.embedSvc.EmbedBatch(texts)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("batch embed: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Compute similarities
|
||||||
|
results := make([]sqlitevec.QueryResult, len(embeddings))
|
||||||
|
for i, emb := range embeddings {
|
||||||
|
similarity := cosineSimilarity(queryEmb, emb)
|
||||||
|
distance := 1.0 - similarity // Convert to distance
|
||||||
|
|
||||||
|
results[i] = sqlitevec.QueryResult{
|
||||||
|
ID: validIDs[i],
|
||||||
|
Distance: float64(distance),
|
||||||
|
Similarity: float64(similarity),
|
||||||
|
Metadata: make(map[string]any),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return results, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// trackAccess records document access for hub detection
|
||||||
|
func (c *Client) trackAccess(results []sqlitevec.QueryResult) {
|
||||||
|
if len(results) == 0 {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
c.mu.Lock()
|
||||||
|
defer c.mu.Unlock()
|
||||||
|
|
||||||
|
now := time.Now()
|
||||||
|
for _, r := range results {
|
||||||
|
c.accessCount[r.ID]++
|
||||||
|
c.lastAccess[r.ID] = now
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// isHub checks if a document qualifies as a hub
|
||||||
|
func (c *Client) isHub(docID string) bool {
|
||||||
|
c.mu.RLock()
|
||||||
|
defer c.mu.RUnlock()
|
||||||
|
|
||||||
|
count := c.accessCount[docID]
|
||||||
|
return count >= c.hubThreshold
|
||||||
|
}
|
||||||
|
|
||||||
|
// getCandidateNonHubs returns IDs of non-hub documents matching filter
|
||||||
|
func (c *Client) getCandidateNonHubs(where map[string]any, limit int) []string {
|
||||||
|
c.cacheMu.RLock()
|
||||||
|
defer c.cacheMu.RUnlock()
|
||||||
|
|
||||||
|
candidates := make([]string, 0, limit)
|
||||||
|
for id := range c.contentCache {
|
||||||
|
if !c.isHub(id) {
|
||||||
|
candidates = append(candidates, id)
|
||||||
|
if len(candidates) >= limit {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return candidates
|
||||||
|
}
|
||||||
|
|
||||||
|
// IsConnected always returns true (wraps base client)
|
||||||
|
func (c *Client) IsConnected() bool {
|
||||||
|
return c.base.IsConnected()
|
||||||
|
}
|
||||||
|
|
||||||
|
// Close releases resources
|
||||||
|
func (c *Client) Close() error {
|
||||||
|
return c.base.Close()
|
||||||
|
}
|
||||||
|
|
||||||
|
// Count returns the total number of vectors in the store
|
||||||
|
func (c *Client) Count(ctx context.Context) (int64, error) {
|
||||||
|
return c.base.Count(ctx)
|
||||||
|
}
|
||||||
|
|
||||||
|
// ModelVersion returns the current embedding model version
|
||||||
|
func (c *Client) ModelVersion() string {
|
||||||
|
return c.base.ModelVersion()
|
||||||
|
}
|
||||||
|
|
||||||
|
// NeedsRebuild checks if vectors need to be rebuilt due to model version change
|
||||||
|
func (c *Client) NeedsRebuild(ctx context.Context) (bool, string) {
|
||||||
|
return c.base.NeedsRebuild(ctx)
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetStaleVectors returns doc_ids of vectors with mismatched or null model versions
|
||||||
|
func (c *Client) GetStaleVectors(ctx context.Context) ([]sqlitevec.StaleVectorInfo, error) {
|
||||||
|
return c.base.GetStaleVectors(ctx)
|
||||||
|
}
|
||||||
|
|
||||||
|
// DeleteVectorsByDocIDs removes vectors by their doc_ids
|
||||||
|
func (c *Client) DeleteVectorsByDocIDs(ctx context.Context, docIDs []string) error {
|
||||||
|
return c.base.DeleteVectorsByDocIDs(ctx, docIDs)
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetStorageStats returns storage efficiency metrics
|
||||||
|
func (c *Client) GetStorageStats(ctx context.Context) (StorageStats, error) {
|
||||||
|
c.mu.RLock()
|
||||||
|
c.cacheMu.RLock()
|
||||||
|
defer c.mu.RUnlock()
|
||||||
|
defer c.cacheMu.RUnlock()
|
||||||
|
|
||||||
|
totalDocs := len(c.contentCache)
|
||||||
|
hubCount := 0
|
||||||
|
for id := range c.contentCache {
|
||||||
|
if c.accessCount[id] >= c.hubThreshold {
|
||||||
|
hubCount++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
storedCount := hubCount
|
||||||
|
if c.strategy == StorageAlways {
|
||||||
|
// Get actual count from database
|
||||||
|
if count, err := c.base.Count(ctx); err == nil {
|
||||||
|
storedCount = int(count)
|
||||||
|
}
|
||||||
|
} else if c.strategy == StorageOnDemand {
|
||||||
|
storedCount = 0
|
||||||
|
}
|
||||||
|
|
||||||
|
embeddingSize := 384 * 4 // 384 dims × 4 bytes (float32)
|
||||||
|
storedBytes := storedCount * embeddingSize
|
||||||
|
potentialBytes := totalDocs * embeddingSize
|
||||||
|
|
||||||
|
savingsPercent := 0.0
|
||||||
|
if potentialBytes > 0 {
|
||||||
|
savingsPercent = (1.0 - float64(storedBytes)/float64(potentialBytes)) * 100
|
||||||
|
}
|
||||||
|
|
||||||
|
return StorageStats{
|
||||||
|
TotalDocuments: totalDocs,
|
||||||
|
HubDocuments: hubCount,
|
||||||
|
StoredEmbeddings: storedCount,
|
||||||
|
StorageBytes: storedBytes,
|
||||||
|
SavingsPercent: savingsPercent,
|
||||||
|
Strategy: c.strategy,
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// StorageStats contains storage efficiency metrics
|
||||||
|
type StorageStats struct {
|
||||||
|
TotalDocuments int
|
||||||
|
HubDocuments int
|
||||||
|
StoredEmbeddings int
|
||||||
|
StorageBytes int
|
||||||
|
SavingsPercent float64
|
||||||
|
Strategy VectorStorageStrategy
|
||||||
|
}
|
||||||
|
|
||||||
|
// Helper functions
|
||||||
|
|
||||||
|
func cosineSimilarity(a, b []float32) float32 {
|
||||||
|
var dotProduct, normA, normB float32
|
||||||
|
for i := range a {
|
||||||
|
dotProduct += a[i] * b[i]
|
||||||
|
normA += a[i] * a[i]
|
||||||
|
normB += b[i] * b[i]
|
||||||
|
}
|
||||||
|
if normA == 0 || normB == 0 {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
return dotProduct / float32(math.Sqrt(float64(normA))*math.Sqrt(float64(normB)))
|
||||||
|
}
|
||||||
|
|
||||||
|
func sortBySimilarity(results []sqlitevec.QueryResult) {
|
||||||
|
// Use a simple but efficient sorting algorithm
|
||||||
|
n := len(results)
|
||||||
|
for i := 0; i < n-1; i++ {
|
||||||
|
for j := 0; j < n-i-1; j++ {
|
||||||
|
if results[j].Similarity < results[j+1].Similarity {
|
||||||
|
results[j], results[j+1] = results[j+1], results[j]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func strategyToString(s VectorStorageStrategy) string {
|
||||||
|
switch s {
|
||||||
|
case StorageAlways:
|
||||||
|
return "always"
|
||||||
|
case StorageHub:
|
||||||
|
return "hub"
|
||||||
|
case StorageOnDemand:
|
||||||
|
return "on_demand"
|
||||||
|
default:
|
||||||
|
return "unknown"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ParseStrategy converts a string to VectorStorageStrategy
|
||||||
|
func ParseStrategy(s string) VectorStorageStrategy {
|
||||||
|
switch s {
|
||||||
|
case "hub":
|
||||||
|
return StorageHub
|
||||||
|
case "on_demand":
|
||||||
|
return StorageOnDemand
|
||||||
|
case "always":
|
||||||
|
return StorageAlways
|
||||||
|
default:
|
||||||
|
return StorageHub // Default to hub strategy
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,186 @@
|
|||||||
|
package hybrid
|
||||||
|
|
||||||
|
import (
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/lukaszraczylo/claude-mnemonic/internal/vector/sqlitevec"
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestParseStrategy(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
input string
|
||||||
|
expected VectorStorageStrategy
|
||||||
|
}{
|
||||||
|
{"hub_strategy", "hub", StorageHub},
|
||||||
|
{"on_demand_strategy", "on_demand", StorageOnDemand},
|
||||||
|
{"always_strategy", "always", StorageAlways},
|
||||||
|
{"invalid_defaults_to_hub", "invalid", StorageHub},
|
||||||
|
{"empty_defaults_to_hub", "", StorageHub},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
result := ParseStrategy(tt.input)
|
||||||
|
assert.Equal(t, tt.expected, result)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestStrategyToString(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
expected string
|
||||||
|
input VectorStorageStrategy
|
||||||
|
}{
|
||||||
|
{"hub_to_string", "hub", StorageHub},
|
||||||
|
{"on_demand_to_string", "on_demand", StorageOnDemand},
|
||||||
|
{"always_to_string", "always", StorageAlways},
|
||||||
|
{"invalid_to_unknown", "unknown", VectorStorageStrategy(99)},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
result := strategyToString(tt.input)
|
||||||
|
assert.Equal(t, tt.expected, result)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCosineSimilarity(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
a []float32
|
||||||
|
b []float32
|
||||||
|
expected float32
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "identical_vectors",
|
||||||
|
a: []float32{1, 0, 0},
|
||||||
|
b: []float32{1, 0, 0},
|
||||||
|
expected: 1.0,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "orthogonal_vectors",
|
||||||
|
a: []float32{1, 0, 0},
|
||||||
|
b: []float32{0, 1, 0},
|
||||||
|
expected: 0.0,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "opposite_vectors",
|
||||||
|
a: []float32{1, 0, 0},
|
||||||
|
b: []float32{-1, 0, 0},
|
||||||
|
expected: -1.0,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "zero_vector",
|
||||||
|
a: []float32{0, 0, 0},
|
||||||
|
b: []float32{1, 1, 1},
|
||||||
|
expected: 0.0,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "parallel_vectors",
|
||||||
|
a: []float32{2, 0, 0},
|
||||||
|
b: []float32{4, 0, 0},
|
||||||
|
expected: 1.0,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
result := cosineSimilarity(tt.a, tt.b)
|
||||||
|
assert.InDelta(t, tt.expected, result, 0.001)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSortBySimilarity(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
input []sqlitevec.QueryResult
|
||||||
|
expected []string // Expected order of IDs
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "already_sorted",
|
||||||
|
input: []sqlitevec.QueryResult{
|
||||||
|
{ID: "doc1", Similarity: 0.9},
|
||||||
|
{ID: "doc2", Similarity: 0.7},
|
||||||
|
{ID: "doc3", Similarity: 0.5},
|
||||||
|
},
|
||||||
|
expected: []string{"doc1", "doc2", "doc3"},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "reverse_sorted",
|
||||||
|
input: []sqlitevec.QueryResult{
|
||||||
|
{ID: "doc1", Similarity: 0.3},
|
||||||
|
{ID: "doc2", Similarity: 0.7},
|
||||||
|
{ID: "doc3", Similarity: 0.9},
|
||||||
|
},
|
||||||
|
expected: []string{"doc3", "doc2", "doc1"},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "random_order",
|
||||||
|
input: []sqlitevec.QueryResult{
|
||||||
|
{ID: "doc1", Similarity: 0.5},
|
||||||
|
{ID: "doc2", Similarity: 0.9},
|
||||||
|
{ID: "doc3", Similarity: 0.3},
|
||||||
|
{ID: "doc4", Similarity: 0.7},
|
||||||
|
},
|
||||||
|
expected: []string{"doc2", "doc4", "doc1", "doc3"},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "identical_similarities",
|
||||||
|
input: []sqlitevec.QueryResult{
|
||||||
|
{ID: "doc1", Similarity: 0.5},
|
||||||
|
{ID: "doc2", Similarity: 0.5},
|
||||||
|
{ID: "doc3", Similarity: 0.5},
|
||||||
|
},
|
||||||
|
expected: []string{"doc1", "doc2", "doc3"},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "empty_list",
|
||||||
|
input: []sqlitevec.QueryResult{},
|
||||||
|
expected: []string{},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "single_element",
|
||||||
|
input: []sqlitevec.QueryResult{
|
||||||
|
{ID: "doc1", Similarity: 0.5},
|
||||||
|
},
|
||||||
|
expected: []string{"doc1"},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
sortBySimilarity(tt.input)
|
||||||
|
|
||||||
|
actual := make([]string, len(tt.input))
|
||||||
|
for i, r := range tt.input {
|
||||||
|
actual[i] = r.ID
|
||||||
|
}
|
||||||
|
|
||||||
|
assert.Equal(t, tt.expected, actual)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSortBySimilarity_PreserveOtherFields(t *testing.T) {
|
||||||
|
input := []sqlitevec.QueryResult{
|
||||||
|
{ID: "doc1", Similarity: 0.3, Distance: 0.7, Metadata: map[string]any{"key": "val1"}},
|
||||||
|
{ID: "doc2", Similarity: 0.9, Distance: 0.1, Metadata: map[string]any{"key": "val2"}},
|
||||||
|
}
|
||||||
|
|
||||||
|
sortBySimilarity(input)
|
||||||
|
|
||||||
|
assert.Equal(t, "doc2", input[0].ID)
|
||||||
|
assert.InDelta(t, 0.9, input[0].Similarity, 0.001)
|
||||||
|
assert.InDelta(t, 0.1, input[0].Distance, 0.001)
|
||||||
|
assert.Equal(t, "val2", input[0].Metadata["key"])
|
||||||
|
|
||||||
|
assert.Equal(t, "doc1", input[1].ID)
|
||||||
|
assert.InDelta(t, 0.3, input[1].Similarity, 0.001)
|
||||||
|
assert.InDelta(t, 0.7, input[1].Distance, 0.001)
|
||||||
|
assert.Equal(t, "val1", input[1].Metadata["key"])
|
||||||
|
}
|
||||||
@@ -0,0 +1,62 @@
|
|||||||
|
package hybrid
|
||||||
|
|
||||||
|
import (
|
||||||
|
"os"
|
||||||
|
"strconv"
|
||||||
|
|
||||||
|
"github.com/rs/zerolog/log"
|
||||||
|
)
|
||||||
|
|
||||||
|
// GetStrategyFromEnv reads CLAUDE_MNEMONIC_VECTOR_STRATEGY from environment
|
||||||
|
func GetStrategyFromEnv() VectorStorageStrategy {
|
||||||
|
strategyStr := os.Getenv("CLAUDE_MNEMONIC_VECTOR_STRATEGY")
|
||||||
|
if strategyStr == "" {
|
||||||
|
// Default to hub strategy for optimal balance
|
||||||
|
return StorageHub
|
||||||
|
}
|
||||||
|
|
||||||
|
strategy := ParseStrategy(strategyStr)
|
||||||
|
log.Info().
|
||||||
|
Str("env_value", strategyStr).
|
||||||
|
Str("strategy", strategyToString(strategy)).
|
||||||
|
Msg("Vector storage strategy from environment")
|
||||||
|
|
||||||
|
return strategy
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetHubThresholdFromEnv reads CLAUDE_MNEMONIC_HUB_THRESHOLD from environment
|
||||||
|
func GetHubThresholdFromEnv() int {
|
||||||
|
thresholdStr := os.Getenv("CLAUDE_MNEMONIC_HUB_THRESHOLD")
|
||||||
|
if thresholdStr == "" {
|
||||||
|
return 5 // Default threshold
|
||||||
|
}
|
||||||
|
|
||||||
|
threshold, err := strconv.Atoi(thresholdStr)
|
||||||
|
if err != nil {
|
||||||
|
log.Warn().
|
||||||
|
Err(err).
|
||||||
|
Str("env_value", thresholdStr).
|
||||||
|
Msg("Invalid hub threshold in environment, using default")
|
||||||
|
return 5
|
||||||
|
}
|
||||||
|
|
||||||
|
if threshold < 1 {
|
||||||
|
log.Warn().
|
||||||
|
Int("env_value", threshold).
|
||||||
|
Msg("Hub threshold too low, using minimum of 1")
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
|
||||||
|
log.Info().
|
||||||
|
Int("threshold", threshold).
|
||||||
|
Msg("Hub threshold from environment")
|
||||||
|
|
||||||
|
return threshold
|
||||||
|
}
|
||||||
|
|
||||||
|
// IsHybridEnabled checks if hybrid storage should be used
|
||||||
|
// Returns false if CLAUDE_MNEMONIC_VECTOR_STRATEGY=always (backwards compat)
|
||||||
|
func IsHybridEnabled() bool {
|
||||||
|
strategy := GetStrategyFromEnv()
|
||||||
|
return strategy != StorageAlways
|
||||||
|
}
|
||||||
@@ -0,0 +1,308 @@
|
|||||||
|
package hybrid
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"sort"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/lukaszraczylo/claude-mnemonic/internal/graph"
|
||||||
|
"github.com/lukaszraczylo/claude-mnemonic/internal/vector/sqlitevec"
|
||||||
|
"github.com/lukaszraczylo/claude-mnemonic/pkg/models"
|
||||||
|
"github.com/rs/zerolog/log"
|
||||||
|
)
|
||||||
|
|
||||||
|
// GraphConfig configures graph-aware search
|
||||||
|
type GraphConfig struct {
|
||||||
|
Enabled bool
|
||||||
|
MaxHops int // Maximum graph traversal depth (default: 2)
|
||||||
|
BranchFactor int // Number of neighbors to expand per node (default: 5)
|
||||||
|
EdgeWeight float64 // Minimum edge weight to follow (default: 0.3)
|
||||||
|
}
|
||||||
|
|
||||||
|
// DefaultGraphConfig returns sensible defaults for graph search
|
||||||
|
func DefaultGraphConfig() GraphConfig {
|
||||||
|
return GraphConfig{
|
||||||
|
Enabled: true,
|
||||||
|
MaxHops: 2,
|
||||||
|
BranchFactor: 5,
|
||||||
|
EdgeWeight: 0.3,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// GraphSearchClient wraps hybrid.Client with graph-aware search
|
||||||
|
type GraphSearchClient struct {
|
||||||
|
*Client
|
||||||
|
graph *graph.ObservationGraph
|
||||||
|
graphConfig GraphConfig
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewGraphSearchClient creates a graph-enhanced hybrid client
|
||||||
|
func NewGraphSearchClient(baseClient *Client, observationGraph *graph.ObservationGraph, cfg GraphConfig) *GraphSearchClient {
|
||||||
|
return &GraphSearchClient{
|
||||||
|
Client: baseClient,
|
||||||
|
graph: observationGraph,
|
||||||
|
graphConfig: cfg,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Query performs graph-aware vector search with two-level traversal
|
||||||
|
func (g *GraphSearchClient) Query(ctx context.Context, query string, limit int, where map[string]any) ([]sqlitevec.QueryResult, error) {
|
||||||
|
if !g.graphConfig.Enabled || g.graph == nil {
|
||||||
|
// Fall back to standard hybrid search
|
||||||
|
return g.Client.Query(ctx, query, limit, where)
|
||||||
|
}
|
||||||
|
|
||||||
|
startTime := time.Now()
|
||||||
|
|
||||||
|
// 1. Generate query embedding
|
||||||
|
queryEmb, err := g.embedSvc.Embed(query)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("embed query: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// 2. Search hub nodes (stored embeddings)
|
||||||
|
hubResults, err := g.base.Query(ctx, query, limit*2, where)
|
||||||
|
if err != nil {
|
||||||
|
// Fall back to standard search on error
|
||||||
|
log.Warn().Err(err).Msg("Hub search failed, falling back to hybrid search")
|
||||||
|
return g.Client.Query(ctx, query, limit, where)
|
||||||
|
}
|
||||||
|
|
||||||
|
// 3. Track hub access
|
||||||
|
g.trackAccess(hubResults)
|
||||||
|
|
||||||
|
// 4. Expand via graph traversal
|
||||||
|
expandedIDs := g.expandFromHubs(hubResults, limit*4)
|
||||||
|
|
||||||
|
// 5. Filter to non-hubs that need recomputation
|
||||||
|
nonHubIDs := make([]string, 0)
|
||||||
|
for _, id := range expandedIDs {
|
||||||
|
if !g.isHub(id) {
|
||||||
|
nonHubIDs = append(nonHubIDs, id)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 6. Batch recompute non-hub embeddings
|
||||||
|
recomputedResults, err := g.recomputeAndScore(ctx, query, nonHubIDs)
|
||||||
|
if err != nil {
|
||||||
|
log.Warn().Err(err).Msg("Recomputation failed, using hub results only")
|
||||||
|
recomputedResults = nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// 7. Apply graph-based ranking boost
|
||||||
|
allResults := g.mergeAndRankWithGraph(hubResults, recomputedResults, queryEmb)
|
||||||
|
|
||||||
|
// 8. Return top K
|
||||||
|
if len(allResults) > limit {
|
||||||
|
allResults = allResults[:limit]
|
||||||
|
}
|
||||||
|
|
||||||
|
duration := time.Since(startTime)
|
||||||
|
log.Debug().
|
||||||
|
Dur("duration_ms", duration).
|
||||||
|
Int("hubs", len(hubResults)).
|
||||||
|
Int("expanded", len(expandedIDs)).
|
||||||
|
Int("recomputed", len(recomputedResults)).
|
||||||
|
Int("results", len(allResults)).
|
||||||
|
Msg("Graph search completed")
|
||||||
|
|
||||||
|
return allResults, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// expandFromHubs traverses graph from hub nodes to find promising candidates
|
||||||
|
func (g *GraphSearchClient) expandFromHubs(hubResults []sqlitevec.QueryResult, maxCandidates int) []string {
|
||||||
|
if g.graph == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
expanded := make(map[string]float64) // doc_id -> relevance score
|
||||||
|
visited := make(map[int64]bool)
|
||||||
|
|
||||||
|
// Start from top hub results
|
||||||
|
for i, result := range hubResults {
|
||||||
|
if i >= g.graphConfig.BranchFactor*2 {
|
||||||
|
break // Limit starting points
|
||||||
|
}
|
||||||
|
|
||||||
|
// Parse observation ID from doc_id
|
||||||
|
obsID := parseObservationID(result.ID)
|
||||||
|
if obsID == 0 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
// Mark as visited with high relevance (direct match)
|
||||||
|
visited[obsID] = true
|
||||||
|
expanded[result.ID] = result.Similarity
|
||||||
|
|
||||||
|
// Traverse graph from this hub
|
||||||
|
g.traverseGraph(obsID, result.Similarity, 0, expanded, visited)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Convert to sorted list
|
||||||
|
type candidate struct {
|
||||||
|
ID string
|
||||||
|
Relevance float64
|
||||||
|
}
|
||||||
|
|
||||||
|
candidates := make([]candidate, 0, len(expanded))
|
||||||
|
for id, rel := range expanded {
|
||||||
|
candidates = append(candidates, candidate{ID: id, Relevance: rel})
|
||||||
|
}
|
||||||
|
|
||||||
|
// Sort by relevance descending
|
||||||
|
sort.Slice(candidates, func(i, j int) bool {
|
||||||
|
return candidates[i].Relevance > candidates[j].Relevance
|
||||||
|
})
|
||||||
|
|
||||||
|
// Return top candidates
|
||||||
|
if len(candidates) > maxCandidates {
|
||||||
|
candidates = candidates[:maxCandidates]
|
||||||
|
}
|
||||||
|
|
||||||
|
result := make([]string, len(candidates))
|
||||||
|
for i, c := range candidates {
|
||||||
|
result[i] = c.ID
|
||||||
|
}
|
||||||
|
|
||||||
|
return result
|
||||||
|
}
|
||||||
|
|
||||||
|
// traverseGraph performs depth-limited graph traversal
|
||||||
|
func (g *GraphSearchClient) traverseGraph(nodeID int64, baseRelevance float64, depth int, expanded map[string]float64, visited map[int64]bool) {
|
||||||
|
if depth >= g.graphConfig.MaxHops {
|
||||||
|
return // Max depth reached
|
||||||
|
}
|
||||||
|
|
||||||
|
// Get neighbors from graph
|
||||||
|
neighbors, weights, err := g.graph.GetNeighbors(nodeID)
|
||||||
|
if err != nil {
|
||||||
|
return // No neighbors or error
|
||||||
|
}
|
||||||
|
|
||||||
|
// Traverse top neighbors by weight
|
||||||
|
type neighborWeight struct {
|
||||||
|
ID int64
|
||||||
|
Weight float32
|
||||||
|
}
|
||||||
|
|
||||||
|
neighborList := make([]neighborWeight, len(neighbors))
|
||||||
|
for i := range neighbors {
|
||||||
|
neighborList[i] = neighborWeight{
|
||||||
|
ID: neighbors[i],
|
||||||
|
Weight: weights[i],
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Sort by weight descending
|
||||||
|
sort.Slice(neighborList, func(i, j int) bool {
|
||||||
|
return neighborList[i].Weight > neighborList[j].Weight
|
||||||
|
})
|
||||||
|
|
||||||
|
// Expand top branch_factor neighbors
|
||||||
|
expanded_count := 0
|
||||||
|
for _, nw := range neighborList {
|
||||||
|
if expanded_count >= g.graphConfig.BranchFactor {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
|
||||||
|
// Skip if edge weight too low
|
||||||
|
if float64(nw.Weight) < g.graphConfig.EdgeWeight {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
// Skip if already visited
|
||||||
|
if visited[nw.ID] {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
visited[nw.ID] = true
|
||||||
|
|
||||||
|
// Calculate propagated relevance (decays with distance)
|
||||||
|
decay := 0.7 // 30% decay per hop
|
||||||
|
propagatedRelevance := baseRelevance * float64(nw.Weight) * decay
|
||||||
|
|
||||||
|
// Add to expanded set
|
||||||
|
docID := formatObservationDocID(nw.ID)
|
||||||
|
if existing, ok := expanded[docID]; !ok || propagatedRelevance > existing {
|
||||||
|
expanded[docID] = propagatedRelevance
|
||||||
|
}
|
||||||
|
|
||||||
|
// Recursively traverse
|
||||||
|
g.traverseGraph(nw.ID, propagatedRelevance, depth+1, expanded, visited)
|
||||||
|
expanded_count++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// mergeAndRankWithGraph combines hub and recomputed results with graph-based ranking
|
||||||
|
func (g *GraphSearchClient) mergeAndRankWithGraph(hubResults, recomputedResults []sqlitevec.QueryResult, queryEmb []float32) []sqlitevec.QueryResult {
|
||||||
|
// Merge results
|
||||||
|
allResults := append(hubResults, recomputedResults...)
|
||||||
|
|
||||||
|
// Apply graph-based re-ranking
|
||||||
|
if g.graph != nil {
|
||||||
|
for i := range allResults {
|
||||||
|
obsID := parseObservationID(allResults[i].ID)
|
||||||
|
if obsID == 0 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
// Boost score based on node degree (hubs are more important)
|
||||||
|
node, err := g.graph.GetNode(obsID)
|
||||||
|
if err == nil && node.Degree > 0 {
|
||||||
|
// Degree boost: up to 10% increase for high-degree nodes
|
||||||
|
degreeBoost := 1.0 + (0.1 * float64(node.Degree) / 20.0)
|
||||||
|
if degreeBoost > 1.1 {
|
||||||
|
degreeBoost = 1.1
|
||||||
|
}
|
||||||
|
allResults[i].Similarity *= degreeBoost
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Sort by adjusted similarity
|
||||||
|
sortBySimilarity(allResults)
|
||||||
|
|
||||||
|
return allResults
|
||||||
|
}
|
||||||
|
|
||||||
|
// parseObservationID extracts observation ID from doc_id
|
||||||
|
// Format: "obs-{id}-{field}"
|
||||||
|
func parseObservationID(docID string) int64 {
|
||||||
|
var obsID int64
|
||||||
|
// Ignore error - returns 0 on parse failure, which callers handle
|
||||||
|
_, _ = fmt.Sscanf(docID, "obs-%d-", &obsID)
|
||||||
|
return obsID
|
||||||
|
}
|
||||||
|
|
||||||
|
// formatObservationDocID creates a doc_id for an observation
|
||||||
|
func formatObservationDocID(obsID int64) string {
|
||||||
|
return fmt.Sprintf("obs-%d-combined", obsID)
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetGraphStats returns statistics about the observation graph
|
||||||
|
func (g *GraphSearchClient) GetGraphStats() graph.GraphStats {
|
||||||
|
if g.graph == nil {
|
||||||
|
return graph.GraphStats{}
|
||||||
|
}
|
||||||
|
return g.graph.Stats()
|
||||||
|
}
|
||||||
|
|
||||||
|
// RebuildGraph rebuilds the observation graph from current observations
|
||||||
|
// This should be called periodically or when observations change significantly
|
||||||
|
func (g *GraphSearchClient) RebuildGraph(ctx context.Context, observations []*models.Observation) error {
|
||||||
|
log.Info().Int("observations", len(observations)).Msg("Rebuilding observation graph")
|
||||||
|
|
||||||
|
newGraph, err := graph.BuildFromObservations(ctx, observations)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("build graph: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
g.graph = newGraph
|
||||||
|
|
||||||
|
log.Info().
|
||||||
|
Int("nodes", newGraph.Stats().NodeCount).
|
||||||
|
Int("edges", newGraph.Stats().EdgeCount).
|
||||||
|
Msg("Graph rebuilt successfully")
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,16 @@
|
|||||||
|
package hybrid
|
||||||
|
|
||||||
|
import (
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/lukaszraczylo/claude-mnemonic/internal/vector"
|
||||||
|
)
|
||||||
|
|
||||||
|
// TestInterfaceImplementation verifies that hybrid clients implement vector.Client interface
|
||||||
|
func TestInterfaceImplementation(t *testing.T) {
|
||||||
|
// Compile-time check that Client implements vector.Client
|
||||||
|
var _ vector.Client = (*Client)(nil)
|
||||||
|
|
||||||
|
// Compile-time check that GraphSearchClient implements vector.Client
|
||||||
|
var _ vector.Client = (*GraphSearchClient)(nil)
|
||||||
|
}
|
||||||
@@ -0,0 +1,272 @@
|
|||||||
|
package hybrid
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"sync"
|
||||||
|
"sync/atomic"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Metrics tracks performance and usage statistics for hybrid vector storage
|
||||||
|
type Metrics struct {
|
||||||
|
startTime time.Time
|
||||||
|
recentLatencies []time.Duration
|
||||||
|
latenciesMu sync.Mutex
|
||||||
|
totalQueries atomic.Int64
|
||||||
|
hubOnlyQueries atomic.Int64
|
||||||
|
hybridQueries atomic.Int64
|
||||||
|
onDemandQueries atomic.Int64
|
||||||
|
graphQueries atomic.Int64
|
||||||
|
totalLatency atomic.Int64 // Sum in microseconds
|
||||||
|
hubLatency atomic.Int64
|
||||||
|
recomputeLatency atomic.Int64
|
||||||
|
totalDocuments atomic.Int64
|
||||||
|
hubDocuments atomic.Int64
|
||||||
|
storedEmbeddings atomic.Int64
|
||||||
|
recomputedCount atomic.Int64
|
||||||
|
cacheHits atomic.Int64
|
||||||
|
cacheMisses atomic.Int64
|
||||||
|
graphTraversals atomic.Int64
|
||||||
|
avgTraversalDepth atomic.Int64
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewMetrics creates a new metrics tracker
|
||||||
|
func NewMetrics() *Metrics {
|
||||||
|
return &Metrics{
|
||||||
|
recentLatencies: make([]time.Duration, 0, 1000),
|
||||||
|
startTime: time.Now(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// RecordQuery records a query execution
|
||||||
|
func (m *Metrics) RecordQuery(queryType string, latency time.Duration, recomputed int) {
|
||||||
|
m.totalQueries.Add(1)
|
||||||
|
m.totalLatency.Add(latency.Microseconds())
|
||||||
|
|
||||||
|
switch queryType {
|
||||||
|
case "hub_only":
|
||||||
|
m.hubOnlyQueries.Add(1)
|
||||||
|
case "hybrid":
|
||||||
|
m.hybridQueries.Add(1)
|
||||||
|
case "on_demand":
|
||||||
|
m.onDemandQueries.Add(1)
|
||||||
|
case "graph":
|
||||||
|
m.graphQueries.Add(1)
|
||||||
|
}
|
||||||
|
|
||||||
|
if recomputed > 0 {
|
||||||
|
m.recomputedCount.Add(int64(recomputed))
|
||||||
|
}
|
||||||
|
|
||||||
|
// Track recent latencies
|
||||||
|
m.latenciesMu.Lock()
|
||||||
|
m.recentLatencies = append(m.recentLatencies, latency)
|
||||||
|
if len(m.recentLatencies) > 1000 {
|
||||||
|
m.recentLatencies = m.recentLatencies[len(m.recentLatencies)-1000:]
|
||||||
|
}
|
||||||
|
m.latenciesMu.Unlock()
|
||||||
|
}
|
||||||
|
|
||||||
|
// RecordHubLatency records time spent in hub search
|
||||||
|
func (m *Metrics) RecordHubLatency(latency time.Duration) {
|
||||||
|
m.hubLatency.Add(latency.Microseconds())
|
||||||
|
}
|
||||||
|
|
||||||
|
// RecordRecomputeLatency records time spent recomputing embeddings
|
||||||
|
func (m *Metrics) RecordRecomputeLatency(latency time.Duration) {
|
||||||
|
m.recomputeLatency.Add(latency.Microseconds())
|
||||||
|
}
|
||||||
|
|
||||||
|
// RecordCacheHit records a content cache hit
|
||||||
|
func (m *Metrics) RecordCacheHit() {
|
||||||
|
m.cacheHits.Add(1)
|
||||||
|
}
|
||||||
|
|
||||||
|
// RecordCacheMiss records a content cache miss
|
||||||
|
func (m *Metrics) RecordCacheMiss() {
|
||||||
|
m.cacheMisses.Add(1)
|
||||||
|
}
|
||||||
|
|
||||||
|
// RecordGraphTraversal records a graph traversal operation
|
||||||
|
func (m *Metrics) RecordGraphTraversal(depth int) {
|
||||||
|
m.graphTraversals.Add(1)
|
||||||
|
m.avgTraversalDepth.Add(int64(depth))
|
||||||
|
}
|
||||||
|
|
||||||
|
// UpdateStorageStats updates current storage statistics
|
||||||
|
func (m *Metrics) UpdateStorageStats(total, hubs, stored int) {
|
||||||
|
m.totalDocuments.Store(int64(total))
|
||||||
|
m.hubDocuments.Store(int64(hubs))
|
||||||
|
m.storedEmbeddings.Store(int64(stored))
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetSnapshot returns current metrics snapshot
|
||||||
|
func (m *Metrics) GetSnapshot() MetricsSnapshot {
|
||||||
|
m.latenciesMu.Lock()
|
||||||
|
defer m.latenciesMu.Unlock()
|
||||||
|
|
||||||
|
totalQueries := m.totalQueries.Load()
|
||||||
|
|
||||||
|
snapshot := MetricsSnapshot{
|
||||||
|
// Query counts
|
||||||
|
TotalQueries: totalQueries,
|
||||||
|
HubOnlyQueries: m.hubOnlyQueries.Load(),
|
||||||
|
HybridQueries: m.hybridQueries.Load(),
|
||||||
|
OnDemandQueries: m.onDemandQueries.Load(),
|
||||||
|
GraphQueries: m.graphQueries.Load(),
|
||||||
|
|
||||||
|
// Storage
|
||||||
|
TotalDocuments: int(m.totalDocuments.Load()),
|
||||||
|
HubDocuments: int(m.hubDocuments.Load()),
|
||||||
|
StoredEmbeddings: int(m.storedEmbeddings.Load()),
|
||||||
|
RecomputedTotal: m.recomputedCount.Load(),
|
||||||
|
|
||||||
|
// Cache
|
||||||
|
CacheHits: m.cacheHits.Load(),
|
||||||
|
CacheMisses: m.cacheMisses.Load(),
|
||||||
|
|
||||||
|
// Graph
|
||||||
|
GraphTraversals: m.graphTraversals.Load(),
|
||||||
|
|
||||||
|
// Runtime
|
||||||
|
Uptime: time.Since(m.startTime),
|
||||||
|
}
|
||||||
|
|
||||||
|
// Calculate latencies
|
||||||
|
if totalQueries > 0 {
|
||||||
|
snapshot.AvgLatency = time.Duration(m.totalLatency.Load()/totalQueries) * time.Microsecond
|
||||||
|
snapshot.AvgHubLatency = time.Duration(m.hubLatency.Load()/totalQueries) * time.Microsecond
|
||||||
|
}
|
||||||
|
|
||||||
|
if m.recomputedCount.Load() > 0 {
|
||||||
|
snapshot.AvgRecomputeLatency = time.Duration(m.recomputeLatency.Load()/m.recomputedCount.Load()) * time.Microsecond
|
||||||
|
}
|
||||||
|
|
||||||
|
// Calculate percentiles
|
||||||
|
if len(m.recentLatencies) > 0 {
|
||||||
|
sorted := make([]time.Duration, len(m.recentLatencies))
|
||||||
|
copy(sorted, m.recentLatencies)
|
||||||
|
sortDurations(sorted)
|
||||||
|
|
||||||
|
snapshot.P50Latency = percentile(sorted, 0.50)
|
||||||
|
snapshot.P95Latency = percentile(sorted, 0.95)
|
||||||
|
snapshot.P99Latency = percentile(sorted, 0.99)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Calculate cache hit rate
|
||||||
|
totalCacheOps := snapshot.CacheHits + snapshot.CacheMisses
|
||||||
|
if totalCacheOps > 0 {
|
||||||
|
snapshot.CacheHitRate = float64(snapshot.CacheHits) / float64(totalCacheOps)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Calculate storage savings
|
||||||
|
if snapshot.TotalDocuments > 0 {
|
||||||
|
embeddingSize := 384 * 4 // 384 dims × 4 bytes
|
||||||
|
fullStorage := snapshot.TotalDocuments * embeddingSize
|
||||||
|
actualStorage := snapshot.StoredEmbeddings * embeddingSize
|
||||||
|
|
||||||
|
if fullStorage > 0 {
|
||||||
|
snapshot.StorageSavingsPercent = (1.0 - float64(actualStorage)/float64(fullStorage)) * 100
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Calculate avg traversal depth
|
||||||
|
if snapshot.GraphTraversals > 0 {
|
||||||
|
snapshot.AvgTraversalDepth = float64(m.avgTraversalDepth.Load()) / float64(snapshot.GraphTraversals)
|
||||||
|
}
|
||||||
|
|
||||||
|
return snapshot
|
||||||
|
}
|
||||||
|
|
||||||
|
// MetricsSnapshot represents a point-in-time metrics snapshot
|
||||||
|
type MetricsSnapshot struct {
|
||||||
|
// Query metrics
|
||||||
|
TotalQueries int64
|
||||||
|
HubOnlyQueries int64
|
||||||
|
HybridQueries int64
|
||||||
|
OnDemandQueries int64
|
||||||
|
GraphQueries int64
|
||||||
|
|
||||||
|
// Latency metrics
|
||||||
|
AvgLatency time.Duration
|
||||||
|
P50Latency time.Duration
|
||||||
|
P95Latency time.Duration
|
||||||
|
P99Latency time.Duration
|
||||||
|
AvgHubLatency time.Duration
|
||||||
|
AvgRecomputeLatency time.Duration
|
||||||
|
|
||||||
|
// Storage metrics
|
||||||
|
TotalDocuments int
|
||||||
|
HubDocuments int
|
||||||
|
StoredEmbeddings int
|
||||||
|
StorageSavingsPercent float64
|
||||||
|
RecomputedTotal int64
|
||||||
|
|
||||||
|
// Cache metrics
|
||||||
|
CacheHits int64
|
||||||
|
CacheMisses int64
|
||||||
|
CacheHitRate float64
|
||||||
|
|
||||||
|
// Graph metrics
|
||||||
|
GraphTraversals int64
|
||||||
|
AvgTraversalDepth float64
|
||||||
|
|
||||||
|
// Runtime
|
||||||
|
Uptime time.Duration
|
||||||
|
}
|
||||||
|
|
||||||
|
// sortDurations sorts a slice of durations in ascending order
|
||||||
|
func sortDurations(durations []time.Duration) {
|
||||||
|
n := len(durations)
|
||||||
|
for i := 0; i < n-1; i++ {
|
||||||
|
for j := 0; j < n-i-1; j++ {
|
||||||
|
if durations[j] > durations[j+1] {
|
||||||
|
durations[j], durations[j+1] = durations[j+1], durations[j]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// percentile calculates the Nth percentile from a sorted slice
|
||||||
|
func percentile(sorted []time.Duration, p float64) time.Duration {
|
||||||
|
if len(sorted) == 0 {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
idx := int(float64(len(sorted)) * p)
|
||||||
|
if idx >= len(sorted) {
|
||||||
|
idx = len(sorted) - 1
|
||||||
|
}
|
||||||
|
|
||||||
|
return sorted[idx]
|
||||||
|
}
|
||||||
|
|
||||||
|
// String returns a human-readable representation of metrics
|
||||||
|
func (s MetricsSnapshot) String() string {
|
||||||
|
return fmt.Sprintf(`Hybrid Vector Storage Metrics:
|
||||||
|
Queries:
|
||||||
|
Total: %d (Hub: %d, Hybrid: %d, OnDemand: %d, Graph: %d)
|
||||||
|
Avg Latency: %v (p50: %v, p95: %v, p99: %v)
|
||||||
|
Hub Latency: %v, Recompute Latency: %v
|
||||||
|
Storage:
|
||||||
|
Documents: %d (Hubs: %d, %.1f%%)
|
||||||
|
Stored Embeddings: %d
|
||||||
|
Savings: %.1f%%
|
||||||
|
Total Recomputed: %d
|
||||||
|
Cache:
|
||||||
|
Hits: %d, Misses: %d (Hit Rate: %.1f%%)
|
||||||
|
Graph:
|
||||||
|
Traversals: %d (Avg Depth: %.2f)
|
||||||
|
Runtime: %v`,
|
||||||
|
s.TotalQueries, s.HubOnlyQueries, s.HybridQueries, s.OnDemandQueries, s.GraphQueries,
|
||||||
|
s.AvgLatency, s.P50Latency, s.P95Latency, s.P99Latency,
|
||||||
|
s.AvgHubLatency, s.AvgRecomputeLatency,
|
||||||
|
s.TotalDocuments, s.HubDocuments, float64(s.HubDocuments)/float64(s.TotalDocuments)*100,
|
||||||
|
s.StoredEmbeddings,
|
||||||
|
s.StorageSavingsPercent,
|
||||||
|
s.RecomputedTotal,
|
||||||
|
s.CacheHits, s.CacheMisses, s.CacheHitRate*100,
|
||||||
|
s.GraphTraversals, s.AvgTraversalDepth,
|
||||||
|
s.Uptime,
|
||||||
|
)
|
||||||
|
}
|
||||||
@@ -0,0 +1,42 @@
|
|||||||
|
// Package vector provides common interfaces for vector storage implementations
|
||||||
|
package vector
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
|
||||||
|
"github.com/lukaszraczylo/claude-mnemonic/internal/vector/sqlitevec"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Client defines the interface for vector storage operations.
|
||||||
|
// Both sqlitevec.Client and hybrid.Client implement this interface.
|
||||||
|
type Client interface {
|
||||||
|
// AddDocuments adds documents with their embeddings to the vector store
|
||||||
|
AddDocuments(ctx context.Context, docs []sqlitevec.Document) error
|
||||||
|
|
||||||
|
// DeleteDocuments removes documents by their IDs
|
||||||
|
DeleteDocuments(ctx context.Context, ids []string) error
|
||||||
|
|
||||||
|
// Query performs a vector similarity search
|
||||||
|
Query(ctx context.Context, query string, limit int, where map[string]any) ([]sqlitevec.QueryResult, error)
|
||||||
|
|
||||||
|
// IsConnected checks if the vector store is available
|
||||||
|
IsConnected() bool
|
||||||
|
|
||||||
|
// Close releases resources
|
||||||
|
Close() error
|
||||||
|
|
||||||
|
// Count returns the total number of vectors in the store
|
||||||
|
Count(ctx context.Context) (int64, error)
|
||||||
|
|
||||||
|
// ModelVersion returns the current embedding model version
|
||||||
|
ModelVersion() string
|
||||||
|
|
||||||
|
// NeedsRebuild checks if vectors need to be rebuilt due to model version change
|
||||||
|
NeedsRebuild(ctx context.Context) (bool, string)
|
||||||
|
|
||||||
|
// GetStaleVectors returns doc_ids of vectors with mismatched or null model versions
|
||||||
|
GetStaleVectors(ctx context.Context) ([]sqlitevec.StaleVectorInfo, error)
|
||||||
|
|
||||||
|
// DeleteVectorsByDocIDs removes vectors by their doc_ids
|
||||||
|
DeleteVectorsByDocIDs(ctx context.Context, docIDs []string) error
|
||||||
|
}
|
||||||
@@ -87,9 +87,9 @@ func (c *Client) AddDocuments(ctx context.Context, docs []Document) error {
|
|||||||
|
|
||||||
for i, doc := range docs {
|
for i, doc := range docs {
|
||||||
// Serialize embedding to blob format
|
// Serialize embedding to blob format
|
||||||
embBlob, err := sqlite_vec.SerializeFloat32(embeddings[i])
|
embBlob, serErr := sqlite_vec.SerializeFloat32(embeddings[i])
|
||||||
if err != nil {
|
if serErr != nil {
|
||||||
return fmt.Errorf("serialize embedding for %s: %w", doc.ID, err)
|
return fmt.Errorf("serialize embedding for %s: %w", doc.ID, serErr)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Extract metadata
|
// Extract metadata
|
||||||
@@ -212,8 +212,8 @@ func (c *Client) Query(ctx context.Context, query string, limit int, where map[s
|
|||||||
var sqliteID int64
|
var sqliteID int64
|
||||||
var docType, fieldType, project, scope sql.NullString
|
var docType, fieldType, project, scope sql.NullString
|
||||||
|
|
||||||
if err := rows.Scan(&r.ID, &r.Distance, &sqliteID, &docType, &fieldType, &project, &scope); err != nil {
|
if scanErr := rows.Scan(&r.ID, &r.Distance, &sqliteID, &docType, &fieldType, &project, &scope); scanErr != nil {
|
||||||
return nil, fmt.Errorf("scan row: %w", err)
|
return nil, fmt.Errorf("scan row: %w", scanErr)
|
||||||
}
|
}
|
||||||
|
|
||||||
r.Similarity = DistanceToSimilarity(r.Distance)
|
r.Similarity = DistanceToSimilarity(r.Distance)
|
||||||
@@ -319,11 +319,11 @@ func (c *Client) NeedsRebuild(ctx context.Context) (bool, string) {
|
|||||||
// StaleVectorInfo contains information about a vector that needs rebuilding.
|
// StaleVectorInfo contains information about a vector that needs rebuilding.
|
||||||
type StaleVectorInfo struct {
|
type StaleVectorInfo struct {
|
||||||
DocID string
|
DocID string
|
||||||
SQLiteID int64
|
|
||||||
DocType string
|
DocType string
|
||||||
FieldType string
|
FieldType string
|
||||||
Project string
|
Project string
|
||||||
Scope string
|
Scope string
|
||||||
|
SQLiteID int64
|
||||||
}
|
}
|
||||||
|
|
||||||
// GetStaleVectors returns doc_ids of vectors with mismatched or null model versions.
|
// GetStaleVectors returns doc_ids of vectors with mismatched or null model versions.
|
||||||
@@ -352,8 +352,8 @@ func (c *Client) GetStaleVectors(ctx context.Context) ([]StaleVectorInfo, error)
|
|||||||
var sqliteID sql.NullInt64
|
var sqliteID sql.NullInt64
|
||||||
var docType, fieldType, project, scope sql.NullString
|
var docType, fieldType, project, scope sql.NullString
|
||||||
|
|
||||||
if err := rows.Scan(&info.DocID, &sqliteID, &docType, &fieldType, &project, &scope); err != nil {
|
if scanErr := rows.Scan(&info.DocID, &sqliteID, &docType, &fieldType, &project, &scope); scanErr != nil {
|
||||||
return nil, fmt.Errorf("scan row: %w", err)
|
return nil, fmt.Errorf("scan row: %w", scanErr)
|
||||||
}
|
}
|
||||||
|
|
||||||
info.SQLiteID = sqliteID.Int64
|
info.SQLiteID = sqliteID.Int64
|
||||||
|
|||||||
@@ -8,21 +8,22 @@ const (
|
|||||||
DocTypeObservation DocType = "observation"
|
DocTypeObservation DocType = "observation"
|
||||||
DocTypeSessionSummary DocType = "session_summary"
|
DocTypeSessionSummary DocType = "session_summary"
|
||||||
DocTypeUserPrompt DocType = "user_prompt"
|
DocTypeUserPrompt DocType = "user_prompt"
|
||||||
|
DocTypeCodeChunk DocType = "code_chunk"
|
||||||
)
|
)
|
||||||
|
|
||||||
// Document represents a document to store with vector embedding.
|
// Document represents a document to store with vector embedding.
|
||||||
type Document struct {
|
type Document struct {
|
||||||
|
Metadata map[string]any
|
||||||
ID string
|
ID string
|
||||||
Content string
|
Content string
|
||||||
Metadata map[string]any
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// QueryResult represents a search result from vector search.
|
// QueryResult represents a search result from vector search.
|
||||||
type QueryResult struct {
|
type QueryResult struct {
|
||||||
|
Metadata map[string]any
|
||||||
ID string
|
ID string
|
||||||
Distance float64
|
Distance float64
|
||||||
Similarity float64 // 1.0 = identical, 0.0 = opposite (derived from distance)
|
Similarity float64
|
||||||
Metadata map[string]any
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// DistanceToSimilarity converts sqlite-vec cosine distance to similarity score.
|
// DistanceToSimilarity converts sqlite-vec cosine distance to similarity score.
|
||||||
|
|||||||
@@ -42,10 +42,10 @@ func TestQueryResult_Fields(t *testing.T) {
|
|||||||
|
|
||||||
func TestBuildWhereFilter(t *testing.T) {
|
func TestBuildWhereFilter(t *testing.T) {
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
|
expected map[string]interface{}
|
||||||
name string
|
name string
|
||||||
docType DocType
|
docType DocType
|
||||||
project string
|
project string
|
||||||
expected map[string]interface{}
|
|
||||||
}{
|
}{
|
||||||
{
|
{
|
||||||
name: "empty_filters",
|
name: "empty_filters",
|
||||||
@@ -474,9 +474,9 @@ func TestCopyMetadataMulti(t *testing.T) {
|
|||||||
func TestJoinStrings(t *testing.T) {
|
func TestJoinStrings(t *testing.T) {
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
name string
|
name string
|
||||||
strs []string
|
|
||||||
sep string
|
sep string
|
||||||
expected string
|
expected string
|
||||||
|
strs []string
|
||||||
}{
|
}{
|
||||||
{
|
{
|
||||||
name: "empty_slice",
|
name: "empty_slice",
|
||||||
@@ -522,8 +522,8 @@ func TestTruncateString(t *testing.T) {
|
|||||||
tests := []struct {
|
tests := []struct {
|
||||||
name string
|
name string
|
||||||
input string
|
input string
|
||||||
maxLen int
|
|
||||||
expected string
|
expected string
|
||||||
|
maxLen int
|
||||||
}{
|
}{
|
||||||
{
|
{
|
||||||
name: "shorter_than_max",
|
name: "shorter_than_max",
|
||||||
|
|||||||
@@ -5,13 +5,15 @@ import (
|
|||||||
"context"
|
"context"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
|
||||||
|
"github.com/lukaszraczylo/claude-mnemonic/internal/chunking"
|
||||||
"github.com/lukaszraczylo/claude-mnemonic/pkg/models"
|
"github.com/lukaszraczylo/claude-mnemonic/pkg/models"
|
||||||
"github.com/rs/zerolog/log"
|
"github.com/rs/zerolog/log"
|
||||||
)
|
)
|
||||||
|
|
||||||
// Sync provides synchronization between SQLite data and vector embeddings.
|
// Sync provides synchronization between SQLite data and vector embeddings.
|
||||||
type Sync struct {
|
type Sync struct {
|
||||||
client *Client
|
client *Client
|
||||||
|
chunkingManager *chunking.Manager
|
||||||
}
|
}
|
||||||
|
|
||||||
// NewSync creates a new sync service.
|
// NewSync creates a new sync service.
|
||||||
@@ -19,9 +21,23 @@ func NewSync(client *Client) *Sync {
|
|||||||
return &Sync{client: client}
|
return &Sync{client: client}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// SetChunkingManager sets the code chunking manager (optional).
|
||||||
|
// If set, observations will include code chunks from tracked files.
|
||||||
|
func (s *Sync) SetChunkingManager(manager *chunking.Manager) {
|
||||||
|
s.chunkingManager = manager
|
||||||
|
}
|
||||||
|
|
||||||
// SyncObservation syncs a single observation to the vector store.
|
// SyncObservation syncs a single observation to the vector store.
|
||||||
|
// If a chunking manager is configured, also chunks tracked code files.
|
||||||
func (s *Sync) SyncObservation(ctx context.Context, obs *models.Observation) error {
|
func (s *Sync) SyncObservation(ctx context.Context, obs *models.Observation) error {
|
||||||
docs := s.formatObservationDocs(obs)
|
docs := s.formatObservationDocs(obs)
|
||||||
|
|
||||||
|
// Add code chunks from tracked files if chunking manager is available
|
||||||
|
if s.chunkingManager != nil {
|
||||||
|
codeChunkDocs := s.formatCodeChunkDocs(ctx, obs)
|
||||||
|
docs = append(docs, codeChunkDocs...)
|
||||||
|
}
|
||||||
|
|
||||||
if len(docs) == 0 {
|
if len(docs) == 0 {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
@@ -99,6 +115,98 @@ func (s *Sync) formatObservationDocs(obs *models.Observation) []Document {
|
|||||||
return docs
|
return docs
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// formatCodeChunkDocs formats code chunks from tracked files into vector documents.
|
||||||
|
// Uses AST-aware chunking to extract semantic code units (functions, classes, methods).
|
||||||
|
func (s *Sync) formatCodeChunkDocs(ctx context.Context, obs *models.Observation) []Document {
|
||||||
|
if s.chunkingManager == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Determine scope for metadata
|
||||||
|
scope := string(obs.Scope)
|
||||||
|
if scope == "" {
|
||||||
|
scope = "project"
|
||||||
|
}
|
||||||
|
|
||||||
|
// Collect all tracked files (read + modified)
|
||||||
|
allFiles := make([]string, 0, len(obs.FilesRead)+len(obs.FilesModified))
|
||||||
|
allFiles = append(allFiles, obs.FilesRead...)
|
||||||
|
allFiles = append(allFiles, obs.FilesModified...)
|
||||||
|
|
||||||
|
// Filter to only files supported by chunking manager
|
||||||
|
var supportedFiles []string
|
||||||
|
for _, file := range allFiles {
|
||||||
|
if s.chunkingManager.SupportsFile(file) {
|
||||||
|
supportedFiles = append(supportedFiles, file)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(supportedFiles) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Chunk all supported files
|
||||||
|
results, errs := s.chunkingManager.ChunkFiles(ctx, supportedFiles)
|
||||||
|
if len(errs) > 0 {
|
||||||
|
// Log errors but don't fail the entire sync
|
||||||
|
for _, err := range errs {
|
||||||
|
log.Warn().Err(err).Msg("Failed to chunk file")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Convert chunks to vector documents
|
||||||
|
docs := make([]Document, 0)
|
||||||
|
chunkIndex := 0
|
||||||
|
|
||||||
|
for filePath, chunks := range results {
|
||||||
|
for _, chunk := range chunks {
|
||||||
|
doc := Document{
|
||||||
|
ID: fmt.Sprintf("obs_%d_chunk_%d", obs.ID, chunkIndex),
|
||||||
|
Content: chunk.SearchableContent(),
|
||||||
|
Metadata: map[string]any{
|
||||||
|
"sqlite_id": obs.ID,
|
||||||
|
"doc_type": string(DocTypeCodeChunk),
|
||||||
|
"field_type": "code_chunk",
|
||||||
|
"sdk_session_id": obs.SDKSessionID,
|
||||||
|
"project": obs.Project,
|
||||||
|
"scope": scope,
|
||||||
|
"created_at_epoch": obs.CreatedAtEpoch,
|
||||||
|
// Code chunk specific metadata
|
||||||
|
"file_path": filePath,
|
||||||
|
"language": string(chunk.Language),
|
||||||
|
"chunk_type": string(chunk.Type),
|
||||||
|
"symbol_name": chunk.Name,
|
||||||
|
"start_line": chunk.StartLine,
|
||||||
|
"end_line": chunk.EndLine,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
// Add parent name if this is a method
|
||||||
|
if chunk.ParentName != "" {
|
||||||
|
doc.Metadata["parent_name"] = chunk.ParentName
|
||||||
|
}
|
||||||
|
|
||||||
|
// Add signature if available
|
||||||
|
if chunk.Signature != "" {
|
||||||
|
doc.Metadata["signature"] = chunk.Signature
|
||||||
|
}
|
||||||
|
|
||||||
|
docs = append(docs, doc)
|
||||||
|
chunkIndex++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(docs) > 0 {
|
||||||
|
log.Debug().
|
||||||
|
Int64("observationId", obs.ID).
|
||||||
|
Int("codeChunks", len(docs)).
|
||||||
|
Int("files", len(results)).
|
||||||
|
Msg("Generated code chunk documents")
|
||||||
|
}
|
||||||
|
|
||||||
|
return docs
|
||||||
|
}
|
||||||
|
|
||||||
// SyncSummary syncs a single session summary to the vector store.
|
// SyncSummary syncs a single session summary to the vector store.
|
||||||
func (s *Sync) SyncSummary(ctx context.Context, summary *models.SessionSummary) error {
|
func (s *Sync) SyncSummary(ctx context.Context, summary *models.SessionSummary) error {
|
||||||
docs := s.formatSummaryDocs(summary)
|
docs := s.formatSummaryDocs(summary)
|
||||||
@@ -191,21 +299,27 @@ func (s *Sync) SyncUserPrompt(ctx context.Context, prompt *models.UserPromptWith
|
|||||||
}
|
}
|
||||||
|
|
||||||
// DeleteObservations removes observation documents from the vector store.
|
// DeleteObservations removes observation documents from the vector store.
|
||||||
|
// Includes both observation fields (narrative, facts) and code chunks.
|
||||||
func (s *Sync) DeleteObservations(ctx context.Context, observationIDs []int64) error {
|
func (s *Sync) DeleteObservations(ctx context.Context, observationIDs []int64) error {
|
||||||
if len(observationIDs) == 0 {
|
if len(observationIDs) == 0 {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// Generate all possible document IDs for these observations
|
// Generate all possible document IDs for these observations
|
||||||
// Pattern: obs_{id}_narrative, obs_{id}_fact_{0..n}
|
// Pattern: obs_{id}_narrative, obs_{id}_fact_{0..n}, obs_{id}_chunk_{0..n}
|
||||||
const maxFactsPerObs = 20
|
const maxFactsPerObs = 20
|
||||||
ids := make([]string, 0, len(observationIDs)*(maxFactsPerObs+1))
|
const maxChunksPerObs = 100 // Reasonable upper bound for code chunks
|
||||||
|
ids := make([]string, 0, len(observationIDs)*(maxFactsPerObs+maxChunksPerObs+1))
|
||||||
|
|
||||||
for _, obsID := range observationIDs {
|
for _, obsID := range observationIDs {
|
||||||
ids = append(ids, fmt.Sprintf("obs_%d_narrative", obsID))
|
ids = append(ids, fmt.Sprintf("obs_%d_narrative", obsID))
|
||||||
for i := 0; i < maxFactsPerObs; i++ {
|
for i := 0; i < maxFactsPerObs; i++ {
|
||||||
ids = append(ids, fmt.Sprintf("obs_%d_fact_%d", obsID, i))
|
ids = append(ids, fmt.Sprintf("obs_%d_fact_%d", obsID, i))
|
||||||
}
|
}
|
||||||
|
// Include code chunk IDs
|
||||||
|
for i := 0; i < maxChunksPerObs; i++ {
|
||||||
|
ids = append(ids, fmt.Sprintf("obs_%d_chunk_%d", obsID, i))
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := s.client.DeleteDocuments(ctx, ids); err != nil {
|
if err := s.client.DeleteDocuments(ctx, ids); err != nil {
|
||||||
|
|||||||
@@ -16,15 +16,15 @@ import (
|
|||||||
// Watcher monitors a file or directory for deletion and calls onDelete when removed.
|
// Watcher monitors a file or directory for deletion and calls onDelete when removed.
|
||||||
// It watches the parent directory since fsnotify cannot watch non-existent files.
|
// It watches the parent directory since fsnotify cannot watch non-existent files.
|
||||||
type Watcher struct {
|
type Watcher struct {
|
||||||
targetPath string // The file/directory to watch for deletion
|
|
||||||
parentPath string // Parent directory (what we actually watch)
|
|
||||||
onDelete func() // Callback when target is deleted
|
|
||||||
watcher *fsnotify.Watcher
|
|
||||||
ctx context.Context
|
ctx context.Context
|
||||||
|
onDelete func()
|
||||||
|
watcher *fsnotify.Watcher
|
||||||
cancel context.CancelFunc
|
cancel context.CancelFunc
|
||||||
|
targetPath string
|
||||||
|
parentPath string
|
||||||
|
debounce time.Duration
|
||||||
mu sync.Mutex
|
mu sync.Mutex
|
||||||
running bool
|
running bool
|
||||||
debounce time.Duration
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// New creates a new Watcher for the given target path.
|
// New creates a new Watcher for the given target path.
|
||||||
|
|||||||
@@ -158,10 +158,10 @@ type SessionInitRequest struct {
|
|||||||
|
|
||||||
// SessionInitResponse is the response for session initialization.
|
// SessionInitResponse is the response for session initialization.
|
||||||
type SessionInitResponse struct {
|
type SessionInitResponse struct {
|
||||||
|
Reason string `json:"reason,omitempty"`
|
||||||
SessionDBID int64 `json:"sessionDbId"`
|
SessionDBID int64 `json:"sessionDbId"`
|
||||||
PromptNumber int `json:"promptNumber"`
|
PromptNumber int `json:"promptNumber"`
|
||||||
Skipped bool `json:"skipped,omitempty"`
|
Skipped bool `json:"skipped,omitempty"`
|
||||||
Reason string `json:"reason,omitempty"`
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// DuplicatePromptWindowSeconds is the time window for detecting duplicate prompt submissions.
|
// DuplicatePromptWindowSeconds is the time window for detecting duplicate prompt submissions.
|
||||||
@@ -296,8 +296,8 @@ func (s *Service) handleSessionStart(w http.ResponseWriter, r *http.Request) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
var req SessionStartRequest
|
var req SessionStartRequest
|
||||||
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
if decodeErr := json.NewDecoder(r.Body).Decode(&req); decodeErr != nil {
|
||||||
http.Error(w, err.Error(), http.StatusBadRequest)
|
http.Error(w, decodeErr.Error(), http.StatusBadRequest)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1312,3 +1312,85 @@ func (s *Service) handleRestart(w http.ResponseWriter, r *http.Request) {
|
|||||||
}
|
}
|
||||||
}()
|
}()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// handleGetGraphStats returns observation graph statistics.
|
||||||
|
func (s *Service) handleGetGraphStats(w http.ResponseWriter, r *http.Request) {
|
||||||
|
if s.graphSearchClient == nil {
|
||||||
|
writeJSON(w, map[string]interface{}{
|
||||||
|
"enabled": false,
|
||||||
|
"message": "Graph search not enabled",
|
||||||
|
})
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
stats := s.graphSearchClient.GetGraphStats()
|
||||||
|
|
||||||
|
response := map[string]interface{}{
|
||||||
|
"enabled": s.config.GraphEnabled,
|
||||||
|
"nodeCount": stats.NodeCount,
|
||||||
|
"edgeCount": stats.EdgeCount,
|
||||||
|
"avgDegree": stats.AvgDegree,
|
||||||
|
"maxDegree": stats.MaxDegree,
|
||||||
|
"minDegree": stats.MinDegree,
|
||||||
|
"medianDegree": stats.MedianDegree,
|
||||||
|
"edgeTypes": stats.EdgeTypes,
|
||||||
|
"config": map[string]interface{}{
|
||||||
|
"maxHops": s.config.GraphMaxHops,
|
||||||
|
"branchFactor": s.config.GraphBranchFactor,
|
||||||
|
"edgeWeight": s.config.GraphEdgeWeight,
|
||||||
|
"rebuildIntervalMin": s.config.GraphRebuildIntervalMin,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
writeJSON(w, response)
|
||||||
|
}
|
||||||
|
|
||||||
|
// handleGetVectorMetrics returns hybrid vector storage metrics.
|
||||||
|
func (s *Service) handleGetVectorMetrics(w http.ResponseWriter, r *http.Request) {
|
||||||
|
if s.hybridMetrics == nil {
|
||||||
|
writeJSON(w, map[string]interface{}{
|
||||||
|
"enabled": false,
|
||||||
|
"message": "Vector metrics not available",
|
||||||
|
})
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
snapshot := s.hybridMetrics.GetSnapshot()
|
||||||
|
|
||||||
|
response := map[string]interface{}{
|
||||||
|
"queries": map[string]interface{}{
|
||||||
|
"total": snapshot.TotalQueries,
|
||||||
|
"hubOnly": snapshot.HubOnlyQueries,
|
||||||
|
"hybrid": snapshot.HybridQueries,
|
||||||
|
"onDemand": snapshot.OnDemandQueries,
|
||||||
|
"graph": snapshot.GraphQueries,
|
||||||
|
},
|
||||||
|
"latency": map[string]interface{}{
|
||||||
|
"avg": snapshot.AvgLatency.String(),
|
||||||
|
"p50": snapshot.P50Latency.String(),
|
||||||
|
"p95": snapshot.P95Latency.String(),
|
||||||
|
"p99": snapshot.P99Latency.String(),
|
||||||
|
"avgHub": snapshot.AvgHubLatency.String(),
|
||||||
|
"avgRecompute": snapshot.AvgRecomputeLatency.String(),
|
||||||
|
},
|
||||||
|
"storage": map[string]interface{}{
|
||||||
|
"totalDocuments": snapshot.TotalDocuments,
|
||||||
|
"hubDocuments": snapshot.HubDocuments,
|
||||||
|
"storedEmbeddings": snapshot.StoredEmbeddings,
|
||||||
|
"savingsPercent": snapshot.StorageSavingsPercent,
|
||||||
|
"recomputedTotal": snapshot.RecomputedTotal,
|
||||||
|
},
|
||||||
|
"cache": map[string]interface{}{
|
||||||
|
"hits": snapshot.CacheHits,
|
||||||
|
"misses": snapshot.CacheMisses,
|
||||||
|
"hitRate": snapshot.CacheHitRate,
|
||||||
|
},
|
||||||
|
"graph": map[string]interface{}{
|
||||||
|
"traversals": snapshot.GraphTraversals,
|
||||||
|
"avgDepth": snapshot.AvgTraversalDepth,
|
||||||
|
},
|
||||||
|
"uptime": snapshot.Uptime.String(),
|
||||||
|
}
|
||||||
|
|
||||||
|
writeJSON(w, response)
|
||||||
|
}
|
||||||
|
|||||||
@@ -46,7 +46,7 @@ func (s *Service) handleGetRelationGraph(w http.ResponseWriter, r *http.Request)
|
|||||||
// Get depth parameter (default 2)
|
// Get depth parameter (default 2)
|
||||||
depth := 2
|
depth := 2
|
||||||
if depthStr := r.URL.Query().Get("depth"); depthStr != "" {
|
if depthStr := r.URL.Query().Get("depth"); depthStr != "" {
|
||||||
if d, err := strconv.Atoi(depthStr); err == nil && d > 0 && d <= 5 {
|
if d, atoiErr := strconv.Atoi(depthStr); atoiErr == nil && d > 0 && d <= 5 {
|
||||||
depth = d
|
depth = d
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -72,7 +72,7 @@ func (s *Service) handleGetRelatedObservations(w http.ResponseWriter, r *http.Re
|
|||||||
// Get minimum confidence parameter (default 0.4)
|
// Get minimum confidence parameter (default 0.4)
|
||||||
minConfidence := 0.4
|
minConfidence := 0.4
|
||||||
if confStr := r.URL.Query().Get("min_confidence"); confStr != "" {
|
if confStr := r.URL.Query().Get("min_confidence"); confStr != "" {
|
||||||
if c, err := strconv.ParseFloat(confStr, 64); err == nil && c >= 0 && c <= 1 {
|
if c, parseErr := strconv.ParseFloat(confStr, 64); parseErr == nil && c >= 0 && c <= 1 {
|
||||||
minConfidence = c
|
minConfidence = c
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -10,6 +10,7 @@ import (
|
|||||||
|
|
||||||
"github.com/go-chi/chi/v5"
|
"github.com/go-chi/chi/v5"
|
||||||
"github.com/lukaszraczylo/claude-mnemonic/pkg/models"
|
"github.com/lukaszraczylo/claude-mnemonic/pkg/models"
|
||||||
|
"github.com/rs/zerolog/log"
|
||||||
)
|
)
|
||||||
|
|
||||||
// FeedbackRequest represents a user feedback submission.
|
// FeedbackRequest represents a user feedback submission.
|
||||||
@@ -65,9 +66,10 @@ func (s *Service) handleObservationFeedback(w http.ResponseWriter, r *http.Reque
|
|||||||
if err == nil && obs != nil {
|
if err == nil && obs != nil {
|
||||||
obs.UserFeedback = req.Feedback // Apply the new feedback
|
obs.UserFeedback = req.Feedback // Apply the new feedback
|
||||||
newScore = scoreCalculator.Calculate(obs, time.Now())
|
newScore = scoreCalculator.Calculate(obs, time.Now())
|
||||||
if err := observationStore.UpdateImportanceScore(r.Context(), id, newScore); err != nil {
|
if scoreErr := observationStore.UpdateImportanceScore(r.Context(), id, newScore); scoreErr != nil {
|
||||||
// Log but don't fail - feedback was recorded
|
// Log but don't fail - feedback was recorded
|
||||||
// Score will be updated on next recalculation cycle
|
// Score will be updated on next recalculation cycle
|
||||||
|
log.Warn().Err(scoreErr).Int64("id", id).Msg("Failed to update importance score after feedback")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -261,8 +263,9 @@ func (s *Service) handleUpdateConceptWeight(w http.ResponseWriter, r *http.Reque
|
|||||||
|
|
||||||
// Refresh concept weights in recalculator
|
// Refresh concept weights in recalculator
|
||||||
if recalculator != nil {
|
if recalculator != nil {
|
||||||
if err := recalculator.RefreshConceptWeights(r.Context()); err != nil {
|
if refreshErr := recalculator.RefreshConceptWeights(r.Context()); refreshErr != nil {
|
||||||
// Log but don't fail - weight was saved
|
// Log but don't fail - weight was saved
|
||||||
|
log.Warn().Err(refreshErr).Str("concept", concept).Msg("Failed to refresh concept weights in recalculator")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -308,8 +311,9 @@ func (s *Service) handleTriggerRecalculation(w http.ResponseWriter, r *http.Requ
|
|||||||
|
|
||||||
// Run recalculation in background
|
// Run recalculation in background
|
||||||
go func() {
|
go func() {
|
||||||
if err := recalculator.RecalculateNow(r.Context()); err != nil {
|
if recalcErr := recalculator.RecalculateNow(r.Context()); recalcErr != nil {
|
||||||
// Log error but don't block response
|
// Log error but don't block response
|
||||||
|
log.Warn().Err(recalcErr).Msg("Failed to trigger score recalculation")
|
||||||
}
|
}
|
||||||
}()
|
}()
|
||||||
|
|
||||||
@@ -347,8 +351,9 @@ func (s *Service) incrementRetrievalCounts(ids []int64) {
|
|||||||
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
|
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
|
||||||
defer cancel()
|
defer cancel()
|
||||||
|
|
||||||
if err := store.IncrementRetrievalCount(ctx, ids); err != nil {
|
if incrErr := store.IncrementRetrievalCount(ctx, ids); incrErr != nil {
|
||||||
// Log but don't fail - this is a background operation
|
// Log but don't fail - this is a background operation
|
||||||
|
log.Warn().Err(incrErr).Msg("Failed to increment retrieval count in background")
|
||||||
}
|
}
|
||||||
}()
|
}()
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1276,8 +1276,6 @@ func TestObservationRequest_Fields(t *testing.T) {
|
|||||||
ClaudeSessionID: "session-abc",
|
ClaudeSessionID: "session-abc",
|
||||||
Project: "my-project",
|
Project: "my-project",
|
||||||
ToolName: "Read",
|
ToolName: "Read",
|
||||||
ToolInput: map[string]string{"path": "/file.go"},
|
|
||||||
ToolResponse: "file contents",
|
|
||||||
CWD: "/home/user/project",
|
CWD: "/home/user/project",
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -77,10 +77,10 @@ func TestParseObservations_TableDriven(t *testing.T) {
|
|||||||
tests := []struct {
|
tests := []struct {
|
||||||
name string
|
name string
|
||||||
input string
|
input string
|
||||||
expectedCount int
|
|
||||||
expectedType models.ObservationType
|
expectedType models.ObservationType
|
||||||
expectedTitle string
|
expectedTitle string
|
||||||
checkConcepts []string
|
checkConcepts []string
|
||||||
|
expectedCount int
|
||||||
}{
|
}{
|
||||||
{
|
{
|
||||||
name: "valid_bugfix_observation",
|
name: "valid_bugfix_observation",
|
||||||
@@ -300,9 +300,9 @@ func TestParseSummary_TableDriven(t *testing.T) {
|
|||||||
tests := []struct {
|
tests := []struct {
|
||||||
name string
|
name string
|
||||||
input string
|
input string
|
||||||
|
expectedRequest string
|
||||||
sessionID int64
|
sessionID int64
|
||||||
expectNil bool
|
expectNil bool
|
||||||
expectedRequest string
|
|
||||||
}{
|
}{
|
||||||
{
|
{
|
||||||
name: "empty_input",
|
name: "empty_input",
|
||||||
|
|||||||
@@ -31,15 +31,14 @@ type SyncSummaryFunc func(summary *models.SessionSummary)
|
|||||||
|
|
||||||
// Processor handles SDK agent processing of observations and summaries using Claude Code CLI.
|
// Processor handles SDK agent processing of observations and summaries using Claude Code CLI.
|
||||||
type Processor struct {
|
type Processor struct {
|
||||||
claudePath string
|
|
||||||
model string
|
|
||||||
observationStore *sqlite.ObservationStore
|
observationStore *sqlite.ObservationStore
|
||||||
summaryStore *sqlite.SummaryStore
|
summaryStore *sqlite.SummaryStore
|
||||||
broadcastFunc BroadcastFunc
|
broadcastFunc BroadcastFunc
|
||||||
syncObservationFunc SyncObservationFunc
|
syncObservationFunc SyncObservationFunc
|
||||||
syncSummaryFunc SyncSummaryFunc
|
syncSummaryFunc SyncSummaryFunc
|
||||||
// Semaphore to limit concurrent Claude CLI calls (prevents API overload)
|
sem chan struct{}
|
||||||
sem chan struct{}
|
claudePath string
|
||||||
|
model string
|
||||||
}
|
}
|
||||||
|
|
||||||
// SetBroadcastFunc sets the broadcast callback for SSE events.
|
// SetBroadcastFunc sets the broadcast callback for SSE events.
|
||||||
|
|||||||
@@ -11,8 +11,8 @@ import (
|
|||||||
|
|
||||||
func TestIsSelfReferentialSummary(t *testing.T) {
|
func TestIsSelfReferentialSummary(t *testing.T) {
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
name string
|
|
||||||
summary *models.ParsedSummary
|
summary *models.ParsedSummary
|
||||||
|
name string
|
||||||
expected bool
|
expected bool
|
||||||
}{
|
}{
|
||||||
{
|
{
|
||||||
@@ -281,8 +281,8 @@ func TestTruncateForLog(t *testing.T) {
|
|||||||
tests := []struct {
|
tests := []struct {
|
||||||
name string
|
name string
|
||||||
input string
|
input string
|
||||||
maxLen int
|
|
||||||
expected string
|
expected string
|
||||||
|
maxLen int
|
||||||
}{
|
}{
|
||||||
{
|
{
|
||||||
name: "shorter_than_max",
|
name: "shorter_than_max",
|
||||||
@@ -371,11 +371,11 @@ func TestCaptureFileMtimes(t *testing.T) {
|
|||||||
file1 := filepath.Join(tmpDir, "file1.txt")
|
file1 := filepath.Join(tmpDir, "file1.txt")
|
||||||
file2 := filepath.Join(tmpDir, "file2.txt")
|
file2 := filepath.Join(tmpDir, "file2.txt")
|
||||||
|
|
||||||
err = os.WriteFile(file1, []byte("content1"), 0644)
|
err = os.WriteFile(file1, []byte("content1"), 0600)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
err = os.WriteFile(file2, []byte("content2"), 0644)
|
err = os.WriteFile(file2, []byte("content2"), 0600)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
@@ -415,7 +415,7 @@ func TestGetFileMtimes(t *testing.T) {
|
|||||||
defer os.RemoveAll(tmpDir)
|
defer os.RemoveAll(tmpDir)
|
||||||
|
|
||||||
testFile := filepath.Join(tmpDir, "test.txt")
|
testFile := filepath.Join(tmpDir, "test.txt")
|
||||||
err = os.WriteFile(testFile, []byte("content"), 0644)
|
err = os.WriteFile(testFile, []byte("content"), 0600)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
@@ -437,7 +437,7 @@ func TestGetFileContent(t *testing.T) {
|
|||||||
t.Run("reads_existing_file", func(t *testing.T) {
|
t.Run("reads_existing_file", func(t *testing.T) {
|
||||||
testFile := filepath.Join(tmpDir, "test.txt")
|
testFile := filepath.Join(tmpDir, "test.txt")
|
||||||
content := "test content"
|
content := "test content"
|
||||||
err := os.WriteFile(testFile, []byte(content), 0644)
|
err := os.WriteFile(testFile, []byte(content), 0600)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
@@ -459,7 +459,7 @@ func TestGetFileContent(t *testing.T) {
|
|||||||
for i := 0; i < 3000; i++ {
|
for i := 0; i < 3000; i++ {
|
||||||
longContent += "x"
|
longContent += "x"
|
||||||
}
|
}
|
||||||
err := os.WriteFile(testFile, []byte(longContent), 0644)
|
err := os.WriteFile(testFile, []byte(longContent), 0600)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
@@ -473,7 +473,7 @@ func TestGetFileContent(t *testing.T) {
|
|||||||
t.Run("resolves_relative_path_with_cwd", func(t *testing.T) {
|
t.Run("resolves_relative_path_with_cwd", func(t *testing.T) {
|
||||||
testFile := filepath.Join(tmpDir, "relative.txt")
|
testFile := filepath.Join(tmpDir, "relative.txt")
|
||||||
content := "relative content"
|
content := "relative content"
|
||||||
err := os.WriteFile(testFile, []byte(content), 0644)
|
err := os.WriteFile(testFile, []byte(content), 0600)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
@@ -719,8 +719,8 @@ func TestShouldSkipTrivialOperation_EdgeCases(t *testing.T) {
|
|||||||
// TestIsSelfReferentialSummary_MoreCases tests additional self-referential detection cases.
|
// TestIsSelfReferentialSummary_MoreCases tests additional self-referential detection cases.
|
||||||
func TestIsSelfReferentialSummary_MoreCases(t *testing.T) {
|
func TestIsSelfReferentialSummary_MoreCases(t *testing.T) {
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
name string
|
|
||||||
summary *models.ParsedSummary
|
summary *models.ParsedSummary
|
||||||
|
name string
|
||||||
expected bool
|
expected bool
|
||||||
}{
|
}{
|
||||||
{
|
{
|
||||||
@@ -932,7 +932,7 @@ func TestCaptureFileMtimes_DuplicatePaths(t *testing.T) {
|
|||||||
defer os.RemoveAll(tmpDir)
|
defer os.RemoveAll(tmpDir)
|
||||||
|
|
||||||
testFile := filepath.Join(tmpDir, "shared.txt")
|
testFile := filepath.Join(tmpDir, "shared.txt")
|
||||||
err = os.WriteFile(testFile, []byte("content"), 0644)
|
err = os.WriteFile(testFile, []byte("content"), 0600)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -24,12 +24,12 @@ var ObservationConcepts = []string{
|
|||||||
|
|
||||||
// ToolExecution represents a tool execution for observation.
|
// ToolExecution represents a tool execution for observation.
|
||||||
type ToolExecution struct {
|
type ToolExecution struct {
|
||||||
ID int64
|
|
||||||
ToolName string
|
ToolName string
|
||||||
ToolInput string
|
ToolInput string
|
||||||
ToolOutput string
|
ToolOutput string
|
||||||
CreatedAtEpoch int64
|
|
||||||
CWD string
|
CWD string
|
||||||
|
ID int64
|
||||||
|
CreatedAtEpoch int64
|
||||||
}
|
}
|
||||||
|
|
||||||
// BuildObservationPrompt builds a prompt for processing a tool observation.
|
// BuildObservationPrompt builds a prompt for processing a tool observation.
|
||||||
@@ -67,12 +67,12 @@ func BuildObservationPrompt(exec ToolExecution) string {
|
|||||||
|
|
||||||
// SummaryRequest contains data for building a summary prompt.
|
// SummaryRequest contains data for building a summary prompt.
|
||||||
type SummaryRequest struct {
|
type SummaryRequest struct {
|
||||||
SessionDBID int64
|
|
||||||
SDKSessionID string
|
SDKSessionID string
|
||||||
Project string
|
Project string
|
||||||
UserPrompt string
|
UserPrompt string
|
||||||
LastUserMessage string
|
LastUserMessage string
|
||||||
LastAssistantMessage string
|
LastAssistantMessage string
|
||||||
|
SessionDBID int64
|
||||||
}
|
}
|
||||||
|
|
||||||
// BuildSummaryPrompt builds a prompt requesting a session summary.
|
// BuildSummaryPrompt builds a prompt requesting a session summary.
|
||||||
|
|||||||
@@ -12,8 +12,8 @@ func TestTruncate(t *testing.T) {
|
|||||||
tests := []struct {
|
tests := []struct {
|
||||||
name string
|
name string
|
||||||
input string
|
input string
|
||||||
maxLen int
|
|
||||||
expected string
|
expected string
|
||||||
|
maxLen int
|
||||||
}{
|
}{
|
||||||
{
|
{
|
||||||
name: "shorter_than_max",
|
name: "shorter_than_max",
|
||||||
@@ -58,10 +58,10 @@ func TestTruncate(t *testing.T) {
|
|||||||
func TestBuildObservationPrompt(t *testing.T) {
|
func TestBuildObservationPrompt(t *testing.T) {
|
||||||
now := time.Now().UnixMilli()
|
now := time.Now().UnixMilli()
|
||||||
|
|
||||||
tests := []struct {
|
tests := []struct { //nolint:govet
|
||||||
name string
|
|
||||||
exec ToolExecution
|
exec ToolExecution
|
||||||
contains []string
|
contains []string
|
||||||
|
name string
|
||||||
}{
|
}{
|
||||||
{
|
{
|
||||||
name: "basic_read_tool",
|
name: "basic_read_tool",
|
||||||
|
|||||||
+255
-93
@@ -12,6 +12,10 @@ import (
|
|||||||
|
|
||||||
"github.com/go-chi/chi/v5"
|
"github.com/go-chi/chi/v5"
|
||||||
"github.com/go-chi/chi/v5/middleware"
|
"github.com/go-chi/chi/v5/middleware"
|
||||||
|
"github.com/lukaszraczylo/claude-mnemonic/internal/chunking"
|
||||||
|
"github.com/lukaszraczylo/claude-mnemonic/internal/chunking/golang"
|
||||||
|
"github.com/lukaszraczylo/claude-mnemonic/internal/chunking/python"
|
||||||
|
"github.com/lukaszraczylo/claude-mnemonic/internal/chunking/typescript"
|
||||||
"github.com/lukaszraczylo/claude-mnemonic/internal/config"
|
"github.com/lukaszraczylo/claude-mnemonic/internal/config"
|
||||||
"github.com/lukaszraczylo/claude-mnemonic/internal/db/sqlite"
|
"github.com/lukaszraczylo/claude-mnemonic/internal/db/sqlite"
|
||||||
"github.com/lukaszraczylo/claude-mnemonic/internal/embedding"
|
"github.com/lukaszraczylo/claude-mnemonic/internal/embedding"
|
||||||
@@ -20,6 +24,8 @@ import (
|
|||||||
"github.com/lukaszraczylo/claude-mnemonic/internal/scoring"
|
"github.com/lukaszraczylo/claude-mnemonic/internal/scoring"
|
||||||
"github.com/lukaszraczylo/claude-mnemonic/internal/search/expansion"
|
"github.com/lukaszraczylo/claude-mnemonic/internal/search/expansion"
|
||||||
"github.com/lukaszraczylo/claude-mnemonic/internal/update"
|
"github.com/lukaszraczylo/claude-mnemonic/internal/update"
|
||||||
|
"github.com/lukaszraczylo/claude-mnemonic/internal/vector"
|
||||||
|
"github.com/lukaszraczylo/claude-mnemonic/internal/vector/hybrid"
|
||||||
"github.com/lukaszraczylo/claude-mnemonic/internal/vector/sqlitevec"
|
"github.com/lukaszraczylo/claude-mnemonic/internal/vector/sqlitevec"
|
||||||
"github.com/lukaszraczylo/claude-mnemonic/internal/watcher"
|
"github.com/lukaszraczylo/claude-mnemonic/internal/watcher"
|
||||||
"github.com/lukaszraczylo/claude-mnemonic/internal/worker/sdk"
|
"github.com/lukaszraczylo/claude-mnemonic/internal/worker/sdk"
|
||||||
@@ -56,80 +62,52 @@ type RetrievalStats struct {
|
|||||||
|
|
||||||
// Service is the main worker service orchestrator.
|
// Service is the main worker service orchestrator.
|
||||||
type Service struct {
|
type Service struct {
|
||||||
// Version of the worker binary
|
startTime time.Time
|
||||||
version string
|
initError error
|
||||||
|
vectorClient vector.Client
|
||||||
// Configuration
|
ctx context.Context
|
||||||
config *config.Config
|
sseBroadcaster *sse.Broadcaster
|
||||||
|
server *http.Server
|
||||||
// Database
|
graphRebuildTicker *time.Ticker
|
||||||
store *sqlite.Store
|
hybridMetrics *hybrid.Metrics
|
||||||
sessionStore *sqlite.SessionStore
|
graphSearchClient *hybrid.GraphSearchClient
|
||||||
observationStore *sqlite.ObservationStore
|
retrievalStats map[string]*RetrievalStats
|
||||||
summaryStore *sqlite.SummaryStore
|
staleQueue chan staleVerifyRequest
|
||||||
promptStore *sqlite.PromptStore
|
queryExpander *expansion.Expander
|
||||||
conflictStore *sqlite.ConflictStore
|
recalculator *scoring.Recalculator
|
||||||
patternStore *sqlite.PatternStore
|
summaryStore *sqlite.SummaryStore
|
||||||
relationStore *sqlite.RelationStore
|
promptStore *sqlite.PromptStore
|
||||||
|
conflictStore *sqlite.ConflictStore
|
||||||
// Pattern detection
|
patternStore *sqlite.PatternStore
|
||||||
patternDetector *pattern.Detector
|
relationStore *sqlite.RelationStore
|
||||||
|
patternDetector *pattern.Detector
|
||||||
// Domain services
|
sessionManager *session.Manager
|
||||||
sessionManager *session.Manager
|
router *chi.Mux
|
||||||
sseBroadcaster *sse.Broadcaster
|
config *config.Config
|
||||||
processor *sdk.Processor
|
store *sqlite.Store
|
||||||
|
vectorSync *sqlitevec.Sync
|
||||||
// Vector database (sqlite-vec with local embeddings)
|
reranker *reranking.Service
|
||||||
embedSvc *embedding.Service
|
updater *update.Updater
|
||||||
vectorClient *sqlitevec.Client
|
observationStore *sqlite.ObservationStore
|
||||||
vectorSync *sqlitevec.Sync
|
scoreCalculator *scoring.Calculator
|
||||||
|
processor *sdk.Processor
|
||||||
// Cross-encoder reranking (for improved search relevance)
|
dbWatcher *watcher.Watcher
|
||||||
reranker *reranking.Service
|
sessionStore *sqlite.SessionStore
|
||||||
|
configWatcher *watcher.Watcher
|
||||||
// Query expansion (for improved search recall)
|
embedSvc *embedding.Service
|
||||||
queryExpander *expansion.Expander
|
cancel context.CancelFunc
|
||||||
|
version string
|
||||||
// Importance scoring
|
wg sync.WaitGroup
|
||||||
scoreCalculator *scoring.Calculator
|
initMu sync.RWMutex
|
||||||
recalculator *scoring.Recalculator
|
retrievalStatsMu sync.RWMutex
|
||||||
|
staleQueueOnce sync.Once
|
||||||
// HTTP server
|
ready atomic.Bool
|
||||||
router *chi.Mux
|
|
||||||
server *http.Server
|
|
||||||
startTime time.Time
|
|
||||||
|
|
||||||
// Retrieval statistics (per-project)
|
|
||||||
retrievalStats map[string]*RetrievalStats
|
|
||||||
retrievalStatsMu sync.RWMutex
|
|
||||||
|
|
||||||
// Lifecycle
|
|
||||||
ctx context.Context
|
|
||||||
cancel context.CancelFunc
|
|
||||||
wg sync.WaitGroup
|
|
||||||
|
|
||||||
// Initialization state (for deferred init)
|
|
||||||
ready atomic.Bool
|
|
||||||
initError error
|
|
||||||
initMu sync.RWMutex
|
|
||||||
|
|
||||||
// Background verification queue for stale observations
|
|
||||||
staleQueue chan staleVerifyRequest
|
|
||||||
staleQueueOnce sync.Once
|
|
||||||
|
|
||||||
// File watchers for auto-recreation on deletion
|
|
||||||
dbWatcher *watcher.Watcher
|
|
||||||
configWatcher *watcher.Watcher
|
|
||||||
|
|
||||||
// Self-updater
|
|
||||||
updater *update.Updater
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// staleVerifyRequest represents a request to verify a stale observation in background
|
// staleVerifyRequest represents a request to verify a stale observation in background
|
||||||
type staleVerifyRequest struct {
|
type staleVerifyRequest struct {
|
||||||
observationID int64
|
|
||||||
cwd string
|
cwd string
|
||||||
|
observationID int64
|
||||||
}
|
}
|
||||||
|
|
||||||
// NewService creates a new worker service with deferred initialization.
|
// NewService creates a new worker service with deferred initialization.
|
||||||
@@ -212,7 +190,7 @@ func (s *Service) initializeAsync() {
|
|||||||
|
|
||||||
// Create embedding service and sqlite-vec client for vector search (optional)
|
// Create embedding service and sqlite-vec client for vector search (optional)
|
||||||
var embedSvc *embedding.Service
|
var embedSvc *embedding.Service
|
||||||
var vectorClient *sqlitevec.Client
|
var vectorClient vector.Client
|
||||||
var vectorSync *sqlitevec.Sync
|
var vectorSync *sqlitevec.Sync
|
||||||
|
|
||||||
var reranker *reranking.Service
|
var reranker *reranking.Service
|
||||||
@@ -223,17 +201,68 @@ func (s *Service) initializeAsync() {
|
|||||||
} else {
|
} else {
|
||||||
embedSvc = emb
|
embedSvc = emb
|
||||||
// Create sqlite-vec client using the same DB connection
|
// Create sqlite-vec client using the same DB connection
|
||||||
client, err := sqlitevec.NewClient(sqlitevec.Config{
|
baseClient, clientErr := sqlitevec.NewClient(sqlitevec.Config{
|
||||||
DB: store.DB(),
|
DB: store.DB(),
|
||||||
}, embedSvc)
|
}, embedSvc)
|
||||||
if err != nil {
|
if clientErr != nil {
|
||||||
log.Warn().Err(err).Msg("sqlite-vec client creation failed - vector search disabled")
|
log.Warn().Err(clientErr).Msg("sqlite-vec client creation failed - vector search disabled")
|
||||||
} else {
|
} else {
|
||||||
vectorClient = client
|
// Wrap with LEANN hybrid storage client
|
||||||
vectorSync = sqlitevec.NewSync(client)
|
strategy := hybrid.GetStrategyFromEnv()
|
||||||
|
hybridClient := hybrid.NewClient(hybrid.Config{
|
||||||
|
BaseClient: baseClient,
|
||||||
|
DB: store.DB(),
|
||||||
|
EmbedSvc: embedSvc,
|
||||||
|
Strategy: strategy,
|
||||||
|
HubThreshold: hybrid.GetHubThresholdFromEnv(),
|
||||||
|
})
|
||||||
|
|
||||||
|
// Wrap with graph-aware search client
|
||||||
|
graphConfig := hybrid.GraphConfig{
|
||||||
|
Enabled: s.config.GraphEnabled,
|
||||||
|
MaxHops: s.config.GraphMaxHops,
|
||||||
|
BranchFactor: s.config.GraphBranchFactor,
|
||||||
|
EdgeWeight: s.config.GraphEdgeWeight,
|
||||||
|
}
|
||||||
|
graphClient := hybrid.NewGraphSearchClient(hybridClient, nil, graphConfig)
|
||||||
|
vectorClient = graphClient
|
||||||
|
s.graphSearchClient = graphClient
|
||||||
|
s.hybridMetrics = hybrid.NewMetrics()
|
||||||
|
|
||||||
|
vectorSync = sqlitevec.NewSync(baseClient)
|
||||||
|
|
||||||
|
// Initialize AST-aware code chunking
|
||||||
|
chunkOpts := chunking.DefaultChunkOptions()
|
||||||
|
chunkers := []chunking.Chunker{
|
||||||
|
golang.NewChunker(chunkOpts),
|
||||||
|
python.NewChunker(chunkOpts),
|
||||||
|
typescript.NewChunker(chunkOpts),
|
||||||
|
}
|
||||||
|
chunkingManager := chunking.NewManager(chunkers, chunkOpts)
|
||||||
|
vectorSync.SetChunkingManager(chunkingManager)
|
||||||
|
|
||||||
|
strategyName := "hub" // default
|
||||||
|
switch strategy {
|
||||||
|
case hybrid.StorageAlways:
|
||||||
|
strategyName = "always"
|
||||||
|
case hybrid.StorageOnDemand:
|
||||||
|
strategyName = "on_demand"
|
||||||
|
}
|
||||||
|
|
||||||
log.Info().
|
log.Info().
|
||||||
Str("model", embedSvc.Version()).
|
Str("model", embedSvc.Version()).
|
||||||
Msg("sqlite-vec vector search enabled")
|
Str("vector_strategy", strategyName).
|
||||||
|
Bool("graph_enabled", s.config.GraphEnabled).
|
||||||
|
Strs("chunkers", []string{"go", "python", "typescript"}).
|
||||||
|
Msg("sqlite-vec vector search with AST-aware code chunking enabled")
|
||||||
|
|
||||||
|
if s.config.GraphEnabled {
|
||||||
|
log.Info().
|
||||||
|
Int("max_hops", s.config.GraphMaxHops).
|
||||||
|
Int("branch_factor", s.config.GraphBranchFactor).
|
||||||
|
Float64("edge_weight", s.config.GraphEdgeWeight).
|
||||||
|
Msg("Graph-aware search configured (graph will be built after initialization)")
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Create cross-encoder reranking service if enabled
|
// Create cross-encoder reranking service if enabled
|
||||||
@@ -243,9 +272,9 @@ func (s *Service) initializeAsync() {
|
|||||||
rerankCfg.Alpha = s.config.RerankingAlpha
|
rerankCfg.Alpha = s.config.RerankingAlpha
|
||||||
}
|
}
|
||||||
|
|
||||||
ranker, err := reranking.NewService(rerankCfg)
|
ranker, rankerErr := reranking.NewService(rerankCfg)
|
||||||
if err != nil {
|
if rankerErr != nil {
|
||||||
log.Warn().Err(err).Msg("Cross-encoder reranking service creation failed - reranking disabled")
|
log.Warn().Err(rankerErr).Msg("Cross-encoder reranking service creation failed - reranking disabled")
|
||||||
} else {
|
} else {
|
||||||
reranker = ranker
|
reranker = ranker
|
||||||
log.Info().
|
log.Info().
|
||||||
@@ -424,6 +453,12 @@ func (s *Service) initializeAsync() {
|
|||||||
// Start file watchers for auto-recreation on deletion
|
// Start file watchers for auto-recreation on deletion
|
||||||
s.startWatchers()
|
s.startWatchers()
|
||||||
|
|
||||||
|
// Build initial observation graph in background if graph search is enabled
|
||||||
|
if s.config.GraphEnabled && s.graphSearchClient != nil {
|
||||||
|
s.wg.Add(1)
|
||||||
|
go s.buildInitialGraph(observationStore)
|
||||||
|
}
|
||||||
|
|
||||||
// Check if vectors need rebuilding (empty or model version mismatch) and trigger background rebuild
|
// Check if vectors need rebuilding (empty or model version mismatch) and trigger background rebuild
|
||||||
if vectorClient != nil && vectorSync != nil {
|
if vectorClient != nil && vectorSync != nil {
|
||||||
needsRebuild, reason := vectorClient.NeedsRebuild(s.ctx)
|
needsRebuild, reason := vectorClient.NeedsRebuild(s.ctx)
|
||||||
@@ -457,8 +492,8 @@ func (s *Service) startWatchers() {
|
|||||||
log.Warn().Err(err).Msg("Failed to create database watcher")
|
log.Warn().Err(err).Msg("Failed to create database watcher")
|
||||||
} else {
|
} else {
|
||||||
s.dbWatcher = dbWatcher
|
s.dbWatcher = dbWatcher
|
||||||
if err := dbWatcher.Start(); err != nil {
|
if startErr := dbWatcher.Start(); startErr != nil {
|
||||||
log.Warn().Err(err).Msg("Failed to start database watcher")
|
log.Warn().Err(startErr).Msg("Failed to start database watcher")
|
||||||
} else {
|
} else {
|
||||||
log.Info().Str("path", s.config.DBPath).Msg("Database file watcher started")
|
log.Info().Str("path", s.config.DBPath).Msg("Database file watcher started")
|
||||||
}
|
}
|
||||||
@@ -559,15 +594,26 @@ func (s *Service) reinitializeDatabase() {
|
|||||||
log.Warn().Err(err).Msg("Embedding service creation failed after reinit")
|
log.Warn().Err(err).Msg("Embedding service creation failed after reinit")
|
||||||
} else {
|
} else {
|
||||||
embedSvc = emb
|
embedSvc = emb
|
||||||
client, err := sqlitevec.NewClient(sqlitevec.Config{
|
client, clientErr := sqlitevec.NewClient(sqlitevec.Config{
|
||||||
DB: store.DB(),
|
DB: store.DB(),
|
||||||
}, embedSvc)
|
}, embedSvc)
|
||||||
if err != nil {
|
if clientErr != nil {
|
||||||
log.Warn().Err(err).Msg("sqlite-vec client creation failed after reinit")
|
log.Warn().Err(clientErr).Msg("sqlite-vec client creation failed after reinit")
|
||||||
} else {
|
} else {
|
||||||
vectorClient = client
|
vectorClient = client
|
||||||
vectorSync = sqlitevec.NewSync(client)
|
vectorSync = sqlitevec.NewSync(client)
|
||||||
log.Info().Msg("sqlite-vec reconnected after reinit")
|
|
||||||
|
// Initialize AST-aware code chunking
|
||||||
|
chunkOpts := chunking.DefaultChunkOptions()
|
||||||
|
chunkers := []chunking.Chunker{
|
||||||
|
golang.NewChunker(chunkOpts),
|
||||||
|
python.NewChunker(chunkOpts),
|
||||||
|
typescript.NewChunker(chunkOpts),
|
||||||
|
}
|
||||||
|
chunkingManager := chunking.NewManager(chunkers, chunkOpts)
|
||||||
|
vectorSync.SetChunkingManager(chunkingManager)
|
||||||
|
|
||||||
|
log.Info().Msg("sqlite-vec with code chunking reconnected after reinit")
|
||||||
}
|
}
|
||||||
|
|
||||||
// Recreate cross-encoder reranking service if enabled
|
// Recreate cross-encoder reranking service if enabled
|
||||||
@@ -577,9 +623,9 @@ func (s *Service) reinitializeDatabase() {
|
|||||||
rerankCfg.Alpha = s.config.RerankingAlpha
|
rerankCfg.Alpha = s.config.RerankingAlpha
|
||||||
}
|
}
|
||||||
|
|
||||||
ranker, err := reranking.NewService(rerankCfg)
|
ranker, rankerErr := reranking.NewService(rerankCfg)
|
||||||
if err != nil {
|
if rankerErr != nil {
|
||||||
log.Warn().Err(err).Msg("Cross-encoder reranking service creation failed after reinit")
|
log.Warn().Err(rankerErr).Msg("Cross-encoder reranking service creation failed after reinit")
|
||||||
} else {
|
} else {
|
||||||
reranker = ranker
|
reranker = ranker
|
||||||
log.Info().Msg("Cross-encoder reranking reconnected after reinit")
|
log.Info().Msg("Cross-encoder reranking reconnected after reinit")
|
||||||
@@ -824,8 +870,8 @@ func (s *Service) rebuildAllVectors(
|
|||||||
log.Error().Err(err).Msg("Failed to fetch observations for vector rebuild")
|
log.Error().Err(err).Msg("Failed to fetch observations for vector rebuild")
|
||||||
} else {
|
} else {
|
||||||
for _, obs := range observations {
|
for _, obs := range observations {
|
||||||
if err := vectorSync.SyncObservation(s.ctx, obs); err != nil {
|
if syncErr := vectorSync.SyncObservation(s.ctx, obs); syncErr != nil {
|
||||||
log.Warn().Err(err).Int64("id", obs.ID).Msg("Failed to sync observation during rebuild")
|
log.Warn().Err(syncErr).Int64("id", obs.ID).Msg("Failed to sync observation during rebuild")
|
||||||
syncErrors++
|
syncErrors++
|
||||||
} else {
|
} else {
|
||||||
totalSynced++
|
totalSynced++
|
||||||
@@ -840,8 +886,8 @@ func (s *Service) rebuildAllVectors(
|
|||||||
log.Error().Err(err).Msg("Failed to fetch summaries for vector rebuild")
|
log.Error().Err(err).Msg("Failed to fetch summaries for vector rebuild")
|
||||||
} else {
|
} else {
|
||||||
for _, summary := range summaries {
|
for _, summary := range summaries {
|
||||||
if err := vectorSync.SyncSummary(s.ctx, summary); err != nil {
|
if syncErr := vectorSync.SyncSummary(s.ctx, summary); syncErr != nil {
|
||||||
log.Warn().Err(err).Int64("id", summary.ID).Msg("Failed to sync summary during rebuild")
|
log.Warn().Err(syncErr).Int64("id", summary.ID).Msg("Failed to sync summary during rebuild")
|
||||||
syncErrors++
|
syncErrors++
|
||||||
} else {
|
} else {
|
||||||
totalSynced++
|
totalSynced++
|
||||||
@@ -880,7 +926,7 @@ func (s *Service) rebuildStaleVectors(
|
|||||||
observationStore *sqlite.ObservationStore,
|
observationStore *sqlite.ObservationStore,
|
||||||
summaryStore *sqlite.SummaryStore,
|
summaryStore *sqlite.SummaryStore,
|
||||||
promptStore *sqlite.PromptStore,
|
promptStore *sqlite.PromptStore,
|
||||||
vectorClient *sqlitevec.Client,
|
vectorClient vector.Client,
|
||||||
vectorSync *sqlitevec.Sync,
|
vectorSync *sqlitevec.Sync,
|
||||||
) {
|
) {
|
||||||
defer s.wg.Done()
|
defer s.wg.Done()
|
||||||
@@ -1045,6 +1091,113 @@ func (s *Service) verifyStaleObservation(req staleVerifyRequest) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// buildInitialGraph builds the observation graph from all observations in background.
|
||||||
|
func (s *Service) buildInitialGraph(observationStore *sqlite.ObservationStore) {
|
||||||
|
defer s.wg.Done()
|
||||||
|
|
||||||
|
log.Info().Msg("Building initial observation graph...")
|
||||||
|
start := time.Now()
|
||||||
|
|
||||||
|
// Fetch all observations
|
||||||
|
observations, err := observationStore.GetAllObservations(s.ctx)
|
||||||
|
if err != nil {
|
||||||
|
log.Error().Err(err).Msg("Failed to fetch observations for graph building")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(observations) == 0 {
|
||||||
|
log.Info().Msg("No observations to build graph from")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// Build graph using RebuildGraph method
|
||||||
|
if err := s.graphSearchClient.RebuildGraph(s.ctx, observations); err != nil {
|
||||||
|
log.Error().Err(err).Msg("Failed to build observation graph")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
elapsed := time.Since(start)
|
||||||
|
stats := s.graphSearchClient.GetGraphStats()
|
||||||
|
|
||||||
|
log.Info().
|
||||||
|
Int("observations", len(observations)).
|
||||||
|
Int("nodes", stats.NodeCount).
|
||||||
|
Int("edges", stats.EdgeCount).
|
||||||
|
Float64("avg_degree", stats.AvgDegree).
|
||||||
|
Int("max_degree", stats.MaxDegree).
|
||||||
|
Dur("elapsed", elapsed).
|
||||||
|
Msg("Initial observation graph built successfully")
|
||||||
|
|
||||||
|
// Start periodic graph rebuild if configured
|
||||||
|
if s.config.GraphRebuildIntervalMin > 0 {
|
||||||
|
s.startGraphRebuildTimer(observationStore)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// startGraphRebuildTimer starts a periodic ticker to rebuild the observation graph.
|
||||||
|
func (s *Service) startGraphRebuildTimer(observationStore *sqlite.ObservationStore) {
|
||||||
|
interval := time.Duration(s.config.GraphRebuildIntervalMin) * time.Minute
|
||||||
|
s.graphRebuildTicker = time.NewTicker(interval)
|
||||||
|
|
||||||
|
log.Info().
|
||||||
|
Dur("interval", interval).
|
||||||
|
Msg("Started periodic graph rebuild timer")
|
||||||
|
|
||||||
|
s.wg.Add(1)
|
||||||
|
go func() {
|
||||||
|
defer s.wg.Done()
|
||||||
|
defer s.graphRebuildTicker.Stop()
|
||||||
|
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-s.ctx.Done():
|
||||||
|
return
|
||||||
|
case <-s.graphRebuildTicker.C:
|
||||||
|
log.Info().Msg("Periodic graph rebuild triggered")
|
||||||
|
s.rebuildGraph(observationStore)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
}
|
||||||
|
|
||||||
|
// rebuildGraph rebuilds the observation graph from current observations.
|
||||||
|
func (s *Service) rebuildGraph(observationStore *sqlite.ObservationStore) {
|
||||||
|
if s.graphSearchClient == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
start := time.Now()
|
||||||
|
|
||||||
|
// Fetch all observations
|
||||||
|
observations, err := observationStore.GetAllObservations(s.ctx)
|
||||||
|
if err != nil {
|
||||||
|
log.Error().Err(err).Msg("Failed to fetch observations for graph rebuild")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(observations) == 0 {
|
||||||
|
log.Debug().Msg("No observations to rebuild graph from")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// Rebuild graph
|
||||||
|
if err := s.graphSearchClient.RebuildGraph(s.ctx, observations); err != nil {
|
||||||
|
log.Error().Err(err).Msg("Failed to rebuild observation graph")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
elapsed := time.Since(start)
|
||||||
|
stats := s.graphSearchClient.GetGraphStats()
|
||||||
|
|
||||||
|
log.Info().
|
||||||
|
Int("observations", len(observations)).
|
||||||
|
Int("nodes", stats.NodeCount).
|
||||||
|
Int("edges", stats.EdgeCount).
|
||||||
|
Float64("avg_degree", stats.AvgDegree).
|
||||||
|
Dur("elapsed", elapsed).
|
||||||
|
Msg("Observation graph rebuilt successfully")
|
||||||
|
}
|
||||||
|
|
||||||
// setupMiddleware configures HTTP middleware.
|
// setupMiddleware configures HTTP middleware.
|
||||||
func (s *Service) setupMiddleware() {
|
func (s *Service) setupMiddleware() {
|
||||||
s.router.Use(middleware.Logger)
|
s.router.Use(middleware.Logger)
|
||||||
@@ -1110,6 +1263,10 @@ func (s *Service) setupRoutes() {
|
|||||||
r.Get("/api/types", s.handleGetTypes)
|
r.Get("/api/types", s.handleGetTypes)
|
||||||
r.Get("/api/models", s.handleGetModels)
|
r.Get("/api/models", s.handleGetModels)
|
||||||
|
|
||||||
|
// Graph and vector metrics routes
|
||||||
|
r.Get("/api/graph/stats", s.handleGetGraphStats)
|
||||||
|
r.Get("/api/vector/metrics", s.handleGetVectorMetrics)
|
||||||
|
|
||||||
// Observation scoring and feedback routes
|
// Observation scoring and feedback routes
|
||||||
r.Post("/api/observations/{id}/feedback", s.handleObservationFeedback)
|
r.Post("/api/observations/{id}/feedback", s.handleObservationFeedback)
|
||||||
r.Get("/api/observations/{id}/score", s.handleExplainScore)
|
r.Get("/api/observations/{id}/score", s.handleExplainScore)
|
||||||
@@ -1376,6 +1533,11 @@ func (s *Service) Shutdown(ctx context.Context) error {
|
|||||||
s.patternDetector.Stop()
|
s.patternDetector.Stop()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Stop graph rebuild ticker
|
||||||
|
if s.graphRebuildTicker != nil {
|
||||||
|
s.graphRebuildTicker.Stop()
|
||||||
|
}
|
||||||
|
|
||||||
// Shutdown all sessions
|
// Shutdown all sessions
|
||||||
s.sessionManager.ShutdownAll(ctx)
|
s.sessionManager.ShutdownAll(ctx)
|
||||||
|
|
||||||
|
|||||||
@@ -21,11 +21,11 @@ const (
|
|||||||
|
|
||||||
// ObservationData contains data for a tool observation.
|
// ObservationData contains data for a tool observation.
|
||||||
type ObservationData struct {
|
type ObservationData struct {
|
||||||
ToolName string
|
|
||||||
ToolInput interface{}
|
ToolInput interface{}
|
||||||
ToolResponse interface{}
|
ToolResponse interface{}
|
||||||
PromptNumber int
|
ToolName string
|
||||||
CWD string
|
CWD string
|
||||||
|
PromptNumber int
|
||||||
}
|
}
|
||||||
|
|
||||||
// SummarizeData contains data for a summarize request.
|
// SummarizeData contains data for a summarize request.
|
||||||
@@ -36,30 +36,28 @@ type SummarizeData struct {
|
|||||||
|
|
||||||
// PendingMessage represents a message queued for SDK processing.
|
// PendingMessage represents a message queued for SDK processing.
|
||||||
type PendingMessage struct {
|
type PendingMessage struct {
|
||||||
Type MessageType
|
|
||||||
Observation *ObservationData
|
Observation *ObservationData
|
||||||
Summarize *SummarizeData
|
Summarize *SummarizeData
|
||||||
|
Type MessageType
|
||||||
}
|
}
|
||||||
|
|
||||||
// ActiveSession represents an in-memory active session being processed.
|
// ActiveSession represents an in-memory active session being processed.
|
||||||
type ActiveSession struct {
|
type ActiveSession struct {
|
||||||
SessionDBID int64
|
StartTime time.Time
|
||||||
ClaudeSessionID string
|
ctx context.Context
|
||||||
SDKSessionID string
|
cancel context.CancelFunc
|
||||||
|
notify chan struct{}
|
||||||
Project string
|
Project string
|
||||||
UserPrompt string
|
UserPrompt string
|
||||||
|
SDKSessionID string
|
||||||
|
ClaudeSessionID string
|
||||||
|
pendingMessages []PendingMessage
|
||||||
LastPromptNumber int
|
LastPromptNumber int
|
||||||
StartTime time.Time
|
|
||||||
CumulativeInputTokens int64
|
CumulativeInputTokens int64
|
||||||
CumulativeOutputTokens int64
|
CumulativeOutputTokens int64
|
||||||
|
SessionDBID int64
|
||||||
// Concurrency control
|
messageMu sync.Mutex
|
||||||
pendingMessages []PendingMessage
|
generatorActive atomic.Bool
|
||||||
messageMu sync.Mutex
|
|
||||||
notify chan struct{}
|
|
||||||
ctx context.Context
|
|
||||||
cancel context.CancelFunc
|
|
||||||
generatorActive atomic.Bool
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// SessionTimeout is how long an inactive session can exist before cleanup.
|
// SessionTimeout is how long an inactive session can exist before cleanup.
|
||||||
@@ -70,15 +68,14 @@ const CleanupInterval = 5 * time.Minute
|
|||||||
|
|
||||||
// Manager manages active session lifecycles.
|
// Manager manages active session lifecycles.
|
||||||
type Manager struct {
|
type Manager struct {
|
||||||
sessionStore *sqlite.SessionStore
|
ctx context.Context
|
||||||
sessions map[int64]*ActiveSession
|
sessionStore *sqlite.SessionStore
|
||||||
mu sync.RWMutex
|
sessions map[int64]*ActiveSession
|
||||||
onCreated func(int64)
|
onCreated func(int64)
|
||||||
onDeleted func(int64)
|
onDeleted func(int64)
|
||||||
ctx context.Context
|
cancel context.CancelFunc
|
||||||
cancel context.CancelFunc
|
|
||||||
// Global notification channel for immediate processing
|
|
||||||
ProcessNotify chan struct{}
|
ProcessNotify chan struct{}
|
||||||
|
mu sync.RWMutex
|
||||||
}
|
}
|
||||||
|
|
||||||
// NewManager creates a new session manager.
|
// NewManager creates a new session manager.
|
||||||
|
|||||||
@@ -47,9 +47,9 @@ func (s *ManagerSuite) TestActiveSession() {
|
|||||||
SDKSessionID: "sdk-123",
|
SDKSessionID: "sdk-123",
|
||||||
Project: "test-project",
|
Project: "test-project",
|
||||||
UserPrompt: "Hello",
|
UserPrompt: "Hello",
|
||||||
StartTime: time.Now(),
|
StartTime: time.Now(), //nolint:govet
|
||||||
pendingMessages: make([]PendingMessage, 0),
|
pendingMessages: make([]PendingMessage, 0), //nolint:govet
|
||||||
notify: make(chan struct{}, 1),
|
notify: make(chan struct{}, 1), //nolint:govet
|
||||||
}
|
}
|
||||||
|
|
||||||
s.Equal(int64(1), session.SessionDBID)
|
s.Equal(int64(1), session.SessionDBID)
|
||||||
@@ -134,7 +134,7 @@ func (s *ManagerSuite) TestDeleteSession() {
|
|||||||
session := &ActiveSession{
|
session := &ActiveSession{
|
||||||
SessionDBID: 1,
|
SessionDBID: 1,
|
||||||
Project: "test-project",
|
Project: "test-project",
|
||||||
StartTime: time.Now(),
|
StartTime: time.Now(), //nolint:govet
|
||||||
pendingMessages: []PendingMessage{},
|
pendingMessages: []PendingMessage{},
|
||||||
ctx: ctx,
|
ctx: ctx,
|
||||||
cancel: cancel,
|
cancel: cancel,
|
||||||
@@ -236,8 +236,8 @@ func TestTimeoutConstants(t *testing.T) {
|
|||||||
func TestObservationData(t *testing.T) {
|
func TestObservationData(t *testing.T) {
|
||||||
data := ObservationData{
|
data := ObservationData{
|
||||||
ToolName: "Read",
|
ToolName: "Read",
|
||||||
ToolInput: map[string]string{"path": "/test/file.go"},
|
ToolInput: map[string]string{"path": "/test/file.go"}, //nolint:govet
|
||||||
ToolResponse: "file content",
|
ToolResponse: "file content", //nolint:govet
|
||||||
PromptNumber: 1,
|
PromptNumber: 1,
|
||||||
CWD: "/test",
|
CWD: "/test",
|
||||||
}
|
}
|
||||||
@@ -333,7 +333,7 @@ func TestConcurrentSessionAccess(t *testing.T) {
|
|||||||
// TestProcessNotifyChannel tests the process notification channel.
|
// TestProcessNotifyChannel tests the process notification channel.
|
||||||
func TestProcessNotifyChannel(t *testing.T) {
|
func TestProcessNotifyChannel(t *testing.T) {
|
||||||
manager := &Manager{
|
manager := &Manager{
|
||||||
sessions: make(map[int64]*ActiveSession),
|
sessions: make(map[int64]*ActiveSession), //nolint:govet
|
||||||
ProcessNotify: make(chan struct{}, 1),
|
ProcessNotify: make(chan struct{}, 1),
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -367,7 +367,7 @@ func TestActiveSessionContext(t *testing.T) {
|
|||||||
ctx, cancel := context.WithCancel(context.Background())
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
|
||||||
session := &ActiveSession{
|
session := &ActiveSession{
|
||||||
SessionDBID: 1,
|
SessionDBID: 1, //nolint:govet
|
||||||
ctx: ctx,
|
ctx: ctx,
|
||||||
cancel: cancel,
|
cancel: cancel,
|
||||||
}
|
}
|
||||||
@@ -438,7 +438,7 @@ func (s *ManagerSuite) TestShutdownAll() {
|
|||||||
s.manager.sessions[i] = &ActiveSession{
|
s.manager.sessions[i] = &ActiveSession{
|
||||||
SessionDBID: i,
|
SessionDBID: i,
|
||||||
Project: "test-project",
|
Project: "test-project",
|
||||||
StartTime: time.Now(),
|
StartTime: time.Now(), //nolint:govet
|
||||||
pendingMessages: []PendingMessage{},
|
pendingMessages: []PendingMessage{},
|
||||||
ctx: ctx,
|
ctx: ctx,
|
||||||
cancel: cancel,
|
cancel: cancel,
|
||||||
@@ -479,7 +479,7 @@ func (s *ManagerSuite) TestDeleteNonExistentSession() {
|
|||||||
// TestLastPromptNumber tests prompt number tracking.
|
// TestLastPromptNumber tests prompt number tracking.
|
||||||
func TestLastPromptNumber(t *testing.T) {
|
func TestLastPromptNumber(t *testing.T) {
|
||||||
session := &ActiveSession{
|
session := &ActiveSession{
|
||||||
SessionDBID: 1,
|
SessionDBID: 1, //nolint:govet
|
||||||
LastPromptNumber: 0,
|
LastPromptNumber: 0,
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -526,7 +526,7 @@ func TestActiveSessionNotifyChannel(t *testing.T) {
|
|||||||
// TestMessageMutex tests message mutex operations.
|
// TestMessageMutex tests message mutex operations.
|
||||||
func TestMessageMutex(t *testing.T) {
|
func TestMessageMutex(t *testing.T) {
|
||||||
session := &ActiveSession{
|
session := &ActiveSession{
|
||||||
pendingMessages: make([]PendingMessage, 0),
|
pendingMessages: make([]PendingMessage, 0), //nolint:govet
|
||||||
}
|
}
|
||||||
|
|
||||||
var wg sync.WaitGroup
|
var wg sync.WaitGroup
|
||||||
@@ -559,7 +559,7 @@ func (s *ManagerSuite) TestQueueDepthMultipleSessions() {
|
|||||||
}
|
}
|
||||||
s.manager.sessions[2] = &ActiveSession{
|
s.manager.sessions[2] = &ActiveSession{
|
||||||
SessionDBID: 2,
|
SessionDBID: 2,
|
||||||
pendingMessages: make([]PendingMessage, 0),
|
pendingMessages: make([]PendingMessage, 0), //nolint:govet
|
||||||
}
|
}
|
||||||
s.manager.sessions[3] = &ActiveSession{
|
s.manager.sessions[3] = &ActiveSession{
|
||||||
SessionDBID: 3,
|
SessionDBID: 3,
|
||||||
@@ -658,7 +658,7 @@ func TestActiveSessionCWD(t *testing.T) {
|
|||||||
for _, tt := range tests {
|
for _, tt := range tests {
|
||||||
t.Run(tt.name, func(t *testing.T) {
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
data := ObservationData{
|
data := ObservationData{
|
||||||
ToolName: "Test",
|
ToolName: "Test", //nolint:govet
|
||||||
CWD: tt.cwd,
|
CWD: tt.cwd,
|
||||||
}
|
}
|
||||||
assert.Equal(t, tt.cwd, data.CWD)
|
assert.Equal(t, tt.cwd, data.CWD)
|
||||||
@@ -669,22 +669,22 @@ func TestActiveSessionCWD(t *testing.T) {
|
|||||||
// TestToolInputResponse tests various tool input/response types.
|
// TestToolInputResponse tests various tool input/response types.
|
||||||
func TestToolInputResponse(t *testing.T) {
|
func TestToolInputResponse(t *testing.T) {
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
name string
|
|
||||||
input interface{}
|
input interface{}
|
||||||
response interface{}
|
response interface{}
|
||||||
|
name string
|
||||||
}{
|
}{
|
||||||
{"nil_values", nil, nil},
|
{name: "nil_values", input: nil, response: nil},
|
||||||
{"string_values", "input string", "response string"},
|
{name: "string_values", input: "input string", response: "response string"},
|
||||||
{"map_values", map[string]string{"key": "value"}, map[string]interface{}{"result": true}},
|
{name: "map_values", input: map[string]string{"key": "value"}, response: map[string]interface{}{"result": true}},
|
||||||
{"slice_values", []string{"a", "b"}, []int{1, 2, 3}},
|
{name: "slice_values", input: []string{"a", "b"}, response: []int{1, 2, 3}},
|
||||||
{"int_values", 42, 100},
|
{name: "int_values", input: 42, response: 100},
|
||||||
{"bool_values", true, false},
|
{name: "bool_values", input: true, response: false},
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, tt := range tests {
|
for _, tt := range tests {
|
||||||
t.Run(tt.name, func(t *testing.T) {
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
data := ObservationData{
|
data := ObservationData{
|
||||||
ToolName: "TestTool",
|
ToolName: "TestTool", //nolint:govet
|
||||||
ToolInput: tt.input,
|
ToolInput: tt.input,
|
||||||
ToolResponse: tt.response,
|
ToolResponse: tt.response,
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -19,10 +19,10 @@ const (
|
|||||||
|
|
||||||
// Client represents a connected SSE client.
|
// Client represents a connected SSE client.
|
||||||
type Client struct {
|
type Client struct {
|
||||||
ID string
|
|
||||||
Writer http.ResponseWriter
|
Writer http.ResponseWriter
|
||||||
Flusher http.Flusher
|
Flusher http.Flusher
|
||||||
Done chan struct{}
|
Done chan struct{}
|
||||||
|
ID string
|
||||||
}
|
}
|
||||||
|
|
||||||
// Broadcaster manages SSE client connections and message broadcasting.
|
// Broadcaster manages SSE client connections and message broadcasting.
|
||||||
|
|||||||
@@ -256,8 +256,8 @@ func TestHandleSSE(t *testing.T) {
|
|||||||
// TestBroadcastJSON tests broadcasting various JSON types.
|
// TestBroadcastJSON tests broadcasting various JSON types.
|
||||||
func TestBroadcastJSON(t *testing.T) {
|
func TestBroadcastJSON(t *testing.T) {
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
name string
|
|
||||||
data interface{}
|
data interface{}
|
||||||
|
name string
|
||||||
wantErr bool
|
wantErr bool
|
||||||
}{
|
}{
|
||||||
{
|
{
|
||||||
|
|||||||
@@ -62,11 +62,11 @@ type BaseInput struct {
|
|||||||
// HookContext provides common context for hook handlers.
|
// HookContext provides common context for hook handlers.
|
||||||
type HookContext struct {
|
type HookContext struct {
|
||||||
HookName string
|
HookName string
|
||||||
Port int
|
|
||||||
Project string
|
Project string
|
||||||
SessionID string
|
SessionID string
|
||||||
CWD string
|
CWD string
|
||||||
RawInput []byte
|
RawInput []byte
|
||||||
|
Port int
|
||||||
}
|
}
|
||||||
|
|
||||||
// HookHandler is a function that handles hook-specific logic.
|
// HookHandler is a function that handles hook-specific logic.
|
||||||
@@ -92,8 +92,8 @@ func RunHook[T any](hookName string, handler HookHandler[T]) {
|
|||||||
|
|
||||||
// Parse input
|
// Parse input
|
||||||
var input T
|
var input T
|
||||||
if err := json.Unmarshal(inputData, &input); err != nil {
|
if unmarshalErr := json.Unmarshal(inputData, &input); unmarshalErr != nil {
|
||||||
WriteError(hookName, err)
|
WriteError(hookName, unmarshalErr)
|
||||||
os.Exit(1)
|
os.Exit(1)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -320,11 +320,11 @@ func TestExtractBaseVersion(t *testing.T) {
|
|||||||
// TestPOST tests the POST function with a mock server.
|
// TestPOST tests the POST function with a mock server.
|
||||||
func TestPOST(t *testing.T) {
|
func TestPOST(t *testing.T) {
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
name string
|
|
||||||
serverHandler func(w http.ResponseWriter, r *http.Request)
|
|
||||||
body interface{}
|
body interface{}
|
||||||
expectError bool
|
serverHandler func(w http.ResponseWriter, r *http.Request)
|
||||||
expectedResult map[string]interface{}
|
expectedResult map[string]interface{}
|
||||||
|
name string
|
||||||
|
expectError bool
|
||||||
}{
|
}{
|
||||||
{
|
{
|
||||||
name: "successful POST with JSON response",
|
name: "successful POST with JSON response",
|
||||||
@@ -393,10 +393,10 @@ func TestPOST(t *testing.T) {
|
|||||||
// TestGET tests the GET function with a mock server.
|
// TestGET tests the GET function with a mock server.
|
||||||
func TestGET(t *testing.T) {
|
func TestGET(t *testing.T) {
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
name string
|
|
||||||
serverHandler func(w http.ResponseWriter, r *http.Request)
|
serverHandler func(w http.ResponseWriter, r *http.Request)
|
||||||
expectError bool
|
|
||||||
expectedResult map[string]interface{}
|
expectedResult map[string]interface{}
|
||||||
|
name string
|
||||||
|
expectError bool
|
||||||
}{
|
}{
|
||||||
{
|
{
|
||||||
name: "successful GET with JSON response",
|
name: "successful GET with JSON response",
|
||||||
@@ -532,8 +532,8 @@ func TestExitCodes(t *testing.T) {
|
|||||||
func TestHookResponse(t *testing.T) {
|
func TestHookResponse(t *testing.T) {
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
name string
|
name string
|
||||||
response HookResponse
|
|
||||||
expected string
|
expected string
|
||||||
|
response HookResponse
|
||||||
}{
|
}{
|
||||||
{
|
{
|
||||||
name: "continue true",
|
name: "continue true",
|
||||||
@@ -597,8 +597,8 @@ func TestHookContext(t *testing.T) {
|
|||||||
// TestIsWorkerRunning_WithServer tests IsWorkerRunning with actual server.
|
// TestIsWorkerRunning_WithServer tests IsWorkerRunning with actual server.
|
||||||
func TestIsWorkerRunning_WithServer(t *testing.T) {
|
func TestIsWorkerRunning_WithServer(t *testing.T) {
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
name string
|
|
||||||
serverHandler func(w http.ResponseWriter, r *http.Request)
|
serverHandler func(w http.ResponseWriter, r *http.Request)
|
||||||
|
name string
|
||||||
expectedResult bool
|
expectedResult bool
|
||||||
}{
|
}{
|
||||||
{
|
{
|
||||||
@@ -828,8 +828,8 @@ func TestBaseInput_PartialFields(t *testing.T) {
|
|||||||
func TestHookResponse_Marshal(t *testing.T) {
|
func TestHookResponse_Marshal(t *testing.T) {
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
name string
|
name string
|
||||||
response HookResponse
|
|
||||||
contains []string
|
contains []string
|
||||||
|
response HookResponse
|
||||||
}{
|
}{
|
||||||
{
|
{
|
||||||
name: "continue true",
|
name: "continue true",
|
||||||
@@ -1177,7 +1177,7 @@ func TestHookContext_RawInput(t *testing.T) {
|
|||||||
for _, tt := range tests {
|
for _, tt := range tests {
|
||||||
t.Run(tt.name, func(t *testing.T) {
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
ctx := HookContext{
|
ctx := HookContext{
|
||||||
HookName: "test",
|
HookName: "test", //nolint:govet
|
||||||
RawInput: tt.rawInput,
|
RawInput: tt.rawInput,
|
||||||
}
|
}
|
||||||
assert.Equal(t, tt.rawInput, ctx.RawInput)
|
assert.Equal(t, tt.rawInput, ctx.RawInput)
|
||||||
|
|||||||
@@ -33,25 +33,25 @@ const (
|
|||||||
|
|
||||||
// ObservationConflict tracks conflicting observations.
|
// ObservationConflict tracks conflicting observations.
|
||||||
type ObservationConflict struct {
|
type ObservationConflict struct {
|
||||||
ID int64 `db:"id" json:"id"`
|
ResolvedAt *string `db:"resolved_at" json:"resolved_at,omitempty"`
|
||||||
NewerObsID int64 `db:"newer_obs_id" json:"newer_obs_id"`
|
|
||||||
OlderObsID int64 `db:"older_obs_id" json:"older_obs_id"`
|
|
||||||
ConflictType ConflictType `db:"conflict_type" json:"conflict_type"`
|
ConflictType ConflictType `db:"conflict_type" json:"conflict_type"`
|
||||||
Resolution ConflictResolution `db:"resolution" json:"resolution"`
|
Resolution ConflictResolution `db:"resolution" json:"resolution"`
|
||||||
Reason string `db:"reason" json:"reason"`
|
Reason string `db:"reason" json:"reason"`
|
||||||
DetectedAt string `db:"detected_at" json:"detected_at"`
|
DetectedAt string `db:"detected_at" json:"detected_at"`
|
||||||
|
ID int64 `db:"id" json:"id"`
|
||||||
|
NewerObsID int64 `db:"newer_obs_id" json:"newer_obs_id"`
|
||||||
|
OlderObsID int64 `db:"older_obs_id" json:"older_obs_id"`
|
||||||
DetectedAtEpoch int64 `db:"detected_at_epoch" json:"detected_at_epoch"`
|
DetectedAtEpoch int64 `db:"detected_at_epoch" json:"detected_at_epoch"`
|
||||||
Resolved bool `db:"resolved" json:"resolved"`
|
Resolved bool `db:"resolved" json:"resolved"`
|
||||||
ResolvedAt *string `db:"resolved_at" json:"resolved_at,omitempty"`
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// ConflictDetectionResult contains the result of conflict detection.
|
// ConflictDetectionResult contains the result of conflict detection.
|
||||||
type ConflictDetectionResult struct {
|
type ConflictDetectionResult struct {
|
||||||
HasConflict bool
|
|
||||||
Type ConflictType
|
Type ConflictType
|
||||||
Resolution ConflictResolution
|
Resolution ConflictResolution
|
||||||
Reason string
|
Reason string
|
||||||
OlderObsIDs []int64 // IDs of observations that conflict with the new one
|
OlderObsIDs []int64
|
||||||
|
HasConflict bool
|
||||||
}
|
}
|
||||||
|
|
||||||
// NewObservationConflict creates a new conflict record.
|
// NewObservationConflict creates a new conflict record.
|
||||||
|
|||||||
@@ -51,8 +51,8 @@ func (s *ConflictSuite) TestDetectExplicitCorrection_TableDriven() {
|
|||||||
tests := []struct {
|
tests := []struct {
|
||||||
name string
|
name string
|
||||||
text string
|
text string
|
||||||
expectMatch bool
|
|
||||||
expectPattern string
|
expectPattern string
|
||||||
|
expectMatch bool
|
||||||
}{
|
}{
|
||||||
{
|
{
|
||||||
name: "actually that was wrong",
|
name: "actually that was wrong",
|
||||||
@@ -128,9 +128,9 @@ func (s *ConflictSuite) TestDetectExplicitCorrection_TableDriven() {
|
|||||||
// TestDetectOpposingFileChanges_TableDriven tests opposing file change detection.
|
// TestDetectOpposingFileChanges_TableDriven tests opposing file change detection.
|
||||||
func (s *ConflictSuite) TestDetectOpposingFileChanges_TableDriven() {
|
func (s *ConflictSuite) TestDetectOpposingFileChanges_TableDriven() {
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
name string
|
|
||||||
newerObs *Observation
|
newerObs *Observation
|
||||||
olderObs *Observation
|
olderObs *Observation
|
||||||
|
name string
|
||||||
expectConflict bool
|
expectConflict bool
|
||||||
}{
|
}{
|
||||||
{
|
{
|
||||||
@@ -202,9 +202,9 @@ func (s *ConflictSuite) TestDetectOpposingFileChanges_TableDriven() {
|
|||||||
// TestDetectConceptTagMismatch_TableDriven tests concept tag mismatch detection.
|
// TestDetectConceptTagMismatch_TableDriven tests concept tag mismatch detection.
|
||||||
func (s *ConflictSuite) TestDetectConceptTagMismatch_TableDriven() {
|
func (s *ConflictSuite) TestDetectConceptTagMismatch_TableDriven() {
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
name string
|
|
||||||
newerObs *Observation
|
newerObs *Observation
|
||||||
olderObs *Observation
|
olderObs *Observation
|
||||||
|
name string
|
||||||
expectConflict bool
|
expectConflict bool
|
||||||
}{
|
}{
|
||||||
{
|
{
|
||||||
@@ -406,9 +406,9 @@ func TestObservationConflict_Fields(t *testing.T) {
|
|||||||
OlderObsID: 5,
|
OlderObsID: 5,
|
||||||
ConflictType: ConflictSuperseded,
|
ConflictType: ConflictSuperseded,
|
||||||
Resolution: ResolutionPreferNewer,
|
Resolution: ResolutionPreferNewer,
|
||||||
Reason: "Test reason",
|
Reason: "Test reason", //nolint:govet
|
||||||
DetectedAt: "2024-01-01T00:00:00Z",
|
DetectedAt: "2024-01-01T00:00:00Z", //nolint:govet
|
||||||
DetectedAtEpoch: 1704067200000,
|
DetectedAtEpoch: 1704067200000, //nolint:govet
|
||||||
Resolved: false,
|
Resolved: false,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
+28
-36
@@ -121,48 +121,44 @@ func (j JSONInt64Map) Value() (driver.Value, error) {
|
|||||||
|
|
||||||
// Observation represents a learning extracted from a Claude Code session.
|
// Observation represents a learning extracted from a Claude Code session.
|
||||||
type Observation struct {
|
type Observation struct {
|
||||||
ID int64 `db:"id" json:"id"`
|
FileMtimes JSONInt64Map `db:"file_mtimes" json:"file_mtimes,omitempty"`
|
||||||
SDKSessionID string `db:"sdk_session_id" json:"sdk_session_id"`
|
SDKSessionID string `db:"sdk_session_id" json:"sdk_session_id"`
|
||||||
Project string `db:"project" json:"project"`
|
Project string `db:"project" json:"project"`
|
||||||
Scope ObservationScope `db:"scope" json:"scope"`
|
Scope ObservationScope `db:"scope" json:"scope"`
|
||||||
Type ObservationType `db:"type" json:"type"`
|
Type ObservationType `db:"type" json:"type"`
|
||||||
Title sql.NullString `db:"title" json:"title,omitempty"`
|
CreatedAt string `db:"created_at" json:"created_at"`
|
||||||
Subtitle sql.NullString `db:"subtitle" json:"subtitle,omitempty"`
|
Subtitle sql.NullString `db:"subtitle" json:"subtitle,omitempty"`
|
||||||
Facts JSONStringArray `db:"facts" json:"facts,omitempty"`
|
Title sql.NullString `db:"title" json:"title,omitempty"`
|
||||||
Narrative sql.NullString `db:"narrative" json:"narrative,omitempty"`
|
Narrative sql.NullString `db:"narrative" json:"narrative,omitempty"`
|
||||||
Concepts JSONStringArray `db:"concepts" json:"concepts,omitempty"`
|
Concepts JSONStringArray `db:"concepts" json:"concepts,omitempty"`
|
||||||
FilesRead JSONStringArray `db:"files_read" json:"files_read,omitempty"`
|
FilesRead JSONStringArray `db:"files_read" json:"files_read,omitempty"`
|
||||||
FilesModified JSONStringArray `db:"files_modified" json:"files_modified,omitempty"`
|
FilesModified JSONStringArray `db:"files_modified" json:"files_modified,omitempty"`
|
||||||
FileMtimes JSONInt64Map `db:"file_mtimes" json:"file_mtimes,omitempty"`
|
Facts JSONStringArray `db:"facts" json:"facts,omitempty"`
|
||||||
PromptNumber sql.NullInt64 `db:"prompt_number" json:"prompt_number,omitempty"`
|
PromptNumber sql.NullInt64 `db:"prompt_number" json:"prompt_number,omitempty"`
|
||||||
|
LastRetrievedAt sql.NullInt64 `db:"last_retrieved_at_epoch" json:"last_retrieved_at_epoch,omitempty"`
|
||||||
|
ScoreUpdatedAt sql.NullInt64 `db:"score_updated_at_epoch" json:"score_updated_at_epoch,omitempty"`
|
||||||
DiscoveryTokens int64 `db:"discovery_tokens" json:"discovery_tokens"`
|
DiscoveryTokens int64 `db:"discovery_tokens" json:"discovery_tokens"`
|
||||||
CreatedAt string `db:"created_at" json:"created_at"`
|
ID int64 `db:"id" json:"id"`
|
||||||
CreatedAtEpoch int64 `db:"created_at_epoch" json:"created_at_epoch"`
|
CreatedAtEpoch int64 `db:"created_at_epoch" json:"created_at_epoch"`
|
||||||
|
ImportanceScore float64 `db:"importance_score" json:"importance_score"`
|
||||||
|
UserFeedback int `db:"user_feedback" json:"user_feedback"`
|
||||||
|
RetrievalCount int `db:"retrieval_count" json:"retrieval_count"`
|
||||||
IsStale bool `db:"-" json:"is_stale,omitempty"`
|
IsStale bool `db:"-" json:"is_stale,omitempty"`
|
||||||
|
IsSuperseded bool `db:"is_superseded" json:"is_superseded,omitempty"`
|
||||||
// Importance scoring fields
|
|
||||||
ImportanceScore float64 `db:"importance_score" json:"importance_score"`
|
|
||||||
UserFeedback int `db:"user_feedback" json:"user_feedback"`
|
|
||||||
RetrievalCount int `db:"retrieval_count" json:"retrieval_count"`
|
|
||||||
LastRetrievedAt sql.NullInt64 `db:"last_retrieved_at_epoch" json:"last_retrieved_at_epoch,omitempty"`
|
|
||||||
ScoreUpdatedAt sql.NullInt64 `db:"score_updated_at_epoch" json:"score_updated_at_epoch,omitempty"`
|
|
||||||
|
|
||||||
// Conflict detection fields
|
|
||||||
IsSuperseded bool `db:"is_superseded" json:"is_superseded,omitempty"`
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// ParsedObservation represents an observation parsed from SDK response XML.
|
// ParsedObservation represents an observation parsed from SDK response XML.
|
||||||
type ParsedObservation struct {
|
type ParsedObservation struct {
|
||||||
|
FileMtimes map[string]int64
|
||||||
Type ObservationType
|
Type ObservationType
|
||||||
Title string
|
Title string
|
||||||
Subtitle string
|
Subtitle string
|
||||||
Facts []string
|
|
||||||
Narrative string
|
Narrative string
|
||||||
|
Scope ObservationScope
|
||||||
|
Facts []string
|
||||||
Concepts []string
|
Concepts []string
|
||||||
FilesRead []string
|
FilesRead []string
|
||||||
FilesModified []string
|
FilesModified []string
|
||||||
FileMtimes map[string]int64 // File path -> mtime epoch ms
|
|
||||||
Scope ObservationScope // Optional: if empty, will be auto-determined
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// ToStoredObservation converts a ParsedObservation to the stored Observation format.
|
// ToStoredObservation converts a ParsedObservation to the stored Observation format.
|
||||||
@@ -197,34 +193,30 @@ func DetermineScope(concepts []string) ObservationScope {
|
|||||||
// ObservationJSON is a JSON-friendly representation of Observation.
|
// ObservationJSON is a JSON-friendly representation of Observation.
|
||||||
// It converts sql.NullString to plain strings for clean JSON output.
|
// It converts sql.NullString to plain strings for clean JSON output.
|
||||||
type ObservationJSON struct {
|
type ObservationJSON struct {
|
||||||
ID int64 `json:"id"`
|
FileMtimes map[string]int64 `json:"file_mtimes,omitempty"`
|
||||||
|
Subtitle string `json:"subtitle,omitempty"`
|
||||||
SDKSessionID string `json:"sdk_session_id"`
|
SDKSessionID string `json:"sdk_session_id"`
|
||||||
Project string `json:"project"`
|
|
||||||
Scope ObservationScope `json:"scope"`
|
Scope ObservationScope `json:"scope"`
|
||||||
Type ObservationType `json:"type"`
|
Type ObservationType `json:"type"`
|
||||||
Title string `json:"title,omitempty"`
|
Title string `json:"title,omitempty"`
|
||||||
Subtitle string `json:"subtitle,omitempty"`
|
CreatedAt string `json:"created_at"`
|
||||||
Facts []string `json:"facts,omitempty"`
|
|
||||||
Narrative string `json:"narrative,omitempty"`
|
Narrative string `json:"narrative,omitempty"`
|
||||||
|
Project string `json:"project"`
|
||||||
Concepts []string `json:"concepts,omitempty"`
|
Concepts []string `json:"concepts,omitempty"`
|
||||||
|
Facts []string `json:"facts,omitempty"`
|
||||||
FilesRead []string `json:"files_read,omitempty"`
|
FilesRead []string `json:"files_read,omitempty"`
|
||||||
FilesModified []string `json:"files_modified,omitempty"`
|
FilesModified []string `json:"files_modified,omitempty"`
|
||||||
FileMtimes map[string]int64 `json:"file_mtimes,omitempty"`
|
|
||||||
PromptNumber int64 `json:"prompt_number,omitempty"`
|
|
||||||
DiscoveryTokens int64 `json:"discovery_tokens"`
|
|
||||||
CreatedAt string `json:"created_at"`
|
|
||||||
CreatedAtEpoch int64 `json:"created_at_epoch"`
|
CreatedAtEpoch int64 `json:"created_at_epoch"`
|
||||||
|
DiscoveryTokens int64 `json:"discovery_tokens"`
|
||||||
|
ID int64 `json:"id"`
|
||||||
|
PromptNumber int64 `json:"prompt_number,omitempty"`
|
||||||
|
ImportanceScore float64 `json:"importance_score"`
|
||||||
|
UserFeedback int `json:"user_feedback"`
|
||||||
|
RetrievalCount int `json:"retrieval_count"`
|
||||||
|
LastRetrievedAt int64 `json:"last_retrieved_at_epoch,omitempty"`
|
||||||
|
ScoreUpdatedAt int64 `json:"score_updated_at_epoch,omitempty"`
|
||||||
IsStale bool `json:"is_stale,omitempty"`
|
IsStale bool `json:"is_stale,omitempty"`
|
||||||
|
IsSuperseded bool `json:"is_superseded,omitempty"`
|
||||||
// Importance scoring fields
|
|
||||||
ImportanceScore float64 `json:"importance_score"`
|
|
||||||
UserFeedback int `json:"user_feedback"`
|
|
||||||
RetrievalCount int `json:"retrieval_count"`
|
|
||||||
LastRetrievedAt int64 `json:"last_retrieved_at_epoch,omitempty"`
|
|
||||||
ScoreUpdatedAt int64 `json:"score_updated_at_epoch,omitempty"`
|
|
||||||
|
|
||||||
// Conflict detection fields
|
|
||||||
IsSuperseded bool `json:"is_superseded,omitempty"`
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// MarshalJSON implements json.Marshaler for Observation.
|
// MarshalJSON implements json.Marshaler for Observation.
|
||||||
|
|||||||
@@ -50,8 +50,8 @@ func (s *ObservationSuite) TestGlobalizableConcepts() {
|
|||||||
func (s *ObservationSuite) TestDetermineScope_TableDriven() {
|
func (s *ObservationSuite) TestDetermineScope_TableDriven() {
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
name string
|
name string
|
||||||
concepts []string
|
|
||||||
expected ObservationScope
|
expected ObservationScope
|
||||||
|
concepts []string
|
||||||
}{
|
}{
|
||||||
{
|
{
|
||||||
name: "empty concepts - project scope",
|
name: "empty concepts - project scope",
|
||||||
@@ -106,8 +106,8 @@ func (s *ObservationSuite) TestDetermineScope_TableDriven() {
|
|||||||
// TestParsedObservation_FileMtimesJSON tests FileMtimes JSON serialization.
|
// TestParsedObservation_FileMtimesJSON tests FileMtimes JSON serialization.
|
||||||
func (s *ObservationSuite) TestParsedObservation_FileMtimesJSON() {
|
func (s *ObservationSuite) TestParsedObservation_FileMtimesJSON() {
|
||||||
obs := &ParsedObservation{
|
obs := &ParsedObservation{
|
||||||
Type: ObsTypeDiscovery,
|
Type: ObsTypeDiscovery, //nolint:govet
|
||||||
Title: "Test",
|
Title: "Test", //nolint:govet
|
||||||
FileMtimes: map[string]int64{"file1.go": 1234567890, "file2.go": 1234567891},
|
FileMtimes: map[string]int64{"file1.go": 1234567890, "file2.go": 1234567891},
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -121,9 +121,9 @@ func (s *ObservationSuite) TestParsedObservation_FileMtimesJSON() {
|
|||||||
// TestObservation_CheckStaleness_TableDriven tests staleness checking.
|
// TestObservation_CheckStaleness_TableDriven tests staleness checking.
|
||||||
func (s *ObservationSuite) TestObservation_CheckStaleness_TableDriven() {
|
func (s *ObservationSuite) TestObservation_CheckStaleness_TableDriven() {
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
name string
|
|
||||||
storedMtimes map[string]int64
|
storedMtimes map[string]int64
|
||||||
currentMtimes map[string]int64
|
currentMtimes map[string]int64
|
||||||
|
name string
|
||||||
expectedStale bool
|
expectedStale bool
|
||||||
}{
|
}{
|
||||||
{
|
{
|
||||||
@@ -221,9 +221,9 @@ func (s *ObservationSuite) TestParsedObservation_Fields() {
|
|||||||
func (s *ObservationSuite) TestObservation_NullFields() {
|
func (s *ObservationSuite) TestObservation_NullFields() {
|
||||||
// Test with null fields
|
// Test with null fields
|
||||||
obs := &Observation{
|
obs := &Observation{
|
||||||
ID: 1,
|
ID: 1, //nolint:govet
|
||||||
Project: "test",
|
Project: "test", //nolint:govet
|
||||||
Type: ObsTypeDiscovery,
|
Type: ObsTypeDiscovery, //nolint:govet
|
||||||
Title: sql.NullString{Valid: false},
|
Title: sql.NullString{Valid: false},
|
||||||
Subtitle: sql.NullString{Valid: false},
|
Subtitle: sql.NullString{Valid: false},
|
||||||
Narrative: sql.NullString{Valid: false},
|
Narrative: sql.NullString{Valid: false},
|
||||||
@@ -235,9 +235,9 @@ func (s *ObservationSuite) TestObservation_NullFields() {
|
|||||||
|
|
||||||
// Test with valid fields
|
// Test with valid fields
|
||||||
obs2 := &Observation{
|
obs2 := &Observation{
|
||||||
ID: 2,
|
ID: 2, //nolint:govet
|
||||||
Project: "test",
|
Project: "test", //nolint:govet
|
||||||
Type: ObsTypeBugfix,
|
Type: ObsTypeBugfix, //nolint:govet
|
||||||
Title: sql.NullString{String: "Fix bug", Valid: true},
|
Title: sql.NullString{String: "Fix bug", Valid: true},
|
||||||
Subtitle: sql.NullString{String: "Memory leak", Valid: true},
|
Subtitle: sql.NullString{String: "Memory leak", Valid: true},
|
||||||
Narrative: sql.NullString{String: "Fixed memory leak in handler", Valid: true},
|
Narrative: sql.NullString{String: "Fixed memory leak in handler", Valid: true},
|
||||||
@@ -300,10 +300,10 @@ func TestParsedObservation_ToStoredObservation(t *testing.T) {
|
|||||||
// TestJSONStringArray tests JSONStringArray scanning.
|
// TestJSONStringArray tests JSONStringArray scanning.
|
||||||
func TestJSONStringArray(t *testing.T) {
|
func TestJSONStringArray(t *testing.T) {
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
name string
|
|
||||||
input interface{}
|
input interface{}
|
||||||
wantErr bool
|
name string
|
||||||
expected JSONStringArray
|
expected JSONStringArray
|
||||||
|
wantErr bool
|
||||||
}{
|
}{
|
||||||
{
|
{
|
||||||
name: "nil input",
|
name: "nil input",
|
||||||
@@ -348,10 +348,10 @@ func TestJSONStringArray(t *testing.T) {
|
|||||||
// TestJSONInt64Map tests JSONInt64Map scanning.
|
// TestJSONInt64Map tests JSONInt64Map scanning.
|
||||||
func TestJSONInt64Map(t *testing.T) {
|
func TestJSONInt64Map(t *testing.T) {
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
name string
|
|
||||||
input interface{}
|
input interface{}
|
||||||
wantErr bool
|
|
||||||
expected JSONInt64Map
|
expected JSONInt64Map
|
||||||
|
name string
|
||||||
|
wantErr bool
|
||||||
}{
|
}{
|
||||||
{
|
{
|
||||||
name: "nil input",
|
name: "nil input",
|
||||||
|
|||||||
+25
-25
@@ -39,21 +39,21 @@ const (
|
|||||||
// Pattern represents a recurring pattern detected across observations.
|
// Pattern represents a recurring pattern detected across observations.
|
||||||
// This enables Claude to reference historical insights: "I've encountered this pattern 12 times."
|
// This enables Claude to reference historical insights: "I've encountered this pattern 12 times."
|
||||||
type Pattern struct {
|
type Pattern struct {
|
||||||
ID int64 `db:"id" json:"id"`
|
Status PatternStatus `db:"status" json:"status"`
|
||||||
Name string `db:"name" json:"name"` // e.g., "State Management Anti-Pattern"
|
Name string `db:"name" json:"name"`
|
||||||
Type PatternType `db:"type" json:"type"` // bug, refactor, architecture, etc.
|
Type PatternType `db:"type" json:"type"`
|
||||||
Description sql.NullString `db:"description" json:"description"` // Detailed description
|
|
||||||
Signature JSONStringArray `db:"signature" json:"signature"` // Keyword clusters for detection
|
|
||||||
Recommendation sql.NullString `db:"recommendation" json:"recommendation"` // What works for this pattern
|
|
||||||
Frequency int `db:"frequency" json:"frequency"` // How many times encountered
|
|
||||||
Projects JSONStringArray `db:"projects" json:"projects"` // Projects where this pattern was seen
|
|
||||||
ObservationIDs JSONInt64Array `db:"observation_ids" json:"observation_ids"` // Source observation IDs
|
|
||||||
Status PatternStatus `db:"status" json:"status"` // active, deprecated, merged
|
|
||||||
MergedIntoID sql.NullInt64 `db:"merged_into_id" json:"merged_into_id,omitempty"`
|
|
||||||
Confidence float64 `db:"confidence" json:"confidence"` // Detection confidence (0.0-1.0)
|
|
||||||
LastSeenAt string `db:"last_seen_at" json:"last_seen_at"` // Last time pattern was detected
|
|
||||||
LastSeenEpoch int64 `db:"last_seen_at_epoch" json:"last_seen_at_epoch"`
|
|
||||||
CreatedAt string `db:"created_at" json:"created_at"`
|
CreatedAt string `db:"created_at" json:"created_at"`
|
||||||
|
LastSeenAt string `db:"last_seen_at" json:"last_seen_at"`
|
||||||
|
Signature JSONStringArray `db:"signature" json:"signature"`
|
||||||
|
Projects JSONStringArray `db:"projects" json:"projects"`
|
||||||
|
ObservationIDs JSONInt64Array `db:"observation_ids" json:"observation_ids"`
|
||||||
|
Recommendation sql.NullString `db:"recommendation" json:"recommendation"`
|
||||||
|
Description sql.NullString `db:"description" json:"description"`
|
||||||
|
MergedIntoID sql.NullInt64 `db:"merged_into_id" json:"merged_into_id,omitempty"`
|
||||||
|
Frequency int `db:"frequency" json:"frequency"`
|
||||||
|
Confidence float64 `db:"confidence" json:"confidence"`
|
||||||
|
ID int64 `db:"id" json:"id"`
|
||||||
|
LastSeenEpoch int64 `db:"last_seen_at_epoch" json:"last_seen_at_epoch"`
|
||||||
CreatedAtEpoch int64 `db:"created_at_epoch" json:"created_at_epoch"`
|
CreatedAtEpoch int64 `db:"created_at_epoch" json:"created_at_epoch"`
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -95,21 +95,21 @@ func (j JSONInt64Array) Value() (driver.Value, error) {
|
|||||||
|
|
||||||
// PatternJSON is a JSON-friendly representation of Pattern.
|
// PatternJSON is a JSON-friendly representation of Pattern.
|
||||||
type PatternJSON struct {
|
type PatternJSON struct {
|
||||||
ID int64 `json:"id"`
|
Status PatternStatus `json:"status"`
|
||||||
Name string `json:"name"`
|
Name string `json:"name"`
|
||||||
Type PatternType `json:"type"`
|
Type PatternType `json:"type"`
|
||||||
Description string `json:"description,omitempty"`
|
Description string `json:"description,omitempty"`
|
||||||
Signature []string `json:"signature,omitempty"`
|
CreatedAt string `json:"created_at"`
|
||||||
Recommendation string `json:"recommendation,omitempty"`
|
Recommendation string `json:"recommendation,omitempty"`
|
||||||
Frequency int `json:"frequency"`
|
LastSeenAt string `json:"last_seen_at"`
|
||||||
Projects []string `json:"projects,omitempty"`
|
Signature []string `json:"signature,omitempty"`
|
||||||
ObservationIDs []int64 `json:"observation_ids,omitempty"`
|
ObservationIDs []int64 `json:"observation_ids,omitempty"`
|
||||||
Status PatternStatus `json:"status"`
|
Projects []string `json:"projects,omitempty"`
|
||||||
MergedIntoID int64 `json:"merged_into_id,omitempty"`
|
MergedIntoID int64 `json:"merged_into_id,omitempty"`
|
||||||
Confidence float64 `json:"confidence"`
|
Confidence float64 `json:"confidence"`
|
||||||
LastSeenAt string `json:"last_seen_at"`
|
Frequency int `json:"frequency"`
|
||||||
LastSeenEpoch int64 `json:"last_seen_at_epoch"`
|
LastSeenEpoch int64 `json:"last_seen_at_epoch"`
|
||||||
CreatedAt string `json:"created_at"`
|
ID int64 `json:"id"`
|
||||||
CreatedAtEpoch int64 `json:"created_at_epoch"`
|
CreatedAtEpoch int64 `json:"created_at_epoch"`
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -214,11 +214,11 @@ func (p *Pattern) updateConfidence() {
|
|||||||
|
|
||||||
// PatternMatch represents a match between an observation and a potential pattern.
|
// PatternMatch represents a match between an observation and a potential pattern.
|
||||||
type PatternMatch struct {
|
type PatternMatch struct {
|
||||||
PatternID int64 `json:"pattern_id"`
|
MatchedOn string `json:"matched_on"`
|
||||||
Score float64 `json:"score"` // Match score (0.0-1.0)
|
|
||||||
MatchedOn string `json:"matched_on"` // What triggered the match (concept, keyword, type, etc.)
|
|
||||||
IsNew bool `json:"is_new"` // Whether this would create a new pattern
|
|
||||||
SuggestedName string `json:"suggested_name,omitempty"`
|
SuggestedName string `json:"suggested_name,omitempty"`
|
||||||
|
PatternID int64 `json:"pattern_id"`
|
||||||
|
Score float64 `json:"score"`
|
||||||
|
IsNew bool `json:"is_new"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// PatternSignatureKeywords are common keywords used in pattern detection.
|
// PatternSignatureKeywords are common keywords used in pattern detection.
|
||||||
|
|||||||
@@ -116,18 +116,18 @@ func TestPattern_ConfidenceCalculation(t *testing.T) {
|
|||||||
|
|
||||||
func TestPatternType_Detection(t *testing.T) {
|
func TestPatternType_Detection(t *testing.T) {
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
concepts []string
|
|
||||||
title string
|
title string
|
||||||
narrative string
|
narrative string
|
||||||
expected PatternType
|
expected PatternType
|
||||||
|
concepts []string
|
||||||
}{
|
}{
|
||||||
{[]string{"anti-pattern"}, "", "", PatternTypeAntiPattern},
|
{concepts: []string{"anti-pattern"}, title: "", narrative: "", expected: PatternTypeAntiPattern},
|
||||||
{[]string{"best-practice"}, "", "", PatternTypeBestPractice},
|
{concepts: []string{"best-practice"}, title: "", narrative: "", expected: PatternTypeBestPractice},
|
||||||
{[]string{"architecture"}, "", "", PatternTypeArchitecture},
|
{concepts: []string{"architecture"}, title: "", narrative: "", expected: PatternTypeArchitecture},
|
||||||
{[]string{"refactor"}, "", "", PatternTypeRefactor},
|
{concepts: []string{"refactor"}, title: "", narrative: "", expected: PatternTypeRefactor},
|
||||||
{[]string{}, "nil pointer bug", "", PatternTypeBug},
|
{concepts: []string{}, title: "nil pointer bug", narrative: "", expected: PatternTypeBug},
|
||||||
{[]string{}, "Deadlock in concurrent code", "", PatternTypeBug},
|
{concepts: []string{}, title: "Deadlock in concurrent code", narrative: "", expected: PatternTypeBug},
|
||||||
{[]string{}, "Extract interface", "", PatternTypeRefactor},
|
{concepts: []string{}, title: "Extract interface", narrative: "", expected: PatternTypeRefactor},
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, tt := range tests {
|
for _, tt := range tests {
|
||||||
|
|||||||
@@ -3,18 +3,18 @@ package models
|
|||||||
|
|
||||||
// UserPrompt represents a user prompt captured during a session.
|
// UserPrompt represents a user prompt captured during a session.
|
||||||
type UserPrompt struct {
|
type UserPrompt struct {
|
||||||
ID int64 `db:"id" json:"id"`
|
|
||||||
ClaudeSessionID string `db:"claude_session_id" json:"claude_session_id"`
|
ClaudeSessionID string `db:"claude_session_id" json:"claude_session_id"`
|
||||||
PromptNumber int `db:"prompt_number" json:"prompt_number"`
|
|
||||||
PromptText string `db:"prompt_text" json:"prompt_text"`
|
PromptText string `db:"prompt_text" json:"prompt_text"`
|
||||||
MatchedObservations int `db:"matched_observations" json:"matched_observations"`
|
|
||||||
CreatedAt string `db:"created_at" json:"created_at"`
|
CreatedAt string `db:"created_at" json:"created_at"`
|
||||||
|
ID int64 `db:"id" json:"id"`
|
||||||
|
PromptNumber int `db:"prompt_number" json:"prompt_number"`
|
||||||
|
MatchedObservations int `db:"matched_observations" json:"matched_observations"`
|
||||||
CreatedAtEpoch int64 `db:"created_at_epoch" json:"created_at_epoch"`
|
CreatedAtEpoch int64 `db:"created_at_epoch" json:"created_at_epoch"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// UserPromptWithSession includes session context for search results.
|
// UserPromptWithSession includes session context for search results.
|
||||||
type UserPromptWithSession struct {
|
type UserPromptWithSession struct {
|
||||||
UserPrompt
|
|
||||||
Project string `db:"project" json:"project"`
|
Project string `db:"project" json:"project"`
|
||||||
SDKSessionID string `db:"sdk_session_id" json:"sdk_session_id"`
|
SDKSessionID string `db:"sdk_session_id" json:"sdk_session_id"`
|
||||||
|
UserPrompt
|
||||||
}
|
}
|
||||||
|
|||||||
+20
-12
@@ -60,14 +60,14 @@ const (
|
|||||||
|
|
||||||
// ObservationRelation represents a directed relationship between two observations.
|
// ObservationRelation represents a directed relationship between two observations.
|
||||||
type ObservationRelation struct {
|
type ObservationRelation struct {
|
||||||
ID int64 `db:"id" json:"id"`
|
|
||||||
SourceID int64 `db:"source_id" json:"source_id"`
|
|
||||||
TargetID int64 `db:"target_id" json:"target_id"`
|
|
||||||
RelationType RelationType `db:"relation_type" json:"relation_type"`
|
RelationType RelationType `db:"relation_type" json:"relation_type"`
|
||||||
Confidence float64 `db:"confidence" json:"confidence"`
|
|
||||||
DetectionSource RelationDetectionSource `db:"detection_source" json:"detection_source"`
|
DetectionSource RelationDetectionSource `db:"detection_source" json:"detection_source"`
|
||||||
Reason string `db:"reason" json:"reason,omitempty"`
|
Reason string `db:"reason" json:"reason,omitempty"`
|
||||||
CreatedAt string `db:"created_at" json:"created_at"`
|
CreatedAt string `db:"created_at" json:"created_at"`
|
||||||
|
ID int64 `db:"id" json:"id"`
|
||||||
|
SourceID int64 `db:"source_id" json:"source_id"`
|
||||||
|
TargetID int64 `db:"target_id" json:"target_id"`
|
||||||
|
Confidence float64 `db:"confidence" json:"confidence"`
|
||||||
CreatedAtEpoch int64 `db:"created_at_epoch" json:"created_at_epoch"`
|
CreatedAtEpoch int64 `db:"created_at_epoch" json:"created_at_epoch"`
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -88,12 +88,12 @@ func NewObservationRelation(sourceID, targetID int64, relType RelationType, conf
|
|||||||
|
|
||||||
// RelationDetectionResult contains the result of relation detection.
|
// RelationDetectionResult contains the result of relation detection.
|
||||||
type RelationDetectionResult struct {
|
type RelationDetectionResult struct {
|
||||||
SourceID int64
|
|
||||||
TargetID int64
|
|
||||||
RelationType RelationType
|
RelationType RelationType
|
||||||
Confidence float64
|
|
||||||
DetectionSource RelationDetectionSource
|
DetectionSource RelationDetectionSource
|
||||||
Reason string
|
Reason string
|
||||||
|
SourceID int64
|
||||||
|
TargetID int64
|
||||||
|
Confidence float64
|
||||||
}
|
}
|
||||||
|
|
||||||
// DetectFileOverlapRelation checks if observations share file references and determines relationship type.
|
// DetectFileOverlapRelation checks if observations share file references and determines relationship type.
|
||||||
@@ -420,21 +420,27 @@ func DetectRelationsWithExisting(newer *Observation, existing []*Observation, mi
|
|||||||
|
|
||||||
// 1. File overlap detection
|
// 1. File overlap detection
|
||||||
if result := DetectFileOverlapRelation(newer, older); result != nil && result.Confidence >= minConfidence {
|
if result := DetectFileOverlapRelation(newer, older); result != nil && result.Confidence >= minConfidence {
|
||||||
if bestResult == nil || result.Confidence > bestResult.Confidence {
|
if bestResult == nil { //nolint:govet
|
||||||
|
bestResult = result
|
||||||
|
} else if result.Confidence > bestResult.Confidence {
|
||||||
bestResult = result
|
bestResult = result
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// 2. Concept overlap detection
|
// 2. Concept overlap detection
|
||||||
if result := DetectConceptOverlapRelation(newer, older); result != nil && result.Confidence >= minConfidence {
|
if result := DetectConceptOverlapRelation(newer, older); result != nil && result.Confidence >= minConfidence {
|
||||||
if bestResult == nil || result.Confidence > bestResult.Confidence {
|
if bestResult == nil {
|
||||||
|
bestResult = result
|
||||||
|
} else if result.Confidence > bestResult.Confidence {
|
||||||
bestResult = result
|
bestResult = result
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// 3. Type progression detection
|
// 3. Type progression detection
|
||||||
if result := DetectTypeProgressionRelation(newer, older); result != nil && result.Confidence >= minConfidence {
|
if result := DetectTypeProgressionRelation(newer, older); result != nil && result.Confidence >= minConfidence {
|
||||||
if bestResult == nil || result.Confidence > bestResult.Confidence {
|
if bestResult == nil {
|
||||||
|
bestResult = result
|
||||||
|
} else if result.Confidence > bestResult.Confidence {
|
||||||
bestResult = result
|
bestResult = result
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -449,7 +455,9 @@ func DetectRelationsWithExisting(newer *Observation, existing []*Observation, mi
|
|||||||
|
|
||||||
// 5. Narrative mention detection (can upgrade relation type)
|
// 5. Narrative mention detection (can upgrade relation type)
|
||||||
if result := DetectNarrativeMentionRelation(newer, older); result != nil && result.Confidence >= minConfidence {
|
if result := DetectNarrativeMentionRelation(newer, older); result != nil && result.Confidence >= minConfidence {
|
||||||
if bestResult == nil || result.Confidence > bestResult.Confidence {
|
if bestResult == nil {
|
||||||
|
bestResult = result
|
||||||
|
} else if result.Confidence > bestResult.Confidence {
|
||||||
bestResult = result
|
bestResult = result
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -484,6 +492,6 @@ type RelationWithDetails struct {
|
|||||||
|
|
||||||
// RelationGraph represents a graph of related observations.
|
// RelationGraph represents a graph of related observations.
|
||||||
type RelationGraph struct {
|
type RelationGraph struct {
|
||||||
CenterID int64 `json:"center_id"`
|
|
||||||
Relations []*RelationWithDetails `json:"relations"`
|
Relations []*RelationWithDetails `json:"relations"`
|
||||||
|
CenterID int64 `json:"center_id"`
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -8,12 +8,12 @@ import (
|
|||||||
|
|
||||||
func TestDetectFileOverlapRelation(t *testing.T) {
|
func TestDetectFileOverlapRelation(t *testing.T) {
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
name string
|
|
||||||
newer *Observation
|
newer *Observation
|
||||||
older *Observation
|
older *Observation
|
||||||
wantRelation bool
|
name string
|
||||||
wantRelType RelationType
|
wantRelType RelationType
|
||||||
wantMinConfid float64
|
wantMinConfid float64
|
||||||
|
wantRelation bool
|
||||||
}{
|
}{
|
||||||
{
|
{
|
||||||
name: "no file overlap",
|
name: "no file overlap",
|
||||||
@@ -105,11 +105,11 @@ func TestDetectFileOverlapRelation(t *testing.T) {
|
|||||||
|
|
||||||
func TestDetectConceptOverlapRelation(t *testing.T) {
|
func TestDetectConceptOverlapRelation(t *testing.T) {
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
name string
|
|
||||||
newer *Observation
|
newer *Observation
|
||||||
older *Observation
|
older *Observation
|
||||||
wantRelation bool
|
name string
|
||||||
wantMinConfid float64
|
wantMinConfid float64
|
||||||
|
wantRelation bool
|
||||||
}{
|
}{
|
||||||
{
|
{
|
||||||
name: "no concept overlap",
|
name: "no concept overlap",
|
||||||
@@ -179,8 +179,8 @@ func TestDetectTypeProgressionRelation(t *testing.T) {
|
|||||||
name string
|
name string
|
||||||
newerType ObservationType
|
newerType ObservationType
|
||||||
olderType ObservationType
|
olderType ObservationType
|
||||||
wantRelation bool
|
|
||||||
wantRelType RelationType
|
wantRelType RelationType
|
||||||
|
wantRelation bool
|
||||||
}{
|
}{
|
||||||
{
|
{
|
||||||
name: "bugfix fixes discovery",
|
name: "bugfix fixes discovery",
|
||||||
@@ -314,8 +314,8 @@ func TestDetectNarrativeMentionRelation(t *testing.T) {
|
|||||||
tests := []struct {
|
tests := []struct {
|
||||||
name string
|
name string
|
||||||
narrative string
|
narrative string
|
||||||
wantRelation bool
|
|
||||||
wantRelType RelationType
|
wantRelType RelationType
|
||||||
|
wantRelation bool
|
||||||
}{
|
}{
|
||||||
{
|
{
|
||||||
name: "fixes language",
|
name: "fixes language",
|
||||||
|
|||||||
+7
-23
@@ -4,8 +4,8 @@ package models
|
|||||||
// ConceptWeight represents a configurable weight for a concept.
|
// ConceptWeight represents a configurable weight for a concept.
|
||||||
type ConceptWeight struct {
|
type ConceptWeight struct {
|
||||||
Concept string `db:"concept" json:"concept"`
|
Concept string `db:"concept" json:"concept"`
|
||||||
Weight float64 `db:"weight" json:"weight"`
|
|
||||||
UpdatedAt string `db:"updated_at" json:"updated_at"`
|
UpdatedAt string `db:"updated_at" json:"updated_at"`
|
||||||
|
Weight float64 `db:"weight" json:"weight"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// UserFeedbackType represents the type of user feedback.
|
// UserFeedbackType represents the type of user feedback.
|
||||||
@@ -62,28 +62,12 @@ var TypeBaseScores = map[ObservationType]float64{
|
|||||||
|
|
||||||
// ScoringConfig contains all scoring weights and parameters.
|
// ScoringConfig contains all scoring weights and parameters.
|
||||||
type ScoringConfig struct {
|
type ScoringConfig struct {
|
||||||
// RecencyHalfLifeDays is the number of days for the importance score to halve.
|
ConceptWeights map[string]float64 `json:"concept_weights"`
|
||||||
// With 7 days, a 7-day old observation has 50% of a new observation's recency score.
|
RecencyHalfLifeDays float64 `json:"recency_half_life_days"`
|
||||||
RecencyHalfLifeDays float64 `json:"recency_half_life_days"`
|
FeedbackWeight float64 `json:"feedback_weight"`
|
||||||
|
ConceptWeight float64 `json:"concept_weight"`
|
||||||
// FeedbackWeight scales the user feedback contribution to final score.
|
RetrievalWeight float64 `json:"retrieval_weight"`
|
||||||
// With 0.30, a thumbs up adds 0.30 to the score, thumbs down subtracts 0.30.
|
MinScore float64 `json:"min_score"`
|
||||||
FeedbackWeight float64 `json:"feedback_weight"`
|
|
||||||
|
|
||||||
// ConceptWeight scales the concept boost contribution.
|
|
||||||
// The sum of matching concept weights is multiplied by this.
|
|
||||||
ConceptWeight float64 `json:"concept_weight"`
|
|
||||||
|
|
||||||
// RetrievalWeight scales the retrieval boost contribution.
|
|
||||||
// Popular observations get a logarithmic bonus.
|
|
||||||
RetrievalWeight float64 `json:"retrieval_weight"`
|
|
||||||
|
|
||||||
// ConceptWeights maps concept names to their importance weights.
|
|
||||||
ConceptWeights map[string]float64 `json:"concept_weights"`
|
|
||||||
|
|
||||||
// MinScore is the minimum allowed importance score.
|
|
||||||
// Prevents observations from completely disappearing.
|
|
||||||
MinScore float64 `json:"min_score"`
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// DefaultScoringConfig returns the default scoring configuration.
|
// DefaultScoringConfig returns the default scoring configuration.
|
||||||
|
|||||||
@@ -17,29 +17,29 @@ const (
|
|||||||
|
|
||||||
// SDKSession represents a Claude Code session tracked by the memory system.
|
// SDKSession represents a Claude Code session tracked by the memory system.
|
||||||
type SDKSession struct {
|
type SDKSession struct {
|
||||||
ID int64 `db:"id" json:"id"`
|
|
||||||
ClaudeSessionID string `db:"claude_session_id" json:"claude_session_id"`
|
ClaudeSessionID string `db:"claude_session_id" json:"claude_session_id"`
|
||||||
SDKSessionID sql.NullString `db:"sdk_session_id" json:"sdk_session_id,omitempty"`
|
|
||||||
Project string `db:"project" json:"project"`
|
Project string `db:"project" json:"project"`
|
||||||
UserPrompt sql.NullString `db:"user_prompt" json:"user_prompt,omitempty"`
|
|
||||||
WorkerPort sql.NullInt64 `db:"worker_port" json:"worker_port,omitempty"`
|
|
||||||
PromptCounter int64 `db:"prompt_counter" json:"prompt_counter"`
|
|
||||||
Status SessionStatus `db:"status" json:"status"`
|
Status SessionStatus `db:"status" json:"status"`
|
||||||
StartedAt string `db:"started_at" json:"started_at"`
|
StartedAt string `db:"started_at" json:"started_at"`
|
||||||
StartedAtEpoch int64 `db:"started_at_epoch" json:"started_at_epoch"`
|
SDKSessionID sql.NullString `db:"sdk_session_id" json:"sdk_session_id,omitempty"`
|
||||||
|
UserPrompt sql.NullString `db:"user_prompt" json:"user_prompt,omitempty"`
|
||||||
CompletedAt sql.NullString `db:"completed_at" json:"completed_at,omitempty"`
|
CompletedAt sql.NullString `db:"completed_at" json:"completed_at,omitempty"`
|
||||||
|
WorkerPort sql.NullInt64 `db:"worker_port" json:"worker_port,omitempty"`
|
||||||
CompletedAtEpoch sql.NullInt64 `db:"completed_at_epoch" json:"completed_at_epoch,omitempty"`
|
CompletedAtEpoch sql.NullInt64 `db:"completed_at_epoch" json:"completed_at_epoch,omitempty"`
|
||||||
|
ID int64 `db:"id" json:"id"`
|
||||||
|
PromptCounter int64 `db:"prompt_counter" json:"prompt_counter"`
|
||||||
|
StartedAtEpoch int64 `db:"started_at_epoch" json:"started_at_epoch"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// ActiveSession represents an in-memory active session being processed.
|
// ActiveSession represents an in-memory active session being processed.
|
||||||
type ActiveSession struct {
|
type ActiveSession struct {
|
||||||
SessionDBID int64
|
StartTime time.Time
|
||||||
ClaudeSessionID string
|
ClaudeSessionID string
|
||||||
SDKSessionID string
|
SDKSessionID string
|
||||||
Project string
|
Project string
|
||||||
UserPrompt string
|
UserPrompt string
|
||||||
|
SessionDBID int64
|
||||||
LastPromptNumber int
|
LastPromptNumber int
|
||||||
StartTime time.Time
|
|
||||||
CumulativeInputTokens int64
|
CumulativeInputTokens int64
|
||||||
CumulativeOutputTokens int64
|
CumulativeOutputTokens int64
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -9,18 +9,18 @@ import (
|
|||||||
|
|
||||||
// SessionSummary represents a summary of a Claude Code session.
|
// SessionSummary represents a summary of a Claude Code session.
|
||||||
type SessionSummary struct {
|
type SessionSummary struct {
|
||||||
ID int64 `db:"id" json:"id"`
|
CreatedAt string `db:"created_at" json:"created_at"`
|
||||||
SDKSessionID string `db:"sdk_session_id" json:"sdk_session_id"`
|
SDKSessionID string `db:"sdk_session_id" json:"sdk_session_id"`
|
||||||
Project string `db:"project" json:"project"`
|
Project string `db:"project" json:"project"`
|
||||||
Request sql.NullString `db:"request" json:"request,omitempty"`
|
Completed sql.NullString `db:"completed" json:"completed,omitempty"`
|
||||||
Investigated sql.NullString `db:"investigated" json:"investigated,omitempty"`
|
Investigated sql.NullString `db:"investigated" json:"investigated,omitempty"`
|
||||||
Learned sql.NullString `db:"learned" json:"learned,omitempty"`
|
Learned sql.NullString `db:"learned" json:"learned,omitempty"`
|
||||||
Completed sql.NullString `db:"completed" json:"completed,omitempty"`
|
|
||||||
NextSteps sql.NullString `db:"next_steps" json:"next_steps,omitempty"`
|
NextSteps sql.NullString `db:"next_steps" json:"next_steps,omitempty"`
|
||||||
Notes sql.NullString `db:"notes" json:"notes,omitempty"`
|
Notes sql.NullString `db:"notes" json:"notes,omitempty"`
|
||||||
|
Request sql.NullString `db:"request" json:"request,omitempty"`
|
||||||
PromptNumber sql.NullInt64 `db:"prompt_number" json:"prompt_number,omitempty"`
|
PromptNumber sql.NullInt64 `db:"prompt_number" json:"prompt_number,omitempty"`
|
||||||
|
ID int64 `db:"id" json:"id"`
|
||||||
DiscoveryTokens int64 `db:"discovery_tokens" json:"discovery_tokens"`
|
DiscoveryTokens int64 `db:"discovery_tokens" json:"discovery_tokens"`
|
||||||
CreatedAt string `db:"created_at" json:"created_at"`
|
|
||||||
CreatedAtEpoch int64 `db:"created_at_epoch" json:"created_at_epoch"`
|
CreatedAtEpoch int64 `db:"created_at_epoch" json:"created_at_epoch"`
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -56,18 +56,18 @@ func NewSessionSummary(sdkSessionID, project string, parsed *ParsedSummary, prom
|
|||||||
// SessionSummaryJSON is a JSON-friendly representation of SessionSummary.
|
// SessionSummaryJSON is a JSON-friendly representation of SessionSummary.
|
||||||
// It converts sql.NullString to plain strings for clean JSON output.
|
// It converts sql.NullString to plain strings for clean JSON output.
|
||||||
type SessionSummaryJSON struct {
|
type SessionSummaryJSON struct {
|
||||||
ID int64 `json:"id"`
|
Completed string `json:"completed,omitempty"`
|
||||||
SDKSessionID string `json:"sdk_session_id"`
|
SDKSessionID string `json:"sdk_session_id"`
|
||||||
Project string `json:"project"`
|
Project string `json:"project"`
|
||||||
Request string `json:"request,omitempty"`
|
Request string `json:"request,omitempty"`
|
||||||
Investigated string `json:"investigated,omitempty"`
|
Investigated string `json:"investigated,omitempty"`
|
||||||
Learned string `json:"learned,omitempty"`
|
Learned string `json:"learned,omitempty"`
|
||||||
Completed string `json:"completed,omitempty"`
|
|
||||||
NextSteps string `json:"next_steps,omitempty"`
|
NextSteps string `json:"next_steps,omitempty"`
|
||||||
Notes string `json:"notes,omitempty"`
|
Notes string `json:"notes,omitempty"`
|
||||||
|
CreatedAt string `json:"created_at"`
|
||||||
|
ID int64 `json:"id"`
|
||||||
PromptNumber int64 `json:"prompt_number,omitempty"`
|
PromptNumber int64 `json:"prompt_number,omitempty"`
|
||||||
DiscoveryTokens int64 `json:"discovery_tokens"`
|
DiscoveryTokens int64 `json:"discovery_tokens"`
|
||||||
CreatedAt string `json:"created_at"`
|
|
||||||
CreatedAtEpoch int64 `json:"created_at_epoch"`
|
CreatedAtEpoch int64 `json:"created_at_epoch"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -184,8 +184,8 @@ func (s *SummarySuite) TestSessionSummaryJSON() {
|
|||||||
Notes: "Notes",
|
Notes: "Notes",
|
||||||
PromptNumber: 5,
|
PromptNumber: 5,
|
||||||
DiscoveryTokens: 1000,
|
DiscoveryTokens: 1000,
|
||||||
CreatedAt: "2024-01-01T00:00:00Z",
|
CreatedAt: "2024-01-01T00:00:00Z", //nolint:govet
|
||||||
CreatedAtEpoch: 1704067200000,
|
CreatedAtEpoch: 1704067200000, //nolint:govet
|
||||||
}
|
}
|
||||||
|
|
||||||
s.Equal(int64(1), j.ID)
|
s.Equal(int64(1), j.ID)
|
||||||
|
|||||||
@@ -12,9 +12,9 @@ import (
|
|||||||
|
|
||||||
func TestJaccardSimilarity(t *testing.T) {
|
func TestJaccardSimilarity(t *testing.T) {
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
name string
|
|
||||||
set1 map[string]bool
|
set1 map[string]bool
|
||||||
set2 map[string]bool
|
set2 map[string]bool
|
||||||
|
name string
|
||||||
expected float64
|
expected float64
|
||||||
}{
|
}{
|
||||||
{
|
{
|
||||||
|
|||||||
Generated
+2
-2
@@ -1,12 +1,12 @@
|
|||||||
{
|
{
|
||||||
"name": "claude-mnemonic-dashboard",
|
"name": "claude-mnemonic-dashboard",
|
||||||
"version": "0ddacaa-dirty",
|
"version": "651732e",
|
||||||
"lockfileVersion": 3,
|
"lockfileVersion": 3,
|
||||||
"requires": true,
|
"requires": true,
|
||||||
"packages": {
|
"packages": {
|
||||||
"": {
|
"": {
|
||||||
"name": "claude-mnemonic-dashboard",
|
"name": "claude-mnemonic-dashboard",
|
||||||
"version": "0ddacaa-dirty",
|
"version": "651732e",
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"vis-data": "^7.1.9",
|
"vis-data": "^7.1.9",
|
||||||
"vis-network": "^9.1.9",
|
"vis-network": "^9.1.9",
|
||||||
|
|||||||
+1
-1
@@ -1,6 +1,6 @@
|
|||||||
{
|
{
|
||||||
"name": "claude-mnemonic-dashboard",
|
"name": "claude-mnemonic-dashboard",
|
||||||
"version": "0ddacaa-dirty",
|
"version": "651732e",
|
||||||
"private": true,
|
"private": true,
|
||||||
"type": "module",
|
"type": "module",
|
||||||
"scripts": {
|
"scripts": {
|
||||||
|
|||||||
@@ -2,6 +2,7 @@
|
|||||||
import { ref, computed } from 'vue'
|
import { ref, computed } from 'vue'
|
||||||
import type { Stats, SelfCheckResponse } from '@/types'
|
import type { Stats, SelfCheckResponse } from '@/types'
|
||||||
import ProjectFilter from './ProjectFilter.vue'
|
import ProjectFilter from './ProjectFilter.vue'
|
||||||
|
import { useGraphMetrics } from '@/composables'
|
||||||
|
|
||||||
const props = defineProps<{
|
const props = defineProps<{
|
||||||
stats: Stats | null
|
stats: Stats | null
|
||||||
@@ -18,12 +19,21 @@ defineEmits<{
|
|||||||
|
|
||||||
// Collapse state - persisted in localStorage
|
// Collapse state - persisted in localStorage
|
||||||
const isCollapsed = ref(localStorage.getItem('sidebar-collapsed') === 'true')
|
const isCollapsed = ref(localStorage.getItem('sidebar-collapsed') === 'true')
|
||||||
|
const metricsExpanded = ref(localStorage.getItem('metrics-expanded') === 'true')
|
||||||
|
|
||||||
|
// Graph metrics composable
|
||||||
|
const { graphStats, vectorMetrics, loading: metricsLoading, refresh: refreshMetrics } = useGraphMetrics()
|
||||||
|
|
||||||
function toggleCollapse() {
|
function toggleCollapse() {
|
||||||
isCollapsed.value = !isCollapsed.value
|
isCollapsed.value = !isCollapsed.value
|
||||||
localStorage.setItem('sidebar-collapsed', String(isCollapsed.value))
|
localStorage.setItem('sidebar-collapsed', String(isCollapsed.value))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function toggleMetrics() {
|
||||||
|
metricsExpanded.value = !metricsExpanded.value
|
||||||
|
localStorage.setItem('metrics-expanded', String(metricsExpanded.value))
|
||||||
|
}
|
||||||
|
|
||||||
function formatNumber(n: number): string {
|
function formatNumber(n: number): string {
|
||||||
if (n >= 1000000) return (n / 1000000).toFixed(1) + 'M'
|
if (n >= 1000000) return (n / 1000000).toFixed(1) + 'M'
|
||||||
if (n >= 1000) return (n / 1000).toFixed(1) + 'K'
|
if (n >= 1000) return (n / 1000).toFixed(1) + 'K'
|
||||||
@@ -205,6 +215,99 @@ function getStatusColor(status: string): string {
|
|||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
<!-- Advanced Metrics -->
|
||||||
|
<div class="bg-slate-800/50 rounded-lg border border-slate-700/50">
|
||||||
|
<button
|
||||||
|
@click="toggleMetrics"
|
||||||
|
class="w-full flex items-center justify-between p-4 hover:bg-slate-700/30 transition-colors rounded-lg"
|
||||||
|
>
|
||||||
|
<div class="flex items-center gap-2">
|
||||||
|
<i class="fas fa-chart-line text-violet-400" />
|
||||||
|
<h3 class="text-sm font-semibold text-white">Advanced Metrics</h3>
|
||||||
|
</div>
|
||||||
|
<i
|
||||||
|
:class="[
|
||||||
|
'fas text-slate-400 transition-transform duration-200',
|
||||||
|
metricsExpanded ? 'fa-chevron-up' : 'fa-chevron-down'
|
||||||
|
]"
|
||||||
|
/>
|
||||||
|
</button>
|
||||||
|
|
||||||
|
<Transition name="expand">
|
||||||
|
<div v-show="metricsExpanded" class="px-4 pb-4 space-y-4">
|
||||||
|
<!-- Loading State -->
|
||||||
|
<div v-if="metricsLoading" class="text-center py-4">
|
||||||
|
<i class="fas fa-spinner fa-spin text-slate-400" />
|
||||||
|
<p class="text-slate-500 text-sm mt-2">Loading metrics...</p>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<!-- Graph Stats -->
|
||||||
|
<div v-else-if="graphStats?.enabled">
|
||||||
|
<div class="flex items-center justify-between mb-2">
|
||||||
|
<span class="text-xs text-slate-400 uppercase tracking-wide">Graph</span>
|
||||||
|
<button
|
||||||
|
@click="refreshMetrics"
|
||||||
|
class="text-xs text-violet-400 hover:text-violet-300 transition-colors"
|
||||||
|
title="Refresh metrics"
|
||||||
|
>
|
||||||
|
<i class="fas fa-sync-alt" />
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
<div class="space-y-2">
|
||||||
|
<div class="flex items-center justify-between">
|
||||||
|
<span class="text-slate-400 text-sm">Nodes</span>
|
||||||
|
<span class="text-white font-medium">{{ formatNumber(graphStats.nodeCount) }}</span>
|
||||||
|
</div>
|
||||||
|
<div class="flex items-center justify-between">
|
||||||
|
<span class="text-slate-400 text-sm">Edges</span>
|
||||||
|
<span class="text-white font-medium">{{ formatNumber(graphStats.edgeCount) }}</span>
|
||||||
|
</div>
|
||||||
|
<div class="flex items-center justify-between">
|
||||||
|
<span class="text-slate-400 text-sm">Avg Degree</span>
|
||||||
|
<span class="text-white font-medium">{{ graphStats.avgDegree.toFixed(1) }}</span>
|
||||||
|
</div>
|
||||||
|
<div class="flex items-center justify-between">
|
||||||
|
<span class="text-slate-400 text-sm">Max Degree</span>
|
||||||
|
<span class="text-white font-medium">{{ graphStats.maxDegree }}</span>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<!-- Vector Metrics -->
|
||||||
|
<div v-if="vectorMetrics?.enabled" class="mt-4 pt-4 border-t border-slate-700/50">
|
||||||
|
<div class="text-xs text-slate-400 uppercase tracking-wide mb-2">Vector Storage</div>
|
||||||
|
<div class="space-y-2">
|
||||||
|
<div class="flex items-center justify-between">
|
||||||
|
<span class="text-slate-400 text-sm">Savings</span>
|
||||||
|
<span class="text-green-400 font-medium">
|
||||||
|
{{ vectorMetrics.storage.savingsPercent.toFixed(1) }}%
|
||||||
|
</span>
|
||||||
|
</div>
|
||||||
|
<div class="flex items-center justify-between">
|
||||||
|
<span class="text-slate-400 text-sm">Queries</span>
|
||||||
|
<span class="text-white font-medium">{{ formatNumber(vectorMetrics.queries.total) }}</span>
|
||||||
|
</div>
|
||||||
|
<div class="flex items-center justify-between">
|
||||||
|
<span class="text-slate-400 text-sm">Cache Hit</span>
|
||||||
|
<span class="text-cyan-400 font-medium">
|
||||||
|
{{ (vectorMetrics.cache.hitRate * 100).toFixed(1) }}%
|
||||||
|
</span>
|
||||||
|
</div>
|
||||||
|
<div class="flex items-center justify-between">
|
||||||
|
<span class="text-slate-400 text-sm">Avg Latency</span>
|
||||||
|
<span class="text-white font-medium text-xs">{{ vectorMetrics.latency.avg }}</span>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<!-- Disabled State -->
|
||||||
|
<div v-else class="text-slate-500 text-sm py-2">
|
||||||
|
{{ graphStats?.message || 'Metrics not available' }}
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</Transition>
|
||||||
|
</div>
|
||||||
|
|
||||||
<!-- Session Info -->
|
<!-- Session Info -->
|
||||||
<div v-if="stats" class="bg-slate-800/50 rounded-lg p-4 border border-slate-700/50">
|
<div v-if="stats" class="bg-slate-800/50 rounded-lg p-4 border border-slate-700/50">
|
||||||
<div class="flex items-center gap-2 mb-3">
|
<div class="flex items-center gap-2 mb-3">
|
||||||
@@ -260,6 +363,30 @@ function getStatusColor(status: string): string {
|
|||||||
>
|
>
|
||||||
<i class="fas fa-search text-cyan-400" />
|
<i class="fas fa-search text-cyan-400" />
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
<!-- Metrics indicator -->
|
||||||
|
<div
|
||||||
|
v-if="graphStats?.enabled"
|
||||||
|
class="bg-slate-800/50 rounded-lg p-3 border border-slate-700/50 flex justify-center"
|
||||||
|
:title="`${graphStats.nodeCount} nodes, ${graphStats.edgeCount} edges`"
|
||||||
|
>
|
||||||
|
<i class="fas fa-chart-line text-violet-400" />
|
||||||
|
</div>
|
||||||
</div>
|
</div>
|
||||||
</aside>
|
</aside>
|
||||||
</template>
|
</template>
|
||||||
|
|
||||||
|
<style scoped>
|
||||||
|
.expand-enter-active,
|
||||||
|
.expand-leave-active {
|
||||||
|
transition: all 0.3s ease;
|
||||||
|
overflow: hidden;
|
||||||
|
max-height: 500px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.expand-enter-from,
|
||||||
|
.expand-leave-to {
|
||||||
|
max-height: 0;
|
||||||
|
opacity: 0;
|
||||||
|
}
|
||||||
|
</style>
|
||||||
|
|||||||
@@ -3,3 +3,4 @@ export { useStats } from './useStats'
|
|||||||
export { useTimeline } from './useTimeline'
|
export { useTimeline } from './useTimeline'
|
||||||
export { useUpdate } from './useUpdate'
|
export { useUpdate } from './useUpdate'
|
||||||
export { useHealth } from './useHealth'
|
export { useHealth } from './useHealth'
|
||||||
|
export { useGraphMetrics } from './useGraphMetrics'
|
||||||
|
|||||||
@@ -0,0 +1,43 @@
|
|||||||
|
import { ref, onMounted } from 'vue'
|
||||||
|
import type { GraphStats, VectorMetrics } from '@/types'
|
||||||
|
import { fetchGraphStats, fetchVectorMetrics } from '@/utils/api'
|
||||||
|
|
||||||
|
export function useGraphMetrics() {
|
||||||
|
const graphStats = ref<GraphStats | null>(null)
|
||||||
|
const vectorMetrics = ref<VectorMetrics | null>(null)
|
||||||
|
const loading = ref(false)
|
||||||
|
const error = ref<string | null>(null)
|
||||||
|
|
||||||
|
const refresh = async () => {
|
||||||
|
loading.value = true
|
||||||
|
error.value = null
|
||||||
|
|
||||||
|
try {
|
||||||
|
// Fetch both in parallel
|
||||||
|
const [graph, vector] = await Promise.all([
|
||||||
|
fetchGraphStats(),
|
||||||
|
fetchVectorMetrics()
|
||||||
|
])
|
||||||
|
|
||||||
|
graphStats.value = graph
|
||||||
|
vectorMetrics.value = vector
|
||||||
|
} catch (err) {
|
||||||
|
error.value = err instanceof Error ? err.message : 'Failed to fetch metrics'
|
||||||
|
console.error('[GraphMetrics] Error:', err)
|
||||||
|
} finally {
|
||||||
|
loading.value = false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
onMounted(() => {
|
||||||
|
refresh()
|
||||||
|
})
|
||||||
|
|
||||||
|
return {
|
||||||
|
graphStats,
|
||||||
|
vectorMetrics,
|
||||||
|
loading,
|
||||||
|
error,
|
||||||
|
refresh
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -63,3 +63,58 @@ export interface SelfCheckResponse {
|
|||||||
uptime: string
|
uptime: string
|
||||||
components: ComponentHealth[]
|
components: ComponentHealth[]
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export interface GraphStats {
|
||||||
|
enabled: boolean
|
||||||
|
nodeCount: number
|
||||||
|
edgeCount: number
|
||||||
|
avgDegree: number
|
||||||
|
maxDegree: number
|
||||||
|
minDegree: number
|
||||||
|
medianDegree: number
|
||||||
|
edgeTypes: Record<string, number>
|
||||||
|
config: {
|
||||||
|
maxHops: number
|
||||||
|
branchFactor: number
|
||||||
|
edgeWeight: number
|
||||||
|
rebuildIntervalMin: number
|
||||||
|
}
|
||||||
|
message?: string
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface VectorMetrics {
|
||||||
|
enabled: boolean
|
||||||
|
queries: {
|
||||||
|
total: number
|
||||||
|
hubOnly: number
|
||||||
|
hybrid: number
|
||||||
|
onDemand: number
|
||||||
|
graph: number
|
||||||
|
}
|
||||||
|
latency: {
|
||||||
|
avg: string
|
||||||
|
p50: string
|
||||||
|
p95: string
|
||||||
|
p99: string
|
||||||
|
avgHub: string
|
||||||
|
avgRecompute: string
|
||||||
|
}
|
||||||
|
storage: {
|
||||||
|
totalDocuments: number
|
||||||
|
hubDocuments: number
|
||||||
|
storedEmbeddings: number
|
||||||
|
savingsPercent: number
|
||||||
|
recomputedTotal: number
|
||||||
|
}
|
||||||
|
cache: {
|
||||||
|
hits: number
|
||||||
|
misses: number
|
||||||
|
hitRate: number
|
||||||
|
}
|
||||||
|
graph: {
|
||||||
|
traversals: number
|
||||||
|
avgDepth: number
|
||||||
|
}
|
||||||
|
uptime: string
|
||||||
|
message?: string
|
||||||
|
}
|
||||||
|
|||||||
+9
-1
@@ -1,4 +1,4 @@
|
|||||||
import type { Observation, UserPrompt, SessionSummary, Stats, FeedItem, ObservationFeedItem, PromptFeedItem, SummaryFeedItem, RelationWithDetails, RelationGraph, RelationStats } from '@/types'
|
import type { Observation, UserPrompt, SessionSummary, Stats, FeedItem, ObservationFeedItem, PromptFeedItem, SummaryFeedItem, RelationWithDetails, RelationGraph, RelationStats, GraphStats, VectorMetrics } from '@/types'
|
||||||
|
|
||||||
const API_BASE = '/api'
|
const API_BASE = '/api'
|
||||||
const DEFAULT_TIMEOUT = 10000 // 10 seconds
|
const DEFAULT_TIMEOUT = 10000 // 10 seconds
|
||||||
@@ -164,3 +164,11 @@ export async function fetchRelatedObservations(observationId: number, minConfide
|
|||||||
export async function fetchRelationStats(signal?: AbortSignal): Promise<RelationStats> {
|
export async function fetchRelationStats(signal?: AbortSignal): Promise<RelationStats> {
|
||||||
return fetchWithRetry<RelationStats>(`${API_BASE}/relations/stats`, { signal })
|
return fetchWithRetry<RelationStats>(`${API_BASE}/relations/stats`, { signal })
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export async function fetchGraphStats(signal?: AbortSignal): Promise<GraphStats> {
|
||||||
|
return fetchWithRetry<GraphStats>(`${API_BASE}/graph/stats`, { signal })
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function fetchVectorMetrics(signal?: AbortSignal): Promise<VectorMetrics> {
|
||||||
|
return fetchWithRetry<VectorMetrics>(`${API_BASE}/vector/metrics`, { signal })
|
||||||
|
}
|
||||||
|
|||||||
@@ -1 +1 @@
|
|||||||
{"root":["./src/main.ts","./src/vite-env.d.ts","./src/components/index.ts","./src/composables/index.ts","./src/composables/usehealth.ts","./src/composables/usesse.ts","./src/composables/usestats.ts","./src/composables/usetimeline.ts","./src/composables/usetypes.ts","./src/composables/useupdate.ts","./src/types/api.ts","./src/types/index.ts","./src/types/observation.ts","./src/types/prompt.ts","./src/types/relation.ts","./src/types/summary.ts","./src/utils/api.ts","./src/utils/formatters.ts","./src/app.vue","./src/components/badge.vue","./src/components/card.vue","./src/components/filtertabs.vue","./src/components/header.vue","./src/components/iconbox.vue","./src/components/observationcard.vue","./src/components/projectfilter.vue","./src/components/promptcard.vue","./src/components/relationgraph.vue","./src/components/sidebar.vue","./src/components/statscards.vue","./src/components/summarycard.vue","./src/components/timeline.vue"],"version":"5.7.3"}
|
{"root":["./src/main.ts","./src/vite-env.d.ts","./src/components/index.ts","./src/composables/index.ts","./src/composables/usegraphmetrics.ts","./src/composables/usehealth.ts","./src/composables/usesse.ts","./src/composables/usestats.ts","./src/composables/usetimeline.ts","./src/composables/usetypes.ts","./src/composables/useupdate.ts","./src/types/api.ts","./src/types/index.ts","./src/types/observation.ts","./src/types/prompt.ts","./src/types/relation.ts","./src/types/summary.ts","./src/utils/api.ts","./src/utils/formatters.ts","./src/app.vue","./src/components/badge.vue","./src/components/card.vue","./src/components/filtertabs.vue","./src/components/header.vue","./src/components/iconbox.vue","./src/components/observationcard.vue","./src/components/projectfilter.vue","./src/components/promptcard.vue","./src/components/relationgraph.vue","./src/components/sidebar.vue","./src/components/statscards.vue","./src/components/summarycard.vue","./src/components/timeline.vue"],"version":"5.7.3"}
|
||||||
Reference in New Issue
Block a user