lukaszraczylo/claude-mnemonic
1
0
Fork 0
mirror of https://github.com/lukaszraczylo/claude-mnemonic.git synced 2026-10-17 05:59:36 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(leann-phase2): implement hybrid vector storage and graph-based search

Browse Source 
- [x] Add AST-aware code chunking for Go, Python, and TypeScript using tree-sitter
- [x] Implement LEANN-inspired hybrid vector storage with hub detection and selective embedding storage (60-80% savings)
- [x] Add observation relationship graph with CSR format and edge detection (file overlap, semantic similarity, temporal, concept)
- [x] Implement graph-aware search with two-level traversal and relationship-based ranking
- [x] Add auto-tuning system for dynamic hub threshold adjustment based on query performance
- [x] Add comprehensive metrics tracking for vector storage, queries, latency, and graph traversals
- [x] Update configuration system with graph and hybrid storage settings
- [x] Add graph stats and vector metrics endpoints to worker service
- [x] Enhance UI sidebar with advanced metrics display and graph visualization
- [x] Optimize struct field alignment throughout codebase for memory efficiency
- [x] Update documentation with LEANN Phase 2 features and performance benefits
- [x] Add tree-sitter dependency for AST parsing
This commit is contained in:
lukaszraczylo
2026-01-07 20:43:10 +00:00
parent 7ab4b07cf2
commit 74ae8ed4c1
83 changed files with 5190 additions and 603 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/search/expansion/expander.go
+9 -9
View File
@@ -30,25 +30,25 @@ const (
// ExpandedQuery represents a query variant with metadata.
type ExpandedQuery struct {
Query string `json:"query"`
Weight float64 `json:"weight"` // Weight for result merging (0.0-1.0)
Source string `json:"source"` // Where this expansion came from
Intent QueryIntent `json:"intent"` // Detected intent
Source string `json:"source"`
Intent QueryIntent `json:"intent"`
Weight float64 `json:"weight"`
}
// Expander provides context-aware query expansion.
type Expander struct {
embedSvc *embedding.Service
vocabulary []VocabEntry // Known vocabulary from observations
vocabVectors [][]float32 // Embeddings for vocabulary entries
vocabMu sync.RWMutex // Protects vocabulary
intentPatterns map[QueryIntent][]*regexp.Regexp
vocabulary []VocabEntry
vocabVectors [][]float32
vocabMu sync.RWMutex
}
// VocabEntry represents a vocabulary term from observations.
type VocabEntry struct {
Term string // The term itself
Weight float64 // How common/important this term is (0.0-1.0)
Source string // Where it came from (title, concept, narrative)
Term string
Source string
Weight float64
}
// Config holds expander configuration.
internal/search/expansion/expander_test.go
+12 -12
View File
@@ -88,16 +88,16 @@ func (s *ExpanderSuite) TestExpand() {
tests := []struct {
name string
query string
expectedIntent QueryIntent
minExpansions int
hasOriginal bool
expectedIntent QueryIntent
}{
{"question", "how do I implement auth", 1, true, IntentQuestion},
{"error", "fix the bug in login", 1, true, IntentError},
{"implementation", "implement user handler", 1, true, IntentImplementation},
{"architecture", "architecture design", 1, true, IntentArchitecture},
{"general", "database connection", 1, true, IntentGeneral},
{"empty", "", 0, false, IntentGeneral},
{name: "question", query: "how do I implement auth", expectedIntent: IntentQuestion, minExpansions: 1, hasOriginal: true},
{name: "error", query: "fix the bug in login", expectedIntent: IntentError, minExpansions: 1, hasOriginal: true},
{name: "implementation", query: "implement user handler", expectedIntent: IntentImplementation, minExpansions: 1, hasOriginal: true},
{name: "architecture", query: "architecture design", expectedIntent: IntentArchitecture, minExpansions: 1, hasOriginal: true},
{name: "general", query: "database connection", expectedIntent: IntentGeneral, minExpansions: 1, hasOriginal: true},
{name: "empty", query: "", expectedIntent: IntentGeneral, minExpansions: 0, hasOriginal: false},
}
for _, tt := range tests {
@@ -392,13 +392,13 @@ func TestTruncate(t *testing.T) {
tests := []struct {
name string
input string
maxLen int
expected string
maxLen int
}{
{"short", "hello", 10, "hello"},
{"exact", "hello", 5, "hello"},
{"long", "hello world", 5, "hello..."},
{"empty", "", 10, ""},
{name: "short", input: "hello", expected: "hello", maxLen: 10},
{name: "exact", input: "hello", expected: "hello", maxLen: 5},
{name: "long", input: "hello world", expected: "hello...", maxLen: 5},
{name: "empty", input: "", expected: "", maxLen: 10},
}
for _, tt := range tests {
internal/search/integration_test.go
+8 -8
View File
@@ -516,16 +516,16 @@ func TestTruncate_TableDriven(t *testing.T) {
tests := []struct {
name string
input string
maxLen int
expected string
maxLen int
}{
{"short_string", "hello", 10, "hello"},
{"exact_length", "hello", 5, "hello"},
{"long_string", "hello world", 5, "hello..."},
{"empty_string", "", 10, ""},
{"whitespace_only", " ", 10, ""},
{"with_leading_space", " hello ", 10, "hello"},
{"very_long", "this is a very long string that should be truncated", 20, "this is a very long ..."},
{name: "short_string", input: "hello", expected: "hello", maxLen: 10},
{name: "exact_length", input: "hello", expected: "hello", maxLen: 5},
{name: "long_string", input: "hello world", expected: "hello...", maxLen: 5},
{name: "empty_string", input: "", expected: "", maxLen: 10},
{name: "whitespace_only", input: " ", expected: "", maxLen: 10},
{name: "with_leading_space", input: " hello ", expected: "hello", maxLen: 10},
{name: "very_long", input: "this is a very long string that should be truncated", expected: "this is a very long ...", maxLen: 20},
}
for _, tt := range tests {
internal/search/manager.go
+15 -15
View File
@@ -35,41 +35,41 @@ func NewManager(
// SearchParams contains parameters for unified search.
type SearchParams struct {
Query string
Type string // "observations", "sessions", "prompts", or empty for all
Format string
Type string
Project string
ObsType string // Observation type filter
ObsType string
Concepts string
Files string
Query string
Scope string
OrderBy string
DateStart int64
DateEnd int64
OrderBy string // "relevance", "date_desc", "date_asc"
Limit int
Offset int
Format string // "index" or "full"
Scope string // "project", "global", or empty for project+global
IncludeGlobal bool // If true, include global observations along with project-scoped
ExcludeSuperseded bool // If true, exclude observations that have been superseded
Limit int
DateEnd int64
IncludeGlobal bool
ExcludeSuperseded bool
}
// SearchResult represents a unified search result.
type SearchResult struct {
Type string `json:"type"` // "observation", "session", "prompt"
ID int64 `json:"id"`
Metadata map[string]interface{} `json:"metadata,omitempty"`
Type string `json:"type"`
Title string `json:"title,omitempty"`
Content string `json:"content,omitempty"`
Project string `json:"project"`
Scope string `json:"scope,omitempty"` // "project" or "global"
Scope string `json:"scope,omitempty"`
ID int64 `json:"id"`
CreatedAt int64 `json:"created_at_epoch"`
Score float64 `json:"score,omitempty"`
Metadata map[string]interface{} `json:"metadata,omitempty"`
}
// UnifiedSearchResult contains the combined search results.
type UnifiedSearchResult struct {
Query string `json:"query,omitempty"`
Results []SearchResult `json:"results"`
TotalCount int `json:"total_count"`
Query string `json:"query,omitempty"`
}
// UnifiedSearch performs a unified search across all document types.
internal/search/manager_test.go
+20 -20
View File
@@ -94,8 +94,8 @@ func TestTruncate(t *testing.T) {
tests := []struct {
name string
input string
maxLen int
expected string
maxLen int
}{
{
name: "short string no truncation",
@@ -148,8 +148,8 @@ func TestObservationToResult(t *testing.T) {
m := NewManager(nil, nil, nil, nil)
tests := []struct {
name string
obs *models.Observation
name string
format string
expected SearchResult
}{
@@ -240,8 +240,8 @@ func TestSummaryToResult(t *testing.T) {
m := NewManager(nil, nil, nil, nil)
tests := []struct {
name string
summary *models.SessionSummary
name string
format string
expected SearchResult
}{
@@ -322,8 +322,8 @@ func TestPromptToResult(t *testing.T) {
m := NewManager(nil, nil, nil, nil)
tests := []struct {
name string
prompt *models.UserPromptWithSession
name string
format string
expected SearchResult
}{
@@ -406,9 +406,9 @@ func TestPromptToResult(t *testing.T) {
func TestSearchParamsValidation(t *testing.T) {
tests := []struct {
name string
expectedOrder string
params SearchParams
expectedLimit int
expectedOrder string
}{
{
name: "default limit applied",
@@ -731,16 +731,16 @@ func TestPromptToResultFormats(t *testing.T) {
func TestSearchParamsDefaults(t *testing.T) {
tests := []struct {
name string
initialLimit int
initialOrder string
expectedLimit int
expectedOrder string
initialLimit int
expectedLimit int
}{
{"zero_limit", 0, "", 20, "date_desc"},
{"negative_limit", -5, "", 20, "date_desc"},
{"over_100_limit", 150, "", 100, "date_desc"},
{"valid_limit_50", 50, "relevance", 50, "relevance"},
{"custom_order", 30, "date_asc", 30, "date_asc"},
{name: "zero_limit", initialOrder: "", expectedOrder: "date_desc", initialLimit: 0, expectedLimit: 20},
{name: "negative_limit", initialOrder: "", expectedOrder: "date_desc", initialLimit: -5, expectedLimit: 20},
{name: "over_100_limit", initialOrder: "", expectedOrder: "date_desc", initialLimit: 150, expectedLimit: 100},
{name: "valid_limit_50", initialOrder: "relevance", expectedOrder: "relevance", initialLimit: 50, expectedLimit: 50},
{name: "custom_order", initialOrder: "date_asc", expectedOrder: "date_asc", initialLimit: 30, expectedLimit: 30},
}
for _, tt := range tests {
@@ -774,18 +774,18 @@ func TestTruncateEdgeCases(t *testing.T) {
tests := []struct {
name string
input string
maxLen int
expected string
maxLen int
}{
// Unicode strings - uses byte length so ensure maxLen accommodates full string
{"unicode_string_no_truncate", "日本語テスト", 20, "日本語テスト"},
{"mixed_unicode_no_truncate", "Hello世界", 15, "Hello世界"},
{name: "unicode_string_no_truncate", input: "日本語テスト", expected: "日本語テスト", maxLen: 20},
{name: "mixed_unicode_no_truncate", input: "Hello世界", expected: "Hello世界", maxLen: 15},
// ASCII truncation
{"ascii_truncate", "Hello World", 5, "Hello..."},
{"only_whitespace", " ", 10, ""},
{"tabs_and_newlines", "\t\n \t", 10, ""},
{"newlines_with_content", "\n\nhello\n\n", 10, "hello"},
{"zero_max_len", "hello", 0, "..."},
{name: "ascii_truncate", input: "Hello World", expected: "Hello...", maxLen: 5},
{name: "only_whitespace", input: " ", expected: "", maxLen: 10},
{name: "tabs_and_newlines", input: "\t\n \t", expected: "", maxLen: 10},
{name: "newlines_with_content", input: "\n\nhello\n\n", expected: "hello", maxLen: 10},
{name: "zero_max_len", input: "hello", expected: "...", maxLen: 0},
}
for _, tt := range tests {