feat(leann-phase2): implement hybrid vector storage and graph-based search

- [x] Add AST-aware code chunking for Go, Python, and TypeScript using tree-sitter
- [x] Implement LEANN-inspired hybrid vector storage with hub detection and selective embedding storage (60-80% savings)
- [x] Add observation relationship graph with CSR format and edge detection (file overlap, semantic similarity, temporal, concept)
- [x] Implement graph-aware search with two-level traversal and relationship-based ranking
- [x] Add auto-tuning system for dynamic hub threshold adjustment based on query performance
- [x] Add comprehensive metrics tracking for vector storage, queries, latency, and graph traversals
- [x] Update configuration system with graph and hybrid storage settings
- [x] Add graph stats and vector metrics endpoints to worker service
- [x] Enhance UI sidebar with advanced metrics display and graph visualization
- [x] Optimize struct field alignment throughout codebase for memory efficiency
- [x] Update documentation with LEANN Phase 2 features and performance benefits
- [x] Add tree-sitter dependency for AST parsing
This commit is contained in:
2026-01-07 20:43:10 +00:00
parent 7ab4b07cf2
commit 74ae8ed4c1
83 changed files with 5190 additions and 603 deletions
+9 -9
View File
@@ -30,25 +30,25 @@ const (
// ExpandedQuery represents a query variant with metadata.
type ExpandedQuery struct {
Query string `json:"query"`
Weight float64 `json:"weight"` // Weight for result merging (0.0-1.0)
Source string `json:"source"` // Where this expansion came from
Intent QueryIntent `json:"intent"` // Detected intent
Source string `json:"source"`
Intent QueryIntent `json:"intent"`
Weight float64 `json:"weight"`
}
// Expander provides context-aware query expansion.
type Expander struct {
embedSvc *embedding.Service
vocabulary []VocabEntry // Known vocabulary from observations
vocabVectors [][]float32 // Embeddings for vocabulary entries
vocabMu sync.RWMutex // Protects vocabulary
intentPatterns map[QueryIntent][]*regexp.Regexp
vocabulary []VocabEntry
vocabVectors [][]float32
vocabMu sync.RWMutex
}
// VocabEntry represents a vocabulary term from observations.
type VocabEntry struct {
Term string // The term itself
Weight float64 // How common/important this term is (0.0-1.0)
Source string // Where it came from (title, concept, narrative)
Term string
Source string
Weight float64
}
// Config holds expander configuration.
+12 -12
View File
@@ -88,16 +88,16 @@ func (s *ExpanderSuite) TestExpand() {
tests := []struct {
name string
query string
expectedIntent QueryIntent
minExpansions int
hasOriginal bool
expectedIntent QueryIntent
}{
{"question", "how do I implement auth", 1, true, IntentQuestion},
{"error", "fix the bug in login", 1, true, IntentError},
{"implementation", "implement user handler", 1, true, IntentImplementation},
{"architecture", "architecture design", 1, true, IntentArchitecture},
{"general", "database connection", 1, true, IntentGeneral},
{"empty", "", 0, false, IntentGeneral},
{name: "question", query: "how do I implement auth", expectedIntent: IntentQuestion, minExpansions: 1, hasOriginal: true},
{name: "error", query: "fix the bug in login", expectedIntent: IntentError, minExpansions: 1, hasOriginal: true},
{name: "implementation", query: "implement user handler", expectedIntent: IntentImplementation, minExpansions: 1, hasOriginal: true},
{name: "architecture", query: "architecture design", expectedIntent: IntentArchitecture, minExpansions: 1, hasOriginal: true},
{name: "general", query: "database connection", expectedIntent: IntentGeneral, minExpansions: 1, hasOriginal: true},
{name: "empty", query: "", expectedIntent: IntentGeneral, minExpansions: 0, hasOriginal: false},
}
for _, tt := range tests {
@@ -392,13 +392,13 @@ func TestTruncate(t *testing.T) {
tests := []struct {
name string
input string
maxLen int
expected string
maxLen int
}{
{"short", "hello", 10, "hello"},
{"exact", "hello", 5, "hello"},
{"long", "hello world", 5, "hello..."},
{"empty", "", 10, ""},
{name: "short", input: "hello", expected: "hello", maxLen: 10},
{name: "exact", input: "hello", expected: "hello", maxLen: 5},
{name: "long", input: "hello world", expected: "hello...", maxLen: 5},
{name: "empty", input: "", expected: "", maxLen: 10},
}
for _, tt := range tests {