lukaszraczylo/claude-mnemonic
1
0
Fork 0
mirror of https://github.com/lukaszraczylo/claude-mnemonic.git synced 2026-10-20 06:27:50 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(chunking): add AST-aware code chunking for Go, Python, TypeScript

Browse Source 
- [x] Add language-specific chunkers with AST parsing (Go, Python, TypeScript)
- [x] Implement chunking manager to dispatch files to appropriate chunkers
- [x] Integrate code chunks into vector sync for semantic search
- [x] Add tree-sitter dependency for Python/TypeScript parsing
- [x] Reorder struct fields for consistency across codebase
- [x] Rename error variables to follow Go conventions (err → unmarshalErr, etc.)
- [x] Add code chunk metadata to vector documents (language, symbol name, line ranges)
- [x] Update worker service to initialize chunking pipeline with all three languages
This commit is contained in:
lukaszraczylo
2026-01-07 13:19:58 +00:00
parent 40a44a71eb
commit 4f4b4ac70f
78 changed files with 2313 additions and 652 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/search/expansion/expander.go
+9 -9
View File
@@ -30,25 +30,25 @@ const (
// ExpandedQuery represents a query variant with metadata.
type ExpandedQuery struct {
Query string `json:"query"`
Weight float64 `json:"weight"` // Weight for result merging (0.0-1.0)
Source string `json:"source"` // Where this expansion came from
Intent QueryIntent `json:"intent"` // Detected intent
Source string `json:"source"`
Intent QueryIntent `json:"intent"`
Weight float64 `json:"weight"`
}
// Expander provides context-aware query expansion.
type Expander struct {
embedSvc *embedding.Service
vocabulary []VocabEntry // Known vocabulary from observations
vocabVectors [][]float32 // Embeddings for vocabulary entries
vocabMu sync.RWMutex // Protects vocabulary
intentPatterns map[QueryIntent][]*regexp.Regexp
vocabulary []VocabEntry
vocabVectors [][]float32
vocabMu sync.RWMutex
}
// VocabEntry represents a vocabulary term from observations.
type VocabEntry struct {
Term string // The term itself
Weight float64 // How common/important this term is (0.0-1.0)
Source string // Where it came from (title, concept, narrative)
Term string
Source string
Weight float64
}
// Config holds expander configuration.
internal/search/expansion/expander_test.go
+12 -12
View File
@@ -88,16 +88,16 @@ func (s *ExpanderSuite) TestExpand() {
tests := []struct {
name string
query string
expectedIntent QueryIntent
minExpansions int
hasOriginal bool
expectedIntent QueryIntent
}{
{"question", "how do I implement auth", 1, true, IntentQuestion},
{"error", "fix the bug in login", 1, true, IntentError},
{"implementation", "implement user handler", 1, true, IntentImplementation},
{"architecture", "architecture design", 1, true, IntentArchitecture},
{"general", "database connection", 1, true, IntentGeneral},
{"empty", "", 0, false, IntentGeneral},
{name: "question", query: "how do I implement auth", expectedIntent: IntentQuestion, minExpansions: 1, hasOriginal: true},
{name: "error", query: "fix the bug in login", expectedIntent: IntentError, minExpansions: 1, hasOriginal: true},
{name: "implementation", query: "implement user handler", expectedIntent: IntentImplementation, minExpansions: 1, hasOriginal: true},
{name: "architecture", query: "architecture design", expectedIntent: IntentArchitecture, minExpansions: 1, hasOriginal: true},
{name: "general", query: "database connection", expectedIntent: IntentGeneral, minExpansions: 1, hasOriginal: true},
{name: "empty", query: "", expectedIntent: IntentGeneral, minExpansions: 0, hasOriginal: false},
}
for _, tt := range tests {
@@ -392,13 +392,13 @@ func TestTruncate(t *testing.T) {
tests := []struct {
name string
input string
maxLen int
expected string
maxLen int
}{
{"short", "hello", 10, "hello"},
{"exact", "hello", 5, "hello"},
{"long", "hello world", 5, "hello..."},
{"empty", "", 10, ""},
{name: "short", input: "hello", maxLen: 10, expected: "hello"},
{name: "exact", input: "hello", maxLen: 5, expected: "hello"},
{name: "long", input: "hello world", maxLen: 5, expected: "hello..."},
{name: "empty", input: "", maxLen: 10, expected: ""},
}
for _, tt := range tests {
internal/search/integration_test.go
+8 -8
View File
@@ -517,16 +517,16 @@ func TestTruncate_TableDriven(t *testing.T) {
tests := []struct {
name string
input string
maxLen int
expected string
maxLen int
}{
{"short_string", "hello", 10, "hello"},
{"exact_length", "hello", 5, "hello"},
{"long_string", "hello world", 5, "hello..."},
{"empty_string", "", 10, ""},
{"whitespace_only", " ", 10, ""},
{"with_leading_space", " hello ", 10, "hello"},
{"very_long", "this is a very long string that should be truncated", 20, "this is a very long ..."},
{name: "short_string", input: "hello", maxLen: 10, expected: "hello"},
{name: "exact_length", input: "hello", maxLen: 5, expected: "hello"},
{name: "long_string", input: "hello world", maxLen: 5, expected: "hello..."},
{name: "empty_string", input: "", maxLen: 10, expected: ""},
{name: "whitespace_only", input: " ", maxLen: 10, expected: ""},
{name: "with_leading_space", input: " hello ", maxLen: 10, expected: "hello"},
{name: "very_long", input: "this is a very long string that should be truncated", maxLen: 20, expected: "this is a very long ..."},
}
for _, tt := range tests {
internal/search/manager.go
+15 -15
View File
@@ -35,41 +35,41 @@ func NewManager(
// SearchParams contains parameters for unified search.
type SearchParams struct {
Query string
Type string // "observations", "sessions", "prompts", or empty for all
Format string
Type string
Project string
ObsType string // Observation type filter
ObsType string
Concepts string
Files string
Query string
Scope string
OrderBy string
DateStart int64
DateEnd int64
OrderBy string // "relevance", "date_desc", "date_asc"
Limit int
Offset int
Format string // "index" or "full"
Scope string // "project", "global", or empty for project+global
IncludeGlobal bool // If true, include global observations along with project-scoped
ExcludeSuperseded bool // If true, exclude observations that have been superseded
Limit int
DateEnd int64
IncludeGlobal bool
ExcludeSuperseded bool
}
// SearchResult represents a unified search result.
type SearchResult struct {
Type string `json:"type"` // "observation", "session", "prompt"
ID int64 `json:"id"`
Metadata map[string]interface{} `json:"metadata,omitempty"`
Type string `json:"type"`
Title string `json:"title,omitempty"`
Content string `json:"content,omitempty"`
Project string `json:"project"`
Scope string `json:"scope,omitempty"` // "project" or "global"
Scope string `json:"scope,omitempty"`
ID int64 `json:"id"`
CreatedAt int64 `json:"created_at_epoch"`
Score float64 `json:"score,omitempty"`
Metadata map[string]interface{} `json:"metadata,omitempty"`
}
// UnifiedSearchResult contains the combined search results.
type UnifiedSearchResult struct {
Query string `json:"query,omitempty"`
Results []SearchResult `json:"results"`
TotalCount int `json:"total_count"`
Query string `json:"query,omitempty"`
}
// UnifiedSearch performs a unified search across all document types.
internal/search/manager_test.go
+37 -49
View File
@@ -4,7 +4,6 @@ package search
import (
"database/sql"
"testing"
"time"
"github.com/lukaszraczylo/claude-mnemonic/pkg/models"
"github.com/stretchr/testify/assert"
@@ -94,8 +93,8 @@ func TestTruncate(t *testing.T) {
tests := []struct {
name string
input string
maxLen int
expected string
maxLen int
}{
{
name: "short string no truncation",
@@ -147,11 +146,11 @@ func TestTruncate(t *testing.T) {
func TestObservationToResult(t *testing.T) {
m := NewManager(nil, nil, nil, nil)
tests := []struct {
name string
tests := []struct { //nolint:govet
obs *models.Observation
format string
expected SearchResult
format string
name string
}{
{
name: "full format with all fields",
@@ -239,11 +238,11 @@ func TestObservationToResult(t *testing.T) {
func TestSummaryToResult(t *testing.T) {
m := NewManager(nil, nil, nil, nil)
tests := []struct {
name string
tests := []struct { //nolint:govet
summary *models.SessionSummary
format string
expected SearchResult
format string
name string
}{
{
name: "full format with all fields",
@@ -321,11 +320,11 @@ func TestSummaryToResult(t *testing.T) {
func TestPromptToResult(t *testing.T) {
m := NewManager(nil, nil, nil, nil)
tests := []struct {
name string
tests := []struct { //nolint:govet
prompt *models.UserPromptWithSession
format string
expected SearchResult
format string
name string
}{
{
name: "full format with content",
@@ -406,9 +405,9 @@ func TestPromptToResult(t *testing.T) {
func TestSearchParamsValidation(t *testing.T) {
tests := []struct {
name string
expectedOrder string
params SearchParams
expectedLimit int
expectedOrder string
}{
{
name: "default limit applied",
@@ -731,23 +730,21 @@ func TestPromptToResultFormats(t *testing.T) {
func TestSearchParamsDefaults(t *testing.T) {
tests := []struct {
name string
initialLimit int
initialOrder string
expectedLimit int
expectedOrder string
initialLimit int
expectedLimit int
}{
{"zero_limit", 0, "", 20, "date_desc"},
{"negative_limit", -5, "", 20, "date_desc"},
{"over_100_limit", 150, "", 100, "date_desc"},
{"valid_limit_50", 50, "relevance", 50, "relevance"},
{"custom_order", 30, "date_asc", 30, "date_asc"},
{name: "zero_limit", initialLimit: 0, initialOrder: "", expectedLimit: 20, expectedOrder: "date_desc"},
{name: "negative_limit", initialLimit: -5, initialOrder: "", expectedLimit: 20, expectedOrder: "date_desc"},
{name: "over_100_limit", initialLimit: 150, initialOrder: "", expectedLimit: 100, expectedOrder: "date_desc"},
{name: "valid_limit_50", initialLimit: 50, initialOrder: "relevance", expectedLimit: 50, expectedOrder: "relevance"},
{name: "custom_order", initialLimit: 30, initialOrder: "date_asc", expectedLimit: 30, expectedOrder: "date_asc"},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
params := SearchParams{
Query: "test",
Project: "project",
Limit: tt.initialLimit,
OrderBy: tt.initialOrder,
}
@@ -774,18 +771,18 @@ func TestTruncateEdgeCases(t *testing.T) {
tests := []struct {
name string
input string
maxLen int
expected string
maxLen int
}{
// Unicode strings - uses byte length so ensure maxLen accommodates full string
{"unicode_string_no_truncate", "日本語テスト", 20, "日本語テスト"},
{"mixed_unicode_no_truncate", "Hello世界", 15, "Hello世界"},
{name: "unicode_string_no_truncate", input: "日本語テスト", maxLen: 20, expected: "日本語テスト"},
{name: "mixed_unicode_no_truncate", input: "Hello世界", maxLen: 15, expected: "Hello世界"},
// ASCII truncation
{"ascii_truncate", "Hello World", 5, "Hello..."},
{"only_whitespace", " ", 10, ""},
{"tabs_and_newlines", "\t\n \t", 10, ""},
{"newlines_with_content", "\n\nhello\n\n", 10, "hello"},
{"zero_max_len", "hello", 0, "..."},
{name: "ascii_truncate", input: "Hello World", maxLen: 5, expected: "Hello..."},
{name: "only_whitespace", input: " ", maxLen: 10, expected: ""},
{name: "tabs_and_newlines", input: "\t\n \t", maxLen: 10, expected: ""},
{name: "newlines_with_content", input: "\n\nhello\n\n", maxLen: 10, expected: "hello"},
{name: "zero_max_len", input: "hello", maxLen: 0, expected: "..."},
}
for _, tt := range tests {
@@ -812,8 +809,6 @@ func TestUnifiedSearchResultEmpty(t *testing.T) {
// TestSearchResultMetadata tests SearchResult metadata handling.
func TestSearchResultMetadata(t *testing.T) {
result := SearchResult{
Type: "observation",
ID: 1,
Metadata: map[string]interface{}{
"obs_type": "discovery",
"scope": "project",
@@ -835,10 +830,7 @@ func TestSearchResultTypes(t *testing.T) {
for _, typ := range types {
t.Run(typ, func(t *testing.T) {
result := SearchResult{
Type: typ,
ID: 1,
Project: "test",
CreatedAt: time.Now().UnixMilli(),
Type: typ,
}
assert.Equal(t, typ, result.Type)
})
@@ -952,8 +944,8 @@ func TestSearchParams_OrderByValues(t *testing.T) {
for _, order := range validOrders {
t.Run("order_"+order, func(t *testing.T) {
params := SearchParams{
Query: "test",
Project: "test",
Query: "test", //nolint:govet
Project: "test", //nolint:govet
OrderBy: order,
}
assert.Equal(t, order, params.OrderBy)
@@ -968,9 +960,7 @@ func TestSearchParams_TypeValues(t *testing.T) {
for _, typ := range validTypes {
t.Run("type_"+typ, func(t *testing.T) {
params := SearchParams{
Query: "test",
Project: "test",
Type: typ,
Type: typ,
}
assert.Equal(t, typ, params.Type)
})
@@ -984,8 +974,8 @@ func TestSearchParams_ScopeValues(t *testing.T) {
for _, scope := range validScopes {
t.Run("scope_"+scope, func(t *testing.T) {
params := SearchParams{
Query: "test",
Project: "test",
Query: "test", //nolint:govet
Project: "test", //nolint:govet
Scope: scope,
}
assert.Equal(t, scope, params.Scope)
@@ -1000,8 +990,8 @@ func TestSearchParams_FormatValues(t *testing.T) {
for _, format := range validFormats {
t.Run("format_"+format, func(t *testing.T) {
params := SearchParams{
Query: "test",
Project: "test",
Query: "test", //nolint:govet
Project: "test", //nolint:govet
Format: format,
}
assert.Equal(t, format, params.Format)
@@ -1020,7 +1010,7 @@ func TestUnifiedSearchResult_MultipleResults(t *testing.T) {
result := UnifiedSearchResult{
Results: results,
TotalCount: 3,
Query: "test query",
Query: "test query", //nolint:govet
}
assert.Len(t, result.Results, 3)
@@ -1040,8 +1030,8 @@ func TestSearchResult_Metadata(t *testing.T) {
}
result := SearchResult{
Type: "observation",
ID: 1,
Type: "observation", //nolint:govet
ID: 1, //nolint:govet
Metadata: metadata,
}
@@ -1066,8 +1056,6 @@ func TestSearchResult_Scores(t *testing.T) {
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
result := SearchResult{
Type: "observation",
ID: 1,
Score: tt.score,
}
assert.Equal(t, tt.score, result.Score)