feat(chunking): add AST-aware code chunking for Go, Python, TypeScript

- [x] Add language-specific chunkers with AST parsing (Go, Python, TypeScript)
- [x] Implement chunking manager to dispatch files to appropriate chunkers
- [x] Integrate code chunks into vector sync for semantic search
- [x] Add tree-sitter dependency for Python/TypeScript parsing
- [x] Reorder struct fields for consistency across codebase
- [x] Rename error variables to follow Go conventions (err → unmarshalErr, etc.)
- [x] Add code chunk metadata to vector documents (language, symbol name, line ranges)
- [x] Update worker service to initialize chunking pipeline with all three languages
This commit is contained in:
2026-01-07 13:19:58 +00:00
parent 40a44a71eb
commit 4f4b4ac70f
78 changed files with 2313 additions and 652 deletions
+19 -30
View File
@@ -37,40 +37,29 @@ var CriticalConcepts = []string{
// Config holds the application configuration.
type Config struct {
// Worker settings
WorkerPort int `json:"worker_port"`
// Database settings
DBPath string `json:"db_path"`
MaxConns int `json:"max_conns"`
// SDK Agent settings
Model string `json:"model"`
ClaudeCodePath string `json:"claude_code_path"`
// Embedding settings
EmbeddingModel string `json:"embedding_model"` // e.g., "bge-v1.5"
// Reranking settings (cross-encoder)
RerankingEnabled bool `json:"reranking_enabled"` // Enable cross-encoder reranking
RerankingCandidates int `json:"reranking_candidates"` // Number of candidates to retrieve before reranking (default 100)
RerankingResults int `json:"reranking_results"` // Number of results to return after reranking (default 10)
RerankingAlpha float64 `json:"reranking_alpha"` // Weight for combining scores: alpha*rerank + (1-alpha)*original (default 0.7)
RerankingMinImprovement float64 `json:"reranking_min_improvement"` // Minimum rank improvement to trigger reranking (default 0, always rerank)
RerankingPureMode bool `json:"reranking_pure_mode"` // Use pure cross-encoder scores without combining with bi-encoder (default false)
// Context injection settings
ContextFullField string `json:"context_full_field"`
DBPath string `json:"db_path"`
Model string `json:"model"`
ClaudeCodePath string `json:"claude_code_path"`
EmbeddingModel string `json:"embedding_model"`
ContextObsConcepts []string `json:"context_obs_concepts"`
ContextObsTypes []string `json:"context_obs_types"`
RerankingMinImprovement float64 `json:"reranking_min_improvement"`
RerankingCandidates int `json:"reranking_candidates"`
RerankingAlpha float64 `json:"reranking_alpha"`
WorkerPort int `json:"worker_port"`
ContextMaxPromptResults int `json:"context_max_prompt_results"`
ContextObservations int `json:"context_observations"`
ContextFullCount int `json:"context_full_count"`
ContextSessionCount int `json:"context_session_count"`
ContextShowReadTokens bool `json:"context_show_read_tokens"`
ContextShowWorkTokens bool `json:"context_show_work_tokens"`
ContextFullField string `json:"context_full_field"`
ContextRelevanceThreshold float64 `json:"context_relevance_threshold"`
MaxConns int `json:"max_conns"`
RerankingResults int `json:"reranking_results"`
ContextShowLastSummary bool `json:"context_show_last_summary"`
ContextObsTypes []string `json:"context_obs_types"`
ContextObsConcepts []string `json:"context_obs_concepts"`
ContextRelevanceThreshold float64 `json:"context_relevance_threshold"` // 0.0-1.0, minimum similarity for inclusion
ContextMaxPromptResults int `json:"context_max_prompt_results"` // Max results per prompt (0 = threshold only)
RerankingEnabled bool `json:"reranking_enabled"`
ContextShowWorkTokens bool `json:"context_show_work_tokens"`
ContextShowReadTokens bool `json:"context_show_read_tokens"`
RerankingPureMode bool `json:"reranking_pure_mode"`
}
var (
+3 -3
View File
@@ -121,8 +121,8 @@ func (s *ConfigSuite) TestLoad_TableDriven() {
tests := []struct {
name string
settingsJSON string
expectedPort int
expectedModel string
expectedPort int
expectedObsObs int
}{
{
@@ -183,12 +183,12 @@ func (s *ConfigSuite) TestLoad_TableDriven() {
s.Require().NoError(err)
if tt.settingsJSON != "" {
err := os.WriteFile(
writeErr := os.WriteFile(
filepath.Join(tempDir, ".claude-mnemonic", "settings.json"),
[]byte(tt.settingsJSON),
0600,
)
s.Require().NoError(err)
s.Require().NoError(writeErr)
}
cfg, err := Load()