feat(chunking): add AST-aware code chunking for Go, Python, TypeScript

- [x] Add language-specific chunkers with AST parsing (Go, Python, TypeScript)
- [x] Implement chunking manager to dispatch files to appropriate chunkers
- [x] Integrate code chunks into vector sync for semantic search
- [x] Add tree-sitter dependency for Python/TypeScript parsing
- [x] Reorder struct fields for consistency across codebase
- [x] Rename error variables to follow Go conventions (err → unmarshalErr, etc.)
- [x] Add code chunk metadata to vector documents (language, symbol name, line ranges)
- [x] Update worker service to initialize chunking pipeline with all three languages
This commit is contained in:
2026-01-07 13:19:58 +00:00
parent 40a44a71eb
commit 4f4b4ac70f
78 changed files with 2313 additions and 652 deletions
+6 -6
View File
@@ -182,13 +182,13 @@ func (s *ConflictStore) CleanupSupersededObservations(ctx context.Context, proje
var toDelete []int64
for rows.Next() {
var id int64
if err := rows.Scan(&id); err != nil {
return nil, err
if scanErr := rows.Scan(&id); scanErr != nil {
return nil, scanErr
}
toDelete = append(toDelete, id)
}
if err := rows.Err(); err != nil {
return nil, err
if rowsErr := rows.Err(); rowsErr != nil {
return nil, rowsErr
}
if len(toDelete) == 0 {
@@ -197,8 +197,8 @@ func (s *ConflictStore) CleanupSupersededObservations(ctx context.Context, proje
// Delete the conflict records first (due to foreign key constraints)
for _, obsID := range toDelete {
if err := s.DeleteConflictsByObservationID(ctx, obsID); err != nil {
return nil, err
if delErr := s.DeleteConflictsByObservationID(ctx, obsID); delErr != nil {
return nil, delErr
}
}
+6 -6
View File
@@ -54,14 +54,14 @@ func TestNullInt(t *testing.T) {
func TestRepeatPlaceholders(t *testing.T) {
tests := []struct {
name string
n int
expected string
n int
}{
{"zero", 0, ""},
{"negative", -1, ""},
{"one", 1, ", ?"},
{"two", 2, ", ?, ?"},
{"three", 3, ", ?, ?, ?"},
{name: "zero", n: 0, expected: ""},
{name: "negative", n: -1, expected: ""},
{name: "one", n: 1, expected: ", ?"},
{name: "two", n: 2, expected: ", ?, ?"},
{name: "three", n: 3, expected: ", ?, ?, ?"},
}
for _, tt := range tests {
+4 -4
View File
@@ -9,9 +9,9 @@ import (
// Migration represents a database schema migration.
type Migration struct {
Version int
Name string
SQL string
Version int
}
// Migrations is the list of all database migrations in order.
@@ -539,11 +539,11 @@ func (m *MigrationManager) ApplyMigration(migration Migration) error {
if err != nil {
return fmt.Errorf("begin transaction: %w", err)
}
defer tx.Rollback()
defer func() { _ = tx.Rollback() }()
// Execute migration SQL
if _, err := tx.Exec(migration.SQL); err != nil {
return fmt.Errorf("execute migration %d (%s): %w", migration.Version, migration.Name, err)
if _, execErr := tx.Exec(migration.SQL); execErr != nil {
return fmt.Errorf("execute migration %d (%s): %w", migration.Version, migration.Name, execErr)
}
// Record migration
+4 -4
View File
@@ -585,13 +585,13 @@ func (s *ObservationStore) CleanupOldObservations(ctx context.Context, project s
var toDelete []int64
for rows.Next() {
var id int64
if err := rows.Scan(&id); err != nil {
return nil, err
if scanErr := rows.Scan(&id); scanErr != nil {
return nil, scanErr
}
toDelete = append(toDelete, id)
}
if err := rows.Err(); err != nil {
return nil, err
if rowsErr := rows.Err(); rowsErr != nil {
return nil, rowsErr
}
if len(toDelete) == 0 {
+2 -2
View File
@@ -65,10 +65,10 @@ func (s *ObservationStoreSuite) TestStoreObservation_TableDriven() {
ctx := context.Background()
tests := []struct {
obs *models.ParsedObservation
name string
sdkSessionID string
project string
obs *models.ParsedObservation
promptNum int
tokens int64
wantErr bool
@@ -308,8 +308,8 @@ func (s *ObservationStoreSuite) TestGetObservationsByIDs() {
tests := []struct {
name string
queryIDs []int64
orderBy string
queryIDs []int64
limit int
wantCount int
}{
+4 -4
View File
@@ -102,13 +102,13 @@ func (s *PromptStore) CleanupOldPrompts(ctx context.Context) ([]int64, error) {
var toDelete []int64
for rows.Next() {
var id int64
if err := rows.Scan(&id); err != nil {
return nil, err
if scanErr := rows.Scan(&id); scanErr != nil {
return nil, scanErr
}
toDelete = append(toDelete, id)
}
if err := rows.Err(); err != nil {
return nil, err
if rowsErr := rows.Err(); rowsErr != nil {
return nil, rowsErr
}
if len(toDelete) == 0 {
+5 -5
View File
@@ -261,14 +261,14 @@ func TestPromptStore_SaveMultiplePrompts(t *testing.T) {
tests := []struct {
claudeSessionID string
promptNum int
text string
promptNum int
matches int
}{
{"claude-1", 1, "First prompt", 5},
{"claude-1", 2, "Second prompt", 3},
{"claude-2", 1, "Third prompt", 0},
{"claude-1", 3, "Fourth prompt", 10},
{claudeSessionID: "claude-1", promptNum: 1, text: "First prompt", matches: 5},
{claudeSessionID: "claude-1", promptNum: 2, text: "Second prompt", matches: 3},
{claudeSessionID: "claude-2", promptNum: 1, text: "Third prompt", matches: 0},
{claudeSessionID: "claude-1", promptNum: 3, text: "Fourth prompt", matches: 10},
}
for _, tt := range tests {
+2 -2
View File
@@ -71,7 +71,7 @@ func (s *ObservationStore) UpdateImportanceScores(ctx context.Context, scores ma
if err != nil {
return err
}
defer tx.Rollback()
defer func() { _ = tx.Rollback() }()
now := time.Now().UnixMilli()
stmt, err := tx.PrepareContext(ctx, `
@@ -171,7 +171,7 @@ func (s *ObservationStore) UpdateConceptWeights(ctx context.Context, weights map
if err != nil {
return err
}
defer tx.Rollback()
defer func() { _ = tx.Rollback() }()
stmt, err := tx.PrepareContext(ctx, `
INSERT INTO concept_weights (concept, weight, updated_at)
+1 -1
View File
@@ -211,8 +211,8 @@ func (s *SessionStoreSuite) TestFindAnySDKSession_Scenarios() {
tests := []struct {
name string
claudeSessionID string
wantFound bool
wantProject string
wantFound bool
}{
{
name: "find existing session 1",
+2 -2
View File
@@ -94,8 +94,8 @@ func (s *Store) GetStmt(query string) (*sql.Stmt, error) {
defer s.stmtMu.Unlock()
// Double-check after acquiring write lock
if stmt, ok := s.stmtCache[query]; ok {
return stmt, nil
if cachedStmt, ok := s.stmtCache[query]; ok {
return cachedStmt, nil
}
stmt, err := s.db.Prepare(query)
+7 -7
View File
@@ -130,13 +130,13 @@ func (s *StoreSuite) TestQueryContext() {
seedSession(s.T(), s.db, "claude-1", "sdk-1", "project-a")
tests := []struct {
setupFunc func()
assertFunc func(rows *sql.Rows)
name string
query string
args []interface{}
wantErr bool
wantRows int
setupFunc func()
assertFunc func(rows *sql.Rows)
wantErr bool
}{
{
name: "query existing session",
@@ -366,8 +366,8 @@ func (s *HelpersSuite) TestNullInt() {
func (s *HelpersSuite) TestRepeatPlaceholders() {
tests := []struct {
name string
input int
expected string
input int
}{
{
name: "zero",
@@ -438,10 +438,10 @@ func TestBuildGetByIDsQuery(t *testing.T) {
tests := []struct {
name string
baseQuery string
ids []int64
orderBy string
limit int
wantQuery string
ids []int64
limit int
wantArgs int
}{
{
@@ -483,10 +483,10 @@ func TestEnsureSessionExists(t *testing.T) {
ctx := context.Background()
tests := []struct {
setup func()
name string
sdkSessionID string
project string
setup func()
wantErr bool
}{
{