mirror of
https://github.com/lukaszraczylo/claude-mnemonic.git
synced 2026-10-19 06:18:13 +00:00
feat(chunking): add AST-aware code chunking for Go, Python, TypeScript
- [x] Add language-specific chunkers with AST parsing (Go, Python, TypeScript) - [x] Implement chunking manager to dispatch files to appropriate chunkers - [x] Integrate code chunks into vector sync for semantic search - [x] Add tree-sitter dependency for Python/TypeScript parsing - [x] Reorder struct fields for consistency across codebase - [x] Rename error variables to follow Go conventions (err → unmarshalErr, etc.) - [x] Add code chunk metadata to vector documents (language, symbol name, line ranges) - [x] Update worker service to initialize chunking pipeline with all three languages
This commit is contained in:
@@ -21,11 +21,11 @@ const (
|
||||
|
||||
// ObservationData contains data for a tool observation.
|
||||
type ObservationData struct {
|
||||
ToolName string
|
||||
ToolInput interface{}
|
||||
ToolResponse interface{}
|
||||
PromptNumber int
|
||||
ToolName string
|
||||
CWD string
|
||||
PromptNumber int
|
||||
}
|
||||
|
||||
// SummarizeData contains data for a summarize request.
|
||||
@@ -36,30 +36,28 @@ type SummarizeData struct {
|
||||
|
||||
// PendingMessage represents a message queued for SDK processing.
|
||||
type PendingMessage struct {
|
||||
Type MessageType
|
||||
Observation *ObservationData
|
||||
Summarize *SummarizeData
|
||||
Type MessageType
|
||||
}
|
||||
|
||||
// ActiveSession represents an in-memory active session being processed.
|
||||
type ActiveSession struct {
|
||||
SessionDBID int64
|
||||
ClaudeSessionID string
|
||||
SDKSessionID string
|
||||
StartTime time.Time
|
||||
ctx context.Context
|
||||
cancel context.CancelFunc
|
||||
notify chan struct{}
|
||||
Project string
|
||||
UserPrompt string
|
||||
SDKSessionID string
|
||||
ClaudeSessionID string
|
||||
pendingMessages []PendingMessage
|
||||
LastPromptNumber int
|
||||
StartTime time.Time
|
||||
CumulativeInputTokens int64
|
||||
CumulativeOutputTokens int64
|
||||
|
||||
// Concurrency control
|
||||
pendingMessages []PendingMessage
|
||||
messageMu sync.Mutex
|
||||
notify chan struct{}
|
||||
ctx context.Context
|
||||
cancel context.CancelFunc
|
||||
generatorActive atomic.Bool
|
||||
SessionDBID int64
|
||||
messageMu sync.Mutex
|
||||
generatorActive atomic.Bool
|
||||
}
|
||||
|
||||
// SessionTimeout is how long an inactive session can exist before cleanup.
|
||||
@@ -70,15 +68,14 @@ const CleanupInterval = 5 * time.Minute
|
||||
|
||||
// Manager manages active session lifecycles.
|
||||
type Manager struct {
|
||||
sessionStore *sqlite.SessionStore
|
||||
sessions map[int64]*ActiveSession
|
||||
mu sync.RWMutex
|
||||
onCreated func(int64)
|
||||
onDeleted func(int64)
|
||||
ctx context.Context
|
||||
cancel context.CancelFunc
|
||||
// Global notification channel for immediate processing
|
||||
ctx context.Context
|
||||
sessionStore *sqlite.SessionStore
|
||||
sessions map[int64]*ActiveSession
|
||||
onCreated func(int64)
|
||||
onDeleted func(int64)
|
||||
cancel context.CancelFunc
|
||||
ProcessNotify chan struct{}
|
||||
mu sync.RWMutex
|
||||
}
|
||||
|
||||
// NewManager creates a new session manager.
|
||||
|
||||
@@ -47,9 +47,9 @@ func (s *ManagerSuite) TestActiveSession() {
|
||||
SDKSessionID: "sdk-123",
|
||||
Project: "test-project",
|
||||
UserPrompt: "Hello",
|
||||
StartTime: time.Now(),
|
||||
pendingMessages: make([]PendingMessage, 0),
|
||||
notify: make(chan struct{}, 1),
|
||||
StartTime: time.Now(), //nolint:govet
|
||||
pendingMessages: make([]PendingMessage, 0), //nolint:govet
|
||||
notify: make(chan struct{}, 1), //nolint:govet
|
||||
}
|
||||
|
||||
s.Equal(int64(1), session.SessionDBID)
|
||||
@@ -134,7 +134,7 @@ func (s *ManagerSuite) TestDeleteSession() {
|
||||
session := &ActiveSession{
|
||||
SessionDBID: 1,
|
||||
Project: "test-project",
|
||||
StartTime: time.Now(),
|
||||
StartTime: time.Now(), //nolint:govet
|
||||
pendingMessages: []PendingMessage{},
|
||||
ctx: ctx,
|
||||
cancel: cancel,
|
||||
@@ -236,8 +236,8 @@ func TestTimeoutConstants(t *testing.T) {
|
||||
func TestObservationData(t *testing.T) {
|
||||
data := ObservationData{
|
||||
ToolName: "Read",
|
||||
ToolInput: map[string]string{"path": "/test/file.go"},
|
||||
ToolResponse: "file content",
|
||||
ToolInput: map[string]string{"path": "/test/file.go"}, //nolint:govet
|
||||
ToolResponse: "file content", //nolint:govet
|
||||
PromptNumber: 1,
|
||||
CWD: "/test",
|
||||
}
|
||||
@@ -333,7 +333,7 @@ func TestConcurrentSessionAccess(t *testing.T) {
|
||||
// TestProcessNotifyChannel tests the process notification channel.
|
||||
func TestProcessNotifyChannel(t *testing.T) {
|
||||
manager := &Manager{
|
||||
sessions: make(map[int64]*ActiveSession),
|
||||
sessions: make(map[int64]*ActiveSession), //nolint:govet
|
||||
ProcessNotify: make(chan struct{}, 1),
|
||||
}
|
||||
|
||||
@@ -367,7 +367,7 @@ func TestActiveSessionContext(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
|
||||
session := &ActiveSession{
|
||||
SessionDBID: 1,
|
||||
SessionDBID: 1, //nolint:govet
|
||||
ctx: ctx,
|
||||
cancel: cancel,
|
||||
}
|
||||
@@ -438,7 +438,7 @@ func (s *ManagerSuite) TestShutdownAll() {
|
||||
s.manager.sessions[i] = &ActiveSession{
|
||||
SessionDBID: i,
|
||||
Project: "test-project",
|
||||
StartTime: time.Now(),
|
||||
StartTime: time.Now(), //nolint:govet
|
||||
pendingMessages: []PendingMessage{},
|
||||
ctx: ctx,
|
||||
cancel: cancel,
|
||||
@@ -479,7 +479,7 @@ func (s *ManagerSuite) TestDeleteNonExistentSession() {
|
||||
// TestLastPromptNumber tests prompt number tracking.
|
||||
func TestLastPromptNumber(t *testing.T) {
|
||||
session := &ActiveSession{
|
||||
SessionDBID: 1,
|
||||
SessionDBID: 1, //nolint:govet
|
||||
LastPromptNumber: 0,
|
||||
}
|
||||
|
||||
@@ -526,7 +526,7 @@ func TestActiveSessionNotifyChannel(t *testing.T) {
|
||||
// TestMessageMutex tests message mutex operations.
|
||||
func TestMessageMutex(t *testing.T) {
|
||||
session := &ActiveSession{
|
||||
pendingMessages: make([]PendingMessage, 0),
|
||||
pendingMessages: make([]PendingMessage, 0), //nolint:govet
|
||||
}
|
||||
|
||||
var wg sync.WaitGroup
|
||||
@@ -559,7 +559,7 @@ func (s *ManagerSuite) TestQueueDepthMultipleSessions() {
|
||||
}
|
||||
s.manager.sessions[2] = &ActiveSession{
|
||||
SessionDBID: 2,
|
||||
pendingMessages: make([]PendingMessage, 0),
|
||||
pendingMessages: make([]PendingMessage, 0), //nolint:govet
|
||||
}
|
||||
s.manager.sessions[3] = &ActiveSession{
|
||||
SessionDBID: 3,
|
||||
@@ -658,7 +658,7 @@ func TestActiveSessionCWD(t *testing.T) {
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
data := ObservationData{
|
||||
ToolName: "Test",
|
||||
ToolName: "Test", //nolint:govet
|
||||
CWD: tt.cwd,
|
||||
}
|
||||
assert.Equal(t, tt.cwd, data.CWD)
|
||||
@@ -669,22 +669,22 @@ func TestActiveSessionCWD(t *testing.T) {
|
||||
// TestToolInputResponse tests various tool input/response types.
|
||||
func TestToolInputResponse(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
input interface{}
|
||||
response interface{}
|
||||
name string
|
||||
}{
|
||||
{"nil_values", nil, nil},
|
||||
{"string_values", "input string", "response string"},
|
||||
{"map_values", map[string]string{"key": "value"}, map[string]interface{}{"result": true}},
|
||||
{"slice_values", []string{"a", "b"}, []int{1, 2, 3}},
|
||||
{"int_values", 42, 100},
|
||||
{"bool_values", true, false},
|
||||
{name: "nil_values", input: nil, response: nil},
|
||||
{name: "string_values", input: "input string", response: "response string"},
|
||||
{name: "map_values", input: map[string]string{"key": "value"}, response: map[string]interface{}{"result": true}},
|
||||
{name: "slice_values", input: []string{"a", "b"}, response: []int{1, 2, 3}},
|
||||
{name: "int_values", input: 42, response: 100},
|
||||
{name: "bool_values", input: true, response: false},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
data := ObservationData{
|
||||
ToolName: "TestTool",
|
||||
ToolName: "TestTool", //nolint:govet
|
||||
ToolInput: tt.input,
|
||||
ToolResponse: tt.response,
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user