lukaszraczylo/claude-mnemonic
1
0
Fork 0
mirror of https://github.com/lukaszraczylo/claude-mnemonic.git synced 2026-10-20 06:27:50 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(chunking): add AST-aware code chunking for Go, Python, TypeScript

Browse Source 
- [x] Add language-specific chunkers with AST parsing (Go, Python, TypeScript)
- [x] Implement chunking manager to dispatch files to appropriate chunkers
- [x] Integrate code chunks into vector sync for semantic search
- [x] Add tree-sitter dependency for Python/TypeScript parsing
- [x] Reorder struct fields for consistency across codebase
- [x] Rename error variables to follow Go conventions (err → unmarshalErr, etc.)
- [x] Add code chunk metadata to vector documents (language, symbol name, line ranges)
- [x] Update worker service to initialize chunking pipeline with all three languages
This commit is contained in:
lukaszraczylo
2026-01-07 13:19:58 +00:00
parent 40a44a71eb
commit 4f4b4ac70f
78 changed files with 2313 additions and 652 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/worker/sdk/parser_test.go
+2 -2
View File
@@ -77,10 +77,10 @@ func TestParseObservations_TableDriven(t *testing.T) {
tests := []struct {
name string
input string
expectedCount int
expectedType models.ObservationType
expectedTitle string
checkConcepts []string
expectedCount int
}{
{
name: "valid_bugfix_observation",
@@ -300,9 +300,9 @@ func TestParseSummary_TableDriven(t *testing.T) {
tests := []struct {
name string
input string
expectedRequest string
sessionID int64
expectNil bool
expectedRequest string
}{
{
name: "empty_input",
internal/worker/sdk/processor.go
+3 -4
View File
@@ -31,15 +31,14 @@ type SyncSummaryFunc func(summary *models.SessionSummary)
// Processor handles SDK agent processing of observations and summaries using Claude Code CLI.
type Processor struct {
claudePath string
model string
observationStore *sqlite.ObservationStore
summaryStore *sqlite.SummaryStore
broadcastFunc BroadcastFunc
syncObservationFunc SyncObservationFunc
syncSummaryFunc SyncSummaryFunc
// Semaphore to limit concurrent Claude CLI calls (prevents API overload)
sem chan struct{}
sem chan struct{}
claudePath string
model string
}
// SetBroadcastFunc sets the broadcast callback for SSE events.
internal/worker/sdk/processor_test.go
+10 -10
View File
@@ -11,8 +11,8 @@ import (
func TestIsSelfReferentialSummary(t *testing.T) {
tests := []struct {
name string
summary *models.ParsedSummary
name string
expected bool
}{
{
@@ -281,8 +281,8 @@ func TestTruncateForLog(t *testing.T) {
tests := []struct {
name string
input string
maxLen int
expected string
maxLen int
}{
{
name: "shorter_than_max",
@@ -371,11 +371,11 @@ func TestCaptureFileMtimes(t *testing.T) {
file1 := filepath.Join(tmpDir, "file1.txt")
file2 := filepath.Join(tmpDir, "file2.txt")
err = os.WriteFile(file1, []byte("content1"), 0644)
err = os.WriteFile(file1, []byte("content1"), 0600)
if err != nil {
t.Fatal(err)
}
err = os.WriteFile(file2, []byte("content2"), 0644)
err = os.WriteFile(file2, []byte("content2"), 0600)
if err != nil {
t.Fatal(err)
}
@@ -415,7 +415,7 @@ func TestGetFileMtimes(t *testing.T) {
defer os.RemoveAll(tmpDir)
testFile := filepath.Join(tmpDir, "test.txt")
err = os.WriteFile(testFile, []byte("content"), 0644)
err = os.WriteFile(testFile, []byte("content"), 0600)
if err != nil {
t.Fatal(err)
}
@@ -437,7 +437,7 @@ func TestGetFileContent(t *testing.T) {
t.Run("reads_existing_file", func(t *testing.T) {
testFile := filepath.Join(tmpDir, "test.txt")
content := "test content"
err := os.WriteFile(testFile, []byte(content), 0644)
err := os.WriteFile(testFile, []byte(content), 0600)
if err != nil {
t.Fatal(err)
}
@@ -459,7 +459,7 @@ func TestGetFileContent(t *testing.T) {
for i := 0; i < 3000; i++ {
longContent += "x"
}
err := os.WriteFile(testFile, []byte(longContent), 0644)
err := os.WriteFile(testFile, []byte(longContent), 0600)
if err != nil {
t.Fatal(err)
}
@@ -473,7 +473,7 @@ func TestGetFileContent(t *testing.T) {
t.Run("resolves_relative_path_with_cwd", func(t *testing.T) {
testFile := filepath.Join(tmpDir, "relative.txt")
content := "relative content"
err := os.WriteFile(testFile, []byte(content), 0644)
err := os.WriteFile(testFile, []byte(content), 0600)
if err != nil {
t.Fatal(err)
}
@@ -719,8 +719,8 @@ func TestShouldSkipTrivialOperation_EdgeCases(t *testing.T) {
// TestIsSelfReferentialSummary_MoreCases tests additional self-referential detection cases.
func TestIsSelfReferentialSummary_MoreCases(t *testing.T) {
tests := []struct {
name string
summary *models.ParsedSummary
name string
expected bool
}{
{
@@ -932,7 +932,7 @@ func TestCaptureFileMtimes_DuplicatePaths(t *testing.T) {
defer os.RemoveAll(tmpDir)
testFile := filepath.Join(tmpDir, "shared.txt")
err = os.WriteFile(testFile, []byte("content"), 0644)
err = os.WriteFile(testFile, []byte("content"), 0600)
if err != nil {
t.Fatal(err)
}
internal/worker/sdk/prompts.go
+3 -3
View File
@@ -24,12 +24,12 @@ var ObservationConcepts = []string{
// ToolExecution represents a tool execution for observation.
type ToolExecution struct {
ID int64
ToolName string
ToolInput string
ToolOutput string
CreatedAtEpoch int64
CWD string
ID int64
CreatedAtEpoch int64
}
// BuildObservationPrompt builds a prompt for processing a tool observation.
@@ -67,12 +67,12 @@ func BuildObservationPrompt(exec ToolExecution) string {
// SummaryRequest contains data for building a summary prompt.
type SummaryRequest struct {
SessionDBID int64
SDKSessionID string
Project string
UserPrompt string
LastUserMessage string
LastAssistantMessage string
SessionDBID int64
}
// BuildSummaryPrompt builds a prompt requesting a session summary.
internal/worker/sdk/prompts_test.go
+3 -3
View File
@@ -12,8 +12,8 @@ func TestTruncate(t *testing.T) {
tests := []struct {
name string
input string
maxLen int
expected string
maxLen int
}{
{
name: "shorter_than_max",
@@ -58,10 +58,10 @@ func TestTruncate(t *testing.T) {
func TestBuildObservationPrompt(t *testing.T) {
now := time.Now().UnixMilli()
tests := []struct {
name string
tests := []struct { //nolint:govet
exec ToolExecution
contains []string
name string
}{
{
name: "basic_read_tool",