feat(chunking): add AST-aware code chunking for Go, Python, TypeScript

- [x] Add language-specific chunkers with AST parsing (Go, Python, TypeScript)
- [x] Implement chunking manager to dispatch files to appropriate chunkers
- [x] Integrate code chunks into vector sync for semantic search
- [x] Add tree-sitter dependency for Python/TypeScript parsing
- [x] Reorder struct fields for consistency across codebase
- [x] Rename error variables to follow Go conventions (err → unmarshalErr, etc.)
- [x] Add code chunk metadata to vector documents (language, symbol name, line ranges)
- [x] Update worker service to initialize chunking pipeline with all three languages
This commit is contained in:
2026-01-07 13:19:58 +00:00
parent 40a44a71eb
commit 4f4b4ac70f
78 changed files with 2313 additions and 652 deletions
+10 -10
View File
@@ -15,10 +15,10 @@ import (
// Server is the MCP server that exposes search tools.
type Server struct {
searchMgr *search.Manager
version string
stdin io.Reader
stdout io.Writer
searchMgr *search.Manager
version string
}
// NewServer creates a new MCP server.
@@ -41,17 +41,17 @@ type Request struct {
// Response represents a JSON-RPC response.
type Response struct {
JSONRPC string `json:"jsonrpc"`
ID any `json:"id"`
Result any `json:"result,omitempty"`
Error *Error `json:"error,omitempty"`
JSONRPC string `json:"jsonrpc"`
}
// Error represents a JSON-RPC error.
type Error struct {
Code int `json:"code"`
Message string `json:"message"`
Data any `json:"data,omitempty"`
Message string `json:"message"`
Code int `json:"code"`
}
// ToolCallParams represents parameters for tools/call method.
@@ -62,9 +62,9 @@ type ToolCallParams struct {
// Tool represents an MCP tool definition.
type Tool struct {
InputSchema map[string]any `json:"inputSchema"`
Name string `json:"name"`
Description string `json:"description"`
InputSchema map[string]any `json:"inputSchema"`
}
// Run starts the MCP server loop.
@@ -440,17 +440,17 @@ func (s *Server) callTool(ctx context.Context, name string, args json.RawMessage
// TimelineParams represents parameters for timeline operations.
type TimelineParams struct {
AnchorID int64 `json:"anchor_id"`
Query string `json:"query"`
Before int `json:"before"`
After int `json:"after"`
Project string `json:"project"`
ObsType string `json:"obs_type"`
Concepts string `json:"concepts"`
Files string `json:"files"`
Format string `json:"format"`
AnchorID int64 `json:"anchor_id"`
Before int `json:"before"`
After int `json:"after"`
DateStart int64 `json:"dateStart"`
DateEnd int64 `json:"dateEnd"`
Format string `json:"format"`
}
// handleTimeline handles timeline requests.
+13 -15
View File
@@ -34,8 +34,8 @@ func (s *ServerSuite) TestNewServer() {
func TestRequest(t *testing.T) {
tests := []struct {
name string
req Request
expected string
req Request
}{
{
name: "initialize request",
@@ -137,10 +137,10 @@ func TestResponse(t *testing.T) {
// TestError tests Error struct.
func TestError(t *testing.T) {
tests := []struct {
name string
tests := []struct { //nolint:govet
err Error
expected string
name string
}{
{
name: "parse error",
@@ -365,11 +365,11 @@ func TestHandleRequest(t *testing.T) {
ctx := context.Background()
tests := []struct {
name string
req *Request
expectError bool
errorCode int
name string
errorMessage string
errorCode int
expectError bool
}{
{
name: "initialize method",
@@ -683,10 +683,8 @@ func TestRun_MultipleRequests(t *testing.T) {
func TestHandleTimeline_Defaults(t *testing.T) {
// Test that handleTimeline sets default before/after values
params := TimelineParams{
AnchorID: 0,
Query: "",
Before: 0,
After: 0,
Before: 0,
After: 0,
}
// Simulate the default value assignment from handleTimeline
@@ -753,13 +751,13 @@ func TestServerStdinStdoutConfig(t *testing.T) {
// TestResponseIDTypes tests that response IDs can be various types.
func TestResponseIDTypes(t *testing.T) {
tests := []struct {
name string
id any
name string
}{
{"integer id", 1},
{"string id", "abc-123"},
{"float id", 1.5},
{"null id", nil},
{name: "integer id", id: 1},
{name: "string id", id: "abc-123"},
{name: "float id", id: 1.5},
{name: "null id", id: nil},
}
for _, tt := range tests {