Move from chroma to sqlitevec with local embedding

internal/vector/sqlitevec/client.go

@@ -0,0 +1,254 @@
+// Package sqlitevec provides sqlite-vec based vector database integration for claude-mnemonic.
+package sqlitevec
+
+import (
+	"context"
+	"database/sql"
+	"fmt"
+	"strings"
+	"sync"
+
+	sqlite_vec "github.com/asg017/sqlite-vec-go-bindings/cgo"
+	"github.com/lukaszraczylo/claude-mnemonic/internal/embedding"
+	"github.com/rs/zerolog/log"
+)
+
+// Client provides vector operations via sqlite-vec.
+type Client struct {
+	db       *sql.DB
+	embedSvc *embedding.Service
+	mu       sync.Mutex
+}
+
+// Config holds configuration for the client.
+type Config struct {
+	DB *sql.DB
+}
+
+// NewClient creates a new sqlite-vec client.
+func NewClient(cfg Config, embedSvc *embedding.Service) (*Client, error) {
+	if cfg.DB == nil {
+		return nil, fmt.Errorf("database connection required")
+	}
+	if embedSvc == nil {
+		return nil, fmt.Errorf("embedding service required")
+	}
+
+	return &Client{
+		db:       cfg.DB,
+		embedSvc: embedSvc,
+	}, nil
+}
+
+// AddDocuments adds documents with their embeddings to the vector store.
+func (c *Client) AddDocuments(ctx context.Context, docs []Document) error {
+	if len(docs) == 0 {
+		return nil
+	}
+
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	// Generate embeddings for all documents
+	texts := make([]string, len(docs))
+	for i, doc := range docs {
+		texts[i] = doc.Content
+	}
+
+	embeddings, err := c.embedSvc.EmbedBatch(texts)
+	if err != nil {
+		return fmt.Errorf("generate embeddings: %w", err)
+	}
+
+	// Insert into vectors table
+	const insertQuery = `
+		INSERT OR REPLACE INTO vectors (doc_id, embedding, sqlite_id, doc_type, field_type, project, scope)
+		VALUES (?, ?, ?, ?, ?, ?, ?)
+	`
+
+	tx, err := c.db.BeginTx(ctx, nil)
+	if err != nil {
+		return fmt.Errorf("begin transaction: %w", err)
+	}
+	defer func() {
+		if err != nil {
+			_ = tx.Rollback()
+		}
+	}()
+
+	stmt, err := tx.PrepareContext(ctx, insertQuery)
+	if err != nil {
+		return fmt.Errorf("prepare statement: %w", err)
+	}
+	defer stmt.Close()
+
+	for i, doc := range docs {
+		// Serialize embedding to blob format
+		embBlob, err := sqlite_vec.SerializeFloat32(embeddings[i])
+		if err != nil {
+			return fmt.Errorf("serialize embedding for %s: %w", doc.ID, err)
+		}
+
+		// Extract metadata
+		sqliteID, _ := doc.Metadata["sqlite_id"].(int64)
+		docType, _ := doc.Metadata["doc_type"].(string)
+		fieldType, _ := doc.Metadata["field_type"].(string)
+		project, _ := doc.Metadata["project"].(string)
+		scope, _ := doc.Metadata["scope"].(string)
+
+		_, err = stmt.ExecContext(ctx,
+			doc.ID,
+			embBlob,
+			sqliteID,
+			docType,
+			fieldType,
+			project,
+			scope,
+		)
+		if err != nil {
+			return fmt.Errorf("insert document %s: %w", doc.ID, err)
+		}
+	}
+
+	if err = tx.Commit(); err != nil {
+		return fmt.Errorf("commit transaction: %w", err)
+	}
+
+	log.Debug().Int("count", len(docs)).Msg("Added documents to sqlite-vec")
+	return nil
+}
+
+// DeleteDocuments removes documents by their IDs.
+func (c *Client) DeleteDocuments(ctx context.Context, ids []string) error {
+	if len(ids) == 0 {
+		return nil
+	}
+
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	// Build placeholder string
+	placeholders := make([]string, len(ids))
+	args := make([]interface{}, len(ids))
+	for i, id := range ids {
+		placeholders[i] = "?"
+		args[i] = id
+	}
+
+	// #nosec G201 -- Placeholders are "?" strings, actual values are parameterized via args
+	query := fmt.Sprintf("DELETE FROM vectors WHERE doc_id IN (%s)",
+		strings.Join(placeholders, ","))
+
+	_, err := c.db.ExecContext(ctx, query, args...)
+	if err != nil {
+		return fmt.Errorf("delete documents: %w", err)
+	}
+
+	log.Debug().Int("count", len(ids)).Msg("Deleted documents from sqlite-vec")
+	return nil
+}
+
+// Query performs a vector similarity search.
+func (c *Client) Query(ctx context.Context, query string, limit int, where map[string]any) ([]QueryResult, error) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	// Generate query embedding
+	queryEmb, err := c.embedSvc.Embed(query)
+	if err != nil {
+		return nil, fmt.Errorf("embed query: %w", err)
+	}
+
+	// Serialize query embedding
+	queryBlob, err := sqlite_vec.SerializeFloat32(queryEmb)
+	if err != nil {
+		return nil, fmt.Errorf("serialize query embedding: %w", err)
+	}
+
+	// Build query with filters
+	// vec0 supports WHERE clauses on metadata columns
+	args := []interface{}{queryBlob}
+
+	sqlQuery := `
+		SELECT
+			doc_id,
+			distance,
+			sqlite_id,
+			doc_type,
+			field_type,
+			project,
+			scope
+		FROM vectors
+		WHERE embedding MATCH ?
+	`
+
+	// Add filters - these work with vec0 metadata columns
+	if docType, ok := where["doc_type"].(string); ok && docType != "" {
+		sqlQuery += " AND doc_type = ?"
+		args = append(args, docType)
+	}
+	if project, ok := where["project"].(string); ok && project != "" {
+		// Include project-specific OR global scope
+		sqlQuery += " AND (project = ? OR scope = 'global')"
+		args = append(args, project)
+	}
+
+	sqlQuery += " ORDER BY distance LIMIT ?"
+	args = append(args, limit)
+
+	rows, err := c.db.QueryContext(ctx, sqlQuery, args...)
+	if err != nil {
+		return nil, fmt.Errorf("query vectors: %w", err)
+	}
+	defer rows.Close()
+
+	var results []QueryResult
+	for rows.Next() {
+		var r QueryResult
+		var sqliteID int64
+		var docType, fieldType, project, scope sql.NullString
+
+		if err := rows.Scan(&r.ID, &r.Distance, &sqliteID, &docType, &fieldType, &project, &scope); err != nil {
+			return nil, fmt.Errorf("scan row: %w", err)
+		}
+
+		r.Metadata = map[string]any{
+			"sqlite_id":  float64(sqliteID), // Keep as float64 for compatibility
+			"doc_type":   docType.String,
+			"field_type": fieldType.String,
+			"project":    project.String,
+			"scope":      scope.String,
+		}
+
+		results = append(results, r)
+	}
+
+	if err = rows.Err(); err != nil {
+		return nil, fmt.Errorf("iterate rows: %w", err)
+	}
+
+	log.Debug().
+		Str("query", truncateString(query, 50)).
+		Int("results", len(results)).
+		Msg("Vector search completed")
+
+	return results, nil
+}
+
+// IsConnected always returns true (no external process).
+func (c *Client) IsConnected() bool {
+	return c.db != nil
+}
+
+// Close is a no-op (db managed externally).
+func (c *Client) Close() error {
+	return nil
+}
+
+// truncateString truncates a string to maxLen characters.
+func truncateString(s string, maxLen int) string {
+	if len(s) <= maxLen {
+		return s
+	}
+	return s[:maxLen] + "..."
+}

internal/vector/sqlitevec/helpers.go

@@ -0,0 +1,234 @@
+// Package sqlitevec provides sqlite-vec based vector database integration for claude-mnemonic.
+package sqlitevec
+
+// DocType represents the type of document stored in the vector table.
+type DocType string
+
+const (
+	DocTypeObservation    DocType = "observation"
+	DocTypeSessionSummary DocType = "session_summary"
+	DocTypeUserPrompt     DocType = "user_prompt"
+)
+
+// Document represents a document to store with vector embedding.
+type Document struct {
+	ID       string
+	Content  string
+	Metadata map[string]any
+}
+
+// QueryResult represents a search result from vector search.
+type QueryResult struct {
+	ID       string
+	Distance float64
+	Metadata map[string]any
+}
+
+// ExtractedIDs contains SQLite IDs extracted from query results, grouped by document type.
+type ExtractedIDs struct {
+	ObservationIDs []int64
+	SummaryIDs     []int64
+	PromptIDs      []int64
+}
+
+// BuildWhereFilter creates a where filter map for vector queries.
+// If docType is empty, no doc_type filter is added.
+func BuildWhereFilter(docType DocType, project string) map[string]interface{} {
+	where := make(map[string]interface{})
+	if docType != "" {
+		where["doc_type"] = string(docType)
+	}
+	if project != "" {
+		where["project"] = project
+	}
+	return where
+}
+
+// ExtractIDsByDocType extracts SQLite IDs from query results,
+// grouped by document type and deduplicated.
+func ExtractIDsByDocType(results []QueryResult) *ExtractedIDs {
+	ids := &ExtractedIDs{}
+	seenObs := make(map[int64]bool)
+	seenSummary := make(map[int64]bool)
+	seenPrompt := make(map[int64]bool)
+
+	for _, result := range results {
+		sqliteID, ok := result.Metadata["sqlite_id"].(float64)
+		if !ok {
+			// Try int64 directly
+			if id, ok := result.Metadata["sqlite_id"].(int64); ok {
+				sqliteID = float64(id)
+			} else {
+				continue
+			}
+		}
+		id := int64(sqliteID)
+
+		docType, _ := result.Metadata["doc_type"].(string)
+		switch docType {
+		case string(DocTypeObservation):
+			if !seenObs[id] {
+				seenObs[id] = true
+				ids.ObservationIDs = append(ids.ObservationIDs, id)
+			}
+		case string(DocTypeSessionSummary):
+			if !seenSummary[id] {
+				seenSummary[id] = true
+				ids.SummaryIDs = append(ids.SummaryIDs, id)
+			}
+		case string(DocTypeUserPrompt):
+			if !seenPrompt[id] {
+				seenPrompt[id] = true
+				ids.PromptIDs = append(ids.PromptIDs, id)
+			}
+		}
+	}
+
+	return ids
+}
+
+// ExtractObservationIDs extracts observation SQLite IDs from query results,
+// optionally filtering by project or including global scope.
+func ExtractObservationIDs(results []QueryResult, project string) []int64 {
+	var ids []int64
+	seen := make(map[int64]bool)
+
+	for _, result := range results {
+		sqliteID, ok := result.Metadata["sqlite_id"].(float64)
+		if !ok {
+			if id, ok := result.Metadata["sqlite_id"].(int64); ok {
+				sqliteID = float64(id)
+			} else {
+				continue
+			}
+		}
+		id := int64(sqliteID)
+
+		docType, _ := result.Metadata["doc_type"].(string)
+		if docType != string(DocTypeObservation) {
+			continue
+		}
+
+		if project != "" {
+			proj, _ := result.Metadata["project"].(string)
+			scope, _ := result.Metadata["scope"].(string)
+			if proj != project && scope != "global" {
+				continue
+			}
+		}
+
+		if !seen[id] {
+			seen[id] = true
+			ids = append(ids, id)
+		}
+	}
+
+	return ids
+}
+
+// ExtractSummaryIDs extracts session summary SQLite IDs from query results.
+func ExtractSummaryIDs(results []QueryResult, project string) []int64 {
+	var ids []int64
+	seen := make(map[int64]bool)
+
+	for _, result := range results {
+		sqliteID, ok := result.Metadata["sqlite_id"].(float64)
+		if !ok {
+			if id, ok := result.Metadata["sqlite_id"].(int64); ok {
+				sqliteID = float64(id)
+			} else {
+				continue
+			}
+		}
+		id := int64(sqliteID)
+
+		docType, _ := result.Metadata["doc_type"].(string)
+		if docType != string(DocTypeSessionSummary) {
+			continue
+		}
+
+		if project != "" {
+			proj, _ := result.Metadata["project"].(string)
+			if proj != project {
+				continue
+			}
+		}
+
+		if !seen[id] {
+			seen[id] = true
+			ids = append(ids, id)
+		}
+	}
+
+	return ids
+}
+
+// ExtractPromptIDs extracts user prompt SQLite IDs from query results.
+func ExtractPromptIDs(results []QueryResult, project string) []int64 {
+	var ids []int64
+	seen := make(map[int64]bool)
+
+	for _, result := range results {
+		sqliteID, ok := result.Metadata["sqlite_id"].(float64)
+		if !ok {
+			if id, ok := result.Metadata["sqlite_id"].(int64); ok {
+				sqliteID = float64(id)
+			} else {
+				continue
+			}
+		}
+		id := int64(sqliteID)
+
+		docType, _ := result.Metadata["doc_type"].(string)
+		if docType != string(DocTypeUserPrompt) {
+			continue
+		}
+
+		if project != "" {
+			proj, _ := result.Metadata["project"].(string)
+			if proj != project {
+				continue
+			}
+		}
+
+		if !seen[id] {
+			seen[id] = true
+			ids = append(ids, id)
+		}
+	}
+
+	return ids
+}
+
+// Helper functions for metadata manipulation
+
+func copyMetadata(base map[string]any, key string, value any) map[string]any {
+	result := make(map[string]any, len(base)+1)
+	for k, v := range base {
+		result[k] = v
+	}
+	result[key] = value
+	return result
+}
+
+func copyMetadataMulti(base map[string]any, extra map[string]any) map[string]any {
+	result := make(map[string]any, len(base)+len(extra))
+	for k, v := range base {
+		result[k] = v
+	}
+	for k, v := range extra {
+		result[k] = v
+	}
+	return result
+}
+
+func joinStrings(strs []string, sep string) string {
+	if len(strs) == 0 {
+		return ""
+	}
+	result := strs[0]
+	for i := 1; i < len(strs); i++ {
+		result += sep + strs[i]
+	}
+	return result
+}

internal/vector/sqlitevec/sync.go

@@ -0,0 +1,242 @@
+// Package sqlitevec provides sqlite-vec based vector database integration for claude-mnemonic.
+package sqlitevec
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/lukaszraczylo/claude-mnemonic/pkg/models"
+	"github.com/rs/zerolog/log"
+)
+
+// Sync provides synchronization between SQLite data and vector embeddings.
+type Sync struct {
+	client *Client
+}
+
+// NewSync creates a new sync service.
+func NewSync(client *Client) *Sync {
+	return &Sync{client: client}
+}
+
+// SyncObservation syncs a single observation to the vector store.
+func (s *Sync) SyncObservation(ctx context.Context, obs *models.Observation) error {
+	docs := s.formatObservationDocs(obs)
+	if len(docs) == 0 {
+		return nil
+	}
+
+	if err := s.client.AddDocuments(ctx, docs); err != nil {
+		return fmt.Errorf("add observation docs: %w", err)
+	}
+
+	log.Debug().
+		Int64("observationId", obs.ID).
+		Int("docCount", len(docs)).
+		Msg("Synced observation to sqlite-vec")
+
+	return nil
+}
+
+// formatObservationDocs formats an observation into vector documents.
+// Each semantic field becomes a separate vector document (granular approach).
+func (s *Sync) formatObservationDocs(obs *models.Observation) []Document {
+	docs := make([]Document, 0, len(obs.Facts)+2)
+
+	// Determine scope for metadata
+	scope := string(obs.Scope)
+	if scope == "" {
+		scope = "project"
+	}
+
+	baseMetadata := map[string]any{
+		"sqlite_id":        obs.ID,
+		"doc_type":         "observation",
+		"sdk_session_id":   obs.SDKSessionID,
+		"project":          obs.Project,
+		"scope":            scope,
+		"created_at_epoch": obs.CreatedAtEpoch,
+		"type":             string(obs.Type),
+	}
+
+	if obs.Title.Valid {
+		baseMetadata["title"] = obs.Title.String
+	}
+	if obs.Subtitle.Valid {
+		baseMetadata["subtitle"] = obs.Subtitle.String
+	}
+	if len(obs.Concepts) > 0 {
+		baseMetadata["concepts"] = joinStrings(obs.Concepts, ",")
+	}
+	if len(obs.FilesRead) > 0 {
+		baseMetadata["files_read"] = joinStrings(obs.FilesRead, ",")
+	}
+	if len(obs.FilesModified) > 0 {
+		baseMetadata["files_modified"] = joinStrings(obs.FilesModified, ",")
+	}
+
+	// Narrative as separate document
+	if obs.Narrative.Valid && obs.Narrative.String != "" {
+		docs = append(docs, Document{
+			ID:       fmt.Sprintf("obs_%d_narrative", obs.ID),
+			Content:  obs.Narrative.String,
+			Metadata: copyMetadata(baseMetadata, "field_type", "narrative"),
+		})
+	}
+
+	// Each fact as separate document
+	for i, fact := range obs.Facts {
+		docs = append(docs, Document{
+			ID:      fmt.Sprintf("obs_%d_fact_%d", obs.ID, i),
+			Content: fact,
+			Metadata: copyMetadataMulti(baseMetadata, map[string]any{
+				"field_type": "fact",
+				"fact_index": i,
+			}),
+		})
+	}
+
+	return docs
+}
+
+// SyncSummary syncs a single session summary to the vector store.
+func (s *Sync) SyncSummary(ctx context.Context, summary *models.SessionSummary) error {
+	docs := s.formatSummaryDocs(summary)
+	if len(docs) == 0 {
+		return nil
+	}
+
+	if err := s.client.AddDocuments(ctx, docs); err != nil {
+		return fmt.Errorf("add summary docs: %w", err)
+	}
+
+	log.Debug().
+		Int64("summaryId", summary.ID).
+		Int("docCount", len(docs)).
+		Msg("Synced summary to sqlite-vec")
+
+	return nil
+}
+
+// formatSummaryDocs formats a session summary into vector documents.
+func (s *Sync) formatSummaryDocs(summary *models.SessionSummary) []Document {
+	docs := make([]Document, 0, 6)
+
+	baseMetadata := map[string]any{
+		"sqlite_id":        summary.ID,
+		"doc_type":         "session_summary",
+		"sdk_session_id":   summary.SDKSessionID,
+		"project":          summary.Project,
+		"scope":            "", // Summaries don't have scope
+		"created_at_epoch": summary.CreatedAtEpoch,
+	}
+
+	if summary.PromptNumber.Valid {
+		baseMetadata["prompt_number"] = summary.PromptNumber.Int64
+	}
+
+	// Each field as separate document
+	fields := []struct {
+		name  string
+		value string
+		valid bool
+	}{
+		{"request", summary.Request.String, summary.Request.Valid},
+		{"investigated", summary.Investigated.String, summary.Investigated.Valid},
+		{"learned", summary.Learned.String, summary.Learned.Valid},
+		{"completed", summary.Completed.String, summary.Completed.Valid},
+		{"next_steps", summary.NextSteps.String, summary.NextSteps.Valid},
+		{"notes", summary.Notes.String, summary.Notes.Valid},
+	}
+
+	for _, field := range fields {
+		if field.valid && field.value != "" {
+			docs = append(docs, Document{
+				ID:       fmt.Sprintf("summary_%d_%s", summary.ID, field.name),
+				Content:  field.value,
+				Metadata: copyMetadata(baseMetadata, "field_type", field.name),
+			})
+		}
+	}
+
+	return docs
+}
+
+// SyncUserPrompt syncs a single user prompt to the vector store.
+func (s *Sync) SyncUserPrompt(ctx context.Context, prompt *models.UserPromptWithSession) error {
+	doc := Document{
+		ID:      fmt.Sprintf("prompt_%d", prompt.ID),
+		Content: prompt.PromptText,
+		Metadata: map[string]any{
+			"sqlite_id":        prompt.ID,
+			"doc_type":         "user_prompt",
+			"sdk_session_id":   prompt.SDKSessionID,
+			"project":          prompt.Project,
+			"scope":            "", // Prompts don't have scope
+			"created_at_epoch": prompt.CreatedAtEpoch,
+			"prompt_number":    prompt.PromptNumber,
+			"field_type":       "prompt",
+		},
+	}
+
+	if err := s.client.AddDocuments(ctx, []Document{doc}); err != nil {
+		return fmt.Errorf("add prompt doc: %w", err)
+	}
+
+	log.Debug().
+		Int64("promptId", prompt.ID).
+		Msg("Synced user prompt to sqlite-vec")
+
+	return nil
+}
+
+// DeleteObservations removes observation documents from the vector store.
+func (s *Sync) DeleteObservations(ctx context.Context, observationIDs []int64) error {
+	if len(observationIDs) == 0 {
+		return nil
+	}
+
+	// Generate all possible document IDs for these observations
+	// Pattern: obs_{id}_narrative, obs_{id}_fact_{0..n}
+	const maxFactsPerObs = 20
+	ids := make([]string, 0, len(observationIDs)*(maxFactsPerObs+1))
+
+	for _, obsID := range observationIDs {
+		ids = append(ids, fmt.Sprintf("obs_%d_narrative", obsID))
+		for i := 0; i < maxFactsPerObs; i++ {
+			ids = append(ids, fmt.Sprintf("obs_%d_fact_%d", obsID, i))
+		}
+	}
+
+	if err := s.client.DeleteDocuments(ctx, ids); err != nil {
+		return fmt.Errorf("delete observation docs: %w", err)
+	}
+
+	log.Debug().
+		Int("observationCount", len(observationIDs)).
+		Msg("Deleted observations from sqlite-vec")
+
+	return nil
+}
+
+// DeleteUserPrompts removes user prompt documents from the vector store.
+func (s *Sync) DeleteUserPrompts(ctx context.Context, promptIDs []int64) error {
+	if len(promptIDs) == 0 {
+		return nil
+	}
+
+	ids := make([]string, len(promptIDs))
+	for i, promptID := range promptIDs {
+		ids[i] = fmt.Sprintf("prompt_%d", promptID)
+	}
+
+	if err := s.client.DeleteDocuments(ctx, ids); err != nil {
+		return fmt.Errorf("delete prompt docs: %w", err)
+	}
+
+	log.Debug().
+		Int("promptCount", len(promptIDs)).
+		Msg("Deleted user prompts from sqlite-vec")
+
+	return nil
+}