Additional abstractions for both sqlite and chroma.

internal/worker/handlers.go

@@ -9,7 +9,9 @@ import (
 	"time"
 
 	"github.com/go-chi/chi/v5"
+	"github.com/lukaszraczylo/claude-mnemonic/internal/db/sqlite"
 	"github.com/lukaszraczylo/claude-mnemonic/internal/privacy"
+	"github.com/lukaszraczylo/claude-mnemonic/internal/vector/chroma"
 	"github.com/lukaszraczylo/claude-mnemonic/internal/worker/sdk"
 	"github.com/lukaszraczylo/claude-mnemonic/internal/worker/session"
 	"github.com/lukaszraczylo/claude-mnemonic/pkg/models"
@@ -22,6 +24,53 @@ const (
 	DefaultObservationsLimit = 100
 
 	// DefaultSummariesLimit is the default number of summaries to return.
+)
+
+// ObservationTypes is the canonical list of observation types.
+// Used by both Go backend and served to frontend.
+var ObservationTypes = []string{
+	"bugfix",
+	"feature",
+	"refactor",
+	"discovery",
+	"decision",
+	"change",
+}
+
+// ConceptTypes is the canonical list of valid concept types.
+// Used by both Go backend and served to frontend.
+var ConceptTypes = []string{
+	// Semantic concepts
+	"how-it-works",
+	"why-it-exists",
+	"what-changed",
+	"problem-solution",
+	"gotcha",
+	"pattern",
+	"trade-off",
+	// Globalizable concepts (from models.GlobalizableConcepts)
+	"best-practice",
+	"anti-pattern",
+	"architecture",
+	"security",
+	"performance",
+	"testing",
+	"debugging",
+	"workflow",
+	"tooling",
+	// Additional useful concepts
+	"refactoring",
+	"api",
+	"database",
+	"configuration",
+	"error-handling",
+	"caching",
+	"logging",
+	"auth",
+	"validation",
+}
+
+const (
 	DefaultSummariesLimit = 50
 
 	// DefaultPromptsLimit is the default number of prompts to return.
@@ -401,25 +450,40 @@ func (s *Service) handleSummarize(w http.ResponseWriter, r *http.Request) {
 }
 
 // handleGetObservations returns recent observations.
+// Supports optional query parameter for semantic search via ChromaDB.
 func (s *Service) handleGetObservations(w http.ResponseWriter, r *http.Request) {
-	limit := DefaultObservationsLimit
-	if l := r.URL.Query().Get("limit"); l != "" {
-		if parsed, err := strconv.Atoi(l); err == nil && parsed > 0 {
-			limit = parsed
-		}
-	}
-
+	limit := sqlite.ParseLimitParam(r, DefaultObservationsLimit)
 	project := r.URL.Query().Get("project")
+	query := r.URL.Query().Get("query")
 
 	var observations []*models.Observation
 	var err error
+	var usedChroma bool
 
-	if project != "" {
-		// Filter by project - includes project-scoped and global observations
-		observations, err = s.observationStore.GetRecentObservations(r.Context(), project, limit)
-	} else {
-		// All projects
-		observations, err = s.observationStore.GetAllRecentObservations(r.Context(), limit)
+	// Use ChromaDB if query is provided and ChromaDB is available
+	if query != "" && s.chromaClient != nil && s.chromaClient.IsConnected() {
+		where := chroma.BuildWhereFilter(chroma.DocTypeObservation, "")
+		chromaResults, chromaErr := s.chromaClient.Query(r.Context(), query, limit*2, where)
+		if chromaErr == nil && len(chromaResults) > 0 {
+			obsIDs := chroma.ExtractObservationIDs(chromaResults, project)
+			if len(obsIDs) > 0 {
+				observations, err = s.observationStore.GetObservationsByIDs(r.Context(), obsIDs, "date_desc", limit)
+				if err == nil {
+					usedChroma = true
+				}
+			}
+		}
+	}
+
+	// Fall back to SQLite if ChromaDB not used
+	if !usedChroma {
+		if project != "" {
+			// Filter by project - includes project-scoped and global observations
+			observations, err = s.observationStore.GetRecentObservations(r.Context(), project, limit)
+		} else {
+			// All projects
+			observations, err = s.observationStore.GetAllRecentObservations(r.Context(), limit)
+		}
 	}
 
 	if err != nil {
@@ -435,23 +499,38 @@ func (s *Service) handleGetObservations(w http.ResponseWriter, r *http.Request)
 }
 
 // handleGetSummaries returns recent summaries.
+// Supports optional query parameter for semantic search via ChromaDB.
 func (s *Service) handleGetSummaries(w http.ResponseWriter, r *http.Request) {
-	limit := DefaultSummariesLimit
-	if l := r.URL.Query().Get("limit"); l != "" {
-		if parsed, err := strconv.Atoi(l); err == nil && parsed > 0 {
-			limit = parsed
-		}
-	}
-
+	limit := sqlite.ParseLimitParam(r, DefaultSummariesLimit)
 	project := r.URL.Query().Get("project")
+	query := r.URL.Query().Get("query")
 
 	var summaries []*models.SessionSummary
 	var err error
+	var usedChroma bool
 
-	if project != "" {
-		summaries, err = s.summaryStore.GetRecentSummaries(r.Context(), project, limit)
-	} else {
-		summaries, err = s.summaryStore.GetAllRecentSummaries(r.Context(), limit)
+	// Use ChromaDB if query is provided and ChromaDB is available
+	if query != "" && s.chromaClient != nil && s.chromaClient.IsConnected() {
+		where := chroma.BuildWhereFilter(chroma.DocTypeSessionSummary, "")
+		chromaResults, chromaErr := s.chromaClient.Query(r.Context(), query, limit*2, where)
+		if chromaErr == nil && len(chromaResults) > 0 {
+			summaryIDs := chroma.ExtractSummaryIDs(chromaResults, project)
+			if len(summaryIDs) > 0 {
+				summaries, err = s.summaryStore.GetSummariesByIDs(r.Context(), summaryIDs, "date_desc", limit)
+				if err == nil {
+					usedChroma = true
+				}
+			}
+		}
+	}
+
+	// Fall back to SQLite if ChromaDB not used
+	if !usedChroma {
+		if project != "" {
+			summaries, err = s.summaryStore.GetRecentSummaries(r.Context(), project, limit)
+		} else {
+			summaries, err = s.summaryStore.GetAllRecentSummaries(r.Context(), limit)
+		}
 	}
 
 	if err != nil {
@@ -467,23 +546,38 @@ func (s *Service) handleGetSummaries(w http.ResponseWriter, r *http.Request) {
 }
 
 // handleGetPrompts returns recent user prompts.
+// Supports optional query parameter for semantic search via ChromaDB.
 func (s *Service) handleGetPrompts(w http.ResponseWriter, r *http.Request) {
-	limit := DefaultPromptsLimit
-	if l := r.URL.Query().Get("limit"); l != "" {
-		if parsed, err := strconv.Atoi(l); err == nil && parsed > 0 {
-			limit = parsed
-		}
-	}
-
+	limit := sqlite.ParseLimitParam(r, DefaultPromptsLimit)
 	project := r.URL.Query().Get("project")
+	query := r.URL.Query().Get("query")
 
 	var prompts []*models.UserPromptWithSession
 	var err error
+	var usedChroma bool
 
-	if project != "" {
-		prompts, err = s.promptStore.GetRecentUserPromptsByProject(r.Context(), project, limit)
-	} else {
-		prompts, err = s.promptStore.GetAllRecentUserPrompts(r.Context(), limit)
+	// Use ChromaDB if query is provided and ChromaDB is available
+	if query != "" && s.chromaClient != nil && s.chromaClient.IsConnected() {
+		where := chroma.BuildWhereFilter(chroma.DocTypeUserPrompt, "")
+		chromaResults, chromaErr := s.chromaClient.Query(r.Context(), query, limit*2, where)
+		if chromaErr == nil && len(chromaResults) > 0 {
+			promptIDs := chroma.ExtractPromptIDs(chromaResults, project)
+			if len(promptIDs) > 0 {
+				prompts, err = s.promptStore.GetPromptsByIDs(r.Context(), promptIDs, "date_desc", limit)
+				if err == nil {
+					usedChroma = true
+				}
+			}
+		}
+	}
+
+	// Fall back to SQLite if ChromaDB not used
+	if !usedChroma {
+		if project != "" {
+			prompts, err = s.promptStore.GetRecentUserPromptsByProject(r.Context(), project, limit)
+		} else {
+			prompts, err = s.promptStore.GetAllRecentUserPrompts(r.Context(), limit)
+		}
 	}
 
 	if err != nil {
@@ -509,6 +603,15 @@ func (s *Service) handleGetProjects(w http.ResponseWriter, r *http.Request) {
 	writeJSON(w, projects)
 }
 
+// handleGetTypes returns the canonical list of observation and concept types.
+// This provides a single source of truth for both backend and frontend.
+func (s *Service) handleGetTypes(w http.ResponseWriter, r *http.Request) {
+	writeJSON(w, map[string]interface{}{
+		"observation_types": ObservationTypes,
+		"concept_types":     ConceptTypes,
+	})
+}
+
 // handleGetStats returns worker statistics.
 func (s *Service) handleGetStats(w http.ResponseWriter, r *http.Request) {
 	retrievalStats := s.GetRetrievalStats()
@@ -576,22 +679,42 @@ func (s *Service) handleSearchByPrompt(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	limit := DefaultSearchLimit
-	if l := r.URL.Query().Get("limit"); l != "" {
-		if parsed, err := strconv.Atoi(l); err == nil && parsed > 0 {
-			limit = parsed
+	limit := sqlite.ParseLimitParam(r, DefaultSearchLimit)
+
+	var observations []*models.Observation
+	var err error
+	var usedChroma bool
+
+	// Try ChromaDB vector search first if available
+	if s.chromaClient != nil && s.chromaClient.IsConnected() {
+		where := chroma.BuildWhereFilter(chroma.DocTypeObservation, "")
+
+		chromaResults, chromaErr := s.chromaClient.Query(r.Context(), query, limit*2, where)
+		if chromaErr == nil && len(chromaResults) > 0 {
+			// Extract observation IDs with project/scope filtering using shared helper
+			obsIDs := chroma.ExtractObservationIDs(chromaResults, project)
+
+			if len(obsIDs) > 0 {
+				// Fetch full observations from SQLite
+				observations, err = s.observationStore.GetObservationsByIDs(r.Context(), obsIDs, "date_desc", limit)
+				if err == nil {
+					usedChroma = true
+				}
+			}
 		}
 	}
 
-	// Search using FTS
-	observations, err := s.observationStore.SearchObservationsFTS(r.Context(), query, project, limit)
-	if err != nil {
-		// FTS might fail if query has special chars, try without
-		log.Warn().Err(err).Str("query", query).Msg("FTS search failed, falling back to recent")
-		observations, err = s.observationStore.GetRecentObservations(r.Context(), project, limit)
+	// Fall back to FTS if ChromaDB not available or returned no results
+	if !usedChroma || len(observations) == 0 {
+		observations, err = s.observationStore.SearchObservationsFTS(r.Context(), query, project, limit)
 		if err != nil {
-			http.Error(w, err.Error(), http.StatusInternalServerError)
-			return
+			// FTS might fail if query has special chars, try without
+			log.Warn().Err(err).Str("query", query).Msg("FTS search failed, falling back to recent")
+			observations, err = s.observationStore.GetRecentObservations(r.Context(), project, limit)
+			if err != nil {
+				http.Error(w, err.Error(), http.StatusInternalServerError)
+				return
+			}
 		}
 	}
 

internal/worker/sdk/parser.go

@@ -27,8 +27,9 @@ var (
 		"decision":  true,
 	}
 
-	// Valid concepts (strict list - no custom tags allowed)
+	// Valid concepts - expanded list matching GlobalizableConcepts and common use cases
 	validConcepts = map[string]bool{
+		// Semantic concepts
 		"how-it-works":     true,
 		"why-it-exists":    true,
 		"what-changed":     true,
@@ -36,6 +37,26 @@ var (
 		"gotcha":           true,
 		"pattern":          true,
 		"trade-off":        true,
+		// Globalizable concepts (from models.GlobalizableConcepts)
+		"best-practice": true,
+		"anti-pattern":  true,
+		"architecture":  true,
+		"security":      true,
+		"performance":   true,
+		"testing":       true,
+		"debugging":     true,
+		"workflow":      true,
+		"tooling":       true,
+		// Additional useful concepts
+		"refactoring":    true,
+		"api":            true,
+		"database":       true,
+		"configuration":  true,
+		"error-handling": true,
+		"caching":        true,
+		"logging":        true,
+		"auth":           true,
+		"validation":     true,
 	}
 )
 

internal/worker/sdk/processor.go

@@ -123,17 +123,9 @@ func (p *Processor) ProcessObservation(ctx context.Context, sdkSessionID, projec
 
 	log.Info().Str("tool", toolName).Msg("Processing tool execution with Claude CLI")
 
-	// Check if we already have observations for this file (skip if covered)
-	if filePath := extractFilePath(toolName, inputStr); filePath != "" {
-		exists, err := p.observationStore.ExistsSimilarObservation(ctx, project, []string{filePath}, nil)
-		if err == nil && exists {
-			log.Debug().
-				Str("tool", toolName).
-				Str("file", filePath).
-				Msg("Skipping - file already has observations")
-			return nil
-		}
-	}
+	// Note: Removed the "file already has observations" check
+	// Each tool execution can produce unique insights even for the same file
+	// Similarity-based deduplication will handle true duplicates
 
 	// Build the prompt
 	exec := ToolExecution{
@@ -413,8 +405,9 @@ func shouldSkipTool(toolName string) bool {
 // shouldSkipTrivialOperation performs local pre-filtering to skip trivial operations
 // without making a Haiku API call. Returns true if the operation is too trivial to process.
 func shouldSkipTrivialOperation(toolName, inputStr, outputStr string) bool {
-	// Skip if output is too small to be meaningful (less than 100 chars)
-	if len(outputStr) < 100 {
+	// Skip if output is too small to be meaningful (less than 50 chars)
+	// Reduced from 100 to capture more meaningful small operations
+	if len(outputStr) < 50 {
 		return true
 	}
 
@@ -477,36 +470,6 @@ func shouldSkipTrivialOperation(toolName, inputStr, outputStr string) bool {
 	return false
 }
 
-// extractFilePath extracts the file path from tool input for deduplication.
-func extractFilePath(toolName, inputStr string) string {
-	if inputStr == "" {
-		return ""
-	}
-
-	var input map[string]interface{}
-	if err := json.Unmarshal([]byte(inputStr), &input); err != nil {
-		return ""
-	}
-
-	// Handle different tool input formats
-	switch toolName {
-	case "Read":
-		if fp, ok := input["file_path"].(string); ok {
-			return fp
-		}
-	case "Grep", "Search":
-		if path, ok := input["path"].(string); ok {
-			return path
-		}
-	case "Edit", "Write":
-		if fp, ok := input["file_path"].(string); ok {
-			return fp
-		}
-	}
-
-	return ""
-}
-
 // toJSONString converts an interface to a JSON string.
 func toJSONString(v interface{}) string {
 	if v == nil {
@@ -767,12 +730,18 @@ func hasMeaningfulContent(assistantMsg string) bool {
 const systemPrompt = `You are a memory extraction agent for Claude Code sessions. Your job is to analyze tool executions and extract meaningful observations that would be useful for future sessions.
 
 GUIDELINES:
-1. Only create observations for SIGNIFICANT learnings - not every tool call needs one
-2. Focus on: decisions made, bugs fixed, patterns discovered, project structure learned
-3. Skip trivial operations like simple file reads without insights
+1. Create observations for any meaningful learnings - be generous, not restrictive
+2. Focus on: decisions made, bugs fixed, patterns discovered, project structure, code changes, refactoring
+3. Even small changes can be worth remembering if they reveal something about the codebase
 4. Be concise but informative in your observations
 5. Use appropriate type tags: decision, bugfix, feature, refactor, discovery, change
 
+CONCEPT TAGS (use 1-3 of these):
+- how-it-works, why-it-exists, what-changed, problem-solution, gotcha
+- pattern, trade-off, best-practice, anti-pattern, architecture
+- security, performance, testing, debugging, workflow, tooling
+- refactoring, api, database, configuration, error-handling
+
 OUTPUT FORMAT:
 When you find something worth remembering, output:
 <observation>
@@ -794,5 +763,7 @@ When you find something worth remembering, output:
 </files_modified>
 </observation>
 
-If the tool execution is not noteworthy, simply respond with:
-<skip reason="not significant"/>`
+If the tool execution is truly trivial (just a directory listing, empty result, etc.), respond with:
+<skip reason="trivial"/>
+
+Prefer creating observations over skipping - memories are valuable for future context!`

internal/worker/service.go

@@ -655,6 +655,7 @@ func (s *Service) setupRoutes() {
 		r.Get("/api/projects", s.handleGetProjects)
 		r.Get("/api/stats", s.handleGetStats)
 		r.Get("/api/stats/retrieval", s.handleGetRetrievalStats)
+		r.Get("/api/types", s.handleGetTypes)
 
 		// Context injection
 		r.Get("/api/context/count", s.handleContextCount)