Additional abstractions for both sqlite and chroma.

internal/db/sqlite/helpers.go

@@ -0,0 +1,160 @@
+// Package sqlite provides SQLite database operations for claude-mnemonic.
+package sqlite
+
+import (
+	"context"
+	"database/sql"
+	"net/http"
+	"strconv"
+	"time"
+
+	"github.com/lukaszraczylo/claude-mnemonic/pkg/models"
+)
+
+// EnsureSessionExists creates a session if it doesn't exist.
+// This is shared between ObservationStore and SummaryStore to avoid duplication.
+func EnsureSessionExists(ctx context.Context, store *Store, sdkSessionID, project string) error {
+	const checkQuery = `SELECT id FROM sdk_sessions WHERE sdk_session_id = ?`
+	var id int64
+	err := store.QueryRowContext(ctx, checkQuery, sdkSessionID).Scan(&id)
+	if err == nil {
+		return nil // Session exists
+	}
+	if err != sql.ErrNoRows {
+		return err
+	}
+
+	// Auto-create session
+	now := time.Now()
+	const insertQuery = `
+		INSERT INTO sdk_sessions
+		(claude_session_id, sdk_session_id, project, started_at, started_at_epoch, status)
+		VALUES (?, ?, ?, ?, ?, 'active')
+	`
+	_, err = store.ExecContext(ctx, insertQuery,
+		sdkSessionID, sdkSessionID, project,
+		now.Format(time.RFC3339), now.UnixMilli(),
+	)
+	return err
+}
+
+// nullString converts a string to sql.NullString.
+func nullString(s string) sql.NullString {
+	return sql.NullString{String: s, Valid: s != ""}
+}
+
+// nullInt converts an int to sql.NullInt64.
+func nullInt(i int) sql.NullInt64 {
+	return sql.NullInt64{Int64: int64(i), Valid: i > 0}
+}
+
+// repeatPlaceholders generates n comma-prefixed placeholders for SQL IN clauses.
+// e.g., repeatPlaceholders(2) returns ", ?, ?"
+func repeatPlaceholders(n int) string {
+	if n <= 0 {
+		return ""
+	}
+	result := ""
+	for i := 0; i < n; i++ {
+		result += ", ?"
+	}
+	return result
+}
+
+// int64SliceToInterface converts []int64 to []interface{} for SQL queries.
+func int64SliceToInterface(ids []int64) []interface{} {
+	args := make([]interface{}, len(ids))
+	for i, id := range ids {
+		args[i] = id
+	}
+	return args
+}
+
+// ParseLimitParam parses a limit query parameter with a default value.
+func ParseLimitParam(r *http.Request, defaultLimit int) int {
+	if l := r.URL.Query().Get("limit"); l != "" {
+		if parsed, err := strconv.Atoi(l); err == nil && parsed > 0 {
+			return parsed
+		}
+	}
+	return defaultLimit
+}
+
+// scanSummary scans a single summary from a row scanner.
+func scanSummary(scanner interface{ Scan(...interface{}) error }) (*models.SessionSummary, error) {
+	var summary models.SessionSummary
+	if err := scanner.Scan(
+		&summary.ID, &summary.SDKSessionID, &summary.Project,
+		&summary.Request, &summary.Investigated, &summary.Learned, &summary.Completed,
+		&summary.NextSteps, &summary.Notes, &summary.PromptNumber, &summary.DiscoveryTokens,
+		&summary.CreatedAt, &summary.CreatedAtEpoch,
+	); err != nil {
+		return nil, err
+	}
+	return &summary, nil
+}
+
+// scanSummaryRows scans multiple summaries from rows.
+func scanSummaryRows(rows *sql.Rows) ([]*models.SessionSummary, error) {
+	var summaries []*models.SessionSummary
+	for rows.Next() {
+		summary, err := scanSummary(rows)
+		if err != nil {
+			return nil, err
+		}
+		summaries = append(summaries, summary)
+	}
+	return summaries, rows.Err()
+}
+
+// scanPromptWithSession scans a single prompt with session info from a row scanner.
+func scanPromptWithSession(scanner interface{ Scan(...interface{}) error }) (*models.UserPromptWithSession, error) {
+	var prompt models.UserPromptWithSession
+	if err := scanner.Scan(
+		&prompt.ID, &prompt.ClaudeSessionID, &prompt.PromptNumber, &prompt.PromptText,
+		&prompt.MatchedObservations, &prompt.CreatedAt, &prompt.CreatedAtEpoch,
+		&prompt.Project, &prompt.SDKSessionID,
+	); err != nil {
+		return nil, err
+	}
+	return &prompt, nil
+}
+
+// scanPromptWithSessionRows scans multiple prompts with session info from rows.
+func scanPromptWithSessionRows(rows *sql.Rows) ([]*models.UserPromptWithSession, error) {
+	var prompts []*models.UserPromptWithSession
+	for rows.Next() {
+		prompt, err := scanPromptWithSession(rows)
+		if err != nil {
+			return nil, err
+		}
+		prompts = append(prompts, prompt)
+	}
+	return prompts, rows.Err()
+}
+
+// BuildGetByIDsQuery builds a query for fetching records by IDs with optional ordering and limit.
+// Returns the query string and args slice.
+func BuildGetByIDsQuery(baseQuery string, ids []int64, orderBy string, limit int) (string, []interface{}) {
+	// Build query with placeholders
+	// #nosec G202 -- query uses parameterized placeholders, not user input
+	query := baseQuery + ` WHERE id IN (?` + repeatPlaceholders(len(ids)-1) + `)
+		ORDER BY created_at_epoch `
+
+	if orderBy == "date_asc" {
+		query += "ASC"
+	} else {
+		query += "DESC"
+	}
+
+	if limit > 0 {
+		query += " LIMIT ?"
+	}
+
+	args := int64SliceToInterface(ids)
+	if limit > 0 {
+		args = append(args, limit)
+	}
+
+	return query, args
+}

internal/db/sqlite/observation.go

@@ -91,28 +91,7 @@ func (s *ObservationStore) StoreObservation(ctx context.Context, sdkSessionID, p
 
 // ensureSessionExists creates a session if it doesn't exist.
 func (s *ObservationStore) ensureSessionExists(ctx context.Context, sdkSessionID, project string) error {
-	const checkQuery = `SELECT id FROM sdk_sessions WHERE sdk_session_id = ?`
-	var id int64
-	err := s.store.QueryRowContext(ctx, checkQuery, sdkSessionID).Scan(&id)
-	if err == nil {
-		return nil // Session exists
-	}
-	if err != sql.ErrNoRows {
-		return err
-	}
-
-	// Auto-create session
-	now := time.Now()
-	const insertQuery = `
-		INSERT INTO sdk_sessions
-		(claude_session_id, sdk_session_id, project, started_at, started_at_epoch, status)
-		VALUES (?, ?, ?, ?, ?, 'active')
-	`
-	_, err = s.store.ExecContext(ctx, insertQuery,
-		sdkSessionID, sdkSessionID, project,
-		now.Format(time.RFC3339), now.UnixMilli(),
-	)
-	return err
+	return EnsureSessionExists(ctx, s.store, sdkSessionID, project)
 }
 
 // GetObservationByID retrieves an observation by ID.
@@ -159,10 +138,7 @@ func (s *ObservationStore) GetObservationsByIDs(ctx context.Context, ids []int64
 	}
 
 	// Convert []int64 to []interface{}
-	args := make([]interface{}, len(ids))
-	for i, id := range ids {
-		args[i] = id
-	}
+	args := int64SliceToInterface(ids)
 	if limit > 0 {
 		args = append(args, limit)
 	}
@@ -355,37 +331,6 @@ func extractKeywords(query string) []string {
 	return keywords
 }
 
-// ExistsSimilarObservation checks if an observation about the same files exists for a project.
-// Used to prevent duplicate observations when re-reading the same files.
-func (s *ObservationStore) ExistsSimilarObservation(ctx context.Context, project string, filesRead, filesModified []string) (bool, error) {
-	// If no files tracked, can't deduplicate
-	if len(filesRead) == 0 && len(filesModified) == 0 {
-		return false, nil
-	}
-
-	// Check if any observation exists with the same primary file
-	// Use the first file as the key identifier
-	var primaryFile string
-	if len(filesRead) > 0 {
-		primaryFile = filesRead[0]
-	} else if len(filesModified) > 0 {
-		primaryFile = filesModified[0]
-	}
-
-	const query = `
-		SELECT COUNT(*) FROM observations
-		WHERE project = ? AND (files_read LIKE ? OR files_modified LIKE ?)
-	`
-	pattern := "%" + primaryFile + "%"
-
-	var count int
-	err := s.store.QueryRowContext(ctx, query, project, pattern, pattern).Scan(&count)
-	if err != nil {
-		return false, err
-	}
-	return count > 0, nil
-}
-
 // DeleteObservations deletes multiple observations by ID.
 func (s *ObservationStore) DeleteObservations(ctx context.Context, ids []int64) (int64, error) {
 	if len(ids) == 0 {
@@ -497,21 +442,4 @@ func scanObservationRows(rows *sql.Rows) ([]*models.Observation, error) {
 	return observations, rows.Err()
 }
 
-func nullString(s string) sql.NullString {
-	return sql.NullString{String: s, Valid: s != ""}
-}
-
-func nullInt(i int) sql.NullInt64 {
-	return sql.NullInt64{Int64: int64(i), Valid: i > 0}
-}
-
-func repeatPlaceholders(n int) string {
-	if n <= 0 {
-		return ""
-	}
-	result := ""
-	for i := 0; i < n; i++ {
-		result += ", ?"
-	}
-	return result
-}
+// Note: nullString, nullInt, and repeatPlaceholders are in helpers.go

internal/db/sqlite/prompt.go

@@ -128,9 +128,12 @@ func (s *PromptStore) GetPromptsByIDs(ctx context.Context, ids []int64, orderBy
 	// #nosec G202 -- query uses parameterized placeholders, not user input
 	query := `
 		SELECT up.id, up.claude_session_id, up.prompt_number, up.prompt_text,
-		       up.created_at, up.created_at_epoch, s.project, s.sdk_session_id
+		       COALESCE(up.matched_observations, 0) as matched_observations,
+		       up.created_at, up.created_at_epoch,
+		       COALESCE(s.project, '') as project,
+		       COALESCE(s.sdk_session_id, '') as sdk_session_id
 		FROM user_prompts up
-		JOIN sdk_sessions s ON up.claude_session_id = s.claude_session_id
+		LEFT JOIN sdk_sessions s ON up.claude_session_id = s.claude_session_id
 		WHERE up.id IN (?` + repeatPlaceholders(len(ids)-1) + `)
 		ORDER BY up.created_at_epoch `
 
@@ -144,11 +147,7 @@ func (s *PromptStore) GetPromptsByIDs(ctx context.Context, ids []int64, orderBy
 		query += " LIMIT ?"
 	}
 
-	// Convert []int64 to []interface{}
-	args := make([]interface{}, len(ids))
-	for i, id := range ids {
-		args[i] = id
-	}
+	args := int64SliceToInterface(ids)
 	if limit > 0 {
 		args = append(args, limit)
 	}
@@ -159,18 +158,7 @@ func (s *PromptStore) GetPromptsByIDs(ctx context.Context, ids []int64, orderBy
 	}
 	defer rows.Close()
 
-	var prompts []*models.UserPromptWithSession
-	for rows.Next() {
-		var prompt models.UserPromptWithSession
-		if err := rows.Scan(
-			&prompt.ID, &prompt.ClaudeSessionID, &prompt.PromptNumber, &prompt.PromptText,
-			&prompt.CreatedAt, &prompt.CreatedAtEpoch, &prompt.Project, &prompt.SDKSessionID,
-		); err != nil {
-			return nil, err
-		}
-		prompts = append(prompts, &prompt)
-	}
-	return prompts, rows.Err()
+	return scanPromptWithSessionRows(rows)
 }
 
 // GetAllRecentUserPrompts retrieves recent user prompts across all sessions.
@@ -193,19 +181,7 @@ func (s *PromptStore) GetAllRecentUserPrompts(ctx context.Context, limit int) ([
 	}
 	defer rows.Close()
 
-	var prompts []*models.UserPromptWithSession
-	for rows.Next() {
-		var prompt models.UserPromptWithSession
-		if err := rows.Scan(
-			&prompt.ID, &prompt.ClaudeSessionID, &prompt.PromptNumber, &prompt.PromptText,
-			&prompt.MatchedObservations, &prompt.CreatedAt, &prompt.CreatedAtEpoch,
-			&prompt.Project, &prompt.SDKSessionID,
-		); err != nil {
-			return nil, err
-		}
-		prompts = append(prompts, &prompt)
-	}
-	return prompts, rows.Err()
+	return scanPromptWithSessionRows(rows)
 }
 
 // GetRecentUserPromptsByProject retrieves recent user prompts for a specific project.
@@ -229,17 +205,5 @@ func (s *PromptStore) GetRecentUserPromptsByProject(ctx context.Context, project
 	}
 	defer rows.Close()
 
-	var prompts []*models.UserPromptWithSession
-	for rows.Next() {
-		var prompt models.UserPromptWithSession
-		if err := rows.Scan(
-			&prompt.ID, &prompt.ClaudeSessionID, &prompt.PromptNumber, &prompt.PromptText,
-			&prompt.MatchedObservations, &prompt.CreatedAt, &prompt.CreatedAtEpoch,
-			&prompt.Project, &prompt.SDKSessionID,
-		); err != nil {
-			return nil, err
-		}
-		prompts = append(prompts, &prompt)
-	}
-	return prompts, rows.Err()
+	return scanPromptWithSessionRows(rows)
 }

internal/db/sqlite/summary.go

@@ -3,7 +3,6 @@ package sqlite
 
 import (
 	"context"
-	"database/sql"
 	"time"
 
 	"github.com/lukaszraczylo/claude-mnemonic/pkg/models"
@@ -54,28 +53,7 @@ func (s *SummaryStore) StoreSummary(ctx context.Context, sdkSessionID, project s
 
 // ensureSessionExists creates a session if it doesn't exist.
 func (s *SummaryStore) ensureSessionExists(ctx context.Context, sdkSessionID, project string) error {
-	const checkQuery = `SELECT id FROM sdk_sessions WHERE sdk_session_id = ?`
-	var id int64
-	err := s.store.QueryRowContext(ctx, checkQuery, sdkSessionID).Scan(&id)
-	if err == nil {
-		return nil // Session exists
-	}
-	if err != sql.ErrNoRows {
-		return err
-	}
-
-	// Auto-create session
-	now := time.Now()
-	const insertQuery = `
-		INSERT INTO sdk_sessions
-		(claude_session_id, sdk_session_id, project, started_at, started_at_epoch, status)
-		VALUES (?, ?, ?, ?, ?, 'active')
-	`
-	_, err = s.store.ExecContext(ctx, insertQuery,
-		sdkSessionID, sdkSessionID, project,
-		now.Format(time.RFC3339), now.UnixMilli(),
-	)
-	return err
+	return EnsureSessionExists(ctx, s.store, sdkSessionID, project)
 }
 
 // GetSummariesByIDs retrieves summaries by a list of IDs.
@@ -84,33 +62,12 @@ func (s *SummaryStore) GetSummariesByIDs(ctx context.Context, ids []int64, order
 		return nil, nil
 	}
 
-	// Build query with placeholders
-	// #nosec G202 -- query uses parameterized placeholders, not user input
-	query := `
+	const baseQuery = `
 		SELECT id, sdk_session_id, project, request, investigated, learned, completed,
 		       next_steps, notes, prompt_number, discovery_tokens, created_at, created_at_epoch
-		FROM session_summaries
-		WHERE id IN (?` + repeatPlaceholders(len(ids)-1) + `)
-		ORDER BY created_at_epoch `
+		FROM session_summaries`
 
-	if orderBy == "date_asc" {
-		query += "ASC"
-	} else {
-		query += "DESC"
-	}
-
-	if limit > 0 {
-		query += " LIMIT ?"
-	}
-
-	// Convert []int64 to []interface{}
-	args := make([]interface{}, len(ids))
-	for i, id := range ids {
-		args[i] = id
-	}
-	if limit > 0 {
-		args = append(args, limit)
-	}
+	query, args := BuildGetByIDsQuery(baseQuery, ids, orderBy, limit)
 
 	rows, err := s.store.db.QueryContext(ctx, query, args...)
 	if err != nil {
@@ -118,20 +75,7 @@ func (s *SummaryStore) GetSummariesByIDs(ctx context.Context, ids []int64, order
 	}
 	defer rows.Close()
 
-	var summaries []*models.SessionSummary
-	for rows.Next() {
-		var summary models.SessionSummary
-		if err := rows.Scan(
-			&summary.ID, &summary.SDKSessionID, &summary.Project,
-			&summary.Request, &summary.Investigated, &summary.Learned, &summary.Completed,
-			&summary.NextSteps, &summary.Notes, &summary.PromptNumber, &summary.DiscoveryTokens,
-			&summary.CreatedAt, &summary.CreatedAtEpoch,
-		); err != nil {
-			return nil, err
-		}
-		summaries = append(summaries, &summary)
-	}
-	return summaries, rows.Err()
+	return scanSummaryRows(rows)
 }
 
 // GetRecentSummaries retrieves recent summaries for a project.
@@ -151,20 +95,7 @@ func (s *SummaryStore) GetRecentSummaries(ctx context.Context, project string, l
 	}
 	defer rows.Close()
 
-	var summaries []*models.SessionSummary
-	for rows.Next() {
-		var summary models.SessionSummary
-		if err := rows.Scan(
-			&summary.ID, &summary.SDKSessionID, &summary.Project,
-			&summary.Request, &summary.Investigated, &summary.Learned, &summary.Completed,
-			&summary.NextSteps, &summary.Notes, &summary.PromptNumber, &summary.DiscoveryTokens,
-			&summary.CreatedAt, &summary.CreatedAtEpoch,
-		); err != nil {
-			return nil, err
-		}
-		summaries = append(summaries, &summary)
-	}
-	return summaries, rows.Err()
+	return scanSummaryRows(rows)
 }
 
 // GetAllRecentSummaries retrieves recent summaries across all projects.
@@ -183,18 +114,5 @@ func (s *SummaryStore) GetAllRecentSummaries(ctx context.Context, limit int) ([]
 	}
 	defer rows.Close()
 
-	var summaries []*models.SessionSummary
-	for rows.Next() {
-		var summary models.SessionSummary
-		if err := rows.Scan(
-			&summary.ID, &summary.SDKSessionID, &summary.Project,
-			&summary.Request, &summary.Investigated, &summary.Learned, &summary.Completed,
-			&summary.NextSteps, &summary.Notes, &summary.PromptNumber, &summary.DiscoveryTokens,
-			&summary.CreatedAt, &summary.CreatedAtEpoch,
-		); err != nil {
-			return nil, err
-		}
-		summaries = append(summaries, &summary)
-	}
-	return summaries, rows.Err()
+	return scanSummaryRows(rows)
 }

internal/search/manager.go

@@ -94,18 +94,17 @@ func (m *Manager) UnifiedSearch(ctx context.Context, params SearchParams) (*Unif
 
 // vectorSearch performs semantic search via ChromaDB.
 func (m *Manager) vectorSearch(ctx context.Context, params SearchParams) (*UnifiedSearchResult, error) {
-	// Build where filter
-	where := make(map[string]interface{})
-	if params.Project != "" {
-		where["project"] = params.Project
-	}
-	if params.Type == "observations" {
-		where["doc_type"] = "observation"
-	} else if params.Type == "sessions" {
-		where["doc_type"] = "session_summary"
-	} else if params.Type == "prompts" {
-		where["doc_type"] = "user_prompt"
+	// Build where filter based on search type
+	var docType chroma.DocType
+	switch params.Type {
+	case "observations":
+		docType = chroma.DocTypeObservation
+	case "sessions":
+		docType = chroma.DocTypeSessionSummary
+	case "prompts":
+		docType = chroma.DocTypeUserPrompt
 	}
+	where := chroma.BuildWhereFilter(docType, params.Project)
 
 	// Query ChromaDB
 	chromaResults, err := m.chromaClient.Query(ctx, params.Query, params.Limit*2, where)
@@ -114,40 +113,11 @@ func (m *Manager) vectorSearch(ctx context.Context, params SearchParams) (*Unifi
 		return m.filterSearch(ctx, params)
 	}
 
-	// Collect unique IDs by type
-	obsIDs := make([]int64, 0)
-	summaryIDs := make([]int64, 0)
-	promptIDs := make([]int64, 0)
-	seenObs := make(map[int64]bool)
-	seenSummary := make(map[int64]bool)
-	seenPrompt := make(map[int64]bool)
-
-	for _, result := range chromaResults {
-		sqliteID, ok := result.Metadata["sqlite_id"].(float64)
-		if !ok {
-			continue
-		}
-		id := int64(sqliteID)
-
-		docType, _ := result.Metadata["doc_type"].(string)
-		switch docType {
-		case "observation":
-			if !seenObs[id] {
-				seenObs[id] = true
-				obsIDs = append(obsIDs, id)
-			}
-		case "session_summary":
-			if !seenSummary[id] {
-				seenSummary[id] = true
-				summaryIDs = append(summaryIDs, id)
-			}
-		case "user_prompt":
-			if !seenPrompt[id] {
-				seenPrompt[id] = true
-				promptIDs = append(promptIDs, id)
-			}
-		}
-	}
+	// Extract IDs grouped by document type using shared helper
+	extracted := chroma.ExtractIDsByDocType(chromaResults)
+	obsIDs := extracted.ObservationIDs
+	summaryIDs := extracted.SummaryIDs
+	promptIDs := extracted.PromptIDs
 
 	// Fetch full records from SQLite
 	var results []SearchResult

internal/vector/chroma/helpers.go

@@ -0,0 +1,175 @@
+// Package chroma provides ChromaDB vector database integration for claude-mnemonic.
+package chroma
+
+// DocType represents the type of document stored in ChromaDB.
+type DocType string
+
+const (
+	DocTypeObservation    DocType = "observation"
+	DocTypeSessionSummary DocType = "session_summary"
+	DocTypeUserPrompt     DocType = "user_prompt"
+)
+
+// ExtractedIDs contains SQLite IDs extracted from ChromaDB results, grouped by document type.
+type ExtractedIDs struct {
+	ObservationIDs []int64
+	SummaryIDs     []int64
+	PromptIDs      []int64
+}
+
+// BuildWhereFilter creates a where filter map for ChromaDB queries.
+// If docType is empty, no doc_type filter is added.
+func BuildWhereFilter(docType DocType, project string) map[string]interface{} {
+	where := make(map[string]interface{})
+	if docType != "" {
+		where["doc_type"] = string(docType)
+	}
+	if project != "" {
+		where["project"] = project
+	}
+	return where
+}
+
+// ExtractIDsByDocType extracts SQLite IDs from ChromaDB query results,
+// grouped by document type and deduplicated.
+func ExtractIDsByDocType(results []QueryResult) *ExtractedIDs {
+	ids := &ExtractedIDs{}
+	seenObs := make(map[int64]bool)
+	seenSummary := make(map[int64]bool)
+	seenPrompt := make(map[int64]bool)
+
+	for _, result := range results {
+		sqliteID, ok := result.Metadata["sqlite_id"].(float64)
+		if !ok {
+			continue
+		}
+		id := int64(sqliteID)
+
+		docType, _ := result.Metadata["doc_type"].(string)
+		switch docType {
+		case string(DocTypeObservation):
+			if !seenObs[id] {
+				seenObs[id] = true
+				ids.ObservationIDs = append(ids.ObservationIDs, id)
+			}
+		case string(DocTypeSessionSummary):
+			if !seenSummary[id] {
+				seenSummary[id] = true
+				ids.SummaryIDs = append(ids.SummaryIDs, id)
+			}
+		case string(DocTypeUserPrompt):
+			if !seenPrompt[id] {
+				seenPrompt[id] = true
+				ids.PromptIDs = append(ids.PromptIDs, id)
+			}
+		}
+	}
+
+	return ids
+}
+
+// ExtractObservationIDs extracts observation SQLite IDs from ChromaDB query results,
+// optionally filtering by project or including global scope.
+// If project is empty, all observation IDs are returned.
+// If project is set, only observations matching the project or with global scope are returned.
+func ExtractObservationIDs(results []QueryResult, project string) []int64 {
+	var ids []int64
+	seen := make(map[int64]bool)
+
+	for _, result := range results {
+		sqliteID, ok := result.Metadata["sqlite_id"].(float64)
+		if !ok {
+			continue
+		}
+		id := int64(sqliteID)
+
+		// Check document type
+		docType, _ := result.Metadata["doc_type"].(string)
+		if docType != string(DocTypeObservation) {
+			continue
+		}
+
+		// Apply project/scope filter if project is specified
+		if project != "" {
+			proj, _ := result.Metadata["project"].(string)
+			scope, _ := result.Metadata["scope"].(string)
+			// Include if project matches OR scope is global
+			if proj != project && scope != "global" {
+				continue
+			}
+		}
+
+		if !seen[id] {
+			seen[id] = true
+			ids = append(ids, id)
+		}
+	}
+
+	return ids
+}
+
+// ExtractSummaryIDs extracts session summary SQLite IDs from ChromaDB query results.
+func ExtractSummaryIDs(results []QueryResult, project string) []int64 {
+	var ids []int64
+	seen := make(map[int64]bool)
+
+	for _, result := range results {
+		sqliteID, ok := result.Metadata["sqlite_id"].(float64)
+		if !ok {
+			continue
+		}
+		id := int64(sqliteID)
+
+		docType, _ := result.Metadata["doc_type"].(string)
+		if docType != string(DocTypeSessionSummary) {
+			continue
+		}
+
+		if project != "" {
+			proj, _ := result.Metadata["project"].(string)
+			if proj != project {
+				continue
+			}
+		}
+
+		if !seen[id] {
+			seen[id] = true
+			ids = append(ids, id)
+		}
+	}
+
+	return ids
+}
+
+// ExtractPromptIDs extracts user prompt SQLite IDs from ChromaDB query results.
+func ExtractPromptIDs(results []QueryResult, project string) []int64 {
+	var ids []int64
+	seen := make(map[int64]bool)
+
+	for _, result := range results {
+		sqliteID, ok := result.Metadata["sqlite_id"].(float64)
+		if !ok {
+			continue
+		}
+		id := int64(sqliteID)
+
+		docType, _ := result.Metadata["doc_type"].(string)
+		if docType != string(DocTypeUserPrompt) {
+			continue
+		}
+
+		if project != "" {
+			proj, _ := result.Metadata["project"].(string)
+			if proj != project {
+				continue
+			}
+		}
+
+		if !seen[id] {
+			seen[id] = true
+			ids = append(ids, id)
+		}
+	}
+
+	return ids
+}

internal/worker/handlers.go

@@ -9,7 +9,9 @@ import (
 	"time"
 
 	"github.com/go-chi/chi/v5"
+	"github.com/lukaszraczylo/claude-mnemonic/internal/db/sqlite"
 	"github.com/lukaszraczylo/claude-mnemonic/internal/privacy"
+	"github.com/lukaszraczylo/claude-mnemonic/internal/vector/chroma"
 	"github.com/lukaszraczylo/claude-mnemonic/internal/worker/sdk"
 	"github.com/lukaszraczylo/claude-mnemonic/internal/worker/session"
 	"github.com/lukaszraczylo/claude-mnemonic/pkg/models"
@@ -22,6 +24,53 @@ const (
 	DefaultObservationsLimit = 100
 
 	// DefaultSummariesLimit is the default number of summaries to return.
+)
+
+// ObservationTypes is the canonical list of observation types.
+// Used by both Go backend and served to frontend.
+var ObservationTypes = []string{
+	"bugfix",
+	"feature",
+	"refactor",
+	"discovery",
+	"decision",
+	"change",
+}
+
+// ConceptTypes is the canonical list of valid concept types.
+// Used by both Go backend and served to frontend.
+var ConceptTypes = []string{
+	// Semantic concepts
+	"how-it-works",
+	"why-it-exists",
+	"what-changed",
+	"problem-solution",
+	"gotcha",
+	"pattern",
+	"trade-off",
+	// Globalizable concepts (from models.GlobalizableConcepts)
+	"best-practice",
+	"anti-pattern",
+	"architecture",
+	"security",
+	"performance",
+	"testing",
+	"debugging",
+	"workflow",
+	"tooling",
+	// Additional useful concepts
+	"refactoring",
+	"api",
+	"database",
+	"configuration",
+	"error-handling",
+	"caching",
+	"logging",
+	"auth",
+	"validation",
+}
+
+const (
 	DefaultSummariesLimit = 50
 
 	// DefaultPromptsLimit is the default number of prompts to return.
@@ -401,25 +450,40 @@ func (s *Service) handleSummarize(w http.ResponseWriter, r *http.Request) {
 }
 
 // handleGetObservations returns recent observations.
+// Supports optional query parameter for semantic search via ChromaDB.
 func (s *Service) handleGetObservations(w http.ResponseWriter, r *http.Request) {
-	limit := DefaultObservationsLimit
-	if l := r.URL.Query().Get("limit"); l != "" {
-		if parsed, err := strconv.Atoi(l); err == nil && parsed > 0 {
-			limit = parsed
-		}
-	}
-
+	limit := sqlite.ParseLimitParam(r, DefaultObservationsLimit)
 	project := r.URL.Query().Get("project")
+	query := r.URL.Query().Get("query")
 
 	var observations []*models.Observation
 	var err error
+	var usedChroma bool
 
-	if project != "" {
-		// Filter by project - includes project-scoped and global observations
-		observations, err = s.observationStore.GetRecentObservations(r.Context(), project, limit)
-	} else {
-		// All projects
-		observations, err = s.observationStore.GetAllRecentObservations(r.Context(), limit)
+	// Use ChromaDB if query is provided and ChromaDB is available
+	if query != "" && s.chromaClient != nil && s.chromaClient.IsConnected() {
+		where := chroma.BuildWhereFilter(chroma.DocTypeObservation, "")
+		chromaResults, chromaErr := s.chromaClient.Query(r.Context(), query, limit*2, where)
+		if chromaErr == nil && len(chromaResults) > 0 {
+			obsIDs := chroma.ExtractObservationIDs(chromaResults, project)
+			if len(obsIDs) > 0 {
+				observations, err = s.observationStore.GetObservationsByIDs(r.Context(), obsIDs, "date_desc", limit)
+				if err == nil {
+					usedChroma = true
+				}
+			}
+		}
+	}
+
+	// Fall back to SQLite if ChromaDB not used
+	if !usedChroma {
+		if project != "" {
+			// Filter by project - includes project-scoped and global observations
+			observations, err = s.observationStore.GetRecentObservations(r.Context(), project, limit)
+		} else {
+			// All projects
+			observations, err = s.observationStore.GetAllRecentObservations(r.Context(), limit)
+		}
 	}
 
 	if err != nil {
@@ -435,23 +499,38 @@ func (s *Service) handleGetObservations(w http.ResponseWriter, r *http.Request)
 }
 
 // handleGetSummaries returns recent summaries.
+// Supports optional query parameter for semantic search via ChromaDB.
 func (s *Service) handleGetSummaries(w http.ResponseWriter, r *http.Request) {
-	limit := DefaultSummariesLimit
-	if l := r.URL.Query().Get("limit"); l != "" {
-		if parsed, err := strconv.Atoi(l); err == nil && parsed > 0 {
-			limit = parsed
-		}
-	}
-
+	limit := sqlite.ParseLimitParam(r, DefaultSummariesLimit)
 	project := r.URL.Query().Get("project")
+	query := r.URL.Query().Get("query")
 
 	var summaries []*models.SessionSummary
 	var err error
+	var usedChroma bool
 
-	if project != "" {
-		summaries, err = s.summaryStore.GetRecentSummaries(r.Context(), project, limit)
-	} else {
-		summaries, err = s.summaryStore.GetAllRecentSummaries(r.Context(), limit)
+	// Use ChromaDB if query is provided and ChromaDB is available
+	if query != "" && s.chromaClient != nil && s.chromaClient.IsConnected() {
+		where := chroma.BuildWhereFilter(chroma.DocTypeSessionSummary, "")
+		chromaResults, chromaErr := s.chromaClient.Query(r.Context(), query, limit*2, where)
+		if chromaErr == nil && len(chromaResults) > 0 {
+			summaryIDs := chroma.ExtractSummaryIDs(chromaResults, project)
+			if len(summaryIDs) > 0 {
+				summaries, err = s.summaryStore.GetSummariesByIDs(r.Context(), summaryIDs, "date_desc", limit)
+				if err == nil {
+					usedChroma = true
+				}
+			}
+		}
+	}
+
+	// Fall back to SQLite if ChromaDB not used
+	if !usedChroma {
+		if project != "" {
+			summaries, err = s.summaryStore.GetRecentSummaries(r.Context(), project, limit)
+		} else {
+			summaries, err = s.summaryStore.GetAllRecentSummaries(r.Context(), limit)
+		}
 	}
 
 	if err != nil {
@@ -467,23 +546,38 @@ func (s *Service) handleGetSummaries(w http.ResponseWriter, r *http.Request) {
 }
 
 // handleGetPrompts returns recent user prompts.
+// Supports optional query parameter for semantic search via ChromaDB.
 func (s *Service) handleGetPrompts(w http.ResponseWriter, r *http.Request) {
-	limit := DefaultPromptsLimit
-	if l := r.URL.Query().Get("limit"); l != "" {
-		if parsed, err := strconv.Atoi(l); err == nil && parsed > 0 {
-			limit = parsed
-		}
-	}
-
+	limit := sqlite.ParseLimitParam(r, DefaultPromptsLimit)
 	project := r.URL.Query().Get("project")
+	query := r.URL.Query().Get("query")
 
 	var prompts []*models.UserPromptWithSession
 	var err error
+	var usedChroma bool
 
-	if project != "" {
-		prompts, err = s.promptStore.GetRecentUserPromptsByProject(r.Context(), project, limit)
-	} else {
-		prompts, err = s.promptStore.GetAllRecentUserPrompts(r.Context(), limit)
+	// Use ChromaDB if query is provided and ChromaDB is available
+	if query != "" && s.chromaClient != nil && s.chromaClient.IsConnected() {
+		where := chroma.BuildWhereFilter(chroma.DocTypeUserPrompt, "")
+		chromaResults, chromaErr := s.chromaClient.Query(r.Context(), query, limit*2, where)
+		if chromaErr == nil && len(chromaResults) > 0 {
+			promptIDs := chroma.ExtractPromptIDs(chromaResults, project)
+			if len(promptIDs) > 0 {
+				prompts, err = s.promptStore.GetPromptsByIDs(r.Context(), promptIDs, "date_desc", limit)
+				if err == nil {
+					usedChroma = true
+				}
+			}
+		}
+	}
+
+	// Fall back to SQLite if ChromaDB not used
+	if !usedChroma {
+		if project != "" {
+			prompts, err = s.promptStore.GetRecentUserPromptsByProject(r.Context(), project, limit)
+		} else {
+			prompts, err = s.promptStore.GetAllRecentUserPrompts(r.Context(), limit)
+		}
 	}
 
 	if err != nil {
@@ -509,6 +603,15 @@ func (s *Service) handleGetProjects(w http.ResponseWriter, r *http.Request) {
 	writeJSON(w, projects)
 }
 
+// handleGetTypes returns the canonical list of observation and concept types.
+// This provides a single source of truth for both backend and frontend.
+func (s *Service) handleGetTypes(w http.ResponseWriter, r *http.Request) {
+	writeJSON(w, map[string]interface{}{
+		"observation_types": ObservationTypes,
+		"concept_types":     ConceptTypes,
+	})
+}
+
 // handleGetStats returns worker statistics.
 func (s *Service) handleGetStats(w http.ResponseWriter, r *http.Request) {
 	retrievalStats := s.GetRetrievalStats()
@@ -576,22 +679,42 @@ func (s *Service) handleSearchByPrompt(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	limit := DefaultSearchLimit
-	if l := r.URL.Query().Get("limit"); l != "" {
-		if parsed, err := strconv.Atoi(l); err == nil && parsed > 0 {
-			limit = parsed
+	limit := sqlite.ParseLimitParam(r, DefaultSearchLimit)
+
+	var observations []*models.Observation
+	var err error
+	var usedChroma bool
+
+	// Try ChromaDB vector search first if available
+	if s.chromaClient != nil && s.chromaClient.IsConnected() {
+		where := chroma.BuildWhereFilter(chroma.DocTypeObservation, "")
+
+		chromaResults, chromaErr := s.chromaClient.Query(r.Context(), query, limit*2, where)
+		if chromaErr == nil && len(chromaResults) > 0 {
+			// Extract observation IDs with project/scope filtering using shared helper
+			obsIDs := chroma.ExtractObservationIDs(chromaResults, project)
+
+			if len(obsIDs) > 0 {
+				// Fetch full observations from SQLite
+				observations, err = s.observationStore.GetObservationsByIDs(r.Context(), obsIDs, "date_desc", limit)
+				if err == nil {
+					usedChroma = true
+				}
+			}
 		}
 	}
 
-	// Search using FTS
-	observations, err := s.observationStore.SearchObservationsFTS(r.Context(), query, project, limit)
-	if err != nil {
-		// FTS might fail if query has special chars, try without
-		log.Warn().Err(err).Str("query", query).Msg("FTS search failed, falling back to recent")
-		observations, err = s.observationStore.GetRecentObservations(r.Context(), project, limit)
+	// Fall back to FTS if ChromaDB not available or returned no results
+	if !usedChroma || len(observations) == 0 {
+		observations, err = s.observationStore.SearchObservationsFTS(r.Context(), query, project, limit)
 		if err != nil {
-			http.Error(w, err.Error(), http.StatusInternalServerError)
-			return
+			// FTS might fail if query has special chars, try without
+			log.Warn().Err(err).Str("query", query).Msg("FTS search failed, falling back to recent")
+			observations, err = s.observationStore.GetRecentObservations(r.Context(), project, limit)
+			if err != nil {
+				http.Error(w, err.Error(), http.StatusInternalServerError)
+				return
+			}
 		}
 	}
 

internal/worker/sdk/parser.go

@@ -27,8 +27,9 @@ var (
 		"decision":  true,
 	}
 
-	// Valid concepts (strict list - no custom tags allowed)
+	// Valid concepts - expanded list matching GlobalizableConcepts and common use cases
 	validConcepts = map[string]bool{
+		// Semantic concepts
 		"how-it-works":     true,
 		"why-it-exists":    true,
 		"what-changed":     true,
@@ -36,6 +37,26 @@ var (
 		"gotcha":           true,
 		"pattern":          true,
 		"trade-off":        true,
+		// Globalizable concepts (from models.GlobalizableConcepts)
+		"best-practice": true,
+		"anti-pattern":  true,
+		"architecture":  true,
+		"security":      true,
+		"performance":   true,
+		"testing":       true,
+		"debugging":     true,
+		"workflow":      true,
+		"tooling":       true,
+		// Additional useful concepts
+		"refactoring":    true,
+		"api":            true,
+		"database":       true,
+		"configuration":  true,
+		"error-handling": true,
+		"caching":        true,
+		"logging":        true,
+		"auth":           true,
+		"validation":     true,
 	}
 )
 

internal/worker/sdk/processor.go

@@ -123,17 +123,9 @@ func (p *Processor) ProcessObservation(ctx context.Context, sdkSessionID, projec
 
 	log.Info().Str("tool", toolName).Msg("Processing tool execution with Claude CLI")
 
-	// Check if we already have observations for this file (skip if covered)
-	if filePath := extractFilePath(toolName, inputStr); filePath != "" {
-		exists, err := p.observationStore.ExistsSimilarObservation(ctx, project, []string{filePath}, nil)
-		if err == nil && exists {
-			log.Debug().
-				Str("tool", toolName).
-				Str("file", filePath).
-				Msg("Skipping - file already has observations")
-			return nil
-		}
-	}
+	// Note: Removed the "file already has observations" check
+	// Each tool execution can produce unique insights even for the same file
+	// Similarity-based deduplication will handle true duplicates
 
 	// Build the prompt
 	exec := ToolExecution{
@@ -413,8 +405,9 @@ func shouldSkipTool(toolName string) bool {
 // shouldSkipTrivialOperation performs local pre-filtering to skip trivial operations
 // without making a Haiku API call. Returns true if the operation is too trivial to process.
 func shouldSkipTrivialOperation(toolName, inputStr, outputStr string) bool {
-	// Skip if output is too small to be meaningful (less than 100 chars)
-	if len(outputStr) < 100 {
+	// Skip if output is too small to be meaningful (less than 50 chars)
+	// Reduced from 100 to capture more meaningful small operations
+	if len(outputStr) < 50 {
 		return true
 	}
 
@@ -477,36 +470,6 @@ func shouldSkipTrivialOperation(toolName, inputStr, outputStr string) bool {
 	return false
 }
 
-// extractFilePath extracts the file path from tool input for deduplication.
-func extractFilePath(toolName, inputStr string) string {
-	if inputStr == "" {
-		return ""
-	}
-
-	var input map[string]interface{}
-	if err := json.Unmarshal([]byte(inputStr), &input); err != nil {
-		return ""
-	}
-
-	// Handle different tool input formats
-	switch toolName {
-	case "Read":
-		if fp, ok := input["file_path"].(string); ok {
-			return fp
-		}
-	case "Grep", "Search":
-		if path, ok := input["path"].(string); ok {
-			return path
-		}
-	case "Edit", "Write":
-		if fp, ok := input["file_path"].(string); ok {
-			return fp
-		}
-	}
-
-	return ""
-}
-
 // toJSONString converts an interface to a JSON string.
 func toJSONString(v interface{}) string {
 	if v == nil {
@@ -767,12 +730,18 @@ func hasMeaningfulContent(assistantMsg string) bool {
 const systemPrompt = `You are a memory extraction agent for Claude Code sessions. Your job is to analyze tool executions and extract meaningful observations that would be useful for future sessions.
 
 GUIDELINES:
-1. Only create observations for SIGNIFICANT learnings - not every tool call needs one
-2. Focus on: decisions made, bugs fixed, patterns discovered, project structure learned
-3. Skip trivial operations like simple file reads without insights
+1. Create observations for any meaningful learnings - be generous, not restrictive
+2. Focus on: decisions made, bugs fixed, patterns discovered, project structure, code changes, refactoring
+3. Even small changes can be worth remembering if they reveal something about the codebase
 4. Be concise but informative in your observations
 5. Use appropriate type tags: decision, bugfix, feature, refactor, discovery, change
 
+CONCEPT TAGS (use 1-3 of these):
+- how-it-works, why-it-exists, what-changed, problem-solution, gotcha
+- pattern, trade-off, best-practice, anti-pattern, architecture
+- security, performance, testing, debugging, workflow, tooling
+- refactoring, api, database, configuration, error-handling
+
 OUTPUT FORMAT:
 When you find something worth remembering, output:
 <observation>
@@ -794,5 +763,7 @@ When you find something worth remembering, output:
 </files_modified>
 </observation>
 
-If the tool execution is not noteworthy, simply respond with:
-<skip reason="not significant"/>`
+If the tool execution is truly trivial (just a directory listing, empty result, etc.), respond with:
+<skip reason="trivial"/>
+
+Prefer creating observations over skipping - memories are valuable for future context!`

internal/worker/service.go

@@ -655,6 +655,7 @@ func (s *Service) setupRoutes() {
 		r.Get("/api/projects", s.handleGetProjects)
 		r.Get("/api/stats", s.handleGetStats)
 		r.Get("/api/stats/retrieval", s.handleGetRetrievalStats)
+		r.Get("/api/types", s.handleGetTypes)
 
 		// Context injection
 		r.Get("/api/context/count", s.handleContextCount)