fix(db): keep archived notes out of search, lists, counts and the vector rebuild (#74)

* fix(db): keep archived notes out of search, lists, counts and the vector rebuild

An archived note is meant to be kept but unused, yet most read paths did not
filter on is_archived: full-text and LIKE search, vector-hit hydration,
context injection, the paginated lists, counts and the vector rebuild all still
returned it. A vector hit for an archived note also still pointed at a live
embedding, because archiving never dropped it.

- add notArchivedFilter/visibleScope and apply them to every read path
- the paginated lists take includeArchived; the API reads
  ?include_archived=true, and exports ask for them so a backup is complete
- archiving drops the note's vectors, unarchiving syncs them back
- add //go:build fts5 to observation_update_test.go so `go vet` without tags
  builds (it referenced a helper from a tagged file)

Adds a store test that archives one of three notes and checks every path, and
that fails if the filter is removed.

Co-Authored-By: Claude Sonnet 5.5 <[email protected]>

* fix(worker): drop the vectors of notes archived through bulk-status too

POST /api/observations/bulk-status with action "archive" archived notes but left
their vectors in the index; only the dedicated archive handler dropped them.
Both paths now call dropArchivedFromVectors.

Co-Authored-By: Claude Sonnet 5.5 <[email protected]>

---------

Co-authored-by: Claude Sonnet 5.5 <[email protected]>

internal/db/gorm/observation_archived_test.go

@@ -0,0 +1,107 @@
+//go:build fts5
+
+package gorm
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/lukaszraczylo/claude-mnemonic/pkg/models"
+)
+
+// An archived note is kept, but it must stay out of search, context, lists, counts and the vector rebuild, and an
+// export (which asks for it) still has it.
+func TestArchivedObservationsAreKeptButHiddenEverywhereExceptAnExport(t *testing.T) {
+	s, store, cleanup := testObservationStore(t)
+	defer cleanup()
+	ctx := context.Background()
+
+	var ids []int64
+	for _, title := range []string{"Zebra ledger reconciliation", "Quokka deployment checklist", "Ocelot retry budget"} {
+		id, _, err := s.StoreObservation(ctx, "claude-1", "proj",
+			&models.ParsedObservation{Type: models.ObsTypeDiscovery, Title: title, Narrative: title + " notes"}, 1, 10)
+		require.NoError(t, err)
+		ids = append(ids, id)
+		time.Sleep(2 * time.Millisecond)
+	}
+	archivedID := ids[0] // the zebra note
+	require.NoError(t, s.ArchiveObservation(ctx, archivedID, "test"))
+
+	has := func(obs []*models.Observation) bool {
+		for _, o := range obs {
+			if o.ID == archivedID {
+				return true
+			}
+		}
+		return false
+	}
+
+	recent, err := s.GetRecentObservations(ctx, "proj", 10)
+	require.NoError(t, err)
+	assert.False(t, has(recent), "context injection does not see it")
+
+	strict, err := s.GetObservationsByProjectStrict(ctx, "proj", 10)
+	require.NoError(t, err)
+	assert.False(t, has(strict))
+
+	all, err := s.GetAllRecentObservations(ctx, 10)
+	require.NoError(t, err)
+	assert.False(t, has(all))
+
+	byID, err := s.GetObservationsByIDs(ctx, ids, "default", 0)
+	require.NoError(t, err)
+	assert.False(t, has(byID), "a vector hit for it is dropped when it is loaded")
+	assert.Len(t, byID, 2)
+
+	ordered, err := s.GetObservationsByIDsPreserveOrder(ctx, ids)
+	require.NoError(t, err)
+	assert.False(t, has(ordered))
+
+	fts, err := s.SearchObservationsFTS(ctx, "zebra ledger", "proj", 10)
+	require.NoError(t, err)
+	assert.False(t, has(fts), "full-text search does not find it")
+	other, err := s.SearchObservationsFTS(ctx, "quokka deployment", "proj", 10)
+	require.NoError(t, err)
+	assert.Len(t, other, 1, "and still finds the others")
+
+	n, err := s.GetObservationCount(ctx, "proj")
+	require.NoError(t, err)
+	assert.Equal(t, 2, n, "the count is of notes that can be used")
+
+	rebuild, err := s.GetAllObservations(ctx)
+	require.NoError(t, err)
+	assert.False(t, has(rebuild), "a vector rebuild does not put it back into the index")
+
+	listed, total, err := s.GetAllRecentObservationsPaginated(ctx, 10, 0, false)
+	require.NoError(t, err)
+	assert.False(t, has(listed))
+	assert.EqualValues(t, 2, total)
+	projListed, projTotal, err := s.GetObservationsByProjectStrictPaginated(ctx, "proj", 10, 0, false)
+	require.NoError(t, err)
+	assert.False(t, has(projListed))
+	assert.EqualValues(t, 2, projTotal)
+
+	exported, total, err := s.GetAllRecentObservationsPaginated(ctx, 10, 0, true)
+	require.NoError(t, err)
+	assert.True(t, has(exported), "an export asks for them")
+	assert.EqualValues(t, 3, total)
+	projExported, _, err := s.GetObservationsByProjectStrictPaginated(ctx, "proj", 10, 0, true)
+	require.NoError(t, err)
+	assert.True(t, has(projExported))
+
+	// Kept: the row is still there, and restoring it makes it visible again.
+	one, err := s.GetObservationByID(ctx, archivedID)
+	require.NoError(t, err)
+	require.NotNil(t, one)
+	require.NoError(t, s.UnarchiveObservation(ctx, archivedID))
+	back, err := s.SearchObservationsFTS(ctx, "zebra ledger", "proj", 10)
+	require.NoError(t, err)
+	assert.True(t, has(back))
+	var rows int64
+	require.NoError(t, store.DB.Model(&Observation{}).Where("project = ?", "proj").Count(&rows).Error)
+	assert.EqualValues(t, 3, rows)
+}

internal/db/gorm/observation_store.go

@@ -290,7 +290,7 @@ func (s *ObservationStore) GetObservationsByIDs(ctx context.Context, ids []int64
 	}
 
 	var dbObservations []Observation
-	query := s.db.WithContext(ctx).Where("id IN ?", ids)
+	query := s.db.WithContext(ctx).Scopes(notArchivedFilter()).Where("id IN ?", ids)
 
 	// Apply ordering
 	switch orderBy {
@@ -329,7 +329,7 @@ func (s *ObservationStore) GetObservationsByIDsPreserveOrder(ctx context.Context
 
 	// Fetch all observations in a single query
 	var dbObservations []Observation
-	err := s.db.WithContext(ctx).Where("id IN ?", ids).Find(&dbObservations).Error
+	err := s.db.WithContext(ctx).Scopes(notArchivedFilter()).Where("id IN ?", ids).Find(&dbObservations).Error
 	if err != nil {
 		return nil, err
 	}
@@ -380,7 +380,7 @@ func (s *ObservationStore) BatchGetObservationsWithScores(ctx context.Context, i
 func (s *ObservationStore) GetRecentObservations(ctx context.Context, project string, limit int) ([]*models.Observation, error) {
 	var dbObservations []Observation
 	err := s.db.WithContext(ctx).
-		Scopes(projectScopeFilter(project), importanceOrdering()).
+		Scopes(projectScopeFilter(project), notArchivedFilter(), importanceOrdering()).
 		Limit(limit).
 		Find(&dbObservations).Error
 
@@ -430,7 +430,7 @@ func (s *ObservationStore) GetObservationsByProjectStrict(ctx context.Context, p
 	var dbObservations []Observation
 	err := s.db.WithContext(ctx).
 		Where("project = ?", project).
-		Scopes(importanceOrdering()).
+		Scopes(notArchivedFilter(), importanceOrdering()).
 		Limit(limit).
 		Find(&dbObservations).Error
 
@@ -447,6 +447,7 @@ func (s *ObservationStore) GetObservationCount(ctx context.Context, project stri
 	err := s.db.WithContext(ctx).
 		Model(&Observation{}).
 		Where("project = ?", project).
+		Scopes(notArchivedFilter()).
 		Count(&count).Error
 
 	return int(count), err
@@ -456,7 +457,7 @@ func (s *ObservationStore) GetObservationCount(ctx context.Context, project stri
 func (s *ObservationStore) GetAllRecentObservations(ctx context.Context, limit int) ([]*models.Observation, error) {
 	var dbObservations []Observation
 	err := s.db.WithContext(ctx).
-		Scopes(importanceOrdering()).
+		Scopes(notArchivedFilter(), importanceOrdering()).
 		Limit(limit).
 		Find(&dbObservations).Error
 
@@ -467,19 +468,21 @@ func (s *ObservationStore) GetAllRecentObservations(ctx context.Context, limit i
 	return toModelObservations(dbObservations), nil
 }
 
-// GetAllRecentObservationsPaginated retrieves recent observations with pagination.
-func (s *ObservationStore) GetAllRecentObservationsPaginated(ctx context.Context, limit, offset int) ([]*models.Observation, int64, error) {
+// GetAllRecentObservationsPaginated retrieves recent observations with pagination. Archived notes are left out unless
+// includeArchived is set (an export asks for them).
+func (s *ObservationStore) GetAllRecentObservationsPaginated(ctx context.Context, limit, offset int, includeArchived bool) ([]*models.Observation, int64, error) {
 	var dbObservations []Observation
 	var total int64
+	visible := visibleScope(includeArchived)
 
 	// Get total count
-	if err := s.db.WithContext(ctx).Model(&Observation{}).Count(&total).Error; err != nil {
+	if err := s.db.WithContext(ctx).Model(&Observation{}).Scopes(visible).Count(&total).Error; err != nil {
 		return nil, 0, err
 	}
 
 	// Get paginated results
 	err := s.db.WithContext(ctx).
-		Scopes(importanceOrdering()).
+		Scopes(visible, importanceOrdering()).
 		Limit(limit).
 		Offset(offset).
 		Find(&dbObservations).Error
@@ -491,20 +494,22 @@ func (s *ObservationStore) GetAllRecentObservationsPaginated(ctx context.Context
 	return toModelObservations(dbObservations), total, nil
 }
 
-// GetObservationsByProjectStrictPaginated retrieves observations strictly from a project with pagination.
-func (s *ObservationStore) GetObservationsByProjectStrictPaginated(ctx context.Context, project string, limit, offset int) ([]*models.Observation, int64, error) {
+// GetObservationsByProjectStrictPaginated retrieves observations strictly from a project with pagination. Archived notes
+// are left out unless includeArchived is set (an export asks for them).
+func (s *ObservationStore) GetObservationsByProjectStrictPaginated(ctx context.Context, project string, limit, offset int, includeArchived bool) ([]*models.Observation, int64, error) {
 	var dbObservations []Observation
 	var total int64
+	visible := visibleScope(includeArchived)
 
 	// Get total count for project
-	if err := s.db.WithContext(ctx).Model(&Observation{}).Where("project = ?", project).Count(&total).Error; err != nil {
+	if err := s.db.WithContext(ctx).Model(&Observation{}).Where("project = ?", project).Scopes(visible).Count(&total).Error; err != nil {
 		return nil, 0, err
 	}
 
 	// Get paginated results
 	err := s.db.WithContext(ctx).
 		Where("project = ?", project).
-		Scopes(importanceOrdering()).
+		Scopes(visible, importanceOrdering()).
 		Limit(limit).
 		Offset(offset).
 		Find(&dbObservations).Error
@@ -516,11 +521,13 @@ func (s *ObservationStore) GetObservationsByProjectStrictPaginated(ctx context.C
 	return toModelObservations(dbObservations), total, nil
 }
 
-// GetAllObservations retrieves all observations (for vector rebuild).
+// GetAllObservations retrieves all observations that are not archived (for vector rebuild: an archived note stays out of
+// the vector index).
 // Note: For large datasets, prefer GetAllObservationsIterator to avoid memory issues.
 func (s *ObservationStore) GetAllObservations(ctx context.Context) ([]*models.Observation, error) {
 	var dbObservations []Observation
 	err := s.db.WithContext(ctx).
+		Scopes(notArchivedFilter()).
 		Order("id").
 		Find(&dbObservations).Error
 
@@ -606,6 +613,7 @@ func (s *ObservationStore) SearchObservationsFTS(ctx context.Context, query, pro
 		JOIN observations_fts fts ON o.id = fts.rowid
 		WHERE observations_fts MATCH ?
 		  AND (o.project = ? OR o.scope = 'global')
+		  AND COALESCE(o.is_archived, 0) = 0
 		ORDER BY rank, COALESCE(o.importance_score, 1.0) DESC
 		LIMIT ?
 	`
@@ -660,7 +668,7 @@ func (s *ObservationStore) searchObservationsLike(ctx context.Context, keywords
 
 	// Build WHERE clause
 	whereClause := strings.Join(conditions, " OR ")
-	fullWhere := "(" + whereClause + ") AND (project = ? OR scope = 'global')"
+	fullWhere := "(" + whereClause + ") AND (project = ? OR scope = 'global') AND COALESCE(is_archived, 0) = 0"
 	args = append(args, project)
 
 	var dbObservations []Observation
@@ -958,6 +966,21 @@ func projectScopeFilter(project string) func(*gorm.DB) *gorm.DB {
 	}
 }
 
+// notArchivedFilter hides archived notes: an archived note is kept, but it must stay out of search, context and lists.
+func notArchivedFilter() func(*gorm.DB) *gorm.DB {
+	return func(db *gorm.DB) *gorm.DB {
+		return db.Where("COALESCE(is_archived, 0) = 0")
+	}
+}
+
+// visibleScope hides archived notes unless they are asked for (an export keeps them; lists and counts do not).
+func visibleScope(includeArchived bool) func(*gorm.DB) *gorm.DB {
+	if includeArchived {
+		return func(db *gorm.DB) *gorm.DB { return db }
+	}
+	return notArchivedFilter()
+}
+
 // activeObservationFilter filters for active (non-archived, non-superseded) observations.
 // This is more efficient than chaining notSupersededFilter + notArchivedFilter
 // as it produces a single WHERE clause for the query optimizer.

internal/worker/handlers_data.go

@@ -28,6 +28,8 @@ func (s *Service) handleGetObservations(w http.ResponseWriter, r *http.Request)
 	pagination := gorm.ParsePaginationParams(r, DefaultObservationsLimit)
 	project := r.URL.Query().Get("project")
 	query := r.URL.Query().Get("query")
+	// Archived notes are not listed unless asked for.
+	includeArchived := r.URL.Query().Get("include_archived") == "true"
 
 	// Validate project name to prevent path traversal
 	if err := ValidateProjectName(project); err != nil {
@@ -60,10 +62,10 @@ func (s *Service) handleGetObservations(w http.ResponseWriter, r *http.Request)
 	if !usedVector {
 		if project != "" {
 			// Strict project filtering for dashboard - only observations from this project
-			observations, total, err = s.observationStore.GetObservationsByProjectStrictPaginated(ctx, project, pagination.Limit, pagination.Offset)
+			observations, total, err = s.observationStore.GetObservationsByProjectStrictPaginated(ctx, project, pagination.Limit, pagination.Offset, includeArchived)
 		} else {
 			// All projects
-			observations, total, err = s.observationStore.GetAllRecentObservationsPaginated(ctx, pagination.Limit, pagination.Offset)
+			observations, total, err = s.observationStore.GetAllRecentObservationsPaginated(ctx, pagination.Limit, pagination.Offset, includeArchived)
 		}
 	}
 

internal/worker/handlers_import_export.go

@@ -300,6 +300,8 @@ func (s *Service) handleArchiveObservations(w http.ResponseWriter, r *http.Reque
 		Int("failed", len(failedIDs)).
 		Msg("Observations archived")
 
+	s.dropArchivedFromVectors(archivedIDs)
+
 	// Invalidate cache if any observations were archived
 	if len(archivedIDs) > 0 {
 		if req.Project != "" {
@@ -322,6 +324,22 @@ func (s *Service) handleArchiveObservations(w http.ResponseWriter, r *http.Reque
 	writeJSON(w, response)
 }
 
+// dropArchivedFromVectors takes archived notes out of the vector index, so search cannot return them (unarchiving puts
+// them back). Every path that archives notes calls it.
+func (s *Service) dropArchivedFromVectors(ids []int64) {
+	if len(ids) == 0 || s.vectorSync == nil {
+		return
+	}
+	ids = append([]int64(nil), ids...)
+	s.asyncVectorSync(func() {
+		ctx, cancel := context.WithTimeout(s.ctx, 10*time.Second)
+		defer cancel()
+		if err := s.vectorSync.DeleteObservations(ctx, ids); err != nil && s.ctx.Err() == nil {
+			log.Warn().Err(err).Int("count", len(ids)).Msg("Failed to drop archived observations from sqlite-vec")
+		}
+	})
+}
+
 // handleUnarchiveObservation restores an archived observation.
 func (s *Service) handleUnarchiveObservation(w http.ResponseWriter, r *http.Request) {
 	idStr := chi.URLParam(r, "id")
@@ -336,6 +354,21 @@ func (s *Service) handleUnarchiveObservation(w http.ResponseWriter, r *http.Requ
 		return
 	}
 
+	// Archiving dropped the note's vectors; put them back so search finds it again.
+	if s.vectorSync != nil {
+		s.asyncVectorSync(func() {
+			ctx, cancel := context.WithTimeout(s.ctx, 10*time.Second)
+			defer cancel()
+			obs, err := s.observationStore.GetObservationByID(ctx, id)
+			if err != nil || obs == nil {
+				return
+			}
+			if err := s.vectorSync.SyncObservation(ctx, obs); err != nil && s.ctx.Err() == nil {
+				log.Warn().Err(err).Int64("id", id).Msg("Failed to restore an unarchived observation to sqlite-vec")
+			}
+		})
+	}
+
 	// Invalidate all caches since we don't know the project
 	s.invalidateAllObsCountCache()
 
@@ -400,9 +433,9 @@ func (s *Service) handleExportObservations(w http.ResponseWriter, r *http.Reques
 	var err error
 
 	if project != "" {
-		observations, _, err = s.observationStore.GetObservationsByProjectStrictPaginated(ctx, project, limit, 0)
+		observations, _, err = s.observationStore.GetObservationsByProjectStrictPaginated(ctx, project, limit, 0, true)
 	} else {
-		observations, _, err = s.observationStore.GetAllRecentObservationsPaginated(ctx, limit, 0)
+		observations, _, err = s.observationStore.GetAllRecentObservationsPaginated(ctx, limit, 0, true)
 	}
 
 	if err != nil {
@@ -531,14 +564,17 @@ func (s *Service) handleBulkStatusUpdate(w http.ResponseWriter, r *http.Request)
 		}
 
 	case "archive":
+		var archivedIDs []int64
 		for _, id := range req.IDs {
 			if err := s.observationStore.ArchiveObservation(ctx, id, req.Reason); err != nil {
 				failed++
 				errors = append(errors, fmt.Sprintf("id %d: %v", id, err))
 			} else {
 				updated++
+				archivedIDs = append(archivedIDs, id)
 			}
 		}
+		s.dropArchivedFromVectors(archivedIDs)
 
 	case "set_feedback":
 		if req.Feedback < -1 || req.Feedback > 1 {
@@ -598,9 +634,9 @@ func (s *Service) handleFindDuplicates(w http.ResponseWriter, r *http.Request) {
 	var err error
 
 	if project != "" {
-		observations, _, err = s.observationStore.GetObservationsByProjectStrictPaginated(ctx, project, limit, 0)
+		observations, _, err = s.observationStore.GetObservationsByProjectStrictPaginated(ctx, project, limit, 0, true)
 	} else {
-		observations, _, err = s.observationStore.GetAllRecentObservationsPaginated(ctx, limit, 0)
+		observations, _, err = s.observationStore.GetAllRecentObservationsPaginated(ctx, limit, 0, true)
 	}
 
 	if err != nil {