fix(db): keep archived notes out of search, lists, counts and the vector rebuild (#74)

* fix(db): keep archived notes out of search, lists, counts and the vector rebuild

An archived note is meant to be kept but unused, yet most read paths did not
filter on is_archived: full-text and LIKE search, vector-hit hydration,
context injection, the paginated lists, counts and the vector rebuild all still
returned it. A vector hit for an archived note also still pointed at a live
embedding, because archiving never dropped it.

- add notArchivedFilter/visibleScope and apply them to every read path
- the paginated lists take includeArchived; the API reads
  ?include_archived=true, and exports ask for them so a backup is complete
- archiving drops the note's vectors, unarchiving syncs them back
- add //go:build fts5 to observation_update_test.go so `go vet` without tags
  builds (it referenced a helper from a tagged file)

Adds a store test that archives one of three notes and checks every path, and
that fails if the filter is removed.

Co-Authored-By: Claude Sonnet 5.5 <[email protected]>

* fix(worker): drop the vectors of notes archived through bulk-status too

POST /api/observations/bulk-status with action "archive" archived notes but left
their vectors in the index; only the dedicated archive handler dropped them.
Both paths now call dropArchivedFromVectors.

Co-Authored-By: Claude Sonnet 5.5 <[email protected]>

---------

Co-authored-by: Claude Sonnet 5.5 <[email protected]>

internal/worker/handlers_data.go

@@ -28,6 +28,8 @@ func (s *Service) handleGetObservations(w http.ResponseWriter, r *http.Request)
 	pagination := gorm.ParsePaginationParams(r, DefaultObservationsLimit)
 	project := r.URL.Query().Get("project")
 	query := r.URL.Query().Get("query")
+	// Archived notes are not listed unless asked for.
+	includeArchived := r.URL.Query().Get("include_archived") == "true"
 
 	// Validate project name to prevent path traversal
 	if err := ValidateProjectName(project); err != nil {
@@ -60,10 +62,10 @@ func (s *Service) handleGetObservations(w http.ResponseWriter, r *http.Request)
 	if !usedVector {
 		if project != "" {
 			// Strict project filtering for dashboard - only observations from this project
-			observations, total, err = s.observationStore.GetObservationsByProjectStrictPaginated(ctx, project, pagination.Limit, pagination.Offset)
+			observations, total, err = s.observationStore.GetObservationsByProjectStrictPaginated(ctx, project, pagination.Limit, pagination.Offset, includeArchived)
 		} else {
 			// All projects
-			observations, total, err = s.observationStore.GetAllRecentObservationsPaginated(ctx, pagination.Limit, pagination.Offset)
+			observations, total, err = s.observationStore.GetAllRecentObservationsPaginated(ctx, pagination.Limit, pagination.Offset, includeArchived)
 		}
 	}
 

internal/worker/handlers_import_export.go

@@ -300,6 +300,8 @@ func (s *Service) handleArchiveObservations(w http.ResponseWriter, r *http.Reque
 		Int("failed", len(failedIDs)).
 		Msg("Observations archived")
 
+	s.dropArchivedFromVectors(archivedIDs)
+
 	// Invalidate cache if any observations were archived
 	if len(archivedIDs) > 0 {
 		if req.Project != "" {
@@ -322,6 +324,22 @@ func (s *Service) handleArchiveObservations(w http.ResponseWriter, r *http.Reque
 	writeJSON(w, response)
 }
 
+// dropArchivedFromVectors takes archived notes out of the vector index, so search cannot return them (unarchiving puts
+// them back). Every path that archives notes calls it.
+func (s *Service) dropArchivedFromVectors(ids []int64) {
+	if len(ids) == 0 || s.vectorSync == nil {
+		return
+	}
+	ids = append([]int64(nil), ids...)
+	s.asyncVectorSync(func() {
+		ctx, cancel := context.WithTimeout(s.ctx, 10*time.Second)
+		defer cancel()
+		if err := s.vectorSync.DeleteObservations(ctx, ids); err != nil && s.ctx.Err() == nil {
+			log.Warn().Err(err).Int("count", len(ids)).Msg("Failed to drop archived observations from sqlite-vec")
+		}
+	})
+}
+
 // handleUnarchiveObservation restores an archived observation.
 func (s *Service) handleUnarchiveObservation(w http.ResponseWriter, r *http.Request) {
 	idStr := chi.URLParam(r, "id")
@@ -336,6 +354,21 @@ func (s *Service) handleUnarchiveObservation(w http.ResponseWriter, r *http.Requ
 		return
 	}
 
+	// Archiving dropped the note's vectors; put them back so search finds it again.
+	if s.vectorSync != nil {
+		s.asyncVectorSync(func() {
+			ctx, cancel := context.WithTimeout(s.ctx, 10*time.Second)
+			defer cancel()
+			obs, err := s.observationStore.GetObservationByID(ctx, id)
+			if err != nil || obs == nil {
+				return
+			}
+			if err := s.vectorSync.SyncObservation(ctx, obs); err != nil && s.ctx.Err() == nil {
+				log.Warn().Err(err).Int64("id", id).Msg("Failed to restore an unarchived observation to sqlite-vec")
+			}
+		})
+	}
+
 	// Invalidate all caches since we don't know the project
 	s.invalidateAllObsCountCache()
 
@@ -400,9 +433,9 @@ func (s *Service) handleExportObservations(w http.ResponseWriter, r *http.Reques
 	var err error
 
 	if project != "" {
-		observations, _, err = s.observationStore.GetObservationsByProjectStrictPaginated(ctx, project, limit, 0)
+		observations, _, err = s.observationStore.GetObservationsByProjectStrictPaginated(ctx, project, limit, 0, true)
 	} else {
-		observations, _, err = s.observationStore.GetAllRecentObservationsPaginated(ctx, limit, 0)
+		observations, _, err = s.observationStore.GetAllRecentObservationsPaginated(ctx, limit, 0, true)
 	}
 
 	if err != nil {
@@ -531,14 +564,17 @@ func (s *Service) handleBulkStatusUpdate(w http.ResponseWriter, r *http.Request)
 		}
 
 	case "archive":
+		var archivedIDs []int64
 		for _, id := range req.IDs {
 			if err := s.observationStore.ArchiveObservation(ctx, id, req.Reason); err != nil {
 				failed++
 				errors = append(errors, fmt.Sprintf("id %d: %v", id, err))
 			} else {
 				updated++
+				archivedIDs = append(archivedIDs, id)
 			}
 		}
+		s.dropArchivedFromVectors(archivedIDs)
 
 	case "set_feedback":
 		if req.Feedback < -1 || req.Feedback > 1 {
@@ -598,9 +634,9 @@ func (s *Service) handleFindDuplicates(w http.ResponseWriter, r *http.Request) {
 	var err error
 
 	if project != "" {
-		observations, _, err = s.observationStore.GetObservationsByProjectStrictPaginated(ctx, project, limit, 0)
+		observations, _, err = s.observationStore.GetObservationsByProjectStrictPaginated(ctx, project, limit, 0, true)
 	} else {
-		observations, _, err = s.observationStore.GetAllRecentObservationsPaginated(ctx, limit, 0)
+		observations, _, err = s.observationStore.GetAllRecentObservationsPaginated(ctx, limit, 0, true)
 	}
 
 	if err != nil {