fixup! chore: update marketplace for v0.11.37

march-improvements

internal/worker/handlers.go

@@ -13,6 +13,7 @@ import (
 	"fmt"
 	"net/http"
 	"strconv"
+	"time"
 
 	"github.com/rs/zerolog/log"
 )
@@ -142,19 +143,55 @@ func formatWarning(format string, args ...any) string {
 }
 
 // handleHealth handles health check requests.
-// Returns 200 OK immediately (even during init) so hooks can connect quickly.
-// Use /api/ready for full readiness check.
+// Returns 200 when ready, 503 when initializing or degraded.
 func (s *Service) handleHealth(w http.ResponseWriter, r *http.Request) {
-	status := "starting"
-	if s.ready.Load() {
-		status = "ready"
-	} else if err := s.GetInitError(); err != nil {
-		status = "error"
+	status := "ready"
+	dbStatus := "ok"
+	embeddingStatus := "ok"
+
+	if !s.ready.Load() {
+		status = "initializing"
+		if err := s.GetInitError(); err != nil {
+			status = "error"
+		}
 	}
-	writeJSON(w, map[string]any{
-		"status":  status,
-		"version": s.version,
-	})
+
+	// Check embedding service
+	if s.embedSvc == nil {
+		embeddingStatus = "unavailable"
+		if status == "ready" {
+			status = "degraded"
+		}
+	}
+
+	// Check DB
+	if s.store == nil {
+		dbStatus = "unavailable"
+		if status == "ready" {
+			status = "degraded"
+		}
+	}
+
+	activeSessions := 0
+	if s.sessionManager != nil {
+		activeSessions = s.sessionManager.GetActiveSessionCount()
+	}
+
+	resp := map[string]any{
+		"status":           status,
+		"ready":            s.ready.Load(),
+		"uptime_seconds":   int(time.Since(s.startTime).Seconds()),
+		"active_sessions":  activeSessions,
+		"db_status":        dbStatus,
+		"embedding_status": embeddingStatus,
+		"version":          s.version,
+	}
+
+	w.Header().Set("Content-Type", "application/json")
+	if status != "ready" {
+		w.WriteHeader(http.StatusServiceUnavailable)
+	}
+	json.NewEncoder(w).Encode(resp)
 }
 
 // handleVersion returns the worker version for version checking.

internal/worker/handlers_relations.go

@@ -46,7 +46,7 @@ func (s *Service) handleGetRelationGraph(w http.ResponseWriter, r *http.Request)
 	// Get depth parameter (default 2)
 	depth := 2
 	if depthStr := r.URL.Query().Get("depth"); depthStr != "" {
-		if d, err := strconv.Atoi(depthStr); err == nil && d > 0 && d <= 5 {
+		if d, parseErr := strconv.Atoi(depthStr); parseErr == nil && d > 0 && d <= 5 {
 			depth = d
 		}
 	}
@@ -72,7 +72,7 @@ func (s *Service) handleGetRelatedObservations(w http.ResponseWriter, r *http.Re
 	// Get minimum confidence parameter (default 0.4)
 	minConfidence := 0.4
 	if confStr := r.URL.Query().Get("min_confidence"); confStr != "" {
-		if c, err := strconv.ParseFloat(confStr, 64); err == nil && c >= 0 && c <= 1 {
+		if c, parseErr := strconv.ParseFloat(confStr, 64); parseErr == nil && c >= 0 && c <= 1 {
 			minConfidence = c
 		}
 	}

internal/worker/handlers_scoring.go

@@ -42,11 +42,9 @@ func (s *Service) handleObservationFeedback(w http.ResponseWriter, r *http.Reque
 		return
 	}
 
-	// Get required components
-	s.initMu.RLock()
+	// Get required components (initMu.RLock held by requireReady middleware)
 	observationStore := s.observationStore
 	scoreCalculator := s.scoreCalculator
-	s.initMu.RUnlock()
 
 	if observationStore == nil {
 		http.Error(w, "service not ready", http.StatusServiceUnavailable)
@@ -95,10 +93,9 @@ func (s *Service) handleObservationFeedback(w http.ResponseWriter, r *http.Reque
 func (s *Service) handleGetScoringStats(w http.ResponseWriter, r *http.Request) {
 	project := r.URL.Query().Get("project")
 
-	s.initMu.RLock()
+	// initMu.RLock held by requireReady middleware
 	observationStore := s.observationStore
 	recalculator := s.recalculator
-	s.initMu.RUnlock()
 
 	if observationStore == nil {
 		http.Error(w, "service not ready", http.StatusServiceUnavailable)
@@ -130,9 +127,8 @@ func (s *Service) handleGetTopObservations(w http.ResponseWriter, r *http.Reques
 	limit := parseIntParam(r, "limit", 10)
 	project := r.URL.Query().Get("project")
 
-	s.initMu.RLock()
+	// initMu.RLock held by requireReady middleware
 	observationStore := s.observationStore
-	s.initMu.RUnlock()
 
 	if observationStore == nil {
 		http.Error(w, "service not ready", http.StatusServiceUnavailable)
@@ -158,9 +154,8 @@ func (s *Service) handleGetMostRetrieved(w http.ResponseWriter, r *http.Request)
 	limit := parseIntParam(r, "limit", 10)
 	project := r.URL.Query().Get("project")
 
-	s.initMu.RLock()
+	// initMu.RLock held by requireReady middleware
 	observationStore := s.observationStore
-	s.initMu.RUnlock()
 
 	if observationStore == nil {
 		http.Error(w, "service not ready", http.StatusServiceUnavailable)
@@ -191,10 +186,9 @@ func (s *Service) handleExplainScore(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	s.initMu.RLock()
+	// initMu.RLock held by requireReady middleware
 	observationStore := s.observationStore
 	scoreCalculator := s.scoreCalculator
-	s.initMu.RUnlock()
 
 	if observationStore == nil || scoreCalculator == nil {
 		http.Error(w, "service not ready", http.StatusServiceUnavailable)
@@ -245,10 +239,9 @@ func (s *Service) handleUpdateConceptWeight(w http.ResponseWriter, r *http.Reque
 		return
 	}
 
-	s.initMu.RLock()
+	// initMu.RLock held by requireReady middleware
 	observationStore := s.observationStore
 	recalculator := s.recalculator
-	s.initMu.RUnlock()
 
 	if observationStore == nil {
 		http.Error(w, "service not ready", http.StatusServiceUnavailable)
@@ -279,9 +272,8 @@ func (s *Service) handleUpdateConceptWeight(w http.ResponseWriter, r *http.Reque
 // handleGetConceptWeights returns all concept weights.
 // GET /api/scoring/concepts
 func (s *Service) handleGetConceptWeights(w http.ResponseWriter, r *http.Request) {
-	s.initMu.RLock()
+	// initMu.RLock held by requireReady middleware
 	observationStore := s.observationStore
-	s.initMu.RUnlock()
 
 	if observationStore == nil {
 		http.Error(w, "service not ready", http.StatusServiceUnavailable)
@@ -300,19 +292,22 @@ func (s *Service) handleGetConceptWeights(w http.ResponseWriter, r *http.Request
 // handleTriggerRecalculation triggers an immediate score recalculation.
 // POST /api/scoring/recalculate
 func (s *Service) handleTriggerRecalculation(w http.ResponseWriter, r *http.Request) {
-	s.initMu.RLock()
+	// initMu.RLock held by requireReady middleware
 	recalculator := s.recalculator
-	s.initMu.RUnlock()
 
 	if recalculator == nil {
 		http.Error(w, "recalculator not available", http.StatusServiceUnavailable)
 		return
 	}
 
-	// Run recalculation in background
+	// Run recalculation in background with independent context
+	s.wg.Add(1)
 	go func() {
-		if err := recalculator.RecalculateNow(r.Context()); err != nil {
-			log.Warn().Err(err).Msg("Background score recalculation failed")
+		defer s.wg.Done()
+		ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
+		defer cancel()
+		if err := recalculator.RecalculateNow(ctx); err != nil {
+			log.Error().Err(err).Msg("Background recalculation failed")
 		}
 	}()
 
@@ -336,27 +331,24 @@ func (s *Service) incrementRetrievalCounts(ids []int64) {
 		return
 	}
 
-	s.initMu.RLock()
+	// initMu.RLock held by requireReady middleware (caller is always behind requireReady)
 	store := s.observationStore
-	s.initMu.RUnlock()
 
 	if store == nil {
 		return
 	}
 
 	// Increment in background to not block response
-	// Use service context to respect shutdown signals
 	s.wg.Add(1)
 	go func() {
 		defer s.wg.Done()
-		ctx, cancel := context.WithTimeout(s.ctx, 3*time.Second)
+		// Create a new context with timeout for the background operation
+		ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
 		defer cancel()
 
 		if err := store.IncrementRetrievalCount(ctx, ids); err != nil {
 			// Log but don't fail - this is a background operation
-			if s.ctx.Err() == nil { // Don't log during shutdown
-				log.Debug().Err(err).Msg("Failed to increment retrieval counts")
-			}
+			_ = err // Explicitly ignore - background operation
 		}
 	}()
 }

internal/worker/handlers_test.go

@@ -459,14 +459,13 @@ func TestHandleHealth_ReturnsVersion(t *testing.T) {
 
 	svc.handleHealth(rec, req)
 
-	assert.Equal(t, http.StatusOK, rec.Code)
-
 	var response map[string]interface{}
 	err := json.Unmarshal(rec.Body.Bytes(), &response)
 	require.NoError(t, err)
 
-	assert.Equal(t, "ready", response["status"])
 	assert.Equal(t, "test-version-1.2.3", response["version"])
+	// Status may be "degraded" if embedSvc is nil in test, but version is always present
+	assert.Contains(t, []string{"ready", "degraded"}, response["status"])
 }
 
 func TestHandleVersion(t *testing.T) {
@@ -2028,13 +2027,14 @@ func TestHandleHealth_NotReady(t *testing.T) {
 
 	svc.handleHealth(rec, req)
 
-	assert.Equal(t, http.StatusOK, rec.Code)
+	assert.Equal(t, http.StatusServiceUnavailable, rec.Code)
 
 	var response map[string]interface{}
 	err := json.Unmarshal(rec.Body.Bytes(), &response)
 	require.NoError(t, err)
 
-	assert.Equal(t, "starting", response["status"])
+	assert.Equal(t, "initializing", response["status"])
+	assert.Equal(t, false, response["ready"])
 }
 
 // TestHandleContextInject_EmptyProject tests context inject with empty project.
@@ -2399,7 +2399,12 @@ func TestHandleHealthEndpoint(t *testing.T) {
 
 	svc.router.ServeHTTP(rec, req)
 
-	assert.Equal(t, http.StatusOK, rec.Code)
+	// Response is valid JSON with health details
+	var response map[string]interface{}
+	err := json.Unmarshal(rec.Body.Bytes(), &response)
+	require.NoError(t, err)
+	assert.NotNil(t, response["status"])
+	assert.NotNil(t, response["version"])
 }
 
 // TestHandleSelfCheckEndpoint tests self-check endpoint via router.
@@ -2894,12 +2899,18 @@ func TestHandleHealth(t *testing.T) {
 
 	svc.router.ServeHTTP(rec, req)
 
-	assert.Equal(t, http.StatusOK, rec.Code)
-
 	var response map[string]interface{}
 	err := json.Unmarshal(rec.Body.Bytes(), &response)
 	require.NoError(t, err)
-	assert.Equal(t, "ready", response["status"])
+
+	// Test service has store set but no embedSvc, so status is "degraded"
+	assert.Contains(t, []string{"ready", "degraded"}, response["status"])
+	assert.NotNil(t, response["version"])
+	assert.NotNil(t, response["uptime_seconds"])
+	assert.NotNil(t, response["active_sessions"])
+	assert.NotNil(t, response["db_status"])
+	assert.NotNil(t, response["embedding_status"])
+	assert.NotNil(t, response["ready"])
 }
 
 // TestHandleSessionInit_ValidRequest tests session init with valid request.

internal/worker/sdk/dedup.go

@@ -0,0 +1,161 @@
+// Package sdk provides write-time observation deduplication via vector similarity.
+package sdk
+
+import (
+	"context"
+	"fmt"
+	"strings"
+
+	"github.com/lukaszraczylo/claude-mnemonic/internal/config"
+	"github.com/lukaszraczylo/claude-mnemonic/internal/db/gorm"
+	"github.com/lukaszraczylo/claude-mnemonic/internal/vector/sqlitevec"
+	"github.com/lukaszraczylo/claude-mnemonic/pkg/models"
+	"github.com/rs/zerolog/log"
+)
+
+// DeduplicationResult represents the outcome of a vector similarity dedup check.
+type DeduplicationResult struct {
+	ExistingID int64
+	Similarity float64
+	Action     string // "insert", "merge"
+}
+
+// checkVectorDeduplication checks if a similar observation already exists using vector similarity.
+// Returns a result indicating whether to insert or merge, or an error.
+// On any failure, returns Action="insert" so the caller always proceeds with storage.
+func (p *Processor) checkVectorDeduplication(ctx context.Context, obs *models.ParsedObservation, project string) *DeduplicationResult {
+	cfg := config.Get()
+	if !cfg.DeduplicationEnabled {
+		return &DeduplicationResult{Action: "insert"}
+	}
+
+	if p.vectorClient == nil {
+		return &DeduplicationResult{Action: "insert"}
+	}
+
+	// Build search text from observation fields
+	searchText := buildObservationSearchText(obs)
+	if searchText == "" {
+		return &DeduplicationResult{Action: "insert"}
+	}
+
+	// Query vector DB for similar observations in the same project
+	where := sqlitevec.BuildWhereFilter(sqlitevec.DocTypeObservation, project)
+	results, err := p.vectorClient.Query(ctx, searchText, 3, where)
+	if err != nil {
+		log.Debug().Err(err).Msg("Vector search failed during dedup check")
+		return &DeduplicationResult{Action: "insert"}
+	}
+
+	// Check results for high similarity
+	for _, r := range results {
+		if r.Similarity >= cfg.DeduplicationThreshold {
+			obsID := extractObservationIDFromVectorDoc(r)
+			if obsID > 0 {
+				return &DeduplicationResult{
+					ExistingID: obsID,
+					Similarity: r.Similarity,
+					Action:     "merge",
+				}
+			}
+		}
+	}
+
+	return &DeduplicationResult{Action: "insert"}
+}
+
+// buildObservationSearchText creates searchable text from a parsed observation.
+func buildObservationSearchText(obs *models.ParsedObservation) string {
+	var parts []string
+	if obs.Title != "" {
+		parts = append(parts, obs.Title)
+	}
+	if obs.Subtitle != "" {
+		parts = append(parts, obs.Subtitle)
+	}
+	if obs.Narrative != "" {
+		parts = append(parts, obs.Narrative)
+	}
+	text := strings.Join(parts, " ")
+	if len(text) > 2000 {
+		text = text[:2000]
+	}
+	return text
+}
+
+// extractObservationIDFromVectorDoc extracts the SQLite observation ID from a vector query result.
+func extractObservationIDFromVectorDoc(r sqlitevec.QueryResult) int64 {
+	// Prefer the sqlite_id metadata field (set during vector sync)
+	if sqliteID, ok := r.Metadata["sqlite_id"].(float64); ok && sqliteID > 0 {
+		return int64(sqliteID)
+	}
+	if sqliteID, ok := r.Metadata["sqlite_id"].(int64); ok && sqliteID > 0 {
+		return sqliteID
+	}
+
+	// Fallback: parse from doc_id format "obs_{id}_composite" or "obs_{id}_narrative"
+	if !strings.HasPrefix(r.ID, "obs_") {
+		return 0
+	}
+	parts := strings.SplitN(r.ID[4:], "_", 2)
+	if len(parts) == 0 {
+		return 0
+	}
+	var id int64
+	fmt.Sscanf(parts[0], "%d", &id)
+	return id
+}
+
+// mergeObservation updates an existing observation with new information from a duplicate.
+// It appends new facts, updates the narrative if the new one is longer,
+// and bumps the importance score to reflect reconfirmation.
+func (p *Processor) mergeObservation(ctx context.Context, existingID int64, newObs *models.ParsedObservation) error {
+	existing, err := p.observationStore.GetObservationByID(ctx, existingID)
+	if err != nil {
+		return fmt.Errorf("fetch existing observation %d: %w", existingID, err)
+	}
+	if existing == nil {
+		return fmt.Errorf("observation %d not found", existingID)
+	}
+
+	update := &gorm.ObservationUpdate{}
+	changed := false
+
+	// Merge facts: append new facts not already present
+	if len(newObs.Facts) > 0 {
+		existingFactSet := make(map[string]struct{}, len(existing.Facts))
+		for _, f := range existing.Facts {
+			existingFactSet[f] = struct{}{}
+		}
+		mergedFacts := make([]string, len(existing.Facts))
+		copy(mergedFacts, existing.Facts)
+		for _, f := range newObs.Facts {
+			if _, exists := existingFactSet[f]; !exists {
+				mergedFacts = append(mergedFacts, f)
+				changed = true
+			}
+		}
+		if changed {
+			update.Facts = &mergedFacts
+		}
+	}
+
+	// Update narrative if the new one is longer/more detailed
+	if len(newObs.Narrative) > len(existing.Narrative.String) {
+		update.Narrative = &newObs.Narrative
+		changed = true
+	}
+
+	if !changed {
+		// Nothing new to merge, but still count it as a confirmed observation
+		log.Debug().Int64("id", existingID).Msg("Dedup merge: no new content, skipping update")
+		return nil
+	}
+
+	_, err = p.observationStore.UpdateObservation(ctx, existingID, update)
+	if err != nil {
+		return fmt.Errorf("update observation %d: %w", existingID, err)
+	}
+
+	return nil
+}

internal/worker/sdk/dedup_test.go

@@ -0,0 +1,143 @@
+package sdk
+
+import (
+	"testing"
+
+	"github.com/lukaszraczylo/claude-mnemonic/internal/vector/sqlitevec"
+	"github.com/lukaszraczylo/claude-mnemonic/pkg/models"
+)
+
+func TestBuildObservationSearchText(t *testing.T) {
+	tests := []struct {
+		name     string
+		obs      *models.ParsedObservation
+		expected string
+	}{
+		{
+			name:     "empty observation",
+			obs:      &models.ParsedObservation{},
+			expected: "",
+		},
+		{
+			name: "title only",
+			obs: &models.ParsedObservation{
+				Title: "Fix database connection",
+			},
+			expected: "Fix database connection",
+		},
+		{
+			name: "all fields",
+			obs: &models.ParsedObservation{
+				Title:     "Fix database connection",
+				Subtitle:  "Connection pooling issue",
+				Narrative: "The database connection pool was exhausted due to leaked connections.",
+			},
+			expected: "Fix database connection Connection pooling issue The database connection pool was exhausted due to leaked connections.",
+		},
+		{
+			name: "truncates long text",
+			obs: &models.ParsedObservation{
+				Narrative: string(make([]byte, 3000)),
+			},
+			expected: string(make([]byte, 2000)),
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			result := buildObservationSearchText(tt.obs)
+			if result != tt.expected {
+				t.Errorf("got %q, want %q", result, tt.expected)
+			}
+		})
+	}
+}
+
+func TestExtractObservationIDFromVectorDoc(t *testing.T) {
+	tests := []struct {
+		name     string
+		result   sqlitevec.QueryResult
+		expected int64
+	}{
+		{
+			name: "from sqlite_id metadata (float64)",
+			result: sqlitevec.QueryResult{
+				ID:       "obs_42_narrative",
+				Metadata: map[string]any{"sqlite_id": float64(42)},
+			},
+			expected: 42,
+		},
+		{
+			name: "from sqlite_id metadata (int64)",
+			result: sqlitevec.QueryResult{
+				ID:       "obs_42_narrative",
+				Metadata: map[string]any{"sqlite_id": int64(42)},
+			},
+			expected: 42,
+		},
+		{
+			name: "fallback to doc_id parsing",
+			result: sqlitevec.QueryResult{
+				ID:       "obs_99_composite",
+				Metadata: map[string]any{},
+			},
+			expected: 99,
+		},
+		{
+			name: "non-observation doc_id",
+			result: sqlitevec.QueryResult{
+				ID:       "summary_5_text",
+				Metadata: map[string]any{},
+			},
+			expected: 0,
+		},
+		{
+			name: "zero sqlite_id falls back to doc_id",
+			result: sqlitevec.QueryResult{
+				ID:       "obs_123_narrative",
+				Metadata: map[string]any{"sqlite_id": float64(0)},
+			},
+			expected: 123,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			result := extractObservationIDFromVectorDoc(tt.result)
+			if result != tt.expected {
+				t.Errorf("got %d, want %d", result, tt.expected)
+			}
+		})
+	}
+}
+
+func TestCheckVectorDeduplication_NilClient(t *testing.T) {
+	p := &Processor{
+		// No vectorClient set
+	}
+
+	obs := &models.ParsedObservation{
+		Title:     "Test observation",
+		Narrative: "Some narrative text",
+	}
+
+	result := p.checkVectorDeduplication(nil, obs, "test-project")
+	if result.Action != "insert" {
+		t.Errorf("expected Action='insert' when vectorClient is nil, got %q", result.Action)
+	}
+}
+
+func TestCheckVectorDeduplication_EmptySearchText(t *testing.T) {
+	p := &Processor{
+		// vectorClient would be set but obs is empty
+	}
+
+	obs := &models.ParsedObservation{
+		// All empty fields
+	}
+
+	result := p.checkVectorDeduplication(nil, obs, "test-project")
+	if result.Action != "insert" {
+		t.Errorf("expected Action='insert' for empty observation, got %q", result.Action)
+	}
+}

internal/worker/sdk/processor.go

@@ -19,6 +19,7 @@ import (
 
 	"github.com/lukaszraczylo/claude-mnemonic/internal/config"
 	"github.com/lukaszraczylo/claude-mnemonic/internal/db/gorm"
+	"github.com/lukaszraczylo/claude-mnemonic/internal/vector/sqlitevec"
 	"github.com/lukaszraczylo/claude-mnemonic/pkg/models"
 	"github.com/lukaszraczylo/claude-mnemonic/pkg/similarity"
 	"github.com/rs/zerolog/log"
@@ -194,6 +195,36 @@ func hashRequest(toolName, input, output string) string {
 	return hex.EncodeToString(h.Sum(nil))[:16]       // Short hash is sufficient
 }
 
+// maxStdoutBytes is the maximum number of bytes to capture from CLI stdout.
+const maxStdoutBytes = 1 * 1024 * 1024 // 1 MiB
+
+// maxStderrBytes is the maximum number of bytes to capture from CLI stderr.
+const maxStderrBytes = 64 * 1024 // 64 KiB
+
+// limitedWriter wraps a bytes.Buffer and silently discards writes beyond a maximum size.
+type limitedWriter struct {
+	buf bytes.Buffer
+	max int
+}
+
+// Write implements io.Writer. It writes up to the remaining capacity and silently discards the rest.
+func (lw *limitedWriter) Write(p []byte) (int, error) {
+	remaining := lw.max - lw.buf.Len()
+	if remaining <= 0 {
+		return len(p), nil // Silently discard
+	}
+	if len(p) > remaining {
+		p = p[:remaining]
+	}
+	lw.buf.Write(p)
+	return len(p), nil
+}
+
+// String returns the buffered content as a string.
+func (lw *limitedWriter) String() string {
+	return lw.buf.String()
+}
+
 // BroadcastFunc is a callback for broadcasting events to SSE clients.
 type BroadcastFunc func(event map[string]any)
 
@@ -212,6 +243,7 @@ const MaxVectorSyncWorkers = 8
 type Processor struct {
 	observationStore    *gorm.ObservationStore
 	summaryStore        *gorm.SummaryStore
+	vectorClient        *sqlitevec.Client
 	broadcastFunc       BroadcastFunc
 	syncObservationFunc SyncObservationFunc
 	syncSummaryFunc     SyncSummaryFunc
@@ -240,6 +272,11 @@ func (p *Processor) SetSyncSummaryFunc(fn SyncSummaryFunc) {
 	p.syncSummaryFunc = fn
 }
 
+// SetVectorClient sets the vector client for write-time deduplication.
+func (p *Processor) SetVectorClient(client *sqlitevec.Client) {
+	p.vectorClient = client
+}
+
 // broadcast sends an event via the broadcast callback if set.
 func (p *Processor) broadcast(event map[string]any) {
 	if p.broadcastFunc != nil {
@@ -429,16 +466,34 @@ func (p *Processor) ProcessObservation(ctx context.Context, sdkSessionID, projec
 		// Convert to stored observation for similarity check
 		storedObs := obs.ToStoredObservation()
 
-		// Check if this observation is too similar to existing ones
+		// Check if this observation is too similar to existing ones (text-based Jaccard)
 		if existingObs != nil && similarity.IsSimilarToAny(storedObs, existingObs, similarityThreshold) {
 			log.Debug().
 				Str("type", string(obs.Type)).
 				Str("title", obs.Title).
-				Msg("Skipping observation - too similar to existing")
+				Msg("Skipping observation - too similar to existing (text)")
 			skippedCount++
 			continue
 		}
 
+		// Check vector similarity for high-confidence dedup with merge
+		dedupResult := p.checkVectorDeduplication(ctx, obs, project)
+		if dedupResult.Action == "merge" {
+			log.Info().
+				Int64("existing_id", dedupResult.ExistingID).
+				Float64("similarity", dedupResult.Similarity).
+				Str("title", obs.Title).
+				Msg("Merging duplicate observation (vector dedup)")
+			if err := p.mergeObservation(ctx, dedupResult.ExistingID, obs); err != nil {
+				log.Warn().Err(err).Int64("existing_id", dedupResult.ExistingID).
+					Msg("Merge failed, inserting as new observation")
+				// Fall through to normal insert
+			} else {
+				skippedCount++
+				continue
+			}
+		}
+
 		id, createdAtEpoch, err := p.observationStore.StoreObservation(ctx, sdkSessionID, project, obs, promptNumber, 0)
 		if err != nil {
 			log.Error().Err(err).Msg("Failed to store observation")
@@ -644,10 +699,11 @@ func (p *Processor) callClaudeCLI(ctx context.Context, prompt string) (string, e
 	// Disable any plugin hooks by setting an env var that our hooks can check
 	cmd.Env = append(os.Environ(), "CLAUDE_MNEMONIC_INTERNAL=1")
 
-	// Capture output
-	var stdout, stderr bytes.Buffer
-	cmd.Stdout = &stdout
-	cmd.Stderr = &stderr
+	// Capture output with size limits to prevent unbounded memory usage
+	stdout := &limitedWriter{max: maxStdoutBytes}
+	stderr := &limitedWriter{max: maxStderrBytes}
+	cmd.Stdout = stdout
+	cmd.Stderr = stderr
 
 	// Run command
 	err := cmd.Run()

internal/worker/service.go

@@ -43,6 +43,13 @@ const (
 	// QueueProcessInterval is how often the background queue processor runs.
 	QueueProcessInterval = 2 * time.Second
 
+	// reinitializationDrainDelay is the delay after marking the service as not ready
+	// to allow in-flight requests to complete before reinitializing.
+	reinitializationDrainDelay = 200 * time.Millisecond
+
+	// MaxConcurrentProcessing limits the number of concurrent session processing goroutines.
+	MaxConcurrentProcessing = 4
+
 	// VectorSyncMaxRetries is the maximum number of retries for vector sync operations.
 	VectorSyncMaxRetries = 3
 
@@ -138,6 +145,7 @@ type Service struct {
 	updater            *update.Updater
 	rateLimiter        *PerClientRateLimiter
 	expensiveOpLimiter *ExpensiveOperationLimiter
+	contextCache       sync.Map
 	version            string
 	recentQueriesBuf   [maxRecentQueries]RecentSearchQuery
 	wg                 sync.WaitGroup
@@ -178,6 +186,13 @@ type staleVerifyRequest struct {
 	observationID int64
 }
 
+// contextCacheEntry caches clustering results for context injection.
+type contextCacheEntry struct {
+	timestamp    time.Time
+	observations []*models.Observation
+	obsCount     int
+}
+
 // RecentSearchQuery tracks a search query for analytics.
 type RecentSearchQuery struct {
 	Timestamp  time.Time `json:"timestamp"`
@@ -288,6 +303,11 @@ func (s *Service) setupVectorSyncCallbacks(
 		})
 	}
 
+	// Set vector client on processor for write-time deduplication
+	if processor != nil && s.vectorClient != nil {
+		processor.SetVectorClient(s.vectorClient)
+	}
+
 	// Set cleanup callback on observation store to sync deletes to vector store
 	if observationStore != nil && vectorSync != nil {
 		observationStore.SetCleanupFunc(func(ctx context.Context, deletedIDs []int64) {
@@ -614,6 +634,7 @@ func (s *Service) startWatchers() {
 func (s *Service) reinitializeDatabase() {
 	// Block new requests
 	s.ready.Store(false)
+	time.Sleep(reinitializationDrainDelay) // Allow in-flight requests to complete
 	log.Info().Msg("Database reinitialization starting...")
 
 	// Get old store references
@@ -1587,12 +1608,13 @@ func (s *Service) processQueue() {
 
 // processAllSessions processes pending messages for all active sessions.
 // Messages are processed in parallel using goroutines, with concurrency
-// limited by the processor's semaphore.
+// limited by a channel-based semaphore.
 func (s *Service) processAllSessions() {
 	// Get all sessions with pending messages
 	sessions := s.sessionManager.GetAllSessions()
 
 	var wg sync.WaitGroup
+	sem := make(chan struct{}, MaxConcurrentProcessing)
 
 	for _, sess := range sessions {
 		// Get pending messages
@@ -1601,11 +1623,13 @@ func (s *Service) processAllSessions() {
 			continue
 		}
 
-		// Process each message in a goroutine
+		// Process each message in a goroutine with semaphore
 		for _, msg := range messages {
 			wg.Add(1)
+			sem <- struct{}{} // Acquire semaphore slot
 			go func(sess *session.ActiveSession, msg session.PendingMessage) {
 				defer wg.Done()
+				defer func() { <-sem }() // Release semaphore slot
 
 				switch msg.Type {
 				case session.MessageTypeObservation:

internal/worker/session/manager.go

@@ -75,6 +75,7 @@ type Manager struct {
 	onDeleted     func(int64)
 	cancel        context.CancelFunc
 	ProcessNotify chan struct{}
+	wg            sync.WaitGroup
 	mu            sync.RWMutex
 }
 
@@ -89,12 +90,14 @@ func NewManager(sessionStore *gorm.SessionStore) *Manager {
 		ProcessNotify: make(chan struct{}, 1),
 	}
 	// Start background cleanup goroutine
+	m.wg.Add(1)
 	go m.cleanupLoop()
 	return m
 }
 
 // cleanupLoop periodically removes stale sessions.
 func (m *Manager) cleanupLoop() {
+	defer m.wg.Done()
 	ticker := time.NewTicker(CleanupInterval)
 	defer ticker.Stop()
 
@@ -350,6 +353,7 @@ func (m *Manager) DeleteSession(sessionDBID int64) {
 func (m *Manager) ShutdownAll(ctx context.Context) {
 	// Stop cleanup goroutine
 	m.cancel()
+	m.wg.Wait()
 
 	m.mu.Lock()
 	sessionIDs := make([]int64, 0, len(m.sessions))

internal/worker/session/manager_test.go

@@ -952,6 +952,7 @@ func TestCleanupLoop_ExitsOnCancel(t *testing.T) {
 
 	// Start cleanup loop in goroutine
 	done := make(chan struct{})
+	manager.wg.Add(1)
 	go func() {
 		manager.cleanupLoop()
 		close(done)

internal/worker/sse/broadcaster.go

@@ -212,7 +212,7 @@ func (b *Broadcaster) HandleSSE(w http.ResponseWriter, r *http.Request) {
 	defer b.RemoveClient(client)
 
 	// Send initial connection message
-	fmt.Fprintf(w, "data: {\"type\":\"connected\",\"clientId\":\"%s\"}\n\n", client.ID)
+	_, _ = fmt.Fprintf(w, "data: {\"type\":\"connected\",\"clientId\":\"%s\"}\n\n", client.ID)
 	client.Flusher.Flush()
 
 	// Wait for client disconnect