fixup! chore: update marketplace for v0.11.37

march-improvements

internal/worker/sdk/dedup.go

@@ -0,0 +1,161 @@
+// Package sdk provides write-time observation deduplication via vector similarity.
+package sdk
+
+import (
+	"context"
+	"fmt"
+	"strings"
+
+	"github.com/lukaszraczylo/claude-mnemonic/internal/config"
+	"github.com/lukaszraczylo/claude-mnemonic/internal/db/gorm"
+	"github.com/lukaszraczylo/claude-mnemonic/internal/vector/sqlitevec"
+	"github.com/lukaszraczylo/claude-mnemonic/pkg/models"
+	"github.com/rs/zerolog/log"
+)
+
+// DeduplicationResult represents the outcome of a vector similarity dedup check.
+type DeduplicationResult struct {
+	ExistingID int64
+	Similarity float64
+	Action     string // "insert", "merge"
+}
+
+// checkVectorDeduplication checks if a similar observation already exists using vector similarity.
+// Returns a result indicating whether to insert or merge, or an error.
+// On any failure, returns Action="insert" so the caller always proceeds with storage.
+func (p *Processor) checkVectorDeduplication(ctx context.Context, obs *models.ParsedObservation, project string) *DeduplicationResult {
+	cfg := config.Get()
+	if !cfg.DeduplicationEnabled {
+		return &DeduplicationResult{Action: "insert"}
+	}
+
+	if p.vectorClient == nil {
+		return &DeduplicationResult{Action: "insert"}
+	}
+
+	// Build search text from observation fields
+	searchText := buildObservationSearchText(obs)
+	if searchText == "" {
+		return &DeduplicationResult{Action: "insert"}
+	}
+
+	// Query vector DB for similar observations in the same project
+	where := sqlitevec.BuildWhereFilter(sqlitevec.DocTypeObservation, project)
+	results, err := p.vectorClient.Query(ctx, searchText, 3, where)
+	if err != nil {
+		log.Debug().Err(err).Msg("Vector search failed during dedup check")
+		return &DeduplicationResult{Action: "insert"}
+	}
+
+	// Check results for high similarity
+	for _, r := range results {
+		if r.Similarity >= cfg.DeduplicationThreshold {
+			obsID := extractObservationIDFromVectorDoc(r)
+			if obsID > 0 {
+				return &DeduplicationResult{
+					ExistingID: obsID,
+					Similarity: r.Similarity,
+					Action:     "merge",
+				}
+			}
+		}
+	}
+
+	return &DeduplicationResult{Action: "insert"}
+}
+
+// buildObservationSearchText creates searchable text from a parsed observation.
+func buildObservationSearchText(obs *models.ParsedObservation) string {
+	var parts []string
+	if obs.Title != "" {
+		parts = append(parts, obs.Title)
+	}
+	if obs.Subtitle != "" {
+		parts = append(parts, obs.Subtitle)
+	}
+	if obs.Narrative != "" {
+		parts = append(parts, obs.Narrative)
+	}
+	text := strings.Join(parts, " ")
+	if len(text) > 2000 {
+		text = text[:2000]
+	}
+	return text
+}
+
+// extractObservationIDFromVectorDoc extracts the SQLite observation ID from a vector query result.
+func extractObservationIDFromVectorDoc(r sqlitevec.QueryResult) int64 {
+	// Prefer the sqlite_id metadata field (set during vector sync)
+	if sqliteID, ok := r.Metadata["sqlite_id"].(float64); ok && sqliteID > 0 {
+		return int64(sqliteID)
+	}
+	if sqliteID, ok := r.Metadata["sqlite_id"].(int64); ok && sqliteID > 0 {
+		return sqliteID
+	}
+
+	// Fallback: parse from doc_id format "obs_{id}_composite" or "obs_{id}_narrative"
+	if !strings.HasPrefix(r.ID, "obs_") {
+		return 0
+	}
+	parts := strings.SplitN(r.ID[4:], "_", 2)
+	if len(parts) == 0 {
+		return 0
+	}
+	var id int64
+	fmt.Sscanf(parts[0], "%d", &id)
+	return id
+}
+
+// mergeObservation updates an existing observation with new information from a duplicate.
+// It appends new facts, updates the narrative if the new one is longer,
+// and bumps the importance score to reflect reconfirmation.
+func (p *Processor) mergeObservation(ctx context.Context, existingID int64, newObs *models.ParsedObservation) error {
+	existing, err := p.observationStore.GetObservationByID(ctx, existingID)
+	if err != nil {
+		return fmt.Errorf("fetch existing observation %d: %w", existingID, err)
+	}
+	if existing == nil {
+		return fmt.Errorf("observation %d not found", existingID)
+	}
+
+	update := &gorm.ObservationUpdate{}
+	changed := false
+
+	// Merge facts: append new facts not already present
+	if len(newObs.Facts) > 0 {
+		existingFactSet := make(map[string]struct{}, len(existing.Facts))
+		for _, f := range existing.Facts {
+			existingFactSet[f] = struct{}{}
+		}
+		mergedFacts := make([]string, len(existing.Facts))
+		copy(mergedFacts, existing.Facts)
+		for _, f := range newObs.Facts {
+			if _, exists := existingFactSet[f]; !exists {
+				mergedFacts = append(mergedFacts, f)
+				changed = true
+			}
+		}
+		if changed {
+			update.Facts = &mergedFacts
+		}
+	}
+
+	// Update narrative if the new one is longer/more detailed
+	if len(newObs.Narrative) > len(existing.Narrative.String) {
+		update.Narrative = &newObs.Narrative
+		changed = true
+	}
+
+	if !changed {
+		// Nothing new to merge, but still count it as a confirmed observation
+		log.Debug().Int64("id", existingID).Msg("Dedup merge: no new content, skipping update")
+		return nil
+	}
+
+	_, err = p.observationStore.UpdateObservation(ctx, existingID, update)
+	if err != nil {
+		return fmt.Errorf("update observation %d: %w", existingID, err)
+	}
+
+	return nil
+}

internal/worker/sdk/dedup_test.go

@@ -0,0 +1,143 @@
+package sdk
+
+import (
+	"testing"
+
+	"github.com/lukaszraczylo/claude-mnemonic/internal/vector/sqlitevec"
+	"github.com/lukaszraczylo/claude-mnemonic/pkg/models"
+)
+
+func TestBuildObservationSearchText(t *testing.T) {
+	tests := []struct {
+		name     string
+		obs      *models.ParsedObservation
+		expected string
+	}{
+		{
+			name:     "empty observation",
+			obs:      &models.ParsedObservation{},
+			expected: "",
+		},
+		{
+			name: "title only",
+			obs: &models.ParsedObservation{
+				Title: "Fix database connection",
+			},
+			expected: "Fix database connection",
+		},
+		{
+			name: "all fields",
+			obs: &models.ParsedObservation{
+				Title:     "Fix database connection",
+				Subtitle:  "Connection pooling issue",
+				Narrative: "The database connection pool was exhausted due to leaked connections.",
+			},
+			expected: "Fix database connection Connection pooling issue The database connection pool was exhausted due to leaked connections.",
+		},
+		{
+			name: "truncates long text",
+			obs: &models.ParsedObservation{
+				Narrative: string(make([]byte, 3000)),
+			},
+			expected: string(make([]byte, 2000)),
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			result := buildObservationSearchText(tt.obs)
+			if result != tt.expected {
+				t.Errorf("got %q, want %q", result, tt.expected)
+			}
+		})
+	}
+}
+
+func TestExtractObservationIDFromVectorDoc(t *testing.T) {
+	tests := []struct {
+		name     string
+		result   sqlitevec.QueryResult
+		expected int64
+	}{
+		{
+			name: "from sqlite_id metadata (float64)",
+			result: sqlitevec.QueryResult{
+				ID:       "obs_42_narrative",
+				Metadata: map[string]any{"sqlite_id": float64(42)},
+			},
+			expected: 42,
+		},
+		{
+			name: "from sqlite_id metadata (int64)",
+			result: sqlitevec.QueryResult{
+				ID:       "obs_42_narrative",
+				Metadata: map[string]any{"sqlite_id": int64(42)},
+			},
+			expected: 42,
+		},
+		{
+			name: "fallback to doc_id parsing",
+			result: sqlitevec.QueryResult{
+				ID:       "obs_99_composite",
+				Metadata: map[string]any{},
+			},
+			expected: 99,
+		},
+		{
+			name: "non-observation doc_id",
+			result: sqlitevec.QueryResult{
+				ID:       "summary_5_text",
+				Metadata: map[string]any{},
+			},
+			expected: 0,
+		},
+		{
+			name: "zero sqlite_id falls back to doc_id",
+			result: sqlitevec.QueryResult{
+				ID:       "obs_123_narrative",
+				Metadata: map[string]any{"sqlite_id": float64(0)},
+			},
+			expected: 123,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			result := extractObservationIDFromVectorDoc(tt.result)
+			if result != tt.expected {
+				t.Errorf("got %d, want %d", result, tt.expected)
+			}
+		})
+	}
+}
+
+func TestCheckVectorDeduplication_NilClient(t *testing.T) {
+	p := &Processor{
+		// No vectorClient set
+	}
+
+	obs := &models.ParsedObservation{
+		Title:     "Test observation",
+		Narrative: "Some narrative text",
+	}
+
+	result := p.checkVectorDeduplication(nil, obs, "test-project")
+	if result.Action != "insert" {
+		t.Errorf("expected Action='insert' when vectorClient is nil, got %q", result.Action)
+	}
+}
+
+func TestCheckVectorDeduplication_EmptySearchText(t *testing.T) {
+	p := &Processor{
+		// vectorClient would be set but obs is empty
+	}
+
+	obs := &models.ParsedObservation{
+		// All empty fields
+	}
+
+	result := p.checkVectorDeduplication(nil, obs, "test-project")
+	if result.Action != "insert" {
+		t.Errorf("expected Action='insert' for empty observation, got %q", result.Action)
+	}
+}

internal/worker/sdk/processor.go

@@ -19,6 +19,7 @@ import (
 
 	"github.com/lukaszraczylo/claude-mnemonic/internal/config"
 	"github.com/lukaszraczylo/claude-mnemonic/internal/db/gorm"
+	"github.com/lukaszraczylo/claude-mnemonic/internal/vector/sqlitevec"
 	"github.com/lukaszraczylo/claude-mnemonic/pkg/models"
 	"github.com/lukaszraczylo/claude-mnemonic/pkg/similarity"
 	"github.com/rs/zerolog/log"
@@ -194,6 +195,36 @@ func hashRequest(toolName, input, output string) string {
 	return hex.EncodeToString(h.Sum(nil))[:16]       // Short hash is sufficient
 }
 
+// maxStdoutBytes is the maximum number of bytes to capture from CLI stdout.
+const maxStdoutBytes = 1 * 1024 * 1024 // 1 MiB
+
+// maxStderrBytes is the maximum number of bytes to capture from CLI stderr.
+const maxStderrBytes = 64 * 1024 // 64 KiB
+
+// limitedWriter wraps a bytes.Buffer and silently discards writes beyond a maximum size.
+type limitedWriter struct {
+	buf bytes.Buffer
+	max int
+}
+
+// Write implements io.Writer. It writes up to the remaining capacity and silently discards the rest.
+func (lw *limitedWriter) Write(p []byte) (int, error) {
+	remaining := lw.max - lw.buf.Len()
+	if remaining <= 0 {
+		return len(p), nil // Silently discard
+	}
+	if len(p) > remaining {
+		p = p[:remaining]
+	}
+	lw.buf.Write(p)
+	return len(p), nil
+}
+
+// String returns the buffered content as a string.
+func (lw *limitedWriter) String() string {
+	return lw.buf.String()
+}
+
 // BroadcastFunc is a callback for broadcasting events to SSE clients.
 type BroadcastFunc func(event map[string]any)
 
@@ -212,6 +243,7 @@ const MaxVectorSyncWorkers = 8
 type Processor struct {
 	observationStore    *gorm.ObservationStore
 	summaryStore        *gorm.SummaryStore
+	vectorClient        *sqlitevec.Client
 	broadcastFunc       BroadcastFunc
 	syncObservationFunc SyncObservationFunc
 	syncSummaryFunc     SyncSummaryFunc
@@ -240,6 +272,11 @@ func (p *Processor) SetSyncSummaryFunc(fn SyncSummaryFunc) {
 	p.syncSummaryFunc = fn
 }
 
+// SetVectorClient sets the vector client for write-time deduplication.
+func (p *Processor) SetVectorClient(client *sqlitevec.Client) {
+	p.vectorClient = client
+}
+
 // broadcast sends an event via the broadcast callback if set.
 func (p *Processor) broadcast(event map[string]any) {
 	if p.broadcastFunc != nil {
@@ -429,16 +466,34 @@ func (p *Processor) ProcessObservation(ctx context.Context, sdkSessionID, projec
 		// Convert to stored observation for similarity check
 		storedObs := obs.ToStoredObservation()
 
-		// Check if this observation is too similar to existing ones
+		// Check if this observation is too similar to existing ones (text-based Jaccard)
 		if existingObs != nil && similarity.IsSimilarToAny(storedObs, existingObs, similarityThreshold) {
 			log.Debug().
 				Str("type", string(obs.Type)).
 				Str("title", obs.Title).
-				Msg("Skipping observation - too similar to existing")
+				Msg("Skipping observation - too similar to existing (text)")
 			skippedCount++
 			continue
 		}
 
+		// Check vector similarity for high-confidence dedup with merge
+		dedupResult := p.checkVectorDeduplication(ctx, obs, project)
+		if dedupResult.Action == "merge" {
+			log.Info().
+				Int64("existing_id", dedupResult.ExistingID).
+				Float64("similarity", dedupResult.Similarity).
+				Str("title", obs.Title).
+				Msg("Merging duplicate observation (vector dedup)")
+			if err := p.mergeObservation(ctx, dedupResult.ExistingID, obs); err != nil {
+				log.Warn().Err(err).Int64("existing_id", dedupResult.ExistingID).
+					Msg("Merge failed, inserting as new observation")
+				// Fall through to normal insert
+			} else {
+				skippedCount++
+				continue
+			}
+		}
+
 		id, createdAtEpoch, err := p.observationStore.StoreObservation(ctx, sdkSessionID, project, obs, promptNumber, 0)
 		if err != nil {
 			log.Error().Err(err).Msg("Failed to store observation")
@@ -644,10 +699,11 @@ func (p *Processor) callClaudeCLI(ctx context.Context, prompt string) (string, e
 	// Disable any plugin hooks by setting an env var that our hooks can check
 	cmd.Env = append(os.Environ(), "CLAUDE_MNEMONIC_INTERNAL=1")
 
-	// Capture output
-	var stdout, stderr bytes.Buffer
-	cmd.Stdout = &stdout
-	cmd.Stderr = &stderr
+	// Capture output with size limits to prevent unbounded memory usage
+	stdout := &limitedWriter{max: maxStdoutBytes}
+	stderr := &limitedWriter{max: maxStderrBytes}
+	cmd.Stdout = stdout
+	cmd.Stderr = stderr
 
 	// Run command
 	err := cmd.Run()