fix(vector): answer project-filtered queries with two searches (#70)

A project filter was '(project = ? OR scope = global)', which vec0 rejects in a nearest-neighbour query ('A LIMIT or k = ? constraint is required'). The only caller that passes a project is the de-duplication of new observations, which treats an error as 'insert', so semantic de-duplication never took effect.

Run one search for the project and one for global documents and merge them by distance without repeats, cut to the limit. QueryMultiField asks for enough field documents and keeps the best per observation; its two skipped tests are enabled. Because de-duplication now runs, canMergeInto skips a superseded or deleted note as a merge target.

Co-authored-by: Claude Sonnet 5.5 <[email protected]>
This commit is contained in:
Holger Reinhardt
2026-10-04 23:16:16 +01:00
committed by GitHub
co-authored by Claude Sonnet 5.5
parent 055103cb3b
commit 9b6e5f7648
4 changed files with 210 additions and 98 deletions
+15 -1
View File
@@ -51,7 +51,7 @@ func (p *Processor) checkVectorDeduplication(ctx context.Context, obs *models.Pa
for _, r := range results {
if r.Similarity >= cfg.DeduplicationThreshold {
obsID := extractObservationIDFromVectorDoc(r)
if obsID > 0 {
if obsID > 0 && p.canMergeInto(ctx, obsID) {
return &DeduplicationResult{
ExistingID: obsID,
Similarity: r.Similarity,
@@ -64,6 +64,20 @@ func (p *Processor) checkVectorDeduplication(ctx context.Context, obs *models.Pa
return &DeduplicationResult{Action: "insert"}
}
// canMergeInto says whether a new observation may be merged into the existing one. A note a person superseded
// is hidden from sessions, so what was merged into it would be hidden as well: such a note is not a target,
// and neither is one that is gone.
func (p *Processor) canMergeInto(ctx context.Context, id int64) bool {
if p.observationStore == nil {
return true
}
existing, err := p.observationStore.GetObservationByID(ctx, id)
if err != nil {
return false
}
return existing != nil && !existing.IsSuperseded
}
// buildObservationSearchText creates searchable text from a parsed observation.
func buildObservationSearchText(obs *models.ParsedObservation) string {
var parts []string
+33
View File
@@ -4,6 +4,7 @@ import (
"context"
"testing"
"github.com/lukaszraczylo/claude-mnemonic/internal/db/gorm"
"github.com/lukaszraczylo/claude-mnemonic/internal/vector/sqlitevec"
"github.com/lukaszraczylo/claude-mnemonic/pkg/models"
)
@@ -142,3 +143,35 @@ func TestCheckVectorDeduplication_EmptySearchText(t *testing.T) {
t.Errorf("expected Action='insert' for empty observation, got %q", result.Action)
}
}
func TestCanMergeInto(t *testing.T) {
ctx := context.Background()
dir := t.TempDir()
store, err := gorm.NewStore(gorm.Config{Path: dir + "/test.db", MaxConns: 1})
if err != nil {
t.Fatal(err)
}
defer store.Close()
obsStore := gorm.NewObservationStore(store, nil, nil, nil)
id, _, err := obsStore.StoreObservation(ctx, "s1", "proj", &models.ParsedObservation{Type: models.ObsTypeDiscovery, Title: "A note"}, 1, 1)
if err != nil {
t.Fatal(err)
}
p := &Processor{observationStore: obsStore}
if !p.canMergeInto(ctx, id) {
t.Error("a live observation is a merge target")
}
if p.canMergeInto(ctx, id+1000) {
t.Error("an observation that does not exist is not")
}
if err := store.DB.Exec(`UPDATE observations SET is_superseded = 1 WHERE id = ?`, id).Error; err != nil {
t.Fatal(err)
}
if p.canMergeInto(ctx, id) {
t.Error("a superseded observation is hidden from sessions, so nothing is merged into it")
}
if !(&Processor{}).canMergeInto(ctx, id) {
t.Error("without an observation store there is nothing to check against")
}
}