fix(embedding): L2-normalise embeddings so vector similarity is valid; add search eval harness

2026-10-11 04:47:22 +00:00 · 2026-09-30 01:36:14 +01:00
parent 0bbba6c078
commit 9e5bcc64a8
3 changed files with 456 additions and 3 deletions
@@ -9,6 +9,7 @@ import (
 	"encoding/hex"
 	"fmt"
 	"io"
+	"math"
 	"os"
 	"path/filepath"
 	"strings"
@@ -26,6 +27,9 @@ const EmbeddingDim = 384

 // Model version constants
 const (
+	// BGEStorageVersion tags stored vectors; bump it to force a rebuild when vectors change.
+	BGEStorageVersion = "bge-v1.5-l2"
+
 	// BGEModelVersion is the version string for bge-small-en-v1.5
 	BGEModelVersion = "bge-v1.5"
 	// BGEModelName is the human-readable name for bge-small-en-v1.5
@@ -128,7 +132,7 @@ func (m *bgeModel) Name() string {

 // Version returns the short version string for storage.
 func (m *bgeModel) Version() string {
-	return BGEModelVersion
+	return BGEStorageVersion
 }

 // Dimensions returns the embedding vector size.
@@ -512,12 +516,28 @@ func meanPooling(embeddings []float32, attentionMask []int64, batchSize, seqLen,
 			}
 		}

-		results[b] = result
+		results[b] = l2Normalize(result)
 	}

 	return results
 }

+// l2Normalize scales v to unit length in place; sqlite-vec distances assume unit vectors.
+func l2Normalize(v []float32) []float32 {
+	var sum float64
+	for _, x := range v {
+		sum += float64(x) * float64(x)
+	}
+	if sum == 0 {
+		return v
+	}
+	inv := float32(1 / math.Sqrt(sum))
+	for i := range v {
+		v[i] *= inv
+	}
+	return v
+}
+
 // clsPooling extracts the [CLS] token embedding (first token).
 // Input shape: [batch, seq_len, hidden]
 // Output shape: [batch, hidden]
@@ -529,7 +549,7 @@ func clsPooling(embeddings []float32, batchSize, seqLen, hiddenSize int) [][]flo
 		// CLS token is at position 0
 		embOffset := b * seqLen * hiddenSize
 		copy(result, embeddings[embOffset:embOffset+hiddenSize])
-		results[b] = result
+		results[b] = l2Normalize(result)
 	}

 	return results
@@ -0,0 +1,276 @@
+{
+ "docs": [
+  {
+   "type": "bugfix",
+   "title": "Fixed race in session cache",
+   "narrative": "Concurrent map writes in the session cache caused panics; guarded with a RWMutex."
+  },
+  {
+   "type": "decision",
+   "title": "Chose SQLite WAL mode",
+   "narrative": "WAL lets readers proceed while a writer commits, so hooks can query during writes."
+  },
+  {
+   "type": "feature",
+   "title": "Added Prometheus metrics endpoint",
+   "narrative": "Exposes request latency histograms and error counters at /metrics for scraping."
+  },
+  {
+   "type": "bugfix",
+   "title": "Fixed memory leak in websocket hub",
+   "narrative": "Goroutines were never released when clients disconnected; added context cancellation."
+  },
+  {
+   "type": "refactor",
+   "title": "Extracted retry helper with backoff",
+   "narrative": "Exponential backoff with jitter now lives in one function used by all HTTP callers."
+  },
+  {
+   "type": "decision",
+   "title": "Use argon2id for password hashing",
+   "narrative": "bcrypt cost was too low for new hardware; argon2id with 64MB memory chosen."
+  },
+  {
+   "type": "feature",
+   "title": "Dark mode toggle in dashboard",
+   "narrative": "Theme preference persists in localStorage and follows prefers-color-scheme by default."
+  },
+  {
+   "type": "discovery",
+   "title": "Kubernetes readiness probe timing",
+   "narrative": "Pods were killed during warmup because the readiness probe fired before the model loaded."
+  },
+  {
+   "type": "bugfix",
+   "title": "Fixed off-by-one in pagination",
+   "narrative": "The last page skipped one row when total count was an exact multiple of page size."
+  },
+  {
+   "type": "decision",
+   "title": "Adopted table-driven tests",
+   "narrative": "Subtests with named cases make failures easier to locate and new cases cheap to add."
+  },
+  {
+   "type": "feature",
+   "title": "Rate limiting middleware",
+   "narrative": "Token bucket per client IP returns 429 with a Retry-After header when exceeded."
+  },
+  {
+   "type": "bugfix",
+   "title": "Fixed timezone bug in scheduler",
+   "narrative": "Cron jobs fired an hour late after daylight saving because times were stored in local zone."
+  },
+  {
+   "type": "discovery",
+   "title": "Slow query caused by missing index",
+   "narrative": "EXPLAIN showed a sequential scan on the events table; a composite index cut it from 900ms to 4ms."
+  },
+  {
+   "type": "refactor",
+   "title": "Split monolithic handler file",
+   "narrative": "The 3000 line handlers file was divided by domain: sessions, observations, search."
+  },
+  {
+   "type": "decision",
+   "title": "Embeddings computed locally with ONNX",
+   "narrative": "Avoids sending user text to third parties and works offline."
+  },
+  {
+   "type": "feature",
+   "title": "Export observations as JSON",
+   "narrative": "New CLI command dumps all stored observations to a portable JSON file for backup."
+  },
+  {
+   "type": "bugfix",
+   "title": "Fixed CORS preflight failure",
+   "narrative": "OPTIONS requests were rejected by auth middleware; moved CORS handling before authentication."
+  },
+  {
+   "type": "discovery",
+   "title": "Docker image size dominated by build cache",
+   "narrative": "Multi-stage build with a distroless runtime reduced the image from 1.2GB to 80MB."
+  },
+  {
+   "type": "decision",
+   "title": "Structured logging with zerolog",
+   "narrative": "JSON logs with fields are easier to filter in aggregation than free-form strings."
+  },
+  {
+   "type": "feature",
+   "title": "Full text search with FTS5",
+   "narrative": "Keyword queries use SQLite FTS5 with porter stemming alongside vector similarity."
+  },
+  {
+   "type": "bugfix",
+   "title": "Fixed panic on nil config",
+   "narrative": "Missing settings file returned a nil pointer; defaults are now loaded instead."
+  },
+  {
+   "type": "discovery",
+   "title": "GC pauses from large slices",
+   "narrative": "Preallocating slice capacity removed repeated growth and lowered p99 latency."
+  },
+  {
+   "type": "decision",
+   "title": "Graceful shutdown on SIGTERM",
+   "narrative": "Server drains in-flight requests for 30 seconds before exiting so deploys drop nothing."
+  },
+  {
+   "type": "refactor",
+   "title": "Replaced global state with dependency injection",
+   "narrative": "Services receive their stores through constructors, which makes tests independent."
+  },
+  {
+   "type": "feature",
+   "title": "OAuth login with GitHub",
+   "narrative": "Users authenticate via the GitHub authorization code flow and receive a session cookie."
+  },
+  {
+   "type": "bugfix",
+   "title": "Fixed flaky integration test",
+   "narrative": "The test relied on sleep; replaced with polling on a readiness condition."
+  },
+  {
+   "type": "discovery",
+   "title": "Git hooks slowed commits",
+   "narrative": "Pre-commit ran the full test suite; limited it to changed packages."
+  },
+  {
+   "type": "decision",
+   "title": "Pinned dependency versions",
+   "narrative": "Lockfile committed and CI uses frozen installs to keep builds reproducible."
+  },
+  {
+   "type": "feature",
+   "title": "Webhook retries with dead letter queue",
+   "narrative": "Failed deliveries retry five times and then land in a dead letter table for inspection."
+  },
+  {
+   "type": "discovery",
+   "title": "Unicode normalization broke search",
+   "narrative": "Accented characters did not match until queries and content were NFC normalized."
+  }
+ ],
+ "queries": [
+  {
+   "doc": 0,
+   "query": "why did concurrent writes to the cache crash the program"
+  },
+  {
+   "doc": 1,
+   "query": "how can readers keep working while the database is being written"
+  },
+  {
+   "doc": 2,
+   "query": "where do I scrape request latency numbers from"
+  },
+  {
+   "doc": 3,
+   "query": "goroutines not freed after clients drop connection"
+  },
+  {
+   "doc": 4,
+   "query": "shared function for repeating failed requests with growing delays"
+  },
+  {
+   "doc": 5,
+   "query": "which algorithm protects stored passwords"
+  },
+  {
+   "doc": 6,
+   "query": "can the UI switch to a dark theme"
+  },
+  {
+   "doc": 7,
+   "query": "containers restarted while the model was still loading"
+  },
+  {
+   "doc": 8,
+   "query": "last page of results missing an item"
+  },
+  {
+   "doc": 9,
+   "query": "why do our tests use named cases"
+  },
+  {
+   "doc": 10,
+   "query": "how are abusive clients throttled"
+  },
+  {
+   "doc": 11,
+   "query": "scheduled jobs running late after clocks changed"
+  },
+  {
+   "doc": 12,
+   "query": "query got fast after adding an index"
+  },
+  {
+   "doc": 13,
+   "query": "large source file was broken up by domain"
+  },
+  {
+   "doc": 14,
+   "query": "do we send text to an external embedding API"
+  },
+  {
+   "doc": 15,
+   "query": "how to back up everything I stored"
+  },
+  {
+   "doc": 16,
+   "query": "browser blocked cross-origin request before login check"
+  },
+  {
+   "doc": 17,
+   "query": "how did we make the container image smaller"
+  },
+  {
+   "doc": 18,
+   "query": "why are logs emitted as JSON"
+  },
+  {
+   "doc": 19,
+   "query": "keyword matching with stemming"
+  },
+  {
+   "doc": 20,
+   "query": "crash when settings file is absent"
+  },
+  {
+   "doc": 21,
+   "query": "reduce latency spikes from memory allocation"
+  },
+  {
+   "doc": 22,
+   "query": "what happens to requests during deployment shutdown"
+  },
+  {
+   "doc": 23,
+   "query": "stop using package level variables for services"
+  },
+  {
+   "doc": 24,
+   "query": "sign in with a third party identity provider"
+  },
+  {
+   "doc": 25,
+   "query": "test that fails randomly because of timing"
+  },
+  {
+   "doc": 26,
+   "query": "commit takes too long because of checks"
+  },
+  {
+   "doc": 27,
+   "query": "make builds reproducible across machines"
+  },
+  {
+   "doc": 28,
+   "query": "what happens when a callback delivery fails repeatedly"
+  },
+  {
+   "doc": 29,
+   "query": "accented letters not found when searching"
+  }
+ ]
+}
@@ -0,0 +1,157 @@
+// Command eval-search measures search quality against a labelled corpus.
+//
+//	eval-search seed <dbdir>        writes the corpus into <dbdir>/claude-mnemonic.db
+//	eval-search run <port>          queries a running worker and prints MRR@10 and recall@5
+package main
+
+import (
+	"context"
+	_ "embed"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"os"
+	"path/filepath"
+	"strconv"
+	"time"
+
+	"github.com/lukaszraczylo/claude-mnemonic/internal/db/gorm"
+	"github.com/lukaszraczylo/claude-mnemonic/pkg/models"
+)
+
+const (
+	project   = "evalproj"
+	sessionID = "eval-session"
+	dbFile    = "claude-mnemonic.db"
+	recallAt  = 5
+	mrrAt     = 10
+)
+
+//go:embed corpus.json
+var corpusJSON []byte
+
+type corpus struct {
+	Docs []struct {
+		Type      string `json:"type"`
+		Title     string `json:"title"`
+		Narrative string `json:"narrative"`
+	} `json:"docs"`
+	Queries []struct {
+		Query string `json:"query"`
+		Doc   int    `json:"doc"`
+	} `json:"queries"`
+}
+
+func main() {
+	if len(os.Args) != 3 {
+		fmt.Fprintln(os.Stderr, "usage: eval-search seed <dbdir> | run <port>")
+		os.Exit(2)
+	}
+	var c corpus
+	if err := json.Unmarshal(corpusJSON, &c); err != nil {
+		fatal(err)
+	}
+	var err error
+	switch os.Args[1] {
+	case "seed":
+		err = seed(c, os.Args[2])
+	case "run":
+		err = run(c, os.Args[2])
+	default:
+		err = fmt.Errorf("unknown mode %q", os.Args[1])
+	}
+	if err != nil {
+		fatal(err)
+	}
+}
+
+func fatal(err error) {
+	fmt.Fprintln(os.Stderr, "error:", err)
+	os.Exit(1)
+}
+
+// seed stores docs in order, so doc i gets observation id i+1.
+func seed(c corpus, dir string) error {
+	if err := os.MkdirAll(dir, 0o750); err != nil {
+		return err
+	}
+	store, err := gorm.NewStore(gorm.Config{Path: filepath.Join(dir, dbFile), MaxConns: 1})
+	if err != nil {
+		return err
+	}
+	defer func() { _ = store.Close() }()
+	obsStore := gorm.NewObservationStore(store, nil, nil, nil)
+	for i, d := range c.Docs {
+		_, _, err := obsStore.StoreObservation(context.Background(), sessionID, project, &models.ParsedObservation{
+			Type:      models.ObservationType(d.Type),
+			Title:     d.Title,
+			Narrative: d.Narrative,
+			Concepts:  []string{"eval"},
+		}, i+1, 0)
+		if err != nil {
+			return fmt.Errorf("store doc %d: %w", i, err)
+		}
+	}
+	return nil
+}
+
+type obsRef struct {
+	ID int64 `json:"id"`
+}
+
+func printRanks(query string, want int64, got []obsRef) {
+	pos := -1
+	for i, o := range got {
+		if o.ID == want {
+			pos = i + 1
+			break
+		}
+	}
+	fmt.Printf("rank=%d results=%d %q\n", pos, len(got), query)
+}
+
+func run(c corpus, port string) error {
+	client := &http.Client{Timeout: 20 * time.Second}
+	var mrr, recall, results float64
+	for _, q := range c.Queries {
+		u := fmt.Sprintf("http://127.0.0.1:%s/api/context/search?project=%s&query=%s", port, project, url.QueryEscape(q.Query))
+		resp, err := client.Get(u)
+		if err != nil {
+			return err
+		}
+		body, err := io.ReadAll(resp.Body)
+		_ = resp.Body.Close()
+		if err != nil {
+			return err
+		}
+		var out struct {
+			Observations []obsRef `json:"observations"`
+		}
+		if err := json.Unmarshal(body, &out); err != nil {
+			return fmt.Errorf("decode %q: %w", q.Query, err)
+		}
+		results += float64(len(out.Observations))
+		want := int64(q.Doc + 1)
+		if os.Getenv("EVAL_VERBOSE") != "" {
+			printRanks(q.Query, want, out.Observations)
+		}
+		for rank, o := range out.Observations {
+			if o.ID != want {
+				continue
+			}
+			if rank < mrrAt {
+				mrr += 1 / float64(rank+1)
+			}
+			if rank < recallAt {
+				recall++
+			}
+			break
+		}
+	}
+	n := float64(len(c.Queries))
+	fmt.Printf("queries=%d MRR@%d=%s recall@%d=%s avg_results=%s\n", len(c.Queries), mrrAt, strconv.FormatFloat(mrr/n, 'f', 4, 64),
+		recallAt, strconv.FormatFloat(recall/n, 'f', 4, 64), strconv.FormatFloat(results/n, 'f', 2, 64))
+	return nil
+}