From 9e5bcc64a825b517025ae7b2277c7b4a84db9398 Mon Sep 17 00:00:00 2001 From: Lukasz Raczylo Date: Wed, 30 Sep 2026 01:36:14 +0100 Subject: [PATCH] fix(embedding): L2-normalise embeddings so vector similarity is valid; add search eval harness --- internal/embedding/service.go | 26 ++- scripts/eval-search/corpus.json | 276 ++++++++++++++++++++++++++++++++ scripts/eval-search/main.go | 157 ++++++++++++++++++ 3 files changed, 456 insertions(+), 3 deletions(-) create mode 100644 scripts/eval-search/corpus.json create mode 100644 scripts/eval-search/main.go diff --git a/internal/embedding/service.go b/internal/embedding/service.go index e1e5fd7..85b0238 100644 --- a/internal/embedding/service.go +++ b/internal/embedding/service.go @@ -9,6 +9,7 @@ import ( "encoding/hex" "fmt" "io" + "math" "os" "path/filepath" "strings" @@ -26,6 +27,9 @@ const EmbeddingDim = 384 // Model version constants const ( + // BGEStorageVersion tags stored vectors; bump it to force a rebuild when vectors change. + BGEStorageVersion = "bge-v1.5-l2" + // BGEModelVersion is the version string for bge-small-en-v1.5 BGEModelVersion = "bge-v1.5" // BGEModelName is the human-readable name for bge-small-en-v1.5 @@ -128,7 +132,7 @@ func (m *bgeModel) Name() string { // Version returns the short version string for storage. func (m *bgeModel) Version() string { - return BGEModelVersion + return BGEStorageVersion } // Dimensions returns the embedding vector size. @@ -512,12 +516,28 @@ func meanPooling(embeddings []float32, attentionMask []int64, batchSize, seqLen, } } - results[b] = result + results[b] = l2Normalize(result) } return results } +// l2Normalize scales v to unit length in place; sqlite-vec distances assume unit vectors. +func l2Normalize(v []float32) []float32 { + var sum float64 + for _, x := range v { + sum += float64(x) * float64(x) + } + if sum == 0 { + return v + } + inv := float32(1 / math.Sqrt(sum)) + for i := range v { + v[i] *= inv + } + return v +} + // clsPooling extracts the [CLS] token embedding (first token). // Input shape: [batch, seq_len, hidden] // Output shape: [batch, hidden] @@ -529,7 +549,7 @@ func clsPooling(embeddings []float32, batchSize, seqLen, hiddenSize int) [][]flo // CLS token is at position 0 embOffset := b * seqLen * hiddenSize copy(result, embeddings[embOffset:embOffset+hiddenSize]) - results[b] = result + results[b] = l2Normalize(result) } return results diff --git a/scripts/eval-search/corpus.json b/scripts/eval-search/corpus.json new file mode 100644 index 0000000..c1ed33d --- /dev/null +++ b/scripts/eval-search/corpus.json @@ -0,0 +1,276 @@ +{ + "docs": [ + { + "type": "bugfix", + "title": "Fixed race in session cache", + "narrative": "Concurrent map writes in the session cache caused panics; guarded with a RWMutex." + }, + { + "type": "decision", + "title": "Chose SQLite WAL mode", + "narrative": "WAL lets readers proceed while a writer commits, so hooks can query during writes." + }, + { + "type": "feature", + "title": "Added Prometheus metrics endpoint", + "narrative": "Exposes request latency histograms and error counters at /metrics for scraping." + }, + { + "type": "bugfix", + "title": "Fixed memory leak in websocket hub", + "narrative": "Goroutines were never released when clients disconnected; added context cancellation." + }, + { + "type": "refactor", + "title": "Extracted retry helper with backoff", + "narrative": "Exponential backoff with jitter now lives in one function used by all HTTP callers." + }, + { + "type": "decision", + "title": "Use argon2id for password hashing", + "narrative": "bcrypt cost was too low for new hardware; argon2id with 64MB memory chosen." + }, + { + "type": "feature", + "title": "Dark mode toggle in dashboard", + "narrative": "Theme preference persists in localStorage and follows prefers-color-scheme by default." + }, + { + "type": "discovery", + "title": "Kubernetes readiness probe timing", + "narrative": "Pods were killed during warmup because the readiness probe fired before the model loaded." + }, + { + "type": "bugfix", + "title": "Fixed off-by-one in pagination", + "narrative": "The last page skipped one row when total count was an exact multiple of page size." + }, + { + "type": "decision", + "title": "Adopted table-driven tests", + "narrative": "Subtests with named cases make failures easier to locate and new cases cheap to add." + }, + { + "type": "feature", + "title": "Rate limiting middleware", + "narrative": "Token bucket per client IP returns 429 with a Retry-After header when exceeded." + }, + { + "type": "bugfix", + "title": "Fixed timezone bug in scheduler", + "narrative": "Cron jobs fired an hour late after daylight saving because times were stored in local zone." + }, + { + "type": "discovery", + "title": "Slow query caused by missing index", + "narrative": "EXPLAIN showed a sequential scan on the events table; a composite index cut it from 900ms to 4ms." + }, + { + "type": "refactor", + "title": "Split monolithic handler file", + "narrative": "The 3000 line handlers file was divided by domain: sessions, observations, search." + }, + { + "type": "decision", + "title": "Embeddings computed locally with ONNX", + "narrative": "Avoids sending user text to third parties and works offline." + }, + { + "type": "feature", + "title": "Export observations as JSON", + "narrative": "New CLI command dumps all stored observations to a portable JSON file for backup." + }, + { + "type": "bugfix", + "title": "Fixed CORS preflight failure", + "narrative": "OPTIONS requests were rejected by auth middleware; moved CORS handling before authentication." + }, + { + "type": "discovery", + "title": "Docker image size dominated by build cache", + "narrative": "Multi-stage build with a distroless runtime reduced the image from 1.2GB to 80MB." + }, + { + "type": "decision", + "title": "Structured logging with zerolog", + "narrative": "JSON logs with fields are easier to filter in aggregation than free-form strings." + }, + { + "type": "feature", + "title": "Full text search with FTS5", + "narrative": "Keyword queries use SQLite FTS5 with porter stemming alongside vector similarity." + }, + { + "type": "bugfix", + "title": "Fixed panic on nil config", + "narrative": "Missing settings file returned a nil pointer; defaults are now loaded instead." + }, + { + "type": "discovery", + "title": "GC pauses from large slices", + "narrative": "Preallocating slice capacity removed repeated growth and lowered p99 latency." + }, + { + "type": "decision", + "title": "Graceful shutdown on SIGTERM", + "narrative": "Server drains in-flight requests for 30 seconds before exiting so deploys drop nothing." + }, + { + "type": "refactor", + "title": "Replaced global state with dependency injection", + "narrative": "Services receive their stores through constructors, which makes tests independent." + }, + { + "type": "feature", + "title": "OAuth login with GitHub", + "narrative": "Users authenticate via the GitHub authorization code flow and receive a session cookie." + }, + { + "type": "bugfix", + "title": "Fixed flaky integration test", + "narrative": "The test relied on sleep; replaced with polling on a readiness condition." + }, + { + "type": "discovery", + "title": "Git hooks slowed commits", + "narrative": "Pre-commit ran the full test suite; limited it to changed packages." + }, + { + "type": "decision", + "title": "Pinned dependency versions", + "narrative": "Lockfile committed and CI uses frozen installs to keep builds reproducible." + }, + { + "type": "feature", + "title": "Webhook retries with dead letter queue", + "narrative": "Failed deliveries retry five times and then land in a dead letter table for inspection." + }, + { + "type": "discovery", + "title": "Unicode normalization broke search", + "narrative": "Accented characters did not match until queries and content were NFC normalized." + } + ], + "queries": [ + { + "doc": 0, + "query": "why did concurrent writes to the cache crash the program" + }, + { + "doc": 1, + "query": "how can readers keep working while the database is being written" + }, + { + "doc": 2, + "query": "where do I scrape request latency numbers from" + }, + { + "doc": 3, + "query": "goroutines not freed after clients drop connection" + }, + { + "doc": 4, + "query": "shared function for repeating failed requests with growing delays" + }, + { + "doc": 5, + "query": "which algorithm protects stored passwords" + }, + { + "doc": 6, + "query": "can the UI switch to a dark theme" + }, + { + "doc": 7, + "query": "containers restarted while the model was still loading" + }, + { + "doc": 8, + "query": "last page of results missing an item" + }, + { + "doc": 9, + "query": "why do our tests use named cases" + }, + { + "doc": 10, + "query": "how are abusive clients throttled" + }, + { + "doc": 11, + "query": "scheduled jobs running late after clocks changed" + }, + { + "doc": 12, + "query": "query got fast after adding an index" + }, + { + "doc": 13, + "query": "large source file was broken up by domain" + }, + { + "doc": 14, + "query": "do we send text to an external embedding API" + }, + { + "doc": 15, + "query": "how to back up everything I stored" + }, + { + "doc": 16, + "query": "browser blocked cross-origin request before login check" + }, + { + "doc": 17, + "query": "how did we make the container image smaller" + }, + { + "doc": 18, + "query": "why are logs emitted as JSON" + }, + { + "doc": 19, + "query": "keyword matching with stemming" + }, + { + "doc": 20, + "query": "crash when settings file is absent" + }, + { + "doc": 21, + "query": "reduce latency spikes from memory allocation" + }, + { + "doc": 22, + "query": "what happens to requests during deployment shutdown" + }, + { + "doc": 23, + "query": "stop using package level variables for services" + }, + { + "doc": 24, + "query": "sign in with a third party identity provider" + }, + { + "doc": 25, + "query": "test that fails randomly because of timing" + }, + { + "doc": 26, + "query": "commit takes too long because of checks" + }, + { + "doc": 27, + "query": "make builds reproducible across machines" + }, + { + "doc": 28, + "query": "what happens when a callback delivery fails repeatedly" + }, + { + "doc": 29, + "query": "accented letters not found when searching" + } + ] +} \ No newline at end of file diff --git a/scripts/eval-search/main.go b/scripts/eval-search/main.go new file mode 100644 index 0000000..1902b3f --- /dev/null +++ b/scripts/eval-search/main.go @@ -0,0 +1,157 @@ +// Command eval-search measures search quality against a labelled corpus. +// +// eval-search seed writes the corpus into /claude-mnemonic.db +// eval-search run queries a running worker and prints MRR@10 and recall@5 +package main + +import ( + "context" + _ "embed" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "os" + "path/filepath" + "strconv" + "time" + + "github.com/lukaszraczylo/claude-mnemonic/internal/db/gorm" + "github.com/lukaszraczylo/claude-mnemonic/pkg/models" +) + +const ( + project = "evalproj" + sessionID = "eval-session" + dbFile = "claude-mnemonic.db" + recallAt = 5 + mrrAt = 10 +) + +//go:embed corpus.json +var corpusJSON []byte + +type corpus struct { + Docs []struct { + Type string `json:"type"` + Title string `json:"title"` + Narrative string `json:"narrative"` + } `json:"docs"` + Queries []struct { + Query string `json:"query"` + Doc int `json:"doc"` + } `json:"queries"` +} + +func main() { + if len(os.Args) != 3 { + fmt.Fprintln(os.Stderr, "usage: eval-search seed | run ") + os.Exit(2) + } + var c corpus + if err := json.Unmarshal(corpusJSON, &c); err != nil { + fatal(err) + } + var err error + switch os.Args[1] { + case "seed": + err = seed(c, os.Args[2]) + case "run": + err = run(c, os.Args[2]) + default: + err = fmt.Errorf("unknown mode %q", os.Args[1]) + } + if err != nil { + fatal(err) + } +} + +func fatal(err error) { + fmt.Fprintln(os.Stderr, "error:", err) + os.Exit(1) +} + +// seed stores docs in order, so doc i gets observation id i+1. +func seed(c corpus, dir string) error { + if err := os.MkdirAll(dir, 0o750); err != nil { + return err + } + store, err := gorm.NewStore(gorm.Config{Path: filepath.Join(dir, dbFile), MaxConns: 1}) + if err != nil { + return err + } + defer func() { _ = store.Close() }() + obsStore := gorm.NewObservationStore(store, nil, nil, nil) + for i, d := range c.Docs { + _, _, err := obsStore.StoreObservation(context.Background(), sessionID, project, &models.ParsedObservation{ + Type: models.ObservationType(d.Type), + Title: d.Title, + Narrative: d.Narrative, + Concepts: []string{"eval"}, + }, i+1, 0) + if err != nil { + return fmt.Errorf("store doc %d: %w", i, err) + } + } + return nil +} + +type obsRef struct { + ID int64 `json:"id"` +} + +func printRanks(query string, want int64, got []obsRef) { + pos := -1 + for i, o := range got { + if o.ID == want { + pos = i + 1 + break + } + } + fmt.Printf("rank=%d results=%d %q\n", pos, len(got), query) +} + +func run(c corpus, port string) error { + client := &http.Client{Timeout: 20 * time.Second} + var mrr, recall, results float64 + for _, q := range c.Queries { + u := fmt.Sprintf("http://127.0.0.1:%s/api/context/search?project=%s&query=%s", port, project, url.QueryEscape(q.Query)) + resp, err := client.Get(u) + if err != nil { + return err + } + body, err := io.ReadAll(resp.Body) + _ = resp.Body.Close() + if err != nil { + return err + } + var out struct { + Observations []obsRef `json:"observations"` + } + if err := json.Unmarshal(body, &out); err != nil { + return fmt.Errorf("decode %q: %w", q.Query, err) + } + results += float64(len(out.Observations)) + want := int64(q.Doc + 1) + if os.Getenv("EVAL_VERBOSE") != "" { + printRanks(q.Query, want, out.Observations) + } + for rank, o := range out.Observations { + if o.ID != want { + continue + } + if rank < mrrAt { + mrr += 1 / float64(rank+1) + } + if rank < recallAt { + recall++ + } + break + } + } + n := float64(len(c.Queries)) + fmt.Printf("queries=%d MRR@%d=%s recall@%d=%s avg_results=%s\n", len(c.Queries), mrrAt, strconv.FormatFloat(mrr/n, 'f', 4, 64), + recallAt, strconv.FormatFloat(recall/n, 'f', 4, 64), strconv.FormatFloat(results/n, 'f', 2, 64)) + return nil +}