mirror of
https://github.com/lukaszraczylo/claude-mnemonic.git
synced 2026-10-12 04:55:18 +00:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
f19c247278 | ||
|
|
8cddc88ab6 | ||
|
|
c9c0d988bc | ||
|
|
3945c187be | ||
|
|
06b8b47584 | ||
|
|
8dc524ea59 | ||
|
|
9dc59d13fe | ||
|
|
d8c49d8d89 | ||
|
|
6e26c4e85d | ||
|
|
8c2c69582a | ||
|
|
fbe938fef2 | ||
|
|
64c2ace22e | ||
|
|
f2886bbedb | ||
|
|
9b6e5f7648 | ||
|
|
055103cb3b | ||
|
|
67de0f0c84 | ||
|
|
eba3136fff | ||
|
|
17f7bffef5 | ||
|
|
0d75b867e6 | ||
|
|
a8e9d0f256 | ||
|
|
8857cc9c72 | ||
|
|
096490a2d3 | ||
|
|
eaf7c4ddca | ||
|
|
2d1f52cdf4 | ||
|
|
9d7959f98f | ||
|
|
daa9959961 | ||
|
|
f8e5b08de7 | ||
|
|
ad718f9292 | ||
|
|
1cfe6936de | ||
|
|
2b2bc47fe5 | ||
|
|
88d316b3ec | ||
|
|
4cef2eb792 | ||
|
|
c3cad756f9 | ||
|
|
c3fdfed512 | ||
|
|
7cb0dfbc13 | ||
|
|
9e5bcc64a8 | ||
|
|
0bbba6c078 | ||
|
|
da1d96dab4 | ||
|
|
443f557000 | ||
|
|
affdd4fab0 | ||
|
|
41b774f973 | ||
|
|
ee285441e4 | ||
|
|
2658694839 | ||
|
|
e71d34912e | ||
|
|
9c832f9ac0 | ||
|
|
7d55a6a493 | ||
|
|
8995781628 | ||
|
|
10ab2a6d41 | ||
|
|
052fef0af8 | ||
|
|
3384bc1d5f | ||
|
|
3caf83b7ea | ||
|
|
93b19d5402 | ||
|
|
6a819918f6 | ||
|
|
98ab1cd3e9 | ||
|
|
55ba09e0c6 | ||
|
|
a369a421aa | ||
|
|
76f31e97ae | ||
|
|
132666a190 | ||
|
|
3a8e3667b3 | ||
|
|
3ea3904875 | ||
|
|
8a8814968e | ||
|
|
6b8ecdd2ec | ||
|
|
aae542565e |
@@ -14,7 +14,6 @@
|
||||
"mcpServers": {
|
||||
"claude-mnemonic": {
|
||||
"command": "${CLAUDE_PLUGIN_ROOT}/mcp-server",
|
||||
"args": ["--project", "${CLAUDE_PROJECT}"],
|
||||
"env": {}
|
||||
}
|
||||
},
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
name: Autoupdate go.mod and go.sum
|
||||
name: Update dependencies
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
@@ -6,16 +6,17 @@ on:
|
||||
- cron: "0 3 * * *"
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
actions: write
|
||||
contents: write
|
||||
issues: write
|
||||
pull-requests: write
|
||||
security-events: write
|
||||
|
||||
jobs:
|
||||
autoupdate:
|
||||
uses: lukaszraczylo/shared-actions/.github/workflows/go-autoupdate.yaml@main
|
||||
update:
|
||||
uses: lukaszraczylo/shared-actions/.github/workflows/renovate-autoupdate.yaml@main
|
||||
with:
|
||||
go-version: ">=1.24"
|
||||
release-workflow: "release.yaml"
|
||||
lfs: false
|
||||
secrets: inherit
|
||||
build-tags: "fts5"
|
||||
node-working-directory: ui
|
||||
require-node-tests: false
|
||||
release-workflow: release.yaml
|
||||
|
||||
@@ -95,3 +95,6 @@ docs/dist
|
||||
plugin/.claude-plugin/plugin.json
|
||||
plugin/.claude-plugin/marketplace.json
|
||||
user-prompt
|
||||
|
||||
# Compressed ONNX runtime libs are produced by scripts/download-onnx-libs.sh
|
||||
internal/embedding/assets/lib/**/*.gz
|
||||
|
||||
@@ -3,9 +3,9 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"time"
|
||||
@@ -13,6 +13,9 @@ import (
|
||||
"github.com/lukaszraczylo/claude-mnemonic/pkg/hooks"
|
||||
)
|
||||
|
||||
// statsTimeout bounds the worker stats request.
|
||||
const statsTimeout = 100 * time.Millisecond
|
||||
|
||||
// StatusInput is the JSON input from Claude Code's statusline feature.
|
||||
type StatusInput struct {
|
||||
HookEventName string `json:"hook_event_name"`
|
||||
@@ -105,26 +108,19 @@ func handleStatusline(input *StatusInput, port int) string {
|
||||
// getWorkerStats fetches stats from the worker service.
|
||||
func getWorkerStats(port int, project string) *WorkerStats {
|
||||
// Build URL with optional project parameter
|
||||
endpoint := fmt.Sprintf("http://127.0.0.1:%d/api/stats", port)
|
||||
endpoint := "/api/stats"
|
||||
if project != "" {
|
||||
endpoint += "?project=" + url.QueryEscape(project)
|
||||
}
|
||||
|
||||
// Create HTTP client with short timeout (statusline must be fast)
|
||||
client := &http.Client{Timeout: 100 * time.Millisecond}
|
||||
|
||||
resp, err := client.Get(endpoint)
|
||||
// Short timeout: statusline must be fast
|
||||
body, err := hooks.GETBody(context.Background(), statsTimeout, port, endpoint)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return nil
|
||||
}
|
||||
|
||||
var stats WorkerStats
|
||||
if err := json.NewDecoder(resp.Body).Decode(&stats); err != nil {
|
||||
if err := json.Unmarshal(body, &stats); err != nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
+14
-3
@@ -8,12 +8,14 @@ import (
|
||||
"net/http"
|
||||
"os"
|
||||
"os/signal"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/lukaszraczylo/claude-mnemonic/internal/config"
|
||||
"github.com/lukaszraczylo/claude-mnemonic/internal/mcp"
|
||||
"github.com/lukaszraczylo/claude-mnemonic/internal/watcher"
|
||||
"github.com/lukaszraczylo/claude-mnemonic/pkg/hooks"
|
||||
"github.com/lukaszraczylo/oss-telemetry"
|
||||
"github.com/rs/zerolog"
|
||||
"github.com/rs/zerolog/log"
|
||||
@@ -22,9 +24,18 @@ import (
|
||||
// Version is set at build time via ldflags.
|
||||
var Version = "dev"
|
||||
|
||||
// defaultProject derives the project ID the same way the hooks do.
|
||||
func defaultProject() string {
|
||||
dir := os.Getenv("CLAUDE_PROJECT_DIR")
|
||||
if dir == "" {
|
||||
dir, _ = os.Getwd()
|
||||
}
|
||||
return hooks.ProjectIDWithName(dir)
|
||||
}
|
||||
|
||||
func main() {
|
||||
// Parse flags
|
||||
project := flag.String("project", "", "Project name (required)")
|
||||
project := flag.String("project", "", "Project ID (default: derived from CLAUDE_PROJECT_DIR or cwd)")
|
||||
debug := flag.Bool("debug", false, "Enable debug logging")
|
||||
flag.Parse()
|
||||
|
||||
@@ -35,8 +46,8 @@ func main() {
|
||||
}
|
||||
log.Logger = log.Output(zerolog.ConsoleWriter{Out: os.Stderr, NoColor: true})
|
||||
|
||||
if *project == "" {
|
||||
log.Fatal().Msg("--project is required")
|
||||
if *project == "" || strings.Contains(*project, "${") {
|
||||
*project = defaultProject()
|
||||
}
|
||||
|
||||
// Get worker port from config
|
||||
|
||||
@@ -1,3 +1,8 @@
|
||||
---
|
||||
description: Restart the Claude Mnemonic worker process
|
||||
allowed-tools: Bash(curl:*)
|
||||
---
|
||||
|
||||
# Restart Claude Mnemonic Worker
|
||||
|
||||
Restart the claude-mnemonic worker process. Use this command when experiencing issues with the memory system.
|
||||
|
||||
@@ -0,0 +1,60 @@
|
||||
# Experiment journal
|
||||
|
||||
## Campaign B: binary size
|
||||
|
||||
Metric: sum of bytes of worker, mcp and 6 hook binaries (release flags, `-s -w -trimpath`, `-tags fts5`). Lower is better.
|
||||
Threshold: at least 3% drop, tests green. Stop: 5 consecutive no-improve trials or 15 trials.
|
||||
Mutate: Go code and build flags. Frozen: public API, DB schema, plugin manifest.
|
||||
|
||||
| id | mutation | total bytes | delta vs base | kept? |
|
||||
|----|----------|-------------|---------------|-------|
|
||||
| 01 | baseline | 102698416 | - | base |
|
||||
| 02 | pkg/hooks: raw HTTP/1.0 client replaces net/http | 87603952 | -14.7% | KEEP |
|
||||
| 03 | statusline: shared client, drop net/http | 84531056 | -3.5% vs 02 | KEEP |
|
||||
| 04 | embed gzipped ONNX runtime libs, gunzip at first use | 57216880 | -32.3% vs 03 | KEEP |
|
||||
| 05 | hooks built with CGO_ENABLED=0 | 57216880 | 0% | revert (not applied) |
|
||||
| - | statically dedupe or compress tokenizer.json | - | projected ~1.2-1.6% | not run, under 3% threshold |
|
||||
| - | drop net/http from mcp | - | none possible | not run, oss-telemetry links net/http |
|
||||
|
||||
Campaign B stopped on diminishing returns: 102698416 -> 57216880 bytes (-44.3%).
|
||||
|
||||
## Campaign A: hook latency (stub worker, 200 runs, p50)
|
||||
|
||||
Harness: hook binary against a local stub worker (100-observation context), same machine, alternating runs. Lower is better.
|
||||
|
||||
| hook | before (ee28544) | after trial 02 | delta |
|
||||
|------|------------------|----------------|-------|
|
||||
| session-start | 4.6-5.6 ms | 3.3 ms | -30% or better |
|
||||
| user-prompt | 4.3-4.7 ms | 3.5-3.7 ms | -15% to -22% |
|
||||
|
||||
Hooks now sit near the process-start floor. No further trials run.
|
||||
|
||||
## Campaign D: reliability
|
||||
|
||||
Metric: failing cases in the extraction fault test (`TestWriteGunzipped`). Lower is better.
|
||||
|
||||
| id | mutation | failing cases | kept? |
|
||||
|----|----------|---------------|-------|
|
||||
| 01 | baseline (truncated archive leaves partial lib.so) | 1 | base |
|
||||
| 02 | extract via temp file and rename | 0 | KEEP |
|
||||
|
||||
## Campaign C: search accuracy
|
||||
|
||||
Harness: `scripts/eval-search` seeds 30 labelled observations, starts a real worker (fresh HOME, model cache copied), waits for the vector rebuild, then runs 30 paraphrased queries against `/api/context/search`. Metric: MRR@10 (higher is better). Threshold: at least +0.02. Runs are deterministic (repeat runs identical).
|
||||
|
||||
| id | mutation | MRR@10 | recall@5 | kept? |
|
||||
|----|----------|--------|----------|-------|
|
||||
| 01 | baseline | 0.7944 | 0.8667 | base |
|
||||
| 02 | L2-normalise embeddings (similarity was negative for every vector hit, so search fell back to FTS) | 0.8944 | 0.9667 | KEEP |
|
||||
| 03 | reranking alpha 0.7 -> 0.5 | 0.9111 | 0.9667 | revert (+0.017 < 0.02) |
|
||||
| 04 | reranking alpha 0.7 -> 0.9 | 0.9111 | 0.9667 | revert (+0.017 < 0.02) |
|
||||
| 05 | reranker off | 0.8889 | 0.9667 | revert |
|
||||
| 06 | reranker pure mode | 0.8714 | 0.9667 | revert |
|
||||
| 07 | CLS pooling instead of mean | 0.8944 | 0.9667 | revert |
|
||||
|
||||
Trial 02 changes stored vectors. `BGEStorageVersion` is bumped to `bge-v1.5-l2`, so existing installs rebuild vectors on next start.
|
||||
Caveat: 30 queries is a small set, so borderline gains such as alpha 0.5 are not distinguishable from noise.
|
||||
|
||||
Notes:
|
||||
- Trial 04 needs `scripts/download-onnx-libs.sh` to produce `.gz` files; existing raw libs trigger a re-download.
|
||||
- Extracted library verified byte-identical to the raw dylib (`cmp`).
|
||||
@@ -1,40 +1,38 @@
|
||||
module github.com/lukaszraczylo/claude-mnemonic
|
||||
|
||||
go 1.25.1
|
||||
go 1.26.0
|
||||
|
||||
replace github.com/sugarme/tokenizer => github.com/clems4ever/tokenizer v0.0.0-20250926133620-9ddc80533c43
|
||||
|
||||
require (
|
||||
github.com/asg017/sqlite-vec-go-bindings v0.1.6
|
||||
github.com/fsnotify/fsnotify v1.10.1
|
||||
github.com/go-chi/chi/v5 v5.3.0
|
||||
github.com/go-gormigrate/gormigrate/v2 v2.1.6
|
||||
github.com/goccy/go-json v0.10.6
|
||||
github.com/lukaszraczylo/oss-telemetry v0.2.3
|
||||
github.com/mattn/go-sqlite3 v1.14.46
|
||||
github.com/go-chi/chi/v5 v5.3.2
|
||||
github.com/go-gormigrate/gormigrate/v2 v2.1.7
|
||||
github.com/goccy/go-json v0.11.2
|
||||
github.com/lukaszraczylo/oss-telemetry v0.2.5
|
||||
github.com/mattn/go-sqlite3 v1.14.52
|
||||
github.com/rs/zerolog v1.35.1
|
||||
github.com/stretchr/testify v1.11.1
|
||||
github.com/stretchr/testify v1.12.1
|
||||
github.com/sugarme/tokenizer v0.3.0
|
||||
github.com/yalue/onnxruntime_go v1.31.0
|
||||
golang.org/x/sync v0.21.0
|
||||
github.com/yalue/onnxruntime_go v1.36.0
|
||||
golang.org/x/sync v0.24.0
|
||||
gorm.io/driver/sqlite v1.6.0
|
||||
gorm.io/gorm v1.31.1
|
||||
gorm.io/gorm v1.31.2
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/davecgh/go-spew v1.1.1 // indirect
|
||||
github.com/emirpasic/gods v1.18.1 // indirect
|
||||
github.com/jinzhu/inflection v1.0.0 // indirect
|
||||
github.com/jinzhu/now v1.1.5 // indirect
|
||||
github.com/mattn/go-colorable v0.1.15 // indirect
|
||||
github.com/mattn/go-isatty v0.0.22 // indirect
|
||||
github.com/mattn/go-colorable v0.1.16 // indirect
|
||||
github.com/mattn/go-isatty v0.0.24 // indirect
|
||||
github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect
|
||||
github.com/patrickmn/go-cache v2.1.0+incompatible // indirect
|
||||
github.com/pmezard/go-difflib v1.0.0 // indirect
|
||||
github.com/rivo/uniseg v0.4.7 // indirect
|
||||
github.com/schollz/progressbar/v2 v2.15.0 // indirect
|
||||
github.com/sugarme/regexpset v0.0.0-20200920021344-4d4ec8eaf93c // indirect
|
||||
golang.org/x/sys v0.46.0 // indirect
|
||||
golang.org/x/text v0.38.0 // indirect
|
||||
gopkg.in/yaml.v3 v3.0.1 // indirect
|
||||
go.yaml.in/yaml/v3 v3.0.5 // indirect
|
||||
golang.org/x/sys v0.49.0 // indirect
|
||||
golang.org/x/text v0.43.0 // indirect
|
||||
)
|
||||
|
||||
@@ -3,35 +3,33 @@ github.com/asg017/sqlite-vec-go-bindings v0.1.6/go.mod h1:A8+cTt/nKFsYCQF6OgzSNp
|
||||
github.com/clems4ever/tokenizer v0.0.0-20250926133620-9ddc80533c43 h1:j8YQypEqa5OjqbGciCNb9hOcYbo1oTVuEjd/iu9U2SY=
|
||||
github.com/clems4ever/tokenizer v0.0.0-20250926133620-9ddc80533c43/go.mod h1:VJ+DLK5ZEZwzvODOWwY0cw+B1dabTd3nCB5HuFCItCc=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc=
|
||||
github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ=
|
||||
github.com/fsnotify/fsnotify v1.10.1 h1:b0/UzAf9yR5rhf3RPm9gf3ehBPpf0oZKIjtpKrx59Ho=
|
||||
github.com/fsnotify/fsnotify v1.10.1/go.mod h1:TLheqan6HD6GBK6PrDWyDPBaEV8LspOxvPSjC+bVfgo=
|
||||
github.com/go-chi/chi/v5 v5.3.0 h1:halUjDxhshgXHMrao5bB8eNBXo/rnzwr8m5m36glehM=
|
||||
github.com/go-chi/chi/v5 v5.3.0/go.mod h1:R+tYY2hNuVUUjxoPtqUdgBqevM9s9njzkTLutVsOCto=
|
||||
github.com/go-gormigrate/gormigrate/v2 v2.1.6 h1:VtX+l1Stj2v5RGubVQk0LS/8EPGXR+ldcOyCmlmKoyg=
|
||||
github.com/go-gormigrate/gormigrate/v2 v2.1.6/go.mod h1:PZpedQc4tWaxn6kvXicwhinh3L0seLpMc5ReKRX5id4=
|
||||
github.com/goccy/go-json v0.10.6 h1:p8HrPJzOakx/mn/bQtjgNjdTcN+/S6FcG2CTtQOrHVU=
|
||||
github.com/goccy/go-json v0.10.6/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M=
|
||||
github.com/go-chi/chi/v5 v5.3.2 h1:5YQkICvTCSZ25hoRsyJazN0scjzKGiu4VAUc7H1o1nY=
|
||||
github.com/go-chi/chi/v5 v5.3.2/go.mod h1:R+tYY2hNuVUUjxoPtqUdgBqevM9s9njzkTLutVsOCto=
|
||||
github.com/go-gormigrate/gormigrate/v2 v2.1.7 h1:PdT4jVPbRb4R+0Ey2R0yJOdctVf4Whiq1Qi4necaZdg=
|
||||
github.com/go-gormigrate/gormigrate/v2 v2.1.7/go.mod h1:3ouXglTuPrKF5+7cQyVGfvAXTU4vLMaYh9+EPl03uog=
|
||||
github.com/goccy/go-json v0.11.2 h1:jdZv93Tt4ioR8yW1CoNsvSxrcZlCXAUU1aZXN7gpXUA=
|
||||
github.com/goccy/go-json v0.11.2/go.mod h1:3NdmfEkZlB7YI5UFw/qdFKq8XN1aiWR0YyRPWZNQltY=
|
||||
github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E=
|
||||
github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
|
||||
github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ=
|
||||
github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
|
||||
github.com/lukaszraczylo/oss-telemetry v0.2.3 h1:xoDtBqeZGmXj7IteiE1M5WMuzeoqag58qEleI0Cf2Ms=
|
||||
github.com/lukaszraczylo/oss-telemetry v0.2.3/go.mod h1:+Cn78qZo8rc3T9eZt0v3oICYRdd75wORtSidc8lNjDQ=
|
||||
github.com/mattn/go-colorable v0.1.15 h1:+u9SLTRGnXv73cEsnsmoZBom+dMU88B2M0aDcWy0/jY=
|
||||
github.com/mattn/go-colorable v0.1.15/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8=
|
||||
github.com/mattn/go-isatty v0.0.22 h1:j8l17JJ9i6VGPUFUYoTUKPSgKe/83EYU2zBC7YNKMw4=
|
||||
github.com/mattn/go-isatty v0.0.22/go.mod h1:ZXfXG4SQHsB/w3ZeOYbR0PrPwLy+n6xiMrJlRFqopa4=
|
||||
github.com/mattn/go-sqlite3 v1.14.46 h1:ZfaNcYO/CGNMRxkN1vvG9qf+Y+uvXfgT9a6MlEw+HmU=
|
||||
github.com/mattn/go-sqlite3 v1.14.46/go.mod h1:6JTjA44L93a0QCyJef5YvlPoKXntQPjzWv5gtm9sB6w=
|
||||
github.com/lukaszraczylo/oss-telemetry v0.2.5 h1:W6Qgwp7IY4yBRO361MmnEAmM59motdQpbAFAx8v6fJM=
|
||||
github.com/lukaszraczylo/oss-telemetry v0.2.5/go.mod h1:+Cn78qZo8rc3T9eZt0v3oICYRdd75wORtSidc8lNjDQ=
|
||||
github.com/mattn/go-colorable v0.1.16 h1:+2YjeAe7xP1OQbZW1jERPWWCDLlNqFNCvjiqGiAt84A=
|
||||
github.com/mattn/go-colorable v0.1.16/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8=
|
||||
github.com/mattn/go-isatty v0.0.24 h1:tGZZoVgT/KiqK1c8ocVLeDS8BSWMRd47J3Lbz7vsReI=
|
||||
github.com/mattn/go-isatty v0.0.24/go.mod h1:nMCL3Zebbrt45jsMDgnfIwz6ydEQApk5oEI3HqDio6A=
|
||||
github.com/mattn/go-sqlite3 v1.14.52 h1:wVbm2Qnf4OXkqhBTSPuCRZDRnxfbVrrmiCEroVdog8U=
|
||||
github.com/mattn/go-sqlite3 v1.14.52/go.mod h1:6JTjA44L93a0QCyJef5YvlPoKXntQPjzWv5gtm9sB6w=
|
||||
github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ=
|
||||
github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw=
|
||||
github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc=
|
||||
github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
|
||||
github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
|
||||
@@ -41,23 +39,21 @@ github.com/schollz/progressbar/v2 v2.15.0 h1:dVzHQ8fHRmtPjD3K10jT3Qgn/+H+92jhPrh
|
||||
github.com/schollz/progressbar/v2 v2.15.0/go.mod h1:UdPq3prGkfQ7MOzZKlDRpYKcFqEMczbD7YmbPgpzKMI=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
||||
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
|
||||
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
|
||||
github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE=
|
||||
github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg=
|
||||
github.com/sugarme/regexpset v0.0.0-20200920021344-4d4ec8eaf93c h1:pwb4kNSHb4K89ymCaN+5lPH/MwnfSVg4rzGDh4d+iy4=
|
||||
github.com/sugarme/regexpset v0.0.0-20200920021344-4d4ec8eaf93c/go.mod h1:2gwkXLWbDGUQWeL3RtpCmcY4mzCtU13kb9UsAg9xMaw=
|
||||
github.com/yalue/onnxruntime_go v1.31.0 h1:1ln4YW1SFOFfGJZXe3jNOb2JUSt+l2pEneZfV8HdtFA=
|
||||
github.com/yalue/onnxruntime_go v1.31.0/go.mod h1:b4X26A8pekNb1ACJ58wAXgNKeUCGEAQ9dmACut9Sm/4=
|
||||
golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM=
|
||||
golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
|
||||
golang.org/x/sys v0.46.0 h1:noSf2Fq6F8DBgS+LysIkx7rIExoNHJsxOAtPp4rthXw=
|
||||
golang.org/x/sys v0.46.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
|
||||
golang.org/x/text v0.38.0 h1:sXmwo9DwP3OK9EZ7PqAdaooSGozfl/3a6/xJcbzPRhE=
|
||||
golang.org/x/text v0.38.0/go.mod h1:YXZt3QhHUKYT53r2lLKFIVi6Ao1jdzrTR/KQ09qyxF4=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
github.com/yalue/onnxruntime_go v1.36.0 h1:iH1Q++DcsyT9sWtN26KYimESlI5hhXpKaChHDS44oV4=
|
||||
github.com/yalue/onnxruntime_go v1.36.0/go.mod h1:b4X26A8pekNb1ACJ58wAXgNKeUCGEAQ9dmACut9Sm/4=
|
||||
go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw=
|
||||
go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg=
|
||||
golang.org/x/sync v0.24.0 h1:iqiDrGoDgyuvAKrhLQthUEzU/R378N78gXbBjxed7oA=
|
||||
golang.org/x/sync v0.24.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0=
|
||||
golang.org/x/sys v0.49.0 h1:XbzkgYJHdqh/8m2Uu0W/dQv8nktxx4BFHp1M0gROTXA=
|
||||
golang.org/x/sys v0.49.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
|
||||
golang.org/x/text v0.43.0 h1:1QivrlhwAsnMOcqgOdl9my7s9OyZl8O5/UGUTJVXoWc=
|
||||
golang.org/x/text v0.43.0/go.mod h1:hbSIYA/amXcRXTFSZqh8tWHEoqRvvbKPJ2uz/KU+a+s=
|
||||
gorm.io/driver/sqlite v1.6.0 h1:WHRRrIiulaPiPFmDcod6prc4l2VGVWHz80KspNsxSfQ=
|
||||
gorm.io/driver/sqlite v1.6.0/go.mod h1:AO9V1qIQddBESngQUKWL9yoH93HIeA1X6V633rBwyT8=
|
||||
gorm.io/gorm v1.31.1 h1:7CA8FTFz/gRfgqgpeKIBcervUn3xSyPUmr6B2WXJ7kg=
|
||||
gorm.io/gorm v1.31.1/go.mod h1:XyQVbO2k6YkOis7C2437jSit3SsDK72s7n7rsSHd+Gs=
|
||||
gorm.io/gorm v1.31.2 h1:3o8FXNo9v9S858gil+3LlZA1LkCOzgb4g5BL64FgaCo=
|
||||
gorm.io/gorm v1.31.2/go.mod h1:XyQVbO2k6YkOis7C2437jSit3SsDK72s7n7rsSHd+Gs=
|
||||
|
||||
+5
-5
@@ -6,7 +6,7 @@
|
||||
"hooks": [
|
||||
{
|
||||
"type": "command",
|
||||
"command": "${CLAUDE_PLUGIN_ROOT}/hooks/session-start",
|
||||
"command": "\"${CLAUDE_PLUGIN_ROOT}/hooks/session-start\"",
|
||||
"timeout": 30
|
||||
}
|
||||
]
|
||||
@@ -17,7 +17,7 @@
|
||||
"hooks": [
|
||||
{
|
||||
"type": "command",
|
||||
"command": "${CLAUDE_PLUGIN_ROOT}/hooks/user-prompt",
|
||||
"command": "\"${CLAUDE_PLUGIN_ROOT}/hooks/user-prompt\"",
|
||||
"timeout": 10
|
||||
}
|
||||
]
|
||||
@@ -29,7 +29,7 @@
|
||||
"hooks": [
|
||||
{
|
||||
"type": "command",
|
||||
"command": "${CLAUDE_PLUGIN_ROOT}/hooks/post-tool-use",
|
||||
"command": "\"${CLAUDE_PLUGIN_ROOT}/hooks/post-tool-use\"",
|
||||
"timeout": 10
|
||||
}
|
||||
]
|
||||
@@ -40,7 +40,7 @@
|
||||
"hooks": [
|
||||
{
|
||||
"type": "command",
|
||||
"command": "${CLAUDE_PLUGIN_ROOT}/hooks/subagent-stop",
|
||||
"command": "\"${CLAUDE_PLUGIN_ROOT}/hooks/subagent-stop\"",
|
||||
"timeout": 10
|
||||
}
|
||||
]
|
||||
@@ -51,7 +51,7 @@
|
||||
"hooks": [
|
||||
{
|
||||
"type": "command",
|
||||
"command": "${CLAUDE_PLUGIN_ROOT}/hooks/stop",
|
||||
"command": "\"${CLAUDE_PLUGIN_ROOT}/hooks/stop\"",
|
||||
"timeout": 30
|
||||
}
|
||||
]
|
||||
|
||||
@@ -252,9 +252,6 @@ func (s *ObservationStore) UpdateObservation(ctx context.Context, id int64, upda
|
||||
updates["scope"] = sql.NullString{String: *update.Scope, Valid: true}
|
||||
}
|
||||
|
||||
// Add updated_at timestamp
|
||||
updates["updated_at_epoch"] = sql.NullInt64{Int64: time.Now().Unix(), Valid: true}
|
||||
|
||||
if len(updates) == 0 {
|
||||
// Nothing to update, just return existing observation
|
||||
return toModelObservation(&dbObs), nil
|
||||
|
||||
@@ -500,15 +500,13 @@ func TestObservationStore_CleanupOldObservations(t *testing.T) {
|
||||
|
||||
// Store observations beyond the limit WITHOUT async cleanup
|
||||
// We disable async cleanup by not setting cleanupFunc
|
||||
var allIDs []int64
|
||||
for i := 0; i < 105; i++ {
|
||||
observation := &models.ParsedObservation{
|
||||
Type: models.ObsTypeDiscovery,
|
||||
Title: "Observation",
|
||||
}
|
||||
id, _, err := observationStore.StoreObservation(ctx, "claude-1", "test-project", observation, i, 10)
|
||||
_, _, err := observationStore.StoreObservation(ctx, "claude-1", "test-project", observation, i, 10)
|
||||
require.NoError(t, err)
|
||||
allIDs = append(allIDs, id)
|
||||
time.Sleep(2 * time.Millisecond) // Ensure different timestamps
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,96 @@
|
||||
//go:build fts5
|
||||
|
||||
package gorm
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/lukaszraczylo/claude-mnemonic/pkg/models"
|
||||
)
|
||||
|
||||
func storeForUpdate(t *testing.T) (*ObservationStore, int64, func()) {
|
||||
t.Helper()
|
||||
obsStore, store, cleanup := testObservationStore(t)
|
||||
ctx := context.Background()
|
||||
sessionID, err := NewSessionStore(store).CreateSDKSession(ctx, "claude-1", "test-project", "")
|
||||
require.NoError(t, err)
|
||||
id, _, err := obsStore.StoreObservation(ctx, "claude-1", "test-project", &models.ParsedObservation{
|
||||
Type: models.ObsTypeDiscovery, Title: "Original title", Subtitle: "Original subtitle", Narrative: "Original narrative",
|
||||
Facts: []string{"fact one"}, Concepts: []string{"cache"}, FilesRead: []string{"a.go"},
|
||||
}, int(sessionID), 1)
|
||||
require.NoError(t, err)
|
||||
return obsStore, id, cleanup
|
||||
}
|
||||
|
||||
func ptr[T any](v T) *T { return &v }
|
||||
|
||||
func TestUpdateObservation_ChangesOnlyTheGivenFields(t *testing.T) {
|
||||
obsStore, id, cleanup := storeForUpdate(t)
|
||||
defer cleanup()
|
||||
ctx := context.Background()
|
||||
|
||||
got, err := obsStore.UpdateObservation(ctx, id, &ObservationUpdate{Title: ptr("New title"), Narrative: ptr("New narrative")})
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "New title", got.Title.String)
|
||||
assert.Equal(t, "New narrative", got.Narrative.String)
|
||||
assert.Equal(t, "Original subtitle", got.Subtitle.String, "a field that was not given stays")
|
||||
assert.Equal(t, []string{"fact one"}, []string(got.Facts))
|
||||
|
||||
stored, err := obsStore.GetObservationByID(ctx, id)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "New title", stored.Title.String, "and it is stored, not only returned")
|
||||
}
|
||||
|
||||
func TestUpdateObservation_ReplacesListsAndScope(t *testing.T) {
|
||||
obsStore, id, cleanup := storeForUpdate(t)
|
||||
defer cleanup()
|
||||
ctx := context.Background()
|
||||
|
||||
got, err := obsStore.UpdateObservation(ctx, id, &ObservationUpdate{
|
||||
Facts: &[]string{"a", "b"}, Concepts: &[]string{}, FilesRead: &[]string{"x.go"}, FilesModified: &[]string{"y.go"}, Scope: ptr("global"),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, []string{"a", "b"}, []string(got.Facts))
|
||||
assert.Empty(t, got.Concepts, "an empty list clears the field")
|
||||
assert.Equal(t, []string{"x.go"}, []string(got.FilesRead))
|
||||
assert.Equal(t, []string{"y.go"}, []string(got.FilesModified))
|
||||
assert.Equal(t, models.ScopeGlobal, got.Scope)
|
||||
}
|
||||
|
||||
func TestUpdateObservation_NothingToChangeReturnsTheObservation(t *testing.T) {
|
||||
obsStore, id, cleanup := storeForUpdate(t)
|
||||
defer cleanup()
|
||||
|
||||
got, err := obsStore.UpdateObservation(context.Background(), id, &ObservationUpdate{})
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "Original title", got.Title.String)
|
||||
}
|
||||
|
||||
func TestUpdateObservation_Refusals(t *testing.T) {
|
||||
obsStore, id, cleanup := storeForUpdate(t)
|
||||
defer cleanup()
|
||||
ctx := context.Background()
|
||||
|
||||
_, err := obsStore.UpdateObservation(ctx, id, nil)
|
||||
assert.Error(t, err, "a nil update")
|
||||
_, err = obsStore.UpdateObservation(ctx, 999999, &ObservationUpdate{Title: ptr("x")})
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "not found")
|
||||
}
|
||||
|
||||
func TestUpdateObservation_KeepsTheSearchIndexInStep(t *testing.T) {
|
||||
obsStore, id, cleanup := storeForUpdate(t)
|
||||
defer cleanup()
|
||||
ctx := context.Background()
|
||||
|
||||
_, err := obsStore.UpdateObservation(ctx, id, &ObservationUpdate{Title: ptr("Zebrafish enclosure")})
|
||||
require.NoError(t, err)
|
||||
var n int64
|
||||
require.NoError(t, obsStore.db.WithContext(ctx).Raw(
|
||||
`SELECT COUNT(*) FROM observations_fts WHERE observations_fts MATCH 'zebrafish'`).Scan(&n).Error)
|
||||
assert.EqualValues(t, 1, n, "the full-text index sees the new title")
|
||||
}
|
||||
@@ -1 +1 @@
|
||||
1.26.0
|
||||
1.30.0
|
||||
|
||||
@@ -1 +1 @@
|
||||
1.26.0
|
||||
1.30.0
|
||||
|
||||
@@ -1 +1 @@
|
||||
1.26.0
|
||||
1.30.0
|
||||
|
||||
@@ -1 +1 @@
|
||||
1.26.0
|
||||
1.30.0
|
||||
|
||||
@@ -6,7 +6,7 @@ import (
|
||||
_ "embed"
|
||||
)
|
||||
|
||||
//go:embed assets/lib/darwin-arm64/libonnxruntime.dylib
|
||||
//go:embed assets/lib/darwin-arm64/libonnxruntime.dylib.gz
|
||||
var onnxRuntimeLib []byte
|
||||
|
||||
const onnxRuntimeLibName = "libonnxruntime.dylib"
|
||||
|
||||
@@ -6,10 +6,10 @@ import (
|
||||
_ "embed"
|
||||
)
|
||||
|
||||
//go:embed assets/lib/linux-amd64/libonnxruntime.so
|
||||
//go:embed assets/lib/linux-amd64/libonnxruntime.so.gz
|
||||
var onnxRuntimeLib []byte
|
||||
|
||||
//go:embed assets/lib/linux-amd64/libonnxruntime_providers_shared.so
|
||||
//go:embed assets/lib/linux-amd64/libonnxruntime_providers_shared.so.gz
|
||||
var onnxRuntimeProvidersLib []byte
|
||||
|
||||
const onnxRuntimeLibName = "libonnxruntime.so"
|
||||
|
||||
@@ -6,10 +6,10 @@ import (
|
||||
_ "embed"
|
||||
)
|
||||
|
||||
//go:embed assets/lib/linux-arm64/libonnxruntime.so
|
||||
//go:embed assets/lib/linux-arm64/libonnxruntime.so.gz
|
||||
var onnxRuntimeLib []byte
|
||||
|
||||
//go:embed assets/lib/linux-arm64/libonnxruntime_providers_shared.so
|
||||
//go:embed assets/lib/linux-arm64/libonnxruntime_providers_shared.so.gz
|
||||
var onnxRuntimeProvidersLib []byte
|
||||
|
||||
const onnxRuntimeLibName = "libonnxruntime.so"
|
||||
|
||||
@@ -6,7 +6,7 @@ import (
|
||||
_ "embed"
|
||||
)
|
||||
|
||||
//go:embed assets/lib/windows-amd64/onnxruntime.dll
|
||||
//go:embed assets/lib/windows-amd64/onnxruntime.dll.gz
|
||||
var onnxRuntimeLib []byte
|
||||
|
||||
const onnxRuntimeLibName = "onnxruntime.dll"
|
||||
|
||||
@@ -3,10 +3,13 @@ package embedding
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"io"
|
||||
"math"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
@@ -24,6 +27,9 @@ const EmbeddingDim = 384
|
||||
|
||||
// Model version constants
|
||||
const (
|
||||
// BGEStorageVersion tags stored vectors; bump it to force a rebuild when vectors change.
|
||||
BGEStorageVersion = "bge-v1.5-l2"
|
||||
|
||||
// BGEModelVersion is the version string for bge-small-en-v1.5
|
||||
BGEModelVersion = "bge-v1.5"
|
||||
// BGEModelName is the human-readable name for bge-small-en-v1.5
|
||||
@@ -126,7 +132,7 @@ func (m *bgeModel) Name() string {
|
||||
|
||||
// Version returns the short version string for storage.
|
||||
func (m *bgeModel) Version() string {
|
||||
return BGEModelVersion
|
||||
return BGEStorageVersion
|
||||
}
|
||||
|
||||
// Dimensions returns the embedding vector size.
|
||||
@@ -134,7 +140,7 @@ func (m *bgeModel) Dimensions() int {
|
||||
return EmbeddingDim
|
||||
}
|
||||
|
||||
// extractONNXLibrary extracts the embedded ONNX runtime library to a temp directory.
|
||||
// extractONNXLibrary gunzips the embedded ONNX runtime library to a temp directory.
|
||||
// Uses content hash to avoid re-extracting if already present.
|
||||
func extractONNXLibrary() (string, error) {
|
||||
// Create a hash of the library content for cache key
|
||||
@@ -157,16 +163,14 @@ func extractONNXLibrary() (string, error) {
|
||||
}
|
||||
|
||||
// Write main library
|
||||
// #nosec G306 -- Shared library needs executable permission (0755) for dynamic linker
|
||||
if err := os.WriteFile(libPath, onnxRuntimeLib, 0755); err != nil {
|
||||
if err := writeGunzipped(libPath, onnxRuntimeLib); err != nil {
|
||||
return "", fmt.Errorf("write library: %w", err)
|
||||
}
|
||||
|
||||
// Write providers library if present (Linux only)
|
||||
if len(onnxRuntimeProvidersLib) > 0 && onnxRuntimeProvidersLibName != "" {
|
||||
providersPath := filepath.Join(cacheDir, onnxRuntimeProvidersLibName)
|
||||
// #nosec G306 -- Shared library needs executable permission (0755) for dynamic linker
|
||||
if err := os.WriteFile(providersPath, onnxRuntimeProvidersLib, 0755); err != nil {
|
||||
if err := writeGunzipped(providersPath, onnxRuntimeProvidersLib); err != nil {
|
||||
return "", fmt.Errorf("write providers library: %w", err)
|
||||
}
|
||||
}
|
||||
@@ -512,12 +516,28 @@ func meanPooling(embeddings []float32, attentionMask []int64, batchSize, seqLen,
|
||||
}
|
||||
}
|
||||
|
||||
results[b] = result
|
||||
results[b] = l2Normalize(result)
|
||||
}
|
||||
|
||||
return results
|
||||
}
|
||||
|
||||
// l2Normalize scales v to unit length in place; sqlite-vec distances assume unit vectors.
|
||||
func l2Normalize(v []float32) []float32 {
|
||||
var sum float64
|
||||
for _, x := range v {
|
||||
sum += float64(x) * float64(x)
|
||||
}
|
||||
if sum == 0 {
|
||||
return v
|
||||
}
|
||||
inv := float32(1 / math.Sqrt(sum))
|
||||
for i := range v {
|
||||
v[i] *= inv
|
||||
}
|
||||
return v
|
||||
}
|
||||
|
||||
// clsPooling extracts the [CLS] token embedding (first token).
|
||||
// Input shape: [batch, seq_len, hidden]
|
||||
// Output shape: [batch, hidden]
|
||||
@@ -529,7 +549,7 @@ func clsPooling(embeddings []float32, batchSize, seqLen, hiddenSize int) [][]flo
|
||||
// CLS token is at position 0
|
||||
embOffset := b * seqLen * hiddenSize
|
||||
copy(result, embeddings[embOffset:embOffset+hiddenSize])
|
||||
results[b] = result
|
||||
results[b] = l2Normalize(result)
|
||||
}
|
||||
|
||||
return results
|
||||
@@ -626,3 +646,38 @@ func (s *Service) EmbedBatchWithContext(ctx context.Context, texts []string) ([]
|
||||
func (s *Service) Close() error {
|
||||
return s.model.Close()
|
||||
}
|
||||
|
||||
// writeGunzipped decompresses gz into path with the executable bit the dynamic linker needs.
|
||||
// It writes a temp file and renames it, so a crash never leaves a truncated library at path.
|
||||
func writeGunzipped(path string, gz []byte) (err error) {
|
||||
zr, err := gzip.NewReader(bytes.NewReader(gz))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() { _ = zr.Close() }()
|
||||
|
||||
f, err := os.CreateTemp(filepath.Dir(path), filepath.Base(path)+".tmp*")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
tmp := f.Name()
|
||||
defer func() {
|
||||
if err != nil {
|
||||
_ = os.Remove(tmp)
|
||||
}
|
||||
}()
|
||||
|
||||
// #nosec G110 -- archive is embedded at build time, not external input
|
||||
if _, err = io.Copy(f, zr); err != nil {
|
||||
_ = f.Close()
|
||||
return err
|
||||
}
|
||||
if err = f.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
// #nosec G302 -- shared library needs the executable bit for the dynamic linker
|
||||
if err = os.Chmod(tmp, 0o755); err != nil {
|
||||
return err
|
||||
}
|
||||
return os.Rename(tmp, path)
|
||||
}
|
||||
|
||||
@@ -1,9 +1,13 @@
|
||||
package embedding
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"errors"
|
||||
"math"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
@@ -475,3 +479,57 @@ func cosineSimilarity(a, b []float32) float64 {
|
||||
|
||||
return dotProduct / (math.Sqrt(normA) * math.Sqrt(normB))
|
||||
}
|
||||
|
||||
func gzipBytes(t *testing.T, data []byte) []byte {
|
||||
t.Helper()
|
||||
var buf bytes.Buffer
|
||||
zw := gzip.NewWriter(&buf)
|
||||
_, err := zw.Write(data)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, zw.Close())
|
||||
return buf.Bytes()
|
||||
}
|
||||
|
||||
func TestWriteGunzipped(t *testing.T) {
|
||||
payload := bytes.Repeat([]byte("onnx"), 50_000)
|
||||
good := gzipBytes(t, payload)
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
gz []byte
|
||||
wantErr bool
|
||||
wantFile bool
|
||||
}{
|
||||
{"valid archive", good, false, true},
|
||||
{"truncated archive leaves no file", good[:len(good)/2], true, false},
|
||||
{"not gzip leaves no file", []byte("plain text"), true, false},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, "lib.so")
|
||||
|
||||
err := writeGunzipped(path, tt.gz)
|
||||
|
||||
if tt.wantErr {
|
||||
require.Error(t, err)
|
||||
} else {
|
||||
require.NoError(t, err)
|
||||
}
|
||||
_, statErr := os.Stat(path)
|
||||
assert.Equal(t, tt.wantFile, statErr == nil)
|
||||
entries, _ := os.ReadDir(dir)
|
||||
if !tt.wantFile {
|
||||
assert.Empty(t, entries, "no partial or temp files may remain")
|
||||
}
|
||||
if tt.wantFile {
|
||||
got, err := os.ReadFile(path)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, payload, got)
|
||||
info, err := os.Stat(path)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, os.FileMode(0o755), info.Mode().Perm())
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,38 @@
|
||||
package mcp
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestIdleTimeoutFromEnv(t *testing.T) {
|
||||
// The idle shutdown is off unless asked for; anything that is not a positive duration leaves it off.
|
||||
for value, want := range map[string]time.Duration{
|
||||
"": 0,
|
||||
"0": 0,
|
||||
"abc": 0,
|
||||
"-5m": 0,
|
||||
"45": 0, // a bare number is not a duration
|
||||
"90m": 90 * time.Minute,
|
||||
"2h": 2 * time.Hour,
|
||||
"30s": 30 * time.Second,
|
||||
" 2h ": 0, // not trimmed: the operator wrote it, so it is wrong rather than guessed
|
||||
} {
|
||||
t.Run(value, func(t *testing.T) {
|
||||
t.Setenv(IdleTimeoutEnv, value)
|
||||
assert.Equal(t, want, idleTimeoutFromEnv())
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewServer_HasNoIdleTimeoutByDefault(t *testing.T) {
|
||||
t.Setenv(IdleTimeoutEnv, "")
|
||||
assert.Zero(t, NewServer(nil, "http://localhost:1", "p", "v").idleTimeout)
|
||||
}
|
||||
|
||||
func TestNewServer_TakesTheIdleTimeoutFromTheEnvironment(t *testing.T) {
|
||||
t.Setenv(IdleTimeoutEnv, "3h")
|
||||
assert.Equal(t, 3*time.Hour, NewServer(nil, "http://localhost:1", "p", "v").idleTimeout)
|
||||
}
|
||||
@@ -0,0 +1,97 @@
|
||||
//go:build unix
|
||||
|
||||
package mcp
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"os"
|
||||
"sync"
|
||||
"syscall"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// blockingPipe returns the two ends of a pipe in blocking mode, the way an inherited stdin is: a Read on it cannot be
|
||||
// woken by closing the file (the standard pipe from os.Pipe is pollable and can, which hid this).
|
||||
func blockingPipe(t *testing.T) (r, w *os.File) {
|
||||
t.Helper()
|
||||
var fds [2]int
|
||||
require.NoError(t, syscall.Pipe(fds[:]))
|
||||
r = os.NewFile(uintptr(fds[0]), "stdin-like")
|
||||
w = os.NewFile(uintptr(fds[1]), "stdin-writer")
|
||||
t.Cleanup(func() { _ = w.Close(); _ = r.Close() })
|
||||
return r, w
|
||||
}
|
||||
|
||||
type lockedBuffer struct {
|
||||
buf bytes.Buffer
|
||||
mu sync.Mutex
|
||||
}
|
||||
|
||||
func (b *lockedBuffer) Write(p []byte) (int, error) {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
return b.buf.Write(p)
|
||||
}
|
||||
|
||||
func (b *lockedBuffer) String() string {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
return b.buf.String()
|
||||
}
|
||||
|
||||
// The idle timeout used to close stdin and hope the read loop ended. On a real (blocking) stdin the read stays blocked,
|
||||
// so the server logged "shutting down" and then lived on with a closed stdin; the next message from the client woke the
|
||||
// read, which failed with "file already closed" and killed the connection mid-call (seen in Claude Desktop after the
|
||||
// connector sat unused for more than 30 minutes). An idle timeout that is on must end Run for real.
|
||||
func TestRun_IdleTimeoutEndsTheServerCleanly(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
stdin, _ := blockingPipe(t)
|
||||
server := &Server{stdin: stdin, stdout: &lockedBuffer{}, idleTimeout: 50 * time.Millisecond, monitorInterval: 10 * time.Millisecond}
|
||||
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- server.Run(context.Background()) }()
|
||||
|
||||
select {
|
||||
case err := <-done:
|
||||
require.NoError(t, err, "an idle shutdown is a clean exit, not an error")
|
||||
case <-time.After(3 * time.Second):
|
||||
t.Fatal("Run did not return after the idle timeout: stdin was closed but the blocked read never ended")
|
||||
}
|
||||
}
|
||||
|
||||
// By default there is no idle timeout: a stdio server lives as long as its client, and a client that talks to it after a
|
||||
// long pause must get an answer.
|
||||
func TestRun_WithoutAnIdleTimeoutTheServerKeepsAnsweringAfterALongPause(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
stdin, stdinWriter := blockingPipe(t)
|
||||
stdout := &lockedBuffer{}
|
||||
server := &Server{stdin: stdin, stdout: stdout, monitorInterval: 5 * time.Millisecond}
|
||||
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- server.Run(context.Background()) }()
|
||||
|
||||
time.Sleep(300 * time.Millisecond) // many monitor ticks with no traffic
|
||||
select {
|
||||
case err := <-done:
|
||||
t.Fatalf("Run ended during a pause with no traffic: %v", err)
|
||||
default:
|
||||
}
|
||||
|
||||
_, err := stdinWriter.WriteString(`{"jsonrpc":"2.0","id":7,"method":"tools/list"}` + "\n")
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.Eventually(t, func() bool { return bytes.Contains([]byte(stdout.String()), []byte(`"id":7`)) }, 3*time.Second, 10*time.Millisecond,
|
||||
"the request after the pause is answered")
|
||||
select {
|
||||
case err := <-done:
|
||||
t.Fatalf("Run ended after answering: %v", err)
|
||||
default:
|
||||
}
|
||||
}
|
||||
+41
-8
@@ -30,6 +30,11 @@ type Server struct {
|
||||
version string
|
||||
writeMu sync.Mutex
|
||||
lastActivity atomic.Int64
|
||||
// idleTimeout ends the server after this long without a message; 0 (the default) never does. A stdio server lives
|
||||
// as long as its client: ending it while the client is still connected makes the client's next call fail.
|
||||
idleTimeout time.Duration
|
||||
// monitorInterval is how often the parent and idle checks run (30s when 0).
|
||||
monitorInterval time.Duration
|
||||
}
|
||||
|
||||
// NewServer creates a new MCP server that proxies to the worker HTTP API.
|
||||
@@ -41,9 +46,24 @@ func NewServer(client *http.Client, workerURL, project, version string) *Server
|
||||
version: version,
|
||||
stdin: os.Stdin,
|
||||
stdout: os.Stdout,
|
||||
|
||||
idleTimeout: idleTimeoutFromEnv(),
|
||||
}
|
||||
}
|
||||
|
||||
// IdleTimeoutEnv names the environment variable that turns the idle shutdown on, as a Go duration such as "2h".
|
||||
// It is off by default.
|
||||
const IdleTimeoutEnv = "CLAUDE_MNEMONIC_MCP_IDLE_TIMEOUT"
|
||||
|
||||
// idleTimeoutFromEnv reads IdleTimeoutEnv; an unset, invalid or non-positive value means no idle shutdown.
|
||||
func idleTimeoutFromEnv() time.Duration {
|
||||
d, err := time.ParseDuration(os.Getenv(IdleTimeoutEnv))
|
||||
if err != nil || d <= 0 {
|
||||
return 0
|
||||
}
|
||||
return d
|
||||
}
|
||||
|
||||
// Request represents a JSON-RPC request.
|
||||
type Request struct {
|
||||
JSONRPC string `json:"jsonrpc"`
|
||||
@@ -82,6 +102,13 @@ type Tool struct {
|
||||
|
||||
// Run starts the MCP server loop.
|
||||
func (s *Server) Run(ctx context.Context) error {
|
||||
// An idle shutdown cancels this context. Closing stdin does not do it: on a real (blocking) stdin the read stays
|
||||
// blocked, the server lived on with a closed stdin, and the client's next message woke the read with "file already
|
||||
// closed", which killed the connection mid-call.
|
||||
ctx, cancel := context.WithCancel(ctx)
|
||||
defer cancel()
|
||||
var idleStopped atomic.Bool
|
||||
|
||||
scanner := bufio.NewScanner(s.stdin)
|
||||
buf := make([]byte, 0, 64*1024)
|
||||
scanner.Buffer(buf, 1024*1024) // 1MB max message size
|
||||
@@ -100,11 +127,15 @@ func (s *Server) Run(ctx context.Context) error {
|
||||
scanErr <- scanner.Err()
|
||||
}()
|
||||
|
||||
// Monitor parent process liveness and idle timeout.
|
||||
// If the parent dies (ppid changes) or no messages arrive for 30 minutes, shut down.
|
||||
// Monitor parent process liveness and, when one is set, the idle timeout.
|
||||
// If the parent dies (ppid changes) or no messages arrive for idleTimeout, shut down.
|
||||
parentPID := os.Getppid()
|
||||
go func() {
|
||||
ticker := time.NewTicker(30 * time.Second)
|
||||
interval := s.monitorInterval
|
||||
if interval <= 0 {
|
||||
interval = 30 * time.Second
|
||||
}
|
||||
ticker := time.NewTicker(interval)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
@@ -118,11 +149,10 @@ func (s *Server) Run(ctx context.Context) error {
|
||||
}
|
||||
return
|
||||
}
|
||||
if time.Since(time.Unix(s.lastActivity.Load(), 0)) > 30*time.Minute {
|
||||
log.Info().Msg("MCP server idle timeout (30m), shutting down")
|
||||
if closer, ok := s.stdin.(io.Closer); ok {
|
||||
_ = closer.Close()
|
||||
}
|
||||
if s.idleTimeout > 0 && time.Since(time.Unix(s.lastActivity.Load(), 0)) > s.idleTimeout {
|
||||
log.Info().Dur("idle", s.idleTimeout).Msg("MCP server idle timeout, shutting down")
|
||||
idleStopped.Store(true)
|
||||
cancel()
|
||||
return
|
||||
}
|
||||
}
|
||||
@@ -140,6 +170,9 @@ func (s *Server) Run(ctx context.Context) error {
|
||||
case <-ctx.Done():
|
||||
// Drain in-flight requests before returning.
|
||||
wg.Wait()
|
||||
if idleStopped.Load() {
|
||||
return nil // an idle shutdown the operator asked for is a clean exit
|
||||
}
|
||||
return ctx.Err()
|
||||
case line, ok := <-lines:
|
||||
if !ok {
|
||||
|
||||
+125
-107
@@ -5,6 +5,7 @@ import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
@@ -139,9 +140,22 @@ func NewClient(cfg Config, embedSvc *embedding.Service) (*Client, error) {
|
||||
return c, nil
|
||||
}
|
||||
|
||||
// AddDocuments adds documents with their embeddings to the vector store.
|
||||
// AddDocuments embeds the documents and inserts them. A document whose ID already exists is an error (the vec0
|
||||
// table does not replace on insert); use ReplaceDocuments to write the documents of a note that may have some.
|
||||
func (c *Client) AddDocuments(ctx context.Context, docs []Document) error {
|
||||
if len(docs) == 0 {
|
||||
return c.writeDocuments(ctx, docs, "", 0)
|
||||
}
|
||||
|
||||
// ReplaceDocuments writes the documents of one note (observation, summary, ...) in place of the ones it had: the old
|
||||
// documents are deleted and the new ones inserted in one transaction, so an edit that removes a fact leaves no stale
|
||||
// document behind and a failure leaves the old ones untouched. docs may be empty, which only deletes.
|
||||
func (c *Client) ReplaceDocuments(ctx context.Context, docType string, sqliteID int64, docs []Document) error {
|
||||
return c.writeDocuments(ctx, docs, docType, sqliteID)
|
||||
}
|
||||
|
||||
// writeDocuments inserts docs; when docType is not empty it first deletes every document of that note.
|
||||
func (c *Client) writeDocuments(ctx context.Context, docs []Document, docType string, sqliteID int64) (err error) {
|
||||
if len(docs) == 0 && docType == "" {
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -154,9 +168,11 @@ func (c *Client) AddDocuments(ctx context.Context, docs []Document) error {
|
||||
// Compute embeddings OUTSIDE the write lock for better concurrency.
|
||||
// Embedding is ONNX inference (slow, mutex-protected internally) — holding
|
||||
// writeMu during inference blocks all concurrent writes and reads.
|
||||
embeddings, err := c.embedSvc.EmbedBatchWithContext(ctx, texts)
|
||||
if err != nil {
|
||||
return fmt.Errorf("generate embeddings: %w", err)
|
||||
var embeddings [][]float32
|
||||
if len(texts) > 0 {
|
||||
if embeddings, err = c.embedSvc.EmbedBatchWithContext(ctx, texts); err != nil {
|
||||
return fmt.Errorf("generate embeddings: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Acquire write lock for DB operations only
|
||||
@@ -165,7 +181,7 @@ func (c *Client) AddDocuments(ctx context.Context, docs []Document) error {
|
||||
|
||||
// Insert into vectors table with model version tracking
|
||||
const insertQuery = `
|
||||
INSERT OR REPLACE INTO vectors (doc_id, embedding, sqlite_id, doc_type, field_type, project, scope, model_version)
|
||||
INSERT INTO vectors (doc_id, embedding, sqlite_id, doc_type, field_type, project, scope, model_version)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
||||
`
|
||||
|
||||
@@ -176,6 +192,8 @@ func (c *Client) AddDocuments(ctx context.Context, docs []Document) error {
|
||||
if err != nil {
|
||||
return fmt.Errorf("begin transaction: %w", err)
|
||||
}
|
||||
// err is the named result, so every failed return below reaches this and the write lock is released. (A shadowed
|
||||
// err in the loop once left the transaction open, and the database locked, after a failed insert.)
|
||||
defer func() {
|
||||
if err != nil {
|
||||
if rbErr := tx.Rollback(); rbErr != nil {
|
||||
@@ -185,6 +203,12 @@ func (c *Client) AddDocuments(ctx context.Context, docs []Document) error {
|
||||
}
|
||||
}()
|
||||
|
||||
if docType != "" {
|
||||
if _, err = tx.ExecContext(ctx, `DELETE FROM vectors WHERE doc_type = ? AND sqlite_id = ?`, docType, sqliteID); err != nil {
|
||||
return fmt.Errorf("delete old documents: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
stmt, err := tx.PrepareContext(ctx, insertQuery)
|
||||
if err != nil {
|
||||
return fmt.Errorf("prepare statement: %w", err)
|
||||
@@ -193,29 +217,28 @@ func (c *Client) AddDocuments(ctx context.Context, docs []Document) error {
|
||||
|
||||
for i, doc := range docs {
|
||||
// Serialize embedding to blob format
|
||||
embBlob, err := sqlite_vec.SerializeFloat32(embeddings[i])
|
||||
if err != nil {
|
||||
var embBlob []byte
|
||||
if embBlob, err = sqlite_vec.SerializeFloat32(embeddings[i]); err != nil {
|
||||
return fmt.Errorf("serialize embedding for %s: %w", doc.ID, err)
|
||||
}
|
||||
|
||||
// Extract metadata
|
||||
sqliteID, _ := doc.Metadata["sqlite_id"].(int64)
|
||||
docType, _ := doc.Metadata["doc_type"].(string)
|
||||
docSQLiteID, _ := doc.Metadata["sqlite_id"].(int64)
|
||||
docDocType, _ := doc.Metadata["doc_type"].(string)
|
||||
fieldType, _ := doc.Metadata["field_type"].(string)
|
||||
project, _ := doc.Metadata["project"].(string)
|
||||
scope, _ := doc.Metadata["scope"].(string)
|
||||
|
||||
_, err = stmt.ExecContext(ctx,
|
||||
if _, err = stmt.ExecContext(ctx,
|
||||
doc.ID,
|
||||
embBlob,
|
||||
sqliteID,
|
||||
docType,
|
||||
docSQLiteID,
|
||||
docDocType,
|
||||
fieldType,
|
||||
project,
|
||||
scope,
|
||||
modelVersion,
|
||||
)
|
||||
if err != nil {
|
||||
); err != nil {
|
||||
return fmt.Errorf("insert document %s: %w", doc.ID, err)
|
||||
}
|
||||
}
|
||||
@@ -292,67 +315,13 @@ func (c *Client) Query(ctx context.Context, query string, limit int, where map[s
|
||||
}
|
||||
defer c.readMu.RUnlock()
|
||||
|
||||
// Build query with filters
|
||||
// vec0 supports WHERE clauses on metadata columns
|
||||
args := []any{queryBlob}
|
||||
|
||||
sqlQuery := `
|
||||
SELECT
|
||||
doc_id,
|
||||
distance,
|
||||
sqlite_id,
|
||||
doc_type,
|
||||
field_type,
|
||||
project,
|
||||
scope
|
||||
FROM vectors
|
||||
WHERE embedding MATCH ?
|
||||
`
|
||||
|
||||
// Add filters - these work with vec0 metadata columns
|
||||
if docType, ok := where["doc_type"].(string); ok && docType != "" {
|
||||
sqlQuery += " AND doc_type = ?"
|
||||
args = append(args, docType)
|
||||
}
|
||||
if project, ok := where["project"].(string); ok && project != "" {
|
||||
// Include project-specific OR global scope
|
||||
sqlQuery += " AND (project = ? OR scope = 'global')"
|
||||
args = append(args, project)
|
||||
}
|
||||
|
||||
sqlQuery += " ORDER BY distance LIMIT ?"
|
||||
args = append(args, limit)
|
||||
|
||||
rows, err := c.db.QueryContext(ctx, sqlQuery, args...)
|
||||
// vec0 answers equality filters on metadata columns, but not an OR between them, so a project filter
|
||||
// (the project's own documents, plus global ones) is asked as two searches and merged.
|
||||
docType, _ := where["doc_type"].(string)
|
||||
project, _ := where["project"].(string)
|
||||
results, err := c.nearest(ctx, queryBlob, docType, project, limit)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("query vectors: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var results []QueryResult
|
||||
for rows.Next() {
|
||||
var r QueryResult
|
||||
var sqliteID int64
|
||||
var docType, fieldType, project, scope sql.NullString
|
||||
|
||||
if err := rows.Scan(&r.ID, &r.Distance, &sqliteID, &docType, &fieldType, &project, &scope); err != nil {
|
||||
return nil, fmt.Errorf("scan row: %w", err)
|
||||
}
|
||||
|
||||
r.Similarity = DistanceToSimilarity(r.Distance)
|
||||
r.Metadata = map[string]any{
|
||||
"sqlite_id": float64(sqliteID), // Keep as float64 for compatibility
|
||||
"doc_type": docType.String,
|
||||
"field_type": fieldType.String,
|
||||
"project": project.String,
|
||||
"scope": scope.String,
|
||||
}
|
||||
|
||||
results = append(results, r)
|
||||
}
|
||||
|
||||
if err = rows.Err(); err != nil {
|
||||
return nil, fmt.Errorf("iterate rows: %w", err)
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Cache the results
|
||||
@@ -507,60 +476,109 @@ func (c *Client) QueryMultiField(ctx context.Context, query string, limit int, d
|
||||
}
|
||||
defer c.readMu.RUnlock()
|
||||
|
||||
// Query with field type aggregation - get best match per document
|
||||
sqlQuery := `
|
||||
WITH ranked_results AS (
|
||||
SELECT
|
||||
doc_id,
|
||||
distance,
|
||||
sqlite_id,
|
||||
doc_type,
|
||||
field_type,
|
||||
project,
|
||||
scope,
|
||||
ROW_NUMBER() OVER (PARTITION BY sqlite_id ORDER BY distance ASC) as rn
|
||||
FROM vectors
|
||||
WHERE embedding MATCH ?
|
||||
AND doc_type = ?
|
||||
AND (project = ? OR scope = 'global')
|
||||
)
|
||||
SELECT doc_id, distance, sqlite_id, doc_type, field_type, project, scope
|
||||
FROM ranked_results
|
||||
WHERE rn = 1
|
||||
ORDER BY distance
|
||||
LIMIT ?
|
||||
`
|
||||
// Each observation has several vector documents (one per field). Ask for enough of them that `limit`
|
||||
// distinct observations are likely to be among them, then keep the best match per observation.
|
||||
found, err := c.nearest(ctx, queryBlob, docType, project, limit*multiFieldFetchFactor)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
results := make([]QueryResult, 0, limit)
|
||||
seen := make(map[float64]bool, limit)
|
||||
for _, r := range found { // best first
|
||||
id, _ := r.Metadata["sqlite_id"].(float64)
|
||||
if seen[id] {
|
||||
continue
|
||||
}
|
||||
seen[id] = true
|
||||
results = append(results, r)
|
||||
if len(results) == limit {
|
||||
break
|
||||
}
|
||||
}
|
||||
return results, nil
|
||||
}
|
||||
|
||||
rows, err := c.db.QueryContext(ctx, sqlQuery, queryBlob, docType, project, limit)
|
||||
// multiFieldFetchFactor is how many vector documents QueryMultiField asks for per wanted observation.
|
||||
const multiFieldFetchFactor = 6
|
||||
|
||||
// nearest runs the nearest-neighbour search for a document type and, when project is not empty, for that
|
||||
// project's documents and the global ones. The caller holds the read lock. Results are best first, at most limit.
|
||||
func (c *Client) nearest(ctx context.Context, queryBlob []byte, docType, project string, limit int) ([]QueryResult, error) {
|
||||
base := " WHERE embedding MATCH ?"
|
||||
baseArgs := []any{queryBlob}
|
||||
if docType != "" {
|
||||
base += " AND doc_type = ?"
|
||||
baseArgs = append(baseArgs, docType)
|
||||
}
|
||||
if project == "" {
|
||||
return c.knn(ctx, base, baseArgs, limit)
|
||||
}
|
||||
own, err := c.knn(ctx, base+" AND project = ?", append(append([]any{}, baseArgs...), project), limit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
global, err := c.knn(ctx, base+" AND scope = 'global'", baseArgs, limit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return mergeNearest(limit, own, global), nil
|
||||
}
|
||||
|
||||
// knn runs one vec0 nearest-neighbour query. where holds the MATCH and filters vec0 can answer on their own.
|
||||
func (c *Client) knn(ctx context.Context, where string, args []any, limit int) ([]QueryResult, error) {
|
||||
// #nosec G202 -- where is built from constant clauses with "?" placeholders only (see nearest); every value goes via args
|
||||
rows, err := c.db.QueryContext(ctx,
|
||||
"SELECT doc_id, distance, sqlite_id, doc_type, field_type, project, scope FROM vectors"+where+" ORDER BY distance LIMIT ?",
|
||||
append(append([]any{}, args...), limit)...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("query vectors: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
// Pre-allocate with limit to avoid repeated slice growth
|
||||
results := make([]QueryResult, 0, limit)
|
||||
var results []QueryResult
|
||||
for rows.Next() {
|
||||
var r QueryResult
|
||||
var sqliteID int64
|
||||
var docTypeVal, fieldType, projectVal, scope sql.NullString
|
||||
var docType, fieldType, project, scope sql.NullString
|
||||
|
||||
if err := rows.Scan(&r.ID, &r.Distance, &sqliteID, &docTypeVal, &fieldType, &projectVal, &scope); err != nil {
|
||||
if err := rows.Scan(&r.ID, &r.Distance, &sqliteID, &docType, &fieldType, &project, &scope); err != nil {
|
||||
return nil, fmt.Errorf("scan row: %w", err)
|
||||
}
|
||||
|
||||
r.Similarity = DistanceToSimilarity(r.Distance)
|
||||
r.Metadata = map[string]any{
|
||||
"sqlite_id": float64(sqliteID),
|
||||
"doc_type": docTypeVal.String,
|
||||
"sqlite_id": float64(sqliteID), // Keep as float64 for compatibility
|
||||
"doc_type": docType.String,
|
||||
"field_type": fieldType.String,
|
||||
"project": projectVal.String,
|
||||
"project": project.String,
|
||||
"scope": scope.String,
|
||||
}
|
||||
|
||||
results = append(results, r)
|
||||
}
|
||||
if err = rows.Err(); err != nil {
|
||||
return nil, fmt.Errorf("iterate rows: %w", err)
|
||||
}
|
||||
return results, nil
|
||||
}
|
||||
|
||||
return results, rows.Err()
|
||||
// mergeNearest combines result lists into one, best first, without repeating a document, at most limit long.
|
||||
func mergeNearest(limit int, lists ...[]QueryResult) []QueryResult {
|
||||
var all []QueryResult
|
||||
seen := map[string]bool{}
|
||||
for _, list := range lists {
|
||||
for _, r := range list {
|
||||
if !seen[r.ID] {
|
||||
seen[r.ID] = true
|
||||
all = append(all, r)
|
||||
}
|
||||
}
|
||||
}
|
||||
sort.SliceStable(all, func(i, j int) bool { return all[i].Distance < all[j].Distance })
|
||||
if len(all) > limit {
|
||||
all = all[:limit]
|
||||
}
|
||||
return all
|
||||
}
|
||||
|
||||
// acquireRLockWithContext acquires a read lock on mu, respecting ctx cancellation.
|
||||
|
||||
@@ -405,6 +405,50 @@ func TestClient_Query_WithProjectFilter(t *testing.T) {
|
||||
results, err := client.Query(context.Background(), "authentication security", 10, nil)
|
||||
require.NoError(t, err)
|
||||
assert.NotEmpty(t, results, "Should find some results")
|
||||
assert.Len(t, results, 3)
|
||||
|
||||
ids := func(rs []QueryResult) []string {
|
||||
var out []string
|
||||
for _, r := range rs {
|
||||
out = append(out, r.ID)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// With a project filter: the project's own documents and the global ones, not another project's.
|
||||
results, err = client.Query(context.Background(), "authentication security", 10, BuildWhereFilter(DocTypeObservation, "project-a"))
|
||||
require.NoError(t, err, "a project filter must be answerable (vec0 rejects an OR between metadata filters)")
|
||||
assert.ElementsMatch(t, []string{"obs-1", "obs-3"}, ids(results), "project-a's note and the global one, not project-b's own")
|
||||
|
||||
results, err = client.Query(context.Background(), "authentication security", 10, BuildWhereFilter(DocTypeObservation, "project-b"))
|
||||
require.NoError(t, err)
|
||||
assert.ElementsMatch(t, []string{"obs-2", "obs-3"}, ids(results))
|
||||
|
||||
// Best first, and the limit applies to the merged list.
|
||||
results, err = client.Query(context.Background(), "authentication security", 1, BuildWhereFilter(DocTypeObservation, "project-a"))
|
||||
require.NoError(t, err)
|
||||
require.Len(t, results, 1)
|
||||
all, err := client.Query(context.Background(), "authentication security", 10, BuildWhereFilter(DocTypeObservation, "project-a"))
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, all[0].ID, results[0].ID, "the one result is the best of the merged list")
|
||||
assert.LessOrEqual(t, all[0].Distance, all[1].Distance)
|
||||
|
||||
// A project nobody wrote to still sees the global documents.
|
||||
results, err = client.Query(context.Background(), "authentication security", 10, BuildWhereFilter(DocTypeObservation, "project-z"))
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, []string{"obs-3"}, ids(results))
|
||||
}
|
||||
|
||||
func TestMergeNearest(t *testing.T) {
|
||||
r := func(id string, d float64) QueryResult { return QueryResult{ID: id, Distance: d} }
|
||||
got := mergeNearest(3, []QueryResult{r("a", 0.5), r("b", 0.9)}, []QueryResult{r("b", 0.9), r("c", 0.1), r("d", 0.7)})
|
||||
var ids []string
|
||||
for _, x := range got {
|
||||
ids = append(ids, x.ID)
|
||||
}
|
||||
assert.Equal(t, []string{"c", "a", "d"}, ids, "best first, a document found twice counts once, cut to the limit")
|
||||
assert.Empty(t, mergeNearest(5))
|
||||
assert.Len(t, mergeNearest(5, []QueryResult{r("a", 1)}, nil), 1)
|
||||
}
|
||||
|
||||
func TestClient_IsConnected(t *testing.T) {
|
||||
@@ -1105,8 +1149,6 @@ func TestClient_QueryBatch_WithContextCancellation(t *testing.T) {
|
||||
// =============================================================================
|
||||
|
||||
func TestClient_QueryMultiField_Basic(t *testing.T) {
|
||||
t.Skip("QueryMultiField SQL query needs 'k' parameter fix for sqlite-vec")
|
||||
|
||||
db, dbCleanup := testDB(t)
|
||||
defer dbCleanup()
|
||||
|
||||
@@ -1173,8 +1215,6 @@ func TestClient_QueryMultiField_Basic(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestClient_QueryMultiField_WithGlobalScope(t *testing.T) {
|
||||
t.Skip("QueryMultiField SQL query needs 'k' parameter fix for sqlite-vec")
|
||||
|
||||
db, dbCleanup := testDB(t)
|
||||
defer dbCleanup()
|
||||
|
||||
@@ -1217,7 +1257,37 @@ func TestClient_QueryMultiField_WithGlobalScope(t *testing.T) {
|
||||
require.NoError(t, err)
|
||||
|
||||
// Should include both project-scoped (matching project) and global
|
||||
assert.NotEmpty(t, results)
|
||||
require.Len(t, results, 2)
|
||||
assert.ElementsMatch(t, []float64{1, 2}, []float64{
|
||||
results[0].Metadata["sqlite_id"].(float64), results[1].Metadata["sqlite_id"].(float64),
|
||||
})
|
||||
}
|
||||
|
||||
// The query the de-duplication of new observations makes: the same project, a few results, and a note that
|
||||
// says nearly the same thing must come back with a high similarity.
|
||||
func TestClient_Query_ProjectFilterFindsANearDuplicate(t *testing.T) {
|
||||
db, dbCleanup := testDB(t)
|
||||
defer dbCleanup()
|
||||
embedSvc, embedCleanup := testEmbeddingService(t)
|
||||
defer embedCleanup()
|
||||
client, err := NewClient(Config{DB: db}, embedSvc)
|
||||
require.NoError(t, err)
|
||||
|
||||
text := "The shipping rate cache lives for sixty minutes and is stored in Redis."
|
||||
require.NoError(t, client.AddDocuments(context.Background(), []Document{
|
||||
{ID: "obs-1", Content: text, Metadata: map[string]any{"sqlite_id": int64(1), "doc_type": "observation", "project": "shop", "scope": "project"}},
|
||||
{ID: "obs-2", Content: text, Metadata: map[string]any{"sqlite_id": int64(2), "doc_type": "observation", "project": "elsewhere", "scope": "project"}},
|
||||
{ID: "obs-3", Content: "Parcel labels are printed as PDF.", Metadata: map[string]any{"sqlite_id": int64(3), "doc_type": "observation", "project": "shop", "scope": "project"}},
|
||||
}))
|
||||
|
||||
results, err := client.Query(context.Background(), text, 3, BuildWhereFilter(DocTypeObservation, "shop"))
|
||||
require.NoError(t, err)
|
||||
require.NotEmpty(t, results)
|
||||
assert.Equal(t, "obs-1", results[0].ID, "the same text in the same project is the best match")
|
||||
assert.GreaterOrEqual(t, results[0].Similarity, 0.99)
|
||||
for _, r := range results {
|
||||
assert.NotEqual(t, "obs-2", r.ID, "another project's note is not a duplicate")
|
||||
}
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
@@ -2152,3 +2222,67 @@ func TestGetOrComputeEmbedding_ContextCancelDuringSingleflight(t *testing.T) {
|
||||
// Wait for first call to finish
|
||||
<-firstDone
|
||||
}
|
||||
|
||||
func TestClient_AddDocuments_FailedBatchIsRolledBackAndDoesNotLockTheDatabase(t *testing.T) {
|
||||
db, dbCleanup := testDB(t)
|
||||
defer dbCleanup()
|
||||
embedSvc, embedCleanup := testEmbeddingService(t)
|
||||
defer embedCleanup()
|
||||
client, err := NewClient(Config{DB: db}, embedSvc)
|
||||
require.NoError(t, err)
|
||||
|
||||
doc := func(id string, sqliteID int64) Document {
|
||||
return Document{ID: id, Content: "text of " + id, Metadata: map[string]any{
|
||||
"sqlite_id": sqliteID, "doc_type": "observation", "field_type": "narrative", "project": "alpha", "scope": "project",
|
||||
}}
|
||||
}
|
||||
require.NoError(t, client.AddDocuments(context.Background(), []Document{doc("obs_1_narrative", 1)}))
|
||||
|
||||
// The second document of the batch exists already, so the batch fails after the first was inserted.
|
||||
err = client.AddDocuments(context.Background(), []Document{doc("obs_2_narrative", 2), doc("obs_1_narrative", 1)})
|
||||
require.Error(t, err)
|
||||
|
||||
var n int
|
||||
require.NoError(t, db.QueryRow("SELECT COUNT(*) FROM vectors WHERE doc_id = 'obs_2_narrative'").Scan(&n))
|
||||
assert.Zero(t, n, "nothing of the failed batch is kept")
|
||||
|
||||
// With the transaction left open, this would wait for the lock until the context ends.
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
require.NoError(t, client.AddDocuments(ctx, []Document{doc("obs_3_narrative", 3)}), "a write after a failed one goes through")
|
||||
}
|
||||
|
||||
func TestClient_ReplaceDocuments_ReplacesOnlyTheDocumentsOfThatNote(t *testing.T) {
|
||||
db, dbCleanup := testDB(t)
|
||||
defer dbCleanup()
|
||||
embedSvc, embedCleanup := testEmbeddingService(t)
|
||||
defer embedCleanup()
|
||||
client, err := NewClient(Config{DB: db}, embedSvc)
|
||||
require.NoError(t, err)
|
||||
ctx := context.Background()
|
||||
|
||||
doc := func(id string, sqliteID int64, docType, scope string) Document {
|
||||
return Document{ID: id, Content: "text of " + id, Metadata: map[string]any{
|
||||
"sqlite_id": sqliteID, "doc_type": docType, "field_type": "narrative", "project": "alpha", "scope": scope,
|
||||
}}
|
||||
}
|
||||
require.NoError(t, client.AddDocuments(ctx, []Document{
|
||||
doc("obs_1_narrative", 1, "observation", "global"), doc("obs_1_fact_0", 1, "observation", "global"),
|
||||
doc("obs_2_narrative", 2, "observation", "global"),
|
||||
doc("summary_1_request", 1, "session_summary", ""), // the same sqlite id, another kind of document
|
||||
}))
|
||||
|
||||
require.NoError(t, client.ReplaceDocuments(ctx, "observation", 1, []Document{doc("obs_1_narrative", 1, "observation", "project")}))
|
||||
|
||||
count := func(where string, args ...any) (n int) {
|
||||
require.NoError(t, db.QueryRow("SELECT COUNT(*) FROM vectors WHERE "+where, args...).Scan(&n))
|
||||
return n
|
||||
}
|
||||
assert.Equal(t, 1, count("doc_type = 'observation' AND sqlite_id = 1"), "the fact document is gone")
|
||||
assert.Equal(t, 1, count("doc_id = 'obs_1_narrative' AND scope = 'project'"), "and the narrative carries the new scope")
|
||||
assert.Equal(t, 1, count("doc_id = 'obs_2_narrative' AND scope = 'global'"), "another note is untouched")
|
||||
assert.Equal(t, 1, count("doc_id = 'summary_1_request'"), "so is a summary with the same id")
|
||||
|
||||
require.NoError(t, client.ReplaceDocuments(ctx, "observation", 1, nil), "no documents left: only the old ones are deleted")
|
||||
assert.Zero(t, count("doc_type = 'observation' AND sqlite_id = 1"))
|
||||
}
|
||||
|
||||
@@ -22,11 +22,8 @@ func NewSync(client *Client) *Sync {
|
||||
// SyncObservation syncs a single observation to the vector store.
|
||||
func (s *Sync) SyncObservation(ctx context.Context, obs *models.Observation) error {
|
||||
docs := s.formatObservationDocs(obs)
|
||||
if len(docs) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
if err := s.client.AddDocuments(ctx, docs); err != nil {
|
||||
// Replace, not add: a note that is synced again (after an edit, a re-scope or a re-import) already has documents.
|
||||
if err := s.client.ReplaceDocuments(ctx, "observation", obs.ID, docs); err != nil {
|
||||
return fmt.Errorf("add observation docs: %w", err)
|
||||
}
|
||||
|
||||
@@ -102,11 +99,7 @@ func (s *Sync) formatObservationDocs(obs *models.Observation) []Document {
|
||||
// SyncSummary syncs a single session summary to the vector store.
|
||||
func (s *Sync) SyncSummary(ctx context.Context, summary *models.SessionSummary) error {
|
||||
docs := s.formatSummaryDocs(summary)
|
||||
if len(docs) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
if err := s.client.AddDocuments(ctx, docs); err != nil {
|
||||
if err := s.client.ReplaceDocuments(ctx, "session_summary", summary.ID, docs); err != nil {
|
||||
return fmt.Errorf("add summary docs: %w", err)
|
||||
}
|
||||
|
||||
|
||||
@@ -346,3 +346,57 @@ func TestSync_FormatObservationDocs_EmptyScope(t *testing.T) {
|
||||
assert.Len(t, docs, 1)
|
||||
assert.Equal(t, "project", docs[0].Metadata["scope"])
|
||||
}
|
||||
|
||||
func vectorCount(t *testing.T, c *Client, where string, args ...any) int {
|
||||
t.Helper()
|
||||
var n int
|
||||
require.NoError(t, c.db.QueryRow("SELECT COUNT(*) FROM vectors WHERE "+where, args...).Scan(&n))
|
||||
return n
|
||||
}
|
||||
|
||||
func TestSync_SyncObservation_AgainReplacesTheDocumentsOfTheNote(t *testing.T) {
|
||||
client, cleanup := testClient(t)
|
||||
defer cleanup()
|
||||
sync := NewSync(client)
|
||||
ctx := context.Background()
|
||||
|
||||
obs := &models.Observation{
|
||||
ID: 7, SDKSessionID: "s", Project: "alpha", Type: models.ObsTypeDiscovery, Scope: models.ScopeGlobal,
|
||||
Narrative: sql.NullString{String: "The queue keeps one consumer per tenant.", Valid: true},
|
||||
Facts: []string{"one consumer per tenant", "retries are capped", "dead letters go to a second queue"},
|
||||
}
|
||||
other := &models.Observation{
|
||||
ID: 8, SDKSessionID: "s", Project: "alpha", Type: models.ObsTypeDiscovery, Scope: models.ScopeProject,
|
||||
Narrative: sql.NullString{String: "Releases are cut on Thursdays.", Valid: true},
|
||||
}
|
||||
require.NoError(t, sync.SyncObservation(ctx, obs))
|
||||
require.NoError(t, sync.SyncObservation(ctx, other))
|
||||
require.Equal(t, 4, vectorCount(t, client, "doc_type = 'observation' AND sqlite_id = ?", 7))
|
||||
|
||||
// The note was edited: one fact fewer, and it is a project note now.
|
||||
obs.Facts = obs.Facts[:2]
|
||||
obs.Scope = models.ScopeProject
|
||||
require.NoError(t, sync.SyncObservation(ctx, obs), "syncing a note that already has documents is not an error")
|
||||
|
||||
assert.Equal(t, 3, vectorCount(t, client, "doc_type = 'observation' AND sqlite_id = ?", 7), "narrative and two facts; the dropped fact's document is gone")
|
||||
assert.Equal(t, 0, vectorCount(t, client, "doc_id = ?", "obs_7_fact_2"))
|
||||
assert.Equal(t, 3, vectorCount(t, client, "sqlite_id = ? AND scope = 'project'", 7), "the new scope is what the vectors carry")
|
||||
assert.Equal(t, 1, vectorCount(t, client, "sqlite_id = ?", 8), "other notes are not touched")
|
||||
}
|
||||
|
||||
func TestSync_SyncSummary_AgainReplacesTheDocumentsOfTheSummary(t *testing.T) {
|
||||
client, cleanup := testClient(t)
|
||||
defer cleanup()
|
||||
sync := NewSync(client)
|
||||
ctx := context.Background()
|
||||
|
||||
summary := &models.SessionSummary{
|
||||
ID: 3, SDKSessionID: "s", Project: "alpha",
|
||||
Request: sql.NullString{String: "Add the queue", Valid: true}, Learned: sql.NullString{String: "One consumer per tenant", Valid: true},
|
||||
}
|
||||
require.NoError(t, sync.SyncSummary(ctx, summary))
|
||||
summary.Learned = sql.NullString{}
|
||||
require.NoError(t, sync.SyncSummary(ctx, summary))
|
||||
|
||||
assert.Equal(t, 1, vectorCount(t, client, "doc_type = 'session_summary' AND sqlite_id = ?", 3), "the field that was emptied has no document any more")
|
||||
}
|
||||
|
||||
@@ -218,9 +218,8 @@ func (s *Service) handleUpdateRestart(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
// Restart in background after response is sent
|
||||
go func() {
|
||||
if err := s.updater.Restart(); err != nil {
|
||||
log.Error().Err(err).Msg("Failed to restart worker")
|
||||
}
|
||||
// Restart only returns when the new worker failed to start.
|
||||
log.Error().Err(s.updater.Restart()).Msg("Failed to restart worker")
|
||||
}()
|
||||
}
|
||||
|
||||
@@ -244,8 +243,7 @@ func (s *Service) handleRestart(w http.ResponseWriter, r *http.Request) {
|
||||
go func() {
|
||||
// Small delay to ensure response is sent
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
if err := s.updater.Restart(); err != nil {
|
||||
log.Error().Err(err).Msg("Failed to restart worker")
|
||||
}
|
||||
// Restart only returns when the new worker failed to start.
|
||||
log.Error().Err(s.updater.Restart()).Msg("Failed to restart worker")
|
||||
}()
|
||||
}
|
||||
|
||||
@@ -0,0 +1,59 @@
|
||||
package worker
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func doUpdateRequest(t *testing.T, svc *Service, method, target string, body any) *httptest.ResponseRecorder {
|
||||
t.Helper()
|
||||
var buf bytes.Buffer
|
||||
if body != nil {
|
||||
require.NoError(t, json.NewEncoder(&buf).Encode(body))
|
||||
}
|
||||
rec := httptest.NewRecorder()
|
||||
svc.router.ServeHTTP(rec, httptest.NewRequest(method, target, &buf))
|
||||
return rec
|
||||
}
|
||||
|
||||
func TestHandleUpdateObservation_EditsAndReturnsTheObservation(t *testing.T) {
|
||||
svc, cleanup := testService(t)
|
||||
defer cleanup()
|
||||
id := createTestObservation(t, svc.observationStore, "proj_aaaaaa", "Old title", "Old narrative", []string{"cache"})
|
||||
|
||||
rec := doUpdateRequest(t, svc, http.MethodPut, fmt.Sprintf("/api/observations/%d", id),
|
||||
map[string]any{"title": "New title", "facts": []string{"one", "two"}})
|
||||
require.Equal(t, http.StatusOK, rec.Code, rec.Body.String())
|
||||
var resp struct {
|
||||
Observation struct {
|
||||
Title string `json:"title"`
|
||||
Narrative string `json:"narrative"`
|
||||
Facts []string `json:"facts"`
|
||||
} `json:"observation"`
|
||||
}
|
||||
require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &resp))
|
||||
assert.Equal(t, "New title", resp.Observation.Title)
|
||||
assert.Equal(t, "Old narrative", resp.Observation.Narrative)
|
||||
assert.Equal(t, []string{"one", "two"}, resp.Observation.Facts)
|
||||
|
||||
rec = doUpdateRequest(t, svc, http.MethodGet, fmt.Sprintf("/api/observations/%d", id), nil)
|
||||
require.Equal(t, http.StatusOK, rec.Code)
|
||||
assert.Contains(t, rec.Body.String(), "New title", "the edit is stored")
|
||||
}
|
||||
|
||||
func TestHandleUpdateObservation_Refusals(t *testing.T) {
|
||||
svc, cleanup := testService(t)
|
||||
defer cleanup()
|
||||
id := createTestObservation(t, svc.observationStore, "proj_aaaaaa", "A title", "A narrative", nil)
|
||||
|
||||
assert.Equal(t, http.StatusNotFound, doUpdateRequest(t, svc, http.MethodPut, "/api/observations/999999", map[string]any{"title": "x"}).Code)
|
||||
assert.Equal(t, http.StatusBadRequest, doUpdateRequest(t, svc, http.MethodPut, fmt.Sprintf("/api/observations/%d", id), map[string]any{"scope": "galaxy"}).Code)
|
||||
assert.Equal(t, http.StatusBadRequest, doUpdateRequest(t, svc, http.MethodPut, "/api/observations/abc", map[string]any{"title": "x"}).Code)
|
||||
}
|
||||
@@ -2,6 +2,7 @@
|
||||
package worker
|
||||
|
||||
import (
|
||||
"net"
|
||||
"net/http"
|
||||
"sync"
|
||||
"time"
|
||||
@@ -206,14 +207,13 @@ func (pcrl *PerClientRateLimiter) Stats() map[string]any {
|
||||
}
|
||||
|
||||
// PerClientRateLimitMiddleware creates middleware that applies per-client rate limiting.
|
||||
// Uses X-Forwarded-For or RemoteAddr to identify clients.
|
||||
// Identifies clients by the peer IP of the connection; forwarding headers are spoofable and ignored.
|
||||
func PerClientRateLimitMiddleware(limiter *PerClientRateLimiter) func(http.Handler) http.Handler {
|
||||
return func(next http.Handler) http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
// Get client identifier (prefer X-Real-IP from RealIP middleware)
|
||||
clientKey := r.RemoteAddr
|
||||
if xff := r.Header.Get("X-Real-IP"); xff != "" {
|
||||
clientKey = xff
|
||||
if host, _, err := net.SplitHostPort(clientKey); err == nil {
|
||||
clientKey = host
|
||||
}
|
||||
|
||||
if !limiter.Allow(clientKey) {
|
||||
|
||||
@@ -0,0 +1,35 @@
|
||||
package worker
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestPerClientRateLimitMiddleware_KeysOnPeerIP(t *testing.T) {
|
||||
h := PerClientRateLimitMiddleware(NewPerClientRateLimiter(0.001, 1))(
|
||||
http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusOK) }))
|
||||
|
||||
do := func(remote, realIP string) int {
|
||||
req := httptest.NewRequest(http.MethodGet, "/", nil)
|
||||
req.RemoteAddr = remote
|
||||
if realIP != "" {
|
||||
req.Header.Set("X-Real-IP", realIP)
|
||||
}
|
||||
rec := httptest.NewRecorder()
|
||||
h.ServeHTTP(rec, req)
|
||||
return rec.Code
|
||||
}
|
||||
|
||||
if got := do("10.0.0.1:1111", "1.1.1.1"); got != http.StatusOK {
|
||||
t.Fatalf("first request: got %d", got)
|
||||
}
|
||||
// Same peer, new port and forged header: must share the exhausted bucket.
|
||||
if got := do("10.0.0.1:2222", "2.2.2.2"); got != http.StatusTooManyRequests {
|
||||
t.Fatalf("spoofed header/new port bypassed limit: got %d", got)
|
||||
}
|
||||
// Different peer is unaffected.
|
||||
if got := do("10.0.0.2:1111", ""); got != http.StatusOK {
|
||||
t.Fatalf("other peer: got %d", got)
|
||||
}
|
||||
}
|
||||
@@ -51,7 +51,7 @@ func (p *Processor) checkVectorDeduplication(ctx context.Context, obs *models.Pa
|
||||
for _, r := range results {
|
||||
if r.Similarity >= cfg.DeduplicationThreshold {
|
||||
obsID := extractObservationIDFromVectorDoc(r)
|
||||
if obsID > 0 {
|
||||
if obsID > 0 && p.canMergeInto(ctx, obsID) {
|
||||
return &DeduplicationResult{
|
||||
ExistingID: obsID,
|
||||
Similarity: r.Similarity,
|
||||
@@ -64,6 +64,20 @@ func (p *Processor) checkVectorDeduplication(ctx context.Context, obs *models.Pa
|
||||
return &DeduplicationResult{Action: "insert"}
|
||||
}
|
||||
|
||||
// canMergeInto says whether a new observation may be merged into the existing one. A note a person superseded
|
||||
// is hidden from sessions, so what was merged into it would be hidden as well: such a note is not a target,
|
||||
// and neither is one that is gone.
|
||||
func (p *Processor) canMergeInto(ctx context.Context, id int64) bool {
|
||||
if p.observationStore == nil {
|
||||
return true
|
||||
}
|
||||
existing, err := p.observationStore.GetObservationByID(ctx, id)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
return existing != nil && !existing.IsSuperseded
|
||||
}
|
||||
|
||||
// buildObservationSearchText creates searchable text from a parsed observation.
|
||||
func buildObservationSearchText(obs *models.ParsedObservation) string {
|
||||
var parts []string
|
||||
|
||||
@@ -1,8 +1,10 @@
|
||||
package sdk
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"github.com/lukaszraczylo/claude-mnemonic/internal/db/gorm"
|
||||
"github.com/lukaszraczylo/claude-mnemonic/internal/vector/sqlitevec"
|
||||
"github.com/lukaszraczylo/claude-mnemonic/pkg/models"
|
||||
)
|
||||
@@ -121,7 +123,7 @@ func TestCheckVectorDeduplication_NilClient(t *testing.T) {
|
||||
Narrative: "Some narrative text",
|
||||
}
|
||||
|
||||
result := p.checkVectorDeduplication(nil, obs, "test-project")
|
||||
result := p.checkVectorDeduplication(context.Background(), obs, "test-project")
|
||||
if result.Action != "insert" {
|
||||
t.Errorf("expected Action='insert' when vectorClient is nil, got %q", result.Action)
|
||||
}
|
||||
@@ -136,8 +138,40 @@ func TestCheckVectorDeduplication_EmptySearchText(t *testing.T) {
|
||||
// All empty fields
|
||||
}
|
||||
|
||||
result := p.checkVectorDeduplication(nil, obs, "test-project")
|
||||
result := p.checkVectorDeduplication(context.Background(), obs, "test-project")
|
||||
if result.Action != "insert" {
|
||||
t.Errorf("expected Action='insert' for empty observation, got %q", result.Action)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCanMergeInto(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
dir := t.TempDir()
|
||||
store, err := gorm.NewStore(gorm.Config{Path: dir + "/test.db", MaxConns: 1})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer store.Close()
|
||||
obsStore := gorm.NewObservationStore(store, nil, nil, nil)
|
||||
id, _, err := obsStore.StoreObservation(ctx, "s1", "proj", &models.ParsedObservation{Type: models.ObsTypeDiscovery, Title: "A note"}, 1, 1)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
p := &Processor{observationStore: obsStore}
|
||||
|
||||
if !p.canMergeInto(ctx, id) {
|
||||
t.Error("a live observation is a merge target")
|
||||
}
|
||||
if p.canMergeInto(ctx, id+1000) {
|
||||
t.Error("an observation that does not exist is not")
|
||||
}
|
||||
if err := store.DB.Exec(`UPDATE observations SET is_superseded = 1 WHERE id = ?`, id).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if p.canMergeInto(ctx, id) {
|
||||
t.Error("a superseded observation is hidden from sessions, so nothing is merged into it")
|
||||
}
|
||||
if !(&Processor{}).canMergeInto(ctx, id) {
|
||||
t.Error("without an observation store there is nothing to check against")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1213,7 +1213,6 @@ func (s *Service) setupMiddleware() {
|
||||
|
||||
s.router.Use(middleware.Logger)
|
||||
s.router.Use(middleware.Recoverer)
|
||||
s.router.Use(middleware.RealIP)
|
||||
|
||||
// Add security headers (X-Frame-Options, X-Content-Type-Options, CSP, etc.)
|
||||
s.router.Use(SecurityHeaders)
|
||||
@@ -1227,7 +1226,7 @@ func (s *Service) setupMiddleware() {
|
||||
// Add gzip compression for responses >1KB (reduces bandwidth ~70% for JSON)
|
||||
s.router.Use(middleware.Compress(5)) // Level 5 = good balance of speed vs compression
|
||||
|
||||
// Apply per-client rate limiting (after RealIP so we get the real client IP)
|
||||
// Apply per-client rate limiting (keyed on the socket peer address; forwarding headers are untrusted)
|
||||
if s.rateLimiter != nil {
|
||||
s.router.Use(PerClientRateLimitMiddleware(s.rateLimiter))
|
||||
}
|
||||
|
||||
+10
-10
@@ -13,7 +13,7 @@
|
||||
{
|
||||
"name": "claude-mnemonic",
|
||||
"description": "Persistent memory system for Claude Code - Go implementation with SQLite and ChromaDB vector search",
|
||||
"version": "0.21.4",
|
||||
"version": "0.21.127",
|
||||
"author": {
|
||||
"name": "lukaszraczylo",
|
||||
"url": "https://github.com/lukaszraczylo"
|
||||
@@ -23,25 +23,25 @@
|
||||
"license": "MIT",
|
||||
"homepage": "https://github.com/lukaszraczylo/claude-mnemonic",
|
||||
"releases": {
|
||||
"latest": "0.21.4",
|
||||
"latest": "0.21.127",
|
||||
"versions": {
|
||||
"0.21.4": {
|
||||
"releaseDate": "2026-06-19",
|
||||
"0.21.127": {
|
||||
"releaseDate": "2026-10-12",
|
||||
"downloads": {
|
||||
"darwin-arm64": {
|
||||
"url": "https://github.com/lukaszraczylo/claude-mnemonic/releases/download/v0.21.4/claude-mnemonic_0.21.4_darwin_arm64.tar.gz",
|
||||
"url": "https://github.com/lukaszraczylo/claude-mnemonic/releases/download/v0.21.127/claude-mnemonic_0.21.127_darwin_arm64.tar.gz",
|
||||
"format": "tar.gz",
|
||||
"sha256": "caf7738e7956de8e41401750d652ea35a788b8cb1d40dfc650f1d34993bde163"
|
||||
"sha256": "4191ab626fb31a046f6eaeb0f3e5febdbb50285bd0d6d4e21a69e626fe5bbda0"
|
||||
},
|
||||
"linux-amd64": {
|
||||
"url": "https://github.com/lukaszraczylo/claude-mnemonic/releases/download/v0.21.4/claude-mnemonic_0.21.4_linux_amd64.tar.gz",
|
||||
"url": "https://github.com/lukaszraczylo/claude-mnemonic/releases/download/v0.21.127/claude-mnemonic_0.21.127_linux_amd64.tar.gz",
|
||||
"format": "tar.gz",
|
||||
"sha256": "2d66980787db911786848680b267c72ef8f77a4c34b5f83c7e4c56375f620632"
|
||||
"sha256": "9245e63dc10ee88bae33d1519d8c31951ab0abeef045cfd940903bfcf5db3272"
|
||||
},
|
||||
"windows-amd64": {
|
||||
"url": "https://github.com/lukaszraczylo/claude-mnemonic/releases/download/v0.21.4/claude-mnemonic_0.21.4_windows_amd64.zip",
|
||||
"url": "https://github.com/lukaszraczylo/claude-mnemonic/releases/download/v0.21.127/claude-mnemonic_0.21.127_windows_amd64.zip",
|
||||
"format": "zip",
|
||||
"sha256": "c2e77ef85755cab568e644d8ba4248bf3ab3c88ac4001a68bb96d7f1149c9530"
|
||||
"sha256": "ced294d3791c4ebd0bfe37375589d1eff091a75f488eadf0b3b019bc87e3ea1d"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,94 @@
|
||||
package hooks
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// maxResponseBytes caps a worker response so a misbehaving peer cannot exhaust memory.
|
||||
const maxResponseBytes = 32 << 20
|
||||
|
||||
// Minimal HTTP/1.0 client over a raw TCP socket. Linking net/http adds ~3 MB to
|
||||
// every hook binary; hooks only speak plain JSON to the local worker. HTTP/1.0
|
||||
// makes the server skip chunked encoding and close the connection, so the body
|
||||
// is simply everything after the header block.
|
||||
type httpResponse struct {
|
||||
StatusText string
|
||||
Body []byte
|
||||
Status int
|
||||
}
|
||||
|
||||
func httpDo(ctx context.Context, timeout time.Duration, method string, port int, path string, body []byte) (*httpResponse, error) {
|
||||
ctx, cancel := context.WithTimeout(ctx, timeout)
|
||||
defer cancel()
|
||||
|
||||
addr := net.JoinHostPort("127.0.0.1", strconv.Itoa(port))
|
||||
var d net.Dialer
|
||||
conn, err := d.DialContext(ctx, "tcp", addr)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer func() { _ = conn.Close() }()
|
||||
|
||||
if dl, ok := ctx.Deadline(); ok {
|
||||
_ = conn.SetDeadline(dl)
|
||||
}
|
||||
stop := context.AfterFunc(ctx, func() { _ = conn.Close() })
|
||||
defer stop()
|
||||
|
||||
var req bytes.Buffer
|
||||
fmt.Fprintf(&req, "%s %s HTTP/1.0\r\nHost: %s\r\n", method, path, addr)
|
||||
if body != nil {
|
||||
fmt.Fprintf(&req, "Content-Type: application/json\r\nContent-Length: %d\r\n", len(body))
|
||||
}
|
||||
req.WriteString("\r\n")
|
||||
req.Write(body)
|
||||
if _, err := conn.Write(req.Bytes()); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
raw, err := io.ReadAll(io.LimitReader(bufio.NewReader(conn), maxResponseBytes))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return parseHTTPResponse(raw)
|
||||
}
|
||||
|
||||
func parseHTTPResponse(raw []byte) (*httpResponse, error) {
|
||||
head, body, found := bytes.Cut(raw, []byte("\r\n\r\n"))
|
||||
if !found {
|
||||
return nil, errors.New("malformed http response")
|
||||
}
|
||||
statusLine, _, _ := strings.Cut(string(head), "\r\n")
|
||||
_, rest, ok := strings.Cut(statusLine, " ")
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("malformed status line %q", statusLine)
|
||||
}
|
||||
codeStr, _, _ := strings.Cut(rest, " ")
|
||||
code, err := strconv.Atoi(codeStr)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("malformed status line %q", statusLine)
|
||||
}
|
||||
return &httpResponse{Status: code, StatusText: rest, Body: body}, nil
|
||||
}
|
||||
|
||||
// GETBody fetches path from the worker within timeout and returns the raw body.
|
||||
// Statuses of 400 and above are returned as errors.
|
||||
func GETBody(ctx context.Context, timeout time.Duration, port int, path string) ([]byte, error) {
|
||||
resp, err := httpDo(ctx, timeout, "GET", port, path, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if resp.Status >= 400 {
|
||||
return nil, fmt.Errorf("request failed: %s", resp.StatusText)
|
||||
}
|
||||
return resp.Body, nil
|
||||
}
|
||||
+39
-137
@@ -2,12 +2,10 @@
|
||||
package hooks
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
@@ -28,6 +26,9 @@ const (
|
||||
// HealthCheckTimeout is the timeout for health checks.
|
||||
HealthCheckTimeout = 2 * time.Second
|
||||
|
||||
// hookTimeout bounds a single hook->worker request.
|
||||
hookTimeout = 10 * time.Second
|
||||
|
||||
// StartupTimeout is the timeout for worker startup.
|
||||
StartupTimeout = 10 * time.Second
|
||||
|
||||
@@ -52,26 +53,6 @@ var (
|
||||
// circuitBreakerMu protects lastStartupFailure.
|
||||
circuitBreakerMu sync.Mutex
|
||||
lastStartupFailure time.Time
|
||||
|
||||
// hookClient is a shared HTTP client for hook->worker requests.
|
||||
// DisableKeepAlives prevents TIME_WAIT connection leaks since each hook
|
||||
// is a separate OS process that exits quickly.
|
||||
hookClient = &http.Client{
|
||||
Timeout: 10 * time.Second,
|
||||
Transport: &http.Transport{
|
||||
DisableKeepAlives: true,
|
||||
MaxIdleConns: 1,
|
||||
},
|
||||
}
|
||||
|
||||
// healthClient is a shared HTTP client for health/version checks.
|
||||
healthClient = &http.Client{
|
||||
Timeout: HealthCheckTimeout,
|
||||
Transport: &http.Transport{
|
||||
DisableKeepAlives: true,
|
||||
MaxIdleConns: 1,
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
// IsWorkerAvailable performs a fast check without network calls.
|
||||
@@ -110,22 +91,21 @@ func GetWorkerPort() int {
|
||||
// Parses the JSON health response to check the "ready" field when available.
|
||||
// Falls back to HTTP status code check for backwards compatibility.
|
||||
func IsWorkerRunning(port int) bool {
|
||||
resp, err := healthClient.Get(fmt.Sprintf("http://127.0.0.1:%d/api/health", port))
|
||||
resp, err := httpDo(context.Background(), HealthCheckTimeout, "GET", port, "/api/health", nil)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
|
||||
// Try to parse JSON response for structured health check
|
||||
var health struct {
|
||||
Ready bool `json:"ready"`
|
||||
}
|
||||
if err := json.NewDecoder(resp.Body).Decode(&health); err == nil {
|
||||
if err := json.Unmarshal(resp.Body, &health); err == nil {
|
||||
return health.Ready
|
||||
}
|
||||
|
||||
// Fallback: treat HTTP 200 as healthy (backwards compatibility)
|
||||
return resp.StatusCode == http.StatusOK
|
||||
return resp.Status == 200
|
||||
}
|
||||
|
||||
// workerCachePath returns the path to the worker cache file.
|
||||
@@ -396,18 +376,17 @@ func updateCacheFromPort(port int) {
|
||||
|
||||
// GetWorkerVersion gets the version of the running worker.
|
||||
func GetWorkerVersion(port int) string {
|
||||
resp, err := healthClient.Get(fmt.Sprintf("http://127.0.0.1:%d/api/version", port))
|
||||
resp, err := httpDo(context.Background(), HealthCheckTimeout, "GET", port, "/api/version", nil)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
if resp.Status != 200 {
|
||||
return ""
|
||||
}
|
||||
|
||||
var result map[string]string
|
||||
if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
|
||||
if err := json.Unmarshal(resp.Body, &result); err != nil {
|
||||
return ""
|
||||
}
|
||||
|
||||
@@ -510,58 +489,46 @@ func findWorkerBinary() string {
|
||||
return ""
|
||||
}
|
||||
|
||||
// POST sends a POST request to the worker.
|
||||
func POST(port int, path string, body interface{}) (map[string]interface{}, error) {
|
||||
jsonBody, err := json.Marshal(body)
|
||||
// request performs one worker call and returns the decoded JSON object.
|
||||
// A non-JSON body yields (nil, nil) when lenient, else the decode error.
|
||||
func request(ctx context.Context, method string, port int, path string, body interface{}, lenient bool) (map[string]interface{}, error) {
|
||||
var payload []byte
|
||||
if body != nil {
|
||||
var err error
|
||||
if payload, err = json.Marshal(body); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
resp, err := httpDo(ctx, hookTimeout, method, port, path, payload)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
resp, err := hookClient.Post(
|
||||
fmt.Sprintf("http://127.0.0.1:%d%s", port, path),
|
||||
"application/json",
|
||||
bytes.NewReader(jsonBody),
|
||||
)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
|
||||
if resp.StatusCode >= 400 {
|
||||
return nil, fmt.Errorf("request failed: %s", resp.Status)
|
||||
if resp.Status >= 400 {
|
||||
return nil, fmt.Errorf("request failed: %s", resp.StatusText)
|
||||
}
|
||||
|
||||
var result map[string]interface{}
|
||||
if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
|
||||
// Not all endpoints return JSON
|
||||
return nil, nil
|
||||
if err := json.Unmarshal(resp.Body, &result); err != nil {
|
||||
if lenient {
|
||||
// Not all endpoints return JSON
|
||||
return nil, nil
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// POST sends a POST request to the worker.
|
||||
func POST(port int, path string, body interface{}) (map[string]interface{}, error) {
|
||||
return request(context.Background(), "POST", port, path, body, true)
|
||||
}
|
||||
|
||||
// POSTWithContext sends a POST request using the provided context.
|
||||
// Used for fire-and-forget calls where we want to control the timeout externally.
|
||||
func POSTWithContext(ctx context.Context, port int, path string, body interface{}) error {
|
||||
jsonBody, err := json.Marshal(body)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPost,
|
||||
fmt.Sprintf("http://127.0.0.1:%d%s", port, path),
|
||||
bytes.NewReader(jsonBody))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
|
||||
resp, err := hookClient.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
return nil
|
||||
_, err := request(ctx, "POST", port, path, body, true)
|
||||
return err
|
||||
}
|
||||
|
||||
// POSTWithContextResult sends a POST request using the provided context and
|
||||
@@ -570,56 +537,12 @@ func POSTWithContext(ctx context.Context, port int, path string, body interface{
|
||||
// aborts at the hook deadline instead of blocking for the full client timeout.
|
||||
// A non-JSON body is returned as (nil, nil), matching POST's behavior.
|
||||
func POSTWithContextResult(ctx context.Context, port int, path string, body interface{}) (map[string]interface{}, error) {
|
||||
jsonBody, err := json.Marshal(body)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPost,
|
||||
fmt.Sprintf("http://127.0.0.1:%d%s", port, path),
|
||||
bytes.NewReader(jsonBody))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
|
||||
resp, err := hookClient.Do(req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
|
||||
if resp.StatusCode >= 400 {
|
||||
return nil, fmt.Errorf("request failed: %s", resp.Status)
|
||||
}
|
||||
|
||||
var result map[string]interface{}
|
||||
if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
|
||||
// Not all endpoints return JSON
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
return result, nil
|
||||
return request(ctx, "POST", port, path, body, true)
|
||||
}
|
||||
|
||||
// GET sends a GET request to the worker.
|
||||
func GET(port int, path string) (map[string]interface{}, error) {
|
||||
resp, err := hookClient.Get(fmt.Sprintf("http://127.0.0.1:%d%s", port, path))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
|
||||
if resp.StatusCode >= 400 {
|
||||
return nil, fmt.Errorf("request failed: %s", resp.Status)
|
||||
}
|
||||
|
||||
var result map[string]interface{}
|
||||
if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return result, nil
|
||||
return request(context.Background(), "GET", port, path, nil, false)
|
||||
}
|
||||
|
||||
// GETWithContext sends a GET request using the provided context and decodes the
|
||||
@@ -627,28 +550,7 @@ func GET(port int, path string) (map[string]interface{}, error) {
|
||||
// Used on the prompt critical path so a wedged worker aborts at the hook
|
||||
// deadline instead of blocking for the full client timeout.
|
||||
func GETWithContext(ctx context.Context, port int, path string) (map[string]interface{}, error) {
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet,
|
||||
fmt.Sprintf("http://127.0.0.1:%d%s", port, path), nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
resp, err := hookClient.Do(req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
|
||||
if resp.StatusCode >= 400 {
|
||||
return nil, fmt.Errorf("request failed: %s", resp.Status)
|
||||
}
|
||||
|
||||
var result map[string]interface{}
|
||||
if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return result, nil
|
||||
return request(ctx, "GET", port, path, nil, false)
|
||||
}
|
||||
|
||||
// versionsCompatible checks if two versions are compatible for dev builds.
|
||||
|
||||
+98
-15
@@ -5,8 +5,12 @@ import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@@ -1481,22 +1485,101 @@ func TestSleepCtx_Cancelled(t *testing.T) {
|
||||
assert.Less(t, elapsed, 500*time.Millisecond)
|
||||
}
|
||||
|
||||
// TestHookClients_DisableKeepAlives asserts shared clients disable keep-alives
|
||||
// to prevent TIME_WAIT connection leaks in short-lived hook processes.
|
||||
func TestHookClients_DisableKeepAlives(t *testing.T) {
|
||||
hTransport, ok := hookClient.Transport.(*http.Transport)
|
||||
require.True(t, ok, "hookClient.Transport should be *http.Transport")
|
||||
assert.True(t, hTransport.DisableKeepAlives, "hookClient must disable keep-alives")
|
||||
assert.Equal(t, 1, hTransport.MaxIdleConns)
|
||||
// TestHTTPDo_Behaviour covers the raw HTTP/1.0 client against a real server.
|
||||
func TestHTTPDo_Behaviour(t *testing.T) {
|
||||
big := strings.Repeat("x", 200_000)
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
switch r.URL.Path {
|
||||
case "/echo":
|
||||
b, _ := io.ReadAll(r.Body)
|
||||
assert.Equal(t, "application/json", r.Header.Get("Content-Type"))
|
||||
_, _ = w.Write(b)
|
||||
case "/big":
|
||||
_, _ = w.Write([]byte(`{"v":"` + big + `"}`))
|
||||
case "/missing":
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
case "/empty":
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
}
|
||||
}))
|
||||
defer srv.Close()
|
||||
port, err := strconv.Atoi(srv.URL[strings.LastIndex(srv.URL, ":")+1:])
|
||||
require.NoError(t, err)
|
||||
|
||||
hcTransport, ok := healthClient.Transport.(*http.Transport)
|
||||
require.True(t, ok, "healthClient.Transport should be *http.Transport")
|
||||
assert.True(t, hcTransport.DisableKeepAlives, "healthClient must disable keep-alives")
|
||||
assert.Equal(t, 1, hcTransport.MaxIdleConns)
|
||||
tests := []struct {
|
||||
name string
|
||||
method string
|
||||
path string
|
||||
body []byte
|
||||
wantStatus int
|
||||
wantLen int
|
||||
}{
|
||||
{"post echoes body", "POST", "/echo", []byte(`{"a":1}`), 200, 7},
|
||||
{"large body over server buffer", "GET", "/big", nil, 200, len(big) + 8},
|
||||
{"not found", "GET", "/missing", nil, 404, 0},
|
||||
{"no content", "GET", "/empty", nil, 204, 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
resp, err := httpDo(context.Background(), time.Second, tt.method, port, tt.path, tt.body)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, tt.wantStatus, resp.Status)
|
||||
assert.Len(t, resp.Body, tt.wantLen)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestHookClient_Timeout verifies hookClient timeout is set.
|
||||
func TestHookClient_Timeout(t *testing.T) {
|
||||
assert.Equal(t, 10*time.Second, hookClient.Timeout)
|
||||
assert.Equal(t, HealthCheckTimeout, healthClient.Timeout)
|
||||
// TestHTTPDo_Failures covers connection, timeout, cancel and malformed replies.
|
||||
func TestHTTPDo_Failures(t *testing.T) {
|
||||
t.Run("connection refused", func(t *testing.T) {
|
||||
l, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
require.NoError(t, err)
|
||||
port := l.Addr().(*net.TCPAddr).Port
|
||||
require.NoError(t, l.Close())
|
||||
_, err = httpDo(context.Background(), time.Second, "GET", port, "/", nil)
|
||||
assert.Error(t, err)
|
||||
})
|
||||
|
||||
t.Run("wedged server times out", func(t *testing.T) {
|
||||
l, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
require.NoError(t, err)
|
||||
defer func() { _ = l.Close() }()
|
||||
go func() {
|
||||
c, err := l.Accept()
|
||||
if err == nil {
|
||||
defer func() { _ = c.Close() }()
|
||||
time.Sleep(2 * time.Second)
|
||||
}
|
||||
}()
|
||||
start := time.Now()
|
||||
_, err = httpDo(context.Background(), 200*time.Millisecond, "GET", l.Addr().(*net.TCPAddr).Port, "/", nil)
|
||||
assert.Error(t, err)
|
||||
assert.Less(t, time.Since(start), time.Second)
|
||||
})
|
||||
|
||||
t.Run("cancelled context aborts", func(t *testing.T) {
|
||||
l, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
require.NoError(t, err)
|
||||
defer func() { _ = l.Close() }()
|
||||
go func() {
|
||||
c, err := l.Accept()
|
||||
if err == nil {
|
||||
defer func() { _ = c.Close() }()
|
||||
time.Sleep(2 * time.Second)
|
||||
}
|
||||
}()
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond)
|
||||
defer cancel()
|
||||
start := time.Now()
|
||||
_, err = httpDo(ctx, 5*time.Second, "GET", l.Addr().(*net.TCPAddr).Port, "/", nil)
|
||||
assert.Error(t, err)
|
||||
assert.Less(t, time.Since(start), time.Second)
|
||||
})
|
||||
|
||||
t.Run("malformed response", func(t *testing.T) {
|
||||
for _, raw := range []string{"", "garbage", "HTTP/1.0\r\n\r\n", "HTTP/1.0 abc OK\r\n\r\n"} {
|
||||
_, err := parseHTTPResponse([]byte(raw))
|
||||
assert.Error(t, err, "input %q", raw)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
@@ -10,7 +10,6 @@
|
||||
"mcpServers": {
|
||||
"claude-mnemonic": {
|
||||
"command": "${CLAUDE_PLUGIN_ROOT}/mcp-server",
|
||||
"args": ["--project", "${CLAUDE_PROJECT}"],
|
||||
"env": {}
|
||||
}
|
||||
},
|
||||
|
||||
@@ -1,3 +1,8 @@
|
||||
---
|
||||
description: Restart the Claude Mnemonic worker process
|
||||
allowed-tools: Bash(curl:*)
|
||||
---
|
||||
|
||||
# Restart Claude Mnemonic Worker
|
||||
|
||||
Restart the claude-mnemonic worker process. Use this command when experiencing issues with the memory system.
|
||||
|
||||
@@ -6,7 +6,7 @@
|
||||
"hooks": [
|
||||
{
|
||||
"type": "command",
|
||||
"command": "${CLAUDE_PLUGIN_ROOT}/hooks/session-start",
|
||||
"command": "\"${CLAUDE_PLUGIN_ROOT}/hooks/session-start\"",
|
||||
"timeout": 30
|
||||
}
|
||||
]
|
||||
@@ -17,7 +17,7 @@
|
||||
"hooks": [
|
||||
{
|
||||
"type": "command",
|
||||
"command": "${CLAUDE_PLUGIN_ROOT}/hooks/user-prompt",
|
||||
"command": "\"${CLAUDE_PLUGIN_ROOT}/hooks/user-prompt\"",
|
||||
"timeout": 10
|
||||
}
|
||||
]
|
||||
@@ -29,7 +29,7 @@
|
||||
"hooks": [
|
||||
{
|
||||
"type": "command",
|
||||
"command": "${CLAUDE_PLUGIN_ROOT}/hooks/post-tool-use",
|
||||
"command": "\"${CLAUDE_PLUGIN_ROOT}/hooks/post-tool-use\"",
|
||||
"timeout": 10
|
||||
}
|
||||
]
|
||||
@@ -40,7 +40,7 @@
|
||||
"hooks": [
|
||||
{
|
||||
"type": "command",
|
||||
"command": "${CLAUDE_PLUGIN_ROOT}/hooks/subagent-stop",
|
||||
"command": "\"${CLAUDE_PLUGIN_ROOT}/hooks/subagent-stop\"",
|
||||
"timeout": 10
|
||||
}
|
||||
]
|
||||
@@ -51,7 +51,7 @@
|
||||
"hooks": [
|
||||
{
|
||||
"type": "command",
|
||||
"command": "${CLAUDE_PLUGIN_ROOT}/hooks/stop",
|
||||
"command": "\"${CLAUDE_PLUGIN_ROOT}/hooks/stop\"",
|
||||
"timeout": 30
|
||||
}
|
||||
]
|
||||
|
||||
@@ -6,7 +6,7 @@
|
||||
|
||||
set -e
|
||||
|
||||
ONNX_VERSION="1.26.0"
|
||||
ONNX_VERSION="1.30.0"
|
||||
ASSETS_DIR="internal/embedding/assets/lib"
|
||||
PLATFORM="${1:-all}"
|
||||
FORCE_DOWNLOAD=false
|
||||
@@ -51,9 +51,22 @@ get_installed_version() {
|
||||
fi
|
||||
}
|
||||
|
||||
# Gzip the libraries: Go embeds the .gz and gunzips once at first use, which
|
||||
# shrinks the worker binary by roughly 25 MB.
|
||||
compress_libs() {
|
||||
local plat="$1"
|
||||
local f
|
||||
for f in "${ASSETS_DIR}/${plat}"/*.dylib "${ASSETS_DIR}/${plat}"/*.so "${ASSETS_DIR}/${plat}"/*.dll; do
|
||||
if [ -f "$f" ]; then
|
||||
gzip -9 -n -f "$f"
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
# Write version file after successful download
|
||||
write_version_file() {
|
||||
local plat="$1"
|
||||
compress_libs "$plat"
|
||||
echo "${ONNX_VERSION}" > "${ASSETS_DIR}/${plat}/.version"
|
||||
}
|
||||
|
||||
@@ -110,9 +123,9 @@ download_windows_amd64() {
|
||||
lib_exists() {
|
||||
local plat="$1"
|
||||
case "$plat" in
|
||||
darwin-*) [ -f "${ASSETS_DIR}/${plat}/libonnxruntime.dylib" ] ;;
|
||||
linux-*) [ -f "${ASSETS_DIR}/${plat}/libonnxruntime.so" ] ;;
|
||||
windows-*) [ -f "${ASSETS_DIR}/${plat}/onnxruntime.dll" ] ;;
|
||||
darwin-*) [ -f "${ASSETS_DIR}/${plat}/libonnxruntime.dylib.gz" ] ;;
|
||||
linux-*) [ -f "${ASSETS_DIR}/${plat}/libonnxruntime.so.gz" ] ;;
|
||||
windows-*) [ -f "${ASSETS_DIR}/${plat}/onnxruntime.dll.gz" ] ;;
|
||||
*) return 1 ;;
|
||||
esac
|
||||
}
|
||||
|
||||
@@ -0,0 +1,276 @@
|
||||
{
|
||||
"docs": [
|
||||
{
|
||||
"type": "bugfix",
|
||||
"title": "Fixed race in session cache",
|
||||
"narrative": "Concurrent map writes in the session cache caused panics; guarded with a RWMutex."
|
||||
},
|
||||
{
|
||||
"type": "decision",
|
||||
"title": "Chose SQLite WAL mode",
|
||||
"narrative": "WAL lets readers proceed while a writer commits, so hooks can query during writes."
|
||||
},
|
||||
{
|
||||
"type": "feature",
|
||||
"title": "Added Prometheus metrics endpoint",
|
||||
"narrative": "Exposes request latency histograms and error counters at /metrics for scraping."
|
||||
},
|
||||
{
|
||||
"type": "bugfix",
|
||||
"title": "Fixed memory leak in websocket hub",
|
||||
"narrative": "Goroutines were never released when clients disconnected; added context cancellation."
|
||||
},
|
||||
{
|
||||
"type": "refactor",
|
||||
"title": "Extracted retry helper with backoff",
|
||||
"narrative": "Exponential backoff with jitter now lives in one function used by all HTTP callers."
|
||||
},
|
||||
{
|
||||
"type": "decision",
|
||||
"title": "Use argon2id for password hashing",
|
||||
"narrative": "bcrypt cost was too low for new hardware; argon2id with 64MB memory chosen."
|
||||
},
|
||||
{
|
||||
"type": "feature",
|
||||
"title": "Dark mode toggle in dashboard",
|
||||
"narrative": "Theme preference persists in localStorage and follows prefers-color-scheme by default."
|
||||
},
|
||||
{
|
||||
"type": "discovery",
|
||||
"title": "Kubernetes readiness probe timing",
|
||||
"narrative": "Pods were killed during warmup because the readiness probe fired before the model loaded."
|
||||
},
|
||||
{
|
||||
"type": "bugfix",
|
||||
"title": "Fixed off-by-one in pagination",
|
||||
"narrative": "The last page skipped one row when total count was an exact multiple of page size."
|
||||
},
|
||||
{
|
||||
"type": "decision",
|
||||
"title": "Adopted table-driven tests",
|
||||
"narrative": "Subtests with named cases make failures easier to locate and new cases cheap to add."
|
||||
},
|
||||
{
|
||||
"type": "feature",
|
||||
"title": "Rate limiting middleware",
|
||||
"narrative": "Token bucket per client IP returns 429 with a Retry-After header when exceeded."
|
||||
},
|
||||
{
|
||||
"type": "bugfix",
|
||||
"title": "Fixed timezone bug in scheduler",
|
||||
"narrative": "Cron jobs fired an hour late after daylight saving because times were stored in local zone."
|
||||
},
|
||||
{
|
||||
"type": "discovery",
|
||||
"title": "Slow query caused by missing index",
|
||||
"narrative": "EXPLAIN showed a sequential scan on the events table; a composite index cut it from 900ms to 4ms."
|
||||
},
|
||||
{
|
||||
"type": "refactor",
|
||||
"title": "Split monolithic handler file",
|
||||
"narrative": "The 3000 line handlers file was divided by domain: sessions, observations, search."
|
||||
},
|
||||
{
|
||||
"type": "decision",
|
||||
"title": "Embeddings computed locally with ONNX",
|
||||
"narrative": "Avoids sending user text to third parties and works offline."
|
||||
},
|
||||
{
|
||||
"type": "feature",
|
||||
"title": "Export observations as JSON",
|
||||
"narrative": "New CLI command dumps all stored observations to a portable JSON file for backup."
|
||||
},
|
||||
{
|
||||
"type": "bugfix",
|
||||
"title": "Fixed CORS preflight failure",
|
||||
"narrative": "OPTIONS requests were rejected by auth middleware; moved CORS handling before authentication."
|
||||
},
|
||||
{
|
||||
"type": "discovery",
|
||||
"title": "Docker image size dominated by build cache",
|
||||
"narrative": "Multi-stage build with a distroless runtime reduced the image from 1.2GB to 80MB."
|
||||
},
|
||||
{
|
||||
"type": "decision",
|
||||
"title": "Structured logging with zerolog",
|
||||
"narrative": "JSON logs with fields are easier to filter in aggregation than free-form strings."
|
||||
},
|
||||
{
|
||||
"type": "feature",
|
||||
"title": "Full text search with FTS5",
|
||||
"narrative": "Keyword queries use SQLite FTS5 with porter stemming alongside vector similarity."
|
||||
},
|
||||
{
|
||||
"type": "bugfix",
|
||||
"title": "Fixed panic on nil config",
|
||||
"narrative": "Missing settings file returned a nil pointer; defaults are now loaded instead."
|
||||
},
|
||||
{
|
||||
"type": "discovery",
|
||||
"title": "GC pauses from large slices",
|
||||
"narrative": "Preallocating slice capacity removed repeated growth and lowered p99 latency."
|
||||
},
|
||||
{
|
||||
"type": "decision",
|
||||
"title": "Graceful shutdown on SIGTERM",
|
||||
"narrative": "Server drains in-flight requests for 30 seconds before exiting so deploys drop nothing."
|
||||
},
|
||||
{
|
||||
"type": "refactor",
|
||||
"title": "Replaced global state with dependency injection",
|
||||
"narrative": "Services receive their stores through constructors, which makes tests independent."
|
||||
},
|
||||
{
|
||||
"type": "feature",
|
||||
"title": "OAuth login with GitHub",
|
||||
"narrative": "Users authenticate via the GitHub authorization code flow and receive a session cookie."
|
||||
},
|
||||
{
|
||||
"type": "bugfix",
|
||||
"title": "Fixed flaky integration test",
|
||||
"narrative": "The test relied on sleep; replaced with polling on a readiness condition."
|
||||
},
|
||||
{
|
||||
"type": "discovery",
|
||||
"title": "Git hooks slowed commits",
|
||||
"narrative": "Pre-commit ran the full test suite; limited it to changed packages."
|
||||
},
|
||||
{
|
||||
"type": "decision",
|
||||
"title": "Pinned dependency versions",
|
||||
"narrative": "Lockfile committed and CI uses frozen installs to keep builds reproducible."
|
||||
},
|
||||
{
|
||||
"type": "feature",
|
||||
"title": "Webhook retries with dead letter queue",
|
||||
"narrative": "Failed deliveries retry five times and then land in a dead letter table for inspection."
|
||||
},
|
||||
{
|
||||
"type": "discovery",
|
||||
"title": "Unicode normalization broke search",
|
||||
"narrative": "Accented characters did not match until queries and content were NFC normalized."
|
||||
}
|
||||
],
|
||||
"queries": [
|
||||
{
|
||||
"doc": 0,
|
||||
"query": "why did concurrent writes to the cache crash the program"
|
||||
},
|
||||
{
|
||||
"doc": 1,
|
||||
"query": "how can readers keep working while the database is being written"
|
||||
},
|
||||
{
|
||||
"doc": 2,
|
||||
"query": "where do I scrape request latency numbers from"
|
||||
},
|
||||
{
|
||||
"doc": 3,
|
||||
"query": "goroutines not freed after clients drop connection"
|
||||
},
|
||||
{
|
||||
"doc": 4,
|
||||
"query": "shared function for repeating failed requests with growing delays"
|
||||
},
|
||||
{
|
||||
"doc": 5,
|
||||
"query": "which algorithm protects stored passwords"
|
||||
},
|
||||
{
|
||||
"doc": 6,
|
||||
"query": "can the UI switch to a dark theme"
|
||||
},
|
||||
{
|
||||
"doc": 7,
|
||||
"query": "containers restarted while the model was still loading"
|
||||
},
|
||||
{
|
||||
"doc": 8,
|
||||
"query": "last page of results missing an item"
|
||||
},
|
||||
{
|
||||
"doc": 9,
|
||||
"query": "why do our tests use named cases"
|
||||
},
|
||||
{
|
||||
"doc": 10,
|
||||
"query": "how are abusive clients throttled"
|
||||
},
|
||||
{
|
||||
"doc": 11,
|
||||
"query": "scheduled jobs running late after clocks changed"
|
||||
},
|
||||
{
|
||||
"doc": 12,
|
||||
"query": "query got fast after adding an index"
|
||||
},
|
||||
{
|
||||
"doc": 13,
|
||||
"query": "large source file was broken up by domain"
|
||||
},
|
||||
{
|
||||
"doc": 14,
|
||||
"query": "do we send text to an external embedding API"
|
||||
},
|
||||
{
|
||||
"doc": 15,
|
||||
"query": "how to back up everything I stored"
|
||||
},
|
||||
{
|
||||
"doc": 16,
|
||||
"query": "browser blocked cross-origin request before login check"
|
||||
},
|
||||
{
|
||||
"doc": 17,
|
||||
"query": "how did we make the container image smaller"
|
||||
},
|
||||
{
|
||||
"doc": 18,
|
||||
"query": "why are logs emitted as JSON"
|
||||
},
|
||||
{
|
||||
"doc": 19,
|
||||
"query": "keyword matching with stemming"
|
||||
},
|
||||
{
|
||||
"doc": 20,
|
||||
"query": "crash when settings file is absent"
|
||||
},
|
||||
{
|
||||
"doc": 21,
|
||||
"query": "reduce latency spikes from memory allocation"
|
||||
},
|
||||
{
|
||||
"doc": 22,
|
||||
"query": "what happens to requests during deployment shutdown"
|
||||
},
|
||||
{
|
||||
"doc": 23,
|
||||
"query": "stop using package level variables for services"
|
||||
},
|
||||
{
|
||||
"doc": 24,
|
||||
"query": "sign in with a third party identity provider"
|
||||
},
|
||||
{
|
||||
"doc": 25,
|
||||
"query": "test that fails randomly because of timing"
|
||||
},
|
||||
{
|
||||
"doc": 26,
|
||||
"query": "commit takes too long because of checks"
|
||||
},
|
||||
{
|
||||
"doc": 27,
|
||||
"query": "make builds reproducible across machines"
|
||||
},
|
||||
{
|
||||
"doc": 28,
|
||||
"query": "what happens when a callback delivery fails repeatedly"
|
||||
},
|
||||
{
|
||||
"doc": 29,
|
||||
"query": "accented letters not found when searching"
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,157 @@
|
||||
// Command eval-search measures search quality against a labelled corpus.
|
||||
//
|
||||
// eval-search seed <dbdir> writes the corpus into <dbdir>/claude-mnemonic.db
|
||||
// eval-search run <port> queries a running worker and prints MRR@10 and recall@5
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
_ "embed"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/lukaszraczylo/claude-mnemonic/internal/db/gorm"
|
||||
"github.com/lukaszraczylo/claude-mnemonic/pkg/models"
|
||||
)
|
||||
|
||||
const (
|
||||
project = "evalproj"
|
||||
sessionID = "eval-session"
|
||||
dbFile = "claude-mnemonic.db"
|
||||
recallAt = 5
|
||||
mrrAt = 10
|
||||
)
|
||||
|
||||
//go:embed corpus.json
|
||||
var corpusJSON []byte
|
||||
|
||||
type corpus struct {
|
||||
Docs []struct {
|
||||
Type string `json:"type"`
|
||||
Title string `json:"title"`
|
||||
Narrative string `json:"narrative"`
|
||||
} `json:"docs"`
|
||||
Queries []struct {
|
||||
Query string `json:"query"`
|
||||
Doc int `json:"doc"`
|
||||
} `json:"queries"`
|
||||
}
|
||||
|
||||
func main() {
|
||||
if len(os.Args) != 3 {
|
||||
fmt.Fprintln(os.Stderr, "usage: eval-search seed <dbdir> | run <port>")
|
||||
os.Exit(2)
|
||||
}
|
||||
var c corpus
|
||||
if err := json.Unmarshal(corpusJSON, &c); err != nil {
|
||||
fatal(err)
|
||||
}
|
||||
var err error
|
||||
switch os.Args[1] {
|
||||
case "seed":
|
||||
err = seed(c, os.Args[2])
|
||||
case "run":
|
||||
err = run(c, os.Args[2])
|
||||
default:
|
||||
err = fmt.Errorf("unknown mode %q", os.Args[1])
|
||||
}
|
||||
if err != nil {
|
||||
fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func fatal(err error) {
|
||||
fmt.Fprintln(os.Stderr, "error:", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
// seed stores docs in order, so doc i gets observation id i+1.
|
||||
func seed(c corpus, dir string) error {
|
||||
if err := os.MkdirAll(dir, 0o750); err != nil {
|
||||
return err
|
||||
}
|
||||
store, err := gorm.NewStore(gorm.Config{Path: filepath.Join(dir, dbFile), MaxConns: 1})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() { _ = store.Close() }()
|
||||
obsStore := gorm.NewObservationStore(store, nil, nil, nil)
|
||||
for i, d := range c.Docs {
|
||||
_, _, err := obsStore.StoreObservation(context.Background(), sessionID, project, &models.ParsedObservation{
|
||||
Type: models.ObservationType(d.Type),
|
||||
Title: d.Title,
|
||||
Narrative: d.Narrative,
|
||||
Concepts: []string{"eval"},
|
||||
}, i+1, 0)
|
||||
if err != nil {
|
||||
return fmt.Errorf("store doc %d: %w", i, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type obsRef struct {
|
||||
ID int64 `json:"id"`
|
||||
}
|
||||
|
||||
func printRanks(query string, want int64, got []obsRef) {
|
||||
pos := -1
|
||||
for i, o := range got {
|
||||
if o.ID == want {
|
||||
pos = i + 1
|
||||
break
|
||||
}
|
||||
}
|
||||
fmt.Printf("rank=%d results=%d %q\n", pos, len(got), query)
|
||||
}
|
||||
|
||||
func run(c corpus, port string) error {
|
||||
client := &http.Client{Timeout: 20 * time.Second}
|
||||
var mrr, recall, results float64
|
||||
for _, q := range c.Queries {
|
||||
u := fmt.Sprintf("http://127.0.0.1:%s/api/context/search?project=%s&query=%s", port, project, url.QueryEscape(q.Query))
|
||||
resp, err := client.Get(u) // #nosec G704 -- dev tool, loopback URL from a numeric port
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
_ = resp.Body.Close()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var out struct {
|
||||
Observations []obsRef `json:"observations"`
|
||||
}
|
||||
if err := json.Unmarshal(body, &out); err != nil {
|
||||
return fmt.Errorf("decode %q: %w", q.Query, err)
|
||||
}
|
||||
results += float64(len(out.Observations))
|
||||
want := int64(q.Doc + 1)
|
||||
if os.Getenv("EVAL_VERBOSE") != "" {
|
||||
printRanks(q.Query, want, out.Observations)
|
||||
}
|
||||
for rank, o := range out.Observations {
|
||||
if o.ID != want {
|
||||
continue
|
||||
}
|
||||
if rank < mrrAt {
|
||||
mrr += 1 / float64(rank+1)
|
||||
}
|
||||
if rank < recallAt {
|
||||
recall++
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
n := float64(len(c.Queries))
|
||||
fmt.Printf("queries=%d MRR@%d=%s recall@%d=%s avg_results=%s\n", len(c.Queries), mrrAt, strconv.FormatFloat(mrr/n, 'f', 4, 64),
|
||||
recallAt, strconv.FormatFloat(recall/n, 'f', 4, 64), strconv.FormatFloat(results/n, 'f', 2, 64))
|
||||
return nil
|
||||
}
|
||||
Generated
+664
-359
File diff suppressed because it is too large
Load Diff
+11
-11
@@ -10,19 +10,19 @@
|
||||
"type-check": "vue-tsc --noEmit"
|
||||
},
|
||||
"dependencies": {
|
||||
"vis-data": "^8.0.4",
|
||||
"vis-network": "^10.1.0",
|
||||
"vue": "^3.5.34"
|
||||
"vis-data": "^8.0.5",
|
||||
"vis-network": "^10.1.2",
|
||||
"vue": "^3.5.43"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@fortawesome/fontawesome-free": "^7.2.0",
|
||||
"@tailwindcss/postcss": "^4.3.0",
|
||||
"@types/node": "^25.9.1",
|
||||
"@vitejs/plugin-vue": "^6.0.7",
|
||||
"postcss": "^8.5.15",
|
||||
"tailwindcss": "^4.3.0",
|
||||
"@fortawesome/fontawesome-free": "^7.3.1",
|
||||
"@tailwindcss/postcss": "^4.3.3",
|
||||
"@types/node": "^26.0.0",
|
||||
"@vitejs/plugin-vue": "^6.0.9",
|
||||
"postcss": "^8.5.28",
|
||||
"tailwindcss": "^4.3.3",
|
||||
"typescript": "~6.0.3",
|
||||
"vite": "^8.0.14",
|
||||
"vue-tsc": "^3.3.1"
|
||||
"vite": "^8.3.0",
|
||||
"vue-tsc": "^3.3.11"
|
||||
}
|
||||
}
|
||||
|
||||
+11
-11
@@ -10,19 +10,19 @@
|
||||
"type-check": "vue-tsc --noEmit"
|
||||
},
|
||||
"dependencies": {
|
||||
"vis-data": "^8.0.4",
|
||||
"vis-network": "^10.1.0",
|
||||
"vue": "^3.5.34"
|
||||
"vis-data": "^8.0.5",
|
||||
"vis-network": "^10.1.2",
|
||||
"vue": "^3.5.43"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@fortawesome/fontawesome-free": "^7.2.0",
|
||||
"@tailwindcss/postcss": "^4.3.0",
|
||||
"@types/node": "^25.9.1",
|
||||
"@vitejs/plugin-vue": "^6.0.7",
|
||||
"postcss": "^8.5.15",
|
||||
"tailwindcss": "^4.3.0",
|
||||
"@fortawesome/fontawesome-free": "^7.3.1",
|
||||
"@tailwindcss/postcss": "^4.3.3",
|
||||
"@types/node": "^26.0.0",
|
||||
"@vitejs/plugin-vue": "^6.0.9",
|
||||
"postcss": "^8.5.28",
|
||||
"tailwindcss": "^4.3.3",
|
||||
"typescript": "~6.0.3",
|
||||
"vite": "^8.0.14",
|
||||
"vue-tsc": "^3.3.1"
|
||||
"vite": "^8.3.0",
|
||||
"vue-tsc": "^3.3.11"
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user