mirror of
https://github.com/lukaszraczylo/claude-mnemonic.git
synced 2026-10-13 04:59:05 +00:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
7a061c85eb | ||
|
|
92a99c7615 | ||
|
|
f8693bde26 | ||
|
|
c99b8f29b8 | ||
|
|
40a44a71eb | ||
|
|
7cbe16ae7e | ||
|
|
f79782a008 | ||
|
|
48957a6c81 | ||
|
|
ed8b5e92e1 | ||
|
|
2f303454af | ||
|
|
cca48962b4 | ||
|
|
7e2e4d7b6b | ||
|
|
c259bb1d18 | ||
|
|
4add030bed | ||
|
|
587cdab9a5 | ||
|
|
2098a38d64 | ||
|
|
d90664d27d | ||
|
|
fbb09ac6f5 | ||
|
|
6680aa7072 | ||
|
|
cbf6f533f3 | ||
|
|
7d856c7330 | ||
|
|
406de5c850 | ||
|
|
7fed76b53e | ||
|
|
9a2f66d37c | ||
|
|
9d12f08c91 | ||
|
|
4440fd0afb | ||
|
|
2730727e8b | ||
|
|
cae0af62e2 | ||
|
|
cc7b458f07 | ||
|
|
916b2354aa | ||
|
|
cc95e92450 | ||
|
|
97fb3a4102 | ||
|
|
d01f51d0d1 | ||
|
|
a45d1dc0e7 | ||
|
|
939ff8573d | ||
|
|
95073ae65e | ||
|
|
17d69bc8fe | ||
|
|
9afd95d0e5 | ||
|
|
47c83a71e2 | ||
|
|
0f78f6af43 | ||
|
|
48e54a6773 | ||
|
|
02169677a9 | ||
|
|
e975419ec7 | ||
|
|
6e4c132651 | ||
|
|
6a685a79c2 | ||
|
|
c40fa7317b | ||
|
|
cded6bb532 | ||
|
|
4dc0a81582 | ||
|
|
0af10165c6 | ||
|
|
091af2d21b | ||
|
|
a71c61932e | ||
|
|
bc4e9e66ae | ||
|
|
be4a7c19d0 | ||
|
|
b431221f53 | ||
|
|
38b3786942 | ||
|
|
64dd58dbfa | ||
|
|
816af36dd3 | ||
|
|
c715413c09 | ||
|
|
7a6182bb3b | ||
|
|
05a5cea5c7 | ||
|
|
9c7fcbc64c | ||
|
|
a9535c0b64 | ||
|
|
7f9a28743e | ||
|
|
85e1dfa7f3 | ||
|
|
d7c20cea54 |
@@ -0,0 +1 @@
|
||||
internal/embedding/assets/*.onnx filter=lfs diff=lfs merge=lfs -text
|
||||
@@ -17,4 +17,5 @@ jobs:
|
||||
with:
|
||||
go-version: ">=1.24"
|
||||
release-workflow: "release.yaml"
|
||||
lfs: true
|
||||
secrets: inherit
|
||||
|
||||
@@ -20,4 +20,5 @@ jobs:
|
||||
uses: lukaszraczylo/shared-actions/.github/workflows/go-pr.yaml@main
|
||||
with:
|
||||
go-version: ">=1.24"
|
||||
lfs: true
|
||||
secrets: inherit
|
||||
|
||||
+103
-102
@@ -1,120 +1,121 @@
|
||||
name: Release
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
paths-ignore:
|
||||
- "**.md"
|
||||
- "docs/**"
|
||||
workflow_dispatch:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
paths-ignore:
|
||||
- "**.md"
|
||||
- "docs/**"
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
id-token: write
|
||||
contents: write
|
||||
packages: write
|
||||
id-token: write
|
||||
contents: write
|
||||
packages: write
|
||||
|
||||
jobs:
|
||||
release:
|
||||
uses: lukaszraczylo/shared-actions/.github/workflows/go-release-cgo.yaml@main
|
||||
with:
|
||||
go-version: ">=1.24"
|
||||
node-enabled: true
|
||||
node-version: "20"
|
||||
node-build-script: "cd ui && npm ci && npm run build"
|
||||
node-cache-dependency-path: "ui/package-lock.json"
|
||||
node-output-path: "ui/dist"
|
||||
node-embed-path: "internal/worker/static"
|
||||
secrets: inherit
|
||||
|
||||
commit-marketplace:
|
||||
needs: release
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
release:
|
||||
uses: lukaszraczylo/shared-actions/.github/workflows/go-release-cgo.yaml@main
|
||||
with:
|
||||
ref: main
|
||||
go-version: ">=1.24"
|
||||
node-enabled: true
|
||||
node-version: "20"
|
||||
node-build-script: "cd ui && npm ci && npm run build"
|
||||
node-cache-dependency-path: "ui/package-lock.json"
|
||||
node-output-path: "ui/dist"
|
||||
node-embed-path: "internal/worker/static"
|
||||
lfs: true
|
||||
secrets: inherit
|
||||
|
||||
- name: Download checksums
|
||||
run: |
|
||||
VERSION="${{ needs.release.outputs.version_tag }}"
|
||||
curl -sSL -o checksums.txt \
|
||||
"https://github.com/lukaszraczylo/claude-mnemonic/releases/download/${VERSION}/checksums.txt"
|
||||
commit-marketplace:
|
||||
needs: release
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
ref: main
|
||||
|
||||
- name: Generate marketplace.json with checksums
|
||||
run: |
|
||||
VERSION="${{ needs.release.outputs.version }}"
|
||||
VERSION_TAG="${{ needs.release.outputs.version_tag }}"
|
||||
TODAY=$(date -u +%Y-%m-%d)
|
||||
- name: Download checksums
|
||||
run: |
|
||||
VERSION="${{ needs.release.outputs.version_tag }}"
|
||||
curl -sSL -o checksums.txt \
|
||||
"https://github.com/lukaszraczylo/claude-mnemonic/releases/download/${VERSION}/checksums.txt"
|
||||
|
||||
# Extract checksums from checksums.txt
|
||||
SHA_DARWIN_ARM64=$(grep "darwin_arm64" checksums.txt | awk '{print $1}')
|
||||
SHA_LINUX_AMD64=$(grep "linux_amd64" checksums.txt | awk '{print $1}')
|
||||
SHA_WINDOWS_AMD64=$(grep "windows_amd64" checksums.txt | awk '{print $1}')
|
||||
- name: Generate marketplace.json with checksums
|
||||
run: |
|
||||
VERSION="${{ needs.release.outputs.version }}"
|
||||
VERSION_TAG="${{ needs.release.outputs.version_tag }}"
|
||||
TODAY=$(date -u +%Y-%m-%d)
|
||||
|
||||
# Generate marketplace.json
|
||||
cat > marketplace.json << EOF
|
||||
{
|
||||
"\$schema": "https://anthropic.com/claude-code/marketplace.schema.json",
|
||||
"name": "claude-mnemonic",
|
||||
"version": "1.0.0",
|
||||
"description": "Persistent memory system for Claude Code - stores observations, session summaries, and user prompts with semantic search",
|
||||
"owner": {
|
||||
"name": "lukaszraczylo",
|
||||
"email": "[email protected]",
|
||||
"url": "https://github.com/lukaszraczylo"
|
||||
},
|
||||
"repository": "https://github.com/lukaszraczylo/claude-mnemonic",
|
||||
"plugins": [
|
||||
{
|
||||
"name": "claude-mnemonic",
|
||||
"description": "Persistent memory system for Claude Code - Go implementation with SQLite and ChromaDB vector search",
|
||||
"version": "${VERSION}",
|
||||
"author": {
|
||||
"name": "lukaszraczylo",
|
||||
"url": "https://github.com/lukaszraczylo"
|
||||
},
|
||||
"category": "productivity",
|
||||
"tags": ["memory", "persistence", "search", "context"],
|
||||
"license": "MIT",
|
||||
"homepage": "https://github.com/lukaszraczylo/claude-mnemonic",
|
||||
"releases": {
|
||||
"latest": "${VERSION}",
|
||||
"versions": {
|
||||
"${VERSION}": {
|
||||
"releaseDate": "${TODAY}",
|
||||
"downloads": {
|
||||
"darwin-arm64": {
|
||||
"url": "https://github.com/lukaszraczylo/claude-mnemonic/releases/download/${VERSION_TAG}/claude-mnemonic_${VERSION}_darwin_arm64.tar.gz",
|
||||
"format": "tar.gz",
|
||||
"sha256": "${SHA_DARWIN_ARM64}"
|
||||
# Extract checksums from checksums.txt
|
||||
SHA_DARWIN_ARM64=$(grep "darwin_arm64" checksums.txt | awk '{print $1}')
|
||||
SHA_LINUX_AMD64=$(grep "linux_amd64" checksums.txt | awk '{print $1}')
|
||||
SHA_WINDOWS_AMD64=$(grep "windows_amd64" checksums.txt | awk '{print $1}')
|
||||
|
||||
# Generate marketplace.json
|
||||
cat > marketplace.json << EOF
|
||||
{
|
||||
"\$schema": "https://anthropic.com/claude-code/marketplace.schema.json",
|
||||
"name": "claude-mnemonic",
|
||||
"version": "1.0.0",
|
||||
"description": "Persistent memory system for Claude Code - stores observations, session summaries, and user prompts with semantic search",
|
||||
"owner": {
|
||||
"name": "lukaszraczylo",
|
||||
"email": "[email protected]",
|
||||
"url": "https://github.com/lukaszraczylo"
|
||||
},
|
||||
"repository": "https://github.com/lukaszraczylo/claude-mnemonic",
|
||||
"plugins": [
|
||||
{
|
||||
"name": "claude-mnemonic",
|
||||
"description": "Persistent memory system for Claude Code - Go implementation with SQLite and ChromaDB vector search",
|
||||
"version": "${VERSION}",
|
||||
"author": {
|
||||
"name": "lukaszraczylo",
|
||||
"url": "https://github.com/lukaszraczylo"
|
||||
},
|
||||
"linux-amd64": {
|
||||
"url": "https://github.com/lukaszraczylo/claude-mnemonic/releases/download/${VERSION_TAG}/claude-mnemonic_${VERSION}_linux_amd64.tar.gz",
|
||||
"format": "tar.gz",
|
||||
"sha256": "${SHA_LINUX_AMD64}"
|
||||
},
|
||||
"windows-amd64": {
|
||||
"url": "https://github.com/lukaszraczylo/claude-mnemonic/releases/download/${VERSION_TAG}/claude-mnemonic_${VERSION}_windows_amd64.zip",
|
||||
"format": "zip",
|
||||
"sha256": "${SHA_WINDOWS_AMD64}"
|
||||
"category": "productivity",
|
||||
"tags": ["memory", "persistence", "search", "context"],
|
||||
"license": "MIT",
|
||||
"homepage": "https://github.com/lukaszraczylo/claude-mnemonic",
|
||||
"releases": {
|
||||
"latest": "${VERSION}",
|
||||
"versions": {
|
||||
"${VERSION}": {
|
||||
"releaseDate": "${TODAY}",
|
||||
"downloads": {
|
||||
"darwin-arm64": {
|
||||
"url": "https://github.com/lukaszraczylo/claude-mnemonic/releases/download/${VERSION_TAG}/claude-mnemonic_${VERSION}_darwin_arm64.tar.gz",
|
||||
"format": "tar.gz",
|
||||
"sha256": "${SHA_DARWIN_ARM64}"
|
||||
},
|
||||
"linux-amd64": {
|
||||
"url": "https://github.com/lukaszraczylo/claude-mnemonic/releases/download/${VERSION_TAG}/claude-mnemonic_${VERSION}_linux_amd64.tar.gz",
|
||||
"format": "tar.gz",
|
||||
"sha256": "${SHA_LINUX_AMD64}"
|
||||
},
|
||||
"windows-amd64": {
|
||||
"url": "https://github.com/lukaszraczylo/claude-mnemonic/releases/download/${VERSION_TAG}/claude-mnemonic_${VERSION}_windows_amd64.zip",
|
||||
"format": "zip",
|
||||
"sha256": "${SHA_WINDOWS_AMD64}"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
EOF
|
||||
EOF
|
||||
|
||||
echo "Generated marketplace.json:"
|
||||
cat marketplace.json
|
||||
echo "Generated marketplace.json:"
|
||||
cat marketplace.json
|
||||
|
||||
- name: Commit and push
|
||||
run: |
|
||||
git config user.name "github-actions[bot]"
|
||||
git config user.email "github-actions[bot]@users.noreply.github.com"
|
||||
git add marketplace.json
|
||||
git diff --staged --quiet || git commit -m "chore: update marketplace for ${{ needs.release.outputs.version_tag }}"
|
||||
git push
|
||||
- name: Commit and push
|
||||
run: |
|
||||
git config user.name "github-actions[bot]"
|
||||
git config user.email "github-actions[bot]@users.noreply.github.com"
|
||||
git add marketplace.json
|
||||
git diff --staged --quiet || git commit -m "chore: update marketplace for ${{ needs.release.outputs.version_tag }}"
|
||||
git push
|
||||
|
||||
@@ -83,3 +83,10 @@ logs/
|
||||
dist/
|
||||
docs/dist
|
||||
.claude-plugin
|
||||
|
||||
# Auto-generated plugin configs (generated by scripts/generate-plugin-config.sh)
|
||||
.claude-plugin/
|
||||
|
||||
# Non-template plugin configs (keep only .tpl files)
|
||||
plugin/.claude-plugin/plugin.json
|
||||
plugin/.claude-plugin/marketplace.json
|
||||
|
||||
@@ -0,0 +1,24 @@
|
||||
# Project-specific golangci-lint configuration for claude-mnemonic
|
||||
# Inherits from global ~/.golangci.yml and adds project-specific exclusions
|
||||
|
||||
issues:
|
||||
exclude-rules:
|
||||
# Project-specific: Exclude unused warnings for public API functions in pkg/models
|
||||
# These detection functions are part of the public API
|
||||
- path: pkg/models/(conflict|relation)\.go
|
||||
linters:
|
||||
- unused
|
||||
text: "(Detect|New)"
|
||||
|
||||
# Project-specific: Test helper method used only in tests
|
||||
- path: internal/db/gorm/store\.go
|
||||
linters:
|
||||
- unused
|
||||
text: "GetDB"
|
||||
|
||||
exclude-dirs:
|
||||
- vendor
|
||||
|
||||
run:
|
||||
timeout: 5m
|
||||
tests: true
|
||||
@@ -221,6 +221,35 @@ builds:
|
||||
- goos: windows
|
||||
goarch: arm64
|
||||
|
||||
# Hook: statusline
|
||||
- id: hook-statusline
|
||||
main: ./cmd/hooks/statusline
|
||||
binary: hooks/statusline
|
||||
env:
|
||||
- CGO_ENABLED=1
|
||||
tags:
|
||||
- fts5
|
||||
flags:
|
||||
- -trimpath
|
||||
ldflags:
|
||||
- -s -w
|
||||
- -X main.Version={{.Version}}
|
||||
- -X github.com/lukaszraczylo/claude-mnemonic/pkg/hooks.Version={{.Version}}
|
||||
goos:
|
||||
- darwin
|
||||
- linux
|
||||
- windows
|
||||
goarch:
|
||||
- amd64
|
||||
- arm64
|
||||
ignore:
|
||||
- goos: darwin
|
||||
goarch: amd64
|
||||
- goos: linux
|
||||
goarch: arm64
|
||||
- goos: windows
|
||||
goarch: arm64
|
||||
|
||||
archives:
|
||||
- id: default
|
||||
formats:
|
||||
@@ -237,6 +266,9 @@ archives:
|
||||
- src: plugin/hooks/hooks.json
|
||||
dst: hooks
|
||||
strip_parent: true
|
||||
- src: plugin/commands/*
|
||||
dst: commands
|
||||
strip_parent: true
|
||||
format_overrides:
|
||||
- goos: windows
|
||||
formats:
|
||||
|
||||
@@ -138,13 +138,16 @@ install: build stop-worker
|
||||
@# Install to marketplaces directory (for direct installs)
|
||||
@mkdir -p $(HOME)/.claude/plugins/marketplaces/claude-mnemonic/hooks
|
||||
@mkdir -p $(HOME)/.claude/plugins/marketplaces/claude-mnemonic/.claude-plugin
|
||||
@mkdir -p $(HOME)/.claude/plugins/marketplaces/claude-mnemonic/commands
|
||||
cp $(BUILD_DIR)/worker $(HOME)/.claude/plugins/marketplaces/claude-mnemonic/
|
||||
cp $(BUILD_DIR)/mcp-server $(HOME)/.claude/plugins/marketplaces/claude-mnemonic/
|
||||
cp $(BUILD_DIR)/hooks/* $(HOME)/.claude/plugins/marketplaces/claude-mnemonic/hooks/
|
||||
cp $(PLUGIN_DIR)/hooks/hooks.json $(HOME)/.claude/plugins/marketplaces/claude-mnemonic/hooks/
|
||||
@# Copy slash commands if they exist
|
||||
@if [ -d "$(PLUGIN_DIR)/commands" ]; then cp -r $(PLUGIN_DIR)/commands/* $(HOME)/.claude/plugins/marketplaces/claude-mnemonic/commands/ 2>/dev/null || true; fi
|
||||
@# Update plugin.json and marketplace.json with current version to prevent stale version directories
|
||||
@sed 's/"version": "[^"]*"/"version": "$(VERSION)"/g' $(PLUGIN_DIR)/.claude-plugin/plugin.json > $(HOME)/.claude/plugins/marketplaces/claude-mnemonic/.claude-plugin/plugin.json
|
||||
@sed 's/"version": "[^"]*"/"version": "$(VERSION)"/g' $(PLUGIN_DIR)/.claude-plugin/marketplace.json > $(HOME)/.claude/plugins/marketplaces/claude-mnemonic/.claude-plugin/marketplace.json
|
||||
@sed 's/{{ .Version }}/$(VERSION)/g; s/{{.Version}}/$(VERSION)/g' $(PLUGIN_DIR)/.claude-plugin/plugin.json.tpl > $(HOME)/.claude/plugins/marketplaces/claude-mnemonic/.claude-plugin/plugin.json
|
||||
@sed 's/{{ .Version }}/$(VERSION)/g; s/{{.Version}}/$(VERSION)/g' $(PLUGIN_DIR)/.claude-plugin/marketplace.json.tpl > $(HOME)/.claude/plugins/marketplaces/claude-mnemonic/.claude-plugin/marketplace.json
|
||||
@echo "Registering plugin with Claude Code..."
|
||||
@./scripts/register-plugin.sh "$(VERSION)"
|
||||
@$(MAKE) start-worker
|
||||
@@ -157,14 +160,15 @@ uninstall: stop-worker
|
||||
rm -rf $(HOME)/.claude/plugins/marketplaces/claude-mnemonic
|
||||
@echo "Uninstallation complete!"
|
||||
|
||||
# Run tests
|
||||
# Run tests (with FTS5 support)
|
||||
test: setup-libs
|
||||
go test -v -race ./...
|
||||
go test $(BUILD_TAGS) -v -race ./...
|
||||
|
||||
# Run tests with coverage
|
||||
test-coverage:
|
||||
go test -v -race -coverprofile=coverage.out ./...
|
||||
# Run tests with coverage (with FTS5 support)
|
||||
test-coverage: setup-libs
|
||||
go test $(BUILD_TAGS) -v -race -coverprofile=coverage.out ./...
|
||||
go tool cover -html=coverage.out -o coverage.html
|
||||
@go tool cover -func=coverage.out | tail -1
|
||||
|
||||
# Run benchmarks
|
||||
bench:
|
||||
|
||||
@@ -12,6 +12,30 @@ Claude Code forgets everything when your session ends. Claude Mnemonic fixes tha
|
||||
|
||||
It captures what Claude learns during your coding sessions - bug fixes, architecture decisions, patterns that work - and brings that knowledge back in future conversations. No more re-explaining your codebase.
|
||||
|
||||

|
||||
|
||||
## What's New in v0.7
|
||||
|
||||
- **Two-Stage Retrieval** - Cross-encoder reranking for dramatically improved search relevance
|
||||
- **Knowledge Graph** - Automatic relationship detection between observations with visual graph in dashboard
|
||||

|
||||
- **Pattern Detection** - Identifies recurring patterns across sessions and projects
|
||||
- **Importance Scoring** - Time decay and voting system to surface the most valuable memories
|
||||
- **Query Expansion** - Reformulates searches to find semantically related content
|
||||
- **Conflict Detection** - Identifies and resolves contradictory observations
|
||||
- **Observation Lifecycle** - Memories can be superseded when better information arrives
|
||||
|
||||
<details>
|
||||
<summary>Previous: v0.6</summary>
|
||||
|
||||
- **Auto-Updates** - Automatically stays up-to-date with the latest version
|
||||
- **Slash Command: `/restart`** - Restart the worker directly from Claude Code
|
||||
- **Local Embeddings** - All semantic search runs locally via ONNX Runtime (no external API calls)
|
||||
- **Async Queue Processing** - Non-blocking observation capture for faster sessions
|
||||
- **Smarter Storage** - Filters out system/agent summaries to keep knowledge relevant
|
||||
- **Improved Reliability** - Better handling of connectivity issues and dead connections
|
||||
</details>
|
||||
|
||||
## Requirements
|
||||
|
||||
| Dependency | Required | Purpose |
|
||||
@@ -73,9 +97,11 @@ cosign verify-blob \
|
||||
| **Persistent Memory** | Observations survive across sessions and restarts |
|
||||
| **Project Isolation** | Each project has its own knowledge base |
|
||||
| **Global Patterns** | Best practices are shared across all projects |
|
||||
| **Semantic Search** | Find relevant context with natural language |
|
||||
| **Semantic Search** | Find relevant context with natural language (local embeddings) |
|
||||
| **Live Statusline** | Real-time metrics in Claude Code: `[mnemonic] ● served:42 | project:28 memories` |
|
||||
| **Web Dashboard** | Browse and manage memories at `localhost:37777` |
|
||||
| **Auto-Updates** | Automatically downloads and applies new versions |
|
||||
| **Slash Commands** | Control the worker directly from Claude Code |
|
||||
|
||||
### How knowledge flows
|
||||
|
||||
@@ -99,16 +125,39 @@ Config file: `~/.claude-mnemonic/settings.json`
|
||||
{
|
||||
"CLAUDE_MNEMONIC_WORKER_PORT": 37777,
|
||||
"CLAUDE_MNEMONIC_CONTEXT_OBSERVATIONS": 100,
|
||||
"CLAUDE_MNEMONIC_CONTEXT_FULL_COUNT": 25
|
||||
"CLAUDE_MNEMONIC_CONTEXT_FULL_COUNT": 25,
|
||||
"CLAUDE_MNEMONIC_RERANKING_ENABLED": true
|
||||
}
|
||||
```
|
||||
|
||||
### Core Settings
|
||||
|
||||
| Variable | Default | What it does |
|
||||
|----------|---------|--------------|
|
||||
| `WORKER_PORT` | `37777` | Dashboard & API port |
|
||||
| `CONTEXT_OBSERVATIONS` | `100` | Max memories per session |
|
||||
| `CONTEXT_FULL_COUNT` | `25` | Full detail memories (rest are condensed to title only) |
|
||||
| `CONTEXT_FULL_COUNT` | `25` | Full detail memories (rest are condensed) |
|
||||
| `CONTEXT_SESSION_COUNT` | `10` | Recent sessions to reference |
|
||||
| `CONTEXT_RELEVANCE_THRESHOLD` | `0.3` | Minimum similarity score (0.0-1.0) for inclusion |
|
||||
| `CONTEXT_MAX_PROMPT_RESULTS` | `10` | Max results per prompt search |
|
||||
|
||||
### Reranking Settings (Two-Stage Retrieval)
|
||||
|
||||
| Variable | Default | What it does |
|
||||
|----------|---------|--------------|
|
||||
| `RERANKING_ENABLED` | `true` | Enable cross-encoder reranking |
|
||||
| `RERANKING_CANDIDATES` | `100` | Candidates to retrieve before reranking |
|
||||
| `RERANKING_RESULTS` | `10` | Final results after reranking |
|
||||
| `RERANKING_ALPHA` | `0.7` | Score blend: alpha×rerank + (1-alpha)×original |
|
||||
| `RERANKING_PURE_MODE` | `false` | Use pure cross-encoder scores only |
|
||||
|
||||
### Embedding Settings
|
||||
|
||||
| Variable | Default | What it does |
|
||||
|----------|---------|--------------|
|
||||
| `EMBEDDING_MODEL` | `bge-v1.5` | Embedding model for semantic search |
|
||||
|
||||
All variables are prefixed with `CLAUDE_MNEMONIC_` in the config file.
|
||||
|
||||
## Project vs Global scope
|
||||
|
||||
@@ -131,6 +180,25 @@ These search tools are available via MCP:
|
||||
- `changes` - find code modifications
|
||||
- `how_it_works` - system understanding queries
|
||||
|
||||
## Slash Commands
|
||||
|
||||
Available commands within Claude Code:
|
||||
|
||||
| Command | Description |
|
||||
|---------|-------------|
|
||||
| `/restart` | Restart the worker process when experiencing issues |
|
||||
|
||||
## Auto-Updates
|
||||
|
||||
Claude Mnemonic automatically checks for updates and applies them. Updates are downloaded in the background and applied on restart.
|
||||
|
||||
- Automatic update checks on startup
|
||||
- Background downloads (up to 250MB)
|
||||
- Seamless restart after update
|
||||
- Manual trigger: `curl -X POST http://127.0.0.1:37777/api/update/apply`
|
||||
|
||||
Check update status: `curl http://127.0.0.1:37777/api/update/status`
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
**Worker won't start?**
|
||||
@@ -144,6 +212,20 @@ cat /tmp/claude-mnemonic-worker.log # view logs
|
||||
rm -f ~/.claude-mnemonic/*.db-wal ~/.claude-mnemonic/*.db-shm
|
||||
```
|
||||
|
||||
**Worker unresponsive?**
|
||||
```bash
|
||||
# Restart via API
|
||||
curl -X POST http://127.0.0.1:37777/api/restart
|
||||
|
||||
# Or use the slash command in Claude Code
|
||||
/restart
|
||||
```
|
||||
|
||||
**Check health status:**
|
||||
```bash
|
||||
curl http://127.0.0.1:37777/api/selfcheck
|
||||
```
|
||||
|
||||
## Uninstall
|
||||
|
||||
```bash
|
||||
@@ -158,10 +240,11 @@ curl -sSL https://raw.githubusercontent.com/lukaszraczylo/claude-mnemonic/main/s
|
||||
|
||||
- **SQLite + FTS5** - Full-text search for exact matches
|
||||
- **sqlite-vec** - Vector database embedded in SQLite
|
||||
- **all-MiniLM-L6-v2** - Local embedding model (384 dimensions) via ONNX Runtime
|
||||
- **Two-Stage Retrieval** - Bi-encoder (embedding) + cross-encoder (reranking) for high accuracy
|
||||
- **Local Models** - all-MiniLM-L6-v2 for embeddings, BGE reranker for relevance scoring
|
||||
- **Go** - Single binary, no external dependencies
|
||||
|
||||
Everything runs locally. No Python. No external vector database. No API calls for embeddings.
|
||||
Everything runs locally. No Python. No external vector database. No API calls.
|
||||
|
||||
## Platform support
|
||||
|
||||
|
||||
@@ -17,11 +17,50 @@ type Input struct {
|
||||
ToolUseID string `json:"tool_use_id"`
|
||||
}
|
||||
|
||||
// skipTools lists tools that never produce useful observations.
|
||||
// Skip the HTTP call entirely for these to reduce overhead during heavy tool usage.
|
||||
var skipTools = map[string]bool{
|
||||
// Internal tracking tools (but NOT TodoWrite - it captures planned work)
|
||||
"Task": true,
|
||||
"TaskOutput": true,
|
||||
|
||||
// File discovery tools (just listings, no insights)
|
||||
"Glob": true,
|
||||
"ListDir": true,
|
||||
"LS": true,
|
||||
"KillShell": true,
|
||||
|
||||
// Question/interaction tools (no code insights)
|
||||
"AskUserQuestion": true,
|
||||
|
||||
// Plan mode tools (planning, not execution)
|
||||
"EnterPlanMode": true,
|
||||
"ExitPlanMode": true,
|
||||
|
||||
// Skill/command execution (meta-operations)
|
||||
"Skill": true,
|
||||
"SlashCommand": true,
|
||||
|
||||
// Read is high-volume and rarely produces insights worth the overhead
|
||||
// The processor would skip most reads anyway after filtering
|
||||
"Read": true,
|
||||
|
||||
// Search tools are for finding, not modifying
|
||||
"Grep": true,
|
||||
"WebSearch": true,
|
||||
}
|
||||
|
||||
func main() {
|
||||
hooks.RunHook("PostToolUse", handlePostToolUse)
|
||||
}
|
||||
|
||||
func handlePostToolUse(ctx *hooks.HookContext, input *Input) (string, error) {
|
||||
// Skip HTTP call entirely for tools that never produce useful observations.
|
||||
// This significantly reduces overhead during heavy tool usage.
|
||||
if skipTools[input.ToolName] {
|
||||
return "", nil
|
||||
}
|
||||
|
||||
fmt.Fprintf(os.Stderr, "[post-tool-use] %s\n", input.ToolName)
|
||||
|
||||
// Send observation to worker
|
||||
|
||||
@@ -2,9 +2,7 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/url"
|
||||
"os"
|
||||
"strings"
|
||||
@@ -14,11 +12,8 @@ import (
|
||||
|
||||
// Input is the hook input from Claude Code.
|
||||
type Input struct {
|
||||
SessionID string `json:"session_id"`
|
||||
CWD string `json:"cwd"`
|
||||
PermissionMode string `json:"permission_mode"`
|
||||
HookEventName string `json:"hook_event_name"`
|
||||
Source string `json:"source"` // "startup", "resume", "clear", "compact"
|
||||
hooks.BaseInput
|
||||
Source string `json:"source"` // "startup", "resume", "clear", "compact"
|
||||
}
|
||||
|
||||
// Observation represents an observation from the API.
|
||||
@@ -32,53 +27,26 @@ type Observation struct {
|
||||
}
|
||||
|
||||
func main() {
|
||||
// Skip if this is an internal call (from SDK processor)
|
||||
if os.Getenv("CLAUDE_MNEMONIC_INTERNAL") == "1" {
|
||||
hooks.WriteResponse("SessionStart", true)
|
||||
return
|
||||
}
|
||||
|
||||
// Read input from stdin
|
||||
inputData, err := io.ReadAll(os.Stdin)
|
||||
if err != nil {
|
||||
hooks.WriteError("SessionStart", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
var input Input
|
||||
if err := json.Unmarshal(inputData, &input); err != nil {
|
||||
hooks.WriteError("SessionStart", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
// Ensure worker is running
|
||||
port, err := hooks.EnsureWorkerRunning()
|
||||
if err != nil {
|
||||
hooks.WriteError("SessionStart", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
// Generate unique project ID from CWD (dirname_hash format)
|
||||
project := hooks.ProjectIDWithName(input.CWD)
|
||||
hooks.RunHook("SessionStart", handleSessionStart)
|
||||
}
|
||||
|
||||
func handleSessionStart(ctx *hooks.HookContext, input *Input) (string, error) {
|
||||
// Fetch observations for context injection
|
||||
endpoint := fmt.Sprintf("/api/context/inject?project=%s&cwd=%s",
|
||||
url.QueryEscape(project),
|
||||
url.QueryEscape(input.CWD))
|
||||
url.QueryEscape(ctx.Project),
|
||||
url.QueryEscape(ctx.CWD))
|
||||
|
||||
result, err := hooks.GET(port, endpoint)
|
||||
result, err := hooks.GET(ctx.Port, endpoint)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "[claude-mnemonic] Warning: context fetch failed: %v\n", err)
|
||||
hooks.WriteResponse("SessionStart", true)
|
||||
return
|
||||
return "", nil
|
||||
}
|
||||
|
||||
// Parse observations from response
|
||||
obsData, ok := result["observations"].([]interface{})
|
||||
if !ok || len(obsData) == 0 {
|
||||
// No observations - just continue normally
|
||||
hooks.WriteResponse("SessionStart", true)
|
||||
return
|
||||
return "", nil
|
||||
}
|
||||
|
||||
// Get full_count from response (how many observations get full detail)
|
||||
@@ -136,17 +104,7 @@ func main() {
|
||||
}
|
||||
|
||||
contextBuilder += "</claude-mnemonic-context>\n"
|
||||
|
||||
// Output context as JSON with additionalContext field
|
||||
response := map[string]interface{}{
|
||||
"continue": true,
|
||||
"hookSpecificOutput": map[string]interface{}{
|
||||
"hookEventName": "SessionStart",
|
||||
"additionalContext": contextBuilder,
|
||||
},
|
||||
}
|
||||
_ = json.NewEncoder(os.Stdout).Encode(response)
|
||||
os.Exit(0)
|
||||
return contextBuilder, nil
|
||||
}
|
||||
|
||||
func getString(m map[string]interface{}, key string) string {
|
||||
|
||||
@@ -5,7 +5,6 @@ package main
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
@@ -72,18 +71,13 @@ const (
|
||||
)
|
||||
|
||||
func main() {
|
||||
// Read input from stdin
|
||||
inputData, err := io.ReadAll(os.Stdin)
|
||||
if err != nil {
|
||||
// On error, output minimal status
|
||||
fmt.Println(formatOffline())
|
||||
return
|
||||
}
|
||||
hooks.RunStatuslineHook(handleStatusline)
|
||||
}
|
||||
|
||||
var input StatusInput
|
||||
if err := json.Unmarshal(inputData, &input); err != nil {
|
||||
fmt.Println(formatOffline())
|
||||
return
|
||||
func handleStatusline(input *StatusInput, port int) string {
|
||||
// Handle error cases (nil input)
|
||||
if input == nil {
|
||||
return formatOffline()
|
||||
}
|
||||
|
||||
// Determine project directory
|
||||
@@ -102,16 +96,14 @@ func main() {
|
||||
}
|
||||
|
||||
// Get worker stats
|
||||
stats := getWorkerStats(project)
|
||||
stats := getWorkerStats(port, project)
|
||||
|
||||
// Format and output statusline
|
||||
fmt.Println(formatStatusLine(stats, input))
|
||||
// Format and return statusline
|
||||
return formatStatusLine(stats, *input)
|
||||
}
|
||||
|
||||
// getWorkerStats fetches stats from the worker service.
|
||||
func getWorkerStats(project string) *WorkerStats {
|
||||
port := hooks.GetWorkerPort()
|
||||
|
||||
func getWorkerStats(port int, project string) *WorkerStats {
|
||||
// Build URL with optional project parameter
|
||||
endpoint := fmt.Sprintf("http://127.0.0.1:%d/api/stats", port)
|
||||
if project != "" {
|
||||
@@ -187,54 +179,45 @@ func formatDefault(stats *WorkerStats, useColors bool) string {
|
||||
}
|
||||
|
||||
// Build status parts with clear labels
|
||||
parts := []string{}
|
||||
parts := []string{
|
||||
prefix,
|
||||
indicator,
|
||||
}
|
||||
|
||||
// Total memories served to Claude this session
|
||||
parts = append(parts, fmt.Sprintf("served:%d", stats.Retrieval.ObservationsServed))
|
||||
|
||||
// Context injections (memories auto-loaded at session start)
|
||||
// Add retrieval stats if available
|
||||
if stats.Retrieval.ObservationsServed > 0 {
|
||||
parts = append(parts, fmt.Sprintf("served:%d", stats.Retrieval.ObservationsServed))
|
||||
}
|
||||
if stats.Retrieval.ContextInjections > 0 {
|
||||
parts = append(parts, fmt.Sprintf("injected:%d", stats.Retrieval.ContextInjections))
|
||||
}
|
||||
|
||||
// Semantic searches performed
|
||||
if stats.Retrieval.SearchRequests > 0 {
|
||||
parts = append(parts, fmt.Sprintf("searches:%d", stats.Retrieval.SearchRequests))
|
||||
}
|
||||
|
||||
// Project-specific memory count
|
||||
// Add project-specific observation count if available
|
||||
if stats.ProjectObservations > 0 {
|
||||
if useColors {
|
||||
parts = append(parts, fmt.Sprintf("%sproject:%d memories%s", colorYellow, stats.ProjectObservations, reset))
|
||||
} else {
|
||||
parts = append(parts, fmt.Sprintf("project:%d memories", stats.ProjectObservations))
|
||||
}
|
||||
parts = append(parts, fmt.Sprintf("project:%d memories", stats.ProjectObservations))
|
||||
}
|
||||
|
||||
// Processing indicator
|
||||
if stats.IsProcessing || stats.QueueDepth > 0 {
|
||||
if useColors {
|
||||
parts = append(parts, colorYellow+"processing..."+colorReset)
|
||||
} else {
|
||||
parts = append(parts, "processing...")
|
||||
}
|
||||
}
|
||||
|
||||
result := prefix + " " + indicator
|
||||
for i, part := range parts {
|
||||
if i == 0 {
|
||||
result += " " + part
|
||||
} else {
|
||||
result += " | " + part
|
||||
// Join with separators
|
||||
result := parts[0] + " " + parts[1]
|
||||
if len(parts) > 2 {
|
||||
for i := 2; i < len(parts); i++ {
|
||||
if useColors {
|
||||
result += colorGray + " | " + reset + parts[i]
|
||||
} else {
|
||||
result += " | " + parts[i]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
// formatCompact returns a compact status line.
|
||||
// formatCompact returns a compact status line format.
|
||||
func formatCompact(stats *WorkerStats, useColors bool) string {
|
||||
// [m] ● 42/5/3 (28)
|
||||
// [m] ● 42/5/3
|
||||
var prefix, indicator string
|
||||
if useColors {
|
||||
prefix = colorCyan + "[m]" + colorReset
|
||||
@@ -244,31 +227,16 @@ func formatCompact(stats *WorkerStats, useColors bool) string {
|
||||
indicator = "●"
|
||||
}
|
||||
|
||||
result := fmt.Sprintf("%s %s %d/%d/%d",
|
||||
return fmt.Sprintf("%s %s %d/%d/%d",
|
||||
prefix, indicator,
|
||||
stats.Retrieval.ObservationsServed,
|
||||
stats.Retrieval.ContextInjections,
|
||||
stats.Retrieval.SearchRequests,
|
||||
)
|
||||
|
||||
if stats.ProjectObservations > 0 {
|
||||
result += fmt.Sprintf(" (%d)", stats.ProjectObservations)
|
||||
}
|
||||
|
||||
if stats.IsProcessing || stats.QueueDepth > 0 {
|
||||
if useColors {
|
||||
result += " " + colorYellow + "⚙" + colorReset
|
||||
} else {
|
||||
result += " ⚙"
|
||||
}
|
||||
}
|
||||
|
||||
return result
|
||||
stats.Retrieval.SearchRequests)
|
||||
}
|
||||
|
||||
// formatMinimal returns a minimal status line.
|
||||
// formatMinimal returns a minimal status line format.
|
||||
func formatMinimal(stats *WorkerStats, useColors bool) string {
|
||||
// ● 42 obs
|
||||
// ● 28 memories
|
||||
var indicator string
|
||||
if useColors {
|
||||
indicator = colorGreen + "●" + colorReset
|
||||
@@ -276,32 +244,31 @@ func formatMinimal(stats *WorkerStats, useColors bool) string {
|
||||
indicator = "●"
|
||||
}
|
||||
|
||||
result := fmt.Sprintf("%s %d", indicator, stats.Retrieval.ObservationsServed)
|
||||
|
||||
if stats.ProjectObservations > 0 {
|
||||
result += fmt.Sprintf("/%d", stats.ProjectObservations)
|
||||
return fmt.Sprintf("%s %d memories", indicator, stats.ProjectObservations)
|
||||
}
|
||||
|
||||
return result
|
||||
return fmt.Sprintf("%s mnemonic ready", indicator)
|
||||
}
|
||||
|
||||
// formatOffline returns the offline status.
|
||||
// formatOffline returns status for when worker is offline.
|
||||
func formatOffline() string {
|
||||
return formatOfflineColored(true)
|
||||
useColors := os.Getenv("NO_COLOR") == "" && os.Getenv("TERM") != "dumb"
|
||||
return formatOfflineColored(useColors)
|
||||
}
|
||||
|
||||
// formatOfflineColored returns the offline status with optional colors.
|
||||
// formatOfflineColored returns colored offline status.
|
||||
func formatOfflineColored(useColors bool) string {
|
||||
if useColors {
|
||||
return colorCyan + "[mnemonic]" + colorReset + " " + colorGray + "○" + colorReset
|
||||
return colorGray + "[mnemonic]" + colorReset + " " + colorGray + "○" + colorReset + " offline"
|
||||
}
|
||||
return "[mnemonic] ○"
|
||||
return "[mnemonic] ○ offline"
|
||||
}
|
||||
|
||||
// formatStartingColored returns the starting status with optional colors.
|
||||
// formatStartingColored returns colored starting status.
|
||||
func formatStartingColored(useColors bool) string {
|
||||
if useColors {
|
||||
return colorCyan + "[mnemonic]" + colorReset + " " + colorYellow + "◐" + colorReset + " starting"
|
||||
return colorYellow + "[mnemonic]" + colorReset + " " + colorYellow + "○" + colorReset + " starting..."
|
||||
}
|
||||
return "[mnemonic] ◐ starting"
|
||||
return "[mnemonic] ○ starting..."
|
||||
}
|
||||
|
||||
+19
-2
@@ -75,13 +75,16 @@ func parseTranscript(path string) (lastUser, lastAssistant string) {
|
||||
continue
|
||||
}
|
||||
|
||||
if msg.Type == "message" {
|
||||
// Transcript entries have type: "user" or "assistant" (not "message")
|
||||
// Check if this is a user/assistant message with content
|
||||
if msg.Type == "user" || msg.Type == "assistant" {
|
||||
text := extractTextContent(msg.Message.Content)
|
||||
if text == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
switch msg.Message.Role {
|
||||
// Use the outer type field for role (message.role may differ)
|
||||
switch msg.Type {
|
||||
case "user":
|
||||
lastUser = text
|
||||
case "assistant":
|
||||
@@ -98,6 +101,9 @@ func main() {
|
||||
}
|
||||
|
||||
func handleStop(ctx *hooks.HookContext, input *Input) (string, error) {
|
||||
// Debug: dump raw input
|
||||
fmt.Fprintf(os.Stderr, "[stop] Raw input: %s\n", string(ctx.RawInput))
|
||||
|
||||
// Find session
|
||||
result, err := hooks.GET(ctx.Port, fmt.Sprintf("/api/sessions?claudeSessionId=%s", ctx.SessionID))
|
||||
if err != nil || result == nil {
|
||||
@@ -116,6 +122,17 @@ func handleStop(ctx *hooks.HookContext, input *Input) (string, error) {
|
||||
lastUser, lastAssistant = parseTranscript(input.TranscriptPath)
|
||||
}
|
||||
|
||||
// Debug: log what we extracted
|
||||
fmt.Fprintf(os.Stderr, "[stop] Transcript path: %s\n", input.TranscriptPath)
|
||||
fmt.Fprintf(os.Stderr, "[stop] Last user message length: %d\n", len(lastUser))
|
||||
fmt.Fprintf(os.Stderr, "[stop] Last assistant message length: %d\n", len(lastAssistant))
|
||||
if len(lastAssistant) > 0 {
|
||||
preview := lastAssistant
|
||||
if len(preview) > 300 {
|
||||
preview = preview[:300] + "..."
|
||||
}
|
||||
fmt.Fprintf(os.Stderr, "[stop] Last assistant preview: %s\n", preview)
|
||||
}
|
||||
fmt.Fprintf(os.Stderr, "[stop] Requesting summary for session %d (transcript: %v)\n", int64(sessionID), input.TranscriptPath != "")
|
||||
|
||||
// Request summary with message context from transcript
|
||||
|
||||
@@ -2,9 +2,7 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/url"
|
||||
"os"
|
||||
|
||||
@@ -13,60 +11,28 @@ import (
|
||||
|
||||
// Input is the hook input from Claude Code.
|
||||
type Input struct {
|
||||
SessionID string `json:"session_id"`
|
||||
CWD string `json:"cwd"`
|
||||
PermissionMode string `json:"permission_mode"`
|
||||
HookEventName string `json:"hook_event_name"`
|
||||
Prompt string `json:"prompt"`
|
||||
hooks.BaseInput
|
||||
Prompt string `json:"prompt"`
|
||||
}
|
||||
|
||||
func main() {
|
||||
// Skip if this is an internal call (from SDK processor)
|
||||
if os.Getenv("CLAUDE_MNEMONIC_INTERNAL") == "1" {
|
||||
hooks.WriteResponse("UserPromptSubmit", true)
|
||||
return
|
||||
}
|
||||
|
||||
// Read input from stdin
|
||||
inputData, err := io.ReadAll(os.Stdin)
|
||||
if err != nil {
|
||||
hooks.WriteError("UserPromptSubmit", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
var input Input
|
||||
if err := json.Unmarshal(inputData, &input); err != nil {
|
||||
hooks.WriteError("UserPromptSubmit", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
// Ensure worker is running
|
||||
port, err := hooks.EnsureWorkerRunning()
|
||||
if err != nil {
|
||||
hooks.WriteError("UserPromptSubmit", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
// Generate unique project ID from CWD
|
||||
project := hooks.ProjectIDWithName(input.CWD)
|
||||
hooks.RunHook("UserPromptSubmit", handleUserPrompt)
|
||||
}
|
||||
|
||||
func handleUserPrompt(ctx *hooks.HookContext, input *Input) (string, error) {
|
||||
// Search for relevant observations based on the prompt
|
||||
searchURL := fmt.Sprintf("/api/context/search?project=%s&query=%s&cwd=%s",
|
||||
url.QueryEscape(project),
|
||||
url.QueryEscape(ctx.Project),
|
||||
url.QueryEscape(input.Prompt),
|
||||
url.QueryEscape(input.CWD))
|
||||
url.QueryEscape(ctx.CWD))
|
||||
|
||||
var contextToInject string
|
||||
var observationCount int
|
||||
|
||||
searchResult, _ := hooks.GET(port, searchURL)
|
||||
searchResult, _ := hooks.GET(ctx.Port, searchURL)
|
||||
if observations, ok := searchResult["observations"].([]interface{}); ok && len(observations) > 0 {
|
||||
// Limit to top 5 most relevant observations
|
||||
maxObs := 5
|
||||
if len(observations) < maxObs {
|
||||
maxObs = len(observations)
|
||||
}
|
||||
observations = observations[:maxObs]
|
||||
// Results are already filtered by relevance threshold and capped by max_results
|
||||
// from the server-side config (ContextRelevanceThreshold, ContextMaxPromptResults)
|
||||
observationCount = len(observations)
|
||||
|
||||
// Build context from search results
|
||||
@@ -108,27 +74,24 @@ func main() {
|
||||
}
|
||||
|
||||
contextBuilder += "</relevant-memory>\n"
|
||||
|
||||
contextToInject = contextBuilder
|
||||
}
|
||||
|
||||
// Initialize session with matched observations count
|
||||
result, err := hooks.POST(port, "/api/sessions/init", map[string]interface{}{
|
||||
"claudeSessionId": input.SessionID,
|
||||
"project": project,
|
||||
result, err := hooks.POST(ctx.Port, "/api/sessions/init", map[string]interface{}{
|
||||
"claudeSessionId": ctx.SessionID,
|
||||
"project": ctx.Project,
|
||||
"prompt": input.Prompt,
|
||||
"matchedObservations": observationCount,
|
||||
})
|
||||
if err != nil {
|
||||
hooks.WriteError("UserPromptSubmit", err)
|
||||
os.Exit(1)
|
||||
return "", err
|
||||
}
|
||||
|
||||
// Check if skipped due to privacy
|
||||
if skipped, ok := result["skipped"].(bool); ok && skipped {
|
||||
fmt.Fprintf(os.Stderr, "[user-prompt] Session skipped (private)\n")
|
||||
hooks.WriteResponse("UserPromptSubmit", true)
|
||||
return
|
||||
return "", nil
|
||||
}
|
||||
|
||||
sessionID := int64(result["sessionDbId"].(float64))
|
||||
@@ -137,30 +100,20 @@ func main() {
|
||||
fmt.Fprintf(os.Stderr, "[user-prompt] Session %d, prompt #%d\n", sessionID, promptNumber)
|
||||
|
||||
// Start SDK agent
|
||||
_, err = hooks.POST(port, fmt.Sprintf("/sessions/%d/init", sessionID), map[string]interface{}{
|
||||
_, err = hooks.POST(ctx.Port, fmt.Sprintf("/sessions/%d/init", sessionID), map[string]interface{}{
|
||||
"userPrompt": input.Prompt,
|
||||
"promptNumber": promptNumber,
|
||||
})
|
||||
if err != nil {
|
||||
hooks.WriteError("UserPromptSubmit", err)
|
||||
os.Exit(1)
|
||||
return "", err
|
||||
}
|
||||
|
||||
// Output results - stdout with exit 0 adds context to Claude's prompt
|
||||
// Return context if we found relevant observations
|
||||
if observationCount > 0 {
|
||||
// Show match count to user via stderr
|
||||
fmt.Fprintf(os.Stderr, "[claude-mnemonic] Found %d relevant memories for this prompt\n", observationCount)
|
||||
// Output context as JSON with additionalContext field
|
||||
response := map[string]interface{}{
|
||||
"continue": true,
|
||||
"hookSpecificOutput": map[string]interface{}{
|
||||
"hookEventName": "UserPromptSubmit",
|
||||
"additionalContext": contextToInject,
|
||||
},
|
||||
}
|
||||
_ = json.NewEncoder(os.Stdout).Encode(response)
|
||||
os.Exit(0)
|
||||
} else {
|
||||
hooks.WriteResponse("UserPromptSubmit", true)
|
||||
return contextToInject, nil
|
||||
}
|
||||
|
||||
return "", nil
|
||||
}
|
||||
|
||||
+34
-12
@@ -10,12 +10,14 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/lukaszraczylo/claude-mnemonic/internal/config"
|
||||
"github.com/lukaszraczylo/claude-mnemonic/internal/db/sqlite"
|
||||
"github.com/lukaszraczylo/claude-mnemonic/internal/db/gorm"
|
||||
"github.com/lukaszraczylo/claude-mnemonic/internal/embedding"
|
||||
"github.com/lukaszraczylo/claude-mnemonic/internal/mcp"
|
||||
"github.com/lukaszraczylo/claude-mnemonic/internal/scoring"
|
||||
"github.com/lukaszraczylo/claude-mnemonic/internal/search"
|
||||
"github.com/lukaszraczylo/claude-mnemonic/internal/vector/sqlitevec"
|
||||
"github.com/lukaszraczylo/claude-mnemonic/internal/watcher"
|
||||
"github.com/lukaszraczylo/claude-mnemonic/pkg/models"
|
||||
"github.com/rs/zerolog"
|
||||
"github.com/rs/zerolog/log"
|
||||
)
|
||||
@@ -71,22 +73,25 @@ func main() {
|
||||
cancel()
|
||||
}()
|
||||
|
||||
// Initialize SQLite store (migrations run automatically)
|
||||
storeCfg := sqlite.StoreConfig{
|
||||
// Initialize database store (migrations run automatically)
|
||||
storeCfg := gorm.Config{
|
||||
Path: dbPath,
|
||||
MaxConns: cfg.MaxConns,
|
||||
WALMode: true,
|
||||
// WALMode is enabled automatically by GORM
|
||||
}
|
||||
store, err := sqlite.NewStore(storeCfg)
|
||||
store, err := gorm.NewStore(storeCfg)
|
||||
if err != nil {
|
||||
log.Fatal().Err(err).Msg("Failed to initialize SQLite store")
|
||||
log.Fatal().Err(err).Msg("Failed to initialize database store")
|
||||
}
|
||||
defer store.Close()
|
||||
|
||||
// Initialize stores
|
||||
observationStore := sqlite.NewObservationStore(store)
|
||||
summaryStore := sqlite.NewSummaryStore(store)
|
||||
promptStore := sqlite.NewPromptStore(store)
|
||||
observationStore := gorm.NewObservationStore(store, nil, nil, nil)
|
||||
summaryStore := gorm.NewSummaryStore(store)
|
||||
promptStore := gorm.NewPromptStore(store, nil)
|
||||
patternStore := gorm.NewPatternStore(store)
|
||||
relationStore := gorm.NewRelationStore(store)
|
||||
sessionStore := gorm.NewSessionStore(store)
|
||||
|
||||
// Initialize embedding service and vector client
|
||||
var vectorClient *sqlitevec.Client
|
||||
@@ -95,7 +100,7 @@ func main() {
|
||||
log.Warn().Err(err).Msg("Embedding service unavailable, vector search disabled")
|
||||
} else {
|
||||
defer embedSvc.Close()
|
||||
vectorClient, err = sqlitevec.NewClient(sqlitevec.Config{DB: store.DB()}, embedSvc)
|
||||
vectorClient, err = sqlitevec.NewClient(sqlitevec.Config{DB: store.GetRawDB()}, embedSvc)
|
||||
if err != nil {
|
||||
log.Warn().Err(err).Msg("Vector client unavailable, vector search disabled")
|
||||
} else {
|
||||
@@ -103,14 +108,31 @@ func main() {
|
||||
}
|
||||
}
|
||||
|
||||
// Initialize scoring components
|
||||
scoreConfig := models.DefaultScoringConfig()
|
||||
scoreCalculator := scoring.NewCalculator(scoreConfig)
|
||||
recalculator := scoring.NewRecalculator(observationStore, scoreCalculator, log.Logger)
|
||||
go recalculator.Start(ctx)
|
||||
defer recalculator.Stop()
|
||||
|
||||
// Initialize search manager
|
||||
searchMgr := search.NewManager(observationStore, summaryStore, promptStore, vectorClient)
|
||||
|
||||
// Start file watchers
|
||||
startWatchers(ctx, dbPath)
|
||||
|
||||
// Create and run MCP server
|
||||
server := mcp.NewServer(searchMgr, Version)
|
||||
// Create and run MCP server with all dependencies
|
||||
server := mcp.NewServer(
|
||||
searchMgr,
|
||||
Version,
|
||||
observationStore,
|
||||
patternStore,
|
||||
relationStore,
|
||||
sessionStore,
|
||||
vectorClient,
|
||||
scoreCalculator,
|
||||
recalculator,
|
||||
)
|
||||
log.Info().Str("project", *project).Str("version", Version).Msg("Starting MCP server")
|
||||
|
||||
if err := server.Run(ctx); err != nil {
|
||||
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 252 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 123 KiB |
+37
-7
@@ -29,6 +29,21 @@
|
||||
subtitle="Stop re-explaining your codebase. Claude Mnemonic captures bug fixes, architecture decisions, and coding patterns - then brings them back exactly when you need them."
|
||||
/>
|
||||
|
||||
<!-- Dashboard Preview -->
|
||||
<section class="py-12 lg:py-16 px-4 sm:px-6 relative">
|
||||
<div class="max-w-6xl mx-auto">
|
||||
<div class="relative rounded-xl overflow-hidden border border-slate-700/50 shadow-2xl shadow-amber-500/5">
|
||||
<div class="absolute inset-0 bg-gradient-to-t from-slate-950 via-transparent to-transparent pointer-events-none z-10"></div>
|
||||
<img
|
||||
src="/claude-mnemonic.jpg"
|
||||
alt="Claude Mnemonic Dashboard"
|
||||
class="w-full h-auto"
|
||||
/>
|
||||
</div>
|
||||
<p class="text-center text-slate-500 text-sm mt-4">The dashboard at localhost:37777 - browse, search, and manage your memories</p>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<!-- Problem Section -->
|
||||
<section class="py-20 lg:py-28 px-4 sm:px-6 relative">
|
||||
<div class="max-w-6xl mx-auto grid lg:grid-cols-2 gap-8 lg:gap-16 items-center">
|
||||
@@ -85,6 +100,21 @@
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<!-- Knowledge Graph Preview -->
|
||||
<section class="py-16 lg:py-20 px-4 sm:px-6 relative">
|
||||
<div class="max-w-5xl mx-auto">
|
||||
<SectionHeader title="See how knowledge connects" subtitle="The knowledge graph reveals relationships between your memories automatically" />
|
||||
<div class="relative rounded-xl overflow-hidden border border-slate-700/50 shadow-2xl shadow-purple-500/5">
|
||||
<img
|
||||
src="/observation-relation-graph.jpg"
|
||||
alt="Knowledge Graph Visualization"
|
||||
class="w-full h-auto"
|
||||
/>
|
||||
</div>
|
||||
<p class="text-center text-slate-500 text-sm mt-4">Click any observation to explore its relationships - see what causes what, what fixes what, and how concepts evolve</p>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<!-- Before/After Section -->
|
||||
<section class="py-20 lg:py-28 px-4 sm:px-6">
|
||||
<div class="max-w-6xl mx-auto">
|
||||
@@ -288,8 +318,8 @@
|
||||
<p class="text-slate-400 text-xs sm:text-sm">Embedded vector database. No external services required.</p>
|
||||
</div>
|
||||
<div class="glass rounded-2xl p-6 sm:p-8 hover:border-amber-500/30 transition-colors">
|
||||
<div class="text-3xl sm:text-4xl font-bold text-amber-500 mb-2">MiniLM</div>
|
||||
<p class="text-slate-400 text-xs sm:text-sm">Local embeddings via ONNX. "Fix auth" finds "JWT issue" automatically.</p>
|
||||
<div class="text-3xl sm:text-4xl font-bold text-amber-500 mb-2">BGE</div>
|
||||
<p class="text-slate-400 text-xs sm:text-sm">Two-stage retrieval: bi-encoder embeddings + cross-encoder reranking for high accuracy.</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
@@ -386,10 +416,10 @@ const activeTab = ref('macos')
|
||||
|
||||
const features = [
|
||||
{ icon: 'fas fa-brain', title: 'Learns as you work', description: 'Every bug fix, every architecture decision, every "aha moment" - captured automatically without breaking your flow.' },
|
||||
{ icon: 'fas fa-search', title: 'Two-stage retrieval', description: 'Cross-encoder reranking delivers highly relevant results. Finds what you need even with vague queries like "that auth thing".' },
|
||||
{ icon: 'fas fa-project-diagram', title: 'Knowledge graph', description: 'Automatically discovers relationships between memories. See how concepts connect in the visual graph dashboard.' },
|
||||
{ icon: 'fas fa-folder-tree', title: 'Project-aware context', description: 'Your React knowledge stays in React projects. Your Go patterns stay in Go projects. No context pollution.' },
|
||||
{ icon: 'fas fa-globe', title: 'Shared best practices', description: 'Security patterns, performance tips, and universal learnings automatically available across all your projects.' },
|
||||
{ icon: 'fas fa-search', title: 'Finds what matters', description: 'Semantic search finds relevant memories even when you don\'t remember the exact words. "That auth thing" just works.' },
|
||||
{ icon: 'fas fa-chart-line', title: 'Live statusline', description: 'Real-time metrics right in Claude Code: memories served, searches performed, and project memory count at a glance.' },
|
||||
{ icon: 'fas fa-chart-line', title: 'Smart scoring', description: 'Importance decay, pattern detection, and conflict resolution ensure the most valuable memories surface first.' },
|
||||
{ icon: 'fas fa-lock', title: '100% private', description: 'Your code context never leaves your machine. No telemetry. No cloud sync. Your memories are yours.' },
|
||||
]
|
||||
|
||||
@@ -415,8 +445,8 @@ const installCommands = {
|
||||
const configOptions = [
|
||||
{ name: 'CLAUDE_MNEMONIC_WORKER_PORT', description: 'HTTP port for the worker service (default: 37777)', icon: 'fas fa-network-wired' },
|
||||
{ name: 'CLAUDE_MNEMONIC_CONTEXT_OBSERVATIONS', description: 'Maximum observations injected per session (default: 100)', icon: 'fas fa-layer-group' },
|
||||
{ name: 'CLAUDE_MNEMONIC_CONTEXT_FULL_COUNT', description: 'Observations with full narrative detail, rest are condensed (default: 25)', icon: 'fas fa-expand' },
|
||||
{ name: 'CLAUDE_MNEMONIC_MODEL', description: 'Model for processing observations (default: haiku)', icon: 'fas fa-microchip' },
|
||||
{ name: 'CLAUDE_MNEMONIC_RERANKING_ENABLED', description: 'Enable cross-encoder reranking for improved search relevance (default: true)', icon: 'fas fa-sort-amount-down' },
|
||||
{ name: 'CLAUDE_MNEMONIC_CONTEXT_RELEVANCE_THRESHOLD', description: 'Minimum similarity score for inclusion, 0.0-1.0 (default: 0.3)', icon: 'fas fa-filter' },
|
||||
]
|
||||
|
||||
const requiredDeps = [
|
||||
|
||||
@@ -9,16 +9,21 @@ require (
|
||||
github.com/fsnotify/fsnotify v1.9.0
|
||||
github.com/go-chi/chi/v5 v5.2.3
|
||||
github.com/goccy/go-json v0.10.5
|
||||
github.com/mattn/go-sqlite3 v1.14.32
|
||||
github.com/mattn/go-sqlite3 v1.14.33
|
||||
github.com/rs/zerolog v1.34.0
|
||||
github.com/stretchr/testify v1.11.1
|
||||
github.com/sugarme/tokenizer v0.3.0
|
||||
github.com/yalue/onnxruntime_go v1.21.0
|
||||
github.com/yalue/onnxruntime_go v1.25.0
|
||||
gorm.io/driver/sqlite v1.5.7
|
||||
gorm.io/gorm v1.26.1
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/davecgh/go-spew v1.1.1 // indirect
|
||||
github.com/emirpasic/gods v1.18.1 // indirect
|
||||
github.com/go-gormigrate/gormigrate/v2 v2.1.5 // indirect
|
||||
github.com/jinzhu/inflection v1.0.0 // indirect
|
||||
github.com/jinzhu/now v1.1.5 // indirect
|
||||
github.com/mattn/go-colorable v0.1.14 // indirect
|
||||
github.com/mattn/go-isatty v0.0.20 // indirect
|
||||
github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect
|
||||
@@ -28,6 +33,6 @@ require (
|
||||
github.com/schollz/progressbar/v2 v2.15.0 // indirect
|
||||
github.com/sugarme/regexpset v0.0.0-20200920021344-4d4ec8eaf93c // indirect
|
||||
golang.org/x/sys v0.39.0 // indirect
|
||||
golang.org/x/text v0.25.0 // indirect
|
||||
golang.org/x/text v0.32.0 // indirect
|
||||
gopkg.in/yaml.v3 v3.0.1 // indirect
|
||||
)
|
||||
|
||||
@@ -12,9 +12,15 @@ github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S
|
||||
github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0=
|
||||
github.com/go-chi/chi/v5 v5.2.3 h1:WQIt9uxdsAbgIYgid+BpYc+liqQZGMHRaUwp0JUcvdE=
|
||||
github.com/go-chi/chi/v5 v5.2.3/go.mod h1:L2yAIGWB3H+phAw1NxKwWM+7eUH/lU8pOMm5hHcoops=
|
||||
github.com/go-gormigrate/gormigrate/v2 v2.1.5 h1:1OyorA5LtdQw12cyJDEHuTrEV3GiXiIhS4/QTTa/SM8=
|
||||
github.com/go-gormigrate/gormigrate/v2 v2.1.5/go.mod h1:mj9ekk/7CPF3VjopaFvWKN2v7fN3D9d3eEOAXRhi/+M=
|
||||
github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4=
|
||||
github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M=
|
||||
github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
|
||||
github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E=
|
||||
github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
|
||||
github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ=
|
||||
github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
|
||||
github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
|
||||
github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE=
|
||||
github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8=
|
||||
@@ -22,8 +28,8 @@ github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/
|
||||
github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
|
||||
github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
|
||||
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
|
||||
github.com/mattn/go-sqlite3 v1.14.32 h1:JD12Ag3oLy1zQA+BNn74xRgaBbdhbNIDYvQUEuuErjs=
|
||||
github.com/mattn/go-sqlite3 v1.14.32/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
|
||||
github.com/mattn/go-sqlite3 v1.14.33 h1:A5blZ5ulQo2AtayQ9/limgHEkFreKj1Dv226a1K73s0=
|
||||
github.com/mattn/go-sqlite3 v1.14.33/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
|
||||
github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ=
|
||||
github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw=
|
||||
github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc=
|
||||
@@ -44,16 +50,22 @@ github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu
|
||||
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
|
||||
github.com/sugarme/regexpset v0.0.0-20200920021344-4d4ec8eaf93c h1:pwb4kNSHb4K89ymCaN+5lPH/MwnfSVg4rzGDh4d+iy4=
|
||||
github.com/sugarme/regexpset v0.0.0-20200920021344-4d4ec8eaf93c/go.mod h1:2gwkXLWbDGUQWeL3RtpCmcY4mzCtU13kb9UsAg9xMaw=
|
||||
github.com/yalue/onnxruntime_go v1.21.0 h1:DdtvfY7OP5gR8mwPDqAOAQckf+KcI30hPNJL8hQaYWI=
|
||||
github.com/yalue/onnxruntime_go v1.21.0/go.mod h1:b4X26A8pekNb1ACJ58wAXgNKeUCGEAQ9dmACut9Sm/4=
|
||||
github.com/yalue/onnxruntime_go v1.25.0 h1:nlhVau1BpLZ/BYr+WpPZCJRD/WES0qo6dK7aKyyAs3g=
|
||||
github.com/yalue/onnxruntime_go v1.25.0/go.mod h1:b4X26A8pekNb1ACJ58wAXgNKeUCGEAQ9dmACut9Sm/4=
|
||||
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.39.0 h1:CvCKL8MeisomCi6qNZ+wbb0DN9E5AATixKsvNtMoMFk=
|
||||
golang.org/x/sys v0.39.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
|
||||
golang.org/x/text v0.25.0 h1:qVyWApTSYLk/drJRO5mDlNYskwQznZmkpV2c8q9zls4=
|
||||
golang.org/x/text v0.25.0/go.mod h1:WEdwpYrmk1qmdHvhkSTNPm3app7v4rsT8F2UD6+VHIA=
|
||||
golang.org/x/text v0.32.0 h1:ZD01bjUt1FQ9WJ0ClOL5vxgxOI/sVCNgX1YtKwcY0mU=
|
||||
golang.org/x/text v0.32.0/go.mod h1:o/rUWzghvpD5TXrTIBuJU77MTaN0ljMWE47kxGJQ7jY=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gorm.io/driver/sqlite v1.5.7 h1:8NvsrhP0ifM7LX9G4zPB97NwovUakUxc+2V2uuf3Z1I=
|
||||
gorm.io/driver/sqlite v1.5.7/go.mod h1:U+J8craQU6Fzkcvu8oLeAQmi50TkwPEhHDEjQZXDah4=
|
||||
gorm.io/gorm v1.25.12 h1:I0u8i2hWQItBq1WfE0o2+WuL9+8L21K9e2HHSTE/0f8=
|
||||
gorm.io/gorm v1.25.12/go.mod h1:xh7N7RHfYlNc5EmcI/El95gXusucDrQnHXe0+CgWcLQ=
|
||||
gorm.io/gorm v1.26.1 h1:ghB2gUI9FkS46luZtn6DLZ0f6ooBJ5IbVej2ENFDjRw=
|
||||
gorm.io/gorm v1.26.1/go.mod h1:8Z33v652h4//uMA76KjeDH8mJXPm1QNCYrMeatR0DOE=
|
||||
|
||||
+74
-22
@@ -48,16 +48,29 @@ type Config struct {
|
||||
Model string `json:"model"`
|
||||
ClaudeCodePath string `json:"claude_code_path"`
|
||||
|
||||
// Embedding settings
|
||||
EmbeddingModel string `json:"embedding_model"` // e.g., "bge-v1.5"
|
||||
|
||||
// Reranking settings (cross-encoder)
|
||||
RerankingEnabled bool `json:"reranking_enabled"` // Enable cross-encoder reranking
|
||||
RerankingCandidates int `json:"reranking_candidates"` // Number of candidates to retrieve before reranking (default 100)
|
||||
RerankingResults int `json:"reranking_results"` // Number of results to return after reranking (default 10)
|
||||
RerankingAlpha float64 `json:"reranking_alpha"` // Weight for combining scores: alpha*rerank + (1-alpha)*original (default 0.7)
|
||||
RerankingMinImprovement float64 `json:"reranking_min_improvement"` // Minimum rank improvement to trigger reranking (default 0, always rerank)
|
||||
RerankingPureMode bool `json:"reranking_pure_mode"` // Use pure cross-encoder scores without combining with bi-encoder (default false)
|
||||
|
||||
// Context injection settings
|
||||
ContextObservations int `json:"context_observations"`
|
||||
ContextFullCount int `json:"context_full_count"`
|
||||
ContextSessionCount int `json:"context_session_count"`
|
||||
ContextShowReadTokens bool `json:"context_show_read_tokens"`
|
||||
ContextShowWorkTokens bool `json:"context_show_work_tokens"`
|
||||
ContextFullField string `json:"context_full_field"`
|
||||
ContextShowLastSummary bool `json:"context_show_last_summary"`
|
||||
ContextObsTypes []string `json:"context_obs_types"`
|
||||
ContextObsConcepts []string `json:"context_obs_concepts"`
|
||||
ContextObservations int `json:"context_observations"`
|
||||
ContextFullCount int `json:"context_full_count"`
|
||||
ContextSessionCount int `json:"context_session_count"`
|
||||
ContextShowReadTokens bool `json:"context_show_read_tokens"`
|
||||
ContextShowWorkTokens bool `json:"context_show_work_tokens"`
|
||||
ContextFullField string `json:"context_full_field"`
|
||||
ContextShowLastSummary bool `json:"context_show_last_summary"`
|
||||
ContextObsTypes []string `json:"context_obs_types"`
|
||||
ContextObsConcepts []string `json:"context_obs_concepts"`
|
||||
ContextRelevanceThreshold float64 `json:"context_relevance_threshold"` // 0.0-1.0, minimum similarity for inclusion
|
||||
ContextMaxPromptResults int `json:"context_max_prompt_results"` // Max results per prompt (0 = threshold only)
|
||||
}
|
||||
|
||||
var (
|
||||
@@ -119,22 +132,33 @@ func EnsureAll() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// DefaultEmbeddingModel is the default embedding model to use.
|
||||
const DefaultEmbeddingModel = "bge-v1.5"
|
||||
|
||||
// Default returns a Config with default values.
|
||||
func Default() *Config {
|
||||
return &Config{
|
||||
WorkerPort: DefaultWorkerPort,
|
||||
DBPath: DBPath(),
|
||||
MaxConns: 4,
|
||||
Model: DefaultModel,
|
||||
ContextObservations: 100,
|
||||
ContextFullCount: 25,
|
||||
ContextSessionCount: 10,
|
||||
ContextShowReadTokens: true,
|
||||
ContextShowWorkTokens: true,
|
||||
ContextFullField: "narrative",
|
||||
ContextShowLastSummary: true,
|
||||
ContextObsTypes: DefaultObservationTypes,
|
||||
ContextObsConcepts: DefaultObservationConcepts,
|
||||
WorkerPort: DefaultWorkerPort,
|
||||
DBPath: DBPath(),
|
||||
MaxConns: 4,
|
||||
Model: DefaultModel,
|
||||
EmbeddingModel: DefaultEmbeddingModel,
|
||||
RerankingEnabled: true, // Enable by default for improved relevance
|
||||
RerankingCandidates: 100, // Retrieve top 100 candidates
|
||||
RerankingResults: 10, // Return top 10 after reranking
|
||||
RerankingAlpha: 0.7, // Favor cross-encoder score
|
||||
RerankingMinImprovement: 0, // Always apply reranking
|
||||
ContextObservations: 100,
|
||||
ContextFullCount: 25,
|
||||
ContextSessionCount: 10,
|
||||
ContextShowReadTokens: true,
|
||||
ContextShowWorkTokens: true,
|
||||
ContextFullField: "narrative",
|
||||
ContextShowLastSummary: true,
|
||||
ContextObsTypes: DefaultObservationTypes,
|
||||
ContextObsConcepts: DefaultObservationConcepts,
|
||||
ContextRelevanceThreshold: 0.3, // Minimum 30% similarity to include
|
||||
ContextMaxPromptResults: 10, // Cap at 10 results max (0 = no cap, threshold only)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -166,6 +190,28 @@ func Load() (*Config, error) {
|
||||
if v, ok := settings["CLAUDE_CODE_PATH"].(string); ok {
|
||||
cfg.ClaudeCodePath = v
|
||||
}
|
||||
if v, ok := settings["CLAUDE_MNEMONIC_EMBEDDING_MODEL"].(string); ok && v != "" {
|
||||
cfg.EmbeddingModel = v
|
||||
}
|
||||
// Reranking settings
|
||||
if v, ok := settings["CLAUDE_MNEMONIC_RERANKING_ENABLED"].(bool); ok {
|
||||
cfg.RerankingEnabled = v
|
||||
}
|
||||
if v, ok := settings["CLAUDE_MNEMONIC_RERANKING_CANDIDATES"].(float64); ok && v > 0 {
|
||||
cfg.RerankingCandidates = int(v)
|
||||
}
|
||||
if v, ok := settings["CLAUDE_MNEMONIC_RERANKING_RESULTS"].(float64); ok && v > 0 {
|
||||
cfg.RerankingResults = int(v)
|
||||
}
|
||||
if v, ok := settings["CLAUDE_MNEMONIC_RERANKING_ALPHA"].(float64); ok && v >= 0 && v <= 1 {
|
||||
cfg.RerankingAlpha = v
|
||||
}
|
||||
if v, ok := settings["CLAUDE_MNEMONIC_RERANKING_MIN_IMPROVEMENT"].(float64); ok && v >= 0 {
|
||||
cfg.RerankingMinImprovement = v
|
||||
}
|
||||
if v, ok := settings["CLAUDE_MNEMONIC_RERANKING_PURE_MODE"].(bool); ok {
|
||||
cfg.RerankingPureMode = v
|
||||
}
|
||||
if v, ok := settings["CLAUDE_MNEMONIC_CONTEXT_OBSERVATIONS"].(float64); ok {
|
||||
cfg.ContextObservations = int(v)
|
||||
}
|
||||
@@ -181,6 +227,12 @@ func Load() (*Config, error) {
|
||||
if v, ok := settings["CLAUDE_MNEMONIC_CONTEXT_OBS_CONCEPTS"].(string); ok && v != "" {
|
||||
cfg.ContextObsConcepts = splitTrim(v)
|
||||
}
|
||||
if v, ok := settings["CLAUDE_MNEMONIC_CONTEXT_RELEVANCE_THRESHOLD"].(float64); ok && v >= 0 && v <= 1 {
|
||||
cfg.ContextRelevanceThreshold = v
|
||||
}
|
||||
if v, ok := settings["CLAUDE_MNEMONIC_CONTEXT_MAX_PROMPT_RESULTS"].(float64); ok && v >= 0 {
|
||||
cfg.ContextMaxPromptResults = int(v)
|
||||
}
|
||||
|
||||
return cfg, nil
|
||||
}
|
||||
|
||||
@@ -0,0 +1,436 @@
|
||||
// Package config provides configuration management for claude-mnemonic.
|
||||
package config
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"github.com/stretchr/testify/suite"
|
||||
)
|
||||
|
||||
// ConfigSuite is a test suite for config operations.
|
||||
type ConfigSuite struct {
|
||||
suite.Suite
|
||||
tempDir string
|
||||
origHomeDir string
|
||||
}
|
||||
|
||||
func (s *ConfigSuite) SetupTest() {
|
||||
var err error
|
||||
s.tempDir, err = os.MkdirTemp("", "config-test-*")
|
||||
s.Require().NoError(err)
|
||||
|
||||
// Save and override HOME
|
||||
s.origHomeDir = os.Getenv("HOME")
|
||||
os.Setenv("HOME", s.tempDir)
|
||||
}
|
||||
|
||||
func (s *ConfigSuite) TearDownTest() {
|
||||
os.Setenv("HOME", s.origHomeDir)
|
||||
os.RemoveAll(s.tempDir)
|
||||
}
|
||||
|
||||
func TestConfigSuite(t *testing.T) {
|
||||
suite.Run(t, new(ConfigSuite))
|
||||
}
|
||||
|
||||
// TestDefault tests default configuration values.
|
||||
func (s *ConfigSuite) TestDefault() {
|
||||
cfg := Default()
|
||||
|
||||
s.Equal(DefaultWorkerPort, cfg.WorkerPort)
|
||||
s.Equal(DefaultModel, cfg.Model)
|
||||
s.Equal(4, cfg.MaxConns)
|
||||
s.Equal(100, cfg.ContextObservations)
|
||||
s.Equal(25, cfg.ContextFullCount)
|
||||
s.Equal(10, cfg.ContextSessionCount)
|
||||
s.True(cfg.ContextShowReadTokens)
|
||||
s.True(cfg.ContextShowWorkTokens)
|
||||
s.Equal("narrative", cfg.ContextFullField)
|
||||
s.True(cfg.ContextShowLastSummary)
|
||||
s.Equal(DefaultObservationTypes, cfg.ContextObsTypes)
|
||||
s.Equal(DefaultObservationConcepts, cfg.ContextObsConcepts)
|
||||
}
|
||||
|
||||
// TestDataDir tests data directory path.
|
||||
func (s *ConfigSuite) TestDataDir() {
|
||||
dir := DataDir()
|
||||
s.Contains(dir, ".claude-mnemonic")
|
||||
}
|
||||
|
||||
// TestDBPath tests database path.
|
||||
func (s *ConfigSuite) TestDBPath() {
|
||||
path := DBPath()
|
||||
s.Contains(path, "claude-mnemonic.db")
|
||||
}
|
||||
|
||||
// TestSettingsPath tests settings file path.
|
||||
func (s *ConfigSuite) TestSettingsPath() {
|
||||
path := SettingsPath()
|
||||
s.Contains(path, "settings.json")
|
||||
}
|
||||
|
||||
// TestEnsureDataDir tests data directory creation.
|
||||
func (s *ConfigSuite) TestEnsureDataDir() {
|
||||
err := EnsureDataDir()
|
||||
s.NoError(err)
|
||||
|
||||
dir := DataDir()
|
||||
info, err := os.Stat(dir)
|
||||
s.NoError(err)
|
||||
s.True(info.IsDir())
|
||||
}
|
||||
|
||||
// TestEnsureSettings tests settings file creation.
|
||||
func (s *ConfigSuite) TestEnsureSettings() {
|
||||
// First ensure data dir exists
|
||||
err := EnsureDataDir()
|
||||
s.NoError(err)
|
||||
|
||||
// Ensure settings creates default file
|
||||
err = EnsureSettings()
|
||||
s.NoError(err)
|
||||
|
||||
path := SettingsPath()
|
||||
info, err := os.Stat(path)
|
||||
s.NoError(err)
|
||||
s.False(info.IsDir())
|
||||
|
||||
// Second call should not error (file exists)
|
||||
err = EnsureSettings()
|
||||
s.NoError(err)
|
||||
}
|
||||
|
||||
// TestEnsureAll tests full initialization.
|
||||
func (s *ConfigSuite) TestEnsureAll() {
|
||||
err := EnsureAll()
|
||||
s.NoError(err)
|
||||
|
||||
// Verify dir and settings exist
|
||||
_, err = os.Stat(DataDir())
|
||||
s.NoError(err)
|
||||
_, err = os.Stat(SettingsPath())
|
||||
s.NoError(err)
|
||||
}
|
||||
|
||||
// TestLoad_TableDriven tests configuration loading with various scenarios.
|
||||
func (s *ConfigSuite) TestLoad_TableDriven() {
|
||||
tests := []struct {
|
||||
name string
|
||||
settingsJSON string
|
||||
expectedPort int
|
||||
expectedModel string
|
||||
expectedObsObs int
|
||||
}{
|
||||
{
|
||||
name: "no settings file",
|
||||
settingsJSON: "",
|
||||
expectedPort: DefaultWorkerPort,
|
||||
expectedModel: DefaultModel,
|
||||
expectedObsObs: 100,
|
||||
},
|
||||
{
|
||||
name: "custom port",
|
||||
settingsJSON: `{"CLAUDE_MNEMONIC_WORKER_PORT": 38888}`,
|
||||
expectedPort: 38888,
|
||||
expectedModel: DefaultModel,
|
||||
expectedObsObs: 100,
|
||||
},
|
||||
{
|
||||
name: "custom model",
|
||||
settingsJSON: `{"CLAUDE_MNEMONIC_MODEL": "sonnet"}`,
|
||||
expectedPort: DefaultWorkerPort,
|
||||
expectedModel: "sonnet",
|
||||
expectedObsObs: 100,
|
||||
},
|
||||
{
|
||||
name: "custom observations",
|
||||
settingsJSON: `{"CLAUDE_MNEMONIC_CONTEXT_OBSERVATIONS": 200}`,
|
||||
expectedPort: DefaultWorkerPort,
|
||||
expectedModel: DefaultModel,
|
||||
expectedObsObs: 200,
|
||||
},
|
||||
{
|
||||
name: "multiple settings",
|
||||
settingsJSON: `{"CLAUDE_MNEMONIC_WORKER_PORT": 39999, "CLAUDE_MNEMONIC_MODEL": "opus", "CLAUDE_MNEMONIC_CONTEXT_OBSERVATIONS": 50}`,
|
||||
expectedPort: 39999,
|
||||
expectedModel: "opus",
|
||||
expectedObsObs: 50,
|
||||
},
|
||||
{
|
||||
name: "invalid JSON returns defaults",
|
||||
settingsJSON: `{invalid}`,
|
||||
expectedPort: DefaultWorkerPort,
|
||||
expectedModel: DefaultModel,
|
||||
expectedObsObs: 100,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
s.Run(tt.name, func() {
|
||||
// Create fresh temp dir
|
||||
tempDir, err := os.MkdirTemp("", "config-test-*")
|
||||
s.Require().NoError(err)
|
||||
defer os.RemoveAll(tempDir)
|
||||
|
||||
os.Setenv("HOME", tempDir)
|
||||
|
||||
// Create data dir
|
||||
err = os.MkdirAll(filepath.Join(tempDir, ".claude-mnemonic"), 0750)
|
||||
s.Require().NoError(err)
|
||||
|
||||
if tt.settingsJSON != "" {
|
||||
err := os.WriteFile(
|
||||
filepath.Join(tempDir, ".claude-mnemonic", "settings.json"),
|
||||
[]byte(tt.settingsJSON),
|
||||
0600,
|
||||
)
|
||||
s.Require().NoError(err)
|
||||
}
|
||||
|
||||
cfg, err := Load()
|
||||
s.NoError(err)
|
||||
s.NotNil(cfg)
|
||||
s.Equal(tt.expectedPort, cfg.WorkerPort)
|
||||
s.Equal(tt.expectedModel, cfg.Model)
|
||||
s.Equal(tt.expectedObsObs, cfg.ContextObservations)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestGetWorkerPort_TableDriven tests worker port retrieval with various scenarios.
|
||||
func TestGetWorkerPort_TableDriven(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
envValue string
|
||||
wantPort int
|
||||
setEnv bool
|
||||
}{
|
||||
{
|
||||
name: "no env, use default",
|
||||
envValue: "",
|
||||
wantPort: DefaultWorkerPort,
|
||||
setEnv: false,
|
||||
},
|
||||
{
|
||||
name: "env set to valid port",
|
||||
envValue: "38888",
|
||||
wantPort: 38888,
|
||||
setEnv: true,
|
||||
},
|
||||
{
|
||||
name: "env set to invalid value",
|
||||
envValue: "invalid",
|
||||
wantPort: DefaultWorkerPort,
|
||||
setEnv: true,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
// Save original env
|
||||
origEnv := os.Getenv("CLAUDE_MNEMONIC_WORKER_PORT")
|
||||
defer os.Setenv("CLAUDE_MNEMONIC_WORKER_PORT", origEnv)
|
||||
|
||||
if tt.setEnv {
|
||||
os.Setenv("CLAUDE_MNEMONIC_WORKER_PORT", tt.envValue)
|
||||
} else {
|
||||
os.Unsetenv("CLAUDE_MNEMONIC_WORKER_PORT")
|
||||
}
|
||||
|
||||
// We can't easily test GetWorkerPort since it uses Get() which caches
|
||||
// So we test the env parsing logic directly
|
||||
if tt.setEnv && tt.envValue != "" {
|
||||
if tt.wantPort != DefaultWorkerPort {
|
||||
assert.Equal(t, tt.envValue, os.Getenv("CLAUDE_MNEMONIC_WORKER_PORT"))
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestSplitTrim tests the splitTrim helper function.
|
||||
func TestSplitTrim(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
input string
|
||||
expected []string
|
||||
}{
|
||||
{
|
||||
name: "empty string",
|
||||
input: "",
|
||||
expected: []string{},
|
||||
},
|
||||
{
|
||||
name: "single value",
|
||||
input: "bugfix",
|
||||
expected: []string{"bugfix"},
|
||||
},
|
||||
{
|
||||
name: "multiple values",
|
||||
input: "bugfix,feature,refactor",
|
||||
expected: []string{"bugfix", "feature", "refactor"},
|
||||
},
|
||||
{
|
||||
name: "values with spaces",
|
||||
input: " bugfix , feature , refactor ",
|
||||
expected: []string{"bugfix", "feature", "refactor"},
|
||||
},
|
||||
{
|
||||
name: "empty values filtered",
|
||||
input: "bugfix,,feature,,",
|
||||
expected: []string{"bugfix", "feature"},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
result := splitTrim(tt.input)
|
||||
assert.Equal(t, tt.expected, result)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestDefaultObservationTypes tests default observation types.
|
||||
func TestDefaultObservationTypes(t *testing.T) {
|
||||
expected := []string{
|
||||
"bugfix", "feature", "refactor", "change", "discovery", "decision",
|
||||
}
|
||||
assert.Equal(t, expected, DefaultObservationTypes)
|
||||
}
|
||||
|
||||
// TestDefaultObservationConcepts tests default observation concepts.
|
||||
func TestDefaultObservationConcepts(t *testing.T) {
|
||||
expected := []string{
|
||||
"how-it-works", "why-it-exists", "what-changed",
|
||||
"problem-solution", "gotcha", "pattern", "trade-off",
|
||||
}
|
||||
assert.Equal(t, expected, DefaultObservationConcepts)
|
||||
}
|
||||
|
||||
// TestCriticalConcepts tests critical concepts list.
|
||||
func TestCriticalConcepts(t *testing.T) {
|
||||
expected := []string{
|
||||
"gotcha", "pattern", "problem-solution", "trade-off",
|
||||
}
|
||||
assert.Equal(t, expected, CriticalConcepts)
|
||||
}
|
||||
|
||||
// TestLoad_ClaudeCodePath tests claude code path loading.
|
||||
func TestLoad_ClaudeCodePath(t *testing.T) {
|
||||
// Create temp dir
|
||||
tempDir, err := os.MkdirTemp("", "config-test-*")
|
||||
require.NoError(t, err)
|
||||
defer os.RemoveAll(tempDir)
|
||||
|
||||
origHome := os.Getenv("HOME")
|
||||
os.Setenv("HOME", tempDir)
|
||||
defer os.Setenv("HOME", origHome)
|
||||
|
||||
// Create data dir and settings
|
||||
err = os.MkdirAll(filepath.Join(tempDir, ".claude-mnemonic"), 0750)
|
||||
require.NoError(t, err)
|
||||
|
||||
settingsJSON := `{"CLAUDE_CODE_PATH": "/usr/local/bin/claude"}`
|
||||
err = os.WriteFile(
|
||||
filepath.Join(tempDir, ".claude-mnemonic", "settings.json"),
|
||||
[]byte(settingsJSON),
|
||||
0600,
|
||||
)
|
||||
require.NoError(t, err)
|
||||
|
||||
cfg, err := Load()
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "/usr/local/bin/claude", cfg.ClaudeCodePath)
|
||||
}
|
||||
|
||||
// TestGet tests the global config getter.
|
||||
func TestGet(t *testing.T) {
|
||||
// Save and restore HOME
|
||||
origHome := os.Getenv("HOME")
|
||||
tempDir, err := os.MkdirTemp("", "config-get-test-*")
|
||||
require.NoError(t, err)
|
||||
defer func() {
|
||||
os.Setenv("HOME", origHome)
|
||||
os.RemoveAll(tempDir)
|
||||
}()
|
||||
os.Setenv("HOME", tempDir)
|
||||
|
||||
// Create data dir
|
||||
err = os.MkdirAll(filepath.Join(tempDir, ".claude-mnemonic"), 0750)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Get() should return a valid config
|
||||
cfg := Get()
|
||||
require.NotNil(t, cfg)
|
||||
assert.Greater(t, cfg.WorkerPort, 0)
|
||||
assert.NotEmpty(t, cfg.Model)
|
||||
}
|
||||
|
||||
// TestGetWorkerPort_WithEnv tests GetWorkerPort with environment variable.
|
||||
func TestGetWorkerPort_WithEnv(t *testing.T) {
|
||||
// Save original env
|
||||
origEnv := os.Getenv("CLAUDE_MNEMONIC_WORKER_PORT")
|
||||
defer os.Setenv("CLAUDE_MNEMONIC_WORKER_PORT", origEnv)
|
||||
|
||||
// Test with valid port in env
|
||||
os.Setenv("CLAUDE_MNEMONIC_WORKER_PORT", "45678")
|
||||
port := GetWorkerPort()
|
||||
assert.Equal(t, 45678, port)
|
||||
|
||||
// Test with invalid port (should fall back to config)
|
||||
os.Setenv("CLAUDE_MNEMONIC_WORKER_PORT", "not-a-number")
|
||||
port = GetWorkerPort()
|
||||
// Should return from Get().WorkerPort, which is default
|
||||
assert.Greater(t, port, 0)
|
||||
|
||||
// Test with zero port (should fall back to config)
|
||||
os.Setenv("CLAUDE_MNEMONIC_WORKER_PORT", "0")
|
||||
port = GetWorkerPort()
|
||||
// Zero is invalid, so should use default
|
||||
assert.Greater(t, port, 0)
|
||||
|
||||
// Test with no env (should use config)
|
||||
os.Unsetenv("CLAUDE_MNEMONIC_WORKER_PORT")
|
||||
port = GetWorkerPort()
|
||||
assert.Greater(t, port, 0)
|
||||
}
|
||||
|
||||
// TestLoad_ContextSettings tests context-related settings loading.
|
||||
func TestLoad_ContextSettings(t *testing.T) {
|
||||
// Create temp dir
|
||||
tempDir, err := os.MkdirTemp("", "config-test-*")
|
||||
require.NoError(t, err)
|
||||
defer os.RemoveAll(tempDir)
|
||||
|
||||
origHome := os.Getenv("HOME")
|
||||
os.Setenv("HOME", tempDir)
|
||||
defer os.Setenv("HOME", origHome)
|
||||
|
||||
// Create data dir and settings
|
||||
err = os.MkdirAll(filepath.Join(tempDir, ".claude-mnemonic"), 0750)
|
||||
require.NoError(t, err)
|
||||
|
||||
settingsJSON := `{
|
||||
"CLAUDE_MNEMONIC_CONTEXT_FULL_COUNT": 50,
|
||||
"CLAUDE_MNEMONIC_CONTEXT_SESSION_COUNT": 20,
|
||||
"CLAUDE_MNEMONIC_CONTEXT_OBS_TYPES": "bugfix,feature",
|
||||
"CLAUDE_MNEMONIC_CONTEXT_OBS_CONCEPTS": "security,performance"
|
||||
}`
|
||||
err = os.WriteFile(
|
||||
filepath.Join(tempDir, ".claude-mnemonic", "settings.json"),
|
||||
[]byte(settingsJSON),
|
||||
0600,
|
||||
)
|
||||
require.NoError(t, err)
|
||||
|
||||
cfg, err := Load()
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 50, cfg.ContextFullCount)
|
||||
assert.Equal(t, 20, cfg.ContextSessionCount)
|
||||
assert.Equal(t, []string{"bugfix", "feature"}, cfg.ContextObsTypes)
|
||||
assert.Equal(t, []string{"security", "performance"}, cfg.ContextObsConcepts)
|
||||
}
|
||||
@@ -0,0 +1,331 @@
|
||||
//go:build fts5
|
||||
|
||||
// Package gorm provides GORM-based database operations for claude-mnemonic.
|
||||
package gorm
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"gorm.io/gorm/logger"
|
||||
|
||||
"github.com/lukaszraczylo/claude-mnemonic/pkg/models"
|
||||
)
|
||||
|
||||
// setupBenchStore creates a temporary store for benchmarking.
|
||||
func setupBenchStore(b *testing.B) (*Store, func()) {
|
||||
b.Helper()
|
||||
|
||||
tmpDir, err := os.MkdirTemp("", "gorm_bench_*")
|
||||
if err != nil {
|
||||
b.Fatalf("create temp dir: %v", err)
|
||||
}
|
||||
|
||||
dbPath := filepath.Join(tmpDir, "bench.db")
|
||||
cfg := Config{
|
||||
Path: dbPath,
|
||||
MaxConns: 4,
|
||||
LogLevel: logger.Silent,
|
||||
}
|
||||
|
||||
store, err := NewStore(cfg)
|
||||
if err != nil {
|
||||
os.RemoveAll(tmpDir)
|
||||
b.Fatalf("NewStore failed: %v", err)
|
||||
}
|
||||
|
||||
cleanup := func() {
|
||||
store.Close()
|
||||
os.RemoveAll(tmpDir)
|
||||
}
|
||||
|
||||
return store, cleanup
|
||||
}
|
||||
|
||||
// BenchmarkSessionStore_CreateSDKSession benchmarks session creation (most frequent operation).
|
||||
func BenchmarkSessionStore_CreateSDKSession(b *testing.B) {
|
||||
store, cleanup := setupBenchStore(b)
|
||||
defer cleanup()
|
||||
|
||||
sessionStore := NewSessionStore(store)
|
||||
ctx := context.Background()
|
||||
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
sessionID := fmt.Sprintf("claude-bench-%d", i)
|
||||
_, err := sessionStore.CreateSDKSession(ctx, sessionID, "bench-project", "test prompt")
|
||||
if err != nil {
|
||||
b.Fatalf("CreateSDKSession failed: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// BenchmarkSessionStore_CreateSDKSession_Idempotent benchmarks idempotent session creation (INSERT OR IGNORE).
|
||||
func BenchmarkSessionStore_CreateSDKSession_Idempotent(b *testing.B) {
|
||||
store, cleanup := setupBenchStore(b)
|
||||
defer cleanup()
|
||||
|
||||
sessionStore := NewSessionStore(store)
|
||||
ctx := context.Background()
|
||||
|
||||
// Pre-create session
|
||||
sessionStore.CreateSDKSession(ctx, "claude-bench", "bench-project", "test prompt")
|
||||
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
_, err := sessionStore.CreateSDKSession(ctx, "claude-bench", "bench-project", "updated prompt")
|
||||
if err != nil {
|
||||
b.Fatalf("CreateSDKSession failed: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// BenchmarkObservationStore_StoreObservation benchmarks observation storage (high frequency).
|
||||
func BenchmarkObservationStore_StoreObservation(b *testing.B) {
|
||||
store, cleanup := setupBenchStore(b)
|
||||
defer cleanup()
|
||||
|
||||
sessionStore := NewSessionStore(store)
|
||||
obsStore := NewObservationStore(store, nil, nil, nil)
|
||||
ctx := context.Background()
|
||||
|
||||
// Create session
|
||||
sessionID, _ := sessionStore.CreateSDKSession(ctx, "claude-bench", "bench-project", "")
|
||||
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
obs := &models.ParsedObservation{
|
||||
Type: models.ObsTypeDiscovery,
|
||||
Title: fmt.Sprintf("Observation %d", i),
|
||||
Narrative: "Benchmark observation content",
|
||||
}
|
||||
_, _, err := obsStore.StoreObservation(ctx, "claude-bench", "bench-project", obs, int(sessionID), int64(i+1))
|
||||
if err != nil {
|
||||
b.Fatalf("StoreObservation failed: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// BenchmarkObservationStore_GetRecentObservations benchmarks recent observation retrieval.
|
||||
func BenchmarkObservationStore_GetRecentObservations(b *testing.B) {
|
||||
store, cleanup := setupBenchStore(b)
|
||||
defer cleanup()
|
||||
|
||||
sessionStore := NewSessionStore(store)
|
||||
obsStore := NewObservationStore(store, nil, nil, nil)
|
||||
ctx := context.Background()
|
||||
|
||||
// Create session and observations
|
||||
sessionID, _ := sessionStore.CreateSDKSession(ctx, "claude-bench", "bench-project", "")
|
||||
for i := 0; i < 100; i++ {
|
||||
obs := &models.ParsedObservation{
|
||||
Type: models.ObsTypeDiscovery,
|
||||
Title: fmt.Sprintf("Observation %d", i),
|
||||
Narrative: "Benchmark observation content",
|
||||
}
|
||||
obsStore.StoreObservation(ctx, "claude-bench", "bench-project", obs, int(sessionID), int64(i+1))
|
||||
}
|
||||
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
_, err := obsStore.GetRecentObservations(ctx, "bench-project", 20)
|
||||
if err != nil {
|
||||
b.Fatalf("GetRecentObservations failed: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// BenchmarkObservationStore_SearchObservationsFTS benchmarks FTS5 search (latency-sensitive).
|
||||
func BenchmarkObservationStore_SearchObservationsFTS(b *testing.B) {
|
||||
store, cleanup := setupBenchStore(b)
|
||||
defer cleanup()
|
||||
|
||||
sessionStore := NewSessionStore(store)
|
||||
obsStore := NewObservationStore(store, nil, nil, nil)
|
||||
ctx := context.Background()
|
||||
|
||||
// Create session and observations with searchable content
|
||||
sessionID, _ := sessionStore.CreateSDKSession(ctx, "claude-bench", "bench-project", "")
|
||||
for i := 0; i < 100; i++ {
|
||||
obs := &models.ParsedObservation{
|
||||
Type: models.ObsTypeDiscovery,
|
||||
Title: fmt.Sprintf("Security best practice %d", i),
|
||||
Narrative: "This observation discusses security patterns and authentication mechanisms",
|
||||
}
|
||||
obsStore.StoreObservation(ctx, "claude-bench", "bench-project", obs, int(sessionID), int64(i+1))
|
||||
}
|
||||
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
_, err := obsStore.SearchObservationsFTS(ctx, "security authentication", "bench-project", 10)
|
||||
if err != nil {
|
||||
b.Fatalf("SearchObservationsFTS failed: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// BenchmarkObservationStore_UpdateImportanceScore benchmarks scoring updates.
|
||||
func BenchmarkObservationStore_UpdateImportanceScore(b *testing.B) {
|
||||
store, cleanup := setupBenchStore(b)
|
||||
defer cleanup()
|
||||
|
||||
sessionStore := NewSessionStore(store)
|
||||
obsStore := NewObservationStore(store, nil, nil, nil)
|
||||
ctx := context.Background()
|
||||
|
||||
// Create session and observation
|
||||
sessionID, _ := sessionStore.CreateSDKSession(ctx, "claude-bench", "bench-project", "")
|
||||
obs := &models.ParsedObservation{Type: models.ObsTypeDiscovery, Title: "Test"}
|
||||
obsID, _, _ := obsStore.StoreObservation(ctx, "claude-bench", "bench-project", obs, int(sessionID), 1)
|
||||
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
score := float64(i%10) + 1.0
|
||||
err := obsStore.UpdateImportanceScore(ctx, obsID, score)
|
||||
if err != nil {
|
||||
b.Fatalf("UpdateImportanceScore failed: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// BenchmarkObservationStore_UpdateImportanceScores_Bulk benchmarks bulk scoring updates.
|
||||
func BenchmarkObservationStore_UpdateImportanceScores_Bulk(b *testing.B) {
|
||||
store, cleanup := setupBenchStore(b)
|
||||
defer cleanup()
|
||||
|
||||
sessionStore := NewSessionStore(store)
|
||||
obsStore := NewObservationStore(store, nil, nil, nil)
|
||||
ctx := context.Background()
|
||||
|
||||
// Create session and 100 observations
|
||||
sessionID, _ := sessionStore.CreateSDKSession(ctx, "claude-bench", "bench-project", "")
|
||||
var obsIDs []int64
|
||||
for i := 0; i < 100; i++ {
|
||||
obs := &models.ParsedObservation{Type: models.ObsTypeDiscovery, Title: fmt.Sprintf("Obs %d", i)}
|
||||
obsID, _, _ := obsStore.StoreObservation(ctx, "claude-bench", "bench-project", obs, int(sessionID), int64(i+1))
|
||||
obsIDs = append(obsIDs, obsID)
|
||||
}
|
||||
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
scores := make(map[int64]float64)
|
||||
for _, id := range obsIDs {
|
||||
scores[id] = float64(i%10) + 1.0
|
||||
}
|
||||
err := obsStore.UpdateImportanceScores(ctx, scores)
|
||||
if err != nil {
|
||||
b.Fatalf("UpdateImportanceScores failed: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// BenchmarkPromptStore_SaveUserPromptWithMatches benchmarks prompt storage with matches.
|
||||
func BenchmarkPromptStore_SaveUserPromptWithMatches(b *testing.B) {
|
||||
store, cleanup := setupBenchStore(b)
|
||||
defer cleanup()
|
||||
|
||||
sessionStore := NewSessionStore(store)
|
||||
promptStore := NewPromptStore(store, nil)
|
||||
ctx := context.Background()
|
||||
|
||||
// Create session
|
||||
sessionID, _ := sessionStore.CreateSDKSession(ctx, "claude-bench", "bench-project", "")
|
||||
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
_, err := promptStore.SaveUserPromptWithMatches(ctx, "claude-bench", int(sessionID), fmt.Sprintf("Prompt %d", i), i+1)
|
||||
if err != nil {
|
||||
b.Fatalf("SaveUserPromptWithMatches failed: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// BenchmarkSummaryStore_StoreSummary benchmarks summary storage.
|
||||
func BenchmarkSummaryStore_StoreSummary(b *testing.B) {
|
||||
store, cleanup := setupBenchStore(b)
|
||||
defer cleanup()
|
||||
|
||||
sessionStore := NewSessionStore(store)
|
||||
summaryStore := NewSummaryStore(store)
|
||||
ctx := context.Background()
|
||||
|
||||
// Create session
|
||||
sessionStore.CreateSDKSession(ctx, "claude-bench", "bench-project", "")
|
||||
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
summary := &models.ParsedSummary{
|
||||
Request: fmt.Sprintf("Request %d", i),
|
||||
Investigated: "Investigation details",
|
||||
Learned: "Learning summary",
|
||||
Completed: "Completion status",
|
||||
}
|
||||
_, _, err := summaryStore.StoreSummary(ctx, "claude-bench", "bench-project", summary, i+1, 100)
|
||||
if err != nil {
|
||||
b.Fatalf("StoreSummary failed: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// BenchmarkRelationStore_StoreRelation benchmarks relation storage.
|
||||
func BenchmarkRelationStore_StoreRelation(b *testing.B) {
|
||||
store, cleanup := setupBenchStore(b)
|
||||
defer cleanup()
|
||||
|
||||
sessionStore := NewSessionStore(store)
|
||||
obsStore := NewObservationStore(store, nil, nil, nil)
|
||||
relationStore := NewRelationStore(store)
|
||||
ctx := context.Background()
|
||||
|
||||
// Create session and observations
|
||||
sessionID, _ := sessionStore.CreateSDKSession(ctx, "claude-bench", "bench-project", "")
|
||||
obs1 := &models.ParsedObservation{Type: models.ObsTypeDiscovery, Title: "Source"}
|
||||
obsID1, _, _ := obsStore.StoreObservation(ctx, "claude-bench", "bench-project", obs1, int(sessionID), 1)
|
||||
obs2 := &models.ParsedObservation{Type: models.ObsTypeDiscovery, Title: "Target"}
|
||||
obsID2, _, _ := obsStore.StoreObservation(ctx, "claude-bench", "bench-project", obs2, int(sessionID), 2)
|
||||
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
relation := &models.ObservationRelation{
|
||||
SourceID: obsID1,
|
||||
TargetID: obsID2,
|
||||
RelationType: models.RelationCauses,
|
||||
Confidence: 0.9,
|
||||
DetectionSource: models.DetectionSourceFileOverlap,
|
||||
}
|
||||
_, err := relationStore.StoreRelation(ctx, relation)
|
||||
if err != nil {
|
||||
b.Fatalf("StoreRelation failed: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// BenchmarkPatternStore_StorePattern benchmarks pattern storage.
|
||||
func BenchmarkPatternStore_StorePattern(b *testing.B) {
|
||||
store, cleanup := setupBenchStore(b)
|
||||
defer cleanup()
|
||||
|
||||
patternStore := NewPatternStore(store)
|
||||
ctx := context.Background()
|
||||
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
pattern := &models.Pattern{
|
||||
Name: fmt.Sprintf("Pattern %d", i),
|
||||
Type: models.PatternTypeBug,
|
||||
Description: sql.NullString{String: "Benchmark pattern", Valid: true},
|
||||
Frequency: 1,
|
||||
Confidence: 0.8,
|
||||
Projects: []string{"bench-project"},
|
||||
Status: models.PatternStatusActive,
|
||||
}
|
||||
_, err := patternStore.StorePattern(ctx, pattern)
|
||||
if err != nil {
|
||||
b.Fatalf("StorePattern failed: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,281 @@
|
||||
// Package gorm provides GORM-based database operations for claude-mnemonic.
|
||||
package gorm
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"time"
|
||||
|
||||
"gorm.io/gorm"
|
||||
|
||||
"github.com/lukaszraczylo/claude-mnemonic/pkg/models"
|
||||
)
|
||||
|
||||
// SupersededRetentionDays is the number of days to keep superseded observations before deletion.
|
||||
const SupersededRetentionDays = 3
|
||||
|
||||
// ConflictStore provides conflict-related database operations using GORM.
|
||||
type ConflictStore struct {
|
||||
db *gorm.DB
|
||||
}
|
||||
|
||||
// NewConflictStore creates a new conflict store.
|
||||
func NewConflictStore(store *Store) *ConflictStore {
|
||||
return &ConflictStore{
|
||||
db: store.DB,
|
||||
}
|
||||
}
|
||||
|
||||
// StoreConflict stores a new observation conflict.
|
||||
func (s *ConflictStore) StoreConflict(ctx context.Context, conflict *models.ObservationConflict) (int64, error) {
|
||||
dbConflict := &ObservationConflict{
|
||||
NewerObsID: conflict.NewerObsID,
|
||||
OlderObsID: conflict.OlderObsID,
|
||||
ConflictType: conflict.ConflictType,
|
||||
Resolution: conflict.Resolution,
|
||||
DetectedAt: conflict.DetectedAt,
|
||||
DetectedAtEpoch: conflict.DetectedAtEpoch,
|
||||
Resolved: 0,
|
||||
}
|
||||
|
||||
// Convert bool to int
|
||||
if conflict.Resolved {
|
||||
dbConflict.Resolved = 1
|
||||
}
|
||||
|
||||
// Handle nullable fields
|
||||
if conflict.Reason != "" {
|
||||
dbConflict.Reason = sql.NullString{String: conflict.Reason, Valid: true}
|
||||
}
|
||||
if conflict.ResolvedAt != nil && *conflict.ResolvedAt != "" {
|
||||
dbConflict.ResolvedAt = sql.NullString{String: *conflict.ResolvedAt, Valid: true}
|
||||
}
|
||||
|
||||
result := s.db.WithContext(ctx).Create(dbConflict)
|
||||
if result.Error != nil {
|
||||
return 0, result.Error
|
||||
}
|
||||
|
||||
return dbConflict.ID, nil
|
||||
}
|
||||
|
||||
// MarkObservationSuperseded marks an observation as superseded.
|
||||
func (s *ConflictStore) MarkObservationSuperseded(ctx context.Context, obsID int64) error {
|
||||
result := s.db.WithContext(ctx).
|
||||
Model(&Observation{}).
|
||||
Where("id = ?", obsID).
|
||||
Update("is_superseded", 1)
|
||||
|
||||
return result.Error
|
||||
}
|
||||
|
||||
// MarkObservationsSuperseded marks multiple observations as superseded.
|
||||
func (s *ConflictStore) MarkObservationsSuperseded(ctx context.Context, obsIDs []int64) error {
|
||||
if len(obsIDs) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
result := s.db.WithContext(ctx).
|
||||
Model(&Observation{}).
|
||||
Where("id IN ?", obsIDs).
|
||||
Update("is_superseded", 1)
|
||||
|
||||
return result.Error
|
||||
}
|
||||
|
||||
// GetConflictsByObservationID retrieves all conflicts involving an observation.
|
||||
func (s *ConflictStore) GetConflictsByObservationID(ctx context.Context, obsID int64) ([]*models.ObservationConflict, error) {
|
||||
var conflicts []ObservationConflict
|
||||
|
||||
err := s.db.WithContext(ctx).
|
||||
Where("newer_obs_id = ? OR older_obs_id = ?", obsID, obsID).
|
||||
Order("detected_at_epoch DESC").
|
||||
Find(&conflicts).Error
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return toModelConflicts(conflicts), nil
|
||||
}
|
||||
|
||||
// GetUnresolvedConflicts retrieves all unresolved conflicts.
|
||||
func (s *ConflictStore) GetUnresolvedConflicts(ctx context.Context, limit int) ([]*models.ObservationConflict, error) {
|
||||
var conflicts []ObservationConflict
|
||||
|
||||
err := s.db.WithContext(ctx).
|
||||
Where("resolved = 0").
|
||||
Order("detected_at_epoch DESC").
|
||||
Limit(limit).
|
||||
Find(&conflicts).Error
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return toModelConflicts(conflicts), nil
|
||||
}
|
||||
|
||||
// GetSupersededObservationIDs returns IDs of all observations that have been superseded.
|
||||
func (s *ConflictStore) GetSupersededObservationIDs(ctx context.Context, project string) ([]int64, error) {
|
||||
var ids []int64
|
||||
|
||||
err := s.db.WithContext(ctx).
|
||||
Table("observation_conflicts oc").
|
||||
Select("DISTINCT oc.older_obs_id").
|
||||
Joins("JOIN observations o ON o.id = oc.older_obs_id").
|
||||
Where("oc.resolution = ?", models.ResolutionPreferNewer).
|
||||
Where("o.project = ? OR o.scope = 'global'", project).
|
||||
Pluck("oc.older_obs_id", &ids).Error
|
||||
|
||||
return ids, err
|
||||
}
|
||||
|
||||
// ResolveConflict marks a conflict as resolved.
|
||||
func (s *ConflictStore) ResolveConflict(ctx context.Context, conflictID int64, resolution models.ConflictResolution) error {
|
||||
now := time.Now().Format(time.RFC3339)
|
||||
|
||||
result := s.db.WithContext(ctx).
|
||||
Model(&ObservationConflict{}).
|
||||
Where("id = ?", conflictID).
|
||||
Updates(map[string]interface{}{
|
||||
"resolved": 1,
|
||||
"resolved_at": now,
|
||||
"resolution": resolution,
|
||||
})
|
||||
|
||||
return result.Error
|
||||
}
|
||||
|
||||
// DeleteConflictsByObservationID deletes all conflicts involving an observation.
|
||||
// Called when an observation is deleted.
|
||||
func (s *ConflictStore) DeleteConflictsByObservationID(ctx context.Context, obsID int64) error {
|
||||
result := s.db.WithContext(ctx).
|
||||
Where("newer_obs_id = ? OR older_obs_id = ?", obsID, obsID).
|
||||
Delete(&ObservationConflict{})
|
||||
|
||||
return result.Error
|
||||
}
|
||||
|
||||
// ConflictWithDetails contains a conflict with its observation details.
|
||||
type ConflictWithDetails struct {
|
||||
Conflict *models.ObservationConflict
|
||||
NewerObsTitle string
|
||||
OlderObsTitle string
|
||||
}
|
||||
|
||||
// CleanupSupersededObservations deletes observations that have been superseded for longer than
|
||||
// SupersededRetentionDays. Returns the IDs of deleted observations for downstream cleanup (e.g., vector DB).
|
||||
func (s *ConflictStore) CleanupSupersededObservations(ctx context.Context, project string) ([]int64, error) {
|
||||
// Calculate cutoff time (3 days ago in milliseconds)
|
||||
cutoffEpoch := time.Now().AddDate(0, 0, -SupersededRetentionDays).UnixMilli()
|
||||
|
||||
var toDelete []int64
|
||||
|
||||
// Use a transaction to prevent TOCTOU race condition
|
||||
err := s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
|
||||
// Find IDs to delete
|
||||
err := tx.Table("observations o").
|
||||
Select("DISTINCT o.id").
|
||||
Joins("JOIN observation_conflicts oc ON o.id = oc.older_obs_id").
|
||||
Where("o.is_superseded = 1").
|
||||
Where("o.project = ?", project).
|
||||
Where("oc.detected_at_epoch < ?", cutoffEpoch).
|
||||
Pluck("o.id", &toDelete).Error
|
||||
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if len(toDelete) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Delete the conflict records first (due to foreign key constraints)
|
||||
for _, obsID := range toDelete {
|
||||
err := tx.Where("newer_obs_id = ? OR older_obs_id = ?", obsID, obsID).
|
||||
Delete(&ObservationConflict{}).Error
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// Delete the observations
|
||||
return tx.Delete(&Observation{}, toDelete).Error
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return toDelete, nil
|
||||
}
|
||||
|
||||
// GetConflictsWithDetails retrieves all conflicts with observation titles for display.
|
||||
func (s *ConflictStore) GetConflictsWithDetails(ctx context.Context, project string, limit int) ([]*ConflictWithDetails, error) {
|
||||
var results []struct {
|
||||
ObservationConflict
|
||||
NewerTitle sql.NullString `gorm:"column:newer_title"`
|
||||
OlderTitle sql.NullString `gorm:"column:older_title"`
|
||||
}
|
||||
|
||||
err := s.db.WithContext(ctx).
|
||||
Table("observation_conflicts oc").
|
||||
Select("oc.*, "+
|
||||
"COALESCE(newer.title, '') as newer_title, "+
|
||||
"COALESCE(older.title, '') as older_title").
|
||||
Joins("JOIN observations newer ON newer.id = oc.newer_obs_id").
|
||||
Joins("JOIN observations older ON older.id = oc.older_obs_id").
|
||||
Where("newer.project = ? OR older.project = ?", project, project).
|
||||
Order("oc.detected_at_epoch DESC").
|
||||
Limit(limit).
|
||||
Scan(&results).Error
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
conflicts := make([]*ConflictWithDetails, len(results))
|
||||
for i, r := range results {
|
||||
conflicts[i] = &ConflictWithDetails{
|
||||
Conflict: toModelConflict(&r.ObservationConflict),
|
||||
NewerObsTitle: r.NewerTitle.String,
|
||||
OlderObsTitle: r.OlderTitle.String,
|
||||
}
|
||||
}
|
||||
|
||||
return conflicts, nil
|
||||
}
|
||||
|
||||
// toModelConflict converts a GORM ObservationConflict to a pkg/models ObservationConflict.
|
||||
func toModelConflict(c *ObservationConflict) *models.ObservationConflict {
|
||||
conflict := &models.ObservationConflict{
|
||||
ID: c.ID,
|
||||
NewerObsID: c.NewerObsID,
|
||||
OlderObsID: c.OlderObsID,
|
||||
ConflictType: c.ConflictType,
|
||||
Resolution: c.Resolution,
|
||||
DetectedAt: c.DetectedAt,
|
||||
DetectedAtEpoch: c.DetectedAtEpoch,
|
||||
Resolved: c.Resolved == 1,
|
||||
}
|
||||
|
||||
if c.Reason.Valid {
|
||||
conflict.Reason = c.Reason.String
|
||||
}
|
||||
if c.ResolvedAt.Valid {
|
||||
s := c.ResolvedAt.String
|
||||
conflict.ResolvedAt = &s
|
||||
}
|
||||
|
||||
return conflict
|
||||
}
|
||||
|
||||
// toModelConflicts converts a slice of GORM ObservationConflicts to pkg/models ObservationConflicts.
|
||||
func toModelConflicts(conflicts []ObservationConflict) []*models.ObservationConflict {
|
||||
result := make([]*models.ObservationConflict, len(conflicts))
|
||||
for i, c := range conflicts {
|
||||
result[i] = toModelConflict(&c)
|
||||
}
|
||||
return result
|
||||
}
|
||||
@@ -0,0 +1,637 @@
|
||||
//go:build fts5
|
||||
|
||||
// Package gorm provides GORM-based database operations for claude-mnemonic.
|
||||
package gorm
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"gorm.io/gorm/logger"
|
||||
|
||||
"github.com/lukaszraczylo/claude-mnemonic/pkg/models"
|
||||
)
|
||||
|
||||
// testConflictStore creates a ConflictStore with a temporary database for testing.
|
||||
func testConflictStore(t *testing.T) (*ConflictStore, *Store, func()) {
|
||||
t.Helper()
|
||||
|
||||
tmpDir, err := os.MkdirTemp("", "gorm_conflict_test_*")
|
||||
if err != nil {
|
||||
t.Fatalf("create temp dir: %v", err)
|
||||
}
|
||||
|
||||
dbPath := filepath.Join(tmpDir, "test.db")
|
||||
cfg := Config{
|
||||
Path: dbPath,
|
||||
MaxConns: 4,
|
||||
LogLevel: logger.Silent,
|
||||
}
|
||||
|
||||
store, err := NewStore(cfg)
|
||||
if err != nil {
|
||||
os.RemoveAll(tmpDir)
|
||||
t.Fatalf("NewStore failed: %v", err)
|
||||
}
|
||||
|
||||
conflictStore := NewConflictStore(store)
|
||||
|
||||
cleanup := func() {
|
||||
store.Close()
|
||||
os.RemoveAll(tmpDir)
|
||||
}
|
||||
|
||||
return conflictStore, store, cleanup
|
||||
}
|
||||
|
||||
func TestConflictStore_StoreConflict(t *testing.T) {
|
||||
conflictStore, store, cleanup := testConflictStore(t)
|
||||
defer cleanup()
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
// Create session for observations
|
||||
sessionStore := NewSessionStore(store)
|
||||
sessionID, err := sessionStore.CreateSDKSession(ctx, "claude-1", "test-project", "")
|
||||
require.NoError(t, err)
|
||||
|
||||
// Create test observations
|
||||
obsStore := NewObservationStore(store, nil, nil, nil)
|
||||
obs1 := &models.ParsedObservation{
|
||||
Type: models.ObsTypeDiscovery,
|
||||
Title: "Newer observation",
|
||||
}
|
||||
obsID1, _, err := obsStore.StoreObservation(ctx, "claude-1", "test-project", obs1, int(sessionID), 1)
|
||||
require.NoError(t, err)
|
||||
|
||||
obs2 := &models.ParsedObservation{
|
||||
Type: models.ObsTypeDiscovery,
|
||||
Title: "Older observation",
|
||||
}
|
||||
obsID2, _, err := obsStore.StoreObservation(ctx, "claude-1", "test-project", obs2, int(sessionID), 2)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Create conflict
|
||||
now := time.Now()
|
||||
conflict := &models.ObservationConflict{
|
||||
NewerObsID: obsID1,
|
||||
OlderObsID: obsID2,
|
||||
ConflictType: models.ConflictContradicts,
|
||||
Resolution: models.ResolutionPreferNewer,
|
||||
Reason: "Newer observation contradicts older one",
|
||||
DetectedAt: now.Format(time.RFC3339),
|
||||
DetectedAtEpoch: now.UnixMilli(),
|
||||
Resolved: false,
|
||||
}
|
||||
|
||||
id, err := conflictStore.StoreConflict(ctx, conflict)
|
||||
require.NoError(t, err)
|
||||
assert.Greater(t, id, int64(0))
|
||||
|
||||
// Verify conflict was stored
|
||||
var count int64
|
||||
store.DB.Model(&ObservationConflict{}).Where("id = ?", id).Count(&count)
|
||||
assert.Equal(t, int64(1), count)
|
||||
}
|
||||
|
||||
func TestConflictStore_MarkObservationSuperseded(t *testing.T) {
|
||||
conflictStore, store, cleanup := testConflictStore(t)
|
||||
defer cleanup()
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
// Create observation
|
||||
sessionStore := NewSessionStore(store)
|
||||
sessionID, err := sessionStore.CreateSDKSession(ctx, "claude-1", "test-project", "")
|
||||
require.NoError(t, err)
|
||||
|
||||
obsStore := NewObservationStore(store, nil, nil, nil)
|
||||
obs := &models.ParsedObservation{
|
||||
Type: models.ObsTypeDiscovery,
|
||||
Title: "Test observation",
|
||||
}
|
||||
obsID, _, err := obsStore.StoreObservation(ctx, "claude-1", "test-project", obs, int(sessionID), 1)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Mark as superseded
|
||||
err = conflictStore.MarkObservationSuperseded(ctx, obsID)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Verify it's marked
|
||||
var dbObs Observation
|
||||
store.DB.First(&dbObs, obsID)
|
||||
assert.Equal(t, 1, dbObs.IsSuperseded)
|
||||
}
|
||||
|
||||
func TestConflictStore_MarkObservationsSuperseded(t *testing.T) {
|
||||
conflictStore, store, cleanup := testConflictStore(t)
|
||||
defer cleanup()
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
obsIDs []int64
|
||||
setup func() []int64
|
||||
}{
|
||||
{
|
||||
name: "empty list",
|
||||
obsIDs: []int64{},
|
||||
setup: func() []int64 { return []int64{} },
|
||||
},
|
||||
{
|
||||
name: "single observation",
|
||||
setup: func() []int64 {
|
||||
sessionStore := NewSessionStore(store)
|
||||
sessionID, _ := sessionStore.CreateSDKSession(ctx, "claude-1", "test-project", "")
|
||||
obsStore := NewObservationStore(store, nil, nil, nil)
|
||||
obs := &models.ParsedObservation{
|
||||
Type: models.ObsTypeDiscovery,
|
||||
Title: "Test",
|
||||
}
|
||||
id, _, _ := obsStore.StoreObservation(ctx, "claude-1", "test-project", obs, int(sessionID), 1)
|
||||
return []int64{id}
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "multiple observations",
|
||||
setup: func() []int64 {
|
||||
sessionStore := NewSessionStore(store)
|
||||
sessionID, _ := sessionStore.CreateSDKSession(ctx, "claude-1", "test-project", "")
|
||||
obsStore := NewObservationStore(store, nil, nil, nil)
|
||||
var ids []int64
|
||||
for i := 0; i < 3; i++ {
|
||||
obs := &models.ParsedObservation{
|
||||
Type: models.ObsTypeDiscovery,
|
||||
Title: "Test",
|
||||
}
|
||||
id, _, _ := obsStore.StoreObservation(ctx, "claude-1", "test-project", obs, int(sessionID), int64(i+1))
|
||||
ids = append(ids, id)
|
||||
}
|
||||
return ids
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
obsIDs := tt.setup()
|
||||
err := conflictStore.MarkObservationsSuperseded(ctx, obsIDs)
|
||||
require.NoError(t, err)
|
||||
|
||||
if len(obsIDs) > 0 {
|
||||
// Verify all are marked
|
||||
for _, id := range obsIDs {
|
||||
var dbObs Observation
|
||||
store.DB.First(&dbObs, id)
|
||||
assert.Equal(t, 1, dbObs.IsSuperseded)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestConflictStore_GetConflictsByObservationID(t *testing.T) {
|
||||
conflictStore, store, cleanup := testConflictStore(t)
|
||||
defer cleanup()
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
// Create observations
|
||||
sessionStore := NewSessionStore(store)
|
||||
sessionID, err := sessionStore.CreateSDKSession(ctx, "claude-1", "test-project", "")
|
||||
require.NoError(t, err)
|
||||
|
||||
obsStore := NewObservationStore(store, nil, nil, nil)
|
||||
var obsIDs []int64
|
||||
for i := 0; i < 3; i++ {
|
||||
obs := &models.ParsedObservation{
|
||||
Type: models.ObsTypeDiscovery,
|
||||
Title: "Test",
|
||||
}
|
||||
id, _, err := obsStore.StoreObservation(ctx, "claude-1", "test-project", obs, int(sessionID), int64(i+1))
|
||||
require.NoError(t, err)
|
||||
obsIDs = append(obsIDs, id)
|
||||
}
|
||||
|
||||
// Create conflicts involving observation 2 (index 1)
|
||||
now := time.Now()
|
||||
conflict1 := &models.ObservationConflict{
|
||||
NewerObsID: obsIDs[0],
|
||||
OlderObsID: obsIDs[1],
|
||||
ConflictType: models.ConflictContradicts,
|
||||
Resolution: models.ResolutionPreferNewer,
|
||||
Reason: "reason1",
|
||||
DetectedAt: now.Format(time.RFC3339),
|
||||
DetectedAtEpoch: now.UnixMilli(),
|
||||
}
|
||||
_, err = conflictStore.StoreConflict(ctx, conflict1)
|
||||
require.NoError(t, err)
|
||||
|
||||
conflict2 := &models.ObservationConflict{
|
||||
NewerObsID: obsIDs[1],
|
||||
OlderObsID: obsIDs[2],
|
||||
ConflictType: models.ConflictSuperseded,
|
||||
Resolution: models.ResolutionPreferNewer,
|
||||
Reason: "reason2",
|
||||
DetectedAt: now.Format(time.RFC3339),
|
||||
DetectedAtEpoch: now.UnixMilli(),
|
||||
}
|
||||
_, err = conflictStore.StoreConflict(ctx, conflict2)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Get conflicts for observation 2 (involved in 2 conflicts)
|
||||
conflicts, err := conflictStore.GetConflictsByObservationID(ctx, obsIDs[1])
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, conflicts, 2)
|
||||
}
|
||||
|
||||
func TestConflictStore_GetUnresolvedConflicts(t *testing.T) {
|
||||
conflictStore, store, cleanup := testConflictStore(t)
|
||||
defer cleanup()
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
// Create observations
|
||||
sessionStore := NewSessionStore(store)
|
||||
sessionID, err := sessionStore.CreateSDKSession(ctx, "claude-1", "test-project", "")
|
||||
require.NoError(t, err)
|
||||
|
||||
obsStore := NewObservationStore(store, nil, nil, nil)
|
||||
obs1 := &models.ParsedObservation{
|
||||
Type: models.ObsTypeDiscovery,
|
||||
Title: "Test1",
|
||||
}
|
||||
obsID1, _, err := obsStore.StoreObservation(ctx, "claude-1", "test-project", obs1, int(sessionID), 1)
|
||||
require.NoError(t, err)
|
||||
|
||||
obs2 := &models.ParsedObservation{
|
||||
Type: models.ObsTypeDiscovery,
|
||||
Title: "Test2",
|
||||
}
|
||||
obsID2, _, err := obsStore.StoreObservation(ctx, "claude-1", "test-project", obs2, int(sessionID), 2)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Create unresolved conflicts
|
||||
now := time.Now()
|
||||
for i := 0; i < 5; i++ {
|
||||
conflict := &models.ObservationConflict{
|
||||
NewerObsID: obsID1,
|
||||
OlderObsID: obsID2,
|
||||
ConflictType: models.ConflictContradicts,
|
||||
Resolution: models.ResolutionPreferNewer,
|
||||
Reason: "reason",
|
||||
DetectedAt: now.Format(time.RFC3339),
|
||||
DetectedAtEpoch: now.UnixMilli(),
|
||||
Resolved: false,
|
||||
}
|
||||
_, err = conflictStore.StoreConflict(ctx, conflict)
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
// Create resolved conflict
|
||||
resolvedAt := now.Format(time.RFC3339)
|
||||
resolvedConflict := &models.ObservationConflict{
|
||||
NewerObsID: obsID1,
|
||||
OlderObsID: obsID2,
|
||||
ConflictType: models.ConflictContradicts,
|
||||
Resolution: models.ResolutionPreferNewer,
|
||||
Reason: "reason",
|
||||
DetectedAt: now.Format(time.RFC3339),
|
||||
DetectedAtEpoch: now.UnixMilli(),
|
||||
Resolved: true,
|
||||
ResolvedAt: &resolvedAt,
|
||||
}
|
||||
_, err = conflictStore.StoreConflict(ctx, resolvedConflict)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Get unresolved conflicts with limit
|
||||
conflicts, err := conflictStore.GetUnresolvedConflicts(ctx, 3)
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, conflicts, 3)
|
||||
|
||||
// Verify all are unresolved
|
||||
for _, c := range conflicts {
|
||||
assert.False(t, c.Resolved)
|
||||
}
|
||||
}
|
||||
|
||||
func TestConflictStore_GetSupersededObservationIDs(t *testing.T) {
|
||||
conflictStore, store, cleanup := testConflictStore(t)
|
||||
defer cleanup()
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
// Create observations
|
||||
sessionStore := NewSessionStore(store)
|
||||
sessionID, err := sessionStore.CreateSDKSession(ctx, "claude-1", "test-project", "")
|
||||
require.NoError(t, err)
|
||||
|
||||
obsStore := NewObservationStore(store, nil, nil, nil)
|
||||
|
||||
// Create newer observations
|
||||
newer1 := &models.ParsedObservation{
|
||||
Type: models.ObsTypeDiscovery,
|
||||
Title: "Newer1",
|
||||
}
|
||||
newerID1, _, err := obsStore.StoreObservation(ctx, "claude-1", "test-project", newer1, int(sessionID), 1)
|
||||
require.NoError(t, err)
|
||||
|
||||
newer2 := &models.ParsedObservation{
|
||||
Type: models.ObsTypeDiscovery,
|
||||
Title: "Newer2",
|
||||
}
|
||||
newerID2, _, err := obsStore.StoreObservation(ctx, "claude-1", "test-project", newer2, int(sessionID), 2)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Create older observations
|
||||
older1 := &models.ParsedObservation{
|
||||
Type: models.ObsTypeDiscovery,
|
||||
Title: "Older1",
|
||||
}
|
||||
olderID1, _, err := obsStore.StoreObservation(ctx, "claude-1", "test-project", older1, int(sessionID), 3)
|
||||
require.NoError(t, err)
|
||||
|
||||
older2 := &models.ParsedObservation{
|
||||
Type: models.ObsTypeDiscovery,
|
||||
Title: "Older2",
|
||||
}
|
||||
olderID2, _, err := obsStore.StoreObservation(ctx, "claude-1", "test-project", older2, int(sessionID), 4)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Mark older observations as superseded
|
||||
err = conflictStore.MarkObservationsSuperseded(ctx, []int64{olderID1, olderID2})
|
||||
require.NoError(t, err)
|
||||
|
||||
// Create conflicts with prefer_newer resolution
|
||||
now := time.Now()
|
||||
conflict1 := &models.ObservationConflict{
|
||||
NewerObsID: newerID1,
|
||||
OlderObsID: olderID1,
|
||||
ConflictType: models.ConflictSuperseded,
|
||||
Resolution: models.ResolutionPreferNewer,
|
||||
Reason: "reason1",
|
||||
DetectedAt: now.Format(time.RFC3339),
|
||||
DetectedAtEpoch: now.UnixMilli(),
|
||||
}
|
||||
_, err = conflictStore.StoreConflict(ctx, conflict1)
|
||||
require.NoError(t, err)
|
||||
|
||||
conflict2 := &models.ObservationConflict{
|
||||
NewerObsID: newerID2,
|
||||
OlderObsID: olderID2,
|
||||
ConflictType: models.ConflictSuperseded,
|
||||
Resolution: models.ResolutionPreferNewer,
|
||||
Reason: "reason2",
|
||||
DetectedAt: now.Format(time.RFC3339),
|
||||
DetectedAtEpoch: now.UnixMilli(),
|
||||
}
|
||||
_, err = conflictStore.StoreConflict(ctx, conflict2)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Get superseded IDs (should return older observation IDs)
|
||||
ids, err := conflictStore.GetSupersededObservationIDs(ctx, "test-project")
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, ids, 2)
|
||||
assert.Contains(t, ids, olderID1)
|
||||
assert.Contains(t, ids, olderID2)
|
||||
}
|
||||
|
||||
func TestConflictStore_ResolveConflict(t *testing.T) {
|
||||
conflictStore, _, cleanup := testConflictStore(t)
|
||||
defer cleanup()
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
// Create a simple conflict by inserting directly to DB
|
||||
conflict := &ObservationConflict{
|
||||
NewerObsID: 1,
|
||||
OlderObsID: 2,
|
||||
ConflictType: models.ConflictContradicts,
|
||||
Resolution: models.ResolutionManual,
|
||||
DetectedAt: time.Now().Format(time.RFC3339),
|
||||
DetectedAtEpoch: time.Now().UnixMilli(),
|
||||
Resolved: 0,
|
||||
}
|
||||
conflictStore.db.Create(conflict)
|
||||
|
||||
// Resolve conflict
|
||||
err := conflictStore.ResolveConflict(ctx, conflict.ID, models.ResolutionPreferNewer)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Verify resolved
|
||||
var resolved ObservationConflict
|
||||
conflictStore.db.First(&resolved, conflict.ID)
|
||||
assert.Equal(t, 1, resolved.Resolved)
|
||||
assert.True(t, resolved.ResolvedAt.Valid)
|
||||
assert.Equal(t, models.ResolutionPreferNewer, resolved.Resolution)
|
||||
}
|
||||
|
||||
func TestConflictStore_DeleteConflictsByObservationID(t *testing.T) {
|
||||
conflictStore, _, cleanup := testConflictStore(t)
|
||||
defer cleanup()
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
// Create conflicts directly in DB
|
||||
now := time.Now()
|
||||
conflicts := []ObservationConflict{
|
||||
{
|
||||
NewerObsID: 1,
|
||||
OlderObsID: 2,
|
||||
ConflictType: models.ConflictContradicts,
|
||||
Resolution: models.ResolutionPreferNewer,
|
||||
DetectedAt: now.Format(time.RFC3339),
|
||||
DetectedAtEpoch: now.UnixMilli(),
|
||||
},
|
||||
{
|
||||
NewerObsID: 3,
|
||||
OlderObsID: 1,
|
||||
ConflictType: models.ConflictContradicts,
|
||||
Resolution: models.ResolutionPreferNewer,
|
||||
DetectedAt: now.Format(time.RFC3339),
|
||||
DetectedAtEpoch: now.UnixMilli(),
|
||||
},
|
||||
{
|
||||
NewerObsID: 2,
|
||||
OlderObsID: 3,
|
||||
ConflictType: models.ConflictContradicts,
|
||||
Resolution: models.ResolutionPreferNewer,
|
||||
DetectedAt: now.Format(time.RFC3339),
|
||||
DetectedAtEpoch: now.UnixMilli(),
|
||||
},
|
||||
}
|
||||
for _, c := range conflicts {
|
||||
conflictStore.db.Create(&c)
|
||||
}
|
||||
|
||||
// Delete conflicts for observation 1
|
||||
err := conflictStore.DeleteConflictsByObservationID(ctx, 1)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Verify only conflicts involving 1 are deleted
|
||||
var count int64
|
||||
conflictStore.db.Model(&ObservationConflict{}).
|
||||
Where("newer_obs_id = 1 OR older_obs_id = 1").
|
||||
Count(&count)
|
||||
assert.Equal(t, int64(0), count)
|
||||
|
||||
// Other conflict should still exist
|
||||
conflictStore.db.Model(&ObservationConflict{}).
|
||||
Where("newer_obs_id = 2 AND older_obs_id = 3").
|
||||
Count(&count)
|
||||
assert.Equal(t, int64(1), count)
|
||||
}
|
||||
|
||||
func TestConflictStore_CleanupSupersededObservations(t *testing.T) {
|
||||
conflictStore, store, cleanup := testConflictStore(t)
|
||||
defer cleanup()
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
// Create observations
|
||||
sessionStore := NewSessionStore(store)
|
||||
sessionID, err := sessionStore.CreateSDKSession(ctx, "claude-1", "test-project", "")
|
||||
require.NoError(t, err)
|
||||
|
||||
obsStore := NewObservationStore(store, nil, nil, nil)
|
||||
|
||||
// Create newer observations
|
||||
newer1 := &models.ParsedObservation{
|
||||
Type: models.ObsTypeDiscovery,
|
||||
Title: "Newer1",
|
||||
}
|
||||
newerID1, _, err := obsStore.StoreObservation(ctx, "claude-1", "test-project", newer1, int(sessionID), 1)
|
||||
require.NoError(t, err)
|
||||
|
||||
newer2 := &models.ParsedObservation{
|
||||
Type: models.ObsTypeDiscovery,
|
||||
Title: "Newer2",
|
||||
}
|
||||
newerID2, _, err := obsStore.StoreObservation(ctx, "claude-1", "test-project", newer2, int(sessionID), 2)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Create older observations
|
||||
older1 := &models.ParsedObservation{
|
||||
Type: models.ObsTypeDiscovery,
|
||||
Title: "OldSuperseded",
|
||||
}
|
||||
oldSupersededID, _, err := obsStore.StoreObservation(ctx, "claude-1", "test-project", older1, int(sessionID), 3)
|
||||
require.NoError(t, err)
|
||||
|
||||
older2 := &models.ParsedObservation{
|
||||
Type: models.ObsTypeDiscovery,
|
||||
Title: "RecentSuperseded",
|
||||
}
|
||||
recentSupersededID, _, err := obsStore.StoreObservation(ctx, "claude-1", "test-project", older2, int(sessionID), 4)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Mark as superseded
|
||||
err = conflictStore.MarkObservationsSuperseded(ctx, []int64{oldSupersededID, recentSupersededID})
|
||||
require.NoError(t, err)
|
||||
|
||||
// Create conflicts
|
||||
oldTime := time.Now().AddDate(0, 0, -SupersededRetentionDays-1)
|
||||
recentTime := time.Now().AddDate(0, 0, -1)
|
||||
|
||||
// Old conflict (should be deleted)
|
||||
oldConflict := &models.ObservationConflict{
|
||||
NewerObsID: newerID1,
|
||||
OlderObsID: oldSupersededID,
|
||||
ConflictType: models.ConflictSuperseded,
|
||||
Resolution: models.ResolutionPreferNewer,
|
||||
Reason: "old",
|
||||
DetectedAt: oldTime.Format(time.RFC3339),
|
||||
DetectedAtEpoch: oldTime.UnixMilli(),
|
||||
}
|
||||
_, err = conflictStore.StoreConflict(ctx, oldConflict)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Recent conflict (should be kept)
|
||||
recentConflict := &models.ObservationConflict{
|
||||
NewerObsID: newerID2,
|
||||
OlderObsID: recentSupersededID,
|
||||
ConflictType: models.ConflictSuperseded,
|
||||
Resolution: models.ResolutionPreferNewer,
|
||||
Reason: "recent",
|
||||
DetectedAt: recentTime.Format(time.RFC3339),
|
||||
DetectedAtEpoch: recentTime.UnixMilli(),
|
||||
}
|
||||
_, err = conflictStore.StoreConflict(ctx, recentConflict)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Cleanup old superseded observations
|
||||
deletedIDs, err := conflictStore.CleanupSupersededObservations(ctx, "test-project")
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, deletedIDs, 1)
|
||||
assert.Contains(t, deletedIDs, oldSupersededID)
|
||||
|
||||
// Verify only old superseded observation was deleted
|
||||
var count int64
|
||||
store.DB.Model(&Observation{}).Count(&count)
|
||||
assert.Equal(t, int64(3), count) // newer1, newer2, recentSuperseded remain
|
||||
|
||||
// Verify old observation was deleted
|
||||
store.DB.Model(&Observation{}).Where("id = ?", oldSupersededID).Count(&count)
|
||||
assert.Equal(t, int64(0), count)
|
||||
}
|
||||
|
||||
func TestConflictStore_GetConflictsWithDetails(t *testing.T) {
|
||||
conflictStore, store, cleanup := testConflictStore(t)
|
||||
defer cleanup()
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
// Create observations
|
||||
sessionStore := NewSessionStore(store)
|
||||
sessionID, err := sessionStore.CreateSDKSession(ctx, "claude-1", "test-project", "")
|
||||
require.NoError(t, err)
|
||||
|
||||
obsStore := NewObservationStore(store, nil, nil, nil)
|
||||
|
||||
newer := &models.ParsedObservation{
|
||||
Type: models.ObsTypeDiscovery,
|
||||
Title: "Newer observation",
|
||||
}
|
||||
newerID, _, err := obsStore.StoreObservation(ctx, "claude-1", "test-project", newer, int(sessionID), 1)
|
||||
require.NoError(t, err)
|
||||
|
||||
older := &models.ParsedObservation{
|
||||
Type: models.ObsTypeDiscovery,
|
||||
Title: "Older observation",
|
||||
}
|
||||
olderID, _, err := obsStore.StoreObservation(ctx, "claude-1", "test-project", older, int(sessionID), 2)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Create conflict
|
||||
now := time.Now()
|
||||
conflict := &models.ObservationConflict{
|
||||
NewerObsID: newerID,
|
||||
OlderObsID: olderID,
|
||||
ConflictType: models.ConflictContradicts,
|
||||
Resolution: models.ResolutionPreferNewer,
|
||||
Reason: "Test conflict",
|
||||
DetectedAt: now.Format(time.RFC3339),
|
||||
DetectedAtEpoch: now.UnixMilli(),
|
||||
}
|
||||
_, err = conflictStore.StoreConflict(ctx, conflict)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Get conflicts with details
|
||||
conflicts, err := conflictStore.GetConflictsWithDetails(ctx, "test-project", 10)
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, conflicts, 1)
|
||||
|
||||
// Verify conflict details
|
||||
assert.Equal(t, newerID, conflicts[0].Conflict.NewerObsID)
|
||||
assert.Equal(t, olderID, conflicts[0].Conflict.OlderObsID)
|
||||
assert.Equal(t, models.ConflictContradicts, conflicts[0].Conflict.ConflictType)
|
||||
assert.Equal(t, "Test conflict", conflicts[0].Conflict.Reason)
|
||||
assert.Equal(t, "Newer observation", conflicts[0].NewerObsTitle)
|
||||
assert.Equal(t, "Older observation", conflicts[0].OlderObsTitle)
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
// Package gorm provides a GORM-based database implementation for claude-mnemonic.
|
||||
//
|
||||
// This is a drop-in replacement for internal/db/sqlite with the following benefits:
|
||||
// - 50% code reduction (8,500 → 4,250 lines)
|
||||
// - Type-safe query building
|
||||
// - Automatic statement caching
|
||||
// - Same performance characteristics
|
||||
// - Zero breaking changes
|
||||
//
|
||||
// Status: Production-ready, not yet integrated
|
||||
//
|
||||
// # Integration
|
||||
//
|
||||
// To use this package instead of internal/db/sqlite:
|
||||
//
|
||||
// import "github.com/lukaszraczylo/claude-mnemonic/internal/db/gorm"
|
||||
//
|
||||
// store, err := gorm.NewStore(gorm.Config{
|
||||
// Path: "/path/to/database.db",
|
||||
// MaxConns: 4,
|
||||
// LogLevel: logger.Silent,
|
||||
// })
|
||||
//
|
||||
// See INTEGRATION_GUIDE.md for complete migration instructions.
|
||||
//
|
||||
// # Testing
|
||||
//
|
||||
// All tests require the fts5 build tag:
|
||||
//
|
||||
// go test -tags "fts5" -v ./internal/db/gorm
|
||||
//
|
||||
// # Performance
|
||||
//
|
||||
// See PERFORMANCE.md for detailed benchmark results.
|
||||
package gorm
|
||||
|
||||
// This file exists for package documentation and to prevent deadcode warnings
|
||||
// on an intentionally unused (but complete and tested) implementation.
|
||||
@@ -0,0 +1,42 @@
|
||||
//go:build fts5
|
||||
|
||||
package gorm
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
_ "github.com/mattn/go-sqlite3"
|
||||
)
|
||||
|
||||
// TestFTS5Available verifies FTS5 is available in mattn/go-sqlite3
|
||||
func TestFTS5Available(t *testing.T) {
|
||||
tmpDir, err := os.MkdirTemp("", "fts5_test_*")
|
||||
if err != nil {
|
||||
t.Fatalf("create temp dir: %v", err)
|
||||
}
|
||||
defer os.RemoveAll(tmpDir)
|
||||
|
||||
dbPath := filepath.Join(tmpDir, "test.db")
|
||||
|
||||
// Open with mattn/go-sqlite3 driver
|
||||
db, err := sql.Open("sqlite3", dbPath)
|
||||
if err != nil {
|
||||
t.Fatalf("open database: %v", err)
|
||||
}
|
||||
defer db.Close()
|
||||
|
||||
// Try to create FTS5 virtual table
|
||||
_, err = db.Exec(`
|
||||
CREATE VIRTUAL TABLE test_fts USING fts5(
|
||||
content
|
||||
)
|
||||
`)
|
||||
if err != nil {
|
||||
t.Fatalf("create FTS5 table failed: %v", err)
|
||||
}
|
||||
|
||||
t.Logf("✅ FTS5 is available in mattn/go-sqlite3")
|
||||
}
|
||||
@@ -0,0 +1,64 @@
|
||||
// Package gorm provides GORM-based database operations for claude-mnemonic.
|
||||
package gorm
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
// EnsureSessionExists creates a session if it doesn't exist.
|
||||
// This is shared between stores to avoid duplication.
|
||||
func EnsureSessionExists(ctx context.Context, db *gorm.DB, sdkSessionID, project string) error {
|
||||
// Check if session exists
|
||||
var count int64
|
||||
err := db.WithContext(ctx).
|
||||
Model(&SDKSession{}).
|
||||
Where("sdk_session_id = ?", sdkSessionID).
|
||||
Count(&count).Error
|
||||
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if count > 0 {
|
||||
return nil // Session exists
|
||||
}
|
||||
|
||||
// Auto-create session
|
||||
now := time.Now()
|
||||
session := &SDKSession{
|
||||
ClaudeSessionID: sdkSessionID,
|
||||
SDKSessionID: sqlNullString(sdkSessionID),
|
||||
Project: project,
|
||||
Status: "active",
|
||||
StartedAt: now.Format(time.RFC3339),
|
||||
StartedAtEpoch: now.UnixMilli(),
|
||||
PromptCounter: 0,
|
||||
}
|
||||
|
||||
return db.WithContext(ctx).Create(session).Error
|
||||
}
|
||||
|
||||
// sqlNullString creates a sql.NullString from a string.
|
||||
func sqlNullString(s string) sql.NullString {
|
||||
if s == "" {
|
||||
return sql.NullString{Valid: false}
|
||||
}
|
||||
return sql.NullString{String: s, Valid: true}
|
||||
}
|
||||
|
||||
// ParseLimitParam parses the "limit" query parameter from an HTTP request.
|
||||
// Returns defaultLimit if the parameter is missing or invalid.
|
||||
func ParseLimitParam(r *http.Request, defaultLimit int) int {
|
||||
if l := r.URL.Query().Get("limit"); l != "" {
|
||||
if parsed, err := strconv.Atoi(l); err == nil && parsed > 0 {
|
||||
return parsed
|
||||
}
|
||||
}
|
||||
return defaultLimit
|
||||
}
|
||||
@@ -0,0 +1,343 @@
|
||||
//go:build fts5
|
||||
|
||||
// Package gorm provides GORM-based database operations for claude-mnemonic.
|
||||
package gorm
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"gorm.io/gorm/logger"
|
||||
|
||||
"github.com/lukaszraczylo/claude-mnemonic/pkg/models"
|
||||
)
|
||||
|
||||
// TestIntegration_EndToEndWorkflow verifies a complete workflow
|
||||
// simulating real usage of the GORM package.
|
||||
func TestIntegration_EndToEndWorkflow(t *testing.T) {
|
||||
tmpDir, err := os.MkdirTemp("", "gorm_integration_test_*")
|
||||
require.NoError(t, err)
|
||||
defer os.RemoveAll(tmpDir)
|
||||
|
||||
dbPath := filepath.Join(tmpDir, "test.db")
|
||||
cfg := Config{
|
||||
Path: dbPath,
|
||||
MaxConns: 4,
|
||||
LogLevel: logger.Silent,
|
||||
}
|
||||
|
||||
// Step 1: Initialize store
|
||||
store, err := NewStore(cfg)
|
||||
require.NoError(t, err)
|
||||
defer store.Close()
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
// Step 2: Create all store types
|
||||
sessionStore := NewSessionStore(store)
|
||||
summaryStore := NewSummaryStore(store)
|
||||
conflictStore := NewConflictStore(store)
|
||||
relationStore := NewRelationStore(store)
|
||||
patternStore := NewPatternStore(store)
|
||||
|
||||
// Create observation store with dependencies
|
||||
observationStore := NewObservationStore(store, nil, conflictStore, relationStore)
|
||||
promptStore := NewPromptStore(store, nil)
|
||||
|
||||
// Step 3: Create a session
|
||||
sessionID, err := sessionStore.CreateSDKSession(ctx, "claude-test", "test-project", "")
|
||||
require.NoError(t, err)
|
||||
assert.Greater(t, sessionID, int64(0))
|
||||
|
||||
// Step 4: Store observations
|
||||
obs1 := &models.ParsedObservation{
|
||||
Type: models.ObsTypeDiscovery,
|
||||
Title: "Test Discovery",
|
||||
Subtitle: "Testing GORM integration",
|
||||
Facts: []string{"Fact 1", "Fact 2"},
|
||||
Concepts: []string{"testing", "integration"},
|
||||
}
|
||||
|
||||
obsID1, _, err := observationStore.StoreObservation(ctx, "claude-test", "test-project", obs1, int(sessionID), 1)
|
||||
require.NoError(t, err)
|
||||
assert.Greater(t, obsID1, int64(0))
|
||||
|
||||
obs2 := &models.ParsedObservation{
|
||||
Type: models.ObsTypeBugfix,
|
||||
Title: "Test Bugfix",
|
||||
Facts: []string{"Fixed bug"},
|
||||
Concepts: []string{"bugfix"},
|
||||
}
|
||||
|
||||
obsID2, _, err := observationStore.StoreObservation(ctx, "claude-test", "test-project", obs2, int(sessionID), 2)
|
||||
require.NoError(t, err)
|
||||
assert.Greater(t, obsID2, int64(0))
|
||||
|
||||
// Step 5: Create relations
|
||||
now := time.Now()
|
||||
relation := &models.ObservationRelation{
|
||||
SourceID: obsID1,
|
||||
TargetID: obsID2,
|
||||
RelationType: models.RelationCauses,
|
||||
Confidence: 0.8,
|
||||
DetectionSource: models.DetectionSourceFileOverlap,
|
||||
CreatedAt: now.Format(time.RFC3339),
|
||||
CreatedAtEpoch: now.UnixMilli(),
|
||||
}
|
||||
|
||||
relID, err := relationStore.StoreRelation(ctx, relation)
|
||||
require.NoError(t, err)
|
||||
assert.Greater(t, relID, int64(0))
|
||||
|
||||
// Step 6: Update importance scores
|
||||
err = observationStore.UpdateImportanceScore(ctx, obsID1, 5.0)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Step 7: Increment retrieval counts
|
||||
err = observationStore.IncrementRetrievalCount(ctx, []int64{obsID1, obsID2})
|
||||
require.NoError(t, err)
|
||||
|
||||
// Step 8: Create a pattern
|
||||
pattern := &models.Pattern{
|
||||
Name: "Test Pattern",
|
||||
Type: models.PatternTypeBug,
|
||||
Signature: []string{"bug", "fix"},
|
||||
Frequency: 1,
|
||||
Projects: []string{"test-project"},
|
||||
ObservationIDs: []int64{obsID1, obsID2},
|
||||
Status: models.PatternStatusActive,
|
||||
Confidence: 0.75,
|
||||
LastSeenAt: now.Format(time.RFC3339),
|
||||
LastSeenEpoch: now.UnixMilli(),
|
||||
CreatedAt: now.Format(time.RFC3339),
|
||||
CreatedAtEpoch: now.UnixMilli(),
|
||||
}
|
||||
|
||||
patternID, err := patternStore.StorePattern(ctx, pattern)
|
||||
require.NoError(t, err)
|
||||
assert.Greater(t, patternID, int64(0))
|
||||
|
||||
// Step 9: Store a prompt
|
||||
promptID, err := promptStore.SaveUserPromptWithMatches(ctx, "claude-test", 1, "Test prompt", 2)
|
||||
require.NoError(t, err)
|
||||
assert.Greater(t, promptID, int64(0))
|
||||
|
||||
// Step 10: Store a summary
|
||||
summary := &models.ParsedSummary{
|
||||
Request: "Test request",
|
||||
Investigated: "Test investigation",
|
||||
Learned: "Test learning",
|
||||
Completed: "Test completion",
|
||||
NextSteps: "Test next steps",
|
||||
Notes: "Test notes",
|
||||
}
|
||||
|
||||
summaryID, _, err := summaryStore.StoreSummary(ctx, "claude-test", "test-project", summary, 1, 100)
|
||||
require.NoError(t, err)
|
||||
assert.Greater(t, summaryID, int64(0))
|
||||
|
||||
// Step 11: Verify data retrieval
|
||||
retrievedObs, err := observationStore.GetObservationByID(ctx, obsID1)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, retrievedObs)
|
||||
assert.Equal(t, "Test Discovery", retrievedObs.Title.String)
|
||||
assert.Equal(t, 5.0, retrievedObs.ImportanceScore)
|
||||
assert.Equal(t, 1, retrievedObs.RetrievalCount)
|
||||
|
||||
// Step 12: Verify relations
|
||||
relations, err := relationStore.GetRelationsByObservationID(ctx, obsID1)
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, relations, 1)
|
||||
assert.Equal(t, obsID2, relations[0].TargetID)
|
||||
|
||||
// Step 13: Verify pattern
|
||||
retrievedPattern, err := patternStore.GetPatternByID(ctx, patternID)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, retrievedPattern)
|
||||
assert.Equal(t, "Test Pattern", retrievedPattern.Name)
|
||||
|
||||
// Step 14: Verify stats
|
||||
stats, err := observationStore.GetObservationFeedbackStats(ctx, "test-project")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 2, stats.Total)
|
||||
|
||||
t.Log("✅ End-to-end integration test passed!")
|
||||
}
|
||||
|
||||
// TestIntegration_StoreCompatibility verifies that Store methods work correctly.
|
||||
func TestIntegration_StoreCompatibility(t *testing.T) {
|
||||
tmpDir, err := os.MkdirTemp("", "gorm_store_test_*")
|
||||
require.NoError(t, err)
|
||||
defer os.RemoveAll(tmpDir)
|
||||
|
||||
dbPath := filepath.Join(tmpDir, "test.db")
|
||||
cfg := Config{
|
||||
Path: dbPath,
|
||||
MaxConns: 4,
|
||||
LogLevel: logger.Silent,
|
||||
}
|
||||
|
||||
store, err := NewStore(cfg)
|
||||
require.NoError(t, err)
|
||||
defer store.Close()
|
||||
|
||||
// Verify raw DB access (needed for vector client)
|
||||
rawDB := store.GetRawDB()
|
||||
require.NotNil(t, rawDB)
|
||||
assert.IsType(t, &sql.DB{}, rawDB)
|
||||
|
||||
// Verify GORM DB access
|
||||
gormDB := store.GetDB()
|
||||
require.NotNil(t, gormDB)
|
||||
|
||||
// Verify Close works
|
||||
err = store.Close()
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
// TestIntegration_ConcurrentAccess verifies thread-safe operations.
|
||||
func TestIntegration_ConcurrentAccess(t *testing.T) {
|
||||
tmpDir, err := os.MkdirTemp("", "gorm_concurrent_test_*")
|
||||
require.NoError(t, err)
|
||||
defer os.RemoveAll(tmpDir)
|
||||
|
||||
dbPath := filepath.Join(tmpDir, "test.db")
|
||||
cfg := Config{
|
||||
Path: dbPath,
|
||||
MaxConns: 4,
|
||||
LogLevel: logger.Silent,
|
||||
}
|
||||
|
||||
store, err := NewStore(cfg)
|
||||
require.NoError(t, err)
|
||||
defer store.Close()
|
||||
|
||||
sessionStore := NewSessionStore(store)
|
||||
ctx := context.Background()
|
||||
|
||||
// Create session
|
||||
sessionID, err := sessionStore.CreateSDKSession(ctx, "claude-concurrent", "test-project", "")
|
||||
require.NoError(t, err)
|
||||
|
||||
// Concurrent prompt counter increments
|
||||
done := make(chan bool)
|
||||
numGoroutines := 10
|
||||
|
||||
for i := 0; i < numGoroutines; i++ {
|
||||
go func() {
|
||||
_, err := sessionStore.IncrementPromptCounter(ctx, sessionID)
|
||||
assert.NoError(t, err)
|
||||
done <- true
|
||||
}()
|
||||
}
|
||||
|
||||
// Wait for all goroutines
|
||||
for i := 0; i < numGoroutines; i++ {
|
||||
<-done
|
||||
}
|
||||
|
||||
// Verify final count
|
||||
session, err := sessionStore.GetSessionByID(ctx, sessionID)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, int64(numGoroutines), int64(session.PromptCounter))
|
||||
|
||||
t.Log("✅ Concurrent access test passed!")
|
||||
}
|
||||
|
||||
// TestIntegration_WALMode verifies WAL mode is enabled.
|
||||
func TestIntegration_WALMode(t *testing.T) {
|
||||
tmpDir, err := os.MkdirTemp("", "gorm_wal_test_*")
|
||||
require.NoError(t, err)
|
||||
defer os.RemoveAll(tmpDir)
|
||||
|
||||
dbPath := filepath.Join(tmpDir, "test.db")
|
||||
cfg := Config{
|
||||
Path: dbPath,
|
||||
MaxConns: 4,
|
||||
LogLevel: logger.Silent,
|
||||
}
|
||||
|
||||
store, err := NewStore(cfg)
|
||||
require.NoError(t, err)
|
||||
defer store.Close()
|
||||
|
||||
// Check WAL mode via raw SQL
|
||||
var journalMode string
|
||||
err = store.GetRawDB().QueryRow("PRAGMA journal_mode").Scan(&journalMode)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "wal", journalMode, "WAL mode should be enabled")
|
||||
|
||||
t.Log("✅ WAL mode verification passed!")
|
||||
}
|
||||
|
||||
// TestIntegration_FTS5Search verifies FTS5 functionality.
|
||||
func TestIntegration_FTS5Search(t *testing.T) {
|
||||
tmpDir, err := os.MkdirTemp("", "gorm_fts5_test_*")
|
||||
require.NoError(t, err)
|
||||
defer os.RemoveAll(tmpDir)
|
||||
|
||||
dbPath := filepath.Join(tmpDir, "test.db")
|
||||
cfg := Config{
|
||||
Path: dbPath,
|
||||
MaxConns: 4,
|
||||
LogLevel: logger.Silent,
|
||||
}
|
||||
|
||||
store, err := NewStore(cfg)
|
||||
require.NoError(t, err)
|
||||
defer store.Close()
|
||||
|
||||
ctx := context.Background()
|
||||
sessionStore := NewSessionStore(store)
|
||||
observationStore := NewObservationStore(store, nil, nil, nil)
|
||||
|
||||
// Create session
|
||||
sessionID, _ := sessionStore.CreateSDKSession(ctx, "claude-fts5", "test-project", "")
|
||||
|
||||
// Store observations with searchable text
|
||||
obs1 := &models.ParsedObservation{
|
||||
Type: models.ObsTypeDiscovery,
|
||||
Title: "Database optimization techniques",
|
||||
Subtitle: "Improving query performance",
|
||||
Facts: []string{"Use indexes", "Optimize queries"},
|
||||
Concepts: []string{"performance", "optimization"},
|
||||
}
|
||||
|
||||
obsID1, _, _ := observationStore.StoreObservation(ctx, "claude-fts5", "test-project", obs1, int(sessionID), 1)
|
||||
|
||||
obs2 := &models.ParsedObservation{
|
||||
Type: models.ObsTypeBugfix,
|
||||
Title: "Fixed memory leak",
|
||||
Facts: []string{"Closed connections properly"},
|
||||
Concepts: []string{"bugfix", "memory"},
|
||||
}
|
||||
|
||||
observationStore.StoreObservation(ctx, "claude-fts5", "test-project", obs2, int(sessionID), 2)
|
||||
|
||||
// Give FTS5 triggers time to process
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
|
||||
// Search using FTS5
|
||||
results, err := observationStore.SearchObservationsFTS(ctx, "optimization", "test-project", 10)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Should find the optimization observation
|
||||
assert.NotEmpty(t, results, "FTS5 search should return results")
|
||||
|
||||
found := false
|
||||
for _, obs := range results {
|
||||
if obs.ID == obsID1 {
|
||||
found = true
|
||||
break
|
||||
}
|
||||
}
|
||||
assert.True(t, found, "FTS5 should find the optimization observation")
|
||||
|
||||
t.Log("✅ FTS5 search test passed!")
|
||||
}
|
||||
@@ -0,0 +1,332 @@
|
||||
// Package gorm provides GORM-based database operations for claude-mnemonic.
|
||||
package gorm
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/go-gormigrate/gormigrate/v2"
|
||||
"gorm.io/gorm"
|
||||
"gorm.io/gorm/clause"
|
||||
)
|
||||
|
||||
// runMigrations runs all database migrations using gormigrate.
|
||||
func runMigrations(db *gorm.DB, sqlDB *sql.DB) error {
|
||||
m := gormigrate.New(db, gormigrate.DefaultOptions, []*gormigrate.Migration{
|
||||
// Migration 001: Core tables (SDKSession, Observation, SessionSummary)
|
||||
{
|
||||
ID: "001_core_tables",
|
||||
Migrate: func(tx *gorm.DB) error {
|
||||
// AutoMigrate creates tables with all indexes from struct tags
|
||||
if err := tx.AutoMigrate(&SDKSession{}); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := tx.AutoMigrate(&Observation{}); err != nil {
|
||||
return err
|
||||
}
|
||||
return tx.AutoMigrate(&SessionSummary{})
|
||||
},
|
||||
Rollback: func(tx *gorm.DB) error {
|
||||
return tx.Migrator().DropTable("sdk_sessions", "observations", "session_summaries")
|
||||
},
|
||||
},
|
||||
|
||||
// Migration 002: User prompts table
|
||||
{
|
||||
ID: "002_user_prompts",
|
||||
Migrate: func(tx *gorm.DB) error {
|
||||
return tx.AutoMigrate(&UserPrompt{})
|
||||
},
|
||||
Rollback: func(tx *gorm.DB) error {
|
||||
return tx.Migrator().DropTable("user_prompts")
|
||||
},
|
||||
},
|
||||
|
||||
// Migration 003: FTS5 virtual table for user prompts
|
||||
{
|
||||
ID: "003_user_prompts_fts",
|
||||
Migrate: func(tx *gorm.DB) error {
|
||||
sqls := []string{
|
||||
`CREATE VIRTUAL TABLE IF NOT EXISTS user_prompts_fts USING fts5(
|
||||
prompt_text,
|
||||
content='user_prompts',
|
||||
content_rowid='id'
|
||||
)`,
|
||||
`CREATE TRIGGER IF NOT EXISTS user_prompts_ai AFTER INSERT ON user_prompts BEGIN
|
||||
INSERT INTO user_prompts_fts(rowid, prompt_text)
|
||||
VALUES (new.id, new.prompt_text);
|
||||
END`,
|
||||
`CREATE TRIGGER IF NOT EXISTS user_prompts_ad AFTER DELETE ON user_prompts BEGIN
|
||||
INSERT INTO user_prompts_fts(user_prompts_fts, rowid, prompt_text)
|
||||
VALUES('delete', old.id, old.prompt_text);
|
||||
END`,
|
||||
`CREATE TRIGGER IF NOT EXISTS user_prompts_au AFTER UPDATE ON user_prompts BEGIN
|
||||
INSERT INTO user_prompts_fts(user_prompts_fts, rowid, prompt_text)
|
||||
VALUES('delete', old.id, old.prompt_text);
|
||||
INSERT INTO user_prompts_fts(rowid, prompt_text)
|
||||
VALUES (new.id, new.prompt_text);
|
||||
END`,
|
||||
}
|
||||
for _, s := range sqls {
|
||||
if err := tx.Exec(s).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
},
|
||||
Rollback: func(tx *gorm.DB) error {
|
||||
sqls := []string{
|
||||
"DROP TRIGGER IF EXISTS user_prompts_au",
|
||||
"DROP TRIGGER IF EXISTS user_prompts_ad",
|
||||
"DROP TRIGGER IF EXISTS user_prompts_ai",
|
||||
"DROP TABLE IF EXISTS user_prompts_fts",
|
||||
}
|
||||
for _, s := range sqls {
|
||||
if err := tx.Exec(s).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
},
|
||||
},
|
||||
|
||||
// Migration 004: FTS5 virtual table for observations
|
||||
{
|
||||
ID: "004_observations_fts",
|
||||
Migrate: func(tx *gorm.DB) error {
|
||||
sqls := []string{
|
||||
`CREATE VIRTUAL TABLE IF NOT EXISTS observations_fts USING fts5(
|
||||
title, subtitle, narrative,
|
||||
content='observations',
|
||||
content_rowid='id'
|
||||
)`,
|
||||
`CREATE TRIGGER IF NOT EXISTS observations_ai AFTER INSERT ON observations BEGIN
|
||||
INSERT INTO observations_fts(rowid, title, subtitle, narrative)
|
||||
VALUES (new.id, new.title, new.subtitle, new.narrative);
|
||||
END`,
|
||||
`CREATE TRIGGER IF NOT EXISTS observations_ad AFTER DELETE ON observations BEGIN
|
||||
INSERT INTO observations_fts(observations_fts, rowid, title, subtitle, narrative)
|
||||
VALUES('delete', old.id, old.title, old.subtitle, old.narrative);
|
||||
END`,
|
||||
`CREATE TRIGGER IF NOT EXISTS observations_au AFTER UPDATE ON observations BEGIN
|
||||
INSERT INTO observations_fts(observations_fts, rowid, title, subtitle, narrative)
|
||||
VALUES('delete', old.id, old.title, old.subtitle, old.narrative);
|
||||
INSERT INTO observations_fts(rowid, title, subtitle, narrative)
|
||||
VALUES (new.id, new.title, new.subtitle, new.narrative);
|
||||
END`,
|
||||
}
|
||||
for _, s := range sqls {
|
||||
if err := tx.Exec(s).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
},
|
||||
Rollback: func(tx *gorm.DB) error {
|
||||
sqls := []string{
|
||||
"DROP TRIGGER IF EXISTS observations_au",
|
||||
"DROP TRIGGER IF EXISTS observations_ad",
|
||||
"DROP TRIGGER IF EXISTS observations_ai",
|
||||
"DROP TABLE IF EXISTS observations_fts",
|
||||
}
|
||||
for _, s := range sqls {
|
||||
if err := tx.Exec(s).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
},
|
||||
},
|
||||
|
||||
// Migration 005: FTS5 virtual table for session summaries
|
||||
{
|
||||
ID: "005_session_summaries_fts",
|
||||
Migrate: func(tx *gorm.DB) error {
|
||||
sqls := []string{
|
||||
`CREATE VIRTUAL TABLE IF NOT EXISTS session_summaries_fts USING fts5(
|
||||
request, investigated, learned, completed, next_steps, notes,
|
||||
content='session_summaries',
|
||||
content_rowid='id'
|
||||
)`,
|
||||
`CREATE TRIGGER IF NOT EXISTS session_summaries_ai AFTER INSERT ON session_summaries BEGIN
|
||||
INSERT INTO session_summaries_fts(rowid, request, investigated, learned, completed, next_steps, notes)
|
||||
VALUES (new.id, new.request, new.investigated, new.learned, new.completed, new.next_steps, new.notes);
|
||||
END`,
|
||||
`CREATE TRIGGER IF NOT EXISTS session_summaries_ad AFTER DELETE ON session_summaries BEGIN
|
||||
INSERT INTO session_summaries_fts(session_summaries_fts, rowid, request, investigated, learned, completed, next_steps, notes)
|
||||
VALUES('delete', old.id, old.request, old.investigated, old.learned, old.completed, old.next_steps, old.notes);
|
||||
END`,
|
||||
`CREATE TRIGGER IF NOT EXISTS session_summaries_au AFTER UPDATE ON session_summaries BEGIN
|
||||
INSERT INTO session_summaries_fts(session_summaries_fts, rowid, request, investigated, learned, completed, next_steps, notes)
|
||||
VALUES('delete', old.id, old.request, old.investigated, old.learned, old.completed, old.next_steps, old.notes);
|
||||
INSERT INTO session_summaries_fts(rowid, request, investigated, learned, completed, next_steps, notes)
|
||||
VALUES (new.id, new.request, new.investigated, new.learned, new.completed, new.next_steps, new.notes);
|
||||
END`,
|
||||
}
|
||||
for _, s := range sqls {
|
||||
if err := tx.Exec(s).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
},
|
||||
Rollback: func(tx *gorm.DB) error {
|
||||
sqls := []string{
|
||||
"DROP TRIGGER IF EXISTS session_summaries_au",
|
||||
"DROP TRIGGER IF EXISTS session_summaries_ad",
|
||||
"DROP TRIGGER IF EXISTS session_summaries_ai",
|
||||
"DROP TABLE IF EXISTS session_summaries_fts",
|
||||
}
|
||||
for _, s := range sqls {
|
||||
if err := tx.Exec(s).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
},
|
||||
},
|
||||
|
||||
// Migration 006: sqlite-vec vectors table
|
||||
{
|
||||
ID: "006_sqlite_vec_vectors",
|
||||
Migrate: func(tx *gorm.DB) error {
|
||||
// Note: Uses bge-small-en-v1.5 embeddings (384 dimensions) with model_version
|
||||
sql := `CREATE VIRTUAL TABLE IF NOT EXISTS vectors USING vec0(
|
||||
doc_id TEXT PRIMARY KEY,
|
||||
embedding float[384],
|
||||
sqlite_id INTEGER,
|
||||
doc_type TEXT,
|
||||
field_type TEXT,
|
||||
project TEXT,
|
||||
scope TEXT,
|
||||
model_version TEXT
|
||||
)`
|
||||
return tx.Exec(sql).Error
|
||||
},
|
||||
Rollback: func(tx *gorm.DB) error {
|
||||
return tx.Exec("DROP TABLE IF EXISTS vectors").Error
|
||||
},
|
||||
},
|
||||
|
||||
// Migration 007: Concept weights table with seed data
|
||||
{
|
||||
ID: "007_concept_weights",
|
||||
Migrate: func(tx *gorm.DB) error {
|
||||
if err := tx.AutoMigrate(&ConceptWeight{}); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Seed default concept weights
|
||||
now := time.Now().Format(time.RFC3339)
|
||||
weights := []ConceptWeight{
|
||||
{Concept: "security", Weight: 0.30, UpdatedAt: now},
|
||||
{Concept: "gotcha", Weight: 0.25, UpdatedAt: now},
|
||||
{Concept: "best-practice", Weight: 0.20, UpdatedAt: now},
|
||||
{Concept: "anti-pattern", Weight: 0.20, UpdatedAt: now},
|
||||
{Concept: "architecture", Weight: 0.15, UpdatedAt: now},
|
||||
{Concept: "performance", Weight: 0.15, UpdatedAt: now},
|
||||
{Concept: "error-handling", Weight: 0.15, UpdatedAt: now},
|
||||
{Concept: "pattern", Weight: 0.10, UpdatedAt: now},
|
||||
{Concept: "testing", Weight: 0.10, UpdatedAt: now},
|
||||
{Concept: "debugging", Weight: 0.10, UpdatedAt: now},
|
||||
{Concept: "workflow", Weight: 0.05, UpdatedAt: now},
|
||||
{Concept: "tooling", Weight: 0.05, UpdatedAt: now},
|
||||
}
|
||||
|
||||
// INSERT OR IGNORE equivalent in GORM
|
||||
return tx.Clauses(clause.OnConflict{DoNothing: true}).Create(&weights).Error
|
||||
},
|
||||
Rollback: func(tx *gorm.DB) error {
|
||||
return tx.Migrator().DropTable("concept_weights")
|
||||
},
|
||||
},
|
||||
|
||||
// Migration 008: Observation conflicts table
|
||||
{
|
||||
ID: "008_observation_conflicts",
|
||||
Migrate: func(tx *gorm.DB) error {
|
||||
return tx.AutoMigrate(&ObservationConflict{})
|
||||
},
|
||||
Rollback: func(tx *gorm.DB) error {
|
||||
return tx.Migrator().DropTable("observation_conflicts")
|
||||
},
|
||||
},
|
||||
|
||||
// Migration 009: Patterns table
|
||||
{
|
||||
ID: "009_patterns",
|
||||
Migrate: func(tx *gorm.DB) error {
|
||||
return tx.AutoMigrate(&Pattern{})
|
||||
},
|
||||
Rollback: func(tx *gorm.DB) error {
|
||||
return tx.Migrator().DropTable("patterns")
|
||||
},
|
||||
},
|
||||
|
||||
// Migration 010: FTS5 virtual table for patterns
|
||||
{
|
||||
ID: "010_patterns_fts",
|
||||
Migrate: func(tx *gorm.DB) error {
|
||||
sqls := []string{
|
||||
`CREATE VIRTUAL TABLE IF NOT EXISTS patterns_fts USING fts5(
|
||||
name, description, recommendation,
|
||||
content='patterns',
|
||||
content_rowid='id'
|
||||
)`,
|
||||
`CREATE TRIGGER IF NOT EXISTS patterns_ai AFTER INSERT ON patterns BEGIN
|
||||
INSERT INTO patterns_fts(rowid, name, description, recommendation)
|
||||
VALUES (new.id, new.name, new.description, new.recommendation);
|
||||
END`,
|
||||
`CREATE TRIGGER IF NOT EXISTS patterns_ad AFTER DELETE ON patterns BEGIN
|
||||
INSERT INTO patterns_fts(patterns_fts, rowid, name, description, recommendation)
|
||||
VALUES('delete', old.id, old.name, old.description, old.recommendation);
|
||||
END`,
|
||||
`CREATE TRIGGER IF NOT EXISTS patterns_au AFTER UPDATE ON patterns BEGIN
|
||||
INSERT INTO patterns_fts(patterns_fts, rowid, name, description, recommendation)
|
||||
VALUES('delete', old.id, old.name, old.description, old.recommendation);
|
||||
INSERT INTO patterns_fts(rowid, name, description, recommendation)
|
||||
VALUES (new.id, new.name, new.description, new.recommendation);
|
||||
END`,
|
||||
}
|
||||
for _, s := range sqls {
|
||||
if err := tx.Exec(s).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
},
|
||||
Rollback: func(tx *gorm.DB) error {
|
||||
sqls := []string{
|
||||
"DROP TRIGGER IF EXISTS patterns_au",
|
||||
"DROP TRIGGER IF EXISTS patterns_ad",
|
||||
"DROP TRIGGER IF EXISTS patterns_ai",
|
||||
"DROP TABLE IF EXISTS patterns_fts",
|
||||
}
|
||||
for _, s := range sqls {
|
||||
if err := tx.Exec(s).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
},
|
||||
},
|
||||
|
||||
// Migration 011: Observation relations table
|
||||
{
|
||||
ID: "011_observation_relations",
|
||||
Migrate: func(tx *gorm.DB) error {
|
||||
return tx.AutoMigrate(&ObservationRelation{})
|
||||
},
|
||||
Rollback: func(tx *gorm.DB) error {
|
||||
return tx.Migrator().DropTable("observation_relations")
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
if err := m.Migrate(); err != nil {
|
||||
return fmt.Errorf("run gormigrate migrations: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,274 @@
|
||||
// Package gorm provides GORM-based database operations for claude-mnemonic.
|
||||
package gorm
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"time"
|
||||
|
||||
"gorm.io/gorm"
|
||||
|
||||
"github.com/lukaszraczylo/claude-mnemonic/pkg/models"
|
||||
)
|
||||
|
||||
// GORM Models
|
||||
|
||||
// Note: JSON types (JSONStringArray, JSONInt64Map) are imported from pkg/models
|
||||
// and already implement sql.Scanner and driver.Valuer interfaces.
|
||||
|
||||
// SDKSession represents a Claude Code session.
|
||||
type SDKSession struct {
|
||||
ID int64 `gorm:"primaryKey;autoIncrement"`
|
||||
ClaudeSessionID string `gorm:"uniqueIndex;not null"`
|
||||
SDKSessionID sql.NullString `gorm:"uniqueIndex"`
|
||||
Project string `gorm:"index;not null"`
|
||||
UserPrompt sql.NullString
|
||||
WorkerPort sql.NullInt64
|
||||
PromptCounter int `gorm:"default:0"`
|
||||
Status string `gorm:"type:text;check:status IN ('active', 'completed', 'failed');default:'active';index"`
|
||||
StartedAt string `gorm:"not null"`
|
||||
StartedAtEpoch int64 `gorm:"index:idx_sessions_started,sort:desc;not null"`
|
||||
CompletedAt sql.NullString
|
||||
CompletedAtEpoch sql.NullInt64
|
||||
}
|
||||
|
||||
func (SDKSession) TableName() string { return "sdk_sessions" }
|
||||
|
||||
// BeforeCreate hook to ensure timestamps are set.
|
||||
func (s *SDKSession) BeforeCreate(tx *gorm.DB) error {
|
||||
if s.StartedAtEpoch == 0 {
|
||||
s.StartedAtEpoch = time.Now().UnixMilli()
|
||||
}
|
||||
if s.StartedAt == "" {
|
||||
s.StartedAt = time.Now().Format(time.RFC3339)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Observation represents a stored observation (learning).
|
||||
type Observation struct {
|
||||
ID int64 `gorm:"primaryKey;autoIncrement"`
|
||||
SDKSessionID string `gorm:"index;not null"`
|
||||
Project string `gorm:"index;not null"`
|
||||
Scope models.ObservationScope `gorm:"type:text;default:'project';check:scope IN ('project', 'global');index:idx_observations_scope;index:idx_observations_project_scope,priority:2"`
|
||||
Type models.ObservationType `gorm:"type:text;check:type IN ('decision', 'bugfix', 'feature', 'refactor', 'discovery', 'change');index;not null"`
|
||||
|
||||
// Content fields
|
||||
Title sql.NullString `gorm:"type:text"`
|
||||
Subtitle sql.NullString `gorm:"type:text"`
|
||||
Facts models.JSONStringArray `gorm:"type:text"` // JSON array
|
||||
Narrative sql.NullString `gorm:"type:text"`
|
||||
Concepts models.JSONStringArray `gorm:"type:text"` // JSON array
|
||||
FilesRead models.JSONStringArray `gorm:"type:text"` // JSON array
|
||||
FilesModified models.JSONStringArray `gorm:"type:text"` // JSON array
|
||||
FileMtimes models.JSONInt64Map `gorm:"type:text"` // JSON object
|
||||
|
||||
// Metadata
|
||||
PromptNumber sql.NullInt64
|
||||
DiscoveryTokens int64 `gorm:"default:0"`
|
||||
CreatedAt string `gorm:"not null"`
|
||||
CreatedAtEpoch int64 `gorm:"index:idx_observations_created,sort:desc;not null"`
|
||||
|
||||
// Importance scoring fields
|
||||
ImportanceScore float64 `gorm:"type:real;default:1.0;index:idx_observations_importance,priority:1,sort:desc"`
|
||||
UserFeedback int `gorm:"default:0"`
|
||||
RetrievalCount int `gorm:"default:0"`
|
||||
LastRetrievedAt sql.NullInt64 `gorm:"column:last_retrieved_at_epoch"`
|
||||
ScoreUpdatedAt sql.NullInt64 `gorm:"column:score_updated_at_epoch;index:idx_observations_score_updated"`
|
||||
IsSuperseded int `gorm:"default:0;index:idx_observations_superseded,priority:1"`
|
||||
}
|
||||
|
||||
func (Observation) TableName() string { return "observations" }
|
||||
|
||||
// BeforeCreate hook to ensure defaults are set.
|
||||
func (o *Observation) BeforeCreate(tx *gorm.DB) error {
|
||||
if o.CreatedAtEpoch == 0 {
|
||||
o.CreatedAtEpoch = time.Now().UnixMilli()
|
||||
}
|
||||
if o.CreatedAt == "" {
|
||||
o.CreatedAt = time.Now().Format(time.RFC3339)
|
||||
}
|
||||
if o.ImportanceScore == 0 {
|
||||
o.ImportanceScore = 1.0
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// SessionSummary represents a session summary.
|
||||
type SessionSummary struct {
|
||||
ID int64 `gorm:"primaryKey;autoIncrement"`
|
||||
SDKSessionID string `gorm:"index;not null"`
|
||||
Project string `gorm:"index;not null"`
|
||||
|
||||
// Summary fields (nullable TEXT)
|
||||
Request sql.NullString
|
||||
Investigated sql.NullString
|
||||
Learned sql.NullString
|
||||
Completed sql.NullString
|
||||
NextSteps sql.NullString `gorm:"column:next_steps"`
|
||||
Notes sql.NullString
|
||||
|
||||
// Metadata
|
||||
PromptNumber sql.NullInt64
|
||||
DiscoveryTokens int64 `gorm:"default:0"`
|
||||
CreatedAt string `gorm:"not null"`
|
||||
CreatedAtEpoch int64 `gorm:"index:idx_summaries_created,sort:desc;not null"`
|
||||
}
|
||||
|
||||
func (SessionSummary) TableName() string { return "session_summaries" }
|
||||
|
||||
// BeforeCreate hook to ensure timestamps are set.
|
||||
func (s *SessionSummary) BeforeCreate(tx *gorm.DB) error {
|
||||
if s.CreatedAtEpoch == 0 {
|
||||
s.CreatedAtEpoch = time.Now().UnixMilli()
|
||||
}
|
||||
if s.CreatedAt == "" {
|
||||
s.CreatedAt = time.Now().Format(time.RFC3339)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// UserPrompt represents a user prompt.
|
||||
type UserPrompt struct {
|
||||
ID int64 `gorm:"primaryKey;autoIncrement"`
|
||||
ClaudeSessionID string `gorm:"index;not null;uniqueIndex:idx_user_prompts_session_number_unique,priority:1"`
|
||||
PromptNumber int `gorm:"index;not null;uniqueIndex:idx_user_prompts_session_number_unique,priority:2"`
|
||||
PromptText string `gorm:"type:text;not null"`
|
||||
MatchedObservations int `gorm:"default:0"`
|
||||
CreatedAt string `gorm:"not null"`
|
||||
CreatedAtEpoch int64 `gorm:"index:idx_prompts_created,sort:desc;not null"`
|
||||
}
|
||||
|
||||
func (UserPrompt) TableName() string { return "user_prompts" }
|
||||
|
||||
// BeforeCreate hook to ensure timestamps are set.
|
||||
func (p *UserPrompt) BeforeCreate(tx *gorm.DB) error {
|
||||
if p.CreatedAtEpoch == 0 {
|
||||
p.CreatedAtEpoch = time.Now().UnixMilli()
|
||||
}
|
||||
if p.CreatedAt == "" {
|
||||
p.CreatedAt = time.Now().Format(time.RFC3339)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ObservationConflict tracks conflicts between observations.
|
||||
type ObservationConflict struct {
|
||||
ID int64 `gorm:"primaryKey;autoIncrement"`
|
||||
NewerObsID int64 `gorm:"index:idx_conflicts_newer;not null"`
|
||||
OlderObsID int64 `gorm:"index:idx_conflicts_older;not null"`
|
||||
ConflictType models.ConflictType `gorm:"type:text;check:conflict_type IN ('superseded', 'contradicts', 'outdated_pattern');not null"`
|
||||
Resolution models.ConflictResolution `gorm:"type:text;check:resolution IN ('prefer_newer', 'prefer_older', 'manual');not null"`
|
||||
Reason sql.NullString `gorm:"type:text"`
|
||||
DetectedAt string `gorm:"not null"`
|
||||
DetectedAtEpoch int64 `gorm:"index:idx_conflicts_unresolved,priority:2,sort:desc;not null"`
|
||||
Resolved int `gorm:"default:0;index:idx_conflicts_unresolved,priority:1"`
|
||||
ResolvedAt sql.NullString
|
||||
}
|
||||
|
||||
func (ObservationConflict) TableName() string { return "observation_conflicts" }
|
||||
|
||||
// BeforeCreate hook to ensure timestamps are set.
|
||||
func (c *ObservationConflict) BeforeCreate(tx *gorm.DB) error {
|
||||
if c.DetectedAtEpoch == 0 {
|
||||
c.DetectedAtEpoch = time.Now().UnixMilli()
|
||||
}
|
||||
if c.DetectedAt == "" {
|
||||
c.DetectedAt = time.Now().Format(time.RFC3339)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ObservationRelation tracks relationships between observations.
|
||||
type ObservationRelation struct {
|
||||
ID int64 `gorm:"primaryKey;autoIncrement"`
|
||||
SourceID int64 `gorm:"index:idx_relations_source;index:idx_relations_both,priority:1;uniqueIndex:idx_relations_unique,priority:1;not null"`
|
||||
TargetID int64 `gorm:"index:idx_relations_target;index:idx_relations_both,priority:2;uniqueIndex:idx_relations_unique,priority:2;not null"`
|
||||
RelationType models.RelationType `gorm:"type:text;check:relation_type IN ('causes', 'fixes', 'supersedes', 'depends_on', 'relates_to', 'evolves_from');index:idx_relations_type;uniqueIndex:idx_relations_unique,priority:3;not null"`
|
||||
Confidence float64 `gorm:"type:real;default:0.5;index:idx_relations_confidence,sort:desc;not null"`
|
||||
DetectionSource models.RelationDetectionSource `gorm:"type:text;check:detection_source IN ('file_overlap', 'embedding_similarity', 'temporal_proximity', 'narrative_mention', 'concept_overlap', 'type_progression');not null"`
|
||||
Reason sql.NullString `gorm:"type:text"`
|
||||
CreatedAt string `gorm:"not null"`
|
||||
CreatedAtEpoch int64 `gorm:"not null"`
|
||||
}
|
||||
|
||||
func (ObservationRelation) TableName() string { return "observation_relations" }
|
||||
|
||||
// BeforeCreate hook to ensure timestamps are set.
|
||||
func (r *ObservationRelation) BeforeCreate(tx *gorm.DB) error {
|
||||
if r.CreatedAtEpoch == 0 {
|
||||
r.CreatedAtEpoch = time.Now().UnixMilli()
|
||||
}
|
||||
if r.CreatedAt == "" {
|
||||
r.CreatedAt = time.Now().Format(time.RFC3339)
|
||||
}
|
||||
if r.Confidence == 0 {
|
||||
r.Confidence = 0.5
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Pattern represents a detected recurring pattern.
|
||||
type Pattern struct {
|
||||
ID int64 `gorm:"primaryKey;autoIncrement"`
|
||||
Name string `gorm:"type:text;not null"`
|
||||
Type models.PatternType `gorm:"type:text;check:type IN ('bug', 'refactor', 'architecture', 'anti-pattern', 'best-practice');index;not null"`
|
||||
Description sql.NullString `gorm:"type:text"`
|
||||
Signature models.JSONStringArray `gorm:"type:text"` // JSON array of keywords
|
||||
Recommendation sql.NullString `gorm:"type:text"`
|
||||
Frequency int `gorm:"default:1;index:idx_patterns_frequency,sort:desc"`
|
||||
Projects models.JSONStringArray `gorm:"type:text"` // JSON array
|
||||
ObservationIDs models.JSONInt64Array `gorm:"type:text"` // JSON array
|
||||
Status models.PatternStatus `gorm:"type:text;default:'active';check:status IN ('active', 'deprecated', 'merged');index"`
|
||||
MergedIntoID sql.NullInt64
|
||||
Confidence float64 `gorm:"type:real;default:0.5;index:idx_patterns_confidence,sort:desc"`
|
||||
LastSeenAt string `gorm:"not null"`
|
||||
LastSeenAtEpoch int64 `gorm:"index:idx_patterns_last_seen,sort:desc;not null"`
|
||||
CreatedAt string `gorm:"not null"`
|
||||
CreatedAtEpoch int64 `gorm:"not null"`
|
||||
}
|
||||
|
||||
func (Pattern) TableName() string { return "patterns" }
|
||||
|
||||
// BeforeCreate hook to ensure timestamps and defaults are set.
|
||||
func (p *Pattern) BeforeCreate(tx *gorm.DB) error {
|
||||
now := time.Now()
|
||||
if p.CreatedAtEpoch == 0 {
|
||||
p.CreatedAtEpoch = now.UnixMilli()
|
||||
}
|
||||
if p.CreatedAt == "" {
|
||||
p.CreatedAt = now.Format(time.RFC3339)
|
||||
}
|
||||
if p.LastSeenAtEpoch == 0 {
|
||||
p.LastSeenAtEpoch = now.UnixMilli()
|
||||
}
|
||||
if p.LastSeenAt == "" {
|
||||
p.LastSeenAt = now.Format(time.RFC3339)
|
||||
}
|
||||
if p.Confidence == 0 {
|
||||
p.Confidence = 0.5
|
||||
}
|
||||
if p.Frequency == 0 {
|
||||
p.Frequency = 1
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ConceptWeight stores configurable weights for importance scoring.
|
||||
type ConceptWeight struct {
|
||||
Concept string `gorm:"primaryKey;type:text"`
|
||||
Weight float64 `gorm:"type:real;not null;default:0.1"`
|
||||
UpdatedAt string `gorm:"not null"`
|
||||
}
|
||||
|
||||
func (ConceptWeight) TableName() string { return "concept_weights" }
|
||||
|
||||
// BeforeCreate hook to ensure timestamp is set.
|
||||
func (c *ConceptWeight) BeforeCreate(tx *gorm.DB) error {
|
||||
if c.UpdatedAt == "" {
|
||||
c.UpdatedAt = time.Now().Format(time.RFC3339)
|
||||
}
|
||||
if c.Weight == 0 {
|
||||
c.Weight = 0.1
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,563 @@
|
||||
// Package gorm provides GORM-based database operations for claude-mnemonic.
|
||||
package gorm
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gorm.io/gorm"
|
||||
|
||||
"github.com/lukaszraczylo/claude-mnemonic/pkg/models"
|
||||
)
|
||||
|
||||
// MaxObservationsPerProject is the maximum number of observations to keep per project.
|
||||
const MaxObservationsPerProject = 100
|
||||
|
||||
// CleanupFunc is a callback for when observations are cleaned up.
|
||||
// Receives the IDs of deleted observations for downstream cleanup (e.g., vector DB).
|
||||
type CleanupFunc func(ctx context.Context, deletedIDs []int64)
|
||||
|
||||
// ObservationStore provides observation-related database operations using GORM.
|
||||
type ObservationStore struct {
|
||||
db *gorm.DB
|
||||
rawDB *sql.DB
|
||||
cleanupFunc CleanupFunc
|
||||
conflictStore interface{} // Placeholder for ConflictStore (Phase 4)
|
||||
relationStore interface{} // Placeholder for RelationStore (Phase 4)
|
||||
}
|
||||
|
||||
// NewObservationStore creates a new observation store.
|
||||
// The conflictStore and relationStore parameters are optional (can be nil) and will be used in Phase 4.
|
||||
func NewObservationStore(store *Store, cleanupFunc CleanupFunc, conflictStore, relationStore interface{}) *ObservationStore {
|
||||
return &ObservationStore{
|
||||
db: store.DB,
|
||||
rawDB: store.GetRawDB(),
|
||||
cleanupFunc: cleanupFunc,
|
||||
conflictStore: conflictStore,
|
||||
relationStore: relationStore,
|
||||
}
|
||||
}
|
||||
|
||||
// SetCleanupFunc sets the callback for when observations are deleted during cleanup.
|
||||
func (s *ObservationStore) SetCleanupFunc(fn CleanupFunc) {
|
||||
s.cleanupFunc = fn
|
||||
}
|
||||
|
||||
// StoreObservation stores a new observation.
|
||||
func (s *ObservationStore) StoreObservation(ctx context.Context, sdkSessionID, project string, obs *models.ParsedObservation, promptNumber int, discoveryTokens int64) (int64, int64, error) {
|
||||
now := time.Now()
|
||||
nowEpoch := now.UnixMilli()
|
||||
|
||||
// Ensure session exists (auto-create if missing)
|
||||
if err := EnsureSessionExists(ctx, s.db, sdkSessionID, project); err != nil {
|
||||
return 0, 0, err
|
||||
}
|
||||
|
||||
// Determine scope: use parsed scope if set, otherwise auto-determine from concepts
|
||||
scope := obs.Scope
|
||||
if scope == "" {
|
||||
scope = models.DetermineScope(obs.Concepts)
|
||||
}
|
||||
|
||||
dbObs := &Observation{
|
||||
SDKSessionID: sdkSessionID,
|
||||
Project: project,
|
||||
Scope: scope,
|
||||
Type: obs.Type,
|
||||
Title: nullString(obs.Title),
|
||||
Subtitle: nullString(obs.Subtitle),
|
||||
Facts: models.JSONStringArray(obs.Facts),
|
||||
Narrative: nullString(obs.Narrative),
|
||||
Concepts: models.JSONStringArray(obs.Concepts),
|
||||
FilesRead: models.JSONStringArray(obs.FilesRead),
|
||||
FilesModified: models.JSONStringArray(obs.FilesModified),
|
||||
FileMtimes: models.JSONInt64Map(obs.FileMtimes),
|
||||
PromptNumber: nullInt64(promptNumber),
|
||||
DiscoveryTokens: discoveryTokens,
|
||||
CreatedAt: now.Format(time.RFC3339),
|
||||
CreatedAtEpoch: nowEpoch,
|
||||
}
|
||||
|
||||
err := s.db.WithContext(ctx).Create(dbObs).Error
|
||||
if err != nil {
|
||||
return 0, 0, err
|
||||
}
|
||||
|
||||
// Cleanup old observations beyond the limit for this project (async to not block handler)
|
||||
if project != "" {
|
||||
go func(proj string) {
|
||||
cleanupCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
deletedIDs, _ := s.CleanupOldObservations(cleanupCtx, proj)
|
||||
if len(deletedIDs) > 0 && s.cleanupFunc != nil {
|
||||
s.cleanupFunc(cleanupCtx, deletedIDs)
|
||||
}
|
||||
}(project)
|
||||
}
|
||||
|
||||
// Note: Conflict and relation detection intentionally omitted for now
|
||||
// Will be added in Phase 4 when ConflictStore and RelationStore are implemented
|
||||
|
||||
return dbObs.ID, nowEpoch, nil
|
||||
}
|
||||
|
||||
// GetObservationByID retrieves an observation by its ID.
|
||||
func (s *ObservationStore) GetObservationByID(ctx context.Context, id int64) (*models.Observation, error) {
|
||||
var dbObs Observation
|
||||
err := s.db.WithContext(ctx).First(&dbObs, id).Error
|
||||
if err == gorm.ErrRecordNotFound {
|
||||
return nil, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return toModelObservation(&dbObs), nil
|
||||
}
|
||||
|
||||
// GetObservationsByIDs retrieves observations by a list of IDs.
|
||||
func (s *ObservationStore) GetObservationsByIDs(ctx context.Context, ids []int64, orderBy string, limit int) ([]*models.Observation, error) {
|
||||
if len(ids) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
var dbObservations []Observation
|
||||
query := s.db.WithContext(ctx).Where("id IN ?", ids)
|
||||
|
||||
// Apply ordering
|
||||
switch orderBy {
|
||||
case "date_asc":
|
||||
query = query.Order("created_at_epoch ASC")
|
||||
case "date_desc":
|
||||
query = query.Order("created_at_epoch DESC")
|
||||
case "importance":
|
||||
query = query.Order("importance_score DESC, created_at_epoch DESC")
|
||||
default:
|
||||
// Default: importance first, then recency
|
||||
query = query.Order("COALESCE(importance_score, 1.0) DESC, created_at_epoch DESC")
|
||||
}
|
||||
|
||||
// Apply limit
|
||||
if limit > 0 {
|
||||
query = query.Limit(limit)
|
||||
}
|
||||
|
||||
err := query.Find(&dbObservations).Error
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return toModelObservations(dbObservations), nil
|
||||
}
|
||||
|
||||
// GetRecentObservations retrieves recent observations for a project.
|
||||
// This includes project-scoped observations for the specified project AND global observations.
|
||||
// Results are ordered by importance_score DESC, then created_at_epoch DESC.
|
||||
func (s *ObservationStore) GetRecentObservations(ctx context.Context, project string, limit int) ([]*models.Observation, error) {
|
||||
var dbObservations []Observation
|
||||
err := s.db.WithContext(ctx).
|
||||
Scopes(projectScopeFilter(project), importanceOrdering()).
|
||||
Limit(limit).
|
||||
Find(&dbObservations).Error
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return toModelObservations(dbObservations), nil
|
||||
}
|
||||
|
||||
// GetActiveObservations retrieves recent non-superseded observations for a project.
|
||||
// This excludes observations that have been marked as superseded by newer ones.
|
||||
// Results are ordered by importance_score DESC, then created_at_epoch DESC.
|
||||
func (s *ObservationStore) GetActiveObservations(ctx context.Context, project string, limit int) ([]*models.Observation, error) {
|
||||
var dbObservations []Observation
|
||||
err := s.db.WithContext(ctx).
|
||||
Scopes(projectScopeFilter(project), notSupersededFilter(), importanceOrdering()).
|
||||
Limit(limit).
|
||||
Find(&dbObservations).Error
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return toModelObservations(dbObservations), nil
|
||||
}
|
||||
|
||||
// GetSupersededObservations retrieves observations that have been superseded by newer ones.
|
||||
// Results are ordered by created_at_epoch DESC.
|
||||
func (s *ObservationStore) GetSupersededObservations(ctx context.Context, project string, limit int) ([]*models.Observation, error) {
|
||||
var dbObservations []Observation
|
||||
err := s.db.WithContext(ctx).
|
||||
Where("project = ? AND COALESCE(is_superseded, 0) = 1", project).
|
||||
Order("created_at_epoch DESC").
|
||||
Limit(limit).
|
||||
Find(&dbObservations).Error
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return toModelObservations(dbObservations), nil
|
||||
}
|
||||
|
||||
// GetObservationsByProjectStrict retrieves observations for a project (strict - no global observations).
|
||||
func (s *ObservationStore) GetObservationsByProjectStrict(ctx context.Context, project string, limit int) ([]*models.Observation, error) {
|
||||
var dbObservations []Observation
|
||||
err := s.db.WithContext(ctx).
|
||||
Where("project = ?", project).
|
||||
Scopes(importanceOrdering()).
|
||||
Limit(limit).
|
||||
Find(&dbObservations).Error
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return toModelObservations(dbObservations), nil
|
||||
}
|
||||
|
||||
// GetObservationCount returns the count of observations for a project.
|
||||
func (s *ObservationStore) GetObservationCount(ctx context.Context, project string) (int, error) {
|
||||
var count int64
|
||||
err := s.db.WithContext(ctx).
|
||||
Model(&Observation{}).
|
||||
Where("project = ?", project).
|
||||
Count(&count).Error
|
||||
|
||||
return int(count), err
|
||||
}
|
||||
|
||||
// GetAllRecentObservations retrieves recent observations across all projects.
|
||||
func (s *ObservationStore) GetAllRecentObservations(ctx context.Context, limit int) ([]*models.Observation, error) {
|
||||
var dbObservations []Observation
|
||||
err := s.db.WithContext(ctx).
|
||||
Scopes(importanceOrdering()).
|
||||
Limit(limit).
|
||||
Find(&dbObservations).Error
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return toModelObservations(dbObservations), nil
|
||||
}
|
||||
|
||||
// GetAllObservations retrieves all observations (for vector rebuild).
|
||||
func (s *ObservationStore) GetAllObservations(ctx context.Context) ([]*models.Observation, error) {
|
||||
var dbObservations []Observation
|
||||
err := s.db.WithContext(ctx).
|
||||
Order("id").
|
||||
Find(&dbObservations).Error
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return toModelObservations(dbObservations), nil
|
||||
}
|
||||
|
||||
// SearchObservationsFTS performs full-text search on observations using FTS5.
|
||||
// Falls back to LIKE search if FTS5 fails.
|
||||
func (s *ObservationStore) SearchObservationsFTS(ctx context.Context, query, project string, limit int) ([]*models.Observation, error) {
|
||||
if limit <= 0 {
|
||||
limit = 10
|
||||
}
|
||||
|
||||
// Extract keywords from the query
|
||||
keywords := extractKeywords(query)
|
||||
if len(keywords) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// Build FTS5 query: keyword1 OR keyword2 OR keyword3
|
||||
ftsTerms := strings.Join(keywords, " OR ")
|
||||
|
||||
// Use FTS5 via raw SQL (GORM can't handle FTS5 MATCH operator)
|
||||
ftsQuery := `
|
||||
SELECT o.id, o.sdk_session_id, o.project, COALESCE(o.scope, 'project') as scope, o.type,
|
||||
o.title, o.subtitle, o.facts, o.narrative, o.concepts, o.files_read, o.files_modified,
|
||||
o.file_mtimes, o.prompt_number, o.discovery_tokens, o.created_at, o.created_at_epoch,
|
||||
COALESCE(o.importance_score, 1.0) as importance_score,
|
||||
COALESCE(o.user_feedback, 0) as user_feedback,
|
||||
COALESCE(o.retrieval_count, 0) as retrieval_count,
|
||||
o.last_retrieved_at_epoch, o.score_updated_at_epoch,
|
||||
COALESCE(o.is_superseded, 0) as is_superseded
|
||||
FROM observations o
|
||||
JOIN observations_fts fts ON o.id = fts.rowid
|
||||
WHERE observations_fts MATCH ?
|
||||
AND (o.project = ? OR o.scope = 'global')
|
||||
ORDER BY rank, COALESCE(o.importance_score, 1.0) DESC
|
||||
LIMIT ?
|
||||
`
|
||||
|
||||
rows, err := s.rawDB.QueryContext(ctx, ftsQuery, ftsTerms, project, limit)
|
||||
if err != nil {
|
||||
// FTS failed, try LIKE fallback
|
||||
return s.searchObservationsLike(ctx, keywords, project, limit)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
observations, err := scanObservationRows(rows)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// If FTS returned nothing, try LIKE search
|
||||
if len(observations) == 0 {
|
||||
return s.searchObservationsLike(ctx, keywords, project, limit)
|
||||
}
|
||||
|
||||
return observations, nil
|
||||
}
|
||||
|
||||
// searchObservationsLike performs fallback LIKE search on observations using GORM.
|
||||
func (s *ObservationStore) searchObservationsLike(ctx context.Context, keywords []string, project string, limit int) ([]*models.Observation, error) {
|
||||
if len(keywords) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// Build LIKE conditions for each keyword
|
||||
var conditions []string
|
||||
var args []interface{}
|
||||
|
||||
for _, kw := range keywords {
|
||||
pattern := "%" + kw + "%"
|
||||
conditions = append(conditions, "(title LIKE ? OR subtitle LIKE ? OR narrative LIKE ?)")
|
||||
args = append(args, pattern, pattern, pattern)
|
||||
}
|
||||
|
||||
// Build WHERE clause
|
||||
whereClause := strings.Join(conditions, " OR ")
|
||||
fullWhere := "(" + whereClause + ") AND (project = ? OR scope = 'global')"
|
||||
args = append(args, project)
|
||||
|
||||
var dbObservations []Observation
|
||||
err := s.db.WithContext(ctx).
|
||||
Where(fullWhere, args...).
|
||||
Scopes(importanceOrdering()).
|
||||
Limit(limit).
|
||||
Find(&dbObservations).Error
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return toModelObservations(dbObservations), nil
|
||||
}
|
||||
|
||||
// DeleteObservations deletes observations by IDs.
|
||||
func (s *ObservationStore) DeleteObservations(ctx context.Context, ids []int64) (int64, error) {
|
||||
if len(ids) == 0 {
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
result := s.db.WithContext(ctx).Delete(&Observation{}, ids)
|
||||
return result.RowsAffected, result.Error
|
||||
}
|
||||
|
||||
// CleanupOldObservations removes observations beyond the limit for a project.
|
||||
// Returns the IDs of deleted observations.
|
||||
func (s *ObservationStore) CleanupOldObservations(ctx context.Context, project string) ([]int64, error) {
|
||||
// Use a transaction to prevent TOCTOU race condition
|
||||
var idsToDelete []int64
|
||||
|
||||
err := s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
|
||||
// Find IDs to keep (most recent MaxObservationsPerProject)
|
||||
var idsToKeep []int64
|
||||
err := tx.Model(&Observation{}).
|
||||
Where("project = ?", project).
|
||||
Order("created_at_epoch DESC").
|
||||
Limit(MaxObservationsPerProject).
|
||||
Pluck("id", &idsToKeep).Error
|
||||
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if len(idsToKeep) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Find IDs to delete (all IDs not in the keep list)
|
||||
// This happens in the same transaction to prevent race conditions
|
||||
err = tx.Model(&Observation{}).
|
||||
Where("project = ? AND id NOT IN ?", project, idsToKeep).
|
||||
Pluck("id", &idsToDelete).Error
|
||||
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if len(idsToDelete) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Delete the observations
|
||||
return tx.Delete(&Observation{}, idsToDelete).Error
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return idsToDelete, nil
|
||||
}
|
||||
|
||||
// ====================
|
||||
// GORM Scopes (Reusable Query Filters)
|
||||
// ====================
|
||||
|
||||
// projectScopeFilter filters observations by project scope.
|
||||
// Includes project-scoped observations for the specified project AND global observations.
|
||||
func projectScopeFilter(project string) func(*gorm.DB) *gorm.DB {
|
||||
return func(db *gorm.DB) *gorm.DB {
|
||||
return db.Where("(project = ? AND (scope IS NULL OR scope = 'project')) OR scope = 'global'", project)
|
||||
}
|
||||
}
|
||||
|
||||
// notSupersededFilter filters out superseded observations.
|
||||
func notSupersededFilter() func(*gorm.DB) *gorm.DB {
|
||||
return func(db *gorm.DB) *gorm.DB {
|
||||
return db.Where("COALESCE(is_superseded, 0) = 0")
|
||||
}
|
||||
}
|
||||
|
||||
// importanceOrdering orders by importance score DESC, then created_at_epoch DESC.
|
||||
func importanceOrdering() func(*gorm.DB) *gorm.DB {
|
||||
return func(db *gorm.DB) *gorm.DB {
|
||||
return db.Order("COALESCE(importance_score, 1.0) DESC, created_at_epoch DESC")
|
||||
}
|
||||
}
|
||||
|
||||
// ====================
|
||||
// Helper Functions
|
||||
// ====================
|
||||
|
||||
// extractKeywords extracts keywords from a search query.
|
||||
func extractKeywords(query string) []string {
|
||||
words := strings.Fields(strings.ToLower(query))
|
||||
var keywords []string
|
||||
|
||||
commonWords := map[string]bool{
|
||||
"the": true, "and": true, "or": true, "but": true, "in": true,
|
||||
"on": true, "at": true, "to": true, "for": true, "of": true,
|
||||
"with": true, "by": true, "from": true, "as": true, "is": true,
|
||||
"was": true, "are": true, "were": true, "be": true, "been": true,
|
||||
"being": true, "have": true, "has": true, "had": true, "do": true,
|
||||
"does": true, "did": true, "will": true, "would": true, "should": true,
|
||||
"could": true, "may": true, "might": true, "must": true, "can": true,
|
||||
}
|
||||
|
||||
for _, word := range words {
|
||||
// Skip short words and common words
|
||||
if len(word) <= 3 || commonWords[word] {
|
||||
continue
|
||||
}
|
||||
keywords = append(keywords, word)
|
||||
}
|
||||
|
||||
return keywords
|
||||
}
|
||||
|
||||
// scanObservationRows scans multiple observations from raw SQL rows.
|
||||
func scanObservationRows(rows *sql.Rows) ([]*models.Observation, error) {
|
||||
var observations []*models.Observation
|
||||
for rows.Next() {
|
||||
obs, err := scanObservation(rows)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
observations = append(observations, obs)
|
||||
}
|
||||
return observations, rows.Err()
|
||||
}
|
||||
|
||||
// scanObservation scans a single observation from a row scanner.
|
||||
func scanObservation(scanner interface{ Scan(...interface{}) error }) (*models.Observation, error) {
|
||||
var obs models.Observation
|
||||
var factsJSON, conceptsJSON, filesReadJSON, filesModifiedJSON, fileMtimesJSON []byte
|
||||
var isSuperseded int
|
||||
|
||||
err := scanner.Scan(
|
||||
&obs.ID, &obs.SDKSessionID, &obs.Project, &obs.Scope, &obs.Type,
|
||||
&obs.Title, &obs.Subtitle, &factsJSON, &obs.Narrative, &conceptsJSON,
|
||||
&filesReadJSON, &filesModifiedJSON, &fileMtimesJSON,
|
||||
&obs.PromptNumber, &obs.DiscoveryTokens, &obs.CreatedAt, &obs.CreatedAtEpoch,
|
||||
&obs.ImportanceScore, &obs.UserFeedback, &obs.RetrievalCount,
|
||||
&obs.LastRetrievedAt, &obs.ScoreUpdatedAt, &isSuperseded,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Unmarshal JSON fields (data comes from DB, should always be valid)
|
||||
if len(factsJSON) > 0 {
|
||||
_ = json.Unmarshal(factsJSON, &obs.Facts)
|
||||
}
|
||||
if len(conceptsJSON) > 0 {
|
||||
_ = json.Unmarshal(conceptsJSON, &obs.Concepts)
|
||||
}
|
||||
if len(filesReadJSON) > 0 {
|
||||
_ = json.Unmarshal(filesReadJSON, &obs.FilesRead)
|
||||
}
|
||||
if len(filesModifiedJSON) > 0 {
|
||||
_ = json.Unmarshal(filesModifiedJSON, &obs.FilesModified)
|
||||
}
|
||||
if len(fileMtimesJSON) > 0 {
|
||||
_ = json.Unmarshal(fileMtimesJSON, &obs.FileMtimes)
|
||||
}
|
||||
|
||||
// Convert int to bool for IsSuperseded
|
||||
obs.IsSuperseded = isSuperseded != 0
|
||||
|
||||
return &obs, nil
|
||||
}
|
||||
|
||||
// toModelObservation converts a GORM Observation to pkg/models.Observation.
|
||||
func toModelObservation(o *Observation) *models.Observation {
|
||||
return &models.Observation{
|
||||
ID: o.ID,
|
||||
SDKSessionID: o.SDKSessionID,
|
||||
Project: o.Project,
|
||||
Scope: o.Scope,
|
||||
Type: o.Type,
|
||||
Title: o.Title,
|
||||
Subtitle: o.Subtitle,
|
||||
Facts: o.Facts,
|
||||
Narrative: o.Narrative,
|
||||
Concepts: o.Concepts,
|
||||
FilesRead: o.FilesRead,
|
||||
FilesModified: o.FilesModified,
|
||||
FileMtimes: o.FileMtimes,
|
||||
PromptNumber: o.PromptNumber,
|
||||
DiscoveryTokens: o.DiscoveryTokens,
|
||||
CreatedAt: o.CreatedAt,
|
||||
CreatedAtEpoch: o.CreatedAtEpoch,
|
||||
ImportanceScore: o.ImportanceScore,
|
||||
UserFeedback: o.UserFeedback,
|
||||
RetrievalCount: o.RetrievalCount,
|
||||
LastRetrievedAt: o.LastRetrievedAt,
|
||||
ScoreUpdatedAt: o.ScoreUpdatedAt,
|
||||
IsSuperseded: o.IsSuperseded != 0, // Convert int to bool
|
||||
}
|
||||
}
|
||||
|
||||
// toModelObservations converts a slice of GORM Observation to pkg/models.Observation.
|
||||
func toModelObservations(observations []Observation) []*models.Observation {
|
||||
result := make([]*models.Observation, len(observations))
|
||||
for i := range observations {
|
||||
result[i] = toModelObservation(&observations[i])
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
// nullInt64 converts an int to sql.NullInt64.
|
||||
func nullInt64(val int) sql.NullInt64 {
|
||||
if val == 0 {
|
||||
return sql.NullInt64{Valid: false}
|
||||
}
|
||||
return sql.NullInt64{Int64: int64(val), Valid: true}
|
||||
}
|
||||
@@ -0,0 +1,593 @@
|
||||
//go:build fts5
|
||||
|
||||
// Package gorm provides GORM-based database operations for claude-mnemonic.
|
||||
package gorm
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"gorm.io/gorm/logger"
|
||||
|
||||
"github.com/lukaszraczylo/claude-mnemonic/pkg/models"
|
||||
)
|
||||
|
||||
// testObservationStore creates an ObservationStore with a temporary database for testing.
|
||||
func testObservationStore(t *testing.T) (*ObservationStore, *Store, func()) {
|
||||
t.Helper()
|
||||
|
||||
tmpDir, err := os.MkdirTemp("", "gorm_observation_test_*")
|
||||
if err != nil {
|
||||
t.Fatalf("create temp dir: %v", err)
|
||||
}
|
||||
|
||||
dbPath := filepath.Join(tmpDir, "test.db")
|
||||
cfg := Config{
|
||||
Path: dbPath,
|
||||
MaxConns: 4,
|
||||
LogLevel: logger.Silent,
|
||||
}
|
||||
|
||||
store, err := NewStore(cfg)
|
||||
if err != nil {
|
||||
os.RemoveAll(tmpDir)
|
||||
t.Fatalf("NewStore failed: %v", err)
|
||||
}
|
||||
|
||||
observationStore := NewObservationStore(store, nil, nil, nil)
|
||||
|
||||
cleanup := func() {
|
||||
store.Close()
|
||||
os.RemoveAll(tmpDir)
|
||||
}
|
||||
|
||||
return observationStore, store, cleanup
|
||||
}
|
||||
|
||||
func TestObservationStore_StoreObservation(t *testing.T) {
|
||||
observationStore, store, cleanup := testObservationStore(t)
|
||||
defer cleanup()
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
// Create a session first
|
||||
sessionStore := NewSessionStore(store)
|
||||
_, err := sessionStore.CreateSDKSession(ctx, "claude-1", "test-project", "")
|
||||
require.NoError(t, err)
|
||||
|
||||
// Store an observation
|
||||
observation := &models.ParsedObservation{
|
||||
Type: models.ObsTypeDecision,
|
||||
Title: "User prefers tabs over spaces",
|
||||
Narrative: "Observed in code formatting",
|
||||
Concepts: []string{"coding-style", "preferences"},
|
||||
}
|
||||
|
||||
id, epoch, err := observationStore.StoreObservation(ctx, "claude-1", "test-project", observation, 1, 100)
|
||||
require.NoError(t, err)
|
||||
assert.Greater(t, id, int64(0))
|
||||
assert.Greater(t, epoch, int64(0))
|
||||
}
|
||||
|
||||
func TestObservationStore_StoreObservation_AutoCreateSession(t *testing.T) {
|
||||
observationStore, _, cleanup := testObservationStore(t)
|
||||
defer cleanup()
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
// Store observation without pre-creating session
|
||||
observation := &models.ParsedObservation{
|
||||
Type: models.ObsTypeDiscovery,
|
||||
Title: "Test auto-create",
|
||||
}
|
||||
|
||||
id, _, err := observationStore.StoreObservation(ctx, "claude-auto", "auto-project", observation, 1, 50)
|
||||
require.NoError(t, err)
|
||||
assert.Greater(t, id, int64(0))
|
||||
}
|
||||
|
||||
func TestObservationStore_StoreObservation_WithScope(t *testing.T) {
|
||||
observationStore, _, cleanup := testObservationStore(t)
|
||||
defer cleanup()
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
tags []string
|
||||
expectedScope models.ObservationScope
|
||||
}{
|
||||
{
|
||||
name: "Global scope - best practice",
|
||||
tags: []string{"best-practice", "testing"},
|
||||
expectedScope: models.ScopeGlobal,
|
||||
},
|
||||
{
|
||||
name: "Global scope - security",
|
||||
tags: []string{"security", "auth"},
|
||||
expectedScope: models.ScopeGlobal,
|
||||
},
|
||||
{
|
||||
name: "Project scope - specific feature",
|
||||
tags: []string{"feature", "implementation"},
|
||||
expectedScope: models.ScopeProject,
|
||||
},
|
||||
{
|
||||
name: "Project scope - no tags",
|
||||
tags: []string{},
|
||||
expectedScope: models.ScopeProject,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
observation := &models.ParsedObservation{
|
||||
Type: models.ObsTypeDiscovery,
|
||||
Title: "Test scope determination",
|
||||
Concepts: tt.tags,
|
||||
}
|
||||
|
||||
id, _, err := observationStore.StoreObservation(ctx, "claude-1", "test-project", observation, 1, 50)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Verify scope was set correctly
|
||||
observations, err := observationStore.GetObservationsByIDs(ctx, []int64{id}, "default", 10)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, observations, 1)
|
||||
assert.Equal(t, tt.expectedScope, observations[0].Scope)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestObservationStore_StoreObservation_AsyncCleanup(t *testing.T) {
|
||||
observationStore, _, cleanup := testObservationStore(t)
|
||||
defer cleanup()
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
// Track cleanup calls
|
||||
var cleanupMutex sync.Mutex
|
||||
cleanupCalled := false
|
||||
var cleanupIDs []int64
|
||||
|
||||
cleanupFunc := func(ctx context.Context, deletedIDs []int64) {
|
||||
cleanupMutex.Lock()
|
||||
defer cleanupMutex.Unlock()
|
||||
cleanupCalled = true
|
||||
cleanupIDs = deletedIDs
|
||||
}
|
||||
|
||||
observationStore.cleanupFunc = cleanupFunc
|
||||
|
||||
// Store observations beyond the limit (MaxObservationsPerProject = 100)
|
||||
for i := 0; i < 105; i++ {
|
||||
observation := &models.ParsedObservation{
|
||||
Type: models.ObsTypeDiscovery,
|
||||
Title: "Observation",
|
||||
}
|
||||
_, _, err := observationStore.StoreObservation(ctx, "claude-1", "test-project", observation, i, 50)
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
// Wait for async cleanup to complete
|
||||
time.Sleep(200 * time.Millisecond)
|
||||
|
||||
// Verify cleanup was called
|
||||
cleanupMutex.Lock()
|
||||
defer cleanupMutex.Unlock()
|
||||
assert.True(t, cleanupCalled, "Cleanup function should have been called")
|
||||
assert.NotEmpty(t, cleanupIDs, "Cleanup should have deleted some observations")
|
||||
}
|
||||
|
||||
func TestObservationStore_GetObservationsByIDs(t *testing.T) {
|
||||
observationStore, _, cleanup := testObservationStore(t)
|
||||
defer cleanup()
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
// Store multiple observations with different importance scores
|
||||
var ids []int64
|
||||
for i := 1; i <= 3; i++ {
|
||||
observation := &models.ParsedObservation{
|
||||
Type: models.ObsTypeDiscovery,
|
||||
Title: "Test",
|
||||
}
|
||||
id, _, err := observationStore.StoreObservation(ctx, "claude-1", "test-project", observation, i, 10)
|
||||
require.NoError(t, err)
|
||||
ids = append(ids, id)
|
||||
|
||||
// Update importance score directly
|
||||
observationStore.db.Model(&Observation{}).Where("id = ?", id).Update("importance_score", float64(i))
|
||||
time.Sleep(10 * time.Millisecond) // Ensure different timestamps
|
||||
}
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
orderBy string
|
||||
expected []int64
|
||||
}{
|
||||
{
|
||||
name: "Default ordering - importance desc",
|
||||
orderBy: "default",
|
||||
expected: []int64{ids[2], ids[1], ids[0]}, // High to low importance
|
||||
},
|
||||
{
|
||||
name: "Importance ordering",
|
||||
orderBy: "importance",
|
||||
expected: []int64{ids[2], ids[1], ids[0]},
|
||||
},
|
||||
{
|
||||
name: "Date ascending",
|
||||
orderBy: "date_asc",
|
||||
expected: []int64{ids[0], ids[1], ids[2]}, // Oldest to newest
|
||||
},
|
||||
{
|
||||
name: "Date descending",
|
||||
orderBy: "date_desc",
|
||||
expected: []int64{ids[2], ids[1], ids[0]}, // Newest to oldest
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
observations, err := observationStore.GetObservationsByIDs(ctx, ids, tt.orderBy, 10)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, observations, 3)
|
||||
|
||||
// Verify ordering
|
||||
for i, obs := range observations {
|
||||
assert.Equal(t, tt.expected[i], obs.ID, "Position %d should have ID %d", i, tt.expected[i])
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestObservationStore_GetObservationsByIDs_Limit(t *testing.T) {
|
||||
observationStore, _, cleanup := testObservationStore(t)
|
||||
defer cleanup()
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
// Store multiple observations
|
||||
var ids []int64
|
||||
for i := 1; i <= 5; i++ {
|
||||
observation := &models.ParsedObservation{
|
||||
Type: models.ObsTypeDiscovery,
|
||||
Title: "Test",
|
||||
}
|
||||
id, _, err := observationStore.StoreObservation(ctx, "claude-1", "test-project", observation, i, 10)
|
||||
require.NoError(t, err)
|
||||
ids = append(ids, id)
|
||||
}
|
||||
|
||||
// Get with limit
|
||||
observations, err := observationStore.GetObservationsByIDs(ctx, ids, "default", 3)
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, observations, 3)
|
||||
}
|
||||
|
||||
func TestObservationStore_GetObservationsByIDs_EmptyInput(t *testing.T) {
|
||||
observationStore, _, cleanup := testObservationStore(t)
|
||||
defer cleanup()
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
// Get with empty IDs
|
||||
observations, err := observationStore.GetObservationsByIDs(ctx, []int64{}, "default", 10)
|
||||
require.NoError(t, err)
|
||||
assert.Nil(t, observations)
|
||||
}
|
||||
|
||||
func TestObservationStore_GetRecentObservations(t *testing.T) {
|
||||
observationStore, _, cleanup := testObservationStore(t)
|
||||
defer cleanup()
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
// Store project-scoped observations
|
||||
for i := 1; i <= 3; i++ {
|
||||
observation := &models.ParsedObservation{
|
||||
Type: models.ObsTypeDiscovery,
|
||||
Title: "Project A fact",
|
||||
}
|
||||
_, _, err := observationStore.StoreObservation(ctx, "claude-1", "project-a", observation, i, 10)
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
// Store global-scoped observation
|
||||
observation := &models.ParsedObservation{
|
||||
Type: models.ObsTypeDiscovery,
|
||||
Title: "Global best practice",
|
||||
Concepts: []string{"best-practice"},
|
||||
}
|
||||
_, _, err := observationStore.StoreObservation(ctx, "claude-2", "project-b", observation, 1, 10)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Store observation for different project
|
||||
observation = &models.ParsedObservation{
|
||||
Type: models.ObsTypeDiscovery,
|
||||
Title: "Project B fact",
|
||||
}
|
||||
_, _, err = observationStore.StoreObservation(ctx, "claude-2", "project-b", observation, 2, 10)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Wait for any async cleanup to complete before querying
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
|
||||
// Get recent observations for project-a (should include project-a + global)
|
||||
observations, err := observationStore.GetRecentObservations(ctx, "project-a", 10)
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, observations, 4) // 3 project-a + 1 global
|
||||
|
||||
// Verify scope filtering
|
||||
projectCount := 0
|
||||
globalCount := 0
|
||||
for _, obs := range observations {
|
||||
if obs.Scope == models.ScopeProject {
|
||||
assert.Equal(t, "project-a", obs.Project)
|
||||
projectCount++
|
||||
} else if obs.Scope == models.ScopeGlobal {
|
||||
globalCount++
|
||||
}
|
||||
}
|
||||
assert.Equal(t, 3, projectCount)
|
||||
assert.Equal(t, 1, globalCount)
|
||||
}
|
||||
|
||||
func TestObservationStore_GetActiveObservations(t *testing.T) {
|
||||
observationStore, _, cleanup := testObservationStore(t)
|
||||
defer cleanup()
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
// Store active observation
|
||||
activeObs := &models.ParsedObservation{
|
||||
Type: models.ObsTypeDiscovery,
|
||||
Title: "Active observation",
|
||||
}
|
||||
activeID, _, err := observationStore.StoreObservation(ctx, "claude-1", "test-project", activeObs, 1, 10)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Store superseded observation
|
||||
supersededObs := &models.ParsedObservation{
|
||||
Type: models.ObsTypeDiscovery,
|
||||
Title: "Superseded observation",
|
||||
}
|
||||
supersededID, _, err := observationStore.StoreObservation(ctx, "claude-1", "test-project", supersededObs, 2, 10)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Mark as superseded
|
||||
observationStore.db.Model(&Observation{}).Where("id = ?", supersededID).Update("is_superseded", 1)
|
||||
|
||||
// Get active observations (should exclude superseded)
|
||||
observations, err := observationStore.GetActiveObservations(ctx, "test-project", 10)
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, observations, 1)
|
||||
assert.Equal(t, activeID, observations[0].ID)
|
||||
assert.False(t, observations[0].IsSuperseded)
|
||||
}
|
||||
|
||||
func TestObservationStore_GetSupersededObservations(t *testing.T) {
|
||||
observationStore, _, cleanup := testObservationStore(t)
|
||||
defer cleanup()
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
// Store active observation
|
||||
activeObs := &models.ParsedObservation{
|
||||
Type: models.ObsTypeDiscovery,
|
||||
Title: "Active observation",
|
||||
}
|
||||
_, _, err := observationStore.StoreObservation(ctx, "claude-1", "test-project", activeObs, 1, 10)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Store superseded observation
|
||||
supersededObs := &models.ParsedObservation{
|
||||
Type: models.ObsTypeDiscovery,
|
||||
Title: "Superseded observation",
|
||||
}
|
||||
supersededID, _, err := observationStore.StoreObservation(ctx, "claude-1", "test-project", supersededObs, 2, 10)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Mark as superseded
|
||||
observationStore.db.Model(&Observation{}).Where("id = ?", supersededID).Update("is_superseded", 1)
|
||||
|
||||
// Get superseded observations (should exclude active)
|
||||
observations, err := observationStore.GetSupersededObservations(ctx, "test-project", 10)
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, observations, 1)
|
||||
assert.Equal(t, supersededID, observations[0].ID)
|
||||
assert.True(t, observations[0].IsSuperseded)
|
||||
}
|
||||
|
||||
func TestObservationStore_GetObservationsByProjectStrict(t *testing.T) {
|
||||
observationStore, _, cleanup := testObservationStore(t)
|
||||
defer cleanup()
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
// Store project-scoped observations
|
||||
for i := 1; i <= 2; i++ {
|
||||
observation := &models.ParsedObservation{
|
||||
Type: models.ObsTypeDiscovery,
|
||||
Title: "Project A fact",
|
||||
}
|
||||
_, _, err := observationStore.StoreObservation(ctx, "claude-1", "project-a", observation, i, 10)
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
// Store global-scoped observation
|
||||
observation := &models.ParsedObservation{
|
||||
Type: models.ObsTypeDiscovery,
|
||||
Title: "Global best practice",
|
||||
Concepts: []string{"best-practice"},
|
||||
}
|
||||
_, _, err := observationStore.StoreObservation(ctx, "claude-2", "project-b", observation, 1, 10)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Get strict project observations (should exclude global)
|
||||
observations, err := observationStore.GetObservationsByProjectStrict(ctx, "project-a", 10)
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, observations, 2) // Only project-a observations
|
||||
|
||||
// Verify all are project-scoped
|
||||
for _, obs := range observations {
|
||||
assert.Equal(t, models.ScopeProject, obs.Scope)
|
||||
assert.Equal(t, "project-a", obs.Project)
|
||||
}
|
||||
}
|
||||
|
||||
func TestObservationStore_SearchObservationsFTS(t *testing.T) {
|
||||
observationStore, _, cleanup := testObservationStore(t)
|
||||
defer cleanup()
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
// Store observations with searchable content
|
||||
observations := []*models.ParsedObservation{
|
||||
{
|
||||
Type: models.ObsTypeDiscovery,
|
||||
Title: "User prefers React for frontend development",
|
||||
Concepts: []string{"frontend", "react"},
|
||||
},
|
||||
{
|
||||
Type: models.ObsTypeDiscovery,
|
||||
Title: "Backend uses Go with chi router",
|
||||
Concepts: []string{"backend", "golang"},
|
||||
},
|
||||
{
|
||||
Type: models.ObsTypeDiscovery,
|
||||
Title: "Database is SQLite with FTS5",
|
||||
Concepts: []string{"database", "sqlite"},
|
||||
},
|
||||
}
|
||||
|
||||
for i, obs := range observations {
|
||||
_, _, err := observationStore.StoreObservation(ctx, "claude-1", "test-project", obs, i+1, 10)
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
// Wait for FTS5 triggers to fire
|
||||
time.Sleep(200 * time.Millisecond)
|
||||
|
||||
// Search for "React frontend"
|
||||
results, err := observationStore.SearchObservationsFTS(ctx, "React frontend", "test-project", 10)
|
||||
require.NoError(t, err)
|
||||
assert.NotEmpty(t, results, "Should find observations matching 'React frontend'")
|
||||
|
||||
// Verify results contain relevant observation
|
||||
found := false
|
||||
for _, obs := range results {
|
||||
if obs.Title.String == "User prefers React for frontend development" {
|
||||
found = true
|
||||
break
|
||||
}
|
||||
}
|
||||
assert.True(t, found, "Should find the React observation")
|
||||
}
|
||||
|
||||
func TestObservationStore_CleanupOldObservations(t *testing.T) {
|
||||
observationStore, _, cleanup := testObservationStore(t)
|
||||
defer cleanup()
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
// Store observations beyond the limit WITHOUT async cleanup
|
||||
// We disable async cleanup by not setting cleanupFunc
|
||||
var allIDs []int64
|
||||
for i := 0; i < 105; i++ {
|
||||
observation := &models.ParsedObservation{
|
||||
Type: models.ObsTypeDiscovery,
|
||||
Title: "Observation",
|
||||
}
|
||||
id, _, err := observationStore.StoreObservation(ctx, "claude-1", "test-project", observation, i, 10)
|
||||
require.NoError(t, err)
|
||||
allIDs = append(allIDs, id)
|
||||
time.Sleep(2 * time.Millisecond) // Ensure different timestamps
|
||||
}
|
||||
|
||||
// Wait for any async cleanups to complete (even though cleanupFunc is nil)
|
||||
time.Sleep(200 * time.Millisecond)
|
||||
|
||||
// Verify we have 105 observations initially (async cleanup should have run but deleted items)
|
||||
initial, err := observationStore.GetRecentObservations(ctx, "test-project", 200)
|
||||
require.NoError(t, err)
|
||||
|
||||
// If async cleanup already happened, we'll have <= 100
|
||||
// Run cleanup manually to ensure cleanup logic works
|
||||
deletedIDs, err := observationStore.CleanupOldObservations(ctx, "test-project")
|
||||
require.NoError(t, err)
|
||||
|
||||
// After cleanup (manual or async), we should have at most 100
|
||||
remaining, err := observationStore.GetRecentObservations(ctx, "test-project", 200)
|
||||
require.NoError(t, err)
|
||||
assert.LessOrEqual(t, len(remaining), 100, "Should have at most 100 observations after cleanup")
|
||||
|
||||
// The number deleted should match how many were over the limit
|
||||
expectedDeleted := len(initial) - len(remaining)
|
||||
assert.Len(t, deletedIDs, expectedDeleted, "Should delete observations beyond limit")
|
||||
}
|
||||
|
||||
func TestObservationStore_DeleteObservations(t *testing.T) {
|
||||
observationStore, _, cleanup := testObservationStore(t)
|
||||
defer cleanup()
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
// Store multiple observations
|
||||
var ids []int64
|
||||
for i := 1; i <= 5; i++ {
|
||||
observation := &models.ParsedObservation{
|
||||
Type: models.ObsTypeDiscovery,
|
||||
Title: "Test",
|
||||
}
|
||||
id, _, err := observationStore.StoreObservation(ctx, "claude-1", "test-project", observation, i, 10)
|
||||
require.NoError(t, err)
|
||||
ids = append(ids, id)
|
||||
}
|
||||
|
||||
// Delete first 3 observations
|
||||
_, err := observationStore.DeleteObservations(ctx, ids[:3])
|
||||
require.NoError(t, err)
|
||||
|
||||
// Verify only 2 remain
|
||||
remaining, err := observationStore.GetRecentObservations(ctx, "test-project", 10)
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, remaining, 2)
|
||||
|
||||
// Verify deleted observations are gone
|
||||
deleted, err := observationStore.GetObservationsByIDs(ctx, ids[:3], "default", 10)
|
||||
require.NoError(t, err)
|
||||
assert.Empty(t, deleted)
|
||||
}
|
||||
|
||||
// Note: TestObservationStore_MarkObservationsSuperseded is omitted because
|
||||
// MarkObservationsSuperseded is a ConflictStore method (Phase 4), not ObservationStore
|
||||
|
||||
func TestObservationStore_GetAllObservations(t *testing.T) {
|
||||
observationStore, _, cleanup := testObservationStore(t)
|
||||
defer cleanup()
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
// Store observations across projects
|
||||
_, _, err := observationStore.StoreObservation(ctx, "claude-1", "project-a", &models.ParsedObservation{Type: models.ObsTypeDiscovery, Title: "A1"}, 1, 10)
|
||||
require.NoError(t, err)
|
||||
|
||||
_, _, err = observationStore.StoreObservation(ctx, "claude-2", "project-b", &models.ParsedObservation{Type: models.ObsTypeDiscovery, Title: "B1"}, 1, 10)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Get all observations (for vector rebuild)
|
||||
all, err := observationStore.GetAllObservations(ctx)
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, all, 2)
|
||||
|
||||
// Verify ordering by ID
|
||||
assert.Less(t, all[0].ID, all[1].ID)
|
||||
}
|
||||
@@ -0,0 +1,420 @@
|
||||
// Package gorm provides GORM-based database operations for claude-mnemonic.
|
||||
package gorm
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"time"
|
||||
|
||||
"gorm.io/gorm"
|
||||
|
||||
"github.com/lukaszraczylo/claude-mnemonic/pkg/models"
|
||||
)
|
||||
|
||||
// PatternCleanupFunc is a callback for when patterns are deleted.
|
||||
type PatternCleanupFunc func(ctx context.Context, deletedIDs []int64)
|
||||
|
||||
// PatternStore provides pattern-related database operations using GORM.
|
||||
type PatternStore struct {
|
||||
db *gorm.DB
|
||||
cleanupFunc PatternCleanupFunc
|
||||
}
|
||||
|
||||
// NewPatternStore creates a new pattern store.
|
||||
func NewPatternStore(store *Store) *PatternStore {
|
||||
return &PatternStore{
|
||||
db: store.DB,
|
||||
}
|
||||
}
|
||||
|
||||
// SetCleanupFunc sets the callback for when patterns are deleted.
|
||||
func (s *PatternStore) SetCleanupFunc(fn PatternCleanupFunc) {
|
||||
s.cleanupFunc = fn
|
||||
}
|
||||
|
||||
// StorePattern stores a new pattern.
|
||||
func (s *PatternStore) StorePattern(ctx context.Context, pattern *models.Pattern) (int64, error) {
|
||||
dbPattern := &Pattern{
|
||||
Name: pattern.Name,
|
||||
Type: pattern.Type,
|
||||
Signature: pattern.Signature,
|
||||
Frequency: pattern.Frequency,
|
||||
Projects: pattern.Projects,
|
||||
ObservationIDs: pattern.ObservationIDs,
|
||||
Status: pattern.Status,
|
||||
Confidence: pattern.Confidence,
|
||||
LastSeenAt: pattern.LastSeenAt,
|
||||
LastSeenAtEpoch: pattern.LastSeenEpoch,
|
||||
CreatedAt: pattern.CreatedAt,
|
||||
CreatedAtEpoch: pattern.CreatedAtEpoch,
|
||||
}
|
||||
|
||||
if pattern.Description.Valid {
|
||||
dbPattern.Description = sql.NullString{String: pattern.Description.String, Valid: true}
|
||||
}
|
||||
|
||||
if pattern.Recommendation.Valid {
|
||||
dbPattern.Recommendation = sql.NullString{String: pattern.Recommendation.String, Valid: true}
|
||||
}
|
||||
|
||||
if pattern.MergedIntoID.Valid {
|
||||
dbPattern.MergedIntoID = sql.NullInt64{Int64: pattern.MergedIntoID.Int64, Valid: true}
|
||||
}
|
||||
|
||||
result := s.db.WithContext(ctx).Create(dbPattern)
|
||||
if result.Error != nil {
|
||||
return 0, result.Error
|
||||
}
|
||||
|
||||
return dbPattern.ID, nil
|
||||
}
|
||||
|
||||
// UpdatePattern updates an existing pattern.
|
||||
func (s *PatternStore) UpdatePattern(ctx context.Context, pattern *models.Pattern) error {
|
||||
updates := map[string]interface{}{
|
||||
"name": pattern.Name,
|
||||
"type": pattern.Type,
|
||||
"signature": pattern.Signature,
|
||||
"frequency": pattern.Frequency,
|
||||
"projects": pattern.Projects,
|
||||
"observation_ids": pattern.ObservationIDs,
|
||||
"status": pattern.Status,
|
||||
"confidence": pattern.Confidence,
|
||||
"last_seen_at": pattern.LastSeenAt,
|
||||
"last_seen_at_epoch": pattern.LastSeenEpoch,
|
||||
}
|
||||
|
||||
if pattern.Description.Valid {
|
||||
updates["description"] = pattern.Description.String
|
||||
} else {
|
||||
updates["description"] = nil
|
||||
}
|
||||
|
||||
if pattern.Recommendation.Valid {
|
||||
updates["recommendation"] = pattern.Recommendation.String
|
||||
} else {
|
||||
updates["recommendation"] = nil
|
||||
}
|
||||
|
||||
if pattern.MergedIntoID.Valid {
|
||||
updates["merged_into_id"] = pattern.MergedIntoID.Int64
|
||||
} else {
|
||||
updates["merged_into_id"] = nil
|
||||
}
|
||||
|
||||
result := s.db.WithContext(ctx).
|
||||
Model(&Pattern{}).
|
||||
Where("id = ?", pattern.ID).
|
||||
Updates(updates)
|
||||
|
||||
return result.Error
|
||||
}
|
||||
|
||||
// GetPatternByID retrieves a pattern by ID.
|
||||
func (s *PatternStore) GetPatternByID(ctx context.Context, id int64) (*models.Pattern, error) {
|
||||
var dbPattern Pattern
|
||||
|
||||
err := s.db.WithContext(ctx).First(&dbPattern, id).Error
|
||||
if err == gorm.ErrRecordNotFound {
|
||||
return nil, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return toModelPattern(&dbPattern), nil
|
||||
}
|
||||
|
||||
// GetPatternByName retrieves a pattern by name.
|
||||
func (s *PatternStore) GetPatternByName(ctx context.Context, name string) (*models.Pattern, error) {
|
||||
var dbPattern Pattern
|
||||
|
||||
err := s.db.WithContext(ctx).
|
||||
Where("name = ? AND status = ?", name, models.PatternStatusActive).
|
||||
First(&dbPattern).Error
|
||||
|
||||
if err == gorm.ErrRecordNotFound {
|
||||
return nil, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return toModelPattern(&dbPattern), nil
|
||||
}
|
||||
|
||||
// GetActivePatterns retrieves all active patterns.
|
||||
func (s *PatternStore) GetActivePatterns(ctx context.Context, limit int) ([]*models.Pattern, error) {
|
||||
var patterns []Pattern
|
||||
|
||||
err := s.db.WithContext(ctx).
|
||||
Where("status = ?", models.PatternStatusActive).
|
||||
Order("frequency DESC, confidence DESC").
|
||||
Limit(limit).
|
||||
Find(&patterns).Error
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return toModelPatterns(patterns), nil
|
||||
}
|
||||
|
||||
// GetPatternsByType retrieves patterns of a specific type.
|
||||
func (s *PatternStore) GetPatternsByType(ctx context.Context, patternType models.PatternType, limit int) ([]*models.Pattern, error) {
|
||||
var patterns []Pattern
|
||||
|
||||
err := s.db.WithContext(ctx).
|
||||
Where("type = ? AND status = ?", patternType, models.PatternStatusActive).
|
||||
Order("frequency DESC, confidence DESC").
|
||||
Limit(limit).
|
||||
Find(&patterns).Error
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return toModelPatterns(patterns), nil
|
||||
}
|
||||
|
||||
// GetPatternsByProject retrieves patterns that have been observed in a specific project.
|
||||
// Uses raw SQL since JSON_EACH is complex in GORM.
|
||||
func (s *PatternStore) GetPatternsByProject(ctx context.Context, project string, limit int) ([]*models.Pattern, error) {
|
||||
var patterns []Pattern
|
||||
|
||||
// Use raw SQL for JSON_EACH query
|
||||
query := `
|
||||
SELECT * FROM patterns
|
||||
WHERE status = 'active'
|
||||
AND EXISTS (
|
||||
SELECT 1 FROM json_each(projects)
|
||||
WHERE json_each.value = ?
|
||||
)
|
||||
ORDER BY frequency DESC, confidence DESC
|
||||
LIMIT ?
|
||||
`
|
||||
|
||||
err := s.db.WithContext(ctx).
|
||||
Raw(query, project, limit).
|
||||
Scan(&patterns).Error
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return toModelPatterns(patterns), nil
|
||||
}
|
||||
|
||||
// FindMatchingPatterns searches for patterns that match a given signature.
|
||||
// Pattern matching is done in Go code for simplicity.
|
||||
func (s *PatternStore) FindMatchingPatterns(ctx context.Context, signature []string, minScore float64) ([]*models.Pattern, error) {
|
||||
// Get all active patterns
|
||||
patterns, err := s.GetActivePatterns(ctx, 100)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Filter by signature match in Go
|
||||
var matches []*models.Pattern
|
||||
for _, pattern := range patterns {
|
||||
score := models.CalculateMatchScore(signature, pattern.Signature)
|
||||
if score >= minScore {
|
||||
matches = append(matches, pattern)
|
||||
}
|
||||
}
|
||||
|
||||
return matches, nil
|
||||
}
|
||||
|
||||
// MarkPatternDeprecated marks a pattern as deprecated.
|
||||
func (s *PatternStore) MarkPatternDeprecated(ctx context.Context, id int64) error {
|
||||
result := s.db.WithContext(ctx).
|
||||
Model(&Pattern{}).
|
||||
Where("id = ?", id).
|
||||
Update("status", models.PatternStatusDeprecated)
|
||||
|
||||
return result.Error
|
||||
}
|
||||
|
||||
// MergePatterns merges a source pattern into a target pattern.
|
||||
func (s *PatternStore) MergePatterns(ctx context.Context, sourceID, targetID int64) error {
|
||||
// Get both patterns
|
||||
source, err := s.GetPatternByID(ctx, sourceID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
target, err := s.GetPatternByID(ctx, targetID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Merge source into target
|
||||
target.Frequency += source.Frequency
|
||||
|
||||
// Merge projects (deduplicate)
|
||||
for _, proj := range source.Projects {
|
||||
found := false
|
||||
for _, existing := range target.Projects {
|
||||
if existing == proj {
|
||||
found = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
target.Projects = append(target.Projects, proj)
|
||||
}
|
||||
}
|
||||
|
||||
// Merge observation IDs (deduplicate)
|
||||
for _, obsID := range source.ObservationIDs {
|
||||
found := false
|
||||
for _, existing := range target.ObservationIDs {
|
||||
if existing == obsID {
|
||||
found = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
target.ObservationIDs = append(target.ObservationIDs, obsID)
|
||||
}
|
||||
}
|
||||
|
||||
// Update target
|
||||
if err := s.UpdatePattern(ctx, target); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Mark source as merged
|
||||
source.Status = models.PatternStatusMerged
|
||||
source.MergedIntoID = sql.NullInt64{Int64: targetID, Valid: true}
|
||||
return s.UpdatePattern(ctx, source)
|
||||
}
|
||||
|
||||
// DeletePattern deletes a pattern by ID.
|
||||
func (s *PatternStore) DeletePattern(ctx context.Context, id int64) error {
|
||||
result := s.db.WithContext(ctx).Delete(&Pattern{}, id)
|
||||
|
||||
if result.Error == nil && s.cleanupFunc != nil {
|
||||
s.cleanupFunc(ctx, []int64{id})
|
||||
}
|
||||
|
||||
return result.Error
|
||||
}
|
||||
|
||||
// SearchPatternsFTS performs full-text search on patterns.
|
||||
// Uses raw SQL for FTS5 query.
|
||||
func (s *PatternStore) SearchPatternsFTS(ctx context.Context, searchQuery string, limit int) ([]*models.Pattern, error) {
|
||||
var patterns []Pattern
|
||||
|
||||
// Use raw SQL for FTS5 MATCH query
|
||||
query := `
|
||||
SELECT p.*
|
||||
FROM patterns p
|
||||
JOIN patterns_fts fts ON p.id = fts.rowid
|
||||
WHERE patterns_fts MATCH ?
|
||||
AND p.status = 'active'
|
||||
ORDER BY rank
|
||||
LIMIT ?
|
||||
`
|
||||
|
||||
err := s.db.WithContext(ctx).
|
||||
Raw(query, searchQuery, limit).
|
||||
Scan(&patterns).Error
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return toModelPatterns(patterns), nil
|
||||
}
|
||||
|
||||
// PatternStats contains aggregate statistics about patterns.
|
||||
type PatternStats struct {
|
||||
Total int `json:"total"`
|
||||
Active int `json:"active"`
|
||||
Deprecated int `json:"deprecated"`
|
||||
Merged int `json:"merged"`
|
||||
TotalOccurrences int `json:"total_occurrences"`
|
||||
AvgConfidence float64 `json:"avg_confidence"`
|
||||
Bugs int `json:"bugs"`
|
||||
Refactors int `json:"refactors"`
|
||||
Architectures int `json:"architectures"`
|
||||
AntiPatterns int `json:"anti_patterns"`
|
||||
BestPractices int `json:"best_practices"`
|
||||
}
|
||||
|
||||
// GetPatternStats returns statistics about patterns.
|
||||
// Uses raw SQL for complex aggregate query.
|
||||
func (s *PatternStore) GetPatternStats(ctx context.Context) (*PatternStats, error) {
|
||||
var stats PatternStats
|
||||
|
||||
query := `
|
||||
SELECT
|
||||
COUNT(*) as total,
|
||||
COUNT(CASE WHEN status = 'active' THEN 1 END) as active,
|
||||
COUNT(CASE WHEN status = 'deprecated' THEN 1 END) as deprecated,
|
||||
COUNT(CASE WHEN status = 'merged' THEN 1 END) as merged,
|
||||
COALESCE(SUM(frequency), 0) as total_occurrences,
|
||||
COALESCE(AVG(confidence), 0) as avg_confidence,
|
||||
COUNT(CASE WHEN type = 'bug' THEN 1 END) as bugs,
|
||||
COUNT(CASE WHEN type = 'refactor' THEN 1 END) as refactors,
|
||||
COUNT(CASE WHEN type = 'architecture' THEN 1 END) as architectures,
|
||||
COUNT(CASE WHEN type = 'anti-pattern' THEN 1 END) as anti_patterns,
|
||||
COUNT(CASE WHEN type = 'best-practice' THEN 1 END) as best_practices
|
||||
FROM patterns
|
||||
`
|
||||
|
||||
err := s.db.WithContext(ctx).Raw(query).Scan(&stats).Error
|
||||
return &stats, err
|
||||
}
|
||||
|
||||
// IncrementPatternFrequency atomically increments a pattern's frequency and updates last_seen.
|
||||
func (s *PatternStore) IncrementPatternFrequency(ctx context.Context, id int64, project string, observationID int64) error {
|
||||
now := time.Now()
|
||||
|
||||
// Get current pattern
|
||||
pattern, err := s.GetPatternByID(ctx, id)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Add occurrence
|
||||
pattern.AddOccurrence(project, observationID)
|
||||
pattern.LastSeenAt = now.Format(time.RFC3339)
|
||||
pattern.LastSeenEpoch = now.UnixMilli()
|
||||
|
||||
return s.UpdatePattern(ctx, pattern)
|
||||
}
|
||||
|
||||
// toModelPattern converts a GORM Pattern to a pkg/models Pattern.
|
||||
func toModelPattern(p *Pattern) *models.Pattern {
|
||||
pattern := &models.Pattern{
|
||||
ID: p.ID,
|
||||
Name: p.Name,
|
||||
Type: p.Type,
|
||||
Description: p.Description,
|
||||
Signature: p.Signature,
|
||||
Recommendation: p.Recommendation,
|
||||
Frequency: p.Frequency,
|
||||
Projects: p.Projects,
|
||||
ObservationIDs: p.ObservationIDs,
|
||||
Status: p.Status,
|
||||
MergedIntoID: p.MergedIntoID,
|
||||
Confidence: p.Confidence,
|
||||
LastSeenAt: p.LastSeenAt,
|
||||
LastSeenEpoch: p.LastSeenAtEpoch,
|
||||
CreatedAt: p.CreatedAt,
|
||||
CreatedAtEpoch: p.CreatedAtEpoch,
|
||||
}
|
||||
|
||||
return pattern
|
||||
}
|
||||
|
||||
// toModelPatterns converts a slice of GORM Patterns to pkg/models Patterns.
|
||||
func toModelPatterns(patterns []Pattern) []*models.Pattern {
|
||||
result := make([]*models.Pattern, len(patterns))
|
||||
for i, p := range patterns {
|
||||
result[i] = toModelPattern(&p)
|
||||
}
|
||||
return result
|
||||
}
|
||||
@@ -0,0 +1,485 @@
|
||||
//go:build fts5
|
||||
|
||||
// Package gorm provides GORM-based database operations for claude-mnemonic.
|
||||
package gorm
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"gorm.io/gorm/logger"
|
||||
|
||||
"github.com/lukaszraczylo/claude-mnemonic/pkg/models"
|
||||
)
|
||||
|
||||
func testPatternStore(t *testing.T) (*PatternStore, *Store, func()) {
|
||||
t.Helper()
|
||||
|
||||
tmpDir, err := os.MkdirTemp("", "gorm_pattern_test_*")
|
||||
if err != nil {
|
||||
t.Fatalf("create temp dir: %v", err)
|
||||
}
|
||||
|
||||
dbPath := filepath.Join(tmpDir, "test.db")
|
||||
cfg := Config{
|
||||
Path: dbPath,
|
||||
MaxConns: 4,
|
||||
LogLevel: logger.Silent,
|
||||
}
|
||||
|
||||
store, err := NewStore(cfg)
|
||||
if err != nil {
|
||||
os.RemoveAll(tmpDir)
|
||||
t.Fatalf("NewStore failed: %v", err)
|
||||
}
|
||||
|
||||
patternStore := NewPatternStore(store)
|
||||
|
||||
cleanup := func() {
|
||||
store.Close()
|
||||
os.RemoveAll(tmpDir)
|
||||
}
|
||||
|
||||
return patternStore, store, cleanup
|
||||
}
|
||||
|
||||
func TestPatternStore_StorePattern(t *testing.T) {
|
||||
patternStore, _, cleanup := testPatternStore(t)
|
||||
defer cleanup()
|
||||
ctx := context.Background()
|
||||
|
||||
now := time.Now()
|
||||
pattern := &models.Pattern{
|
||||
Name: "Test Pattern",
|
||||
Type: models.PatternTypeBug,
|
||||
Description: sql.NullString{String: "Test description", Valid: true},
|
||||
Signature: []string{"bug", "error"},
|
||||
Recommendation: sql.NullString{String: "Fix it", Valid: true},
|
||||
Frequency: 1,
|
||||
Projects: []string{"test-project"},
|
||||
ObservationIDs: []int64{1, 2, 3},
|
||||
Status: models.PatternStatusActive,
|
||||
Confidence: 0.8,
|
||||
LastSeenAt: now.Format(time.RFC3339),
|
||||
LastSeenEpoch: now.UnixMilli(),
|
||||
CreatedAt: now.Format(time.RFC3339),
|
||||
CreatedAtEpoch: now.UnixMilli(),
|
||||
}
|
||||
|
||||
id, err := patternStore.StorePattern(ctx, pattern)
|
||||
require.NoError(t, err)
|
||||
assert.Greater(t, id, int64(0))
|
||||
|
||||
// Verify pattern was stored
|
||||
retrieved, err := patternStore.GetPatternByID(ctx, id)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, pattern.Name, retrieved.Name)
|
||||
assert.Equal(t, pattern.Type, retrieved.Type)
|
||||
assert.Equal(t, pattern.Signature, retrieved.Signature)
|
||||
assert.Equal(t, pattern.Frequency, retrieved.Frequency)
|
||||
assert.Equal(t, pattern.Status, retrieved.Status)
|
||||
assert.Equal(t, pattern.Confidence, retrieved.Confidence)
|
||||
}
|
||||
|
||||
func TestPatternStore_UpdatePattern(t *testing.T) {
|
||||
patternStore, _, cleanup := testPatternStore(t)
|
||||
defer cleanup()
|
||||
ctx := context.Background()
|
||||
|
||||
now := time.Now()
|
||||
pattern := &models.Pattern{
|
||||
Name: "Original",
|
||||
Type: models.PatternTypeBug,
|
||||
Frequency: 1,
|
||||
Status: models.PatternStatusActive,
|
||||
Confidence: 0.5,
|
||||
LastSeenAt: now.Format(time.RFC3339),
|
||||
LastSeenEpoch: now.UnixMilli(),
|
||||
CreatedAt: now.Format(time.RFC3339),
|
||||
CreatedAtEpoch: now.UnixMilli(),
|
||||
}
|
||||
|
||||
id, err := patternStore.StorePattern(ctx, pattern)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Update pattern
|
||||
pattern.ID = id
|
||||
pattern.Name = "Updated"
|
||||
pattern.Frequency = 5
|
||||
pattern.Confidence = 0.9
|
||||
|
||||
err = patternStore.UpdatePattern(ctx, pattern)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Verify update
|
||||
retrieved, err := patternStore.GetPatternByID(ctx, id)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "Updated", retrieved.Name)
|
||||
assert.Equal(t, 5, retrieved.Frequency)
|
||||
assert.Equal(t, 0.9, retrieved.Confidence)
|
||||
}
|
||||
|
||||
func TestPatternStore_GetPatternByName(t *testing.T) {
|
||||
patternStore, _, cleanup := testPatternStore(t)
|
||||
defer cleanup()
|
||||
ctx := context.Background()
|
||||
|
||||
now := time.Now()
|
||||
pattern := &models.Pattern{
|
||||
Name: "Unique Pattern",
|
||||
Type: models.PatternTypeRefactor,
|
||||
Frequency: 1,
|
||||
Status: models.PatternStatusActive,
|
||||
Confidence: 0.7,
|
||||
LastSeenAt: now.Format(time.RFC3339),
|
||||
LastSeenEpoch: now.UnixMilli(),
|
||||
CreatedAt: now.Format(time.RFC3339),
|
||||
CreatedAtEpoch: now.UnixMilli(),
|
||||
}
|
||||
|
||||
_, err := patternStore.StorePattern(ctx, pattern)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Retrieve by name
|
||||
retrieved, err := patternStore.GetPatternByName(ctx, "Unique Pattern")
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, retrieved)
|
||||
assert.Equal(t, "Unique Pattern", retrieved.Name)
|
||||
assert.Equal(t, models.PatternTypeRefactor, retrieved.Type)
|
||||
|
||||
// Non-existent pattern
|
||||
notFound, err := patternStore.GetPatternByName(ctx, "Nonexistent")
|
||||
require.NoError(t, err)
|
||||
assert.Nil(t, notFound)
|
||||
}
|
||||
|
||||
func TestPatternStore_GetActivePatterns(t *testing.T) {
|
||||
patternStore, _, cleanup := testPatternStore(t)
|
||||
defer cleanup()
|
||||
ctx := context.Background()
|
||||
|
||||
now := time.Now()
|
||||
|
||||
// Create active patterns
|
||||
for i := 0; i < 3; i++ {
|
||||
pattern := &models.Pattern{
|
||||
Name: "Pattern " + string(rune('A'+i)),
|
||||
Type: models.PatternTypeBug,
|
||||
Frequency: i + 1, // Different frequencies for sorting
|
||||
Status: models.PatternStatusActive,
|
||||
Confidence: 0.8,
|
||||
LastSeenAt: now.Format(time.RFC3339),
|
||||
LastSeenEpoch: now.UnixMilli(),
|
||||
CreatedAt: now.Format(time.RFC3339),
|
||||
CreatedAtEpoch: now.UnixMilli(),
|
||||
}
|
||||
_, err := patternStore.StorePattern(ctx, pattern)
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
// Create deprecated pattern (should not be included)
|
||||
deprecatedPattern := &models.Pattern{
|
||||
Name: "Deprecated Pattern",
|
||||
Type: models.PatternTypeBug,
|
||||
Frequency: 100,
|
||||
Status: models.PatternStatusDeprecated,
|
||||
Confidence: 0.9,
|
||||
LastSeenAt: now.Format(time.RFC3339),
|
||||
LastSeenEpoch: now.UnixMilli(),
|
||||
CreatedAt: now.Format(time.RFC3339),
|
||||
CreatedAtEpoch: now.UnixMilli(),
|
||||
}
|
||||
_, err := patternStore.StorePattern(ctx, deprecatedPattern)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Get active patterns
|
||||
patterns, err := patternStore.GetActivePatterns(ctx, 10)
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, patterns, 3) // Only active patterns
|
||||
|
||||
// Verify sorted by frequency DESC
|
||||
assert.Equal(t, 3, patterns[0].Frequency)
|
||||
assert.Equal(t, 2, patterns[1].Frequency)
|
||||
assert.Equal(t, 1, patterns[2].Frequency)
|
||||
}
|
||||
|
||||
func TestPatternStore_GetPatternsByType(t *testing.T) {
|
||||
patternStore, _, cleanup := testPatternStore(t)
|
||||
defer cleanup()
|
||||
ctx := context.Background()
|
||||
|
||||
now := time.Now()
|
||||
|
||||
// Create patterns of different types
|
||||
bugPattern := &models.Pattern{
|
||||
Name: "Bug Pattern",
|
||||
Type: models.PatternTypeBug,
|
||||
Frequency: 1,
|
||||
Status: models.PatternStatusActive,
|
||||
Confidence: 0.8,
|
||||
LastSeenAt: now.Format(time.RFC3339),
|
||||
LastSeenEpoch: now.UnixMilli(),
|
||||
CreatedAt: now.Format(time.RFC3339),
|
||||
CreatedAtEpoch: now.UnixMilli(),
|
||||
}
|
||||
_, err := patternStore.StorePattern(ctx, bugPattern)
|
||||
require.NoError(t, err)
|
||||
|
||||
refactorPattern := &models.Pattern{
|
||||
Name: "Refactor Pattern",
|
||||
Type: models.PatternTypeRefactor,
|
||||
Frequency: 1,
|
||||
Status: models.PatternStatusActive,
|
||||
Confidence: 0.7,
|
||||
LastSeenAt: now.Format(time.RFC3339),
|
||||
LastSeenEpoch: now.UnixMilli(),
|
||||
CreatedAt: now.Format(time.RFC3339),
|
||||
CreatedAtEpoch: now.UnixMilli(),
|
||||
}
|
||||
_, err = patternStore.StorePattern(ctx, refactorPattern)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Get only bug patterns
|
||||
bugPatterns, err := patternStore.GetPatternsByType(ctx, models.PatternTypeBug, 10)
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, bugPatterns, 1)
|
||||
assert.Equal(t, "Bug Pattern", bugPatterns[0].Name)
|
||||
assert.Equal(t, models.PatternTypeBug, bugPatterns[0].Type)
|
||||
|
||||
// Get only refactor patterns
|
||||
refactorPatterns, err := patternStore.GetPatternsByType(ctx, models.PatternTypeRefactor, 10)
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, refactorPatterns, 1)
|
||||
assert.Equal(t, "Refactor Pattern", refactorPatterns[0].Name)
|
||||
}
|
||||
|
||||
func TestPatternStore_MarkPatternDeprecated(t *testing.T) {
|
||||
patternStore, _, cleanup := testPatternStore(t)
|
||||
defer cleanup()
|
||||
ctx := context.Background()
|
||||
|
||||
now := time.Now()
|
||||
pattern := &models.Pattern{
|
||||
Name: "To Deprecate",
|
||||
Type: models.PatternTypeBug,
|
||||
Frequency: 1,
|
||||
Status: models.PatternStatusActive,
|
||||
Confidence: 0.5,
|
||||
LastSeenAt: now.Format(time.RFC3339),
|
||||
LastSeenEpoch: now.UnixMilli(),
|
||||
CreatedAt: now.Format(time.RFC3339),
|
||||
CreatedAtEpoch: now.UnixMilli(),
|
||||
}
|
||||
|
||||
id, err := patternStore.StorePattern(ctx, pattern)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Mark as deprecated
|
||||
err = patternStore.MarkPatternDeprecated(ctx, id)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Verify status changed
|
||||
retrieved, err := patternStore.GetPatternByID(ctx, id)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, models.PatternStatusDeprecated, retrieved.Status)
|
||||
}
|
||||
|
||||
func TestPatternStore_MergePatterns(t *testing.T) {
|
||||
patternStore, _, cleanup := testPatternStore(t)
|
||||
defer cleanup()
|
||||
ctx := context.Background()
|
||||
|
||||
now := time.Now()
|
||||
|
||||
// Create source pattern
|
||||
source := &models.Pattern{
|
||||
Name: "Source Pattern",
|
||||
Type: models.PatternTypeBug,
|
||||
Frequency: 5,
|
||||
Projects: []string{"project-a", "project-b"},
|
||||
ObservationIDs: []int64{1, 2, 3},
|
||||
Status: models.PatternStatusActive,
|
||||
Confidence: 0.7,
|
||||
LastSeenAt: now.Format(time.RFC3339),
|
||||
LastSeenEpoch: now.UnixMilli(),
|
||||
CreatedAt: now.Format(time.RFC3339),
|
||||
CreatedAtEpoch: now.UnixMilli(),
|
||||
}
|
||||
sourceID, err := patternStore.StorePattern(ctx, source)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Create target pattern
|
||||
target := &models.Pattern{
|
||||
Name: "Target Pattern",
|
||||
Type: models.PatternTypeBug,
|
||||
Frequency: 10,
|
||||
Projects: []string{"project-b", "project-c"},
|
||||
ObservationIDs: []int64{3, 4, 5},
|
||||
Status: models.PatternStatusActive,
|
||||
Confidence: 0.8,
|
||||
LastSeenAt: now.Format(time.RFC3339),
|
||||
LastSeenEpoch: now.UnixMilli(),
|
||||
CreatedAt: now.Format(time.RFC3339),
|
||||
CreatedAtEpoch: now.UnixMilli(),
|
||||
}
|
||||
targetID, err := patternStore.StorePattern(ctx, target)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Merge source into target
|
||||
err = patternStore.MergePatterns(ctx, sourceID, targetID)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Verify target was updated
|
||||
mergedTarget, err := patternStore.GetPatternByID(ctx, targetID)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 15, mergedTarget.Frequency) // 5 + 10
|
||||
assert.ElementsMatch(t, []string{"project-a", "project-b", "project-c"}, mergedTarget.Projects)
|
||||
assert.ElementsMatch(t, []int64{1, 2, 3, 4, 5}, mergedTarget.ObservationIDs)
|
||||
|
||||
// Verify source was marked as merged
|
||||
mergedSource, err := patternStore.GetPatternByID(ctx, sourceID)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, models.PatternStatusMerged, mergedSource.Status)
|
||||
assert.True(t, mergedSource.MergedIntoID.Valid)
|
||||
assert.Equal(t, targetID, mergedSource.MergedIntoID.Int64)
|
||||
}
|
||||
|
||||
func TestPatternStore_DeletePattern(t *testing.T) {
|
||||
patternStore, _, cleanup := testPatternStore(t)
|
||||
defer cleanup()
|
||||
ctx := context.Background()
|
||||
|
||||
now := time.Now()
|
||||
pattern := &models.Pattern{
|
||||
Name: "To Delete",
|
||||
Type: models.PatternTypeBug,
|
||||
Frequency: 1,
|
||||
Status: models.PatternStatusActive,
|
||||
Confidence: 0.5,
|
||||
LastSeenAt: now.Format(time.RFC3339),
|
||||
LastSeenEpoch: now.UnixMilli(),
|
||||
CreatedAt: now.Format(time.RFC3339),
|
||||
CreatedAtEpoch: now.UnixMilli(),
|
||||
}
|
||||
|
||||
id, err := patternStore.StorePattern(ctx, pattern)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Delete pattern
|
||||
err = patternStore.DeletePattern(ctx, id)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Verify deleted
|
||||
deleted, err := patternStore.GetPatternByID(ctx, id)
|
||||
require.NoError(t, err)
|
||||
assert.Nil(t, deleted)
|
||||
}
|
||||
|
||||
func TestPatternStore_IncrementPatternFrequency(t *testing.T) {
|
||||
patternStore, _, cleanup := testPatternStore(t)
|
||||
defer cleanup()
|
||||
ctx := context.Background()
|
||||
|
||||
now := time.Now()
|
||||
pattern := &models.Pattern{
|
||||
Name: "Frequency Test",
|
||||
Type: models.PatternTypeBug,
|
||||
Frequency: 1,
|
||||
Projects: []string{"project-a"},
|
||||
ObservationIDs: []int64{},
|
||||
Status: models.PatternStatusActive,
|
||||
Confidence: 0.7,
|
||||
LastSeenAt: now.Format(time.RFC3339),
|
||||
LastSeenEpoch: now.UnixMilli(),
|
||||
CreatedAt: now.Format(time.RFC3339),
|
||||
CreatedAtEpoch: now.UnixMilli(),
|
||||
}
|
||||
|
||||
id, err := patternStore.StorePattern(ctx, pattern)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Increment frequency with new project and observation
|
||||
err = patternStore.IncrementPatternFrequency(ctx, id, "project-b", 42)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Verify frequency incremented and new data added
|
||||
updated, err := patternStore.GetPatternByID(ctx, id)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 2, updated.Frequency)
|
||||
assert.ElementsMatch(t, []string{"project-a", "project-b"}, updated.Projects)
|
||||
assert.Contains(t, updated.ObservationIDs, int64(42))
|
||||
|
||||
// Last seen should be updated (rough check - within last 5 seconds)
|
||||
updatedTime, _ := time.Parse(time.RFC3339, updated.LastSeenAt)
|
||||
assert.WithinDuration(t, time.Now(), updatedTime, 5*time.Second)
|
||||
}
|
||||
|
||||
func TestPatternStore_GetPatternStats(t *testing.T) {
|
||||
patternStore, _, cleanup := testPatternStore(t)
|
||||
defer cleanup()
|
||||
ctx := context.Background()
|
||||
|
||||
now := time.Now()
|
||||
|
||||
// Create patterns with different statuses and types
|
||||
patterns := []*models.Pattern{
|
||||
{
|
||||
Name: "Bug 1",
|
||||
Type: models.PatternTypeBug,
|
||||
Frequency: 10,
|
||||
Status: models.PatternStatusActive,
|
||||
Confidence: 0.8,
|
||||
LastSeenAt: now.Format(time.RFC3339),
|
||||
LastSeenEpoch: now.UnixMilli(),
|
||||
CreatedAt: now.Format(time.RFC3339),
|
||||
CreatedAtEpoch: now.UnixMilli(),
|
||||
},
|
||||
{
|
||||
Name: "Refactor 1",
|
||||
Type: models.PatternTypeRefactor,
|
||||
Frequency: 5,
|
||||
Status: models.PatternStatusActive,
|
||||
Confidence: 0.7,
|
||||
LastSeenAt: now.Format(time.RFC3339),
|
||||
LastSeenEpoch: now.UnixMilli(),
|
||||
CreatedAt: now.Format(time.RFC3339),
|
||||
CreatedAtEpoch: now.UnixMilli(),
|
||||
},
|
||||
{
|
||||
Name: "Deprecated 1",
|
||||
Type: models.PatternTypeBestPractice,
|
||||
Frequency: 3,
|
||||
Status: models.PatternStatusDeprecated,
|
||||
Confidence: 0.6,
|
||||
LastSeenAt: now.Format(time.RFC3339),
|
||||
LastSeenEpoch: now.UnixMilli(),
|
||||
CreatedAt: now.Format(time.RFC3339),
|
||||
CreatedAtEpoch: now.UnixMilli(),
|
||||
},
|
||||
}
|
||||
|
||||
for _, p := range patterns {
|
||||
_, err := patternStore.StorePattern(ctx, p)
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
// Get stats
|
||||
stats, err := patternStore.GetPatternStats(ctx)
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.Equal(t, 3, stats.Total)
|
||||
assert.Equal(t, 2, stats.Active)
|
||||
assert.Equal(t, 1, stats.Deprecated)
|
||||
assert.Equal(t, 0, stats.Merged)
|
||||
assert.Equal(t, 18, stats.TotalOccurrences) // 10 + 5 + 3
|
||||
assert.InDelta(t, 0.7, stats.AvgConfidence, 0.05) // (0.8 + 0.7 + 0.6) / 3
|
||||
assert.Equal(t, 1, stats.Bugs)
|
||||
assert.Equal(t, 1, stats.Refactors)
|
||||
assert.Equal(t, 1, stats.BestPractices)
|
||||
}
|
||||
@@ -0,0 +1,317 @@
|
||||
// Package gorm provides GORM-based database operations for claude-mnemonic.
|
||||
package gorm
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"time"
|
||||
|
||||
"gorm.io/gorm"
|
||||
"gorm.io/gorm/clause"
|
||||
|
||||
"github.com/lukaszraczylo/claude-mnemonic/pkg/models"
|
||||
)
|
||||
|
||||
// PromptCleanupFunc is a callback for when prompts are cleaned up.
|
||||
// Receives the IDs of deleted prompts for downstream cleanup (e.g., vector DB).
|
||||
type PromptCleanupFunc func(ctx context.Context, deletedIDs []int64)
|
||||
|
||||
// MaxPromptsGlobal is the hard limit of prompts across all projects.
|
||||
const MaxPromptsGlobal = 500
|
||||
|
||||
// PromptStore provides user prompt-related database operations using GORM.
|
||||
type PromptStore struct {
|
||||
db *gorm.DB
|
||||
cleanupFunc PromptCleanupFunc
|
||||
}
|
||||
|
||||
// NewPromptStore creates a new prompt store.
|
||||
func NewPromptStore(store *Store, cleanupFunc PromptCleanupFunc) *PromptStore {
|
||||
return &PromptStore{
|
||||
db: store.DB,
|
||||
cleanupFunc: cleanupFunc,
|
||||
}
|
||||
}
|
||||
|
||||
// SetCleanupFunc sets the callback for when prompts are deleted during cleanup.
|
||||
func (s *PromptStore) SetCleanupFunc(fn PromptCleanupFunc) {
|
||||
s.cleanupFunc = fn
|
||||
}
|
||||
|
||||
// SaveUserPromptWithMatches saves a user prompt with matched observation count.
|
||||
// Uses INSERT OR IGNORE to be idempotent - duplicate (session, prompt_number) pairs are silently ignored.
|
||||
// This prevents duplicate prompts when the user-prompt hook fires multiple times.
|
||||
func (s *PromptStore) SaveUserPromptWithMatches(ctx context.Context, claudeSessionID string, promptNumber int, promptText string, matchedObservations int) (int64, error) {
|
||||
now := time.Now()
|
||||
|
||||
prompt := &UserPrompt{
|
||||
ClaudeSessionID: claudeSessionID,
|
||||
PromptNumber: promptNumber,
|
||||
PromptText: promptText,
|
||||
MatchedObservations: matchedObservations,
|
||||
CreatedAt: now.Format(time.RFC3339),
|
||||
CreatedAtEpoch: now.UnixMilli(),
|
||||
}
|
||||
|
||||
// INSERT OR IGNORE using OnConflict
|
||||
result := s.db.WithContext(ctx).
|
||||
Clauses(clause.OnConflict{
|
||||
Columns: []clause.Column{{Name: "claude_session_id"}, {Name: "prompt_number"}},
|
||||
DoNothing: true,
|
||||
}).
|
||||
Create(prompt)
|
||||
|
||||
if result.Error != nil {
|
||||
return 0, result.Error
|
||||
}
|
||||
|
||||
// If RowsAffected is 0, the insert was ignored (duplicate) - fetch the existing ID
|
||||
if result.RowsAffected == 0 {
|
||||
var existing UserPrompt
|
||||
err := s.db.Where("claude_session_id = ? AND prompt_number = ?", claudeSessionID, promptNumber).
|
||||
First(&existing).Error
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
// Return existing ID without triggering cleanup (already handled when first inserted)
|
||||
return existing.ID, nil
|
||||
}
|
||||
|
||||
// Cleanup old prompts beyond the global limit (async to not block handler)
|
||||
go func() {
|
||||
cleanupCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
deletedIDs, _ := s.CleanupOldPrompts(cleanupCtx)
|
||||
if len(deletedIDs) > 0 && s.cleanupFunc != nil {
|
||||
s.cleanupFunc(cleanupCtx, deletedIDs)
|
||||
}
|
||||
}()
|
||||
|
||||
return prompt.ID, nil
|
||||
}
|
||||
|
||||
// CleanupOldPrompts deletes prompts beyond the global limit.
|
||||
// Keeps the most recent MaxPromptsGlobal prompts.
|
||||
// Returns the IDs of deleted prompts for downstream cleanup (e.g., vector DB).
|
||||
func (s *PromptStore) CleanupOldPrompts(ctx context.Context) ([]int64, error) {
|
||||
// Use a transaction to prevent TOCTOU race condition
|
||||
var idsToDelete []int64
|
||||
|
||||
err := s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
|
||||
// Find IDs to keep (most recent MaxPromptsGlobal)
|
||||
var idsToKeep []int64
|
||||
err := tx.Model(&UserPrompt{}).
|
||||
Order("created_at_epoch DESC").
|
||||
Limit(MaxPromptsGlobal).
|
||||
Pluck("id", &idsToKeep).Error
|
||||
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if len(idsToKeep) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Find IDs to delete (all IDs not in the keep list)
|
||||
// This happens in the same transaction to prevent race conditions
|
||||
err = tx.Model(&UserPrompt{}).
|
||||
Where("id NOT IN ?", idsToKeep).
|
||||
Pluck("id", &idsToDelete).Error
|
||||
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if len(idsToDelete) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Delete the prompts
|
||||
return tx.Delete(&UserPrompt{}, idsToDelete).Error
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return idsToDelete, nil
|
||||
}
|
||||
|
||||
// GetPromptsByIDs retrieves user prompts by a list of IDs.
|
||||
func (s *PromptStore) GetPromptsByIDs(ctx context.Context, ids []int64, orderBy string, limit int) ([]*models.UserPromptWithSession, error) {
|
||||
if len(ids) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
var results []struct {
|
||||
UserPrompt
|
||||
Project sql.NullString `gorm:"column:project"`
|
||||
SDKSessionID sql.NullString `gorm:"column:sdk_session_id"`
|
||||
}
|
||||
|
||||
query := s.db.WithContext(ctx).
|
||||
Table("user_prompts up").
|
||||
Select("up.id, up.claude_session_id, up.prompt_number, up.prompt_text, "+
|
||||
"COALESCE(up.matched_observations, 0) as matched_observations, "+
|
||||
"up.created_at, up.created_at_epoch, "+
|
||||
"COALESCE(s.project, '') as project, "+
|
||||
"COALESCE(s.sdk_session_id, '') as sdk_session_id").
|
||||
Joins("LEFT JOIN sdk_sessions s ON up.claude_session_id = s.claude_session_id").
|
||||
Where("up.id IN ?", ids)
|
||||
|
||||
// Apply ordering
|
||||
switch orderBy {
|
||||
case "date_asc":
|
||||
query = query.Order("up.created_at_epoch ASC")
|
||||
case "date_desc", "default", "":
|
||||
query = query.Order("up.created_at_epoch DESC")
|
||||
}
|
||||
|
||||
// Apply limit
|
||||
if limit > 0 {
|
||||
query = query.Limit(limit)
|
||||
}
|
||||
|
||||
err := query.Scan(&results).Error
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return toModelUserPromptsWithSession(results), nil
|
||||
}
|
||||
|
||||
// GetAllRecentUserPrompts retrieves recent user prompts across all projects.
|
||||
func (s *PromptStore) GetAllRecentUserPrompts(ctx context.Context, limit int) ([]*models.UserPromptWithSession, error) {
|
||||
var results []struct {
|
||||
UserPrompt
|
||||
Project sql.NullString `gorm:"column:project"`
|
||||
SDKSessionID sql.NullString `gorm:"column:sdk_session_id"`
|
||||
}
|
||||
|
||||
query := s.db.WithContext(ctx).
|
||||
Table("user_prompts up").
|
||||
Select("up.id, up.claude_session_id, up.prompt_number, up.prompt_text, " +
|
||||
"COALESCE(up.matched_observations, 0) as matched_observations, " +
|
||||
"up.created_at, up.created_at_epoch, " +
|
||||
"COALESCE(s.project, '') as project, " +
|
||||
"COALESCE(s.sdk_session_id, '') as sdk_session_id").
|
||||
Joins("LEFT JOIN sdk_sessions s ON up.claude_session_id = s.claude_session_id").
|
||||
Order("up.created_at_epoch DESC").
|
||||
Limit(limit)
|
||||
|
||||
err := query.Scan(&results).Error
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return toModelUserPromptsWithSession(results), nil
|
||||
}
|
||||
|
||||
// GetAllPrompts retrieves all user prompts (for vector rebuild).
|
||||
func (s *PromptStore) GetAllPrompts(ctx context.Context) ([]*models.UserPromptWithSession, error) {
|
||||
var results []struct {
|
||||
UserPrompt
|
||||
Project sql.NullString `gorm:"column:project"`
|
||||
SDKSessionID sql.NullString `gorm:"column:sdk_session_id"`
|
||||
}
|
||||
|
||||
query := s.db.WithContext(ctx).
|
||||
Table("user_prompts up").
|
||||
Select("up.id, up.claude_session_id, up.prompt_number, up.prompt_text, " +
|
||||
"COALESCE(up.matched_observations, 0) as matched_observations, " +
|
||||
"up.created_at, up.created_at_epoch, " +
|
||||
"COALESCE(s.project, '') as project, " +
|
||||
"COALESCE(s.sdk_session_id, '') as sdk_session_id").
|
||||
Joins("LEFT JOIN sdk_sessions s ON up.claude_session_id = s.claude_session_id").
|
||||
Order("up.id")
|
||||
|
||||
err := query.Scan(&results).Error
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return toModelUserPromptsWithSession(results), nil
|
||||
}
|
||||
|
||||
// FindRecentPromptByText finds a recent prompt by exact text match within a time window.
|
||||
// Returns (promptID, promptNumber, found).
|
||||
func (s *PromptStore) FindRecentPromptByText(ctx context.Context, claudeSessionID, promptText string, withinSeconds int) (int64, int, bool) {
|
||||
cutoffEpoch := time.Now().Add(-time.Duration(withinSeconds) * time.Second).UnixMilli()
|
||||
|
||||
var prompt UserPrompt
|
||||
err := s.db.WithContext(ctx).
|
||||
Where("claude_session_id = ? AND prompt_text = ? AND created_at_epoch >= ?",
|
||||
claudeSessionID, promptText, cutoffEpoch).
|
||||
Order("created_at_epoch DESC").
|
||||
First(&prompt).Error
|
||||
|
||||
if err != nil {
|
||||
return 0, 0, false
|
||||
}
|
||||
|
||||
return prompt.ID, prompt.PromptNumber, true
|
||||
}
|
||||
|
||||
// GetRecentUserPromptsByProject retrieves recent user prompts for a specific project.
|
||||
func (s *PromptStore) GetRecentUserPromptsByProject(ctx context.Context, project string, limit int) ([]*models.UserPromptWithSession, error) {
|
||||
var results []struct {
|
||||
UserPrompt
|
||||
Project sql.NullString `gorm:"column:project"`
|
||||
SDKSessionID sql.NullString `gorm:"column:sdk_session_id"`
|
||||
}
|
||||
|
||||
query := s.db.WithContext(ctx).
|
||||
Table("user_prompts up").
|
||||
Select("up.id, up.claude_session_id, up.prompt_number, up.prompt_text, "+
|
||||
"COALESCE(up.matched_observations, 0) as matched_observations, "+
|
||||
"up.created_at, up.created_at_epoch, "+
|
||||
"COALESCE(s.project, '') as project, "+
|
||||
"COALESCE(s.sdk_session_id, '') as sdk_session_id").
|
||||
Joins("LEFT JOIN sdk_sessions s ON up.claude_session_id = s.claude_session_id").
|
||||
Where("s.project = ?", project).
|
||||
Order("up.created_at_epoch DESC").
|
||||
Limit(limit)
|
||||
|
||||
err := query.Scan(&results).Error
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return toModelUserPromptsWithSession(results), nil
|
||||
}
|
||||
|
||||
// toModelUserPromptsWithSession converts query results to pkg/models.UserPromptWithSession.
|
||||
func toModelUserPromptsWithSession(results []struct {
|
||||
UserPrompt
|
||||
Project sql.NullString `gorm:"column:project"`
|
||||
SDKSessionID sql.NullString `gorm:"column:sdk_session_id"`
|
||||
}) []*models.UserPromptWithSession {
|
||||
prompts := make([]*models.UserPromptWithSession, len(results))
|
||||
for i, r := range results {
|
||||
project := ""
|
||||
if r.Project.Valid {
|
||||
project = r.Project.String
|
||||
}
|
||||
|
||||
sdkSessionID := ""
|
||||
if r.SDKSessionID.Valid {
|
||||
sdkSessionID = r.SDKSessionID.String
|
||||
}
|
||||
|
||||
prompts[i] = &models.UserPromptWithSession{
|
||||
UserPrompt: models.UserPrompt{
|
||||
ID: r.ID,
|
||||
ClaudeSessionID: r.ClaudeSessionID,
|
||||
PromptNumber: r.PromptNumber,
|
||||
PromptText: r.PromptText,
|
||||
MatchedObservations: r.MatchedObservations,
|
||||
CreatedAt: r.CreatedAt,
|
||||
CreatedAtEpoch: r.CreatedAtEpoch,
|
||||
},
|
||||
Project: project,
|
||||
SDKSessionID: sdkSessionID,
|
||||
}
|
||||
}
|
||||
return prompts
|
||||
}
|
||||
@@ -0,0 +1,396 @@
|
||||
//go:build fts5
|
||||
|
||||
// Package gorm provides GORM-based database operations for claude-mnemonic.
|
||||
package gorm
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"gorm.io/gorm/logger"
|
||||
)
|
||||
|
||||
// testPromptStore creates a PromptStore with a temporary database for testing.
|
||||
func testPromptStore(t *testing.T) (*PromptStore, *Store, func()) {
|
||||
t.Helper()
|
||||
|
||||
tmpDir, err := os.MkdirTemp("", "gorm_prompt_test_*")
|
||||
if err != nil {
|
||||
t.Fatalf("create temp dir: %v", err)
|
||||
}
|
||||
|
||||
dbPath := filepath.Join(tmpDir, "test.db")
|
||||
cfg := Config{
|
||||
Path: dbPath,
|
||||
MaxConns: 4,
|
||||
LogLevel: logger.Silent,
|
||||
}
|
||||
|
||||
store, err := NewStore(cfg)
|
||||
if err != nil {
|
||||
os.RemoveAll(tmpDir)
|
||||
t.Fatalf("NewStore failed: %v", err)
|
||||
}
|
||||
|
||||
promptStore := NewPromptStore(store, nil)
|
||||
|
||||
cleanup := func() {
|
||||
store.Close()
|
||||
os.RemoveAll(tmpDir)
|
||||
}
|
||||
|
||||
return promptStore, store, cleanup
|
||||
}
|
||||
|
||||
func TestPromptStore_SaveUserPromptWithMatches(t *testing.T) {
|
||||
promptStore, store, cleanup := testPromptStore(t)
|
||||
defer cleanup()
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
// Create a session first
|
||||
sessionStore := NewSessionStore(store)
|
||||
_, err := sessionStore.CreateSDKSession(ctx, "claude-1", "test-project", "")
|
||||
require.NoError(t, err)
|
||||
|
||||
// Save a prompt
|
||||
id, err := promptStore.SaveUserPromptWithMatches(ctx, "claude-1", 1, "What is the codebase structure?", 5)
|
||||
require.NoError(t, err)
|
||||
assert.Greater(t, id, int64(0))
|
||||
}
|
||||
|
||||
func TestPromptStore_SaveUserPromptWithMatches_Idempotency(t *testing.T) {
|
||||
promptStore, _, cleanup := testPromptStore(t)
|
||||
defer cleanup()
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
// Save the same prompt twice (same claudeSessionID + promptNumber)
|
||||
id1, err := promptStore.SaveUserPromptWithMatches(ctx, "claude-1", 1, "Test prompt", 3)
|
||||
require.NoError(t, err)
|
||||
|
||||
id2, err := promptStore.SaveUserPromptWithMatches(ctx, "claude-1", 1, "Different text", 5)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Should return the same ID (INSERT OR IGNORE)
|
||||
assert.Equal(t, id1, id2, "Duplicate prompts should return same ID")
|
||||
}
|
||||
|
||||
func TestPromptStore_SaveUserPromptWithMatches_AsyncCleanup(t *testing.T) {
|
||||
promptStore, _, cleanup := testPromptStore(t)
|
||||
defer cleanup()
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
// Track cleanup calls
|
||||
var cleanupMutex sync.Mutex
|
||||
cleanupCalled := false
|
||||
var cleanupIDs []int64
|
||||
|
||||
cleanupFunc := func(ctx context.Context, deletedIDs []int64) {
|
||||
cleanupMutex.Lock()
|
||||
defer cleanupMutex.Unlock()
|
||||
cleanupCalled = true
|
||||
cleanupIDs = deletedIDs
|
||||
}
|
||||
|
||||
promptStore.cleanupFunc = cleanupFunc
|
||||
|
||||
// Save prompts beyond the global limit (MaxPromptsGlobal = 500)
|
||||
// Insert with slower pacing to avoid database lock contention
|
||||
for i := 0; i < 505; i++ {
|
||||
_, err := promptStore.SaveUserPromptWithMatches(ctx, "claude-1", i+1, "Prompt", 1)
|
||||
require.NoError(t, err)
|
||||
if i > 500 {
|
||||
time.Sleep(5 * time.Millisecond) // Slow down after hitting limit
|
||||
}
|
||||
}
|
||||
|
||||
// Wait longer for async cleanup to complete
|
||||
time.Sleep(500 * time.Millisecond)
|
||||
|
||||
// Verify cleanup was called
|
||||
cleanupMutex.Lock()
|
||||
defer cleanupMutex.Unlock()
|
||||
assert.True(t, cleanupCalled, "Cleanup function should have been called")
|
||||
assert.NotEmpty(t, cleanupIDs, "Cleanup should have deleted some prompts")
|
||||
}
|
||||
|
||||
func TestPromptStore_CleanupOldPrompts(t *testing.T) {
|
||||
promptStore, _, cleanup := testPromptStore(t)
|
||||
defer cleanup()
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
// Save prompts beyond the limit
|
||||
// Async cleanup should fire after each insert beyond 500
|
||||
for i := 0; i < 505; i++ {
|
||||
_, err := promptStore.SaveUserPromptWithMatches(ctx, "claude-1", i+1, "Prompt", 1)
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
// Wait for all async cleanups to complete
|
||||
time.Sleep(1 * time.Second)
|
||||
|
||||
// After async cleanup, we should have at most 500 prompts
|
||||
remaining, err := promptStore.GetAllPrompts(ctx)
|
||||
require.NoError(t, err)
|
||||
assert.LessOrEqual(t, len(remaining), MaxPromptsGlobal, "Should have at most %d prompts after async cleanup", MaxPromptsGlobal)
|
||||
}
|
||||
|
||||
func TestPromptStore_GetPromptsByIDs(t *testing.T) {
|
||||
promptStore, _, cleanup := testPromptStore(t)
|
||||
defer cleanup()
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
// Save multiple prompts
|
||||
var ids []int64
|
||||
for i := 1; i <= 3; i++ {
|
||||
id, err := promptStore.SaveUserPromptWithMatches(ctx, "claude-1", i, "Prompt", i)
|
||||
require.NoError(t, err)
|
||||
ids = append(ids, id)
|
||||
time.Sleep(10 * time.Millisecond) // Ensure different timestamps
|
||||
}
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
orderBy string
|
||||
expected []int64
|
||||
}{
|
||||
{
|
||||
name: "Default ordering - date desc",
|
||||
orderBy: "default",
|
||||
expected: []int64{ids[2], ids[1], ids[0]}, // Newest to oldest
|
||||
},
|
||||
{
|
||||
name: "Date ascending",
|
||||
orderBy: "date_asc",
|
||||
expected: []int64{ids[0], ids[1], ids[2]}, // Oldest to newest
|
||||
},
|
||||
{
|
||||
name: "Date descending",
|
||||
orderBy: "date_desc",
|
||||
expected: []int64{ids[2], ids[1], ids[0]}, // Newest to oldest
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
prompts, err := promptStore.GetPromptsByIDs(ctx, ids, tt.orderBy, 10)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, prompts, 3)
|
||||
|
||||
// Verify ordering
|
||||
for i, prompt := range prompts {
|
||||
assert.Equal(t, tt.expected[i], prompt.ID, "Position %d should have ID %d", i, tt.expected[i])
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestPromptStore_GetPromptsByIDs_Limit(t *testing.T) {
|
||||
promptStore, _, cleanup := testPromptStore(t)
|
||||
defer cleanup()
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
// Save multiple prompts
|
||||
var ids []int64
|
||||
for i := 1; i <= 5; i++ {
|
||||
id, err := promptStore.SaveUserPromptWithMatches(ctx, "claude-1", i, "Prompt", i)
|
||||
require.NoError(t, err)
|
||||
ids = append(ids, id)
|
||||
}
|
||||
|
||||
// Get with limit
|
||||
prompts, err := promptStore.GetPromptsByIDs(ctx, ids, "default", 3)
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, prompts, 3)
|
||||
}
|
||||
|
||||
func TestPromptStore_GetPromptsByIDs_EmptyInput(t *testing.T) {
|
||||
promptStore, _, cleanup := testPromptStore(t)
|
||||
defer cleanup()
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
// Get with empty IDs
|
||||
prompts, err := promptStore.GetPromptsByIDs(ctx, []int64{}, "default", 10)
|
||||
require.NoError(t, err)
|
||||
assert.Nil(t, prompts)
|
||||
}
|
||||
|
||||
func TestPromptStore_GetPromptsByIDs_WithSession(t *testing.T) {
|
||||
promptStore, store, cleanup := testPromptStore(t)
|
||||
defer cleanup()
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
// Create a session
|
||||
sessionStore := NewSessionStore(store)
|
||||
_, err := sessionStore.CreateSDKSession(ctx, "claude-1", "test-project", "")
|
||||
require.NoError(t, err)
|
||||
|
||||
// Save a prompt
|
||||
id, err := promptStore.SaveUserPromptWithMatches(ctx, "claude-1", 1, "Test prompt", 5)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Get with session join
|
||||
prompts, err := promptStore.GetPromptsByIDs(ctx, []int64{id}, "default", 10)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, prompts, 1)
|
||||
|
||||
// Verify session data is populated
|
||||
assert.Equal(t, "test-project", prompts[0].Project)
|
||||
assert.NotEmpty(t, prompts[0].SDKSessionID)
|
||||
}
|
||||
|
||||
func TestPromptStore_GetAllRecentUserPrompts(t *testing.T) {
|
||||
promptStore, _, cleanup := testPromptStore(t)
|
||||
defer cleanup()
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
// Save prompts across multiple sessions with timestamps
|
||||
for i := 1; i <= 3; i++ {
|
||||
_, err := promptStore.SaveUserPromptWithMatches(ctx, "claude-1", i, "Prompt A", i)
|
||||
require.NoError(t, err)
|
||||
time.Sleep(10 * time.Millisecond) // Ensure different timestamps
|
||||
}
|
||||
|
||||
for i := 1; i <= 2; i++ {
|
||||
_, err := promptStore.SaveUserPromptWithMatches(ctx, "claude-2", i, "Prompt B", i)
|
||||
require.NoError(t, err)
|
||||
time.Sleep(10 * time.Millisecond) // Ensure different timestamps
|
||||
}
|
||||
|
||||
// Get all recent prompts
|
||||
prompts, err := promptStore.GetAllRecentUserPrompts(ctx, 10)
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, prompts, 5)
|
||||
|
||||
// Verify ordering (most recent first) - last inserted should be first
|
||||
assert.Equal(t, "claude-2", prompts[0].ClaudeSessionID)
|
||||
assert.Equal(t, 2, prompts[0].PromptNumber)
|
||||
}
|
||||
|
||||
func TestPromptStore_GetAllPrompts(t *testing.T) {
|
||||
promptStore, _, cleanup := testPromptStore(t)
|
||||
defer cleanup()
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
// Save prompts
|
||||
for i := 1; i <= 5; i++ {
|
||||
_, err := promptStore.SaveUserPromptWithMatches(ctx, "claude-1", i, "Prompt", i)
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
// Wait for any async cleanup to complete (longer wait for race detector)
|
||||
time.Sleep(500 * time.Millisecond)
|
||||
|
||||
// Get all prompts (for vector rebuild)
|
||||
prompts, err := promptStore.GetAllPrompts(ctx)
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, prompts, 5)
|
||||
|
||||
// Verify ordering by ID
|
||||
for i := 0; i < len(prompts)-1; i++ {
|
||||
assert.Less(t, prompts[i].ID, prompts[i+1].ID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPromptStore_FindRecentPromptByText(t *testing.T) {
|
||||
promptStore, _, cleanup := testPromptStore(t)
|
||||
defer cleanup()
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
// Save a prompt
|
||||
id, err := promptStore.SaveUserPromptWithMatches(ctx, "claude-1", 1, "What is the architecture?", 3)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Find by exact text match within time window
|
||||
foundID, foundNumber, found := promptStore.FindRecentPromptByText(ctx, "claude-1", "What is the architecture?", 60)
|
||||
assert.True(t, found, "Should find the prompt")
|
||||
assert.Equal(t, id, foundID)
|
||||
assert.Equal(t, 1, foundNumber)
|
||||
|
||||
// Try to find with different text
|
||||
_, _, notFound := promptStore.FindRecentPromptByText(ctx, "claude-1", "Different text", 60)
|
||||
assert.False(t, notFound, "Should not find a different prompt")
|
||||
|
||||
// Try to find outside time window
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
_, _, notFound = promptStore.FindRecentPromptByText(ctx, "claude-1", "What is the architecture?", 0)
|
||||
assert.False(t, notFound, "Should not find prompt outside time window")
|
||||
}
|
||||
|
||||
func TestPromptStore_GetRecentUserPromptsByProject(t *testing.T) {
|
||||
promptStore, store, cleanup := testPromptStore(t)
|
||||
defer cleanup()
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
// Create sessions for different projects
|
||||
sessionStore := NewSessionStore(store)
|
||||
_, err := sessionStore.CreateSDKSession(ctx, "claude-1", "project-a", "")
|
||||
require.NoError(t, err)
|
||||
_, err = sessionStore.CreateSDKSession(ctx, "claude-2", "project-b", "")
|
||||
require.NoError(t, err)
|
||||
|
||||
// Save prompts for project-a
|
||||
for i := 1; i <= 3; i++ {
|
||||
_, err := promptStore.SaveUserPromptWithMatches(ctx, "claude-1", i, "Prompt A", i)
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
// Save prompts for project-b
|
||||
for i := 1; i <= 2; i++ {
|
||||
_, err := promptStore.SaveUserPromptWithMatches(ctx, "claude-2", i, "Prompt B", i)
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
// Get prompts for project-a
|
||||
prompts, err := promptStore.GetRecentUserPromptsByProject(ctx, "project-a", 10)
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, prompts, 3)
|
||||
|
||||
// Verify all prompts are from project-a
|
||||
for _, prompt := range prompts {
|
||||
assert.Equal(t, "project-a", prompt.Project)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPromptStore_GetRecentUserPromptsByProject_Limit(t *testing.T) {
|
||||
promptStore, store, cleanup := testPromptStore(t)
|
||||
defer cleanup()
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
// Create session
|
||||
sessionStore := NewSessionStore(store)
|
||||
_, err := sessionStore.CreateSDKSession(ctx, "claude-1", "test-project", "")
|
||||
require.NoError(t, err)
|
||||
|
||||
// Save multiple prompts
|
||||
for i := 1; i <= 10; i++ {
|
||||
_, err := promptStore.SaveUserPromptWithMatches(ctx, "claude-1", i, "Prompt", i)
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
// Wait for any async cleanup to complete
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
|
||||
// Get with limit
|
||||
prompts, err := promptStore.GetRecentUserPromptsByProject(ctx, "test-project", 5)
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, prompts, 5)
|
||||
}
|
||||
@@ -0,0 +1,383 @@
|
||||
// Package gorm provides GORM-based database operations for claude-mnemonic.
|
||||
package gorm
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
|
||||
"gorm.io/gorm"
|
||||
"gorm.io/gorm/clause"
|
||||
|
||||
"github.com/lukaszraczylo/claude-mnemonic/pkg/models"
|
||||
)
|
||||
|
||||
// RelationStore provides relation-related database operations using GORM.
|
||||
type RelationStore struct {
|
||||
db *gorm.DB
|
||||
}
|
||||
|
||||
// NewRelationStore creates a new relation store.
|
||||
func NewRelationStore(store *Store) *RelationStore {
|
||||
return &RelationStore{
|
||||
db: store.DB,
|
||||
}
|
||||
}
|
||||
|
||||
// StoreRelation stores a new observation relation.
|
||||
// Uses INSERT OR IGNORE to handle duplicate (source_id, target_id, relation_type) combinations.
|
||||
func (s *RelationStore) StoreRelation(ctx context.Context, relation *models.ObservationRelation) (int64, error) {
|
||||
dbRelation := &ObservationRelation{
|
||||
SourceID: relation.SourceID,
|
||||
TargetID: relation.TargetID,
|
||||
RelationType: relation.RelationType,
|
||||
Confidence: relation.Confidence,
|
||||
DetectionSource: relation.DetectionSource,
|
||||
CreatedAt: relation.CreatedAt,
|
||||
CreatedAtEpoch: relation.CreatedAtEpoch,
|
||||
}
|
||||
|
||||
// Handle nullable fields
|
||||
if relation.Reason != "" {
|
||||
dbRelation.Reason = sql.NullString{String: relation.Reason, Valid: true}
|
||||
}
|
||||
|
||||
// INSERT OR IGNORE using OnConflict
|
||||
result := s.db.WithContext(ctx).
|
||||
Clauses(clause.OnConflict{
|
||||
Columns: []clause.Column{{Name: "source_id"}, {Name: "target_id"}, {Name: "relation_type"}},
|
||||
DoNothing: true,
|
||||
}).
|
||||
Create(dbRelation)
|
||||
|
||||
if result.Error != nil {
|
||||
return 0, result.Error
|
||||
}
|
||||
|
||||
// If RowsAffected is 0, the insert was ignored (duplicate)
|
||||
if result.RowsAffected == 0 {
|
||||
var existing ObservationRelation
|
||||
err := s.db.Where("source_id = ? AND target_id = ? AND relation_type = ?",
|
||||
relation.SourceID, relation.TargetID, relation.RelationType).
|
||||
First(&existing).Error
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return existing.ID, nil
|
||||
}
|
||||
|
||||
return dbRelation.ID, nil
|
||||
}
|
||||
|
||||
// StoreRelations stores multiple relations in a single transaction.
|
||||
func (s *RelationStore) StoreRelations(ctx context.Context, relations []*models.ObservationRelation) error {
|
||||
if len(relations) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
return s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
|
||||
for _, rel := range relations {
|
||||
dbRelation := &ObservationRelation{
|
||||
SourceID: rel.SourceID,
|
||||
TargetID: rel.TargetID,
|
||||
RelationType: rel.RelationType,
|
||||
Confidence: rel.Confidence,
|
||||
DetectionSource: rel.DetectionSource,
|
||||
CreatedAt: rel.CreatedAt,
|
||||
CreatedAtEpoch: rel.CreatedAtEpoch,
|
||||
}
|
||||
|
||||
if rel.Reason != "" {
|
||||
dbRelation.Reason = sql.NullString{String: rel.Reason, Valid: true}
|
||||
}
|
||||
|
||||
result := tx.Clauses(clause.OnConflict{
|
||||
Columns: []clause.Column{{Name: "source_id"}, {Name: "target_id"}, {Name: "relation_type"}},
|
||||
DoNothing: true,
|
||||
}).Create(dbRelation)
|
||||
|
||||
if result.Error != nil {
|
||||
return result.Error
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
// GetRelationsByObservationID retrieves all relations involving an observation (as source or target).
|
||||
func (s *RelationStore) GetRelationsByObservationID(ctx context.Context, obsID int64) ([]*models.ObservationRelation, error) {
|
||||
var relations []ObservationRelation
|
||||
|
||||
err := s.db.WithContext(ctx).
|
||||
Where("source_id = ? OR target_id = ?", obsID, obsID).
|
||||
Order("confidence DESC, created_at_epoch DESC").
|
||||
Find(&relations).Error
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return toModelRelations(relations), nil
|
||||
}
|
||||
|
||||
// GetOutgoingRelations retrieves relations where the observation is the source.
|
||||
func (s *RelationStore) GetOutgoingRelations(ctx context.Context, obsID int64) ([]*models.ObservationRelation, error) {
|
||||
var relations []ObservationRelation
|
||||
|
||||
err := s.db.WithContext(ctx).
|
||||
Where("source_id = ?", obsID).
|
||||
Order("confidence DESC, created_at_epoch DESC").
|
||||
Find(&relations).Error
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return toModelRelations(relations), nil
|
||||
}
|
||||
|
||||
// GetIncomingRelations retrieves relations where the observation is the target.
|
||||
func (s *RelationStore) GetIncomingRelations(ctx context.Context, obsID int64) ([]*models.ObservationRelation, error) {
|
||||
var relations []ObservationRelation
|
||||
|
||||
err := s.db.WithContext(ctx).
|
||||
Where("target_id = ?", obsID).
|
||||
Order("confidence DESC, created_at_epoch DESC").
|
||||
Find(&relations).Error
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return toModelRelations(relations), nil
|
||||
}
|
||||
|
||||
// GetRelationsByType retrieves all relations of a specific type.
|
||||
func (s *RelationStore) GetRelationsByType(ctx context.Context, relationType models.RelationType, limit int) ([]*models.ObservationRelation, error) {
|
||||
var relations []ObservationRelation
|
||||
|
||||
err := s.db.WithContext(ctx).
|
||||
Where("relation_type = ?", relationType).
|
||||
Order("confidence DESC, created_at_epoch DESC").
|
||||
Limit(limit).
|
||||
Find(&relations).Error
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return toModelRelations(relations), nil
|
||||
}
|
||||
|
||||
// GetRelationsWithDetails retrieves relations with observation titles for display.
|
||||
func (s *RelationStore) GetRelationsWithDetails(ctx context.Context, obsID int64) ([]*models.RelationWithDetails, error) {
|
||||
var results []struct {
|
||||
ObservationRelation
|
||||
SourceTitle sql.NullString `gorm:"column:source_title"`
|
||||
TargetTitle sql.NullString `gorm:"column:target_title"`
|
||||
SourceType string `gorm:"column:source_type"`
|
||||
TargetType string `gorm:"column:target_type"`
|
||||
}
|
||||
|
||||
err := s.db.WithContext(ctx).
|
||||
Table("observation_relations r").
|
||||
Select("r.*, "+
|
||||
"COALESCE(src.title, '') as source_title, "+
|
||||
"COALESCE(tgt.title, '') as target_title, "+
|
||||
"src.type as source_type, "+
|
||||
"tgt.type as target_type").
|
||||
Joins("JOIN observations src ON src.id = r.source_id").
|
||||
Joins("JOIN observations tgt ON tgt.id = r.target_id").
|
||||
Where("r.source_id = ? OR r.target_id = ?", obsID, obsID).
|
||||
Order("r.confidence DESC, r.created_at_epoch DESC").
|
||||
Scan(&results).Error
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
relations := make([]*models.RelationWithDetails, len(results))
|
||||
for i, r := range results {
|
||||
relations[i] = &models.RelationWithDetails{
|
||||
Relation: toModelRelation(&r.ObservationRelation),
|
||||
SourceTitle: r.SourceTitle.String,
|
||||
TargetTitle: r.TargetTitle.String,
|
||||
SourceType: models.ObservationType(r.SourceType),
|
||||
TargetType: models.ObservationType(r.TargetType),
|
||||
}
|
||||
}
|
||||
|
||||
return relations, nil
|
||||
}
|
||||
|
||||
// GetRelationGraph retrieves a relation graph centered on an observation.
|
||||
// This returns all observations within N hops from the center.
|
||||
func (s *RelationStore) GetRelationGraph(ctx context.Context, centerID int64, maxDepth int) (*models.RelationGraph, error) {
|
||||
// Get all relations involving the center observation
|
||||
relations, err := s.GetRelationsWithDetails(ctx, centerID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
graph := &models.RelationGraph{
|
||||
CenterID: centerID,
|
||||
Relations: relations,
|
||||
}
|
||||
|
||||
// If depth > 1, recursively get relations for connected observations
|
||||
if maxDepth > 1 {
|
||||
visited := map[int64]bool{centerID: true}
|
||||
toVisit := make([]int64, 0)
|
||||
|
||||
// Collect IDs of directly connected observations
|
||||
for _, r := range relations {
|
||||
if !visited[r.Relation.SourceID] {
|
||||
toVisit = append(toVisit, r.Relation.SourceID)
|
||||
visited[r.Relation.SourceID] = true
|
||||
}
|
||||
if !visited[r.Relation.TargetID] {
|
||||
toVisit = append(toVisit, r.Relation.TargetID)
|
||||
visited[r.Relation.TargetID] = true
|
||||
}
|
||||
}
|
||||
|
||||
// Get relations for connected observations (depth - 1)
|
||||
for depth := 1; depth < maxDepth && len(toVisit) > 0; depth++ {
|
||||
nextLevel := make([]int64, 0)
|
||||
for _, obsID := range toVisit {
|
||||
moreRelations, err := s.GetRelationsWithDetails(ctx, obsID)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
for _, r := range moreRelations {
|
||||
// Avoid duplicates
|
||||
exists := false
|
||||
for _, existing := range graph.Relations {
|
||||
if existing.Relation.ID == r.Relation.ID {
|
||||
exists = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !exists {
|
||||
graph.Relations = append(graph.Relations, r)
|
||||
}
|
||||
|
||||
// Queue next level
|
||||
if !visited[r.Relation.SourceID] {
|
||||
nextLevel = append(nextLevel, r.Relation.SourceID)
|
||||
visited[r.Relation.SourceID] = true
|
||||
}
|
||||
if !visited[r.Relation.TargetID] {
|
||||
nextLevel = append(nextLevel, r.Relation.TargetID)
|
||||
visited[r.Relation.TargetID] = true
|
||||
}
|
||||
}
|
||||
}
|
||||
toVisit = nextLevel
|
||||
}
|
||||
}
|
||||
|
||||
return graph, nil
|
||||
}
|
||||
|
||||
// DeleteRelationsByObservationID deletes all relations involving an observation.
|
||||
// Called when an observation is deleted.
|
||||
func (s *RelationStore) DeleteRelationsByObservationID(ctx context.Context, obsID int64) error {
|
||||
result := s.db.WithContext(ctx).
|
||||
Where("source_id = ? OR target_id = ?", obsID, obsID).
|
||||
Delete(&ObservationRelation{})
|
||||
|
||||
return result.Error
|
||||
}
|
||||
|
||||
// GetRelationCount returns the count of relations for an observation.
|
||||
func (s *RelationStore) GetRelationCount(ctx context.Context, obsID int64) (int, error) {
|
||||
var count int64
|
||||
err := s.db.WithContext(ctx).
|
||||
Model(&ObservationRelation{}).
|
||||
Where("source_id = ? OR target_id = ?", obsID, obsID).
|
||||
Count(&count).Error
|
||||
|
||||
return int(count), err
|
||||
}
|
||||
|
||||
// GetTotalRelationCount returns the total count of all relations.
|
||||
func (s *RelationStore) GetTotalRelationCount(ctx context.Context) (int, error) {
|
||||
var count int64
|
||||
err := s.db.WithContext(ctx).
|
||||
Model(&ObservationRelation{}).
|
||||
Count(&count).Error
|
||||
|
||||
return int(count), err
|
||||
}
|
||||
|
||||
// GetHighConfidenceRelations retrieves relations with confidence above threshold.
|
||||
func (s *RelationStore) GetHighConfidenceRelations(ctx context.Context, minConfidence float64, limit int) ([]*models.ObservationRelation, error) {
|
||||
var relations []ObservationRelation
|
||||
|
||||
err := s.db.WithContext(ctx).
|
||||
Where("confidence >= ?", minConfidence).
|
||||
Order("confidence DESC, created_at_epoch DESC").
|
||||
Limit(limit).
|
||||
Find(&relations).Error
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return toModelRelations(relations), nil
|
||||
}
|
||||
|
||||
// UpdateRelationConfidence updates the confidence of a relation.
|
||||
func (s *RelationStore) UpdateRelationConfidence(ctx context.Context, relationID int64, newConfidence float64) error {
|
||||
result := s.db.WithContext(ctx).
|
||||
Model(&ObservationRelation{}).
|
||||
Where("id = ?", relationID).
|
||||
Update("confidence", newConfidence)
|
||||
|
||||
return result.Error
|
||||
}
|
||||
|
||||
// GetRelatedObservationIDs returns IDs of observations related to the given one.
|
||||
// This is useful for expanding search results.
|
||||
// Uses CASE expression for bidirectional ID lookup (GORM doesn't support this well, so we use raw SQL).
|
||||
func (s *RelationStore) GetRelatedObservationIDs(ctx context.Context, obsID int64, minConfidence float64) ([]int64, error) {
|
||||
var ids []int64
|
||||
|
||||
err := s.db.WithContext(ctx).
|
||||
Raw("SELECT DISTINCT CASE WHEN source_id = ? THEN target_id ELSE source_id END as related_id "+
|
||||
"FROM observation_relations "+
|
||||
"WHERE (source_id = ? OR target_id = ?) AND confidence >= ?",
|
||||
obsID, obsID, obsID, minConfidence).
|
||||
Pluck("related_id", &ids).Error
|
||||
|
||||
return ids, err
|
||||
}
|
||||
|
||||
// toModelRelation converts a GORM ObservationRelation to a pkg/models ObservationRelation.
|
||||
func toModelRelation(r *ObservationRelation) *models.ObservationRelation {
|
||||
relation := &models.ObservationRelation{
|
||||
ID: r.ID,
|
||||
SourceID: r.SourceID,
|
||||
TargetID: r.TargetID,
|
||||
RelationType: r.RelationType,
|
||||
Confidence: r.Confidence,
|
||||
DetectionSource: r.DetectionSource,
|
||||
CreatedAt: r.CreatedAt,
|
||||
CreatedAtEpoch: r.CreatedAtEpoch,
|
||||
}
|
||||
|
||||
if r.Reason.Valid {
|
||||
relation.Reason = r.Reason.String
|
||||
}
|
||||
|
||||
return relation
|
||||
}
|
||||
|
||||
// toModelRelations converts a slice of GORM ObservationRelations to pkg/models ObservationRelations.
|
||||
func toModelRelations(relations []ObservationRelation) []*models.ObservationRelation {
|
||||
result := make([]*models.ObservationRelation, len(relations))
|
||||
for i, r := range relations {
|
||||
result[i] = toModelRelation(&r)
|
||||
}
|
||||
return result
|
||||
}
|
||||
@@ -0,0 +1,306 @@
|
||||
//go:build fts5
|
||||
|
||||
// Package gorm provides GORM-based database operations for claude-mnemonic.
|
||||
package gorm
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"gorm.io/gorm/logger"
|
||||
|
||||
"github.com/lukaszraczylo/claude-mnemonic/pkg/models"
|
||||
)
|
||||
|
||||
func testRelationStore(t *testing.T) (*RelationStore, *Store, func()) {
|
||||
t.Helper()
|
||||
|
||||
tmpDir, err := os.MkdirTemp("", "gorm_relation_test_*")
|
||||
if err != nil {
|
||||
t.Fatalf("create temp dir: %v", err)
|
||||
}
|
||||
|
||||
dbPath := filepath.Join(tmpDir, "test.db")
|
||||
cfg := Config{
|
||||
Path: dbPath,
|
||||
MaxConns: 4,
|
||||
LogLevel: logger.Silent,
|
||||
}
|
||||
|
||||
store, err := NewStore(cfg)
|
||||
if err != nil {
|
||||
os.RemoveAll(tmpDir)
|
||||
t.Fatalf("NewStore failed: %v", err)
|
||||
}
|
||||
|
||||
relationStore := NewRelationStore(store)
|
||||
|
||||
cleanup := func() {
|
||||
store.Close()
|
||||
os.RemoveAll(tmpDir)
|
||||
}
|
||||
|
||||
return relationStore, store, cleanup
|
||||
}
|
||||
|
||||
func TestRelationStore_StoreRelation(t *testing.T) {
|
||||
relationStore, _, cleanup := testRelationStore(t)
|
||||
defer cleanup()
|
||||
|
||||
ctx := context.Background()
|
||||
now := time.Now()
|
||||
|
||||
relation := &models.ObservationRelation{
|
||||
SourceID: 1,
|
||||
TargetID: 2,
|
||||
RelationType: models.RelationCauses,
|
||||
Confidence: 0.8,
|
||||
DetectionSource: models.DetectionSourceFileOverlap,
|
||||
Reason: "Test relation",
|
||||
CreatedAt: now.Format(time.RFC3339),
|
||||
CreatedAtEpoch: now.UnixMilli(),
|
||||
}
|
||||
|
||||
id, err := relationStore.StoreRelation(ctx, relation)
|
||||
require.NoError(t, err)
|
||||
assert.Greater(t, id, int64(0))
|
||||
}
|
||||
|
||||
func TestRelationStore_StoreRelation_Idempotency(t *testing.T) {
|
||||
relationStore, _, cleanup := testRelationStore(t)
|
||||
defer cleanup()
|
||||
|
||||
ctx := context.Background()
|
||||
now := time.Now()
|
||||
|
||||
relation := &models.ObservationRelation{
|
||||
SourceID: 1,
|
||||
TargetID: 2,
|
||||
RelationType: models.RelationCauses,
|
||||
Confidence: 0.8,
|
||||
DetectionSource: models.DetectionSourceFileOverlap,
|
||||
CreatedAt: now.Format(time.RFC3339),
|
||||
CreatedAtEpoch: now.UnixMilli(),
|
||||
}
|
||||
|
||||
id1, err := relationStore.StoreRelation(ctx, relation)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Store again with same source/target/type - should return same ID
|
||||
id2, err := relationStore.StoreRelation(ctx, relation)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, id1, id2)
|
||||
}
|
||||
|
||||
func TestRelationStore_StoreRelations(t *testing.T) {
|
||||
relationStore, _, cleanup := testRelationStore(t)
|
||||
defer cleanup()
|
||||
|
||||
ctx := context.Background()
|
||||
now := time.Now()
|
||||
|
||||
relations := []*models.ObservationRelation{
|
||||
{
|
||||
SourceID: 1,
|
||||
TargetID: 2,
|
||||
RelationType: models.RelationCauses,
|
||||
Confidence: 0.8,
|
||||
DetectionSource: models.DetectionSourceFileOverlap,
|
||||
CreatedAt: now.Format(time.RFC3339),
|
||||
CreatedAtEpoch: now.UnixMilli(),
|
||||
},
|
||||
{
|
||||
SourceID: 2,
|
||||
TargetID: 3,
|
||||
RelationType: models.RelationFixes,
|
||||
Confidence: 0.9,
|
||||
DetectionSource: models.DetectionSourceTemporalProximity,
|
||||
CreatedAt: now.Format(time.RFC3339),
|
||||
CreatedAtEpoch: now.UnixMilli(),
|
||||
},
|
||||
}
|
||||
|
||||
err := relationStore.StoreRelations(ctx, relations)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Verify both were stored
|
||||
count, err := relationStore.GetTotalRelationCount(ctx)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 2, count)
|
||||
}
|
||||
|
||||
func TestRelationStore_GetRelationsByObservationID(t *testing.T) {
|
||||
relationStore, _, cleanup := testRelationStore(t)
|
||||
defer cleanup()
|
||||
|
||||
ctx := context.Background()
|
||||
now := time.Now()
|
||||
|
||||
// Create relations involving observation 2
|
||||
relations := []*models.ObservationRelation{
|
||||
{
|
||||
SourceID: 1,
|
||||
TargetID: 2,
|
||||
RelationType: models.RelationCauses,
|
||||
Confidence: 0.8,
|
||||
DetectionSource: models.DetectionSourceFileOverlap,
|
||||
CreatedAt: now.Format(time.RFC3339),
|
||||
CreatedAtEpoch: now.UnixMilli(),
|
||||
},
|
||||
{
|
||||
SourceID: 2,
|
||||
TargetID: 3,
|
||||
RelationType: models.RelationFixes,
|
||||
Confidence: 0.9,
|
||||
DetectionSource: models.DetectionSourceTemporalProximity,
|
||||
CreatedAt: now.Format(time.RFC3339),
|
||||
CreatedAtEpoch: now.UnixMilli(),
|
||||
},
|
||||
}
|
||||
|
||||
err := relationStore.StoreRelations(ctx, relations)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Get relations for observation 2 (involved in both)
|
||||
result, err := relationStore.GetRelationsByObservationID(ctx, 2)
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, result, 2)
|
||||
}
|
||||
|
||||
func TestRelationStore_GetOutgoingAndIncomingRelations(t *testing.T) {
|
||||
relationStore, _, cleanup := testRelationStore(t)
|
||||
defer cleanup()
|
||||
|
||||
ctx := context.Background()
|
||||
now := time.Now()
|
||||
|
||||
relations := []*models.ObservationRelation{
|
||||
{
|
||||
SourceID: 2,
|
||||
TargetID: 1,
|
||||
RelationType: models.RelationCauses,
|
||||
Confidence: 0.8,
|
||||
DetectionSource: models.DetectionSourceFileOverlap,
|
||||
CreatedAt: now.Format(time.RFC3339),
|
||||
CreatedAtEpoch: now.UnixMilli(),
|
||||
},
|
||||
{
|
||||
SourceID: 3,
|
||||
TargetID: 2,
|
||||
RelationType: models.RelationFixes,
|
||||
Confidence: 0.9,
|
||||
DetectionSource: models.DetectionSourceTemporalProximity,
|
||||
CreatedAt: now.Format(time.RFC3339),
|
||||
CreatedAtEpoch: now.UnixMilli(),
|
||||
},
|
||||
}
|
||||
|
||||
err := relationStore.StoreRelations(ctx, relations)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Observation 2 has 1 outgoing (to 1) and 1 incoming (from 3)
|
||||
outgoing, err := relationStore.GetOutgoingRelations(ctx, 2)
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, outgoing, 1)
|
||||
assert.Equal(t, int64(1), outgoing[0].TargetID)
|
||||
|
||||
incoming, err := relationStore.GetIncomingRelations(ctx, 2)
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, incoming, 1)
|
||||
assert.Equal(t, int64(3), incoming[0].SourceID)
|
||||
}
|
||||
|
||||
func TestRelationStore_GetRelationCount(t *testing.T) {
|
||||
relationStore, _, cleanup := testRelationStore(t)
|
||||
defer cleanup()
|
||||
|
||||
ctx := context.Background()
|
||||
now := time.Now()
|
||||
|
||||
relations := []*models.ObservationRelation{
|
||||
{
|
||||
SourceID: 1,
|
||||
TargetID: 2,
|
||||
RelationType: models.RelationCauses,
|
||||
Confidence: 0.8,
|
||||
DetectionSource: models.DetectionSourceFileOverlap,
|
||||
CreatedAt: now.Format(time.RFC3339),
|
||||
CreatedAtEpoch: now.UnixMilli(),
|
||||
},
|
||||
{
|
||||
SourceID: 2,
|
||||
TargetID: 3,
|
||||
RelationType: models.RelationFixes,
|
||||
Confidence: 0.9,
|
||||
DetectionSource: models.DetectionSourceTemporalProximity,
|
||||
CreatedAt: now.Format(time.RFC3339),
|
||||
CreatedAtEpoch: now.UnixMilli(),
|
||||
},
|
||||
}
|
||||
|
||||
err := relationStore.StoreRelations(ctx, relations)
|
||||
require.NoError(t, err)
|
||||
|
||||
count, err := relationStore.GetRelationCount(ctx, 2)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 2, count)
|
||||
|
||||
count, err = relationStore.GetRelationCount(ctx, 1)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 1, count)
|
||||
}
|
||||
|
||||
func TestRelationStore_DeleteRelationsByObservationID(t *testing.T) {
|
||||
relationStore, _, cleanup := testRelationStore(t)
|
||||
defer cleanup()
|
||||
|
||||
ctx := context.Background()
|
||||
now := time.Now()
|
||||
|
||||
relations := []*models.ObservationRelation{
|
||||
{
|
||||
SourceID: 1,
|
||||
TargetID: 2,
|
||||
RelationType: models.RelationCauses,
|
||||
Confidence: 0.8,
|
||||
DetectionSource: models.DetectionSourceFileOverlap,
|
||||
CreatedAt: now.Format(time.RFC3339),
|
||||
CreatedAtEpoch: now.UnixMilli(),
|
||||
},
|
||||
{
|
||||
SourceID: 2,
|
||||
TargetID: 3,
|
||||
RelationType: models.RelationFixes,
|
||||
Confidence: 0.9,
|
||||
DetectionSource: models.DetectionSourceTemporalProximity,
|
||||
CreatedAt: now.Format(time.RFC3339),
|
||||
CreatedAtEpoch: now.UnixMilli(),
|
||||
},
|
||||
{
|
||||
SourceID: 4,
|
||||
TargetID: 5,
|
||||
RelationType: models.RelationRelatesTo,
|
||||
Confidence: 0.7,
|
||||
DetectionSource: models.DetectionSourceConceptOverlap,
|
||||
CreatedAt: now.Format(time.RFC3339),
|
||||
CreatedAtEpoch: now.UnixMilli(),
|
||||
},
|
||||
}
|
||||
|
||||
err := relationStore.StoreRelations(ctx, relations)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Delete relations involving observation 2
|
||||
err = relationStore.DeleteRelationsByObservationID(ctx, 2)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Verify only 1 relation remains (4->5)
|
||||
total, err := relationStore.GetTotalRelationCount(ctx)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 1, total)
|
||||
}
|
||||
@@ -0,0 +1,260 @@
|
||||
// Package gorm provides GORM-based database operations for claude-mnemonic.
|
||||
package gorm
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"gorm.io/gorm"
|
||||
|
||||
"github.com/lukaszraczylo/claude-mnemonic/pkg/models"
|
||||
)
|
||||
|
||||
// UpdateObservationFeedback updates the user feedback for an observation.
|
||||
// Feedback values: -1 (thumbs down), 0 (neutral), 1 (thumbs up).
|
||||
func (s *ObservationStore) UpdateObservationFeedback(ctx context.Context, id int64, feedback int) error {
|
||||
now := time.Now().UnixMilli()
|
||||
|
||||
result := s.db.WithContext(ctx).
|
||||
Model(&Observation{}).
|
||||
Where("id = ?", id).
|
||||
Updates(map[string]interface{}{
|
||||
"user_feedback": feedback,
|
||||
"score_updated_at_epoch": now,
|
||||
})
|
||||
|
||||
return result.Error
|
||||
}
|
||||
|
||||
// IncrementRetrievalCount increments the retrieval counter for the given observation IDs.
|
||||
// This is called when observations are returned in search results.
|
||||
func (s *ObservationStore) IncrementRetrievalCount(ctx context.Context, ids []int64) error {
|
||||
if len(ids) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
now := time.Now().UnixMilli()
|
||||
|
||||
// Use raw SQL for increment expression
|
||||
result := s.db.WithContext(ctx).
|
||||
Exec("UPDATE observations SET retrieval_count = COALESCE(retrieval_count, 0) + 1, last_retrieved_at_epoch = ? WHERE id IN ?",
|
||||
now, ids)
|
||||
|
||||
return result.Error
|
||||
}
|
||||
|
||||
// UpdateImportanceScore updates the importance score for a single observation.
|
||||
func (s *ObservationStore) UpdateImportanceScore(ctx context.Context, id int64, score float64) error {
|
||||
now := time.Now().UnixMilli()
|
||||
|
||||
result := s.db.WithContext(ctx).
|
||||
Model(&Observation{}).
|
||||
Where("id = ?", id).
|
||||
Updates(map[string]interface{}{
|
||||
"importance_score": score,
|
||||
"score_updated_at_epoch": now,
|
||||
})
|
||||
|
||||
return result.Error
|
||||
}
|
||||
|
||||
// UpdateImportanceScores bulk updates importance scores for multiple observations.
|
||||
// This is more efficient than individual updates for batch recalculation.
|
||||
func (s *ObservationStore) UpdateImportanceScores(ctx context.Context, scores map[int64]float64) error {
|
||||
if len(scores) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
return s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
|
||||
now := time.Now().UnixMilli()
|
||||
|
||||
for id, score := range scores {
|
||||
err := tx.Model(&Observation{}).
|
||||
Where("id = ?", id).
|
||||
Updates(map[string]interface{}{
|
||||
"importance_score": score,
|
||||
"score_updated_at_epoch": now,
|
||||
}).Error
|
||||
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
// GetObservationsNeedingScoreUpdate returns observations that need their importance score recalculated.
|
||||
// Returns observations where score_updated_at_epoch is NULL or older than the threshold.
|
||||
func (s *ObservationStore) GetObservationsNeedingScoreUpdate(ctx context.Context, threshold time.Duration, limit int) ([]*models.Observation, error) {
|
||||
cutoff := time.Now().Add(-threshold).UnixMilli()
|
||||
|
||||
var observations []Observation
|
||||
|
||||
err := s.db.WithContext(ctx).
|
||||
Where("score_updated_at_epoch IS NULL OR score_updated_at_epoch < ?", cutoff).
|
||||
Order("created_at_epoch DESC").
|
||||
Limit(limit).
|
||||
Find(&observations).Error
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return toModelObservations(observations), nil
|
||||
}
|
||||
|
||||
// GetConceptWeights returns all concept weights from the database.
|
||||
func (s *ObservationStore) GetConceptWeights(ctx context.Context) (map[string]float64, error) {
|
||||
var weights []struct {
|
||||
Concept string
|
||||
Weight float64
|
||||
}
|
||||
|
||||
err := s.db.WithContext(ctx).
|
||||
Table("concept_weights").
|
||||
Select("concept, weight").
|
||||
Scan(&weights).Error
|
||||
|
||||
if err != nil {
|
||||
return models.DefaultConceptWeights, nil
|
||||
}
|
||||
|
||||
if len(weights) == 0 {
|
||||
return models.DefaultConceptWeights, nil
|
||||
}
|
||||
|
||||
result := make(map[string]float64, len(weights))
|
||||
for _, w := range weights {
|
||||
result[w.Concept] = w.Weight
|
||||
}
|
||||
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// SetConceptWeights stores concept weights in the database using UPSERT.
|
||||
func (s *ObservationStore) SetConceptWeights(ctx context.Context, weights map[string]float64) error {
|
||||
if len(weights) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
return s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
|
||||
for concept, weight := range weights {
|
||||
// UPSERT using raw SQL since GORM's ON CONFLICT is complex for this case
|
||||
err := tx.Exec(`
|
||||
INSERT INTO concept_weights (concept, weight, updated_at)
|
||||
VALUES (?, ?, datetime('now'))
|
||||
ON CONFLICT(concept) DO UPDATE SET weight = excluded.weight, updated_at = excluded.updated_at
|
||||
`, concept, weight).Error
|
||||
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
// UpdateConceptWeight updates a single concept weight in the database using UPSERT.
|
||||
func (s *ObservationStore) UpdateConceptWeight(ctx context.Context, concept string, weight float64) error {
|
||||
return s.db.WithContext(ctx).Exec(`
|
||||
INSERT INTO concept_weights (concept, weight, updated_at)
|
||||
VALUES (?, ?, datetime('now'))
|
||||
ON CONFLICT(concept) DO UPDATE SET weight = excluded.weight, updated_at = excluded.updated_at
|
||||
`, concept, weight).Error
|
||||
}
|
||||
|
||||
// FeedbackStats contains statistics about observation feedback and scoring.
|
||||
type FeedbackStats struct {
|
||||
Total int `json:"total"`
|
||||
Positive int `json:"positive"`
|
||||
Negative int `json:"negative"`
|
||||
Neutral int `json:"neutral"`
|
||||
AvgScore float64 `json:"avg_score"`
|
||||
AvgRetrieval float64 `json:"avg_retrieval"`
|
||||
}
|
||||
|
||||
// GetObservationFeedbackStats returns statistics about user feedback.
|
||||
func (s *ObservationStore) GetObservationFeedbackStats(ctx context.Context, project string) (*FeedbackStats, error) {
|
||||
var stats FeedbackStats
|
||||
|
||||
query := s.db.WithContext(ctx).
|
||||
Model(&Observation{}).
|
||||
Select(`
|
||||
COUNT(*) as total,
|
||||
COALESCE(SUM(CASE WHEN user_feedback = 1 THEN 1 ELSE 0 END), 0) as positive,
|
||||
COALESCE(SUM(CASE WHEN user_feedback = -1 THEN 1 ELSE 0 END), 0) as negative,
|
||||
COALESCE(SUM(CASE WHEN user_feedback = 0 THEN 1 ELSE 0 END), 0) as neutral,
|
||||
COALESCE(AVG(COALESCE(importance_score, 1.0)), 0) as avg_score,
|
||||
COALESCE(AVG(COALESCE(retrieval_count, 0)), 0) as avg_retrieval
|
||||
`)
|
||||
|
||||
if project != "" {
|
||||
query = query.Where("project = ? OR scope = 'global'", project)
|
||||
}
|
||||
|
||||
err := query.Scan(&stats).Error
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &stats, nil
|
||||
}
|
||||
|
||||
// GetTopScoringObservations returns the highest-scoring observations.
|
||||
func (s *ObservationStore) GetTopScoringObservations(ctx context.Context, project string, limit int) ([]*models.Observation, error) {
|
||||
var observations []Observation
|
||||
|
||||
query := s.db.WithContext(ctx).
|
||||
Order("COALESCE(importance_score, 1.0) DESC, created_at_epoch DESC").
|
||||
Limit(limit)
|
||||
|
||||
if project != "" {
|
||||
query = query.Where("project = ? OR scope = 'global'", project)
|
||||
}
|
||||
|
||||
err := query.Find(&observations).Error
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return toModelObservations(observations), nil
|
||||
}
|
||||
|
||||
// GetMostRetrievedObservations returns the most frequently retrieved observations.
|
||||
func (s *ObservationStore) GetMostRetrievedObservations(ctx context.Context, project string, limit int) ([]*models.Observation, error) {
|
||||
var observations []Observation
|
||||
|
||||
query := s.db.WithContext(ctx).
|
||||
Where("retrieval_count > 0").
|
||||
Order("retrieval_count DESC, created_at_epoch DESC").
|
||||
Limit(limit)
|
||||
|
||||
if project != "" {
|
||||
query = query.Where("project = ? OR scope = 'global'", project)
|
||||
}
|
||||
|
||||
err := query.Find(&observations).Error
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return toModelObservations(observations), nil
|
||||
}
|
||||
|
||||
// ResetObservationScores resets all observation scores to their default values.
|
||||
// This is useful for testing or when changing the scoring algorithm.
|
||||
func (s *ObservationStore) ResetObservationScores(ctx context.Context) error {
|
||||
// Use Where("1 = 1") to explicitly allow bulk update of all rows
|
||||
result := s.db.WithContext(ctx).
|
||||
Model(&Observation{}).
|
||||
Where("1 = 1").
|
||||
Updates(map[string]interface{}{
|
||||
"importance_score": 1.0,
|
||||
"score_updated_at_epoch": nil,
|
||||
})
|
||||
|
||||
return result.Error
|
||||
}
|
||||
@@ -0,0 +1,355 @@
|
||||
//go:build fts5
|
||||
|
||||
// Package gorm provides GORM-based database operations for claude-mnemonic.
|
||||
package gorm
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/lukaszraczylo/claude-mnemonic/pkg/models"
|
||||
)
|
||||
|
||||
func TestObservationStore_UpdateImportanceScore(t *testing.T) {
|
||||
obsStore, store, cleanup := testObservationStore(t)
|
||||
defer cleanup()
|
||||
ctx := context.Background()
|
||||
|
||||
// Create observation
|
||||
sessionStore := NewSessionStore(store)
|
||||
sessionID, _ := sessionStore.CreateSDKSession(ctx, "claude-1", "test-project", "")
|
||||
obs := &models.ParsedObservation{Type: models.ObsTypeDiscovery, Title: "Test"}
|
||||
obsID, _, _ := obsStore.StoreObservation(ctx, "claude-1", "test-project", obs, int(sessionID), 1)
|
||||
|
||||
// Update score
|
||||
err := obsStore.UpdateImportanceScore(ctx, obsID, 5.0)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Verify
|
||||
var dbObs Observation
|
||||
store.DB.First(&dbObs, obsID)
|
||||
assert.Equal(t, 5.0, dbObs.ImportanceScore)
|
||||
}
|
||||
|
||||
func TestObservationStore_IncrementRetrievalCount(t *testing.T) {
|
||||
obsStore, store, cleanup := testObservationStore(t)
|
||||
defer cleanup()
|
||||
ctx := context.Background()
|
||||
|
||||
sessionStore := NewSessionStore(store)
|
||||
sessionID, _ := sessionStore.CreateSDKSession(ctx, "claude-1", "test-project", "")
|
||||
obs := &models.ParsedObservation{Type: models.ObsTypeDiscovery, Title: "Test"}
|
||||
obsID, _, _ := obsStore.StoreObservation(ctx, "claude-1", "test-project", obs, int(sessionID), 1)
|
||||
|
||||
err := obsStore.IncrementRetrievalCount(ctx, []int64{obsID})
|
||||
require.NoError(t, err)
|
||||
|
||||
var dbObs Observation
|
||||
store.DB.First(&dbObs, obsID)
|
||||
assert.Equal(t, 1, dbObs.RetrievalCount)
|
||||
}
|
||||
|
||||
func TestObservationStore_IncrementRetrievalCount_Multiple(t *testing.T) {
|
||||
obsStore, store, cleanup := testObservationStore(t)
|
||||
defer cleanup()
|
||||
ctx := context.Background()
|
||||
|
||||
sessionStore := NewSessionStore(store)
|
||||
sessionID, _ := sessionStore.CreateSDKSession(ctx, "claude-1", "test-project", "")
|
||||
|
||||
// Create 3 observations
|
||||
ids := make([]int64, 3)
|
||||
for i := 0; i < 3; i++ {
|
||||
obs := &models.ParsedObservation{Type: models.ObsTypeDiscovery, Title: "Test"}
|
||||
obsID, _, _ := obsStore.StoreObservation(ctx, "claude-1", "test-project", obs, int(sessionID), 1)
|
||||
ids[i] = obsID
|
||||
}
|
||||
|
||||
// Increment all
|
||||
err := obsStore.IncrementRetrievalCount(ctx, ids)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Verify all were incremented
|
||||
for _, id := range ids {
|
||||
var dbObs Observation
|
||||
store.DB.First(&dbObs, id)
|
||||
assert.Equal(t, 1, dbObs.RetrievalCount)
|
||||
}
|
||||
|
||||
// Increment again
|
||||
err = obsStore.IncrementRetrievalCount(ctx, ids)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Verify all are now 2
|
||||
for _, id := range ids {
|
||||
var dbObs Observation
|
||||
store.DB.First(&dbObs, id)
|
||||
assert.Equal(t, 2, dbObs.RetrievalCount)
|
||||
}
|
||||
}
|
||||
|
||||
func TestObservationStore_UpdateImportanceScores_Bulk(t *testing.T) {
|
||||
obsStore, store, cleanup := testObservationStore(t)
|
||||
defer cleanup()
|
||||
ctx := context.Background()
|
||||
|
||||
sessionStore := NewSessionStore(store)
|
||||
sessionID, _ := sessionStore.CreateSDKSession(ctx, "claude-1", "test-project", "")
|
||||
|
||||
// Create 3 observations
|
||||
ids := make([]int64, 3)
|
||||
for i := 0; i < 3; i++ {
|
||||
obs := &models.ParsedObservation{Type: models.ObsTypeDiscovery, Title: "Test"}
|
||||
obsID, _, _ := obsStore.StoreObservation(ctx, "claude-1", "test-project", obs, int(sessionID), 1)
|
||||
ids[i] = obsID
|
||||
}
|
||||
|
||||
// Bulk update scores
|
||||
scores := map[int64]float64{
|
||||
ids[0]: 2.5,
|
||||
ids[1]: 3.7,
|
||||
ids[2]: 1.2,
|
||||
}
|
||||
|
||||
err := obsStore.UpdateImportanceScores(ctx, scores)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Verify scores
|
||||
for id, expectedScore := range scores {
|
||||
var dbObs Observation
|
||||
store.DB.First(&dbObs, id)
|
||||
assert.Equal(t, expectedScore, dbObs.ImportanceScore)
|
||||
}
|
||||
}
|
||||
|
||||
func TestObservationStore_UpdateObservationFeedback(t *testing.T) {
|
||||
obsStore, store, cleanup := testObservationStore(t)
|
||||
defer cleanup()
|
||||
ctx := context.Background()
|
||||
|
||||
sessionStore := NewSessionStore(store)
|
||||
sessionID, _ := sessionStore.CreateSDKSession(ctx, "claude-1", "test-project", "")
|
||||
obs := &models.ParsedObservation{Type: models.ObsTypeDiscovery, Title: "Test"}
|
||||
obsID, _, _ := obsStore.StoreObservation(ctx, "claude-1", "test-project", obs, int(sessionID), 1)
|
||||
|
||||
// Set thumbs up
|
||||
err := obsStore.UpdateObservationFeedback(ctx, obsID, 1)
|
||||
require.NoError(t, err)
|
||||
|
||||
var dbObs Observation
|
||||
store.DB.First(&dbObs, obsID)
|
||||
assert.Equal(t, 1, dbObs.UserFeedback)
|
||||
|
||||
// Set thumbs down
|
||||
err = obsStore.UpdateObservationFeedback(ctx, obsID, -1)
|
||||
require.NoError(t, err)
|
||||
|
||||
store.DB.First(&dbObs, obsID)
|
||||
assert.Equal(t, -1, dbObs.UserFeedback)
|
||||
}
|
||||
|
||||
func TestObservationStore_GetObservationFeedbackStats(t *testing.T) {
|
||||
obsStore, store, cleanup := testObservationStore(t)
|
||||
defer cleanup()
|
||||
ctx := context.Background()
|
||||
|
||||
sessionStore := NewSessionStore(store)
|
||||
sessionID, _ := sessionStore.CreateSDKSession(ctx, "claude-1", "test-project", "")
|
||||
|
||||
// Create observations with different feedback
|
||||
obs1 := &models.ParsedObservation{Type: models.ObsTypeDiscovery, Title: "Test1"}
|
||||
obsID1, _, _ := obsStore.StoreObservation(ctx, "claude-1", "test-project", obs1, int(sessionID), 1)
|
||||
obsStore.UpdateObservationFeedback(ctx, obsID1, 1) // thumbs up
|
||||
obsStore.UpdateImportanceScore(ctx, obsID1, 3.0)
|
||||
|
||||
obs2 := &models.ParsedObservation{Type: models.ObsTypeBugfix, Title: "Test2"}
|
||||
obsID2, _, _ := obsStore.StoreObservation(ctx, "claude-1", "test-project", obs2, int(sessionID), 2)
|
||||
obsStore.UpdateObservationFeedback(ctx, obsID2, -1) // thumbs down
|
||||
obsStore.UpdateImportanceScore(ctx, obsID2, 2.0)
|
||||
|
||||
obs3 := &models.ParsedObservation{Type: models.ObsTypeFeature, Title: "Test3"}
|
||||
obsID3, _, _ := obsStore.StoreObservation(ctx, "claude-1", "test-project", obs3, int(sessionID), 3)
|
||||
// neutral (0)
|
||||
obsStore.UpdateImportanceScore(ctx, obsID3, 1.5)
|
||||
obsStore.IncrementRetrievalCount(ctx, []int64{obsID1, obsID2, obsID3})
|
||||
|
||||
// Get stats
|
||||
stats, err := obsStore.GetObservationFeedbackStats(ctx, "test-project")
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.Equal(t, 3, stats.Total)
|
||||
assert.Equal(t, 1, stats.Positive)
|
||||
assert.Equal(t, 1, stats.Negative)
|
||||
assert.Equal(t, 1, stats.Neutral)
|
||||
assert.InDelta(t, 2.166, stats.AvgScore, 0.01) // (3.0 + 2.0 + 1.5) / 3
|
||||
assert.InDelta(t, 1.0, stats.AvgRetrieval, 0.01)
|
||||
}
|
||||
|
||||
func TestObservationStore_GetTopScoringObservations(t *testing.T) {
|
||||
obsStore, store, cleanup := testObservationStore(t)
|
||||
defer cleanup()
|
||||
ctx := context.Background()
|
||||
|
||||
sessionStore := NewSessionStore(store)
|
||||
sessionID, _ := sessionStore.CreateSDKSession(ctx, "claude-1", "test-project", "")
|
||||
|
||||
// Create observations with different scores
|
||||
obs1 := &models.ParsedObservation{Type: models.ObsTypeDiscovery, Title: "High"}
|
||||
obsID1, _, _ := obsStore.StoreObservation(ctx, "claude-1", "test-project", obs1, int(sessionID), 1)
|
||||
obsStore.UpdateImportanceScore(ctx, obsID1, 5.0)
|
||||
|
||||
obs2 := &models.ParsedObservation{Type: models.ObsTypeBugfix, Title: "Low"}
|
||||
obsID2, _, _ := obsStore.StoreObservation(ctx, "claude-1", "test-project", obs2, int(sessionID), 2)
|
||||
obsStore.UpdateImportanceScore(ctx, obsID2, 1.0)
|
||||
|
||||
obs3 := &models.ParsedObservation{Type: models.ObsTypeFeature, Title: "Medium"}
|
||||
obsID3, _, _ := obsStore.StoreObservation(ctx, "claude-1", "test-project", obs3, int(sessionID), 3)
|
||||
obsStore.UpdateImportanceScore(ctx, obsID3, 3.0)
|
||||
|
||||
// Get top 2
|
||||
topObs, err := obsStore.GetTopScoringObservations(ctx, "test-project", 2)
|
||||
require.NoError(t, err)
|
||||
|
||||
require.Len(t, topObs, 2)
|
||||
assert.True(t, topObs[0].Title.Valid)
|
||||
assert.Equal(t, "High", topObs[0].Title.String)
|
||||
assert.Equal(t, 5.0, topObs[0].ImportanceScore)
|
||||
assert.True(t, topObs[1].Title.Valid)
|
||||
assert.Equal(t, "Medium", topObs[1].Title.String)
|
||||
assert.Equal(t, 3.0, topObs[1].ImportanceScore)
|
||||
}
|
||||
|
||||
func TestObservationStore_GetMostRetrievedObservations(t *testing.T) {
|
||||
obsStore, store, cleanup := testObservationStore(t)
|
||||
defer cleanup()
|
||||
ctx := context.Background()
|
||||
|
||||
sessionStore := NewSessionStore(store)
|
||||
sessionID, _ := sessionStore.CreateSDKSession(ctx, "claude-1", "test-project", "")
|
||||
|
||||
// Create observations
|
||||
obs1 := &models.ParsedObservation{Type: models.ObsTypeDiscovery, Title: "Popular"}
|
||||
obsID1, _, _ := obsStore.StoreObservation(ctx, "claude-1", "test-project", obs1, int(sessionID), 1)
|
||||
|
||||
obs2 := &models.ParsedObservation{Type: models.ObsTypeBugfix, Title: "Unpopular"}
|
||||
obsID2, _, _ := obsStore.StoreObservation(ctx, "claude-1", "test-project", obs2, int(sessionID), 2)
|
||||
|
||||
// Increment retrieval counts - each call increments by 1
|
||||
obsStore.IncrementRetrievalCount(ctx, []int64{obsID1}) // increment by 1
|
||||
obsStore.IncrementRetrievalCount(ctx, []int64{obsID1}) // increment by 1 again (total: 2)
|
||||
obsStore.IncrementRetrievalCount(ctx, []int64{obsID1}) // increment by 1 again (total: 3)
|
||||
obsStore.IncrementRetrievalCount(ctx, []int64{obsID2}) // increment by 1 (total: 1)
|
||||
|
||||
// Get most retrieved - should return obsID1 (Popular) with count 3
|
||||
topObs, err := obsStore.GetMostRetrievedObservations(ctx, "test-project", 2)
|
||||
require.NoError(t, err)
|
||||
|
||||
require.Len(t, topObs, 2)
|
||||
assert.True(t, topObs[0].Title.Valid)
|
||||
assert.Equal(t, "Popular", topObs[0].Title.String)
|
||||
assert.Equal(t, 3, topObs[0].RetrievalCount)
|
||||
assert.True(t, topObs[1].Title.Valid)
|
||||
assert.Equal(t, "Unpopular", topObs[1].Title.String)
|
||||
assert.Equal(t, 1, topObs[1].RetrievalCount)
|
||||
}
|
||||
|
||||
func TestObservationStore_SetConceptWeights(t *testing.T) {
|
||||
obsStore, _, cleanup := testObservationStore(t)
|
||||
defer cleanup()
|
||||
ctx := context.Background()
|
||||
|
||||
// Set weights
|
||||
weights := map[string]float64{
|
||||
"security": 2.0,
|
||||
"performance": 1.5,
|
||||
"best-practice": 1.8,
|
||||
}
|
||||
|
||||
err := obsStore.SetConceptWeights(ctx, weights)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Get weights back
|
||||
retrieved, err := obsStore.GetConceptWeights(ctx)
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.Equal(t, 2.0, retrieved["security"])
|
||||
assert.Equal(t, 1.5, retrieved["performance"])
|
||||
assert.Equal(t, 1.8, retrieved["best-practice"])
|
||||
|
||||
// Update weights (UPSERT)
|
||||
weights["security"] = 2.5
|
||||
weights["scalability"] = 1.2
|
||||
|
||||
err = obsStore.SetConceptWeights(ctx, weights)
|
||||
require.NoError(t, err)
|
||||
|
||||
retrieved, err = obsStore.GetConceptWeights(ctx)
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.Equal(t, 2.5, retrieved["security"]) // updated
|
||||
assert.Equal(t, 1.5, retrieved["performance"]) // unchanged
|
||||
assert.Equal(t, 1.2, retrieved["scalability"]) // new
|
||||
assert.Equal(t, 1.8, retrieved["best-practice"]) // unchanged
|
||||
}
|
||||
|
||||
func TestObservationStore_GetObservationsNeedingScoreUpdate(t *testing.T) {
|
||||
obsStore, store, cleanup := testObservationStore(t)
|
||||
defer cleanup()
|
||||
ctx := context.Background()
|
||||
|
||||
sessionStore := NewSessionStore(store)
|
||||
sessionID, _ := sessionStore.CreateSDKSession(ctx, "claude-1", "test-project", "")
|
||||
|
||||
// Create observation with no score update
|
||||
obs1 := &models.ParsedObservation{Type: models.ObsTypeDiscovery, Title: "Needs Update"}
|
||||
obsID1, _, _ := obsStore.StoreObservation(ctx, "claude-1", "test-project", obs1, int(sessionID), 1)
|
||||
|
||||
// Create observation with recent score update
|
||||
obs2 := &models.ParsedObservation{Type: models.ObsTypeBugfix, Title: "Recently Updated"}
|
||||
obsID2, _, _ := obsStore.StoreObservation(ctx, "claude-1", "test-project", obs2, int(sessionID), 2)
|
||||
obsStore.UpdateImportanceScore(ctx, obsID2, 2.0)
|
||||
|
||||
// Get observations needing update (within 1 hour threshold)
|
||||
needsUpdate, err := obsStore.GetObservationsNeedingScoreUpdate(ctx, 1*time.Hour, 10)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Only obs1 should need update (obs2 was just updated)
|
||||
assert.Len(t, needsUpdate, 1)
|
||||
assert.Equal(t, obsID1, needsUpdate[0].ID)
|
||||
assert.True(t, needsUpdate[0].Title.Valid)
|
||||
assert.Equal(t, "Needs Update", needsUpdate[0].Title.String)
|
||||
}
|
||||
|
||||
func TestObservationStore_ResetObservationScores(t *testing.T) {
|
||||
obsStore, store, cleanup := testObservationStore(t)
|
||||
defer cleanup()
|
||||
ctx := context.Background()
|
||||
|
||||
sessionStore := NewSessionStore(store)
|
||||
sessionID, _ := sessionStore.CreateSDKSession(ctx, "claude-1", "test-project", "")
|
||||
|
||||
// Create observations with custom scores
|
||||
obs1 := &models.ParsedObservation{Type: models.ObsTypeDiscovery, Title: "Test1"}
|
||||
obsID1, _, _ := obsStore.StoreObservation(ctx, "claude-1", "test-project", obs1, int(sessionID), 1)
|
||||
obsStore.UpdateImportanceScore(ctx, obsID1, 5.0)
|
||||
|
||||
obs2 := &models.ParsedObservation{Type: models.ObsTypeBugfix, Title: "Test2"}
|
||||
obsID2, _, _ := obsStore.StoreObservation(ctx, "claude-1", "test-project", obs2, int(sessionID), 2)
|
||||
obsStore.UpdateImportanceScore(ctx, obsID2, 3.0)
|
||||
|
||||
// Reset all scores
|
||||
err := obsStore.ResetObservationScores(ctx)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Verify all scores are 1.0
|
||||
var dbObs1, dbObs2 Observation
|
||||
store.DB.First(&dbObs1, obsID1)
|
||||
store.DB.First(&dbObs2, obsID2)
|
||||
|
||||
assert.Equal(t, 1.0, dbObs1.ImportanceScore)
|
||||
assert.Equal(t, 1.0, dbObs2.ImportanceScore)
|
||||
}
|
||||
@@ -0,0 +1,198 @@
|
||||
// Package gorm provides GORM-based database operations for claude-mnemonic.
|
||||
package gorm
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"time"
|
||||
|
||||
"gorm.io/gorm"
|
||||
"gorm.io/gorm/clause"
|
||||
|
||||
"github.com/lukaszraczylo/claude-mnemonic/pkg/models"
|
||||
)
|
||||
|
||||
// SessionStore provides session-related database operations using GORM.
|
||||
type SessionStore struct {
|
||||
db *gorm.DB
|
||||
}
|
||||
|
||||
// NewSessionStore creates a new session store.
|
||||
func NewSessionStore(store *Store) *SessionStore {
|
||||
return &SessionStore{db: store.DB}
|
||||
}
|
||||
|
||||
// CreateSDKSession creates a new SDK session (idempotent - returns existing ID if exists).
|
||||
// This is the KEY to how claude-mnemonic stays unified across hooks.
|
||||
func (s *SessionStore) CreateSDKSession(ctx context.Context, claudeSessionID, project, userPrompt string) (int64, error) {
|
||||
now := time.Now()
|
||||
|
||||
session := &SDKSession{
|
||||
ClaudeSessionID: claudeSessionID,
|
||||
SDKSessionID: func() sql.NullString {
|
||||
return sql.NullString{String: claudeSessionID, Valid: true}
|
||||
}(),
|
||||
Project: project,
|
||||
UserPrompt: func() sql.NullString {
|
||||
if userPrompt != "" {
|
||||
return sql.NullString{String: userPrompt, Valid: true}
|
||||
}
|
||||
return sql.NullString{Valid: false}
|
||||
}(),
|
||||
Status: "active",
|
||||
StartedAt: now.Format(time.RFC3339),
|
||||
StartedAtEpoch: now.UnixMilli(),
|
||||
}
|
||||
|
||||
// CRITICAL: INSERT OR IGNORE makes this idempotent
|
||||
// Use OnConflict with DoNothing to achieve INSERT OR IGNORE behavior
|
||||
result := s.db.WithContext(ctx).
|
||||
Clauses(clause.OnConflict{
|
||||
Columns: []clause.Column{{Name: "claude_session_id"}},
|
||||
DoNothing: true,
|
||||
}).
|
||||
Create(session)
|
||||
|
||||
if result.Error != nil {
|
||||
return 0, result.Error
|
||||
}
|
||||
|
||||
// Check if insert happened
|
||||
if result.RowsAffected == 0 {
|
||||
// Session exists - UPDATE project and user_prompt if we have non-empty values
|
||||
if project != "" {
|
||||
updates := map[string]interface{}{
|
||||
"project": project,
|
||||
}
|
||||
if userPrompt != "" {
|
||||
updates["user_prompt"] = userPrompt
|
||||
}
|
||||
s.db.WithContext(ctx).
|
||||
Model(&SDKSession{}).
|
||||
Where("claude_session_id = ?", claudeSessionID).
|
||||
Updates(updates)
|
||||
}
|
||||
|
||||
// Fetch existing session
|
||||
var existing SDKSession
|
||||
err := s.db.WithContext(ctx).
|
||||
Where("claude_session_id = ?", claudeSessionID).
|
||||
First(&existing).Error
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return existing.ID, nil
|
||||
}
|
||||
|
||||
return session.ID, nil
|
||||
}
|
||||
|
||||
// GetSessionByID retrieves a session by its database ID.
|
||||
func (s *SessionStore) GetSessionByID(ctx context.Context, id int64) (*models.SDKSession, error) {
|
||||
var sess SDKSession
|
||||
err := s.db.WithContext(ctx).First(&sess, id).Error
|
||||
if err == gorm.ErrRecordNotFound {
|
||||
return nil, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return toModelSDKSession(&sess), nil
|
||||
}
|
||||
|
||||
// FindAnySDKSession finds any session by Claude session ID (any status).
|
||||
func (s *SessionStore) FindAnySDKSession(ctx context.Context, claudeSessionID string) (*models.SDKSession, error) {
|
||||
var sess SDKSession
|
||||
err := s.db.WithContext(ctx).
|
||||
Where("claude_session_id = ?", claudeSessionID).
|
||||
First(&sess).Error
|
||||
if err == gorm.ErrRecordNotFound {
|
||||
return nil, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return toModelSDKSession(&sess), nil
|
||||
}
|
||||
|
||||
// IncrementPromptCounter increments the prompt counter and returns the new value.
|
||||
func (s *SessionStore) IncrementPromptCounter(ctx context.Context, id int64) (int, error) {
|
||||
// Atomic increment using GORM expression
|
||||
err := s.db.WithContext(ctx).
|
||||
Model(&SDKSession{}).
|
||||
Where("id = ?", id).
|
||||
Update("prompt_counter", gorm.Expr("COALESCE(prompt_counter, 0) + 1")).Error
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
// Fetch updated value
|
||||
var sess SDKSession
|
||||
err = s.db.WithContext(ctx).
|
||||
Select("prompt_counter").
|
||||
First(&sess, id).Error
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
return sess.PromptCounter, nil
|
||||
}
|
||||
|
||||
// GetPromptCounter returns the current prompt counter for a session.
|
||||
func (s *SessionStore) GetPromptCounter(ctx context.Context, id int64) (int, error) {
|
||||
var sess SDKSession
|
||||
err := s.db.WithContext(ctx).
|
||||
Select("prompt_counter").
|
||||
First(&sess, id).Error
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return sess.PromptCounter, nil
|
||||
}
|
||||
|
||||
// GetSessionsToday returns the count of sessions started today.
|
||||
func (s *SessionStore) GetSessionsToday(ctx context.Context) (int, error) {
|
||||
// Get start of today in milliseconds
|
||||
now := time.Now()
|
||||
startOfDay := time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, now.Location())
|
||||
startEpoch := startOfDay.UnixMilli()
|
||||
|
||||
var count int64
|
||||
err := s.db.WithContext(ctx).
|
||||
Model(&SDKSession{}).
|
||||
Where("started_at_epoch >= ?", startEpoch).
|
||||
Count(&count).Error
|
||||
|
||||
return int(count), err
|
||||
}
|
||||
|
||||
// GetAllProjects returns all unique project names.
|
||||
func (s *SessionStore) GetAllProjects(ctx context.Context) ([]string, error) {
|
||||
var projects []string
|
||||
err := s.db.WithContext(ctx).
|
||||
Model(&SDKSession{}).
|
||||
Distinct("project").
|
||||
Where("project IS NOT NULL AND project != ''").
|
||||
Order("project ASC").
|
||||
Pluck("project", &projects).Error
|
||||
|
||||
return projects, err
|
||||
}
|
||||
|
||||
// toModelSDKSession converts a GORM SDKSession to pkg/models.SDKSession.
|
||||
func toModelSDKSession(sess *SDKSession) *models.SDKSession {
|
||||
return &models.SDKSession{
|
||||
ID: sess.ID,
|
||||
ClaudeSessionID: sess.ClaudeSessionID,
|
||||
SDKSessionID: sess.SDKSessionID,
|
||||
Project: sess.Project,
|
||||
UserPrompt: sess.UserPrompt,
|
||||
WorkerPort: sess.WorkerPort,
|
||||
PromptCounter: int64(sess.PromptCounter),
|
||||
Status: models.SessionStatus(sess.Status),
|
||||
StartedAt: sess.StartedAt,
|
||||
StartedAtEpoch: sess.StartedAtEpoch,
|
||||
CompletedAt: sess.CompletedAt,
|
||||
CompletedAtEpoch: sess.CompletedAtEpoch,
|
||||
}
|
||||
}
|
||||
@@ -1,24 +1,50 @@
|
||||
package sqlite
|
||||
//go:build fts5
|
||||
|
||||
// Package gorm provides GORM-based database operations for claude-mnemonic.
|
||||
package gorm
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/lukaszraczylo/claude-mnemonic/pkg/models"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"gorm.io/gorm/logger"
|
||||
|
||||
"github.com/lukaszraczylo/claude-mnemonic/pkg/models"
|
||||
)
|
||||
|
||||
// testSessionStore creates a SessionStore with a temporary database for testing.
|
||||
func testSessionStore(t *testing.T) (*SessionStore, *Store, func()) {
|
||||
t.Helper()
|
||||
|
||||
db, _, cleanup := testDB(t)
|
||||
createAllTables(t, db)
|
||||
tmpDir, err := os.MkdirTemp("", "gorm_session_test_*")
|
||||
if err != nil {
|
||||
t.Fatalf("create temp dir: %v", err)
|
||||
}
|
||||
|
||||
dbPath := filepath.Join(tmpDir, "test.db")
|
||||
cfg := Config{
|
||||
Path: dbPath,
|
||||
MaxConns: 4,
|
||||
LogLevel: logger.Silent,
|
||||
}
|
||||
|
||||
store, err := NewStore(cfg)
|
||||
if err != nil {
|
||||
os.RemoveAll(tmpDir)
|
||||
t.Fatalf("NewStore failed: %v", err)
|
||||
}
|
||||
|
||||
store := newStoreFromDB(db)
|
||||
sessionStore := NewSessionStore(store)
|
||||
|
||||
cleanup := func() {
|
||||
store.Close()
|
||||
os.RemoveAll(tmpDir)
|
||||
}
|
||||
|
||||
return sessionStore, store, cleanup
|
||||
}
|
||||
|
||||
@@ -40,6 +66,8 @@ func TestSessionStore_CreateSDKSession(t *testing.T) {
|
||||
assert.Equal(t, "claude-1", sess.ClaudeSessionID)
|
||||
assert.Equal(t, "test-project", sess.Project)
|
||||
assert.Equal(t, models.SessionStatusActive, sess.Status)
|
||||
assert.True(t, sess.UserPrompt.Valid)
|
||||
assert.Equal(t, "initial prompt", sess.UserPrompt.String)
|
||||
}
|
||||
|
||||
func TestSessionStore_CreateSDKSession_Idempotent(t *testing.T) {
|
||||
@@ -63,6 +91,24 @@ func TestSessionStore_CreateSDKSession_Idempotent(t *testing.T) {
|
||||
sess, err := sessionStore.GetSessionByID(ctx, id1)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "project-b", sess.Project)
|
||||
assert.Equal(t, "prompt 2", sess.UserPrompt.String)
|
||||
}
|
||||
|
||||
func TestSessionStore_CreateSDKSession_EmptyPrompt(t *testing.T) {
|
||||
sessionStore, _, cleanup := testSessionStore(t)
|
||||
defer cleanup()
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
// Create session with empty prompt
|
||||
id, err := sessionStore.CreateSDKSession(ctx, "claude-2", "test-project", "")
|
||||
require.NoError(t, err)
|
||||
assert.Greater(t, id, int64(0))
|
||||
|
||||
// Verify prompt is NULL
|
||||
sess, err := sessionStore.GetSessionByID(ctx, id)
|
||||
require.NoError(t, err)
|
||||
assert.False(t, sess.UserPrompt.Valid)
|
||||
}
|
||||
|
||||
func TestSessionStore_FindAnySDKSession(t *testing.T) {
|
||||
@@ -87,6 +133,18 @@ func TestSessionStore_FindAnySDKSession(t *testing.T) {
|
||||
assert.Nil(t, sess)
|
||||
}
|
||||
|
||||
func TestSessionStore_GetSessionByID_NotFound(t *testing.T) {
|
||||
sessionStore, _, cleanup := testSessionStore(t)
|
||||
defer cleanup()
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
// Try to get non-existent session
|
||||
sess, err := sessionStore.GetSessionByID(ctx, 99999)
|
||||
require.NoError(t, err)
|
||||
assert.Nil(t, sess)
|
||||
}
|
||||
|
||||
func TestSessionStore_IncrementPromptCounter(t *testing.T) {
|
||||
sessionStore, _, cleanup := testSessionStore(t)
|
||||
defer cleanup()
|
||||
@@ -124,23 +182,22 @@ func TestSessionStore_GetSessionsToday(t *testing.T) {
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
// Initially no sessions today
|
||||
// Initially should be 0
|
||||
count, err := sessionStore.GetSessionsToday(ctx)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 0, count)
|
||||
|
||||
// Create some sessions
|
||||
_, err = sessionStore.CreateSDKSession(ctx, "claude-1", "project-a", "")
|
||||
require.NoError(t, err)
|
||||
_, err = sessionStore.CreateSDKSession(ctx, "claude-2", "project-b", "")
|
||||
require.NoError(t, err)
|
||||
_, err = sessionStore.CreateSDKSession(ctx, "claude-3", "project-c", "")
|
||||
_, err = sessionStore.CreateSDKSession(ctx, "claude-1", "project-1", "")
|
||||
require.NoError(t, err)
|
||||
|
||||
// Should have 3 sessions today
|
||||
_, err = sessionStore.CreateSDKSession(ctx, "claude-2", "project-2", "")
|
||||
require.NoError(t, err)
|
||||
|
||||
// Should now have 2 sessions today
|
||||
count, err = sessionStore.GetSessionsToday(ctx)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 3, count)
|
||||
assert.Equal(t, 2, count)
|
||||
}
|
||||
|
||||
func TestSessionStore_GetAllProjects(t *testing.T) {
|
||||
@@ -149,57 +206,35 @@ func TestSessionStore_GetAllProjects(t *testing.T) {
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
// Create sessions for different projects
|
||||
_, err := sessionStore.CreateSDKSession(ctx, "claude-1", "alpha-project", "")
|
||||
require.NoError(t, err)
|
||||
_, err = sessionStore.CreateSDKSession(ctx, "claude-2", "beta-project", "")
|
||||
require.NoError(t, err)
|
||||
_, err = sessionStore.CreateSDKSession(ctx, "claude-3", "alpha-project", "") // duplicate
|
||||
require.NoError(t, err)
|
||||
_, err = sessionStore.CreateSDKSession(ctx, "claude-4", "gamma-project", "")
|
||||
require.NoError(t, err)
|
||||
|
||||
// Get all projects
|
||||
// Initially should be empty
|
||||
projects, err := sessionStore.GetAllProjects(ctx)
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, projects, 3)
|
||||
assert.Contains(t, projects, "alpha-project")
|
||||
assert.Contains(t, projects, "beta-project")
|
||||
assert.Contains(t, projects, "gamma-project")
|
||||
assert.Empty(t, projects)
|
||||
|
||||
// Should be sorted alphabetically
|
||||
assert.Equal(t, "alpha-project", projects[0])
|
||||
// Create sessions with different projects
|
||||
_, err = sessionStore.CreateSDKSession(ctx, "claude-1", "project-a", "")
|
||||
require.NoError(t, err)
|
||||
|
||||
_, err = sessionStore.CreateSDKSession(ctx, "claude-2", "project-b", "")
|
||||
require.NoError(t, err)
|
||||
|
||||
_, err = sessionStore.CreateSDKSession(ctx, "claude-3", "project-a", "") // Duplicate project
|
||||
require.NoError(t, err)
|
||||
|
||||
// Should get distinct projects in alphabetical order
|
||||
projects, err = sessionStore.GetAllProjects(ctx)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, []string{"project-a", "project-b"}, projects)
|
||||
}
|
||||
|
||||
func TestSessionStore_GetSessionByID_NotFound(t *testing.T) {
|
||||
func TestSessionStore_SessionFields(t *testing.T) {
|
||||
sessionStore, _, cleanup := testSessionStore(t)
|
||||
defer cleanup()
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
// Non-existent ID should return nil, nil (not an error)
|
||||
sess, err := sessionStore.GetSessionByID(ctx, 999)
|
||||
require.NoError(t, err)
|
||||
assert.Nil(t, sess)
|
||||
}
|
||||
|
||||
func TestSessionStore_SessionFields(t *testing.T) {
|
||||
sessionStore, store, cleanup := testSessionStore(t)
|
||||
defer cleanup()
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
// Create a session with full details
|
||||
id, err := sessionStore.CreateSDKSession(ctx, "claude-full", "full-project", "full user prompt")
|
||||
require.NoError(t, err)
|
||||
|
||||
// Manually update additional fields for testing
|
||||
now := time.Now()
|
||||
_, err = storeDB(store).Exec(`
|
||||
UPDATE sdk_sessions
|
||||
SET worker_port = ?, completed_at = ?, completed_at_epoch = ?, status = 'completed'
|
||||
WHERE id = ?
|
||||
`, 37777, now.Format(time.RFC3339), now.UnixMilli(), id)
|
||||
// Create a session
|
||||
id, err := sessionStore.CreateSDKSession(ctx, "claude-1", "test-project", "test prompt")
|
||||
require.NoError(t, err)
|
||||
|
||||
// Retrieve and verify all fields
|
||||
@@ -207,12 +242,18 @@ func TestSessionStore_SessionFields(t *testing.T) {
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, sess)
|
||||
|
||||
// Verify all fields
|
||||
assert.Equal(t, id, sess.ID)
|
||||
assert.Equal(t, "claude-full", sess.ClaudeSessionID)
|
||||
assert.Equal(t, "full-project", sess.Project)
|
||||
assert.Equal(t, models.SessionStatusCompleted, sess.Status)
|
||||
assert.True(t, sess.WorkerPort.Valid)
|
||||
assert.Equal(t, int64(37777), sess.WorkerPort.Int64)
|
||||
assert.True(t, sess.CompletedAt.Valid)
|
||||
assert.True(t, sess.CompletedAtEpoch.Valid)
|
||||
assert.Equal(t, "claude-1", sess.ClaudeSessionID)
|
||||
assert.True(t, sess.SDKSessionID.Valid)
|
||||
assert.Equal(t, "claude-1", sess.SDKSessionID.String) // Should be same as ClaudeSessionID
|
||||
assert.Equal(t, "test-project", sess.Project)
|
||||
assert.True(t, sess.UserPrompt.Valid)
|
||||
assert.Equal(t, "test prompt", sess.UserPrompt.String)
|
||||
assert.Equal(t, int64(0), sess.PromptCounter)
|
||||
assert.Equal(t, models.SessionStatusActive, sess.Status)
|
||||
assert.NotEmpty(t, sess.StartedAt)
|
||||
assert.Greater(t, sess.StartedAtEpoch, int64(0))
|
||||
assert.False(t, sess.CompletedAt.Valid) // Should be NULL
|
||||
assert.False(t, sess.CompletedAtEpoch.Valid) // Should be NULL
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
//go:build !sqlite_omit_load_extension
|
||||
// +build !sqlite_omit_load_extension
|
||||
|
||||
// Package gorm provides GORM-based database operations for claude-mnemonic.
|
||||
package gorm
|
||||
|
||||
// This file ensures mattn/go-sqlite3 is built with FTS5 and other extensions enabled.
|
||||
// The build tag ensures extensions are not omitted.
|
||||
@@ -0,0 +1,117 @@
|
||||
// Package gorm provides GORM-based database operations for claude-mnemonic.
|
||||
package gorm
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"fmt"
|
||||
|
||||
sqlite_vec "github.com/asg017/sqlite-vec-go-bindings/cgo"
|
||||
_ "github.com/mattn/go-sqlite3" // Import SQLite driver with FTS5 support
|
||||
"gorm.io/driver/sqlite"
|
||||
"gorm.io/gorm"
|
||||
"gorm.io/gorm/logger"
|
||||
)
|
||||
|
||||
// Store represents the GORM database connection with sqlite-vec support.
|
||||
type Store struct {
|
||||
DB *gorm.DB
|
||||
sqlDB *sql.DB // For FTS5 and sqlite-vec operations that require raw SQL
|
||||
}
|
||||
|
||||
// Config holds database configuration.
|
||||
type Config struct {
|
||||
Path string // Path to SQLite database file
|
||||
MaxConns int // Maximum number of open connections (default: 4)
|
||||
LogLevel logger.LogLevel // GORM log level (logger.Silent for production)
|
||||
}
|
||||
|
||||
// NewStore creates a new Store with WAL mode enabled and sqlite-vec registered.
|
||||
// CRITICAL: WAL mode and foreign keys are enabled via pragmas for concurrent reads.
|
||||
func NewStore(cfg Config) (*Store, error) {
|
||||
// 1. Register sqlite-vec extension (must be done before opening database)
|
||||
sqlite_vec.Auto()
|
||||
|
||||
// 2. Build connection string (foreign keys enabled in DSN)
|
||||
// Use sqlite3 driver (mattn/go-sqlite3) which has FTS5 support
|
||||
dsn := cfg.Path + "?_foreign_keys=ON"
|
||||
|
||||
// 3. Open raw database connection with mattn/go-sqlite3 (has FTS5 support)
|
||||
sqlDB, err := sql.Open("sqlite3", dsn)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("open database: %w", err)
|
||||
}
|
||||
|
||||
// 4. Wrap with GORM using existing connection
|
||||
db, err := gorm.Open(sqlite.Dialector{
|
||||
Conn: sqlDB,
|
||||
}, &gorm.Config{
|
||||
Logger: logger.Default.LogMode(cfg.LogLevel),
|
||||
// PrepareStmt enables prepared statement caching for performance
|
||||
PrepareStmt: true,
|
||||
// Disable default timestamp fields (we manage created_at manually)
|
||||
NowFunc: nil,
|
||||
})
|
||||
if err != nil {
|
||||
_ = sqlDB.Close() // Explicitly ignore close error during cleanup
|
||||
return nil, fmt.Errorf("open gorm: %w", err)
|
||||
}
|
||||
|
||||
// 5. Configure connection pool (same settings as current implementation)
|
||||
maxConns := cfg.MaxConns
|
||||
if maxConns <= 0 {
|
||||
maxConns = 4
|
||||
}
|
||||
sqlDB.SetMaxOpenConns(maxConns)
|
||||
sqlDB.SetMaxIdleConns(maxConns)
|
||||
sqlDB.SetConnMaxLifetime(0) // Never expire (SQLite connections are cheap)
|
||||
|
||||
// 6. Verify connection
|
||||
if err := sqlDB.Ping(); err != nil {
|
||||
return nil, fmt.Errorf("ping database: %w", err)
|
||||
}
|
||||
|
||||
store := &Store{
|
||||
DB: db,
|
||||
sqlDB: sqlDB,
|
||||
}
|
||||
|
||||
// 7. Run migrations FIRST (before PRAGMA commands)
|
||||
if err := runMigrations(db, sqlDB); err != nil {
|
||||
return nil, fmt.Errorf("run migrations: %w", err)
|
||||
}
|
||||
|
||||
// 8. CRITICAL: Set WAL mode and synchronous mode via raw SQL
|
||||
// Use raw sqlDB to avoid GORM transaction issues
|
||||
if _, err := sqlDB.Exec("PRAGMA journal_mode=WAL"); err != nil {
|
||||
return nil, fmt.Errorf("set WAL mode: %w", err)
|
||||
}
|
||||
if _, err := sqlDB.Exec("PRAGMA synchronous=NORMAL"); err != nil {
|
||||
return nil, fmt.Errorf("set synchronous mode: %w", err)
|
||||
}
|
||||
|
||||
return store, nil
|
||||
}
|
||||
|
||||
// Close closes the database connection.
|
||||
func (s *Store) Close() error {
|
||||
return s.sqlDB.Close()
|
||||
}
|
||||
|
||||
// Ping verifies the database connection is alive.
|
||||
func (s *Store) Ping() error {
|
||||
return s.sqlDB.Ping()
|
||||
}
|
||||
|
||||
// GetRawDB returns the underlying *sql.DB for operations GORM can't handle.
|
||||
// Use this for:
|
||||
// - FTS5 full-text search queries (MATCH operator)
|
||||
// - sqlite-vec vector operations
|
||||
// - Complex raw SQL queries
|
||||
func (s *Store) GetRawDB() *sql.DB {
|
||||
return s.sqlDB
|
||||
}
|
||||
|
||||
// GetDB returns the GORM DB instance for standard queries.
|
||||
func (s *Store) GetDB() *gorm.DB {
|
||||
return s.DB
|
||||
}
|
||||
@@ -0,0 +1,152 @@
|
||||
//go:build fts5
|
||||
|
||||
// Package gorm provides GORM-based database operations for claude-mnemonic.
|
||||
package gorm
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"gorm.io/gorm/logger"
|
||||
)
|
||||
|
||||
func TestNewStore(t *testing.T) {
|
||||
// Create temporary directory for test database
|
||||
tmpDir, err := os.MkdirTemp("", "gorm_test_*")
|
||||
if err != nil {
|
||||
t.Fatalf("create temp dir: %v", err)
|
||||
}
|
||||
defer os.RemoveAll(tmpDir)
|
||||
|
||||
dbPath := filepath.Join(tmpDir, "test.db")
|
||||
|
||||
// Create store with migrations
|
||||
cfg := Config{
|
||||
Path: dbPath,
|
||||
MaxConns: 4,
|
||||
LogLevel: logger.Silent,
|
||||
}
|
||||
|
||||
store, err := NewStore(cfg)
|
||||
if err != nil {
|
||||
t.Fatalf("NewStore failed: %v", err)
|
||||
}
|
||||
defer store.Close()
|
||||
|
||||
// Verify connection works
|
||||
sqlDB := store.GetRawDB()
|
||||
if err := sqlDB.Ping(); err != nil {
|
||||
t.Fatalf("ping failed: %v", err)
|
||||
}
|
||||
|
||||
// Verify WAL mode is enabled
|
||||
var journalMode string
|
||||
err = store.DB.Raw("PRAGMA journal_mode").Scan(&journalMode).Error
|
||||
if err != nil {
|
||||
t.Fatalf("query journal_mode failed: %v", err)
|
||||
}
|
||||
if journalMode != "wal" {
|
||||
t.Errorf("expected WAL mode, got %q", journalMode)
|
||||
}
|
||||
|
||||
// Verify core tables exist
|
||||
tables := []string{
|
||||
"sdk_sessions",
|
||||
"observations",
|
||||
"session_summaries",
|
||||
"user_prompts",
|
||||
"observation_conflicts",
|
||||
"observation_relations",
|
||||
"patterns",
|
||||
"concept_weights",
|
||||
}
|
||||
|
||||
for _, table := range tables {
|
||||
exists := store.DB.Migrator().HasTable(table)
|
||||
if !exists {
|
||||
t.Errorf("table %q does not exist", table)
|
||||
}
|
||||
}
|
||||
|
||||
// Verify FTS5 virtual tables exist (cannot use Migrator().HasTable for virtual tables)
|
||||
ftsTables := []string{
|
||||
"user_prompts_fts",
|
||||
"observations_fts",
|
||||
"session_summaries_fts",
|
||||
"patterns_fts",
|
||||
}
|
||||
|
||||
for _, table := range ftsTables {
|
||||
var count int
|
||||
err := store.DB.Raw("SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name=?", table).Scan(&count).Error
|
||||
if err != nil {
|
||||
t.Errorf("check FTS table %q failed: %v", table, err)
|
||||
}
|
||||
if count != 1 {
|
||||
t.Errorf("FTS table %q does not exist", table)
|
||||
}
|
||||
}
|
||||
|
||||
// Verify vectors table exists (virtual table)
|
||||
var vectorsCount int
|
||||
err = store.DB.Raw("SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name='vectors'").Scan(&vectorsCount).Error
|
||||
if err != nil {
|
||||
t.Errorf("check vectors table failed: %v", err)
|
||||
}
|
||||
if vectorsCount != 1 {
|
||||
t.Errorf("vectors table does not exist")
|
||||
}
|
||||
|
||||
// Verify concept_weights seed data exists
|
||||
var conceptCount int64
|
||||
store.DB.Model(&ConceptWeight{}).Count(&conceptCount)
|
||||
if conceptCount != 12 {
|
||||
t.Errorf("expected 12 concept weights, got %d", conceptCount)
|
||||
}
|
||||
|
||||
t.Logf("✅ Phase 1 Foundation: All migrations successful")
|
||||
t.Logf(" - Core tables: %d", len(tables))
|
||||
t.Logf(" - FTS5 tables: %d", len(ftsTables))
|
||||
t.Logf(" - Vector table: 1")
|
||||
t.Logf(" - Seed data: %d concept weights", conceptCount)
|
||||
}
|
||||
|
||||
func TestMigrationIdempotency(t *testing.T) {
|
||||
// Create temporary directory for test database
|
||||
tmpDir, err := os.MkdirTemp("", "gorm_idempotency_*")
|
||||
if err != nil {
|
||||
t.Fatalf("create temp dir: %v", err)
|
||||
}
|
||||
defer os.RemoveAll(tmpDir)
|
||||
|
||||
dbPath := filepath.Join(tmpDir, "test.db")
|
||||
cfg := Config{
|
||||
Path: dbPath,
|
||||
MaxConns: 4,
|
||||
LogLevel: logger.Silent,
|
||||
}
|
||||
|
||||
// Run migrations first time
|
||||
store1, err := NewStore(cfg)
|
||||
if err != nil {
|
||||
t.Fatalf("NewStore (first) failed: %v", err)
|
||||
}
|
||||
store1.Close()
|
||||
|
||||
// Run migrations second time (should be idempotent)
|
||||
store2, err := NewStore(cfg)
|
||||
if err != nil {
|
||||
t.Fatalf("NewStore (second) failed: %v", err)
|
||||
}
|
||||
defer store2.Close()
|
||||
|
||||
// Verify concept_weights seed data is still exactly 12 (INSERT OR IGNORE)
|
||||
var conceptCount int64
|
||||
store2.DB.Model(&ConceptWeight{}).Count(&conceptCount)
|
||||
if conceptCount != 12 {
|
||||
t.Errorf("expected 12 concept weights after second migration, got %d", conceptCount)
|
||||
}
|
||||
|
||||
t.Logf("✅ Migrations are idempotent")
|
||||
}
|
||||
@@ -0,0 +1,171 @@
|
||||
// Package gorm provides GORM-based database operations for claude-mnemonic.
|
||||
package gorm
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"time"
|
||||
|
||||
"gorm.io/gorm"
|
||||
|
||||
"github.com/lukaszraczylo/claude-mnemonic/pkg/models"
|
||||
)
|
||||
|
||||
// SummaryStore provides summary-related database operations using GORM.
|
||||
type SummaryStore struct {
|
||||
db *gorm.DB
|
||||
}
|
||||
|
||||
// NewSummaryStore creates a new summary store.
|
||||
func NewSummaryStore(store *Store) *SummaryStore {
|
||||
return &SummaryStore{db: store.DB}
|
||||
}
|
||||
|
||||
// StoreSummary stores a new session summary.
|
||||
func (s *SummaryStore) StoreSummary(ctx context.Context, sdkSessionID, project string, summary *models.ParsedSummary, promptNumber int, discoveryTokens int64) (int64, int64, error) {
|
||||
now := time.Now()
|
||||
nowEpoch := now.UnixMilli()
|
||||
|
||||
// Ensure session exists (auto-create if missing)
|
||||
if err := EnsureSessionExists(ctx, s.db, sdkSessionID, project); err != nil {
|
||||
return 0, 0, err
|
||||
}
|
||||
|
||||
dbSummary := &SessionSummary{
|
||||
SDKSessionID: sdkSessionID,
|
||||
Project: project,
|
||||
Request: nullString(summary.Request),
|
||||
Investigated: nullString(summary.Investigated),
|
||||
Learned: nullString(summary.Learned),
|
||||
Completed: nullString(summary.Completed),
|
||||
NextSteps: nullString(summary.NextSteps),
|
||||
Notes: nullString(summary.Notes),
|
||||
PromptNumber: func() sql.NullInt64 {
|
||||
if promptNumber > 0 {
|
||||
return sql.NullInt64{Int64: int64(promptNumber), Valid: true}
|
||||
}
|
||||
return sql.NullInt64{Valid: false}
|
||||
}(),
|
||||
DiscoveryTokens: discoveryTokens,
|
||||
CreatedAt: now.Format(time.RFC3339),
|
||||
CreatedAtEpoch: nowEpoch,
|
||||
}
|
||||
|
||||
err := s.db.WithContext(ctx).Create(dbSummary).Error
|
||||
if err != nil {
|
||||
return 0, 0, err
|
||||
}
|
||||
|
||||
return dbSummary.ID, nowEpoch, nil
|
||||
}
|
||||
|
||||
// GetSummariesByIDs retrieves summaries by a list of IDs.
|
||||
func (s *SummaryStore) GetSummariesByIDs(ctx context.Context, ids []int64, orderBy string, limit int) ([]*models.SessionSummary, error) {
|
||||
if len(ids) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
var dbSummaries []SessionSummary
|
||||
query := s.db.WithContext(ctx).Where("id IN ?", ids)
|
||||
|
||||
// Apply ordering
|
||||
switch orderBy {
|
||||
case "date_asc":
|
||||
query = query.Order("created_at_epoch ASC")
|
||||
case "date_desc", "default", "":
|
||||
query = query.Order("created_at_epoch DESC")
|
||||
}
|
||||
|
||||
// Apply limit
|
||||
if limit > 0 {
|
||||
query = query.Limit(limit)
|
||||
}
|
||||
|
||||
err := query.Find(&dbSummaries).Error
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return toModelSessionSummaries(dbSummaries), nil
|
||||
}
|
||||
|
||||
// GetRecentSummaries retrieves recent summaries for a project.
|
||||
func (s *SummaryStore) GetRecentSummaries(ctx context.Context, project string, limit int) ([]*models.SessionSummary, error) {
|
||||
var dbSummaries []SessionSummary
|
||||
err := s.db.WithContext(ctx).
|
||||
Where("project = ?", project).
|
||||
Order("created_at_epoch DESC").
|
||||
Limit(limit).
|
||||
Find(&dbSummaries).Error
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return toModelSessionSummaries(dbSummaries), nil
|
||||
}
|
||||
|
||||
// GetAllRecentSummaries retrieves recent summaries across all projects.
|
||||
func (s *SummaryStore) GetAllRecentSummaries(ctx context.Context, limit int) ([]*models.SessionSummary, error) {
|
||||
var dbSummaries []SessionSummary
|
||||
err := s.db.WithContext(ctx).
|
||||
Order("created_at_epoch DESC").
|
||||
Limit(limit).
|
||||
Find(&dbSummaries).Error
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return toModelSessionSummaries(dbSummaries), nil
|
||||
}
|
||||
|
||||
// GetAllSummaries retrieves all summaries (for vector rebuild).
|
||||
func (s *SummaryStore) GetAllSummaries(ctx context.Context) ([]*models.SessionSummary, error) {
|
||||
var dbSummaries []SessionSummary
|
||||
err := s.db.WithContext(ctx).
|
||||
Order("id").
|
||||
Find(&dbSummaries).Error
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return toModelSessionSummaries(dbSummaries), nil
|
||||
}
|
||||
|
||||
// toModelSessionSummary converts a GORM SessionSummary to pkg/models.SessionSummary.
|
||||
func toModelSessionSummary(s *SessionSummary) *models.SessionSummary {
|
||||
return &models.SessionSummary{
|
||||
ID: s.ID,
|
||||
SDKSessionID: s.SDKSessionID,
|
||||
Project: s.Project,
|
||||
Request: s.Request,
|
||||
Investigated: s.Investigated,
|
||||
Learned: s.Learned,
|
||||
Completed: s.Completed,
|
||||
NextSteps: s.NextSteps,
|
||||
Notes: s.Notes,
|
||||
PromptNumber: s.PromptNumber,
|
||||
DiscoveryTokens: s.DiscoveryTokens,
|
||||
CreatedAt: s.CreatedAt,
|
||||
CreatedAtEpoch: s.CreatedAtEpoch,
|
||||
}
|
||||
}
|
||||
|
||||
// toModelSessionSummaries converts a slice of GORM SessionSummary to pkg/models.SessionSummary.
|
||||
func toModelSessionSummaries(summaries []SessionSummary) []*models.SessionSummary {
|
||||
result := make([]*models.SessionSummary, len(summaries))
|
||||
for i := range summaries {
|
||||
result[i] = toModelSessionSummary(&summaries[i])
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
// nullString converts a string to sql.NullString.
|
||||
func nullString(s string) sql.NullString {
|
||||
if s == "" {
|
||||
return sql.NullString{Valid: false}
|
||||
}
|
||||
return sql.NullString{String: s, Valid: true}
|
||||
}
|
||||
@@ -0,0 +1,278 @@
|
||||
//go:build fts5
|
||||
|
||||
// Package gorm provides GORM-based database operations for claude-mnemonic.
|
||||
package gorm
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"gorm.io/gorm/logger"
|
||||
|
||||
"github.com/lukaszraczylo/claude-mnemonic/pkg/models"
|
||||
)
|
||||
|
||||
// testSummaryStore creates a SummaryStore with a temporary database for testing.
|
||||
func testSummaryStore(t *testing.T) (*SummaryStore, *Store, func()) {
|
||||
t.Helper()
|
||||
|
||||
tmpDir, err := os.MkdirTemp("", "gorm_summary_test_*")
|
||||
if err != nil {
|
||||
t.Fatalf("create temp dir: %v", err)
|
||||
}
|
||||
|
||||
dbPath := filepath.Join(tmpDir, "test.db")
|
||||
cfg := Config{
|
||||
Path: dbPath,
|
||||
MaxConns: 4,
|
||||
LogLevel: logger.Silent,
|
||||
}
|
||||
|
||||
store, err := NewStore(cfg)
|
||||
if err != nil {
|
||||
os.RemoveAll(tmpDir)
|
||||
t.Fatalf("NewStore failed: %v", err)
|
||||
}
|
||||
|
||||
summaryStore := NewSummaryStore(store)
|
||||
|
||||
cleanup := func() {
|
||||
store.Close()
|
||||
os.RemoveAll(tmpDir)
|
||||
}
|
||||
|
||||
return summaryStore, store, cleanup
|
||||
}
|
||||
|
||||
func TestSummaryStore_StoreSummary(t *testing.T) {
|
||||
summaryStore, store, cleanup := testSummaryStore(t)
|
||||
defer cleanup()
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
// Create a session first
|
||||
sessionStore := NewSessionStore(store)
|
||||
_, err := sessionStore.CreateSDKSession(ctx, "claude-1", "test-project", "")
|
||||
require.NoError(t, err)
|
||||
|
||||
// Store a summary
|
||||
summary := &models.ParsedSummary{
|
||||
Request: "Build a feature",
|
||||
Investigated: "Examined the codebase",
|
||||
Learned: "Discovered patterns",
|
||||
Completed: "Implemented solution",
|
||||
NextSteps: "Write tests",
|
||||
Notes: "Additional notes",
|
||||
}
|
||||
|
||||
id, epoch, err := summaryStore.StoreSummary(ctx, "claude-1", "test-project", summary, 1, 100)
|
||||
require.NoError(t, err)
|
||||
assert.Greater(t, id, int64(0))
|
||||
assert.Greater(t, epoch, int64(0))
|
||||
}
|
||||
|
||||
func TestSummaryStore_StoreSummary_AutoCreateSession(t *testing.T) {
|
||||
summaryStore, _, cleanup := testSummaryStore(t)
|
||||
defer cleanup()
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
// Store summary without pre-creating session
|
||||
summary := &models.ParsedSummary{
|
||||
Request: "Test auto-create",
|
||||
}
|
||||
|
||||
id, _, err := summaryStore.StoreSummary(ctx, "claude-auto", "auto-project", summary, 1, 50)
|
||||
require.NoError(t, err)
|
||||
assert.Greater(t, id, int64(0))
|
||||
}
|
||||
|
||||
func TestSummaryStore_GetRecentSummaries(t *testing.T) {
|
||||
summaryStore, _, cleanup := testSummaryStore(t)
|
||||
defer cleanup()
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
// Store multiple summaries
|
||||
for i := 1; i <= 5; i++ {
|
||||
summary := &models.ParsedSummary{
|
||||
Request: "Request " + string(rune('0'+i)),
|
||||
}
|
||||
_, _, err := summaryStore.StoreSummary(ctx, "claude-1", "project-a", summary, i, 10)
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
// Store summary for different project
|
||||
summary := &models.ParsedSummary{Request: "Other project"}
|
||||
_, _, err := summaryStore.StoreSummary(ctx, "claude-2", "project-b", summary, 1, 10)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Get recent summaries for project-a
|
||||
summaries, err := summaryStore.GetRecentSummaries(ctx, "project-a", 10)
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, summaries, 5)
|
||||
|
||||
// Verify ordering (most recent first)
|
||||
assert.Equal(t, "project-a", summaries[0].Project)
|
||||
}
|
||||
|
||||
func TestSummaryStore_GetAllRecentSummaries(t *testing.T) {
|
||||
summaryStore, _, cleanup := testSummaryStore(t)
|
||||
defer cleanup()
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
// Store summaries across projects
|
||||
_, _, err := summaryStore.StoreSummary(ctx, "claude-1", "project-a", &models.ParsedSummary{Request: "A1"}, 1, 10)
|
||||
require.NoError(t, err)
|
||||
|
||||
_, _, err = summaryStore.StoreSummary(ctx, "claude-2", "project-b", &models.ParsedSummary{Request: "B1"}, 1, 10)
|
||||
require.NoError(t, err)
|
||||
|
||||
_, _, err = summaryStore.StoreSummary(ctx, "claude-3", "project-c", &models.ParsedSummary{Request: "C1"}, 1, 10)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Get all recent summaries
|
||||
summaries, err := summaryStore.GetAllRecentSummaries(ctx, 10)
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, summaries, 3)
|
||||
}
|
||||
|
||||
func TestSummaryStore_GetSummariesByIDs(t *testing.T) {
|
||||
summaryStore, _, cleanup := testSummaryStore(t)
|
||||
defer cleanup()
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
// Store multiple summaries
|
||||
var ids []int64
|
||||
for i := 1; i <= 3; i++ {
|
||||
id, _, err := summaryStore.StoreSummary(ctx, "claude-1", "project-a", &models.ParsedSummary{Request: "Test"}, i, 10)
|
||||
require.NoError(t, err)
|
||||
ids = append(ids, id)
|
||||
}
|
||||
|
||||
// Get by IDs
|
||||
summaries, err := summaryStore.GetSummariesByIDs(ctx, ids, "date_desc", 10)
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, summaries, 3)
|
||||
|
||||
// Get with limit
|
||||
summaries, err = summaryStore.GetSummariesByIDs(ctx, ids, "date_desc", 2)
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, summaries, 2)
|
||||
}
|
||||
|
||||
func TestSummaryStore_GetSummariesByIDs_EmptyInput(t *testing.T) {
|
||||
summaryStore, _, cleanup := testSummaryStore(t)
|
||||
defer cleanup()
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
// Get with empty IDs
|
||||
summaries, err := summaryStore.GetSummariesByIDs(ctx, []int64{}, "date_desc", 10)
|
||||
require.NoError(t, err)
|
||||
assert.Nil(t, summaries)
|
||||
}
|
||||
|
||||
func TestSummaryStore_SummaryFields(t *testing.T) {
|
||||
summaryStore, _, cleanup := testSummaryStore(t)
|
||||
defer cleanup()
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
// Store a summary with all fields
|
||||
summary := &models.ParsedSummary{
|
||||
Request: "Full request",
|
||||
Investigated: "Full investigation",
|
||||
Learned: "Full learning",
|
||||
Completed: "Full completion",
|
||||
NextSteps: "Full next steps",
|
||||
Notes: "Full notes",
|
||||
}
|
||||
|
||||
id, epoch, err := summaryStore.StoreSummary(ctx, "claude-1", "test-project", summary, 5, 200)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Retrieve and verify all fields
|
||||
summaries, err := summaryStore.GetSummariesByIDs(ctx, []int64{id}, "date_desc", 1)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, summaries, 1)
|
||||
|
||||
s := summaries[0]
|
||||
assert.Equal(t, id, s.ID)
|
||||
assert.Equal(t, "claude-1", s.SDKSessionID)
|
||||
assert.Equal(t, "test-project", s.Project)
|
||||
assert.True(t, s.Request.Valid)
|
||||
assert.Equal(t, "Full request", s.Request.String)
|
||||
assert.True(t, s.Investigated.Valid)
|
||||
assert.Equal(t, "Full investigation", s.Investigated.String)
|
||||
assert.True(t, s.Learned.Valid)
|
||||
assert.Equal(t, "Full learning", s.Learned.String)
|
||||
assert.True(t, s.Completed.Valid)
|
||||
assert.Equal(t, "Full completion", s.Completed.String)
|
||||
assert.True(t, s.NextSteps.Valid)
|
||||
assert.Equal(t, "Full next steps", s.NextSteps.String)
|
||||
assert.True(t, s.Notes.Valid)
|
||||
assert.Equal(t, "Full notes", s.Notes.String)
|
||||
assert.True(t, s.PromptNumber.Valid)
|
||||
assert.Equal(t, int64(5), s.PromptNumber.Int64)
|
||||
assert.Equal(t, int64(200), s.DiscoveryTokens)
|
||||
assert.NotEmpty(t, s.CreatedAt)
|
||||
assert.Equal(t, epoch, s.CreatedAtEpoch)
|
||||
}
|
||||
|
||||
func TestSummaryStore_EmptySummary(t *testing.T) {
|
||||
summaryStore, _, cleanup := testSummaryStore(t)
|
||||
defer cleanup()
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
// Store a summary with empty fields
|
||||
summary := &models.ParsedSummary{}
|
||||
|
||||
id, _, err := summaryStore.StoreSummary(ctx, "claude-1", "test-project", summary, 0, 0)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Retrieve and verify NULL fields
|
||||
summaries, err := summaryStore.GetSummariesByIDs(ctx, []int64{id}, "date_desc", 1)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, summaries, 1)
|
||||
|
||||
s := summaries[0]
|
||||
assert.False(t, s.Request.Valid)
|
||||
assert.False(t, s.Investigated.Valid)
|
||||
assert.False(t, s.Learned.Valid)
|
||||
assert.False(t, s.Completed.Valid)
|
||||
assert.False(t, s.NextSteps.Valid)
|
||||
assert.False(t, s.Notes.Valid)
|
||||
assert.False(t, s.PromptNumber.Valid)
|
||||
assert.Equal(t, int64(0), s.DiscoveryTokens)
|
||||
}
|
||||
|
||||
func TestSummaryStore_GetAllSummaries(t *testing.T) {
|
||||
summaryStore, _, cleanup := testSummaryStore(t)
|
||||
defer cleanup()
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
// Store multiple summaries
|
||||
for i := 1; i <= 5; i++ {
|
||||
_, _, err := summaryStore.StoreSummary(ctx, "claude-1", "project-a", &models.ParsedSummary{Request: "Test"}, i, 10)
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
// Get all summaries
|
||||
summaries, err := summaryStore.GetAllSummaries(ctx)
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, summaries, 5)
|
||||
|
||||
// Verify ordering by ID
|
||||
for i := 0; i < len(summaries)-1; i++ {
|
||||
assert.Less(t, summaries[i].ID, summaries[i+1].ID)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,71 @@
|
||||
// Package db defines database interfaces for the claude-mnemonic stores.
|
||||
package db
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/lukaszraczylo/claude-mnemonic/pkg/models"
|
||||
)
|
||||
|
||||
// ObservationReader defines read operations for observations.
|
||||
type ObservationReader interface {
|
||||
GetObservationByID(ctx context.Context, id int64) (*models.Observation, error)
|
||||
GetObservationsByIDs(ctx context.Context, ids []int64, orderBy string, limit int) ([]*models.Observation, error)
|
||||
GetRecentObservations(ctx context.Context, project string, limit int) ([]*models.Observation, error)
|
||||
GetActiveObservations(ctx context.Context, project string, limit int) ([]*models.Observation, error)
|
||||
GetAllRecentObservations(ctx context.Context, limit int) ([]*models.Observation, error)
|
||||
GetAllObservations(ctx context.Context) ([]*models.Observation, error)
|
||||
SearchObservationsFTS(ctx context.Context, query, project string, limit int) ([]*models.Observation, error)
|
||||
GetObservationCount(ctx context.Context, project string) (int, error)
|
||||
}
|
||||
|
||||
// ObservationWriter defines write operations for observations.
|
||||
type ObservationWriter interface {
|
||||
StoreObservation(ctx context.Context, sdkSessionID, project string, obs *models.ParsedObservation, promptNumber int, discoveryTokens int64) (int64, int64, error)
|
||||
DeleteObservations(ctx context.Context, ids []int64) (int64, error)
|
||||
}
|
||||
|
||||
// ObservationStore combines read and write operations for observations.
|
||||
type ObservationStore interface {
|
||||
ObservationReader
|
||||
ObservationWriter
|
||||
}
|
||||
|
||||
// SummaryReader defines read operations for summaries.
|
||||
type SummaryReader interface {
|
||||
GetSummariesByIDs(ctx context.Context, ids []int64, orderBy string, limit int) ([]*models.SessionSummary, error)
|
||||
GetRecentSummaries(ctx context.Context, project string, limit int) ([]*models.SessionSummary, error)
|
||||
GetAllRecentSummaries(ctx context.Context, limit int) ([]*models.SessionSummary, error)
|
||||
GetAllSummaries(ctx context.Context) ([]*models.SessionSummary, error)
|
||||
}
|
||||
|
||||
// SummaryWriter defines write operations for summaries.
|
||||
type SummaryWriter interface {
|
||||
StoreSummary(ctx context.Context, sdkSessionID, project string, summary *models.ParsedSummary, promptNumber int, discoveryTokens int64) (int64, int64, error)
|
||||
}
|
||||
|
||||
// SummaryStore combines read and write operations for summaries.
|
||||
type SummaryStore interface {
|
||||
SummaryReader
|
||||
SummaryWriter
|
||||
}
|
||||
|
||||
// PromptReader defines read operations for prompts.
|
||||
type PromptReader interface {
|
||||
GetPromptsByIDs(ctx context.Context, ids []int64, orderBy string, limit int) ([]*models.UserPromptWithSession, error)
|
||||
GetAllRecentUserPrompts(ctx context.Context, limit int) ([]*models.UserPromptWithSession, error)
|
||||
GetAllPrompts(ctx context.Context) ([]*models.UserPromptWithSession, error)
|
||||
GetRecentUserPromptsByProject(ctx context.Context, project string, limit int) ([]*models.UserPromptWithSession, error)
|
||||
FindRecentPromptByText(ctx context.Context, claudeSessionID, promptText string, withinSeconds int) (int64, int, bool)
|
||||
}
|
||||
|
||||
// PromptWriter defines write operations for prompts.
|
||||
type PromptWriter interface {
|
||||
SaveUserPromptWithMatches(ctx context.Context, claudeSessionID string, promptNumber int, promptText string, matchedObservations int) (int64, error)
|
||||
}
|
||||
|
||||
// PromptStore combines read and write operations for prompts.
|
||||
type PromptStore interface {
|
||||
PromptReader
|
||||
PromptWriter
|
||||
}
|
||||
@@ -1,160 +0,0 @@
|
||||
// Package sqlite provides SQLite database operations for claude-mnemonic.
|
||||
package sqlite
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/lukaszraczylo/claude-mnemonic/pkg/models"
|
||||
)
|
||||
|
||||
// EnsureSessionExists creates a session if it doesn't exist.
|
||||
// This is shared between ObservationStore and SummaryStore to avoid duplication.
|
||||
func EnsureSessionExists(ctx context.Context, store *Store, sdkSessionID, project string) error {
|
||||
const checkQuery = `SELECT id FROM sdk_sessions WHERE sdk_session_id = ?`
|
||||
var id int64
|
||||
err := store.QueryRowContext(ctx, checkQuery, sdkSessionID).Scan(&id)
|
||||
if err == nil {
|
||||
return nil // Session exists
|
||||
}
|
||||
if err != sql.ErrNoRows {
|
||||
return err
|
||||
}
|
||||
|
||||
// Auto-create session
|
||||
now := time.Now()
|
||||
const insertQuery = `
|
||||
INSERT INTO sdk_sessions
|
||||
(claude_session_id, sdk_session_id, project, started_at, started_at_epoch, status)
|
||||
VALUES (?, ?, ?, ?, ?, 'active')
|
||||
`
|
||||
_, err = store.ExecContext(ctx, insertQuery,
|
||||
sdkSessionID, sdkSessionID, project,
|
||||
now.Format(time.RFC3339), now.UnixMilli(),
|
||||
)
|
||||
return err
|
||||
}
|
||||
|
||||
// nullString converts a string to sql.NullString.
|
||||
func nullString(s string) sql.NullString {
|
||||
return sql.NullString{String: s, Valid: s != ""}
|
||||
}
|
||||
|
||||
// nullInt converts an int to sql.NullInt64.
|
||||
func nullInt(i int) sql.NullInt64 {
|
||||
return sql.NullInt64{Int64: int64(i), Valid: i > 0}
|
||||
}
|
||||
|
||||
// repeatPlaceholders generates n comma-prefixed placeholders for SQL IN clauses.
|
||||
// e.g., repeatPlaceholders(2) returns ", ?, ?"
|
||||
func repeatPlaceholders(n int) string {
|
||||
if n <= 0 {
|
||||
return ""
|
||||
}
|
||||
result := ""
|
||||
for i := 0; i < n; i++ {
|
||||
result += ", ?"
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
// int64SliceToInterface converts []int64 to []interface{} for SQL queries.
|
||||
func int64SliceToInterface(ids []int64) []interface{} {
|
||||
args := make([]interface{}, len(ids))
|
||||
for i, id := range ids {
|
||||
args[i] = id
|
||||
}
|
||||
return args
|
||||
}
|
||||
|
||||
// ParseLimitParam parses a limit query parameter with a default value.
|
||||
func ParseLimitParam(r *http.Request, defaultLimit int) int {
|
||||
if l := r.URL.Query().Get("limit"); l != "" {
|
||||
if parsed, err := strconv.Atoi(l); err == nil && parsed > 0 {
|
||||
return parsed
|
||||
}
|
||||
}
|
||||
return defaultLimit
|
||||
}
|
||||
|
||||
// scanSummary scans a single summary from a row scanner.
|
||||
func scanSummary(scanner interface{ Scan(...interface{}) error }) (*models.SessionSummary, error) {
|
||||
var summary models.SessionSummary
|
||||
if err := scanner.Scan(
|
||||
&summary.ID, &summary.SDKSessionID, &summary.Project,
|
||||
&summary.Request, &summary.Investigated, &summary.Learned, &summary.Completed,
|
||||
&summary.NextSteps, &summary.Notes, &summary.PromptNumber, &summary.DiscoveryTokens,
|
||||
&summary.CreatedAt, &summary.CreatedAtEpoch,
|
||||
); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &summary, nil
|
||||
}
|
||||
|
||||
// scanSummaryRows scans multiple summaries from rows.
|
||||
func scanSummaryRows(rows *sql.Rows) ([]*models.SessionSummary, error) {
|
||||
var summaries []*models.SessionSummary
|
||||
for rows.Next() {
|
||||
summary, err := scanSummary(rows)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
summaries = append(summaries, summary)
|
||||
}
|
||||
return summaries, rows.Err()
|
||||
}
|
||||
|
||||
// scanPromptWithSession scans a single prompt with session info from a row scanner.
|
||||
func scanPromptWithSession(scanner interface{ Scan(...interface{}) error }) (*models.UserPromptWithSession, error) {
|
||||
var prompt models.UserPromptWithSession
|
||||
if err := scanner.Scan(
|
||||
&prompt.ID, &prompt.ClaudeSessionID, &prompt.PromptNumber, &prompt.PromptText,
|
||||
&prompt.MatchedObservations, &prompt.CreatedAt, &prompt.CreatedAtEpoch,
|
||||
&prompt.Project, &prompt.SDKSessionID,
|
||||
); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &prompt, nil
|
||||
}
|
||||
|
||||
// scanPromptWithSessionRows scans multiple prompts with session info from rows.
|
||||
func scanPromptWithSessionRows(rows *sql.Rows) ([]*models.UserPromptWithSession, error) {
|
||||
var prompts []*models.UserPromptWithSession
|
||||
for rows.Next() {
|
||||
prompt, err := scanPromptWithSession(rows)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
prompts = append(prompts, prompt)
|
||||
}
|
||||
return prompts, rows.Err()
|
||||
}
|
||||
|
||||
// BuildGetByIDsQuery builds a query for fetching records by IDs with optional ordering and limit.
|
||||
// Returns the query string and args slice.
|
||||
func BuildGetByIDsQuery(baseQuery string, ids []int64, orderBy string, limit int) (string, []interface{}) {
|
||||
// Build query with placeholders
|
||||
// #nosec G202 -- query uses parameterized placeholders, not user input
|
||||
query := baseQuery + ` WHERE id IN (?` + repeatPlaceholders(len(ids)-1) + `)
|
||||
ORDER BY created_at_epoch `
|
||||
|
||||
if orderBy == "date_asc" {
|
||||
query += "ASC"
|
||||
} else {
|
||||
query += "DESC"
|
||||
}
|
||||
|
||||
if limit > 0 {
|
||||
query += " LIMIT ?"
|
||||
}
|
||||
|
||||
args := int64SliceToInterface(ids)
|
||||
if limit > 0 {
|
||||
args = append(args, limit)
|
||||
}
|
||||
|
||||
return query, args
|
||||
}
|
||||
@@ -1,365 +0,0 @@
|
||||
// Package sqlite provides SQLite database operations for claude-mnemonic.
|
||||
package sqlite
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Migration represents a database schema migration.
|
||||
type Migration struct {
|
||||
Version int
|
||||
Name string
|
||||
SQL string
|
||||
}
|
||||
|
||||
// Migrations is the list of all database migrations in order.
|
||||
var Migrations = []Migration{
|
||||
{
|
||||
Version: 4,
|
||||
Name: "sdk_agent_architecture",
|
||||
SQL: `
|
||||
-- SDK Sessions (main session tracking)
|
||||
CREATE TABLE IF NOT EXISTS sdk_sessions (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
claude_session_id TEXT UNIQUE NOT NULL,
|
||||
sdk_session_id TEXT UNIQUE,
|
||||
project TEXT NOT NULL,
|
||||
user_prompt TEXT,
|
||||
started_at TEXT NOT NULL,
|
||||
started_at_epoch INTEGER NOT NULL,
|
||||
completed_at TEXT,
|
||||
completed_at_epoch INTEGER,
|
||||
status TEXT CHECK(status IN ('active', 'completed', 'failed')) NOT NULL DEFAULT 'active'
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_sdk_sessions_claude_id ON sdk_sessions(claude_session_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_sdk_sessions_sdk_id ON sdk_sessions(sdk_session_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_sdk_sessions_project ON sdk_sessions(project);
|
||||
CREATE INDEX IF NOT EXISTS idx_sdk_sessions_status ON sdk_sessions(status);
|
||||
CREATE INDEX IF NOT EXISTS idx_sdk_sessions_started ON sdk_sessions(started_at_epoch DESC);
|
||||
|
||||
-- Observations (extracted learnings)
|
||||
CREATE TABLE IF NOT EXISTS observations (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
sdk_session_id TEXT NOT NULL,
|
||||
project TEXT NOT NULL,
|
||||
text TEXT,
|
||||
type TEXT NOT NULL CHECK(type IN ('decision', 'bugfix', 'feature', 'refactor', 'discovery', 'change')),
|
||||
created_at TEXT NOT NULL,
|
||||
created_at_epoch INTEGER NOT NULL,
|
||||
FOREIGN KEY(sdk_session_id) REFERENCES sdk_sessions(sdk_session_id) ON DELETE CASCADE
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_observations_sdk_session ON observations(sdk_session_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_observations_project ON observations(project);
|
||||
CREATE INDEX IF NOT EXISTS idx_observations_type ON observations(type);
|
||||
CREATE INDEX IF NOT EXISTS idx_observations_created ON observations(created_at_epoch DESC);
|
||||
|
||||
-- Session Summaries
|
||||
CREATE TABLE IF NOT EXISTS session_summaries (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
sdk_session_id TEXT NOT NULL,
|
||||
project TEXT NOT NULL,
|
||||
request TEXT,
|
||||
investigated TEXT,
|
||||
learned TEXT,
|
||||
completed TEXT,
|
||||
next_steps TEXT,
|
||||
files_read TEXT,
|
||||
files_edited TEXT,
|
||||
notes TEXT,
|
||||
created_at TEXT NOT NULL,
|
||||
created_at_epoch INTEGER NOT NULL,
|
||||
FOREIGN KEY(sdk_session_id) REFERENCES sdk_sessions(sdk_session_id) ON DELETE CASCADE
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_session_summaries_sdk_session ON session_summaries(sdk_session_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_session_summaries_project ON session_summaries(project);
|
||||
CREATE INDEX IF NOT EXISTS idx_session_summaries_created ON session_summaries(created_at_epoch DESC);
|
||||
`,
|
||||
},
|
||||
{
|
||||
Version: 5,
|
||||
Name: "worker_port_column",
|
||||
SQL: `ALTER TABLE sdk_sessions ADD COLUMN worker_port INTEGER;`,
|
||||
},
|
||||
{
|
||||
Version: 6,
|
||||
Name: "prompt_tracking_columns",
|
||||
SQL: `
|
||||
ALTER TABLE sdk_sessions ADD COLUMN prompt_counter INTEGER DEFAULT 0;
|
||||
ALTER TABLE observations ADD COLUMN prompt_number INTEGER;
|
||||
ALTER TABLE session_summaries ADD COLUMN prompt_number INTEGER;
|
||||
`,
|
||||
},
|
||||
{
|
||||
Version: 8,
|
||||
Name: "observation_hierarchical_fields",
|
||||
SQL: `
|
||||
ALTER TABLE observations ADD COLUMN title TEXT;
|
||||
ALTER TABLE observations ADD COLUMN subtitle TEXT;
|
||||
ALTER TABLE observations ADD COLUMN facts TEXT;
|
||||
ALTER TABLE observations ADD COLUMN narrative TEXT;
|
||||
ALTER TABLE observations ADD COLUMN concepts TEXT;
|
||||
ALTER TABLE observations ADD COLUMN files_read TEXT;
|
||||
ALTER TABLE observations ADD COLUMN files_modified TEXT;
|
||||
`,
|
||||
},
|
||||
{
|
||||
Version: 10,
|
||||
Name: "user_prompts_table",
|
||||
SQL: `
|
||||
-- User prompts table
|
||||
CREATE TABLE IF NOT EXISTS user_prompts (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
claude_session_id TEXT NOT NULL,
|
||||
prompt_number INTEGER NOT NULL,
|
||||
prompt_text TEXT NOT NULL,
|
||||
created_at TEXT NOT NULL,
|
||||
created_at_epoch INTEGER NOT NULL,
|
||||
FOREIGN KEY(claude_session_id) REFERENCES sdk_sessions(claude_session_id) ON DELETE CASCADE
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_user_prompts_claude_session ON user_prompts(claude_session_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_user_prompts_created ON user_prompts(created_at_epoch DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_user_prompts_prompt_number ON user_prompts(prompt_number);
|
||||
CREATE INDEX IF NOT EXISTS idx_user_prompts_lookup ON user_prompts(claude_session_id, prompt_number);
|
||||
|
||||
-- FTS5 virtual table for user prompts
|
||||
CREATE VIRTUAL TABLE IF NOT EXISTS user_prompts_fts USING fts5(
|
||||
prompt_text,
|
||||
content='user_prompts',
|
||||
content_rowid='id'
|
||||
);
|
||||
|
||||
-- Triggers for FTS5 sync
|
||||
CREATE TRIGGER IF NOT EXISTS user_prompts_ai AFTER INSERT ON user_prompts BEGIN
|
||||
INSERT INTO user_prompts_fts(rowid, prompt_text)
|
||||
VALUES (new.id, new.prompt_text);
|
||||
END;
|
||||
|
||||
CREATE TRIGGER IF NOT EXISTS user_prompts_ad AFTER DELETE ON user_prompts BEGIN
|
||||
INSERT INTO user_prompts_fts(user_prompts_fts, rowid, prompt_text)
|
||||
VALUES('delete', old.id, old.prompt_text);
|
||||
END;
|
||||
|
||||
CREATE TRIGGER IF NOT EXISTS user_prompts_au AFTER UPDATE ON user_prompts BEGIN
|
||||
INSERT INTO user_prompts_fts(user_prompts_fts, rowid, prompt_text)
|
||||
VALUES('delete', old.id, old.prompt_text);
|
||||
INSERT INTO user_prompts_fts(rowid, prompt_text)
|
||||
VALUES (new.id, new.prompt_text);
|
||||
END;
|
||||
`,
|
||||
},
|
||||
{
|
||||
Version: 11,
|
||||
Name: "discovery_tokens_column",
|
||||
SQL: `
|
||||
ALTER TABLE observations ADD COLUMN discovery_tokens INTEGER DEFAULT 0;
|
||||
ALTER TABLE session_summaries ADD COLUMN discovery_tokens INTEGER DEFAULT 0;
|
||||
`,
|
||||
},
|
||||
{
|
||||
Version: 12,
|
||||
Name: "observations_fts",
|
||||
SQL: `
|
||||
-- FTS5 virtual table for observations
|
||||
CREATE VIRTUAL TABLE IF NOT EXISTS observations_fts USING fts5(
|
||||
title, subtitle, narrative,
|
||||
content='observations',
|
||||
content_rowid='id'
|
||||
);
|
||||
|
||||
-- Triggers for FTS5 sync
|
||||
CREATE TRIGGER IF NOT EXISTS observations_ai AFTER INSERT ON observations BEGIN
|
||||
INSERT INTO observations_fts(rowid, title, subtitle, narrative)
|
||||
VALUES (new.id, new.title, new.subtitle, new.narrative);
|
||||
END;
|
||||
|
||||
CREATE TRIGGER IF NOT EXISTS observations_ad AFTER DELETE ON observations BEGIN
|
||||
INSERT INTO observations_fts(observations_fts, rowid, title, subtitle, narrative)
|
||||
VALUES('delete', old.id, old.title, old.subtitle, old.narrative);
|
||||
END;
|
||||
|
||||
CREATE TRIGGER IF NOT EXISTS observations_au AFTER UPDATE ON observations BEGIN
|
||||
INSERT INTO observations_fts(observations_fts, rowid, title, subtitle, narrative)
|
||||
VALUES('delete', old.id, old.title, old.subtitle, old.narrative);
|
||||
INSERT INTO observations_fts(rowid, title, subtitle, narrative)
|
||||
VALUES (new.id, new.title, new.subtitle, new.narrative);
|
||||
END;
|
||||
`,
|
||||
},
|
||||
{
|
||||
Version: 13,
|
||||
Name: "session_summaries_fts",
|
||||
SQL: `
|
||||
-- FTS5 virtual table for session summaries
|
||||
CREATE VIRTUAL TABLE IF NOT EXISTS session_summaries_fts USING fts5(
|
||||
request, investigated, learned, completed, next_steps, notes,
|
||||
content='session_summaries',
|
||||
content_rowid='id'
|
||||
);
|
||||
|
||||
-- Triggers for FTS5 sync
|
||||
CREATE TRIGGER IF NOT EXISTS session_summaries_ai AFTER INSERT ON session_summaries BEGIN
|
||||
INSERT INTO session_summaries_fts(rowid, request, investigated, learned, completed, next_steps, notes)
|
||||
VALUES (new.id, new.request, new.investigated, new.learned, new.completed, new.next_steps, new.notes);
|
||||
END;
|
||||
|
||||
CREATE TRIGGER IF NOT EXISTS session_summaries_ad AFTER DELETE ON session_summaries BEGIN
|
||||
INSERT INTO session_summaries_fts(session_summaries_fts, rowid, request, investigated, learned, completed, next_steps, notes)
|
||||
VALUES('delete', old.id, old.request, old.investigated, old.learned, old.completed, old.next_steps, old.notes);
|
||||
END;
|
||||
|
||||
CREATE TRIGGER IF NOT EXISTS session_summaries_au AFTER UPDATE ON session_summaries BEGIN
|
||||
INSERT INTO session_summaries_fts(session_summaries_fts, rowid, request, investigated, learned, completed, next_steps, notes)
|
||||
VALUES('delete', old.id, old.request, old.investigated, old.learned, old.completed, old.next_steps, old.notes);
|
||||
INSERT INTO session_summaries_fts(rowid, request, investigated, learned, completed, next_steps, notes)
|
||||
VALUES (new.id, new.request, new.investigated, new.learned, new.completed, new.next_steps, new.notes);
|
||||
END;
|
||||
`,
|
||||
},
|
||||
{
|
||||
Version: 14,
|
||||
Name: "observation_scope_column",
|
||||
SQL: `
|
||||
-- Add scope column for project isolation
|
||||
-- 'project' = only visible within same project (default)
|
||||
-- 'global' = visible across all projects (best practices, patterns)
|
||||
ALTER TABLE observations ADD COLUMN scope TEXT DEFAULT 'project' CHECK(scope IN ('project', 'global'));
|
||||
|
||||
-- Index for efficient scope-based queries
|
||||
CREATE INDEX IF NOT EXISTS idx_observations_scope ON observations(scope);
|
||||
CREATE INDEX IF NOT EXISTS idx_observations_project_scope ON observations(project, scope);
|
||||
`,
|
||||
},
|
||||
{
|
||||
Version: 15,
|
||||
Name: "observation_file_mtimes",
|
||||
SQL: `
|
||||
-- Store file modification times at observation creation
|
||||
-- JSON object: {"path": mtime_epoch_ms, ...}
|
||||
-- Used to detect staleness when files change
|
||||
ALTER TABLE observations ADD COLUMN file_mtimes TEXT;
|
||||
`,
|
||||
},
|
||||
{
|
||||
Version: 16,
|
||||
Name: "prompt_matched_observations",
|
||||
SQL: `
|
||||
-- Track how many observations were found relevant for each prompt
|
||||
-- Displayed in dashboard timeline
|
||||
ALTER TABLE user_prompts ADD COLUMN matched_observations INTEGER DEFAULT 0;
|
||||
`,
|
||||
},
|
||||
{
|
||||
Version: 17,
|
||||
Name: "sqlite_vec_vectors",
|
||||
SQL: `
|
||||
-- Vector embeddings table using sqlite-vec
|
||||
-- Each document (narrative, fact, summary field, prompt) gets one vector
|
||||
-- Uses all-MiniLM-L6-v2 embeddings (384 dimensions)
|
||||
CREATE VIRTUAL TABLE IF NOT EXISTS vectors USING vec0(
|
||||
doc_id TEXT PRIMARY KEY,
|
||||
embedding float[384],
|
||||
sqlite_id INTEGER,
|
||||
doc_type TEXT,
|
||||
field_type TEXT,
|
||||
project TEXT,
|
||||
scope TEXT
|
||||
);
|
||||
`,
|
||||
},
|
||||
}
|
||||
|
||||
// MigrationManager handles database schema migrations.
|
||||
type MigrationManager struct {
|
||||
db *sql.DB
|
||||
}
|
||||
|
||||
// NewMigrationManager creates a new migration manager.
|
||||
func NewMigrationManager(db *sql.DB) *MigrationManager {
|
||||
return &MigrationManager{db: db}
|
||||
}
|
||||
|
||||
// EnsureSchemaVersionsTable creates the schema_versions table if it doesn't exist.
|
||||
func (m *MigrationManager) EnsureSchemaVersionsTable() error {
|
||||
_, err := m.db.Exec(`
|
||||
CREATE TABLE IF NOT EXISTS schema_versions (
|
||||
id INTEGER PRIMARY KEY,
|
||||
version INTEGER UNIQUE NOT NULL,
|
||||
applied_at TEXT NOT NULL
|
||||
)
|
||||
`)
|
||||
return err
|
||||
}
|
||||
|
||||
// GetAppliedVersions returns all applied migration versions.
|
||||
func (m *MigrationManager) GetAppliedVersions() (map[int]bool, error) {
|
||||
rows, err := m.db.Query("SELECT version FROM schema_versions ORDER BY version")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
versions := make(map[int]bool)
|
||||
for rows.Next() {
|
||||
var version int
|
||||
if err := rows.Scan(&version); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
versions[version] = true
|
||||
}
|
||||
return versions, rows.Err()
|
||||
}
|
||||
|
||||
// ApplyMigration applies a single migration.
|
||||
func (m *MigrationManager) ApplyMigration(migration Migration) error {
|
||||
tx, err := m.db.Begin()
|
||||
if err != nil {
|
||||
return fmt.Errorf("begin transaction: %w", err)
|
||||
}
|
||||
defer tx.Rollback()
|
||||
|
||||
// Execute migration SQL
|
||||
if _, err := tx.Exec(migration.SQL); err != nil {
|
||||
return fmt.Errorf("execute migration %d (%s): %w", migration.Version, migration.Name, err)
|
||||
}
|
||||
|
||||
// Record migration
|
||||
_, err = tx.Exec(
|
||||
"INSERT INTO schema_versions (version, applied_at) VALUES (?, ?)",
|
||||
migration.Version, time.Now().Format(time.RFC3339),
|
||||
)
|
||||
if err != nil {
|
||||
return fmt.Errorf("record migration %d: %w", migration.Version, err)
|
||||
}
|
||||
|
||||
return tx.Commit()
|
||||
}
|
||||
|
||||
// RunMigrations applies all pending migrations.
|
||||
func (m *MigrationManager) RunMigrations() error {
|
||||
if err := m.EnsureSchemaVersionsTable(); err != nil {
|
||||
return fmt.Errorf("ensure schema_versions table: %w", err)
|
||||
}
|
||||
|
||||
applied, err := m.GetAppliedVersions()
|
||||
if err != nil {
|
||||
return fmt.Errorf("get applied versions: %w", err)
|
||||
}
|
||||
|
||||
for _, migration := range Migrations {
|
||||
if applied[migration.Version] {
|
||||
continue
|
||||
}
|
||||
|
||||
if err := m.ApplyMigration(migration); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -1,445 +0,0 @@
|
||||
// Package sqlite provides SQLite database operations for claude-mnemonic.
|
||||
package sqlite
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/lukaszraczylo/claude-mnemonic/pkg/models"
|
||||
)
|
||||
|
||||
// CleanupFunc is a callback for when observations are cleaned up.
|
||||
// Receives the IDs of deleted observations for downstream cleanup (e.g., vector DB).
|
||||
type CleanupFunc func(ctx context.Context, deletedIDs []int64)
|
||||
|
||||
// ObservationStore provides observation-related database operations.
|
||||
type ObservationStore struct {
|
||||
store *Store
|
||||
cleanupFunc CleanupFunc
|
||||
}
|
||||
|
||||
// NewObservationStore creates a new observation store.
|
||||
func NewObservationStore(store *Store) *ObservationStore {
|
||||
return &ObservationStore{store: store}
|
||||
}
|
||||
|
||||
// SetCleanupFunc sets the callback for when observations are deleted during cleanup.
|
||||
func (s *ObservationStore) SetCleanupFunc(fn CleanupFunc) {
|
||||
s.cleanupFunc = fn
|
||||
}
|
||||
|
||||
// StoreObservation stores a new observation.
|
||||
func (s *ObservationStore) StoreObservation(ctx context.Context, sdkSessionID, project string, obs *models.ParsedObservation, promptNumber int, discoveryTokens int64) (int64, int64, error) {
|
||||
now := time.Now()
|
||||
nowEpoch := now.UnixMilli()
|
||||
|
||||
// Ensure session exists (auto-create if missing)
|
||||
if err := s.ensureSessionExists(ctx, sdkSessionID, project); err != nil {
|
||||
return 0, 0, err
|
||||
}
|
||||
|
||||
// Determine scope: use parsed scope if set, otherwise auto-determine from concepts
|
||||
scope := obs.Scope
|
||||
if scope == "" {
|
||||
scope = models.DetermineScope(obs.Concepts)
|
||||
}
|
||||
|
||||
factsJSON, _ := json.Marshal(obs.Facts)
|
||||
conceptsJSON, _ := json.Marshal(obs.Concepts)
|
||||
filesReadJSON, _ := json.Marshal(obs.FilesRead)
|
||||
filesModifiedJSON, _ := json.Marshal(obs.FilesModified)
|
||||
fileMtimesJSON, _ := json.Marshal(obs.FileMtimes)
|
||||
|
||||
const query = `
|
||||
INSERT INTO observations
|
||||
(sdk_session_id, project, scope, type, title, subtitle, facts, narrative, concepts,
|
||||
files_read, files_modified, file_mtimes, prompt_number, discovery_tokens, created_at, created_at_epoch)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
`
|
||||
|
||||
result, err := s.store.ExecContext(ctx, query,
|
||||
sdkSessionID, project, string(scope), string(obs.Type),
|
||||
nullString(obs.Title), nullString(obs.Subtitle),
|
||||
string(factsJSON), nullString(obs.Narrative), string(conceptsJSON),
|
||||
string(filesReadJSON), string(filesModifiedJSON), string(fileMtimesJSON),
|
||||
nullInt(promptNumber), discoveryTokens,
|
||||
now.Format(time.RFC3339), nowEpoch,
|
||||
)
|
||||
if err != nil {
|
||||
return 0, 0, err
|
||||
}
|
||||
|
||||
id, _ := result.LastInsertId()
|
||||
|
||||
// Cleanup old observations beyond the limit for this project (async to not block handler)
|
||||
if project != "" {
|
||||
go func(proj string) {
|
||||
cleanupCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
deletedIDs, _ := s.CleanupOldObservations(cleanupCtx, proj)
|
||||
if len(deletedIDs) > 0 && s.cleanupFunc != nil {
|
||||
s.cleanupFunc(cleanupCtx, deletedIDs)
|
||||
}
|
||||
}(project)
|
||||
}
|
||||
|
||||
return id, nowEpoch, nil
|
||||
}
|
||||
|
||||
// ensureSessionExists creates a session if it doesn't exist.
|
||||
func (s *ObservationStore) ensureSessionExists(ctx context.Context, sdkSessionID, project string) error {
|
||||
return EnsureSessionExists(ctx, s.store, sdkSessionID, project)
|
||||
}
|
||||
|
||||
// GetObservationByID retrieves an observation by ID.
|
||||
func (s *ObservationStore) GetObservationByID(ctx context.Context, id int64) (*models.Observation, error) {
|
||||
const query = `
|
||||
SELECT id, sdk_session_id, project, COALESCE(scope, 'project') as scope, type, title, subtitle, facts, narrative,
|
||||
concepts, files_read, files_modified, file_mtimes, prompt_number, discovery_tokens,
|
||||
created_at, created_at_epoch
|
||||
FROM observations
|
||||
WHERE id = ?
|
||||
`
|
||||
|
||||
obs, err := scanObservation(s.store.QueryRowContext(ctx, query, id))
|
||||
if err == sql.ErrNoRows {
|
||||
return nil, nil
|
||||
}
|
||||
return obs, err
|
||||
}
|
||||
|
||||
// GetObservationsByIDs retrieves observations by a list of IDs.
|
||||
func (s *ObservationStore) GetObservationsByIDs(ctx context.Context, ids []int64, orderBy string, limit int) ([]*models.Observation, error) {
|
||||
if len(ids) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// Build query with placeholders
|
||||
// #nosec G202 -- query uses parameterized placeholders, not user input
|
||||
query := `
|
||||
SELECT id, sdk_session_id, project, COALESCE(scope, 'project') as scope, type, title, subtitle, facts, narrative,
|
||||
concepts, files_read, files_modified, file_mtimes, prompt_number, discovery_tokens,
|
||||
created_at, created_at_epoch
|
||||
FROM observations
|
||||
WHERE id IN (?` + repeatPlaceholders(len(ids)-1) + `)
|
||||
ORDER BY created_at_epoch `
|
||||
|
||||
if orderBy == "date_asc" {
|
||||
query += "ASC"
|
||||
} else {
|
||||
query += "DESC"
|
||||
}
|
||||
|
||||
if limit > 0 {
|
||||
query += " LIMIT ?"
|
||||
}
|
||||
|
||||
// Convert []int64 to []interface{}
|
||||
args := int64SliceToInterface(ids)
|
||||
if limit > 0 {
|
||||
args = append(args, limit)
|
||||
}
|
||||
|
||||
rows, err := s.store.db.QueryContext(ctx, query, args...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
return scanObservationRows(rows)
|
||||
}
|
||||
|
||||
// GetRecentObservations retrieves recent observations for a project.
|
||||
// This includes project-scoped observations for the specified project AND global observations.
|
||||
func (s *ObservationStore) GetRecentObservations(ctx context.Context, project string, limit int) ([]*models.Observation, error) {
|
||||
const query = `
|
||||
SELECT id, sdk_session_id, project, COALESCE(scope, 'project') as scope, type, title, subtitle, facts, narrative,
|
||||
concepts, files_read, files_modified, file_mtimes, prompt_number, discovery_tokens,
|
||||
created_at, created_at_epoch
|
||||
FROM observations
|
||||
WHERE (project = ? AND (scope IS NULL OR scope = 'project'))
|
||||
OR scope = 'global'
|
||||
ORDER BY created_at_epoch DESC
|
||||
LIMIT ?
|
||||
`
|
||||
|
||||
rows, err := s.store.QueryContext(ctx, query, project, limit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
return scanObservationRows(rows)
|
||||
}
|
||||
|
||||
// GetObservationCount returns the count of observations for a project (including global).
|
||||
func (s *ObservationStore) GetObservationCount(ctx context.Context, project string) (int, error) {
|
||||
const query = `
|
||||
SELECT COUNT(*) FROM observations
|
||||
WHERE project = ? OR scope = 'global'
|
||||
`
|
||||
var count int
|
||||
err := s.store.QueryRowContext(ctx, query, project).Scan(&count)
|
||||
return count, err
|
||||
}
|
||||
|
||||
// GetAllRecentObservations retrieves recent observations across all projects.
|
||||
func (s *ObservationStore) GetAllRecentObservations(ctx context.Context, limit int) ([]*models.Observation, error) {
|
||||
const query = `
|
||||
SELECT id, sdk_session_id, project, COALESCE(scope, 'project') as scope, type, title, subtitle, facts, narrative,
|
||||
concepts, files_read, files_modified, file_mtimes, prompt_number, discovery_tokens,
|
||||
created_at, created_at_epoch
|
||||
FROM observations
|
||||
ORDER BY created_at_epoch DESC
|
||||
LIMIT ?
|
||||
`
|
||||
|
||||
rows, err := s.store.QueryContext(ctx, query, limit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
return scanObservationRows(rows)
|
||||
}
|
||||
|
||||
// SearchObservationsFTS performs full-text search on observations.
|
||||
func (s *ObservationStore) SearchObservationsFTS(ctx context.Context, query, project string, limit int) ([]*models.Observation, error) {
|
||||
if limit <= 0 {
|
||||
limit = 10
|
||||
}
|
||||
|
||||
// Extract keywords from the query (words > 3 chars, not common)
|
||||
keywords := extractKeywords(query)
|
||||
if len(keywords) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// Build FTS5 query: keyword1 OR keyword2 OR keyword3
|
||||
ftsTerms := strings.Join(keywords, " OR ")
|
||||
|
||||
// Use FTS5 to search title, subtitle, and narrative
|
||||
const ftsQuery = `
|
||||
SELECT o.id, o.sdk_session_id, o.project, COALESCE(o.scope, 'project') as scope, o.type,
|
||||
o.title, o.subtitle, o.facts, o.narrative, o.concepts, o.files_read, o.files_modified,
|
||||
o.file_mtimes, o.prompt_number, o.discovery_tokens, o.created_at, o.created_at_epoch
|
||||
FROM observations o
|
||||
JOIN observations_fts fts ON o.id = fts.rowid
|
||||
WHERE observations_fts MATCH ?
|
||||
AND (o.project = ? OR o.scope = 'global')
|
||||
ORDER BY rank
|
||||
LIMIT ?
|
||||
`
|
||||
|
||||
rows, err := s.store.QueryContext(ctx, ftsQuery, ftsTerms, project, limit)
|
||||
if err != nil {
|
||||
// FTS failed, try LIKE fallback
|
||||
return s.searchObservationsLike(ctx, keywords, project, limit)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
observations, err := scanObservationRows(rows)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// If FTS returned nothing, try LIKE search
|
||||
if len(observations) == 0 {
|
||||
return s.searchObservationsLike(ctx, keywords, project, limit)
|
||||
}
|
||||
|
||||
return observations, nil
|
||||
}
|
||||
|
||||
// searchObservationsLike performs fallback LIKE search on observations.
|
||||
func (s *ObservationStore) searchObservationsLike(ctx context.Context, keywords []string, project string, limit int) ([]*models.Observation, error) {
|
||||
if len(keywords) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// Build LIKE conditions for each keyword
|
||||
var conditions []string
|
||||
var args []interface{}
|
||||
|
||||
for _, kw := range keywords {
|
||||
pattern := "%" + kw + "%"
|
||||
conditions = append(conditions, "(title LIKE ? OR subtitle LIKE ? OR narrative LIKE ?)")
|
||||
args = append(args, pattern, pattern, pattern)
|
||||
}
|
||||
|
||||
// #nosec G202 -- query uses parameterized placeholders, not user input
|
||||
query := `
|
||||
SELECT id, sdk_session_id, project, COALESCE(scope, 'project') as scope, type,
|
||||
title, subtitle, facts, narrative, concepts, files_read, files_modified,
|
||||
file_mtimes, prompt_number, discovery_tokens, created_at, created_at_epoch
|
||||
FROM observations
|
||||
WHERE (` + strings.Join(conditions, " OR ") + `)
|
||||
AND (project = ? OR scope = 'global')
|
||||
ORDER BY created_at_epoch DESC
|
||||
LIMIT ?
|
||||
`
|
||||
args = append(args, project, limit)
|
||||
|
||||
rows, err := s.store.db.QueryContext(ctx, query, args...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
return scanObservationRows(rows)
|
||||
}
|
||||
|
||||
// extractKeywords extracts significant words from a query.
|
||||
func extractKeywords(query string) []string {
|
||||
// Common words to skip
|
||||
stopWords := map[string]bool{
|
||||
"what": true, "is": true, "the": true, "a": true, "an": true,
|
||||
"how": true, "does": true, "do": true, "can": true, "could": true,
|
||||
"would": true, "should": true, "where": true, "when": true, "why": true,
|
||||
"which": true, "who": true, "this": true, "that": true, "these": true,
|
||||
"those": true, "it": true, "its": true, "for": true, "from": true,
|
||||
"with": true, "about": true, "into": true, "through": true, "during": true,
|
||||
"before": true, "after": true, "above": true, "below": true, "to": true,
|
||||
"of": true, "in": true, "on": true, "at": true, "by": true, "and": true,
|
||||
"or": true, "but": true, "if": true, "then": true, "else": true,
|
||||
"function": true, "method": true, "class": true, "file": true,
|
||||
"code": true, "work": true, "works": true, "working": true,
|
||||
"please": true, "help": true, "me": true, "my": true, "i": true,
|
||||
"tell": true, "show": true, "explain": true, "describe": true,
|
||||
}
|
||||
|
||||
// Split and filter
|
||||
words := strings.FieldsFunc(strings.ToLower(query), func(r rune) bool {
|
||||
return !((r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9') || r == '_')
|
||||
})
|
||||
|
||||
var keywords []string
|
||||
seen := make(map[string]bool)
|
||||
|
||||
for _, word := range words {
|
||||
// Skip short words, stop words, and duplicates
|
||||
if len(word) < 4 || stopWords[word] || seen[word] {
|
||||
continue
|
||||
}
|
||||
seen[word] = true
|
||||
keywords = append(keywords, word)
|
||||
}
|
||||
|
||||
return keywords
|
||||
}
|
||||
|
||||
// DeleteObservations deletes multiple observations by ID.
|
||||
func (s *ObservationStore) DeleteObservations(ctx context.Context, ids []int64) (int64, error) {
|
||||
if len(ids) == 0 {
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
query := `DELETE FROM observations WHERE id IN (?` + repeatPlaceholders(len(ids)-1) + `)` // #nosec G202 -- uses parameterized placeholders
|
||||
|
||||
args := make([]interface{}, len(ids))
|
||||
for i, id := range ids {
|
||||
args[i] = id
|
||||
}
|
||||
|
||||
result, err := s.store.db.ExecContext(ctx, query, args...)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return result.RowsAffected()
|
||||
}
|
||||
|
||||
// MaxObservationsPerProject is the hard limit of observations per project.
|
||||
const MaxObservationsPerProject = 100
|
||||
|
||||
// CleanupOldObservations deletes observations beyond the limit for a project.
|
||||
// Keeps the most recent MaxObservationsPerProject observations per project.
|
||||
// Returns the IDs of deleted observations for downstream cleanup (e.g., vector DB).
|
||||
func (s *ObservationStore) CleanupOldObservations(ctx context.Context, project string) ([]int64, error) {
|
||||
// First, find IDs that will be deleted
|
||||
const selectQuery = `
|
||||
SELECT id FROM observations
|
||||
WHERE project = ? AND id NOT IN (
|
||||
SELECT id FROM observations
|
||||
WHERE project = ?
|
||||
ORDER BY created_at_epoch DESC
|
||||
LIMIT ?
|
||||
)
|
||||
`
|
||||
|
||||
rows, err := s.store.QueryContext(ctx, selectQuery, project, project, MaxObservationsPerProject)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var toDelete []int64
|
||||
for rows.Next() {
|
||||
var id int64
|
||||
if err := rows.Scan(&id); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
toDelete = append(toDelete, id)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if len(toDelete) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// Delete the observations
|
||||
const deleteQuery = `
|
||||
DELETE FROM observations
|
||||
WHERE project = ? AND id NOT IN (
|
||||
SELECT id FROM observations
|
||||
WHERE project = ?
|
||||
ORDER BY created_at_epoch DESC
|
||||
LIMIT ?
|
||||
)
|
||||
`
|
||||
|
||||
_, err = s.store.ExecContext(ctx, deleteQuery, project, project, MaxObservationsPerProject)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return toDelete, nil
|
||||
}
|
||||
|
||||
// Helper functions
|
||||
|
||||
// scanObservation scans a single observation from a row scanner.
|
||||
// This reduces code duplication across all observation query methods.
|
||||
func scanObservation(scanner interface{ Scan(...interface{}) error }) (*models.Observation, error) {
|
||||
var obs models.Observation
|
||||
if err := scanner.Scan(
|
||||
&obs.ID, &obs.SDKSessionID, &obs.Project, &obs.Scope, &obs.Type,
|
||||
&obs.Title, &obs.Subtitle, &obs.Facts, &obs.Narrative,
|
||||
&obs.Concepts, &obs.FilesRead, &obs.FilesModified, &obs.FileMtimes,
|
||||
&obs.PromptNumber, &obs.DiscoveryTokens,
|
||||
&obs.CreatedAt, &obs.CreatedAtEpoch,
|
||||
); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &obs, nil
|
||||
}
|
||||
|
||||
// scanObservationRows scans multiple observations from rows.
|
||||
// Caller must close rows after calling this function.
|
||||
func scanObservationRows(rows *sql.Rows) ([]*models.Observation, error) {
|
||||
var observations []*models.Observation
|
||||
for rows.Next() {
|
||||
obs, err := scanObservation(rows)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
observations = append(observations, obs)
|
||||
}
|
||||
return observations, rows.Err()
|
||||
}
|
||||
|
||||
// Note: nullString, nullInt, and repeatPlaceholders are in helpers.go
|
||||
@@ -1,374 +0,0 @@
|
||||
// Package sqlite provides SQLite database operations for claude-mnemonic.
|
||||
package sqlite
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/lukaszraczylo/claude-mnemonic/pkg/models"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// testObservationStore creates an ObservationStore with a test database including FTS5.
|
||||
func testObservationStore(t *testing.T) (*ObservationStore, *Store, func()) {
|
||||
t.Helper()
|
||||
|
||||
db, _, cleanup := testDB(t)
|
||||
createAllTables(t, db)
|
||||
|
||||
store := newStoreFromDB(db)
|
||||
obsStore := NewObservationStore(store)
|
||||
|
||||
return obsStore, store, cleanup
|
||||
}
|
||||
|
||||
func TestObservationStore_StoreAndRetrieve(t *testing.T) {
|
||||
obsStore, _, cleanup := testObservationStore(t)
|
||||
defer cleanup()
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
obs := &models.ParsedObservation{
|
||||
Type: models.ObsTypeDiscovery,
|
||||
Title: "Test Observation",
|
||||
Subtitle: "A subtitle",
|
||||
Narrative: "This is a test observation about testing",
|
||||
Facts: []string{"Fact 1", "Fact 2"},
|
||||
Concepts: []string{"testing", "golang"},
|
||||
FilesRead: []string{"test.go"},
|
||||
FilesModified: []string{},
|
||||
}
|
||||
|
||||
id, epoch, err := obsStore.StoreObservation(ctx, "session-1", "project-a", obs, 1, 100)
|
||||
require.NoError(t, err)
|
||||
assert.Greater(t, id, int64(0))
|
||||
assert.Greater(t, epoch, int64(0))
|
||||
|
||||
// Retrieve by ID
|
||||
retrieved, err := obsStore.GetObservationByID(ctx, id)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, retrieved)
|
||||
|
||||
assert.Equal(t, id, retrieved.ID)
|
||||
assert.Equal(t, "session-1", retrieved.SDKSessionID)
|
||||
assert.Equal(t, "project-a", retrieved.Project)
|
||||
assert.Equal(t, models.ObsTypeDiscovery, retrieved.Type)
|
||||
assert.Equal(t, "Test Observation", retrieved.Title.String)
|
||||
assert.Equal(t, "A subtitle", retrieved.Subtitle.String)
|
||||
assert.Equal(t, "This is a test observation about testing", retrieved.Narrative.String)
|
||||
assert.Equal(t, []string{"Fact 1", "Fact 2"}, []string(retrieved.Facts))
|
||||
assert.Equal(t, []string{"testing", "golang"}, []string(retrieved.Concepts))
|
||||
}
|
||||
|
||||
func TestObservationStore_GetRecentObservations(t *testing.T) {
|
||||
obsStore, _, cleanup := testObservationStore(t)
|
||||
defer cleanup()
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
// Create multiple observations
|
||||
for i := 0; i < 10; i++ {
|
||||
obs := &models.ParsedObservation{
|
||||
Type: models.ObsTypeDiscovery,
|
||||
Title: "Observation " + string(rune('A'+i)),
|
||||
Narrative: "Content " + string(rune('A'+i)),
|
||||
Concepts: []string{"test"},
|
||||
}
|
||||
_, _, err := obsStore.StoreObservation(ctx, "session-1", "project-a", obs, i+1, 100)
|
||||
require.NoError(t, err)
|
||||
time.Sleep(time.Millisecond) // Ensure different timestamps
|
||||
}
|
||||
|
||||
// Get recent with limit 5
|
||||
recent, err := obsStore.GetRecentObservations(ctx, "project-a", 5)
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, recent, 5)
|
||||
|
||||
// Get recent with limit 20 (more than exists)
|
||||
recent, err = obsStore.GetRecentObservations(ctx, "project-a", 20)
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, recent, 10)
|
||||
}
|
||||
|
||||
func TestObservationStore_SearchObservationsFTS(t *testing.T) {
|
||||
obsStore, _, cleanup := testObservationStore(t)
|
||||
defer cleanup()
|
||||
|
||||
// FTS5 tables are created by testObservationStore via testutil.CreateAllTables
|
||||
ctx := context.Background()
|
||||
|
||||
// Create observations with different content
|
||||
observations := []struct {
|
||||
title string
|
||||
narrative string
|
||||
}{
|
||||
{"Authentication implementation", "JWT based authentication flow"},
|
||||
{"Database setup", "PostgreSQL configuration and migrations"},
|
||||
{"Caching layer", "Redis caching implementation"},
|
||||
{"User authentication fix", "Fixed authentication bug in login"},
|
||||
{"API endpoints", "REST API implementation details"},
|
||||
}
|
||||
|
||||
for _, o := range observations {
|
||||
obs := &models.ParsedObservation{
|
||||
Type: models.ObsTypeDiscovery,
|
||||
Title: o.title,
|
||||
Narrative: o.narrative,
|
||||
}
|
||||
_, _, err := obsStore.StoreObservation(ctx, "session-1", "project-a", obs, 1, 100)
|
||||
require.NoError(t, err)
|
||||
time.Sleep(time.Millisecond)
|
||||
}
|
||||
|
||||
// Search for authentication - should find 2 observations
|
||||
results, err := obsStore.SearchObservationsFTS(ctx, "authentication", "project-a", 50)
|
||||
require.NoError(t, err)
|
||||
assert.GreaterOrEqual(t, len(results), 2, "should find at least 2 authentication-related observations")
|
||||
|
||||
// Search for database - should find 1 observation
|
||||
results, err = obsStore.SearchObservationsFTS(ctx, "database PostgreSQL", "project-a", 50)
|
||||
require.NoError(t, err)
|
||||
assert.GreaterOrEqual(t, len(results), 1, "should find at least 1 database-related observation")
|
||||
}
|
||||
|
||||
func TestObservationStore_SearchObservationsFTS_LimitRespected(t *testing.T) {
|
||||
obsStore, _, cleanup := testObservationStore(t)
|
||||
defer cleanup()
|
||||
|
||||
// FTS5 tables are created by testObservationStore via testutil.CreateAllTables
|
||||
ctx := context.Background()
|
||||
|
||||
// Create 20 observations with similar content
|
||||
for i := 0; i < 20; i++ {
|
||||
obs := &models.ParsedObservation{
|
||||
Type: models.ObsTypeDiscovery,
|
||||
Title: "Testing observation " + string(rune('A'+i)),
|
||||
Narrative: "This is about testing and quality assurance " + string(rune('A'+i)),
|
||||
}
|
||||
_, _, err := obsStore.StoreObservation(ctx, "session-1", "project-a", obs, 1, 100)
|
||||
require.NoError(t, err)
|
||||
time.Sleep(time.Millisecond)
|
||||
}
|
||||
|
||||
// Search with limit 5
|
||||
results, err := obsStore.SearchObservationsFTS(ctx, "testing quality", "project-a", 5)
|
||||
require.NoError(t, err)
|
||||
assert.LessOrEqual(t, len(results), 5, "should respect limit of 5")
|
||||
|
||||
// Search with limit 15
|
||||
results, err = obsStore.SearchObservationsFTS(ctx, "testing quality", "project-a", 15)
|
||||
require.NoError(t, err)
|
||||
assert.LessOrEqual(t, len(results), 15, "should respect limit of 15")
|
||||
|
||||
// Search with limit 50 (our new default)
|
||||
results, err = obsStore.SearchObservationsFTS(ctx, "testing quality", "project-a", 50)
|
||||
require.NoError(t, err)
|
||||
assert.LessOrEqual(t, len(results), 50, "should respect limit of 50")
|
||||
assert.Equal(t, 20, len(results), "should return all 20 matching observations")
|
||||
}
|
||||
|
||||
func TestObservationStore_GlobalScope(t *testing.T) {
|
||||
obsStore, _, cleanup := testObservationStore(t)
|
||||
defer cleanup()
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
// Create a project-scoped observation
|
||||
projectObs := &models.ParsedObservation{
|
||||
Type: models.ObsTypeDiscovery,
|
||||
Title: "Project specific code",
|
||||
Narrative: "This is specific to project-a",
|
||||
Concepts: []string{"project-specific"},
|
||||
}
|
||||
_, _, err := obsStore.StoreObservation(ctx, "session-1", "project-a", projectObs, 1, 100)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Create a global-scoped observation (has a globalizable concept)
|
||||
globalObs := &models.ParsedObservation{
|
||||
Type: models.ObsTypeDiscovery,
|
||||
Title: "Security best practice",
|
||||
Narrative: "Always validate user input",
|
||||
Concepts: []string{"security", "best-practice"}, // "security" is in GlobalizableConcepts
|
||||
}
|
||||
_, _, err = obsStore.StoreObservation(ctx, "session-1", "project-a", globalObs, 1, 100)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Get recent for project-a - should see both
|
||||
results, err := obsStore.GetRecentObservations(ctx, "project-a", 10)
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, results, 2)
|
||||
|
||||
// Get recent for project-b - should only see global observation
|
||||
results, err = obsStore.GetRecentObservations(ctx, "project-b", 10)
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, results, 1)
|
||||
assert.Equal(t, "Security best practice", results[0].Title.String)
|
||||
assert.Equal(t, models.ScopeGlobal, results[0].Scope)
|
||||
}
|
||||
|
||||
func TestObservationStore_DeleteObservations(t *testing.T) {
|
||||
obsStore, _, cleanup := testObservationStore(t)
|
||||
defer cleanup()
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
// Create observations
|
||||
var ids []int64
|
||||
for i := 0; i < 5; i++ {
|
||||
obs := &models.ParsedObservation{
|
||||
Type: models.ObsTypeDiscovery,
|
||||
Title: "Observation " + string(rune('A'+i)),
|
||||
}
|
||||
id, _, err := obsStore.StoreObservation(ctx, "session-1", "project-a", obs, 1, 100)
|
||||
require.NoError(t, err)
|
||||
ids = append(ids, id)
|
||||
}
|
||||
|
||||
// Verify all exist
|
||||
all, err := obsStore.GetRecentObservations(ctx, "project-a", 10)
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, all, 5)
|
||||
|
||||
// Delete first 3
|
||||
deleted, err := obsStore.DeleteObservations(ctx, ids[:3])
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, int64(3), deleted)
|
||||
|
||||
// Verify only 2 remain
|
||||
remaining, err := obsStore.GetRecentObservations(ctx, "project-a", 10)
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, remaining, 2)
|
||||
}
|
||||
|
||||
func TestObservationStore_GetObservationCount(t *testing.T) {
|
||||
obsStore, _, cleanup := testObservationStore(t)
|
||||
defer cleanup()
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
// Create observations for different projects
|
||||
for i := 0; i < 5; i++ {
|
||||
obs := &models.ParsedObservation{
|
||||
Type: models.ObsTypeDiscovery,
|
||||
Title: "Project A observation " + string(rune('0'+i)),
|
||||
}
|
||||
_, _, err := obsStore.StoreObservation(ctx, "session-1", "project-a", obs, 1, 100)
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
for i := 0; i < 3; i++ {
|
||||
obs := &models.ParsedObservation{
|
||||
Type: models.ObsTypeDiscovery,
|
||||
Title: "Project B observation " + string(rune('0'+i)),
|
||||
}
|
||||
_, _, err := obsStore.StoreObservation(ctx, "session-1", "project-b", obs, 1, 100)
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
// Create a global observation
|
||||
globalObs := &models.ParsedObservation{
|
||||
Type: models.ObsTypeDiscovery,
|
||||
Title: "Global observation",
|
||||
Concepts: []string{"best-practice"}, // Makes it global
|
||||
}
|
||||
_, _, err := obsStore.StoreObservation(ctx, "session-1", "project-a", globalObs, 1, 100)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Count for project-a includes its own + global
|
||||
count, err := obsStore.GetObservationCount(ctx, "project-a")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 6, count) // 5 project-a + 1 global
|
||||
|
||||
// Count for project-b includes its own + global
|
||||
count, err = obsStore.GetObservationCount(ctx, "project-b")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 4, count) // 3 project-b + 1 global
|
||||
}
|
||||
|
||||
func TestObservationStore_CleanupOldObservations(t *testing.T) {
|
||||
obsStore, _, cleanup := testObservationStore(t)
|
||||
defer cleanup()
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
// Create more observations than the limit (MaxObservationsPerProject = 100)
|
||||
// We'll create a smaller number and verify the logic works
|
||||
for i := 0; i < 10; i++ {
|
||||
obs := &models.ParsedObservation{
|
||||
Type: models.ObsTypeDiscovery,
|
||||
Title: "Observation " + string(rune('A'+i)),
|
||||
}
|
||||
_, _, err := obsStore.StoreObservation(ctx, "session-1", "project-a", obs, i+1, 100)
|
||||
require.NoError(t, err)
|
||||
time.Sleep(time.Millisecond)
|
||||
}
|
||||
|
||||
// Cleanup should return empty since we're under the limit
|
||||
deletedIDs, err := obsStore.CleanupOldObservations(ctx, "project-a")
|
||||
require.NoError(t, err)
|
||||
assert.Empty(t, deletedIDs)
|
||||
|
||||
// All 10 should still exist
|
||||
count, err := obsStore.GetObservationCount(ctx, "project-a")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 10, count)
|
||||
}
|
||||
|
||||
func TestObservationStore_SetCleanupFunc(t *testing.T) {
|
||||
obsStore, _, cleanup := testObservationStore(t)
|
||||
defer cleanup()
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
// Track cleanup calls
|
||||
var cleanupCalledWith []int64
|
||||
obsStore.SetCleanupFunc(func(ctx context.Context, deletedIDs []int64) {
|
||||
cleanupCalledWith = deletedIDs
|
||||
})
|
||||
|
||||
// Store an observation (should trigger cleanup, but won't delete anything under limit)
|
||||
obs := &models.ParsedObservation{
|
||||
Type: models.ObsTypeDiscovery,
|
||||
Title: "Test observation",
|
||||
}
|
||||
_, _, err := obsStore.StoreObservation(ctx, "session-1", "project-a", obs, 1, 100)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Cleanup func should not have been called since nothing was deleted
|
||||
assert.Empty(t, cleanupCalledWith)
|
||||
}
|
||||
|
||||
func TestExtractKeywords(t *testing.T) {
|
||||
tests := []struct {
|
||||
query string
|
||||
expected []string
|
||||
}{
|
||||
{
|
||||
query: "What is the authentication flow?",
|
||||
expected: []string{"authentication", "flow"},
|
||||
},
|
||||
{
|
||||
query: "How does the database connection work?",
|
||||
expected: []string{"database", "connection"},
|
||||
},
|
||||
{
|
||||
query: "JWT token validation",
|
||||
expected: []string{"token", "validation"},
|
||||
},
|
||||
{
|
||||
query: "the a an is are", // All stop words
|
||||
expected: nil,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.query, func(t *testing.T) {
|
||||
keywords := extractKeywords(tt.query)
|
||||
for _, exp := range tt.expected {
|
||||
assert.Contains(t, keywords, exp, "should contain keyword: "+exp)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1,209 +0,0 @@
|
||||
// Package sqlite provides SQLite database operations for claude-mnemonic.
|
||||
package sqlite
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"github.com/lukaszraczylo/claude-mnemonic/pkg/models"
|
||||
)
|
||||
|
||||
// PromptCleanupFunc is a callback for when prompts are cleaned up.
|
||||
// Receives the IDs of deleted prompts for downstream cleanup (e.g., vector DB).
|
||||
type PromptCleanupFunc func(ctx context.Context, deletedIDs []int64)
|
||||
|
||||
// MaxPromptsGlobal is the hard limit of prompts across all projects.
|
||||
const MaxPromptsGlobal = 500
|
||||
|
||||
// PromptStore provides user prompt-related database operations.
|
||||
type PromptStore struct {
|
||||
store *Store
|
||||
cleanupFunc PromptCleanupFunc
|
||||
}
|
||||
|
||||
// NewPromptStore creates a new prompt store.
|
||||
func NewPromptStore(store *Store) *PromptStore {
|
||||
return &PromptStore{store: store}
|
||||
}
|
||||
|
||||
// SetCleanupFunc sets the callback for when prompts are deleted during cleanup.
|
||||
func (s *PromptStore) SetCleanupFunc(fn PromptCleanupFunc) {
|
||||
s.cleanupFunc = fn
|
||||
}
|
||||
|
||||
// SaveUserPromptWithMatches saves a user prompt with matched observation count.
|
||||
func (s *PromptStore) SaveUserPromptWithMatches(ctx context.Context, claudeSessionID string, promptNumber int, promptText string, matchedObservations int) (int64, error) {
|
||||
now := time.Now()
|
||||
|
||||
const query = `
|
||||
INSERT INTO user_prompts
|
||||
(claude_session_id, prompt_number, prompt_text, matched_observations, created_at, created_at_epoch)
|
||||
VALUES (?, ?, ?, ?, ?, ?)
|
||||
`
|
||||
|
||||
result, err := s.store.ExecContext(ctx, query,
|
||||
claudeSessionID, promptNumber, promptText, matchedObservations,
|
||||
now.Format(time.RFC3339), now.UnixMilli(),
|
||||
)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
id, _ := result.LastInsertId()
|
||||
|
||||
// Cleanup old prompts beyond the global limit (async to not block handler)
|
||||
go func() {
|
||||
cleanupCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
deletedIDs, _ := s.CleanupOldPrompts(cleanupCtx)
|
||||
if len(deletedIDs) > 0 && s.cleanupFunc != nil {
|
||||
s.cleanupFunc(cleanupCtx, deletedIDs)
|
||||
}
|
||||
}()
|
||||
|
||||
return id, nil
|
||||
}
|
||||
|
||||
// CleanupOldPrompts deletes prompts beyond the global limit.
|
||||
// Keeps the most recent MaxPromptsGlobal prompts.
|
||||
// Returns the IDs of deleted prompts for downstream cleanup (e.g., vector DB).
|
||||
func (s *PromptStore) CleanupOldPrompts(ctx context.Context) ([]int64, error) {
|
||||
// First, find IDs that will be deleted
|
||||
const selectQuery = `
|
||||
SELECT id FROM user_prompts
|
||||
WHERE id NOT IN (
|
||||
SELECT id FROM user_prompts
|
||||
ORDER BY created_at_epoch DESC
|
||||
LIMIT ?
|
||||
)
|
||||
`
|
||||
|
||||
rows, err := s.store.QueryContext(ctx, selectQuery, MaxPromptsGlobal)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var toDelete []int64
|
||||
for rows.Next() {
|
||||
var id int64
|
||||
if err := rows.Scan(&id); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
toDelete = append(toDelete, id)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if len(toDelete) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// Delete the prompts
|
||||
const deleteQuery = `
|
||||
DELETE FROM user_prompts
|
||||
WHERE id NOT IN (
|
||||
SELECT id FROM user_prompts
|
||||
ORDER BY created_at_epoch DESC
|
||||
LIMIT ?
|
||||
)
|
||||
`
|
||||
|
||||
_, err = s.store.ExecContext(ctx, deleteQuery, MaxPromptsGlobal)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return toDelete, nil
|
||||
}
|
||||
|
||||
// GetPromptsByIDs retrieves user prompts by a list of IDs.
|
||||
func (s *PromptStore) GetPromptsByIDs(ctx context.Context, ids []int64, orderBy string, limit int) ([]*models.UserPromptWithSession, error) {
|
||||
if len(ids) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// Build query with placeholders
|
||||
// #nosec G202 -- query uses parameterized placeholders, not user input
|
||||
query := `
|
||||
SELECT up.id, up.claude_session_id, up.prompt_number, up.prompt_text,
|
||||
COALESCE(up.matched_observations, 0) as matched_observations,
|
||||
up.created_at, up.created_at_epoch,
|
||||
COALESCE(s.project, '') as project,
|
||||
COALESCE(s.sdk_session_id, '') as sdk_session_id
|
||||
FROM user_prompts up
|
||||
LEFT JOIN sdk_sessions s ON up.claude_session_id = s.claude_session_id
|
||||
WHERE up.id IN (?` + repeatPlaceholders(len(ids)-1) + `)
|
||||
ORDER BY up.created_at_epoch `
|
||||
|
||||
if orderBy == "date_asc" {
|
||||
query += "ASC"
|
||||
} else {
|
||||
query += "DESC"
|
||||
}
|
||||
|
||||
if limit > 0 {
|
||||
query += " LIMIT ?"
|
||||
}
|
||||
|
||||
args := int64SliceToInterface(ids)
|
||||
if limit > 0 {
|
||||
args = append(args, limit)
|
||||
}
|
||||
|
||||
rows, err := s.store.db.QueryContext(ctx, query, args...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
return scanPromptWithSessionRows(rows)
|
||||
}
|
||||
|
||||
// GetAllRecentUserPrompts retrieves recent user prompts across all sessions.
|
||||
func (s *PromptStore) GetAllRecentUserPrompts(ctx context.Context, limit int) ([]*models.UserPromptWithSession, error) {
|
||||
const query = `
|
||||
SELECT up.id, up.claude_session_id, up.prompt_number, up.prompt_text,
|
||||
COALESCE(up.matched_observations, 0) as matched_observations,
|
||||
up.created_at, up.created_at_epoch,
|
||||
COALESCE(s.project, '') as project,
|
||||
COALESCE(s.sdk_session_id, '') as sdk_session_id
|
||||
FROM user_prompts up
|
||||
LEFT JOIN sdk_sessions s ON up.claude_session_id = s.claude_session_id
|
||||
ORDER BY up.created_at_epoch DESC
|
||||
LIMIT ?
|
||||
`
|
||||
|
||||
rows, err := s.store.QueryContext(ctx, query, limit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
return scanPromptWithSessionRows(rows)
|
||||
}
|
||||
|
||||
// GetRecentUserPromptsByProject retrieves recent user prompts for a specific project.
|
||||
func (s *PromptStore) GetRecentUserPromptsByProject(ctx context.Context, project string, limit int) ([]*models.UserPromptWithSession, error) {
|
||||
const query = `
|
||||
SELECT up.id, up.claude_session_id, up.prompt_number, up.prompt_text,
|
||||
COALESCE(up.matched_observations, 0) as matched_observations,
|
||||
up.created_at, up.created_at_epoch,
|
||||
COALESCE(s.project, '') as project,
|
||||
COALESCE(s.sdk_session_id, '') as sdk_session_id
|
||||
FROM user_prompts up
|
||||
LEFT JOIN sdk_sessions s ON up.claude_session_id = s.claude_session_id
|
||||
WHERE s.project = ?
|
||||
ORDER BY up.created_at_epoch DESC
|
||||
LIMIT ?
|
||||
`
|
||||
|
||||
rows, err := s.store.QueryContext(ctx, query, project, limit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
return scanPromptWithSessionRows(rows)
|
||||
}
|
||||
@@ -1,196 +0,0 @@
|
||||
package sqlite
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func testPromptStore(t *testing.T) (*PromptStore, *Store, func()) {
|
||||
t.Helper()
|
||||
|
||||
db, _, cleanup := testDB(t)
|
||||
createAllTables(t, db)
|
||||
|
||||
store := newStoreFromDB(db)
|
||||
promptStore := NewPromptStore(store)
|
||||
|
||||
return promptStore, store, cleanup
|
||||
}
|
||||
|
||||
func TestPromptStore_SaveUserPromptWithMatches(t *testing.T) {
|
||||
promptStore, store, cleanup := testPromptStore(t)
|
||||
defer cleanup()
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
// Create a session first
|
||||
seedSession(t, storeDB(store), "claude-1", "sdk-1", "test-project")
|
||||
|
||||
// Save a prompt
|
||||
id, err := promptStore.SaveUserPromptWithMatches(ctx, "claude-1", 1, "Help me fix this bug", 5)
|
||||
require.NoError(t, err)
|
||||
assert.Greater(t, id, int64(0))
|
||||
|
||||
// Verify it was saved
|
||||
var count int
|
||||
err = storeDB(store).QueryRow("SELECT COUNT(*) FROM user_prompts WHERE id = ?", id).Scan(&count)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 1, count)
|
||||
}
|
||||
|
||||
func TestPromptStore_GetAllRecentUserPrompts(t *testing.T) {
|
||||
promptStore, store, cleanup := testPromptStore(t)
|
||||
defer cleanup()
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
// Create a session
|
||||
seedSession(t, storeDB(store), "claude-1", "sdk-1", "test-project")
|
||||
|
||||
// Save multiple prompts
|
||||
for i := 1; i <= 5; i++ {
|
||||
_, err := promptStore.SaveUserPromptWithMatches(ctx, "claude-1", i, "Prompt "+string(rune('A'+i-1)), i)
|
||||
require.NoError(t, err)
|
||||
time.Sleep(time.Millisecond) // Ensure different timestamps
|
||||
}
|
||||
|
||||
// Get recent prompts
|
||||
prompts, err := promptStore.GetAllRecentUserPrompts(ctx, 3)
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, prompts, 3)
|
||||
|
||||
// Should be in descending order (most recent first)
|
||||
assert.Equal(t, 5, prompts[0].PromptNumber)
|
||||
}
|
||||
|
||||
func TestPromptStore_GetRecentUserPromptsByProject(t *testing.T) {
|
||||
promptStore, store, cleanup := testPromptStore(t)
|
||||
defer cleanup()
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
// Create sessions for different projects
|
||||
seedSession(t, storeDB(store), "claude-1", "sdk-1", "project-a")
|
||||
seedSession(t, storeDB(store), "claude-2", "sdk-2", "project-b")
|
||||
|
||||
// Save prompts for both projects
|
||||
for i := 1; i <= 3; i++ {
|
||||
_, err := promptStore.SaveUserPromptWithMatches(ctx, "claude-1", i, "Project A prompt", 0)
|
||||
require.NoError(t, err)
|
||||
}
|
||||
for i := 1; i <= 2; i++ {
|
||||
_, err := promptStore.SaveUserPromptWithMatches(ctx, "claude-2", i, "Project B prompt", 0)
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
// Get prompts for project-a
|
||||
prompts, err := promptStore.GetRecentUserPromptsByProject(ctx, "project-a", 10)
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, prompts, 3)
|
||||
|
||||
// Get prompts for project-b
|
||||
prompts, err = promptStore.GetRecentUserPromptsByProject(ctx, "project-b", 10)
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, prompts, 2)
|
||||
}
|
||||
|
||||
func TestPromptStore_CleanupOldPrompts(t *testing.T) {
|
||||
promptStore, store, cleanup := testPromptStore(t)
|
||||
defer cleanup()
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
// Create a session
|
||||
seedSession(t, storeDB(store), "claude-1", "sdk-1", "test-project")
|
||||
|
||||
// Save more prompts than the limit
|
||||
// Note: MaxPromptsGlobal is 500, but we'll test with a smaller number
|
||||
// by directly calling CleanupOldPrompts
|
||||
for i := 1; i <= 10; i++ {
|
||||
_, err := storeDB(store).Exec(`
|
||||
INSERT INTO user_prompts (claude_session_id, prompt_number, prompt_text, created_at, created_at_epoch)
|
||||
VALUES (?, ?, ?, datetime('now'), ?)
|
||||
`, "claude-1", i, "Prompt "+string(rune('A'+i-1)), time.Now().UnixMilli()+int64(i))
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
// Verify we have 10 prompts
|
||||
var count int
|
||||
err := storeDB(store).QueryRow("SELECT COUNT(*) FROM user_prompts").Scan(&count)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 10, count)
|
||||
|
||||
// Cleanup should return empty since we're under the limit
|
||||
deletedIDs, err := promptStore.CleanupOldPrompts(ctx)
|
||||
require.NoError(t, err)
|
||||
assert.Empty(t, deletedIDs)
|
||||
}
|
||||
|
||||
func TestPromptStore_SetCleanupFunc(t *testing.T) {
|
||||
promptStore, store, cleanup := testPromptStore(t)
|
||||
defer cleanup()
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
// Track cleanup calls
|
||||
var cleanupCalledWith []int64
|
||||
promptStore.SetCleanupFunc(func(ctx context.Context, deletedIDs []int64) {
|
||||
cleanupCalledWith = deletedIDs
|
||||
})
|
||||
|
||||
// Create a session
|
||||
seedSession(t, storeDB(store), "claude-1", "sdk-1", "test-project")
|
||||
|
||||
// Save a prompt (should trigger cleanup, but won't delete anything under limit)
|
||||
_, err := promptStore.SaveUserPromptWithMatches(ctx, "claude-1", 1, "Test prompt", 0)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Cleanup func should not have been called since nothing was deleted
|
||||
assert.Empty(t, cleanupCalledWith)
|
||||
}
|
||||
|
||||
func TestPromptStore_GetPromptsByIDs(t *testing.T) {
|
||||
promptStore, store, cleanup := testPromptStore(t)
|
||||
defer cleanup()
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
// Create a session
|
||||
seedSession(t, storeDB(store), "claude-1", "sdk-1", "test-project")
|
||||
|
||||
// Save some prompts and collect their IDs
|
||||
var ids []int64
|
||||
for i := 1; i <= 5; i++ {
|
||||
id, err := promptStore.SaveUserPromptWithMatches(ctx, "claude-1", i, "Prompt "+string(rune('A'+i-1)), 0)
|
||||
require.NoError(t, err)
|
||||
ids = append(ids, id)
|
||||
time.Sleep(time.Millisecond)
|
||||
}
|
||||
|
||||
// Get specific prompts by ID
|
||||
prompts, err := promptStore.GetPromptsByIDs(ctx, ids[:3], "date_desc", 10)
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, prompts, 3)
|
||||
|
||||
// Test with ascending order
|
||||
prompts, err = promptStore.GetPromptsByIDs(ctx, ids, "date_asc", 2)
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, prompts, 2)
|
||||
assert.Equal(t, 1, prompts[0].PromptNumber)
|
||||
}
|
||||
|
||||
func TestPromptStore_GetPromptsByIDs_EmptyInput(t *testing.T) {
|
||||
promptStore, _, cleanup := testPromptStore(t)
|
||||
defer cleanup()
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
// Empty IDs should return nil
|
||||
prompts, err := promptStore.GetPromptsByIDs(ctx, []int64{}, "date_desc", 10)
|
||||
require.NoError(t, err)
|
||||
assert.Nil(t, prompts)
|
||||
}
|
||||
@@ -1,184 +0,0 @@
|
||||
// Package sqlite provides SQLite database operations for claude-mnemonic.
|
||||
package sqlite
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"time"
|
||||
|
||||
"github.com/lukaszraczylo/claude-mnemonic/pkg/models"
|
||||
)
|
||||
|
||||
// SessionStore provides session-related database operations.
|
||||
type SessionStore struct {
|
||||
store *Store
|
||||
}
|
||||
|
||||
// NewSessionStore creates a new session store.
|
||||
func NewSessionStore(store *Store) *SessionStore {
|
||||
return &SessionStore{store: store}
|
||||
}
|
||||
|
||||
// CreateSDKSession creates a new SDK session (idempotent - returns existing ID if exists).
|
||||
// This is the KEY to how claude-mnemonic stays unified across hooks.
|
||||
func (s *SessionStore) CreateSDKSession(ctx context.Context, claudeSessionID, project, userPrompt string) (int64, error) {
|
||||
now := time.Now()
|
||||
|
||||
// CRITICAL: INSERT OR IGNORE makes this idempotent
|
||||
const query = `
|
||||
INSERT OR IGNORE INTO sdk_sessions
|
||||
(claude_session_id, sdk_session_id, project, user_prompt, started_at, started_at_epoch, status)
|
||||
VALUES (?, ?, ?, ?, ?, ?, 'active')
|
||||
`
|
||||
|
||||
result, err := s.store.ExecContext(ctx, query,
|
||||
claudeSessionID, claudeSessionID, project, userPrompt,
|
||||
now.Format(time.RFC3339), now.UnixMilli(),
|
||||
)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
// Check if insert happened
|
||||
rowsAffected, _ := result.RowsAffected()
|
||||
if rowsAffected == 0 {
|
||||
// Session exists - UPDATE project and user_prompt if we have non-empty values
|
||||
if project != "" {
|
||||
const updateQuery = `
|
||||
UPDATE sdk_sessions
|
||||
SET project = ?, user_prompt = ?
|
||||
WHERE claude_session_id = ?
|
||||
`
|
||||
_, _ = s.store.ExecContext(ctx, updateQuery, project, userPrompt, claudeSessionID)
|
||||
}
|
||||
|
||||
// Fetch existing ID
|
||||
var id int64
|
||||
const selectQuery = `SELECT id FROM sdk_sessions WHERE claude_session_id = ? LIMIT 1`
|
||||
err := s.store.QueryRowContext(ctx, selectQuery, claudeSessionID).Scan(&id)
|
||||
return id, err
|
||||
}
|
||||
|
||||
id, _ := result.LastInsertId()
|
||||
return id, nil
|
||||
}
|
||||
|
||||
// GetSessionByID retrieves a session by its database ID.
|
||||
func (s *SessionStore) GetSessionByID(ctx context.Context, id int64) (*models.SDKSession, error) {
|
||||
const query = `
|
||||
SELECT id, claude_session_id, sdk_session_id, project, user_prompt,
|
||||
worker_port, prompt_counter, status, started_at, started_at_epoch,
|
||||
completed_at, completed_at_epoch
|
||||
FROM sdk_sessions
|
||||
WHERE id = ?
|
||||
LIMIT 1
|
||||
`
|
||||
|
||||
var sess models.SDKSession
|
||||
err := s.store.QueryRowContext(ctx, query, id).Scan(
|
||||
&sess.ID, &sess.ClaudeSessionID, &sess.SDKSessionID, &sess.Project, &sess.UserPrompt,
|
||||
&sess.WorkerPort, &sess.PromptCounter, &sess.Status, &sess.StartedAt, &sess.StartedAtEpoch,
|
||||
&sess.CompletedAt, &sess.CompletedAtEpoch,
|
||||
)
|
||||
if err == sql.ErrNoRows {
|
||||
return nil, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &sess, nil
|
||||
}
|
||||
|
||||
// FindAnySDKSession finds any session by Claude session ID (any status).
|
||||
func (s *SessionStore) FindAnySDKSession(ctx context.Context, claudeSessionID string) (*models.SDKSession, error) {
|
||||
const query = `
|
||||
SELECT id, claude_session_id, sdk_session_id, project, user_prompt,
|
||||
worker_port, prompt_counter, status, started_at, started_at_epoch,
|
||||
completed_at, completed_at_epoch
|
||||
FROM sdk_sessions
|
||||
WHERE claude_session_id = ?
|
||||
LIMIT 1
|
||||
`
|
||||
|
||||
var sess models.SDKSession
|
||||
err := s.store.QueryRowContext(ctx, query, claudeSessionID).Scan(
|
||||
&sess.ID, &sess.ClaudeSessionID, &sess.SDKSessionID, &sess.Project, &sess.UserPrompt,
|
||||
&sess.WorkerPort, &sess.PromptCounter, &sess.Status, &sess.StartedAt, &sess.StartedAtEpoch,
|
||||
&sess.CompletedAt, &sess.CompletedAtEpoch,
|
||||
)
|
||||
if err == sql.ErrNoRows {
|
||||
return nil, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &sess, nil
|
||||
}
|
||||
|
||||
// IncrementPromptCounter increments the prompt counter and returns the new value.
|
||||
func (s *SessionStore) IncrementPromptCounter(ctx context.Context, id int64) (int, error) {
|
||||
const updateQuery = `
|
||||
UPDATE sdk_sessions
|
||||
SET prompt_counter = COALESCE(prompt_counter, 0) + 1
|
||||
WHERE id = ?
|
||||
`
|
||||
if _, err := s.store.ExecContext(ctx, updateQuery, id); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
const selectQuery = `SELECT prompt_counter FROM sdk_sessions WHERE id = ?`
|
||||
var counter int
|
||||
err := s.store.QueryRowContext(ctx, selectQuery, id).Scan(&counter)
|
||||
return counter, err
|
||||
}
|
||||
|
||||
// GetPromptCounter returns the current prompt counter for a session.
|
||||
func (s *SessionStore) GetPromptCounter(ctx context.Context, id int64) (int, error) {
|
||||
const query = `SELECT COALESCE(prompt_counter, 0) FROM sdk_sessions WHERE id = ?`
|
||||
var counter int
|
||||
err := s.store.QueryRowContext(ctx, query, id).Scan(&counter)
|
||||
return counter, err
|
||||
}
|
||||
|
||||
// GetSessionsToday returns the count of sessions started today.
|
||||
func (s *SessionStore) GetSessionsToday(ctx context.Context) (int, error) {
|
||||
// Get start of today in milliseconds
|
||||
now := time.Now()
|
||||
startOfDay := time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, now.Location())
|
||||
startEpoch := startOfDay.UnixMilli()
|
||||
|
||||
const query = `SELECT COUNT(*) FROM sdk_sessions WHERE started_at_epoch >= ?`
|
||||
|
||||
var count int
|
||||
err := s.store.QueryRowContext(ctx, query, startEpoch).Scan(&count)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return count, nil
|
||||
}
|
||||
|
||||
// GetAllProjects returns all unique project names.
|
||||
func (s *SessionStore) GetAllProjects(ctx context.Context) ([]string, error) {
|
||||
const query = `
|
||||
SELECT DISTINCT project
|
||||
FROM sdk_sessions
|
||||
WHERE project IS NOT NULL AND project != ''
|
||||
ORDER BY project ASC
|
||||
`
|
||||
|
||||
rows, err := s.store.QueryContext(ctx, query)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var projects []string
|
||||
for rows.Next() {
|
||||
var project string
|
||||
if err := rows.Scan(&project); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
projects = append(projects, project)
|
||||
}
|
||||
return projects, rows.Err()
|
||||
}
|
||||
@@ -1,149 +0,0 @@
|
||||
// Package sqlite provides SQLite database operations for claude-mnemonic.
|
||||
package sqlite
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"sync"
|
||||
|
||||
sqlite_vec "github.com/asg017/sqlite-vec-go-bindings/cgo"
|
||||
_ "github.com/mattn/go-sqlite3"
|
||||
)
|
||||
|
||||
// Store provides database operations with connection pooling and prepared statements.
|
||||
type Store struct {
|
||||
db *sql.DB
|
||||
stmtCache map[string]*sql.Stmt
|
||||
stmtMu sync.RWMutex
|
||||
}
|
||||
|
||||
// StoreConfig holds configuration for the database store.
|
||||
type StoreConfig struct {
|
||||
Path string
|
||||
MaxConns int
|
||||
WALMode bool
|
||||
}
|
||||
|
||||
// NewStore creates a new database store with the given configuration.
|
||||
func NewStore(cfg StoreConfig) (*Store, error) {
|
||||
// Register sqlite-vec extension for vector operations
|
||||
sqlite_vec.Auto()
|
||||
|
||||
// Build connection string with pragmas
|
||||
connStr := cfg.Path + "?_journal_mode=WAL&_synchronous=NORMAL&_foreign_keys=ON"
|
||||
|
||||
db, err := sql.Open("sqlite3", connStr)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("open database: %w", err)
|
||||
}
|
||||
|
||||
// Configure connection pool
|
||||
maxConns := cfg.MaxConns
|
||||
if maxConns <= 0 {
|
||||
maxConns = 4
|
||||
}
|
||||
db.SetMaxOpenConns(maxConns)
|
||||
db.SetMaxIdleConns(maxConns)
|
||||
db.SetConnMaxLifetime(0) // Never expire - SQLite connections are cheap
|
||||
|
||||
// Verify connection
|
||||
if err := db.Ping(); err != nil {
|
||||
_ = db.Close()
|
||||
return nil, fmt.Errorf("ping database: %w", err)
|
||||
}
|
||||
|
||||
store := &Store{
|
||||
db: db,
|
||||
stmtCache: make(map[string]*sql.Stmt),
|
||||
}
|
||||
|
||||
// Run migrations
|
||||
mgr := NewMigrationManager(db)
|
||||
if err := mgr.RunMigrations(); err != nil {
|
||||
_ = db.Close()
|
||||
return nil, fmt.Errorf("run migrations: %w", err)
|
||||
}
|
||||
|
||||
return store, nil
|
||||
}
|
||||
|
||||
// Close closes the database connection and all cached statements.
|
||||
func (s *Store) Close() error {
|
||||
s.stmtMu.Lock()
|
||||
defer s.stmtMu.Unlock()
|
||||
|
||||
for _, stmt := range s.stmtCache {
|
||||
_ = stmt.Close()
|
||||
}
|
||||
s.stmtCache = nil
|
||||
|
||||
return s.db.Close()
|
||||
}
|
||||
|
||||
// GetStmt returns a cached prepared statement, creating it if necessary.
|
||||
func (s *Store) GetStmt(query string) (*sql.Stmt, error) {
|
||||
s.stmtMu.RLock()
|
||||
stmt, ok := s.stmtCache[query]
|
||||
s.stmtMu.RUnlock()
|
||||
if ok {
|
||||
return stmt, nil
|
||||
}
|
||||
|
||||
s.stmtMu.Lock()
|
||||
defer s.stmtMu.Unlock()
|
||||
|
||||
// Double-check after acquiring write lock
|
||||
if stmt, ok := s.stmtCache[query]; ok {
|
||||
return stmt, nil
|
||||
}
|
||||
|
||||
stmt, err := s.db.Prepare(query)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
s.stmtCache[query] = stmt
|
||||
return stmt, nil
|
||||
}
|
||||
|
||||
// ExecContext executes a query that doesn't return rows.
|
||||
func (s *Store) ExecContext(ctx context.Context, query string, args ...interface{}) (sql.Result, error) {
|
||||
stmt, err := s.GetStmt(query)
|
||||
if err != nil {
|
||||
// Fall back to direct execution
|
||||
return s.db.ExecContext(ctx, query, args...)
|
||||
}
|
||||
return stmt.ExecContext(ctx, args...)
|
||||
}
|
||||
|
||||
// QueryContext executes a query that returns rows.
|
||||
func (s *Store) QueryContext(ctx context.Context, query string, args ...interface{}) (*sql.Rows, error) {
|
||||
stmt, err := s.GetStmt(query)
|
||||
if err != nil {
|
||||
// Fall back to direct execution
|
||||
return s.db.QueryContext(ctx, query, args...)
|
||||
}
|
||||
return stmt.QueryContext(ctx, args...)
|
||||
}
|
||||
|
||||
// QueryRowContext executes a query that returns a single row.
|
||||
func (s *Store) QueryRowContext(ctx context.Context, query string, args ...interface{}) *sql.Row {
|
||||
stmt, err := s.GetStmt(query)
|
||||
if err != nil {
|
||||
// Fall back to direct execution
|
||||
return s.db.QueryRowContext(ctx, query, args...)
|
||||
}
|
||||
return stmt.QueryRowContext(ctx, args...)
|
||||
}
|
||||
|
||||
// Ping checks if the database connection is alive.
|
||||
func (s *Store) Ping() error {
|
||||
return s.db.Ping()
|
||||
}
|
||||
|
||||
// DB returns the underlying database connection for direct access.
|
||||
// Use this sparingly - prefer the store methods for most operations.
|
||||
func (s *Store) DB() *sql.DB {
|
||||
return s.db
|
||||
}
|
||||
@@ -1,118 +0,0 @@
|
||||
// Package sqlite provides SQLite database operations for claude-mnemonic.
|
||||
package sqlite
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"github.com/lukaszraczylo/claude-mnemonic/pkg/models"
|
||||
)
|
||||
|
||||
// SummaryStore provides summary-related database operations.
|
||||
type SummaryStore struct {
|
||||
store *Store
|
||||
}
|
||||
|
||||
// NewSummaryStore creates a new summary store.
|
||||
func NewSummaryStore(store *Store) *SummaryStore {
|
||||
return &SummaryStore{store: store}
|
||||
}
|
||||
|
||||
// StoreSummary stores a new session summary.
|
||||
func (s *SummaryStore) StoreSummary(ctx context.Context, sdkSessionID, project string, summary *models.ParsedSummary, promptNumber int, discoveryTokens int64) (int64, int64, error) {
|
||||
now := time.Now()
|
||||
nowEpoch := now.UnixMilli()
|
||||
|
||||
// Ensure session exists (auto-create if missing)
|
||||
if err := s.ensureSessionExists(ctx, sdkSessionID, project); err != nil {
|
||||
return 0, 0, err
|
||||
}
|
||||
|
||||
const query = `
|
||||
INSERT INTO session_summaries
|
||||
(sdk_session_id, project, request, investigated, learned, completed,
|
||||
next_steps, notes, prompt_number, discovery_tokens, created_at, created_at_epoch)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
`
|
||||
|
||||
result, err := s.store.ExecContext(ctx, query,
|
||||
sdkSessionID, project,
|
||||
nullString(summary.Request), nullString(summary.Investigated),
|
||||
nullString(summary.Learned), nullString(summary.Completed),
|
||||
nullString(summary.NextSteps), nullString(summary.Notes),
|
||||
nullInt(promptNumber), discoveryTokens,
|
||||
now.Format(time.RFC3339), nowEpoch,
|
||||
)
|
||||
if err != nil {
|
||||
return 0, 0, err
|
||||
}
|
||||
|
||||
id, _ := result.LastInsertId()
|
||||
return id, nowEpoch, nil
|
||||
}
|
||||
|
||||
// ensureSessionExists creates a session if it doesn't exist.
|
||||
func (s *SummaryStore) ensureSessionExists(ctx context.Context, sdkSessionID, project string) error {
|
||||
return EnsureSessionExists(ctx, s.store, sdkSessionID, project)
|
||||
}
|
||||
|
||||
// GetSummariesByIDs retrieves summaries by a list of IDs.
|
||||
func (s *SummaryStore) GetSummariesByIDs(ctx context.Context, ids []int64, orderBy string, limit int) ([]*models.SessionSummary, error) {
|
||||
if len(ids) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
const baseQuery = `
|
||||
SELECT id, sdk_session_id, project, request, investigated, learned, completed,
|
||||
next_steps, notes, prompt_number, discovery_tokens, created_at, created_at_epoch
|
||||
FROM session_summaries`
|
||||
|
||||
query, args := BuildGetByIDsQuery(baseQuery, ids, orderBy, limit)
|
||||
|
||||
rows, err := s.store.db.QueryContext(ctx, query, args...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
return scanSummaryRows(rows)
|
||||
}
|
||||
|
||||
// GetRecentSummaries retrieves recent summaries for a project.
|
||||
func (s *SummaryStore) GetRecentSummaries(ctx context.Context, project string, limit int) ([]*models.SessionSummary, error) {
|
||||
const query = `
|
||||
SELECT id, sdk_session_id, project, request, investigated, learned, completed,
|
||||
next_steps, notes, prompt_number, discovery_tokens, created_at, created_at_epoch
|
||||
FROM session_summaries
|
||||
WHERE project = ?
|
||||
ORDER BY created_at_epoch DESC
|
||||
LIMIT ?
|
||||
`
|
||||
|
||||
rows, err := s.store.QueryContext(ctx, query, project, limit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
return scanSummaryRows(rows)
|
||||
}
|
||||
|
||||
// GetAllRecentSummaries retrieves recent summaries across all projects.
|
||||
func (s *SummaryStore) GetAllRecentSummaries(ctx context.Context, limit int) ([]*models.SessionSummary, error) {
|
||||
const query = `
|
||||
SELECT id, sdk_session_id, project, request, investigated, learned, completed,
|
||||
next_steps, notes, prompt_number, discovery_tokens, created_at, created_at_epoch
|
||||
FROM session_summaries
|
||||
ORDER BY created_at_epoch DESC
|
||||
LIMIT ?
|
||||
`
|
||||
|
||||
rows, err := s.store.QueryContext(ctx, query, limit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
return scanSummaryRows(rows)
|
||||
}
|
||||
@@ -1,242 +0,0 @@
|
||||
package sqlite
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/lukaszraczylo/claude-mnemonic/pkg/models"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func testSummaryStore(t *testing.T) (*SummaryStore, *Store, func()) {
|
||||
t.Helper()
|
||||
|
||||
db, _, cleanup := testDB(t)
|
||||
createAllTables(t, db)
|
||||
|
||||
store := newStoreFromDB(db)
|
||||
summaryStore := NewSummaryStore(store)
|
||||
|
||||
return summaryStore, store, cleanup
|
||||
}
|
||||
|
||||
func TestSummaryStore_StoreSummary(t *testing.T) {
|
||||
summaryStore, store, cleanup := testSummaryStore(t)
|
||||
defer cleanup()
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
// Create a session first
|
||||
seedSession(t, storeDB(store), "claude-1", "sdk-1", "test-project")
|
||||
|
||||
summary := &models.ParsedSummary{
|
||||
Request: "Add new feature",
|
||||
Investigated: "Looked at existing code",
|
||||
Learned: "Found the pattern to follow",
|
||||
Completed: "Implemented the feature",
|
||||
NextSteps: "Add tests",
|
||||
Notes: "Some additional notes",
|
||||
}
|
||||
|
||||
id, epoch, err := summaryStore.StoreSummary(ctx, "sdk-1", "test-project", summary, 1, 100)
|
||||
require.NoError(t, err)
|
||||
assert.Greater(t, id, int64(0))
|
||||
assert.Greater(t, epoch, int64(0))
|
||||
|
||||
// Verify it was saved
|
||||
var count int
|
||||
err = storeDB(store).QueryRow("SELECT COUNT(*) FROM session_summaries WHERE id = ?", id).Scan(&count)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 1, count)
|
||||
}
|
||||
|
||||
func TestSummaryStore_StoreSummary_AutoCreateSession(t *testing.T) {
|
||||
summaryStore, store, cleanup := testSummaryStore(t)
|
||||
defer cleanup()
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
// Don't create session beforehand - should be auto-created
|
||||
summary := &models.ParsedSummary{
|
||||
Request: "Test request",
|
||||
}
|
||||
|
||||
id, _, err := summaryStore.StoreSummary(ctx, "auto-session", "test-project", summary, 1, 0)
|
||||
require.NoError(t, err)
|
||||
assert.Greater(t, id, int64(0))
|
||||
|
||||
// Verify session was auto-created
|
||||
var sessionCount int
|
||||
err = storeDB(store).QueryRow("SELECT COUNT(*) FROM sdk_sessions WHERE sdk_session_id = ?", "auto-session").Scan(&sessionCount)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 1, sessionCount)
|
||||
}
|
||||
|
||||
func TestSummaryStore_GetRecentSummaries(t *testing.T) {
|
||||
summaryStore, store, cleanup := testSummaryStore(t)
|
||||
defer cleanup()
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
// Create a session
|
||||
seedSession(t, storeDB(store), "claude-1", "sdk-1", "test-project")
|
||||
|
||||
// Store multiple summaries
|
||||
for i := 0; i < 5; i++ {
|
||||
summary := &models.ParsedSummary{
|
||||
Request: "Request " + string(rune('A'+i)),
|
||||
}
|
||||
_, _, err := summaryStore.StoreSummary(ctx, "sdk-1", "test-project", summary, i+1, 0)
|
||||
require.NoError(t, err)
|
||||
time.Sleep(time.Millisecond) // Ensure different timestamps
|
||||
}
|
||||
|
||||
// Get recent summaries with limit
|
||||
summaries, err := summaryStore.GetRecentSummaries(ctx, "test-project", 3)
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, summaries, 3)
|
||||
|
||||
// Should be in descending order
|
||||
assert.Equal(t, int64(5), summaries[0].PromptNumber.Int64)
|
||||
}
|
||||
|
||||
func TestSummaryStore_GetAllRecentSummaries(t *testing.T) {
|
||||
summaryStore, store, cleanup := testSummaryStore(t)
|
||||
defer cleanup()
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
// Create sessions for different projects
|
||||
seedSession(t, storeDB(store), "claude-1", "sdk-1", "project-a")
|
||||
seedSession(t, storeDB(store), "claude-2", "sdk-2", "project-b")
|
||||
|
||||
// Store summaries for both projects
|
||||
for i := 0; i < 3; i++ {
|
||||
summary := &models.ParsedSummary{Request: "Project A request"}
|
||||
_, _, err := summaryStore.StoreSummary(ctx, "sdk-1", "project-a", summary, i+1, 0)
|
||||
require.NoError(t, err)
|
||||
}
|
||||
for i := 0; i < 2; i++ {
|
||||
summary := &models.ParsedSummary{Request: "Project B request"}
|
||||
_, _, err := summaryStore.StoreSummary(ctx, "sdk-2", "project-b", summary, i+1, 0)
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
// Get all summaries (should include both projects)
|
||||
summaries, err := summaryStore.GetAllRecentSummaries(ctx, 10)
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, summaries, 5)
|
||||
}
|
||||
|
||||
func TestSummaryStore_GetSummariesByIDs(t *testing.T) {
|
||||
summaryStore, store, cleanup := testSummaryStore(t)
|
||||
defer cleanup()
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
// Create a session
|
||||
seedSession(t, storeDB(store), "claude-1", "sdk-1", "test-project")
|
||||
|
||||
// Store summaries and collect IDs
|
||||
var ids []int64
|
||||
for i := 0; i < 5; i++ {
|
||||
summary := &models.ParsedSummary{Request: "Request " + string(rune('A'+i))}
|
||||
id, _, err := summaryStore.StoreSummary(ctx, "sdk-1", "test-project", summary, i+1, 0)
|
||||
require.NoError(t, err)
|
||||
ids = append(ids, id)
|
||||
time.Sleep(time.Millisecond)
|
||||
}
|
||||
|
||||
// Get specific summaries by ID
|
||||
summaries, err := summaryStore.GetSummariesByIDs(ctx, ids[:3], "date_desc", 10)
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, summaries, 3)
|
||||
|
||||
// Test with ascending order
|
||||
summaries, err = summaryStore.GetSummariesByIDs(ctx, ids, "date_asc", 2)
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, summaries, 2)
|
||||
assert.Equal(t, int64(1), summaries[0].PromptNumber.Int64)
|
||||
}
|
||||
|
||||
func TestSummaryStore_GetSummariesByIDs_EmptyInput(t *testing.T) {
|
||||
summaryStore, _, cleanup := testSummaryStore(t)
|
||||
defer cleanup()
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
// Empty IDs should return nil
|
||||
summaries, err := summaryStore.GetSummariesByIDs(ctx, []int64{}, "date_desc", 10)
|
||||
require.NoError(t, err)
|
||||
assert.Nil(t, summaries)
|
||||
}
|
||||
|
||||
func TestSummaryStore_SummaryFields(t *testing.T) {
|
||||
summaryStore, store, cleanup := testSummaryStore(t)
|
||||
defer cleanup()
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
// Create a session
|
||||
seedSession(t, storeDB(store), "claude-1", "sdk-1", "test-project")
|
||||
|
||||
// Store a summary with all fields
|
||||
summary := &models.ParsedSummary{
|
||||
Request: "Add authentication",
|
||||
Investigated: "Reviewed existing auth code",
|
||||
Learned: "OAuth is preferred",
|
||||
Completed: "Implemented OAuth flow",
|
||||
NextSteps: "Add refresh token support",
|
||||
Notes: "Consider rate limiting",
|
||||
}
|
||||
|
||||
id, _, err := summaryStore.StoreSummary(ctx, "sdk-1", "test-project", summary, 5, 1500)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Retrieve and verify all fields
|
||||
summaries, err := summaryStore.GetSummariesByIDs(ctx, []int64{id}, "date_desc", 1)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, summaries, 1)
|
||||
|
||||
s := summaries[0]
|
||||
assert.Equal(t, id, s.ID)
|
||||
assert.Equal(t, "sdk-1", s.SDKSessionID)
|
||||
assert.Equal(t, "test-project", s.Project)
|
||||
assert.Equal(t, "Add authentication", s.Request.String)
|
||||
assert.Equal(t, "Reviewed existing auth code", s.Investigated.String)
|
||||
assert.Equal(t, "OAuth is preferred", s.Learned.String)
|
||||
assert.Equal(t, "Implemented OAuth flow", s.Completed.String)
|
||||
assert.Equal(t, "Add refresh token support", s.NextSteps.String)
|
||||
assert.Equal(t, "Consider rate limiting", s.Notes.String)
|
||||
assert.Equal(t, int64(5), s.PromptNumber.Int64)
|
||||
assert.Equal(t, int64(1500), s.DiscoveryTokens)
|
||||
}
|
||||
|
||||
func TestSummaryStore_EmptySummary(t *testing.T) {
|
||||
summaryStore, store, cleanup := testSummaryStore(t)
|
||||
defer cleanup()
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
// Create a session
|
||||
seedSession(t, storeDB(store), "claude-1", "sdk-1", "test-project")
|
||||
|
||||
// Store an empty summary
|
||||
summary := &models.ParsedSummary{}
|
||||
|
||||
id, _, err := summaryStore.StoreSummary(ctx, "sdk-1", "test-project", summary, 0, 0)
|
||||
require.NoError(t, err)
|
||||
assert.Greater(t, id, int64(0))
|
||||
|
||||
// Retrieve and verify null fields
|
||||
summaries, err := summaryStore.GetSummariesByIDs(ctx, []int64{id}, "date_desc", 1)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, summaries, 1)
|
||||
|
||||
s := summaries[0]
|
||||
assert.False(t, s.Request.Valid || s.Request.String != "")
|
||||
assert.False(t, s.Investigated.Valid || s.Investigated.String != "")
|
||||
assert.False(t, s.Learned.Valid || s.Learned.String != "")
|
||||
}
|
||||
@@ -1,315 +0,0 @@
|
||||
package sqlite
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
_ "github.com/mattn/go-sqlite3"
|
||||
)
|
||||
|
||||
// newStoreFromDB creates a Store from an existing database connection for testing.
|
||||
func newStoreFromDB(db *sql.DB) *Store {
|
||||
return &Store{
|
||||
db: db,
|
||||
stmtCache: make(map[string]*sql.Stmt),
|
||||
}
|
||||
}
|
||||
|
||||
// storeDB returns the underlying database connection from a store for testing.
|
||||
func storeDB(s *Store) *sql.DB {
|
||||
return s.db
|
||||
}
|
||||
|
||||
// testDB creates a temporary SQLite database for testing.
|
||||
// Returns the database, path, and a cleanup function.
|
||||
func testDB(t *testing.T) (*sql.DB, string, func()) {
|
||||
t.Helper()
|
||||
|
||||
tmpDir, err := os.MkdirTemp("", "claude-mnemonic-test-*")
|
||||
if err != nil {
|
||||
t.Fatalf("create temp dir: %v", err)
|
||||
}
|
||||
|
||||
dbPath := tmpDir + "/test.db"
|
||||
connStr := dbPath + "?_journal_mode=WAL&_synchronous=NORMAL&_foreign_keys=ON"
|
||||
|
||||
db, err := sql.Open("sqlite3", connStr)
|
||||
if err != nil {
|
||||
_ = os.RemoveAll(tmpDir)
|
||||
t.Fatalf("open database: %v", err)
|
||||
}
|
||||
|
||||
cleanup := func() {
|
||||
_ = db.Close()
|
||||
_ = os.RemoveAll(tmpDir)
|
||||
}
|
||||
|
||||
return db, dbPath, cleanup
|
||||
}
|
||||
|
||||
// createBaseTables creates the base tables without FTS5 for unit testing.
|
||||
func createBaseTables(t *testing.T, db *sql.DB) {
|
||||
t.Helper()
|
||||
|
||||
_, err := db.Exec(`
|
||||
CREATE TABLE IF NOT EXISTS schema_versions (
|
||||
id INTEGER PRIMARY KEY,
|
||||
version INTEGER UNIQUE NOT NULL,
|
||||
applied_at TEXT NOT NULL
|
||||
)
|
||||
`)
|
||||
if err != nil {
|
||||
t.Fatalf("create schema_versions: %v", err)
|
||||
}
|
||||
|
||||
_, err = db.Exec(`
|
||||
CREATE TABLE IF NOT EXISTS sdk_sessions (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
claude_session_id TEXT UNIQUE NOT NULL,
|
||||
sdk_session_id TEXT UNIQUE,
|
||||
project TEXT NOT NULL,
|
||||
user_prompt TEXT,
|
||||
started_at TEXT NOT NULL,
|
||||
started_at_epoch INTEGER NOT NULL,
|
||||
completed_at TEXT,
|
||||
completed_at_epoch INTEGER,
|
||||
status TEXT CHECK(status IN ('active', 'completed', 'failed')) NOT NULL DEFAULT 'active',
|
||||
worker_port INTEGER,
|
||||
prompt_counter INTEGER DEFAULT 0
|
||||
)
|
||||
`)
|
||||
if err != nil {
|
||||
t.Fatalf("create sdk_sessions: %v", err)
|
||||
}
|
||||
|
||||
_, err = db.Exec(`
|
||||
CREATE TABLE IF NOT EXISTS observations (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
sdk_session_id TEXT NOT NULL,
|
||||
project TEXT NOT NULL,
|
||||
text TEXT,
|
||||
type TEXT NOT NULL CHECK(type IN ('decision', 'bugfix', 'feature', 'refactor', 'discovery', 'change')),
|
||||
title TEXT,
|
||||
subtitle TEXT,
|
||||
facts TEXT,
|
||||
narrative TEXT,
|
||||
concepts TEXT,
|
||||
files_read TEXT,
|
||||
files_modified TEXT,
|
||||
file_mtimes TEXT,
|
||||
scope TEXT DEFAULT 'project' CHECK(scope IN ('project', 'global')),
|
||||
prompt_number INTEGER,
|
||||
discovery_tokens INTEGER DEFAULT 0,
|
||||
created_at TEXT NOT NULL,
|
||||
created_at_epoch INTEGER NOT NULL,
|
||||
FOREIGN KEY(sdk_session_id) REFERENCES sdk_sessions(sdk_session_id) ON DELETE CASCADE
|
||||
)
|
||||
`)
|
||||
if err != nil {
|
||||
t.Fatalf("create observations: %v", err)
|
||||
}
|
||||
|
||||
_, err = db.Exec(`
|
||||
CREATE TABLE IF NOT EXISTS session_summaries (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
sdk_session_id TEXT NOT NULL,
|
||||
project TEXT NOT NULL,
|
||||
request TEXT,
|
||||
investigated TEXT,
|
||||
learned TEXT,
|
||||
completed TEXT,
|
||||
next_steps TEXT,
|
||||
files_read TEXT,
|
||||
files_edited TEXT,
|
||||
notes TEXT,
|
||||
prompt_number INTEGER,
|
||||
discovery_tokens INTEGER DEFAULT 0,
|
||||
created_at TEXT NOT NULL,
|
||||
created_at_epoch INTEGER NOT NULL,
|
||||
FOREIGN KEY(sdk_session_id) REFERENCES sdk_sessions(sdk_session_id) ON DELETE CASCADE
|
||||
)
|
||||
`)
|
||||
if err != nil {
|
||||
t.Fatalf("create session_summaries: %v", err)
|
||||
}
|
||||
|
||||
_, err = db.Exec(`
|
||||
CREATE TABLE IF NOT EXISTS user_prompts (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
claude_session_id TEXT NOT NULL,
|
||||
prompt_number INTEGER NOT NULL,
|
||||
prompt_text TEXT NOT NULL,
|
||||
matched_observations INTEGER DEFAULT 0,
|
||||
created_at TEXT NOT NULL,
|
||||
created_at_epoch INTEGER NOT NULL,
|
||||
FOREIGN KEY(claude_session_id) REFERENCES sdk_sessions(claude_session_id) ON DELETE CASCADE
|
||||
)
|
||||
`)
|
||||
if err != nil {
|
||||
t.Fatalf("create user_prompts: %v", err)
|
||||
}
|
||||
|
||||
indexes := []string{
|
||||
`CREATE INDEX IF NOT EXISTS idx_sdk_sessions_claude_id ON sdk_sessions(claude_session_id)`,
|
||||
`CREATE INDEX IF NOT EXISTS idx_sdk_sessions_sdk_id ON sdk_sessions(sdk_session_id)`,
|
||||
`CREATE INDEX IF NOT EXISTS idx_sdk_sessions_project ON sdk_sessions(project)`,
|
||||
`CREATE INDEX IF NOT EXISTS idx_observations_sdk_session ON observations(sdk_session_id)`,
|
||||
`CREATE INDEX IF NOT EXISTS idx_observations_project ON observations(project)`,
|
||||
`CREATE INDEX IF NOT EXISTS idx_observations_scope ON observations(scope)`,
|
||||
`CREATE INDEX IF NOT EXISTS idx_observations_created ON observations(created_at_epoch DESC)`,
|
||||
`CREATE INDEX IF NOT EXISTS idx_session_summaries_sdk_session ON session_summaries(sdk_session_id)`,
|
||||
`CREATE INDEX IF NOT EXISTS idx_session_summaries_project ON session_summaries(project)`,
|
||||
`CREATE INDEX IF NOT EXISTS idx_user_prompts_claude_session ON user_prompts(claude_session_id)`,
|
||||
`CREATE INDEX IF NOT EXISTS idx_user_prompts_created ON user_prompts(created_at_epoch DESC)`,
|
||||
}
|
||||
for _, idx := range indexes {
|
||||
if _, err := db.Exec(idx); err != nil {
|
||||
t.Fatalf("create index: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// seedSession creates a test session in the database.
|
||||
func seedSession(t *testing.T, db *sql.DB, claudeSessionID, sdkSessionID, project string) {
|
||||
t.Helper()
|
||||
|
||||
_, err := db.Exec(`
|
||||
INSERT INTO sdk_sessions (claude_session_id, sdk_session_id, project, started_at, started_at_epoch, status)
|
||||
VALUES (?, ?, ?, datetime('now'), strftime('%s', 'now') * 1000, 'active')
|
||||
`, claudeSessionID, sdkSessionID, project)
|
||||
if err != nil {
|
||||
t.Fatalf("seed session: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// hasFTS5 checks if FTS5 is available in the SQLite build.
|
||||
func hasFTS5(db *sql.DB) bool {
|
||||
_, err := db.Exec("CREATE VIRTUAL TABLE IF NOT EXISTS fts5_test USING fts5(content)")
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
_, _ = db.Exec("DROP TABLE IF EXISTS fts5_test")
|
||||
return true
|
||||
}
|
||||
|
||||
// createFTSTables creates FTS5 virtual tables and triggers for full-text search.
|
||||
func createFTSTables(t *testing.T, db *sql.DB) {
|
||||
t.Helper()
|
||||
|
||||
if !hasFTS5(db) {
|
||||
t.Skip("FTS5 not available in this SQLite build")
|
||||
}
|
||||
|
||||
_, err := db.Exec(`
|
||||
CREATE VIRTUAL TABLE IF NOT EXISTS observations_fts USING fts5(
|
||||
title, subtitle, narrative,
|
||||
content='observations',
|
||||
content_rowid='id'
|
||||
)
|
||||
`)
|
||||
if err != nil {
|
||||
t.Fatalf("create observations_fts: %v", err)
|
||||
}
|
||||
|
||||
_, err = db.Exec(`
|
||||
CREATE TRIGGER IF NOT EXISTS observations_ai AFTER INSERT ON observations BEGIN
|
||||
INSERT INTO observations_fts(rowid, title, subtitle, narrative)
|
||||
VALUES (new.id, new.title, new.subtitle, new.narrative);
|
||||
END
|
||||
`)
|
||||
if err != nil {
|
||||
t.Fatalf("create observations_ai trigger: %v", err)
|
||||
}
|
||||
|
||||
_, err = db.Exec(`
|
||||
CREATE TRIGGER IF NOT EXISTS observations_ad AFTER DELETE ON observations BEGIN
|
||||
INSERT INTO observations_fts(observations_fts, rowid, title, subtitle, narrative)
|
||||
VALUES ('delete', old.id, old.title, old.subtitle, old.narrative);
|
||||
END
|
||||
`)
|
||||
if err != nil {
|
||||
t.Fatalf("create observations_ad trigger: %v", err)
|
||||
}
|
||||
|
||||
_, err = db.Exec(`
|
||||
CREATE TRIGGER IF NOT EXISTS observations_au AFTER UPDATE ON observations BEGIN
|
||||
INSERT INTO observations_fts(observations_fts, rowid, title, subtitle, narrative)
|
||||
VALUES ('delete', old.id, old.title, old.subtitle, old.narrative);
|
||||
INSERT INTO observations_fts(rowid, title, subtitle, narrative)
|
||||
VALUES (new.id, new.title, new.subtitle, new.narrative);
|
||||
END
|
||||
`)
|
||||
if err != nil {
|
||||
t.Fatalf("create observations_au trigger: %v", err)
|
||||
}
|
||||
|
||||
_, err = db.Exec(`
|
||||
CREATE VIRTUAL TABLE IF NOT EXISTS session_summaries_fts USING fts5(
|
||||
request, investigated, learned, completed, next_steps, notes,
|
||||
content='session_summaries',
|
||||
content_rowid='id'
|
||||
)
|
||||
`)
|
||||
if err != nil {
|
||||
t.Fatalf("create session_summaries_fts: %v", err)
|
||||
}
|
||||
|
||||
_, err = db.Exec(`
|
||||
CREATE TRIGGER IF NOT EXISTS summaries_ai AFTER INSERT ON session_summaries BEGIN
|
||||
INSERT INTO session_summaries_fts(rowid, request, investigated, learned, completed, next_steps, notes)
|
||||
VALUES (new.id, new.request, new.investigated, new.learned, new.completed, new.next_steps, new.notes);
|
||||
END
|
||||
`)
|
||||
if err != nil {
|
||||
t.Fatalf("create summaries_ai trigger: %v", err)
|
||||
}
|
||||
|
||||
_, err = db.Exec(`
|
||||
CREATE TRIGGER IF NOT EXISTS summaries_ad AFTER DELETE ON session_summaries BEGIN
|
||||
INSERT INTO session_summaries_fts(session_summaries_fts, rowid, request, investigated, learned, completed, next_steps, notes)
|
||||
VALUES ('delete', old.id, old.request, old.investigated, old.learned, old.completed, old.next_steps, old.notes);
|
||||
END
|
||||
`)
|
||||
if err != nil {
|
||||
t.Fatalf("create summaries_ad trigger: %v", err)
|
||||
}
|
||||
|
||||
_, err = db.Exec(`
|
||||
CREATE VIRTUAL TABLE IF NOT EXISTS user_prompts_fts USING fts5(
|
||||
prompt_text,
|
||||
content='user_prompts',
|
||||
content_rowid='id'
|
||||
)
|
||||
`)
|
||||
if err != nil {
|
||||
t.Fatalf("create user_prompts_fts: %v", err)
|
||||
}
|
||||
|
||||
_, err = db.Exec(`
|
||||
CREATE TRIGGER IF NOT EXISTS prompts_ai AFTER INSERT ON user_prompts BEGIN
|
||||
INSERT INTO user_prompts_fts(rowid, prompt_text)
|
||||
VALUES (new.id, new.prompt_text);
|
||||
END
|
||||
`)
|
||||
if err != nil {
|
||||
t.Fatalf("create prompts_ai trigger: %v", err)
|
||||
}
|
||||
|
||||
_, err = db.Exec(`
|
||||
CREATE TRIGGER IF NOT EXISTS prompts_ad AFTER DELETE ON user_prompts BEGIN
|
||||
INSERT INTO user_prompts_fts(user_prompts_fts, rowid, prompt_text)
|
||||
VALUES ('delete', old.id, old.prompt_text);
|
||||
END
|
||||
`)
|
||||
if err != nil {
|
||||
t.Fatalf("create prompts_ad trigger: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// createAllTables creates all tables including FTS5 for comprehensive testing.
|
||||
func createAllTables(t *testing.T, db *sql.DB) {
|
||||
t.Helper()
|
||||
createBaseTables(t, db)
|
||||
createFTSTables(t, db)
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
bge-small-en-v1.5
|
||||
@@ -0,0 +1 @@
|
||||
1.23.2
|
||||
@@ -0,0 +1 @@
|
||||
1.23.2
|
||||
@@ -0,0 +1 @@
|
||||
1.23.2
|
||||
@@ -0,0 +1 @@
|
||||
1.23.2
|
||||
Binary file not shown.
@@ -1,21 +1,7 @@
|
||||
{
|
||||
"version": "1.0",
|
||||
"truncation": {
|
||||
"direction": "Right",
|
||||
"max_length": 128,
|
||||
"strategy": "LongestFirst",
|
||||
"stride": 0
|
||||
},
|
||||
"padding": {
|
||||
"strategy": {
|
||||
"Fixed": 128
|
||||
},
|
||||
"direction": "Right",
|
||||
"pad_to_multiple_of": null,
|
||||
"pad_id": 0,
|
||||
"pad_type_id": 0,
|
||||
"pad_token": "[PAD]"
|
||||
},
|
||||
"truncation": null,
|
||||
"padding": null,
|
||||
"added_tokens": [
|
||||
{
|
||||
"id": 0,
|
||||
|
||||
@@ -0,0 +1,157 @@
|
||||
// Package embedding provides text embedding generation with swappable models.
|
||||
package embedding
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sync"
|
||||
)
|
||||
|
||||
// PoolingStrategy defines how to pool token embeddings into sentence embeddings.
|
||||
type PoolingStrategy string
|
||||
|
||||
const (
|
||||
// PoolingNone means the model already outputs sentence embeddings directly.
|
||||
PoolingNone PoolingStrategy = "none"
|
||||
// PoolingMean averages all token embeddings (weighted by attention mask).
|
||||
PoolingMean PoolingStrategy = "mean"
|
||||
// PoolingCLS uses only the [CLS] token embedding.
|
||||
PoolingCLS PoolingStrategy = "cls"
|
||||
)
|
||||
|
||||
// ONNXConfig describes ONNX-specific model configuration.
|
||||
// This allows different models to specify their tensor names and pooling needs.
|
||||
type ONNXConfig struct {
|
||||
// InputNames are the ONNX input tensor names in order.
|
||||
InputNames []string
|
||||
// OutputNames are the ONNX output tensor names.
|
||||
OutputNames []string
|
||||
// Pooling specifies how to convert token embeddings to sentence embeddings.
|
||||
// If PoolingNone, the model outputs sentence embeddings directly.
|
||||
Pooling PoolingStrategy
|
||||
// HiddenSize is the embedding dimension (used for pooling calculations).
|
||||
HiddenSize int
|
||||
}
|
||||
|
||||
// EmbeddingModel represents a text embedding model.
|
||||
type EmbeddingModel interface {
|
||||
// Name returns the human-readable model name (e.g., "bge-small-en-v1.5").
|
||||
Name() string
|
||||
|
||||
// Version returns a short version string for storage (e.g., "bge-v1.5").
|
||||
Version() string
|
||||
|
||||
// Dimensions returns the embedding vector size.
|
||||
Dimensions() int
|
||||
|
||||
// Embed generates an embedding for a single text.
|
||||
Embed(text string) ([]float32, error)
|
||||
|
||||
// EmbedBatch generates embeddings for multiple texts.
|
||||
EmbedBatch(texts []string) ([][]float32, error)
|
||||
|
||||
// Close releases model resources.
|
||||
Close() error
|
||||
}
|
||||
|
||||
// ONNXConfigurer is an optional interface that models can implement
|
||||
// to expose their ONNX configuration for introspection.
|
||||
type ONNXConfigurer interface {
|
||||
// ONNXConfig returns the model's ONNX configuration.
|
||||
ONNXConfig() ONNXConfig
|
||||
}
|
||||
|
||||
// ModelMetadata describes an embedding model for UI/config.
|
||||
type ModelMetadata struct {
|
||||
Name string `json:"name"` // Human-readable name
|
||||
Version string `json:"version"` // Short ID for DB storage
|
||||
Dimensions int `json:"dimensions"` // Vector size
|
||||
Description string `json:"description"` // Brief description
|
||||
Default bool `json:"default"` // Is this the default model?
|
||||
}
|
||||
|
||||
// ModelFactory creates a new instance of an embedding model.
|
||||
type ModelFactory func() (EmbeddingModel, error)
|
||||
|
||||
// ModelRegistry provides model lookup by version.
|
||||
type ModelRegistry struct {
|
||||
mu sync.RWMutex
|
||||
models map[string]ModelFactory
|
||||
metadata map[string]ModelMetadata
|
||||
defaultModel string
|
||||
}
|
||||
|
||||
// NewModelRegistry creates a new model registry.
|
||||
func NewModelRegistry() *ModelRegistry {
|
||||
return &ModelRegistry{
|
||||
models: make(map[string]ModelFactory),
|
||||
metadata: make(map[string]ModelMetadata),
|
||||
}
|
||||
}
|
||||
|
||||
// Register adds a model factory to the registry.
|
||||
func (r *ModelRegistry) Register(meta ModelMetadata, factory ModelFactory) {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
|
||||
r.models[meta.Version] = factory
|
||||
r.metadata[meta.Version] = meta
|
||||
|
||||
if meta.Default {
|
||||
r.defaultModel = meta.Version
|
||||
}
|
||||
}
|
||||
|
||||
// Get creates a new instance of the model with the given version.
|
||||
func (r *ModelRegistry) Get(version string) (EmbeddingModel, error) {
|
||||
r.mu.RLock()
|
||||
factory, ok := r.models[version]
|
||||
r.mu.RUnlock()
|
||||
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("unknown model version: %s", version)
|
||||
}
|
||||
|
||||
return factory()
|
||||
}
|
||||
|
||||
// Default returns the default model version.
|
||||
func (r *ModelRegistry) Default() string {
|
||||
r.mu.RLock()
|
||||
defer r.mu.RUnlock()
|
||||
return r.defaultModel
|
||||
}
|
||||
|
||||
// List returns metadata for all registered models.
|
||||
func (r *ModelRegistry) List() []ModelMetadata {
|
||||
r.mu.RLock()
|
||||
defer r.mu.RUnlock()
|
||||
|
||||
result := make([]ModelMetadata, 0, len(r.metadata))
|
||||
for _, meta := range r.metadata {
|
||||
result = append(result, meta)
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
// DefaultRegistry is the global model registry with all available models.
|
||||
var DefaultRegistry = NewModelRegistry()
|
||||
|
||||
// RegisterModel adds a model to the default registry.
|
||||
func RegisterModel(meta ModelMetadata, factory ModelFactory) {
|
||||
DefaultRegistry.Register(meta, factory)
|
||||
}
|
||||
|
||||
// GetModel creates a model instance from the default registry.
|
||||
func GetModel(version string) (EmbeddingModel, error) {
|
||||
return DefaultRegistry.Get(version)
|
||||
}
|
||||
|
||||
// GetDefaultModel returns the default model version from the default registry.
|
||||
func GetDefaultModel() string {
|
||||
return DefaultRegistry.Default()
|
||||
}
|
||||
|
||||
// ListModels returns metadata for all models in the default registry.
|
||||
func ListModels() []ModelMetadata {
|
||||
return DefaultRegistry.List()
|
||||
}
|
||||
+280
-59
@@ -1,4 +1,4 @@
|
||||
// Package embedding provides text embedding generation using all-MiniLM-L6-v2.
|
||||
// Package embedding provides text embedding generation with swappable models.
|
||||
package embedding
|
||||
|
||||
import (
|
||||
@@ -15,19 +15,50 @@ import (
|
||||
ort "github.com/yalue/onnxruntime_go"
|
||||
)
|
||||
|
||||
// EmbeddingDim is the dimension of embeddings produced by all-MiniLM-L6-v2.
|
||||
// EmbeddingDim is the dimension of embeddings produced by the current model.
|
||||
// Both all-MiniLM-L6-v2 and bge-small-en-v1.5 produce 384-dimensional embeddings.
|
||||
const EmbeddingDim = 384
|
||||
|
||||
// Service provides thread-safe text embedding generation.
|
||||
type Service struct {
|
||||
// Model version constants
|
||||
const (
|
||||
// BGEModelVersion is the version string for bge-small-en-v1.5
|
||||
BGEModelVersion = "bge-v1.5"
|
||||
// BGEModelName is the human-readable name for bge-small-en-v1.5
|
||||
BGEModelName = "bge-small-en-v1.5"
|
||||
// DefaultModelVersion is the default model to use
|
||||
DefaultModelVersion = BGEModelVersion
|
||||
)
|
||||
|
||||
// MaxSequenceLength is the maximum token sequence length for the model.
|
||||
const MaxSequenceLength = 512
|
||||
|
||||
// bgeONNXConfig defines the ONNX configuration for BGE models.
|
||||
// BGE outputs last_hidden_state and requires mean pooling.
|
||||
var bgeONNXConfig = ONNXConfig{
|
||||
InputNames: []string{"input_ids", "attention_mask", "token_type_ids"},
|
||||
OutputNames: []string{"last_hidden_state"},
|
||||
Pooling: PoolingMean,
|
||||
HiddenSize: EmbeddingDim,
|
||||
}
|
||||
|
||||
// bgeModel is the ONNX-based embedding model implementation.
|
||||
// Currently supports bge-small-en-v1.5 (previously all-MiniLM-L6-v2).
|
||||
type bgeModel struct {
|
||||
tk *tokenizer.Tokenizer
|
||||
session *ort.DynamicAdvancedSession
|
||||
mu sync.Mutex
|
||||
libDir string // temp directory containing extracted libraries
|
||||
libDir string // temp directory containing extracted libraries
|
||||
config ONNXConfig // ONNX configuration for this model
|
||||
}
|
||||
|
||||
// NewService creates a new embedding service using bundled ONNX runtime and model.
|
||||
func NewService() (*Service, error) {
|
||||
// Compile-time check that bgeModel implements EmbeddingModel
|
||||
var _ EmbeddingModel = (*bgeModel)(nil)
|
||||
|
||||
// Compile-time check that bgeModel implements ONNXConfigurer
|
||||
var _ ONNXConfigurer = (*bgeModel)(nil)
|
||||
|
||||
// newBGEModel creates a new BGE embedding model using bundled ONNX runtime and model.
|
||||
func newBGEModel() (EmbeddingModel, error) {
|
||||
// Extract ONNX runtime library to temp directory
|
||||
libDir, err := extractONNXLibrary()
|
||||
if err != nil {
|
||||
@@ -49,22 +80,41 @@ func NewService() (*Service, error) {
|
||||
return nil, fmt.Errorf("load tokenizer: %w", err)
|
||||
}
|
||||
|
||||
// Create ONNX session with embedded model
|
||||
inputNames := []string{"input_ids", "attention_mask", "token_type_ids"}
|
||||
outputNames := []string{"sentence_embedding"}
|
||||
|
||||
session, err := ort.NewDynamicAdvancedSessionWithONNXData(modelData, inputNames, outputNames, nil)
|
||||
// Create ONNX session using model-specific configuration
|
||||
config := bgeONNXConfig
|
||||
session, err := ort.NewDynamicAdvancedSessionWithONNXData(modelData, config.InputNames, config.OutputNames, nil)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create ONNX session: %w", err)
|
||||
}
|
||||
|
||||
return &Service{
|
||||
return &bgeModel{
|
||||
tk: tk,
|
||||
session: session,
|
||||
libDir: libDir,
|
||||
config: config,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// ONNXConfig returns the model's ONNX configuration.
|
||||
func (m *bgeModel) ONNXConfig() ONNXConfig {
|
||||
return m.config
|
||||
}
|
||||
|
||||
// Name returns the human-readable model name.
|
||||
func (m *bgeModel) Name() string {
|
||||
return BGEModelName
|
||||
}
|
||||
|
||||
// Version returns the short version string for storage.
|
||||
func (m *bgeModel) Version() string {
|
||||
return BGEModelVersion
|
||||
}
|
||||
|
||||
// Dimensions returns the embedding vector size.
|
||||
func (m *bgeModel) Dimensions() int {
|
||||
return EmbeddingDim
|
||||
}
|
||||
|
||||
// extractONNXLibrary extracts the embedded ONNX runtime library to a temp directory.
|
||||
// Uses content hash to avoid re-extracting if already present.
|
||||
func extractONNXLibrary() (string, error) {
|
||||
@@ -107,15 +157,15 @@ func extractONNXLibrary() (string, error) {
|
||||
|
||||
// Embed generates an embedding for a single text.
|
||||
// Returns a 384-dimensional float32 vector.
|
||||
func (s *Service) Embed(text string) ([]float32, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
func (m *bgeModel) Embed(text string) ([]float32, error) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
|
||||
if text == "" {
|
||||
return make([]float32, EmbeddingDim), nil
|
||||
}
|
||||
|
||||
results, err := s.computeBatch([]string{text})
|
||||
results, err := m.computeBatch([]string{text})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -127,13 +177,13 @@ func (s *Service) Embed(text string) ([]float32, error) {
|
||||
|
||||
// EmbedBatch generates embeddings for multiple texts.
|
||||
// Returns slice of 384-dimensional float32 vectors.
|
||||
func (s *Service) EmbedBatch(texts []string) ([][]float32, error) {
|
||||
func (m *bgeModel) EmbedBatch(texts []string) ([][]float32, error) {
|
||||
if len(texts) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
|
||||
// Filter out empty texts and track indices
|
||||
nonEmpty := make([]string, 0, len(texts))
|
||||
@@ -155,7 +205,7 @@ func (s *Service) EmbedBatch(texts []string) ([][]float32, error) {
|
||||
}
|
||||
|
||||
// Compute embeddings for non-empty texts
|
||||
embeddings, err := s.computeBatch(nonEmpty)
|
||||
embeddings, err := m.computeBatch(nonEmpty)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("compute batch embeddings: %w", err)
|
||||
}
|
||||
@@ -173,7 +223,7 @@ func (s *Service) EmbedBatch(texts []string) ([][]float32, error) {
|
||||
}
|
||||
|
||||
// computeBatch runs inference on a batch of texts. Must be called with lock held.
|
||||
func (s *Service) computeBatch(sentences []string) ([][]float32, error) {
|
||||
func (m *bgeModel) computeBatch(sentences []string) ([][]float32, error) {
|
||||
if len(sentences) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
@@ -184,31 +234,57 @@ func (s *Service) computeBatch(sentences []string) ([][]float32, error) {
|
||||
inputBatch[i] = tokenizer.NewSingleEncodeInput(tokenizer.NewRawInputSequence(sent))
|
||||
}
|
||||
|
||||
encodings, err := s.tk.EncodeBatch(inputBatch, true)
|
||||
encodings, err := m.tk.EncodeBatch(inputBatch, true)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("tokenize: %w", err)
|
||||
}
|
||||
|
||||
batchSize := len(encodings)
|
||||
seqLength := len(encodings[0].Ids)
|
||||
hiddenSize := EmbeddingDim
|
||||
hiddenSize := m.config.HiddenSize
|
||||
|
||||
// Find max sequence length across all encodings (tokenizer may not pad uniformly)
|
||||
// Also enforce MaxSequenceLength to prevent model errors
|
||||
seqLength := 0
|
||||
for _, enc := range encodings {
|
||||
if len(enc.Ids) > seqLength {
|
||||
seqLength = len(enc.Ids)
|
||||
}
|
||||
}
|
||||
// Truncate to max model sequence length
|
||||
if seqLength > MaxSequenceLength {
|
||||
seqLength = MaxSequenceLength
|
||||
}
|
||||
|
||||
inputShape := ort.NewShape(int64(batchSize), int64(seqLength))
|
||||
|
||||
// Create input tensors
|
||||
// Create input tensors (pre-filled with zeros for padding)
|
||||
inputIdsData := make([]int64, batchSize*seqLength)
|
||||
attentionMaskData := make([]int64, batchSize*seqLength)
|
||||
tokenTypeIdsData := make([]int64, batchSize*seqLength)
|
||||
|
||||
for b := 0; b < batchSize; b++ {
|
||||
for i, id := range encodings[b].Ids {
|
||||
inputIdsData[b*seqLength+i] = int64(id)
|
||||
// Copy actual token data (rest remains 0 as padding)
|
||||
// Truncate to seqLength to handle long inputs
|
||||
copyLen := len(encodings[b].Ids)
|
||||
if copyLen > seqLength {
|
||||
copyLen = seqLength
|
||||
}
|
||||
for i, mask := range encodings[b].AttentionMask {
|
||||
attentionMaskData[b*seqLength+i] = int64(mask)
|
||||
for i := 0; i < copyLen; i++ {
|
||||
inputIdsData[b*seqLength+i] = int64(encodings[b].Ids[i])
|
||||
}
|
||||
for i, typeId := range encodings[b].TypeIds {
|
||||
tokenTypeIdsData[b*seqLength+i] = int64(typeId)
|
||||
copyLen = len(encodings[b].AttentionMask)
|
||||
if copyLen > seqLength {
|
||||
copyLen = seqLength
|
||||
}
|
||||
for i := 0; i < copyLen; i++ {
|
||||
attentionMaskData[b*seqLength+i] = int64(encodings[b].AttentionMask[i])
|
||||
}
|
||||
copyLen = len(encodings[b].TypeIds)
|
||||
if copyLen > seqLength {
|
||||
copyLen = seqLength
|
||||
}
|
||||
for i := 0; i < copyLen; i++ {
|
||||
tokenTypeIdsData[b*seqLength+i] = int64(encodings[b].TypeIds[i])
|
||||
}
|
||||
}
|
||||
|
||||
@@ -216,65 +292,145 @@ func (s *Service) computeBatch(sentences []string) ([][]float32, error) {
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create input_ids tensor: %w", err)
|
||||
}
|
||||
defer inputIdsTensor.Destroy()
|
||||
defer func() { _ = inputIdsTensor.Destroy() }()
|
||||
|
||||
attentionMaskTensor, err := ort.NewTensor(inputShape, attentionMaskData)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create attention_mask tensor: %w", err)
|
||||
}
|
||||
defer attentionMaskTensor.Destroy()
|
||||
defer func() { _ = attentionMaskTensor.Destroy() }()
|
||||
|
||||
tokenTypeIdsTensor, err := ort.NewTensor(inputShape, tokenTypeIdsData)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create token_type_ids tensor: %w", err)
|
||||
}
|
||||
defer tokenTypeIdsTensor.Destroy()
|
||||
defer func() { _ = tokenTypeIdsTensor.Destroy() }()
|
||||
|
||||
sentenceOutputShape := ort.NewShape(int64(batchSize), int64(hiddenSize))
|
||||
sentenceOutputTensor, err := ort.NewEmptyTensor[float32](sentenceOutputShape)
|
||||
// Create output tensor based on pooling strategy
|
||||
var outputShape ort.Shape
|
||||
|
||||
switch m.config.Pooling {
|
||||
case PoolingNone:
|
||||
// Direct sentence embedding output: [batch, hidden]
|
||||
outputShape = ort.NewShape(int64(batchSize), int64(hiddenSize))
|
||||
case PoolingMean, PoolingCLS:
|
||||
// Token-level output: [batch, seq_len, hidden]
|
||||
outputShape = ort.NewShape(int64(batchSize), int64(seqLength), int64(hiddenSize))
|
||||
default:
|
||||
outputShape = ort.NewShape(int64(batchSize), int64(hiddenSize))
|
||||
}
|
||||
|
||||
outputTensor, err := ort.NewEmptyTensor[float32](outputShape)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create output tensor: %w", err)
|
||||
}
|
||||
defer sentenceOutputTensor.Destroy()
|
||||
defer func() { _ = outputTensor.Destroy() }()
|
||||
|
||||
// Run inference
|
||||
inputTensors := []ort.Value{inputIdsTensor, attentionMaskTensor, tokenTypeIdsTensor}
|
||||
outputTensors := []ort.Value{sentenceOutputTensor}
|
||||
outputTensors := []ort.Value{outputTensor}
|
||||
|
||||
if err := s.session.Run(inputTensors, outputTensors); err != nil {
|
||||
if err := m.session.Run(inputTensors, outputTensors); err != nil {
|
||||
return nil, fmt.Errorf("run inference: %w", err)
|
||||
}
|
||||
|
||||
// Extract results
|
||||
flatOutput := sentenceOutputTensor.GetData()
|
||||
expectedSize := batchSize * hiddenSize
|
||||
if len(flatOutput) != expectedSize {
|
||||
return nil, fmt.Errorf("unexpected output size: got %d, expected %d", len(flatOutput), expectedSize)
|
||||
}
|
||||
// Extract and pool results based on strategy
|
||||
flatOutput := outputTensor.GetData()
|
||||
|
||||
switch m.config.Pooling {
|
||||
case PoolingNone:
|
||||
// Direct output, no pooling needed
|
||||
expectedSize := batchSize * hiddenSize
|
||||
if len(flatOutput) != expectedSize {
|
||||
return nil, fmt.Errorf("unexpected output size: got %d, expected %d", len(flatOutput), expectedSize)
|
||||
}
|
||||
results := make([][]float32, batchSize)
|
||||
for i := 0; i < batchSize; i++ {
|
||||
start := i * hiddenSize
|
||||
end := start + hiddenSize
|
||||
results[i] = make([]float32, hiddenSize)
|
||||
copy(results[i], flatOutput[start:end])
|
||||
}
|
||||
return results, nil
|
||||
|
||||
case PoolingMean:
|
||||
// Mean pooling over tokens (weighted by attention mask)
|
||||
return meanPooling(flatOutput, attentionMaskData, batchSize, seqLength, hiddenSize), nil
|
||||
|
||||
case PoolingCLS:
|
||||
// CLS token pooling (first token of each sequence)
|
||||
return clsPooling(flatOutput, batchSize, seqLength, hiddenSize), nil
|
||||
|
||||
default:
|
||||
return nil, fmt.Errorf("unknown pooling strategy: %s", m.config.Pooling)
|
||||
}
|
||||
}
|
||||
|
||||
// meanPooling applies mean pooling over token embeddings, weighted by attention mask.
|
||||
// Input shape: [batch, seq_len, hidden], attention mask: [batch, seq_len]
|
||||
// Output shape: [batch, hidden]
|
||||
func meanPooling(embeddings []float32, attentionMask []int64, batchSize, seqLen, hiddenSize int) [][]float32 {
|
||||
results := make([][]float32, batchSize)
|
||||
for i := 0; i < batchSize; i++ {
|
||||
start := i * hiddenSize
|
||||
end := start + hiddenSize
|
||||
results[i] = make([]float32, hiddenSize)
|
||||
copy(results[i], flatOutput[start:end])
|
||||
|
||||
for b := 0; b < batchSize; b++ {
|
||||
result := make([]float32, hiddenSize)
|
||||
var maskSum float32
|
||||
|
||||
// Sum embeddings weighted by attention mask
|
||||
for s := 0; s < seqLen; s++ {
|
||||
maskVal := float32(attentionMask[b*seqLen+s])
|
||||
maskSum += maskVal
|
||||
|
||||
if maskVal > 0 {
|
||||
embOffset := (b*seqLen + s) * hiddenSize
|
||||
for h := 0; h < hiddenSize; h++ {
|
||||
result[h] += embeddings[embOffset+h] * maskVal
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Normalize by mask sum (avoid division by zero)
|
||||
if maskSum > 0 {
|
||||
for h := 0; h < hiddenSize; h++ {
|
||||
result[h] /= maskSum
|
||||
}
|
||||
}
|
||||
|
||||
results[b] = result
|
||||
}
|
||||
|
||||
return results, nil
|
||||
return results
|
||||
}
|
||||
|
||||
// clsPooling extracts the [CLS] token embedding (first token).
|
||||
// Input shape: [batch, seq_len, hidden]
|
||||
// Output shape: [batch, hidden]
|
||||
func clsPooling(embeddings []float32, batchSize, seqLen, hiddenSize int) [][]float32 {
|
||||
results := make([][]float32, batchSize)
|
||||
|
||||
for b := 0; b < batchSize; b++ {
|
||||
result := make([]float32, hiddenSize)
|
||||
// CLS token is at position 0
|
||||
embOffset := b * seqLen * hiddenSize
|
||||
copy(result, embeddings[embOffset:embOffset+hiddenSize])
|
||||
results[b] = result
|
||||
}
|
||||
|
||||
return results
|
||||
}
|
||||
|
||||
// Close releases model resources.
|
||||
func (s *Service) Close() error {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
func (m *bgeModel) Close() error {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
|
||||
var errs []error
|
||||
|
||||
if s.session != nil {
|
||||
if err := s.session.Destroy(); err != nil {
|
||||
if m.session != nil {
|
||||
if err := m.session.Destroy(); err != nil {
|
||||
errs = append(errs, fmt.Errorf("destroy session: %w", err))
|
||||
}
|
||||
s.session = nil
|
||||
m.session = nil
|
||||
}
|
||||
|
||||
if err := ort.DestroyEnvironment(); err != nil {
|
||||
@@ -282,10 +438,75 @@ func (s *Service) Close() error {
|
||||
}
|
||||
|
||||
// Optionally clean up extracted library (leave for caching)
|
||||
// os.RemoveAll(s.libDir)
|
||||
// os.RemoveAll(m.libDir)
|
||||
|
||||
if len(errs) > 0 {
|
||||
return errs[0]
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Register the BGE model with the default registry at init time
|
||||
func init() {
|
||||
RegisterModel(ModelMetadata{
|
||||
Name: BGEModelName,
|
||||
Version: BGEModelVersion,
|
||||
Dimensions: EmbeddingDim,
|
||||
Description: "High-quality semantic search model",
|
||||
Default: true,
|
||||
}, newBGEModel)
|
||||
}
|
||||
|
||||
// Service provides thread-safe text embedding generation with model abstraction.
|
||||
type Service struct {
|
||||
model EmbeddingModel
|
||||
}
|
||||
|
||||
// NewService creates a new embedding service using the default model.
|
||||
func NewService() (*Service, error) {
|
||||
return NewServiceWithModel(DefaultModelVersion)
|
||||
}
|
||||
|
||||
// NewServiceWithModel creates a new embedding service using the specified model.
|
||||
func NewServiceWithModel(version string) (*Service, error) {
|
||||
if version == "" {
|
||||
version = DefaultModelVersion
|
||||
}
|
||||
|
||||
model, err := GetModel(version)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("get model %s: %w", version, err)
|
||||
}
|
||||
|
||||
return &Service{model: model}, nil
|
||||
}
|
||||
|
||||
// Name returns the human-readable model name.
|
||||
func (s *Service) Name() string {
|
||||
return s.model.Name()
|
||||
}
|
||||
|
||||
// Version returns the short version string for storage.
|
||||
func (s *Service) Version() string {
|
||||
return s.model.Version()
|
||||
}
|
||||
|
||||
// Dimensions returns the embedding vector size.
|
||||
func (s *Service) Dimensions() int {
|
||||
return s.model.Dimensions()
|
||||
}
|
||||
|
||||
// Embed generates an embedding for a single text.
|
||||
func (s *Service) Embed(text string) ([]float32, error) {
|
||||
return s.model.Embed(text)
|
||||
}
|
||||
|
||||
// EmbedBatch generates embeddings for multiple texts.
|
||||
func (s *Service) EmbedBatch(texts []string) ([][]float32, error) {
|
||||
return s.model.EmbedBatch(texts)
|
||||
}
|
||||
|
||||
// Close releases model resources.
|
||||
func (s *Service) Close() error {
|
||||
return s.model.Close()
|
||||
}
|
||||
|
||||
@@ -0,0 +1,334 @@
|
||||
package embedding
|
||||
|
||||
import (
|
||||
"math"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// TestEmbeddingDim verifies the embedding dimension constant.
|
||||
func TestEmbeddingDim(t *testing.T) {
|
||||
assert.Equal(t, 384, EmbeddingDim)
|
||||
}
|
||||
|
||||
// TestNewService tests creating a new embedding service.
|
||||
func TestNewService(t *testing.T) {
|
||||
svc, err := NewService()
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, svc)
|
||||
|
||||
defer svc.Close()
|
||||
|
||||
// Verify the service is properly initialized via public methods
|
||||
assert.NotEmpty(t, svc.Name())
|
||||
assert.NotEmpty(t, svc.Version())
|
||||
assert.Equal(t, EmbeddingDim, svc.Dimensions())
|
||||
}
|
||||
|
||||
// TestEmbed_SingleText tests embedding a single text.
|
||||
func TestEmbed_SingleText(t *testing.T) {
|
||||
svc, err := NewService()
|
||||
require.NoError(t, err)
|
||||
defer svc.Close()
|
||||
|
||||
embedding, err := svc.Embed("Hello, world!")
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.Len(t, embedding, EmbeddingDim)
|
||||
|
||||
// Verify non-zero embedding
|
||||
var sum float32
|
||||
for _, v := range embedding {
|
||||
sum += v * v
|
||||
}
|
||||
assert.Greater(t, sum, float32(0), "Embedding should not be all zeros")
|
||||
}
|
||||
|
||||
// TestEmbed_EmptyText tests embedding an empty string.
|
||||
func TestEmbed_EmptyText(t *testing.T) {
|
||||
svc, err := NewService()
|
||||
require.NoError(t, err)
|
||||
defer svc.Close()
|
||||
|
||||
embedding, err := svc.Embed("")
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.Len(t, embedding, EmbeddingDim)
|
||||
|
||||
// Empty text should return zero vector
|
||||
for _, v := range embedding {
|
||||
assert.Equal(t, float32(0), v)
|
||||
}
|
||||
}
|
||||
|
||||
// TestEmbed_SimilarTexts tests that similar texts produce similar embeddings.
|
||||
func TestEmbed_SimilarTexts(t *testing.T) {
|
||||
svc, err := NewService()
|
||||
require.NoError(t, err)
|
||||
defer svc.Close()
|
||||
|
||||
emb1, err := svc.Embed("The quick brown fox jumps over the lazy dog.")
|
||||
require.NoError(t, err)
|
||||
|
||||
emb2, err := svc.Embed("A fast brown fox leaps over a sleepy dog.")
|
||||
require.NoError(t, err)
|
||||
|
||||
emb3, err := svc.Embed("Go programming language concurrency patterns.")
|
||||
require.NoError(t, err)
|
||||
|
||||
// Calculate cosine similarity
|
||||
sim12 := cosineSimilarity(emb1, emb2)
|
||||
sim13 := cosineSimilarity(emb1, emb3)
|
||||
|
||||
// Similar texts should have higher similarity
|
||||
assert.Greater(t, sim12, sim13, "Similar sentences should have higher similarity than dissimilar ones")
|
||||
assert.Greater(t, sim12, float64(0.7), "Similar sentences should have high similarity")
|
||||
}
|
||||
|
||||
// TestEmbedBatch_MultipleTexts tests batch embedding.
|
||||
func TestEmbedBatch_MultipleTexts(t *testing.T) {
|
||||
svc, err := NewService()
|
||||
require.NoError(t, err)
|
||||
defer svc.Close()
|
||||
|
||||
texts := []string{
|
||||
"First text about programming.",
|
||||
"Second text about databases.",
|
||||
"Third text about machine learning.",
|
||||
}
|
||||
|
||||
embeddings, err := svc.EmbedBatch(texts)
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.Len(t, embeddings, len(texts))
|
||||
for i, emb := range embeddings {
|
||||
assert.Len(t, emb, EmbeddingDim, "Embedding %d should have correct dimension", i)
|
||||
}
|
||||
}
|
||||
|
||||
// TestEmbedBatch_EmptySlice tests batch embedding with empty slice.
|
||||
func TestEmbedBatch_EmptySlice(t *testing.T) {
|
||||
svc, err := NewService()
|
||||
require.NoError(t, err)
|
||||
defer svc.Close()
|
||||
|
||||
embeddings, err := svc.EmbedBatch([]string{})
|
||||
require.NoError(t, err)
|
||||
assert.Nil(t, embeddings)
|
||||
}
|
||||
|
||||
// TestEmbedBatch_WithEmptyTexts tests batch embedding with some empty texts.
|
||||
func TestEmbedBatch_WithEmptyTexts(t *testing.T) {
|
||||
svc, err := NewService()
|
||||
require.NoError(t, err)
|
||||
defer svc.Close()
|
||||
|
||||
texts := []string{
|
||||
"Valid text one.",
|
||||
"",
|
||||
"Valid text two.",
|
||||
"",
|
||||
}
|
||||
|
||||
embeddings, err := svc.EmbedBatch(texts)
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.Len(t, embeddings, 4)
|
||||
|
||||
// Non-empty texts should have non-zero embeddings
|
||||
var sum0 float32
|
||||
for _, v := range embeddings[0] {
|
||||
sum0 += v * v
|
||||
}
|
||||
assert.Greater(t, sum0, float32(0))
|
||||
|
||||
// Empty texts should have zero embeddings
|
||||
for _, v := range embeddings[1] {
|
||||
assert.Equal(t, float32(0), v)
|
||||
}
|
||||
|
||||
var sum2 float32
|
||||
for _, v := range embeddings[2] {
|
||||
sum2 += v * v
|
||||
}
|
||||
assert.Greater(t, sum2, float32(0))
|
||||
|
||||
for _, v := range embeddings[3] {
|
||||
assert.Equal(t, float32(0), v)
|
||||
}
|
||||
}
|
||||
|
||||
// TestEmbedBatch_AllEmpty tests batch embedding with all empty texts.
|
||||
func TestEmbedBatch_AllEmpty(t *testing.T) {
|
||||
svc, err := NewService()
|
||||
require.NoError(t, err)
|
||||
defer svc.Close()
|
||||
|
||||
texts := []string{"", "", ""}
|
||||
|
||||
embeddings, err := svc.EmbedBatch(texts)
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.Len(t, embeddings, 3)
|
||||
for i, emb := range embeddings {
|
||||
assert.Len(t, emb, EmbeddingDim, "Embedding %d should have correct dimension", i)
|
||||
for j, v := range emb {
|
||||
assert.Equal(t, float32(0), v, "Embedding %d[%d] should be zero", i, j)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestEmbed_Concurrent tests concurrent embedding calls.
|
||||
func TestEmbed_Concurrent(t *testing.T) {
|
||||
svc, err := NewService()
|
||||
require.NoError(t, err)
|
||||
defer svc.Close()
|
||||
|
||||
var wg sync.WaitGroup
|
||||
numGoroutines := 10
|
||||
|
||||
errors := make(chan error, numGoroutines)
|
||||
embeddings := make([][]float32, numGoroutines)
|
||||
|
||||
for i := 0; i < numGoroutines; i++ {
|
||||
wg.Add(1)
|
||||
go func(idx int) {
|
||||
defer wg.Done()
|
||||
|
||||
text := "Test text for concurrent embedding test"
|
||||
emb, err := svc.Embed(text)
|
||||
if err != nil {
|
||||
errors <- err
|
||||
return
|
||||
}
|
||||
embeddings[idx] = emb
|
||||
}(i)
|
||||
}
|
||||
|
||||
wg.Wait()
|
||||
close(errors)
|
||||
|
||||
for err := range errors {
|
||||
t.Errorf("Concurrent embedding error: %v", err)
|
||||
}
|
||||
|
||||
// All embeddings should be valid
|
||||
for i, emb := range embeddings {
|
||||
if emb != nil {
|
||||
assert.Len(t, emb, EmbeddingDim, "Embedding %d should have correct dimension", i)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestEmbed_SpecialCharacters tests embedding text with special characters.
|
||||
func TestEmbed_SpecialCharacters(t *testing.T) {
|
||||
svc, err := NewService()
|
||||
require.NoError(t, err)
|
||||
defer svc.Close()
|
||||
|
||||
texts := []string{
|
||||
"Text with unicode: 你好世界 🎉",
|
||||
"Text with newlines:\nLine 1\nLine 2",
|
||||
"Text with tabs:\tColumn1\tColumn2",
|
||||
"Text with quotes: \"quoted\" and 'single'",
|
||||
"Text with code: func main() { fmt.Println(\"hello\") }",
|
||||
}
|
||||
|
||||
for _, text := range texts {
|
||||
t.Run(text[:20], func(t *testing.T) {
|
||||
emb, err := svc.Embed(text)
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, emb, EmbeddingDim)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestEmbed_LongText tests embedding long text.
|
||||
func TestEmbed_LongText(t *testing.T) {
|
||||
svc, err := NewService()
|
||||
require.NoError(t, err)
|
||||
defer svc.Close()
|
||||
|
||||
// Create a long text (tokenizer should truncate appropriately)
|
||||
longText := ""
|
||||
for i := 0; i < 100; i++ {
|
||||
longText += "This is a sentence to make the text very long. "
|
||||
}
|
||||
|
||||
emb, err := svc.Embed(longText)
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, emb, EmbeddingDim)
|
||||
}
|
||||
|
||||
// TestClose tests closing the service.
|
||||
func TestClose(t *testing.T) {
|
||||
svc, err := NewService()
|
||||
require.NoError(t, err)
|
||||
|
||||
err = svc.Close()
|
||||
require.NoError(t, err)
|
||||
|
||||
// After close, embedding should fail (model resources released)
|
||||
// Note: This behavior is model-specific; some models may still work after close
|
||||
}
|
||||
|
||||
// TestEmbedBatch_SingleItem tests batch embedding with single item.
|
||||
func TestEmbedBatch_SingleItem(t *testing.T) {
|
||||
svc, err := NewService()
|
||||
require.NoError(t, err)
|
||||
defer svc.Close()
|
||||
|
||||
texts := []string{"Single text for batch embedding."}
|
||||
|
||||
embeddings, err := svc.EmbedBatch(texts)
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.Len(t, embeddings, 1)
|
||||
assert.Len(t, embeddings[0], EmbeddingDim)
|
||||
}
|
||||
|
||||
// TestEmbed_Deterministic tests that embedding is deterministic.
|
||||
func TestEmbed_Deterministic(t *testing.T) {
|
||||
svc, err := NewService()
|
||||
require.NoError(t, err)
|
||||
defer svc.Close()
|
||||
|
||||
text := "Test text for deterministic embedding."
|
||||
|
||||
emb1, err := svc.Embed(text)
|
||||
require.NoError(t, err)
|
||||
|
||||
emb2, err := svc.Embed(text)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Same text should produce same embedding
|
||||
for i := 0; i < EmbeddingDim; i++ {
|
||||
assert.Equal(t, emb1[i], emb2[i], "Embedding should be deterministic at index %d", i)
|
||||
}
|
||||
}
|
||||
|
||||
// Helper function to calculate cosine similarity
|
||||
func cosineSimilarity(a, b []float32) float64 {
|
||||
if len(a) != len(b) {
|
||||
return 0
|
||||
}
|
||||
|
||||
var dotProduct float64
|
||||
var normA float64
|
||||
var normB float64
|
||||
|
||||
for i := range a {
|
||||
dotProduct += float64(a[i]) * float64(b[i])
|
||||
normA += float64(a[i]) * float64(a[i])
|
||||
normB += float64(b[i]) * float64(b[i])
|
||||
}
|
||||
|
||||
if normA == 0 || normB == 0 {
|
||||
return 0
|
||||
}
|
||||
|
||||
return dotProduct / (math.Sqrt(normA) * math.Sqrt(normB))
|
||||
}
|
||||
+120
-5
@@ -9,7 +9,11 @@ import (
|
||||
"io"
|
||||
"os"
|
||||
|
||||
"github.com/lukaszraczylo/claude-mnemonic/internal/db/gorm"
|
||||
"github.com/lukaszraczylo/claude-mnemonic/internal/scoring"
|
||||
"github.com/lukaszraczylo/claude-mnemonic/internal/search"
|
||||
"github.com/lukaszraczylo/claude-mnemonic/internal/vector/sqlitevec"
|
||||
"github.com/lukaszraczylo/claude-mnemonic/pkg/models"
|
||||
"github.com/rs/zerolog/log"
|
||||
)
|
||||
|
||||
@@ -19,15 +23,41 @@ type Server struct {
|
||||
version string
|
||||
stdin io.Reader
|
||||
stdout io.Writer
|
||||
|
||||
// Store dependencies for enhanced tools
|
||||
observationStore *gorm.ObservationStore
|
||||
patternStore *gorm.PatternStore
|
||||
relationStore *gorm.RelationStore
|
||||
sessionStore *gorm.SessionStore
|
||||
vectorClient *sqlitevec.Client
|
||||
scoreCalculator *scoring.Calculator
|
||||
recalculator *scoring.Recalculator
|
||||
}
|
||||
|
||||
// NewServer creates a new MCP server.
|
||||
func NewServer(searchMgr *search.Manager, version string) *Server {
|
||||
func NewServer(
|
||||
searchMgr *search.Manager,
|
||||
version string,
|
||||
observationStore *gorm.ObservationStore,
|
||||
patternStore *gorm.PatternStore,
|
||||
relationStore *gorm.RelationStore,
|
||||
sessionStore *gorm.SessionStore,
|
||||
vectorClient *sqlitevec.Client,
|
||||
scoreCalculator *scoring.Calculator,
|
||||
recalculator *scoring.Recalculator,
|
||||
) *Server {
|
||||
return &Server{
|
||||
searchMgr: searchMgr,
|
||||
version: version,
|
||||
stdin: os.Stdin,
|
||||
stdout: os.Stdout,
|
||||
searchMgr: searchMgr,
|
||||
version: version,
|
||||
stdin: os.Stdin,
|
||||
stdout: os.Stdout,
|
||||
observationStore: observationStore,
|
||||
patternStore: patternStore,
|
||||
relationStore: relationStore,
|
||||
sessionStore: sessionStore,
|
||||
vectorClient: vectorClient,
|
||||
scoreCalculator: scoreCalculator,
|
||||
recalculator: recalculator,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -333,6 +363,19 @@ func (s *Server) handleToolsList(req *Request) *Response {
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "find_related_observations",
|
||||
Description: "Find observations related to a given observation ID filtered by confidence threshold. Returns related observations sorted by confidence score. Useful for discovering relevant context.",
|
||||
InputSchema: map[string]any{
|
||||
"type": "object",
|
||||
"required": []string{"id"},
|
||||
"properties": map[string]any{
|
||||
"id": map[string]any{"type": "number", "description": "Observation ID"},
|
||||
"min_confidence": map[string]any{"type": "number", "default": 0.5, "minimum": 0.0, "maximum": 1.0, "description": "Minimum confidence threshold"},
|
||||
"limit": map[string]any{"type": "number", "default": 20, "minimum": 1, "maximum": 100},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
return &Response{
|
||||
@@ -388,6 +431,12 @@ func (s *Server) handleToolsCall(ctx context.Context, req *Request) *Response {
|
||||
|
||||
// callTool dispatches to the appropriate tool handler.
|
||||
func (s *Server) callTool(ctx context.Context, name string, args json.RawMessage) (string, error) {
|
||||
// Relation discovery tool
|
||||
if name == "find_related_observations" {
|
||||
return s.handleFindRelatedObservations(ctx, args)
|
||||
}
|
||||
|
||||
// Original search-based tools
|
||||
var params search.SearchParams
|
||||
if err := json.Unmarshal(args, ¶ms); err != nil {
|
||||
return "", fmt.Errorf("invalid arguments: %w", err)
|
||||
@@ -537,6 +586,72 @@ func (s *Server) handleTimelineByQuery(ctx context.Context, args json.RawMessage
|
||||
return s.handleTimeline(ctx, args)
|
||||
}
|
||||
|
||||
// handleFindRelatedObservations finds observations related to a given observation ID.
|
||||
func (s *Server) handleFindRelatedObservations(ctx context.Context, args json.RawMessage) (string, error) {
|
||||
var params struct {
|
||||
ID int64 `json:"id"`
|
||||
MinConfidence float64 `json:"min_confidence"`
|
||||
Limit int `json:"limit"`
|
||||
}
|
||||
if err := json.Unmarshal(args, ¶ms); err != nil {
|
||||
return "", fmt.Errorf("invalid arguments: %w", err)
|
||||
}
|
||||
|
||||
if params.ID == 0 {
|
||||
return "", fmt.Errorf("id is required")
|
||||
}
|
||||
|
||||
if params.MinConfidence == 0 {
|
||||
params.MinConfidence = 0.5
|
||||
}
|
||||
|
||||
if params.Limit == 0 {
|
||||
params.Limit = 20
|
||||
}
|
||||
if params.Limit > 100 {
|
||||
params.Limit = 100
|
||||
}
|
||||
|
||||
// Get related observation IDs with confidence filter
|
||||
relatedIDs, err := s.relationStore.GetRelatedObservationIDs(ctx, params.ID, params.MinConfidence)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to get related observations: %w", err)
|
||||
}
|
||||
|
||||
if relatedIDs == nil {
|
||||
relatedIDs = []int64{}
|
||||
}
|
||||
|
||||
// Limit results
|
||||
if len(relatedIDs) > params.Limit {
|
||||
relatedIDs = relatedIDs[:params.Limit]
|
||||
}
|
||||
|
||||
// Fetch full observations
|
||||
observations := make([]*models.Observation, 0, len(relatedIDs))
|
||||
for _, id := range relatedIDs {
|
||||
obs, err := s.observationStore.GetObservationByID(ctx, id)
|
||||
if err != nil {
|
||||
continue // Skip errors for individual observations
|
||||
}
|
||||
if obs != nil {
|
||||
observations = append(observations, obs)
|
||||
}
|
||||
}
|
||||
|
||||
response := map[string]any{
|
||||
"observations": observations,
|
||||
"count": len(observations),
|
||||
}
|
||||
|
||||
output, err := json.Marshal(response)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("marshal response: %w", err)
|
||||
}
|
||||
|
||||
return string(output), nil
|
||||
}
|
||||
|
||||
// sendResponse sends a JSON-RPC response.
|
||||
func (s *Server) sendResponse(resp *Response) {
|
||||
data, err := json.Marshal(resp)
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,438 @@
|
||||
// Package pattern provides pattern detection and recognition functionality.
|
||||
package pattern
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/lukaszraczylo/claude-mnemonic/internal/db/gorm"
|
||||
"github.com/lukaszraczylo/claude-mnemonic/pkg/models"
|
||||
"github.com/rs/zerolog/log"
|
||||
)
|
||||
|
||||
// DetectorConfig contains configuration for the pattern detector.
|
||||
type DetectorConfig struct {
|
||||
// MinMatchScore is the minimum similarity score to consider a match (0.0-1.0).
|
||||
MinMatchScore float64
|
||||
// MinFrequencyForPattern is the minimum occurrences before creating a pattern.
|
||||
MinFrequencyForPattern int
|
||||
// AnalysisInterval is how often to run background pattern analysis.
|
||||
AnalysisInterval time.Duration
|
||||
// MaxPatternsToTrack is the maximum number of active patterns.
|
||||
MaxPatternsToTrack int
|
||||
}
|
||||
|
||||
// DefaultConfig returns the default detector configuration.
|
||||
func DefaultConfig() DetectorConfig {
|
||||
return DetectorConfig{
|
||||
MinMatchScore: 0.3, // 30% similarity threshold
|
||||
MinFrequencyForPattern: 2, // At least 2 occurrences to form a pattern
|
||||
AnalysisInterval: 5 * time.Minute,
|
||||
MaxPatternsToTrack: 1000,
|
||||
}
|
||||
}
|
||||
|
||||
// PatternSyncFunc is a callback for syncing patterns to vector store.
|
||||
type PatternSyncFunc func(pattern *models.Pattern)
|
||||
|
||||
// Detector detects and tracks recurring patterns across observations.
|
||||
type Detector struct {
|
||||
config DetectorConfig
|
||||
patternStore *gorm.PatternStore
|
||||
observationStore *gorm.ObservationStore
|
||||
|
||||
// Vector sync callback
|
||||
syncFunc PatternSyncFunc
|
||||
|
||||
// Candidate tracking (patterns not yet confirmed)
|
||||
candidates map[string]*candidatePattern
|
||||
candidatesMu sync.RWMutex
|
||||
|
||||
// Background analysis
|
||||
ctx context.Context
|
||||
cancel context.CancelFunc
|
||||
wg sync.WaitGroup
|
||||
}
|
||||
|
||||
// SetSyncFunc sets the callback for syncing patterns to vector store.
|
||||
func (d *Detector) SetSyncFunc(fn PatternSyncFunc) {
|
||||
d.syncFunc = fn
|
||||
}
|
||||
|
||||
// candidatePattern tracks a potential pattern before it reaches frequency threshold.
|
||||
type candidatePattern struct {
|
||||
signature []string
|
||||
observationIDs []int64
|
||||
projects []string
|
||||
patternType models.PatternType
|
||||
title string
|
||||
lastSeenEpoch int64
|
||||
}
|
||||
|
||||
// NewDetector creates a new pattern detector.
|
||||
func NewDetector(patternStore *gorm.PatternStore, observationStore *gorm.ObservationStore, config DetectorConfig) *Detector {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
return &Detector{
|
||||
config: config,
|
||||
patternStore: patternStore,
|
||||
observationStore: observationStore,
|
||||
candidates: make(map[string]*candidatePattern),
|
||||
ctx: ctx,
|
||||
cancel: cancel,
|
||||
}
|
||||
}
|
||||
|
||||
// Start begins background pattern analysis.
|
||||
func (d *Detector) Start() {
|
||||
d.wg.Add(1)
|
||||
go d.backgroundAnalysis()
|
||||
log.Info().Dur("interval", d.config.AnalysisInterval).Msg("Pattern detector started")
|
||||
}
|
||||
|
||||
// Stop stops background pattern analysis.
|
||||
func (d *Detector) Stop() {
|
||||
d.cancel()
|
||||
d.wg.Wait()
|
||||
log.Info().Msg("Pattern detector stopped")
|
||||
}
|
||||
|
||||
// backgroundAnalysis runs periodic pattern analysis.
|
||||
func (d *Detector) backgroundAnalysis() {
|
||||
defer d.wg.Done()
|
||||
|
||||
ticker := time.NewTicker(d.config.AnalysisInterval)
|
||||
defer ticker.Stop()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-d.ctx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
if err := d.AnalyzeRecentObservations(d.ctx); err != nil {
|
||||
log.Warn().Err(err).Msg("Background pattern analysis failed")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// AnalyzeObservation processes a new observation for pattern detection.
|
||||
// This is called synchronously when a new observation is stored.
|
||||
func (d *Detector) AnalyzeObservation(ctx context.Context, obs *models.Observation) (*DetectionResult, error) {
|
||||
result := &DetectionResult{}
|
||||
|
||||
// Extract signature from observation
|
||||
signature := models.ExtractSignature(
|
||||
obs.Concepts,
|
||||
obs.Title.String,
|
||||
obs.Narrative.String,
|
||||
)
|
||||
if len(signature) == 0 {
|
||||
return result, nil // Nothing to detect
|
||||
}
|
||||
|
||||
// Check against existing patterns
|
||||
matches, err := d.patternStore.FindMatchingPatterns(ctx, signature, d.config.MinMatchScore)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if len(matches) > 0 {
|
||||
// Found existing pattern match
|
||||
bestMatch := matches[0]
|
||||
for _, m := range matches[1:] {
|
||||
if models.CalculateMatchScore(signature, m.Signature) > models.CalculateMatchScore(signature, bestMatch.Signature) {
|
||||
bestMatch = m
|
||||
}
|
||||
}
|
||||
|
||||
// Update the pattern with new occurrence
|
||||
if err := d.patternStore.IncrementPatternFrequency(ctx, bestMatch.ID, obs.Project, obs.ID); err != nil {
|
||||
log.Warn().Err(err).Int64("pattern_id", bestMatch.ID).Msg("Failed to update pattern frequency")
|
||||
}
|
||||
|
||||
result.MatchedPattern = bestMatch
|
||||
result.MatchScore = models.CalculateMatchScore(signature, bestMatch.Signature)
|
||||
result.IsNewPattern = false
|
||||
|
||||
log.Debug().
|
||||
Int64("pattern_id", bestMatch.ID).
|
||||
Str("pattern_name", bestMatch.Name).
|
||||
Float64("score", result.MatchScore).
|
||||
Msg("Observation matched existing pattern")
|
||||
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// No existing pattern match - check candidates
|
||||
candidateKey := generateCandidateKey(signature)
|
||||
d.candidatesMu.Lock()
|
||||
defer d.candidatesMu.Unlock()
|
||||
|
||||
if candidate, exists := d.candidates[candidateKey]; exists {
|
||||
// Update existing candidate
|
||||
candidate.observationIDs = append(candidate.observationIDs, obs.ID)
|
||||
|
||||
// Add project if not already present
|
||||
found := false
|
||||
for _, p := range candidate.projects {
|
||||
if p == obs.Project {
|
||||
found = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
candidate.projects = append(candidate.projects, obs.Project)
|
||||
}
|
||||
candidate.lastSeenEpoch = time.Now().UnixMilli()
|
||||
|
||||
// Check if candidate should be promoted to pattern
|
||||
if len(candidate.observationIDs) >= d.config.MinFrequencyForPattern {
|
||||
pattern, err := d.promoteCandidate(ctx, candidateKey, candidate)
|
||||
if err != nil {
|
||||
log.Warn().Err(err).Msg("Failed to promote candidate to pattern")
|
||||
} else {
|
||||
result.MatchedPattern = pattern
|
||||
result.IsNewPattern = true
|
||||
log.Info().
|
||||
Str("pattern_name", pattern.Name).
|
||||
Int("frequency", pattern.Frequency).
|
||||
Msg("New pattern detected and stored")
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// Create new candidate
|
||||
patternType := models.DetectPatternType(obs.Concepts, obs.Title.String, obs.Narrative.String)
|
||||
d.candidates[candidateKey] = &candidatePattern{
|
||||
signature: signature,
|
||||
observationIDs: []int64{obs.ID},
|
||||
projects: []string{obs.Project},
|
||||
patternType: patternType,
|
||||
title: obs.Title.String,
|
||||
lastSeenEpoch: time.Now().UnixMilli(),
|
||||
}
|
||||
log.Debug().
|
||||
Str("candidate_key", candidateKey).
|
||||
Strs("signature", signature).
|
||||
Msg("New pattern candidate created")
|
||||
}
|
||||
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// promoteCandidate converts a candidate to a stored pattern.
|
||||
func (d *Detector) promoteCandidate(ctx context.Context, key string, candidate *candidatePattern) (*models.Pattern, error) {
|
||||
// Generate pattern name from signature
|
||||
name := generatePatternName(candidate.patternType, candidate.signature, candidate.title)
|
||||
|
||||
// Create base pattern using NewPattern with first observation
|
||||
firstProject := ""
|
||||
if len(candidate.projects) > 0 {
|
||||
firstProject = candidate.projects[0]
|
||||
}
|
||||
var firstObsID int64
|
||||
if len(candidate.observationIDs) > 0 {
|
||||
firstObsID = candidate.observationIDs[0]
|
||||
}
|
||||
pattern := models.NewPattern(
|
||||
name,
|
||||
candidate.patternType,
|
||||
"Automatically detected pattern from recurring observations",
|
||||
candidate.signature,
|
||||
firstProject,
|
||||
firstObsID,
|
||||
)
|
||||
|
||||
// Add remaining projects and observations
|
||||
for i := 1; i < len(candidate.projects); i++ {
|
||||
pattern.Projects = append(pattern.Projects, candidate.projects[i])
|
||||
}
|
||||
for i := 1; i < len(candidate.observationIDs); i++ {
|
||||
pattern.ObservationIDs = append(pattern.ObservationIDs, candidate.observationIDs[i])
|
||||
}
|
||||
pattern.Frequency = len(candidate.observationIDs)
|
||||
|
||||
id, err := d.patternStore.StorePattern(ctx, pattern)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
pattern.ID = id
|
||||
|
||||
// Sync to vector store if callback is set
|
||||
if d.syncFunc != nil {
|
||||
d.syncFunc(pattern)
|
||||
}
|
||||
|
||||
// Remove from candidates
|
||||
delete(d.candidates, key)
|
||||
|
||||
return pattern, nil
|
||||
}
|
||||
|
||||
// AnalyzeRecentObservations analyzes recent observations for pattern detection.
|
||||
// This is used for background batch analysis.
|
||||
func (d *Detector) AnalyzeRecentObservations(ctx context.Context) error {
|
||||
// Get observations from the last 24 hours that haven't been analyzed
|
||||
observations, err := d.observationStore.GetRecentObservations(ctx, "", 100)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
analyzed := 0
|
||||
patternsFound := 0
|
||||
for _, obs := range observations {
|
||||
result, err := d.AnalyzeObservation(ctx, obs)
|
||||
if err != nil {
|
||||
log.Warn().Err(err).Int64("obs_id", obs.ID).Msg("Failed to analyze observation")
|
||||
continue
|
||||
}
|
||||
analyzed++
|
||||
if result.MatchedPattern != nil {
|
||||
patternsFound++
|
||||
}
|
||||
}
|
||||
|
||||
if analyzed > 0 {
|
||||
log.Info().
|
||||
Int("analyzed", analyzed).
|
||||
Int("patterns_found", patternsFound).
|
||||
Msg("Background pattern analysis completed")
|
||||
}
|
||||
|
||||
// Clean up old candidates (older than 7 days)
|
||||
d.cleanupOldCandidates()
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// cleanupOldCandidates removes candidates that haven't been seen recently.
|
||||
func (d *Detector) cleanupOldCandidates() {
|
||||
d.candidatesMu.Lock()
|
||||
defer d.candidatesMu.Unlock()
|
||||
|
||||
threshold := time.Now().Add(-7 * 24 * time.Hour).UnixMilli()
|
||||
for key, candidate := range d.candidates {
|
||||
if candidate.lastSeenEpoch < threshold {
|
||||
delete(d.candidates, key)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// GetPatternInsight returns a formatted insight string for a pattern.
|
||||
func (d *Detector) GetPatternInsight(ctx context.Context, patternID int64) (string, error) {
|
||||
pattern, err := d.patternStore.GetPatternByID(ctx, patternID)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
return formatPatternInsight(pattern), nil
|
||||
}
|
||||
|
||||
// DetectionResult contains the result of pattern detection.
|
||||
type DetectionResult struct {
|
||||
MatchedPattern *models.Pattern
|
||||
MatchScore float64
|
||||
IsNewPattern bool
|
||||
}
|
||||
|
||||
// generateCandidateKey creates a unique key for a signature.
|
||||
func generateCandidateKey(signature []string) string {
|
||||
if len(signature) == 0 {
|
||||
return ""
|
||||
}
|
||||
key := ""
|
||||
for _, s := range signature {
|
||||
key += s + "|"
|
||||
}
|
||||
return key
|
||||
}
|
||||
|
||||
// generatePatternName creates a human-readable name for a pattern.
|
||||
func generatePatternName(patternType models.PatternType, signature []string, title string) string {
|
||||
// Use title if available and meaningful
|
||||
if title != "" && len(title) < 60 {
|
||||
return title
|
||||
}
|
||||
|
||||
// Otherwise generate from type and signature
|
||||
prefix := ""
|
||||
switch patternType {
|
||||
case models.PatternTypeBug:
|
||||
prefix = "Bug Pattern: "
|
||||
case models.PatternTypeRefactor:
|
||||
prefix = "Refactor Pattern: "
|
||||
case models.PatternTypeArchitecture:
|
||||
prefix = "Architecture Pattern: "
|
||||
case models.PatternTypeAntiPattern:
|
||||
prefix = "Anti-Pattern: "
|
||||
case models.PatternTypeBestPractice:
|
||||
prefix = "Best Practice: "
|
||||
}
|
||||
|
||||
// Use first few signature elements
|
||||
if len(signature) > 0 {
|
||||
name := prefix
|
||||
for i, s := range signature {
|
||||
if i >= 3 {
|
||||
break
|
||||
}
|
||||
if i > 0 {
|
||||
name += ", "
|
||||
}
|
||||
name += s
|
||||
}
|
||||
return name
|
||||
}
|
||||
|
||||
return prefix + "Unnamed"
|
||||
}
|
||||
|
||||
// formatPatternInsight creates a human-readable insight from a pattern.
|
||||
func formatPatternInsight(pattern *models.Pattern) string {
|
||||
insight := "I've encountered this pattern " +
|
||||
itoa(pattern.Frequency) + " times"
|
||||
|
||||
if len(pattern.Projects) > 1 {
|
||||
insight += " across " + itoa(len(pattern.Projects)) + " projects"
|
||||
}
|
||||
|
||||
insight += ". "
|
||||
|
||||
if pattern.Recommendation.Valid && pattern.Recommendation.String != "" {
|
||||
insight += "What works: " + pattern.Recommendation.String
|
||||
} else {
|
||||
switch pattern.Type {
|
||||
case models.PatternTypeBug:
|
||||
insight += "This appears to be a recurring bug pattern."
|
||||
case models.PatternTypeAntiPattern:
|
||||
insight += "This is an identified anti-pattern to avoid."
|
||||
case models.PatternTypeBestPractice:
|
||||
insight += "This is a validated best practice."
|
||||
default:
|
||||
insight += "This is a recognized pattern in the codebase."
|
||||
}
|
||||
}
|
||||
|
||||
return insight
|
||||
}
|
||||
|
||||
// itoa converts int to string without importing strconv.
|
||||
func itoa(n int) string {
|
||||
if n == 0 {
|
||||
return "0"
|
||||
}
|
||||
negative := false
|
||||
if n < 0 {
|
||||
negative = true
|
||||
n = -n
|
||||
}
|
||||
var digits []byte
|
||||
for n > 0 {
|
||||
digits = append([]byte{byte('0' + n%10)}, digits...)
|
||||
n /= 10
|
||||
}
|
||||
if negative {
|
||||
digits = append([]byte{'-'}, digits...)
|
||||
}
|
||||
return string(digits)
|
||||
}
|
||||
@@ -0,0 +1,449 @@
|
||||
package pattern
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/lukaszraczylo/claude-mnemonic/internal/db/gorm"
|
||||
"github.com/lukaszraczylo/claude-mnemonic/pkg/models"
|
||||
)
|
||||
|
||||
func TestNewDetector(t *testing.T) {
|
||||
store := setupTestStore(t)
|
||||
defer store.Close()
|
||||
|
||||
patternStore := gorm.NewPatternStore(store)
|
||||
observationStore := gorm.NewObservationStore(store, nil, nil, nil)
|
||||
config := DefaultConfig()
|
||||
|
||||
detector := NewDetector(patternStore, observationStore, config)
|
||||
if detector == nil {
|
||||
t.Fatal("Expected non-nil detector")
|
||||
}
|
||||
|
||||
if detector.config.MinMatchScore != config.MinMatchScore {
|
||||
t.Errorf("Expected MinMatchScore %f, got %f",
|
||||
config.MinMatchScore, detector.config.MinMatchScore)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDetector_StartStop(t *testing.T) {
|
||||
store := setupTestStore(t)
|
||||
defer store.Close()
|
||||
|
||||
patternStore := gorm.NewPatternStore(store)
|
||||
observationStore := gorm.NewObservationStore(store, nil, nil, nil)
|
||||
config := DefaultConfig()
|
||||
config.AnalysisInterval = 100 * time.Millisecond // Short interval for testing
|
||||
|
||||
detector := NewDetector(patternStore, observationStore, config)
|
||||
|
||||
// Start
|
||||
detector.Start()
|
||||
|
||||
// Wait a bit to ensure background goroutine is running
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
|
||||
// Stop
|
||||
detector.Stop()
|
||||
|
||||
// Verify we can stop without hanging
|
||||
// (if this test hangs, the Stop logic is broken)
|
||||
}
|
||||
|
||||
func TestDetector_AnalyzeObservation_NewCandidate(t *testing.T) {
|
||||
store := setupTestStore(t)
|
||||
defer store.Close()
|
||||
|
||||
patternStore := gorm.NewPatternStore(store)
|
||||
observationStore := gorm.NewObservationStore(store, nil, nil, nil)
|
||||
config := DefaultConfig()
|
||||
config.MinFrequencyForPattern = 2
|
||||
|
||||
detector := NewDetector(patternStore, observationStore, config)
|
||||
ctx := context.Background()
|
||||
|
||||
// Create an observation
|
||||
obs := createTestObservation(1, "nil-handling", []string{"nil", "error-handling"})
|
||||
|
||||
// Analyze first observation
|
||||
result, err := detector.AnalyzeObservation(ctx, obs)
|
||||
if err != nil {
|
||||
t.Fatalf("AnalyzeObservation() error = %v", err)
|
||||
}
|
||||
|
||||
// First observation should create a candidate, not a pattern
|
||||
if result.MatchedPattern != nil {
|
||||
t.Errorf("Expected no pattern match for first observation")
|
||||
}
|
||||
if result.IsNewPattern {
|
||||
t.Errorf("Expected IsNewPattern to be false for first observation")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDetector_AnalyzeObservation_PromoteToPattern(t *testing.T) {
|
||||
store := setupTestStore(t)
|
||||
defer store.Close()
|
||||
|
||||
patternStore := gorm.NewPatternStore(store)
|
||||
observationStore := gorm.NewObservationStore(store, nil, nil, nil)
|
||||
config := DefaultConfig()
|
||||
config.MinFrequencyForPattern = 2
|
||||
|
||||
detector := NewDetector(patternStore, observationStore, config)
|
||||
ctx := context.Background()
|
||||
|
||||
// Create two similar observations
|
||||
obs1 := createTestObservation(1, "Nil pointer handling", []string{"nil", "error-handling"})
|
||||
obs2 := createTestObservation(2, "Nil pointer handling", []string{"nil", "error-handling"})
|
||||
|
||||
// Analyze first observation
|
||||
_, err := detector.AnalyzeObservation(ctx, obs1)
|
||||
if err != nil {
|
||||
t.Fatalf("AnalyzeObservation(obs1) error = %v", err)
|
||||
}
|
||||
|
||||
// Analyze second observation - should promote to pattern
|
||||
result, err := detector.AnalyzeObservation(ctx, obs2)
|
||||
if err != nil {
|
||||
t.Fatalf("AnalyzeObservation(obs2) error = %v", err)
|
||||
}
|
||||
|
||||
if result.MatchedPattern == nil {
|
||||
t.Fatal("Expected pattern to be created after second occurrence")
|
||||
}
|
||||
if !result.IsNewPattern {
|
||||
t.Errorf("Expected IsNewPattern to be true for newly promoted pattern")
|
||||
}
|
||||
if result.MatchedPattern.Frequency != 2 {
|
||||
t.Errorf("Expected frequency 2, got %d", result.MatchedPattern.Frequency)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDetector_AnalyzeObservation_MatchExisting(t *testing.T) {
|
||||
store := setupTestStore(t)
|
||||
defer store.Close()
|
||||
|
||||
patternStore := gorm.NewPatternStore(store)
|
||||
observationStore := gorm.NewObservationStore(store, nil, nil, nil)
|
||||
config := DefaultConfig()
|
||||
|
||||
detector := NewDetector(patternStore, observationStore, config)
|
||||
ctx := context.Background()
|
||||
|
||||
// Create an existing pattern
|
||||
pattern := &models.Pattern{
|
||||
Name: "Existing Pattern",
|
||||
Type: models.PatternTypeBug,
|
||||
Signature: []string{"nil", "error-handling", "pointer"},
|
||||
Frequency: 5,
|
||||
Projects: []string{"proj1"},
|
||||
ObservationIDs: []int64{1, 2, 3, 4, 5},
|
||||
Status: models.PatternStatusActive,
|
||||
Confidence: 0.7,
|
||||
LastSeenAt: time.Now().Format(time.RFC3339),
|
||||
LastSeenEpoch: time.Now().UnixMilli(),
|
||||
CreatedAt: time.Now().Format(time.RFC3339),
|
||||
CreatedAtEpoch: time.Now().UnixMilli(),
|
||||
}
|
||||
_, _ = patternStore.StorePattern(ctx, pattern)
|
||||
|
||||
// Create observation with similar signature
|
||||
obs := createTestObservation(10, "Nil check", []string{"nil", "error-handling"})
|
||||
|
||||
// Analyze - should match existing pattern
|
||||
result, err := detector.AnalyzeObservation(ctx, obs)
|
||||
if err != nil {
|
||||
t.Fatalf("AnalyzeObservation() error = %v", err)
|
||||
}
|
||||
|
||||
if result.MatchedPattern == nil {
|
||||
t.Fatal("Expected to match existing pattern")
|
||||
}
|
||||
if result.IsNewPattern {
|
||||
t.Errorf("Expected IsNewPattern to be false for existing pattern")
|
||||
}
|
||||
if result.MatchScore < 0.3 {
|
||||
t.Errorf("Expected match score >= 0.3, got %f", result.MatchScore)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDetector_AnalyzeObservation_NoMatch(t *testing.T) {
|
||||
store := setupTestStore(t)
|
||||
defer store.Close()
|
||||
|
||||
patternStore := gorm.NewPatternStore(store)
|
||||
observationStore := gorm.NewObservationStore(store, nil, nil, nil)
|
||||
config := DefaultConfig()
|
||||
config.MinMatchScore = 0.5 // Higher threshold
|
||||
|
||||
detector := NewDetector(patternStore, observationStore, config)
|
||||
ctx := context.Background()
|
||||
|
||||
// Create an existing pattern with specific signature
|
||||
pattern := &models.Pattern{
|
||||
Name: "Specific Pattern",
|
||||
Type: models.PatternTypeBug,
|
||||
Signature: []string{"database", "connection", "pool"},
|
||||
Frequency: 3,
|
||||
Projects: []string{"proj1"},
|
||||
ObservationIDs: []int64{1, 2, 3},
|
||||
Status: models.PatternStatusActive,
|
||||
Confidence: 0.6,
|
||||
LastSeenAt: time.Now().Format(time.RFC3339),
|
||||
LastSeenEpoch: time.Now().UnixMilli(),
|
||||
CreatedAt: time.Now().Format(time.RFC3339),
|
||||
CreatedAtEpoch: time.Now().UnixMilli(),
|
||||
}
|
||||
_, _ = patternStore.StorePattern(ctx, pattern)
|
||||
|
||||
// Create observation with completely different signature
|
||||
obs := createTestObservation(10, "UI Component", []string{"frontend", "react", "component"})
|
||||
|
||||
// Analyze - should not match
|
||||
result, err := detector.AnalyzeObservation(ctx, obs)
|
||||
if err != nil {
|
||||
t.Fatalf("AnalyzeObservation() error = %v", err)
|
||||
}
|
||||
|
||||
if result.MatchedPattern != nil {
|
||||
t.Errorf("Expected no match for unrelated observation")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDetector_CandidateCleanup(t *testing.T) {
|
||||
store := setupTestStore(t)
|
||||
defer store.Close()
|
||||
|
||||
patternStore := gorm.NewPatternStore(store)
|
||||
observationStore := gorm.NewObservationStore(store, nil, nil, nil)
|
||||
config := DefaultConfig()
|
||||
config.MinFrequencyForPattern = 3 // Higher threshold
|
||||
|
||||
detector := NewDetector(patternStore, observationStore, config)
|
||||
|
||||
// Add an old candidate manually
|
||||
oldKey := "old|candidate|"
|
||||
detector.candidates[oldKey] = &candidatePattern{
|
||||
signature: []string{"old", "candidate"},
|
||||
observationIDs: []int64{1},
|
||||
projects: []string{"proj1"},
|
||||
patternType: models.PatternTypeBug,
|
||||
title: "Old Candidate",
|
||||
lastSeenEpoch: time.Now().Add(-8 * 24 * time.Hour).UnixMilli(), // 8 days ago
|
||||
}
|
||||
|
||||
// Add a recent candidate
|
||||
recentKey := "recent|candidate|"
|
||||
detector.candidates[recentKey] = &candidatePattern{
|
||||
signature: []string{"recent", "candidate"},
|
||||
observationIDs: []int64{2},
|
||||
projects: []string{"proj1"},
|
||||
patternType: models.PatternTypeBug,
|
||||
title: "Recent Candidate",
|
||||
lastSeenEpoch: time.Now().UnixMilli(),
|
||||
}
|
||||
|
||||
// Run cleanup
|
||||
detector.cleanupOldCandidates()
|
||||
|
||||
// Old candidate should be removed
|
||||
if _, exists := detector.candidates[oldKey]; exists {
|
||||
t.Errorf("Expected old candidate to be cleaned up")
|
||||
}
|
||||
|
||||
// Recent candidate should remain
|
||||
if _, exists := detector.candidates[recentKey]; !exists {
|
||||
t.Errorf("Expected recent candidate to remain")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDetector_GetPatternInsight(t *testing.T) {
|
||||
store := setupTestStore(t)
|
||||
defer store.Close()
|
||||
|
||||
patternStore := gorm.NewPatternStore(store)
|
||||
observationStore := gorm.NewObservationStore(store, nil, nil, nil)
|
||||
config := DefaultConfig()
|
||||
|
||||
detector := NewDetector(patternStore, observationStore, config)
|
||||
ctx := context.Background()
|
||||
|
||||
// Create pattern with recommendation
|
||||
pattern := &models.Pattern{
|
||||
Name: "Insight Test Pattern",
|
||||
Type: models.PatternTypeBestPractice,
|
||||
Signature: []string{"test"},
|
||||
Recommendation: sql.NullString{String: "Always write tests first", Valid: true},
|
||||
Frequency: 12,
|
||||
Projects: []string{"proj1", "proj2", "proj3"},
|
||||
ObservationIDs: []int64{1},
|
||||
Status: models.PatternStatusActive,
|
||||
Confidence: 0.8,
|
||||
LastSeenAt: time.Now().Format(time.RFC3339),
|
||||
LastSeenEpoch: time.Now().UnixMilli(),
|
||||
CreatedAt: time.Now().Format(time.RFC3339),
|
||||
CreatedAtEpoch: time.Now().UnixMilli(),
|
||||
}
|
||||
id, _ := patternStore.StorePattern(ctx, pattern)
|
||||
|
||||
// Get insight
|
||||
insight, err := detector.GetPatternInsight(ctx, id)
|
||||
if err != nil {
|
||||
t.Fatalf("GetPatternInsight() error = %v", err)
|
||||
}
|
||||
|
||||
// Verify insight contains expected elements
|
||||
if insight == "" {
|
||||
t.Error("Expected non-empty insight")
|
||||
}
|
||||
if !containsString(insight, "12 times") {
|
||||
t.Errorf("Expected insight to contain frequency, got: %s", insight)
|
||||
}
|
||||
if !containsString(insight, "3 projects") {
|
||||
t.Errorf("Expected insight to contain project count, got: %s", insight)
|
||||
}
|
||||
if !containsString(insight, "Always write tests first") {
|
||||
t.Errorf("Expected insight to contain recommendation, got: %s", insight)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDefaultConfig(t *testing.T) {
|
||||
config := DefaultConfig()
|
||||
|
||||
if config.MinMatchScore <= 0 || config.MinMatchScore > 1 {
|
||||
t.Errorf("Invalid MinMatchScore: %f", config.MinMatchScore)
|
||||
}
|
||||
if config.MinFrequencyForPattern < 1 {
|
||||
t.Errorf("Invalid MinFrequencyForPattern: %d", config.MinFrequencyForPattern)
|
||||
}
|
||||
if config.AnalysisInterval <= 0 {
|
||||
t.Errorf("Invalid AnalysisInterval: %v", config.AnalysisInterval)
|
||||
}
|
||||
if config.MaxPatternsToTrack <= 0 {
|
||||
t.Errorf("Invalid MaxPatternsToTrack: %d", config.MaxPatternsToTrack)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGeneratePatternName(t *testing.T) {
|
||||
tests := []struct {
|
||||
patternType models.PatternType
|
||||
signature []string
|
||||
title string
|
||||
wantPrefix string
|
||||
}{
|
||||
{models.PatternTypeBug, []string{"nil", "error"}, "", "Bug Pattern:"},
|
||||
{models.PatternTypeRefactor, []string{"extract"}, "", "Refactor Pattern:"},
|
||||
{models.PatternTypeArchitecture, []string{"service"}, "", "Architecture Pattern:"},
|
||||
{models.PatternTypeAntiPattern, []string{"god-class"}, "", "Anti-Pattern:"},
|
||||
{models.PatternTypeBestPractice, []string{"testing"}, "", "Best Practice:"},
|
||||
{models.PatternTypeBug, []string{}, "Short Title", "Short Title"}, // Use title directly
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
name := generatePatternName(tt.patternType, tt.signature, tt.title)
|
||||
if !hasPrefix(name, tt.wantPrefix) {
|
||||
t.Errorf("generatePatternName(%v, %v, %q) = %q, want prefix %q",
|
||||
tt.patternType, tt.signature, tt.title, name, tt.wantPrefix)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestFormatPatternInsight(t *testing.T) {
|
||||
// Pattern without recommendation
|
||||
pattern1 := &models.Pattern{
|
||||
Type: models.PatternTypeBug,
|
||||
Frequency: 5,
|
||||
Projects: []string{"proj1"},
|
||||
}
|
||||
insight1 := formatPatternInsight(pattern1)
|
||||
if !containsString(insight1, "5 times") {
|
||||
t.Errorf("Expected insight to contain frequency")
|
||||
}
|
||||
if !containsString(insight1, "recurring bug pattern") {
|
||||
t.Errorf("Expected bug pattern description")
|
||||
}
|
||||
|
||||
// Pattern with recommendation
|
||||
pattern2 := &models.Pattern{
|
||||
Type: models.PatternTypeBestPractice,
|
||||
Frequency: 10,
|
||||
Projects: []string{"proj1", "proj2"},
|
||||
Recommendation: sql.NullString{String: "Do this", Valid: true},
|
||||
}
|
||||
insight2 := formatPatternInsight(pattern2)
|
||||
if !containsString(insight2, "10 times") {
|
||||
t.Errorf("Expected insight to contain frequency")
|
||||
}
|
||||
if !containsString(insight2, "2 projects") {
|
||||
t.Errorf("Expected insight to contain project count")
|
||||
}
|
||||
if !containsString(insight2, "Do this") {
|
||||
t.Errorf("Expected insight to contain recommendation")
|
||||
}
|
||||
}
|
||||
|
||||
// Helper functions
|
||||
|
||||
func setupTestStore(t *testing.T) *gorm.Store {
|
||||
t.Helper()
|
||||
|
||||
// Create temp database file
|
||||
tmpFile, err := os.CreateTemp("", "pattern_test_*.db")
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to create temp file: %v", err)
|
||||
}
|
||||
tmpFile.Close()
|
||||
|
||||
t.Cleanup(func() {
|
||||
os.Remove(tmpFile.Name())
|
||||
})
|
||||
|
||||
store, err := gorm.NewStore(gorm.Config{
|
||||
Path: tmpFile.Name(),
|
||||
MaxConns: 1,
|
||||
})
|
||||
if err != nil {
|
||||
// Check if this is an FTS5 related error
|
||||
if containsString(err.Error(), "fts5") || containsString(err.Error(), "no such module") {
|
||||
t.Skip("FTS5 not available in this SQLite build")
|
||||
}
|
||||
t.Fatalf("Failed to create store: %v", err)
|
||||
}
|
||||
|
||||
return store
|
||||
}
|
||||
|
||||
func createTestObservation(id int64, title string, concepts []string) *models.Observation {
|
||||
return &models.Observation{
|
||||
ID: id,
|
||||
SDKSessionID: "test-session",
|
||||
Project: "test-project",
|
||||
Scope: models.ScopeProject,
|
||||
Type: models.ObsTypeBugfix,
|
||||
Title: sql.NullString{String: title, Valid: true},
|
||||
Narrative: sql.NullString{String: "Test narrative", Valid: true},
|
||||
Concepts: concepts,
|
||||
CreatedAt: time.Now().Format(time.RFC3339),
|
||||
CreatedAtEpoch: time.Now().UnixMilli(),
|
||||
}
|
||||
}
|
||||
|
||||
func containsString(s, substr string) bool {
|
||||
for i := 0; i <= len(s)-len(substr); i++ {
|
||||
if s[i:i+len(substr)] == substr {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func hasPrefix(s, prefix string) bool {
|
||||
if len(s) < len(prefix) {
|
||||
return false
|
||||
}
|
||||
return s[:len(prefix)] == prefix
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
// Package reranking provides cross-encoder reranking for search results.
|
||||
package reranking
|
||||
|
||||
import (
|
||||
_ "embed"
|
||||
)
|
||||
|
||||
// Cross-encoder model and tokenizer files - embedded for all platforms
|
||||
//
|
||||
//go:embed assets/model.onnx
|
||||
var crossEncoderModelData []byte
|
||||
|
||||
//go:embed assets/tokenizer.json
|
||||
var crossEncoderTokenizerData []byte
|
||||
Binary file not shown.
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,57 @@
|
||||
{
|
||||
"added_tokens_decoder": {
|
||||
"0": {
|
||||
"content": "[PAD]",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"100": {
|
||||
"content": "[UNK]",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"101": {
|
||||
"content": "[CLS]",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"102": {
|
||||
"content": "[SEP]",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"103": {
|
||||
"content": "[MASK]",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
}
|
||||
},
|
||||
"clean_up_tokenization_spaces": true,
|
||||
"cls_token": "[CLS]",
|
||||
"do_basic_tokenize": true,
|
||||
"do_lower_case": true,
|
||||
"mask_token": "[MASK]",
|
||||
"model_max_length": 512,
|
||||
"never_split": null,
|
||||
"pad_token": "[PAD]",
|
||||
"sep_token": "[SEP]",
|
||||
"strip_accents": null,
|
||||
"tokenize_chinese_chars": true,
|
||||
"tokenizer_class": "BertTokenizer",
|
||||
"unk_token": "[UNK]"
|
||||
}
|
||||
@@ -0,0 +1,380 @@
|
||||
// Package reranking provides cross-encoder reranking for search results.
|
||||
// Uses MS-MARCO MiniLM L6 v2 cross-encoder model for relevance scoring.
|
||||
package reranking
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"math"
|
||||
"sort"
|
||||
"sync"
|
||||
|
||||
"github.com/rs/zerolog/log"
|
||||
"github.com/sugarme/tokenizer"
|
||||
"github.com/sugarme/tokenizer/pretrained"
|
||||
ort "github.com/yalue/onnxruntime_go"
|
||||
)
|
||||
|
||||
const (
|
||||
// ModelName is the human-readable name for the cross-encoder model
|
||||
ModelName = "ms-marco-MiniLM-L6-v2"
|
||||
// ModelVersion is the short version string for identification
|
||||
ModelVersion = "msmarco-v2"
|
||||
// MaxSequenceLength is the maximum combined query+document token length
|
||||
MaxSequenceLength = 512
|
||||
// DefaultCandidateLimit is the default number of candidates to rerank
|
||||
DefaultCandidateLimit = 100
|
||||
// DefaultResultLimit is the default number of results to return after reranking
|
||||
DefaultResultLimit = 10
|
||||
)
|
||||
|
||||
// Candidate represents a search result candidate for reranking.
|
||||
type Candidate struct {
|
||||
ID string // Document ID
|
||||
Content string // Document text content for scoring
|
||||
Score float64 // Original bi-encoder similarity score
|
||||
Metadata map[string]any // Preserved metadata
|
||||
RerankInfo map[string]float64 // Reranking debug info (optional)
|
||||
}
|
||||
|
||||
// RerankResult represents a reranked search result.
|
||||
type RerankResult struct {
|
||||
ID string // Document ID
|
||||
Content string // Document text content
|
||||
OriginalScore float64 // Original bi-encoder score
|
||||
RerankScore float64 // Cross-encoder relevance score
|
||||
CombinedScore float64 // Weighted combination of scores
|
||||
Metadata map[string]any // Preserved metadata
|
||||
OriginalRank int // Position before reranking (1-indexed)
|
||||
RerankRank int // Position after reranking (1-indexed)
|
||||
RankImprovement int // How much the rank improved (positive = moved up)
|
||||
}
|
||||
|
||||
// Service provides cross-encoder reranking functionality.
|
||||
type Service struct {
|
||||
tk *tokenizer.Tokenizer
|
||||
session *ort.DynamicAdvancedSession
|
||||
mu sync.Mutex
|
||||
|
||||
// Weight for combining scores: combined = alpha*rerank + (1-alpha)*original
|
||||
// Default 0.7 favors cross-encoder score
|
||||
Alpha float64
|
||||
}
|
||||
|
||||
// Config holds configuration for the reranking service.
|
||||
type Config struct {
|
||||
// Alpha is the weight for combining scores (0.0-1.0)
|
||||
// Higher values favor cross-encoder scores, lower values favor bi-encoder scores
|
||||
Alpha float64
|
||||
}
|
||||
|
||||
// DefaultConfig returns sensible defaults for reranking.
|
||||
func DefaultConfig() Config {
|
||||
return Config{
|
||||
Alpha: 0.7, // Favor cross-encoder by default
|
||||
}
|
||||
}
|
||||
|
||||
// NewService creates a new cross-encoder reranking service.
|
||||
// Note: ONNX runtime must be initialized before calling this (via embedding.NewService).
|
||||
func NewService(cfg Config) (*Service, error) {
|
||||
// Load tokenizer from embedded data
|
||||
tk, err := pretrained.FromReader(bytes.NewReader(crossEncoderTokenizerData))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("load cross-encoder tokenizer: %w", err)
|
||||
}
|
||||
|
||||
// Configure tokenizer for sequence classification (pairs)
|
||||
tk.WithTruncation(&tokenizer.TruncationParams{
|
||||
MaxLength: MaxSequenceLength,
|
||||
Strategy: tokenizer.LongestFirst,
|
||||
Stride: 0,
|
||||
})
|
||||
|
||||
// Cross-encoder outputs a single logit for relevance scoring
|
||||
inputNames := []string{"input_ids", "attention_mask", "token_type_ids"}
|
||||
outputNames := []string{"logits"}
|
||||
|
||||
session, err := ort.NewDynamicAdvancedSessionWithONNXData(
|
||||
crossEncoderModelData,
|
||||
inputNames,
|
||||
outputNames,
|
||||
nil,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create cross-encoder ONNX session: %w", err)
|
||||
}
|
||||
|
||||
alpha := cfg.Alpha
|
||||
if alpha <= 0 || alpha > 1 {
|
||||
alpha = 0.7
|
||||
}
|
||||
|
||||
return &Service{
|
||||
tk: tk,
|
||||
session: session,
|
||||
Alpha: alpha,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Rerank reranks candidates using the cross-encoder model.
|
||||
// Takes a query and list of candidates, returns reranked results.
|
||||
func (s *Service) Rerank(query string, candidates []Candidate, limit int) ([]RerankResult, error) {
|
||||
if len(candidates) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
if limit <= 0 {
|
||||
limit = DefaultResultLimit
|
||||
}
|
||||
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
|
||||
// Score all query-document pairs
|
||||
scores, err := s.scoreAll(query, candidates)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("score candidates: %w", err)
|
||||
}
|
||||
|
||||
// Build results with combined scores
|
||||
results := make([]RerankResult, len(candidates))
|
||||
for i, c := range candidates {
|
||||
// Normalize cross-encoder score to 0-1 range using sigmoid
|
||||
normalizedRerank := sigmoid(scores[i])
|
||||
|
||||
results[i] = RerankResult{
|
||||
ID: c.ID,
|
||||
Content: c.Content,
|
||||
OriginalScore: c.Score,
|
||||
RerankScore: normalizedRerank,
|
||||
CombinedScore: s.Alpha*normalizedRerank + (1-s.Alpha)*c.Score,
|
||||
Metadata: c.Metadata,
|
||||
OriginalRank: i + 1,
|
||||
}
|
||||
}
|
||||
|
||||
// Sort by combined score (descending)
|
||||
sort.Slice(results, func(i, j int) bool {
|
||||
return results[i].CombinedScore > results[j].CombinedScore
|
||||
})
|
||||
|
||||
// Assign rerank positions and calculate improvement
|
||||
for i := range results {
|
||||
results[i].RerankRank = i + 1
|
||||
results[i].RankImprovement = results[i].OriginalRank - results[i].RerankRank
|
||||
}
|
||||
|
||||
// Apply limit
|
||||
if len(results) > limit {
|
||||
results = results[:limit]
|
||||
}
|
||||
|
||||
log.Debug().
|
||||
Int("candidates", len(candidates)).
|
||||
Int("returned", len(results)).
|
||||
Float64("alpha", s.Alpha).
|
||||
Msg("Cross-encoder reranking completed")
|
||||
|
||||
return results, nil
|
||||
}
|
||||
|
||||
// RerankByScore reranks candidates and returns sorted by pure cross-encoder score.
|
||||
// Useful when you want to completely replace bi-encoder ranking.
|
||||
func (s *Service) RerankByScore(query string, candidates []Candidate, limit int) ([]RerankResult, error) {
|
||||
if len(candidates) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
if limit <= 0 {
|
||||
limit = DefaultResultLimit
|
||||
}
|
||||
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
|
||||
scores, err := s.scoreAll(query, candidates)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("score candidates: %w", err)
|
||||
}
|
||||
|
||||
results := make([]RerankResult, len(candidates))
|
||||
for i, c := range candidates {
|
||||
normalizedRerank := sigmoid(scores[i])
|
||||
results[i] = RerankResult{
|
||||
ID: c.ID,
|
||||
Content: c.Content,
|
||||
OriginalScore: c.Score,
|
||||
RerankScore: normalizedRerank,
|
||||
CombinedScore: normalizedRerank, // Use pure rerank score
|
||||
Metadata: c.Metadata,
|
||||
OriginalRank: i + 1,
|
||||
}
|
||||
}
|
||||
|
||||
// Sort by rerank score only
|
||||
sort.Slice(results, func(i, j int) bool {
|
||||
return results[i].RerankScore > results[j].RerankScore
|
||||
})
|
||||
|
||||
for i := range results {
|
||||
results[i].RerankRank = i + 1
|
||||
results[i].RankImprovement = results[i].OriginalRank - results[i].RerankRank
|
||||
}
|
||||
|
||||
if len(results) > limit {
|
||||
results = results[:limit]
|
||||
}
|
||||
|
||||
return results, nil
|
||||
}
|
||||
|
||||
// scoreAll scores all query-document pairs using the cross-encoder.
|
||||
// Returns raw logits (before sigmoid normalization).
|
||||
func (s *Service) scoreAll(query string, candidates []Candidate) ([]float64, error) {
|
||||
batchSize := len(candidates)
|
||||
|
||||
// Tokenize all query-document pairs
|
||||
pairs := make([]tokenizer.EncodeInput, batchSize)
|
||||
for i, c := range candidates {
|
||||
// Cross-encoder takes query and document as a pair
|
||||
pairs[i] = tokenizer.NewDualEncodeInput(
|
||||
tokenizer.NewRawInputSequence(query),
|
||||
tokenizer.NewRawInputSequence(c.Content),
|
||||
)
|
||||
}
|
||||
|
||||
encodings, err := s.tk.EncodeBatch(pairs, true)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("tokenize pairs: %w", err)
|
||||
}
|
||||
|
||||
// Find max sequence length
|
||||
seqLength := 0
|
||||
for _, enc := range encodings {
|
||||
if len(enc.Ids) > seqLength {
|
||||
seqLength = len(enc.Ids)
|
||||
}
|
||||
}
|
||||
if seqLength > MaxSequenceLength {
|
||||
seqLength = MaxSequenceLength
|
||||
}
|
||||
|
||||
inputShape := ort.NewShape(int64(batchSize), int64(seqLength))
|
||||
|
||||
// Create input tensors
|
||||
inputIdsData := make([]int64, batchSize*seqLength)
|
||||
attentionMaskData := make([]int64, batchSize*seqLength)
|
||||
tokenTypeIdsData := make([]int64, batchSize*seqLength)
|
||||
|
||||
for b := 0; b < batchSize; b++ {
|
||||
copyLen := len(encodings[b].Ids)
|
||||
if copyLen > seqLength {
|
||||
copyLen = seqLength
|
||||
}
|
||||
for i := 0; i < copyLen; i++ {
|
||||
inputIdsData[b*seqLength+i] = int64(encodings[b].Ids[i])
|
||||
}
|
||||
|
||||
copyLen = len(encodings[b].AttentionMask)
|
||||
if copyLen > seqLength {
|
||||
copyLen = seqLength
|
||||
}
|
||||
for i := 0; i < copyLen; i++ {
|
||||
attentionMaskData[b*seqLength+i] = int64(encodings[b].AttentionMask[i])
|
||||
}
|
||||
|
||||
copyLen = len(encodings[b].TypeIds)
|
||||
if copyLen > seqLength {
|
||||
copyLen = seqLength
|
||||
}
|
||||
for i := 0; i < copyLen; i++ {
|
||||
tokenTypeIdsData[b*seqLength+i] = int64(encodings[b].TypeIds[i])
|
||||
}
|
||||
}
|
||||
|
||||
inputIdsTensor, err := ort.NewTensor(inputShape, inputIdsData)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create input_ids tensor: %w", err)
|
||||
}
|
||||
defer func() { _ = inputIdsTensor.Destroy() }()
|
||||
|
||||
attentionMaskTensor, err := ort.NewTensor(inputShape, attentionMaskData)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create attention_mask tensor: %w", err)
|
||||
}
|
||||
defer func() { _ = attentionMaskTensor.Destroy() }()
|
||||
|
||||
tokenTypeIdsTensor, err := ort.NewTensor(inputShape, tokenTypeIdsData)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create token_type_ids tensor: %w", err)
|
||||
}
|
||||
defer func() { _ = tokenTypeIdsTensor.Destroy() }()
|
||||
|
||||
// Cross-encoder outputs [batch, 1] logits
|
||||
outputShape := ort.NewShape(int64(batchSize), 1)
|
||||
outputTensor, err := ort.NewEmptyTensor[float32](outputShape)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create output tensor: %w", err)
|
||||
}
|
||||
defer func() { _ = outputTensor.Destroy() }()
|
||||
|
||||
// Run inference
|
||||
inputTensors := []ort.Value{inputIdsTensor, attentionMaskTensor, tokenTypeIdsTensor}
|
||||
outputTensors := []ort.Value{outputTensor}
|
||||
|
||||
if err := s.session.Run(inputTensors, outputTensors); err != nil {
|
||||
return nil, fmt.Errorf("run cross-encoder inference: %w", err)
|
||||
}
|
||||
|
||||
// Extract scores
|
||||
flatOutput := outputTensor.GetData()
|
||||
scores := make([]float64, batchSize)
|
||||
for i := 0; i < batchSize; i++ {
|
||||
scores[i] = float64(flatOutput[i])
|
||||
}
|
||||
|
||||
return scores, nil
|
||||
}
|
||||
|
||||
// Score scores a single query-document pair.
|
||||
// Returns the raw cross-encoder logit and normalized score.
|
||||
func (s *Service) Score(query, document string) (rawScore, normalizedScore float64, err error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
|
||||
scores, err := s.scoreAll(query, []Candidate{{Content: document}})
|
||||
if err != nil {
|
||||
return 0, 0, err
|
||||
}
|
||||
|
||||
rawScore = scores[0]
|
||||
normalizedScore = sigmoid(rawScore)
|
||||
return rawScore, normalizedScore, nil
|
||||
}
|
||||
|
||||
// Close releases model resources.
|
||||
func (s *Service) Close() error {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
|
||||
if s.session != nil {
|
||||
if err := s.session.Destroy(); err != nil {
|
||||
return fmt.Errorf("destroy cross-encoder session: %w", err)
|
||||
}
|
||||
s.session = nil
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// sigmoid applies the sigmoid function to normalize scores to 0-1 range.
|
||||
func sigmoid(x float64) float64 {
|
||||
if x > 20 {
|
||||
return 1.0
|
||||
}
|
||||
if x < -20 {
|
||||
return 0.0
|
||||
}
|
||||
return 1.0 / (1.0 + math.Exp(-x))
|
||||
}
|
||||
@@ -0,0 +1,448 @@
|
||||
package reranking
|
||||
|
||||
import (
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"github.com/lukaszraczylo/claude-mnemonic/internal/embedding"
|
||||
)
|
||||
|
||||
// initONNX initializes the ONNX runtime via the embedding service.
|
||||
// Must be called before creating reranking service.
|
||||
func initONNX(t *testing.T) func() {
|
||||
t.Helper()
|
||||
embSvc, err := embedding.NewService()
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to initialize ONNX via embedding service: %v", err)
|
||||
}
|
||||
return func() {
|
||||
embSvc.Close()
|
||||
}
|
||||
}
|
||||
|
||||
// TestSigmoid tests the sigmoid normalization function.
|
||||
func TestSigmoid(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
input float64
|
||||
wantMin float64
|
||||
wantMax float64
|
||||
}{
|
||||
{"positive large", 10, 0.9999, 1.0},
|
||||
{"positive small", 1, 0.7, 0.8},
|
||||
{"zero", 0, 0.4999, 0.5001},
|
||||
{"negative small", -1, 0.2, 0.3},
|
||||
{"negative large", -10, 0, 0.0001},
|
||||
{"very positive", 25, 0.999999, 1.0},
|
||||
{"very negative", -25, 0, 0.000001},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := sigmoid(tt.input)
|
||||
if got < tt.wantMin || got > tt.wantMax {
|
||||
t.Errorf("sigmoid(%v) = %v, want in range [%v, %v]",
|
||||
tt.input, got, tt.wantMin, tt.wantMax)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestDefaultConfig tests the default configuration values.
|
||||
func TestDefaultConfig(t *testing.T) {
|
||||
cfg := DefaultConfig()
|
||||
|
||||
if cfg.Alpha < 0 || cfg.Alpha > 1 {
|
||||
t.Errorf("DefaultConfig().Alpha = %v, want in range [0, 1]", cfg.Alpha)
|
||||
}
|
||||
if cfg.Alpha != 0.7 {
|
||||
t.Errorf("DefaultConfig().Alpha = %v, want 0.7", cfg.Alpha)
|
||||
}
|
||||
}
|
||||
|
||||
// TestNewService tests service creation.
|
||||
func TestNewService(t *testing.T) {
|
||||
cleanup := initONNX(t)
|
||||
defer cleanup()
|
||||
|
||||
cfg := DefaultConfig()
|
||||
svc, err := NewService(cfg)
|
||||
if err != nil {
|
||||
t.Fatalf("NewService() error = %v", err)
|
||||
}
|
||||
defer svc.Close()
|
||||
|
||||
if svc == nil {
|
||||
t.Fatal("NewService() returned nil")
|
||||
}
|
||||
|
||||
if svc.Alpha != cfg.Alpha {
|
||||
t.Errorf("Service.Alpha = %v, want %v", svc.Alpha, cfg.Alpha)
|
||||
}
|
||||
}
|
||||
|
||||
// TestScore tests single pair scoring.
|
||||
func TestScore(t *testing.T) {
|
||||
cleanup := initONNX(t)
|
||||
defer cleanup()
|
||||
|
||||
cfg := DefaultConfig()
|
||||
svc, err := NewService(cfg)
|
||||
if err != nil {
|
||||
t.Fatalf("NewService() error = %v", err)
|
||||
}
|
||||
defer svc.Close()
|
||||
|
||||
query := "What is the capital of France?"
|
||||
relevant := "Paris is the capital and largest city of France."
|
||||
irrelevant := "Dogs are popular pets known for their loyalty."
|
||||
|
||||
// Score relevant document
|
||||
_, relevantNorm, err := svc.Score(query, relevant)
|
||||
if err != nil {
|
||||
t.Fatalf("Score(relevant) error = %v", err)
|
||||
}
|
||||
|
||||
// Score irrelevant document
|
||||
_, irrelevantNorm, err := svc.Score(query, irrelevant)
|
||||
if err != nil {
|
||||
t.Fatalf("Score(irrelevant) error = %v", err)
|
||||
}
|
||||
|
||||
// Relevant document should score higher
|
||||
if relevantNorm <= irrelevantNorm {
|
||||
t.Errorf("Expected relevant (%v) > irrelevant (%v)",
|
||||
relevantNorm, irrelevantNorm)
|
||||
}
|
||||
}
|
||||
|
||||
// TestRerank tests the reranking functionality.
|
||||
func TestRerank(t *testing.T) {
|
||||
cleanup := initONNX(t)
|
||||
defer cleanup()
|
||||
|
||||
cfg := DefaultConfig()
|
||||
svc, err := NewService(cfg)
|
||||
if err != nil {
|
||||
t.Fatalf("NewService() error = %v", err)
|
||||
}
|
||||
defer svc.Close()
|
||||
|
||||
query := "How to handle errors in Go?"
|
||||
candidates := []Candidate{
|
||||
{
|
||||
ID: "1",
|
||||
Content: "Python exception handling with try/except blocks.",
|
||||
Score: 0.8, // High bi-encoder score but irrelevant
|
||||
},
|
||||
{
|
||||
ID: "2",
|
||||
Content: "Go error handling uses explicit return values. Functions return error as the last value.",
|
||||
Score: 0.6, // Lower bi-encoder score but relevant
|
||||
},
|
||||
{
|
||||
ID: "3",
|
||||
Content: "JavaScript uses Promise.catch for async error handling.",
|
||||
Score: 0.7,
|
||||
},
|
||||
}
|
||||
|
||||
results, err := svc.Rerank(query, candidates, 3)
|
||||
if err != nil {
|
||||
t.Fatalf("Rerank() error = %v", err)
|
||||
}
|
||||
|
||||
if len(results) != 3 {
|
||||
t.Fatalf("Rerank() returned %d results, want 3", len(results))
|
||||
}
|
||||
|
||||
// The Go error handling document should rank higher after reranking
|
||||
var goRank int
|
||||
for i, r := range results {
|
||||
if r.ID == "2" {
|
||||
goRank = i + 1
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if goRank == 0 {
|
||||
t.Error("Go document not found in results")
|
||||
}
|
||||
|
||||
// Verify all results have required fields populated
|
||||
for i, r := range results {
|
||||
if r.ID == "" {
|
||||
t.Errorf("Result %d has empty ID", i)
|
||||
}
|
||||
if r.Content == "" {
|
||||
t.Errorf("Result %d has empty Content", i)
|
||||
}
|
||||
if r.RerankRank != i+1 {
|
||||
t.Errorf("Result %d has RerankRank %d, want %d", i, r.RerankRank, i+1)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestRerankEmpty tests reranking with empty input.
|
||||
func TestRerankEmpty(t *testing.T) {
|
||||
cleanup := initONNX(t)
|
||||
defer cleanup()
|
||||
|
||||
cfg := DefaultConfig()
|
||||
svc, err := NewService(cfg)
|
||||
if err != nil {
|
||||
t.Fatalf("NewService() error = %v", err)
|
||||
}
|
||||
defer svc.Close()
|
||||
|
||||
results, err := svc.Rerank("test query", nil, 10)
|
||||
if err != nil {
|
||||
t.Fatalf("Rerank(nil) error = %v", err)
|
||||
}
|
||||
|
||||
if results != nil {
|
||||
t.Errorf("Rerank(nil) = %v, want nil", results)
|
||||
}
|
||||
|
||||
results, err = svc.Rerank("test query", []Candidate{}, 10)
|
||||
if err != nil {
|
||||
t.Fatalf("Rerank([]) error = %v", err)
|
||||
}
|
||||
|
||||
if results != nil {
|
||||
t.Errorf("Rerank([]) = %v, want nil", results)
|
||||
}
|
||||
}
|
||||
|
||||
// TestRerankLimit tests that limit is respected.
|
||||
func TestRerankLimit(t *testing.T) {
|
||||
cleanup := initONNX(t)
|
||||
defer cleanup()
|
||||
|
||||
cfg := DefaultConfig()
|
||||
svc, err := NewService(cfg)
|
||||
if err != nil {
|
||||
t.Fatalf("NewService() error = %v", err)
|
||||
}
|
||||
defer svc.Close()
|
||||
|
||||
candidates := make([]Candidate, 20)
|
||||
for i := range candidates {
|
||||
candidates[i] = Candidate{
|
||||
ID: string(rune('A' + i)),
|
||||
Content: "Test document content for ranking.",
|
||||
Score: 0.5,
|
||||
}
|
||||
}
|
||||
|
||||
results, err := svc.Rerank("test query", candidates, 5)
|
||||
if err != nil {
|
||||
t.Fatalf("Rerank() error = %v", err)
|
||||
}
|
||||
|
||||
if len(results) != 5 {
|
||||
t.Errorf("Rerank() returned %d results, want 5", len(results))
|
||||
}
|
||||
}
|
||||
|
||||
// TestRerankByScore tests pure cross-encoder ranking.
|
||||
func TestRerankByScore(t *testing.T) {
|
||||
cleanup := initONNX(t)
|
||||
defer cleanup()
|
||||
|
||||
cfg := DefaultConfig()
|
||||
svc, err := NewService(cfg)
|
||||
if err != nil {
|
||||
t.Fatalf("NewService() error = %v", err)
|
||||
}
|
||||
defer svc.Close()
|
||||
|
||||
query := "machine learning algorithms"
|
||||
candidates := []Candidate{
|
||||
{
|
||||
ID: "1",
|
||||
Content: "Cooking recipes for Italian pasta dishes.",
|
||||
Score: 0.9, // High original score
|
||||
},
|
||||
{
|
||||
ID: "2",
|
||||
Content: "Neural networks are a type of machine learning algorithm.",
|
||||
Score: 0.3, // Low original score
|
||||
},
|
||||
}
|
||||
|
||||
results, err := svc.RerankByScore(query, candidates, 2)
|
||||
if err != nil {
|
||||
t.Fatalf("RerankByScore() error = %v", err)
|
||||
}
|
||||
|
||||
// Document 2 should rank first since it's about ML
|
||||
if results[0].ID != "2" {
|
||||
t.Errorf("Expected ML document to rank first, got %v", results[0].ID)
|
||||
}
|
||||
|
||||
// CombinedScore should equal RerankScore when using RerankByScore
|
||||
for _, r := range results {
|
||||
if r.CombinedScore != r.RerankScore {
|
||||
t.Errorf("RerankByScore: CombinedScore (%v) != RerankScore (%v)",
|
||||
r.CombinedScore, r.RerankScore)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestRankImprovement tests that rank improvement is calculated correctly.
|
||||
func TestRankImprovement(t *testing.T) {
|
||||
cleanup := initONNX(t)
|
||||
defer cleanup()
|
||||
|
||||
cfg := DefaultConfig()
|
||||
svc, err := NewService(cfg)
|
||||
if err != nil {
|
||||
t.Fatalf("NewService() error = %v", err)
|
||||
}
|
||||
defer svc.Close()
|
||||
|
||||
// Create candidates where we know the expected reranking
|
||||
candidates := []Candidate{
|
||||
{ID: "A", Content: "Unrelated content about weather forecasting.", Score: 0.9},
|
||||
{ID: "B", Content: "How to fix memory leaks in Go programs.", Score: 0.8},
|
||||
{ID: "C", Content: "More unrelated content about gardening tips.", Score: 0.7},
|
||||
}
|
||||
|
||||
results, err := svc.Rerank("debugging memory issues in Go", candidates, 3)
|
||||
if err != nil {
|
||||
t.Fatalf("Rerank() error = %v", err)
|
||||
}
|
||||
|
||||
for _, r := range results {
|
||||
// RankImprovement = OriginalRank - RerankRank
|
||||
// Positive means moved up, negative means moved down
|
||||
expectedImprovement := r.OriginalRank - r.RerankRank
|
||||
if r.RankImprovement != expectedImprovement {
|
||||
t.Errorf("ID %s: RankImprovement = %d, want %d (orig=%d, new=%d)",
|
||||
r.ID, r.RankImprovement, expectedImprovement,
|
||||
r.OriginalRank, r.RerankRank)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestConcurrentRerank tests concurrent reranking calls.
|
||||
func TestConcurrentRerank(t *testing.T) {
|
||||
cleanup := initONNX(t)
|
||||
defer cleanup()
|
||||
|
||||
cfg := DefaultConfig()
|
||||
svc, err := NewService(cfg)
|
||||
if err != nil {
|
||||
t.Fatalf("NewService() error = %v", err)
|
||||
}
|
||||
defer svc.Close()
|
||||
|
||||
candidates := []Candidate{
|
||||
{ID: "1", Content: "Test document one.", Score: 0.5},
|
||||
{ID: "2", Content: "Test document two.", Score: 0.5},
|
||||
}
|
||||
|
||||
var wg sync.WaitGroup
|
||||
errors := make(chan error, 10)
|
||||
|
||||
for i := 0; i < 10; i++ {
|
||||
wg.Add(1)
|
||||
go func(idx int) {
|
||||
defer wg.Done()
|
||||
_, err := svc.Rerank("concurrent test query", candidates, 2)
|
||||
if err != nil {
|
||||
errors <- err
|
||||
}
|
||||
}(i)
|
||||
}
|
||||
|
||||
wg.Wait()
|
||||
close(errors)
|
||||
|
||||
for err := range errors {
|
||||
t.Errorf("Concurrent Rerank error: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestClose tests service cleanup.
|
||||
func TestClose(t *testing.T) {
|
||||
cleanup := initONNX(t)
|
||||
defer cleanup()
|
||||
|
||||
cfg := DefaultConfig()
|
||||
svc, err := NewService(cfg)
|
||||
if err != nil {
|
||||
t.Fatalf("NewService() error = %v", err)
|
||||
}
|
||||
|
||||
err = svc.Close()
|
||||
if err != nil {
|
||||
t.Errorf("Close() error = %v", err)
|
||||
}
|
||||
|
||||
// Double close should not panic
|
||||
err = svc.Close()
|
||||
if err != nil {
|
||||
t.Errorf("Close() on closed service error = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestMetadataPreserved tests that metadata is preserved through reranking.
|
||||
func TestMetadataPreserved(t *testing.T) {
|
||||
cleanup := initONNX(t)
|
||||
defer cleanup()
|
||||
|
||||
cfg := DefaultConfig()
|
||||
svc, err := NewService(cfg)
|
||||
if err != nil {
|
||||
t.Fatalf("NewService() error = %v", err)
|
||||
}
|
||||
defer svc.Close()
|
||||
|
||||
candidates := []Candidate{
|
||||
{
|
||||
ID: "1",
|
||||
Content: "Test content.",
|
||||
Score: 0.5,
|
||||
Metadata: map[string]any{"custom": "value1", "num": 42},
|
||||
},
|
||||
{
|
||||
ID: "2",
|
||||
Content: "Another test.",
|
||||
Score: 0.5,
|
||||
Metadata: map[string]any{"custom": "value2"},
|
||||
},
|
||||
}
|
||||
|
||||
results, err := svc.Rerank("query", candidates, 2)
|
||||
if err != nil {
|
||||
t.Fatalf("Rerank() error = %v", err)
|
||||
}
|
||||
|
||||
for _, r := range results {
|
||||
if r.Metadata == nil {
|
||||
t.Errorf("Result %s has nil metadata", r.ID)
|
||||
continue
|
||||
}
|
||||
|
||||
// Find original candidate
|
||||
var original *Candidate
|
||||
for i := range candidates {
|
||||
if candidates[i].ID == r.ID {
|
||||
original = &candidates[i]
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if original == nil {
|
||||
t.Errorf("Could not find original for result %s", r.ID)
|
||||
continue
|
||||
}
|
||||
|
||||
// Check metadata preserved
|
||||
if original.Metadata["custom"] != r.Metadata["custom"] {
|
||||
t.Errorf("Metadata not preserved for %s", r.ID)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,168 @@
|
||||
// Package scoring provides importance score calculation for observations.
|
||||
package scoring
|
||||
|
||||
import (
|
||||
"math"
|
||||
"time"
|
||||
|
||||
"github.com/lukaszraczylo/claude-mnemonic/pkg/models"
|
||||
)
|
||||
|
||||
// Calculator computes importance scores for observations.
|
||||
type Calculator struct {
|
||||
config *models.ScoringConfig
|
||||
}
|
||||
|
||||
// NewCalculator creates a new scoring calculator.
|
||||
// If config is nil, uses the default configuration.
|
||||
func NewCalculator(config *models.ScoringConfig) *Calculator {
|
||||
if config == nil {
|
||||
config = models.DefaultScoringConfig()
|
||||
}
|
||||
return &Calculator{config: config}
|
||||
}
|
||||
|
||||
// Calculate computes the importance score for an observation at the given time.
|
||||
//
|
||||
// The scoring formula:
|
||||
//
|
||||
// FinalScore = (BaseScore × TypeWeight × RecencyDecay) + FeedbackContrib + ConceptContrib + RetrievalContrib
|
||||
//
|
||||
// Where:
|
||||
// - BaseScore = 1.0
|
||||
// - TypeWeight = observation type multiplier (e.g., bugfix=1.3, change=0.9)
|
||||
// - RecencyDecay = 0.5^(age_days / half_life_days) - halves every 7 days by default
|
||||
// - FeedbackContrib = user_feedback × feedback_weight
|
||||
// - ConceptContrib = sum(concept_weights) × concept_weight_factor
|
||||
// - RetrievalContrib = log2(retrieval_count + 1) × 0.1 × retrieval_weight
|
||||
func (c *Calculator) Calculate(obs *models.Observation, now time.Time) float64 {
|
||||
// 1. Get base type weight
|
||||
typeWeight := models.TypeBaseScore(obs.Type)
|
||||
|
||||
// 2. Calculate recency decay: 0.5^(age_days / half_life_days)
|
||||
ageDays := now.Sub(time.UnixMilli(obs.CreatedAtEpoch)).Hours() / 24.0
|
||||
if ageDays < 0 {
|
||||
ageDays = 0 // Handle future timestamps gracefully
|
||||
}
|
||||
recencyDecay := math.Pow(0.5, ageDays/c.config.RecencyHalfLifeDays)
|
||||
|
||||
// Core score = 1.0 × type_weight × recency_decay
|
||||
coreScore := 1.0 * typeWeight * recencyDecay
|
||||
|
||||
// 3. User feedback contribution: feedback × weight
|
||||
feedbackContrib := float64(obs.UserFeedback) * c.config.FeedbackWeight
|
||||
|
||||
// 4. Concept boost contribution: sum of matching concept weights × factor
|
||||
conceptBoost := 0.0
|
||||
for _, concept := range obs.Concepts {
|
||||
if weight, ok := c.config.ConceptWeights[concept]; ok {
|
||||
conceptBoost += weight
|
||||
}
|
||||
}
|
||||
conceptContrib := conceptBoost * c.config.ConceptWeight
|
||||
|
||||
// 5. Retrieval boost: log2(count + 1) × 0.1 × weight (diminishing returns)
|
||||
retrievalContrib := 0.0
|
||||
if obs.RetrievalCount > 0 {
|
||||
// log2(count + 1) gives diminishing returns: 1→1, 3→2, 7→3, 15→4, etc.
|
||||
retrievalBoost := math.Log2(float64(obs.RetrievalCount)+1) * 0.1
|
||||
retrievalContrib = retrievalBoost * c.config.RetrievalWeight
|
||||
}
|
||||
|
||||
// Final score with minimum threshold
|
||||
finalScore := coreScore + feedbackContrib + conceptContrib + retrievalContrib
|
||||
if finalScore < c.config.MinScore {
|
||||
finalScore = c.config.MinScore
|
||||
}
|
||||
|
||||
return finalScore
|
||||
}
|
||||
|
||||
// CalculateComponents returns the individual components of the importance score.
|
||||
// Useful for debugging and explaining scores to users.
|
||||
func (c *Calculator) CalculateComponents(obs *models.Observation, now time.Time) ScoreComponents {
|
||||
typeWeight := models.TypeBaseScore(obs.Type)
|
||||
|
||||
ageDays := now.Sub(time.UnixMilli(obs.CreatedAtEpoch)).Hours() / 24.0
|
||||
if ageDays < 0 {
|
||||
ageDays = 0
|
||||
}
|
||||
recencyDecay := math.Pow(0.5, ageDays/c.config.RecencyHalfLifeDays)
|
||||
|
||||
coreScore := 1.0 * typeWeight * recencyDecay
|
||||
feedbackContrib := float64(obs.UserFeedback) * c.config.FeedbackWeight
|
||||
|
||||
conceptBoost := 0.0
|
||||
for _, concept := range obs.Concepts {
|
||||
if weight, ok := c.config.ConceptWeights[concept]; ok {
|
||||
conceptBoost += weight
|
||||
}
|
||||
}
|
||||
conceptContrib := conceptBoost * c.config.ConceptWeight
|
||||
|
||||
retrievalContrib := 0.0
|
||||
if obs.RetrievalCount > 0 {
|
||||
retrievalBoost := math.Log2(float64(obs.RetrievalCount)+1) * 0.1
|
||||
retrievalContrib = retrievalBoost * c.config.RetrievalWeight
|
||||
}
|
||||
|
||||
finalScore := coreScore + feedbackContrib + conceptContrib + retrievalContrib
|
||||
if finalScore < c.config.MinScore {
|
||||
finalScore = c.config.MinScore
|
||||
}
|
||||
|
||||
return ScoreComponents{
|
||||
TypeWeight: typeWeight,
|
||||
RecencyDecay: recencyDecay,
|
||||
CoreScore: coreScore,
|
||||
FeedbackContrib: feedbackContrib,
|
||||
ConceptContrib: conceptContrib,
|
||||
RetrievalContrib: retrievalContrib,
|
||||
FinalScore: finalScore,
|
||||
AgeDays: ageDays,
|
||||
}
|
||||
}
|
||||
|
||||
// ScoreComponents contains the breakdown of an importance score calculation.
|
||||
type ScoreComponents struct {
|
||||
TypeWeight float64 `json:"type_weight"`
|
||||
RecencyDecay float64 `json:"recency_decay"`
|
||||
CoreScore float64 `json:"core_score"`
|
||||
FeedbackContrib float64 `json:"feedback_contrib"`
|
||||
ConceptContrib float64 `json:"concept_contrib"`
|
||||
RetrievalContrib float64 `json:"retrieval_contrib"`
|
||||
FinalScore float64 `json:"final_score"`
|
||||
AgeDays float64 `json:"age_days"`
|
||||
}
|
||||
|
||||
// BatchCalculate computes scores for multiple observations.
|
||||
// Returns a map of observation ID to calculated score.
|
||||
func (c *Calculator) BatchCalculate(observations []*models.Observation, now time.Time) map[int64]float64 {
|
||||
scores := make(map[int64]float64, len(observations))
|
||||
for _, obs := range observations {
|
||||
scores[obs.ID] = c.Calculate(obs, now)
|
||||
}
|
||||
return scores
|
||||
}
|
||||
|
||||
// RecalculateThreshold returns the minimum duration before an observation
|
||||
// should have its score recalculated. This prevents excessive recalculation
|
||||
// while ensuring scores stay reasonably fresh.
|
||||
func (c *Calculator) RecalculateThreshold() time.Duration {
|
||||
// Recalculate at most every 6 hours
|
||||
// This balances freshness with performance
|
||||
return 6 * time.Hour
|
||||
}
|
||||
|
||||
// UpdateConfig updates the calculator's scoring configuration.
|
||||
// This allows runtime tuning of scoring parameters.
|
||||
func (c *Calculator) UpdateConfig(config *models.ScoringConfig) {
|
||||
if config != nil {
|
||||
c.config = config
|
||||
}
|
||||
}
|
||||
|
||||
// GetConfig returns the current scoring configuration.
|
||||
func (c *Calculator) GetConfig() *models.ScoringConfig {
|
||||
return c.config
|
||||
}
|
||||
@@ -0,0 +1,638 @@
|
||||
// Package scoring provides importance score calculation for observations.
|
||||
package scoring
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/lukaszraczylo/claude-mnemonic/pkg/models"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"github.com/stretchr/testify/suite"
|
||||
)
|
||||
|
||||
// CalculatorSuite is a test suite for the Calculator.
|
||||
type CalculatorSuite struct {
|
||||
suite.Suite
|
||||
calc *Calculator
|
||||
config *models.ScoringConfig
|
||||
now time.Time
|
||||
}
|
||||
|
||||
func (s *CalculatorSuite) SetupTest() {
|
||||
s.config = models.DefaultScoringConfig()
|
||||
s.calc = NewCalculator(s.config)
|
||||
s.now = time.Date(2025, 1, 15, 12, 0, 0, 0, time.UTC)
|
||||
}
|
||||
|
||||
func TestCalculatorSuite(t *testing.T) {
|
||||
suite.Run(t, new(CalculatorSuite))
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
// GOOD SCENARIOS - Expected normal operations
|
||||
// =============================================================================
|
||||
|
||||
func (s *CalculatorSuite) TestCalculate_GoodScenarios_NewObservation() {
|
||||
// A brand new observation should have score close to type weight
|
||||
obs := &models.Observation{
|
||||
ID: 1,
|
||||
Type: models.ObsTypeBugfix,
|
||||
CreatedAtEpoch: s.now.UnixMilli(),
|
||||
}
|
||||
|
||||
score := s.calc.Calculate(obs, s.now)
|
||||
|
||||
// Expected: 1.0 × 1.3 (bugfix weight) × 1.0 (no decay) = 1.3
|
||||
s.InDelta(1.3, score, 0.01, "new bugfix should score ~1.3")
|
||||
}
|
||||
|
||||
func (s *CalculatorSuite) TestCalculate_GoodScenarios_OneWeekOld() {
|
||||
// One week old observation should have half the recency score
|
||||
obs := &models.Observation{
|
||||
ID: 1,
|
||||
Type: models.ObsTypeDiscovery,
|
||||
CreatedAtEpoch: s.now.Add(-7 * 24 * time.Hour).UnixMilli(),
|
||||
}
|
||||
|
||||
score := s.calc.Calculate(obs, s.now)
|
||||
|
||||
// Expected: 1.0 × 1.1 (discovery) × 0.5 (7 days half-life) = 0.55
|
||||
s.InDelta(0.55, score, 0.05, "7-day old discovery should score ~0.55")
|
||||
}
|
||||
|
||||
func (s *CalculatorSuite) TestCalculate_GoodScenarios_TwoWeeksOld() {
|
||||
// Two weeks old should have 1/4 recency score
|
||||
obs := &models.Observation{
|
||||
ID: 1,
|
||||
Type: models.ObsTypeFeature,
|
||||
CreatedAtEpoch: s.now.Add(-14 * 24 * time.Hour).UnixMilli(),
|
||||
}
|
||||
|
||||
score := s.calc.Calculate(obs, s.now)
|
||||
|
||||
// Expected: 1.0 × 1.2 (feature) × 0.25 (14 days = 2 half-lives) = 0.30
|
||||
s.InDelta(0.30, score, 0.05, "14-day old feature should score ~0.30")
|
||||
}
|
||||
|
||||
func (s *CalculatorSuite) TestCalculate_GoodScenarios_PositiveFeedback() {
|
||||
// Positive feedback should boost score
|
||||
obs := &models.Observation{
|
||||
ID: 1,
|
||||
Type: models.ObsTypeChange,
|
||||
CreatedAtEpoch: s.now.UnixMilli(),
|
||||
UserFeedback: 1, // thumbs up
|
||||
}
|
||||
|
||||
score := s.calc.Calculate(obs, s.now)
|
||||
|
||||
// Expected: (1.0 × 0.9) + 0.30 (feedback) = 1.20
|
||||
s.InDelta(1.20, score, 0.01, "thumbs up should boost score by 0.30")
|
||||
}
|
||||
|
||||
func (s *CalculatorSuite) TestCalculate_GoodScenarios_NegativeFeedback() {
|
||||
// Negative feedback should reduce score
|
||||
obs := &models.Observation{
|
||||
ID: 1,
|
||||
Type: models.ObsTypeChange,
|
||||
CreatedAtEpoch: s.now.UnixMilli(),
|
||||
UserFeedback: -1, // thumbs down
|
||||
}
|
||||
|
||||
score := s.calc.Calculate(obs, s.now)
|
||||
|
||||
// Expected: (1.0 × 0.9) - 0.30 (feedback) = 0.60
|
||||
s.InDelta(0.60, score, 0.01, "thumbs down should reduce score by 0.30")
|
||||
}
|
||||
|
||||
func (s *CalculatorSuite) TestCalculate_GoodScenarios_WithConcepts() {
|
||||
// Observation with valuable concepts should get boost
|
||||
obs := &models.Observation{
|
||||
ID: 1,
|
||||
Type: models.ObsTypeBugfix,
|
||||
CreatedAtEpoch: s.now.UnixMilli(),
|
||||
Concepts: []string{"security", "gotcha"},
|
||||
}
|
||||
|
||||
score := s.calc.Calculate(obs, s.now)
|
||||
|
||||
// Concept boost: (0.30 + 0.25) × 0.20 = 0.11
|
||||
// Expected: 1.3 + 0.11 = 1.41
|
||||
s.InDelta(1.41, score, 0.05, "security+gotcha concepts should boost score")
|
||||
}
|
||||
|
||||
func (s *CalculatorSuite) TestCalculate_GoodScenarios_WithRetrievals() {
|
||||
// Popular observations should get retrieval boost
|
||||
obs := &models.Observation{
|
||||
ID: 1,
|
||||
Type: models.ObsTypeDiscovery,
|
||||
CreatedAtEpoch: s.now.UnixMilli(),
|
||||
RetrievalCount: 7, // log2(8) = 3
|
||||
}
|
||||
|
||||
score := s.calc.Calculate(obs, s.now)
|
||||
|
||||
// Retrieval boost: log2(7+1) × 0.1 × 0.15 = 3 × 0.1 × 0.15 = 0.045
|
||||
// Expected: 1.1 + 0.045 ≈ 1.145
|
||||
s.InDelta(1.145, score, 0.05, "7 retrievals should add small boost")
|
||||
}
|
||||
|
||||
func (s *CalculatorSuite) TestCalculate_GoodScenarios_CombinedFactors() {
|
||||
// Test with all factors combined
|
||||
obs := &models.Observation{
|
||||
ID: 1,
|
||||
Type: models.ObsTypeBugfix,
|
||||
CreatedAtEpoch: s.now.Add(-7 * 24 * time.Hour).UnixMilli(), // 7 days old
|
||||
UserFeedback: 1,
|
||||
Concepts: []string{"security"},
|
||||
RetrievalCount: 3,
|
||||
}
|
||||
|
||||
score := s.calc.Calculate(obs, s.now)
|
||||
|
||||
// Core: 1.0 × 1.3 × 0.5 = 0.65
|
||||
// Feedback: 0.30
|
||||
// Concept: 0.30 × 0.20 = 0.06
|
||||
// Retrieval: log2(4) × 0.1 × 0.15 = 2 × 0.1 × 0.15 = 0.03
|
||||
// Total ≈ 1.04
|
||||
s.InDelta(1.04, score, 0.1, "combined factors should result in ~1.04")
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
// WORSE SCENARIOS - Degraded but acceptable operations
|
||||
// =============================================================================
|
||||
|
||||
func (s *CalculatorSuite) TestCalculate_WorseScenarios_VeryOldObservation() {
|
||||
// Very old observation should have low but non-zero score
|
||||
obs := &models.Observation{
|
||||
ID: 1,
|
||||
Type: models.ObsTypeChange,
|
||||
CreatedAtEpoch: s.now.Add(-90 * 24 * time.Hour).UnixMilli(), // 90 days old
|
||||
}
|
||||
|
||||
score := s.calc.Calculate(obs, s.now)
|
||||
|
||||
// 90 days = ~12.86 half-lives → decay ≈ 0.00014
|
||||
// Core: 1.0 × 0.9 × 0.00014 = 0.000126
|
||||
// But minimum score is 0.01
|
||||
s.GreaterOrEqual(score, 0.01, "very old observation should still meet minimum")
|
||||
s.Less(score, 0.1, "very old observation should be low scoring")
|
||||
}
|
||||
|
||||
func (s *CalculatorSuite) TestCalculate_WorseScenarios_NegativeFeedbackOld() {
|
||||
// Old observation with negative feedback should still have minimum score
|
||||
obs := &models.Observation{
|
||||
ID: 1,
|
||||
Type: models.ObsTypeChange,
|
||||
CreatedAtEpoch: s.now.Add(-60 * 24 * time.Hour).UnixMilli(),
|
||||
UserFeedback: -1,
|
||||
}
|
||||
|
||||
score := s.calc.Calculate(obs, s.now)
|
||||
|
||||
s.GreaterOrEqual(score, s.config.MinScore, "should never go below minimum score")
|
||||
}
|
||||
|
||||
func (s *CalculatorSuite) TestCalculate_WorseScenarios_UnknownConcepts() {
|
||||
// Unknown concepts should not affect score negatively
|
||||
obs := &models.Observation{
|
||||
ID: 1,
|
||||
Type: models.ObsTypeDiscovery,
|
||||
CreatedAtEpoch: s.now.UnixMilli(),
|
||||
Concepts: []string{"unknown-concept", "another-unknown"},
|
||||
}
|
||||
|
||||
score := s.calc.Calculate(obs, s.now)
|
||||
|
||||
// Should just be the base score without concept boost
|
||||
s.InDelta(1.1, score, 0.01, "unknown concepts should not affect score")
|
||||
}
|
||||
|
||||
func (s *CalculatorSuite) TestCalculate_WorseScenarios_MixedConcepts() {
|
||||
// Mix of known and unknown concepts
|
||||
obs := &models.Observation{
|
||||
ID: 1,
|
||||
Type: models.ObsTypeDiscovery,
|
||||
CreatedAtEpoch: s.now.UnixMilli(),
|
||||
Concepts: []string{"security", "unknown-concept"},
|
||||
}
|
||||
|
||||
score := s.calc.Calculate(obs, s.now)
|
||||
|
||||
// Only security should contribute
|
||||
// Expected: 1.1 + (0.30 × 0.20) = 1.16
|
||||
s.InDelta(1.16, score, 0.05, "only known concepts should boost score")
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
// BAD SCENARIOS - Edge cases and error conditions
|
||||
// =============================================================================
|
||||
|
||||
func (s *CalculatorSuite) TestCalculate_BadScenarios_FutureTimestamp() {
|
||||
// Observation created in the future (clock skew)
|
||||
obs := &models.Observation{
|
||||
ID: 1,
|
||||
Type: models.ObsTypeBugfix,
|
||||
CreatedAtEpoch: s.now.Add(24 * time.Hour).UnixMilli(), // 1 day in future
|
||||
}
|
||||
|
||||
score := s.calc.Calculate(obs, s.now)
|
||||
|
||||
// Should handle gracefully - age should be 0
|
||||
s.InDelta(1.3, score, 0.01, "future timestamp should be treated as now")
|
||||
}
|
||||
|
||||
func (s *CalculatorSuite) TestCalculate_BadScenarios_ZeroEpoch() {
|
||||
// Missing creation timestamp
|
||||
obs := &models.Observation{
|
||||
ID: 1,
|
||||
Type: models.ObsTypeDiscovery,
|
||||
CreatedAtEpoch: 0, // Missing timestamp
|
||||
}
|
||||
|
||||
score := s.calc.Calculate(obs, s.now)
|
||||
|
||||
// This will be treated as very old (1970)
|
||||
s.GreaterOrEqual(score, s.config.MinScore, "should still meet minimum")
|
||||
}
|
||||
|
||||
func (s *CalculatorSuite) TestCalculate_BadScenarios_EmptyObservation() {
|
||||
// Minimal observation with defaults
|
||||
obs := &models.Observation{
|
||||
ID: 1,
|
||||
Type: "", // Empty type
|
||||
CreatedAtEpoch: s.now.UnixMilli(),
|
||||
}
|
||||
|
||||
score := s.calc.Calculate(obs, s.now)
|
||||
|
||||
// Unknown type should default to 1.0 weight
|
||||
s.InDelta(1.0, score, 0.01, "empty type should use default weight 1.0")
|
||||
}
|
||||
|
||||
func (s *CalculatorSuite) TestCalculate_BadScenarios_ExtremeRetrievalCount() {
|
||||
// Very high retrieval count
|
||||
obs := &models.Observation{
|
||||
ID: 1,
|
||||
Type: models.ObsTypeDiscovery,
|
||||
CreatedAtEpoch: s.now.UnixMilli(),
|
||||
RetrievalCount: 1000000, // Extreme value
|
||||
}
|
||||
|
||||
score := s.calc.Calculate(obs, s.now)
|
||||
|
||||
// log2(1000001) ≈ 19.93, so boost = 19.93 × 0.1 × 0.15 ≈ 0.30
|
||||
// Score should be reasonable, not exploding
|
||||
s.Less(score, 2.0, "extreme retrieval count should not explode score")
|
||||
}
|
||||
|
||||
func (s *CalculatorSuite) TestCalculate_BadScenarios_NegativeRetrievalCount() {
|
||||
// Negative retrieval count (should not happen but test defensively)
|
||||
obs := &models.Observation{
|
||||
ID: 1,
|
||||
Type: models.ObsTypeDiscovery,
|
||||
CreatedAtEpoch: s.now.UnixMilli(),
|
||||
RetrievalCount: -5,
|
||||
}
|
||||
|
||||
score := s.calc.Calculate(obs, s.now)
|
||||
|
||||
// Should not panic and should give base score
|
||||
s.InDelta(1.1, score, 0.01, "negative retrieval should be ignored")
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
// EDGE CASES - Boundary conditions
|
||||
// =============================================================================
|
||||
|
||||
func (s *CalculatorSuite) TestCalculate_EdgeCases_ExactlyOneHalfLife() {
|
||||
obs := &models.Observation{
|
||||
ID: 1,
|
||||
Type: models.ObsTypeChange, // 0.9 weight
|
||||
CreatedAtEpoch: s.now.Add(-7 * 24 * time.Hour).UnixMilli(),
|
||||
}
|
||||
|
||||
score := s.calc.Calculate(obs, s.now)
|
||||
s.InDelta(0.45, score, 0.01, "exactly 7 days should give 0.5 decay")
|
||||
}
|
||||
|
||||
func (s *CalculatorSuite) TestCalculate_EdgeCases_ExactlyTwoHalfLives() {
|
||||
obs := &models.Observation{
|
||||
ID: 1,
|
||||
Type: models.ObsTypeChange,
|
||||
CreatedAtEpoch: s.now.Add(-14 * 24 * time.Hour).UnixMilli(),
|
||||
}
|
||||
|
||||
score := s.calc.Calculate(obs, s.now)
|
||||
s.InDelta(0.225, score, 0.01, "exactly 14 days should give 0.25 decay")
|
||||
}
|
||||
|
||||
func (s *CalculatorSuite) TestCalculate_EdgeCases_AllTypeWeights() {
|
||||
types := []struct {
|
||||
t models.ObservationType
|
||||
weight float64
|
||||
}{
|
||||
{models.ObsTypeBugfix, 1.3},
|
||||
{models.ObsTypeFeature, 1.2},
|
||||
{models.ObsTypeDiscovery, 1.1},
|
||||
{models.ObsTypeDecision, 1.1},
|
||||
{models.ObsTypeRefactor, 1.0},
|
||||
{models.ObsTypeChange, 0.9},
|
||||
}
|
||||
|
||||
for _, tt := range types {
|
||||
s.Run(string(tt.t), func() {
|
||||
obs := &models.Observation{
|
||||
ID: 1,
|
||||
Type: tt.t,
|
||||
CreatedAtEpoch: s.now.UnixMilli(),
|
||||
}
|
||||
score := s.calc.Calculate(obs, s.now)
|
||||
s.InDelta(tt.weight, score, 0.01)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func (s *CalculatorSuite) TestCalculate_EdgeCases_MinimumScoreEnforced() {
|
||||
// Create worst case scenario
|
||||
obs := &models.Observation{
|
||||
ID: 1,
|
||||
Type: models.ObsTypeChange, // Lowest weight 0.9
|
||||
CreatedAtEpoch: time.Date(2000, 1, 1, 0, 0, 0, 0, time.UTC).UnixMilli(), // Very old
|
||||
UserFeedback: -1, // Negative feedback
|
||||
}
|
||||
|
||||
score := s.calc.Calculate(obs, s.now)
|
||||
|
||||
s.Equal(s.config.MinScore, score, "should be exactly minimum score")
|
||||
}
|
||||
|
||||
func (s *CalculatorSuite) TestCalculate_EdgeCases_AllConceptsMaxWeight() {
|
||||
// Observation with all high-value concepts
|
||||
obs := &models.Observation{
|
||||
ID: 1,
|
||||
Type: models.ObsTypeBugfix,
|
||||
CreatedAtEpoch: s.now.UnixMilli(),
|
||||
Concepts: []string{"security", "gotcha", "best-practice", "anti-pattern"},
|
||||
}
|
||||
|
||||
score := s.calc.Calculate(obs, s.now)
|
||||
|
||||
// security=0.30, gotcha=0.25, best-practice=0.20, anti-pattern=0.20 = 0.95
|
||||
// Concept contrib: 0.95 × 0.20 = 0.19
|
||||
// Total: 1.3 + 0.19 = 1.49
|
||||
s.InDelta(1.49, score, 0.05, "all high-value concepts should boost significantly")
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
// CALCULATE COMPONENTS TESTS
|
||||
// =============================================================================
|
||||
|
||||
func (s *CalculatorSuite) TestCalculateComponents_ReturnsAllComponents() {
|
||||
obs := &models.Observation{
|
||||
ID: 1,
|
||||
Type: models.ObsTypeBugfix,
|
||||
CreatedAtEpoch: s.now.Add(-7 * 24 * time.Hour).UnixMilli(),
|
||||
UserFeedback: 1,
|
||||
Concepts: []string{"security"},
|
||||
RetrievalCount: 7,
|
||||
}
|
||||
|
||||
components := s.calc.CalculateComponents(obs, s.now)
|
||||
|
||||
s.InDelta(1.3, components.TypeWeight, 0.01)
|
||||
s.InDelta(0.5, components.RecencyDecay, 0.01)
|
||||
s.InDelta(0.65, components.CoreScore, 0.05)
|
||||
s.InDelta(0.30, components.FeedbackContrib, 0.01)
|
||||
s.InDelta(0.06, components.ConceptContrib, 0.02)
|
||||
s.Greater(components.RetrievalContrib, 0.0)
|
||||
s.InDelta(7.0, components.AgeDays, 0.1)
|
||||
s.Greater(components.FinalScore, 0.0)
|
||||
}
|
||||
|
||||
func (s *CalculatorSuite) TestCalculateComponents_MatchesCalculate() {
|
||||
obs := &models.Observation{
|
||||
ID: 1,
|
||||
Type: models.ObsTypeFeature,
|
||||
CreatedAtEpoch: s.now.Add(-3 * 24 * time.Hour).UnixMilli(),
|
||||
UserFeedback: -1,
|
||||
Concepts: []string{"performance", "architecture"},
|
||||
RetrievalCount: 15,
|
||||
}
|
||||
|
||||
score := s.calc.Calculate(obs, s.now)
|
||||
components := s.calc.CalculateComponents(obs, s.now)
|
||||
|
||||
s.InDelta(score, components.FinalScore, 0.001, "Calculate and CalculateComponents should match")
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
// BATCH CALCULATE TESTS
|
||||
// =============================================================================
|
||||
|
||||
func (s *CalculatorSuite) TestBatchCalculate_Empty() {
|
||||
scores := s.calc.BatchCalculate(nil, s.now)
|
||||
s.Empty(scores)
|
||||
|
||||
scores = s.calc.BatchCalculate([]*models.Observation{}, s.now)
|
||||
s.Empty(scores)
|
||||
}
|
||||
|
||||
func (s *CalculatorSuite) TestBatchCalculate_Multiple() {
|
||||
obs := []*models.Observation{
|
||||
{ID: 1, Type: models.ObsTypeBugfix, CreatedAtEpoch: s.now.UnixMilli()},
|
||||
{ID: 2, Type: models.ObsTypeFeature, CreatedAtEpoch: s.now.Add(-7 * 24 * time.Hour).UnixMilli()},
|
||||
{ID: 3, Type: models.ObsTypeChange, CreatedAtEpoch: s.now.Add(-14 * 24 * time.Hour).UnixMilli()},
|
||||
}
|
||||
|
||||
scores := s.calc.BatchCalculate(obs, s.now)
|
||||
|
||||
s.Len(scores, 3)
|
||||
s.Contains(scores, int64(1))
|
||||
s.Contains(scores, int64(2))
|
||||
s.Contains(scores, int64(3))
|
||||
|
||||
s.InDelta(1.3, scores[1], 0.01) // New bugfix
|
||||
s.InDelta(0.6, scores[2], 0.1) // 7-day feature
|
||||
s.InDelta(0.225, scores[3], 0.05) // 14-day change
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
// CONFIGURATION TESTS
|
||||
// =============================================================================
|
||||
|
||||
func (s *CalculatorSuite) TestNewCalculator_NilConfig() {
|
||||
calc := NewCalculator(nil)
|
||||
s.NotNil(calc)
|
||||
s.NotNil(calc.config)
|
||||
s.Equal(7.0, calc.config.RecencyHalfLifeDays)
|
||||
}
|
||||
|
||||
func (s *CalculatorSuite) TestUpdateConfig() {
|
||||
newConfig := &models.ScoringConfig{
|
||||
RecencyHalfLifeDays: 14.0, // Changed from 7
|
||||
FeedbackWeight: 0.50,
|
||||
ConceptWeight: 0.10,
|
||||
RetrievalWeight: 0.05,
|
||||
MinScore: 0.001,
|
||||
ConceptWeights: map[string]float64{"test": 0.5},
|
||||
}
|
||||
|
||||
s.calc.UpdateConfig(newConfig)
|
||||
|
||||
obs := &models.Observation{
|
||||
ID: 1,
|
||||
Type: models.ObsTypeChange,
|
||||
CreatedAtEpoch: s.now.Add(-14 * 24 * time.Hour).UnixMilli(),
|
||||
}
|
||||
|
||||
score := s.calc.Calculate(obs, s.now)
|
||||
|
||||
// With 14-day half-life, 14 days = exactly one half-life
|
||||
// Expected: 1.0 × 0.9 × 0.5 = 0.45
|
||||
s.InDelta(0.45, score, 0.01)
|
||||
}
|
||||
|
||||
func (s *CalculatorSuite) TestUpdateConfig_NilIgnored() {
|
||||
originalConfig := s.calc.GetConfig()
|
||||
s.calc.UpdateConfig(nil)
|
||||
s.Equal(originalConfig, s.calc.GetConfig())
|
||||
}
|
||||
|
||||
func (s *CalculatorSuite) TestGetConfig() {
|
||||
config := s.calc.GetConfig()
|
||||
s.NotNil(config)
|
||||
s.Equal(7.0, config.RecencyHalfLifeDays)
|
||||
}
|
||||
|
||||
func (s *CalculatorSuite) TestRecalculateThreshold() {
|
||||
threshold := s.calc.RecalculateThreshold()
|
||||
s.Equal(6*time.Hour, threshold)
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
// STANDALONE TESTS (non-suite)
|
||||
// =============================================================================
|
||||
|
||||
func TestNewCalculator_DefaultConfig(t *testing.T) {
|
||||
calc := NewCalculator(nil)
|
||||
require.NotNil(t, calc)
|
||||
assert.Equal(t, 7.0, calc.config.RecencyHalfLifeDays)
|
||||
assert.Equal(t, 0.30, calc.config.FeedbackWeight)
|
||||
assert.Equal(t, 0.01, calc.config.MinScore)
|
||||
}
|
||||
|
||||
func TestCalculator_ConcurrentAccess(t *testing.T) {
|
||||
calc := NewCalculator(nil)
|
||||
now := time.Now()
|
||||
|
||||
// Test that calculator is safe for concurrent reads
|
||||
done := make(chan bool, 10)
|
||||
for i := 0; i < 10; i++ {
|
||||
go func(id int64) {
|
||||
obs := &models.Observation{
|
||||
ID: id,
|
||||
Type: models.ObsTypeBugfix,
|
||||
CreatedAtEpoch: now.UnixMilli(),
|
||||
}
|
||||
score := calc.Calculate(obs, now)
|
||||
assert.Greater(t, score, 0.0)
|
||||
done <- true
|
||||
}(int64(i))
|
||||
}
|
||||
|
||||
for i := 0; i < 10; i++ {
|
||||
<-done
|
||||
}
|
||||
}
|
||||
|
||||
func TestCalculator_DecayPrecision(t *testing.T) {
|
||||
calc := NewCalculator(nil)
|
||||
now := time.Now()
|
||||
|
||||
// Test that decay is mathematically correct
|
||||
testCases := []struct {
|
||||
days int
|
||||
expectedDecay float64
|
||||
}{
|
||||
{0, 1.0},
|
||||
{7, 0.5},
|
||||
{14, 0.25},
|
||||
{21, 0.125},
|
||||
{28, 0.0625},
|
||||
}
|
||||
|
||||
for _, tc := range testCases {
|
||||
t.Run(string(rune('0'+tc.days/7))+"_half_lives", func(t *testing.T) {
|
||||
obs := &models.Observation{
|
||||
ID: 1,
|
||||
Type: models.ObsTypeRefactor, // 1.0 weight
|
||||
CreatedAtEpoch: now.Add(-time.Duration(tc.days) * 24 * time.Hour).UnixMilli(),
|
||||
}
|
||||
components := calc.CalculateComponents(obs, now)
|
||||
assert.InDelta(t, tc.expectedDecay, components.RecencyDecay, 0.001)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestTypeBaseScore_UnknownType(t *testing.T) {
|
||||
score := models.TypeBaseScore("unknown-type")
|
||||
assert.Equal(t, 1.0, score, "unknown type should default to 1.0")
|
||||
}
|
||||
|
||||
func TestTypeBaseScore_AllKnownTypes(t *testing.T) {
|
||||
expected := map[models.ObservationType]float64{
|
||||
models.ObsTypeBugfix: 1.3,
|
||||
models.ObsTypeFeature: 1.2,
|
||||
models.ObsTypeDiscovery: 1.1,
|
||||
models.ObsTypeDecision: 1.1,
|
||||
models.ObsTypeRefactor: 1.0,
|
||||
models.ObsTypeChange: 0.9,
|
||||
}
|
||||
|
||||
for obsType, expectedScore := range expected {
|
||||
t.Run(string(obsType), func(t *testing.T) {
|
||||
score := models.TypeBaseScore(obsType)
|
||||
assert.Equal(t, expectedScore, score)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestCalculator_RetrievalBoostDiminishingReturns(t *testing.T) {
|
||||
calc := NewCalculator(nil)
|
||||
now := time.Now()
|
||||
|
||||
// Test that retrieval boost has diminishing returns
|
||||
// When retrieval count doubles, the boost should NOT double (log2 gives diminishing returns)
|
||||
|
||||
// Collect boosts for different counts
|
||||
boosts := make([]float64, 0)
|
||||
retrievalCounts := []int{1, 3, 7, 15, 31, 63, 127}
|
||||
|
||||
for _, count := range retrievalCounts {
|
||||
obs := &models.Observation{
|
||||
ID: 1,
|
||||
Type: models.ObsTypeRefactor,
|
||||
CreatedAtEpoch: now.UnixMilli(),
|
||||
RetrievalCount: count,
|
||||
}
|
||||
components := calc.CalculateComponents(obs, now)
|
||||
boosts = append(boosts, components.RetrievalContrib)
|
||||
}
|
||||
|
||||
// Verify boost increases but at a decreasing rate
|
||||
for i := 1; i < len(boosts); i++ {
|
||||
// Each boost should be higher than the previous
|
||||
assert.Greater(t, boosts[i], boosts[i-1],
|
||||
"boost should increase with more retrievals")
|
||||
|
||||
// But not proportionally - calculate the ratios
|
||||
if i >= 2 {
|
||||
ratio1 := boosts[i-1] / boosts[i-2]
|
||||
ratio2 := boosts[i] / boosts[i-1]
|
||||
// The growth ratio should be decreasing (diminishing returns)
|
||||
assert.Less(t, ratio2, ratio1+0.01, // Allow small floating point tolerance
|
||||
"growth rate should decrease (diminishing returns)")
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,186 @@
|
||||
// Package scoring provides importance score calculation for observations.
|
||||
package scoring
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/rs/zerolog"
|
||||
|
||||
"github.com/lukaszraczylo/claude-mnemonic/internal/db/gorm"
|
||||
"github.com/lukaszraczylo/claude-mnemonic/pkg/models"
|
||||
)
|
||||
|
||||
// ObservationStore defines the interface for observation storage operations needed by the recalculator.
|
||||
type ObservationStore interface {
|
||||
GetObservationsNeedingScoreUpdate(ctx context.Context, threshold time.Duration, limit int) ([]*models.Observation, error)
|
||||
UpdateImportanceScores(ctx context.Context, scores map[int64]float64) error
|
||||
GetConceptWeights(ctx context.Context) (map[string]float64, error)
|
||||
}
|
||||
|
||||
// Recalculator periodically recalculates importance scores for observations.
|
||||
type Recalculator struct {
|
||||
store ObservationStore
|
||||
calculator *Calculator
|
||||
log zerolog.Logger
|
||||
interval time.Duration
|
||||
batchSize int
|
||||
stopCh chan struct{}
|
||||
doneCh chan struct{}
|
||||
mu sync.Mutex
|
||||
running bool
|
||||
}
|
||||
|
||||
// NewRecalculator creates a new background recalculator.
|
||||
func NewRecalculator(store ObservationStore, calc *Calculator, log zerolog.Logger) *Recalculator {
|
||||
return &Recalculator{
|
||||
store: store,
|
||||
calculator: calc,
|
||||
log: log.With().Str("component", "recalculator").Logger(),
|
||||
interval: 1 * time.Hour, // Run every hour
|
||||
batchSize: 500, // Process 500 observations at a time
|
||||
stopCh: make(chan struct{}),
|
||||
doneCh: make(chan struct{}),
|
||||
}
|
||||
}
|
||||
|
||||
// Start begins the background recalculation loop.
|
||||
// This should be called in a goroutine.
|
||||
func (r *Recalculator) Start(ctx context.Context) {
|
||||
r.mu.Lock()
|
||||
if r.running {
|
||||
r.mu.Unlock()
|
||||
return
|
||||
}
|
||||
r.running = true
|
||||
r.mu.Unlock()
|
||||
|
||||
defer func() {
|
||||
r.mu.Lock()
|
||||
r.running = false
|
||||
r.mu.Unlock()
|
||||
close(r.doneCh)
|
||||
}()
|
||||
|
||||
// Initial run
|
||||
r.recalculate(ctx)
|
||||
|
||||
r.mu.Lock()
|
||||
interval := r.interval
|
||||
r.mu.Unlock()
|
||||
|
||||
ticker := time.NewTicker(interval)
|
||||
defer ticker.Stop()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
r.log.Info().Msg("recalculator shutting down due to context cancellation")
|
||||
return
|
||||
case <-r.stopCh:
|
||||
r.log.Info().Msg("recalculator stopping")
|
||||
return
|
||||
case <-ticker.C:
|
||||
r.recalculate(ctx)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Stop stops the background recalculation loop.
|
||||
func (r *Recalculator) Stop() {
|
||||
r.mu.Lock()
|
||||
if !r.running {
|
||||
r.mu.Unlock()
|
||||
return
|
||||
}
|
||||
r.mu.Unlock()
|
||||
|
||||
close(r.stopCh)
|
||||
<-r.doneCh
|
||||
}
|
||||
|
||||
// recalculate performs a single recalculation batch.
|
||||
func (r *Recalculator) recalculate(ctx context.Context) {
|
||||
now := time.Now()
|
||||
threshold := r.calculator.RecalculateThreshold()
|
||||
|
||||
r.mu.Lock()
|
||||
batchSize := r.batchSize
|
||||
r.mu.Unlock()
|
||||
|
||||
observations, err := r.store.GetObservationsNeedingScoreUpdate(ctx, threshold, batchSize)
|
||||
if err != nil {
|
||||
r.log.Error().Err(err).Msg("failed to get observations for score update")
|
||||
return
|
||||
}
|
||||
|
||||
if len(observations) == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
scores := r.calculator.BatchCalculate(observations, now)
|
||||
|
||||
if err := r.store.UpdateImportanceScores(ctx, scores); err != nil {
|
||||
r.log.Error().Err(err).Msg("failed to update importance scores")
|
||||
return
|
||||
}
|
||||
|
||||
r.log.Info().
|
||||
Int("count", len(scores)).
|
||||
Dur("elapsed", time.Since(now)).
|
||||
Msg("recalculated importance scores")
|
||||
}
|
||||
|
||||
// RecalculateNow triggers an immediate recalculation.
|
||||
// This is useful for testing or when scores need to be updated urgently.
|
||||
func (r *Recalculator) RecalculateNow(ctx context.Context) error {
|
||||
r.recalculate(ctx)
|
||||
return nil
|
||||
}
|
||||
|
||||
// RefreshConceptWeights reloads concept weights from the database.
|
||||
// Call this after updating concept weights to apply changes.
|
||||
func (r *Recalculator) RefreshConceptWeights(ctx context.Context) error {
|
||||
weights, err := r.store.GetConceptWeights(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
config := r.calculator.GetConfig()
|
||||
config.ConceptWeights = weights
|
||||
r.calculator.UpdateConfig(config)
|
||||
|
||||
r.log.Info().Int("count", len(weights)).Msg("refreshed concept weights")
|
||||
return nil
|
||||
}
|
||||
|
||||
// Stats returns statistics about the recalculator.
|
||||
type Stats struct {
|
||||
Running bool `json:"running"`
|
||||
Interval time.Duration `json:"interval"`
|
||||
BatchSize int `json:"batch_size"`
|
||||
HalfLife float64 `json:"half_life_days"`
|
||||
MinScore float64 `json:"min_score"`
|
||||
ConceptsLen int `json:"concepts_count"`
|
||||
}
|
||||
|
||||
// GetStats returns current recalculator statistics.
|
||||
func (r *Recalculator) GetStats() Stats {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
|
||||
config := r.calculator.GetConfig()
|
||||
|
||||
return Stats{
|
||||
Running: r.running,
|
||||
Interval: r.interval,
|
||||
BatchSize: r.batchSize,
|
||||
HalfLife: config.RecencyHalfLifeDays,
|
||||
MinScore: config.MinScore,
|
||||
ConceptsLen: len(config.ConceptWeights),
|
||||
}
|
||||
}
|
||||
|
||||
// Ensure ObservationStore satisfies the interface
|
||||
var _ ObservationStore = (*gorm.ObservationStore)(nil)
|
||||
@@ -0,0 +1,447 @@
|
||||
// Package scoring provides importance score calculation for observations.
|
||||
package scoring
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/rs/zerolog"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/lukaszraczylo/claude-mnemonic/pkg/models"
|
||||
)
|
||||
|
||||
// MockObservationStore is a mock implementation of ObservationStore for testing.
|
||||
type MockObservationStore struct {
|
||||
mu sync.Mutex
|
||||
observations []*models.Observation
|
||||
scores map[int64]float64
|
||||
conceptWeights map[string]float64
|
||||
updateErr error
|
||||
getErr error
|
||||
getConceptsErr error
|
||||
updateScoresCalls int
|
||||
}
|
||||
|
||||
func NewMockObservationStore() *MockObservationStore {
|
||||
return &MockObservationStore{
|
||||
observations: []*models.Observation{},
|
||||
scores: make(map[int64]float64),
|
||||
conceptWeights: make(map[string]float64),
|
||||
}
|
||||
}
|
||||
|
||||
func (m *MockObservationStore) GetObservationsNeedingScoreUpdate(ctx context.Context, threshold time.Duration, limit int) ([]*models.Observation, error) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
|
||||
if m.getErr != nil {
|
||||
return nil, m.getErr
|
||||
}
|
||||
|
||||
// Return observations that haven't been updated within threshold
|
||||
now := time.Now()
|
||||
var result []*models.Observation
|
||||
for _, obs := range m.observations {
|
||||
if !obs.ScoreUpdatedAt.Valid || now.Sub(time.Unix(obs.ScoreUpdatedAt.Int64, 0)) > threshold {
|
||||
result = append(result, obs)
|
||||
if len(result) >= limit {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (m *MockObservationStore) UpdateImportanceScores(ctx context.Context, scores map[int64]float64) error {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
|
||||
m.updateScoresCalls++
|
||||
if m.updateErr != nil {
|
||||
return m.updateErr
|
||||
}
|
||||
|
||||
for id, score := range scores {
|
||||
m.scores[id] = score
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *MockObservationStore) GetConceptWeights(ctx context.Context) (map[string]float64, error) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
|
||||
if m.getConceptsErr != nil {
|
||||
return nil, m.getConceptsErr
|
||||
}
|
||||
return m.conceptWeights, nil
|
||||
}
|
||||
|
||||
func (m *MockObservationStore) AddObservation(obs *models.Observation) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
m.observations = append(m.observations, obs)
|
||||
}
|
||||
|
||||
func (m *MockObservationStore) SetConceptWeights(weights map[string]float64) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
m.conceptWeights = weights
|
||||
}
|
||||
|
||||
func (m *MockObservationStore) GetScore(id int64) (float64, bool) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
score, ok := m.scores[id]
|
||||
return score, ok
|
||||
}
|
||||
|
||||
func (m *MockObservationStore) GetUpdateScoresCalls() int {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
return m.updateScoresCalls
|
||||
}
|
||||
|
||||
func (m *MockObservationStore) SetUpdateError(err error) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
m.updateErr = err
|
||||
}
|
||||
|
||||
func (m *MockObservationStore) SetGetError(err error) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
m.getErr = err
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
// RECALCULATOR TESTS
|
||||
// =============================================================================
|
||||
|
||||
func TestNewRecalculator(t *testing.T) {
|
||||
store := NewMockObservationStore()
|
||||
calc := NewCalculator(nil)
|
||||
log := zerolog.Nop()
|
||||
|
||||
recalc := NewRecalculator(store, calc, log)
|
||||
|
||||
require.NotNil(t, recalc)
|
||||
assert.NotNil(t, recalc.store)
|
||||
assert.NotNil(t, recalc.calculator)
|
||||
assert.Equal(t, 1*time.Hour, recalc.interval)
|
||||
assert.Equal(t, 500, recalc.batchSize)
|
||||
assert.False(t, recalc.running)
|
||||
}
|
||||
|
||||
func TestRecalculator_RecalculateNow(t *testing.T) {
|
||||
store := NewMockObservationStore()
|
||||
calc := NewCalculator(nil)
|
||||
log := zerolog.Nop()
|
||||
recalc := NewRecalculator(store, calc, log)
|
||||
|
||||
// Add observations
|
||||
now := time.Now()
|
||||
store.AddObservation(&models.Observation{
|
||||
ID: 1,
|
||||
Type: models.ObsTypeBugfix,
|
||||
CreatedAtEpoch: now.UnixMilli(),
|
||||
})
|
||||
store.AddObservation(&models.Observation{
|
||||
ID: 2,
|
||||
Type: models.ObsTypeFeature,
|
||||
CreatedAtEpoch: now.Add(-7 * 24 * time.Hour).UnixMilli(),
|
||||
})
|
||||
|
||||
ctx := context.Background()
|
||||
err := recalc.RecalculateNow(ctx)
|
||||
|
||||
require.NoError(t, err)
|
||||
|
||||
// Verify scores were calculated
|
||||
score1, ok := store.GetScore(1)
|
||||
assert.True(t, ok)
|
||||
assert.Greater(t, score1, 0.0)
|
||||
|
||||
score2, ok := store.GetScore(2)
|
||||
assert.True(t, ok)
|
||||
assert.Greater(t, score2, 0.0)
|
||||
assert.Less(t, score2, score1, "older observation should have lower score")
|
||||
}
|
||||
|
||||
func TestRecalculator_RefreshConceptWeights(t *testing.T) {
|
||||
store := NewMockObservationStore()
|
||||
calc := NewCalculator(nil)
|
||||
log := zerolog.Nop()
|
||||
recalc := NewRecalculator(store, calc, log)
|
||||
|
||||
// Set up custom weights in store
|
||||
customWeights := map[string]float64{
|
||||
"security": 0.50,
|
||||
"performance": 0.25,
|
||||
}
|
||||
store.SetConceptWeights(customWeights)
|
||||
|
||||
ctx := context.Background()
|
||||
err := recalc.RefreshConceptWeights(ctx)
|
||||
|
||||
require.NoError(t, err)
|
||||
|
||||
// Verify weights were updated in calculator config
|
||||
config := calc.GetConfig()
|
||||
assert.Equal(t, 0.50, config.ConceptWeights["security"])
|
||||
assert.Equal(t, 0.25, config.ConceptWeights["performance"])
|
||||
}
|
||||
|
||||
func TestRecalculator_RefreshConceptWeights_Error(t *testing.T) {
|
||||
store := NewMockObservationStore()
|
||||
calc := NewCalculator(nil)
|
||||
log := zerolog.Nop()
|
||||
recalc := NewRecalculator(store, calc, log)
|
||||
|
||||
store.getConceptsErr = assert.AnError
|
||||
|
||||
ctx := context.Background()
|
||||
err := recalc.RefreshConceptWeights(ctx)
|
||||
|
||||
require.Error(t, err)
|
||||
}
|
||||
|
||||
func TestRecalculator_GetStats(t *testing.T) {
|
||||
store := NewMockObservationStore()
|
||||
calc := NewCalculator(nil)
|
||||
log := zerolog.Nop()
|
||||
recalc := NewRecalculator(store, calc, log)
|
||||
|
||||
// Set fields directly for testing
|
||||
recalc.interval = 2 * time.Hour
|
||||
recalc.batchSize = 250
|
||||
|
||||
stats := recalc.GetStats()
|
||||
|
||||
assert.False(t, stats.Running)
|
||||
assert.Equal(t, 2*time.Hour, stats.Interval)
|
||||
assert.Equal(t, 250, stats.BatchSize)
|
||||
assert.Equal(t, 7.0, stats.HalfLife)
|
||||
assert.Equal(t, 0.01, stats.MinScore)
|
||||
}
|
||||
|
||||
func TestRecalculator_StartStop(t *testing.T) {
|
||||
store := NewMockObservationStore()
|
||||
calc := NewCalculator(nil)
|
||||
log := zerolog.Nop()
|
||||
recalc := NewRecalculator(store, calc, log)
|
||||
|
||||
// Use a short interval for testing
|
||||
recalc.interval = 50 * time.Millisecond
|
||||
|
||||
// Add an observation
|
||||
store.AddObservation(&models.Observation{
|
||||
ID: 1,
|
||||
Type: models.ObsTypeBugfix,
|
||||
CreatedAtEpoch: time.Now().UnixMilli(),
|
||||
})
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
|
||||
// Start in goroutine
|
||||
go recalc.Start(ctx)
|
||||
|
||||
// Wait for initial run and at least one tick
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
|
||||
// Verify it ran
|
||||
calls := store.GetUpdateScoresCalls()
|
||||
assert.GreaterOrEqual(t, calls, 1, "should have run at least once")
|
||||
|
||||
// Stop via context cancellation
|
||||
cancel()
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
|
||||
// Verify stopped
|
||||
stats := recalc.GetStats()
|
||||
assert.False(t, stats.Running)
|
||||
}
|
||||
|
||||
func TestRecalculator_StartTwice(t *testing.T) {
|
||||
store := NewMockObservationStore()
|
||||
calc := NewCalculator(nil)
|
||||
log := zerolog.Nop()
|
||||
recalc := NewRecalculator(store, calc, log)
|
||||
|
||||
recalc.interval = 1 * time.Hour // Long interval so it doesn't tick during test
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
|
||||
// Start first time
|
||||
go recalc.Start(ctx)
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
|
||||
// Try to start second time (should return immediately)
|
||||
go recalc.Start(ctx)
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
|
||||
// Should still be running (only once)
|
||||
stats := recalc.GetStats()
|
||||
assert.True(t, stats.Running)
|
||||
|
||||
cancel()
|
||||
}
|
||||
|
||||
func TestRecalculator_StopWhenNotRunning(t *testing.T) {
|
||||
store := NewMockObservationStore()
|
||||
calc := NewCalculator(nil)
|
||||
log := zerolog.Nop()
|
||||
recalc := NewRecalculator(store, calc, log)
|
||||
|
||||
// Stop without starting - should not panic
|
||||
recalc.Stop()
|
||||
|
||||
stats := recalc.GetStats()
|
||||
assert.False(t, stats.Running)
|
||||
}
|
||||
|
||||
func TestRecalculator_EmptyStore(t *testing.T) {
|
||||
store := NewMockObservationStore()
|
||||
calc := NewCalculator(nil)
|
||||
log := zerolog.Nop()
|
||||
recalc := NewRecalculator(store, calc, log)
|
||||
|
||||
ctx := context.Background()
|
||||
err := recalc.RecalculateNow(ctx)
|
||||
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 0, store.GetUpdateScoresCalls(), "should not call update with no observations")
|
||||
}
|
||||
|
||||
func TestRecalculator_GetObservationsError(t *testing.T) {
|
||||
store := NewMockObservationStore()
|
||||
calc := NewCalculator(nil)
|
||||
log := zerolog.Nop()
|
||||
recalc := NewRecalculator(store, calc, log)
|
||||
|
||||
store.SetGetError(assert.AnError)
|
||||
|
||||
ctx := context.Background()
|
||||
err := recalc.RecalculateNow(ctx)
|
||||
|
||||
// Should not return error (logs it instead)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 0, store.GetUpdateScoresCalls())
|
||||
}
|
||||
|
||||
func TestRecalculator_UpdateScoresError(t *testing.T) {
|
||||
store := NewMockObservationStore()
|
||||
calc := NewCalculator(nil)
|
||||
log := zerolog.Nop()
|
||||
recalc := NewRecalculator(store, calc, log)
|
||||
|
||||
store.AddObservation(&models.Observation{
|
||||
ID: 1,
|
||||
Type: models.ObsTypeBugfix,
|
||||
CreatedAtEpoch: time.Now().UnixMilli(),
|
||||
})
|
||||
|
||||
store.SetUpdateError(assert.AnError)
|
||||
|
||||
ctx := context.Background()
|
||||
err := recalc.RecalculateNow(ctx)
|
||||
|
||||
// Should not return error (logs it instead)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 1, store.GetUpdateScoresCalls())
|
||||
}
|
||||
|
||||
func TestRecalculator_BatchProcessing(t *testing.T) {
|
||||
store := NewMockObservationStore()
|
||||
calc := NewCalculator(nil)
|
||||
log := zerolog.Nop()
|
||||
recalc := NewRecalculator(store, calc, log)
|
||||
|
||||
// Set small batch size
|
||||
recalc.batchSize = 3
|
||||
|
||||
// Add 5 observations
|
||||
now := time.Now()
|
||||
for i := 1; i <= 5; i++ {
|
||||
store.AddObservation(&models.Observation{
|
||||
ID: int64(i),
|
||||
Type: models.ObsTypeBugfix,
|
||||
CreatedAtEpoch: now.UnixMilli(),
|
||||
})
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
err := recalc.RecalculateNow(ctx)
|
||||
|
||||
require.NoError(t, err)
|
||||
|
||||
// Should only process batch size (3)
|
||||
scores := 0
|
||||
for i := 1; i <= 5; i++ {
|
||||
if _, ok := store.GetScore(int64(i)); ok {
|
||||
scores++
|
||||
}
|
||||
}
|
||||
assert.Equal(t, 3, scores, "should only process batch size observations")
|
||||
}
|
||||
|
||||
func TestRecalculator_ConcurrentAccess(t *testing.T) {
|
||||
store := NewMockObservationStore()
|
||||
calc := NewCalculator(nil)
|
||||
log := zerolog.Nop()
|
||||
recalc := NewRecalculator(store, calc, log)
|
||||
|
||||
// Add observations
|
||||
now := time.Now()
|
||||
for i := 1; i <= 10; i++ {
|
||||
store.AddObservation(&models.Observation{
|
||||
ID: int64(i),
|
||||
Type: models.ObsTypeBugfix,
|
||||
CreatedAtEpoch: now.UnixMilli(),
|
||||
})
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
var wg sync.WaitGroup
|
||||
|
||||
// Run multiple recalculations concurrently
|
||||
for i := 0; i < 5; i++ {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
_ = recalc.RecalculateNow(ctx)
|
||||
}()
|
||||
}
|
||||
|
||||
wg.Wait()
|
||||
|
||||
// Should complete without race conditions
|
||||
// (use -race flag to verify)
|
||||
assert.GreaterOrEqual(t, store.GetUpdateScoresCalls(), 1)
|
||||
}
|
||||
|
||||
func TestRecalculator_StatsThreadSafe(t *testing.T) {
|
||||
store := NewMockObservationStore()
|
||||
calc := NewCalculator(nil)
|
||||
log := zerolog.Nop()
|
||||
recalc := NewRecalculator(store, calc, log)
|
||||
|
||||
var wg sync.WaitGroup
|
||||
|
||||
// Concurrent reads (use -race flag to verify)
|
||||
for i := 0; i < 10; i++ {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
_ = recalc.GetStats()
|
||||
}()
|
||||
}
|
||||
|
||||
wg.Wait()
|
||||
}
|
||||
@@ -0,0 +1,448 @@
|
||||
// Package expansion provides context-aware query expansion for improved search recall.
|
||||
package expansion
|
||||
|
||||
import (
|
||||
"context"
|
||||
"regexp"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"github.com/lukaszraczylo/claude-mnemonic/internal/embedding"
|
||||
"github.com/rs/zerolog/log"
|
||||
)
|
||||
|
||||
// QueryIntent represents the detected intent of a query.
|
||||
type QueryIntent string
|
||||
|
||||
const (
|
||||
// IntentQuestion indicates a question-type query (how, why, what, etc.)
|
||||
IntentQuestion QueryIntent = "question"
|
||||
// IntentError indicates an error/debugging query
|
||||
IntentError QueryIntent = "error"
|
||||
// IntentImplementation indicates an implementation/coding query
|
||||
IntentImplementation QueryIntent = "implementation"
|
||||
// IntentArchitecture indicates an architecture/design query
|
||||
IntentArchitecture QueryIntent = "architecture"
|
||||
// IntentGeneral indicates a general lookup query
|
||||
IntentGeneral QueryIntent = "general"
|
||||
)
|
||||
|
||||
// ExpandedQuery represents a query variant with metadata.
|
||||
type ExpandedQuery struct {
|
||||
Query string `json:"query"`
|
||||
Weight float64 `json:"weight"` // Weight for result merging (0.0-1.0)
|
||||
Source string `json:"source"` // Where this expansion came from
|
||||
Intent QueryIntent `json:"intent"` // Detected intent
|
||||
}
|
||||
|
||||
// Expander provides context-aware query expansion.
|
||||
type Expander struct {
|
||||
embedSvc *embedding.Service
|
||||
vocabulary []VocabEntry // Known vocabulary from observations
|
||||
vocabVectors [][]float32 // Embeddings for vocabulary entries
|
||||
vocabMu sync.RWMutex // Protects vocabulary
|
||||
intentPatterns map[QueryIntent][]*regexp.Regexp
|
||||
}
|
||||
|
||||
// VocabEntry represents a vocabulary term from observations.
|
||||
type VocabEntry struct {
|
||||
Term string // The term itself
|
||||
Weight float64 // How common/important this term is (0.0-1.0)
|
||||
Source string // Where it came from (title, concept, narrative)
|
||||
}
|
||||
|
||||
// Config holds expander configuration.
|
||||
type Config struct {
|
||||
// MaxExpansions limits the number of expanded queries returned
|
||||
MaxExpansions int
|
||||
// MinSimilarity is the minimum similarity score for vocabulary expansion
|
||||
MinSimilarity float64
|
||||
// EnableVocabularyExpansion enables finding related terms from observations
|
||||
EnableVocabularyExpansion bool
|
||||
}
|
||||
|
||||
// DefaultConfig returns sensible default configuration.
|
||||
func DefaultConfig() Config {
|
||||
return Config{
|
||||
MaxExpansions: 4,
|
||||
MinSimilarity: 0.5,
|
||||
EnableVocabularyExpansion: true,
|
||||
}
|
||||
}
|
||||
|
||||
// NewExpander creates a new query expander.
|
||||
func NewExpander(embedSvc *embedding.Service) *Expander {
|
||||
e := &Expander{
|
||||
embedSvc: embedSvc,
|
||||
intentPatterns: buildIntentPatterns(),
|
||||
}
|
||||
return e
|
||||
}
|
||||
|
||||
// buildIntentPatterns creates regex patterns for intent detection.
|
||||
func buildIntentPatterns() map[QueryIntent][]*regexp.Regexp {
|
||||
patterns := make(map[QueryIntent][]*regexp.Regexp)
|
||||
|
||||
// Question patterns
|
||||
patterns[IntentQuestion] = []*regexp.Regexp{
|
||||
regexp.MustCompile(`(?i)^(how|why|what|when|where|which|who)\b`),
|
||||
regexp.MustCompile(`(?i)\?$`),
|
||||
regexp.MustCompile(`(?i)\b(explain|describe|understand)\b`),
|
||||
}
|
||||
|
||||
// Error/debugging patterns
|
||||
patterns[IntentError] = []*regexp.Regexp{
|
||||
regexp.MustCompile(`(?i)\b(error|bug|issue|problem|fail|crash|exception|panic)\b`),
|
||||
regexp.MustCompile(`(?i)\b(fix|debug|troubleshoot|resolve)\b`),
|
||||
regexp.MustCompile(`(?i)\b(doesn't work|not working|broken)\b`),
|
||||
}
|
||||
|
||||
// Implementation patterns
|
||||
patterns[IntentImplementation] = []*regexp.Regexp{
|
||||
regexp.MustCompile(`(?i)\b(implement|add|create|build|write|code)\b`),
|
||||
regexp.MustCompile(`(?i)\b(function|method|handler|endpoint|api)\b`),
|
||||
regexp.MustCompile(`(?i)\b(feature|functionality)\b`),
|
||||
}
|
||||
|
||||
// Architecture patterns
|
||||
patterns[IntentArchitecture] = []*regexp.Regexp{
|
||||
regexp.MustCompile(`(?i)\b(architecture|design|pattern|structure)\b`),
|
||||
regexp.MustCompile(`(?i)\b(component|module|layer|service)\b`),
|
||||
regexp.MustCompile(`(?i)\b(flow|pipeline|workflow)\b`),
|
||||
}
|
||||
|
||||
return patterns
|
||||
}
|
||||
|
||||
// DetectIntent analyzes a query to determine its intent.
|
||||
func (e *Expander) DetectIntent(query string) QueryIntent {
|
||||
query = strings.TrimSpace(query)
|
||||
if query == "" {
|
||||
return IntentGeneral
|
||||
}
|
||||
|
||||
// Check patterns in priority order
|
||||
intentOrder := []QueryIntent{IntentError, IntentQuestion, IntentImplementation, IntentArchitecture}
|
||||
|
||||
for _, intent := range intentOrder {
|
||||
patterns := e.intentPatterns[intent]
|
||||
for _, pattern := range patterns {
|
||||
if pattern.MatchString(query) {
|
||||
return intent
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return IntentGeneral
|
||||
}
|
||||
|
||||
// Expand generates expanded query variants based on the original query.
|
||||
func (e *Expander) Expand(ctx context.Context, query string, cfg Config) []ExpandedQuery {
|
||||
query = strings.TrimSpace(query)
|
||||
if query == "" {
|
||||
return nil
|
||||
}
|
||||
|
||||
intent := e.DetectIntent(query)
|
||||
expansions := make([]ExpandedQuery, 0, cfg.MaxExpansions)
|
||||
|
||||
// Always include the original query with highest weight
|
||||
expansions = append(expansions, ExpandedQuery{
|
||||
Query: query,
|
||||
Weight: 1.0,
|
||||
Source: "original",
|
||||
Intent: intent,
|
||||
})
|
||||
|
||||
// Generate intent-based expansions
|
||||
intentExpansions := e.expandByIntent(query, intent)
|
||||
expansions = append(expansions, intentExpansions...)
|
||||
|
||||
// Generate vocabulary-based expansions if enabled and we have vocabulary
|
||||
if cfg.EnableVocabularyExpansion && e.embedSvc != nil && len(e.vocabulary) > 0 {
|
||||
vocabExpansions := e.expandByVocabulary(ctx, query, cfg.MinSimilarity)
|
||||
expansions = append(expansions, vocabExpansions...)
|
||||
}
|
||||
|
||||
// Deduplicate and limit
|
||||
expansions = deduplicateExpansions(expansions)
|
||||
if len(expansions) > cfg.MaxExpansions {
|
||||
expansions = expansions[:cfg.MaxExpansions]
|
||||
}
|
||||
|
||||
log.Debug().
|
||||
Str("query", truncate(query, 50)).
|
||||
Str("intent", string(intent)).
|
||||
Int("expansions", len(expansions)).
|
||||
Msg("Query expanded")
|
||||
|
||||
return expansions
|
||||
}
|
||||
|
||||
// expandByIntent generates expansions based on detected query intent.
|
||||
func (e *Expander) expandByIntent(query string, intent QueryIntent) []ExpandedQuery {
|
||||
var expansions []ExpandedQuery
|
||||
|
||||
// Extract key terms from query for context-aware expansion
|
||||
keyTerms := extractKeyTerms(query)
|
||||
|
||||
switch intent {
|
||||
case IntentQuestion:
|
||||
// For questions, create a declarative variant
|
||||
declarative := makeDeclarative(query)
|
||||
if declarative != query {
|
||||
expansions = append(expansions, ExpandedQuery{
|
||||
Query: declarative,
|
||||
Weight: 0.85,
|
||||
Source: "intent:declarative",
|
||||
Intent: intent,
|
||||
})
|
||||
}
|
||||
|
||||
case IntentError:
|
||||
// For errors, expand with solution-oriented terms
|
||||
if len(keyTerms) > 0 {
|
||||
solutionQuery := strings.Join(keyTerms, " ") + " solution fix"
|
||||
expansions = append(expansions, ExpandedQuery{
|
||||
Query: solutionQuery,
|
||||
Weight: 0.8,
|
||||
Source: "intent:solution",
|
||||
Intent: intent,
|
||||
})
|
||||
}
|
||||
|
||||
case IntentImplementation:
|
||||
// For implementation queries, focus on the what/how
|
||||
if len(keyTerms) > 0 {
|
||||
howQuery := "how " + strings.Join(keyTerms, " ")
|
||||
expansions = append(expansions, ExpandedQuery{
|
||||
Query: howQuery,
|
||||
Weight: 0.75,
|
||||
Source: "intent:how",
|
||||
Intent: intent,
|
||||
})
|
||||
}
|
||||
|
||||
case IntentArchitecture:
|
||||
// For architecture queries, expand with design context
|
||||
if len(keyTerms) > 0 {
|
||||
designQuery := strings.Join(keyTerms, " ") + " design structure"
|
||||
expansions = append(expansions, ExpandedQuery{
|
||||
Query: designQuery,
|
||||
Weight: 0.75,
|
||||
Source: "intent:design",
|
||||
Intent: intent,
|
||||
})
|
||||
}
|
||||
|
||||
case IntentGeneral:
|
||||
// For general queries, try noun phrase extraction
|
||||
// No additional expansion - rely on vocabulary expansion
|
||||
}
|
||||
|
||||
return expansions
|
||||
}
|
||||
|
||||
// expandByVocabulary finds similar terms from the observation vocabulary.
|
||||
func (e *Expander) expandByVocabulary(ctx context.Context, query string, minSimilarity float64) []ExpandedQuery {
|
||||
e.vocabMu.RLock()
|
||||
defer e.vocabMu.RUnlock()
|
||||
|
||||
if len(e.vocabulary) == 0 || e.embedSvc == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Embed the query
|
||||
queryEmb, err := e.embedSvc.Embed(query)
|
||||
if err != nil {
|
||||
log.Warn().Err(err).Msg("Failed to embed query for vocabulary expansion")
|
||||
return nil
|
||||
}
|
||||
|
||||
// Find similar vocabulary terms
|
||||
type scoredTerm struct {
|
||||
entry VocabEntry
|
||||
score float64
|
||||
}
|
||||
|
||||
var similar []scoredTerm
|
||||
for i, entry := range e.vocabulary {
|
||||
if i >= len(e.vocabVectors) {
|
||||
break
|
||||
}
|
||||
|
||||
score := cosineSimilarity(queryEmb, e.vocabVectors[i])
|
||||
if score >= minSimilarity {
|
||||
similar = append(similar, scoredTerm{entry: entry, score: score})
|
||||
}
|
||||
}
|
||||
|
||||
if len(similar) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Sort by score (descending) using bubble sort
|
||||
for i := 0; i < len(similar)-1; i++ {
|
||||
for j := i + 1; j < len(similar); j++ {
|
||||
if similar[j].score > similar[i].score {
|
||||
similar[i], similar[j] = similar[j], similar[i]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Create expansion by combining top similar terms with query
|
||||
var expansions []ExpandedQuery
|
||||
if len(similar) > 0 {
|
||||
// Take top 2 similar terms and combine with original key terms
|
||||
keyTerms := extractKeyTerms(query)
|
||||
for i := 0; i < min(2, len(similar)); i++ {
|
||||
term := similar[i].entry.Term
|
||||
// Don't add if term is already in query
|
||||
if strings.Contains(strings.ToLower(query), strings.ToLower(term)) {
|
||||
continue
|
||||
}
|
||||
|
||||
combinedQuery := strings.Join(keyTerms, " ") + " " + term
|
||||
expansions = append(expansions, ExpandedQuery{
|
||||
Query: combinedQuery,
|
||||
Weight: 0.7 * similar[i].score * similar[i].entry.Weight,
|
||||
Source: "vocabulary:" + term,
|
||||
Intent: IntentGeneral,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
return expansions
|
||||
}
|
||||
|
||||
// Helper functions
|
||||
|
||||
// extractKeyTerms extracts meaningful terms from a query.
|
||||
func extractKeyTerms(query string) []string {
|
||||
// Common stop words to filter out
|
||||
stopWords := map[string]bool{
|
||||
"a": true, "an": true, "the": true, "is": true, "are": true,
|
||||
"was": true, "were": true, "be": true, "been": true, "being": true,
|
||||
"have": true, "has": true, "had": true, "do": true, "does": true,
|
||||
"did": true, "will": true, "would": true, "could": true, "should": true,
|
||||
"may": true, "might": true, "must": true, "can": true,
|
||||
"i": true, "me": true, "my": true, "we": true, "our": true,
|
||||
"you": true, "your": true, "it": true, "its": true,
|
||||
"this": true, "that": true, "these": true, "those": true,
|
||||
"what": true, "which": true, "who": true, "whom": true,
|
||||
"how": true, "why": true, "when": true, "where": true,
|
||||
"to": true, "for": true, "with": true, "about": true, "from": true,
|
||||
"in": true, "on": true, "at": true, "by": true, "of": true,
|
||||
"and": true, "or": true, "but": true, "if": true, "then": true,
|
||||
}
|
||||
|
||||
// Split and filter
|
||||
words := strings.Fields(strings.ToLower(query))
|
||||
var terms []string
|
||||
|
||||
for _, word := range words {
|
||||
// Remove punctuation
|
||||
word = strings.Trim(word, ".,?!;:'\"()[]{}")
|
||||
if len(word) < 2 {
|
||||
continue
|
||||
}
|
||||
if stopWords[word] {
|
||||
continue
|
||||
}
|
||||
terms = append(terms, word)
|
||||
}
|
||||
|
||||
return terms
|
||||
}
|
||||
|
||||
// makeDeclarative converts a question to a declarative statement.
|
||||
func makeDeclarative(query string) string {
|
||||
query = strings.TrimSpace(query)
|
||||
|
||||
// Remove question mark
|
||||
query = strings.TrimSuffix(query, "?")
|
||||
|
||||
// Handle common question patterns
|
||||
patterns := []struct {
|
||||
prefix string
|
||||
replacement string
|
||||
}{
|
||||
{"how do i ", ""},
|
||||
{"how to ", ""},
|
||||
{"how does ", ""},
|
||||
{"how is ", ""},
|
||||
{"what is ", ""},
|
||||
{"what are ", ""},
|
||||
{"why does ", ""},
|
||||
{"why is ", ""},
|
||||
{"where is ", ""},
|
||||
{"where are ", ""},
|
||||
{"when does ", ""},
|
||||
{"when is ", ""},
|
||||
}
|
||||
|
||||
lower := strings.ToLower(query)
|
||||
for _, p := range patterns {
|
||||
if strings.HasPrefix(lower, p.prefix) {
|
||||
return strings.TrimSpace(query[len(p.prefix):])
|
||||
}
|
||||
}
|
||||
|
||||
return query
|
||||
}
|
||||
|
||||
// deduplicateExpansions removes duplicate queries while preserving order.
|
||||
func deduplicateExpansions(expansions []ExpandedQuery) []ExpandedQuery {
|
||||
seen := make(map[string]bool)
|
||||
result := make([]ExpandedQuery, 0, len(expansions))
|
||||
|
||||
for _, exp := range expansions {
|
||||
normalized := strings.ToLower(strings.TrimSpace(exp.Query))
|
||||
if !seen[normalized] {
|
||||
seen[normalized] = true
|
||||
result = append(result, exp)
|
||||
}
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
// cosineSimilarity computes cosine similarity between two vectors.
|
||||
func cosineSimilarity(a, b []float32) float64 {
|
||||
if len(a) != len(b) || len(a) == 0 {
|
||||
return 0
|
||||
}
|
||||
|
||||
var dot, normA, normB float64
|
||||
for i := range a {
|
||||
dot += float64(a[i]) * float64(b[i])
|
||||
normA += float64(a[i]) * float64(a[i])
|
||||
normB += float64(b[i]) * float64(b[i])
|
||||
}
|
||||
|
||||
if normA == 0 || normB == 0 {
|
||||
return 0
|
||||
}
|
||||
|
||||
return dot / (sqrt(normA) * sqrt(normB))
|
||||
}
|
||||
|
||||
// sqrt is a simple square root implementation.
|
||||
func sqrt(x float64) float64 {
|
||||
if x <= 0 {
|
||||
return 0
|
||||
}
|
||||
z := x
|
||||
for i := 0; i < 10; i++ {
|
||||
z = (z + x/z) / 2
|
||||
}
|
||||
return z
|
||||
}
|
||||
|
||||
// truncate truncates a string to maxLen characters.
|
||||
func truncate(s string, maxLen int) string {
|
||||
if len(s) <= maxLen {
|
||||
return s
|
||||
}
|
||||
return s[:maxLen] + "..."
|
||||
}
|
||||
@@ -0,0 +1,518 @@
|
||||
// Package expansion provides context-aware query expansion for improved search recall.
|
||||
package expansion
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/suite"
|
||||
)
|
||||
|
||||
// ExpanderSuite tests the Expander functionality.
|
||||
type ExpanderSuite struct {
|
||||
suite.Suite
|
||||
expander *Expander
|
||||
}
|
||||
|
||||
func TestExpanderSuite(t *testing.T) {
|
||||
suite.Run(t, new(ExpanderSuite))
|
||||
}
|
||||
|
||||
func (s *ExpanderSuite) SetupTest() {
|
||||
// Create expander without embedding service for basic tests
|
||||
s.expander = NewExpander(nil)
|
||||
}
|
||||
|
||||
// TestNewExpander tests expander creation.
|
||||
func (s *ExpanderSuite) TestNewExpander() {
|
||||
e := NewExpander(nil)
|
||||
s.NotNil(e)
|
||||
s.NotNil(e.intentPatterns)
|
||||
s.Nil(e.embedSvc)
|
||||
}
|
||||
|
||||
// TestDetectIntent tests intent detection.
|
||||
func (s *ExpanderSuite) TestDetectIntent() {
|
||||
tests := []struct {
|
||||
name string
|
||||
query string
|
||||
expected QueryIntent
|
||||
}{
|
||||
// Question intents
|
||||
{"how_question", "how do I implement auth?", IntentQuestion},
|
||||
{"why_question", "why does this fail?", IntentError}, // "fail" triggers error first
|
||||
{"what_question", "what is the purpose of this function?", IntentQuestion},
|
||||
{"question_mark", "the handler for auth?", IntentQuestion},
|
||||
{"explain", "explain the architecture", IntentQuestion},
|
||||
|
||||
// Error intents
|
||||
{"error_word", "authentication error in login", IntentError},
|
||||
{"bug_word", "bug in user registration", IntentError},
|
||||
{"fix_word", "fix the memory leak", IntentError},
|
||||
{"not_working", "login not working", IntentError},
|
||||
{"crash", "application crash on startup", IntentError},
|
||||
|
||||
// Implementation intents
|
||||
{"implement", "implement user authentication", IntentImplementation},
|
||||
{"add_feature", "add new endpoint for users", IntentImplementation},
|
||||
{"create", "create a handler for uploads", IntentImplementation},
|
||||
{"function", "function to validate input", IntentImplementation},
|
||||
|
||||
// Architecture intents
|
||||
{"architecture", "architecture of the system", IntentArchitecture},
|
||||
{"design", "design pattern for observers", IntentArchitecture},
|
||||
{"component", "component structure", IntentArchitecture},
|
||||
{"flow", "data flow in the pipeline", IntentArchitecture},
|
||||
|
||||
// General intents
|
||||
{"general", "user authentication", IntentGeneral},
|
||||
{"empty", "", IntentGeneral},
|
||||
{"simple", "database", IntentGeneral},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
s.Run(tt.name, func() {
|
||||
result := s.expander.DetectIntent(tt.query)
|
||||
s.Equal(tt.expected, result, "Query: %s", tt.query)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestExpand tests basic query expansion.
|
||||
func (s *ExpanderSuite) TestExpand() {
|
||||
ctx := context.Background()
|
||||
cfg := DefaultConfig()
|
||||
cfg.EnableVocabularyExpansion = false // Disable for unit test
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
query string
|
||||
minExpansions int
|
||||
hasOriginal bool
|
||||
expectedIntent QueryIntent
|
||||
}{
|
||||
{"question", "how do I implement auth", 1, true, IntentQuestion},
|
||||
{"error", "fix the bug in login", 1, true, IntentError},
|
||||
{"implementation", "implement user handler", 1, true, IntentImplementation},
|
||||
{"architecture", "architecture design", 1, true, IntentArchitecture},
|
||||
{"general", "database connection", 1, true, IntentGeneral},
|
||||
{"empty", "", 0, false, IntentGeneral},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
s.Run(tt.name, func() {
|
||||
expansions := s.expander.Expand(ctx, tt.query, cfg)
|
||||
|
||||
if tt.minExpansions == 0 {
|
||||
s.Empty(expansions)
|
||||
return
|
||||
}
|
||||
|
||||
s.GreaterOrEqual(len(expansions), tt.minExpansions)
|
||||
|
||||
if tt.hasOriginal {
|
||||
// First expansion should be the original
|
||||
s.Equal(tt.query, expansions[0].Query)
|
||||
s.Equal(1.0, expansions[0].Weight)
|
||||
s.Equal("original", expansions[0].Source)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestExpandWithConfig tests expansion with custom config.
|
||||
func (s *ExpanderSuite) TestExpandWithConfig() {
|
||||
ctx := context.Background()
|
||||
|
||||
cfg := Config{
|
||||
MaxExpansions: 2,
|
||||
MinSimilarity: 0.7,
|
||||
EnableVocabularyExpansion: false,
|
||||
}
|
||||
|
||||
expansions := s.expander.Expand(ctx, "how to implement authentication", cfg)
|
||||
s.LessOrEqual(len(expansions), cfg.MaxExpansions)
|
||||
}
|
||||
|
||||
// TestExpandDeduplication tests that duplicates are removed.
|
||||
func (s *ExpanderSuite) TestExpandDeduplication() {
|
||||
ctx := context.Background()
|
||||
cfg := DefaultConfig()
|
||||
cfg.EnableVocabularyExpansion = false
|
||||
|
||||
// Query that might generate duplicate expansions
|
||||
query := "how to fix authentication"
|
||||
expansions := s.expander.Expand(ctx, query, cfg)
|
||||
|
||||
// Check for duplicates
|
||||
seen := make(map[string]bool)
|
||||
for _, exp := range expansions {
|
||||
normalized := exp.Query
|
||||
s.False(seen[normalized], "Duplicate expansion found: %s", exp.Query)
|
||||
seen[normalized] = true
|
||||
}
|
||||
}
|
||||
|
||||
// TestExtractKeyTerms tests key term extraction.
|
||||
func TestExtractKeyTerms(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
query string
|
||||
expected []string
|
||||
}{
|
||||
{
|
||||
name: "simple",
|
||||
query: "user authentication handler",
|
||||
expected: []string{"user", "authentication", "handler"},
|
||||
},
|
||||
{
|
||||
name: "with_stop_words",
|
||||
query: "how to implement the user login",
|
||||
expected: []string{"implement", "user", "login"},
|
||||
},
|
||||
{
|
||||
name: "with_punctuation",
|
||||
query: "fix the bug, please!",
|
||||
expected: []string{"fix", "bug", "please"},
|
||||
},
|
||||
{
|
||||
name: "empty",
|
||||
query: "",
|
||||
expected: nil,
|
||||
},
|
||||
{
|
||||
name: "only_stop_words",
|
||||
query: "the a an is are",
|
||||
expected: nil,
|
||||
},
|
||||
{
|
||||
name: "short_words_filtered",
|
||||
query: "a b c auth",
|
||||
expected: []string{"auth"},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
result := extractKeyTerms(tt.query)
|
||||
if tt.expected == nil {
|
||||
assert.Empty(t, result)
|
||||
} else {
|
||||
assert.Equal(t, tt.expected, result)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestMakeDeclarative tests question to declarative conversion.
|
||||
func TestMakeDeclarative(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
query string
|
||||
expected string
|
||||
}{
|
||||
{
|
||||
name: "how_do_i",
|
||||
query: "how do I implement auth?",
|
||||
expected: "implement auth",
|
||||
},
|
||||
{
|
||||
name: "how_to",
|
||||
query: "how to fix the bug",
|
||||
expected: "fix the bug",
|
||||
},
|
||||
{
|
||||
name: "what_is",
|
||||
query: "what is the purpose of this?",
|
||||
expected: "the purpose of this",
|
||||
},
|
||||
{
|
||||
name: "why_does",
|
||||
query: "why does this fail?",
|
||||
expected: "this fail",
|
||||
},
|
||||
{
|
||||
name: "already_declarative",
|
||||
query: "user authentication",
|
||||
expected: "user authentication",
|
||||
},
|
||||
{
|
||||
name: "question_mark_only",
|
||||
query: "authentication?",
|
||||
expected: "authentication",
|
||||
},
|
||||
{
|
||||
name: "case_insensitive",
|
||||
query: "How To Fix Auth?",
|
||||
expected: "Fix Auth",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
result := makeDeclarative(tt.query)
|
||||
assert.Equal(t, tt.expected, result)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestDeduplicateExpansions tests deduplication.
|
||||
func TestDeduplicateExpansions(t *testing.T) {
|
||||
expansions := []ExpandedQuery{
|
||||
{Query: "auth handler", Weight: 1.0},
|
||||
{Query: "AUTH HANDLER", Weight: 0.8}, // Duplicate (case insensitive)
|
||||
{Query: "auth handler ", Weight: 0.7}, // Duplicate (whitespace)
|
||||
{Query: "user auth", Weight: 0.6},
|
||||
}
|
||||
|
||||
result := deduplicateExpansions(expansions)
|
||||
assert.Len(t, result, 2) // "auth handler" and "user auth"
|
||||
assert.Equal(t, "auth handler", result[0].Query)
|
||||
assert.Equal(t, 1.0, result[0].Weight) // First one preserved
|
||||
assert.Equal(t, "user auth", result[1].Query)
|
||||
}
|
||||
|
||||
// TestCosineSimilarity tests cosine similarity calculation.
|
||||
func TestCosineSimilarity(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
a []float32
|
||||
b []float32
|
||||
expected float64
|
||||
delta float64
|
||||
}{
|
||||
{
|
||||
name: "identical_vectors",
|
||||
a: []float32{1, 0, 0},
|
||||
b: []float32{1, 0, 0},
|
||||
expected: 1.0,
|
||||
delta: 0.001,
|
||||
},
|
||||
{
|
||||
name: "orthogonal_vectors",
|
||||
a: []float32{1, 0, 0},
|
||||
b: []float32{0, 1, 0},
|
||||
expected: 0.0,
|
||||
delta: 0.001,
|
||||
},
|
||||
{
|
||||
name: "opposite_vectors",
|
||||
a: []float32{1, 0, 0},
|
||||
b: []float32{-1, 0, 0},
|
||||
expected: -1.0,
|
||||
delta: 0.001,
|
||||
},
|
||||
{
|
||||
name: "similar_vectors",
|
||||
a: []float32{1, 1, 0},
|
||||
b: []float32{1, 0, 0},
|
||||
expected: 0.707,
|
||||
delta: 0.01,
|
||||
},
|
||||
{
|
||||
name: "empty_vectors",
|
||||
a: []float32{},
|
||||
b: []float32{},
|
||||
expected: 0.0,
|
||||
delta: 0.001,
|
||||
},
|
||||
{
|
||||
name: "different_lengths",
|
||||
a: []float32{1, 0},
|
||||
b: []float32{1, 0, 0},
|
||||
expected: 0.0,
|
||||
delta: 0.001,
|
||||
},
|
||||
{
|
||||
name: "zero_vector",
|
||||
a: []float32{0, 0, 0},
|
||||
b: []float32{1, 1, 1},
|
||||
expected: 0.0,
|
||||
delta: 0.001,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
result := cosineSimilarity(tt.a, tt.b)
|
||||
assert.InDelta(t, tt.expected, result, tt.delta)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestDefaultConfig tests default configuration.
|
||||
func TestDefaultConfig(t *testing.T) {
|
||||
cfg := DefaultConfig()
|
||||
|
||||
assert.Equal(t, 4, cfg.MaxExpansions)
|
||||
assert.Equal(t, 0.5, cfg.MinSimilarity)
|
||||
assert.True(t, cfg.EnableVocabularyExpansion)
|
||||
}
|
||||
|
||||
// TestExpandedQueryStruct tests ExpandedQuery struct.
|
||||
func TestExpandedQueryStruct(t *testing.T) {
|
||||
eq := ExpandedQuery{
|
||||
Query: "test query",
|
||||
Weight: 0.85,
|
||||
Source: "vocabulary:auth",
|
||||
Intent: IntentQuestion,
|
||||
}
|
||||
|
||||
assert.Equal(t, "test query", eq.Query)
|
||||
assert.Equal(t, 0.85, eq.Weight)
|
||||
assert.Equal(t, "vocabulary:auth", eq.Source)
|
||||
assert.Equal(t, IntentQuestion, eq.Intent)
|
||||
}
|
||||
|
||||
// TestVocabEntry tests VocabEntry struct.
|
||||
func TestVocabEntry(t *testing.T) {
|
||||
ve := VocabEntry{
|
||||
Term: "authentication",
|
||||
Weight: 0.9,
|
||||
Source: "concept",
|
||||
}
|
||||
|
||||
assert.Equal(t, "authentication", ve.Term)
|
||||
assert.Equal(t, 0.9, ve.Weight)
|
||||
assert.Equal(t, "concept", ve.Source)
|
||||
}
|
||||
|
||||
// TestIntentConstants tests intent constant values.
|
||||
func TestIntentConstants(t *testing.T) {
|
||||
assert.Equal(t, QueryIntent("question"), IntentQuestion)
|
||||
assert.Equal(t, QueryIntent("error"), IntentError)
|
||||
assert.Equal(t, QueryIntent("implementation"), IntentImplementation)
|
||||
assert.Equal(t, QueryIntent("architecture"), IntentArchitecture)
|
||||
assert.Equal(t, QueryIntent("general"), IntentGeneral)
|
||||
}
|
||||
|
||||
// TestTruncate tests the truncate helper.
|
||||
func TestTruncate(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
input string
|
||||
maxLen int
|
||||
expected string
|
||||
}{
|
||||
{"short", "hello", 10, "hello"},
|
||||
{"exact", "hello", 5, "hello"},
|
||||
{"long", "hello world", 5, "hello..."},
|
||||
{"empty", "", 10, ""},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
result := truncate(tt.input, tt.maxLen)
|
||||
assert.Equal(t, tt.expected, result)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestSqrt tests the sqrt helper.
|
||||
func TestSqrt(t *testing.T) {
|
||||
tests := []struct {
|
||||
input float64
|
||||
expected float64
|
||||
delta float64
|
||||
}{
|
||||
{4.0, 2.0, 0.001},
|
||||
{9.0, 3.0, 0.001},
|
||||
{16.0, 4.0, 0.001},
|
||||
{2.0, 1.414, 0.01},
|
||||
{0.0, 0.0, 0.001},
|
||||
{-1.0, 0.0, 0.001},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run("", func(t *testing.T) {
|
||||
result := sqrt(tt.input)
|
||||
assert.InDelta(t, tt.expected, result, tt.delta)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestExpandByIntentError tests error intent expansion.
|
||||
func (s *ExpanderSuite) TestExpandByIntentError() {
|
||||
expansions := s.expander.expandByIntent("fix authentication bug", IntentError)
|
||||
s.NotEmpty(expansions)
|
||||
|
||||
// Should have solution-oriented expansion
|
||||
hasSolution := false
|
||||
for _, exp := range expansions {
|
||||
if exp.Source == "intent:solution" {
|
||||
hasSolution = true
|
||||
break
|
||||
}
|
||||
}
|
||||
s.True(hasSolution)
|
||||
}
|
||||
|
||||
// TestExpandByIntentQuestion tests question intent expansion.
|
||||
func (s *ExpanderSuite) TestExpandByIntentQuestion() {
|
||||
expansions := s.expander.expandByIntent("how do I implement auth", IntentQuestion)
|
||||
s.NotEmpty(expansions)
|
||||
|
||||
// Should have declarative expansion
|
||||
hasDeclarative := false
|
||||
for _, exp := range expansions {
|
||||
if exp.Source == "intent:declarative" {
|
||||
hasDeclarative = true
|
||||
break
|
||||
}
|
||||
}
|
||||
s.True(hasDeclarative)
|
||||
}
|
||||
|
||||
// TestExpandByIntentImplementation tests implementation intent expansion.
|
||||
func (s *ExpanderSuite) TestExpandByIntentImplementation() {
|
||||
expansions := s.expander.expandByIntent("implement user handler", IntentImplementation)
|
||||
s.NotEmpty(expansions)
|
||||
|
||||
// Should have how expansion
|
||||
hasHow := false
|
||||
for _, exp := range expansions {
|
||||
if exp.Source == "intent:how" {
|
||||
hasHow = true
|
||||
break
|
||||
}
|
||||
}
|
||||
s.True(hasHow)
|
||||
}
|
||||
|
||||
// TestExpandByIntentArchitecture tests architecture intent expansion.
|
||||
func (s *ExpanderSuite) TestExpandByIntentArchitecture() {
|
||||
expansions := s.expander.expandByIntent("system architecture design", IntentArchitecture)
|
||||
s.NotEmpty(expansions)
|
||||
|
||||
// Should have design expansion
|
||||
hasDesign := false
|
||||
for _, exp := range expansions {
|
||||
if exp.Source == "intent:design" {
|
||||
hasDesign = true
|
||||
break
|
||||
}
|
||||
}
|
||||
s.True(hasDesign)
|
||||
}
|
||||
|
||||
// TestExpandByIntentGeneral tests general intent returns no expansions.
|
||||
func (s *ExpanderSuite) TestExpandByIntentGeneral() {
|
||||
expansions := s.expander.expandByIntent("database", IntentGeneral)
|
||||
s.Empty(expansions) // General intent doesn't add intent-based expansions
|
||||
}
|
||||
|
||||
// TestEmptyVocabulary tests expansion with empty vocabulary.
|
||||
func (s *ExpanderSuite) TestEmptyVocabulary() {
|
||||
ctx := context.Background()
|
||||
expansions := s.expander.expandByVocabulary(ctx, "test query", 0.5)
|
||||
s.Empty(expansions)
|
||||
}
|
||||
|
||||
// TestIntentPatternsExist tests that all intents have patterns.
|
||||
func (s *ExpanderSuite) TestIntentPatternsExist() {
|
||||
s.NotEmpty(s.expander.intentPatterns[IntentQuestion])
|
||||
s.NotEmpty(s.expander.intentPatterns[IntentError])
|
||||
s.NotEmpty(s.expander.intentPatterns[IntentImplementation])
|
||||
s.NotEmpty(s.expander.intentPatterns[IntentArchitecture])
|
||||
}
|
||||
@@ -0,0 +1,645 @@
|
||||
// Package search provides unified search capabilities for claude-mnemonic.
|
||||
package search
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"github.com/lukaszraczylo/claude-mnemonic/internal/db/gorm"
|
||||
"github.com/lukaszraczylo/claude-mnemonic/pkg/models"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"github.com/stretchr/testify/suite"
|
||||
|
||||
// Import sqlite driver
|
||||
_ "github.com/mattn/go-sqlite3"
|
||||
)
|
||||
|
||||
// Ensure context is used (for later tests)
|
||||
var _ = context.Background
|
||||
|
||||
// hasFTS5 checks if FTS5 is available in the SQLite build.
|
||||
func hasFTS5(t *testing.T) bool {
|
||||
t.Helper()
|
||||
|
||||
tmpDir, err := os.MkdirTemp("", "fts5-check-*")
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
defer func() { _ = os.RemoveAll(tmpDir) }()
|
||||
|
||||
dbPath := tmpDir + "/check.db"
|
||||
db, err := sql.Open("sqlite3", dbPath)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
defer func() { _ = db.Close() }()
|
||||
|
||||
_, err = db.Exec("CREATE VIRTUAL TABLE IF NOT EXISTS fts5_test USING fts5(content)")
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
_, _ = db.Exec("DROP TABLE IF EXISTS fts5_test")
|
||||
return true
|
||||
}
|
||||
|
||||
// testStore creates a gorm.Store with a temporary database for testing.
|
||||
func testStore(t *testing.T) (*gorm.Store, func()) {
|
||||
t.Helper()
|
||||
|
||||
if !hasFTS5(t) {
|
||||
t.Skip("FTS5 not available in this SQLite build")
|
||||
}
|
||||
|
||||
tmpDir, err := os.MkdirTemp("", "search-integration-test-*")
|
||||
require.NoError(t, err)
|
||||
|
||||
dbPath := tmpDir + "/test.db"
|
||||
|
||||
store, err := gorm.NewStore(gorm.Config{
|
||||
Path: dbPath,
|
||||
MaxConns: 1,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
cleanup := func() {
|
||||
_ = store.Close()
|
||||
_ = os.RemoveAll(tmpDir)
|
||||
}
|
||||
|
||||
return store, cleanup
|
||||
}
|
||||
|
||||
// SearchIntegrationSuite tests search with real SQLite stores.
|
||||
type SearchIntegrationSuite struct {
|
||||
suite.Suite
|
||||
store *gorm.Store
|
||||
cleanup func()
|
||||
manager *Manager
|
||||
obsStore *gorm.ObservationStore
|
||||
sumStore *gorm.SummaryStore
|
||||
prmStore *gorm.PromptStore
|
||||
}
|
||||
|
||||
func (s *SearchIntegrationSuite) SetupTest() {
|
||||
if !hasFTS5(s.T()) {
|
||||
s.T().Skip("FTS5 not available in this SQLite build")
|
||||
}
|
||||
|
||||
s.store, s.cleanup = testStore(s.T())
|
||||
|
||||
// Create real stores backed by SQLite
|
||||
s.obsStore = gorm.NewObservationStore(s.store, nil, nil, nil)
|
||||
s.sumStore = gorm.NewSummaryStore(s.store)
|
||||
s.prmStore = gorm.NewPromptStore(s.store, nil)
|
||||
|
||||
// Create search manager with real stores (no vector client for now)
|
||||
s.manager = NewManager(s.obsStore, s.sumStore, s.prmStore, nil)
|
||||
}
|
||||
|
||||
func (s *SearchIntegrationSuite) TearDownTest() {
|
||||
if s.cleanup != nil {
|
||||
s.cleanup()
|
||||
}
|
||||
}
|
||||
|
||||
func TestSearchIntegrationSuite(t *testing.T) {
|
||||
suite.Run(t, new(SearchIntegrationSuite))
|
||||
}
|
||||
|
||||
// seedObservations inserts test observations into the database.
|
||||
func (s *SearchIntegrationSuite) seedObservations(ctx context.Context) []int64 {
|
||||
var ids []int64
|
||||
|
||||
// Observation 1: Authentication bug fix
|
||||
obs1 := &models.ParsedObservation{
|
||||
Type: models.ObsTypeBugfix,
|
||||
Scope: models.ScopeProject,
|
||||
Title: "Fixed Authentication Bug",
|
||||
Narrative: "Resolved JWT token validation issue that caused intermittent login failures",
|
||||
Concepts: []string{"authentication", "jwt", "security"},
|
||||
FilesRead: []string{"auth/handler.go", "auth/jwt.go"},
|
||||
}
|
||||
id1, _, err := s.obsStore.StoreObservation(ctx, "sdk-sess-1", "test-project", obs1, 1, 100)
|
||||
s.Require().NoError(err)
|
||||
ids = append(ids, id1)
|
||||
|
||||
// Observation 2: Database optimization decision
|
||||
obs2 := &models.ParsedObservation{
|
||||
Type: models.ObsTypeDecision,
|
||||
Scope: models.ScopeProject,
|
||||
Title: "Database Query Optimization Decision",
|
||||
Narrative: "Decided to add indexes on user_id and created_at columns for better performance",
|
||||
Concepts: []string{"database", "performance", "decision"},
|
||||
FilesRead: []string{"db/migrations/001.sql"},
|
||||
}
|
||||
id2, _, err := s.obsStore.StoreObservation(ctx, "sdk-sess-1", "test-project", obs2, 2, 150)
|
||||
s.Require().NoError(err)
|
||||
ids = append(ids, id2)
|
||||
|
||||
// Observation 3: Global best practice
|
||||
obs3 := &models.ParsedObservation{
|
||||
Type: models.ObsTypeDiscovery,
|
||||
Scope: models.ScopeGlobal,
|
||||
Title: "Error Handling Best Practice",
|
||||
Narrative: "Use wrapped errors with context for better debugging: errors.Wrap(err, context)",
|
||||
Concepts: []string{"best-practice", "errors", "patterns"},
|
||||
FilesRead: []string{"pkg/errors/errors.go"},
|
||||
}
|
||||
id3, _, err := s.obsStore.StoreObservation(ctx, "sdk-sess-2", "other-project", obs3, 1, 80)
|
||||
s.Require().NoError(err)
|
||||
ids = append(ids, id3)
|
||||
|
||||
// Observation 4: Code change/refactoring
|
||||
obs4 := &models.ParsedObservation{
|
||||
Type: models.ObsTypeChange,
|
||||
Scope: models.ScopeProject,
|
||||
Title: "Refactored User Service",
|
||||
Narrative: "Changed user service to use repository pattern, modified interfaces for better testability",
|
||||
Concepts: []string{"refactoring", "architecture"},
|
||||
FilesModified: []string{"services/user.go", "services/user_test.go"},
|
||||
}
|
||||
id4, _, err := s.obsStore.StoreObservation(ctx, "sdk-sess-1", "test-project", obs4, 3, 200)
|
||||
s.Require().NoError(err)
|
||||
ids = append(ids, id4)
|
||||
|
||||
return ids
|
||||
}
|
||||
|
||||
// seedSummaries inserts test session summaries into the database.
|
||||
func (s *SearchIntegrationSuite) seedSummaries(ctx context.Context) []int64 {
|
||||
var ids []int64
|
||||
|
||||
// Summary 1
|
||||
sum1 := &models.ParsedSummary{
|
||||
Request: "Fix authentication bug",
|
||||
Investigated: "JWT token validation and session handling",
|
||||
Learned: "JWT validation requires algorithm check to prevent alg:none attacks",
|
||||
Completed: "Fixed JWT validation, added tests",
|
||||
NextSteps: "Review other security endpoints",
|
||||
}
|
||||
id1, _, err := s.sumStore.StoreSummary(ctx, "sdk-sess-1", "test-project", sum1, 1, 100)
|
||||
s.Require().NoError(err)
|
||||
ids = append(ids, id1)
|
||||
|
||||
// Summary 2
|
||||
sum2 := &models.ParsedSummary{
|
||||
Request: "Optimize database queries",
|
||||
Investigated: "Query execution plans and index usage",
|
||||
Learned: "Composite indexes work better for range queries",
|
||||
Completed: "Added indexes, verified performance improvement",
|
||||
NextSteps: "Monitor query times in production",
|
||||
}
|
||||
id2, _, err := s.sumStore.StoreSummary(ctx, "sdk-sess-1", "test-project", sum2, 2, 150)
|
||||
s.Require().NoError(err)
|
||||
ids = append(ids, id2)
|
||||
|
||||
return ids
|
||||
}
|
||||
|
||||
// TestFilterSearch_WithRealStores tests filterSearch with seeded data.
|
||||
func (s *SearchIntegrationSuite) TestFilterSearch_WithRealStores() {
|
||||
ctx := context.Background()
|
||||
|
||||
// Seed test data
|
||||
obsIDs := s.seedObservations(ctx)
|
||||
sumIDs := s.seedSummaries(ctx)
|
||||
s.Require().Len(obsIDs, 4)
|
||||
s.Require().Len(sumIDs, 2)
|
||||
|
||||
// Test filter search for observations only
|
||||
result, err := s.manager.filterSearch(ctx, SearchParams{
|
||||
Project: "test-project",
|
||||
Type: "observations",
|
||||
Limit: 10,
|
||||
Format: "full",
|
||||
})
|
||||
s.Require().NoError(err)
|
||||
s.NotNil(result)
|
||||
|
||||
// Should return project observations + global observation (4 total: 3 project + 1 global)
|
||||
s.GreaterOrEqual(len(result.Results), 3)
|
||||
|
||||
// Verify result types
|
||||
for _, r := range result.Results {
|
||||
s.Equal("observation", r.Type)
|
||||
}
|
||||
}
|
||||
|
||||
// TestFilterSearch_SessionsOnly tests filterSearch for sessions.
|
||||
func (s *SearchIntegrationSuite) TestFilterSearch_SessionsOnly() {
|
||||
ctx := context.Background()
|
||||
|
||||
// Seed test data
|
||||
_ = s.seedObservations(ctx)
|
||||
sumIDs := s.seedSummaries(ctx)
|
||||
s.Require().Len(sumIDs, 2)
|
||||
|
||||
// Test filter search for sessions only
|
||||
result, err := s.manager.filterSearch(ctx, SearchParams{
|
||||
Project: "test-project",
|
||||
Type: "sessions",
|
||||
Limit: 10,
|
||||
Format: "full",
|
||||
})
|
||||
s.Require().NoError(err)
|
||||
s.NotNil(result)
|
||||
|
||||
// Should return 2 summaries
|
||||
s.Len(result.Results, 2)
|
||||
|
||||
// Verify result types
|
||||
for _, r := range result.Results {
|
||||
s.Equal("session", r.Type)
|
||||
s.NotEmpty(r.Title) // Title should be populated from Request
|
||||
}
|
||||
}
|
||||
|
||||
// TestFilterSearch_AllTypes tests filterSearch for all types.
|
||||
func (s *SearchIntegrationSuite) TestFilterSearch_AllTypes() {
|
||||
ctx := context.Background()
|
||||
|
||||
// Seed test data
|
||||
obsIDs := s.seedObservations(ctx)
|
||||
sumIDs := s.seedSummaries(ctx)
|
||||
s.Require().Len(obsIDs, 4)
|
||||
s.Require().Len(sumIDs, 2)
|
||||
|
||||
// Test filter search for all types (Type = "")
|
||||
result, err := s.manager.filterSearch(ctx, SearchParams{
|
||||
Project: "test-project",
|
||||
Type: "", // All types
|
||||
Limit: 20,
|
||||
Format: "full",
|
||||
})
|
||||
s.Require().NoError(err)
|
||||
s.NotNil(result)
|
||||
|
||||
// Should return both observations and sessions
|
||||
hasObservations := false
|
||||
hasSessions := false
|
||||
for _, r := range result.Results {
|
||||
if r.Type == "observation" {
|
||||
hasObservations = true
|
||||
}
|
||||
if r.Type == "session" {
|
||||
hasSessions = true
|
||||
}
|
||||
}
|
||||
s.True(hasObservations, "Should have observation results")
|
||||
s.True(hasSessions, "Should have session results")
|
||||
}
|
||||
|
||||
// TestUnifiedSearch_DefaultLimit tests UnifiedSearch with default limit.
|
||||
func (s *SearchIntegrationSuite) TestUnifiedSearch_DefaultLimit() {
|
||||
ctx := context.Background()
|
||||
|
||||
// Seed test data
|
||||
s.seedObservations(ctx)
|
||||
s.seedSummaries(ctx)
|
||||
|
||||
// Test with no limit specified (should default to 20)
|
||||
result, err := s.manager.UnifiedSearch(ctx, SearchParams{
|
||||
Project: "test-project",
|
||||
})
|
||||
s.Require().NoError(err)
|
||||
s.NotNil(result)
|
||||
s.LessOrEqual(len(result.Results), 20)
|
||||
}
|
||||
|
||||
// TestUnifiedSearch_LimitCapping tests UnifiedSearch limit capping.
|
||||
func (s *SearchIntegrationSuite) TestUnifiedSearch_LimitCapping() {
|
||||
ctx := context.Background()
|
||||
|
||||
// Seed test data
|
||||
s.seedObservations(ctx)
|
||||
s.seedSummaries(ctx)
|
||||
|
||||
// Test with limit > 100 (should be capped to 100)
|
||||
result, err := s.manager.UnifiedSearch(ctx, SearchParams{
|
||||
Project: "test-project",
|
||||
Limit: 500,
|
||||
})
|
||||
s.Require().NoError(err)
|
||||
s.NotNil(result)
|
||||
s.LessOrEqual(len(result.Results), 100)
|
||||
}
|
||||
|
||||
// TestDecisions_WithRealStores tests the Decisions method falls back to filterSearch.
|
||||
func (s *SearchIntegrationSuite) TestDecisions_WithRealStores() {
|
||||
ctx := context.Background()
|
||||
|
||||
// Seed test data
|
||||
s.seedObservations(ctx)
|
||||
|
||||
// Test Decisions search (without vector client, falls back to filterSearch)
|
||||
result, err := s.manager.Decisions(ctx, SearchParams{
|
||||
Project: "test-project",
|
||||
Query: "database",
|
||||
Limit: 10,
|
||||
})
|
||||
s.Require().NoError(err)
|
||||
s.NotNil(result)
|
||||
|
||||
// Without vector client, falls back to filterSearch which returns observations
|
||||
// All results should be observations (type is forced to "observations" in Decisions)
|
||||
for _, r := range result.Results {
|
||||
s.Equal("observation", r.Type)
|
||||
}
|
||||
}
|
||||
|
||||
// TestChanges_WithRealStores tests the Changes method falls back to filterSearch.
|
||||
func (s *SearchIntegrationSuite) TestChanges_WithRealStores() {
|
||||
ctx := context.Background()
|
||||
|
||||
// Seed test data
|
||||
s.seedObservations(ctx)
|
||||
|
||||
// Test Changes search (without vector client, falls back to filterSearch)
|
||||
result, err := s.manager.Changes(ctx, SearchParams{
|
||||
Project: "test-project",
|
||||
Query: "user service",
|
||||
Limit: 10,
|
||||
})
|
||||
s.Require().NoError(err)
|
||||
s.NotNil(result)
|
||||
|
||||
// Without vector client, falls back to filterSearch which returns observations
|
||||
// All results should be observations (type is forced to "observations" in Changes)
|
||||
for _, r := range result.Results {
|
||||
s.Equal("observation", r.Type)
|
||||
}
|
||||
}
|
||||
|
||||
// TestHowItWorks_WithRealStores tests the HowItWorks method falls back to filterSearch.
|
||||
func (s *SearchIntegrationSuite) TestHowItWorks_WithRealStores() {
|
||||
ctx := context.Background()
|
||||
|
||||
// Seed test data
|
||||
s.seedObservations(ctx)
|
||||
|
||||
// Test HowItWorks search (without vector client, falls back to filterSearch)
|
||||
result, err := s.manager.HowItWorks(ctx, SearchParams{
|
||||
Project: "test-project",
|
||||
Query: "authentication",
|
||||
Limit: 10,
|
||||
})
|
||||
s.Require().NoError(err)
|
||||
s.NotNil(result)
|
||||
|
||||
// Without vector client, falls back to filterSearch which returns observations
|
||||
// All results should be observations (type is forced to "observations" in HowItWorks)
|
||||
for _, r := range result.Results {
|
||||
s.Equal("observation", r.Type)
|
||||
}
|
||||
}
|
||||
|
||||
// TestObservationToResult tests observation to result conversion with full format.
|
||||
func (s *SearchIntegrationSuite) TestObservationToResult_FullFormat() {
|
||||
ctx := context.Background()
|
||||
|
||||
// Insert single observation
|
||||
obs := &models.ParsedObservation{
|
||||
Type: models.ObsTypeDiscovery,
|
||||
Scope: models.ScopeProject,
|
||||
Title: "Test Title",
|
||||
Narrative: "Detailed narrative content for testing",
|
||||
Concepts: []string{"testing", "content"},
|
||||
}
|
||||
id, _, err := s.obsStore.StoreObservation(ctx, "sdk-test", "test-project", obs, 1, 50)
|
||||
s.Require().NoError(err)
|
||||
|
||||
// Retrieve and convert
|
||||
retrieved, err := s.obsStore.GetObservationByID(ctx, id)
|
||||
s.Require().NoError(err)
|
||||
s.Require().NotNil(retrieved)
|
||||
|
||||
result := s.manager.observationToResult(retrieved, "full")
|
||||
|
||||
s.Equal("observation", result.Type)
|
||||
s.Equal(id, result.ID)
|
||||
s.Equal("Test Title", result.Title)
|
||||
s.Equal("Detailed narrative content for testing", result.Content)
|
||||
s.Equal("test-project", result.Project)
|
||||
s.Equal("project", result.Scope)
|
||||
s.NotNil(result.Metadata)
|
||||
s.Equal("discovery", result.Metadata["obs_type"])
|
||||
}
|
||||
|
||||
// TestObservationToResult_IndexFormat tests index format (no content).
|
||||
func (s *SearchIntegrationSuite) TestObservationToResult_IndexFormat() {
|
||||
ctx := context.Background()
|
||||
|
||||
obs := &models.ParsedObservation{
|
||||
Type: models.ObsTypeBugfix,
|
||||
Scope: models.ScopeGlobal,
|
||||
Title: "Bug Fix Title",
|
||||
Narrative: "This should not appear in index format",
|
||||
}
|
||||
id, _, err := s.obsStore.StoreObservation(ctx, "sdk-test", "test-project", obs, 1, 50)
|
||||
s.Require().NoError(err)
|
||||
|
||||
retrieved, err := s.obsStore.GetObservationByID(ctx, id)
|
||||
s.Require().NoError(err)
|
||||
|
||||
result := s.manager.observationToResult(retrieved, "index")
|
||||
|
||||
s.Equal("observation", result.Type)
|
||||
s.Equal("Bug Fix Title", result.Title)
|
||||
s.Empty(result.Content, "Index format should not include content")
|
||||
s.Equal("global", result.Scope)
|
||||
}
|
||||
|
||||
// TestSummaryToResult_FullFormat tests summary to result conversion.
|
||||
func (s *SearchIntegrationSuite) TestSummaryToResult_FullFormat() {
|
||||
ctx := context.Background()
|
||||
|
||||
sum := &models.ParsedSummary{
|
||||
Request: "Implement new feature",
|
||||
Learned: "Learned important lessons about testing",
|
||||
}
|
||||
id, _, err := s.sumStore.StoreSummary(ctx, "sdk-test", "test-project", sum, 1, 50)
|
||||
s.Require().NoError(err)
|
||||
|
||||
// Retrieve via GetRecentSummaries since there's no GetByID
|
||||
summaries, err := s.sumStore.GetRecentSummaries(ctx, "test-project", 10)
|
||||
s.Require().NoError(err)
|
||||
s.Require().NotEmpty(summaries)
|
||||
|
||||
var retrieved *models.SessionSummary
|
||||
for _, s := range summaries {
|
||||
if s.ID == id {
|
||||
retrieved = s
|
||||
break
|
||||
}
|
||||
}
|
||||
s.Require().NotNil(retrieved)
|
||||
|
||||
result := s.manager.summaryToResult(retrieved, "full")
|
||||
|
||||
s.Equal("session", result.Type)
|
||||
s.Equal(id, result.ID)
|
||||
s.Contains(result.Title, "Implement new feature")
|
||||
s.Equal("Learned important lessons about testing", result.Content)
|
||||
s.Equal("test-project", result.Project)
|
||||
}
|
||||
|
||||
// TestPromptToResult_FullFormat tests prompt to result conversion.
|
||||
func (s *SearchIntegrationSuite) TestPromptToResult_FullFormat() {
|
||||
// First create a session
|
||||
ctx := context.Background()
|
||||
sessionStore := gorm.NewSessionStore(s.store)
|
||||
_, err := sessionStore.CreateSDKSession(ctx, "sdk-prompt-test", "test-project", "initial prompt")
|
||||
s.Require().NoError(err)
|
||||
|
||||
// Save a user prompt
|
||||
promptID, err := s.prmStore.SaveUserPromptWithMatches(ctx, "sdk-prompt-test", 1, "Help me fix this authentication bug", 3)
|
||||
s.Require().NoError(err)
|
||||
|
||||
// Retrieve prompts
|
||||
prompts, err := s.prmStore.GetPromptsByIDs(ctx, []int64{promptID}, "date_desc", 10)
|
||||
s.Require().NoError(err)
|
||||
s.Require().NotEmpty(prompts)
|
||||
|
||||
result := s.manager.promptToResult(prompts[0], "full")
|
||||
|
||||
s.Equal("prompt", result.Type)
|
||||
s.Equal(promptID, result.ID)
|
||||
s.Contains(result.Title, "Help me fix")
|
||||
s.Equal("Help me fix this authentication bug", result.Content)
|
||||
}
|
||||
|
||||
// TestTruncate_TableDriven tests truncation with various inputs.
|
||||
func TestTruncate_TableDriven(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
input string
|
||||
maxLen int
|
||||
expected string
|
||||
}{
|
||||
{"short_string", "hello", 10, "hello"},
|
||||
{"exact_length", "hello", 5, "hello"},
|
||||
{"long_string", "hello world", 5, "hello..."},
|
||||
{"empty_string", "", 10, ""},
|
||||
{"whitespace_only", " ", 10, ""},
|
||||
{"with_leading_space", " hello ", 10, "hello"},
|
||||
{"very_long", "this is a very long string that should be truncated", 20, "this is a very long ..."},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
result := truncate(tt.input, tt.maxLen)
|
||||
assert.Equal(t, tt.expected, result)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestManagerWithNilStores tests that Manager handles nil stores gracefully.
|
||||
func TestManagerWithNilStores(t *testing.T) {
|
||||
m := NewManager(nil, nil, nil, nil)
|
||||
assert.NotNil(t, m)
|
||||
assert.Nil(t, m.observationStore)
|
||||
assert.Nil(t, m.summaryStore)
|
||||
assert.Nil(t, m.promptStore)
|
||||
assert.Nil(t, m.vectorClient)
|
||||
}
|
||||
|
||||
// TestSearchResultMetadataFields tests all metadata fields with real data.
|
||||
func (s *SearchIntegrationSuite) TestSearchResultMetadataFields() {
|
||||
ctx := context.Background()
|
||||
|
||||
obs := &models.ParsedObservation{
|
||||
Type: models.ObsTypeDecision,
|
||||
Scope: models.ScopeGlobal,
|
||||
Title: "Architecture Decision",
|
||||
Concepts: []string{"auth", "security"},
|
||||
FilesRead: []string{"handler.go", "auth.go"},
|
||||
}
|
||||
id, _, err := s.obsStore.StoreObservation(ctx, "sdk-meta-test", "test-project", obs, 1, 50)
|
||||
s.Require().NoError(err)
|
||||
|
||||
retrieved, err := s.obsStore.GetObservationByID(ctx, id)
|
||||
s.Require().NoError(err)
|
||||
|
||||
result := s.manager.observationToResult(retrieved, "full")
|
||||
|
||||
// Check metadata fields
|
||||
s.NotNil(result.Metadata)
|
||||
s.Equal("decision", result.Metadata["obs_type"])
|
||||
s.Equal("global", result.Metadata["scope"])
|
||||
s.Equal("global", result.Scope)
|
||||
}
|
||||
|
||||
// TestObservationToResult_AllFormats tests different format options.
|
||||
func TestObservationToResult_AllFormats(t *testing.T) {
|
||||
m := NewManager(nil, nil, nil, nil)
|
||||
|
||||
obs := &models.Observation{
|
||||
ID: 1,
|
||||
Project: "test",
|
||||
Type: models.ObsTypeBugfix,
|
||||
Scope: models.ScopeProject,
|
||||
Title: sql.NullString{String: "Bug Fix Title", Valid: true},
|
||||
Narrative: sql.NullString{String: "Detailed bug fix narrative", Valid: true},
|
||||
CreatedAtEpoch: 1704067200000,
|
||||
}
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
format string
|
||||
expectContent bool
|
||||
}{
|
||||
{"full_format", "full", true},
|
||||
{"index_format", "index", false},
|
||||
{"empty_format", "", false},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
result := m.observationToResult(obs, tt.format)
|
||||
assert.Equal(t, "observation", result.Type)
|
||||
assert.Equal(t, int64(1), result.ID)
|
||||
if tt.expectContent {
|
||||
assert.NotEmpty(t, result.Content)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestSummaryToResult_AllFormats tests different format options for summaries.
|
||||
func TestSummaryToResult_AllFormats(t *testing.T) {
|
||||
m := NewManager(nil, nil, nil, nil)
|
||||
|
||||
summary := &models.SessionSummary{
|
||||
ID: 1,
|
||||
Project: "test",
|
||||
Request: sql.NullString{String: "Test request", Valid: true},
|
||||
Learned: sql.NullString{String: "Test learned", Valid: true},
|
||||
Completed: sql.NullString{String: "Test completed", Valid: true},
|
||||
NextSteps: sql.NullString{String: "Test next steps", Valid: true},
|
||||
CreatedAtEpoch: 1704067200000,
|
||||
}
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
format string
|
||||
expectContent bool
|
||||
}{
|
||||
{"full_format", "full", true},
|
||||
{"index_format", "index", false},
|
||||
{"empty_format", "", false},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
result := m.summaryToResult(summary, tt.format)
|
||||
assert.Equal(t, "session", result.Type)
|
||||
assert.Equal(t, int64(1), result.ID)
|
||||
if tt.expectContent {
|
||||
assert.NotEmpty(t, result.Content)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
+36
-22
@@ -5,24 +5,24 @@ import (
|
||||
"context"
|
||||
"strings"
|
||||
|
||||
"github.com/lukaszraczylo/claude-mnemonic/internal/db/sqlite"
|
||||
"github.com/lukaszraczylo/claude-mnemonic/internal/db/gorm"
|
||||
"github.com/lukaszraczylo/claude-mnemonic/internal/vector/sqlitevec"
|
||||
"github.com/lukaszraczylo/claude-mnemonic/pkg/models"
|
||||
)
|
||||
|
||||
// Manager provides unified search across SQLite and sqlite-vec.
|
||||
type Manager struct {
|
||||
observationStore *sqlite.ObservationStore
|
||||
summaryStore *sqlite.SummaryStore
|
||||
promptStore *sqlite.PromptStore
|
||||
observationStore *gorm.ObservationStore
|
||||
summaryStore *gorm.SummaryStore
|
||||
promptStore *gorm.PromptStore
|
||||
vectorClient *sqlitevec.Client
|
||||
}
|
||||
|
||||
// NewManager creates a new search manager.
|
||||
func NewManager(
|
||||
observationStore *sqlite.ObservationStore,
|
||||
summaryStore *sqlite.SummaryStore,
|
||||
promptStore *sqlite.PromptStore,
|
||||
observationStore *gorm.ObservationStore,
|
||||
summaryStore *gorm.SummaryStore,
|
||||
promptStore *gorm.PromptStore,
|
||||
vectorClient *sqlitevec.Client,
|
||||
) *Manager {
|
||||
return &Manager{
|
||||
@@ -35,20 +35,21 @@ func NewManager(
|
||||
|
||||
// SearchParams contains parameters for unified search.
|
||||
type SearchParams struct {
|
||||
Query string
|
||||
Type string // "observations", "sessions", "prompts", or empty for all
|
||||
Project string
|
||||
ObsType string // Observation type filter
|
||||
Concepts string
|
||||
Files string
|
||||
DateStart int64
|
||||
DateEnd int64
|
||||
OrderBy string // "relevance", "date_desc", "date_asc"
|
||||
Limit int
|
||||
Offset int
|
||||
Format string // "index" or "full"
|
||||
Scope string // "project", "global", or empty for project+global
|
||||
IncludeGlobal bool // If true, include global observations along with project-scoped
|
||||
Query string
|
||||
Type string // "observations", "sessions", "prompts", or empty for all
|
||||
Project string
|
||||
ObsType string // Observation type filter
|
||||
Concepts string
|
||||
Files string
|
||||
DateStart int64
|
||||
DateEnd int64
|
||||
OrderBy string // "relevance", "date_desc", "date_asc"
|
||||
Limit int
|
||||
Offset int
|
||||
Format string // "index" or "full"
|
||||
Scope string // "project", "global", or empty for project+global
|
||||
IncludeGlobal bool // If true, include global observations along with project-scoped
|
||||
ExcludeSuperseded bool // If true, exclude observations that have been superseded
|
||||
}
|
||||
|
||||
// SearchResult represents a unified search result.
|
||||
@@ -126,6 +127,10 @@ func (m *Manager) vectorSearch(ctx context.Context, params SearchParams) (*Unifi
|
||||
obs, err := m.observationStore.GetObservationsByIDs(ctx, obsIDs, params.OrderBy, 0)
|
||||
if err == nil {
|
||||
for _, o := range obs {
|
||||
// Skip superseded observations when requested
|
||||
if params.ExcludeSuperseded && o.IsSuperseded {
|
||||
continue
|
||||
}
|
||||
results = append(results, m.observationToResult(o, params.Format))
|
||||
}
|
||||
}
|
||||
@@ -167,7 +172,16 @@ func (m *Manager) filterSearch(ctx context.Context, params SearchParams) (*Unifi
|
||||
|
||||
// Search observations
|
||||
if params.Type == "" || params.Type == "observations" {
|
||||
obs, err := m.observationStore.GetRecentObservations(ctx, params.Project, params.Limit)
|
||||
var obs []*models.Observation
|
||||
var err error
|
||||
|
||||
// Use active observations (excluding superseded) when requested
|
||||
if params.ExcludeSuperseded {
|
||||
obs, err = m.observationStore.GetActiveObservations(ctx, params.Project, params.Limit)
|
||||
} else {
|
||||
obs, err = m.observationStore.GetRecentObservations(ctx, params.Project, params.Limit)
|
||||
}
|
||||
|
||||
if err == nil {
|
||||
for _, o := range obs {
|
||||
results = append(results, m.observationToResult(o, params.Format))
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
+114
-53
@@ -27,7 +27,7 @@ const (
|
||||
GitHubRepo = "lukaszraczylo/claude-mnemonic"
|
||||
ReleasesAPI = "https://api.github.com/repos/" + GitHubRepo + "/releases/latest"
|
||||
CheckInterval = 24 * time.Hour
|
||||
MaxExtractedSize = 100 * 1024 * 1024 // 100MB max per extracted file
|
||||
MaxExtractedSize = 250 * 1024 * 1024 // 250MB max per extracted file
|
||||
RestartDelay = 500 * time.Millisecond
|
||||
)
|
||||
|
||||
@@ -49,22 +49,36 @@ type Asset struct {
|
||||
|
||||
// UpdateInfo contains information about an available update.
|
||||
type UpdateInfo struct {
|
||||
Available bool `json:"available"`
|
||||
CurrentVersion string `json:"current_version"`
|
||||
LatestVersion string `json:"latest_version"`
|
||||
ReleaseNotes string `json:"release_notes,omitempty"`
|
||||
PublishedAt time.Time `json:"published_at,omitempty"`
|
||||
DownloadURL string `json:"download_url,omitempty"`
|
||||
ChecksumsURL string `json:"checksums_url,omitempty"`
|
||||
BundleURL string `json:"bundle_url,omitempty"` // Sigstore bundle (.sigstore.json)
|
||||
Available bool `json:"available"`
|
||||
CurrentVersion string `json:"current_version"`
|
||||
LatestVersion string `json:"latest_version"`
|
||||
ReleaseNotes string `json:"release_notes,omitempty"`
|
||||
PublishedAt time.Time `json:"published_at,omitempty"`
|
||||
DownloadURL string `json:"download_url,omitempty"`
|
||||
ChecksumsURL string `json:"checksums_url,omitempty"`
|
||||
BundleURL string `json:"bundle_url,omitempty"` // Sigstore bundle (.sigstore.json)
|
||||
ManualUpdateCommand string `json:"manual_update_command,omitempty"`
|
||||
}
|
||||
|
||||
// InstallScriptURL is the URL to the remote installation script.
|
||||
const InstallScriptURL = "https://raw.githubusercontent.com/" + GitHubRepo + "/main/scripts/install.sh"
|
||||
|
||||
// GetManualUpdateCommand returns the curl command for manual update.
|
||||
// If version is empty, it installs the latest version.
|
||||
func GetManualUpdateCommand(version string) string {
|
||||
if version == "" {
|
||||
return fmt.Sprintf("curl -sSL %s | bash", InstallScriptURL)
|
||||
}
|
||||
return fmt.Sprintf("curl -sSL %s | bash -s -- %s", InstallScriptURL, version)
|
||||
}
|
||||
|
||||
// UpdateStatus represents the current update status.
|
||||
type UpdateStatus struct {
|
||||
State string `json:"state"` // "idle", "checking", "downloading", "verifying", "applying", "done", "error"
|
||||
Progress float64 `json:"progress"`
|
||||
Message string `json:"message"`
|
||||
Error string `json:"error,omitempty"`
|
||||
State string `json:"state"` // "idle", "checking", "downloading", "verifying", "applying", "done", "error"
|
||||
Progress float64 `json:"progress"`
|
||||
Message string `json:"message"`
|
||||
Error string `json:"error,omitempty"`
|
||||
ManualUpdateCommand string `json:"manual_update_command,omitempty"` // Shown when update fails
|
||||
}
|
||||
|
||||
// Updater handles self-updates.
|
||||
@@ -112,9 +126,10 @@ func (u *Updater) setError(err error) {
|
||||
u.mu.Lock()
|
||||
defer u.mu.Unlock()
|
||||
u.status = UpdateStatus{
|
||||
State: "error",
|
||||
Message: "Update failed",
|
||||
Error: err.Error(),
|
||||
State: "error",
|
||||
Message: "Update failed",
|
||||
Error: err.Error(),
|
||||
ManualUpdateCommand: GetManualUpdateCommand(""), // Always provide fallback command
|
||||
}
|
||||
}
|
||||
|
||||
@@ -169,6 +184,9 @@ func (u *Updater) CheckForUpdate(ctx context.Context) (*UpdateInfo, error) {
|
||||
// Compare versions
|
||||
info.Available = isNewerVersion(info.LatestVersion, u.currentVersion)
|
||||
|
||||
// Always include manual update command as an alternative option
|
||||
info.ManualUpdateCommand = GetManualUpdateCommand("v" + info.LatestVersion)
|
||||
|
||||
if info.Available {
|
||||
// Find download URLs for current platform
|
||||
platform := getPlatform()
|
||||
@@ -451,57 +469,100 @@ func (u *Updater) extractTarGz(archivePath, destDir string) error {
|
||||
}
|
||||
|
||||
func (u *Updater) replaceBinaries(extractDir string) error {
|
||||
// Files to replace
|
||||
binaries := []struct {
|
||||
src string
|
||||
dest string
|
||||
}{
|
||||
{"worker", filepath.Join(u.installDir, "worker")},
|
||||
{"mcp-server", filepath.Join(u.installDir, "mcp-server")},
|
||||
{"hooks/session-start", filepath.Join(u.installDir, "hooks", "session-start")},
|
||||
{"hooks/user-prompt", filepath.Join(u.installDir, "hooks", "user-prompt")},
|
||||
{"hooks/post-tool-use", filepath.Join(u.installDir, "hooks", "post-tool-use")},
|
||||
{"hooks/stop", filepath.Join(u.installDir, "hooks", "stop")},
|
||||
{"hooks/subagent-stop", filepath.Join(u.installDir, "hooks", "subagent-stop")},
|
||||
// Binary files to replace
|
||||
binaryFiles := []string{
|
||||
"worker",
|
||||
"mcp-server",
|
||||
"hooks/session-start",
|
||||
"hooks/user-prompt",
|
||||
"hooks/post-tool-use",
|
||||
"hooks/stop",
|
||||
"hooks/subagent-stop",
|
||||
"hooks/statusline",
|
||||
}
|
||||
|
||||
for _, b := range binaries {
|
||||
src := filepath.Join(extractDir, b.src)
|
||||
if _, err := os.Stat(src); os.IsNotExist(err) {
|
||||
continue // Skip if not in archive
|
||||
}
|
||||
// Get all install directories (marketplaces and cache directories)
|
||||
installDirs := u.getInstallDirectories()
|
||||
|
||||
// Backup existing binary
|
||||
if _, err := os.Stat(b.dest); err == nil {
|
||||
backup := b.dest + ".bak"
|
||||
if err := os.Rename(b.dest, backup); err != nil {
|
||||
return fmt.Errorf("failed to backup %s: %w", b.dest, err)
|
||||
for _, installDir := range installDirs {
|
||||
log.Debug().Str("dir", installDir).Msg("Installing binaries to directory")
|
||||
|
||||
for _, binaryFile := range binaryFiles {
|
||||
src := filepath.Join(extractDir, binaryFile)
|
||||
if _, err := os.Stat(src); os.IsNotExist(err) {
|
||||
continue // Skip if not in archive
|
||||
}
|
||||
defer func(backup, dest string) {
|
||||
// Clean up backup on success, restore on failure
|
||||
if _, err := os.Stat(dest); err == nil {
|
||||
_ = os.Remove(backup)
|
||||
|
||||
dest := filepath.Join(installDir, binaryFile)
|
||||
|
||||
// Backup existing binary
|
||||
if _, err := os.Stat(dest); err == nil {
|
||||
backup := dest + ".bak"
|
||||
if err := os.Rename(dest, backup); err != nil {
|
||||
log.Warn().Err(err).Str("file", dest).Msg("Failed to backup, continuing anyway")
|
||||
} else {
|
||||
_ = os.Rename(backup, dest)
|
||||
defer func(backup, dest string) {
|
||||
// Clean up backup on success, restore on failure
|
||||
if _, err := os.Stat(dest); err == nil {
|
||||
_ = os.Remove(backup)
|
||||
} else {
|
||||
_ = os.Rename(backup, dest)
|
||||
}
|
||||
}(backup, dest)
|
||||
}
|
||||
}(backup, b.dest)
|
||||
}
|
||||
}
|
||||
|
||||
// Copy new binary
|
||||
if err := copyFile(src, b.dest); err != nil {
|
||||
return fmt.Errorf("failed to install %s: %w", b.dest, err)
|
||||
}
|
||||
// Copy new binary
|
||||
if err := copyFile(src, dest); err != nil {
|
||||
return fmt.Errorf("failed to install %s: %w", dest, err)
|
||||
}
|
||||
|
||||
// Make executable
|
||||
// #nosec G302 -- executables require 0755 permissions
|
||||
if err := os.Chmod(b.dest, 0755); err != nil {
|
||||
return fmt.Errorf("failed to chmod %s: %w", b.dest, err)
|
||||
// Make executable
|
||||
// #nosec G302 -- executables require 0755 permissions
|
||||
if err := os.Chmod(dest, 0755); err != nil {
|
||||
return fmt.Errorf("failed to chmod %s: %w", dest, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// getInstallDirectories returns all directories where binaries should be installed.
|
||||
// This includes the marketplaces directory and any cache directories.
|
||||
func (u *Updater) getInstallDirectories() []string {
|
||||
dirs := []string{u.installDir}
|
||||
|
||||
// Also check cache directories where Claude Code looks for plugins
|
||||
home, err := os.UserHomeDir()
|
||||
if err != nil {
|
||||
return dirs
|
||||
}
|
||||
|
||||
// Look for cache directories under ~/.claude/plugins/cache/claude-mnemonic/claude-mnemonic/
|
||||
cacheBase := filepath.Join(home, ".claude/plugins/cache/claude-mnemonic/claude-mnemonic")
|
||||
entries, err := os.ReadDir(cacheBase)
|
||||
if err != nil {
|
||||
return dirs
|
||||
}
|
||||
|
||||
for _, entry := range entries {
|
||||
if entry.IsDir() {
|
||||
cacheDir := filepath.Join(cacheBase, entry.Name())
|
||||
// Only add if it's different from installDir and contains a worker binary
|
||||
if cacheDir != u.installDir {
|
||||
workerPath := filepath.Join(cacheDir, "worker")
|
||||
if _, err := os.Stat(workerPath); err == nil {
|
||||
dirs = append(dirs, cacheDir)
|
||||
log.Debug().Str("dir", cacheDir).Msg("Found cache directory to update")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return dirs
|
||||
}
|
||||
|
||||
func copyFile(src, dst string) error {
|
||||
// Ensure parent directory exists
|
||||
if err := os.MkdirAll(filepath.Dir(dst), 0750); err != nil {
|
||||
|
||||
@@ -60,12 +60,15 @@ func (c *Client) AddDocuments(ctx context.Context, docs []Document) error {
|
||||
return fmt.Errorf("generate embeddings: %w", err)
|
||||
}
|
||||
|
||||
// Insert into vectors table
|
||||
// Insert into vectors table with model version tracking
|
||||
const insertQuery = `
|
||||
INSERT OR REPLACE INTO vectors (doc_id, embedding, sqlite_id, doc_type, field_type, project, scope)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?)
|
||||
INSERT OR REPLACE INTO vectors (doc_id, embedding, sqlite_id, doc_type, field_type, project, scope, model_version)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
||||
`
|
||||
|
||||
// Get current model version for tracking
|
||||
modelVersion := c.embedSvc.Version()
|
||||
|
||||
tx, err := c.db.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return fmt.Errorf("begin transaction: %w", err)
|
||||
@@ -104,6 +107,7 @@ func (c *Client) AddDocuments(ctx context.Context, docs []Document) error {
|
||||
fieldType,
|
||||
project,
|
||||
scope,
|
||||
modelVersion,
|
||||
)
|
||||
if err != nil {
|
||||
return fmt.Errorf("insert document %s: %w", doc.ID, err)
|
||||
@@ -114,7 +118,7 @@ func (c *Client) AddDocuments(ctx context.Context, docs []Document) error {
|
||||
return fmt.Errorf("commit transaction: %w", err)
|
||||
}
|
||||
|
||||
log.Debug().Int("count", len(docs)).Msg("Added documents to sqlite-vec")
|
||||
log.Debug().Int("count", len(docs)).Str("model", modelVersion).Msg("Added documents to sqlite-vec")
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -212,6 +216,7 @@ func (c *Client) Query(ctx context.Context, query string, limit int, where map[s
|
||||
return nil, fmt.Errorf("scan row: %w", err)
|
||||
}
|
||||
|
||||
r.Similarity = DistanceToSimilarity(r.Distance)
|
||||
r.Metadata = map[string]any{
|
||||
"sqlite_id": float64(sqliteID), // Keep as float64 for compatibility
|
||||
"doc_type": docType.String,
|
||||
@@ -252,3 +257,148 @@ func truncateString(s string, maxLen int) string {
|
||||
}
|
||||
return s[:maxLen] + "..."
|
||||
}
|
||||
|
||||
// Count returns the total number of vectors in the store.
|
||||
func (c *Client) Count(ctx context.Context) (int64, error) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
|
||||
var count int64
|
||||
err := c.db.QueryRowContext(ctx, "SELECT COUNT(*) FROM vectors").Scan(&count)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("count vectors: %w", err)
|
||||
}
|
||||
return count, nil
|
||||
}
|
||||
|
||||
// ModelVersion returns the current embedding model version.
|
||||
func (c *Client) ModelVersion() string {
|
||||
return c.embedSvc.Version()
|
||||
}
|
||||
|
||||
// NeedsRebuild checks if vectors need to be rebuilt due to model version change.
|
||||
// Returns true if:
|
||||
// - The vectors table is empty
|
||||
// - Any vectors have a different model_version than the current model
|
||||
func (c *Client) NeedsRebuild(ctx context.Context) (bool, string) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
|
||||
currentModel := c.embedSvc.Version()
|
||||
|
||||
// Check total count
|
||||
var totalCount int64
|
||||
err := c.db.QueryRowContext(ctx, "SELECT COUNT(*) FROM vectors").Scan(&totalCount)
|
||||
if err != nil {
|
||||
log.Warn().Err(err).Msg("Failed to count vectors for rebuild check")
|
||||
return false, ""
|
||||
}
|
||||
|
||||
if totalCount == 0 {
|
||||
return true, "empty"
|
||||
}
|
||||
|
||||
// Check for vectors with different model version
|
||||
var staleCount int64
|
||||
err = c.db.QueryRowContext(ctx,
|
||||
"SELECT COUNT(*) FROM vectors WHERE model_version != ? OR model_version IS NULL",
|
||||
currentModel,
|
||||
).Scan(&staleCount)
|
||||
if err != nil {
|
||||
log.Warn().Err(err).Msg("Failed to count stale vectors")
|
||||
return false, ""
|
||||
}
|
||||
|
||||
if staleCount > 0 {
|
||||
return true, fmt.Sprintf("model_mismatch:%d", staleCount)
|
||||
}
|
||||
|
||||
return false, ""
|
||||
}
|
||||
|
||||
// StaleVectorInfo contains information about a vector that needs rebuilding.
|
||||
type StaleVectorInfo struct {
|
||||
DocID string
|
||||
SQLiteID int64
|
||||
DocType string
|
||||
FieldType string
|
||||
Project string
|
||||
Scope string
|
||||
}
|
||||
|
||||
// GetStaleVectors returns doc_ids of vectors with mismatched or null model versions.
|
||||
// This enables granular rebuild - only re-embedding documents that need updating.
|
||||
func (c *Client) GetStaleVectors(ctx context.Context) ([]StaleVectorInfo, error) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
|
||||
currentModel := c.embedSvc.Version()
|
||||
|
||||
query := `
|
||||
SELECT doc_id, sqlite_id, doc_type, field_type, project, scope
|
||||
FROM vectors
|
||||
WHERE model_version != ? OR model_version IS NULL
|
||||
`
|
||||
|
||||
rows, err := c.db.QueryContext(ctx, query, currentModel)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("query stale vectors: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var results []StaleVectorInfo
|
||||
for rows.Next() {
|
||||
var info StaleVectorInfo
|
||||
var sqliteID sql.NullInt64
|
||||
var docType, fieldType, project, scope sql.NullString
|
||||
|
||||
if err := rows.Scan(&info.DocID, &sqliteID, &docType, &fieldType, &project, &scope); err != nil {
|
||||
return nil, fmt.Errorf("scan row: %w", err)
|
||||
}
|
||||
|
||||
info.SQLiteID = sqliteID.Int64
|
||||
info.DocType = docType.String
|
||||
info.FieldType = fieldType.String
|
||||
info.Project = project.String
|
||||
info.Scope = scope.String
|
||||
|
||||
results = append(results, info)
|
||||
}
|
||||
|
||||
if err = rows.Err(); err != nil {
|
||||
return nil, fmt.Errorf("iterate rows: %w", err)
|
||||
}
|
||||
|
||||
return results, nil
|
||||
}
|
||||
|
||||
// DeleteVectorsByDocIDs removes vectors by their doc_ids.
|
||||
// Used for granular rebuild - delete stale vectors before re-adding.
|
||||
func (c *Client) DeleteVectorsByDocIDs(ctx context.Context, docIDs []string) error {
|
||||
if len(docIDs) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
|
||||
// Build placeholder string
|
||||
placeholders := make([]string, len(docIDs))
|
||||
args := make([]interface{}, len(docIDs))
|
||||
for i, id := range docIDs {
|
||||
placeholders[i] = "?"
|
||||
args[i] = id
|
||||
}
|
||||
|
||||
// #nosec G201 -- Placeholders are "?" strings, actual values are parameterized via args
|
||||
query := fmt.Sprintf("DELETE FROM vectors WHERE doc_id IN (%s)",
|
||||
strings.Join(placeholders, ","))
|
||||
|
||||
_, err := c.db.ExecContext(ctx, query, args...)
|
||||
if err != nil {
|
||||
return fmt.Errorf("delete vectors by doc_ids: %w", err)
|
||||
}
|
||||
|
||||
log.Debug().Int("count", len(docIDs)).Msg("Deleted stale vectors by doc_id")
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -0,0 +1,754 @@
|
||||
package sqlitevec
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
sqlite_vec "github.com/asg017/sqlite-vec-go-bindings/cgo"
|
||||
"github.com/lukaszraczylo/claude-mnemonic/internal/embedding"
|
||||
_ "github.com/mattn/go-sqlite3"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// testDB creates a test SQLite database with the vectors table.
|
||||
func testDB(t *testing.T) (*sql.DB, func()) {
|
||||
t.Helper()
|
||||
|
||||
// Create temp directory
|
||||
tmpDir, err := os.MkdirTemp("", "sqlitevec-test-*")
|
||||
require.NoError(t, err)
|
||||
|
||||
dbPath := filepath.Join(tmpDir, "test.db")
|
||||
db, err := sql.Open("sqlite3", dbPath)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Enable sqlite-vec
|
||||
sqlite_vec.Auto()
|
||||
|
||||
// Create vectors table (matches production schema)
|
||||
_, err = db.Exec(`
|
||||
CREATE VIRTUAL TABLE IF NOT EXISTS vectors USING vec0(
|
||||
doc_id TEXT PRIMARY KEY,
|
||||
embedding float[384],
|
||||
sqlite_id INTEGER,
|
||||
doc_type TEXT,
|
||||
field_type TEXT,
|
||||
project TEXT,
|
||||
scope TEXT,
|
||||
model_version TEXT
|
||||
)
|
||||
`)
|
||||
require.NoError(t, err)
|
||||
|
||||
cleanup := func() {
|
||||
db.Close()
|
||||
os.RemoveAll(tmpDir)
|
||||
}
|
||||
|
||||
return db, cleanup
|
||||
}
|
||||
|
||||
// testEmbeddingService creates a test embedding service.
|
||||
func testEmbeddingService(t *testing.T) (*embedding.Service, func()) {
|
||||
t.Helper()
|
||||
|
||||
svc, err := embedding.NewService()
|
||||
require.NoError(t, err)
|
||||
|
||||
cleanup := func() {
|
||||
svc.Close()
|
||||
}
|
||||
|
||||
return svc, cleanup
|
||||
}
|
||||
|
||||
func TestNewClient_Success(t *testing.T) {
|
||||
db, dbCleanup := testDB(t)
|
||||
defer dbCleanup()
|
||||
|
||||
embedSvc, embedCleanup := testEmbeddingService(t)
|
||||
defer embedCleanup()
|
||||
|
||||
client, err := NewClient(Config{DB: db}, embedSvc)
|
||||
require.NoError(t, err)
|
||||
assert.NotNil(t, client)
|
||||
}
|
||||
|
||||
func TestNewClient_NilDB(t *testing.T) {
|
||||
embedSvc, embedCleanup := testEmbeddingService(t)
|
||||
defer embedCleanup()
|
||||
|
||||
client, err := NewClient(Config{DB: nil}, embedSvc)
|
||||
assert.Error(t, err)
|
||||
assert.Nil(t, client)
|
||||
assert.Contains(t, err.Error(), "database connection required")
|
||||
}
|
||||
|
||||
func TestNewClient_NilEmbedding(t *testing.T) {
|
||||
db, dbCleanup := testDB(t)
|
||||
defer dbCleanup()
|
||||
|
||||
client, err := NewClient(Config{DB: db}, nil)
|
||||
assert.Error(t, err)
|
||||
assert.Nil(t, client)
|
||||
assert.Contains(t, err.Error(), "embedding service required")
|
||||
}
|
||||
|
||||
func TestClient_AddDocuments_Empty(t *testing.T) {
|
||||
db, dbCleanup := testDB(t)
|
||||
defer dbCleanup()
|
||||
|
||||
embedSvc, embedCleanup := testEmbeddingService(t)
|
||||
defer embedCleanup()
|
||||
|
||||
client, err := NewClient(Config{DB: db}, embedSvc)
|
||||
require.NoError(t, err)
|
||||
|
||||
err = client.AddDocuments(context.Background(), []Document{})
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
func TestClient_AddDocuments_Single(t *testing.T) {
|
||||
db, dbCleanup := testDB(t)
|
||||
defer dbCleanup()
|
||||
|
||||
embedSvc, embedCleanup := testEmbeddingService(t)
|
||||
defer embedCleanup()
|
||||
|
||||
client, err := NewClient(Config{DB: db}, embedSvc)
|
||||
require.NoError(t, err)
|
||||
|
||||
docs := []Document{
|
||||
{
|
||||
ID: "obs-1-title",
|
||||
Content: "This is a test observation about authentication.",
|
||||
Metadata: map[string]any{
|
||||
"sqlite_id": int64(1),
|
||||
"doc_type": "observation",
|
||||
"field_type": "title",
|
||||
"project": "test-project",
|
||||
"scope": "project",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
err = client.AddDocuments(context.Background(), docs)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Verify document was inserted
|
||||
var count int
|
||||
err = db.QueryRow("SELECT COUNT(*) FROM vectors WHERE doc_id = ?", "obs-1-title").Scan(&count)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 1, count)
|
||||
}
|
||||
|
||||
func TestClient_AddDocuments_Multiple(t *testing.T) {
|
||||
db, dbCleanup := testDB(t)
|
||||
defer dbCleanup()
|
||||
|
||||
embedSvc, embedCleanup := testEmbeddingService(t)
|
||||
defer embedCleanup()
|
||||
|
||||
client, err := NewClient(Config{DB: db}, embedSvc)
|
||||
require.NoError(t, err)
|
||||
|
||||
docs := []Document{
|
||||
{
|
||||
ID: "obs-1-title",
|
||||
Content: "Authentication flow implementation.",
|
||||
Metadata: map[string]any{
|
||||
"sqlite_id": int64(1),
|
||||
"doc_type": "observation",
|
||||
"field_type": "title",
|
||||
"project": "test-project",
|
||||
"scope": "project",
|
||||
},
|
||||
},
|
||||
{
|
||||
ID: "obs-1-narrative",
|
||||
Content: "We implemented JWT-based authentication.",
|
||||
Metadata: map[string]any{
|
||||
"sqlite_id": int64(1),
|
||||
"doc_type": "observation",
|
||||
"field_type": "narrative",
|
||||
"project": "test-project",
|
||||
"scope": "project",
|
||||
},
|
||||
},
|
||||
{
|
||||
ID: "obs-2-title",
|
||||
Content: "Database optimization.",
|
||||
Metadata: map[string]any{
|
||||
"sqlite_id": int64(2),
|
||||
"doc_type": "observation",
|
||||
"field_type": "title",
|
||||
"project": "test-project",
|
||||
"scope": "global",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
err = client.AddDocuments(context.Background(), docs)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Verify all documents were inserted
|
||||
var count int
|
||||
err = db.QueryRow("SELECT COUNT(*) FROM vectors").Scan(&count)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 3, count)
|
||||
}
|
||||
|
||||
func TestClient_DeleteDocuments_Empty(t *testing.T) {
|
||||
db, dbCleanup := testDB(t)
|
||||
defer dbCleanup()
|
||||
|
||||
embedSvc, embedCleanup := testEmbeddingService(t)
|
||||
defer embedCleanup()
|
||||
|
||||
client, err := NewClient(Config{DB: db}, embedSvc)
|
||||
require.NoError(t, err)
|
||||
|
||||
err = client.DeleteDocuments(context.Background(), []string{})
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
func TestClient_DeleteDocuments_Existing(t *testing.T) {
|
||||
db, dbCleanup := testDB(t)
|
||||
defer dbCleanup()
|
||||
|
||||
embedSvc, embedCleanup := testEmbeddingService(t)
|
||||
defer embedCleanup()
|
||||
|
||||
client, err := NewClient(Config{DB: db}, embedSvc)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Add documents first
|
||||
docs := []Document{
|
||||
{
|
||||
ID: "doc-1",
|
||||
Content: "First document.",
|
||||
Metadata: map[string]any{
|
||||
"sqlite_id": int64(1),
|
||||
"doc_type": "observation",
|
||||
},
|
||||
},
|
||||
{
|
||||
ID: "doc-2",
|
||||
Content: "Second document.",
|
||||
Metadata: map[string]any{
|
||||
"sqlite_id": int64(2),
|
||||
"doc_type": "observation",
|
||||
},
|
||||
},
|
||||
}
|
||||
err = client.AddDocuments(context.Background(), docs)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Delete one document
|
||||
err = client.DeleteDocuments(context.Background(), []string{"doc-1"})
|
||||
require.NoError(t, err)
|
||||
|
||||
// Verify only one remains
|
||||
var count int
|
||||
err = db.QueryRow("SELECT COUNT(*) FROM vectors").Scan(&count)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 1, count)
|
||||
}
|
||||
|
||||
func TestClient_Query_Basic(t *testing.T) {
|
||||
db, dbCleanup := testDB(t)
|
||||
defer dbCleanup()
|
||||
|
||||
embedSvc, embedCleanup := testEmbeddingService(t)
|
||||
defer embedCleanup()
|
||||
|
||||
client, err := NewClient(Config{DB: db}, embedSvc)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Add some test documents
|
||||
docs := []Document{
|
||||
{
|
||||
ID: "obs-1",
|
||||
Content: "Authentication and login security implementation.",
|
||||
Metadata: map[string]any{
|
||||
"sqlite_id": int64(1),
|
||||
"doc_type": "observation",
|
||||
"project": "test-project",
|
||||
"scope": "project",
|
||||
},
|
||||
},
|
||||
{
|
||||
ID: "obs-2",
|
||||
Content: "Database query optimization techniques.",
|
||||
Metadata: map[string]any{
|
||||
"sqlite_id": int64(2),
|
||||
"doc_type": "observation",
|
||||
"project": "test-project",
|
||||
"scope": "project",
|
||||
},
|
||||
},
|
||||
}
|
||||
err = client.AddDocuments(context.Background(), docs)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Query for authentication-related content
|
||||
results, err := client.Query(context.Background(), "login authentication", 10, nil)
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.NotEmpty(t, results)
|
||||
assert.LessOrEqual(t, len(results), 10)
|
||||
|
||||
// First result should be the authentication document (higher similarity)
|
||||
assert.Equal(t, "obs-1", results[0].ID)
|
||||
}
|
||||
|
||||
func TestClient_Query_WithDocTypeFilter(t *testing.T) {
|
||||
db, dbCleanup := testDB(t)
|
||||
defer dbCleanup()
|
||||
|
||||
embedSvc, embedCleanup := testEmbeddingService(t)
|
||||
defer embedCleanup()
|
||||
|
||||
client, err := NewClient(Config{DB: db}, embedSvc)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Add documents of different types
|
||||
docs := []Document{
|
||||
{
|
||||
ID: "obs-1",
|
||||
Content: "Test content for observation.",
|
||||
Metadata: map[string]any{
|
||||
"sqlite_id": int64(1),
|
||||
"doc_type": "observation",
|
||||
"project": "test-project",
|
||||
},
|
||||
},
|
||||
{
|
||||
ID: "summary-1",
|
||||
Content: "Test content for summary.",
|
||||
Metadata: map[string]any{
|
||||
"sqlite_id": int64(10),
|
||||
"doc_type": "session_summary",
|
||||
"project": "test-project",
|
||||
},
|
||||
},
|
||||
}
|
||||
err = client.AddDocuments(context.Background(), docs)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Query with doc_type filter
|
||||
where := map[string]any{"doc_type": "observation"}
|
||||
results, err := client.Query(context.Background(), "test content", 10, where)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Should only return observation documents
|
||||
for _, r := range results {
|
||||
docType, _ := r.Metadata["doc_type"].(string)
|
||||
assert.Equal(t, "observation", docType)
|
||||
}
|
||||
}
|
||||
|
||||
func TestClient_Query_WithProjectFilter(t *testing.T) {
|
||||
db, dbCleanup := testDB(t)
|
||||
defer dbCleanup()
|
||||
|
||||
embedSvc, embedCleanup := testEmbeddingService(t)
|
||||
defer embedCleanup()
|
||||
|
||||
client, err := NewClient(Config{DB: db}, embedSvc)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Add documents from different projects
|
||||
docs := []Document{
|
||||
{
|
||||
ID: "obs-1",
|
||||
Content: "Project A authentication content.",
|
||||
Metadata: map[string]any{
|
||||
"sqlite_id": int64(1),
|
||||
"doc_type": "observation",
|
||||
"project": "project-a",
|
||||
"scope": "project",
|
||||
},
|
||||
},
|
||||
{
|
||||
ID: "obs-2",
|
||||
Content: "Project B database content.",
|
||||
Metadata: map[string]any{
|
||||
"sqlite_id": int64(2),
|
||||
"doc_type": "observation",
|
||||
"project": "project-b",
|
||||
"scope": "project",
|
||||
},
|
||||
},
|
||||
{
|
||||
ID: "obs-3",
|
||||
Content: "Global security best practices.",
|
||||
Metadata: map[string]any{
|
||||
"sqlite_id": int64(3),
|
||||
"doc_type": "observation",
|
||||
"project": "project-b",
|
||||
"scope": "global",
|
||||
},
|
||||
},
|
||||
}
|
||||
err = client.AddDocuments(context.Background(), docs)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Query without project filter to verify all docs are there
|
||||
results, err := client.Query(context.Background(), "authentication security", 10, nil)
|
||||
require.NoError(t, err)
|
||||
assert.NotEmpty(t, results, "Should find some results")
|
||||
}
|
||||
|
||||
func TestClient_IsConnected(t *testing.T) {
|
||||
db, dbCleanup := testDB(t)
|
||||
defer dbCleanup()
|
||||
|
||||
embedSvc, embedCleanup := testEmbeddingService(t)
|
||||
defer embedCleanup()
|
||||
|
||||
client, err := NewClient(Config{DB: db}, embedSvc)
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.True(t, client.IsConnected())
|
||||
}
|
||||
|
||||
func TestClient_Close(t *testing.T) {
|
||||
db, dbCleanup := testDB(t)
|
||||
defer dbCleanup()
|
||||
|
||||
embedSvc, embedCleanup := testEmbeddingService(t)
|
||||
defer embedCleanup()
|
||||
|
||||
client, err := NewClient(Config{DB: db}, embedSvc)
|
||||
require.NoError(t, err)
|
||||
|
||||
err = client.Close()
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
func TestConfig_Fields(t *testing.T) {
|
||||
db, dbCleanup := testDB(t)
|
||||
defer dbCleanup()
|
||||
|
||||
cfg := Config{DB: db}
|
||||
assert.Equal(t, db, cfg.DB)
|
||||
}
|
||||
|
||||
func TestClient_UpdateDocument_DeleteThenAdd(t *testing.T) {
|
||||
db, dbCleanup := testDB(t)
|
||||
defer dbCleanup()
|
||||
|
||||
embedSvc, embedCleanup := testEmbeddingService(t)
|
||||
defer embedCleanup()
|
||||
|
||||
client, err := NewClient(Config{DB: db}, embedSvc)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Add document
|
||||
docs1 := []Document{
|
||||
{
|
||||
ID: "doc-1",
|
||||
Content: "Original content.",
|
||||
Metadata: map[string]any{
|
||||
"sqlite_id": int64(1),
|
||||
"doc_type": "observation",
|
||||
},
|
||||
},
|
||||
}
|
||||
err = client.AddDocuments(context.Background(), docs1)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Delete then add with new content (proper update pattern)
|
||||
err = client.DeleteDocuments(context.Background(), []string{"doc-1"})
|
||||
require.NoError(t, err)
|
||||
|
||||
docs2 := []Document{
|
||||
{
|
||||
ID: "doc-1",
|
||||
Content: "Updated content.",
|
||||
Metadata: map[string]any{
|
||||
"sqlite_id": int64(1),
|
||||
"doc_type": "observation",
|
||||
},
|
||||
},
|
||||
}
|
||||
err = client.AddDocuments(context.Background(), docs2)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Should have exactly 1 document
|
||||
var count int
|
||||
err = db.QueryRow("SELECT COUNT(*) FROM vectors WHERE doc_id = ?", "doc-1").Scan(&count)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 1, count)
|
||||
}
|
||||
|
||||
func TestClient_DeleteDocuments_NonExistent(t *testing.T) {
|
||||
db, dbCleanup := testDB(t)
|
||||
defer dbCleanup()
|
||||
|
||||
embedSvc, embedCleanup := testEmbeddingService(t)
|
||||
defer embedCleanup()
|
||||
|
||||
client, err := NewClient(Config{DB: db}, embedSvc)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Deleting non-existent document should not error
|
||||
err = client.DeleteDocuments(context.Background(), []string{"non-existent-id"})
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
func TestClient_Count_Empty(t *testing.T) {
|
||||
db, dbCleanup := testDB(t)
|
||||
defer dbCleanup()
|
||||
|
||||
embedSvc, embedCleanup := testEmbeddingService(t)
|
||||
defer embedCleanup()
|
||||
|
||||
client, err := NewClient(Config{DB: db}, embedSvc)
|
||||
require.NoError(t, err)
|
||||
|
||||
count, err := client.Count(context.Background())
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, int64(0), count)
|
||||
}
|
||||
|
||||
func TestClient_Count_WithVectors(t *testing.T) {
|
||||
db, dbCleanup := testDB(t)
|
||||
defer dbCleanup()
|
||||
|
||||
embedSvc, embedCleanup := testEmbeddingService(t)
|
||||
defer embedCleanup()
|
||||
|
||||
client, err := NewClient(Config{DB: db}, embedSvc)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Add some documents
|
||||
docs := []Document{
|
||||
{ID: "doc-1", Content: "test content 1"},
|
||||
{ID: "doc-2", Content: "test content 2"},
|
||||
{ID: "doc-3", Content: "test content 3"},
|
||||
}
|
||||
err = client.AddDocuments(context.Background(), docs)
|
||||
require.NoError(t, err)
|
||||
|
||||
count, err := client.Count(context.Background())
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, int64(3), count)
|
||||
}
|
||||
|
||||
func TestClient_ModelVersion(t *testing.T) {
|
||||
db, dbCleanup := testDB(t)
|
||||
defer dbCleanup()
|
||||
|
||||
embedSvc, embedCleanup := testEmbeddingService(t)
|
||||
defer embedCleanup()
|
||||
|
||||
client, err := NewClient(Config{DB: db}, embedSvc)
|
||||
require.NoError(t, err)
|
||||
|
||||
version := client.ModelVersion()
|
||||
assert.NotEmpty(t, version)
|
||||
// Should match the embedding service version
|
||||
assert.Equal(t, embedSvc.Version(), version)
|
||||
}
|
||||
|
||||
func TestClient_NeedsRebuild_EmptyDatabase(t *testing.T) {
|
||||
db, dbCleanup := testDB(t)
|
||||
defer dbCleanup()
|
||||
|
||||
embedSvc, embedCleanup := testEmbeddingService(t)
|
||||
defer embedCleanup()
|
||||
|
||||
client, err := NewClient(Config{DB: db}, embedSvc)
|
||||
require.NoError(t, err)
|
||||
|
||||
needsRebuild, reason := client.NeedsRebuild(context.Background())
|
||||
assert.True(t, needsRebuild)
|
||||
assert.Equal(t, "empty", reason)
|
||||
}
|
||||
|
||||
func TestClient_NeedsRebuild_ModelMismatch(t *testing.T) {
|
||||
db, dbCleanup := testDB(t)
|
||||
defer dbCleanup()
|
||||
|
||||
embedSvc, embedCleanup := testEmbeddingService(t)
|
||||
defer embedCleanup()
|
||||
|
||||
client, err := NewClient(Config{DB: db}, embedSvc)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Insert vectors with wrong model version
|
||||
embedding := make([]float32, 384)
|
||||
for i := range embedding {
|
||||
embedding[i] = 0.1
|
||||
}
|
||||
embeddingBytes, err := sqlite_vec.SerializeFloat32(embedding)
|
||||
require.NoError(t, err)
|
||||
|
||||
_, err = db.Exec(`
|
||||
INSERT INTO vectors (doc_id, embedding, model_version, sqlite_id, doc_type, field_type, project, scope)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
||||
`, "doc-1", embeddingBytes, "old-model-v1", 1, "observation", "content", "test", "project")
|
||||
require.NoError(t, err)
|
||||
|
||||
_, err = db.Exec(`
|
||||
INSERT INTO vectors (doc_id, embedding, model_version, sqlite_id, doc_type, field_type, project, scope)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
||||
`, "doc-2", embeddingBytes, "old-model-v1", 2, "observation", "content", "test", "project")
|
||||
require.NoError(t, err)
|
||||
|
||||
needsRebuild, reason := client.NeedsRebuild(context.Background())
|
||||
assert.True(t, needsRebuild)
|
||||
assert.Contains(t, reason, "model_mismatch:2")
|
||||
}
|
||||
|
||||
func TestClient_NeedsRebuild_CurrentModel(t *testing.T) {
|
||||
db, dbCleanup := testDB(t)
|
||||
defer dbCleanup()
|
||||
|
||||
embedSvc, embedCleanup := testEmbeddingService(t)
|
||||
defer embedCleanup()
|
||||
|
||||
client, err := NewClient(Config{DB: db}, embedSvc)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Add documents with current model version
|
||||
docs := []Document{
|
||||
{ID: "doc-1", Content: "test content 1"},
|
||||
{ID: "doc-2", Content: "test content 2"},
|
||||
}
|
||||
err = client.AddDocuments(context.Background(), docs)
|
||||
require.NoError(t, err)
|
||||
|
||||
needsRebuild, reason := client.NeedsRebuild(context.Background())
|
||||
assert.False(t, needsRebuild)
|
||||
assert.Empty(t, reason)
|
||||
}
|
||||
|
||||
func TestClient_GetStaleVectors_Empty(t *testing.T) {
|
||||
db, dbCleanup := testDB(t)
|
||||
defer dbCleanup()
|
||||
|
||||
embedSvc, embedCleanup := testEmbeddingService(t)
|
||||
defer embedCleanup()
|
||||
|
||||
client, err := NewClient(Config{DB: db}, embedSvc)
|
||||
require.NoError(t, err)
|
||||
|
||||
stale, err := client.GetStaleVectors(context.Background())
|
||||
require.NoError(t, err)
|
||||
assert.Empty(t, stale)
|
||||
}
|
||||
|
||||
func TestClient_GetStaleVectors_WithMismatch(t *testing.T) {
|
||||
db, dbCleanup := testDB(t)
|
||||
defer dbCleanup()
|
||||
|
||||
embedSvc, embedCleanup := testEmbeddingService(t)
|
||||
defer embedCleanup()
|
||||
|
||||
client, err := NewClient(Config{DB: db}, embedSvc)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Insert vectors with wrong model version
|
||||
embedding := make([]float32, 384)
|
||||
embeddingBytes, err := sqlite_vec.SerializeFloat32(embedding)
|
||||
require.NoError(t, err)
|
||||
|
||||
_, err = db.Exec(`
|
||||
INSERT INTO vectors (doc_id, embedding, model_version, sqlite_id, doc_type, field_type, project, scope)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
||||
`, "doc-1", embeddingBytes, "old-model", 1, "observation", "content", "project-1", "project")
|
||||
require.NoError(t, err)
|
||||
|
||||
_, err = db.Exec(`
|
||||
INSERT INTO vectors (doc_id, embedding, model_version, sqlite_id, doc_type, field_type, project, scope)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
||||
`, "doc-2", embeddingBytes, embedSvc.Version(), 2, "observation", "title", "project-1", "project")
|
||||
require.NoError(t, err)
|
||||
|
||||
stale, err := client.GetStaleVectors(context.Background())
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, stale, 1)
|
||||
assert.Equal(t, "doc-1", stale[0].DocID)
|
||||
assert.Equal(t, int64(1), stale[0].SQLiteID)
|
||||
assert.Equal(t, "observation", stale[0].DocType)
|
||||
assert.Equal(t, "content", stale[0].FieldType)
|
||||
assert.Equal(t, "project-1", stale[0].Project)
|
||||
assert.Equal(t, "project", stale[0].Scope)
|
||||
}
|
||||
|
||||
func TestClient_DeleteVectorsByDocIDs_Empty(t *testing.T) {
|
||||
db, dbCleanup := testDB(t)
|
||||
defer dbCleanup()
|
||||
|
||||
embedSvc, embedCleanup := testEmbeddingService(t)
|
||||
defer embedCleanup()
|
||||
|
||||
client, err := NewClient(Config{DB: db}, embedSvc)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Deleting empty slice should not error
|
||||
err = client.DeleteVectorsByDocIDs(context.Background(), []string{})
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
func TestClient_DeleteVectorsByDocIDs_Success(t *testing.T) {
|
||||
db, dbCleanup := testDB(t)
|
||||
defer dbCleanup()
|
||||
|
||||
embedSvc, embedCleanup := testEmbeddingService(t)
|
||||
defer embedCleanup()
|
||||
|
||||
client, err := NewClient(Config{DB: db}, embedSvc)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Add documents
|
||||
docs := []Document{
|
||||
{ID: "doc-1", Content: "test 1"},
|
||||
{ID: "doc-2", Content: "test 2"},
|
||||
{ID: "doc-3", Content: "test 3"},
|
||||
}
|
||||
err = client.AddDocuments(context.Background(), docs)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Verify 3 documents exist
|
||||
count, err := client.Count(context.Background())
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, int64(3), count)
|
||||
|
||||
// Delete doc-1 and doc-3
|
||||
err = client.DeleteVectorsByDocIDs(context.Background(), []string{"doc-1", "doc-3"})
|
||||
require.NoError(t, err)
|
||||
|
||||
// Should have 1 document remaining
|
||||
count, err = client.Count(context.Background())
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, int64(1), count)
|
||||
|
||||
// Verify doc-2 still exists
|
||||
var exists int
|
||||
err = db.QueryRow("SELECT COUNT(*) FROM vectors WHERE doc_id = ?", "doc-2").Scan(&exists)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 1, exists)
|
||||
}
|
||||
|
||||
func TestClient_DeleteVectorsByDocIDs_NonExistent(t *testing.T) {
|
||||
db, dbCleanup := testDB(t)
|
||||
defer dbCleanup()
|
||||
|
||||
embedSvc, embedCleanup := testEmbeddingService(t)
|
||||
defer embedCleanup()
|
||||
|
||||
client, err := NewClient(Config{DB: db}, embedSvc)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Deleting non-existent IDs should not error
|
||||
err = client.DeleteVectorsByDocIDs(context.Background(), []string{"non-existent-1", "non-existent-2"})
|
||||
require.NoError(t, err)
|
||||
}
|
||||
@@ -19,9 +19,32 @@ type Document struct {
|
||||
|
||||
// QueryResult represents a search result from vector search.
|
||||
type QueryResult struct {
|
||||
ID string
|
||||
Distance float64
|
||||
Metadata map[string]any
|
||||
ID string
|
||||
Distance float64
|
||||
Similarity float64 // 1.0 = identical, 0.0 = opposite (derived from distance)
|
||||
Metadata map[string]any
|
||||
}
|
||||
|
||||
// DistanceToSimilarity converts sqlite-vec cosine distance to similarity score.
|
||||
// Cosine distance: 0 = identical, 2 = opposite
|
||||
// Similarity: 1.0 = identical, 0.0 = opposite
|
||||
func DistanceToSimilarity(distance float64) float64 {
|
||||
return 1.0 - (distance / 2.0)
|
||||
}
|
||||
|
||||
// FilterByThreshold filters results to only include those above the similarity threshold.
|
||||
// If maxResults > 0, also caps the number of results.
|
||||
func FilterByThreshold(results []QueryResult, threshold float64, maxResults int) []QueryResult {
|
||||
var filtered []QueryResult
|
||||
for _, r := range results {
|
||||
if r.Similarity >= threshold {
|
||||
filtered = append(filtered, r)
|
||||
if maxResults > 0 && len(filtered) >= maxResults {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
return filtered
|
||||
}
|
||||
|
||||
// ExtractedIDs contains SQLite IDs extracted from query results, grouped by document type.
|
||||
|
||||
@@ -0,0 +1,690 @@
|
||||
package sqlitevec
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestDocTypes(t *testing.T) {
|
||||
assert.Equal(t, DocType("observation"), DocTypeObservation)
|
||||
assert.Equal(t, DocType("session_summary"), DocTypeSessionSummary)
|
||||
assert.Equal(t, DocType("user_prompt"), DocTypeUserPrompt)
|
||||
}
|
||||
|
||||
func TestDocument_Fields(t *testing.T) {
|
||||
doc := Document{
|
||||
ID: "doc-123",
|
||||
Content: "test content",
|
||||
Metadata: map[string]any{
|
||||
"key": "value",
|
||||
},
|
||||
}
|
||||
|
||||
assert.Equal(t, "doc-123", doc.ID)
|
||||
assert.Equal(t, "test content", doc.Content)
|
||||
assert.Equal(t, "value", doc.Metadata["key"])
|
||||
}
|
||||
|
||||
func TestQueryResult_Fields(t *testing.T) {
|
||||
result := QueryResult{
|
||||
ID: "result-123",
|
||||
Distance: 0.5,
|
||||
Metadata: map[string]any{
|
||||
"sqlite_id": float64(42),
|
||||
},
|
||||
}
|
||||
|
||||
assert.Equal(t, "result-123", result.ID)
|
||||
assert.Equal(t, 0.5, result.Distance)
|
||||
assert.Equal(t, float64(42), result.Metadata["sqlite_id"])
|
||||
}
|
||||
|
||||
func TestBuildWhereFilter(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
docType DocType
|
||||
project string
|
||||
expected map[string]interface{}
|
||||
}{
|
||||
{
|
||||
name: "empty_filters",
|
||||
docType: "",
|
||||
project: "",
|
||||
expected: map[string]interface{}{},
|
||||
},
|
||||
{
|
||||
name: "doc_type_only",
|
||||
docType: DocTypeObservation,
|
||||
project: "",
|
||||
expected: map[string]interface{}{
|
||||
"doc_type": "observation",
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "project_only",
|
||||
docType: "",
|
||||
project: "my-project",
|
||||
expected: map[string]interface{}{
|
||||
"project": "my-project",
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "both_filters",
|
||||
docType: DocTypeSessionSummary,
|
||||
project: "test-project",
|
||||
expected: map[string]interface{}{
|
||||
"doc_type": "session_summary",
|
||||
"project": "test-project",
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "user_prompt_type",
|
||||
docType: DocTypeUserPrompt,
|
||||
project: "prompt-project",
|
||||
expected: map[string]interface{}{
|
||||
"doc_type": "user_prompt",
|
||||
"project": "prompt-project",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
result := BuildWhereFilter(tt.docType, tt.project)
|
||||
assert.Equal(t, tt.expected, result)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractIDsByDocType_Empty(t *testing.T) {
|
||||
results := []QueryResult{}
|
||||
ids := ExtractIDsByDocType(results)
|
||||
|
||||
assert.Empty(t, ids.ObservationIDs)
|
||||
assert.Empty(t, ids.SummaryIDs)
|
||||
assert.Empty(t, ids.PromptIDs)
|
||||
}
|
||||
|
||||
func TestExtractIDsByDocType_AllTypes(t *testing.T) {
|
||||
results := []QueryResult{
|
||||
{
|
||||
ID: "obs-1",
|
||||
Distance: 0.1,
|
||||
Metadata: map[string]any{
|
||||
"sqlite_id": float64(1),
|
||||
"doc_type": "observation",
|
||||
},
|
||||
},
|
||||
{
|
||||
ID: "obs-2",
|
||||
Distance: 0.2,
|
||||
Metadata: map[string]any{
|
||||
"sqlite_id": float64(2),
|
||||
"doc_type": "observation",
|
||||
},
|
||||
},
|
||||
{
|
||||
ID: "summary-1",
|
||||
Distance: 0.3,
|
||||
Metadata: map[string]any{
|
||||
"sqlite_id": float64(10),
|
||||
"doc_type": "session_summary",
|
||||
},
|
||||
},
|
||||
{
|
||||
ID: "prompt-1",
|
||||
Distance: 0.4,
|
||||
Metadata: map[string]any{
|
||||
"sqlite_id": float64(20),
|
||||
"doc_type": "user_prompt",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
ids := ExtractIDsByDocType(results)
|
||||
|
||||
assert.Equal(t, []int64{1, 2}, ids.ObservationIDs)
|
||||
assert.Equal(t, []int64{10}, ids.SummaryIDs)
|
||||
assert.Equal(t, []int64{20}, ids.PromptIDs)
|
||||
}
|
||||
|
||||
func TestExtractIDsByDocType_Deduplication(t *testing.T) {
|
||||
results := []QueryResult{
|
||||
{
|
||||
ID: "obs-1-field1",
|
||||
Distance: 0.1,
|
||||
Metadata: map[string]any{
|
||||
"sqlite_id": float64(1),
|
||||
"doc_type": "observation",
|
||||
},
|
||||
},
|
||||
{
|
||||
ID: "obs-1-field2",
|
||||
Distance: 0.2,
|
||||
Metadata: map[string]any{
|
||||
"sqlite_id": float64(1), // Same ID, different field
|
||||
"doc_type": "observation",
|
||||
},
|
||||
},
|
||||
{
|
||||
ID: "obs-2",
|
||||
Distance: 0.3,
|
||||
Metadata: map[string]any{
|
||||
"sqlite_id": float64(2),
|
||||
"doc_type": "observation",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
ids := ExtractIDsByDocType(results)
|
||||
|
||||
assert.Equal(t, []int64{1, 2}, ids.ObservationIDs) // Should be deduplicated
|
||||
}
|
||||
|
||||
func TestExtractIDsByDocType_Int64Fallback(t *testing.T) {
|
||||
results := []QueryResult{
|
||||
{
|
||||
ID: "obs-1",
|
||||
Distance: 0.1,
|
||||
Metadata: map[string]any{
|
||||
"sqlite_id": int64(42), // int64 instead of float64
|
||||
"doc_type": "observation",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
ids := ExtractIDsByDocType(results)
|
||||
|
||||
assert.Equal(t, []int64{42}, ids.ObservationIDs)
|
||||
}
|
||||
|
||||
func TestExtractIDsByDocType_MissingSqliteID(t *testing.T) {
|
||||
results := []QueryResult{
|
||||
{
|
||||
ID: "obs-1",
|
||||
Distance: 0.1,
|
||||
Metadata: map[string]any{
|
||||
"doc_type": "observation",
|
||||
// Missing sqlite_id
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
ids := ExtractIDsByDocType(results)
|
||||
|
||||
assert.Empty(t, ids.ObservationIDs)
|
||||
}
|
||||
|
||||
func TestExtractIDsByDocType_UnknownType(t *testing.T) {
|
||||
results := []QueryResult{
|
||||
{
|
||||
ID: "unknown-1",
|
||||
Distance: 0.1,
|
||||
Metadata: map[string]any{
|
||||
"sqlite_id": float64(1),
|
||||
"doc_type": "unknown_type",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
ids := ExtractIDsByDocType(results)
|
||||
|
||||
// Should not be added to any category
|
||||
assert.Empty(t, ids.ObservationIDs)
|
||||
assert.Empty(t, ids.SummaryIDs)
|
||||
assert.Empty(t, ids.PromptIDs)
|
||||
}
|
||||
|
||||
func TestExtractObservationIDs_NoFilter(t *testing.T) {
|
||||
results := []QueryResult{
|
||||
{
|
||||
ID: "obs-1",
|
||||
Metadata: map[string]any{
|
||||
"sqlite_id": float64(1),
|
||||
"doc_type": "observation",
|
||||
"project": "proj-a",
|
||||
},
|
||||
},
|
||||
{
|
||||
ID: "obs-2",
|
||||
Metadata: map[string]any{
|
||||
"sqlite_id": float64(2),
|
||||
"doc_type": "observation",
|
||||
"project": "proj-b",
|
||||
},
|
||||
},
|
||||
{
|
||||
ID: "summary-1",
|
||||
Metadata: map[string]any{
|
||||
"sqlite_id": float64(10),
|
||||
"doc_type": "session_summary",
|
||||
"project": "proj-a",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
ids := ExtractObservationIDs(results, "")
|
||||
|
||||
assert.Equal(t, []int64{1, 2}, ids)
|
||||
}
|
||||
|
||||
func TestExtractObservationIDs_WithProjectFilter(t *testing.T) {
|
||||
results := []QueryResult{
|
||||
{
|
||||
ID: "obs-1",
|
||||
Metadata: map[string]any{
|
||||
"sqlite_id": float64(1),
|
||||
"doc_type": "observation",
|
||||
"project": "proj-a",
|
||||
"scope": "project",
|
||||
},
|
||||
},
|
||||
{
|
||||
ID: "obs-2",
|
||||
Metadata: map[string]any{
|
||||
"sqlite_id": float64(2),
|
||||
"doc_type": "observation",
|
||||
"project": "proj-b",
|
||||
"scope": "project",
|
||||
},
|
||||
},
|
||||
{
|
||||
ID: "obs-global",
|
||||
Metadata: map[string]any{
|
||||
"sqlite_id": float64(3),
|
||||
"doc_type": "observation",
|
||||
"project": "proj-b",
|
||||
"scope": "global",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
ids := ExtractObservationIDs(results, "proj-a")
|
||||
|
||||
// Should include proj-a and global scope observations
|
||||
assert.Equal(t, []int64{1, 3}, ids)
|
||||
}
|
||||
|
||||
func TestExtractObservationIDs_Deduplication(t *testing.T) {
|
||||
results := []QueryResult{
|
||||
{
|
||||
ID: "obs-1-field1",
|
||||
Metadata: map[string]any{
|
||||
"sqlite_id": float64(1),
|
||||
"doc_type": "observation",
|
||||
},
|
||||
},
|
||||
{
|
||||
ID: "obs-1-field2",
|
||||
Metadata: map[string]any{
|
||||
"sqlite_id": float64(1), // Same ID
|
||||
"doc_type": "observation",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
ids := ExtractObservationIDs(results, "")
|
||||
|
||||
assert.Equal(t, []int64{1}, ids)
|
||||
}
|
||||
|
||||
func TestExtractSummaryIDs_NoFilter(t *testing.T) {
|
||||
results := []QueryResult{
|
||||
{
|
||||
ID: "summary-1",
|
||||
Metadata: map[string]any{
|
||||
"sqlite_id": float64(10),
|
||||
"doc_type": "session_summary",
|
||||
"project": "proj-a",
|
||||
},
|
||||
},
|
||||
{
|
||||
ID: "summary-2",
|
||||
Metadata: map[string]any{
|
||||
"sqlite_id": float64(20),
|
||||
"doc_type": "session_summary",
|
||||
"project": "proj-b",
|
||||
},
|
||||
},
|
||||
{
|
||||
ID: "obs-1",
|
||||
Metadata: map[string]any{
|
||||
"sqlite_id": float64(1),
|
||||
"doc_type": "observation",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
ids := ExtractSummaryIDs(results, "")
|
||||
|
||||
assert.Equal(t, []int64{10, 20}, ids)
|
||||
}
|
||||
|
||||
func TestExtractSummaryIDs_WithProjectFilter(t *testing.T) {
|
||||
results := []QueryResult{
|
||||
{
|
||||
ID: "summary-1",
|
||||
Metadata: map[string]any{
|
||||
"sqlite_id": float64(10),
|
||||
"doc_type": "session_summary",
|
||||
"project": "proj-a",
|
||||
},
|
||||
},
|
||||
{
|
||||
ID: "summary-2",
|
||||
Metadata: map[string]any{
|
||||
"sqlite_id": float64(20),
|
||||
"doc_type": "session_summary",
|
||||
"project": "proj-b",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
ids := ExtractSummaryIDs(results, "proj-a")
|
||||
|
||||
assert.Equal(t, []int64{10}, ids)
|
||||
}
|
||||
|
||||
func TestExtractPromptIDs_NoFilter(t *testing.T) {
|
||||
results := []QueryResult{
|
||||
{
|
||||
ID: "prompt-1",
|
||||
Metadata: map[string]any{
|
||||
"sqlite_id": float64(100),
|
||||
"doc_type": "user_prompt",
|
||||
"project": "proj-a",
|
||||
},
|
||||
},
|
||||
{
|
||||
ID: "prompt-2",
|
||||
Metadata: map[string]any{
|
||||
"sqlite_id": float64(200),
|
||||
"doc_type": "user_prompt",
|
||||
"project": "proj-b",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
ids := ExtractPromptIDs(results, "")
|
||||
|
||||
assert.Equal(t, []int64{100, 200}, ids)
|
||||
}
|
||||
|
||||
func TestExtractPromptIDs_WithProjectFilter(t *testing.T) {
|
||||
results := []QueryResult{
|
||||
{
|
||||
ID: "prompt-1",
|
||||
Metadata: map[string]any{
|
||||
"sqlite_id": float64(100),
|
||||
"doc_type": "user_prompt",
|
||||
"project": "proj-a",
|
||||
},
|
||||
},
|
||||
{
|
||||
ID: "prompt-2",
|
||||
Metadata: map[string]any{
|
||||
"sqlite_id": float64(200),
|
||||
"doc_type": "user_prompt",
|
||||
"project": "proj-b",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
ids := ExtractPromptIDs(results, "proj-b")
|
||||
|
||||
assert.Equal(t, []int64{200}, ids)
|
||||
}
|
||||
|
||||
func TestCopyMetadata(t *testing.T) {
|
||||
base := map[string]any{
|
||||
"key1": "value1",
|
||||
"key2": 42,
|
||||
}
|
||||
|
||||
result := copyMetadata(base, "key3", "value3")
|
||||
|
||||
// Original should be unchanged
|
||||
assert.Len(t, base, 2)
|
||||
|
||||
// Result should have new key
|
||||
assert.Len(t, result, 3)
|
||||
assert.Equal(t, "value1", result["key1"])
|
||||
assert.Equal(t, 42, result["key2"])
|
||||
assert.Equal(t, "value3", result["key3"])
|
||||
}
|
||||
|
||||
func TestCopyMetadataMulti(t *testing.T) {
|
||||
base := map[string]any{
|
||||
"key1": "value1",
|
||||
}
|
||||
extra := map[string]any{
|
||||
"key2": "value2",
|
||||
"key3": "value3",
|
||||
}
|
||||
|
||||
result := copyMetadataMulti(base, extra)
|
||||
|
||||
assert.Len(t, result, 3)
|
||||
assert.Equal(t, "value1", result["key1"])
|
||||
assert.Equal(t, "value2", result["key2"])
|
||||
assert.Equal(t, "value3", result["key3"])
|
||||
}
|
||||
|
||||
func TestJoinStrings(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
strs []string
|
||||
sep string
|
||||
expected string
|
||||
}{
|
||||
{
|
||||
name: "empty_slice",
|
||||
strs: []string{},
|
||||
sep: ", ",
|
||||
expected: "",
|
||||
},
|
||||
{
|
||||
name: "single_element",
|
||||
strs: []string{"one"},
|
||||
sep: ", ",
|
||||
expected: "one",
|
||||
},
|
||||
{
|
||||
name: "multiple_elements",
|
||||
strs: []string{"one", "two", "three"},
|
||||
sep: ", ",
|
||||
expected: "one, two, three",
|
||||
},
|
||||
{
|
||||
name: "different_separator",
|
||||
strs: []string{"a", "b", "c"},
|
||||
sep: "-",
|
||||
expected: "a-b-c",
|
||||
},
|
||||
{
|
||||
name: "empty_separator",
|
||||
strs: []string{"a", "b", "c"},
|
||||
sep: "",
|
||||
expected: "abc",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
result := joinStrings(tt.strs, tt.sep)
|
||||
assert.Equal(t, tt.expected, result)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestTruncateString(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
input string
|
||||
maxLen int
|
||||
expected string
|
||||
}{
|
||||
{
|
||||
name: "shorter_than_max",
|
||||
input: "hello",
|
||||
maxLen: 10,
|
||||
expected: "hello",
|
||||
},
|
||||
{
|
||||
name: "equal_to_max",
|
||||
input: "hello",
|
||||
maxLen: 5,
|
||||
expected: "hello",
|
||||
},
|
||||
{
|
||||
name: "longer_than_max",
|
||||
input: "hello world",
|
||||
maxLen: 5,
|
||||
expected: "hello...",
|
||||
},
|
||||
{
|
||||
name: "empty_string",
|
||||
input: "",
|
||||
maxLen: 5,
|
||||
expected: "",
|
||||
},
|
||||
{
|
||||
name: "zero_max_length",
|
||||
input: "hello",
|
||||
maxLen: 0,
|
||||
expected: "...",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
result := truncateString(tt.input, tt.maxLen)
|
||||
assert.Equal(t, tt.expected, result)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractedIDs_Empty(t *testing.T) {
|
||||
ids := &ExtractedIDs{}
|
||||
|
||||
assert.Nil(t, ids.ObservationIDs)
|
||||
assert.Nil(t, ids.SummaryIDs)
|
||||
assert.Nil(t, ids.PromptIDs)
|
||||
}
|
||||
|
||||
func TestFilterByThreshold(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
results []QueryResult
|
||||
threshold float64
|
||||
maxResults int
|
||||
expectedLen int
|
||||
expectedIDs []string
|
||||
}{
|
||||
{
|
||||
name: "empty_results",
|
||||
results: []QueryResult{},
|
||||
threshold: 0.5,
|
||||
maxResults: 0,
|
||||
expectedLen: 0,
|
||||
},
|
||||
{
|
||||
name: "all_above_threshold",
|
||||
results: []QueryResult{
|
||||
{ID: "doc-1", Similarity: 0.9},
|
||||
{ID: "doc-2", Similarity: 0.8},
|
||||
{ID: "doc-3", Similarity: 0.7},
|
||||
},
|
||||
threshold: 0.5,
|
||||
maxResults: 0,
|
||||
expectedLen: 3,
|
||||
expectedIDs: []string{"doc-1", "doc-2", "doc-3"},
|
||||
},
|
||||
{
|
||||
name: "some_below_threshold",
|
||||
results: []QueryResult{
|
||||
{ID: "doc-1", Similarity: 0.9},
|
||||
{ID: "doc-2", Similarity: 0.4},
|
||||
{ID: "doc-3", Similarity: 0.7},
|
||||
{ID: "doc-4", Similarity: 0.3},
|
||||
},
|
||||
threshold: 0.5,
|
||||
maxResults: 0,
|
||||
expectedLen: 2,
|
||||
expectedIDs: []string{"doc-1", "doc-3"},
|
||||
},
|
||||
{
|
||||
name: "all_below_threshold",
|
||||
results: []QueryResult{
|
||||
{ID: "doc-1", Similarity: 0.3},
|
||||
{ID: "doc-2", Similarity: 0.2},
|
||||
},
|
||||
threshold: 0.5,
|
||||
maxResults: 0,
|
||||
expectedLen: 0,
|
||||
},
|
||||
{
|
||||
name: "max_results_limit",
|
||||
results: []QueryResult{
|
||||
{ID: "doc-1", Similarity: 0.9},
|
||||
{ID: "doc-2", Similarity: 0.8},
|
||||
{ID: "doc-3", Similarity: 0.7},
|
||||
{ID: "doc-4", Similarity: 0.6},
|
||||
},
|
||||
threshold: 0.5,
|
||||
maxResults: 2,
|
||||
expectedLen: 2,
|
||||
expectedIDs: []string{"doc-1", "doc-2"},
|
||||
},
|
||||
{
|
||||
name: "max_results_with_threshold",
|
||||
results: []QueryResult{
|
||||
{ID: "doc-1", Similarity: 0.9},
|
||||
{ID: "doc-2", Similarity: 0.3},
|
||||
{ID: "doc-3", Similarity: 0.8},
|
||||
{ID: "doc-4", Similarity: 0.2},
|
||||
{ID: "doc-5", Similarity: 0.7},
|
||||
},
|
||||
threshold: 0.5,
|
||||
maxResults: 2,
|
||||
expectedLen: 2,
|
||||
expectedIDs: []string{"doc-1", "doc-3"},
|
||||
},
|
||||
{
|
||||
name: "exact_threshold_included",
|
||||
results: []QueryResult{
|
||||
{ID: "doc-1", Similarity: 0.5},
|
||||
{ID: "doc-2", Similarity: 0.4},
|
||||
},
|
||||
threshold: 0.5,
|
||||
maxResults: 0,
|
||||
expectedLen: 1,
|
||||
expectedIDs: []string{"doc-1"},
|
||||
},
|
||||
{
|
||||
name: "zero_threshold",
|
||||
results: []QueryResult{
|
||||
{ID: "doc-1", Similarity: 0.1},
|
||||
{ID: "doc-2", Similarity: 0.0},
|
||||
},
|
||||
threshold: 0.0,
|
||||
maxResults: 0,
|
||||
expectedLen: 2,
|
||||
expectedIDs: []string{"doc-1", "doc-2"},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
filtered := FilterByThreshold(tt.results, tt.threshold, tt.maxResults)
|
||||
assert.Len(t, filtered, tt.expectedLen)
|
||||
|
||||
if tt.expectedLen > 0 {
|
||||
for i, id := range tt.expectedIDs {
|
||||
assert.Equal(t, id, filtered[i].ID)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -240,3 +240,101 @@ func (s *Sync) DeleteUserPrompts(ctx context.Context, promptIDs []int64) error {
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// SyncPattern syncs a single pattern to the vector store.
|
||||
func (s *Sync) SyncPattern(ctx context.Context, pattern *models.Pattern) error {
|
||||
docs := s.formatPatternDocs(pattern)
|
||||
if len(docs) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
if err := s.client.AddDocuments(ctx, docs); err != nil {
|
||||
return fmt.Errorf("add pattern docs: %w", err)
|
||||
}
|
||||
|
||||
log.Debug().
|
||||
Int64("patternId", pattern.ID).
|
||||
Int("docCount", len(docs)).
|
||||
Msg("Synced pattern to sqlite-vec")
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// formatPatternDocs formats a pattern into vector documents.
|
||||
func (s *Sync) formatPatternDocs(pattern *models.Pattern) []Document {
|
||||
docs := make([]Document, 0, 3)
|
||||
|
||||
baseMetadata := map[string]any{
|
||||
"sqlite_id": pattern.ID,
|
||||
"doc_type": "pattern",
|
||||
"pattern_type": string(pattern.Type),
|
||||
"status": string(pattern.Status),
|
||||
"scope": "global", // Patterns are always global
|
||||
"frequency": pattern.Frequency,
|
||||
"confidence": pattern.Confidence,
|
||||
"created_at_epoch": pattern.CreatedAtEpoch,
|
||||
}
|
||||
|
||||
if len(pattern.Signature) > 0 {
|
||||
baseMetadata["signature"] = joinStrings(pattern.Signature, ",")
|
||||
}
|
||||
if len(pattern.Projects) > 0 {
|
||||
baseMetadata["projects"] = joinStrings(pattern.Projects, ",")
|
||||
}
|
||||
|
||||
// Pattern name as document
|
||||
if pattern.Name != "" {
|
||||
docs = append(docs, Document{
|
||||
ID: fmt.Sprintf("pattern_%d_name", pattern.ID),
|
||||
Content: pattern.Name,
|
||||
Metadata: copyMetadata(baseMetadata, "field_type", "name"),
|
||||
})
|
||||
}
|
||||
|
||||
// Pattern description as document
|
||||
if pattern.Description.Valid && pattern.Description.String != "" {
|
||||
docs = append(docs, Document{
|
||||
ID: fmt.Sprintf("pattern_%d_description", pattern.ID),
|
||||
Content: pattern.Description.String,
|
||||
Metadata: copyMetadata(baseMetadata, "field_type", "description"),
|
||||
})
|
||||
}
|
||||
|
||||
// Pattern recommendation as document
|
||||
if pattern.Recommendation.Valid && pattern.Recommendation.String != "" {
|
||||
docs = append(docs, Document{
|
||||
ID: fmt.Sprintf("pattern_%d_recommendation", pattern.ID),
|
||||
Content: pattern.Recommendation.String,
|
||||
Metadata: copyMetadata(baseMetadata, "field_type", "recommendation"),
|
||||
})
|
||||
}
|
||||
|
||||
return docs
|
||||
}
|
||||
|
||||
// DeletePatterns removes pattern documents from the vector store.
|
||||
func (s *Sync) DeletePatterns(ctx context.Context, patternIDs []int64) error {
|
||||
if len(patternIDs) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Generate all possible document IDs for these patterns
|
||||
// Pattern: pattern_{id}_name, pattern_{id}_description, pattern_{id}_recommendation
|
||||
ids := make([]string, 0, len(patternIDs)*3)
|
||||
|
||||
for _, patternID := range patternIDs {
|
||||
ids = append(ids, fmt.Sprintf("pattern_%d_name", patternID))
|
||||
ids = append(ids, fmt.Sprintf("pattern_%d_description", patternID))
|
||||
ids = append(ids, fmt.Sprintf("pattern_%d_recommendation", patternID))
|
||||
}
|
||||
|
||||
if err := s.client.DeleteDocuments(ctx, ids); err != nil {
|
||||
return fmt.Errorf("delete pattern docs: %w", err)
|
||||
}
|
||||
|
||||
log.Debug().
|
||||
Int("patternCount", len(patternIDs)).
|
||||
Msg("Deleted patterns from sqlite-vec")
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -0,0 +1,348 @@
|
||||
package sqlitevec
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"testing"
|
||||
|
||||
"github.com/lukaszraczylo/claude-mnemonic/pkg/models"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// testClient creates a test client for sync tests.
|
||||
func testClient(t *testing.T) (*Client, func()) {
|
||||
t.Helper()
|
||||
|
||||
db, dbCleanup := testDB(t)
|
||||
embedSvc, embedCleanup := testEmbeddingService(t)
|
||||
|
||||
client, err := NewClient(Config{DB: db}, embedSvc)
|
||||
require.NoError(t, err)
|
||||
|
||||
cleanup := func() {
|
||||
embedCleanup()
|
||||
dbCleanup()
|
||||
}
|
||||
|
||||
return client, cleanup
|
||||
}
|
||||
|
||||
func TestNewSync(t *testing.T) {
|
||||
client, cleanup := testClient(t)
|
||||
defer cleanup()
|
||||
|
||||
sync := NewSync(client)
|
||||
assert.NotNil(t, sync)
|
||||
}
|
||||
|
||||
func TestSync_SyncObservation_Empty(t *testing.T) {
|
||||
client, cleanup := testClient(t)
|
||||
defer cleanup()
|
||||
|
||||
sync := NewSync(client)
|
||||
|
||||
// Observation with no content should be handled gracefully
|
||||
obs := &models.Observation{
|
||||
ID: 1,
|
||||
SDKSessionID: "test-session",
|
||||
Project: "test-project",
|
||||
Type: models.ObsTypeDiscovery,
|
||||
}
|
||||
|
||||
err := sync.SyncObservation(context.Background(), obs)
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
func TestSync_SyncObservation_WithContent(t *testing.T) {
|
||||
client, cleanup := testClient(t)
|
||||
defer cleanup()
|
||||
|
||||
sync := NewSync(client)
|
||||
|
||||
obs := &models.Observation{
|
||||
ID: 1,
|
||||
SDKSessionID: "test-session",
|
||||
Project: "test-project",
|
||||
Type: models.ObsTypeDiscovery,
|
||||
Scope: models.ScopeProject,
|
||||
Title: sql.NullString{String: "Authentication bug fix", Valid: true},
|
||||
Subtitle: sql.NullString{String: "Fixed JWT validation", Valid: true},
|
||||
Narrative: sql.NullString{String: "Fixed the JWT token validation to handle expired tokens correctly.", Valid: true},
|
||||
Facts: []string{"JWT tokens expire after 24 hours", "Refresh tokens are used for renewal"},
|
||||
Concepts: []string{"authentication", "security"},
|
||||
FilesRead: []string{"auth.go"},
|
||||
FilesModified: []string{"handler.go"},
|
||||
}
|
||||
|
||||
err := sync.SyncObservation(context.Background(), obs)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Verify documents were added
|
||||
results, err := client.Query(context.Background(), "authentication", 10, nil)
|
||||
require.NoError(t, err)
|
||||
assert.GreaterOrEqual(t, len(results), 1)
|
||||
}
|
||||
|
||||
func TestSync_SyncObservation_DefaultScope(t *testing.T) {
|
||||
client, cleanup := testClient(t)
|
||||
defer cleanup()
|
||||
|
||||
sync := NewSync(client)
|
||||
|
||||
// Observation without explicit scope
|
||||
obs := &models.Observation{
|
||||
ID: 2,
|
||||
SDKSessionID: "test-session",
|
||||
Project: "test-project",
|
||||
Type: models.ObsTypeBugfix,
|
||||
Narrative: sql.NullString{String: "Fixed a null pointer exception.", Valid: true},
|
||||
}
|
||||
|
||||
err := sync.SyncObservation(context.Background(), obs)
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
func TestSync_SyncSummary_Empty(t *testing.T) {
|
||||
client, cleanup := testClient(t)
|
||||
defer cleanup()
|
||||
|
||||
sync := NewSync(client)
|
||||
|
||||
// Summary with no content
|
||||
summary := &models.SessionSummary{
|
||||
ID: 1,
|
||||
SDKSessionID: "test-session",
|
||||
Project: "test-project",
|
||||
}
|
||||
|
||||
err := sync.SyncSummary(context.Background(), summary)
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
func TestSync_SyncSummary_WithContent(t *testing.T) {
|
||||
client, cleanup := testClient(t)
|
||||
defer cleanup()
|
||||
|
||||
sync := NewSync(client)
|
||||
|
||||
summary := &models.SessionSummary{
|
||||
ID: 1,
|
||||
SDKSessionID: "test-session",
|
||||
Project: "test-project",
|
||||
Request: sql.NullString{String: "Help me fix the authentication bug", Valid: true},
|
||||
Investigated: sql.NullString{String: "Looked at auth.go and handler.go", Valid: true},
|
||||
Learned: sql.NullString{String: "JWT tokens were not being validated properly", Valid: true},
|
||||
Completed: sql.NullString{String: "Fixed the JWT validation logic", Valid: true},
|
||||
NextSteps: sql.NullString{String: "Add tests for edge cases", Valid: true},
|
||||
Notes: sql.NullString{String: "Consider using a library for JWT handling", Valid: true},
|
||||
PromptNumber: sql.NullInt64{Int64: 1, Valid: true},
|
||||
}
|
||||
|
||||
err := sync.SyncSummary(context.Background(), summary)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Verify documents were added
|
||||
results, err := client.Query(context.Background(), "authentication", 10, nil)
|
||||
require.NoError(t, err)
|
||||
assert.GreaterOrEqual(t, len(results), 1)
|
||||
}
|
||||
|
||||
func TestSync_SyncUserPrompt(t *testing.T) {
|
||||
client, cleanup := testClient(t)
|
||||
defer cleanup()
|
||||
|
||||
sync := NewSync(client)
|
||||
|
||||
prompt := &models.UserPromptWithSession{
|
||||
UserPrompt: models.UserPrompt{
|
||||
ID: 1,
|
||||
PromptNumber: 1,
|
||||
PromptText: "Help me fix the authentication bug in the login handler",
|
||||
CreatedAtEpoch: 1234567890,
|
||||
},
|
||||
SDKSessionID: "test-session",
|
||||
Project: "test-project",
|
||||
}
|
||||
|
||||
err := sync.SyncUserPrompt(context.Background(), prompt)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Verify document was added
|
||||
results, err := client.Query(context.Background(), "authentication", 10, nil)
|
||||
require.NoError(t, err)
|
||||
assert.GreaterOrEqual(t, len(results), 1)
|
||||
}
|
||||
|
||||
func TestSync_DeleteObservations_Empty(t *testing.T) {
|
||||
client, cleanup := testClient(t)
|
||||
defer cleanup()
|
||||
|
||||
sync := NewSync(client)
|
||||
|
||||
// Should handle empty list
|
||||
err := sync.DeleteObservations(context.Background(), []int64{})
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
func TestSync_DeleteObservations_WithData(t *testing.T) {
|
||||
client, cleanup := testClient(t)
|
||||
defer cleanup()
|
||||
|
||||
sync := NewSync(client)
|
||||
|
||||
// First add an observation
|
||||
obs := &models.Observation{
|
||||
ID: 10,
|
||||
SDKSessionID: "test-session",
|
||||
Project: "test-project",
|
||||
Type: models.ObsTypeDiscovery,
|
||||
Narrative: sql.NullString{String: "This observation should be deleted.", Valid: true},
|
||||
Facts: []string{"Fact 1", "Fact 2"},
|
||||
}
|
||||
|
||||
err := sync.SyncObservation(context.Background(), obs)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Then delete it
|
||||
err = sync.DeleteObservations(context.Background(), []int64{10})
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
func TestSync_DeleteUserPrompts_Empty(t *testing.T) {
|
||||
client, cleanup := testClient(t)
|
||||
defer cleanup()
|
||||
|
||||
sync := NewSync(client)
|
||||
|
||||
// Should handle empty list
|
||||
err := sync.DeleteUserPrompts(context.Background(), []int64{})
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
func TestSync_DeleteUserPrompts_WithData(t *testing.T) {
|
||||
client, cleanup := testClient(t)
|
||||
defer cleanup()
|
||||
|
||||
sync := NewSync(client)
|
||||
|
||||
// First add a prompt
|
||||
prompt := &models.UserPromptWithSession{
|
||||
UserPrompt: models.UserPrompt{
|
||||
ID: 20,
|
||||
PromptNumber: 1,
|
||||
PromptText: "This prompt should be deleted.",
|
||||
CreatedAtEpoch: 1234567890,
|
||||
},
|
||||
SDKSessionID: "test-session",
|
||||
Project: "test-project",
|
||||
}
|
||||
|
||||
err := sync.SyncUserPrompt(context.Background(), prompt)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Then delete it
|
||||
err = sync.DeleteUserPrompts(context.Background(), []int64{20})
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
func TestSync_FormatObservationDocs_AllFields(t *testing.T) {
|
||||
client, cleanup := testClient(t)
|
||||
defer cleanup()
|
||||
|
||||
sync := NewSync(client)
|
||||
|
||||
obs := &models.Observation{
|
||||
ID: 100,
|
||||
SDKSessionID: "sdk-123",
|
||||
Project: "my-project",
|
||||
Type: models.ObsTypeFeature,
|
||||
Scope: models.ScopeGlobal,
|
||||
Title: sql.NullString{String: "Feature Title", Valid: true},
|
||||
Subtitle: sql.NullString{String: "Feature Subtitle", Valid: true},
|
||||
Narrative: sql.NullString{String: "Feature narrative content", Valid: true},
|
||||
Facts: []string{"Fact A", "Fact B", "Fact C"},
|
||||
Concepts: []string{"api", "performance"},
|
||||
FilesRead: []string{"file1.go", "file2.go"},
|
||||
FilesModified: []string{"file3.go"},
|
||||
}
|
||||
|
||||
docs := sync.formatObservationDocs(obs)
|
||||
|
||||
// Should have 1 narrative + 3 facts = 4 docs
|
||||
assert.Len(t, docs, 4)
|
||||
|
||||
// Check narrative doc
|
||||
var narrativeDoc *Document
|
||||
for i := range docs {
|
||||
if docs[i].ID == "obs_100_narrative" {
|
||||
narrativeDoc = &docs[i]
|
||||
break
|
||||
}
|
||||
}
|
||||
require.NotNil(t, narrativeDoc)
|
||||
assert.Equal(t, "Feature narrative content", narrativeDoc.Content)
|
||||
assert.Equal(t, int64(100), narrativeDoc.Metadata["sqlite_id"])
|
||||
assert.Equal(t, "observation", narrativeDoc.Metadata["doc_type"])
|
||||
assert.Equal(t, "global", narrativeDoc.Metadata["scope"])
|
||||
assert.Equal(t, "narrative", narrativeDoc.Metadata["field_type"])
|
||||
}
|
||||
|
||||
func TestSync_FormatSummaryDocs_AllFields(t *testing.T) {
|
||||
client, cleanup := testClient(t)
|
||||
defer cleanup()
|
||||
|
||||
sync := NewSync(client)
|
||||
|
||||
summary := &models.SessionSummary{
|
||||
ID: 200,
|
||||
SDKSessionID: "sdk-456",
|
||||
Project: "summary-project",
|
||||
Request: sql.NullString{String: "Request content", Valid: true},
|
||||
Investigated: sql.NullString{String: "Investigated content", Valid: true},
|
||||
Learned: sql.NullString{String: "Learned content", Valid: true},
|
||||
Completed: sql.NullString{String: "Completed content", Valid: true},
|
||||
NextSteps: sql.NullString{String: "Next steps content", Valid: true},
|
||||
Notes: sql.NullString{String: "Notes content", Valid: true},
|
||||
PromptNumber: sql.NullInt64{Int64: 5, Valid: true},
|
||||
}
|
||||
|
||||
docs := sync.formatSummaryDocs(summary)
|
||||
|
||||
// Should have 6 docs (one for each field)
|
||||
assert.Len(t, docs, 6)
|
||||
|
||||
// Check request doc
|
||||
var requestDoc *Document
|
||||
for i := range docs {
|
||||
if docs[i].ID == "summary_200_request" {
|
||||
requestDoc = &docs[i]
|
||||
break
|
||||
}
|
||||
}
|
||||
require.NotNil(t, requestDoc)
|
||||
assert.Equal(t, "Request content", requestDoc.Content)
|
||||
assert.Equal(t, int64(200), requestDoc.Metadata["sqlite_id"])
|
||||
assert.Equal(t, "session_summary", requestDoc.Metadata["doc_type"])
|
||||
assert.Equal(t, int64(5), requestDoc.Metadata["prompt_number"])
|
||||
}
|
||||
|
||||
func TestSync_FormatObservationDocs_EmptyScope(t *testing.T) {
|
||||
client, cleanup := testClient(t)
|
||||
defer cleanup()
|
||||
|
||||
sync := NewSync(client)
|
||||
|
||||
obs := &models.Observation{
|
||||
ID: 300,
|
||||
SDKSessionID: "sdk-789",
|
||||
Project: "scope-test",
|
||||
Type: models.ObsTypeDecision,
|
||||
Narrative: sql.NullString{String: "Test narrative", Valid: true},
|
||||
// Scope intentionally left empty
|
||||
}
|
||||
|
||||
docs := sync.formatObservationDocs(obs)
|
||||
assert.Len(t, docs, 1)
|
||||
assert.Equal(t, "project", docs[0].Metadata["scope"])
|
||||
}
|
||||
+309
-18
@@ -4,13 +4,18 @@ package worker
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"sort"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
"github.com/lukaszraczylo/claude-mnemonic/internal/db/sqlite"
|
||||
"github.com/lukaszraczylo/claude-mnemonic/internal/db/gorm"
|
||||
"github.com/lukaszraczylo/claude-mnemonic/internal/embedding"
|
||||
"github.com/lukaszraczylo/claude-mnemonic/internal/privacy"
|
||||
"github.com/lukaszraczylo/claude-mnemonic/internal/reranking"
|
||||
"github.com/lukaszraczylo/claude-mnemonic/internal/search/expansion"
|
||||
"github.com/lukaszraczylo/claude-mnemonic/internal/vector/sqlitevec"
|
||||
"github.com/lukaszraczylo/claude-mnemonic/internal/worker/sdk"
|
||||
"github.com/lukaszraczylo/claude-mnemonic/internal/worker/session"
|
||||
@@ -159,7 +164,13 @@ type SessionInitResponse struct {
|
||||
Reason string `json:"reason,omitempty"`
|
||||
}
|
||||
|
||||
// DuplicatePromptWindowSeconds is the time window for detecting duplicate prompt submissions.
|
||||
// If the same prompt text is seen within this window, it's considered a duplicate hook invocation.
|
||||
const DuplicatePromptWindowSeconds = 10
|
||||
|
||||
// handleSessionInit handles session initialization from user-prompt hook.
|
||||
// This handler is idempotent - duplicate requests within a short time window
|
||||
// return the existing prompt data without creating duplicates.
|
||||
func (s *Service) handleSessionInit(w http.ResponseWriter, r *http.Request) {
|
||||
var req SessionInitRequest
|
||||
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
||||
@@ -182,8 +193,31 @@ func (s *Service) handleSessionInit(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
|
||||
// Clean prompt and create session
|
||||
// Clean prompt
|
||||
cleanedPrompt := privacy.Clean(req.Prompt)
|
||||
|
||||
// DUPLICATE DETECTION: Check if this exact prompt was already saved recently.
|
||||
// This prevents the bug where the hook fires multiple times for the same user action,
|
||||
// creating many duplicate prompts with incrementing numbers.
|
||||
if existingID, existingNum, found := s.promptStore.FindRecentPromptByText(r.Context(), req.ClaudeSessionID, cleanedPrompt, DuplicatePromptWindowSeconds); found {
|
||||
// Get or create session (idempotent)
|
||||
sessionID, _ := s.sessionStore.CreateSDKSession(r.Context(), req.ClaudeSessionID, req.Project, cleanedPrompt)
|
||||
|
||||
log.Debug().
|
||||
Int64("sessionId", sessionID).
|
||||
Int("promptNumber", existingNum).
|
||||
Int64("promptId", existingID).
|
||||
Msg("Duplicate prompt detected - returning existing")
|
||||
|
||||
// Return existing prompt data without incrementing or saving again
|
||||
writeJSON(w, SessionInitResponse{
|
||||
SessionDBID: sessionID,
|
||||
PromptNumber: existingNum,
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
// Create session (idempotent)
|
||||
sessionID, err := s.sessionStore.CreateSDKSession(r.Context(), req.ClaudeSessionID, req.Project, cleanedPrompt)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
@@ -452,7 +486,7 @@ func (s *Service) handleSummarize(w http.ResponseWriter, r *http.Request) {
|
||||
// handleGetObservations returns recent observations.
|
||||
// Supports optional query parameter for semantic search via sqlite-vec.
|
||||
func (s *Service) handleGetObservations(w http.ResponseWriter, r *http.Request) {
|
||||
limit := sqlite.ParseLimitParam(r, DefaultObservationsLimit)
|
||||
limit := gorm.ParseLimitParam(r, DefaultObservationsLimit)
|
||||
project := r.URL.Query().Get("project")
|
||||
query := r.URL.Query().Get("query")
|
||||
|
||||
@@ -478,8 +512,8 @@ func (s *Service) handleGetObservations(w http.ResponseWriter, r *http.Request)
|
||||
// Fall back to SQLite if vector search not used
|
||||
if !usedVector {
|
||||
if project != "" {
|
||||
// Filter by project - includes project-scoped and global observations
|
||||
observations, err = s.observationStore.GetRecentObservations(r.Context(), project, limit)
|
||||
// Strict project filtering for dashboard - only observations from this project
|
||||
observations, err = s.observationStore.GetObservationsByProjectStrict(r.Context(), project, limit)
|
||||
} else {
|
||||
// All projects
|
||||
observations, err = s.observationStore.GetAllRecentObservations(r.Context(), limit)
|
||||
@@ -501,7 +535,7 @@ func (s *Service) handleGetObservations(w http.ResponseWriter, r *http.Request)
|
||||
// handleGetSummaries returns recent summaries.
|
||||
// Supports optional query parameter for semantic search via sqlite-vec.
|
||||
func (s *Service) handleGetSummaries(w http.ResponseWriter, r *http.Request) {
|
||||
limit := sqlite.ParseLimitParam(r, DefaultSummariesLimit)
|
||||
limit := gorm.ParseLimitParam(r, DefaultSummariesLimit)
|
||||
project := r.URL.Query().Get("project")
|
||||
query := r.URL.Query().Get("query")
|
||||
|
||||
@@ -548,7 +582,7 @@ func (s *Service) handleGetSummaries(w http.ResponseWriter, r *http.Request) {
|
||||
// handleGetPrompts returns recent user prompts.
|
||||
// Supports optional query parameter for semantic search via sqlite-vec.
|
||||
func (s *Service) handleGetPrompts(w http.ResponseWriter, r *http.Request) {
|
||||
limit := sqlite.ParseLimitParam(r, DefaultPromptsLimit)
|
||||
limit := gorm.ParseLimitParam(r, DefaultPromptsLimit)
|
||||
project := r.URL.Query().Get("project")
|
||||
query := r.URL.Query().Get("query")
|
||||
|
||||
@@ -612,9 +646,22 @@ func (s *Service) handleGetTypes(w http.ResponseWriter, r *http.Request) {
|
||||
})
|
||||
}
|
||||
|
||||
// handleGetModels returns available embedding models.
|
||||
func (s *Service) handleGetModels(w http.ResponseWriter, _ *http.Request) {
|
||||
models := embedding.ListModels()
|
||||
defaultModel := embedding.GetDefaultModel()
|
||||
|
||||
writeJSON(w, map[string]interface{}{
|
||||
"models": models,
|
||||
"default": defaultModel,
|
||||
"current": s.embedSvc.Version(),
|
||||
})
|
||||
}
|
||||
|
||||
// handleGetStats returns worker statistics.
|
||||
func (s *Service) handleGetStats(w http.ResponseWriter, r *http.Request) {
|
||||
retrievalStats := s.GetRetrievalStats()
|
||||
project := r.URL.Query().Get("project")
|
||||
retrievalStats := s.GetRetrievalStats(project)
|
||||
sessionsToday, _ := s.sessionStore.GetSessionsToday(r.Context())
|
||||
|
||||
response := map[string]interface{}{
|
||||
@@ -628,8 +675,24 @@ func (s *Service) handleGetStats(w http.ResponseWriter, r *http.Request) {
|
||||
"ready": s.ready.Load(),
|
||||
}
|
||||
|
||||
// Add embedding model info
|
||||
if s.embedSvc != nil {
|
||||
response["embeddingModel"] = map[string]interface{}{
|
||||
"name": s.embedSvc.Name(),
|
||||
"version": s.embedSvc.Version(),
|
||||
"dimensions": s.embedSvc.Dimensions(),
|
||||
}
|
||||
}
|
||||
|
||||
// Add vector count
|
||||
if s.vectorClient != nil {
|
||||
if count, err := s.vectorClient.Count(r.Context()); err == nil {
|
||||
response["vectorCount"] = count
|
||||
}
|
||||
}
|
||||
|
||||
// Include project-specific observation count if project is specified
|
||||
if project := r.URL.Query().Get("project"); project != "" {
|
||||
if project != "" {
|
||||
count, err := s.observationStore.GetObservationCount(r.Context(), project)
|
||||
if err == nil {
|
||||
response["projectObservations"] = count
|
||||
@@ -642,7 +705,8 @@ func (s *Service) handleGetStats(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
// handleGetRetrievalStats returns detailed retrieval statistics.
|
||||
func (s *Service) handleGetRetrievalStats(w http.ResponseWriter, r *http.Request) {
|
||||
stats := s.GetRetrievalStats()
|
||||
project := r.URL.Query().Get("project")
|
||||
stats := s.GetRetrievalStats(project)
|
||||
writeJSON(w, stats)
|
||||
}
|
||||
|
||||
@@ -679,20 +743,74 @@ func (s *Service) handleSearchByPrompt(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
|
||||
limit := sqlite.ParseLimitParam(r, DefaultSearchLimit)
|
||||
limit := gorm.ParseLimitParam(r, DefaultSearchLimit)
|
||||
|
||||
var observations []*models.Observation
|
||||
var err error
|
||||
var usedVector bool
|
||||
similarityScores := make(map[int64]float64) // Track similarity per observation
|
||||
|
||||
// Get threshold settings from config
|
||||
threshold := s.config.ContextRelevanceThreshold
|
||||
maxResults := s.config.ContextMaxPromptResults
|
||||
|
||||
// Generate expanded queries if query expander is available
|
||||
var expandedQueries []expansion.ExpandedQuery
|
||||
var detectedIntent string
|
||||
if s.queryExpander != nil {
|
||||
cfg := expansion.DefaultConfig()
|
||||
cfg.EnableVocabularyExpansion = false // Vocabulary expansion is optional
|
||||
expandedQueries = s.queryExpander.Expand(r.Context(), query, cfg)
|
||||
if len(expandedQueries) > 0 {
|
||||
detectedIntent = string(expandedQueries[0].Intent)
|
||||
}
|
||||
}
|
||||
if len(expandedQueries) == 0 {
|
||||
// Fallback to just the original query
|
||||
expandedQueries = []expansion.ExpandedQuery{
|
||||
{Query: query, Weight: 1.0, Source: "original"},
|
||||
}
|
||||
}
|
||||
|
||||
// Try vector search first if available
|
||||
if s.vectorClient != nil && s.vectorClient.IsConnected() {
|
||||
where := sqlitevec.BuildWhereFilter(sqlitevec.DocTypeObservation, "")
|
||||
|
||||
vectorResults, vecErr := s.vectorClient.Query(r.Context(), query, limit*2, where)
|
||||
if vecErr == nil && len(vectorResults) > 0 {
|
||||
// Search with each expanded query and merge results
|
||||
allVectorResults := make([]sqlitevec.QueryResult, 0)
|
||||
queryWeights := make(map[string]float64) // Track weights for score merging
|
||||
|
||||
for _, eq := range expandedQueries {
|
||||
vectorResults, vecErr := s.vectorClient.Query(r.Context(), eq.Query, limit*2, where)
|
||||
if vecErr == nil && len(vectorResults) > 0 {
|
||||
// Apply weight to similarity scores before merging
|
||||
for i := range vectorResults {
|
||||
vectorResults[i].Similarity *= eq.Weight
|
||||
}
|
||||
allVectorResults = append(allVectorResults, vectorResults...)
|
||||
queryWeights[eq.Query] = eq.Weight
|
||||
}
|
||||
}
|
||||
|
||||
if len(allVectorResults) > 0 {
|
||||
// Filter by relevance threshold before extracting IDs
|
||||
// Use a slightly lower threshold for expanded queries
|
||||
effectiveThreshold := threshold * 0.9 // Allow slightly lower scores for expanded queries
|
||||
filteredResults := sqlitevec.FilterByThreshold(allVectorResults, effectiveThreshold, 0)
|
||||
|
||||
// Build similarity map for filtered results (keeping highest weighted score per observation)
|
||||
for _, vr := range filteredResults {
|
||||
if sqliteID, ok := vr.Metadata["sqlite_id"].(float64); ok {
|
||||
id := int64(sqliteID)
|
||||
// Keep the highest score for each observation
|
||||
if existing, exists := similarityScores[id]; !exists || vr.Similarity > existing {
|
||||
similarityScores[id] = vr.Similarity
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Extract observation IDs with project/scope filtering using shared helper
|
||||
obsIDs := sqlitevec.ExtractObservationIDs(vectorResults, project)
|
||||
obsIDs := sqlitevec.ExtractObservationIDs(filteredResults, project)
|
||||
|
||||
if len(obsIDs) > 0 {
|
||||
// Fetch full observations from SQLite
|
||||
@@ -742,23 +860,132 @@ func (s *Service) handleSearchByPrompt(w http.ResponseWriter, r *http.Request) {
|
||||
freshObservations = append(freshObservations, obs)
|
||||
}
|
||||
|
||||
// Apply cross-encoder reranking if available
|
||||
var reranked bool
|
||||
if s.reranker != nil && len(freshObservations) > 0 && usedVector {
|
||||
// Build candidates from observations with their bi-encoder scores
|
||||
candidates := make([]reranking.Candidate, len(freshObservations))
|
||||
for i, obs := range freshObservations {
|
||||
content := obs.Title.String
|
||||
if obs.Narrative.Valid && obs.Narrative.String != "" {
|
||||
content = content + " " + obs.Narrative.String
|
||||
}
|
||||
candidates[i] = reranking.Candidate{
|
||||
ID: fmt.Sprintf("%d", obs.ID),
|
||||
Content: content,
|
||||
Score: similarityScores[obs.ID],
|
||||
Metadata: map[string]any{"obs_idx": i},
|
||||
}
|
||||
}
|
||||
|
||||
// Rerank using cross-encoder - use pure mode or combined scores
|
||||
var rerankResults []reranking.RerankResult
|
||||
var rerankErr error
|
||||
if s.config.RerankingPureMode {
|
||||
rerankResults, rerankErr = s.reranker.RerankByScore(query, candidates, s.config.RerankingResults)
|
||||
} else {
|
||||
rerankResults, rerankErr = s.reranker.Rerank(query, candidates, s.config.RerankingResults)
|
||||
}
|
||||
if rerankErr != nil {
|
||||
log.Warn().Err(rerankErr).Msg("Cross-encoder reranking failed, using original order")
|
||||
} else if len(rerankResults) > 0 {
|
||||
// Update similarity scores with reranked scores
|
||||
for _, rr := range rerankResults {
|
||||
if id, err := strconv.ParseInt(rr.ID, 10, 64); err == nil {
|
||||
similarityScores[id] = rr.CombinedScore
|
||||
}
|
||||
}
|
||||
|
||||
// Reorder observations based on rerank results
|
||||
reorderedObs := make([]*models.Observation, 0, len(rerankResults))
|
||||
obsMap := make(map[int64]*models.Observation)
|
||||
for _, obs := range freshObservations {
|
||||
obsMap[obs.ID] = obs
|
||||
}
|
||||
for _, rr := range rerankResults {
|
||||
if id, err := strconv.ParseInt(rr.ID, 10, 64); err == nil {
|
||||
if obs, ok := obsMap[id]; ok {
|
||||
reorderedObs = append(reorderedObs, obs)
|
||||
}
|
||||
}
|
||||
}
|
||||
freshObservations = reorderedObs
|
||||
reranked = true
|
||||
|
||||
log.Debug().
|
||||
Int("candidates", len(candidates)).
|
||||
Int("returned", len(rerankResults)).
|
||||
Msg("Cross-encoder reranking complete")
|
||||
}
|
||||
}
|
||||
|
||||
// Cluster similar observations to remove duplicates
|
||||
clusteredObservations := clusterObservations(freshObservations, 0.4)
|
||||
|
||||
// Sort by similarity score (highest first) if we have scores and didn't rerank
|
||||
if len(similarityScores) > 0 && len(clusteredObservations) > 0 && !reranked {
|
||||
sort.Slice(clusteredObservations, func(i, j int) bool {
|
||||
scoreI := similarityScores[clusteredObservations[i].ID]
|
||||
scoreJ := similarityScores[clusteredObservations[j].ID]
|
||||
return scoreI > scoreJ
|
||||
})
|
||||
}
|
||||
|
||||
// Apply max results cap if configured
|
||||
if maxResults > 0 && len(clusteredObservations) > maxResults {
|
||||
clusteredObservations = clusteredObservations[:maxResults]
|
||||
}
|
||||
|
||||
// Record retrieval stats (no verification done, so verified=0, deleted=0)
|
||||
s.recordRetrievalStats(int64(len(clusteredObservations)), 0, 0, true)
|
||||
s.recordRetrievalStats(project, int64(len(clusteredObservations)), 0, 0, true)
|
||||
|
||||
// Increment retrieval counts for scoring (async, non-blocking)
|
||||
if len(clusteredObservations) > 0 {
|
||||
ids := make([]int64, len(clusteredObservations))
|
||||
for i, obs := range clusteredObservations {
|
||||
ids[i] = obs.ID
|
||||
}
|
||||
s.incrementRetrievalCounts(ids)
|
||||
}
|
||||
|
||||
log.Info().
|
||||
Str("project", project).
|
||||
Str("query", query).
|
||||
Str("intent", detectedIntent).
|
||||
Int("expansions", len(expandedQueries)).
|
||||
Int("found", len(clusteredObservations)).
|
||||
Int("stale_excluded", staleCount).
|
||||
Float64("threshold", threshold).
|
||||
Msg("Prompt-based observation search")
|
||||
|
||||
// Build response with similarity scores
|
||||
obsWithScores := make([]map[string]interface{}, len(clusteredObservations))
|
||||
for i, obs := range clusteredObservations {
|
||||
obsMap := obs.ToMap()
|
||||
if score, ok := similarityScores[obs.ID]; ok {
|
||||
obsMap["similarity"] = score
|
||||
}
|
||||
obsWithScores[i] = obsMap
|
||||
}
|
||||
|
||||
// Build expansion info for response
|
||||
expansionInfo := make([]map[string]interface{}, len(expandedQueries))
|
||||
for i, eq := range expandedQueries {
|
||||
expansionInfo[i] = map[string]interface{}{
|
||||
"query": eq.Query,
|
||||
"weight": eq.Weight,
|
||||
"source": eq.Source,
|
||||
}
|
||||
}
|
||||
|
||||
writeJSON(w, map[string]interface{}{
|
||||
"project": project,
|
||||
"query": query,
|
||||
"observations": clusteredObservations,
|
||||
"intent": detectedIntent,
|
||||
"expansions": expansionInfo,
|
||||
"observations": obsWithScores,
|
||||
"threshold": threshold,
|
||||
"max_results": maxResults,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -824,7 +1051,16 @@ func (s *Service) handleContextInject(w http.ResponseWriter, r *http.Request) {
|
||||
duplicatesRemoved := len(freshObservations) - len(clusteredObservations)
|
||||
|
||||
// Record retrieval stats (no verification done)
|
||||
s.recordRetrievalStats(int64(len(clusteredObservations)), 0, 0, false)
|
||||
s.recordRetrievalStats(project, int64(len(clusteredObservations)), 0, 0, false)
|
||||
|
||||
// Increment retrieval counts for scoring (async, non-blocking)
|
||||
if len(clusteredObservations) > 0 {
|
||||
ids := make([]int64, len(clusteredObservations))
|
||||
for i, obs := range clusteredObservations {
|
||||
ids[i] = obs.ID
|
||||
}
|
||||
s.incrementRetrievalCounts(ids)
|
||||
}
|
||||
|
||||
log.Info().
|
||||
Str("project", project).
|
||||
@@ -984,6 +1220,35 @@ func (s *Service) handleSelfCheck(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
components = append(components, sseStatus)
|
||||
|
||||
// Check Cross-Encoder Reranker
|
||||
rerankerStatus := ComponentHealth{Name: "Cross-Encoder Reranker", Status: "healthy"}
|
||||
if !s.config.RerankingEnabled {
|
||||
rerankerStatus.Status = "degraded"
|
||||
rerankerStatus.Message = "Disabled in config"
|
||||
if overall == "healthy" {
|
||||
overall = "degraded"
|
||||
}
|
||||
} else if s.reranker == nil {
|
||||
rerankerStatus.Status = "degraded"
|
||||
rerankerStatus.Message = "Not initialized"
|
||||
if overall == "healthy" {
|
||||
overall = "degraded"
|
||||
}
|
||||
} else {
|
||||
// Verify reranker is functional using Score
|
||||
_, normalizedScore, err := s.reranker.Score("test query", "test document")
|
||||
if err != nil {
|
||||
rerankerStatus.Status = "unhealthy"
|
||||
rerankerStatus.Message = fmt.Sprintf("Score check failed: %v", err)
|
||||
if overall == "healthy" {
|
||||
overall = "degraded"
|
||||
}
|
||||
} else {
|
||||
rerankerStatus.Message = fmt.Sprintf("Score check passed (%.4f)", normalizedScore)
|
||||
}
|
||||
}
|
||||
components = append(components, rerankerStatus)
|
||||
|
||||
// Calculate uptime
|
||||
uptime := time.Since(s.startTime).Round(time.Second).String()
|
||||
|
||||
@@ -995,7 +1260,7 @@ func (s *Service) handleSelfCheck(w http.ResponseWriter, r *http.Request) {
|
||||
})
|
||||
}
|
||||
|
||||
// handleUpdateRestart restarts the worker with the new binary.
|
||||
// handleUpdateRestart restarts the worker with the new binary (after update).
|
||||
func (s *Service) handleUpdateRestart(w http.ResponseWriter, r *http.Request) {
|
||||
status := s.updater.GetStatus()
|
||||
if status.State != "done" {
|
||||
@@ -1021,3 +1286,29 @@ func (s *Service) handleUpdateRestart(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
// handleRestart restarts the worker process (general restart, not tied to update).
|
||||
func (s *Service) handleRestart(w http.ResponseWriter, r *http.Request) {
|
||||
log.Info().Msg("Manual restart requested via API")
|
||||
|
||||
// Send response before restarting
|
||||
writeJSON(w, map[string]interface{}{
|
||||
"success": true,
|
||||
"message": "Restarting worker...",
|
||||
"version": s.version,
|
||||
})
|
||||
|
||||
// Flush the response
|
||||
if f, ok := w.(http.Flusher); ok {
|
||||
f.Flush()
|
||||
}
|
||||
|
||||
// Restart in background after response is sent
|
||||
go func() {
|
||||
// Small delay to ensure response is sent
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
if err := s.updater.Restart(); err != nil {
|
||||
log.Error().Err(err).Msg("Failed to restart worker")
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
@@ -0,0 +1,292 @@
|
||||
// Package worker provides the main worker service for claude-mnemonic.
|
||||
package worker
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"strconv"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
"github.com/lukaszraczylo/claude-mnemonic/pkg/models"
|
||||
)
|
||||
|
||||
// DefaultPatternsLimit is the default number of patterns to return.
|
||||
const DefaultPatternsLimit = 100
|
||||
|
||||
// handleGetPatterns returns all active patterns, optionally filtered by type or project.
|
||||
func (s *Service) handleGetPatterns(w http.ResponseWriter, r *http.Request) {
|
||||
s.initMu.RLock()
|
||||
store := s.patternStore
|
||||
s.initMu.RUnlock()
|
||||
|
||||
if store == nil {
|
||||
http.Error(w, "pattern store not initialized", http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
|
||||
// Parse query parameters
|
||||
limit := DefaultPatternsLimit
|
||||
if l := r.URL.Query().Get("limit"); l != "" {
|
||||
if parsed, err := strconv.Atoi(l); err == nil && parsed > 0 {
|
||||
limit = parsed
|
||||
}
|
||||
}
|
||||
|
||||
patternType := r.URL.Query().Get("type")
|
||||
project := r.URL.Query().Get("project")
|
||||
|
||||
var patterns []*models.Pattern
|
||||
var err error
|
||||
|
||||
if patternType != "" {
|
||||
// Filter by type
|
||||
patterns, err = store.GetPatternsByType(r.Context(), models.PatternType(patternType), limit)
|
||||
} else if project != "" {
|
||||
// Filter by project
|
||||
patterns, err = store.GetPatternsByProject(r.Context(), project, limit)
|
||||
} else {
|
||||
// Get all active patterns
|
||||
patterns, err = store.GetActivePatterns(r.Context(), limit)
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
writeJSON(w, patterns)
|
||||
}
|
||||
|
||||
// handleGetPatternStats returns aggregate statistics about patterns.
|
||||
func (s *Service) handleGetPatternStats(w http.ResponseWriter, r *http.Request) {
|
||||
s.initMu.RLock()
|
||||
store := s.patternStore
|
||||
s.initMu.RUnlock()
|
||||
|
||||
if store == nil {
|
||||
http.Error(w, "pattern store not initialized", http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
|
||||
stats, err := store.GetPatternStats(r.Context())
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
writeJSON(w, stats)
|
||||
}
|
||||
|
||||
// handleGetPatternByID returns a single pattern by ID.
|
||||
func (s *Service) handleGetPatternByID(w http.ResponseWriter, r *http.Request) {
|
||||
s.initMu.RLock()
|
||||
store := s.patternStore
|
||||
s.initMu.RUnlock()
|
||||
|
||||
if store == nil {
|
||||
http.Error(w, "pattern store not initialized", http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
|
||||
idStr := chi.URLParam(r, "id")
|
||||
id, err := strconv.ParseInt(idStr, 10, 64)
|
||||
if err != nil {
|
||||
http.Error(w, "invalid pattern ID", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
pattern, err := store.GetPatternByID(r.Context(), id)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
if pattern == nil {
|
||||
http.Error(w, "pattern not found", http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
|
||||
writeJSON(w, pattern)
|
||||
}
|
||||
|
||||
// handleGetPatternInsight returns a formatted insight string for a pattern.
|
||||
func (s *Service) handleGetPatternInsight(w http.ResponseWriter, r *http.Request) {
|
||||
s.initMu.RLock()
|
||||
detector := s.patternDetector
|
||||
s.initMu.RUnlock()
|
||||
|
||||
if detector == nil {
|
||||
http.Error(w, "pattern detector not initialized", http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
|
||||
idStr := chi.URLParam(r, "id")
|
||||
id, err := strconv.ParseInt(idStr, 10, 64)
|
||||
if err != nil {
|
||||
http.Error(w, "invalid pattern ID", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
insight, err := detector.GetPatternInsight(r.Context(), id)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
writeJSON(w, map[string]string{"insight": insight})
|
||||
}
|
||||
|
||||
// handleDeletePattern deletes a pattern by ID.
|
||||
func (s *Service) handleDeletePattern(w http.ResponseWriter, r *http.Request) {
|
||||
s.initMu.RLock()
|
||||
store := s.patternStore
|
||||
s.initMu.RUnlock()
|
||||
|
||||
if store == nil {
|
||||
http.Error(w, "pattern store not initialized", http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
|
||||
idStr := chi.URLParam(r, "id")
|
||||
id, err := strconv.ParseInt(idStr, 10, 64)
|
||||
if err != nil {
|
||||
http.Error(w, "invalid pattern ID", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
if err := store.DeletePattern(r.Context(), id); err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
writeJSON(w, map[string]string{"status": "deleted"})
|
||||
}
|
||||
|
||||
// handleDeprecatePattern marks a pattern as deprecated.
|
||||
func (s *Service) handleDeprecatePattern(w http.ResponseWriter, r *http.Request) {
|
||||
s.initMu.RLock()
|
||||
store := s.patternStore
|
||||
s.initMu.RUnlock()
|
||||
|
||||
if store == nil {
|
||||
http.Error(w, "pattern store not initialized", http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
|
||||
idStr := chi.URLParam(r, "id")
|
||||
id, err := strconv.ParseInt(idStr, 10, 64)
|
||||
if err != nil {
|
||||
http.Error(w, "invalid pattern ID", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
if err := store.MarkPatternDeprecated(r.Context(), id); err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
writeJSON(w, map[string]string{"status": "deprecated"})
|
||||
}
|
||||
|
||||
// MergePatternsRequest is the request body for merging patterns.
|
||||
type MergePatternsRequest struct {
|
||||
SourceID int64 `json:"source_id"`
|
||||
TargetID int64 `json:"target_id"`
|
||||
}
|
||||
|
||||
// handleSearchPatterns performs full-text search on patterns.
|
||||
func (s *Service) handleSearchPatterns(w http.ResponseWriter, r *http.Request) {
|
||||
s.initMu.RLock()
|
||||
store := s.patternStore
|
||||
s.initMu.RUnlock()
|
||||
|
||||
if store == nil {
|
||||
http.Error(w, "pattern store not initialized", http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
|
||||
query := r.URL.Query().Get("q")
|
||||
if query == "" {
|
||||
http.Error(w, "query parameter 'q' is required", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
limit := DefaultPatternsLimit
|
||||
if l := r.URL.Query().Get("limit"); l != "" {
|
||||
if parsed, err := strconv.Atoi(l); err == nil && parsed > 0 {
|
||||
limit = parsed
|
||||
}
|
||||
}
|
||||
|
||||
patterns, err := store.SearchPatternsFTS(r.Context(), query, limit)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
writeJSON(w, patterns)
|
||||
}
|
||||
|
||||
// handleGetPatternByName returns a pattern by its name.
|
||||
func (s *Service) handleGetPatternByName(w http.ResponseWriter, r *http.Request) {
|
||||
s.initMu.RLock()
|
||||
store := s.patternStore
|
||||
s.initMu.RUnlock()
|
||||
|
||||
if store == nil {
|
||||
http.Error(w, "pattern store not initialized", http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
|
||||
name := r.URL.Query().Get("name")
|
||||
if name == "" {
|
||||
http.Error(w, "query parameter 'name' is required", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
pattern, err := store.GetPatternByName(r.Context(), name)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
if pattern == nil {
|
||||
http.Error(w, "pattern not found", http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
|
||||
writeJSON(w, pattern)
|
||||
}
|
||||
|
||||
// handleMergePatterns merges a source pattern into a target pattern.
|
||||
func (s *Service) handleMergePatterns(w http.ResponseWriter, r *http.Request) {
|
||||
s.initMu.RLock()
|
||||
store := s.patternStore
|
||||
s.initMu.RUnlock()
|
||||
|
||||
if store == nil {
|
||||
http.Error(w, "pattern store not initialized", http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
|
||||
var req MergePatternsRequest
|
||||
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
||||
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
if req.SourceID == 0 || req.TargetID == 0 {
|
||||
http.Error(w, "source_id and target_id are required", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
if req.SourceID == req.TargetID {
|
||||
http.Error(w, "source_id and target_id cannot be the same", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
if err := store.MergePatterns(r.Context(), req.SourceID, req.TargetID); err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
writeJSON(w, map[string]string{"status": "merged"})
|
||||
}
|
||||
@@ -0,0 +1,174 @@
|
||||
// Package worker provides the main worker service for claude-mnemonic.
|
||||
package worker
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"strconv"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
"github.com/lukaszraczylo/claude-mnemonic/pkg/models"
|
||||
)
|
||||
|
||||
// DefaultRelationsLimit is the default number of relations to return.
|
||||
const DefaultRelationsLimit = 50
|
||||
|
||||
// handleGetRelations returns relations for an observation.
|
||||
func (s *Service) handleGetRelations(w http.ResponseWriter, r *http.Request) {
|
||||
idStr := chi.URLParam(r, "id")
|
||||
id, err := strconv.ParseInt(idStr, 10, 64)
|
||||
if err != nil {
|
||||
http.Error(w, "invalid observation id", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
relations, err := s.relationStore.GetRelationsWithDetails(r.Context(), id)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
if relations == nil {
|
||||
relations = []*models.RelationWithDetails{}
|
||||
}
|
||||
|
||||
writeJSON(w, relations)
|
||||
}
|
||||
|
||||
// handleGetRelationGraph returns the relation graph for an observation.
|
||||
func (s *Service) handleGetRelationGraph(w http.ResponseWriter, r *http.Request) {
|
||||
idStr := chi.URLParam(r, "id")
|
||||
id, err := strconv.ParseInt(idStr, 10, 64)
|
||||
if err != nil {
|
||||
http.Error(w, "invalid observation id", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
// Get depth parameter (default 2)
|
||||
depth := 2
|
||||
if depthStr := r.URL.Query().Get("depth"); depthStr != "" {
|
||||
if d, err := strconv.Atoi(depthStr); err == nil && d > 0 && d <= 5 {
|
||||
depth = d
|
||||
}
|
||||
}
|
||||
|
||||
graph, err := s.relationStore.GetRelationGraph(r.Context(), id, depth)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
writeJSON(w, graph)
|
||||
}
|
||||
|
||||
// handleGetRelatedObservations returns observations related to a given one.
|
||||
func (s *Service) handleGetRelatedObservations(w http.ResponseWriter, r *http.Request) {
|
||||
idStr := chi.URLParam(r, "id")
|
||||
id, err := strconv.ParseInt(idStr, 10, 64)
|
||||
if err != nil {
|
||||
http.Error(w, "invalid observation id", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
// Get minimum confidence parameter (default 0.4)
|
||||
minConfidence := 0.4
|
||||
if confStr := r.URL.Query().Get("min_confidence"); confStr != "" {
|
||||
if c, err := strconv.ParseFloat(confStr, 64); err == nil && c >= 0 && c <= 1 {
|
||||
minConfidence = c
|
||||
}
|
||||
}
|
||||
|
||||
// Get related observation IDs
|
||||
relatedIDs, err := s.relationStore.GetRelatedObservationIDs(r.Context(), id, minConfidence)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
if len(relatedIDs) == 0 {
|
||||
writeJSON(w, []*models.Observation{})
|
||||
return
|
||||
}
|
||||
|
||||
// Fetch full observations
|
||||
observations, err := s.observationStore.GetObservationsByIDs(r.Context(), relatedIDs, "importance", 50)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
if observations == nil {
|
||||
observations = []*models.Observation{}
|
||||
}
|
||||
|
||||
writeJSON(w, observations)
|
||||
}
|
||||
|
||||
// handleGetRelationsByType returns all relations of a specific type.
|
||||
func (s *Service) handleGetRelationsByType(w http.ResponseWriter, r *http.Request) {
|
||||
relType := chi.URLParam(r, "type")
|
||||
|
||||
// Validate relation type
|
||||
validType := false
|
||||
for _, t := range models.AllRelationTypes {
|
||||
if string(t) == relType {
|
||||
validType = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !validType {
|
||||
http.Error(w, "invalid relation type", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
limit := DefaultRelationsLimit
|
||||
if limitStr := r.URL.Query().Get("limit"); limitStr != "" {
|
||||
if l, err := strconv.Atoi(limitStr); err == nil && l > 0 && l <= 100 {
|
||||
limit = l
|
||||
}
|
||||
}
|
||||
|
||||
relations, err := s.relationStore.GetRelationsByType(r.Context(), models.RelationType(relType), limit)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
if relations == nil {
|
||||
relations = []*models.ObservationRelation{}
|
||||
}
|
||||
|
||||
writeJSON(w, relations)
|
||||
}
|
||||
|
||||
// handleGetRelationStats returns statistics about relations.
|
||||
func (s *Service) handleGetRelationStats(w http.ResponseWriter, r *http.Request) {
|
||||
// Get total relation count
|
||||
totalCount, err := s.relationStore.GetTotalRelationCount(r.Context())
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
// Get high confidence relations count
|
||||
highConfRelations, err := s.relationStore.GetHighConfidenceRelations(r.Context(), 0.7, 1000)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
// Count by relation type
|
||||
typeCounts := make(map[string]int)
|
||||
for _, t := range models.AllRelationTypes {
|
||||
relations, err := s.relationStore.GetRelationsByType(r.Context(), t, 1000)
|
||||
if err == nil {
|
||||
typeCounts[string(t)] = len(relations)
|
||||
}
|
||||
}
|
||||
|
||||
writeJSON(w, map[string]interface{}{
|
||||
"total_count": totalCount,
|
||||
"high_confidence": len(highConfRelations),
|
||||
"by_type": typeCounts,
|
||||
"min_confidence_used": 0.4,
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,358 @@
|
||||
// Package worker provides the main worker service for claude-mnemonic.
|
||||
package worker
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
"github.com/lukaszraczylo/claude-mnemonic/pkg/models"
|
||||
)
|
||||
|
||||
// FeedbackRequest represents a user feedback submission.
|
||||
type FeedbackRequest struct {
|
||||
Feedback int `json:"feedback"` // -1 (thumbs down), 0 (neutral), 1 (thumbs up)
|
||||
}
|
||||
|
||||
// handleObservationFeedback handles user feedback (thumbs up/down) for an observation.
|
||||
// POST /api/observations/{id}/feedback
|
||||
func (s *Service) handleObservationFeedback(w http.ResponseWriter, r *http.Request) {
|
||||
// Parse observation ID
|
||||
idStr := chi.URLParam(r, "id")
|
||||
id, err := strconv.ParseInt(idStr, 10, 64)
|
||||
if err != nil {
|
||||
http.Error(w, "invalid observation id", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
// Parse request body
|
||||
var req FeedbackRequest
|
||||
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
||||
http.Error(w, "invalid request body", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
// Validate feedback value
|
||||
if req.Feedback < -1 || req.Feedback > 1 {
|
||||
http.Error(w, "feedback must be -1, 0, or 1", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
// Get required components
|
||||
s.initMu.RLock()
|
||||
observationStore := s.observationStore
|
||||
scoreCalculator := s.scoreCalculator
|
||||
s.initMu.RUnlock()
|
||||
|
||||
if observationStore == nil {
|
||||
http.Error(w, "service not ready", http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
|
||||
// Update feedback in database
|
||||
if err := observationStore.UpdateObservationFeedback(r.Context(), id, req.Feedback); err != nil {
|
||||
http.Error(w, "failed to update feedback", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
// Recalculate score immediately if calculator is available
|
||||
var newScore float64
|
||||
if scoreCalculator != nil {
|
||||
obs, err := observationStore.GetObservationByID(r.Context(), id)
|
||||
if err == nil && obs != nil {
|
||||
obs.UserFeedback = req.Feedback // Apply the new feedback
|
||||
newScore = scoreCalculator.Calculate(obs, time.Now())
|
||||
if err := observationStore.UpdateImportanceScore(r.Context(), id, newScore); err != nil {
|
||||
// Log but don't fail - feedback was recorded
|
||||
// Score will be updated on next recalculation cycle
|
||||
_ = err // Explicitly ignore - non-critical operation
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Broadcast update via SSE
|
||||
s.sseBroadcaster.Broadcast(map[string]interface{}{
|
||||
"type": "observation_feedback",
|
||||
"id": id,
|
||||
"feedback": req.Feedback,
|
||||
"score": newScore,
|
||||
})
|
||||
|
||||
writeJSON(w, map[string]interface{}{
|
||||
"status": "ok",
|
||||
"id": id,
|
||||
"feedback": req.Feedback,
|
||||
"score": newScore,
|
||||
})
|
||||
}
|
||||
|
||||
// handleGetScoringStats returns scoring statistics and configuration.
|
||||
// GET /api/scoring/stats
|
||||
func (s *Service) handleGetScoringStats(w http.ResponseWriter, r *http.Request) {
|
||||
project := r.URL.Query().Get("project")
|
||||
|
||||
s.initMu.RLock()
|
||||
observationStore := s.observationStore
|
||||
recalculator := s.recalculator
|
||||
s.initMu.RUnlock()
|
||||
|
||||
if observationStore == nil {
|
||||
http.Error(w, "service not ready", http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
|
||||
// Get feedback statistics
|
||||
feedbackStats, err := observationStore.GetObservationFeedbackStats(r.Context(), project)
|
||||
if err != nil {
|
||||
http.Error(w, "failed to get feedback stats", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
response := map[string]interface{}{
|
||||
"feedback": feedbackStats,
|
||||
}
|
||||
|
||||
// Add recalculator stats if available
|
||||
if recalculator != nil {
|
||||
response["recalculator"] = recalculator.GetStats()
|
||||
}
|
||||
|
||||
writeJSON(w, response)
|
||||
}
|
||||
|
||||
// handleGetTopObservations returns the highest-scoring observations.
|
||||
// GET /api/observations/top
|
||||
func (s *Service) handleGetTopObservations(w http.ResponseWriter, r *http.Request) {
|
||||
limit := parseIntParam(r, "limit", 10)
|
||||
project := r.URL.Query().Get("project")
|
||||
|
||||
s.initMu.RLock()
|
||||
observationStore := s.observationStore
|
||||
s.initMu.RUnlock()
|
||||
|
||||
if observationStore == nil {
|
||||
http.Error(w, "service not ready", http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
|
||||
observations, err := observationStore.GetTopScoringObservations(r.Context(), project, limit)
|
||||
if err != nil {
|
||||
http.Error(w, "failed to get top observations", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
if observations == nil {
|
||||
observations = []*models.Observation{}
|
||||
}
|
||||
|
||||
writeJSON(w, observations)
|
||||
}
|
||||
|
||||
// handleGetMostRetrieved returns the most frequently retrieved observations.
|
||||
// GET /api/observations/most-retrieved
|
||||
func (s *Service) handleGetMostRetrieved(w http.ResponseWriter, r *http.Request) {
|
||||
limit := parseIntParam(r, "limit", 10)
|
||||
project := r.URL.Query().Get("project")
|
||||
|
||||
s.initMu.RLock()
|
||||
observationStore := s.observationStore
|
||||
s.initMu.RUnlock()
|
||||
|
||||
if observationStore == nil {
|
||||
http.Error(w, "service not ready", http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
|
||||
observations, err := observationStore.GetMostRetrievedObservations(r.Context(), project, limit)
|
||||
if err != nil {
|
||||
http.Error(w, "failed to get most retrieved observations", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
if observations == nil {
|
||||
observations = []*models.Observation{}
|
||||
}
|
||||
|
||||
writeJSON(w, observations)
|
||||
}
|
||||
|
||||
// handleExplainScore returns a breakdown of how an observation's score was calculated.
|
||||
// GET /api/observations/{id}/score
|
||||
func (s *Service) handleExplainScore(w http.ResponseWriter, r *http.Request) {
|
||||
// Parse observation ID
|
||||
idStr := chi.URLParam(r, "id")
|
||||
id, err := strconv.ParseInt(idStr, 10, 64)
|
||||
if err != nil {
|
||||
http.Error(w, "invalid observation id", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
s.initMu.RLock()
|
||||
observationStore := s.observationStore
|
||||
scoreCalculator := s.scoreCalculator
|
||||
s.initMu.RUnlock()
|
||||
|
||||
if observationStore == nil || scoreCalculator == nil {
|
||||
http.Error(w, "service not ready", http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
|
||||
// Get observation
|
||||
obs, err := observationStore.GetObservationByID(r.Context(), id)
|
||||
if err != nil {
|
||||
http.Error(w, "failed to get observation", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
if obs == nil {
|
||||
http.Error(w, "observation not found", http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
|
||||
// Calculate score components
|
||||
components := scoreCalculator.CalculateComponents(obs, time.Now())
|
||||
|
||||
writeJSON(w, map[string]interface{}{
|
||||
"id": id,
|
||||
"components": components,
|
||||
"config": scoreCalculator.GetConfig(),
|
||||
})
|
||||
}
|
||||
|
||||
// handleUpdateConceptWeight updates a concept weight.
|
||||
// PUT /api/scoring/concepts/{concept}
|
||||
func (s *Service) handleUpdateConceptWeight(w http.ResponseWriter, r *http.Request) {
|
||||
concept := chi.URLParam(r, "concept")
|
||||
if concept == "" {
|
||||
http.Error(w, "concept is required", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
var req struct {
|
||||
Weight float64 `json:"weight"`
|
||||
}
|
||||
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
||||
http.Error(w, "invalid request body", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
// Validate weight
|
||||
if req.Weight < 0 || req.Weight > 1 {
|
||||
http.Error(w, "weight must be between 0 and 1", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
s.initMu.RLock()
|
||||
observationStore := s.observationStore
|
||||
recalculator := s.recalculator
|
||||
s.initMu.RUnlock()
|
||||
|
||||
if observationStore == nil {
|
||||
http.Error(w, "service not ready", http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
|
||||
// Update in database
|
||||
if err := observationStore.UpdateConceptWeight(r.Context(), concept, req.Weight); err != nil {
|
||||
http.Error(w, "failed to update concept weight", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
// Refresh concept weights in recalculator
|
||||
if recalculator != nil {
|
||||
if err := recalculator.RefreshConceptWeights(r.Context()); err != nil {
|
||||
// Log but don't fail - weight was saved
|
||||
_ = err // Explicitly ignore - non-critical operation
|
||||
}
|
||||
}
|
||||
|
||||
writeJSON(w, map[string]interface{}{
|
||||
"status": "ok",
|
||||
"concept": concept,
|
||||
"weight": req.Weight,
|
||||
})
|
||||
}
|
||||
|
||||
// handleGetConceptWeights returns all concept weights.
|
||||
// GET /api/scoring/concepts
|
||||
func (s *Service) handleGetConceptWeights(w http.ResponseWriter, r *http.Request) {
|
||||
s.initMu.RLock()
|
||||
observationStore := s.observationStore
|
||||
s.initMu.RUnlock()
|
||||
|
||||
if observationStore == nil {
|
||||
http.Error(w, "service not ready", http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
|
||||
weights, err := observationStore.GetConceptWeights(r.Context())
|
||||
if err != nil {
|
||||
http.Error(w, "failed to get concept weights", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
writeJSON(w, weights)
|
||||
}
|
||||
|
||||
// handleTriggerRecalculation triggers an immediate score recalculation.
|
||||
// POST /api/scoring/recalculate
|
||||
func (s *Service) handleTriggerRecalculation(w http.ResponseWriter, r *http.Request) {
|
||||
s.initMu.RLock()
|
||||
recalculator := s.recalculator
|
||||
s.initMu.RUnlock()
|
||||
|
||||
if recalculator == nil {
|
||||
http.Error(w, "recalculator not available", http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
|
||||
// Run recalculation in background
|
||||
go func() {
|
||||
if err := recalculator.RecalculateNow(r.Context()); err != nil {
|
||||
// Log error but don't block response
|
||||
_ = err // Explicitly ignore - background operation
|
||||
}
|
||||
}()
|
||||
|
||||
writeJSON(w, map[string]string{"status": "recalculation triggered"})
|
||||
}
|
||||
|
||||
// parseIntParam parses an integer query parameter with a default value.
|
||||
func parseIntParam(r *http.Request, name string, defaultVal int) int {
|
||||
if val := r.URL.Query().Get(name); val != "" {
|
||||
if parsed, err := strconv.Atoi(val); err == nil && parsed > 0 {
|
||||
return parsed
|
||||
}
|
||||
}
|
||||
return defaultVal
|
||||
}
|
||||
|
||||
// incrementRetrievalCounts increments retrieval counts for observations.
|
||||
// Called after search results are returned to track popularity.
|
||||
func (s *Service) incrementRetrievalCounts(ids []int64) {
|
||||
if len(ids) == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
s.initMu.RLock()
|
||||
store := s.observationStore
|
||||
s.initMu.RUnlock()
|
||||
|
||||
if store == nil {
|
||||
return
|
||||
}
|
||||
|
||||
// Increment in background to not block response
|
||||
go func() {
|
||||
// Create a new context with timeout for the background operation
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
|
||||
defer cancel()
|
||||
|
||||
if err := store.IncrementRetrievalCount(ctx, ids); err != nil {
|
||||
// Log but don't fail - this is a background operation
|
||||
_ = err // Explicitly ignore - background operation
|
||||
}
|
||||
}()
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user