mirror of
https://github.com/lukaszraczylo/claude-mnemonic.git
synced 2026-10-12 04:55:18 +00:00
Compare commits
36
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
21bcbfae07 | ||
|
|
03e366da0f | ||
|
|
f8d98f7b6b | ||
|
|
7ce7b80650 | ||
|
|
f23df63059 | ||
|
|
f56037a49a | ||
|
|
d5ccbd641e | ||
|
|
862ff7c3ac | ||
|
|
e7b58312c5 | ||
|
|
e5f4a100ab | ||
|
|
31d67ceb2a | ||
|
|
5c7b623585 | ||
|
|
db528b2544 | ||
|
|
f452591971 | ||
|
|
7fe679f83b | ||
|
|
6c28ecb22a | ||
|
|
7e49113d24 | ||
|
|
e52f3284ba | ||
|
|
f0c35eef0e | ||
|
|
085c2cb3d0 | ||
|
|
cd14ba8915 | ||
|
|
0a0dfc1c2f | ||
|
|
d68a700917 | ||
|
|
c705449467 | ||
|
|
ad8aef92e1 | ||
|
|
268c286859 | ||
|
|
c664ad5fd8 | ||
|
|
0eca679139 | ||
|
|
c14677f1f2 | ||
|
|
ca711a8a7e | ||
|
|
0769feaca5 | ||
|
|
bbd6c3f1ff | ||
|
|
637412e8bb | ||
|
|
a1c1a5cad5 | ||
|
|
3a223da320 | ||
|
|
feb7f251b4 |
@@ -1 +0,0 @@
|
||||
internal/embedding/assets/*.onnx filter=lfs diff=lfs merge=lfs -text
|
||||
@@ -17,5 +17,4 @@ jobs:
|
||||
with:
|
||||
go-version: ">=1.24"
|
||||
release-workflow: "release.yaml"
|
||||
lfs: true
|
||||
secrets: inherit
|
||||
|
||||
@@ -20,5 +20,4 @@ jobs:
|
||||
uses: lukaszraczylo/shared-actions/.github/workflows/go-pr.yaml@main
|
||||
with:
|
||||
go-version: ">=1.24"
|
||||
lfs: true
|
||||
secrets: inherit
|
||||
|
||||
+102
-103
@@ -1,121 +1,120 @@
|
||||
name: Release
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
paths-ignore:
|
||||
- "**.md"
|
||||
- "docs/**"
|
||||
workflow_dispatch:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
paths-ignore:
|
||||
- "**.md"
|
||||
- "docs/**"
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
id-token: write
|
||||
contents: write
|
||||
packages: write
|
||||
id-token: write
|
||||
contents: write
|
||||
packages: write
|
||||
|
||||
jobs:
|
||||
release:
|
||||
uses: lukaszraczylo/shared-actions/.github/workflows/go-release-cgo.yaml@main
|
||||
release:
|
||||
uses: lukaszraczylo/shared-actions/.github/workflows/go-release-cgo.yaml@main
|
||||
with:
|
||||
go-version: ">=1.24"
|
||||
node-enabled: true
|
||||
node-version: "20"
|
||||
node-build-script: "cd ui && npm ci && npm run build"
|
||||
node-cache-dependency-path: "ui/package-lock.json"
|
||||
node-output-path: "ui/dist"
|
||||
node-embed-path: "internal/worker/static"
|
||||
secrets: inherit
|
||||
|
||||
commit-marketplace:
|
||||
needs: release
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
go-version: ">=1.24"
|
||||
node-enabled: true
|
||||
node-version: "20"
|
||||
node-build-script: "cd ui && npm ci && npm run build"
|
||||
node-cache-dependency-path: "ui/package-lock.json"
|
||||
node-output-path: "ui/dist"
|
||||
node-embed-path: "internal/worker/static"
|
||||
lfs: true
|
||||
secrets: inherit
|
||||
ref: main
|
||||
|
||||
commit-marketplace:
|
||||
needs: release
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
ref: main
|
||||
- name: Download checksums
|
||||
run: |
|
||||
VERSION="${{ needs.release.outputs.version_tag }}"
|
||||
curl -sSL -o checksums.txt \
|
||||
"https://github.com/lukaszraczylo/claude-mnemonic/releases/download/${VERSION}/checksums.txt"
|
||||
|
||||
- name: Download checksums
|
||||
run: |
|
||||
VERSION="${{ needs.release.outputs.version_tag }}"
|
||||
curl -sSL -o checksums.txt \
|
||||
"https://github.com/lukaszraczylo/claude-mnemonic/releases/download/${VERSION}/checksums.txt"
|
||||
- name: Generate marketplace.json with checksums
|
||||
run: |
|
||||
VERSION="${{ needs.release.outputs.version }}"
|
||||
VERSION_TAG="${{ needs.release.outputs.version_tag }}"
|
||||
TODAY=$(date -u +%Y-%m-%d)
|
||||
|
||||
- name: Generate marketplace.json with checksums
|
||||
run: |
|
||||
VERSION="${{ needs.release.outputs.version }}"
|
||||
VERSION_TAG="${{ needs.release.outputs.version_tag }}"
|
||||
TODAY=$(date -u +%Y-%m-%d)
|
||||
# Extract checksums from checksums.txt
|
||||
SHA_DARWIN_ARM64=$(grep "darwin_arm64" checksums.txt | awk '{print $1}')
|
||||
SHA_LINUX_AMD64=$(grep "linux_amd64" checksums.txt | awk '{print $1}')
|
||||
SHA_WINDOWS_AMD64=$(grep "windows_amd64" checksums.txt | awk '{print $1}')
|
||||
|
||||
# Extract checksums from checksums.txt
|
||||
SHA_DARWIN_ARM64=$(grep "darwin_arm64" checksums.txt | awk '{print $1}')
|
||||
SHA_LINUX_AMD64=$(grep "linux_amd64" checksums.txt | awk '{print $1}')
|
||||
SHA_WINDOWS_AMD64=$(grep "windows_amd64" checksums.txt | awk '{print $1}')
|
||||
|
||||
# Generate marketplace.json
|
||||
cat > marketplace.json << EOF
|
||||
{
|
||||
"\$schema": "https://anthropic.com/claude-code/marketplace.schema.json",
|
||||
"name": "claude-mnemonic",
|
||||
"version": "1.0.0",
|
||||
"description": "Persistent memory system for Claude Code - stores observations, session summaries, and user prompts with semantic search",
|
||||
"owner": {
|
||||
"name": "lukaszraczylo",
|
||||
"email": "[email protected]",
|
||||
"url": "https://github.com/lukaszraczylo"
|
||||
},
|
||||
"repository": "https://github.com/lukaszraczylo/claude-mnemonic",
|
||||
"plugins": [
|
||||
{
|
||||
"name": "claude-mnemonic",
|
||||
"description": "Persistent memory system for Claude Code - Go implementation with SQLite and ChromaDB vector search",
|
||||
"version": "${VERSION}",
|
||||
"author": {
|
||||
"name": "lukaszraczylo",
|
||||
"url": "https://github.com/lukaszraczylo"
|
||||
# Generate marketplace.json
|
||||
cat > marketplace.json << EOF
|
||||
{
|
||||
"\$schema": "https://anthropic.com/claude-code/marketplace.schema.json",
|
||||
"name": "claude-mnemonic",
|
||||
"version": "1.0.0",
|
||||
"description": "Persistent memory system for Claude Code - stores observations, session summaries, and user prompts with semantic search",
|
||||
"owner": {
|
||||
"name": "lukaszraczylo",
|
||||
"email": "[email protected]",
|
||||
"url": "https://github.com/lukaszraczylo"
|
||||
},
|
||||
"repository": "https://github.com/lukaszraczylo/claude-mnemonic",
|
||||
"plugins": [
|
||||
{
|
||||
"name": "claude-mnemonic",
|
||||
"description": "Persistent memory system for Claude Code - Go implementation with SQLite and ChromaDB vector search",
|
||||
"version": "${VERSION}",
|
||||
"author": {
|
||||
"name": "lukaszraczylo",
|
||||
"url": "https://github.com/lukaszraczylo"
|
||||
},
|
||||
"category": "productivity",
|
||||
"tags": ["memory", "persistence", "search", "context"],
|
||||
"license": "MIT",
|
||||
"homepage": "https://github.com/lukaszraczylo/claude-mnemonic",
|
||||
"releases": {
|
||||
"latest": "${VERSION}",
|
||||
"versions": {
|
||||
"${VERSION}": {
|
||||
"releaseDate": "${TODAY}",
|
||||
"downloads": {
|
||||
"darwin-arm64": {
|
||||
"url": "https://github.com/lukaszraczylo/claude-mnemonic/releases/download/${VERSION_TAG}/claude-mnemonic_${VERSION}_darwin_arm64.tar.gz",
|
||||
"format": "tar.gz",
|
||||
"sha256": "${SHA_DARWIN_ARM64}"
|
||||
},
|
||||
"category": "productivity",
|
||||
"tags": ["memory", "persistence", "search", "context"],
|
||||
"license": "MIT",
|
||||
"homepage": "https://github.com/lukaszraczylo/claude-mnemonic",
|
||||
"releases": {
|
||||
"latest": "${VERSION}",
|
||||
"versions": {
|
||||
"${VERSION}": {
|
||||
"releaseDate": "${TODAY}",
|
||||
"downloads": {
|
||||
"darwin-arm64": {
|
||||
"url": "https://github.com/lukaszraczylo/claude-mnemonic/releases/download/${VERSION_TAG}/claude-mnemonic_${VERSION}_darwin_arm64.tar.gz",
|
||||
"format": "tar.gz",
|
||||
"sha256": "${SHA_DARWIN_ARM64}"
|
||||
},
|
||||
"linux-amd64": {
|
||||
"url": "https://github.com/lukaszraczylo/claude-mnemonic/releases/download/${VERSION_TAG}/claude-mnemonic_${VERSION}_linux_amd64.tar.gz",
|
||||
"format": "tar.gz",
|
||||
"sha256": "${SHA_LINUX_AMD64}"
|
||||
},
|
||||
"windows-amd64": {
|
||||
"url": "https://github.com/lukaszraczylo/claude-mnemonic/releases/download/${VERSION_TAG}/claude-mnemonic_${VERSION}_windows_amd64.zip",
|
||||
"format": "zip",
|
||||
"sha256": "${SHA_WINDOWS_AMD64}"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
"linux-amd64": {
|
||||
"url": "https://github.com/lukaszraczylo/claude-mnemonic/releases/download/${VERSION_TAG}/claude-mnemonic_${VERSION}_linux_amd64.tar.gz",
|
||||
"format": "tar.gz",
|
||||
"sha256": "${SHA_LINUX_AMD64}"
|
||||
},
|
||||
"windows-amd64": {
|
||||
"url": "https://github.com/lukaszraczylo/claude-mnemonic/releases/download/${VERSION_TAG}/claude-mnemonic_${VERSION}_windows_amd64.zip",
|
||||
"format": "zip",
|
||||
"sha256": "${SHA_WINDOWS_AMD64}"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
EOF
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
EOF
|
||||
|
||||
echo "Generated marketplace.json:"
|
||||
cat marketplace.json
|
||||
echo "Generated marketplace.json:"
|
||||
cat marketplace.json
|
||||
|
||||
- name: Commit and push
|
||||
run: |
|
||||
git config user.name "github-actions[bot]"
|
||||
git config user.email "github-actions[bot]@users.noreply.github.com"
|
||||
git add marketplace.json
|
||||
git diff --staged --quiet || git commit -m "chore: update marketplace for ${{ needs.release.outputs.version_tag }}"
|
||||
git push
|
||||
- name: Commit and push
|
||||
run: |
|
||||
git config user.name "github-actions[bot]"
|
||||
git config user.email "github-actions[bot]@users.noreply.github.com"
|
||||
git add marketplace.json
|
||||
git diff --staged --quiet || git commit -m "chore: update marketplace for ${{ needs.release.outputs.version_tag }}"
|
||||
git push
|
||||
|
||||
@@ -1,35 +0,0 @@
|
||||
linters-settings:
|
||||
govet:
|
||||
enable:
|
||||
- fieldalignment
|
||||
errcheck:
|
||||
# Ignore error checks in test files for common test helpers
|
||||
exclude-functions:
|
||||
- (io.Closer).Close
|
||||
- (*encoding/json.Encoder).Encode
|
||||
- (io.Writer).Write
|
||||
|
||||
linters:
|
||||
enable:
|
||||
- errcheck
|
||||
- gosec
|
||||
- govet
|
||||
- gofmt
|
||||
- staticcheck
|
||||
- unused
|
||||
- ineffassign
|
||||
- typecheck
|
||||
|
||||
issues:
|
||||
exclude-dirs:
|
||||
- vendor
|
||||
# Exclude some linters from running on test files
|
||||
exclude-rules:
|
||||
- path: _test\.go
|
||||
linters:
|
||||
- errcheck
|
||||
- gosec
|
||||
|
||||
run:
|
||||
timeout: 5m
|
||||
tests: true
|
||||
@@ -221,35 +221,6 @@ builds:
|
||||
- goos: windows
|
||||
goarch: arm64
|
||||
|
||||
# Hook: statusline
|
||||
- id: hook-statusline
|
||||
main: ./cmd/hooks/statusline
|
||||
binary: hooks/statusline
|
||||
env:
|
||||
- CGO_ENABLED=1
|
||||
tags:
|
||||
- fts5
|
||||
flags:
|
||||
- -trimpath
|
||||
ldflags:
|
||||
- -s -w
|
||||
- -X main.Version={{.Version}}
|
||||
- -X github.com/lukaszraczylo/claude-mnemonic/pkg/hooks.Version={{.Version}}
|
||||
goos:
|
||||
- darwin
|
||||
- linux
|
||||
- windows
|
||||
goarch:
|
||||
- amd64
|
||||
- arm64
|
||||
ignore:
|
||||
- goos: darwin
|
||||
goarch: amd64
|
||||
- goos: linux
|
||||
goarch: arm64
|
||||
- goos: windows
|
||||
goarch: arm64
|
||||
|
||||
archives:
|
||||
- id: default
|
||||
formats:
|
||||
@@ -266,9 +237,6 @@ archives:
|
||||
- src: plugin/hooks/hooks.json
|
||||
dst: hooks
|
||||
strip_parent: true
|
||||
- src: plugin/commands/*
|
||||
dst: commands
|
||||
strip_parent: true
|
||||
format_overrides:
|
||||
- goos: windows
|
||||
formats:
|
||||
|
||||
@@ -138,13 +138,10 @@ install: build stop-worker
|
||||
@# Install to marketplaces directory (for direct installs)
|
||||
@mkdir -p $(HOME)/.claude/plugins/marketplaces/claude-mnemonic/hooks
|
||||
@mkdir -p $(HOME)/.claude/plugins/marketplaces/claude-mnemonic/.claude-plugin
|
||||
@mkdir -p $(HOME)/.claude/plugins/marketplaces/claude-mnemonic/commands
|
||||
cp $(BUILD_DIR)/worker $(HOME)/.claude/plugins/marketplaces/claude-mnemonic/
|
||||
cp $(BUILD_DIR)/mcp-server $(HOME)/.claude/plugins/marketplaces/claude-mnemonic/
|
||||
cp $(BUILD_DIR)/hooks/* $(HOME)/.claude/plugins/marketplaces/claude-mnemonic/hooks/
|
||||
cp $(PLUGIN_DIR)/hooks/hooks.json $(HOME)/.claude/plugins/marketplaces/claude-mnemonic/hooks/
|
||||
@# Copy slash commands if they exist
|
||||
@if [ -d "$(PLUGIN_DIR)/commands" ]; then cp -r $(PLUGIN_DIR)/commands/* $(HOME)/.claude/plugins/marketplaces/claude-mnemonic/commands/ 2>/dev/null || true; fi
|
||||
@# Update plugin.json and marketplace.json with current version to prevent stale version directories
|
||||
@sed 's/"version": "[^"]*"/"version": "$(VERSION)"/g' $(PLUGIN_DIR)/.claude-plugin/plugin.json > $(HOME)/.claude/plugins/marketplaces/claude-mnemonic/.claude-plugin/plugin.json
|
||||
@sed 's/"version": "[^"]*"/"version": "$(VERSION)"/g' $(PLUGIN_DIR)/.claude-plugin/marketplace.json > $(HOME)/.claude/plugins/marketplaces/claude-mnemonic/.claude-plugin/marketplace.json
|
||||
@@ -160,15 +157,14 @@ uninstall: stop-worker
|
||||
rm -rf $(HOME)/.claude/plugins/marketplaces/claude-mnemonic
|
||||
@echo "Uninstallation complete!"
|
||||
|
||||
# Run tests (with FTS5 support)
|
||||
# Run tests
|
||||
test: setup-libs
|
||||
go test $(BUILD_TAGS) -v -race ./...
|
||||
go test -v -race ./...
|
||||
|
||||
# Run tests with coverage (with FTS5 support)
|
||||
test-coverage: setup-libs
|
||||
go test $(BUILD_TAGS) -v -race -coverprofile=coverage.out ./...
|
||||
# Run tests with coverage
|
||||
test-coverage:
|
||||
go test -v -race -coverprofile=coverage.out ./...
|
||||
go tool cover -html=coverage.out -o coverage.html
|
||||
@go tool cover -func=coverage.out | tail -1
|
||||
|
||||
# Run benchmarks
|
||||
bench:
|
||||
|
||||
@@ -12,30 +12,6 @@ Claude Code forgets everything when your session ends. Claude Mnemonic fixes tha
|
||||
|
||||
It captures what Claude learns during your coding sessions - bug fixes, architecture decisions, patterns that work - and brings that knowledge back in future conversations. No more re-explaining your codebase.
|
||||
|
||||

|
||||
|
||||
## What's New in v0.7
|
||||
|
||||
- **Two-Stage Retrieval** - Cross-encoder reranking for dramatically improved search relevance
|
||||
- **Knowledge Graph** - Automatic relationship detection between observations with visual graph in dashboard
|
||||

|
||||
- **Pattern Detection** - Identifies recurring patterns across sessions and projects
|
||||
- **Importance Scoring** - Time decay and voting system to surface the most valuable memories
|
||||
- **Query Expansion** - Reformulates searches to find semantically related content
|
||||
- **Conflict Detection** - Identifies and resolves contradictory observations
|
||||
- **Observation Lifecycle** - Memories can be superseded when better information arrives
|
||||
|
||||
<details>
|
||||
<summary>Previous: v0.6</summary>
|
||||
|
||||
- **Auto-Updates** - Automatically stays up-to-date with the latest version
|
||||
- **Slash Command: `/restart`** - Restart the worker directly from Claude Code
|
||||
- **Local Embeddings** - All semantic search runs locally via ONNX Runtime (no external API calls)
|
||||
- **Async Queue Processing** - Non-blocking observation capture for faster sessions
|
||||
- **Smarter Storage** - Filters out system/agent summaries to keep knowledge relevant
|
||||
- **Improved Reliability** - Better handling of connectivity issues and dead connections
|
||||
</details>
|
||||
|
||||
## Requirements
|
||||
|
||||
| Dependency | Required | Purpose |
|
||||
@@ -97,11 +73,9 @@ cosign verify-blob \
|
||||
| **Persistent Memory** | Observations survive across sessions and restarts |
|
||||
| **Project Isolation** | Each project has its own knowledge base |
|
||||
| **Global Patterns** | Best practices are shared across all projects |
|
||||
| **Semantic Search** | Find relevant context with natural language (local embeddings) |
|
||||
| **Semantic Search** | Find relevant context with natural language |
|
||||
| **Live Statusline** | Real-time metrics in Claude Code: `[mnemonic] ● served:42 | project:28 memories` |
|
||||
| **Web Dashboard** | Browse and manage memories at `localhost:37777` |
|
||||
| **Auto-Updates** | Automatically downloads and applies new versions |
|
||||
| **Slash Commands** | Control the worker directly from Claude Code |
|
||||
|
||||
### How knowledge flows
|
||||
|
||||
@@ -125,39 +99,16 @@ Config file: `~/.claude-mnemonic/settings.json`
|
||||
{
|
||||
"CLAUDE_MNEMONIC_WORKER_PORT": 37777,
|
||||
"CLAUDE_MNEMONIC_CONTEXT_OBSERVATIONS": 100,
|
||||
"CLAUDE_MNEMONIC_CONTEXT_FULL_COUNT": 25,
|
||||
"CLAUDE_MNEMONIC_RERANKING_ENABLED": true
|
||||
"CLAUDE_MNEMONIC_CONTEXT_FULL_COUNT": 25
|
||||
}
|
||||
```
|
||||
|
||||
### Core Settings
|
||||
|
||||
| Variable | Default | What it does |
|
||||
|----------|---------|--------------|
|
||||
| `WORKER_PORT` | `37777` | Dashboard & API port |
|
||||
| `CONTEXT_OBSERVATIONS` | `100` | Max memories per session |
|
||||
| `CONTEXT_FULL_COUNT` | `25` | Full detail memories (rest are condensed) |
|
||||
| `CONTEXT_FULL_COUNT` | `25` | Full detail memories (rest are condensed to title only) |
|
||||
| `CONTEXT_SESSION_COUNT` | `10` | Recent sessions to reference |
|
||||
| `CONTEXT_RELEVANCE_THRESHOLD` | `0.3` | Minimum similarity score (0.0-1.0) for inclusion |
|
||||
| `CONTEXT_MAX_PROMPT_RESULTS` | `10` | Max results per prompt search |
|
||||
|
||||
### Reranking Settings (Two-Stage Retrieval)
|
||||
|
||||
| Variable | Default | What it does |
|
||||
|----------|---------|--------------|
|
||||
| `RERANKING_ENABLED` | `true` | Enable cross-encoder reranking |
|
||||
| `RERANKING_CANDIDATES` | `100` | Candidates to retrieve before reranking |
|
||||
| `RERANKING_RESULTS` | `10` | Final results after reranking |
|
||||
| `RERANKING_ALPHA` | `0.7` | Score blend: alpha×rerank + (1-alpha)×original |
|
||||
| `RERANKING_PURE_MODE` | `false` | Use pure cross-encoder scores only |
|
||||
|
||||
### Embedding Settings
|
||||
|
||||
| Variable | Default | What it does |
|
||||
|----------|---------|--------------|
|
||||
| `EMBEDDING_MODEL` | `bge-v1.5` | Embedding model for semantic search |
|
||||
|
||||
All variables are prefixed with `CLAUDE_MNEMONIC_` in the config file.
|
||||
|
||||
## Project vs Global scope
|
||||
|
||||
@@ -180,25 +131,6 @@ These search tools are available via MCP:
|
||||
- `changes` - find code modifications
|
||||
- `how_it_works` - system understanding queries
|
||||
|
||||
## Slash Commands
|
||||
|
||||
Available commands within Claude Code:
|
||||
|
||||
| Command | Description |
|
||||
|---------|-------------|
|
||||
| `/restart` | Restart the worker process when experiencing issues |
|
||||
|
||||
## Auto-Updates
|
||||
|
||||
Claude Mnemonic automatically checks for updates and applies them. Updates are downloaded in the background and applied on restart.
|
||||
|
||||
- Automatic update checks on startup
|
||||
- Background downloads (up to 250MB)
|
||||
- Seamless restart after update
|
||||
- Manual trigger: `curl -X POST http://127.0.0.1:37777/api/update/apply`
|
||||
|
||||
Check update status: `curl http://127.0.0.1:37777/api/update/status`
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
**Worker won't start?**
|
||||
@@ -212,20 +144,6 @@ cat /tmp/claude-mnemonic-worker.log # view logs
|
||||
rm -f ~/.claude-mnemonic/*.db-wal ~/.claude-mnemonic/*.db-shm
|
||||
```
|
||||
|
||||
**Worker unresponsive?**
|
||||
```bash
|
||||
# Restart via API
|
||||
curl -X POST http://127.0.0.1:37777/api/restart
|
||||
|
||||
# Or use the slash command in Claude Code
|
||||
/restart
|
||||
```
|
||||
|
||||
**Check health status:**
|
||||
```bash
|
||||
curl http://127.0.0.1:37777/api/selfcheck
|
||||
```
|
||||
|
||||
## Uninstall
|
||||
|
||||
```bash
|
||||
@@ -240,11 +158,10 @@ curl -sSL https://raw.githubusercontent.com/lukaszraczylo/claude-mnemonic/main/s
|
||||
|
||||
- **SQLite + FTS5** - Full-text search for exact matches
|
||||
- **sqlite-vec** - Vector database embedded in SQLite
|
||||
- **Two-Stage Retrieval** - Bi-encoder (embedding) + cross-encoder (reranking) for high accuracy
|
||||
- **Local Models** - all-MiniLM-L6-v2 for embeddings, BGE reranker for relevance scoring
|
||||
- **all-MiniLM-L6-v2** - Local embedding model (384 dimensions) via ONNX Runtime
|
||||
- **Go** - Single binary, no external dependencies
|
||||
|
||||
Everything runs locally. No Python. No external vector database. No API calls.
|
||||
Everything runs locally. No Python. No external vector database. No API calls for embeddings.
|
||||
|
||||
## Platform support
|
||||
|
||||
|
||||
@@ -17,50 +17,11 @@ type Input struct {
|
||||
ToolUseID string `json:"tool_use_id"`
|
||||
}
|
||||
|
||||
// skipTools lists tools that never produce useful observations.
|
||||
// Skip the HTTP call entirely for these to reduce overhead during heavy tool usage.
|
||||
var skipTools = map[string]bool{
|
||||
// Internal tracking tools (but NOT TodoWrite - it captures planned work)
|
||||
"Task": true,
|
||||
"TaskOutput": true,
|
||||
|
||||
// File discovery tools (just listings, no insights)
|
||||
"Glob": true,
|
||||
"ListDir": true,
|
||||
"LS": true,
|
||||
"KillShell": true,
|
||||
|
||||
// Question/interaction tools (no code insights)
|
||||
"AskUserQuestion": true,
|
||||
|
||||
// Plan mode tools (planning, not execution)
|
||||
"EnterPlanMode": true,
|
||||
"ExitPlanMode": true,
|
||||
|
||||
// Skill/command execution (meta-operations)
|
||||
"Skill": true,
|
||||
"SlashCommand": true,
|
||||
|
||||
// Read is high-volume and rarely produces insights worth the overhead
|
||||
// The processor would skip most reads anyway after filtering
|
||||
"Read": true,
|
||||
|
||||
// Search tools are for finding, not modifying
|
||||
"Grep": true,
|
||||
"WebSearch": true,
|
||||
}
|
||||
|
||||
func main() {
|
||||
hooks.RunHook("PostToolUse", handlePostToolUse)
|
||||
}
|
||||
|
||||
func handlePostToolUse(ctx *hooks.HookContext, input *Input) (string, error) {
|
||||
// Skip HTTP call entirely for tools that never produce useful observations.
|
||||
// This significantly reduces overhead during heavy tool usage.
|
||||
if skipTools[input.ToolName] {
|
||||
return "", nil
|
||||
}
|
||||
|
||||
fmt.Fprintf(os.Stderr, "[post-tool-use] %s\n", input.ToolName)
|
||||
|
||||
// Send observation to worker
|
||||
|
||||
@@ -23,12 +23,12 @@ type Input struct {
|
||||
|
||||
// Observation represents an observation from the API.
|
||||
type Observation struct {
|
||||
ID int64 `json:"id"`
|
||||
Type string `json:"type"`
|
||||
Title string `json:"title"`
|
||||
Subtitle string `json:"subtitle"`
|
||||
Narrative string `json:"narrative"`
|
||||
Facts []string `json:"facts"`
|
||||
ID int64 `json:"id"`
|
||||
}
|
||||
|
||||
func main() {
|
||||
@@ -46,8 +46,8 @@ func main() {
|
||||
}
|
||||
|
||||
var input Input
|
||||
if unmarshalErr := json.Unmarshal(inputData, &input); unmarshalErr != nil {
|
||||
hooks.WriteError("SessionStart", unmarshalErr)
|
||||
if err := json.Unmarshal(inputData, &input); err != nil {
|
||||
hooks.WriteError("SessionStart", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
|
||||
@@ -44,21 +44,21 @@ type StatusInput struct {
|
||||
|
||||
// WorkerStats is the response from the worker's /api/stats endpoint.
|
||||
type WorkerStats struct {
|
||||
Uptime string `json:"uptime"`
|
||||
Project string `json:"project,omitempty"`
|
||||
Retrieval struct {
|
||||
Uptime string `json:"uptime"`
|
||||
ActiveSessions int `json:"activeSessions"`
|
||||
QueueDepth int `json:"queueDepth"`
|
||||
IsProcessing bool `json:"isProcessing"`
|
||||
ConnectedClients int `json:"connectedClients"`
|
||||
SessionsToday int `json:"sessionsToday"`
|
||||
Ready bool `json:"ready"`
|
||||
Project string `json:"project,omitempty"`
|
||||
ProjectObservations int `json:"projectObservations,omitempty"`
|
||||
Retrieval struct {
|
||||
TotalRequests int64 `json:"TotalRequests"`
|
||||
ObservationsServed int64 `json:"ObservationsServed"`
|
||||
SearchRequests int64 `json:"SearchRequests"`
|
||||
ContextInjections int64 `json:"ContextInjections"`
|
||||
} `json:"retrieval"`
|
||||
ActiveSessions int `json:"activeSessions"`
|
||||
QueueDepth int `json:"queueDepth"`
|
||||
ConnectedClients int `json:"connectedClients"`
|
||||
SessionsToday int `json:"sessionsToday"`
|
||||
ProjectObservations int `json:"projectObservations,omitempty"`
|
||||
IsProcessing bool `json:"isProcessing"`
|
||||
Ready bool `json:"ready"`
|
||||
}
|
||||
|
||||
// ANSI color codes
|
||||
|
||||
+5
-22
@@ -14,17 +14,17 @@ import (
|
||||
// Input is the hook input from Claude Code.
|
||||
type Input struct {
|
||||
hooks.BaseInput
|
||||
TranscriptPath string `json:"transcript_path"`
|
||||
StopHookActive bool `json:"stop_hook_active"`
|
||||
TranscriptPath string `json:"transcript_path"`
|
||||
}
|
||||
|
||||
// TranscriptMessage represents a message in the transcript JSONL file.
|
||||
type TranscriptMessage struct {
|
||||
Type string `json:"type"`
|
||||
Message struct {
|
||||
Content any `json:"content"`
|
||||
Role string `json:"role"`
|
||||
Content any `json:"content"` // Can be string or array
|
||||
} `json:"message"`
|
||||
Type string `json:"type"` // Can be string or array
|
||||
}
|
||||
|
||||
// extractTextContent extracts text content from message content (handles both string and array formats).
|
||||
@@ -75,16 +75,13 @@ func parseTranscript(path string) (lastUser, lastAssistant string) {
|
||||
continue
|
||||
}
|
||||
|
||||
// Transcript entries have type: "user" or "assistant" (not "message")
|
||||
// Check if this is a user/assistant message with content
|
||||
if msg.Type == "user" || msg.Type == "assistant" {
|
||||
if msg.Type == "message" {
|
||||
text := extractTextContent(msg.Message.Content)
|
||||
if text == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
// Use the outer type field for role (message.role may differ)
|
||||
switch msg.Type {
|
||||
switch msg.Message.Role {
|
||||
case "user":
|
||||
lastUser = text
|
||||
case "assistant":
|
||||
@@ -101,9 +98,6 @@ func main() {
|
||||
}
|
||||
|
||||
func handleStop(ctx *hooks.HookContext, input *Input) (string, error) {
|
||||
// Debug: dump raw input
|
||||
fmt.Fprintf(os.Stderr, "[stop] Raw input: %s\n", string(ctx.RawInput))
|
||||
|
||||
// Find session
|
||||
result, err := hooks.GET(ctx.Port, fmt.Sprintf("/api/sessions?claudeSessionId=%s", ctx.SessionID))
|
||||
if err != nil || result == nil {
|
||||
@@ -122,17 +116,6 @@ func handleStop(ctx *hooks.HookContext, input *Input) (string, error) {
|
||||
lastUser, lastAssistant = parseTranscript(input.TranscriptPath)
|
||||
}
|
||||
|
||||
// Debug: log what we extracted
|
||||
fmt.Fprintf(os.Stderr, "[stop] Transcript path: %s\n", input.TranscriptPath)
|
||||
fmt.Fprintf(os.Stderr, "[stop] Last user message length: %d\n", len(lastUser))
|
||||
fmt.Fprintf(os.Stderr, "[stop] Last assistant message length: %d\n", len(lastAssistant))
|
||||
if len(lastAssistant) > 0 {
|
||||
preview := lastAssistant
|
||||
if len(preview) > 300 {
|
||||
preview = preview[:300] + "..."
|
||||
}
|
||||
fmt.Fprintf(os.Stderr, "[stop] Last assistant preview: %s\n", preview)
|
||||
}
|
||||
fmt.Fprintf(os.Stderr, "[stop] Requesting summary for session %d (transcript: %v)\n", int64(sessionID), input.TranscriptPath != "")
|
||||
|
||||
// Request summary with message context from transcript
|
||||
|
||||
@@ -35,8 +35,8 @@ func main() {
|
||||
}
|
||||
|
||||
var input Input
|
||||
if unmarshalErr := json.Unmarshal(inputData, &input); unmarshalErr != nil {
|
||||
hooks.WriteError("UserPromptSubmit", unmarshalErr)
|
||||
if err := json.Unmarshal(inputData, &input); err != nil {
|
||||
hooks.WriteError("UserPromptSubmit", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
@@ -61,8 +61,12 @@ func main() {
|
||||
|
||||
searchResult, _ := hooks.GET(port, searchURL)
|
||||
if observations, ok := searchResult["observations"].([]interface{}); ok && len(observations) > 0 {
|
||||
// Results are already filtered by relevance threshold and capped by max_results
|
||||
// from the server-side config (ContextRelevanceThreshold, ContextMaxPromptResults)
|
||||
// Limit to top 5 most relevant observations
|
||||
maxObs := 5
|
||||
if len(observations) < maxObs {
|
||||
maxObs = len(observations)
|
||||
}
|
||||
observations = observations[:maxObs]
|
||||
observationCount = len(observations)
|
||||
|
||||
// Build context from search results
|
||||
|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 252 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 123 KiB |
+11
-58
@@ -29,21 +29,6 @@
|
||||
subtitle="Stop re-explaining your codebase. Claude Mnemonic captures bug fixes, architecture decisions, and coding patterns - then brings them back exactly when you need them."
|
||||
/>
|
||||
|
||||
<!-- Dashboard Preview -->
|
||||
<section class="py-12 lg:py-16 px-4 sm:px-6 relative">
|
||||
<div class="max-w-6xl mx-auto">
|
||||
<div class="relative rounded-xl overflow-hidden border border-slate-700/50 shadow-2xl shadow-amber-500/5">
|
||||
<div class="absolute inset-0 bg-gradient-to-t from-slate-950 via-transparent to-transparent pointer-events-none z-10"></div>
|
||||
<img
|
||||
src="/claude-mnemonic.jpg"
|
||||
alt="Claude Mnemonic Dashboard"
|
||||
class="w-full h-auto"
|
||||
/>
|
||||
</div>
|
||||
<p class="text-center text-slate-500 text-sm mt-4">The dashboard at localhost:37777 - browse, search, and manage your memories. View graph stats, vector metrics, storage savings, and performance analytics.</p>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<!-- Problem Section -->
|
||||
<section class="py-20 lg:py-28 px-4 sm:px-6 relative">
|
||||
<div class="max-w-6xl mx-auto grid lg:grid-cols-2 gap-8 lg:gap-16 items-center">
|
||||
@@ -100,21 +85,6 @@
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<!-- Knowledge Graph Preview -->
|
||||
<section class="py-16 lg:py-20 px-4 sm:px-6 relative">
|
||||
<div class="max-w-5xl mx-auto">
|
||||
<SectionHeader title="See how knowledge connects" subtitle="The knowledge graph reveals relationships between your memories automatically" />
|
||||
<div class="relative rounded-xl overflow-hidden border border-slate-700/50 shadow-2xl shadow-purple-500/5">
|
||||
<img
|
||||
src="/observation-relation-graph.jpg"
|
||||
alt="Knowledge Graph Visualization"
|
||||
class="w-full h-auto"
|
||||
/>
|
||||
</div>
|
||||
<p class="text-center text-slate-500 text-sm mt-4">Click any observation to explore its relationships - see what causes what, what fixes what, and how concepts evolve</p>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<!-- Before/After Section -->
|
||||
<section class="py-20 lg:py-28 px-4 sm:px-6">
|
||||
<div class="max-w-6xl mx-auto">
|
||||
@@ -304,7 +274,7 @@
|
||||
<section class="py-20 lg:py-28 px-4 sm:px-6">
|
||||
<div class="max-w-6xl mx-auto">
|
||||
<SectionHeader title="Under the hood" subtitle="Built with simplicity and performance in mind" />
|
||||
<div class="grid sm:grid-cols-2 lg:grid-cols-3 gap-4 sm:gap-6 text-center">
|
||||
<div class="grid sm:grid-cols-2 lg:grid-cols-4 gap-4 sm:gap-6 text-center">
|
||||
<div class="glass rounded-2xl p-6 sm:p-8 hover:border-amber-500/30 transition-colors">
|
||||
<div class="text-3xl sm:text-4xl font-bold text-amber-500 mb-2">Go</div>
|
||||
<p class="text-slate-400 text-xs sm:text-sm">Single binary. Fast startup, low memory. Zero runtime dependencies.</p>
|
||||
@@ -315,19 +285,11 @@
|
||||
</div>
|
||||
<div class="glass rounded-2xl p-6 sm:p-8 hover:border-amber-500/30 transition-colors">
|
||||
<div class="text-3xl sm:text-4xl font-bold text-amber-500 mb-2">sqlite-vec</div>
|
||||
<p class="text-slate-400 text-xs sm:text-sm">Hybrid vector storage with LEANN-inspired selective embeddings. 60-80% storage reduction.</p>
|
||||
<p class="text-slate-400 text-xs sm:text-sm">Embedded vector database. No external services required.</p>
|
||||
</div>
|
||||
<div class="glass rounded-2xl p-6 sm:p-8 hover:border-amber-500/30 transition-colors">
|
||||
<div class="text-3xl sm:text-4xl font-bold text-amber-500 mb-2">BGE</div>
|
||||
<p class="text-slate-400 text-xs sm:text-sm">Two-stage retrieval: bi-encoder embeddings + cross-encoder reranking for high accuracy.</p>
|
||||
</div>
|
||||
<div class="glass rounded-2xl p-6 sm:p-8 hover:border-amber-500/30 transition-colors">
|
||||
<div class="text-3xl sm:text-4xl font-bold text-amber-500 mb-2">Tree-sitter</div>
|
||||
<p class="text-slate-400 text-xs sm:text-sm">AST-aware code chunking respects function boundaries for Go, Python, and TypeScript.</p>
|
||||
</div>
|
||||
<div class="glass rounded-2xl p-6 sm:p-8 hover:border-amber-500/30 transition-colors">
|
||||
<div class="text-3xl sm:text-4xl font-bold text-amber-500 mb-2">CSR Graph</div>
|
||||
<p class="text-slate-400 text-xs sm:text-sm">Memory-efficient observation relationship graph with edge detection and hub identification.</p>
|
||||
<div class="text-3xl sm:text-4xl font-bold text-amber-500 mb-2">MiniLM</div>
|
||||
<p class="text-slate-400 text-xs sm:text-sm">Local embeddings via ONNX. "Fix auth" finds "JWT issue" automatically.</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
@@ -424,13 +386,10 @@ const activeTab = ref('macos')
|
||||
|
||||
const features = [
|
||||
{ icon: 'fas fa-brain', title: 'Learns as you work', description: 'Every bug fix, every architecture decision, every "aha moment" - captured automatically without breaking your flow.' },
|
||||
{ icon: 'fas fa-search', title: 'Two-stage retrieval', description: 'Cross-encoder reranking delivers highly relevant results. Finds what you need even with vague queries like "that auth thing".' },
|
||||
{ icon: 'fas fa-project-diagram', title: 'Graph-based search', description: 'LEANN Phase 2: Graph relationships between observations (file overlap, semantic similarity, temporal proximity) for smarter context retrieval.' },
|
||||
{ icon: 'fas fa-microchip', title: 'AST-aware chunking', description: 'Intelligent code splitting respects function boundaries. Go, Python, and TypeScript code is chunked at semantic boundaries, not arbitrary line counts.' },
|
||||
{ icon: 'fas fa-database', title: 'Hybrid vector storage', description: 'LEANN-inspired selective storage: frequently-accessed "hub" observations store embeddings, others recompute on-demand. 60-80% storage savings with <50ms latency.' },
|
||||
{ icon: 'fas fa-folder-tree', title: 'Project-aware context', description: 'Your React knowledge stays in React projects. Your Go patterns stay in Go projects. No context pollution.' },
|
||||
{ icon: 'fas fa-chart-line', title: 'Smart scoring', description: 'Importance decay, pattern detection, and conflict resolution ensure the most valuable memories surface first.' },
|
||||
{ icon: 'fas fa-gauge-high', title: 'Auto-tuning', description: 'Dynamic hub threshold adjustment based on query performance. Automatically balances storage efficiency with search latency for your workload.' },
|
||||
{ icon: 'fas fa-globe', title: 'Shared best practices', description: 'Security patterns, performance tips, and universal learnings automatically available across all your projects.' },
|
||||
{ icon: 'fas fa-search', title: 'Finds what matters', description: 'Semantic search finds relevant memories even when you don\'t remember the exact words. "That auth thing" just works.' },
|
||||
{ icon: 'fas fa-chart-line', title: 'Live statusline', description: 'Real-time metrics right in Claude Code: memories served, searches performed, and project memory count at a glance.' },
|
||||
{ icon: 'fas fa-lock', title: '100% private', description: 'Your code context never leaves your machine. No telemetry. No cloud sync. Your memories are yours.' },
|
||||
]
|
||||
|
||||
@@ -456,12 +415,8 @@ const installCommands = {
|
||||
const configOptions = [
|
||||
{ name: 'CLAUDE_MNEMONIC_WORKER_PORT', description: 'HTTP port for the worker service (default: 37777)', icon: 'fas fa-network-wired' },
|
||||
{ name: 'CLAUDE_MNEMONIC_CONTEXT_OBSERVATIONS', description: 'Maximum observations injected per session (default: 100)', icon: 'fas fa-layer-group' },
|
||||
{ name: 'CLAUDE_MNEMONIC_RERANKING_ENABLED', description: 'Enable cross-encoder reranking for improved search relevance (default: true)', icon: 'fas fa-sort-amount-down' },
|
||||
{ name: 'CLAUDE_MNEMONIC_CONTEXT_RELEVANCE_THRESHOLD', description: 'Minimum similarity score for inclusion, 0.0-1.0 (default: 0.3)', icon: 'fas fa-filter' },
|
||||
{ name: 'CLAUDE_MNEMONIC_VECTOR_STORAGE_STRATEGY', description: 'Storage strategy: "hub" (default), "always", or "on_demand"', icon: 'fas fa-database' },
|
||||
{ name: 'CLAUDE_MNEMONIC_GRAPH_ENABLED', description: 'Enable graph-based search with observation relationships (default: true)', icon: 'fas fa-project-diagram' },
|
||||
{ name: 'CLAUDE_MNEMONIC_GRAPH_MAX_HOPS', description: 'Maximum graph traversal depth for search expansion (default: 2)', icon: 'fas fa-route' },
|
||||
{ name: 'CLAUDE_MNEMONIC_GRAPH_REBUILD_INTERVAL_MIN', description: 'How often to rebuild the observation graph in minutes (default: 60)', icon: 'fas fa-clock' },
|
||||
{ name: 'CLAUDE_MNEMONIC_CONTEXT_FULL_COUNT', description: 'Observations with full narrative detail, rest are condensed (default: 25)', icon: 'fas fa-expand' },
|
||||
{ name: 'CLAUDE_MNEMONIC_MODEL', description: 'Model for processing observations (default: haiku)', icon: 'fas fa-microchip' },
|
||||
]
|
||||
|
||||
const requiredDeps = [
|
||||
@@ -472,11 +427,9 @@ const requiredDeps = [
|
||||
const faqs = [
|
||||
{ question: 'Will it confuse Claude with wrong context?', answer: 'No. Mnemonic uses project isolation and semantic relevance scoring. Only memories from the current project (or global best practices) are injected, and only when they\'re actually relevant to your prompt.' },
|
||||
{ question: 'What exactly gets saved?', answer: 'Bug fixes with context ("Fixed race condition by adding mutex"), architecture decisions ("Using repository pattern for data access"), conventions ("All API routes prefixed with /api/v1"), and learnings you want to preserve.' },
|
||||
{ question: 'How does hybrid vector storage work?', answer: 'LEANN-inspired selective storage: frequently-accessed "hub" observations (identified by access patterns and graph centrality) store embeddings. Infrequently-accessed observations recompute embeddings on-demand during search. This reduces storage by 60-80% with minimal latency impact (<50ms).' },
|
||||
{ question: 'Can I delete or edit memories?', answer: 'Yes. The web dashboard at localhost:37777 lets you browse, search, edit, and delete any memory. You can also view graph relationships, storage metrics, and performance analytics. You\'re always in control.' },
|
||||
{ question: 'Can I delete or edit memories?', answer: 'Yes. The web dashboard at localhost:37777 lets you browse, search, edit, and delete any memory. You\'re always in control.' },
|
||||
{ question: 'Does it work with my existing Claude Code setup?', answer: 'Yes. Mnemonic installs as a Claude Code plugin with hooks. Your existing workflows, settings, and shortcuts remain unchanged.' },
|
||||
{ question: 'What if I switch between projects frequently?', answer: 'That\'s the point. Each project has isolated memories. Switch from your Python ML project to your TypeScript app - context switches automatically.' },
|
||||
{ question: 'Is there a performance impact?', answer: 'Minimal. The Go worker is lightweight (typically under 30MB RAM). Hybrid storage and auto-tuning optimize for your workload. Context injection at session start takes milliseconds for most projects.' },
|
||||
{ question: 'What is AST-aware chunking?', answer: 'When processing code observations, Mnemonic uses Tree-sitter parsers to respect function and class boundaries instead of arbitrary line limits. Go, Python, and TypeScript code is chunked at semantic boundaries for better search accuracy.' },
|
||||
{ question: 'Is there a performance impact?', answer: 'Minimal. The Go worker is lightweight (typically under 30MB RAM). Context injection at session start takes milliseconds for most projects.' },
|
||||
]
|
||||
</script>
|
||||
|
||||
@@ -11,10 +11,9 @@ require (
|
||||
github.com/goccy/go-json v0.10.5
|
||||
github.com/mattn/go-sqlite3 v1.14.32
|
||||
github.com/rs/zerolog v1.34.0
|
||||
github.com/smacker/go-tree-sitter v0.0.0-20240827094217-dd81d9e9be82
|
||||
github.com/stretchr/testify v1.11.1
|
||||
github.com/sugarme/tokenizer v0.3.0
|
||||
github.com/yalue/onnxruntime_go v1.25.0
|
||||
github.com/yalue/onnxruntime_go v1.21.0
|
||||
)
|
||||
|
||||
require (
|
||||
@@ -27,9 +26,8 @@ require (
|
||||
github.com/pmezard/go-difflib v1.0.0 // indirect
|
||||
github.com/rivo/uniseg v0.4.7 // indirect
|
||||
github.com/schollz/progressbar/v2 v2.15.0 // indirect
|
||||
github.com/stretchr/objx v0.5.2 // indirect
|
||||
github.com/sugarme/regexpset v0.0.0-20200920021344-4d4ec8eaf93c // indirect
|
||||
golang.org/x/sys v0.39.0 // indirect
|
||||
golang.org/x/text v0.32.0 // indirect
|
||||
golang.org/x/text v0.25.0 // indirect
|
||||
gopkg.in/yaml.v3 v3.0.1 // indirect
|
||||
)
|
||||
|
||||
@@ -38,25 +38,21 @@ github.com/rs/zerolog v1.34.0 h1:k43nTLIwcTVQAncfCw4KZ2VY6ukYoZaBPNOE8txlOeY=
|
||||
github.com/rs/zerolog v1.34.0/go.mod h1:bJsvje4Z08ROH4Nhs5iH600c3IkWhwp44iRc54W6wYQ=
|
||||
github.com/schollz/progressbar/v2 v2.15.0 h1:dVzHQ8fHRmtPjD3K10jT3Qgn/+H+92jhPrhmxIJfDz8=
|
||||
github.com/schollz/progressbar/v2 v2.15.0/go.mod h1:UdPq3prGkfQ7MOzZKlDRpYKcFqEMczbD7YmbPgpzKMI=
|
||||
github.com/smacker/go-tree-sitter v0.0.0-20240827094217-dd81d9e9be82 h1:6C8qej6f1bStuePVkLSFxoU22XBS165D3klxlzRg8F4=
|
||||
github.com/smacker/go-tree-sitter v0.0.0-20240827094217-dd81d9e9be82/go.mod h1:xe4pgH49k4SsmkQq5OT8abwhWmnzkhpgnXeekbx2efw=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY=
|
||||
github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
|
||||
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
||||
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
|
||||
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
|
||||
github.com/sugarme/regexpset v0.0.0-20200920021344-4d4ec8eaf93c h1:pwb4kNSHb4K89ymCaN+5lPH/MwnfSVg4rzGDh4d+iy4=
|
||||
github.com/sugarme/regexpset v0.0.0-20200920021344-4d4ec8eaf93c/go.mod h1:2gwkXLWbDGUQWeL3RtpCmcY4mzCtU13kb9UsAg9xMaw=
|
||||
github.com/yalue/onnxruntime_go v1.25.0 h1:nlhVau1BpLZ/BYr+WpPZCJRD/WES0qo6dK7aKyyAs3g=
|
||||
github.com/yalue/onnxruntime_go v1.25.0/go.mod h1:b4X26A8pekNb1ACJ58wAXgNKeUCGEAQ9dmACut9Sm/4=
|
||||
github.com/yalue/onnxruntime_go v1.21.0 h1:DdtvfY7OP5gR8mwPDqAOAQckf+KcI30hPNJL8hQaYWI=
|
||||
github.com/yalue/onnxruntime_go v1.21.0/go.mod h1:b4X26A8pekNb1ACJ58wAXgNKeUCGEAQ9dmACut9Sm/4=
|
||||
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.39.0 h1:CvCKL8MeisomCi6qNZ+wbb0DN9E5AATixKsvNtMoMFk=
|
||||
golang.org/x/sys v0.39.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
|
||||
golang.org/x/text v0.32.0 h1:ZD01bjUt1FQ9WJ0ClOL5vxgxOI/sVCNgX1YtKwcY0mU=
|
||||
golang.org/x/text v0.32.0/go.mod h1:o/rUWzghvpD5TXrTIBuJU77MTaN0ljMWE47kxGJQ7jY=
|
||||
golang.org/x/text v0.25.0 h1:qVyWApTSYLk/drJRO5mDlNYskwQznZmkpV2c8q9zls4=
|
||||
golang.org/x/text v0.25.0/go.mod h1:WEdwpYrmk1qmdHvhkSTNPm3app7v4rsT8F2UD6+VHIA=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
|
||||
@@ -1,285 +0,0 @@
|
||||
// Package golang provides AST-aware chunking for Go source files.
|
||||
package golang
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"go/ast"
|
||||
"go/parser"
|
||||
"go/token"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"github.com/lukaszraczylo/claude-mnemonic/internal/chunking"
|
||||
)
|
||||
|
||||
// Chunker implements AST-aware chunking for Go files.
|
||||
type Chunker struct {
|
||||
options chunking.ChunkOptions
|
||||
}
|
||||
|
||||
// NewChunker creates a new Go chunker.
|
||||
func NewChunker(options chunking.ChunkOptions) *Chunker {
|
||||
return &Chunker{options: options}
|
||||
}
|
||||
|
||||
// Language returns the language this chunker supports.
|
||||
func (c *Chunker) Language() chunking.Language {
|
||||
return chunking.LanguageGo
|
||||
}
|
||||
|
||||
// SupportedExtensions returns the file extensions this chunker handles.
|
||||
func (c *Chunker) SupportedExtensions() []string {
|
||||
return []string{".go"}
|
||||
}
|
||||
|
||||
// Chunk parses a Go source file and returns semantic code chunks.
|
||||
func (c *Chunker) Chunk(ctx context.Context, filePath string) ([]chunking.Chunk, error) {
|
||||
// Read file content
|
||||
content, err := os.ReadFile(filePath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read file: %w", err)
|
||||
}
|
||||
|
||||
// Parse the Go file
|
||||
fset := token.NewFileSet()
|
||||
file, err := parser.ParseFile(fset, filePath, content, parser.ParseComments)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("parse Go file: %w", err)
|
||||
}
|
||||
|
||||
chunks := make([]chunking.Chunk, 0)
|
||||
sourceLines := strings.Split(string(content), "\n")
|
||||
|
||||
// Extract chunks from declarations
|
||||
for _, decl := range file.Decls {
|
||||
switch d := decl.(type) {
|
||||
case *ast.FuncDecl:
|
||||
chunk := c.extractFunction(fset, d, sourceLines, filePath)
|
||||
if chunk != nil {
|
||||
chunks = append(chunks, *chunk)
|
||||
}
|
||||
case *ast.GenDecl:
|
||||
extracted := c.extractGenDecl(fset, d, sourceLines, filePath)
|
||||
chunks = append(chunks, extracted...)
|
||||
}
|
||||
}
|
||||
|
||||
return chunks, nil
|
||||
}
|
||||
|
||||
// extractFunction extracts a function or method declaration as a chunk.
|
||||
func (c *Chunker) extractFunction(fset *token.FileSet, fn *ast.FuncDecl, sourceLines []string, filePath string) *chunking.Chunk {
|
||||
// Skip unexported if configured
|
||||
if !c.options.IncludePrivate && !fn.Name.IsExported() {
|
||||
return nil
|
||||
}
|
||||
|
||||
startPos := fset.Position(fn.Pos())
|
||||
endPos := fset.Position(fn.End())
|
||||
|
||||
chunk := &chunking.Chunk{
|
||||
FilePath: filePath,
|
||||
Language: chunking.LanguageGo,
|
||||
Name: fn.Name.Name,
|
||||
StartLine: startPos.Line,
|
||||
EndLine: endPos.Line,
|
||||
}
|
||||
|
||||
// Determine if this is a method or a function
|
||||
if fn.Recv != nil && len(fn.Recv.List) > 0 {
|
||||
chunk.Type = chunking.ChunkTypeMethod
|
||||
chunk.ParentName = c.extractReceiverType(fn.Recv)
|
||||
} else {
|
||||
chunk.Type = chunking.ChunkTypeFunction
|
||||
}
|
||||
|
||||
// Extract content
|
||||
chunk.Content = c.extractLines(sourceLines, startPos.Line, endPos.Line)
|
||||
|
||||
// Extract signature (function declaration without body)
|
||||
chunk.Signature = c.extractFunctionSignature(fn, fset, sourceLines)
|
||||
|
||||
// Extract doc comment
|
||||
if c.options.IncludeDocComments && fn.Doc != nil {
|
||||
chunk.DocComment = strings.TrimSpace(fn.Doc.Text())
|
||||
}
|
||||
|
||||
return chunk
|
||||
}
|
||||
|
||||
// extractGenDecl extracts general declarations (type, const, var).
|
||||
func (c *Chunker) extractGenDecl(fset *token.FileSet, gd *ast.GenDecl, sourceLines []string, filePath string) []chunking.Chunk {
|
||||
var chunks []chunking.Chunk
|
||||
|
||||
for _, spec := range gd.Specs {
|
||||
switch s := spec.(type) {
|
||||
case *ast.TypeSpec:
|
||||
chunk := c.extractTypeSpec(fset, gd, s, sourceLines, filePath)
|
||||
if chunk != nil {
|
||||
chunks = append(chunks, *chunk)
|
||||
}
|
||||
case *ast.ValueSpec:
|
||||
// Handle const and var declarations
|
||||
chunk := c.extractValueSpec(fset, gd, s, sourceLines, filePath)
|
||||
if chunk != nil {
|
||||
chunks = append(chunks, *chunk)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return chunks
|
||||
}
|
||||
|
||||
// extractTypeSpec extracts a type declaration (struct, interface, type alias).
|
||||
func (c *Chunker) extractTypeSpec(fset *token.FileSet, gd *ast.GenDecl, ts *ast.TypeSpec, sourceLines []string, filePath string) *chunking.Chunk {
|
||||
// Skip unexported if configured
|
||||
if !c.options.IncludePrivate && !ts.Name.IsExported() {
|
||||
return nil
|
||||
}
|
||||
|
||||
startPos := fset.Position(gd.Pos())
|
||||
endPos := fset.Position(gd.End())
|
||||
|
||||
chunk := &chunking.Chunk{
|
||||
FilePath: filePath,
|
||||
Language: chunking.LanguageGo,
|
||||
Name: ts.Name.Name,
|
||||
StartLine: startPos.Line,
|
||||
EndLine: endPos.Line,
|
||||
Content: c.extractLines(sourceLines, startPos.Line, endPos.Line),
|
||||
}
|
||||
|
||||
// Determine chunk type based on type expression
|
||||
switch ts.Type.(type) {
|
||||
case *ast.StructType:
|
||||
chunk.Type = chunking.ChunkTypeClass // Treat struct as class
|
||||
case *ast.InterfaceType:
|
||||
chunk.Type = chunking.ChunkTypeInterface
|
||||
default:
|
||||
chunk.Type = chunking.ChunkTypeType
|
||||
}
|
||||
|
||||
// Extract doc comment
|
||||
if c.options.IncludeDocComments && gd.Doc != nil {
|
||||
chunk.DocComment = strings.TrimSpace(gd.Doc.Text())
|
||||
}
|
||||
|
||||
return chunk
|
||||
}
|
||||
|
||||
// extractValueSpec extracts const or var declarations.
|
||||
func (c *Chunker) extractValueSpec(fset *token.FileSet, gd *ast.GenDecl, vs *ast.ValueSpec, sourceLines []string, filePath string) *chunking.Chunk {
|
||||
// Skip if all names are unexported and we're excluding private
|
||||
if !c.options.IncludePrivate {
|
||||
allUnexported := true
|
||||
for _, name := range vs.Names {
|
||||
if name.IsExported() {
|
||||
allUnexported = false
|
||||
break
|
||||
}
|
||||
}
|
||||
if allUnexported {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
startPos := fset.Position(gd.Pos())
|
||||
endPos := fset.Position(gd.End())
|
||||
|
||||
// Use first name as the chunk name, join multiple if present
|
||||
names := make([]string, len(vs.Names))
|
||||
for i, name := range vs.Names {
|
||||
names[i] = name.Name
|
||||
}
|
||||
|
||||
chunk := &chunking.Chunk{
|
||||
FilePath: filePath,
|
||||
Language: chunking.LanguageGo,
|
||||
Name: strings.Join(names, ", "),
|
||||
StartLine: startPos.Line,
|
||||
EndLine: endPos.Line,
|
||||
Content: c.extractLines(sourceLines, startPos.Line, endPos.Line),
|
||||
}
|
||||
|
||||
// Set type based on token
|
||||
if gd.Tok == token.CONST {
|
||||
chunk.Type = chunking.ChunkTypeConst
|
||||
} else {
|
||||
chunk.Type = chunking.ChunkTypeVar
|
||||
}
|
||||
|
||||
// Extract doc comment
|
||||
if c.options.IncludeDocComments && gd.Doc != nil {
|
||||
chunk.DocComment = strings.TrimSpace(gd.Doc.Text())
|
||||
}
|
||||
|
||||
return chunk
|
||||
}
|
||||
|
||||
// extractReceiverType extracts the receiver type name from a method.
|
||||
func (c *Chunker) extractReceiverType(recv *ast.FieldList) string {
|
||||
if len(recv.List) == 0 {
|
||||
return ""
|
||||
}
|
||||
|
||||
field := recv.List[0]
|
||||
switch t := field.Type.(type) {
|
||||
case *ast.Ident:
|
||||
return t.Name
|
||||
case *ast.StarExpr:
|
||||
if ident, ok := t.X.(*ast.Ident); ok {
|
||||
return ident.Name
|
||||
}
|
||||
}
|
||||
|
||||
return ""
|
||||
}
|
||||
|
||||
// extractFunctionSignature extracts the function signature without the body.
|
||||
func (c *Chunker) extractFunctionSignature(fn *ast.FuncDecl, fset *token.FileSet, sourceLines []string) string {
|
||||
if fn.Body == nil {
|
||||
// No body, return entire declaration
|
||||
startPos := fset.Position(fn.Pos())
|
||||
endPos := fset.Position(fn.End())
|
||||
return c.extractLines(sourceLines, startPos.Line, endPos.Line)
|
||||
}
|
||||
|
||||
// Extract from start of function to just before body
|
||||
startPos := fset.Position(fn.Pos())
|
||||
bodyPos := fset.Position(fn.Body.Pos())
|
||||
|
||||
// If body is on the same line, extract just that line up to the opening brace
|
||||
if startPos.Line == bodyPos.Line {
|
||||
line := sourceLines[startPos.Line-1]
|
||||
// Find the opening brace position
|
||||
if idx := strings.Index(line[startPos.Column-1:], "{"); idx >= 0 {
|
||||
return strings.TrimSpace(line[startPos.Column-1 : startPos.Column-1+idx])
|
||||
}
|
||||
return strings.TrimSpace(line[startPos.Column-1:])
|
||||
}
|
||||
|
||||
// Get lines from start to the line containing the opening brace
|
||||
sig := c.extractLines(sourceLines, startPos.Line, bodyPos.Line)
|
||||
// Remove the opening brace and anything after it
|
||||
if idx := strings.Index(sig, "{"); idx >= 0 {
|
||||
sig = sig[:idx]
|
||||
}
|
||||
return strings.TrimSpace(sig)
|
||||
}
|
||||
|
||||
// extractLines extracts a range of lines from source (1-indexed, inclusive).
|
||||
func (c *Chunker) extractLines(lines []string, start, end int) string {
|
||||
if start < 1 || end < start || start > len(lines) {
|
||||
return ""
|
||||
}
|
||||
|
||||
// Adjust for 0-indexed array (start and end are 1-indexed)
|
||||
startIdx := start - 1
|
||||
endIdx := end
|
||||
if endIdx > len(lines) {
|
||||
endIdx = len(lines)
|
||||
}
|
||||
|
||||
return strings.Join(lines[startIdx:endIdx], "\n")
|
||||
}
|
||||
@@ -1,214 +0,0 @@
|
||||
package golang
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/lukaszraczylo/claude-mnemonic/internal/chunking"
|
||||
)
|
||||
|
||||
func TestGoChunker_BasicFunctions(t *testing.T) {
|
||||
// Create temp test file
|
||||
tmpDir := t.TempDir()
|
||||
testFile := filepath.Join(tmpDir, "test.go")
|
||||
|
||||
testCode := `package main
|
||||
|
||||
import "fmt"
|
||||
|
||||
// Greet prints a greeting message
|
||||
func Greet(name string) {
|
||||
fmt.Printf("Hello, %s!\n", name)
|
||||
}
|
||||
|
||||
// Add adds two numbers
|
||||
func Add(a, b int) int {
|
||||
return a + b
|
||||
}
|
||||
|
||||
// unexported function should be included by default
|
||||
func helper() string {
|
||||
return "helper"
|
||||
}
|
||||
`
|
||||
|
||||
if err := os.WriteFile(testFile, []byte(testCode), 0600); err != nil {
|
||||
t.Fatalf("Failed to create test file: %v", err)
|
||||
}
|
||||
|
||||
// Create chunker with default options
|
||||
chunker := NewChunker(chunking.DefaultChunkOptions())
|
||||
|
||||
// Chunk the file
|
||||
chunks, err := chunker.Chunk(context.Background(), testFile)
|
||||
if err != nil {
|
||||
t.Fatalf("Chunk() failed: %v", err)
|
||||
}
|
||||
|
||||
// Verify we got all functions
|
||||
if len(chunks) != 3 {
|
||||
t.Errorf("Expected 3 chunks (Greet, Add, helper), got %d", len(chunks))
|
||||
}
|
||||
|
||||
// Verify chunk details
|
||||
expectedNames := map[string]bool{
|
||||
"Greet": false,
|
||||
"Add": false,
|
||||
"helper": false,
|
||||
}
|
||||
|
||||
for _, chunk := range chunks {
|
||||
if chunk.Type != chunking.ChunkTypeFunction {
|
||||
t.Errorf("Expected chunk type 'function', got '%s'", chunk.Type)
|
||||
}
|
||||
|
||||
if chunk.Language != chunking.LanguageGo {
|
||||
t.Errorf("Expected language 'go', got '%s'", chunk.Language)
|
||||
}
|
||||
|
||||
if _, ok := expectedNames[chunk.Name]; !ok {
|
||||
t.Errorf("Unexpected function name: %s", chunk.Name)
|
||||
} else {
|
||||
expectedNames[chunk.Name] = true
|
||||
}
|
||||
|
||||
// Verify content is non-empty
|
||||
if chunk.Content == "" {
|
||||
t.Errorf("Chunk %s has empty content", chunk.Name)
|
||||
}
|
||||
|
||||
// Verify signature is present for functions
|
||||
if chunk.Signature == "" {
|
||||
t.Errorf("Chunk %s has empty signature", chunk.Name)
|
||||
}
|
||||
}
|
||||
|
||||
// Verify all expected functions were found
|
||||
for name, found := range expectedNames {
|
||||
if !found {
|
||||
t.Errorf("Expected function %s not found", name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestGoChunker_StructsAndMethods(t *testing.T) {
|
||||
tmpDir := t.TempDir()
|
||||
testFile := filepath.Join(tmpDir, "test.go")
|
||||
|
||||
testCode := `package main
|
||||
|
||||
// User represents a user
|
||||
type User struct {
|
||||
ID int
|
||||
Name string
|
||||
}
|
||||
|
||||
// GetName returns the user's name
|
||||
func (u *User) GetName() string {
|
||||
return u.Name
|
||||
}
|
||||
|
||||
// SetName sets the user's name
|
||||
func (u *User) SetName(name string) {
|
||||
u.Name = name
|
||||
}
|
||||
`
|
||||
|
||||
if err := os.WriteFile(testFile, []byte(testCode), 0600); err != nil {
|
||||
t.Fatalf("Failed to create test file: %v", err)
|
||||
}
|
||||
|
||||
chunker := NewChunker(chunking.DefaultChunkOptions())
|
||||
chunks, err := chunker.Chunk(context.Background(), testFile)
|
||||
if err != nil {
|
||||
t.Fatalf("Chunk() failed: %v", err)
|
||||
}
|
||||
|
||||
// Should have 1 struct + 2 methods = 3 chunks
|
||||
if len(chunks) != 3 {
|
||||
t.Errorf("Expected 3 chunks (User struct, GetName, SetName), got %d", len(chunks))
|
||||
}
|
||||
|
||||
// Find the struct and methods
|
||||
var structChunk, getNameChunk, setNameChunk *chunking.Chunk
|
||||
for i := range chunks {
|
||||
switch chunks[i].Name {
|
||||
case "User":
|
||||
structChunk = &chunks[i]
|
||||
case "GetName":
|
||||
getNameChunk = &chunks[i]
|
||||
case "SetName":
|
||||
setNameChunk = &chunks[i]
|
||||
}
|
||||
}
|
||||
|
||||
// Verify struct
|
||||
if structChunk == nil {
|
||||
t.Fatal("User struct not found")
|
||||
}
|
||||
if structChunk.Type != chunking.ChunkTypeClass {
|
||||
t.Errorf("Expected User to be ChunkTypeClass, got %s", structChunk.Type)
|
||||
}
|
||||
|
||||
// Verify methods
|
||||
if getNameChunk == nil {
|
||||
t.Fatal("GetName method not found")
|
||||
}
|
||||
if getNameChunk.Type != chunking.ChunkTypeMethod {
|
||||
t.Errorf("Expected GetName to be ChunkTypeMethod, got %s", getNameChunk.Type)
|
||||
}
|
||||
if getNameChunk.ParentName != "User" {
|
||||
t.Errorf("Expected GetName parent to be 'User', got '%s'", getNameChunk.ParentName)
|
||||
}
|
||||
|
||||
if setNameChunk == nil {
|
||||
t.Fatal("SetName method not found")
|
||||
}
|
||||
if setNameChunk.Type != chunking.ChunkTypeMethod {
|
||||
t.Errorf("Expected SetName to be ChunkTypeMethod, got %s", setNameChunk.Type)
|
||||
}
|
||||
if setNameChunk.ParentName != "User" {
|
||||
t.Errorf("Expected SetName parent to be 'User', got '%s'", setNameChunk.ParentName)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGoChunker_DocComments(t *testing.T) {
|
||||
tmpDir := t.TempDir()
|
||||
testFile := filepath.Join(tmpDir, "test.go")
|
||||
|
||||
testCode := `package main
|
||||
|
||||
// Calculate performs a calculation.
|
||||
// It takes two integers and returns their sum.
|
||||
func Calculate(a, b int) int {
|
||||
return a + b
|
||||
}
|
||||
`
|
||||
|
||||
if err := os.WriteFile(testFile, []byte(testCode), 0600); err != nil {
|
||||
t.Fatalf("Failed to create test file: %v", err)
|
||||
}
|
||||
|
||||
chunker := NewChunker(chunking.DefaultChunkOptions())
|
||||
chunks, err := chunker.Chunk(context.Background(), testFile)
|
||||
if err != nil {
|
||||
t.Fatalf("Chunk() failed: %v", err)
|
||||
}
|
||||
|
||||
if len(chunks) != 1 {
|
||||
t.Fatalf("Expected 1 chunk, got %d", len(chunks))
|
||||
}
|
||||
|
||||
chunk := chunks[0]
|
||||
if chunk.DocComment == "" {
|
||||
t.Error("Expected doc comment to be present")
|
||||
}
|
||||
|
||||
// Doc comment should contain the comment text
|
||||
expectedComment := "Calculate performs a calculation.\nIt takes two integers and returns their sum."
|
||||
if chunk.DocComment != expectedComment {
|
||||
t.Errorf("Expected doc comment '%s', got '%s'", expectedComment, chunk.DocComment)
|
||||
}
|
||||
}
|
||||
@@ -1,106 +0,0 @@
|
||||
package chunking
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Manager dispatches files to appropriate language-specific chunkers.
|
||||
type Manager struct {
|
||||
chunkers map[string]Chunker // extension -> chunker
|
||||
options ChunkOptions
|
||||
}
|
||||
|
||||
// NewManager creates a new chunking manager with the given chunkers.
|
||||
func NewManager(chunkers []Chunker, options ChunkOptions) *Manager {
|
||||
m := &Manager{
|
||||
chunkers: make(map[string]Chunker),
|
||||
options: options,
|
||||
}
|
||||
|
||||
// Register chunkers by their supported extensions
|
||||
for _, chunker := range chunkers {
|
||||
for _, ext := range chunker.SupportedExtensions() {
|
||||
m.chunkers[ext] = chunker
|
||||
}
|
||||
}
|
||||
|
||||
return m
|
||||
}
|
||||
|
||||
// ChunkFile chunks a single file using the appropriate language chunker.
|
||||
// Returns an error if no chunker is found for the file extension.
|
||||
func (m *Manager) ChunkFile(ctx context.Context, filePath string) ([]Chunk, error) {
|
||||
ext := strings.ToLower(filepath.Ext(filePath))
|
||||
chunker, ok := m.chunkers[ext]
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("no chunker for extension %s", ext)
|
||||
}
|
||||
|
||||
chunks, err := chunker.Chunk(ctx, filePath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("chunk %s: %w", filePath, err)
|
||||
}
|
||||
|
||||
// Apply options-based filtering
|
||||
filtered := make([]Chunk, 0, len(chunks))
|
||||
for _, chunk := range chunks {
|
||||
// Filter by minimum lines
|
||||
if m.options.MinLines > 0 {
|
||||
lineCount := chunk.EndLine - chunk.StartLine + 1
|
||||
if lineCount < m.options.MinLines {
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
// Filter by maximum chunk size
|
||||
if m.options.MaxChunkSize > 0 && len(chunk.Content) > m.options.MaxChunkSize {
|
||||
// TODO: Consider splitting large chunks intelligently
|
||||
// For now, skip chunks that are too large
|
||||
continue
|
||||
}
|
||||
|
||||
filtered = append(filtered, chunk)
|
||||
}
|
||||
|
||||
return filtered, nil
|
||||
}
|
||||
|
||||
// ChunkFiles chunks multiple files in parallel.
|
||||
// Returns a map of file path to chunks, and any errors encountered.
|
||||
// Errors for individual files do not stop processing of other files.
|
||||
func (m *Manager) ChunkFiles(ctx context.Context, filePaths []string) (map[string][]Chunk, []error) {
|
||||
results := make(map[string][]Chunk)
|
||||
var errors []error
|
||||
|
||||
for _, filePath := range filePaths {
|
||||
chunks, err := m.ChunkFile(ctx, filePath)
|
||||
if err != nil {
|
||||
errors = append(errors, fmt.Errorf("%s: %w", filePath, err))
|
||||
continue
|
||||
}
|
||||
if len(chunks) > 0 {
|
||||
results[filePath] = chunks
|
||||
}
|
||||
}
|
||||
|
||||
return results, errors
|
||||
}
|
||||
|
||||
// SupportsFile checks if the manager can chunk the given file based on extension.
|
||||
func (m *Manager) SupportsFile(filePath string) bool {
|
||||
ext := strings.ToLower(filepath.Ext(filePath))
|
||||
_, ok := m.chunkers[ext]
|
||||
return ok
|
||||
}
|
||||
|
||||
// SupportedExtensions returns all file extensions supported by registered chunkers.
|
||||
func (m *Manager) SupportedExtensions() []string {
|
||||
exts := make([]string, 0, len(m.chunkers))
|
||||
for ext := range m.chunkers {
|
||||
exts = append(exts, ext)
|
||||
}
|
||||
return exts
|
||||
}
|
||||
@@ -1,162 +0,0 @@
|
||||
package chunking
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// mockChunker is a test chunker that returns dummy chunks
|
||||
type mockChunker struct{}
|
||||
|
||||
func (m *mockChunker) Chunk(ctx context.Context, filePath string) ([]Chunk, error) {
|
||||
// Just return an empty chunk for testing
|
||||
return []Chunk{
|
||||
{
|
||||
FilePath: filePath,
|
||||
Language: LanguageGo,
|
||||
Type: ChunkTypeFunction,
|
||||
Name: "TestFunc",
|
||||
StartLine: 1,
|
||||
EndLine: 1,
|
||||
Content: "test",
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (m *mockChunker) Language() Language {
|
||||
return LanguageGo
|
||||
}
|
||||
|
||||
func (m *mockChunker) SupportedExtensions() []string {
|
||||
return []string{".go", ".py", ".ts"}
|
||||
}
|
||||
|
||||
func TestManager_ChunkMultipleFiles(t *testing.T) {
|
||||
tmpDir := t.TempDir()
|
||||
|
||||
// Create a Go file
|
||||
goFile := filepath.Join(tmpDir, "test.go")
|
||||
goCode := `package main
|
||||
|
||||
func Hello() string {
|
||||
return "hello"
|
||||
}
|
||||
`
|
||||
if err := os.WriteFile(goFile, []byte(goCode), 0600); err != nil {
|
||||
t.Fatalf("Failed to create Go file: %v", err)
|
||||
}
|
||||
|
||||
// Create a Python file
|
||||
pyFile := filepath.Join(tmpDir, "test.py")
|
||||
pyCode := `def greet(name):
|
||||
return f"Hello, {name}!"
|
||||
|
||||
class User:
|
||||
def __init__(self, name):
|
||||
self.name = name
|
||||
`
|
||||
if err := os.WriteFile(pyFile, []byte(pyCode), 0600); err != nil {
|
||||
t.Fatalf("Failed to create Python file: %v", err)
|
||||
}
|
||||
|
||||
// Create a TypeScript file
|
||||
tsFile := filepath.Join(tmpDir, "test.ts")
|
||||
tsCode := `function add(a: number, b: number): number {
|
||||
return a + b;
|
||||
}
|
||||
|
||||
class Calculator {
|
||||
multiply(a: number, b: number): number {
|
||||
return a * b;
|
||||
}
|
||||
}
|
||||
`
|
||||
if err := os.WriteFile(tsFile, []byte(tsCode), 0600); err != nil {
|
||||
t.Fatalf("Failed to create TypeScript file: %v", err)
|
||||
}
|
||||
|
||||
// Create manager
|
||||
manager := NewManager([]Chunker{&mockChunker{}}, DefaultChunkOptions())
|
||||
|
||||
// Test SupportsFile
|
||||
if !manager.SupportsFile(goFile) {
|
||||
t.Error("Manager should support .go files")
|
||||
}
|
||||
if !manager.SupportsFile(pyFile) {
|
||||
t.Error("Manager should support .py files")
|
||||
}
|
||||
if !manager.SupportsFile(tsFile) {
|
||||
t.Error("Manager should support .ts files")
|
||||
}
|
||||
|
||||
unsupportedFile := filepath.Join(tmpDir, "test.txt")
|
||||
if manager.SupportsFile(unsupportedFile) {
|
||||
t.Error("Manager should not support .txt files")
|
||||
}
|
||||
|
||||
// Test ChunkFiles
|
||||
results, errs := manager.ChunkFiles(context.Background(), []string{goFile, pyFile, tsFile})
|
||||
if len(errs) > 0 {
|
||||
t.Errorf("ChunkFiles returned errors: %v", errs)
|
||||
}
|
||||
|
||||
if len(results) != 3 {
|
||||
t.Errorf("Expected results for 3 files, got %d", len(results))
|
||||
}
|
||||
|
||||
// Verify each file has chunks
|
||||
for _, file := range []string{goFile, pyFile, tsFile} {
|
||||
if chunks, ok := results[file]; !ok || len(chunks) == 0 {
|
||||
t.Errorf("No chunks found for file %s", file)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// mockChunkerWithExts is a test chunker with configurable extensions
|
||||
type mockChunkerWithExts struct {
|
||||
exts []string
|
||||
}
|
||||
|
||||
func (m *mockChunkerWithExts) Chunk(ctx context.Context, filePath string) ([]Chunk, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (m *mockChunkerWithExts) Language() Language {
|
||||
return LanguageGo
|
||||
}
|
||||
|
||||
func (m *mockChunkerWithExts) SupportedExtensions() []string {
|
||||
return m.exts
|
||||
}
|
||||
|
||||
func TestManager_SupportedExtensions(t *testing.T) {
|
||||
|
||||
// Create manager with mock chunkers
|
||||
manager := NewManager([]Chunker{
|
||||
&mockChunkerWithExts{exts: []string{".go"}},
|
||||
&mockChunkerWithExts{exts: []string{".py", ".pyw"}},
|
||||
}, DefaultChunkOptions())
|
||||
|
||||
exts := manager.SupportedExtensions()
|
||||
expectedExts := map[string]bool{
|
||||
".go": false,
|
||||
".py": false,
|
||||
".pyw": false,
|
||||
}
|
||||
|
||||
for _, ext := range exts {
|
||||
if _, ok := expectedExts[ext]; ok {
|
||||
expectedExts[ext] = true
|
||||
} else {
|
||||
t.Errorf("Unexpected extension: %s", ext)
|
||||
}
|
||||
}
|
||||
|
||||
for ext, found := range expectedExts {
|
||||
if !found {
|
||||
t.Errorf("Expected extension %s not found", ext)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,291 +0,0 @@
|
||||
// Package python provides AST-aware chunking for Python source files using tree-sitter.
|
||||
package python
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
sitter "github.com/smacker/go-tree-sitter"
|
||||
"github.com/smacker/go-tree-sitter/python"
|
||||
|
||||
"github.com/lukaszraczylo/claude-mnemonic/internal/chunking"
|
||||
)
|
||||
|
||||
// Chunker implements AST-aware chunking for Python files.
|
||||
type Chunker struct {
|
||||
parser *sitter.Parser
|
||||
options chunking.ChunkOptions
|
||||
}
|
||||
|
||||
// NewChunker creates a new Python chunker.
|
||||
func NewChunker(options chunking.ChunkOptions) *Chunker {
|
||||
parser := sitter.NewParser()
|
||||
parser.SetLanguage(python.GetLanguage())
|
||||
|
||||
return &Chunker{
|
||||
options: options,
|
||||
parser: parser,
|
||||
}
|
||||
}
|
||||
|
||||
// Language returns the language this chunker supports.
|
||||
func (c *Chunker) Language() chunking.Language {
|
||||
return chunking.LanguagePython
|
||||
}
|
||||
|
||||
// SupportedExtensions returns the file extensions this chunker handles.
|
||||
func (c *Chunker) SupportedExtensions() []string {
|
||||
return []string{".py"}
|
||||
}
|
||||
|
||||
// Chunk parses a Python source file and returns semantic code chunks.
|
||||
func (c *Chunker) Chunk(ctx context.Context, filePath string) ([]chunking.Chunk, error) {
|
||||
// Read file content
|
||||
content, err := os.ReadFile(filePath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read file: %w", err)
|
||||
}
|
||||
|
||||
// Parse the Python file
|
||||
tree, err := c.parser.ParseCtx(ctx, nil, content)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("parse Python file: %w", err)
|
||||
}
|
||||
defer tree.Close()
|
||||
|
||||
sourceLines := strings.Split(string(content), "\n")
|
||||
chunks := make([]chunking.Chunk, 0)
|
||||
|
||||
// Walk the AST and extract chunks
|
||||
c.walkNode(tree.RootNode(), content, sourceLines, filePath, "", &chunks)
|
||||
|
||||
return chunks, nil
|
||||
}
|
||||
|
||||
// walkNode recursively walks the tree-sitter AST and extracts chunks.
|
||||
func (c *Chunker) walkNode(node *sitter.Node, source []byte, sourceLines []string, filePath string, parentName string, chunks *[]chunking.Chunk) {
|
||||
nodeType := node.Type()
|
||||
|
||||
switch nodeType {
|
||||
case "function_definition":
|
||||
chunk := c.extractFunction(node, source, sourceLines, filePath, parentName)
|
||||
if chunk != nil {
|
||||
*chunks = append(*chunks, *chunk)
|
||||
}
|
||||
|
||||
case "class_definition":
|
||||
chunk := c.extractClass(node, source, sourceLines, filePath)
|
||||
if chunk != nil {
|
||||
*chunks = append(*chunks, *chunk)
|
||||
|
||||
// Walk class body to find methods
|
||||
for i := 0; i < int(node.ChildCount()); i++ {
|
||||
child := node.Child(i)
|
||||
if child.Type() == "block" {
|
||||
c.walkNode(child, source, sourceLines, filePath, chunk.Name, chunks)
|
||||
}
|
||||
}
|
||||
}
|
||||
return // Don't walk children again
|
||||
|
||||
case "block":
|
||||
// Walk statements in block
|
||||
for i := 0; i < int(node.ChildCount()); i++ {
|
||||
c.walkNode(node.Child(i), source, sourceLines, filePath, parentName, chunks)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// Walk all children
|
||||
for i := 0; i < int(node.ChildCount()); i++ {
|
||||
c.walkNode(node.Child(i), source, sourceLines, filePath, parentName, chunks)
|
||||
}
|
||||
}
|
||||
|
||||
// extractFunction extracts a function definition chunk.
|
||||
func (c *Chunker) extractFunction(node *sitter.Node, source []byte, sourceLines []string, filePath string, parentName string) *chunking.Chunk {
|
||||
// Find function name
|
||||
var nameNode *sitter.Node
|
||||
for i := 0; i < int(node.ChildCount()); i++ {
|
||||
child := node.Child(i)
|
||||
if child.Type() == "identifier" {
|
||||
nameNode = child
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if nameNode == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
name := nameNode.Content(source)
|
||||
|
||||
// Skip private functions if configured
|
||||
if !c.options.IncludePrivate && strings.HasPrefix(name, "_") && !strings.HasPrefix(name, "__") {
|
||||
return nil
|
||||
}
|
||||
|
||||
startLine := int(node.StartPoint().Row) + 1
|
||||
endLine := int(node.EndPoint().Row) + 1
|
||||
|
||||
chunk := &chunking.Chunk{
|
||||
FilePath: filePath,
|
||||
Language: chunking.LanguagePython,
|
||||
Name: name,
|
||||
ParentName: parentName,
|
||||
StartLine: startLine,
|
||||
EndLine: endLine,
|
||||
Content: c.extractLines(sourceLines, startLine, endLine),
|
||||
}
|
||||
|
||||
// Determine if this is a method or function
|
||||
if parentName != "" {
|
||||
chunk.Type = chunking.ChunkTypeMethod
|
||||
} else {
|
||||
chunk.Type = chunking.ChunkTypeFunction
|
||||
}
|
||||
|
||||
// Extract signature (def line)
|
||||
chunk.Signature = c.extractFunctionSignature(node, source, sourceLines)
|
||||
|
||||
// Extract docstring as doc comment
|
||||
if c.options.IncludeDocComments {
|
||||
chunk.DocComment = c.extractDocstring(node, source)
|
||||
}
|
||||
|
||||
return chunk
|
||||
}
|
||||
|
||||
// extractClass extracts a class definition chunk.
|
||||
func (c *Chunker) extractClass(node *sitter.Node, source []byte, sourceLines []string, filePath string) *chunking.Chunk {
|
||||
// Find class name
|
||||
var nameNode *sitter.Node
|
||||
for i := 0; i < int(node.ChildCount()); i++ {
|
||||
child := node.Child(i)
|
||||
if child.Type() == "identifier" {
|
||||
nameNode = child
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if nameNode == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
name := nameNode.Content(source)
|
||||
|
||||
// Skip private classes if configured
|
||||
if !c.options.IncludePrivate && strings.HasPrefix(name, "_") && !strings.HasPrefix(name, "__") {
|
||||
return nil
|
||||
}
|
||||
|
||||
startLine := int(node.StartPoint().Row) + 1
|
||||
endLine := int(node.EndPoint().Row) + 1
|
||||
|
||||
chunk := &chunking.Chunk{
|
||||
FilePath: filePath,
|
||||
Language: chunking.LanguagePython,
|
||||
Type: chunking.ChunkTypeClass,
|
||||
Name: name,
|
||||
StartLine: startLine,
|
||||
EndLine: endLine,
|
||||
Content: c.extractLines(sourceLines, startLine, endLine),
|
||||
}
|
||||
|
||||
// Extract class signature (class line)
|
||||
chunk.Signature = c.extractClassSignature(node, source, sourceLines)
|
||||
|
||||
// Extract docstring as doc comment
|
||||
if c.options.IncludeDocComments {
|
||||
chunk.DocComment = c.extractDocstring(node, source)
|
||||
}
|
||||
|
||||
return chunk
|
||||
}
|
||||
|
||||
// extractFunctionSignature extracts the function definition line.
|
||||
func (c *Chunker) extractFunctionSignature(node *sitter.Node, source []byte, sourceLines []string) string {
|
||||
startLine := int(node.StartPoint().Row) + 1
|
||||
|
||||
// Find the colon that ends the signature
|
||||
for i := 0; i < int(node.ChildCount()); i++ {
|
||||
child := node.Child(i)
|
||||
if child.Type() == ":" {
|
||||
endLine := int(child.EndPoint().Row) + 1
|
||||
return strings.TrimSpace(c.extractLines(sourceLines, startLine, endLine))
|
||||
}
|
||||
}
|
||||
|
||||
// Fallback: just return first line
|
||||
return strings.TrimSpace(c.extractLines(sourceLines, startLine, startLine))
|
||||
}
|
||||
|
||||
// extractClassSignature extracts the class definition line.
|
||||
func (c *Chunker) extractClassSignature(node *sitter.Node, source []byte, sourceLines []string) string {
|
||||
startLine := int(node.StartPoint().Row) + 1
|
||||
|
||||
// Find the colon that ends the signature
|
||||
for i := 0; i < int(node.ChildCount()); i++ {
|
||||
child := node.Child(i)
|
||||
if child.Type() == ":" {
|
||||
endLine := int(child.EndPoint().Row) + 1
|
||||
return strings.TrimSpace(c.extractLines(sourceLines, startLine, endLine))
|
||||
}
|
||||
}
|
||||
|
||||
// Fallback: just return first line
|
||||
return strings.TrimSpace(c.extractLines(sourceLines, startLine, startLine))
|
||||
}
|
||||
|
||||
// extractDocstring extracts the docstring from a function or class.
|
||||
func (c *Chunker) extractDocstring(node *sitter.Node, source []byte) string {
|
||||
// Find the block
|
||||
var blockNode *sitter.Node
|
||||
for i := 0; i < int(node.ChildCount()); i++ {
|
||||
child := node.Child(i)
|
||||
if child.Type() == "block" {
|
||||
blockNode = child
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if blockNode == nil {
|
||||
return ""
|
||||
}
|
||||
|
||||
// Check if first statement in block is a string (docstring)
|
||||
for i := 0; i < int(blockNode.ChildCount()); i++ {
|
||||
child := blockNode.Child(i)
|
||||
if child.Type() == "expression_statement" {
|
||||
// Check if it contains a string
|
||||
for j := 0; j < int(child.ChildCount()); j++ {
|
||||
grandchild := child.Child(j)
|
||||
if grandchild.Type() == "string" {
|
||||
docstring := grandchild.Content(source)
|
||||
// Remove quotes
|
||||
docstring = strings.Trim(docstring, `"'`)
|
||||
return strings.TrimSpace(docstring)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return ""
|
||||
}
|
||||
|
||||
// extractLines extracts a range of lines from source (1-indexed, inclusive).
|
||||
func (c *Chunker) extractLines(lines []string, start, end int) string {
|
||||
if start < 1 || end < start || start > len(lines) {
|
||||
return ""
|
||||
}
|
||||
|
||||
startIdx := start - 1
|
||||
endIdx := end
|
||||
if endIdx > len(lines) {
|
||||
endIdx = len(lines)
|
||||
}
|
||||
|
||||
return strings.Join(lines[startIdx:endIdx], "\n")
|
||||
}
|
||||
@@ -1,140 +0,0 @@
|
||||
// Package chunking provides AST-aware code chunking for semantic code search.
|
||||
// Chunks code files into logical units (functions, classes, methods) that preserve
|
||||
// semantic boundaries for better vector embedding and retrieval.
|
||||
package chunking
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// ChunkType represents the type of code chunk.
|
||||
type ChunkType string
|
||||
|
||||
const (
|
||||
// ChunkTypeFunction represents a standalone function.
|
||||
ChunkTypeFunction ChunkType = "function"
|
||||
// ChunkTypeMethod represents a method on a class/struct/type.
|
||||
ChunkTypeMethod ChunkType = "method"
|
||||
// ChunkTypeClass represents a class or struct definition.
|
||||
ChunkTypeClass ChunkType = "class"
|
||||
// ChunkTypeInterface represents an interface definition.
|
||||
ChunkTypeInterface ChunkType = "interface"
|
||||
// ChunkTypeType represents a type alias or type definition.
|
||||
ChunkTypeType ChunkType = "type"
|
||||
// ChunkTypeConst represents constant declarations.
|
||||
ChunkTypeConst ChunkType = "const"
|
||||
// ChunkTypeVar represents variable declarations.
|
||||
ChunkTypeVar ChunkType = "var"
|
||||
)
|
||||
|
||||
// Language represents a programming language.
|
||||
type Language string
|
||||
|
||||
const (
|
||||
// LanguageGo represents the Go programming language.
|
||||
LanguageGo Language = "go"
|
||||
// LanguagePython represents the Python programming language.
|
||||
LanguagePython Language = "python"
|
||||
// LanguageTypeScript represents the TypeScript programming language.
|
||||
LanguageTypeScript Language = "typescript"
|
||||
// LanguageJavaScript represents the JavaScript programming language.
|
||||
LanguageJavaScript Language = "javascript"
|
||||
)
|
||||
|
||||
// Chunk represents a semantic code chunk with AST-derived boundaries.
|
||||
type Chunk struct {
|
||||
Metadata map[string]interface{}
|
||||
FilePath string
|
||||
Language Language
|
||||
Type ChunkType
|
||||
Name string
|
||||
ParentName string
|
||||
Content string
|
||||
Signature string
|
||||
DocComment string
|
||||
StartLine int
|
||||
EndLine int
|
||||
}
|
||||
|
||||
// Identifier returns a human-readable identifier for this chunk.
|
||||
// Format: "ParentName.Name" for methods, "Name" for top-level.
|
||||
func (c *Chunk) Identifier() string {
|
||||
if c.ParentName != "" {
|
||||
return fmt.Sprintf("%s.%s", c.ParentName, c.Name)
|
||||
}
|
||||
return c.Name
|
||||
}
|
||||
|
||||
// LineRange returns a human-readable line range.
|
||||
// Format: "L123-L456"
|
||||
func (c *Chunk) LineRange() string {
|
||||
return fmt.Sprintf("L%d-L%d", c.StartLine, c.EndLine)
|
||||
}
|
||||
|
||||
// SearchableContent returns content optimized for semantic search.
|
||||
// Combines signature, doc comment, and content in a structured format.
|
||||
func (c *Chunk) SearchableContent() string {
|
||||
var parts []string
|
||||
|
||||
// Include signature for functions/methods
|
||||
if c.Signature != "" {
|
||||
parts = append(parts, c.Signature)
|
||||
}
|
||||
|
||||
// Include doc comment
|
||||
if c.DocComment != "" {
|
||||
parts = append(parts, c.DocComment)
|
||||
}
|
||||
|
||||
// Include actual content
|
||||
if c.Content != "" {
|
||||
parts = append(parts, c.Content)
|
||||
}
|
||||
|
||||
return strings.Join(parts, "\n\n")
|
||||
}
|
||||
|
||||
// Chunker is the interface for language-specific code chunkers.
|
||||
type Chunker interface {
|
||||
// Chunk parses a source file and returns semantic code chunks.
|
||||
// Returns an error if the file cannot be parsed or read.
|
||||
Chunk(ctx context.Context, filePath string) ([]Chunk, error)
|
||||
|
||||
// Language returns the language this chunker supports.
|
||||
Language() Language
|
||||
|
||||
// SupportedExtensions returns file extensions this chunker handles.
|
||||
// Example: []string{".go"} for Go chunker
|
||||
SupportedExtensions() []string
|
||||
}
|
||||
|
||||
// ChunkOptions provides options for chunking behavior.
|
||||
type ChunkOptions struct {
|
||||
// MaxChunkSize is the maximum size of a chunk in bytes.
|
||||
// Chunks larger than this will be split (respecting boundaries where possible).
|
||||
// 0 means no limit.
|
||||
MaxChunkSize int
|
||||
|
||||
// IncludeDocComments controls whether to include documentation comments.
|
||||
IncludeDocComments bool
|
||||
|
||||
// IncludePrivate controls whether to include private/unexported symbols.
|
||||
IncludePrivate bool
|
||||
|
||||
// MinLines is the minimum number of lines for a chunk to be included.
|
||||
// Chunks smaller than this will be skipped.
|
||||
// 0 means no minimum.
|
||||
MinLines int
|
||||
}
|
||||
|
||||
// DefaultChunkOptions returns sensible default options.
|
||||
func DefaultChunkOptions() ChunkOptions {
|
||||
return ChunkOptions{
|
||||
MaxChunkSize: 8192, // ~8KB per chunk (well under token limit)
|
||||
IncludeDocComments: true,
|
||||
IncludePrivate: true, // Include all symbols for comprehensive search
|
||||
MinLines: 0, // No minimum - include even single-line functions
|
||||
}
|
||||
}
|
||||
@@ -1,403 +0,0 @@
|
||||
// Package typescript provides AST-aware chunking for TypeScript and JavaScript source files using tree-sitter.
|
||||
package typescript
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
sitter "github.com/smacker/go-tree-sitter"
|
||||
"github.com/smacker/go-tree-sitter/typescript/typescript"
|
||||
|
||||
"github.com/lukaszraczylo/claude-mnemonic/internal/chunking"
|
||||
)
|
||||
|
||||
// Chunker implements AST-aware chunking for TypeScript/JavaScript files.
|
||||
type Chunker struct {
|
||||
parser *sitter.Parser
|
||||
options chunking.ChunkOptions
|
||||
}
|
||||
|
||||
// NewChunker creates a new TypeScript chunker.
|
||||
func NewChunker(options chunking.ChunkOptions) *Chunker {
|
||||
parser := sitter.NewParser()
|
||||
parser.SetLanguage(typescript.GetLanguage())
|
||||
|
||||
return &Chunker{
|
||||
options: options,
|
||||
parser: parser,
|
||||
}
|
||||
}
|
||||
|
||||
// Language returns the language this chunker supports.
|
||||
func (c *Chunker) Language() chunking.Language {
|
||||
return chunking.LanguageTypeScript
|
||||
}
|
||||
|
||||
// SupportedExtensions returns the file extensions this chunker handles.
|
||||
func (c *Chunker) SupportedExtensions() []string {
|
||||
return []string{".ts", ".tsx", ".js", ".jsx"}
|
||||
}
|
||||
|
||||
// Chunk parses a TypeScript/JavaScript source file and returns semantic code chunks.
|
||||
func (c *Chunker) Chunk(ctx context.Context, filePath string) ([]chunking.Chunk, error) {
|
||||
// Read file content
|
||||
content, err := os.ReadFile(filePath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read file: %w", err)
|
||||
}
|
||||
|
||||
// Parse the file
|
||||
tree, err := c.parser.ParseCtx(ctx, nil, content)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("parse TypeScript file: %w", err)
|
||||
}
|
||||
defer tree.Close()
|
||||
|
||||
sourceLines := strings.Split(string(content), "\n")
|
||||
chunks := make([]chunking.Chunk, 0)
|
||||
|
||||
// Walk the AST and extract chunks
|
||||
c.walkNode(tree.RootNode(), content, sourceLines, filePath, "", &chunks)
|
||||
|
||||
return chunks, nil
|
||||
}
|
||||
|
||||
// walkNode recursively walks the tree-sitter AST and extracts chunks.
|
||||
func (c *Chunker) walkNode(node *sitter.Node, source []byte, sourceLines []string, filePath string, parentName string, chunks *[]chunking.Chunk) {
|
||||
nodeType := node.Type()
|
||||
|
||||
switch nodeType {
|
||||
case "function_declaration":
|
||||
chunk := c.extractFunction(node, source, sourceLines, filePath, parentName)
|
||||
if chunk != nil {
|
||||
*chunks = append(*chunks, *chunk)
|
||||
}
|
||||
|
||||
case "method_definition":
|
||||
chunk := c.extractMethod(node, source, sourceLines, filePath, parentName)
|
||||
if chunk != nil {
|
||||
*chunks = append(*chunks, *chunk)
|
||||
}
|
||||
|
||||
case "arrow_function", "function_expression":
|
||||
// Handle arrow functions and function expressions assigned to variables
|
||||
chunk := c.extractFunctionExpression(node, source, sourceLines, filePath, parentName)
|
||||
if chunk != nil {
|
||||
*chunks = append(*chunks, *chunk)
|
||||
}
|
||||
|
||||
case "class_declaration":
|
||||
chunk := c.extractClass(node, source, sourceLines, filePath)
|
||||
if chunk != nil {
|
||||
*chunks = append(*chunks, *chunk)
|
||||
|
||||
// Walk class body to find methods
|
||||
for i := 0; i < int(node.ChildCount()); i++ {
|
||||
child := node.Child(i)
|
||||
if child.Type() == "class_body" {
|
||||
c.walkNode(child, source, sourceLines, filePath, chunk.Name, chunks)
|
||||
}
|
||||
}
|
||||
}
|
||||
return // Don't walk children again
|
||||
|
||||
case "interface_declaration":
|
||||
chunk := c.extractInterface(node, source, sourceLines, filePath)
|
||||
if chunk != nil {
|
||||
*chunks = append(*chunks, *chunk)
|
||||
}
|
||||
|
||||
case "type_alias_declaration":
|
||||
chunk := c.extractTypeAlias(node, source, sourceLines, filePath)
|
||||
if chunk != nil {
|
||||
*chunks = append(*chunks, *chunk)
|
||||
}
|
||||
}
|
||||
|
||||
// Walk all children
|
||||
for i := 0; i < int(node.ChildCount()); i++ {
|
||||
c.walkNode(node.Child(i), source, sourceLines, filePath, parentName, chunks)
|
||||
}
|
||||
}
|
||||
|
||||
// extractFunction extracts a function declaration.
|
||||
func (c *Chunker) extractFunction(node *sitter.Node, source []byte, sourceLines []string, filePath string, parentName string) *chunking.Chunk {
|
||||
name := c.findChildContent(node, "identifier", source)
|
||||
if name == "" {
|
||||
return nil
|
||||
}
|
||||
|
||||
startLine := int(node.StartPoint().Row) + 1
|
||||
endLine := int(node.EndPoint().Row) + 1
|
||||
|
||||
chunk := &chunking.Chunk{
|
||||
FilePath: filePath,
|
||||
Language: chunking.LanguageTypeScript,
|
||||
Type: chunking.ChunkTypeFunction,
|
||||
Name: name,
|
||||
ParentName: parentName,
|
||||
StartLine: startLine,
|
||||
EndLine: endLine,
|
||||
Content: c.extractLines(sourceLines, startLine, endLine),
|
||||
Signature: c.extractFunctionSignature(node, source, sourceLines),
|
||||
}
|
||||
|
||||
// Extract JSDoc comment
|
||||
if c.options.IncludeDocComments {
|
||||
chunk.DocComment = c.extractComment(node, source)
|
||||
}
|
||||
|
||||
return chunk
|
||||
}
|
||||
|
||||
// extractMethod extracts a method definition from a class.
|
||||
func (c *Chunker) extractMethod(node *sitter.Node, source []byte, sourceLines []string, filePath string, parentName string) *chunking.Chunk {
|
||||
name := c.findChildContent(node, "property_identifier", source)
|
||||
if name == "" {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Skip private methods if configured
|
||||
if !c.options.IncludePrivate && strings.HasPrefix(name, "_") {
|
||||
return nil
|
||||
}
|
||||
|
||||
startLine := int(node.StartPoint().Row) + 1
|
||||
endLine := int(node.EndPoint().Row) + 1
|
||||
|
||||
chunk := &chunking.Chunk{
|
||||
FilePath: filePath,
|
||||
Language: chunking.LanguageTypeScript,
|
||||
Type: chunking.ChunkTypeMethod,
|
||||
Name: name,
|
||||
ParentName: parentName,
|
||||
StartLine: startLine,
|
||||
EndLine: endLine,
|
||||
Content: c.extractLines(sourceLines, startLine, endLine),
|
||||
Signature: c.extractMethodSignature(node, source, sourceLines),
|
||||
}
|
||||
|
||||
// Extract JSDoc comment
|
||||
if c.options.IncludeDocComments {
|
||||
chunk.DocComment = c.extractComment(node, source)
|
||||
}
|
||||
|
||||
return chunk
|
||||
}
|
||||
|
||||
// extractFunctionExpression extracts arrow functions and function expressions.
|
||||
func (c *Chunker) extractFunctionExpression(node *sitter.Node, source []byte, sourceLines []string, filePath string, parentName string) *chunking.Chunk {
|
||||
// Try to find the variable name from parent
|
||||
parent := node.Parent()
|
||||
if parent == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
var name string
|
||||
if parent.Type() == "variable_declarator" {
|
||||
name = c.findChildContent(parent, "identifier", source)
|
||||
} else if parent.Type() == "assignment_expression" {
|
||||
// Handle const foo = () => {}
|
||||
for i := 0; i < int(parent.ChildCount()); i++ {
|
||||
child := parent.Child(i)
|
||||
if child.Type() == "identifier" || child.Type() == "member_expression" {
|
||||
name = child.Content(source)
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if name == "" {
|
||||
return nil // Anonymous function, skip
|
||||
}
|
||||
|
||||
startLine := int(node.StartPoint().Row) + 1
|
||||
endLine := int(node.EndPoint().Row) + 1
|
||||
|
||||
chunk := &chunking.Chunk{
|
||||
FilePath: filePath,
|
||||
Language: chunking.LanguageTypeScript,
|
||||
Type: chunking.ChunkTypeFunction,
|
||||
Name: name,
|
||||
ParentName: parentName,
|
||||
StartLine: startLine,
|
||||
EndLine: endLine,
|
||||
Content: c.extractLines(sourceLines, startLine, endLine),
|
||||
}
|
||||
|
||||
return chunk
|
||||
}
|
||||
|
||||
// extractClass extracts a class declaration.
|
||||
func (c *Chunker) extractClass(node *sitter.Node, source []byte, sourceLines []string, filePath string) *chunking.Chunk {
|
||||
name := c.findChildContent(node, "type_identifier", source)
|
||||
if name == "" {
|
||||
return nil
|
||||
}
|
||||
|
||||
startLine := int(node.StartPoint().Row) + 1
|
||||
endLine := int(node.EndPoint().Row) + 1
|
||||
|
||||
chunk := &chunking.Chunk{
|
||||
FilePath: filePath,
|
||||
Language: chunking.LanguageTypeScript,
|
||||
Type: chunking.ChunkTypeClass,
|
||||
Name: name,
|
||||
StartLine: startLine,
|
||||
EndLine: endLine,
|
||||
Content: c.extractLines(sourceLines, startLine, endLine),
|
||||
Signature: c.extractClassSignature(node, source, sourceLines),
|
||||
}
|
||||
|
||||
// Extract JSDoc comment
|
||||
if c.options.IncludeDocComments {
|
||||
chunk.DocComment = c.extractComment(node, source)
|
||||
}
|
||||
|
||||
return chunk
|
||||
}
|
||||
|
||||
// extractInterface extracts an interface declaration.
|
||||
func (c *Chunker) extractInterface(node *sitter.Node, source []byte, sourceLines []string, filePath string) *chunking.Chunk {
|
||||
name := c.findChildContent(node, "type_identifier", source)
|
||||
if name == "" {
|
||||
return nil
|
||||
}
|
||||
|
||||
startLine := int(node.StartPoint().Row) + 1
|
||||
endLine := int(node.EndPoint().Row) + 1
|
||||
|
||||
chunk := &chunking.Chunk{
|
||||
FilePath: filePath,
|
||||
Language: chunking.LanguageTypeScript,
|
||||
Type: chunking.ChunkTypeInterface,
|
||||
Name: name,
|
||||
StartLine: startLine,
|
||||
EndLine: endLine,
|
||||
Content: c.extractLines(sourceLines, startLine, endLine),
|
||||
}
|
||||
|
||||
// Extract JSDoc comment
|
||||
if c.options.IncludeDocComments {
|
||||
chunk.DocComment = c.extractComment(node, source)
|
||||
}
|
||||
|
||||
return chunk
|
||||
}
|
||||
|
||||
// extractTypeAlias extracts a type alias declaration.
|
||||
func (c *Chunker) extractTypeAlias(node *sitter.Node, source []byte, sourceLines []string, filePath string) *chunking.Chunk {
|
||||
name := c.findChildContent(node, "type_identifier", source)
|
||||
if name == "" {
|
||||
return nil
|
||||
}
|
||||
|
||||
startLine := int(node.StartPoint().Row) + 1
|
||||
endLine := int(node.EndPoint().Row) + 1
|
||||
|
||||
chunk := &chunking.Chunk{
|
||||
FilePath: filePath,
|
||||
Language: chunking.LanguageTypeScript,
|
||||
Type: chunking.ChunkTypeType,
|
||||
Name: name,
|
||||
StartLine: startLine,
|
||||
EndLine: endLine,
|
||||
Content: c.extractLines(sourceLines, startLine, endLine),
|
||||
}
|
||||
|
||||
return chunk
|
||||
}
|
||||
|
||||
// findChildContent finds the first child of the given type and returns its content.
|
||||
func (c *Chunker) findChildContent(node *sitter.Node, childType string, source []byte) string {
|
||||
for i := 0; i < int(node.ChildCount()); i++ {
|
||||
child := node.Child(i)
|
||||
if child.Type() == childType {
|
||||
return child.Content(source)
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// extractFunctionSignature extracts the function signature.
|
||||
func (c *Chunker) extractFunctionSignature(node *sitter.Node, source []byte, sourceLines []string) string {
|
||||
startLine := int(node.StartPoint().Row) + 1
|
||||
|
||||
// Find the opening brace of the body
|
||||
for i := 0; i < int(node.ChildCount()); i++ {
|
||||
child := node.Child(i)
|
||||
if child.Type() == "statement_block" {
|
||||
endLine := int(child.StartPoint().Row) + 1
|
||||
return strings.TrimSpace(c.extractLines(sourceLines, startLine, endLine-1))
|
||||
}
|
||||
}
|
||||
|
||||
// Fallback: just return first line
|
||||
return strings.TrimSpace(c.extractLines(sourceLines, startLine, startLine))
|
||||
}
|
||||
|
||||
// extractMethodSignature extracts the method signature.
|
||||
func (c *Chunker) extractMethodSignature(node *sitter.Node, source []byte, sourceLines []string) string {
|
||||
startLine := int(node.StartPoint().Row) + 1
|
||||
|
||||
// Find the opening brace of the body
|
||||
for i := 0; i < int(node.ChildCount()); i++ {
|
||||
child := node.Child(i)
|
||||
if child.Type() == "statement_block" {
|
||||
endLine := int(child.StartPoint().Row) + 1
|
||||
return strings.TrimSpace(c.extractLines(sourceLines, startLine, endLine-1))
|
||||
}
|
||||
}
|
||||
|
||||
return strings.TrimSpace(c.extractLines(sourceLines, startLine, startLine))
|
||||
}
|
||||
|
||||
// extractClassSignature extracts the class declaration line.
|
||||
func (c *Chunker) extractClassSignature(node *sitter.Node, source []byte, sourceLines []string) string {
|
||||
startLine := int(node.StartPoint().Row) + 1
|
||||
|
||||
// Find the opening brace of the class body
|
||||
for i := 0; i < int(node.ChildCount()); i++ {
|
||||
child := node.Child(i)
|
||||
if child.Type() == "class_body" {
|
||||
endLine := int(child.StartPoint().Row) + 1
|
||||
return strings.TrimSpace(c.extractLines(sourceLines, startLine, endLine-1))
|
||||
}
|
||||
}
|
||||
|
||||
return strings.TrimSpace(c.extractLines(sourceLines, startLine, startLine))
|
||||
}
|
||||
|
||||
// extractComment extracts JSDoc or other comments from a node.
|
||||
func (c *Chunker) extractComment(node *sitter.Node, source []byte) string {
|
||||
// Check previous sibling for comment
|
||||
prevSibling := node.PrevSibling()
|
||||
if prevSibling != nil && prevSibling.Type() == "comment" {
|
||||
comment := prevSibling.Content(source)
|
||||
// Remove comment markers
|
||||
comment = strings.TrimPrefix(comment, "/**")
|
||||
comment = strings.TrimPrefix(comment, "/*")
|
||||
comment = strings.TrimSuffix(comment, "*/")
|
||||
comment = strings.TrimPrefix(comment, "//")
|
||||
return strings.TrimSpace(comment)
|
||||
}
|
||||
|
||||
return ""
|
||||
}
|
||||
|
||||
// extractLines extracts a range of lines from source (1-indexed, inclusive).
|
||||
func (c *Chunker) extractLines(lines []string, start, end int) string {
|
||||
if start < 1 || end < start || start > len(lines) {
|
||||
return ""
|
||||
}
|
||||
|
||||
startIdx := start - 1
|
||||
endIdx := end
|
||||
if endIdx > len(lines) {
|
||||
endIdx = len(lines)
|
||||
}
|
||||
|
||||
return strings.Join(lines[startIdx:endIdx], "\n")
|
||||
}
|
||||
+34
-101
@@ -37,34 +37,27 @@ var CriticalConcepts = []string{
|
||||
|
||||
// Config holds the application configuration.
|
||||
type Config struct {
|
||||
ContextFullField string `json:"context_full_field"`
|
||||
DBPath string `json:"db_path"`
|
||||
Model string `json:"model"`
|
||||
ClaudeCodePath string `json:"claude_code_path"`
|
||||
EmbeddingModel string `json:"embedding_model"`
|
||||
ContextObsConcepts []string `json:"context_obs_concepts"`
|
||||
ContextObsTypes []string `json:"context_obs_types"`
|
||||
RerankingMinImprovement float64 `json:"reranking_min_improvement"`
|
||||
RerankingCandidates int `json:"reranking_candidates"`
|
||||
RerankingAlpha float64 `json:"reranking_alpha"`
|
||||
GraphEdgeWeight float64 `json:"graph_edge_weight"`
|
||||
WorkerPort int `json:"worker_port"`
|
||||
ContextMaxPromptResults int `json:"context_max_prompt_results"`
|
||||
ContextObservations int `json:"context_observations"`
|
||||
ContextFullCount int `json:"context_full_count"`
|
||||
ContextSessionCount int `json:"context_session_count"`
|
||||
ContextRelevanceThreshold float64 `json:"context_relevance_threshold"`
|
||||
MaxConns int `json:"max_conns"`
|
||||
RerankingResults int `json:"reranking_results"`
|
||||
GraphMaxHops int `json:"graph_max_hops"`
|
||||
GraphBranchFactor int `json:"graph_branch_factor"`
|
||||
GraphRebuildIntervalMin int `json:"graph_rebuild_interval_min"`
|
||||
ContextShowLastSummary bool `json:"context_show_last_summary"`
|
||||
RerankingEnabled bool `json:"reranking_enabled"`
|
||||
ContextShowWorkTokens bool `json:"context_show_work_tokens"`
|
||||
ContextShowReadTokens bool `json:"context_show_read_tokens"`
|
||||
RerankingPureMode bool `json:"reranking_pure_mode"`
|
||||
GraphEnabled bool `json:"graph_enabled"`
|
||||
// Worker settings
|
||||
WorkerPort int `json:"worker_port"`
|
||||
|
||||
// Database settings
|
||||
DBPath string `json:"db_path"`
|
||||
MaxConns int `json:"max_conns"`
|
||||
|
||||
// SDK Agent settings
|
||||
Model string `json:"model"`
|
||||
ClaudeCodePath string `json:"claude_code_path"`
|
||||
|
||||
// Context injection settings
|
||||
ContextObservations int `json:"context_observations"`
|
||||
ContextFullCount int `json:"context_full_count"`
|
||||
ContextSessionCount int `json:"context_session_count"`
|
||||
ContextShowReadTokens bool `json:"context_show_read_tokens"`
|
||||
ContextShowWorkTokens bool `json:"context_show_work_tokens"`
|
||||
ContextFullField string `json:"context_full_field"`
|
||||
ContextShowLastSummary bool `json:"context_show_last_summary"`
|
||||
ContextObsTypes []string `json:"context_obs_types"`
|
||||
ContextObsConcepts []string `json:"context_obs_concepts"`
|
||||
}
|
||||
|
||||
var (
|
||||
@@ -126,38 +119,22 @@ func EnsureAll() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// DefaultEmbeddingModel is the default embedding model to use.
|
||||
const DefaultEmbeddingModel = "bge-v1.5"
|
||||
|
||||
// Default returns a Config with default values.
|
||||
func Default() *Config {
|
||||
return &Config{
|
||||
WorkerPort: DefaultWorkerPort,
|
||||
DBPath: DBPath(),
|
||||
MaxConns: 4,
|
||||
Model: DefaultModel,
|
||||
EmbeddingModel: DefaultEmbeddingModel,
|
||||
RerankingEnabled: true, // Enable by default for improved relevance
|
||||
RerankingCandidates: 100, // Retrieve top 100 candidates
|
||||
RerankingResults: 10, // Return top 10 after reranking
|
||||
RerankingAlpha: 0.7, // Favor cross-encoder score
|
||||
RerankingMinImprovement: 0, // Always apply reranking
|
||||
GraphEnabled: true, // Enable graph-aware search by default
|
||||
GraphMaxHops: 2, // Two-hop traversal
|
||||
GraphBranchFactor: 5, // Expand top 5 neighbors per node
|
||||
GraphEdgeWeight: 0.3, // Minimum edge weight to follow
|
||||
GraphRebuildIntervalMin: 60, // Rebuild graph every 60 minutes
|
||||
ContextObservations: 100,
|
||||
ContextFullCount: 25,
|
||||
ContextSessionCount: 10,
|
||||
ContextShowReadTokens: true,
|
||||
ContextShowWorkTokens: true,
|
||||
ContextFullField: "narrative",
|
||||
ContextShowLastSummary: true,
|
||||
ContextObsTypes: DefaultObservationTypes,
|
||||
ContextObsConcepts: DefaultObservationConcepts,
|
||||
ContextRelevanceThreshold: 0.3, // Minimum 30% similarity to include
|
||||
ContextMaxPromptResults: 10, // Cap at 10 results max (0 = no cap, threshold only)
|
||||
WorkerPort: DefaultWorkerPort,
|
||||
DBPath: DBPath(),
|
||||
MaxConns: 4,
|
||||
Model: DefaultModel,
|
||||
ContextObservations: 100,
|
||||
ContextFullCount: 25,
|
||||
ContextSessionCount: 10,
|
||||
ContextShowReadTokens: true,
|
||||
ContextShowWorkTokens: true,
|
||||
ContextFullField: "narrative",
|
||||
ContextShowLastSummary: true,
|
||||
ContextObsTypes: DefaultObservationTypes,
|
||||
ContextObsConcepts: DefaultObservationConcepts,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -189,28 +166,6 @@ func Load() (*Config, error) {
|
||||
if v, ok := settings["CLAUDE_CODE_PATH"].(string); ok {
|
||||
cfg.ClaudeCodePath = v
|
||||
}
|
||||
if v, ok := settings["CLAUDE_MNEMONIC_EMBEDDING_MODEL"].(string); ok && v != "" {
|
||||
cfg.EmbeddingModel = v
|
||||
}
|
||||
// Reranking settings
|
||||
if v, ok := settings["CLAUDE_MNEMONIC_RERANKING_ENABLED"].(bool); ok {
|
||||
cfg.RerankingEnabled = v
|
||||
}
|
||||
if v, ok := settings["CLAUDE_MNEMONIC_RERANKING_CANDIDATES"].(float64); ok && v > 0 {
|
||||
cfg.RerankingCandidates = int(v)
|
||||
}
|
||||
if v, ok := settings["CLAUDE_MNEMONIC_RERANKING_RESULTS"].(float64); ok && v > 0 {
|
||||
cfg.RerankingResults = int(v)
|
||||
}
|
||||
if v, ok := settings["CLAUDE_MNEMONIC_RERANKING_ALPHA"].(float64); ok && v >= 0 && v <= 1 {
|
||||
cfg.RerankingAlpha = v
|
||||
}
|
||||
if v, ok := settings["CLAUDE_MNEMONIC_RERANKING_MIN_IMPROVEMENT"].(float64); ok && v >= 0 {
|
||||
cfg.RerankingMinImprovement = v
|
||||
}
|
||||
if v, ok := settings["CLAUDE_MNEMONIC_RERANKING_PURE_MODE"].(bool); ok {
|
||||
cfg.RerankingPureMode = v
|
||||
}
|
||||
if v, ok := settings["CLAUDE_MNEMONIC_CONTEXT_OBSERVATIONS"].(float64); ok {
|
||||
cfg.ContextObservations = int(v)
|
||||
}
|
||||
@@ -226,28 +181,6 @@ func Load() (*Config, error) {
|
||||
if v, ok := settings["CLAUDE_MNEMONIC_CONTEXT_OBS_CONCEPTS"].(string); ok && v != "" {
|
||||
cfg.ContextObsConcepts = splitTrim(v)
|
||||
}
|
||||
if v, ok := settings["CLAUDE_MNEMONIC_CONTEXT_RELEVANCE_THRESHOLD"].(float64); ok && v >= 0 && v <= 1 {
|
||||
cfg.ContextRelevanceThreshold = v
|
||||
}
|
||||
if v, ok := settings["CLAUDE_MNEMONIC_CONTEXT_MAX_PROMPT_RESULTS"].(float64); ok && v >= 0 {
|
||||
cfg.ContextMaxPromptResults = int(v)
|
||||
}
|
||||
// Graph settings
|
||||
if v, ok := settings["CLAUDE_MNEMONIC_GRAPH_ENABLED"].(bool); ok {
|
||||
cfg.GraphEnabled = v
|
||||
}
|
||||
if v, ok := settings["CLAUDE_MNEMONIC_GRAPH_MAX_HOPS"].(float64); ok && v > 0 {
|
||||
cfg.GraphMaxHops = int(v)
|
||||
}
|
||||
if v, ok := settings["CLAUDE_MNEMONIC_GRAPH_BRANCH_FACTOR"].(float64); ok && v > 0 {
|
||||
cfg.GraphBranchFactor = int(v)
|
||||
}
|
||||
if v, ok := settings["CLAUDE_MNEMONIC_GRAPH_EDGE_WEIGHT"].(float64); ok && v >= 0 && v <= 1 {
|
||||
cfg.GraphEdgeWeight = v
|
||||
}
|
||||
if v, ok := settings["CLAUDE_MNEMONIC_GRAPH_REBUILD_INTERVAL_MIN"].(float64); ok && v > 0 {
|
||||
cfg.GraphRebuildIntervalMin = int(v)
|
||||
}
|
||||
|
||||
return cfg, nil
|
||||
}
|
||||
|
||||
@@ -1,436 +0,0 @@
|
||||
// Package config provides configuration management for claude-mnemonic.
|
||||
package config
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"github.com/stretchr/testify/suite"
|
||||
)
|
||||
|
||||
// ConfigSuite is a test suite for config operations.
|
||||
type ConfigSuite struct {
|
||||
suite.Suite
|
||||
tempDir string
|
||||
origHomeDir string
|
||||
}
|
||||
|
||||
func (s *ConfigSuite) SetupTest() {
|
||||
var err error
|
||||
s.tempDir, err = os.MkdirTemp("", "config-test-*")
|
||||
s.Require().NoError(err)
|
||||
|
||||
// Save and override HOME
|
||||
s.origHomeDir = os.Getenv("HOME")
|
||||
os.Setenv("HOME", s.tempDir)
|
||||
}
|
||||
|
||||
func (s *ConfigSuite) TearDownTest() {
|
||||
os.Setenv("HOME", s.origHomeDir)
|
||||
os.RemoveAll(s.tempDir)
|
||||
}
|
||||
|
||||
func TestConfigSuite(t *testing.T) {
|
||||
suite.Run(t, new(ConfigSuite))
|
||||
}
|
||||
|
||||
// TestDefault tests default configuration values.
|
||||
func (s *ConfigSuite) TestDefault() {
|
||||
cfg := Default()
|
||||
|
||||
s.Equal(DefaultWorkerPort, cfg.WorkerPort)
|
||||
s.Equal(DefaultModel, cfg.Model)
|
||||
s.Equal(4, cfg.MaxConns)
|
||||
s.Equal(100, cfg.ContextObservations)
|
||||
s.Equal(25, cfg.ContextFullCount)
|
||||
s.Equal(10, cfg.ContextSessionCount)
|
||||
s.True(cfg.ContextShowReadTokens)
|
||||
s.True(cfg.ContextShowWorkTokens)
|
||||
s.Equal("narrative", cfg.ContextFullField)
|
||||
s.True(cfg.ContextShowLastSummary)
|
||||
s.Equal(DefaultObservationTypes, cfg.ContextObsTypes)
|
||||
s.Equal(DefaultObservationConcepts, cfg.ContextObsConcepts)
|
||||
}
|
||||
|
||||
// TestDataDir tests data directory path.
|
||||
func (s *ConfigSuite) TestDataDir() {
|
||||
dir := DataDir()
|
||||
s.Contains(dir, ".claude-mnemonic")
|
||||
}
|
||||
|
||||
// TestDBPath tests database path.
|
||||
func (s *ConfigSuite) TestDBPath() {
|
||||
path := DBPath()
|
||||
s.Contains(path, "claude-mnemonic.db")
|
||||
}
|
||||
|
||||
// TestSettingsPath tests settings file path.
|
||||
func (s *ConfigSuite) TestSettingsPath() {
|
||||
path := SettingsPath()
|
||||
s.Contains(path, "settings.json")
|
||||
}
|
||||
|
||||
// TestEnsureDataDir tests data directory creation.
|
||||
func (s *ConfigSuite) TestEnsureDataDir() {
|
||||
err := EnsureDataDir()
|
||||
s.NoError(err)
|
||||
|
||||
dir := DataDir()
|
||||
info, err := os.Stat(dir)
|
||||
s.NoError(err)
|
||||
s.True(info.IsDir())
|
||||
}
|
||||
|
||||
// TestEnsureSettings tests settings file creation.
|
||||
func (s *ConfigSuite) TestEnsureSettings() {
|
||||
// First ensure data dir exists
|
||||
err := EnsureDataDir()
|
||||
s.NoError(err)
|
||||
|
||||
// Ensure settings creates default file
|
||||
err = EnsureSettings()
|
||||
s.NoError(err)
|
||||
|
||||
path := SettingsPath()
|
||||
info, err := os.Stat(path)
|
||||
s.NoError(err)
|
||||
s.False(info.IsDir())
|
||||
|
||||
// Second call should not error (file exists)
|
||||
err = EnsureSettings()
|
||||
s.NoError(err)
|
||||
}
|
||||
|
||||
// TestEnsureAll tests full initialization.
|
||||
func (s *ConfigSuite) TestEnsureAll() {
|
||||
err := EnsureAll()
|
||||
s.NoError(err)
|
||||
|
||||
// Verify dir and settings exist
|
||||
_, err = os.Stat(DataDir())
|
||||
s.NoError(err)
|
||||
_, err = os.Stat(SettingsPath())
|
||||
s.NoError(err)
|
||||
}
|
||||
|
||||
// TestLoad_TableDriven tests configuration loading with various scenarios.
|
||||
func (s *ConfigSuite) TestLoad_TableDriven() {
|
||||
tests := []struct {
|
||||
name string
|
||||
settingsJSON string
|
||||
expectedModel string
|
||||
expectedPort int
|
||||
expectedObsObs int
|
||||
}{
|
||||
{
|
||||
name: "no settings file",
|
||||
settingsJSON: "",
|
||||
expectedPort: DefaultWorkerPort,
|
||||
expectedModel: DefaultModel,
|
||||
expectedObsObs: 100,
|
||||
},
|
||||
{
|
||||
name: "custom port",
|
||||
settingsJSON: `{"CLAUDE_MNEMONIC_WORKER_PORT": 38888}`,
|
||||
expectedPort: 38888,
|
||||
expectedModel: DefaultModel,
|
||||
expectedObsObs: 100,
|
||||
},
|
||||
{
|
||||
name: "custom model",
|
||||
settingsJSON: `{"CLAUDE_MNEMONIC_MODEL": "sonnet"}`,
|
||||
expectedPort: DefaultWorkerPort,
|
||||
expectedModel: "sonnet",
|
||||
expectedObsObs: 100,
|
||||
},
|
||||
{
|
||||
name: "custom observations",
|
||||
settingsJSON: `{"CLAUDE_MNEMONIC_CONTEXT_OBSERVATIONS": 200}`,
|
||||
expectedPort: DefaultWorkerPort,
|
||||
expectedModel: DefaultModel,
|
||||
expectedObsObs: 200,
|
||||
},
|
||||
{
|
||||
name: "multiple settings",
|
||||
settingsJSON: `{"CLAUDE_MNEMONIC_WORKER_PORT": 39999, "CLAUDE_MNEMONIC_MODEL": "opus", "CLAUDE_MNEMONIC_CONTEXT_OBSERVATIONS": 50}`,
|
||||
expectedPort: 39999,
|
||||
expectedModel: "opus",
|
||||
expectedObsObs: 50,
|
||||
},
|
||||
{
|
||||
name: "invalid JSON returns defaults",
|
||||
settingsJSON: `{invalid}`,
|
||||
expectedPort: DefaultWorkerPort,
|
||||
expectedModel: DefaultModel,
|
||||
expectedObsObs: 100,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
s.Run(tt.name, func() {
|
||||
// Create fresh temp dir
|
||||
tempDir, err := os.MkdirTemp("", "config-test-*")
|
||||
s.Require().NoError(err)
|
||||
defer os.RemoveAll(tempDir)
|
||||
|
||||
os.Setenv("HOME", tempDir)
|
||||
|
||||
// Create data dir
|
||||
err = os.MkdirAll(filepath.Join(tempDir, ".claude-mnemonic"), 0750)
|
||||
s.Require().NoError(err)
|
||||
|
||||
if tt.settingsJSON != "" {
|
||||
writeErr := os.WriteFile(
|
||||
filepath.Join(tempDir, ".claude-mnemonic", "settings.json"),
|
||||
[]byte(tt.settingsJSON),
|
||||
0600,
|
||||
)
|
||||
s.Require().NoError(writeErr)
|
||||
}
|
||||
|
||||
cfg, err := Load()
|
||||
s.NoError(err)
|
||||
s.NotNil(cfg)
|
||||
s.Equal(tt.expectedPort, cfg.WorkerPort)
|
||||
s.Equal(tt.expectedModel, cfg.Model)
|
||||
s.Equal(tt.expectedObsObs, cfg.ContextObservations)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestGetWorkerPort_TableDriven tests worker port retrieval with various scenarios.
|
||||
func TestGetWorkerPort_TableDriven(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
envValue string
|
||||
wantPort int
|
||||
setEnv bool
|
||||
}{
|
||||
{
|
||||
name: "no env, use default",
|
||||
envValue: "",
|
||||
wantPort: DefaultWorkerPort,
|
||||
setEnv: false,
|
||||
},
|
||||
{
|
||||
name: "env set to valid port",
|
||||
envValue: "38888",
|
||||
wantPort: 38888,
|
||||
setEnv: true,
|
||||
},
|
||||
{
|
||||
name: "env set to invalid value",
|
||||
envValue: "invalid",
|
||||
wantPort: DefaultWorkerPort,
|
||||
setEnv: true,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
// Save original env
|
||||
origEnv := os.Getenv("CLAUDE_MNEMONIC_WORKER_PORT")
|
||||
defer os.Setenv("CLAUDE_MNEMONIC_WORKER_PORT", origEnv)
|
||||
|
||||
if tt.setEnv {
|
||||
os.Setenv("CLAUDE_MNEMONIC_WORKER_PORT", tt.envValue)
|
||||
} else {
|
||||
os.Unsetenv("CLAUDE_MNEMONIC_WORKER_PORT")
|
||||
}
|
||||
|
||||
// We can't easily test GetWorkerPort since it uses Get() which caches
|
||||
// So we test the env parsing logic directly
|
||||
if tt.setEnv && tt.envValue != "" {
|
||||
if tt.wantPort != DefaultWorkerPort {
|
||||
assert.Equal(t, tt.envValue, os.Getenv("CLAUDE_MNEMONIC_WORKER_PORT"))
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestSplitTrim tests the splitTrim helper function.
|
||||
func TestSplitTrim(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
input string
|
||||
expected []string
|
||||
}{
|
||||
{
|
||||
name: "empty string",
|
||||
input: "",
|
||||
expected: []string{},
|
||||
},
|
||||
{
|
||||
name: "single value",
|
||||
input: "bugfix",
|
||||
expected: []string{"bugfix"},
|
||||
},
|
||||
{
|
||||
name: "multiple values",
|
||||
input: "bugfix,feature,refactor",
|
||||
expected: []string{"bugfix", "feature", "refactor"},
|
||||
},
|
||||
{
|
||||
name: "values with spaces",
|
||||
input: " bugfix , feature , refactor ",
|
||||
expected: []string{"bugfix", "feature", "refactor"},
|
||||
},
|
||||
{
|
||||
name: "empty values filtered",
|
||||
input: "bugfix,,feature,,",
|
||||
expected: []string{"bugfix", "feature"},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
result := splitTrim(tt.input)
|
||||
assert.Equal(t, tt.expected, result)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestDefaultObservationTypes tests default observation types.
|
||||
func TestDefaultObservationTypes(t *testing.T) {
|
||||
expected := []string{
|
||||
"bugfix", "feature", "refactor", "change", "discovery", "decision",
|
||||
}
|
||||
assert.Equal(t, expected, DefaultObservationTypes)
|
||||
}
|
||||
|
||||
// TestDefaultObservationConcepts tests default observation concepts.
|
||||
func TestDefaultObservationConcepts(t *testing.T) {
|
||||
expected := []string{
|
||||
"how-it-works", "why-it-exists", "what-changed",
|
||||
"problem-solution", "gotcha", "pattern", "trade-off",
|
||||
}
|
||||
assert.Equal(t, expected, DefaultObservationConcepts)
|
||||
}
|
||||
|
||||
// TestCriticalConcepts tests critical concepts list.
|
||||
func TestCriticalConcepts(t *testing.T) {
|
||||
expected := []string{
|
||||
"gotcha", "pattern", "problem-solution", "trade-off",
|
||||
}
|
||||
assert.Equal(t, expected, CriticalConcepts)
|
||||
}
|
||||
|
||||
// TestLoad_ClaudeCodePath tests claude code path loading.
|
||||
func TestLoad_ClaudeCodePath(t *testing.T) {
|
||||
// Create temp dir
|
||||
tempDir, err := os.MkdirTemp("", "config-test-*")
|
||||
require.NoError(t, err)
|
||||
defer os.RemoveAll(tempDir)
|
||||
|
||||
origHome := os.Getenv("HOME")
|
||||
os.Setenv("HOME", tempDir)
|
||||
defer os.Setenv("HOME", origHome)
|
||||
|
||||
// Create data dir and settings
|
||||
err = os.MkdirAll(filepath.Join(tempDir, ".claude-mnemonic"), 0750)
|
||||
require.NoError(t, err)
|
||||
|
||||
settingsJSON := `{"CLAUDE_CODE_PATH": "/usr/local/bin/claude"}`
|
||||
err = os.WriteFile(
|
||||
filepath.Join(tempDir, ".claude-mnemonic", "settings.json"),
|
||||
[]byte(settingsJSON),
|
||||
0600,
|
||||
)
|
||||
require.NoError(t, err)
|
||||
|
||||
cfg, err := Load()
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "/usr/local/bin/claude", cfg.ClaudeCodePath)
|
||||
}
|
||||
|
||||
// TestGet tests the global config getter.
|
||||
func TestGet(t *testing.T) {
|
||||
// Save and restore HOME
|
||||
origHome := os.Getenv("HOME")
|
||||
tempDir, err := os.MkdirTemp("", "config-get-test-*")
|
||||
require.NoError(t, err)
|
||||
defer func() {
|
||||
os.Setenv("HOME", origHome)
|
||||
os.RemoveAll(tempDir)
|
||||
}()
|
||||
os.Setenv("HOME", tempDir)
|
||||
|
||||
// Create data dir
|
||||
err = os.MkdirAll(filepath.Join(tempDir, ".claude-mnemonic"), 0750)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Get() should return a valid config
|
||||
cfg := Get()
|
||||
require.NotNil(t, cfg)
|
||||
assert.Greater(t, cfg.WorkerPort, 0)
|
||||
assert.NotEmpty(t, cfg.Model)
|
||||
}
|
||||
|
||||
// TestGetWorkerPort_WithEnv tests GetWorkerPort with environment variable.
|
||||
func TestGetWorkerPort_WithEnv(t *testing.T) {
|
||||
// Save original env
|
||||
origEnv := os.Getenv("CLAUDE_MNEMONIC_WORKER_PORT")
|
||||
defer os.Setenv("CLAUDE_MNEMONIC_WORKER_PORT", origEnv)
|
||||
|
||||
// Test with valid port in env
|
||||
os.Setenv("CLAUDE_MNEMONIC_WORKER_PORT", "45678")
|
||||
port := GetWorkerPort()
|
||||
assert.Equal(t, 45678, port)
|
||||
|
||||
// Test with invalid port (should fall back to config)
|
||||
os.Setenv("CLAUDE_MNEMONIC_WORKER_PORT", "not-a-number")
|
||||
port = GetWorkerPort()
|
||||
// Should return from Get().WorkerPort, which is default
|
||||
assert.Greater(t, port, 0)
|
||||
|
||||
// Test with zero port (should fall back to config)
|
||||
os.Setenv("CLAUDE_MNEMONIC_WORKER_PORT", "0")
|
||||
port = GetWorkerPort()
|
||||
// Zero is invalid, so should use default
|
||||
assert.Greater(t, port, 0)
|
||||
|
||||
// Test with no env (should use config)
|
||||
os.Unsetenv("CLAUDE_MNEMONIC_WORKER_PORT")
|
||||
port = GetWorkerPort()
|
||||
assert.Greater(t, port, 0)
|
||||
}
|
||||
|
||||
// TestLoad_ContextSettings tests context-related settings loading.
|
||||
func TestLoad_ContextSettings(t *testing.T) {
|
||||
// Create temp dir
|
||||
tempDir, err := os.MkdirTemp("", "config-test-*")
|
||||
require.NoError(t, err)
|
||||
defer os.RemoveAll(tempDir)
|
||||
|
||||
origHome := os.Getenv("HOME")
|
||||
os.Setenv("HOME", tempDir)
|
||||
defer os.Setenv("HOME", origHome)
|
||||
|
||||
// Create data dir and settings
|
||||
err = os.MkdirAll(filepath.Join(tempDir, ".claude-mnemonic"), 0750)
|
||||
require.NoError(t, err)
|
||||
|
||||
settingsJSON := `{
|
||||
"CLAUDE_MNEMONIC_CONTEXT_FULL_COUNT": 50,
|
||||
"CLAUDE_MNEMONIC_CONTEXT_SESSION_COUNT": 20,
|
||||
"CLAUDE_MNEMONIC_CONTEXT_OBS_TYPES": "bugfix,feature",
|
||||
"CLAUDE_MNEMONIC_CONTEXT_OBS_CONCEPTS": "security,performance"
|
||||
}`
|
||||
err = os.WriteFile(
|
||||
filepath.Join(tempDir, ".claude-mnemonic", "settings.json"),
|
||||
[]byte(settingsJSON),
|
||||
0600,
|
||||
)
|
||||
require.NoError(t, err)
|
||||
|
||||
cfg, err := Load()
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 50, cfg.ContextFullCount)
|
||||
assert.Equal(t, 20, cfg.ContextSessionCount)
|
||||
assert.Equal(t, []string{"bugfix", "feature"}, cfg.ContextObsTypes)
|
||||
assert.Equal(t, []string{"security", "performance"}, cfg.ContextObsConcepts)
|
||||
}
|
||||
@@ -1,276 +0,0 @@
|
||||
// Package sqlite provides SQLite database operations for claude-mnemonic.
|
||||
package sqlite
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"github.com/lukaszraczylo/claude-mnemonic/pkg/models"
|
||||
)
|
||||
|
||||
// SupersededRetentionDays is the number of days to keep superseded observations before deletion.
|
||||
const SupersededRetentionDays = 3
|
||||
|
||||
// ConflictStore provides conflict-related database operations.
|
||||
type ConflictStore struct {
|
||||
store *Store
|
||||
}
|
||||
|
||||
// NewConflictStore creates a new conflict store.
|
||||
func NewConflictStore(store *Store) *ConflictStore {
|
||||
return &ConflictStore{store: store}
|
||||
}
|
||||
|
||||
// StoreConflict stores a new observation conflict.
|
||||
func (s *ConflictStore) StoreConflict(ctx context.Context, conflict *models.ObservationConflict) (int64, error) {
|
||||
const query = `
|
||||
INSERT INTO observation_conflicts
|
||||
(newer_obs_id, older_obs_id, conflict_type, resolution, reason, detected_at, detected_at_epoch, resolved, resolved_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
`
|
||||
|
||||
result, err := s.store.ExecContext(ctx, query,
|
||||
conflict.NewerObsID, conflict.OlderObsID,
|
||||
string(conflict.ConflictType), string(conflict.Resolution),
|
||||
conflict.Reason, conflict.DetectedAt, conflict.DetectedAtEpoch,
|
||||
conflict.Resolved, conflict.ResolvedAt,
|
||||
)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
return result.LastInsertId()
|
||||
}
|
||||
|
||||
// MarkObservationSuperseded marks an observation as superseded.
|
||||
func (s *ConflictStore) MarkObservationSuperseded(ctx context.Context, obsID int64) error {
|
||||
const query = `UPDATE observations SET is_superseded = 1 WHERE id = ?`
|
||||
_, err := s.store.ExecContext(ctx, query, obsID)
|
||||
return err
|
||||
}
|
||||
|
||||
// MarkObservationsSuperseded marks multiple observations as superseded.
|
||||
func (s *ConflictStore) MarkObservationsSuperseded(ctx context.Context, obsIDs []int64) error {
|
||||
if len(obsIDs) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
query := `UPDATE observations SET is_superseded = 1 WHERE id IN (?` + repeatPlaceholders(len(obsIDs)-1) + `)` // #nosec G202 -- uses parameterized placeholders
|
||||
args := int64SliceToInterface(obsIDs)
|
||||
_, err := s.store.db.ExecContext(ctx, query, args...)
|
||||
return err
|
||||
}
|
||||
|
||||
// GetConflictsByObservationID retrieves all conflicts involving an observation.
|
||||
func (s *ConflictStore) GetConflictsByObservationID(ctx context.Context, obsID int64) ([]*models.ObservationConflict, error) {
|
||||
const query = `
|
||||
SELECT id, newer_obs_id, older_obs_id, conflict_type, resolution, reason,
|
||||
detected_at, detected_at_epoch, resolved, resolved_at
|
||||
FROM observation_conflicts
|
||||
WHERE newer_obs_id = ? OR older_obs_id = ?
|
||||
ORDER BY detected_at_epoch DESC
|
||||
`
|
||||
|
||||
rows, err := s.store.QueryContext(ctx, query, obsID, obsID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
return s.scanConflictRows(rows)
|
||||
}
|
||||
|
||||
// GetUnresolvedConflicts retrieves all unresolved conflicts.
|
||||
func (s *ConflictStore) GetUnresolvedConflicts(ctx context.Context, limit int) ([]*models.ObservationConflict, error) {
|
||||
const query = `
|
||||
SELECT id, newer_obs_id, older_obs_id, conflict_type, resolution, reason,
|
||||
detected_at, detected_at_epoch, resolved, resolved_at
|
||||
FROM observation_conflicts
|
||||
WHERE resolved = 0
|
||||
ORDER BY detected_at_epoch DESC
|
||||
LIMIT ?
|
||||
`
|
||||
|
||||
rows, err := s.store.QueryContext(ctx, query, limit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
return s.scanConflictRows(rows)
|
||||
}
|
||||
|
||||
// GetSupersededObservationIDs returns IDs of all observations that have been superseded.
|
||||
func (s *ConflictStore) GetSupersededObservationIDs(ctx context.Context, project string) ([]int64, error) {
|
||||
const query = `
|
||||
SELECT DISTINCT older_obs_id
|
||||
FROM observation_conflicts oc
|
||||
JOIN observations o ON o.id = oc.older_obs_id
|
||||
WHERE oc.resolution = 'prefer_newer'
|
||||
AND (o.project = ? OR o.scope = 'global')
|
||||
`
|
||||
|
||||
rows, err := s.store.QueryContext(ctx, query, project)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var ids []int64
|
||||
for rows.Next() {
|
||||
var id int64
|
||||
if err := rows.Scan(&id); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ids = append(ids, id)
|
||||
}
|
||||
return ids, rows.Err()
|
||||
}
|
||||
|
||||
// ResolveConflict marks a conflict as resolved.
|
||||
func (s *ConflictStore) ResolveConflict(ctx context.Context, conflictID int64, resolution models.ConflictResolution) error {
|
||||
now := time.Now().Format(time.RFC3339)
|
||||
const query = `
|
||||
UPDATE observation_conflicts
|
||||
SET resolved = 1, resolved_at = ?, resolution = ?
|
||||
WHERE id = ?
|
||||
`
|
||||
_, err := s.store.ExecContext(ctx, query, now, string(resolution), conflictID)
|
||||
return err
|
||||
}
|
||||
|
||||
// DeleteConflictsByObservationID deletes all conflicts involving an observation.
|
||||
// Called when an observation is deleted.
|
||||
func (s *ConflictStore) DeleteConflictsByObservationID(ctx context.Context, obsID int64) error {
|
||||
const query = `DELETE FROM observation_conflicts WHERE newer_obs_id = ? OR older_obs_id = ?`
|
||||
_, err := s.store.ExecContext(ctx, query, obsID, obsID)
|
||||
return err
|
||||
}
|
||||
|
||||
// ConflictWithDetails contains a conflict with its observation details.
|
||||
type ConflictWithDetails struct {
|
||||
Conflict *models.ObservationConflict
|
||||
NewerObsTitle string
|
||||
OlderObsTitle string
|
||||
}
|
||||
|
||||
// CleanupSupersededObservations deletes observations that have been superseded for longer than
|
||||
// SupersededRetentionDays. Returns the IDs of deleted observations for downstream cleanup (e.g., vector DB).
|
||||
func (s *ConflictStore) CleanupSupersededObservations(ctx context.Context, project string) ([]int64, error) {
|
||||
// Calculate cutoff time (3 days ago in milliseconds)
|
||||
cutoffEpoch := time.Now().AddDate(0, 0, -SupersededRetentionDays).UnixMilli()
|
||||
|
||||
// First, find the IDs that will be deleted
|
||||
// We delete observations that:
|
||||
// 1. Are marked as superseded
|
||||
// 2. Have a conflict record where they are the older observation
|
||||
// 3. The conflict was detected more than 3 days ago
|
||||
const selectQuery = `
|
||||
SELECT DISTINCT o.id FROM observations o
|
||||
JOIN observation_conflicts oc ON o.id = oc.older_obs_id
|
||||
WHERE o.is_superseded = 1
|
||||
AND o.project = ?
|
||||
AND oc.detected_at_epoch < ?
|
||||
`
|
||||
|
||||
rows, err := s.store.QueryContext(ctx, selectQuery, project, cutoffEpoch)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var toDelete []int64
|
||||
for rows.Next() {
|
||||
var id int64
|
||||
if scanErr := rows.Scan(&id); scanErr != nil {
|
||||
return nil, scanErr
|
||||
}
|
||||
toDelete = append(toDelete, id)
|
||||
}
|
||||
if rowsErr := rows.Err(); rowsErr != nil {
|
||||
return nil, rowsErr
|
||||
}
|
||||
|
||||
if len(toDelete) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// Delete the conflict records first (due to foreign key constraints)
|
||||
for _, obsID := range toDelete {
|
||||
if delErr := s.DeleteConflictsByObservationID(ctx, obsID); delErr != nil {
|
||||
return nil, delErr
|
||||
}
|
||||
}
|
||||
|
||||
// Delete the observations
|
||||
deleteQuery := `DELETE FROM observations WHERE id IN (?` + repeatPlaceholders(len(toDelete)-1) + `)` // #nosec G202 -- uses parameterized placeholders
|
||||
args := int64SliceToInterface(toDelete)
|
||||
_, err = s.store.db.ExecContext(ctx, deleteQuery, args...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return toDelete, nil
|
||||
}
|
||||
|
||||
// GetConflictsWithDetails retrieves all conflicts with observation titles for display.
|
||||
func (s *ConflictStore) GetConflictsWithDetails(ctx context.Context, project string, limit int) ([]*ConflictWithDetails, error) {
|
||||
const query = `
|
||||
SELECT oc.id, oc.newer_obs_id, oc.older_obs_id, oc.conflict_type, oc.resolution, oc.reason,
|
||||
oc.detected_at, oc.detected_at_epoch, oc.resolved, oc.resolved_at,
|
||||
COALESCE(newer.title, '') as newer_title,
|
||||
COALESCE(older.title, '') as older_title
|
||||
FROM observation_conflicts oc
|
||||
JOIN observations newer ON newer.id = oc.newer_obs_id
|
||||
JOIN observations older ON older.id = oc.older_obs_id
|
||||
WHERE newer.project = ? OR older.project = ?
|
||||
ORDER BY oc.detected_at_epoch DESC
|
||||
LIMIT ?
|
||||
`
|
||||
|
||||
rows, err := s.store.QueryContext(ctx, query, project, project, limit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var results []*ConflictWithDetails
|
||||
for rows.Next() {
|
||||
var c models.ObservationConflict
|
||||
var cwd ConflictWithDetails
|
||||
if err := rows.Scan(
|
||||
&c.ID, &c.NewerObsID, &c.OlderObsID,
|
||||
&c.ConflictType, &c.Resolution, &c.Reason,
|
||||
&c.DetectedAt, &c.DetectedAtEpoch,
|
||||
&c.Resolved, &c.ResolvedAt,
|
||||
&cwd.NewerObsTitle, &cwd.OlderObsTitle,
|
||||
); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
cwd.Conflict = &c
|
||||
results = append(results, &cwd)
|
||||
}
|
||||
return results, rows.Err()
|
||||
}
|
||||
|
||||
// scanConflictRows scans multiple conflicts from rows.
|
||||
func (s *ConflictStore) scanConflictRows(rows interface {
|
||||
Next() bool
|
||||
Scan(...interface{}) error
|
||||
Err() error
|
||||
}) ([]*models.ObservationConflict, error) {
|
||||
var conflicts []*models.ObservationConflict
|
||||
for rows.Next() {
|
||||
var c models.ObservationConflict
|
||||
if err := rows.Scan(
|
||||
&c.ID, &c.NewerObsID, &c.OlderObsID,
|
||||
&c.ConflictType, &c.Resolution, &c.Reason,
|
||||
&c.DetectedAt, &c.DetectedAtEpoch,
|
||||
&c.Resolved, &c.ResolvedAt,
|
||||
); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
conflicts = append(conflicts, &c)
|
||||
}
|
||||
return conflicts, rows.Err()
|
||||
}
|
||||
@@ -1,254 +0,0 @@
|
||||
package sqlite
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestNullString(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
input string
|
||||
expected string
|
||||
valid bool
|
||||
}{
|
||||
{"empty_string", "", "", false},
|
||||
{"non_empty_string", "hello", "hello", true},
|
||||
{"whitespace", " ", " ", true},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
result := nullString(tt.input)
|
||||
assert.Equal(t, tt.expected, result.String)
|
||||
assert.Equal(t, tt.valid, result.Valid)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestNullInt(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
input int
|
||||
expected int64
|
||||
valid bool
|
||||
}{
|
||||
{"zero", 0, 0, false},
|
||||
{"positive", 42, 42, true},
|
||||
{"negative", -1, -1, false},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
result := nullInt(tt.input)
|
||||
assert.Equal(t, tt.expected, result.Int64)
|
||||
assert.Equal(t, tt.valid, result.Valid)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRepeatPlaceholders(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
expected string
|
||||
n int
|
||||
}{
|
||||
{name: "zero", n: 0, expected: ""},
|
||||
{name: "negative", n: -1, expected: ""},
|
||||
{name: "one", n: 1, expected: ", ?"},
|
||||
{name: "two", n: 2, expected: ", ?, ?"},
|
||||
{name: "three", n: 3, expected: ", ?, ?, ?"},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
result := repeatPlaceholders(tt.n)
|
||||
assert.Equal(t, tt.expected, result)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestInt64SliceToInterface(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
input []int64
|
||||
expected []interface{}
|
||||
}{
|
||||
{"empty", []int64{}, []interface{}{}},
|
||||
{"single", []int64{42}, []interface{}{int64(42)}},
|
||||
{"multiple", []int64{1, 2, 3}, []interface{}{int64(1), int64(2), int64(3)}},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
result := int64SliceToInterface(tt.input)
|
||||
assert.Equal(t, tt.expected, result)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseLimitParam(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
query string
|
||||
defaultLimit int
|
||||
expected int
|
||||
}{
|
||||
{"no_param_uses_default", "", 10, 10},
|
||||
{"valid_limit", "limit=20", 10, 20},
|
||||
{"invalid_limit_uses_default", "limit=abc", 10, 10},
|
||||
{"zero_limit_uses_default", "limit=0", 10, 10},
|
||||
{"negative_limit_uses_default", "limit=-5", 10, 10},
|
||||
{"large_limit", "limit=1000", 10, 1000},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
req := httptest.NewRequest("GET", "/?"+tt.query, nil)
|
||||
result := ParseLimitParam(req, tt.defaultLimit)
|
||||
assert.Equal(t, tt.expected, result)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestScanSummary(t *testing.T) {
|
||||
db, _, cleanup := testDB(t)
|
||||
defer cleanup()
|
||||
createBaseTables(t, db)
|
||||
seedSession(t, db, "claude-123", "sdk-123", "test-project")
|
||||
|
||||
// Insert a test summary
|
||||
_, err := db.Exec(`
|
||||
INSERT INTO session_summaries (sdk_session_id, project, request, investigated, learned, completed, next_steps, notes, prompt_number, discovery_tokens, created_at, created_at_epoch)
|
||||
VALUES ('sdk-123', 'test-project', 'test request', 'test investigated', 'test learned', 'test completed', 'test next steps', 'test notes', 1, 100, '2025-01-01T00:00:00Z', 1704067200000)
|
||||
`)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Query and scan
|
||||
row := db.QueryRow(`
|
||||
SELECT id, sdk_session_id, project, request, investigated, learned, completed, next_steps, notes, prompt_number, discovery_tokens, created_at, created_at_epoch
|
||||
FROM session_summaries WHERE sdk_session_id = ?
|
||||
`, "sdk-123")
|
||||
|
||||
summary, err := scanSummary(row)
|
||||
require.NoError(t, err)
|
||||
assert.NotNil(t, summary)
|
||||
assert.Equal(t, "sdk-123", summary.SDKSessionID)
|
||||
assert.Equal(t, "test-project", summary.Project)
|
||||
assert.Equal(t, "test request", summary.Request.String)
|
||||
assert.Equal(t, "test investigated", summary.Investigated.String)
|
||||
assert.Equal(t, "test learned", summary.Learned.String)
|
||||
assert.Equal(t, "test completed", summary.Completed.String)
|
||||
assert.Equal(t, "test next steps", summary.NextSteps.String)
|
||||
assert.Equal(t, "test notes", summary.Notes.String)
|
||||
}
|
||||
|
||||
func TestScanSummaryRows(t *testing.T) {
|
||||
db, _, cleanup := testDB(t)
|
||||
defer cleanup()
|
||||
createBaseTables(t, db)
|
||||
seedSession(t, db, "claude-123", "sdk-123", "test-project")
|
||||
|
||||
// Insert multiple summaries
|
||||
_, err := db.Exec(`
|
||||
INSERT INTO session_summaries (sdk_session_id, project, request, investigated, learned, completed, next_steps, notes, prompt_number, discovery_tokens, created_at, created_at_epoch)
|
||||
VALUES
|
||||
('sdk-123', 'test-project', 'request 1', '', '', '', '', '', 1, 0, '2025-01-01T00:00:00Z', 1704067200000),
|
||||
('sdk-123', 'test-project', 'request 2', '', '', '', '', '', 2, 0, '2025-01-02T00:00:00Z', 1704153600000)
|
||||
`)
|
||||
require.NoError(t, err)
|
||||
|
||||
rows, err := db.Query(`
|
||||
SELECT id, sdk_session_id, project, request, investigated, learned, completed, next_steps, notes, prompt_number, discovery_tokens, created_at, created_at_epoch
|
||||
FROM session_summaries WHERE sdk_session_id = ? ORDER BY id
|
||||
`, "sdk-123")
|
||||
require.NoError(t, err)
|
||||
defer rows.Close()
|
||||
|
||||
summaries, err := scanSummaryRows(rows)
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, summaries, 2)
|
||||
assert.Equal(t, "request 1", summaries[0].Request.String)
|
||||
assert.Equal(t, "request 2", summaries[1].Request.String)
|
||||
}
|
||||
|
||||
func TestScanPromptWithSession(t *testing.T) {
|
||||
db, _, cleanup := testDB(t)
|
||||
defer cleanup()
|
||||
createBaseTables(t, db)
|
||||
seedSession(t, db, "claude-123", "sdk-123", "test-project")
|
||||
|
||||
// Insert a test prompt
|
||||
_, err := db.Exec(`
|
||||
INSERT INTO user_prompts (claude_session_id, prompt_number, prompt_text, matched_observations, created_at, created_at_epoch)
|
||||
VALUES ('claude-123', 1, 'test prompt', 5, '2025-01-01T00:00:00Z', 1704067200000)
|
||||
`)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Query with session join
|
||||
row := db.QueryRow(`
|
||||
SELECT p.id, p.claude_session_id, p.prompt_number, p.prompt_text, p.matched_observations, p.created_at, p.created_at_epoch, s.project, s.sdk_session_id
|
||||
FROM user_prompts p
|
||||
JOIN sdk_sessions s ON p.claude_session_id = s.claude_session_id
|
||||
WHERE p.claude_session_id = ?
|
||||
`, "claude-123")
|
||||
|
||||
prompt, err := scanPromptWithSession(row)
|
||||
require.NoError(t, err)
|
||||
assert.NotNil(t, prompt)
|
||||
assert.Equal(t, "claude-123", prompt.ClaudeSessionID)
|
||||
assert.Equal(t, 1, prompt.PromptNumber)
|
||||
assert.Equal(t, "test prompt", prompt.PromptText)
|
||||
assert.Equal(t, 5, prompt.MatchedObservations)
|
||||
assert.Equal(t, "test-project", prompt.Project)
|
||||
assert.Equal(t, "sdk-123", prompt.SDKSessionID)
|
||||
}
|
||||
|
||||
func TestScanPromptWithSessionRows(t *testing.T) {
|
||||
db, _, cleanup := testDB(t)
|
||||
defer cleanup()
|
||||
createBaseTables(t, db)
|
||||
seedSession(t, db, "claude-123", "sdk-123", "test-project")
|
||||
|
||||
// Insert multiple prompts
|
||||
_, err := db.Exec(`
|
||||
INSERT INTO user_prompts (claude_session_id, prompt_number, prompt_text, matched_observations, created_at, created_at_epoch)
|
||||
VALUES
|
||||
('claude-123', 1, 'prompt one', 3, '2025-01-01T00:00:00Z', 1704067200000),
|
||||
('claude-123', 2, 'prompt two', 5, '2025-01-02T00:00:00Z', 1704153600000)
|
||||
`)
|
||||
require.NoError(t, err)
|
||||
|
||||
rows, err := db.Query(`
|
||||
SELECT p.id, p.claude_session_id, p.prompt_number, p.prompt_text, p.matched_observations, p.created_at, p.created_at_epoch, s.project, s.sdk_session_id
|
||||
FROM user_prompts p
|
||||
JOIN sdk_sessions s ON p.claude_session_id = s.claude_session_id
|
||||
WHERE p.claude_session_id = ? ORDER BY p.id
|
||||
`, "claude-123")
|
||||
require.NoError(t, err)
|
||||
defer rows.Close()
|
||||
|
||||
prompts, err := scanPromptWithSessionRows(rows)
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, prompts, 2)
|
||||
assert.Equal(t, "prompt one", prompts[0].PromptText)
|
||||
assert.Equal(t, "prompt two", prompts[1].PromptText)
|
||||
}
|
||||
|
||||
func TestParseLimitParam_HTTPRequest(t *testing.T) {
|
||||
// Test with an actual HTTP request
|
||||
handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
limit := ParseLimitParam(r, 25)
|
||||
if limit != 50 {
|
||||
t.Errorf("Expected limit 50, got %d", limit)
|
||||
}
|
||||
})
|
||||
|
||||
req := httptest.NewRequest("GET", "http://example.com/api?limit=50", nil)
|
||||
w := httptest.NewRecorder()
|
||||
handler.ServeHTTP(w, req)
|
||||
}
|
||||
@@ -9,9 +9,9 @@ import (
|
||||
|
||||
// Migration represents a database schema migration.
|
||||
type Migration struct {
|
||||
Version int
|
||||
Name string
|
||||
SQL string
|
||||
Version int
|
||||
}
|
||||
|
||||
// Migrations is the list of all database migrations in order.
|
||||
@@ -272,224 +272,6 @@ var Migrations = []Migration{
|
||||
);
|
||||
`,
|
||||
},
|
||||
{
|
||||
Version: 18,
|
||||
Name: "user_prompts_unique_constraint",
|
||||
SQL: `
|
||||
-- Add unique constraint to prevent duplicate prompts
|
||||
-- This fixes a bug where the user-prompt hook could fire multiple times
|
||||
-- creating duplicate prompt records with incrementing numbers
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS idx_user_prompts_session_number_unique
|
||||
ON user_prompts(claude_session_id, prompt_number);
|
||||
`,
|
||||
},
|
||||
{
|
||||
Version: 19,
|
||||
Name: "vectors_with_model_version",
|
||||
SQL: `
|
||||
-- Drop old vectors table (virtual tables cannot be altered)
|
||||
DROP TABLE IF EXISTS vectors;
|
||||
|
||||
-- Recreate vectors table with model_version column
|
||||
-- Uses bge-small-en-v1.5 embeddings (384 dimensions)
|
||||
CREATE VIRTUAL TABLE IF NOT EXISTS vectors USING vec0(
|
||||
doc_id TEXT PRIMARY KEY,
|
||||
embedding float[384],
|
||||
sqlite_id INTEGER,
|
||||
doc_type TEXT,
|
||||
field_type TEXT,
|
||||
project TEXT,
|
||||
scope TEXT,
|
||||
model_version TEXT
|
||||
);
|
||||
`,
|
||||
},
|
||||
{
|
||||
Version: 20,
|
||||
Name: "importance_scoring",
|
||||
SQL: `
|
||||
-- Importance scoring system for observations
|
||||
-- Implements multi-factor scoring: type weight, recency decay, user feedback, concept weights, retrieval boost
|
||||
|
||||
-- Cached importance score (recalculated periodically)
|
||||
ALTER TABLE observations ADD COLUMN importance_score REAL DEFAULT 1.0;
|
||||
|
||||
-- User feedback: -1 = thumbs down, 0 = neutral, 1 = thumbs up
|
||||
ALTER TABLE observations ADD COLUMN user_feedback INTEGER DEFAULT 0;
|
||||
|
||||
-- Retrieval tracking: how many times this observation was returned in searches
|
||||
ALTER TABLE observations ADD COLUMN retrieval_count INTEGER DEFAULT 0;
|
||||
|
||||
-- Last time this observation was retrieved (for analytics)
|
||||
ALTER TABLE observations ADD COLUMN last_retrieved_at_epoch INTEGER;
|
||||
|
||||
-- Timestamp of last score recalculation
|
||||
ALTER TABLE observations ADD COLUMN score_updated_at_epoch INTEGER;
|
||||
|
||||
-- Index for importance-based sorting (primary ordering strategy)
|
||||
CREATE INDEX IF NOT EXISTS idx_observations_importance
|
||||
ON observations(importance_score DESC, created_at_epoch DESC);
|
||||
|
||||
-- Index for finding observations needing score recalculation
|
||||
CREATE INDEX IF NOT EXISTS idx_observations_score_updated
|
||||
ON observations(score_updated_at_epoch);
|
||||
|
||||
-- Configurable concept weights table
|
||||
-- Allows runtime tuning of how much each concept contributes to importance
|
||||
CREATE TABLE IF NOT EXISTS concept_weights (
|
||||
concept TEXT PRIMARY KEY,
|
||||
weight REAL NOT NULL DEFAULT 0.1,
|
||||
updated_at TEXT NOT NULL
|
||||
);
|
||||
|
||||
-- Seed default concept weights (security highest, tooling lowest)
|
||||
INSERT OR IGNORE INTO concept_weights (concept, weight, updated_at) VALUES
|
||||
('security', 0.30, datetime('now')),
|
||||
('gotcha', 0.25, datetime('now')),
|
||||
('best-practice', 0.20, datetime('now')),
|
||||
('anti-pattern', 0.20, datetime('now')),
|
||||
('architecture', 0.15, datetime('now')),
|
||||
('performance', 0.15, datetime('now')),
|
||||
('error-handling', 0.15, datetime('now')),
|
||||
('pattern', 0.10, datetime('now')),
|
||||
('testing', 0.10, datetime('now')),
|
||||
('debugging', 0.10, datetime('now')),
|
||||
('workflow', 0.05, datetime('now')),
|
||||
('tooling', 0.05, datetime('now'));
|
||||
`,
|
||||
},
|
||||
{
|
||||
Version: 21,
|
||||
Name: "observation_conflicts",
|
||||
SQL: `
|
||||
-- Observation conflicts table for tracking contradictions and superseded observations
|
||||
-- Implements Issue #5: Contradiction & Obsolescence Detection
|
||||
CREATE TABLE IF NOT EXISTS observation_conflicts (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
newer_obs_id INTEGER NOT NULL,
|
||||
older_obs_id INTEGER NOT NULL,
|
||||
conflict_type TEXT NOT NULL CHECK(conflict_type IN ('superseded', 'contradicts', 'outdated_pattern')),
|
||||
resolution TEXT NOT NULL CHECK(resolution IN ('prefer_newer', 'prefer_older', 'manual')),
|
||||
reason TEXT,
|
||||
detected_at TEXT NOT NULL,
|
||||
detected_at_epoch INTEGER NOT NULL,
|
||||
resolved INTEGER DEFAULT 0,
|
||||
resolved_at TEXT,
|
||||
FOREIGN KEY(newer_obs_id) REFERENCES observations(id) ON DELETE CASCADE,
|
||||
FOREIGN KEY(older_obs_id) REFERENCES observations(id) ON DELETE CASCADE
|
||||
);
|
||||
|
||||
-- Index for looking up conflicts by observation ID
|
||||
CREATE INDEX IF NOT EXISTS idx_conflicts_newer ON observation_conflicts(newer_obs_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_conflicts_older ON observation_conflicts(older_obs_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_conflicts_unresolved ON observation_conflicts(resolved, detected_at_epoch DESC);
|
||||
|
||||
-- Add is_superseded column to observations for quick filtering
|
||||
-- Set to 1 when this observation has been superseded by a newer one
|
||||
ALTER TABLE observations ADD COLUMN is_superseded INTEGER DEFAULT 0;
|
||||
|
||||
-- Index for filtering out superseded observations in queries
|
||||
CREATE INDEX IF NOT EXISTS idx_observations_superseded ON observations(is_superseded, importance_score DESC);
|
||||
`,
|
||||
},
|
||||
{
|
||||
Version: 22,
|
||||
Name: "patterns_table",
|
||||
SQL: `
|
||||
-- Pattern Recognition Engine (Issue #7)
|
||||
-- Tracks recurring patterns detected across observations
|
||||
-- Enables Claude to reference historical insights: "I've encountered this pattern 12 times."
|
||||
CREATE TABLE IF NOT EXISTS patterns (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
name TEXT NOT NULL,
|
||||
type TEXT NOT NULL CHECK(type IN ('bug', 'refactor', 'architecture', 'anti-pattern', 'best-practice')),
|
||||
description TEXT,
|
||||
signature TEXT, -- JSON array of keywords/concepts for detection
|
||||
recommendation TEXT, -- What works for this pattern
|
||||
frequency INTEGER DEFAULT 1, -- How many times encountered
|
||||
projects TEXT, -- JSON array of projects where seen
|
||||
observation_ids TEXT, -- JSON array of source observation IDs
|
||||
status TEXT DEFAULT 'active' CHECK(status IN ('active', 'deprecated', 'merged')),
|
||||
merged_into_id INTEGER, -- If status is 'merged', which pattern it merged into
|
||||
confidence REAL DEFAULT 0.5, -- Detection confidence (0.0-1.0)
|
||||
last_seen_at TEXT NOT NULL,
|
||||
last_seen_at_epoch INTEGER NOT NULL,
|
||||
created_at TEXT NOT NULL,
|
||||
created_at_epoch INTEGER NOT NULL,
|
||||
FOREIGN KEY(merged_into_id) REFERENCES patterns(id) ON DELETE SET NULL
|
||||
);
|
||||
|
||||
-- Indexes for efficient pattern queries
|
||||
CREATE INDEX IF NOT EXISTS idx_patterns_type ON patterns(type);
|
||||
CREATE INDEX IF NOT EXISTS idx_patterns_status ON patterns(status);
|
||||
CREATE INDEX IF NOT EXISTS idx_patterns_frequency ON patterns(frequency DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_patterns_confidence ON patterns(confidence DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_patterns_last_seen ON patterns(last_seen_at_epoch DESC);
|
||||
|
||||
-- FTS5 virtual table for pattern search
|
||||
CREATE VIRTUAL TABLE IF NOT EXISTS patterns_fts USING fts5(
|
||||
name, description, recommendation,
|
||||
content='patterns',
|
||||
content_rowid='id'
|
||||
);
|
||||
|
||||
-- Triggers for FTS5 sync
|
||||
CREATE TRIGGER IF NOT EXISTS patterns_ai AFTER INSERT ON patterns BEGIN
|
||||
INSERT INTO patterns_fts(rowid, name, description, recommendation)
|
||||
VALUES (new.id, new.name, new.description, new.recommendation);
|
||||
END;
|
||||
|
||||
CREATE TRIGGER IF NOT EXISTS patterns_ad AFTER DELETE ON patterns BEGIN
|
||||
INSERT INTO patterns_fts(patterns_fts, rowid, name, description, recommendation)
|
||||
VALUES('delete', old.id, old.name, old.description, old.recommendation);
|
||||
END;
|
||||
|
||||
CREATE TRIGGER IF NOT EXISTS patterns_au AFTER UPDATE ON patterns BEGIN
|
||||
INSERT INTO patterns_fts(patterns_fts, rowid, name, description, recommendation)
|
||||
VALUES('delete', old.id, old.name, old.description, old.recommendation);
|
||||
INSERT INTO patterns_fts(rowid, name, description, recommendation)
|
||||
VALUES (new.id, new.name, new.description, new.recommendation);
|
||||
END;
|
||||
`,
|
||||
},
|
||||
{
|
||||
Version: 23,
|
||||
Name: "observation_relations",
|
||||
SQL: `
|
||||
-- Knowledge Graph: Observation Relations (Issue #4)
|
||||
-- Tracks explicit relationships between observations for knowledge graph navigation.
|
||||
-- Enables queries like "What caused this bug?" or "What depends on this decision?"
|
||||
CREATE TABLE IF NOT EXISTS observation_relations (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
source_id INTEGER NOT NULL,
|
||||
target_id INTEGER NOT NULL,
|
||||
relation_type TEXT NOT NULL CHECK(relation_type IN ('causes', 'fixes', 'supersedes', 'depends_on', 'relates_to', 'evolves_from')),
|
||||
confidence REAL NOT NULL DEFAULT 0.5,
|
||||
detection_source TEXT NOT NULL CHECK(detection_source IN ('file_overlap', 'embedding_similarity', 'temporal_proximity', 'narrative_mention', 'concept_overlap', 'type_progression')),
|
||||
reason TEXT,
|
||||
created_at TEXT NOT NULL,
|
||||
created_at_epoch INTEGER NOT NULL,
|
||||
FOREIGN KEY(source_id) REFERENCES observations(id) ON DELETE CASCADE,
|
||||
FOREIGN KEY(target_id) REFERENCES observations(id) ON DELETE CASCADE,
|
||||
UNIQUE(source_id, target_id, relation_type)
|
||||
);
|
||||
|
||||
-- Index for finding relations by source observation
|
||||
CREATE INDEX IF NOT EXISTS idx_relations_source ON observation_relations(source_id);
|
||||
|
||||
-- Index for finding relations by target observation
|
||||
CREATE INDEX IF NOT EXISTS idx_relations_target ON observation_relations(target_id);
|
||||
|
||||
-- Index for relation type queries
|
||||
CREATE INDEX IF NOT EXISTS idx_relations_type ON observation_relations(relation_type);
|
||||
|
||||
-- Index for confidence-based filtering
|
||||
CREATE INDEX IF NOT EXISTS idx_relations_confidence ON observation_relations(confidence DESC);
|
||||
|
||||
-- Index for finding all relations involving an observation (either direction)
|
||||
CREATE INDEX IF NOT EXISTS idx_relations_both ON observation_relations(source_id, target_id);
|
||||
`,
|
||||
},
|
||||
}
|
||||
|
||||
// MigrationManager handles database schema migrations.
|
||||
@@ -539,11 +321,11 @@ func (m *MigrationManager) ApplyMigration(migration Migration) error {
|
||||
if err != nil {
|
||||
return fmt.Errorf("begin transaction: %w", err)
|
||||
}
|
||||
defer func() { _ = tx.Rollback() }()
|
||||
defer tx.Rollback()
|
||||
|
||||
// Execute migration SQL
|
||||
if _, execErr := tx.Exec(migration.SQL); execErr != nil {
|
||||
return fmt.Errorf("execute migration %d (%s): %w", migration.Version, migration.Name, execErr)
|
||||
if _, err := tx.Exec(migration.SQL); err != nil {
|
||||
return fmt.Errorf("execute migration %d (%s): %w", migration.Version, migration.Name, err)
|
||||
}
|
||||
|
||||
// Record migration
|
||||
|
||||
@@ -1,196 +0,0 @@
|
||||
package sqlite
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestNewMigrationManager(t *testing.T) {
|
||||
db, _, cleanup := testDB(t)
|
||||
defer cleanup()
|
||||
|
||||
manager := NewMigrationManager(db)
|
||||
require.NotNil(t, manager)
|
||||
assert.Equal(t, db, manager.db)
|
||||
}
|
||||
|
||||
func TestMigrationManager_EnsureSchemaVersionsTable(t *testing.T) {
|
||||
db, _, cleanup := testDB(t)
|
||||
defer cleanup()
|
||||
|
||||
manager := NewMigrationManager(db)
|
||||
|
||||
// Should create table without error
|
||||
err := manager.EnsureSchemaVersionsTable()
|
||||
require.NoError(t, err)
|
||||
|
||||
// Table should exist
|
||||
var count int
|
||||
err = db.QueryRow("SELECT COUNT(*) FROM schema_versions").Scan(&count)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 0, count) // Empty table
|
||||
|
||||
// Calling again should not error (IF NOT EXISTS)
|
||||
err = manager.EnsureSchemaVersionsTable()
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
func TestMigrationManager_GetAppliedVersions_Empty(t *testing.T) {
|
||||
db, _, cleanup := testDB(t)
|
||||
defer cleanup()
|
||||
|
||||
manager := NewMigrationManager(db)
|
||||
err := manager.EnsureSchemaVersionsTable()
|
||||
require.NoError(t, err)
|
||||
|
||||
versions, err := manager.GetAppliedVersions()
|
||||
require.NoError(t, err)
|
||||
assert.Empty(t, versions)
|
||||
}
|
||||
|
||||
func TestMigrationManager_GetAppliedVersions_WithVersions(t *testing.T) {
|
||||
db, _, cleanup := testDB(t)
|
||||
defer cleanup()
|
||||
|
||||
manager := NewMigrationManager(db)
|
||||
err := manager.EnsureSchemaVersionsTable()
|
||||
require.NoError(t, err)
|
||||
|
||||
// Insert some versions
|
||||
_, err = db.Exec("INSERT INTO schema_versions (version, applied_at) VALUES (1, '2025-01-01T00:00:00Z')")
|
||||
require.NoError(t, err)
|
||||
_, err = db.Exec("INSERT INTO schema_versions (version, applied_at) VALUES (2, '2025-01-02T00:00:00Z')")
|
||||
require.NoError(t, err)
|
||||
|
||||
versions, err := manager.GetAppliedVersions()
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, versions, 2)
|
||||
assert.True(t, versions[1])
|
||||
assert.True(t, versions[2])
|
||||
assert.False(t, versions[3]) // Not applied
|
||||
}
|
||||
|
||||
func TestMigrationManager_ApplyMigration(t *testing.T) {
|
||||
db, _, cleanup := testDB(t)
|
||||
defer cleanup()
|
||||
|
||||
manager := NewMigrationManager(db)
|
||||
err := manager.EnsureSchemaVersionsTable()
|
||||
require.NoError(t, err)
|
||||
|
||||
// Apply a simple migration
|
||||
migration := Migration{
|
||||
Version: 100,
|
||||
Name: "test_migration",
|
||||
SQL: "CREATE TABLE test_table (id INTEGER PRIMARY KEY, name TEXT)",
|
||||
}
|
||||
|
||||
err = manager.ApplyMigration(migration)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Verify table was created
|
||||
var count int
|
||||
err = db.QueryRow("SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name='test_table'").Scan(&count)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 1, count)
|
||||
|
||||
// Verify migration was recorded
|
||||
var version int
|
||||
err = db.QueryRow("SELECT version FROM schema_versions WHERE version = 100").Scan(&version)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 100, version)
|
||||
}
|
||||
|
||||
func TestMigrationManager_ApplyMigration_InvalidSQL(t *testing.T) {
|
||||
db, _, cleanup := testDB(t)
|
||||
defer cleanup()
|
||||
|
||||
manager := NewMigrationManager(db)
|
||||
err := manager.EnsureSchemaVersionsTable()
|
||||
require.NoError(t, err)
|
||||
|
||||
// Try to apply invalid migration
|
||||
migration := Migration{
|
||||
Version: 100,
|
||||
Name: "invalid_migration",
|
||||
SQL: "INVALID SQL SYNTAX",
|
||||
}
|
||||
|
||||
err = manager.ApplyMigration(migration)
|
||||
assert.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "execute migration 100")
|
||||
}
|
||||
|
||||
func TestMigrationManager_RunMigrations_SingleMigration(t *testing.T) {
|
||||
db, _, cleanup := testDB(t)
|
||||
defer cleanup()
|
||||
|
||||
// Create a test migration manager with a subset of migrations
|
||||
manager := NewMigrationManager(db)
|
||||
|
||||
// First ensure schema versions table exists
|
||||
err := manager.EnsureSchemaVersionsTable()
|
||||
require.NoError(t, err)
|
||||
|
||||
// Apply first migration manually
|
||||
err = manager.ApplyMigration(Migrations[0])
|
||||
require.NoError(t, err)
|
||||
|
||||
// Verify the first migration version was recorded
|
||||
versions, err := manager.GetAppliedVersions()
|
||||
require.NoError(t, err)
|
||||
assert.True(t, versions[Migrations[0].Version])
|
||||
}
|
||||
|
||||
func TestMigrationManager_RunMigrations_SkipsApplied(t *testing.T) {
|
||||
db, _, cleanup := testDB(t)
|
||||
defer cleanup()
|
||||
|
||||
manager := NewMigrationManager(db)
|
||||
err := manager.EnsureSchemaVersionsTable()
|
||||
require.NoError(t, err)
|
||||
|
||||
// Mark some migrations as already applied
|
||||
_, err = db.Exec("INSERT INTO schema_versions (version, applied_at) VALUES (4, '2025-01-01T00:00:00Z')")
|
||||
require.NoError(t, err)
|
||||
|
||||
// Get applied versions
|
||||
versions, err := manager.GetAppliedVersions()
|
||||
require.NoError(t, err)
|
||||
assert.True(t, versions[4])
|
||||
}
|
||||
|
||||
func TestMigration_Struct(t *testing.T) {
|
||||
m := Migration{
|
||||
Version: 1,
|
||||
Name: "test",
|
||||
SQL: "SELECT 1",
|
||||
}
|
||||
|
||||
assert.Equal(t, 1, m.Version)
|
||||
assert.Equal(t, "test", m.Name)
|
||||
assert.Equal(t, "SELECT 1", m.SQL)
|
||||
}
|
||||
|
||||
func TestMigrations_List(t *testing.T) {
|
||||
// Verify migrations are ordered correctly
|
||||
assert.NotEmpty(t, Migrations)
|
||||
|
||||
// Verify all migrations have required fields
|
||||
for i, m := range Migrations {
|
||||
assert.Greater(t, m.Version, 0, "Migration %d has invalid version", i)
|
||||
assert.NotEmpty(t, m.Name, "Migration %d has empty name", i)
|
||||
assert.NotEmpty(t, m.SQL, "Migration %d has empty SQL", i)
|
||||
}
|
||||
|
||||
// Verify key migrations exist
|
||||
versionSet := make(map[int]bool)
|
||||
for _, m := range Migrations {
|
||||
versionSet[m.Version] = true
|
||||
}
|
||||
|
||||
assert.True(t, versionSet[4], "Should have sdk_agent_architecture migration")
|
||||
assert.True(t, versionSet[17], "Should have sqlite_vec_vectors migration")
|
||||
}
|
||||
@@ -11,27 +11,14 @@ import (
|
||||
"github.com/lukaszraczylo/claude-mnemonic/pkg/models"
|
||||
)
|
||||
|
||||
// observationColumns is the standard list of columns to select for observations.
|
||||
// This ensures consistency across all observation queries and includes importance scoring fields.
|
||||
const observationColumns = `id, sdk_session_id, project, COALESCE(scope, 'project') as scope, type,
|
||||
title, subtitle, facts, narrative, concepts, files_read, files_modified, file_mtimes,
|
||||
prompt_number, discovery_tokens, created_at, created_at_epoch,
|
||||
COALESCE(importance_score, 1.0) as importance_score,
|
||||
COALESCE(user_feedback, 0) as user_feedback,
|
||||
COALESCE(retrieval_count, 0) as retrieval_count,
|
||||
last_retrieved_at_epoch, score_updated_at_epoch,
|
||||
COALESCE(is_superseded, 0) as is_superseded`
|
||||
|
||||
// CleanupFunc is a callback for when observations are cleaned up.
|
||||
// Receives the IDs of deleted observations for downstream cleanup (e.g., vector DB).
|
||||
type CleanupFunc func(ctx context.Context, deletedIDs []int64)
|
||||
|
||||
// ObservationStore provides observation-related database operations.
|
||||
type ObservationStore struct {
|
||||
store *Store
|
||||
cleanupFunc CleanupFunc
|
||||
conflictStore *ConflictStore
|
||||
relationStore *RelationStore
|
||||
store *Store
|
||||
cleanupFunc CleanupFunc
|
||||
}
|
||||
|
||||
// NewObservationStore creates a new observation store.
|
||||
@@ -44,16 +31,6 @@ func (s *ObservationStore) SetCleanupFunc(fn CleanupFunc) {
|
||||
s.cleanupFunc = fn
|
||||
}
|
||||
|
||||
// SetConflictStore sets the conflict store for conflict detection.
|
||||
func (s *ObservationStore) SetConflictStore(conflictStore *ConflictStore) {
|
||||
s.conflictStore = conflictStore
|
||||
}
|
||||
|
||||
// SetRelationStore sets the relation store for relationship detection.
|
||||
func (s *ObservationStore) SetRelationStore(relationStore *RelationStore) {
|
||||
s.relationStore = relationStore
|
||||
}
|
||||
|
||||
// StoreObservation stores a new observation.
|
||||
func (s *ObservationStore) StoreObservation(ctx context.Context, sdkSessionID, project string, obs *models.ParsedObservation, promptNumber int, discoveryTokens int64) (int64, int64, error) {
|
||||
now := time.Now()
|
||||
@@ -109,112 +86,9 @@ func (s *ObservationStore) StoreObservation(ctx context.Context, sdkSessionID, p
|
||||
}(project)
|
||||
}
|
||||
|
||||
// Detect conflicts with existing observations (async to not block handler)
|
||||
if s.conflictStore != nil && project != "" {
|
||||
go func(newObsID int64, proj string, parsedObs *models.ParsedObservation) {
|
||||
conflictCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
s.detectAndStoreConflicts(conflictCtx, newObsID, proj, parsedObs)
|
||||
}(id, project, obs)
|
||||
}
|
||||
|
||||
// Detect relationships with existing observations (async to not block handler)
|
||||
if s.relationStore != nil && project != "" {
|
||||
go func(newObsID int64, proj string) {
|
||||
relationCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
s.detectAndStoreRelations(relationCtx, newObsID, proj)
|
||||
}(id, project)
|
||||
}
|
||||
|
||||
return id, nowEpoch, nil
|
||||
}
|
||||
|
||||
// detectAndStoreConflicts detects conflicts between a new observation and existing ones.
|
||||
func (s *ObservationStore) detectAndStoreConflicts(ctx context.Context, newObsID int64, project string, parsedObs *models.ParsedObservation) {
|
||||
// Fetch the newly stored observation
|
||||
newObs, err := s.GetObservationByID(ctx, newObsID)
|
||||
if err != nil || newObs == nil {
|
||||
return
|
||||
}
|
||||
|
||||
// Fetch recent observations from the same project to check for conflicts
|
||||
existing, err := s.GetRecentObservations(ctx, project, 50)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
// Detect conflicts
|
||||
conflicts := models.DetectConflictsWithExisting(newObs, existing)
|
||||
|
||||
// Store conflicts and mark superseded observations
|
||||
var supersededIDs []int64
|
||||
for _, result := range conflicts {
|
||||
for _, olderID := range result.OlderObsIDs {
|
||||
conflict := models.NewObservationConflict(
|
||||
newObsID, olderID,
|
||||
result.Type, result.Resolution, result.Reason,
|
||||
)
|
||||
if _, err := s.conflictStore.StoreConflict(ctx, conflict); err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
// If resolution is to prefer newer, mark older as superseded
|
||||
if result.Resolution == models.ResolutionPreferNewer {
|
||||
supersededIDs = append(supersededIDs, olderID)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Mark superseded observations
|
||||
if len(supersededIDs) > 0 {
|
||||
_ = s.conflictStore.MarkObservationsSuperseded(ctx, supersededIDs)
|
||||
}
|
||||
|
||||
// Cleanup old superseded observations (older than 3 days)
|
||||
deletedIDs, _ := s.conflictStore.CleanupSupersededObservations(ctx, project)
|
||||
if len(deletedIDs) > 0 && s.cleanupFunc != nil {
|
||||
s.cleanupFunc(ctx, deletedIDs)
|
||||
}
|
||||
}
|
||||
|
||||
// MinRelationConfidence is the minimum confidence threshold for storing relations.
|
||||
const MinRelationConfidence = 0.4
|
||||
|
||||
// detectAndStoreRelations detects relationships between a new observation and existing ones.
|
||||
func (s *ObservationStore) detectAndStoreRelations(ctx context.Context, newObsID int64, project string) {
|
||||
// Fetch the newly stored observation
|
||||
newObs, err := s.GetObservationByID(ctx, newObsID)
|
||||
if err != nil || newObs == nil {
|
||||
return
|
||||
}
|
||||
|
||||
// Fetch recent observations from the same project to check for relations
|
||||
existing, err := s.GetRecentObservations(ctx, project, 50)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
// Detect relationships using the models package detection logic
|
||||
results := models.DetectRelationsWithExisting(newObs, existing, MinRelationConfidence)
|
||||
if len(results) == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
// Convert detection results to relation objects
|
||||
relations := make([]*models.ObservationRelation, len(results))
|
||||
for i, r := range results {
|
||||
relations[i] = models.NewObservationRelation(
|
||||
r.SourceID, r.TargetID,
|
||||
r.RelationType, r.Confidence,
|
||||
r.DetectionSource, r.Reason,
|
||||
)
|
||||
}
|
||||
|
||||
// Store all relations
|
||||
_ = s.relationStore.StoreRelations(ctx, relations)
|
||||
}
|
||||
|
||||
// ensureSessionExists creates a session if it doesn't exist.
|
||||
func (s *ObservationStore) ensureSessionExists(ctx context.Context, sdkSessionID, project string) error {
|
||||
return EnsureSessionExists(ctx, s.store, sdkSessionID, project)
|
||||
@@ -222,7 +96,13 @@ func (s *ObservationStore) ensureSessionExists(ctx context.Context, sdkSessionID
|
||||
|
||||
// GetObservationByID retrieves an observation by ID.
|
||||
func (s *ObservationStore) GetObservationByID(ctx context.Context, id int64) (*models.Observation, error) {
|
||||
query := `SELECT ` + observationColumns + ` FROM observations WHERE id = ?`
|
||||
const query = `
|
||||
SELECT id, sdk_session_id, project, COALESCE(scope, 'project') as scope, type, title, subtitle, facts, narrative,
|
||||
concepts, files_read, files_modified, file_mtimes, prompt_number, discovery_tokens,
|
||||
created_at, created_at_epoch
|
||||
FROM observations
|
||||
WHERE id = ?
|
||||
`
|
||||
|
||||
obs, err := scanObservation(s.store.QueryRowContext(ctx, query, id))
|
||||
if err == sql.ErrNoRows {
|
||||
@@ -232,7 +112,6 @@ func (s *ObservationStore) GetObservationByID(ctx context.Context, id int64) (*m
|
||||
}
|
||||
|
||||
// GetObservationsByIDs retrieves observations by a list of IDs.
|
||||
// Results are ordered by importance_score DESC by default, with created_at_epoch as secondary sort.
|
||||
func (s *ObservationStore) GetObservationsByIDs(ctx context.Context, ids []int64, orderBy string, limit int) ([]*models.Observation, error) {
|
||||
if len(ids) == 0 {
|
||||
return nil, nil
|
||||
@@ -240,22 +119,18 @@ func (s *ObservationStore) GetObservationsByIDs(ctx context.Context, ids []int64
|
||||
|
||||
// Build query with placeholders
|
||||
// #nosec G202 -- query uses parameterized placeholders, not user input
|
||||
query := `SELECT ` + observationColumns + `
|
||||
query := `
|
||||
SELECT id, sdk_session_id, project, COALESCE(scope, 'project') as scope, type, title, subtitle, facts, narrative,
|
||||
concepts, files_read, files_modified, file_mtimes, prompt_number, discovery_tokens,
|
||||
created_at, created_at_epoch
|
||||
FROM observations
|
||||
WHERE id IN (?` + repeatPlaceholders(len(ids)-1) + `)
|
||||
ORDER BY `
|
||||
ORDER BY created_at_epoch `
|
||||
|
||||
// Default to importance-based ordering
|
||||
switch orderBy {
|
||||
case "date_asc":
|
||||
query += "created_at_epoch ASC"
|
||||
case "date_desc":
|
||||
query += "created_at_epoch DESC"
|
||||
case "importance":
|
||||
query += "importance_score DESC, created_at_epoch DESC"
|
||||
default:
|
||||
// Default: importance first, then recency
|
||||
query += "COALESCE(importance_score, 1.0) DESC, created_at_epoch DESC"
|
||||
if orderBy == "date_asc" {
|
||||
query += "ASC"
|
||||
} else {
|
||||
query += "DESC"
|
||||
}
|
||||
|
||||
if limit > 0 {
|
||||
@@ -279,56 +154,14 @@ func (s *ObservationStore) GetObservationsByIDs(ctx context.Context, ids []int64
|
||||
|
||||
// GetRecentObservations retrieves recent observations for a project.
|
||||
// This includes project-scoped observations for the specified project AND global observations.
|
||||
// Results are ordered by importance_score DESC, then created_at_epoch DESC.
|
||||
func (s *ObservationStore) GetRecentObservations(ctx context.Context, project string, limit int) ([]*models.Observation, error) {
|
||||
query := `SELECT ` + observationColumns + `
|
||||
const query = `
|
||||
SELECT id, sdk_session_id, project, COALESCE(scope, 'project') as scope, type, title, subtitle, facts, narrative,
|
||||
concepts, files_read, files_modified, file_mtimes, prompt_number, discovery_tokens,
|
||||
created_at, created_at_epoch
|
||||
FROM observations
|
||||
WHERE (project = ? AND (scope IS NULL OR scope = 'project'))
|
||||
OR scope = 'global'
|
||||
ORDER BY COALESCE(importance_score, 1.0) DESC, created_at_epoch DESC
|
||||
LIMIT ?
|
||||
`
|
||||
|
||||
rows, err := s.store.QueryContext(ctx, query, project, limit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
return scanObservationRows(rows)
|
||||
}
|
||||
|
||||
// GetActiveObservations retrieves recent non-superseded observations for a project.
|
||||
// This excludes observations that have been marked as superseded by newer ones.
|
||||
// Use this for context injection where you want to avoid outdated/contradicted advice.
|
||||
// Results are ordered by importance_score DESC, then created_at_epoch DESC.
|
||||
func (s *ObservationStore) GetActiveObservations(ctx context.Context, project string, limit int) ([]*models.Observation, error) {
|
||||
query := `SELECT ` + observationColumns + `
|
||||
FROM observations
|
||||
WHERE ((project = ? AND (scope IS NULL OR scope = 'project'))
|
||||
OR scope = 'global')
|
||||
AND COALESCE(is_superseded, 0) = 0
|
||||
ORDER BY COALESCE(importance_score, 1.0) DESC, created_at_epoch DESC
|
||||
LIMIT ?
|
||||
`
|
||||
|
||||
rows, err := s.store.QueryContext(ctx, query, project, limit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
return scanObservationRows(rows)
|
||||
}
|
||||
|
||||
// GetSupersededObservations retrieves observations that have been superseded by newer ones.
|
||||
// Use this for verification/debugging to see which observations were marked as outdated.
|
||||
// Results are ordered by created_at_epoch DESC.
|
||||
func (s *ObservationStore) GetSupersededObservations(ctx context.Context, project string, limit int) ([]*models.Observation, error) {
|
||||
query := `SELECT ` + observationColumns + `
|
||||
FROM observations
|
||||
WHERE project = ?
|
||||
AND COALESCE(is_superseded, 0) = 1
|
||||
ORDER BY created_at_epoch DESC
|
||||
LIMIT ?
|
||||
`
|
||||
@@ -342,27 +175,6 @@ func (s *ObservationStore) GetSupersededObservations(ctx context.Context, projec
|
||||
return scanObservationRows(rows)
|
||||
}
|
||||
|
||||
// GetObservationsByProjectStrict retrieves observations strictly for a specific project.
|
||||
// Unlike GetRecentObservations, this does NOT include global observations from other projects.
|
||||
// Use this for dashboard filtering where the user expects to see only that project's data.
|
||||
// Results are ordered by importance_score DESC, then created_at_epoch DESC.
|
||||
func (s *ObservationStore) GetObservationsByProjectStrict(ctx context.Context, project string, limit int) ([]*models.Observation, error) {
|
||||
query := `SELECT ` + observationColumns + `
|
||||
FROM observations
|
||||
WHERE project = ?
|
||||
ORDER BY COALESCE(importance_score, 1.0) DESC, created_at_epoch DESC
|
||||
LIMIT ?
|
||||
`
|
||||
|
||||
rows, err := s.store.QueryContext(ctx, query, project, limit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
return scanObservationRows(rows)
|
||||
}
|
||||
|
||||
// GetObservationCount returns the count of observations for a project (including global).
|
||||
func (s *ObservationStore) GetObservationCount(ctx context.Context, project string) (int, error) {
|
||||
const query = `
|
||||
@@ -375,11 +187,13 @@ func (s *ObservationStore) GetObservationCount(ctx context.Context, project stri
|
||||
}
|
||||
|
||||
// GetAllRecentObservations retrieves recent observations across all projects.
|
||||
// Results are ordered by importance_score DESC, then created_at_epoch DESC.
|
||||
func (s *ObservationStore) GetAllRecentObservations(ctx context.Context, limit int) ([]*models.Observation, error) {
|
||||
query := `SELECT ` + observationColumns + `
|
||||
const query = `
|
||||
SELECT id, sdk_session_id, project, COALESCE(scope, 'project') as scope, type, title, subtitle, facts, narrative,
|
||||
concepts, files_read, files_modified, file_mtimes, prompt_number, discovery_tokens,
|
||||
created_at, created_at_epoch
|
||||
FROM observations
|
||||
ORDER BY COALESCE(importance_score, 1.0) DESC, created_at_epoch DESC
|
||||
ORDER BY created_at_epoch DESC
|
||||
LIMIT ?
|
||||
`
|
||||
|
||||
@@ -392,24 +206,7 @@ func (s *ObservationStore) GetAllRecentObservations(ctx context.Context, limit i
|
||||
return scanObservationRows(rows)
|
||||
}
|
||||
|
||||
// GetAllObservations retrieves all observations (for vector rebuild).
|
||||
func (s *ObservationStore) GetAllObservations(ctx context.Context) ([]*models.Observation, error) {
|
||||
query := `SELECT ` + observationColumns + `
|
||||
FROM observations
|
||||
ORDER BY id
|
||||
`
|
||||
|
||||
rows, err := s.store.QueryContext(ctx, query)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
return scanObservationRows(rows)
|
||||
}
|
||||
|
||||
// SearchObservationsFTS performs full-text search on observations.
|
||||
// Results are ordered by FTS rank (relevance), then by importance_score.
|
||||
func (s *ObservationStore) SearchObservationsFTS(ctx context.Context, query, project string, limit int) ([]*models.Observation, error) {
|
||||
if limit <= 0 {
|
||||
limit = 10
|
||||
@@ -425,21 +222,15 @@ func (s *ObservationStore) SearchObservationsFTS(ctx context.Context, query, pro
|
||||
ftsTerms := strings.Join(keywords, " OR ")
|
||||
|
||||
// Use FTS5 to search title, subtitle, and narrative
|
||||
// Include importance scoring columns and order by rank then importance
|
||||
ftsQuery := `
|
||||
const ftsQuery = `
|
||||
SELECT o.id, o.sdk_session_id, o.project, COALESCE(o.scope, 'project') as scope, o.type,
|
||||
o.title, o.subtitle, o.facts, o.narrative, o.concepts, o.files_read, o.files_modified,
|
||||
o.file_mtimes, o.prompt_number, o.discovery_tokens, o.created_at, o.created_at_epoch,
|
||||
COALESCE(o.importance_score, 1.0) as importance_score,
|
||||
COALESCE(o.user_feedback, 0) as user_feedback,
|
||||
COALESCE(o.retrieval_count, 0) as retrieval_count,
|
||||
o.last_retrieved_at_epoch, o.score_updated_at_epoch,
|
||||
COALESCE(o.is_superseded, 0) as is_superseded
|
||||
o.file_mtimes, o.prompt_number, o.discovery_tokens, o.created_at, o.created_at_epoch
|
||||
FROM observations o
|
||||
JOIN observations_fts fts ON o.id = fts.rowid
|
||||
WHERE observations_fts MATCH ?
|
||||
AND (o.project = ? OR o.scope = 'global')
|
||||
ORDER BY rank, COALESCE(o.importance_score, 1.0) DESC
|
||||
ORDER BY rank
|
||||
LIMIT ?
|
||||
`
|
||||
|
||||
@@ -464,7 +255,6 @@ func (s *ObservationStore) SearchObservationsFTS(ctx context.Context, query, pro
|
||||
}
|
||||
|
||||
// searchObservationsLike performs fallback LIKE search on observations.
|
||||
// Results are ordered by importance_score DESC, then created_at_epoch DESC.
|
||||
func (s *ObservationStore) searchObservationsLike(ctx context.Context, keywords []string, project string, limit int) ([]*models.Observation, error) {
|
||||
if len(keywords) == 0 {
|
||||
return nil, nil
|
||||
@@ -481,11 +271,14 @@ func (s *ObservationStore) searchObservationsLike(ctx context.Context, keywords
|
||||
}
|
||||
|
||||
// #nosec G202 -- query uses parameterized placeholders, not user input
|
||||
query := `SELECT ` + observationColumns + `
|
||||
query := `
|
||||
SELECT id, sdk_session_id, project, COALESCE(scope, 'project') as scope, type,
|
||||
title, subtitle, facts, narrative, concepts, files_read, files_modified,
|
||||
file_mtimes, prompt_number, discovery_tokens, created_at, created_at_epoch
|
||||
FROM observations
|
||||
WHERE (` + strings.Join(conditions, " OR ") + `)
|
||||
AND (project = ? OR scope = 'global')
|
||||
ORDER BY COALESCE(importance_score, 1.0) DESC, created_at_epoch DESC
|
||||
ORDER BY created_at_epoch DESC
|
||||
LIMIT ?
|
||||
`
|
||||
args = append(args, project, limit)
|
||||
@@ -585,13 +378,13 @@ func (s *ObservationStore) CleanupOldObservations(ctx context.Context, project s
|
||||
var toDelete []int64
|
||||
for rows.Next() {
|
||||
var id int64
|
||||
if scanErr := rows.Scan(&id); scanErr != nil {
|
||||
return nil, scanErr
|
||||
if err := rows.Scan(&id); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
toDelete = append(toDelete, id)
|
||||
}
|
||||
if rowsErr := rows.Err(); rowsErr != nil {
|
||||
return nil, rowsErr
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if len(toDelete) == 0 {
|
||||
@@ -629,11 +422,6 @@ func scanObservation(scanner interface{ Scan(...interface{}) error }) (*models.O
|
||||
&obs.Concepts, &obs.FilesRead, &obs.FilesModified, &obs.FileMtimes,
|
||||
&obs.PromptNumber, &obs.DiscoveryTokens,
|
||||
&obs.CreatedAt, &obs.CreatedAtEpoch,
|
||||
// Importance scoring fields
|
||||
&obs.ImportanceScore, &obs.UserFeedback, &obs.RetrievalCount,
|
||||
&obs.LastRetrievedAt, &obs.ScoreUpdatedAt,
|
||||
// Conflict detection fields
|
||||
&obs.IsSuperseded,
|
||||
); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
@@ -9,22 +9,8 @@ import (
|
||||
"github.com/lukaszraczylo/claude-mnemonic/pkg/models"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"github.com/stretchr/testify/suite"
|
||||
)
|
||||
|
||||
// testObservationStoreBasic creates an ObservationStore with base tables (no FTS5).
|
||||
func testObservationStoreBasic(t *testing.T) (*ObservationStore, *Store, func()) {
|
||||
t.Helper()
|
||||
|
||||
db, _, cleanup := testDB(t)
|
||||
createBaseTables(t, db)
|
||||
|
||||
store := newStoreFromDB(db)
|
||||
obsStore := NewObservationStore(store)
|
||||
|
||||
return obsStore, store, cleanup
|
||||
}
|
||||
|
||||
// testObservationStore creates an ObservationStore with a test database including FTS5.
|
||||
func testObservationStore(t *testing.T) (*ObservationStore, *Store, func()) {
|
||||
t.Helper()
|
||||
@@ -38,420 +24,6 @@ func testObservationStore(t *testing.T) (*ObservationStore, *Store, func()) {
|
||||
return obsStore, store, cleanup
|
||||
}
|
||||
|
||||
// ObservationStoreSuite is a test suite for ObservationStore operations.
|
||||
type ObservationStoreSuite struct {
|
||||
suite.Suite
|
||||
obsStore *ObservationStore
|
||||
store *Store
|
||||
cleanup func()
|
||||
}
|
||||
|
||||
func (s *ObservationStoreSuite) SetupTest() {
|
||||
s.obsStore, s.store, s.cleanup = testObservationStoreBasic(s.T())
|
||||
}
|
||||
|
||||
func (s *ObservationStoreSuite) TearDownTest() {
|
||||
if s.cleanup != nil {
|
||||
s.cleanup()
|
||||
}
|
||||
}
|
||||
|
||||
func TestObservationStoreSuite(t *testing.T) {
|
||||
suite.Run(t, new(ObservationStoreSuite))
|
||||
}
|
||||
|
||||
// TestStoreObservation_TableDriven tests observation storage with various scenarios.
|
||||
func (s *ObservationStoreSuite) TestStoreObservation_TableDriven() {
|
||||
ctx := context.Background()
|
||||
|
||||
tests := []struct {
|
||||
obs *models.ParsedObservation
|
||||
name string
|
||||
sdkSessionID string
|
||||
project string
|
||||
promptNum int
|
||||
tokens int64
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "basic discovery observation",
|
||||
sdkSessionID: "session-basic",
|
||||
project: "project-a",
|
||||
obs: &models.ParsedObservation{
|
||||
Type: models.ObsTypeDiscovery,
|
||||
Title: "Test Title",
|
||||
Subtitle: "Test Subtitle",
|
||||
Narrative: "Test narrative content",
|
||||
Facts: []string{"Fact 1", "Fact 2"},
|
||||
Concepts: []string{"testing", "golang"},
|
||||
},
|
||||
promptNum: 1,
|
||||
tokens: 100,
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "bugfix observation",
|
||||
sdkSessionID: "session-bugfix",
|
||||
project: "project-b",
|
||||
obs: &models.ParsedObservation{
|
||||
Type: models.ObsTypeBugfix,
|
||||
Title: "Fixed null pointer",
|
||||
Narrative: "Fixed null pointer exception in handler",
|
||||
FilesModified: []string{"handler.go"},
|
||||
},
|
||||
promptNum: 2,
|
||||
tokens: 50,
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "global scope observation",
|
||||
sdkSessionID: "session-global",
|
||||
project: "project-c",
|
||||
obs: &models.ParsedObservation{
|
||||
Type: models.ObsTypeDiscovery,
|
||||
Title: "Security best practice",
|
||||
Narrative: "Always validate user input",
|
||||
Concepts: []string{"security", "best-practice"},
|
||||
},
|
||||
promptNum: 1,
|
||||
tokens: 75,
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "observation with files",
|
||||
sdkSessionID: "session-files",
|
||||
project: "project-d",
|
||||
obs: &models.ParsedObservation{
|
||||
Type: models.ObsTypeFeature,
|
||||
Title: "Added authentication",
|
||||
Narrative: "Implemented JWT authentication",
|
||||
FilesRead: []string{"config.go", "auth.go"},
|
||||
FilesModified: []string{"handler.go", "middleware.go"},
|
||||
FileMtimes: map[string]int64{"handler.go": 1234567890, "middleware.go": 1234567891},
|
||||
},
|
||||
promptNum: 3,
|
||||
tokens: 200,
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "minimal observation",
|
||||
sdkSessionID: "session-minimal",
|
||||
project: "project-e",
|
||||
obs: &models.ParsedObservation{
|
||||
Type: models.ObsTypeChange,
|
||||
},
|
||||
promptNum: 0,
|
||||
tokens: 0,
|
||||
wantErr: false,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
s.Run(tt.name, func() {
|
||||
id, epoch, err := s.obsStore.StoreObservation(ctx, tt.sdkSessionID, tt.project, tt.obs, tt.promptNum, tt.tokens)
|
||||
if tt.wantErr {
|
||||
s.Error(err)
|
||||
return
|
||||
}
|
||||
|
||||
s.NoError(err)
|
||||
s.Greater(id, int64(0))
|
||||
s.Greater(epoch, int64(0))
|
||||
|
||||
// Retrieve and verify
|
||||
retrieved, err := s.obsStore.GetObservationByID(ctx, id)
|
||||
s.NoError(err)
|
||||
s.NotNil(retrieved)
|
||||
s.Equal(id, retrieved.ID)
|
||||
s.Equal(tt.project, retrieved.Project)
|
||||
s.Equal(tt.obs.Type, retrieved.Type)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestGetObservationByID_NotFound tests retrieval of non-existent observation.
|
||||
func (s *ObservationStoreSuite) TestGetObservationByID_NotFound() {
|
||||
ctx := context.Background()
|
||||
|
||||
obs, err := s.obsStore.GetObservationByID(ctx, 99999)
|
||||
s.NoError(err)
|
||||
s.Nil(obs)
|
||||
}
|
||||
|
||||
// TestGetRecentObservations_TableDriven tests recent observations retrieval.
|
||||
func (s *ObservationStoreSuite) TestGetRecentObservations_TableDriven() {
|
||||
ctx := context.Background()
|
||||
|
||||
// Create 15 observations
|
||||
for i := 0; i < 15; i++ {
|
||||
obs := &models.ParsedObservation{
|
||||
Type: models.ObsTypeDiscovery,
|
||||
Title: "Observation " + string(rune('A'+i)),
|
||||
}
|
||||
_, _, err := s.obsStore.StoreObservation(ctx, "session-"+string(rune('0'+i)), "project-a", obs, i, 10)
|
||||
s.NoError(err)
|
||||
time.Sleep(time.Millisecond) // Ensure different timestamps
|
||||
}
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
project string
|
||||
limit int
|
||||
wantCount int
|
||||
}{
|
||||
{
|
||||
name: "limit 5",
|
||||
project: "project-a",
|
||||
limit: 5,
|
||||
wantCount: 5,
|
||||
},
|
||||
{
|
||||
name: "limit 10",
|
||||
project: "project-a",
|
||||
limit: 10,
|
||||
wantCount: 10,
|
||||
},
|
||||
{
|
||||
name: "limit higher than count",
|
||||
project: "project-a",
|
||||
limit: 50,
|
||||
wantCount: 15,
|
||||
},
|
||||
{
|
||||
name: "different project (no results)",
|
||||
project: "project-b",
|
||||
limit: 10,
|
||||
wantCount: 0,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
s.Run(tt.name, func() {
|
||||
observations, err := s.obsStore.GetRecentObservations(ctx, tt.project, tt.limit)
|
||||
s.NoError(err)
|
||||
s.Len(observations, tt.wantCount)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestDeleteObservations_TableDriven tests observation deletion.
|
||||
func (s *ObservationStoreSuite) TestDeleteObservations_TableDriven() {
|
||||
ctx := context.Background()
|
||||
|
||||
// Create observations
|
||||
var ids []int64
|
||||
for i := 0; i < 5; i++ {
|
||||
obs := &models.ParsedObservation{
|
||||
Type: models.ObsTypeDiscovery,
|
||||
Title: "To delete " + string(rune('A'+i)),
|
||||
}
|
||||
id, _, err := s.obsStore.StoreObservation(ctx, "session-del", "project-del", obs, i, 10)
|
||||
s.NoError(err)
|
||||
ids = append(ids, id)
|
||||
}
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
toDelete []int64
|
||||
wantDeleted int64
|
||||
wantRemain int
|
||||
}{
|
||||
{
|
||||
name: "delete none",
|
||||
toDelete: []int64{},
|
||||
wantDeleted: 0,
|
||||
wantRemain: 5,
|
||||
},
|
||||
{
|
||||
name: "delete one",
|
||||
toDelete: ids[0:1],
|
||||
wantDeleted: 1,
|
||||
wantRemain: 4,
|
||||
},
|
||||
{
|
||||
name: "delete remaining",
|
||||
toDelete: ids[1:],
|
||||
wantDeleted: 4,
|
||||
wantRemain: 0,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
s.Run(tt.name, func() {
|
||||
deleted, err := s.obsStore.DeleteObservations(ctx, tt.toDelete)
|
||||
s.NoError(err)
|
||||
s.Equal(tt.wantDeleted, deleted)
|
||||
|
||||
remaining, err := s.obsStore.GetAllRecentObservations(ctx, 100)
|
||||
s.NoError(err)
|
||||
s.Len(remaining, tt.wantRemain)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestGetObservationsByIDs tests retrieval by multiple IDs.
|
||||
func (s *ObservationStoreSuite) TestGetObservationsByIDs() {
|
||||
ctx := context.Background()
|
||||
|
||||
// Create observations
|
||||
var ids []int64
|
||||
for i := 0; i < 5; i++ {
|
||||
obs := &models.ParsedObservation{
|
||||
Type: models.ObsTypeDiscovery,
|
||||
Title: "By ID " + string(rune('A'+i)),
|
||||
}
|
||||
id, _, err := s.obsStore.StoreObservation(ctx, "session-byid", "project-byid", obs, i, 10)
|
||||
s.NoError(err)
|
||||
ids = append(ids, id)
|
||||
time.Sleep(time.Millisecond)
|
||||
}
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
orderBy string
|
||||
queryIDs []int64
|
||||
limit int
|
||||
wantCount int
|
||||
}{
|
||||
{
|
||||
name: "empty IDs",
|
||||
queryIDs: []int64{},
|
||||
orderBy: "date_desc",
|
||||
limit: 10,
|
||||
wantCount: 0,
|
||||
},
|
||||
{
|
||||
name: "single ID",
|
||||
queryIDs: ids[0:1],
|
||||
orderBy: "date_desc",
|
||||
limit: 10,
|
||||
wantCount: 1,
|
||||
},
|
||||
{
|
||||
name: "all IDs",
|
||||
queryIDs: ids,
|
||||
orderBy: "date_desc",
|
||||
limit: 10,
|
||||
wantCount: 5,
|
||||
},
|
||||
{
|
||||
name: "with limit less than IDs",
|
||||
queryIDs: ids,
|
||||
orderBy: "date_desc",
|
||||
limit: 3,
|
||||
wantCount: 3,
|
||||
},
|
||||
{
|
||||
name: "ascending order",
|
||||
queryIDs: ids,
|
||||
orderBy: "date_asc",
|
||||
limit: 10,
|
||||
wantCount: 5,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
s.Run(tt.name, func() {
|
||||
observations, err := s.obsStore.GetObservationsByIDs(ctx, tt.queryIDs, tt.orderBy, tt.limit)
|
||||
if tt.wantCount == 0 {
|
||||
s.NoError(err)
|
||||
s.Nil(observations)
|
||||
} else {
|
||||
s.NoError(err)
|
||||
s.Len(observations, tt.wantCount)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestGlobalScope tests global vs project scope.
|
||||
func (s *ObservationStoreSuite) TestGlobalScope() {
|
||||
ctx := context.Background()
|
||||
|
||||
// Create project-scoped observation
|
||||
projectObs := &models.ParsedObservation{
|
||||
Type: models.ObsTypeDiscovery,
|
||||
Title: "Project specific",
|
||||
Concepts: []string{"project-specific"},
|
||||
}
|
||||
_, _, err := s.obsStore.StoreObservation(ctx, "session-scope", "project-a", projectObs, 1, 10)
|
||||
s.NoError(err)
|
||||
|
||||
// Create global-scoped observation (security concept triggers global)
|
||||
globalObs := &models.ParsedObservation{
|
||||
Type: models.ObsTypeDiscovery,
|
||||
Title: "Global security",
|
||||
Concepts: []string{"security"},
|
||||
}
|
||||
_, _, err = s.obsStore.StoreObservation(ctx, "session-scope", "project-a", globalObs, 2, 10)
|
||||
s.NoError(err)
|
||||
|
||||
// Project-a should see both
|
||||
resultsA, err := s.obsStore.GetRecentObservations(ctx, "project-a", 10)
|
||||
s.NoError(err)
|
||||
s.Len(resultsA, 2)
|
||||
|
||||
// Project-b should only see global
|
||||
resultsB, err := s.obsStore.GetRecentObservations(ctx, "project-b", 10)
|
||||
s.NoError(err)
|
||||
s.Len(resultsB, 1)
|
||||
s.Equal("Global security", resultsB[0].Title.String)
|
||||
s.Equal(models.ScopeGlobal, resultsB[0].Scope)
|
||||
}
|
||||
|
||||
// TestSetCleanupFunc tests the cleanup function callback.
|
||||
func (s *ObservationStoreSuite) TestSetCleanupFunc() {
|
||||
ctx := context.Background()
|
||||
|
||||
var calledWith []int64
|
||||
s.obsStore.SetCleanupFunc(func(ctx context.Context, deletedIDs []int64) {
|
||||
calledWith = deletedIDs
|
||||
})
|
||||
|
||||
// Store an observation
|
||||
obs := &models.ParsedObservation{
|
||||
Type: models.ObsTypeDiscovery,
|
||||
Title: "Test cleanup",
|
||||
}
|
||||
_, _, err := s.obsStore.StoreObservation(ctx, "session-cleanup", "project-cleanup", obs, 1, 10)
|
||||
s.NoError(err)
|
||||
|
||||
// Cleanup should not have been called since nothing was deleted
|
||||
s.Empty(calledWith)
|
||||
}
|
||||
|
||||
// TestGetObservationCount tests observation counting.
|
||||
func (s *ObservationStoreSuite) TestGetObservationCount() {
|
||||
ctx := context.Background()
|
||||
|
||||
// Create observations for project-a
|
||||
for i := 0; i < 5; i++ {
|
||||
obs := &models.ParsedObservation{
|
||||
Type: models.ObsTypeDiscovery,
|
||||
}
|
||||
_, _, err := s.obsStore.StoreObservation(ctx, "session-count", "project-a", obs, i, 10)
|
||||
s.NoError(err)
|
||||
}
|
||||
|
||||
// Create global observation
|
||||
globalObs := &models.ParsedObservation{
|
||||
Type: models.ObsTypeDiscovery,
|
||||
Concepts: []string{"security"},
|
||||
}
|
||||
_, _, err := s.obsStore.StoreObservation(ctx, "session-count", "project-a", globalObs, 6, 10)
|
||||
s.NoError(err)
|
||||
|
||||
// Project-a should count 6 (5 project + 1 global)
|
||||
count, err := s.obsStore.GetObservationCount(ctx, "project-a")
|
||||
s.NoError(err)
|
||||
s.Equal(6, count)
|
||||
|
||||
// Project-b should count 1 (only global)
|
||||
count, err = s.obsStore.GetObservationCount(ctx, "project-b")
|
||||
s.NoError(err)
|
||||
s.Equal(1, count)
|
||||
}
|
||||
|
||||
func TestObservationStore_StoreAndRetrieve(t *testing.T) {
|
||||
obsStore, _, cleanup := testObservationStore(t)
|
||||
defer cleanup()
|
||||
@@ -800,148 +372,3 @@ func TestExtractKeywords(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestObservationStore_GetObservationsByProjectStrict(t *testing.T) {
|
||||
obsStore, _, cleanup := testObservationStore(t)
|
||||
defer cleanup()
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
// Create project-scoped observation for project-a
|
||||
projectObs := &models.ParsedObservation{
|
||||
Type: models.ObsTypeDiscovery,
|
||||
Title: "Project A specific",
|
||||
Narrative: "Only for project-a",
|
||||
Concepts: []string{"local-concept"},
|
||||
}
|
||||
_, _, err := obsStore.StoreObservation(ctx, "session-1", "project-a", projectObs, 1, 100)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Create global observation from project-a
|
||||
globalObs := &models.ParsedObservation{
|
||||
Type: models.ObsTypeDiscovery,
|
||||
Title: "Global security practice",
|
||||
Narrative: "Best practice for all",
|
||||
Concepts: []string{"security", "best-practice"},
|
||||
}
|
||||
_, _, err = obsStore.StoreObservation(ctx, "session-1", "project-a", globalObs, 2, 100)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Create observation for project-b
|
||||
projectBObs := &models.ParsedObservation{
|
||||
Type: models.ObsTypeDiscovery,
|
||||
Title: "Project B specific",
|
||||
Narrative: "Only for project-b",
|
||||
}
|
||||
_, _, err = obsStore.StoreObservation(ctx, "session-1", "project-b", projectBObs, 1, 100)
|
||||
require.NoError(t, err)
|
||||
|
||||
// GetObservationsByProjectStrict for project-a should only return project-a observations
|
||||
// This is different from GetRecentObservations which includes globals from other projects
|
||||
results, err := obsStore.GetObservationsByProjectStrict(ctx, "project-a", 10)
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, results, 2) // Only observations created in project-a
|
||||
|
||||
// Verify both are from project-a
|
||||
for _, obs := range results {
|
||||
assert.Equal(t, "project-a", obs.Project)
|
||||
}
|
||||
|
||||
// GetObservationsByProjectStrict for project-b should only return project-b observations
|
||||
results, err = obsStore.GetObservationsByProjectStrict(ctx, "project-b", 10)
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, results, 1)
|
||||
assert.Equal(t, "Project B specific", results[0].Title.String)
|
||||
}
|
||||
|
||||
func TestObservationStore_SearchObservationsFTS_EmptyQuery(t *testing.T) {
|
||||
obsStore, _, cleanup := testObservationStore(t)
|
||||
defer cleanup()
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
// Create an observation
|
||||
obs := &models.ParsedObservation{
|
||||
Type: models.ObsTypeDiscovery,
|
||||
Title: "Test observation",
|
||||
Narrative: "Some content here",
|
||||
}
|
||||
_, _, err := obsStore.StoreObservation(ctx, "session-1", "project-a", obs, 1, 100)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Search with only stop words (should return nil)
|
||||
results, err := obsStore.SearchObservationsFTS(ctx, "the a an is are", "project-a", 10)
|
||||
require.NoError(t, err)
|
||||
assert.Nil(t, results)
|
||||
|
||||
// Search with empty query
|
||||
results, err = obsStore.SearchObservationsFTS(ctx, "", "project-a", 10)
|
||||
require.NoError(t, err)
|
||||
assert.Nil(t, results)
|
||||
}
|
||||
|
||||
func TestObservationStore_SearchObservationsFTS_DefaultLimit(t *testing.T) {
|
||||
obsStore, _, cleanup := testObservationStore(t)
|
||||
defer cleanup()
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
// Create observations
|
||||
for i := 0; i < 15; i++ {
|
||||
obs := &models.ParsedObservation{
|
||||
Type: models.ObsTypeDiscovery,
|
||||
Title: "Authentication test " + string(rune('A'+i)),
|
||||
Narrative: "Auth related content",
|
||||
}
|
||||
_, _, err := obsStore.StoreObservation(ctx, "session-1", "project-a", obs, i+1, 100)
|
||||
require.NoError(t, err)
|
||||
time.Sleep(time.Millisecond)
|
||||
}
|
||||
|
||||
// Search with limit 0 (should default to 10)
|
||||
results, err := obsStore.SearchObservationsFTS(ctx, "authentication", "project-a", 0)
|
||||
require.NoError(t, err)
|
||||
assert.LessOrEqual(t, len(results), 10)
|
||||
|
||||
// Search with negative limit (should default to 10)
|
||||
results, err = obsStore.SearchObservationsFTS(ctx, "authentication", "project-a", -5)
|
||||
require.NoError(t, err)
|
||||
assert.LessOrEqual(t, len(results), 10)
|
||||
}
|
||||
|
||||
func TestObservationStore_GetAllRecentObservations(t *testing.T) {
|
||||
obsStore, _, cleanup := testObservationStore(t)
|
||||
defer cleanup()
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
// Create observations across different projects
|
||||
projects := []string{"project-a", "project-b", "project-c"}
|
||||
for _, proj := range projects {
|
||||
for i := 0; i < 3; i++ {
|
||||
obs := &models.ParsedObservation{
|
||||
Type: models.ObsTypeDiscovery,
|
||||
Title: proj + " observation " + string(rune('A'+i)),
|
||||
Narrative: "Content for " + proj,
|
||||
}
|
||||
_, _, err := obsStore.StoreObservation(ctx, "session-1", proj, obs, i+1, 100)
|
||||
require.NoError(t, err)
|
||||
time.Sleep(time.Millisecond)
|
||||
}
|
||||
}
|
||||
|
||||
// Get all recent observations
|
||||
results, err := obsStore.GetAllRecentObservations(ctx, 100)
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, results, 9) // 3 projects * 3 observations
|
||||
|
||||
// Verify they are in descending order by epoch
|
||||
for i := 1; i < len(results); i++ {
|
||||
assert.GreaterOrEqual(t, results[i-1].CreatedAtEpoch, results[i].CreatedAtEpoch)
|
||||
}
|
||||
|
||||
// Test with limit
|
||||
results, err = obsStore.GetAllRecentObservations(ctx, 5)
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, results, 5)
|
||||
}
|
||||
|
||||
@@ -1,370 +0,0 @@
|
||||
// Package sqlite provides SQLite database operations for claude-mnemonic.
|
||||
package sqlite
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"time"
|
||||
|
||||
"github.com/lukaszraczylo/claude-mnemonic/pkg/models"
|
||||
)
|
||||
|
||||
// patternColumns is the standard list of columns to select for patterns.
|
||||
const patternColumns = `id, name, type, description, signature, recommendation,
|
||||
frequency, projects, observation_ids, status, merged_into_id, confidence,
|
||||
last_seen_at, last_seen_at_epoch, created_at, created_at_epoch`
|
||||
|
||||
// PatternCleanupFunc is a callback for when patterns are deleted.
|
||||
type PatternCleanupFunc func(ctx context.Context, deletedIDs []int64)
|
||||
|
||||
// PatternStore provides pattern-related database operations.
|
||||
type PatternStore struct {
|
||||
store *Store
|
||||
cleanupFunc PatternCleanupFunc
|
||||
}
|
||||
|
||||
// NewPatternStore creates a new pattern store.
|
||||
func NewPatternStore(store *Store) *PatternStore {
|
||||
return &PatternStore{store: store}
|
||||
}
|
||||
|
||||
// SetCleanupFunc sets the callback for when patterns are deleted.
|
||||
func (s *PatternStore) SetCleanupFunc(fn PatternCleanupFunc) {
|
||||
s.cleanupFunc = fn
|
||||
}
|
||||
|
||||
// StorePattern stores a new pattern.
|
||||
func (s *PatternStore) StorePattern(ctx context.Context, pattern *models.Pattern) (int64, error) {
|
||||
signatureJSON, _ := json.Marshal(pattern.Signature)
|
||||
projectsJSON, _ := json.Marshal(pattern.Projects)
|
||||
obsIDsJSON, _ := json.Marshal(pattern.ObservationIDs)
|
||||
|
||||
const query = `
|
||||
INSERT INTO patterns
|
||||
(name, type, description, signature, recommendation, frequency, projects,
|
||||
observation_ids, status, merged_into_id, confidence,
|
||||
last_seen_at, last_seen_at_epoch, created_at, created_at_epoch)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
`
|
||||
|
||||
result, err := s.store.ExecContext(ctx, query,
|
||||
pattern.Name, string(pattern.Type),
|
||||
nullString(pattern.Description.String), string(signatureJSON),
|
||||
nullString(pattern.Recommendation.String),
|
||||
pattern.Frequency, string(projectsJSON), string(obsIDsJSON),
|
||||
string(pattern.Status), nullInt64(pattern.MergedIntoID),
|
||||
pattern.Confidence, pattern.LastSeenAt, pattern.LastSeenEpoch,
|
||||
pattern.CreatedAt, pattern.CreatedAtEpoch,
|
||||
)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
return result.LastInsertId()
|
||||
}
|
||||
|
||||
// UpdatePattern updates an existing pattern.
|
||||
func (s *PatternStore) UpdatePattern(ctx context.Context, pattern *models.Pattern) error {
|
||||
signatureJSON, _ := json.Marshal(pattern.Signature)
|
||||
projectsJSON, _ := json.Marshal(pattern.Projects)
|
||||
obsIDsJSON, _ := json.Marshal(pattern.ObservationIDs)
|
||||
|
||||
const query = `
|
||||
UPDATE patterns SET
|
||||
name = ?, type = ?, description = ?, signature = ?, recommendation = ?,
|
||||
frequency = ?, projects = ?, observation_ids = ?, status = ?,
|
||||
merged_into_id = ?, confidence = ?, last_seen_at = ?, last_seen_at_epoch = ?
|
||||
WHERE id = ?
|
||||
`
|
||||
|
||||
_, err := s.store.ExecContext(ctx, query,
|
||||
pattern.Name, string(pattern.Type),
|
||||
nullString(pattern.Description.String), string(signatureJSON),
|
||||
nullString(pattern.Recommendation.String),
|
||||
pattern.Frequency, string(projectsJSON), string(obsIDsJSON),
|
||||
string(pattern.Status), nullInt64(pattern.MergedIntoID),
|
||||
pattern.Confidence, pattern.LastSeenAt, pattern.LastSeenEpoch,
|
||||
pattern.ID,
|
||||
)
|
||||
return err
|
||||
}
|
||||
|
||||
// GetPatternByID retrieves a pattern by ID.
|
||||
func (s *PatternStore) GetPatternByID(ctx context.Context, id int64) (*models.Pattern, error) {
|
||||
query := `SELECT ` + patternColumns + ` FROM patterns WHERE id = ?`
|
||||
|
||||
row := s.store.QueryRowContext(ctx, query, id)
|
||||
return scanPattern(row)
|
||||
}
|
||||
|
||||
// GetPatternByName retrieves a pattern by name.
|
||||
func (s *PatternStore) GetPatternByName(ctx context.Context, name string) (*models.Pattern, error) {
|
||||
query := `SELECT ` + patternColumns + ` FROM patterns WHERE name = ? AND status = 'active'`
|
||||
|
||||
row := s.store.QueryRowContext(ctx, query, name)
|
||||
pattern, err := scanPattern(row)
|
||||
if err == sql.ErrNoRows {
|
||||
return nil, nil
|
||||
}
|
||||
return pattern, err
|
||||
}
|
||||
|
||||
// GetActivePatterns retrieves all active patterns.
|
||||
func (s *PatternStore) GetActivePatterns(ctx context.Context, limit int) ([]*models.Pattern, error) {
|
||||
query := `SELECT ` + patternColumns + `
|
||||
FROM patterns
|
||||
WHERE status = 'active'
|
||||
ORDER BY frequency DESC, confidence DESC
|
||||
LIMIT ?`
|
||||
|
||||
rows, err := s.store.QueryContext(ctx, query, limit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
return scanPatternRows(rows)
|
||||
}
|
||||
|
||||
// GetPatternsByType retrieves patterns of a specific type.
|
||||
func (s *PatternStore) GetPatternsByType(ctx context.Context, patternType models.PatternType, limit int) ([]*models.Pattern, error) {
|
||||
query := `SELECT ` + patternColumns + `
|
||||
FROM patterns
|
||||
WHERE type = ? AND status = 'active'
|
||||
ORDER BY frequency DESC, confidence DESC
|
||||
LIMIT ?`
|
||||
|
||||
rows, err := s.store.QueryContext(ctx, query, string(patternType), limit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
return scanPatternRows(rows)
|
||||
}
|
||||
|
||||
// GetPatternsByProject retrieves patterns that have been observed in a specific project.
|
||||
func (s *PatternStore) GetPatternsByProject(ctx context.Context, project string, limit int) ([]*models.Pattern, error) {
|
||||
// Use JSON path to search within the projects array
|
||||
query := `SELECT ` + patternColumns + `
|
||||
FROM patterns
|
||||
WHERE status = 'active'
|
||||
AND EXISTS (
|
||||
SELECT 1 FROM json_each(projects)
|
||||
WHERE json_each.value = ?
|
||||
)
|
||||
ORDER BY frequency DESC, confidence DESC
|
||||
LIMIT ?`
|
||||
|
||||
rows, err := s.store.QueryContext(ctx, query, project, limit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
return scanPatternRows(rows)
|
||||
}
|
||||
|
||||
// FindMatchingPatterns searches for patterns that match a given signature.
|
||||
func (s *PatternStore) FindMatchingPatterns(ctx context.Context, signature []string, minScore float64) ([]*models.Pattern, error) {
|
||||
// Get all active patterns and filter by signature match in Go
|
||||
// This is simpler than complex SQL for JSON array matching
|
||||
patterns, err := s.GetActivePatterns(ctx, 100)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var matches []*models.Pattern
|
||||
for _, pattern := range patterns {
|
||||
score := models.CalculateMatchScore(signature, pattern.Signature)
|
||||
if score >= minScore {
|
||||
matches = append(matches, pattern)
|
||||
}
|
||||
}
|
||||
return matches, nil
|
||||
}
|
||||
|
||||
// MarkPatternDeprecated marks a pattern as deprecated.
|
||||
func (s *PatternStore) MarkPatternDeprecated(ctx context.Context, id int64) error {
|
||||
const query = `UPDATE patterns SET status = 'deprecated' WHERE id = ?`
|
||||
_, err := s.store.ExecContext(ctx, query, id)
|
||||
return err
|
||||
}
|
||||
|
||||
// MergePatterns merges a source pattern into a target pattern.
|
||||
func (s *PatternStore) MergePatterns(ctx context.Context, sourceID, targetID int64) error {
|
||||
// Get both patterns
|
||||
source, err := s.GetPatternByID(ctx, sourceID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
target, err := s.GetPatternByID(ctx, targetID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Merge source into target
|
||||
target.Frequency += source.Frequency
|
||||
for _, proj := range source.Projects {
|
||||
found := false
|
||||
for _, existing := range target.Projects {
|
||||
if existing == proj {
|
||||
found = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
target.Projects = append(target.Projects, proj)
|
||||
}
|
||||
}
|
||||
for _, obsID := range source.ObservationIDs {
|
||||
found := false
|
||||
for _, existing := range target.ObservationIDs {
|
||||
if existing == obsID {
|
||||
found = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
target.ObservationIDs = append(target.ObservationIDs, obsID)
|
||||
}
|
||||
}
|
||||
|
||||
// Update target
|
||||
if err := s.UpdatePattern(ctx, target); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Mark source as merged
|
||||
source.Status = models.PatternStatusMerged
|
||||
source.MergedIntoID = sql.NullInt64{Int64: targetID, Valid: true}
|
||||
return s.UpdatePattern(ctx, source)
|
||||
}
|
||||
|
||||
// DeletePattern deletes a pattern by ID.
|
||||
func (s *PatternStore) DeletePattern(ctx context.Context, id int64) error {
|
||||
const query = `DELETE FROM patterns WHERE id = ?`
|
||||
_, err := s.store.ExecContext(ctx, query, id)
|
||||
if err == nil && s.cleanupFunc != nil {
|
||||
s.cleanupFunc(ctx, []int64{id})
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// SearchPatternsFTS performs full-text search on patterns.
|
||||
func (s *PatternStore) SearchPatternsFTS(ctx context.Context, searchQuery string, limit int) ([]*models.Pattern, error) {
|
||||
query := `SELECT p.` + patternColumns + `
|
||||
FROM patterns p
|
||||
JOIN patterns_fts fts ON p.id = fts.rowid
|
||||
WHERE patterns_fts MATCH ?
|
||||
AND p.status = 'active'
|
||||
ORDER BY rank
|
||||
LIMIT ?`
|
||||
|
||||
rows, err := s.store.QueryContext(ctx, query, searchQuery, limit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
return scanPatternRows(rows)
|
||||
}
|
||||
|
||||
// GetPatternStats returns statistics about patterns.
|
||||
func (s *PatternStore) GetPatternStats(ctx context.Context) (*PatternStats, error) {
|
||||
const query = `
|
||||
SELECT
|
||||
COUNT(*) as total,
|
||||
COUNT(CASE WHEN status = 'active' THEN 1 END) as active,
|
||||
COUNT(CASE WHEN status = 'deprecated' THEN 1 END) as deprecated,
|
||||
COUNT(CASE WHEN status = 'merged' THEN 1 END) as merged,
|
||||
COALESCE(SUM(frequency), 0) as total_occurrences,
|
||||
COALESCE(AVG(confidence), 0) as avg_confidence,
|
||||
COUNT(CASE WHEN type = 'bug' THEN 1 END) as bugs,
|
||||
COUNT(CASE WHEN type = 'refactor' THEN 1 END) as refactors,
|
||||
COUNT(CASE WHEN type = 'architecture' THEN 1 END) as architectures,
|
||||
COUNT(CASE WHEN type = 'anti-pattern' THEN 1 END) as anti_patterns,
|
||||
COUNT(CASE WHEN type = 'best-practice' THEN 1 END) as best_practices
|
||||
FROM patterns
|
||||
`
|
||||
|
||||
var stats PatternStats
|
||||
err := s.store.QueryRowContext(ctx, query).Scan(
|
||||
&stats.Total, &stats.Active, &stats.Deprecated, &stats.Merged,
|
||||
&stats.TotalOccurrences, &stats.AvgConfidence,
|
||||
&stats.Bugs, &stats.Refactors, &stats.Architectures,
|
||||
&stats.AntiPatterns, &stats.BestPractices,
|
||||
)
|
||||
return &stats, err
|
||||
}
|
||||
|
||||
// PatternStats contains aggregate statistics about patterns.
|
||||
type PatternStats struct {
|
||||
Total int `json:"total"`
|
||||
Active int `json:"active"`
|
||||
Deprecated int `json:"deprecated"`
|
||||
Merged int `json:"merged"`
|
||||
TotalOccurrences int `json:"total_occurrences"`
|
||||
AvgConfidence float64 `json:"avg_confidence"`
|
||||
Bugs int `json:"bugs"`
|
||||
Refactors int `json:"refactors"`
|
||||
Architectures int `json:"architectures"`
|
||||
AntiPatterns int `json:"anti_patterns"`
|
||||
BestPractices int `json:"best_practices"`
|
||||
}
|
||||
|
||||
// scanPattern scans a single pattern from a row scanner.
|
||||
func scanPattern(scanner interface{ Scan(...interface{}) error }) (*models.Pattern, error) {
|
||||
var pattern models.Pattern
|
||||
if err := scanner.Scan(
|
||||
&pattern.ID, &pattern.Name, &pattern.Type,
|
||||
&pattern.Description, &pattern.Signature, &pattern.Recommendation,
|
||||
&pattern.Frequency, &pattern.Projects, &pattern.ObservationIDs,
|
||||
&pattern.Status, &pattern.MergedIntoID, &pattern.Confidence,
|
||||
&pattern.LastSeenAt, &pattern.LastSeenEpoch,
|
||||
&pattern.CreatedAt, &pattern.CreatedAtEpoch,
|
||||
); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &pattern, nil
|
||||
}
|
||||
|
||||
// scanPatternRows scans multiple patterns from rows.
|
||||
func scanPatternRows(rows *sql.Rows) ([]*models.Pattern, error) {
|
||||
var patterns []*models.Pattern
|
||||
for rows.Next() {
|
||||
pattern, err := scanPattern(rows)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
patterns = append(patterns, pattern)
|
||||
}
|
||||
return patterns, rows.Err()
|
||||
}
|
||||
|
||||
// nullInt64 converts sql.NullInt64 to the value needed for database insertion.
|
||||
func nullInt64(n sql.NullInt64) interface{} {
|
||||
if n.Valid {
|
||||
return n.Int64
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// IncrementPatternFrequency atomically increments a pattern's frequency and updates last_seen.
|
||||
func (s *PatternStore) IncrementPatternFrequency(ctx context.Context, id int64, project string, observationID int64) error {
|
||||
now := time.Now()
|
||||
|
||||
// Get current pattern
|
||||
pattern, err := s.GetPatternByID(ctx, id)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Add occurrence
|
||||
pattern.AddOccurrence(project, observationID)
|
||||
pattern.LastSeenAt = now.Format(time.RFC3339)
|
||||
pattern.LastSeenEpoch = now.UnixMilli()
|
||||
|
||||
return s.UpdatePattern(ctx, pattern)
|
||||
}
|
||||
@@ -1,507 +0,0 @@
|
||||
package sqlite
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/lukaszraczylo/claude-mnemonic/pkg/models"
|
||||
)
|
||||
|
||||
// setupPatternTestStore creates a test store with patterns table.
|
||||
func setupPatternTestStore(t *testing.T) *Store {
|
||||
t.Helper()
|
||||
db, _, cleanup := testDB(t)
|
||||
t.Cleanup(cleanup)
|
||||
createBaseTables(t, db)
|
||||
return newStoreFromDB(db)
|
||||
}
|
||||
|
||||
func TestPatternStore_StoreAndGet(t *testing.T) {
|
||||
store := setupPatternTestStore(t)
|
||||
|
||||
patternStore := NewPatternStore(store)
|
||||
ctx := context.Background()
|
||||
|
||||
// Create test pattern
|
||||
pattern := &models.Pattern{
|
||||
Name: "Test Pattern",
|
||||
Type: models.PatternTypeBug,
|
||||
Description: sql.NullString{String: "A test pattern", Valid: true},
|
||||
Signature: []string{"nil", "error"},
|
||||
Recommendation: sql.NullString{String: "Always check for nil", Valid: true},
|
||||
Frequency: 1,
|
||||
Projects: []string{"project1"},
|
||||
ObservationIDs: []int64{1, 2},
|
||||
Status: models.PatternStatusActive,
|
||||
Confidence: 0.5,
|
||||
LastSeenAt: time.Now().Format(time.RFC3339),
|
||||
LastSeenEpoch: time.Now().UnixMilli(),
|
||||
CreatedAt: time.Now().Format(time.RFC3339),
|
||||
CreatedAtEpoch: time.Now().UnixMilli(),
|
||||
}
|
||||
|
||||
// Store pattern
|
||||
id, err := patternStore.StorePattern(ctx, pattern)
|
||||
if err != nil {
|
||||
t.Fatalf("StorePattern() error = %v", err)
|
||||
}
|
||||
if id <= 0 {
|
||||
t.Errorf("Expected positive ID, got %d", id)
|
||||
}
|
||||
|
||||
// Get pattern by ID
|
||||
retrieved, err := patternStore.GetPatternByID(ctx, id)
|
||||
if err != nil {
|
||||
t.Fatalf("GetPatternByID() error = %v", err)
|
||||
}
|
||||
|
||||
if retrieved.Name != pattern.Name {
|
||||
t.Errorf("Expected name %s, got %s", pattern.Name, retrieved.Name)
|
||||
}
|
||||
if retrieved.Type != pattern.Type {
|
||||
t.Errorf("Expected type %s, got %s", pattern.Type, retrieved.Type)
|
||||
}
|
||||
if len(retrieved.Signature) != len(pattern.Signature) {
|
||||
t.Errorf("Expected %d signature elements, got %d",
|
||||
len(pattern.Signature), len(retrieved.Signature))
|
||||
}
|
||||
}
|
||||
|
||||
func TestPatternStore_GetByName(t *testing.T) {
|
||||
store := setupPatternTestStore(t)
|
||||
|
||||
patternStore := NewPatternStore(store)
|
||||
ctx := context.Background()
|
||||
|
||||
pattern := createTestPattern("Unique Name Pattern")
|
||||
_, err := patternStore.StorePattern(ctx, pattern)
|
||||
if err != nil {
|
||||
t.Fatalf("StorePattern() error = %v", err)
|
||||
}
|
||||
|
||||
// Get by name
|
||||
retrieved, err := patternStore.GetPatternByName(ctx, "Unique Name Pattern")
|
||||
if err != nil {
|
||||
t.Fatalf("GetPatternByName() error = %v", err)
|
||||
}
|
||||
if retrieved == nil {
|
||||
t.Fatal("Expected pattern, got nil")
|
||||
}
|
||||
if retrieved.Name != "Unique Name Pattern" {
|
||||
t.Errorf("Expected name 'Unique Name Pattern', got '%s'", retrieved.Name)
|
||||
}
|
||||
|
||||
// Get non-existent pattern
|
||||
nonExistent, err := patternStore.GetPatternByName(ctx, "Non Existent")
|
||||
if err != nil {
|
||||
t.Fatalf("GetPatternByName() error = %v", err)
|
||||
}
|
||||
if nonExistent != nil {
|
||||
t.Errorf("Expected nil for non-existent pattern, got %v", nonExistent)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPatternStore_GetActivePatterns(t *testing.T) {
|
||||
store := setupPatternTestStore(t)
|
||||
|
||||
patternStore := NewPatternStore(store)
|
||||
ctx := context.Background()
|
||||
|
||||
// Create multiple patterns with different statuses
|
||||
active1 := createTestPattern("Active 1")
|
||||
active1.Frequency = 5
|
||||
active2 := createTestPattern("Active 2")
|
||||
active2.Frequency = 3
|
||||
deprecated := createTestPattern("Deprecated")
|
||||
deprecated.Status = models.PatternStatusDeprecated
|
||||
|
||||
patternStore.StorePattern(ctx, active1)
|
||||
patternStore.StorePattern(ctx, active2)
|
||||
patternStore.StorePattern(ctx, deprecated)
|
||||
|
||||
// Get active patterns
|
||||
patterns, err := patternStore.GetActivePatterns(ctx, 10)
|
||||
if err != nil {
|
||||
t.Fatalf("GetActivePatterns() error = %v", err)
|
||||
}
|
||||
|
||||
if len(patterns) != 2 {
|
||||
t.Errorf("Expected 2 active patterns, got %d", len(patterns))
|
||||
}
|
||||
|
||||
// Check order (should be by frequency descending)
|
||||
if len(patterns) >= 2 {
|
||||
if patterns[0].Frequency < patterns[1].Frequency {
|
||||
t.Errorf("Patterns not ordered by frequency descending")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestPatternStore_GetPatternsByType(t *testing.T) {
|
||||
store := setupPatternTestStore(t)
|
||||
|
||||
patternStore := NewPatternStore(store)
|
||||
ctx := context.Background()
|
||||
|
||||
// Create patterns of different types
|
||||
bugPattern := createTestPattern("Bug Pattern")
|
||||
bugPattern.Type = models.PatternTypeBug
|
||||
|
||||
refactorPattern := createTestPattern("Refactor Pattern")
|
||||
refactorPattern.Type = models.PatternTypeRefactor
|
||||
|
||||
patternStore.StorePattern(ctx, bugPattern)
|
||||
patternStore.StorePattern(ctx, refactorPattern)
|
||||
|
||||
// Get by type
|
||||
bugs, err := patternStore.GetPatternsByType(ctx, models.PatternTypeBug, 10)
|
||||
if err != nil {
|
||||
t.Fatalf("GetPatternsByType() error = %v", err)
|
||||
}
|
||||
if len(bugs) != 1 {
|
||||
t.Errorf("Expected 1 bug pattern, got %d", len(bugs))
|
||||
}
|
||||
|
||||
refactors, err := patternStore.GetPatternsByType(ctx, models.PatternTypeRefactor, 10)
|
||||
if err != nil {
|
||||
t.Fatalf("GetPatternsByType() error = %v", err)
|
||||
}
|
||||
if len(refactors) != 1 {
|
||||
t.Errorf("Expected 1 refactor pattern, got %d", len(refactors))
|
||||
}
|
||||
}
|
||||
|
||||
func TestPatternStore_GetPatternsByProject(t *testing.T) {
|
||||
store := setupPatternTestStore(t)
|
||||
|
||||
patternStore := NewPatternStore(store)
|
||||
ctx := context.Background()
|
||||
|
||||
// Create patterns with different projects
|
||||
pattern1 := createTestPattern("Pattern 1")
|
||||
pattern1.Projects = []string{"project-a", "project-b"}
|
||||
|
||||
pattern2 := createTestPattern("Pattern 2")
|
||||
pattern2.Projects = []string{"project-b", "project-c"}
|
||||
|
||||
patternStore.StorePattern(ctx, pattern1)
|
||||
patternStore.StorePattern(ctx, pattern2)
|
||||
|
||||
// Get by project
|
||||
projectA, err := patternStore.GetPatternsByProject(ctx, "project-a", 10)
|
||||
if err != nil {
|
||||
t.Fatalf("GetPatternsByProject() error = %v", err)
|
||||
}
|
||||
if len(projectA) != 1 {
|
||||
t.Errorf("Expected 1 pattern for project-a, got %d", len(projectA))
|
||||
}
|
||||
|
||||
projectB, err := patternStore.GetPatternsByProject(ctx, "project-b", 10)
|
||||
if err != nil {
|
||||
t.Fatalf("GetPatternsByProject() error = %v", err)
|
||||
}
|
||||
if len(projectB) != 2 {
|
||||
t.Errorf("Expected 2 patterns for project-b, got %d", len(projectB))
|
||||
}
|
||||
}
|
||||
|
||||
func TestPatternStore_UpdatePattern(t *testing.T) {
|
||||
store := setupPatternTestStore(t)
|
||||
|
||||
patternStore := NewPatternStore(store)
|
||||
ctx := context.Background()
|
||||
|
||||
// Create and store pattern
|
||||
pattern := createTestPattern("Original Name")
|
||||
id, _ := patternStore.StorePattern(ctx, pattern)
|
||||
|
||||
// Update pattern
|
||||
pattern.ID = id
|
||||
pattern.Name = "Updated Name"
|
||||
pattern.Frequency = 10
|
||||
pattern.Confidence = 0.9
|
||||
|
||||
err := patternStore.UpdatePattern(ctx, pattern)
|
||||
if err != nil {
|
||||
t.Fatalf("UpdatePattern() error = %v", err)
|
||||
}
|
||||
|
||||
// Verify update
|
||||
updated, _ := patternStore.GetPatternByID(ctx, id)
|
||||
if updated.Name != "Updated Name" {
|
||||
t.Errorf("Expected name 'Updated Name', got '%s'", updated.Name)
|
||||
}
|
||||
if updated.Frequency != 10 {
|
||||
t.Errorf("Expected frequency 10, got %d", updated.Frequency)
|
||||
}
|
||||
if updated.Confidence != 0.9 {
|
||||
t.Errorf("Expected confidence 0.9, got %f", updated.Confidence)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPatternStore_DeletePattern(t *testing.T) {
|
||||
store := setupPatternTestStore(t)
|
||||
|
||||
patternStore := NewPatternStore(store)
|
||||
ctx := context.Background()
|
||||
|
||||
// Create and store pattern
|
||||
pattern := createTestPattern("To Delete")
|
||||
id, _ := patternStore.StorePattern(ctx, pattern)
|
||||
|
||||
// Delete pattern
|
||||
err := patternStore.DeletePattern(ctx, id)
|
||||
if err != nil {
|
||||
t.Fatalf("DeletePattern() error = %v", err)
|
||||
}
|
||||
|
||||
// Verify deletion
|
||||
deleted, err := patternStore.GetPatternByID(ctx, id)
|
||||
if err != sql.ErrNoRows {
|
||||
t.Errorf("Expected ErrNoRows, got %v", err)
|
||||
}
|
||||
if deleted != nil {
|
||||
t.Errorf("Expected nil for deleted pattern")
|
||||
}
|
||||
}
|
||||
|
||||
func TestPatternStore_MarkPatternDeprecated(t *testing.T) {
|
||||
store := setupPatternTestStore(t)
|
||||
|
||||
patternStore := NewPatternStore(store)
|
||||
ctx := context.Background()
|
||||
|
||||
// Create and store pattern
|
||||
pattern := createTestPattern("To Deprecate")
|
||||
id, _ := patternStore.StorePattern(ctx, pattern)
|
||||
|
||||
// Mark as deprecated
|
||||
err := patternStore.MarkPatternDeprecated(ctx, id)
|
||||
if err != nil {
|
||||
t.Fatalf("MarkPatternDeprecated() error = %v", err)
|
||||
}
|
||||
|
||||
// Verify status
|
||||
deprecated, _ := patternStore.GetPatternByID(ctx, id)
|
||||
if deprecated.Status != models.PatternStatusDeprecated {
|
||||
t.Errorf("Expected status 'deprecated', got '%s'", deprecated.Status)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPatternStore_MergePatterns(t *testing.T) {
|
||||
store := setupPatternTestStore(t)
|
||||
|
||||
patternStore := NewPatternStore(store)
|
||||
ctx := context.Background()
|
||||
|
||||
// Create source and target patterns
|
||||
source := createTestPattern("Source Pattern")
|
||||
source.Frequency = 3
|
||||
source.Projects = []string{"proj1", "proj2"}
|
||||
source.ObservationIDs = []int64{1, 2, 3}
|
||||
|
||||
target := createTestPattern("Target Pattern")
|
||||
target.Frequency = 2
|
||||
target.Projects = []string{"proj2", "proj3"}
|
||||
target.ObservationIDs = []int64{4, 5}
|
||||
|
||||
sourceID, _ := patternStore.StorePattern(ctx, source)
|
||||
targetID, _ := patternStore.StorePattern(ctx, target)
|
||||
|
||||
// Merge
|
||||
err := patternStore.MergePatterns(ctx, sourceID, targetID)
|
||||
if err != nil {
|
||||
t.Fatalf("MergePatterns() error = %v", err)
|
||||
}
|
||||
|
||||
// Verify source is marked as merged
|
||||
mergedSource, _ := patternStore.GetPatternByID(ctx, sourceID)
|
||||
if mergedSource.Status != models.PatternStatusMerged {
|
||||
t.Errorf("Expected source status 'merged', got '%s'", mergedSource.Status)
|
||||
}
|
||||
if !mergedSource.MergedIntoID.Valid || mergedSource.MergedIntoID.Int64 != targetID {
|
||||
t.Errorf("Expected source merged_into_id to be %d", targetID)
|
||||
}
|
||||
|
||||
// Verify target has combined data
|
||||
mergedTarget, _ := patternStore.GetPatternByID(ctx, targetID)
|
||||
expectedFrequency := 5 // 3 + 2
|
||||
if mergedTarget.Frequency != expectedFrequency {
|
||||
t.Errorf("Expected merged frequency %d, got %d", expectedFrequency, mergedTarget.Frequency)
|
||||
}
|
||||
// Should have 3 unique projects: proj1, proj2, proj3
|
||||
if len(mergedTarget.Projects) != 3 {
|
||||
t.Errorf("Expected 3 projects after merge, got %d", len(mergedTarget.Projects))
|
||||
}
|
||||
// Should have 5 observation IDs
|
||||
if len(mergedTarget.ObservationIDs) != 5 {
|
||||
t.Errorf("Expected 5 observation IDs after merge, got %d", len(mergedTarget.ObservationIDs))
|
||||
}
|
||||
}
|
||||
|
||||
func TestPatternStore_FindMatchingPatterns(t *testing.T) {
|
||||
store := setupPatternTestStore(t)
|
||||
|
||||
patternStore := NewPatternStore(store)
|
||||
ctx := context.Background()
|
||||
|
||||
// Create patterns with known signatures
|
||||
pattern1 := createTestPattern("Pattern 1")
|
||||
pattern1.Signature = []string{"nil", "error", "handling"}
|
||||
|
||||
pattern2 := createTestPattern("Pattern 2")
|
||||
pattern2.Signature = []string{"nil", "pointer", "check"}
|
||||
|
||||
pattern3 := createTestPattern("Pattern 3")
|
||||
pattern3.Signature = []string{"refactor", "extract", "method"}
|
||||
|
||||
patternStore.StorePattern(ctx, pattern1)
|
||||
patternStore.StorePattern(ctx, pattern2)
|
||||
patternStore.StorePattern(ctx, pattern3)
|
||||
|
||||
// Find patterns matching "nil" related signature
|
||||
matches, err := patternStore.FindMatchingPatterns(ctx, []string{"nil", "error"}, 0.3)
|
||||
if err != nil {
|
||||
t.Fatalf("FindMatchingPatterns() error = %v", err)
|
||||
}
|
||||
|
||||
if len(matches) < 1 {
|
||||
t.Errorf("Expected at least 1 match, got %d", len(matches))
|
||||
}
|
||||
|
||||
// Verify no match for unrelated signature
|
||||
noMatches, err := patternStore.FindMatchingPatterns(ctx, []string{"completely", "different"}, 0.5)
|
||||
if err != nil {
|
||||
t.Fatalf("FindMatchingPatterns() error = %v", err)
|
||||
}
|
||||
if len(noMatches) != 0 {
|
||||
t.Errorf("Expected 0 matches for unrelated signature, got %d", len(noMatches))
|
||||
}
|
||||
}
|
||||
|
||||
func TestPatternStore_IncrementPatternFrequency(t *testing.T) {
|
||||
store := setupPatternTestStore(t)
|
||||
|
||||
patternStore := NewPatternStore(store)
|
||||
ctx := context.Background()
|
||||
|
||||
// Create pattern
|
||||
pattern := createTestPattern("Frequency Test")
|
||||
pattern.Frequency = 1
|
||||
pattern.Projects = []string{"proj1"}
|
||||
pattern.ObservationIDs = []int64{1}
|
||||
|
||||
id, _ := patternStore.StorePattern(ctx, pattern)
|
||||
|
||||
// Increment frequency
|
||||
err := patternStore.IncrementPatternFrequency(ctx, id, "proj2", 2)
|
||||
if err != nil {
|
||||
t.Fatalf("IncrementPatternFrequency() error = %v", err)
|
||||
}
|
||||
|
||||
// Verify
|
||||
updated, _ := patternStore.GetPatternByID(ctx, id)
|
||||
if updated.Frequency != 2 {
|
||||
t.Errorf("Expected frequency 2, got %d", updated.Frequency)
|
||||
}
|
||||
if len(updated.Projects) != 2 {
|
||||
t.Errorf("Expected 2 projects, got %d", len(updated.Projects))
|
||||
}
|
||||
if len(updated.ObservationIDs) != 2 {
|
||||
t.Errorf("Expected 2 observation IDs, got %d", len(updated.ObservationIDs))
|
||||
}
|
||||
}
|
||||
|
||||
func TestPatternStore_GetPatternStats(t *testing.T) {
|
||||
store := setupPatternTestStore(t)
|
||||
|
||||
patternStore := NewPatternStore(store)
|
||||
ctx := context.Background()
|
||||
|
||||
// Create patterns with different types and statuses
|
||||
bug := createTestPattern("Bug")
|
||||
bug.Type = models.PatternTypeBug
|
||||
bug.Frequency = 5
|
||||
|
||||
refactor := createTestPattern("Refactor")
|
||||
refactor.Type = models.PatternTypeRefactor
|
||||
refactor.Frequency = 3
|
||||
|
||||
deprecated := createTestPattern("Deprecated")
|
||||
deprecated.Type = models.PatternTypeArchitecture
|
||||
deprecated.Status = models.PatternStatusDeprecated
|
||||
|
||||
patternStore.StorePattern(ctx, bug)
|
||||
patternStore.StorePattern(ctx, refactor)
|
||||
patternStore.StorePattern(ctx, deprecated)
|
||||
|
||||
// Get stats
|
||||
stats, err := patternStore.GetPatternStats(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("GetPatternStats() error = %v", err)
|
||||
}
|
||||
|
||||
if stats.Total != 3 {
|
||||
t.Errorf("Expected total 3, got %d", stats.Total)
|
||||
}
|
||||
if stats.Active != 2 {
|
||||
t.Errorf("Expected 2 active, got %d", stats.Active)
|
||||
}
|
||||
if stats.Deprecated != 1 {
|
||||
t.Errorf("Expected 1 deprecated, got %d", stats.Deprecated)
|
||||
}
|
||||
if stats.Bugs != 1 {
|
||||
t.Errorf("Expected 1 bug, got %d", stats.Bugs)
|
||||
}
|
||||
if stats.Refactors != 1 {
|
||||
t.Errorf("Expected 1 refactor, got %d", stats.Refactors)
|
||||
}
|
||||
if stats.TotalOccurrences != 9 { // 5 + 3 + 1
|
||||
t.Errorf("Expected 9 total occurrences, got %d", stats.TotalOccurrences)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPatternStore_CleanupCallback(t *testing.T) {
|
||||
store := setupPatternTestStore(t)
|
||||
|
||||
patternStore := NewPatternStore(store)
|
||||
ctx := context.Background()
|
||||
|
||||
var deletedIDs []int64
|
||||
patternStore.SetCleanupFunc(func(ctx context.Context, ids []int64) {
|
||||
deletedIDs = ids
|
||||
})
|
||||
|
||||
// Create and delete pattern
|
||||
pattern := createTestPattern("Cleanup Test")
|
||||
id, _ := patternStore.StorePattern(ctx, pattern)
|
||||
|
||||
patternStore.DeletePattern(ctx, id)
|
||||
|
||||
if len(deletedIDs) != 1 || deletedIDs[0] != id {
|
||||
t.Errorf("Expected cleanup callback with ID %d, got %v", id, deletedIDs)
|
||||
}
|
||||
}
|
||||
|
||||
// Helper function to create a test pattern
|
||||
func createTestPattern(name string) *models.Pattern {
|
||||
now := time.Now()
|
||||
return &models.Pattern{
|
||||
Name: name,
|
||||
Type: models.PatternTypeBug,
|
||||
Description: sql.NullString{String: "Test description", Valid: true},
|
||||
Signature: []string{"test", "pattern"},
|
||||
Recommendation: sql.NullString{String: "Test recommendation", Valid: true},
|
||||
Frequency: 1,
|
||||
Projects: []string{"test-project"},
|
||||
ObservationIDs: []int64{1},
|
||||
Status: models.PatternStatusActive,
|
||||
Confidence: 0.5,
|
||||
LastSeenAt: now.Format(time.RFC3339),
|
||||
LastSeenEpoch: now.UnixMilli(),
|
||||
CreatedAt: now.Format(time.RFC3339),
|
||||
CreatedAtEpoch: now.UnixMilli(),
|
||||
}
|
||||
}
|
||||
@@ -32,15 +32,11 @@ func (s *PromptStore) SetCleanupFunc(fn PromptCleanupFunc) {
|
||||
}
|
||||
|
||||
// SaveUserPromptWithMatches saves a user prompt with matched observation count.
|
||||
// Uses INSERT OR IGNORE to be idempotent - duplicate (session, prompt_number) pairs are silently ignored.
|
||||
// This prevents duplicate prompts when the user-prompt hook fires multiple times.
|
||||
func (s *PromptStore) SaveUserPromptWithMatches(ctx context.Context, claudeSessionID string, promptNumber int, promptText string, matchedObservations int) (int64, error) {
|
||||
now := time.Now()
|
||||
|
||||
// Use INSERT OR IGNORE for idempotency - if (claude_session_id, prompt_number) already exists,
|
||||
// the insert is silently ignored. This handles concurrent/duplicate hook invocations.
|
||||
const query = `
|
||||
INSERT OR IGNORE INTO user_prompts
|
||||
INSERT INTO user_prompts
|
||||
(claude_session_id, prompt_number, prompt_text, matched_observations, created_at, created_at_epoch)
|
||||
VALUES (?, ?, ?, ?, ?, ?)
|
||||
`
|
||||
@@ -55,17 +51,6 @@ func (s *PromptStore) SaveUserPromptWithMatches(ctx context.Context, claudeSessi
|
||||
|
||||
id, _ := result.LastInsertId()
|
||||
|
||||
// If id is 0, the insert was ignored (duplicate) - fetch the existing ID
|
||||
if id == 0 {
|
||||
const selectQuery = `SELECT id FROM user_prompts WHERE claude_session_id = ? AND prompt_number = ?`
|
||||
row := s.store.QueryRowContext(ctx, selectQuery, claudeSessionID, promptNumber)
|
||||
if err := row.Scan(&id); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
// Return existing ID without triggering cleanup (already handled when first inserted)
|
||||
return id, nil
|
||||
}
|
||||
|
||||
// Cleanup old prompts beyond the global limit (async to not block handler)
|
||||
go func() {
|
||||
cleanupCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
@@ -102,13 +87,13 @@ func (s *PromptStore) CleanupOldPrompts(ctx context.Context) ([]int64, error) {
|
||||
var toDelete []int64
|
||||
for rows.Next() {
|
||||
var id int64
|
||||
if scanErr := rows.Scan(&id); scanErr != nil {
|
||||
return nil, scanErr
|
||||
if err := rows.Scan(&id); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
toDelete = append(toDelete, id)
|
||||
}
|
||||
if rowsErr := rows.Err(); rowsErr != nil {
|
||||
return nil, rowsErr
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if len(toDelete) == 0 {
|
||||
@@ -199,53 +184,6 @@ func (s *PromptStore) GetAllRecentUserPrompts(ctx context.Context, limit int) ([
|
||||
return scanPromptWithSessionRows(rows)
|
||||
}
|
||||
|
||||
// GetAllPrompts retrieves all user prompts (for vector rebuild).
|
||||
func (s *PromptStore) GetAllPrompts(ctx context.Context) ([]*models.UserPromptWithSession, error) {
|
||||
const query = `
|
||||
SELECT up.id, up.claude_session_id, up.prompt_number, up.prompt_text,
|
||||
COALESCE(up.matched_observations, 0) as matched_observations,
|
||||
up.created_at, up.created_at_epoch,
|
||||
COALESCE(s.project, '') as project,
|
||||
COALESCE(s.sdk_session_id, '') as sdk_session_id
|
||||
FROM user_prompts up
|
||||
LEFT JOIN sdk_sessions s ON up.claude_session_id = s.claude_session_id
|
||||
ORDER BY up.id
|
||||
`
|
||||
|
||||
rows, err := s.store.QueryContext(ctx, query)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
return scanPromptWithSessionRows(rows)
|
||||
}
|
||||
|
||||
// FindRecentPromptByText finds a prompt with the same text for a session within the last few seconds.
|
||||
// This is used to detect duplicate hook invocations.
|
||||
// Returns (promptID, promptNumber, found).
|
||||
func (s *PromptStore) FindRecentPromptByText(ctx context.Context, claudeSessionID, promptText string, withinSeconds int) (int64, int, bool) {
|
||||
// Look for an existing prompt with the same text within the time window
|
||||
// This catches duplicate hook invocations that happen in quick succession
|
||||
const query = `
|
||||
SELECT id, prompt_number FROM user_prompts
|
||||
WHERE claude_session_id = ? AND prompt_text = ?
|
||||
AND created_at_epoch > ?
|
||||
ORDER BY created_at_epoch DESC
|
||||
LIMIT 1
|
||||
`
|
||||
|
||||
cutoff := time.Now().Add(-time.Duration(withinSeconds) * time.Second).UnixMilli()
|
||||
|
||||
var id int64
|
||||
var promptNumber int
|
||||
err := s.store.QueryRowContext(ctx, query, claudeSessionID, promptText, cutoff).Scan(&id, &promptNumber)
|
||||
if err != nil {
|
||||
return 0, 0, false
|
||||
}
|
||||
return id, promptNumber, true
|
||||
}
|
||||
|
||||
// GetRecentUserPromptsByProject retrieves recent user prompts for a specific project.
|
||||
func (s *PromptStore) GetRecentUserPromptsByProject(ctx context.Context, project string, limit int) ([]*models.UserPromptWithSession, error) {
|
||||
const query = `
|
||||
|
||||
@@ -194,96 +194,3 @@ func TestPromptStore_GetPromptsByIDs_EmptyInput(t *testing.T) {
|
||||
require.NoError(t, err)
|
||||
assert.Nil(t, prompts)
|
||||
}
|
||||
|
||||
func TestPromptStore_FindRecentPromptByText(t *testing.T) {
|
||||
promptStore, store, cleanup := testPromptStore(t)
|
||||
defer cleanup()
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
// Create a session
|
||||
seedSession(t, storeDB(store), "claude-1", "sdk-1", "test-project")
|
||||
|
||||
// Save a prompt
|
||||
_, err := promptStore.SaveUserPromptWithMatches(ctx, "claude-1", 1, "Help me fix this bug in the code", 0)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Find the prompt by text - returns (id, promptNumber, found)
|
||||
id, promptNum, found := promptStore.FindRecentPromptByText(ctx, "claude-1", "Help me fix this bug in the code", 60)
|
||||
assert.True(t, found, "should find the exact prompt text")
|
||||
assert.Greater(t, id, int64(0))
|
||||
assert.Equal(t, 1, promptNum)
|
||||
|
||||
// Try to find non-existent prompt
|
||||
_, _, found = promptStore.FindRecentPromptByText(ctx, "claude-1", "This prompt does not exist", 60)
|
||||
assert.False(t, found, "should not find non-existent prompt")
|
||||
|
||||
// Try with different session
|
||||
_, _, found = promptStore.FindRecentPromptByText(ctx, "claude-2", "Help me fix this bug in the code", 60)
|
||||
assert.False(t, found, "should not find prompt for different session")
|
||||
}
|
||||
|
||||
func TestPromptStore_FindRecentPromptByText_WindowSeconds(t *testing.T) {
|
||||
promptStore, store, cleanup := testPromptStore(t)
|
||||
defer cleanup()
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
// Create a session
|
||||
seedSession(t, storeDB(store), "claude-1", "sdk-1", "test-project")
|
||||
|
||||
// Save a prompt with an old timestamp
|
||||
oldEpoch := time.Now().Add(-2 * time.Hour).UnixMilli()
|
||||
_, err := storeDB(store).Exec(`
|
||||
INSERT INTO user_prompts (claude_session_id, prompt_number, prompt_text, created_at, created_at_epoch)
|
||||
VALUES (?, ?, ?, datetime('now'), ?)
|
||||
`, "claude-1", 1, "Old prompt text", oldEpoch)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Search within last hour - should not find old prompt
|
||||
_, _, found := promptStore.FindRecentPromptByText(ctx, "claude-1", "Old prompt text", 3600)
|
||||
assert.False(t, found, "should not find prompt outside window")
|
||||
|
||||
// Search within last 3 hours - should find old prompt
|
||||
_, _, found = promptStore.FindRecentPromptByText(ctx, "claude-1", "Old prompt text", 3*3600)
|
||||
assert.True(t, found, "should find prompt within extended window")
|
||||
}
|
||||
|
||||
func TestPromptStore_SaveMultiplePrompts(t *testing.T) {
|
||||
promptStore, store, cleanup := testPromptStore(t)
|
||||
defer cleanup()
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
// Create sessions
|
||||
seedSession(t, storeDB(store), "claude-1", "sdk-1", "project-x")
|
||||
seedSession(t, storeDB(store), "claude-2", "sdk-2", "project-y")
|
||||
|
||||
tests := []struct {
|
||||
claudeSessionID string
|
||||
text string
|
||||
promptNum int
|
||||
matches int
|
||||
}{
|
||||
{claudeSessionID: "claude-1", promptNum: 1, text: "First prompt", matches: 5},
|
||||
{claudeSessionID: "claude-1", promptNum: 2, text: "Second prompt", matches: 3},
|
||||
{claudeSessionID: "claude-2", promptNum: 1, text: "Third prompt", matches: 0},
|
||||
{claudeSessionID: "claude-1", promptNum: 3, text: "Fourth prompt", matches: 10},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
id, err := promptStore.SaveUserPromptWithMatches(ctx, tt.claudeSessionID, tt.promptNum, tt.text, tt.matches)
|
||||
require.NoError(t, err)
|
||||
assert.Greater(t, id, int64(0))
|
||||
}
|
||||
|
||||
// Verify counts
|
||||
var count int
|
||||
err := storeDB(store).QueryRow("SELECT COUNT(*) FROM user_prompts WHERE claude_session_id = 'claude-1'").Scan(&count)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 3, count)
|
||||
|
||||
err = storeDB(store).QueryRow("SELECT COUNT(*) FROM user_prompts WHERE claude_session_id = 'claude-2'").Scan(&count)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 1, count)
|
||||
}
|
||||
|
||||
@@ -1,377 +0,0 @@
|
||||
// Package sqlite provides SQLite database operations for claude-mnemonic.
|
||||
package sqlite
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
|
||||
"github.com/lukaszraczylo/claude-mnemonic/pkg/models"
|
||||
)
|
||||
|
||||
// RelationStore provides relation-related database operations.
|
||||
type RelationStore struct {
|
||||
store *Store
|
||||
}
|
||||
|
||||
// NewRelationStore creates a new relation store.
|
||||
func NewRelationStore(store *Store) *RelationStore {
|
||||
return &RelationStore{store: store}
|
||||
}
|
||||
|
||||
// StoreRelation stores a new observation relation.
|
||||
// Uses INSERT OR IGNORE to handle duplicate (source_id, target_id, relation_type) combinations.
|
||||
func (s *RelationStore) StoreRelation(ctx context.Context, relation *models.ObservationRelation) (int64, error) {
|
||||
const query = `
|
||||
INSERT OR IGNORE INTO observation_relations
|
||||
(source_id, target_id, relation_type, confidence, detection_source, reason, created_at, created_at_epoch)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
||||
`
|
||||
|
||||
result, err := s.store.ExecContext(ctx, query,
|
||||
relation.SourceID, relation.TargetID,
|
||||
string(relation.RelationType), relation.Confidence,
|
||||
string(relation.DetectionSource), relation.Reason,
|
||||
relation.CreatedAt, relation.CreatedAtEpoch,
|
||||
)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
return result.LastInsertId()
|
||||
}
|
||||
|
||||
// StoreRelations stores multiple relations in a single transaction.
|
||||
func (s *RelationStore) StoreRelations(ctx context.Context, relations []*models.ObservationRelation) error {
|
||||
if len(relations) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
tx, err := s.store.db.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() {
|
||||
if err != nil {
|
||||
_ = tx.Rollback()
|
||||
}
|
||||
}()
|
||||
|
||||
const query = `
|
||||
INSERT OR IGNORE INTO observation_relations
|
||||
(source_id, target_id, relation_type, confidence, detection_source, reason, created_at, created_at_epoch)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
||||
`
|
||||
|
||||
stmt, err := tx.PrepareContext(ctx, query)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer stmt.Close()
|
||||
|
||||
for _, rel := range relations {
|
||||
_, err = stmt.ExecContext(ctx,
|
||||
rel.SourceID, rel.TargetID,
|
||||
string(rel.RelationType), rel.Confidence,
|
||||
string(rel.DetectionSource), rel.Reason,
|
||||
rel.CreatedAt, rel.CreatedAtEpoch,
|
||||
)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return tx.Commit()
|
||||
}
|
||||
|
||||
// GetRelationsByObservationID retrieves all relations involving an observation (as source or target).
|
||||
func (s *RelationStore) GetRelationsByObservationID(ctx context.Context, obsID int64) ([]*models.ObservationRelation, error) {
|
||||
const query = `
|
||||
SELECT id, source_id, target_id, relation_type, confidence, detection_source, reason,
|
||||
created_at, created_at_epoch
|
||||
FROM observation_relations
|
||||
WHERE source_id = ? OR target_id = ?
|
||||
ORDER BY confidence DESC, created_at_epoch DESC
|
||||
`
|
||||
|
||||
rows, err := s.store.QueryContext(ctx, query, obsID, obsID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
return s.scanRelationRows(rows)
|
||||
}
|
||||
|
||||
// GetOutgoingRelations retrieves relations where the observation is the source.
|
||||
func (s *RelationStore) GetOutgoingRelations(ctx context.Context, obsID int64) ([]*models.ObservationRelation, error) {
|
||||
const query = `
|
||||
SELECT id, source_id, target_id, relation_type, confidence, detection_source, reason,
|
||||
created_at, created_at_epoch
|
||||
FROM observation_relations
|
||||
WHERE source_id = ?
|
||||
ORDER BY confidence DESC, created_at_epoch DESC
|
||||
`
|
||||
|
||||
rows, err := s.store.QueryContext(ctx, query, obsID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
return s.scanRelationRows(rows)
|
||||
}
|
||||
|
||||
// GetIncomingRelations retrieves relations where the observation is the target.
|
||||
func (s *RelationStore) GetIncomingRelations(ctx context.Context, obsID int64) ([]*models.ObservationRelation, error) {
|
||||
const query = `
|
||||
SELECT id, source_id, target_id, relation_type, confidence, detection_source, reason,
|
||||
created_at, created_at_epoch
|
||||
FROM observation_relations
|
||||
WHERE target_id = ?
|
||||
ORDER BY confidence DESC, created_at_epoch DESC
|
||||
`
|
||||
|
||||
rows, err := s.store.QueryContext(ctx, query, obsID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
return s.scanRelationRows(rows)
|
||||
}
|
||||
|
||||
// GetRelationsByType retrieves all relations of a specific type.
|
||||
func (s *RelationStore) GetRelationsByType(ctx context.Context, relationType models.RelationType, limit int) ([]*models.ObservationRelation, error) {
|
||||
const query = `
|
||||
SELECT id, source_id, target_id, relation_type, confidence, detection_source, reason,
|
||||
created_at, created_at_epoch
|
||||
FROM observation_relations
|
||||
WHERE relation_type = ?
|
||||
ORDER BY confidence DESC, created_at_epoch DESC
|
||||
LIMIT ?
|
||||
`
|
||||
|
||||
rows, err := s.store.QueryContext(ctx, query, string(relationType), limit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
return s.scanRelationRows(rows)
|
||||
}
|
||||
|
||||
// GetRelationsWithDetails retrieves relations with observation titles for display.
|
||||
func (s *RelationStore) GetRelationsWithDetails(ctx context.Context, obsID int64) ([]*models.RelationWithDetails, error) {
|
||||
const query = `
|
||||
SELECT r.id, r.source_id, r.target_id, r.relation_type, r.confidence, r.detection_source, r.reason,
|
||||
r.created_at, r.created_at_epoch,
|
||||
COALESCE(src.title, '') as source_title,
|
||||
COALESCE(tgt.title, '') as target_title,
|
||||
src.type as source_type,
|
||||
tgt.type as target_type
|
||||
FROM observation_relations r
|
||||
JOIN observations src ON src.id = r.source_id
|
||||
JOIN observations tgt ON tgt.id = r.target_id
|
||||
WHERE r.source_id = ? OR r.target_id = ?
|
||||
ORDER BY r.confidence DESC, r.created_at_epoch DESC
|
||||
`
|
||||
|
||||
rows, err := s.store.QueryContext(ctx, query, obsID, obsID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var results []*models.RelationWithDetails
|
||||
for rows.Next() {
|
||||
var r models.ObservationRelation
|
||||
var rwd models.RelationWithDetails
|
||||
var reason sql.NullString
|
||||
if err := rows.Scan(
|
||||
&r.ID, &r.SourceID, &r.TargetID,
|
||||
&r.RelationType, &r.Confidence, &r.DetectionSource, &reason,
|
||||
&r.CreatedAt, &r.CreatedAtEpoch,
|
||||
&rwd.SourceTitle, &rwd.TargetTitle,
|
||||
&rwd.SourceType, &rwd.TargetType,
|
||||
); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if reason.Valid {
|
||||
r.Reason = reason.String
|
||||
}
|
||||
rwd.Relation = &r
|
||||
results = append(results, &rwd)
|
||||
}
|
||||
return results, rows.Err()
|
||||
}
|
||||
|
||||
// GetRelationGraph retrieves a relation graph centered on an observation.
|
||||
// This returns all observations within N hops from the center.
|
||||
func (s *RelationStore) GetRelationGraph(ctx context.Context, centerID int64, maxDepth int) (*models.RelationGraph, error) {
|
||||
// Get all relations involving the center observation
|
||||
relations, err := s.GetRelationsWithDetails(ctx, centerID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
graph := &models.RelationGraph{
|
||||
CenterID: centerID,
|
||||
Relations: relations,
|
||||
}
|
||||
|
||||
// If depth > 1, recursively get relations for connected observations
|
||||
if maxDepth > 1 {
|
||||
visited := map[int64]bool{centerID: true}
|
||||
toVisit := make([]int64, 0)
|
||||
|
||||
// Collect IDs of directly connected observations
|
||||
for _, r := range relations {
|
||||
if !visited[r.Relation.SourceID] {
|
||||
toVisit = append(toVisit, r.Relation.SourceID)
|
||||
visited[r.Relation.SourceID] = true
|
||||
}
|
||||
if !visited[r.Relation.TargetID] {
|
||||
toVisit = append(toVisit, r.Relation.TargetID)
|
||||
visited[r.Relation.TargetID] = true
|
||||
}
|
||||
}
|
||||
|
||||
// Get relations for connected observations (depth - 1)
|
||||
for depth := 1; depth < maxDepth && len(toVisit) > 0; depth++ {
|
||||
nextLevel := make([]int64, 0)
|
||||
for _, obsID := range toVisit {
|
||||
moreRelations, err := s.GetRelationsWithDetails(ctx, obsID)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
for _, r := range moreRelations {
|
||||
// Avoid duplicates
|
||||
exists := false
|
||||
for _, existing := range graph.Relations {
|
||||
if existing.Relation.ID == r.Relation.ID {
|
||||
exists = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !exists {
|
||||
graph.Relations = append(graph.Relations, r)
|
||||
}
|
||||
|
||||
// Queue next level
|
||||
if !visited[r.Relation.SourceID] {
|
||||
nextLevel = append(nextLevel, r.Relation.SourceID)
|
||||
visited[r.Relation.SourceID] = true
|
||||
}
|
||||
if !visited[r.Relation.TargetID] {
|
||||
nextLevel = append(nextLevel, r.Relation.TargetID)
|
||||
visited[r.Relation.TargetID] = true
|
||||
}
|
||||
}
|
||||
}
|
||||
toVisit = nextLevel
|
||||
}
|
||||
}
|
||||
|
||||
return graph, nil
|
||||
}
|
||||
|
||||
// DeleteRelationsByObservationID deletes all relations involving an observation.
|
||||
// Called when an observation is deleted.
|
||||
func (s *RelationStore) DeleteRelationsByObservationID(ctx context.Context, obsID int64) error {
|
||||
const query = `DELETE FROM observation_relations WHERE source_id = ? OR target_id = ?`
|
||||
_, err := s.store.ExecContext(ctx, query, obsID, obsID)
|
||||
return err
|
||||
}
|
||||
|
||||
// GetRelationCount returns the count of relations for an observation.
|
||||
func (s *RelationStore) GetRelationCount(ctx context.Context, obsID int64) (int, error) {
|
||||
const query = `
|
||||
SELECT COUNT(*) FROM observation_relations
|
||||
WHERE source_id = ? OR target_id = ?
|
||||
`
|
||||
var count int
|
||||
err := s.store.QueryRowContext(ctx, query, obsID, obsID).Scan(&count)
|
||||
return count, err
|
||||
}
|
||||
|
||||
// GetTotalRelationCount returns the total count of all relations.
|
||||
func (s *RelationStore) GetTotalRelationCount(ctx context.Context) (int, error) {
|
||||
const query = `SELECT COUNT(*) FROM observation_relations`
|
||||
var count int
|
||||
err := s.store.QueryRowContext(ctx, query).Scan(&count)
|
||||
return count, err
|
||||
}
|
||||
|
||||
// GetHighConfidenceRelations retrieves relations with confidence above threshold.
|
||||
func (s *RelationStore) GetHighConfidenceRelations(ctx context.Context, minConfidence float64, limit int) ([]*models.ObservationRelation, error) {
|
||||
const query = `
|
||||
SELECT id, source_id, target_id, relation_type, confidence, detection_source, reason,
|
||||
created_at, created_at_epoch
|
||||
FROM observation_relations
|
||||
WHERE confidence >= ?
|
||||
ORDER BY confidence DESC, created_at_epoch DESC
|
||||
LIMIT ?
|
||||
`
|
||||
|
||||
rows, err := s.store.QueryContext(ctx, query, minConfidence, limit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
return s.scanRelationRows(rows)
|
||||
}
|
||||
|
||||
// UpdateRelationConfidence updates the confidence of a relation.
|
||||
func (s *RelationStore) UpdateRelationConfidence(ctx context.Context, relationID int64, newConfidence float64) error {
|
||||
const query = `UPDATE observation_relations SET confidence = ? WHERE id = ?`
|
||||
_, err := s.store.ExecContext(ctx, query, newConfidence, relationID)
|
||||
return err
|
||||
}
|
||||
|
||||
// GetRelatedObservationIDs returns IDs of observations related to the given one.
|
||||
// This is useful for expanding search results.
|
||||
func (s *RelationStore) GetRelatedObservationIDs(ctx context.Context, obsID int64, minConfidence float64) ([]int64, error) {
|
||||
const query = `
|
||||
SELECT DISTINCT CASE WHEN source_id = ? THEN target_id ELSE source_id END as related_id
|
||||
FROM observation_relations
|
||||
WHERE (source_id = ? OR target_id = ?) AND confidence >= ?
|
||||
`
|
||||
|
||||
rows, err := s.store.QueryContext(ctx, query, obsID, obsID, obsID, minConfidence)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var ids []int64
|
||||
for rows.Next() {
|
||||
var id int64
|
||||
if err := rows.Scan(&id); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ids = append(ids, id)
|
||||
}
|
||||
return ids, rows.Err()
|
||||
}
|
||||
|
||||
// scanRelationRows scans multiple relations from rows.
|
||||
func (s *RelationStore) scanRelationRows(rows *sql.Rows) ([]*models.ObservationRelation, error) {
|
||||
var relations []*models.ObservationRelation
|
||||
for rows.Next() {
|
||||
var r models.ObservationRelation
|
||||
var reason sql.NullString
|
||||
if err := rows.Scan(
|
||||
&r.ID, &r.SourceID, &r.TargetID,
|
||||
&r.RelationType, &r.Confidence, &r.DetectionSource, &reason,
|
||||
&r.CreatedAt, &r.CreatedAtEpoch,
|
||||
); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if reason.Valid {
|
||||
r.Reason = reason.String
|
||||
}
|
||||
relations = append(relations, &r)
|
||||
}
|
||||
return relations, rows.Err()
|
||||
}
|
||||
@@ -1,324 +0,0 @@
|
||||
// Package sqlite provides SQLite database operations for claude-mnemonic.
|
||||
package sqlite
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"time"
|
||||
|
||||
"github.com/lukaszraczylo/claude-mnemonic/pkg/models"
|
||||
)
|
||||
|
||||
// UpdateObservationFeedback updates the user feedback for an observation.
|
||||
// Feedback values: -1 (thumbs down), 0 (neutral), 1 (thumbs up).
|
||||
func (s *ObservationStore) UpdateObservationFeedback(ctx context.Context, id int64, feedback int) error {
|
||||
const query = `
|
||||
UPDATE observations
|
||||
SET user_feedback = ?, score_updated_at_epoch = ?
|
||||
WHERE id = ?
|
||||
`
|
||||
_, err := s.store.ExecContext(ctx, query, feedback, time.Now().UnixMilli(), id)
|
||||
return err
|
||||
}
|
||||
|
||||
// IncrementRetrievalCount increments the retrieval counter for the given observation IDs.
|
||||
// This is called when observations are returned in search results.
|
||||
func (s *ObservationStore) IncrementRetrievalCount(ctx context.Context, ids []int64) error {
|
||||
if len(ids) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
now := time.Now().UnixMilli()
|
||||
|
||||
// Build query with placeholders
|
||||
// #nosec G202 -- query uses parameterized placeholders, not user input
|
||||
query := `
|
||||
UPDATE observations
|
||||
SET retrieval_count = COALESCE(retrieval_count, 0) + 1,
|
||||
last_retrieved_at_epoch = ?
|
||||
WHERE id IN (?` + repeatPlaceholders(len(ids)-1) + `)
|
||||
`
|
||||
|
||||
args := make([]interface{}, 0, len(ids)+1)
|
||||
args = append(args, now)
|
||||
for _, id := range ids {
|
||||
args = append(args, id)
|
||||
}
|
||||
|
||||
_, err := s.store.db.ExecContext(ctx, query, args...)
|
||||
return err
|
||||
}
|
||||
|
||||
// UpdateImportanceScore updates the importance score for a single observation.
|
||||
func (s *ObservationStore) UpdateImportanceScore(ctx context.Context, id int64, score float64) error {
|
||||
const query = `
|
||||
UPDATE observations
|
||||
SET importance_score = ?, score_updated_at_epoch = ?
|
||||
WHERE id = ?
|
||||
`
|
||||
_, err := s.store.ExecContext(ctx, query, score, time.Now().UnixMilli(), id)
|
||||
return err
|
||||
}
|
||||
|
||||
// UpdateImportanceScores bulk updates importance scores for multiple observations.
|
||||
// This is more efficient than individual updates for batch recalculation.
|
||||
func (s *ObservationStore) UpdateImportanceScores(ctx context.Context, scores map[int64]float64) error {
|
||||
if len(scores) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
tx, err := s.store.db.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() { _ = tx.Rollback() }()
|
||||
|
||||
now := time.Now().UnixMilli()
|
||||
stmt, err := tx.PrepareContext(ctx, `
|
||||
UPDATE observations
|
||||
SET importance_score = ?, score_updated_at_epoch = ?
|
||||
WHERE id = ?
|
||||
`)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer stmt.Close()
|
||||
|
||||
for id, score := range scores {
|
||||
if _, err := stmt.ExecContext(ctx, score, now, id); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return tx.Commit()
|
||||
}
|
||||
|
||||
// GetObservationsNeedingScoreUpdate returns observations that need their importance score recalculated.
|
||||
// Returns observations where score_updated_at_epoch is NULL or older than the threshold.
|
||||
func (s *ObservationStore) GetObservationsNeedingScoreUpdate(ctx context.Context, threshold time.Duration, limit int) ([]*models.Observation, error) {
|
||||
cutoff := time.Now().Add(-threshold).UnixMilli()
|
||||
|
||||
query := `SELECT ` + observationColumns + `
|
||||
FROM observations
|
||||
WHERE score_updated_at_epoch IS NULL OR score_updated_at_epoch < ?
|
||||
ORDER BY created_at_epoch DESC
|
||||
LIMIT ?
|
||||
`
|
||||
|
||||
rows, err := s.store.QueryContext(ctx, query, cutoff, limit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
return scanObservationRows(rows)
|
||||
}
|
||||
|
||||
// GetConceptWeights returns all concept weights from the database.
|
||||
func (s *ObservationStore) GetConceptWeights(ctx context.Context) (map[string]float64, error) {
|
||||
const query = `SELECT concept, weight FROM concept_weights`
|
||||
|
||||
rows, err := s.store.QueryContext(ctx, query)
|
||||
if err != nil {
|
||||
// Table might not exist in older databases
|
||||
if err == sql.ErrNoRows {
|
||||
return models.DefaultConceptWeights, nil
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
weights := make(map[string]float64)
|
||||
for rows.Next() {
|
||||
var concept string
|
||||
var weight float64
|
||||
if err := rows.Scan(&concept, &weight); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
weights[concept] = weight
|
||||
}
|
||||
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// If no weights found, use defaults
|
||||
if len(weights) == 0 {
|
||||
return models.DefaultConceptWeights, nil
|
||||
}
|
||||
|
||||
return weights, nil
|
||||
}
|
||||
|
||||
// UpdateConceptWeight updates a single concept weight.
|
||||
func (s *ObservationStore) UpdateConceptWeight(ctx context.Context, concept string, weight float64) error {
|
||||
const query = `
|
||||
INSERT INTO concept_weights (concept, weight, updated_at)
|
||||
VALUES (?, ?, datetime('now'))
|
||||
ON CONFLICT(concept) DO UPDATE SET weight = excluded.weight, updated_at = excluded.updated_at
|
||||
`
|
||||
_, err := s.store.ExecContext(ctx, query, concept, weight)
|
||||
return err
|
||||
}
|
||||
|
||||
// UpdateConceptWeights bulk updates multiple concept weights.
|
||||
func (s *ObservationStore) UpdateConceptWeights(ctx context.Context, weights map[string]float64) error {
|
||||
if len(weights) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
tx, err := s.store.db.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() { _ = tx.Rollback() }()
|
||||
|
||||
stmt, err := tx.PrepareContext(ctx, `
|
||||
INSERT INTO concept_weights (concept, weight, updated_at)
|
||||
VALUES (?, ?, datetime('now'))
|
||||
ON CONFLICT(concept) DO UPDATE SET weight = excluded.weight, updated_at = excluded.updated_at
|
||||
`)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer stmt.Close()
|
||||
|
||||
for concept, weight := range weights {
|
||||
if _, err := stmt.ExecContext(ctx, concept, weight); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return tx.Commit()
|
||||
}
|
||||
|
||||
// GetObservationFeedbackStats returns statistics about user feedback.
|
||||
func (s *ObservationStore) GetObservationFeedbackStats(ctx context.Context, project string) (*FeedbackStats, error) {
|
||||
var query string
|
||||
var args []interface{}
|
||||
|
||||
if project == "" {
|
||||
query = `
|
||||
SELECT
|
||||
COUNT(*) as total,
|
||||
COALESCE(SUM(CASE WHEN user_feedback = 1 THEN 1 ELSE 0 END), 0) as positive,
|
||||
COALESCE(SUM(CASE WHEN user_feedback = -1 THEN 1 ELSE 0 END), 0) as negative,
|
||||
COALESCE(SUM(CASE WHEN user_feedback = 0 THEN 1 ELSE 0 END), 0) as neutral,
|
||||
COALESCE(AVG(COALESCE(importance_score, 1.0)), 0) as avg_score,
|
||||
COALESCE(AVG(COALESCE(retrieval_count, 0)), 0) as avg_retrieval
|
||||
FROM observations
|
||||
`
|
||||
} else {
|
||||
query = `
|
||||
SELECT
|
||||
COUNT(*) as total,
|
||||
COALESCE(SUM(CASE WHEN user_feedback = 1 THEN 1 ELSE 0 END), 0) as positive,
|
||||
COALESCE(SUM(CASE WHEN user_feedback = -1 THEN 1 ELSE 0 END), 0) as negative,
|
||||
COALESCE(SUM(CASE WHEN user_feedback = 0 THEN 1 ELSE 0 END), 0) as neutral,
|
||||
COALESCE(AVG(COALESCE(importance_score, 1.0)), 0) as avg_score,
|
||||
COALESCE(AVG(COALESCE(retrieval_count, 0)), 0) as avg_retrieval
|
||||
FROM observations
|
||||
WHERE project = ? OR scope = 'global'
|
||||
`
|
||||
args = append(args, project)
|
||||
}
|
||||
|
||||
var stats FeedbackStats
|
||||
err := s.store.QueryRowContext(ctx, query, args...).Scan(
|
||||
&stats.Total,
|
||||
&stats.Positive,
|
||||
&stats.Negative,
|
||||
&stats.Neutral,
|
||||
&stats.AvgScore,
|
||||
&stats.AvgRetrieval,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &stats, nil
|
||||
}
|
||||
|
||||
// FeedbackStats contains statistics about observation feedback and scoring.
|
||||
type FeedbackStats struct {
|
||||
Total int `json:"total"`
|
||||
Positive int `json:"positive"`
|
||||
Negative int `json:"negative"`
|
||||
Neutral int `json:"neutral"`
|
||||
AvgScore float64 `json:"avg_score"`
|
||||
AvgRetrieval float64 `json:"avg_retrieval"`
|
||||
}
|
||||
|
||||
// GetTopScoringObservations returns the highest-scoring observations.
|
||||
func (s *ObservationStore) GetTopScoringObservations(ctx context.Context, project string, limit int) ([]*models.Observation, error) {
|
||||
var query string
|
||||
var args []interface{}
|
||||
|
||||
if project == "" {
|
||||
query = `SELECT ` + observationColumns + `
|
||||
FROM observations
|
||||
ORDER BY COALESCE(importance_score, 1.0) DESC, created_at_epoch DESC
|
||||
LIMIT ?
|
||||
`
|
||||
args = append(args, limit)
|
||||
} else {
|
||||
query = `SELECT ` + observationColumns + `
|
||||
FROM observations
|
||||
WHERE project = ? OR scope = 'global'
|
||||
ORDER BY COALESCE(importance_score, 1.0) DESC, created_at_epoch DESC
|
||||
LIMIT ?
|
||||
`
|
||||
args = append(args, project, limit)
|
||||
}
|
||||
|
||||
rows, err := s.store.QueryContext(ctx, query, args...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
return scanObservationRows(rows)
|
||||
}
|
||||
|
||||
// GetMostRetrievedObservations returns the most frequently retrieved observations.
|
||||
func (s *ObservationStore) GetMostRetrievedObservations(ctx context.Context, project string, limit int) ([]*models.Observation, error) {
|
||||
var query string
|
||||
var args []interface{}
|
||||
|
||||
if project == "" {
|
||||
query = `SELECT ` + observationColumns + `
|
||||
FROM observations
|
||||
WHERE retrieval_count > 0
|
||||
ORDER BY retrieval_count DESC, created_at_epoch DESC
|
||||
LIMIT ?
|
||||
`
|
||||
args = append(args, limit)
|
||||
} else {
|
||||
query = `SELECT ` + observationColumns + `
|
||||
FROM observations
|
||||
WHERE (project = ? OR scope = 'global') AND retrieval_count > 0
|
||||
ORDER BY retrieval_count DESC, created_at_epoch DESC
|
||||
LIMIT ?
|
||||
`
|
||||
args = append(args, project, limit)
|
||||
}
|
||||
|
||||
rows, err := s.store.QueryContext(ctx, query, args...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
return scanObservationRows(rows)
|
||||
}
|
||||
|
||||
// ResetObservationScores resets all observation scores to their default values.
|
||||
// This is useful for testing or when changing the scoring algorithm.
|
||||
func (s *ObservationStore) ResetObservationScores(ctx context.Context) error {
|
||||
const query = `
|
||||
UPDATE observations
|
||||
SET importance_score = 1.0, score_updated_at_epoch = NULL
|
||||
`
|
||||
_, err := s.store.ExecContext(ctx, query)
|
||||
return err
|
||||
}
|
||||
@@ -1,698 +0,0 @@
|
||||
// Package sqlite provides SQLite database operations for claude-mnemonic.
|
||||
package sqlite
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/lukaszraczylo/claude-mnemonic/pkg/models"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"github.com/stretchr/testify/suite"
|
||||
)
|
||||
|
||||
// testScoringObservationStore creates an ObservationStore with scoring columns for testing.
|
||||
func testScoringObservationStore(t *testing.T) (*ObservationStore, *Store, func()) {
|
||||
t.Helper()
|
||||
|
||||
db, _, cleanup := testDB(t)
|
||||
createBaseTables(t, db)
|
||||
createConceptWeightsTable(t, db)
|
||||
|
||||
// Add importance index if not exists (columns already in createBaseTables)
|
||||
if _, err := db.Exec(`CREATE INDEX IF NOT EXISTS idx_observations_importance ON observations(importance_score DESC, created_at_epoch DESC)`); err != nil {
|
||||
t.Fatalf("create importance index: %v", err)
|
||||
}
|
||||
|
||||
store := newStoreFromDB(db)
|
||||
obsStore := NewObservationStore(store)
|
||||
|
||||
return obsStore, store, cleanup
|
||||
}
|
||||
|
||||
// createConceptWeightsTable creates the concept_weights table for testing.
|
||||
func createConceptWeightsTable(t *testing.T, db *sql.DB) {
|
||||
t.Helper()
|
||||
|
||||
_, err := db.Exec(`
|
||||
CREATE TABLE IF NOT EXISTS concept_weights (
|
||||
concept TEXT PRIMARY KEY,
|
||||
weight REAL NOT NULL DEFAULT 0.1,
|
||||
updated_at TEXT NOT NULL
|
||||
)
|
||||
`)
|
||||
if err != nil {
|
||||
t.Fatalf("create concept_weights: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// ScoringStoreSuite is a test suite for scoring-related database operations.
|
||||
type ScoringStoreSuite struct {
|
||||
suite.Suite
|
||||
obsStore *ObservationStore
|
||||
store *Store
|
||||
cleanup func()
|
||||
ctx context.Context
|
||||
}
|
||||
|
||||
func (s *ScoringStoreSuite) SetupTest() {
|
||||
s.obsStore, s.store, s.cleanup = testScoringObservationStore(s.T())
|
||||
s.ctx = context.Background()
|
||||
}
|
||||
|
||||
func (s *ScoringStoreSuite) TearDownTest() {
|
||||
if s.cleanup != nil {
|
||||
s.cleanup()
|
||||
}
|
||||
}
|
||||
|
||||
func TestScoringStoreSuite(t *testing.T) {
|
||||
suite.Run(t, new(ScoringStoreSuite))
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
// FEEDBACK TESTS
|
||||
// =============================================================================
|
||||
|
||||
func (s *ScoringStoreSuite) TestUpdateObservationFeedback_Positive() {
|
||||
// Create observation
|
||||
obs := &models.ParsedObservation{
|
||||
Type: models.ObsTypeBugfix,
|
||||
Title: "Test feedback",
|
||||
}
|
||||
id, _, err := s.obsStore.StoreObservation(s.ctx, "session-1", "project-a", obs, 1, 100)
|
||||
s.NoError(err)
|
||||
|
||||
// Update feedback to positive
|
||||
err = s.obsStore.UpdateObservationFeedback(s.ctx, id, 1)
|
||||
s.NoError(err)
|
||||
|
||||
// Verify
|
||||
retrieved, err := s.obsStore.GetObservationByID(s.ctx, id)
|
||||
s.NoError(err)
|
||||
s.Equal(1, retrieved.UserFeedback)
|
||||
s.True(retrieved.ScoreUpdatedAt.Valid)
|
||||
}
|
||||
|
||||
func (s *ScoringStoreSuite) TestUpdateObservationFeedback_Negative() {
|
||||
obs := &models.ParsedObservation{
|
||||
Type: models.ObsTypeDiscovery,
|
||||
}
|
||||
id, _, err := s.obsStore.StoreObservation(s.ctx, "session-1", "project-a", obs, 1, 100)
|
||||
s.NoError(err)
|
||||
|
||||
err = s.obsStore.UpdateObservationFeedback(s.ctx, id, -1)
|
||||
s.NoError(err)
|
||||
|
||||
retrieved, err := s.obsStore.GetObservationByID(s.ctx, id)
|
||||
s.NoError(err)
|
||||
s.Equal(-1, retrieved.UserFeedback)
|
||||
}
|
||||
|
||||
func (s *ScoringStoreSuite) TestUpdateObservationFeedback_Neutral() {
|
||||
obs := &models.ParsedObservation{
|
||||
Type: models.ObsTypeChange,
|
||||
}
|
||||
id, _, err := s.obsStore.StoreObservation(s.ctx, "session-1", "project-a", obs, 1, 100)
|
||||
s.NoError(err)
|
||||
|
||||
// First set to positive
|
||||
err = s.obsStore.UpdateObservationFeedback(s.ctx, id, 1)
|
||||
s.NoError(err)
|
||||
|
||||
// Then reset to neutral
|
||||
err = s.obsStore.UpdateObservationFeedback(s.ctx, id, 0)
|
||||
s.NoError(err)
|
||||
|
||||
retrieved, err := s.obsStore.GetObservationByID(s.ctx, id)
|
||||
s.NoError(err)
|
||||
s.Equal(0, retrieved.UserFeedback)
|
||||
}
|
||||
|
||||
func (s *ScoringStoreSuite) TestUpdateObservationFeedback_NonExistent() {
|
||||
// Updating non-existent observation should not fail (just no rows affected)
|
||||
err := s.obsStore.UpdateObservationFeedback(s.ctx, 99999, 1)
|
||||
s.NoError(err)
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
// RETRIEVAL COUNT TESTS
|
||||
// =============================================================================
|
||||
|
||||
func (s *ScoringStoreSuite) TestIncrementRetrievalCount_Single() {
|
||||
obs := &models.ParsedObservation{
|
||||
Type: models.ObsTypeBugfix,
|
||||
}
|
||||
id, _, err := s.obsStore.StoreObservation(s.ctx, "session-1", "project-a", obs, 1, 100)
|
||||
s.NoError(err)
|
||||
|
||||
err = s.obsStore.IncrementRetrievalCount(s.ctx, []int64{id})
|
||||
s.NoError(err)
|
||||
|
||||
retrieved, err := s.obsStore.GetObservationByID(s.ctx, id)
|
||||
s.NoError(err)
|
||||
s.Equal(1, retrieved.RetrievalCount)
|
||||
s.True(retrieved.LastRetrievedAt.Valid)
|
||||
}
|
||||
|
||||
func (s *ScoringStoreSuite) TestIncrementRetrievalCount_Multiple() {
|
||||
var ids []int64
|
||||
for i := 0; i < 3; i++ {
|
||||
obs := &models.ParsedObservation{
|
||||
Type: models.ObsTypeDiscovery,
|
||||
}
|
||||
id, _, err := s.obsStore.StoreObservation(s.ctx, "session-1", "project-a", obs, i, 100)
|
||||
s.NoError(err)
|
||||
ids = append(ids, id)
|
||||
}
|
||||
|
||||
err := s.obsStore.IncrementRetrievalCount(s.ctx, ids)
|
||||
s.NoError(err)
|
||||
|
||||
for _, id := range ids {
|
||||
retrieved, err := s.obsStore.GetObservationByID(s.ctx, id)
|
||||
s.NoError(err)
|
||||
s.Equal(1, retrieved.RetrievalCount)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *ScoringStoreSuite) TestIncrementRetrievalCount_Cumulative() {
|
||||
obs := &models.ParsedObservation{
|
||||
Type: models.ObsTypeBugfix,
|
||||
}
|
||||
id, _, err := s.obsStore.StoreObservation(s.ctx, "session-1", "project-a", obs, 1, 100)
|
||||
s.NoError(err)
|
||||
|
||||
// Increment multiple times
|
||||
for i := 0; i < 5; i++ {
|
||||
err = s.obsStore.IncrementRetrievalCount(s.ctx, []int64{id})
|
||||
s.NoError(err)
|
||||
}
|
||||
|
||||
retrieved, err := s.obsStore.GetObservationByID(s.ctx, id)
|
||||
s.NoError(err)
|
||||
s.Equal(5, retrieved.RetrievalCount)
|
||||
}
|
||||
|
||||
func (s *ScoringStoreSuite) TestIncrementRetrievalCount_Empty() {
|
||||
err := s.obsStore.IncrementRetrievalCount(s.ctx, []int64{})
|
||||
s.NoError(err)
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
// IMPORTANCE SCORE TESTS
|
||||
// =============================================================================
|
||||
|
||||
func (s *ScoringStoreSuite) TestUpdateImportanceScore_Single() {
|
||||
obs := &models.ParsedObservation{
|
||||
Type: models.ObsTypeBugfix,
|
||||
}
|
||||
id, _, err := s.obsStore.StoreObservation(s.ctx, "session-1", "project-a", obs, 1, 100)
|
||||
s.NoError(err)
|
||||
|
||||
err = s.obsStore.UpdateImportanceScore(s.ctx, id, 1.5)
|
||||
s.NoError(err)
|
||||
|
||||
retrieved, err := s.obsStore.GetObservationByID(s.ctx, id)
|
||||
s.NoError(err)
|
||||
s.InDelta(1.5, retrieved.ImportanceScore, 0.001)
|
||||
}
|
||||
|
||||
func (s *ScoringStoreSuite) TestUpdateImportanceScores_Batch() {
|
||||
var ids []int64
|
||||
for i := 0; i < 5; i++ {
|
||||
obs := &models.ParsedObservation{
|
||||
Type: models.ObsTypeDiscovery,
|
||||
}
|
||||
id, _, err := s.obsStore.StoreObservation(s.ctx, "session-1", "project-a", obs, i, 100)
|
||||
s.NoError(err)
|
||||
ids = append(ids, id)
|
||||
}
|
||||
|
||||
scores := map[int64]float64{
|
||||
ids[0]: 1.5,
|
||||
ids[1]: 0.8,
|
||||
ids[2]: 1.2,
|
||||
ids[3]: 0.5,
|
||||
ids[4]: 2.0,
|
||||
}
|
||||
|
||||
err := s.obsStore.UpdateImportanceScores(s.ctx, scores)
|
||||
s.NoError(err)
|
||||
|
||||
for id, expectedScore := range scores {
|
||||
retrieved, err := s.obsStore.GetObservationByID(s.ctx, id)
|
||||
s.NoError(err)
|
||||
s.InDelta(expectedScore, retrieved.ImportanceScore, 0.001)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *ScoringStoreSuite) TestUpdateImportanceScores_Empty() {
|
||||
err := s.obsStore.UpdateImportanceScores(s.ctx, map[int64]float64{})
|
||||
s.NoError(err)
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
// OBSERVATIONS NEEDING SCORE UPDATE TESTS
|
||||
// =============================================================================
|
||||
|
||||
func (s *ScoringStoreSuite) TestGetObservationsNeedingScoreUpdate_NeverUpdated() {
|
||||
// Observations without score_updated_at_epoch should need update
|
||||
obs := &models.ParsedObservation{
|
||||
Type: models.ObsTypeBugfix,
|
||||
}
|
||||
id, _, err := s.obsStore.StoreObservation(s.ctx, "session-1", "project-a", obs, 1, 100)
|
||||
s.NoError(err)
|
||||
|
||||
observations, err := s.obsStore.GetObservationsNeedingScoreUpdate(s.ctx, 6*time.Hour, 100)
|
||||
s.NoError(err)
|
||||
s.Len(observations, 1)
|
||||
s.Equal(id, observations[0].ID)
|
||||
}
|
||||
|
||||
func (s *ScoringStoreSuite) TestGetObservationsNeedingScoreUpdate_RecentlyUpdated() {
|
||||
obs := &models.ParsedObservation{
|
||||
Type: models.ObsTypeBugfix,
|
||||
}
|
||||
id, _, err := s.obsStore.StoreObservation(s.ctx, "session-1", "project-a", obs, 1, 100)
|
||||
s.NoError(err)
|
||||
|
||||
// Update score (this sets score_updated_at_epoch)
|
||||
err = s.obsStore.UpdateImportanceScore(s.ctx, id, 1.5)
|
||||
s.NoError(err)
|
||||
|
||||
// Should not need update (just updated)
|
||||
observations, err := s.obsStore.GetObservationsNeedingScoreUpdate(s.ctx, 6*time.Hour, 100)
|
||||
s.NoError(err)
|
||||
s.Empty(observations)
|
||||
}
|
||||
|
||||
func (s *ScoringStoreSuite) TestGetObservationsNeedingScoreUpdate_Limit() {
|
||||
// Create 10 observations
|
||||
for i := 0; i < 10; i++ {
|
||||
obs := &models.ParsedObservation{
|
||||
Type: models.ObsTypeDiscovery,
|
||||
}
|
||||
_, _, err := s.obsStore.StoreObservation(s.ctx, "session-1", "project-a", obs, i, 100)
|
||||
s.NoError(err)
|
||||
}
|
||||
|
||||
// Request only 5
|
||||
observations, err := s.obsStore.GetObservationsNeedingScoreUpdate(s.ctx, 6*time.Hour, 5)
|
||||
s.NoError(err)
|
||||
s.Len(observations, 5)
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
// CONCEPT WEIGHTS TESTS
|
||||
// =============================================================================
|
||||
|
||||
func (s *ScoringStoreSuite) TestGetConceptWeights_Empty() {
|
||||
weights, err := s.obsStore.GetConceptWeights(s.ctx)
|
||||
s.NoError(err)
|
||||
s.Equal(models.DefaultConceptWeights, weights)
|
||||
}
|
||||
|
||||
func (s *ScoringStoreSuite) TestUpdateConceptWeight_NewConcept() {
|
||||
err := s.obsStore.UpdateConceptWeight(s.ctx, "new-concept", 0.42)
|
||||
s.NoError(err)
|
||||
|
||||
weights, err := s.obsStore.GetConceptWeights(s.ctx)
|
||||
s.NoError(err)
|
||||
s.Equal(0.42, weights["new-concept"])
|
||||
}
|
||||
|
||||
func (s *ScoringStoreSuite) TestUpdateConceptWeight_UpdateExisting() {
|
||||
// Insert first
|
||||
err := s.obsStore.UpdateConceptWeight(s.ctx, "test-concept", 0.1)
|
||||
s.NoError(err)
|
||||
|
||||
// Update
|
||||
err = s.obsStore.UpdateConceptWeight(s.ctx, "test-concept", 0.9)
|
||||
s.NoError(err)
|
||||
|
||||
weights, err := s.obsStore.GetConceptWeights(s.ctx)
|
||||
s.NoError(err)
|
||||
s.Equal(0.9, weights["test-concept"])
|
||||
}
|
||||
|
||||
func (s *ScoringStoreSuite) TestUpdateConceptWeights_Batch() {
|
||||
weightsToSet := map[string]float64{
|
||||
"security": 0.5,
|
||||
"performance": 0.3,
|
||||
"testing": 0.2,
|
||||
}
|
||||
|
||||
err := s.obsStore.UpdateConceptWeights(s.ctx, weightsToSet)
|
||||
s.NoError(err)
|
||||
|
||||
retrieved, err := s.obsStore.GetConceptWeights(s.ctx)
|
||||
s.NoError(err)
|
||||
|
||||
for concept, expected := range weightsToSet {
|
||||
s.Equal(expected, retrieved[concept])
|
||||
}
|
||||
}
|
||||
|
||||
func (s *ScoringStoreSuite) TestUpdateConceptWeights_Empty() {
|
||||
err := s.obsStore.UpdateConceptWeights(s.ctx, map[string]float64{})
|
||||
s.NoError(err)
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
// FEEDBACK STATS TESTS
|
||||
// =============================================================================
|
||||
|
||||
func (s *ScoringStoreSuite) TestGetObservationFeedbackStats_Empty() {
|
||||
stats, err := s.obsStore.GetObservationFeedbackStats(s.ctx, "")
|
||||
s.NoError(err)
|
||||
s.Equal(0, stats.Total)
|
||||
s.Equal(0, stats.Positive)
|
||||
s.Equal(0, stats.Negative)
|
||||
s.Equal(0, stats.Neutral)
|
||||
}
|
||||
|
||||
func (s *ScoringStoreSuite) TestGetObservationFeedbackStats_WithData() {
|
||||
// Create observations with different feedback
|
||||
feedbacks := []int{1, 1, 1, -1, -1, 0, 0, 0, 0, 0}
|
||||
for i, fb := range feedbacks {
|
||||
obs := &models.ParsedObservation{
|
||||
Type: models.ObsTypeDiscovery,
|
||||
}
|
||||
id, _, err := s.obsStore.StoreObservation(s.ctx, "session-1", "project-a", obs, i, 100)
|
||||
s.NoError(err)
|
||||
if fb != 0 {
|
||||
err = s.obsStore.UpdateObservationFeedback(s.ctx, id, fb)
|
||||
s.NoError(err)
|
||||
}
|
||||
}
|
||||
|
||||
stats, err := s.obsStore.GetObservationFeedbackStats(s.ctx, "")
|
||||
s.NoError(err)
|
||||
s.Equal(10, stats.Total)
|
||||
s.Equal(3, stats.Positive)
|
||||
s.Equal(2, stats.Negative)
|
||||
s.Equal(5, stats.Neutral)
|
||||
}
|
||||
|
||||
func (s *ScoringStoreSuite) TestGetObservationFeedbackStats_ByProject() {
|
||||
// Project A observations
|
||||
for i := 0; i < 5; i++ {
|
||||
obs := &models.ParsedObservation{
|
||||
Type: models.ObsTypeBugfix,
|
||||
}
|
||||
id, _, err := s.obsStore.StoreObservation(s.ctx, "session-1", "project-a", obs, i, 100)
|
||||
s.NoError(err)
|
||||
_ = s.obsStore.UpdateObservationFeedback(s.ctx, id, 1)
|
||||
}
|
||||
|
||||
// Project B observations
|
||||
for i := 0; i < 3; i++ {
|
||||
obs := &models.ParsedObservation{
|
||||
Type: models.ObsTypeFeature,
|
||||
}
|
||||
id, _, err := s.obsStore.StoreObservation(s.ctx, "session-1", "project-b", obs, i, 100)
|
||||
s.NoError(err)
|
||||
_ = s.obsStore.UpdateObservationFeedback(s.ctx, id, -1)
|
||||
}
|
||||
|
||||
// Check project A stats
|
||||
statsA, err := s.obsStore.GetObservationFeedbackStats(s.ctx, "project-a")
|
||||
s.NoError(err)
|
||||
s.Equal(5, statsA.Total)
|
||||
s.Equal(5, statsA.Positive)
|
||||
|
||||
// Check project B stats
|
||||
statsB, err := s.obsStore.GetObservationFeedbackStats(s.ctx, "project-b")
|
||||
s.NoError(err)
|
||||
s.Equal(3, statsB.Total)
|
||||
s.Equal(3, statsB.Negative)
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
// TOP SCORING OBSERVATIONS TESTS
|
||||
// =============================================================================
|
||||
|
||||
func (s *ScoringStoreSuite) TestGetTopScoringObservations() {
|
||||
// Create observations with different scores
|
||||
for i := 0; i < 5; i++ {
|
||||
obs := &models.ParsedObservation{
|
||||
Type: models.ObsTypeDiscovery,
|
||||
}
|
||||
id, _, err := s.obsStore.StoreObservation(s.ctx, "session-1", "project-a", obs, i, 100)
|
||||
s.NoError(err)
|
||||
// Set different scores
|
||||
err = s.obsStore.UpdateImportanceScore(s.ctx, id, float64(i+1)*0.5)
|
||||
s.NoError(err)
|
||||
}
|
||||
|
||||
// Get top 3
|
||||
top, err := s.obsStore.GetTopScoringObservations(s.ctx, "", 3)
|
||||
s.NoError(err)
|
||||
s.Len(top, 3)
|
||||
|
||||
// Verify ordered by score descending
|
||||
s.GreaterOrEqual(top[0].ImportanceScore, top[1].ImportanceScore)
|
||||
s.GreaterOrEqual(top[1].ImportanceScore, top[2].ImportanceScore)
|
||||
}
|
||||
|
||||
func (s *ScoringStoreSuite) TestGetTopScoringObservations_ByProject() {
|
||||
// Project A with high scores
|
||||
for i := 0; i < 3; i++ {
|
||||
obs := &models.ParsedObservation{
|
||||
Type: models.ObsTypeBugfix,
|
||||
}
|
||||
id, _, err := s.obsStore.StoreObservation(s.ctx, "session-1", "project-a", obs, i, 100)
|
||||
s.NoError(err)
|
||||
_ = s.obsStore.UpdateImportanceScore(s.ctx, id, 2.0)
|
||||
}
|
||||
|
||||
// Project B with low scores
|
||||
for i := 0; i < 3; i++ {
|
||||
obs := &models.ParsedObservation{
|
||||
Type: models.ObsTypeChange,
|
||||
}
|
||||
id, _, err := s.obsStore.StoreObservation(s.ctx, "session-1", "project-b", obs, i, 100)
|
||||
s.NoError(err)
|
||||
_ = s.obsStore.UpdateImportanceScore(s.ctx, id, 0.5)
|
||||
}
|
||||
|
||||
// Get top for project A
|
||||
topA, err := s.obsStore.GetTopScoringObservations(s.ctx, "project-a", 10)
|
||||
s.NoError(err)
|
||||
s.Len(topA, 3)
|
||||
for _, obs := range topA {
|
||||
s.Equal("project-a", obs.Project)
|
||||
}
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
// MOST RETRIEVED OBSERVATIONS TESTS
|
||||
// =============================================================================
|
||||
|
||||
func (s *ScoringStoreSuite) TestGetMostRetrievedObservations() {
|
||||
// Create observations with different retrieval counts
|
||||
var ids []int64
|
||||
for i := 0; i < 5; i++ {
|
||||
obs := &models.ParsedObservation{
|
||||
Type: models.ObsTypeDiscovery,
|
||||
}
|
||||
id, _, err := s.obsStore.StoreObservation(s.ctx, "session-1", "project-a", obs, i, 100)
|
||||
s.NoError(err)
|
||||
ids = append(ids, id)
|
||||
}
|
||||
|
||||
// Set different retrieval counts
|
||||
for i := 0; i < 10; i++ {
|
||||
_ = s.obsStore.IncrementRetrievalCount(s.ctx, []int64{ids[0]}) // 10 retrievals
|
||||
}
|
||||
for i := 0; i < 5; i++ {
|
||||
_ = s.obsStore.IncrementRetrievalCount(s.ctx, []int64{ids[1]}) // 5 retrievals
|
||||
}
|
||||
for i := 0; i < 3; i++ {
|
||||
_ = s.obsStore.IncrementRetrievalCount(s.ctx, []int64{ids[2]}) // 3 retrievals
|
||||
}
|
||||
// ids[3] and ids[4] have 0 retrievals
|
||||
|
||||
// Get top 3
|
||||
most, err := s.obsStore.GetMostRetrievedObservations(s.ctx, "", 3)
|
||||
s.NoError(err)
|
||||
s.Len(most, 3)
|
||||
|
||||
// Verify ordered by retrieval count descending
|
||||
s.Equal(10, most[0].RetrievalCount)
|
||||
s.Equal(5, most[1].RetrievalCount)
|
||||
s.Equal(3, most[2].RetrievalCount)
|
||||
}
|
||||
|
||||
func (s *ScoringStoreSuite) TestGetMostRetrievedObservations_NoRetrievals() {
|
||||
// Create observations without any retrievals
|
||||
obs := &models.ParsedObservation{
|
||||
Type: models.ObsTypeDiscovery,
|
||||
}
|
||||
_, _, err := s.obsStore.StoreObservation(s.ctx, "session-1", "project-a", obs, 1, 100)
|
||||
s.NoError(err)
|
||||
|
||||
most, err := s.obsStore.GetMostRetrievedObservations(s.ctx, "", 10)
|
||||
s.NoError(err)
|
||||
s.Empty(most) // No observations with retrieval_count > 0
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
// RESET OBSERVATION SCORES TESTS
|
||||
// =============================================================================
|
||||
|
||||
func (s *ScoringStoreSuite) TestResetObservationScores() {
|
||||
// Create observations with various scores
|
||||
for i := 0; i < 5; i++ {
|
||||
obs := &models.ParsedObservation{
|
||||
Type: models.ObsTypeDiscovery,
|
||||
}
|
||||
id, _, err := s.obsStore.StoreObservation(s.ctx, "session-1", "project-a", obs, i, 100)
|
||||
s.NoError(err)
|
||||
_ = s.obsStore.UpdateImportanceScore(s.ctx, id, float64(i+1))
|
||||
}
|
||||
|
||||
// Reset all scores
|
||||
err := s.obsStore.ResetObservationScores(s.ctx)
|
||||
s.NoError(err)
|
||||
|
||||
// Verify all scores are reset to 1.0
|
||||
observations, err := s.obsStore.GetAllRecentObservations(s.ctx, 100)
|
||||
s.NoError(err)
|
||||
for _, obs := range observations {
|
||||
s.InDelta(1.0, obs.ImportanceScore, 0.001)
|
||||
s.False(obs.ScoreUpdatedAt.Valid)
|
||||
}
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
// EDGE CASES
|
||||
// =============================================================================
|
||||
|
||||
func (s *ScoringStoreSuite) TestScoring_ZeroScore() {
|
||||
obs := &models.ParsedObservation{
|
||||
Type: models.ObsTypeBugfix,
|
||||
}
|
||||
id, _, err := s.obsStore.StoreObservation(s.ctx, "session-1", "project-a", obs, 1, 100)
|
||||
s.NoError(err)
|
||||
|
||||
// Set score to 0
|
||||
err = s.obsStore.UpdateImportanceScore(s.ctx, id, 0.0)
|
||||
s.NoError(err)
|
||||
|
||||
retrieved, err := s.obsStore.GetObservationByID(s.ctx, id)
|
||||
s.NoError(err)
|
||||
s.InDelta(0.0, retrieved.ImportanceScore, 0.001)
|
||||
}
|
||||
|
||||
func (s *ScoringStoreSuite) TestScoring_NegativeScore() {
|
||||
obs := &models.ParsedObservation{
|
||||
Type: models.ObsTypeBugfix,
|
||||
}
|
||||
id, _, err := s.obsStore.StoreObservation(s.ctx, "session-1", "project-a", obs, 1, 100)
|
||||
s.NoError(err)
|
||||
|
||||
// Set negative score (calculator shouldn't produce this, but test DB handling)
|
||||
err = s.obsStore.UpdateImportanceScore(s.ctx, id, -0.5)
|
||||
s.NoError(err)
|
||||
|
||||
retrieved, err := s.obsStore.GetObservationByID(s.ctx, id)
|
||||
s.NoError(err)
|
||||
s.InDelta(-0.5, retrieved.ImportanceScore, 0.001)
|
||||
}
|
||||
|
||||
func (s *ScoringStoreSuite) TestScoring_LargeScore() {
|
||||
obs := &models.ParsedObservation{
|
||||
Type: models.ObsTypeBugfix,
|
||||
}
|
||||
id, _, err := s.obsStore.StoreObservation(s.ctx, "session-1", "project-a", obs, 1, 100)
|
||||
s.NoError(err)
|
||||
|
||||
// Set very large score
|
||||
err = s.obsStore.UpdateImportanceScore(s.ctx, id, 999.999)
|
||||
s.NoError(err)
|
||||
|
||||
retrieved, err := s.obsStore.GetObservationByID(s.ctx, id)
|
||||
s.NoError(err)
|
||||
s.InDelta(999.999, retrieved.ImportanceScore, 0.001)
|
||||
}
|
||||
|
||||
func (s *ScoringStoreSuite) TestConceptWeight_ZeroWeight() {
|
||||
err := s.obsStore.UpdateConceptWeight(s.ctx, "zero-concept", 0.0)
|
||||
s.NoError(err)
|
||||
|
||||
weights, err := s.obsStore.GetConceptWeights(s.ctx)
|
||||
s.NoError(err)
|
||||
s.Equal(0.0, weights["zero-concept"])
|
||||
}
|
||||
|
||||
func (s *ScoringStoreSuite) TestConceptWeight_ExactBoundary() {
|
||||
err := s.obsStore.UpdateConceptWeight(s.ctx, "max-concept", 1.0)
|
||||
s.NoError(err)
|
||||
|
||||
weights, err := s.obsStore.GetConceptWeights(s.ctx)
|
||||
s.NoError(err)
|
||||
s.Equal(1.0, weights["max-concept"])
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
// STANDALONE TESTS
|
||||
// =============================================================================
|
||||
|
||||
func TestFeedbackStats_Structure(t *testing.T) {
|
||||
stats := FeedbackStats{
|
||||
Total: 100,
|
||||
Positive: 30,
|
||||
Negative: 10,
|
||||
Neutral: 60,
|
||||
AvgScore: 1.5,
|
||||
AvgRetrieval: 5.0,
|
||||
}
|
||||
|
||||
assert.Equal(t, 100, stats.Total)
|
||||
assert.Equal(t, 30, stats.Positive)
|
||||
assert.Equal(t, 10, stats.Negative)
|
||||
assert.Equal(t, 60, stats.Neutral)
|
||||
assert.Equal(t, 1.5, stats.AvgScore)
|
||||
assert.Equal(t, 5.0, stats.AvgRetrieval)
|
||||
}
|
||||
|
||||
func TestScoringStore_Integration(t *testing.T) {
|
||||
obsStore, _, cleanup := testScoringObservationStore(t)
|
||||
defer cleanup()
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
// Full integration test: store, feedback, retrieval, score update
|
||||
obs := &models.ParsedObservation{
|
||||
Type: models.ObsTypeBugfix,
|
||||
Title: "Integration test observation",
|
||||
Concepts: []string{"security"},
|
||||
}
|
||||
id, _, err := obsStore.StoreObservation(ctx, "session-int", "project-int", obs, 1, 100)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Add feedback
|
||||
err = obsStore.UpdateObservationFeedback(ctx, id, 1)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Increment retrieval
|
||||
err = obsStore.IncrementRetrievalCount(ctx, []int64{id})
|
||||
require.NoError(t, err)
|
||||
|
||||
// Update score
|
||||
err = obsStore.UpdateImportanceScore(ctx, id, 1.75)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Verify final state
|
||||
retrieved, err := obsStore.GetObservationByID(ctx, id)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 1, retrieved.UserFeedback)
|
||||
assert.Equal(t, 1, retrieved.RetrievalCount)
|
||||
assert.InDelta(t, 1.75, retrieved.ImportanceScore, 0.001)
|
||||
assert.True(t, retrieved.ScoreUpdatedAt.Valid)
|
||||
assert.True(t, retrieved.LastRetrievedAt.Valid)
|
||||
}
|
||||
@@ -8,14 +8,13 @@ import (
|
||||
"github.com/lukaszraczylo/claude-mnemonic/pkg/models"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"github.com/stretchr/testify/suite"
|
||||
)
|
||||
|
||||
func testSessionStore(t *testing.T) (*SessionStore, *Store, func()) {
|
||||
t.Helper()
|
||||
|
||||
db, _, cleanup := testDB(t)
|
||||
createBaseTables(t, db) // Use base tables without FTS5 for session tests
|
||||
createAllTables(t, db)
|
||||
|
||||
store := newStoreFromDB(db)
|
||||
sessionStore := NewSessionStore(store)
|
||||
@@ -23,236 +22,6 @@ func testSessionStore(t *testing.T) (*SessionStore, *Store, func()) {
|
||||
return sessionStore, store, cleanup
|
||||
}
|
||||
|
||||
// SessionStoreSuite is a test suite for SessionStore operations.
|
||||
type SessionStoreSuite struct {
|
||||
suite.Suite
|
||||
sessionStore *SessionStore
|
||||
store *Store
|
||||
cleanup func()
|
||||
}
|
||||
|
||||
func (s *SessionStoreSuite) SetupTest() {
|
||||
s.sessionStore, s.store, s.cleanup = testSessionStore(s.T())
|
||||
}
|
||||
|
||||
func (s *SessionStoreSuite) TearDownTest() {
|
||||
if s.cleanup != nil {
|
||||
s.cleanup()
|
||||
}
|
||||
}
|
||||
|
||||
func TestSessionStoreSuite(t *testing.T) {
|
||||
suite.Run(t, new(SessionStoreSuite))
|
||||
}
|
||||
|
||||
// TestCreateSDKSession_TableDriven tests session creation with various scenarios.
|
||||
func (s *SessionStoreSuite) TestCreateSDKSession_TableDriven() {
|
||||
ctx := context.Background()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
claudeSessionID string
|
||||
project string
|
||||
userPrompt string
|
||||
wantErr bool
|
||||
wantID bool
|
||||
}{
|
||||
{
|
||||
name: "basic session creation",
|
||||
claudeSessionID: "claude-basic",
|
||||
project: "project-a",
|
||||
userPrompt: "hello world",
|
||||
wantErr: false,
|
||||
wantID: true,
|
||||
},
|
||||
{
|
||||
name: "empty user prompt",
|
||||
claudeSessionID: "claude-noprompt",
|
||||
project: "project-b",
|
||||
userPrompt: "",
|
||||
wantErr: false,
|
||||
wantID: true,
|
||||
},
|
||||
{
|
||||
name: "long project name",
|
||||
claudeSessionID: "claude-longproj",
|
||||
project: "/Users/test/Documents/very/long/path/to/some/project/directory",
|
||||
userPrompt: "test",
|
||||
wantErr: false,
|
||||
wantID: true,
|
||||
},
|
||||
{
|
||||
name: "unicode project name",
|
||||
claudeSessionID: "claude-unicode",
|
||||
project: "项目名称-プロジェクト",
|
||||
userPrompt: "测试 テスト",
|
||||
wantErr: false,
|
||||
wantID: true,
|
||||
},
|
||||
{
|
||||
name: "special characters in prompt",
|
||||
claudeSessionID: "claude-special",
|
||||
project: "project-special",
|
||||
userPrompt: "Fix the bug in file.go:123 with \"quotes\" and 'apostrophes'",
|
||||
wantErr: false,
|
||||
wantID: true,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
s.Run(tt.name, func() {
|
||||
id, err := s.sessionStore.CreateSDKSession(ctx, tt.claudeSessionID, tt.project, tt.userPrompt)
|
||||
if tt.wantErr {
|
||||
s.Error(err)
|
||||
} else {
|
||||
s.NoError(err)
|
||||
if tt.wantID {
|
||||
s.Greater(id, int64(0))
|
||||
}
|
||||
|
||||
// Verify created session
|
||||
sess, err := s.sessionStore.GetSessionByID(ctx, id)
|
||||
s.NoError(err)
|
||||
s.NotNil(sess)
|
||||
s.Equal(tt.claudeSessionID, sess.ClaudeSessionID)
|
||||
s.Equal(tt.project, sess.Project)
|
||||
s.Equal(models.SessionStatusActive, sess.Status)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestIdempotentSession tests that session creation is idempotent.
|
||||
func (s *SessionStoreSuite) TestIdempotentSession() {
|
||||
ctx := context.Background()
|
||||
|
||||
// Create initial session
|
||||
id1, err := s.sessionStore.CreateSDKSession(ctx, "claude-idem", "project-1", "prompt-1")
|
||||
s.NoError(err)
|
||||
s.Greater(id1, int64(0))
|
||||
|
||||
// Create with same claude_session_id - should return same ID
|
||||
id2, err := s.sessionStore.CreateSDKSession(ctx, "claude-idem", "project-2", "prompt-2")
|
||||
s.NoError(err)
|
||||
s.Equal(id1, id2)
|
||||
|
||||
// Verify project was updated
|
||||
sess, err := s.sessionStore.GetSessionByID(ctx, id1)
|
||||
s.NoError(err)
|
||||
s.Equal("project-2", sess.Project)
|
||||
}
|
||||
|
||||
// TestPromptCounterOperations tests prompt counter increment and retrieval.
|
||||
func (s *SessionStoreSuite) TestPromptCounterOperations() {
|
||||
ctx := context.Background()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
increments int
|
||||
expectedCount int
|
||||
}{
|
||||
{
|
||||
name: "no increments",
|
||||
increments: 0,
|
||||
expectedCount: 0,
|
||||
},
|
||||
{
|
||||
name: "single increment",
|
||||
increments: 1,
|
||||
expectedCount: 1,
|
||||
},
|
||||
{
|
||||
name: "multiple increments",
|
||||
increments: 5,
|
||||
expectedCount: 5,
|
||||
},
|
||||
{
|
||||
name: "many increments",
|
||||
increments: 100,
|
||||
expectedCount: 100,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
s.Run(tt.name, func() {
|
||||
// Create fresh session for each test
|
||||
id, err := s.sessionStore.CreateSDKSession(ctx, "claude-counter-"+tt.name, "project", "")
|
||||
s.NoError(err)
|
||||
|
||||
// Increment specified number of times
|
||||
var lastCount int
|
||||
for i := 0; i < tt.increments; i++ {
|
||||
lastCount, err = s.sessionStore.IncrementPromptCounter(ctx, id)
|
||||
s.NoError(err)
|
||||
}
|
||||
|
||||
// Get final count
|
||||
finalCount, err := s.sessionStore.GetPromptCounter(ctx, id)
|
||||
s.NoError(err)
|
||||
s.Equal(tt.expectedCount, finalCount)
|
||||
|
||||
if tt.increments > 0 {
|
||||
s.Equal(tt.expectedCount, lastCount)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestFindAnySDKSession tests session lookup scenarios.
|
||||
func (s *SessionStoreSuite) TestFindAnySDKSession_Scenarios() {
|
||||
ctx := context.Background()
|
||||
|
||||
// Create test sessions
|
||||
_, err := s.sessionStore.CreateSDKSession(ctx, "session-find-1", "project-a", "")
|
||||
s.NoError(err)
|
||||
_, err = s.sessionStore.CreateSDKSession(ctx, "session-find-2", "project-b", "")
|
||||
s.NoError(err)
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
claudeSessionID string
|
||||
wantProject string
|
||||
wantFound bool
|
||||
}{
|
||||
{
|
||||
name: "find existing session 1",
|
||||
claudeSessionID: "session-find-1",
|
||||
wantFound: true,
|
||||
wantProject: "project-a",
|
||||
},
|
||||
{
|
||||
name: "find existing session 2",
|
||||
claudeSessionID: "session-find-2",
|
||||
wantFound: true,
|
||||
wantProject: "project-b",
|
||||
},
|
||||
{
|
||||
name: "find non-existent session",
|
||||
claudeSessionID: "session-nonexistent",
|
||||
wantFound: false,
|
||||
},
|
||||
{
|
||||
name: "find with empty string",
|
||||
claudeSessionID: "",
|
||||
wantFound: false,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
s.Run(tt.name, func() {
|
||||
sess, err := s.sessionStore.FindAnySDKSession(ctx, tt.claudeSessionID)
|
||||
s.NoError(err) // FindAnySDKSession returns nil,nil for not found
|
||||
|
||||
if tt.wantFound {
|
||||
s.NotNil(sess)
|
||||
s.Equal(tt.wantProject, sess.Project)
|
||||
} else {
|
||||
s.Nil(sess)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestSessionStore_CreateSDKSession(t *testing.T) {
|
||||
sessionStore, _, cleanup := testSessionStore(t)
|
||||
defer cleanup()
|
||||
|
||||
@@ -94,8 +94,8 @@ func (s *Store) GetStmt(query string) (*sql.Stmt, error) {
|
||||
defer s.stmtMu.Unlock()
|
||||
|
||||
// Double-check after acquiring write lock
|
||||
if cachedStmt, ok := s.stmtCache[query]; ok {
|
||||
return cachedStmt, nil
|
||||
if stmt, ok := s.stmtCache[query]; ok {
|
||||
return stmt, nil
|
||||
}
|
||||
|
||||
stmt, err := s.db.Prepare(query)
|
||||
|
||||
@@ -1,529 +0,0 @@
|
||||
// Package sqlite provides SQLite database operations for claude-mnemonic.
|
||||
package sqlite
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"github.com/stretchr/testify/suite"
|
||||
)
|
||||
|
||||
// StoreSuite is a test suite for Store operations.
|
||||
type StoreSuite struct {
|
||||
suite.Suite
|
||||
db *sql.DB
|
||||
store *Store
|
||||
cleanup func()
|
||||
}
|
||||
|
||||
// SetupTest creates a fresh database before each test.
|
||||
func (s *StoreSuite) SetupTest() {
|
||||
s.db, _, s.cleanup = testDB(s.T())
|
||||
createBaseTables(s.T(), s.db)
|
||||
s.store = newStoreFromDB(s.db)
|
||||
}
|
||||
|
||||
// TearDownTest cleans up after each test.
|
||||
func (s *StoreSuite) TearDownTest() {
|
||||
if s.cleanup != nil {
|
||||
s.cleanup()
|
||||
}
|
||||
}
|
||||
|
||||
func TestStoreSuite(t *testing.T) {
|
||||
suite.Run(t, new(StoreSuite))
|
||||
}
|
||||
|
||||
// TestGetStmt tests prepared statement caching.
|
||||
func (s *StoreSuite) TestGetStmt() {
|
||||
tests := []struct {
|
||||
name string
|
||||
query string
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "valid simple query",
|
||||
query: "SELECT 1",
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "valid query with parameter",
|
||||
query: "SELECT * FROM sdk_sessions WHERE id = ?",
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "invalid query syntax",
|
||||
query: "SELECT * FROM nonexistent_table WHERE",
|
||||
wantErr: true,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
s.Run(tt.name, func() {
|
||||
stmt, err := s.store.GetStmt(tt.query)
|
||||
if tt.wantErr {
|
||||
s.Error(err)
|
||||
s.Nil(stmt)
|
||||
} else {
|
||||
s.NoError(err)
|
||||
s.NotNil(stmt)
|
||||
|
||||
// Second call should return cached statement
|
||||
stmt2, err := s.store.GetStmt(tt.query)
|
||||
s.NoError(err)
|
||||
s.Same(stmt, stmt2)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestExecContext tests query execution.
|
||||
func (s *StoreSuite) TestExecContext() {
|
||||
ctx := context.Background()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
query string
|
||||
args []interface{}
|
||||
wantErr bool
|
||||
wantAffected int64
|
||||
}{
|
||||
{
|
||||
name: "insert session",
|
||||
query: `INSERT INTO sdk_sessions (claude_session_id, sdk_session_id, project, started_at, started_at_epoch, status)
|
||||
VALUES (?, ?, ?, datetime('now'), strftime('%s', 'now') * 1000, 'active')`,
|
||||
args: []interface{}{"claude-1", "sdk-1", "test-project"},
|
||||
wantErr: false,
|
||||
wantAffected: 1,
|
||||
},
|
||||
{
|
||||
name: "invalid query",
|
||||
query: "INSERT INTO nonexistent_table VALUES (?)",
|
||||
args: []interface{}{"test"},
|
||||
wantErr: true,
|
||||
wantAffected: 0,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
s.Run(tt.name, func() {
|
||||
result, err := s.store.ExecContext(ctx, tt.query, tt.args...)
|
||||
if tt.wantErr {
|
||||
s.Error(err)
|
||||
} else {
|
||||
s.NoError(err)
|
||||
affected, _ := result.RowsAffected()
|
||||
s.Equal(tt.wantAffected, affected)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestQueryContext tests query execution that returns rows.
|
||||
func (s *StoreSuite) TestQueryContext() {
|
||||
ctx := context.Background()
|
||||
|
||||
// Seed data
|
||||
seedSession(s.T(), s.db, "claude-1", "sdk-1", "project-a")
|
||||
|
||||
tests := []struct {
|
||||
setupFunc func()
|
||||
assertFunc func(rows *sql.Rows)
|
||||
name string
|
||||
query string
|
||||
args []interface{}
|
||||
wantRows int
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "query existing session",
|
||||
query: "SELECT id, project FROM sdk_sessions WHERE claude_session_id = ?",
|
||||
args: []interface{}{"claude-1"},
|
||||
wantErr: false,
|
||||
wantRows: 1,
|
||||
},
|
||||
{
|
||||
name: "query non-existent session",
|
||||
query: "SELECT id, project FROM sdk_sessions WHERE claude_session_id = ?",
|
||||
args: []interface{}{"nonexistent"},
|
||||
wantErr: false,
|
||||
wantRows: 0,
|
||||
},
|
||||
{
|
||||
name: "query all sessions",
|
||||
query: "SELECT id, project FROM sdk_sessions",
|
||||
args: nil,
|
||||
wantErr: false,
|
||||
wantRows: 1,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
s.Run(tt.name, func() {
|
||||
rows, err := s.store.QueryContext(ctx, tt.query, tt.args...)
|
||||
if tt.wantErr {
|
||||
s.Error(err)
|
||||
return
|
||||
}
|
||||
|
||||
s.NoError(err)
|
||||
defer rows.Close()
|
||||
|
||||
count := 0
|
||||
for rows.Next() {
|
||||
count++
|
||||
}
|
||||
s.Equal(tt.wantRows, count)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestQueryRowContext tests single row query execution.
|
||||
func (s *StoreSuite) TestQueryRowContext() {
|
||||
ctx := context.Background()
|
||||
|
||||
// Seed data
|
||||
seedSession(s.T(), s.db, "claude-1", "sdk-1", "project-a")
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
query string
|
||||
args []interface{}
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "query existing session",
|
||||
query: "SELECT id FROM sdk_sessions WHERE claude_session_id = ?",
|
||||
args: []interface{}{"claude-1"},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "query non-existent session",
|
||||
query: "SELECT id FROM sdk_sessions WHERE claude_session_id = ?",
|
||||
args: []interface{}{"nonexistent"},
|
||||
wantErr: true, // sql.ErrNoRows
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
s.Run(tt.name, func() {
|
||||
row := s.store.QueryRowContext(ctx, tt.query, tt.args...)
|
||||
var id int64
|
||||
err := row.Scan(&id)
|
||||
if tt.wantErr {
|
||||
s.Error(err)
|
||||
} else {
|
||||
s.NoError(err)
|
||||
s.Greater(id, int64(0))
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestPing tests database connection health check.
|
||||
func (s *StoreSuite) TestPing() {
|
||||
err := s.store.Ping()
|
||||
s.NoError(err)
|
||||
}
|
||||
|
||||
// TestDB tests getting the underlying database connection.
|
||||
func (s *StoreSuite) TestDB() {
|
||||
db := s.store.DB()
|
||||
s.NotNil(db)
|
||||
s.Same(s.db, db)
|
||||
}
|
||||
|
||||
// TestClose tests closing the store.
|
||||
func (s *StoreSuite) TestClose() {
|
||||
// Create a separate store for close test
|
||||
db, _, cleanup := testDB(s.T())
|
||||
defer cleanup()
|
||||
|
||||
store := newStoreFromDB(db)
|
||||
|
||||
// Cache a statement first
|
||||
_, err := store.GetStmt("SELECT 1")
|
||||
s.NoError(err)
|
||||
|
||||
// Close should not error
|
||||
err = store.Close()
|
||||
s.NoError(err)
|
||||
|
||||
// Operations after close should fail
|
||||
err = store.Ping()
|
||||
s.Error(err)
|
||||
}
|
||||
|
||||
// TestConcurrentStmtCache tests concurrent access to statement cache.
|
||||
func (s *StoreSuite) TestConcurrentStmtCache() {
|
||||
ctx := context.Background()
|
||||
queries := []string{
|
||||
"SELECT 1",
|
||||
"SELECT 2",
|
||||
"SELECT id FROM sdk_sessions",
|
||||
"SELECT project FROM sdk_sessions",
|
||||
}
|
||||
|
||||
done := make(chan struct{})
|
||||
for i := 0; i < 10; i++ {
|
||||
go func(i int) {
|
||||
query := queries[i%len(queries)]
|
||||
_, _ = s.store.GetStmt(query)
|
||||
_, _ = s.store.ExecContext(ctx, "SELECT 1")
|
||||
done <- struct{}{}
|
||||
}(i)
|
||||
}
|
||||
|
||||
for i := 0; i < 10; i++ {
|
||||
<-done
|
||||
}
|
||||
}
|
||||
|
||||
// HelpersSuite tests helper functions.
|
||||
type HelpersSuite struct {
|
||||
suite.Suite
|
||||
}
|
||||
|
||||
func TestHelpersSuite(t *testing.T) {
|
||||
suite.Run(t, new(HelpersSuite))
|
||||
}
|
||||
|
||||
func (s *HelpersSuite) TestNullString() {
|
||||
tests := []struct {
|
||||
name string
|
||||
input string
|
||||
wantStr string
|
||||
wantBool bool
|
||||
}{
|
||||
{
|
||||
name: "empty string",
|
||||
input: "",
|
||||
wantStr: "",
|
||||
wantBool: false,
|
||||
},
|
||||
{
|
||||
name: "non-empty string",
|
||||
input: "test",
|
||||
wantStr: "test",
|
||||
wantBool: true,
|
||||
},
|
||||
{
|
||||
name: "whitespace string",
|
||||
input: " ",
|
||||
wantStr: " ",
|
||||
wantBool: true,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
s.Run(tt.name, func() {
|
||||
result := nullString(tt.input)
|
||||
s.Equal(tt.wantStr, result.String)
|
||||
s.Equal(tt.wantBool, result.Valid)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func (s *HelpersSuite) TestNullInt() {
|
||||
tests := []struct {
|
||||
name string
|
||||
input int
|
||||
wantInt int64
|
||||
wantBool bool
|
||||
}{
|
||||
{
|
||||
name: "zero",
|
||||
input: 0,
|
||||
wantInt: 0,
|
||||
wantBool: false,
|
||||
},
|
||||
{
|
||||
name: "negative",
|
||||
input: -1,
|
||||
wantInt: -1,
|
||||
wantBool: false,
|
||||
},
|
||||
{
|
||||
name: "positive",
|
||||
input: 42,
|
||||
wantInt: 42,
|
||||
wantBool: true,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
s.Run(tt.name, func() {
|
||||
result := nullInt(tt.input)
|
||||
s.Equal(tt.wantInt, result.Int64)
|
||||
s.Equal(tt.wantBool, result.Valid)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func (s *HelpersSuite) TestRepeatPlaceholders() {
|
||||
tests := []struct {
|
||||
name string
|
||||
expected string
|
||||
input int
|
||||
}{
|
||||
{
|
||||
name: "zero",
|
||||
input: 0,
|
||||
expected: "",
|
||||
},
|
||||
{
|
||||
name: "negative",
|
||||
input: -1,
|
||||
expected: "",
|
||||
},
|
||||
{
|
||||
name: "one",
|
||||
input: 1,
|
||||
expected: ", ?",
|
||||
},
|
||||
{
|
||||
name: "three",
|
||||
input: 3,
|
||||
expected: ", ?, ?, ?",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
s.Run(tt.name, func() {
|
||||
result := repeatPlaceholders(tt.input)
|
||||
s.Equal(tt.expected, result)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func (s *HelpersSuite) TestInt64SliceToInterface() {
|
||||
tests := []struct {
|
||||
name string
|
||||
input []int64
|
||||
expected int
|
||||
}{
|
||||
{
|
||||
name: "empty slice",
|
||||
input: []int64{},
|
||||
expected: 0,
|
||||
},
|
||||
{
|
||||
name: "single element",
|
||||
input: []int64{42},
|
||||
expected: 1,
|
||||
},
|
||||
{
|
||||
name: "multiple elements",
|
||||
input: []int64{1, 2, 3, 4, 5},
|
||||
expected: 5,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
s.Run(tt.name, func() {
|
||||
result := int64SliceToInterface(tt.input)
|
||||
s.Len(result, tt.expected)
|
||||
for i, v := range result {
|
||||
s.Equal(tt.input[i], v)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestBuildGetByIDsQuery tests the shared query builder.
|
||||
func TestBuildGetByIDsQuery(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
baseQuery string
|
||||
orderBy string
|
||||
wantQuery string
|
||||
ids []int64
|
||||
limit int
|
||||
wantArgs int
|
||||
}{
|
||||
{
|
||||
name: "single id, no limit, desc order",
|
||||
baseQuery: "SELECT * FROM test",
|
||||
ids: []int64{1},
|
||||
orderBy: "date_desc",
|
||||
limit: 0,
|
||||
wantQuery: "SELECT * FROM test WHERE id IN (?)\n\t\tORDER BY created_at_epoch DESC",
|
||||
wantArgs: 1,
|
||||
},
|
||||
{
|
||||
name: "multiple ids with limit and asc order",
|
||||
baseQuery: "SELECT * FROM test",
|
||||
ids: []int64{1, 2, 3},
|
||||
orderBy: "date_asc",
|
||||
limit: 10,
|
||||
wantQuery: "SELECT * FROM test WHERE id IN (?, ?, ?)\n\t\tORDER BY created_at_epoch ASC LIMIT ?",
|
||||
wantArgs: 4,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
query, args := BuildGetByIDsQuery(tt.baseQuery, tt.ids, tt.orderBy, tt.limit)
|
||||
assert.Contains(t, query, "WHERE id IN")
|
||||
assert.Len(t, args, tt.wantArgs)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestEnsureSessionExists tests session auto-creation.
|
||||
func TestEnsureSessionExists(t *testing.T) {
|
||||
db, _, cleanup := testDB(t)
|
||||
defer cleanup()
|
||||
createBaseTables(t, db)
|
||||
|
||||
store := newStoreFromDB(db)
|
||||
ctx := context.Background()
|
||||
|
||||
tests := []struct {
|
||||
setup func()
|
||||
name string
|
||||
sdkSessionID string
|
||||
project string
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "create new session",
|
||||
sdkSessionID: "sdk-new",
|
||||
project: "project-a",
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "session already exists",
|
||||
sdkSessionID: "sdk-existing",
|
||||
project: "project-b",
|
||||
setup: func() {
|
||||
seedSession(t, db, "sdk-existing", "sdk-existing", "project-b")
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
if tt.setup != nil {
|
||||
tt.setup()
|
||||
}
|
||||
|
||||
err := EnsureSessionExists(ctx, store, tt.sdkSessionID, tt.project)
|
||||
if tt.wantErr {
|
||||
require.Error(t, err)
|
||||
} else {
|
||||
require.NoError(t, err)
|
||||
|
||||
// Verify session exists
|
||||
var id int64
|
||||
err := db.QueryRow("SELECT id FROM sdk_sessions WHERE sdk_session_id = ?", tt.sdkSessionID).Scan(&id)
|
||||
require.NoError(t, err)
|
||||
assert.Greater(t, id, int64(0))
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -116,21 +116,3 @@ func (s *SummaryStore) GetAllRecentSummaries(ctx context.Context, limit int) ([]
|
||||
|
||||
return scanSummaryRows(rows)
|
||||
}
|
||||
|
||||
// GetAllSummaries retrieves all summaries (for vector rebuild).
|
||||
func (s *SummaryStore) GetAllSummaries(ctx context.Context) ([]*models.SessionSummary, error) {
|
||||
const query = `
|
||||
SELECT id, sdk_session_id, project, request, investigated, learned, completed,
|
||||
next_steps, notes, prompt_number, discovery_tokens, created_at, created_at_epoch
|
||||
FROM session_summaries
|
||||
ORDER BY id
|
||||
`
|
||||
|
||||
rows, err := s.store.QueryContext(ctx, query)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
return scanSummaryRows(rows)
|
||||
}
|
||||
|
||||
@@ -103,12 +103,6 @@ func createBaseTables(t *testing.T, db *sql.DB) {
|
||||
discovery_tokens INTEGER DEFAULT 0,
|
||||
created_at TEXT NOT NULL,
|
||||
created_at_epoch INTEGER NOT NULL,
|
||||
importance_score REAL DEFAULT 1.0,
|
||||
user_feedback INTEGER DEFAULT 0,
|
||||
retrieval_count INTEGER DEFAULT 0,
|
||||
last_retrieved_at_epoch INTEGER,
|
||||
score_updated_at_epoch INTEGER,
|
||||
is_superseded INTEGER DEFAULT 0,
|
||||
FOREIGN KEY(sdk_session_id) REFERENCES sdk_sessions(sdk_session_id) ON DELETE CASCADE
|
||||
)
|
||||
`)
|
||||
@@ -116,27 +110,6 @@ func createBaseTables(t *testing.T, db *sql.DB) {
|
||||
t.Fatalf("create observations: %v", err)
|
||||
}
|
||||
|
||||
// Create observation_conflicts table for conflict detection
|
||||
_, err = db.Exec(`
|
||||
CREATE TABLE IF NOT EXISTS observation_conflicts (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
newer_obs_id INTEGER NOT NULL,
|
||||
older_obs_id INTEGER NOT NULL,
|
||||
conflict_type TEXT NOT NULL CHECK(conflict_type IN ('superseded', 'contradicts', 'outdated_pattern')),
|
||||
resolution TEXT NOT NULL CHECK(resolution IN ('prefer_newer', 'prefer_older', 'manual')),
|
||||
reason TEXT,
|
||||
detected_at TEXT NOT NULL,
|
||||
detected_at_epoch INTEGER NOT NULL,
|
||||
resolved INTEGER DEFAULT 0,
|
||||
resolved_at TEXT,
|
||||
FOREIGN KEY(newer_obs_id) REFERENCES observations(id) ON DELETE CASCADE,
|
||||
FOREIGN KEY(older_obs_id) REFERENCES observations(id) ON DELETE CASCADE
|
||||
)
|
||||
`)
|
||||
if err != nil {
|
||||
t.Fatalf("create observation_conflicts: %v", err)
|
||||
}
|
||||
|
||||
_, err = db.Exec(`
|
||||
CREATE TABLE IF NOT EXISTS session_summaries (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
@@ -177,31 +150,6 @@ func createBaseTables(t *testing.T, db *sql.DB) {
|
||||
t.Fatalf("create user_prompts: %v", err)
|
||||
}
|
||||
|
||||
_, err = db.Exec(`
|
||||
CREATE TABLE IF NOT EXISTS patterns (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
name TEXT NOT NULL,
|
||||
type TEXT NOT NULL CHECK(type IN ('bug', 'refactor', 'architecture', 'anti-pattern', 'best-practice')),
|
||||
description TEXT,
|
||||
signature TEXT,
|
||||
recommendation TEXT,
|
||||
frequency INTEGER DEFAULT 1,
|
||||
projects TEXT,
|
||||
observation_ids TEXT,
|
||||
status TEXT DEFAULT 'active' CHECK(status IN ('active', 'deprecated', 'merged')),
|
||||
merged_into_id INTEGER,
|
||||
confidence REAL DEFAULT 0.5,
|
||||
last_seen_at TEXT NOT NULL,
|
||||
last_seen_at_epoch INTEGER NOT NULL,
|
||||
created_at TEXT NOT NULL,
|
||||
created_at_epoch INTEGER NOT NULL,
|
||||
FOREIGN KEY(merged_into_id) REFERENCES patterns(id) ON DELETE SET NULL
|
||||
)
|
||||
`)
|
||||
if err != nil {
|
||||
t.Fatalf("create patterns: %v", err)
|
||||
}
|
||||
|
||||
indexes := []string{
|
||||
`CREATE INDEX IF NOT EXISTS idx_sdk_sessions_claude_id ON sdk_sessions(claude_session_id)`,
|
||||
`CREATE INDEX IF NOT EXISTS idx_sdk_sessions_sdk_id ON sdk_sessions(sdk_session_id)`,
|
||||
|
||||
@@ -1 +0,0 @@
|
||||
bge-small-en-v1.5
|
||||
@@ -1 +0,0 @@
|
||||
1.23.2
|
||||
@@ -1 +0,0 @@
|
||||
1.23.2
|
||||
@@ -1 +0,0 @@
|
||||
1.23.2
|
||||
@@ -1 +0,0 @@
|
||||
1.23.2
|
||||
Binary file not shown.
@@ -1,7 +1,21 @@
|
||||
{
|
||||
"version": "1.0",
|
||||
"truncation": null,
|
||||
"padding": null,
|
||||
"truncation": {
|
||||
"direction": "Right",
|
||||
"max_length": 128,
|
||||
"strategy": "LongestFirst",
|
||||
"stride": 0
|
||||
},
|
||||
"padding": {
|
||||
"strategy": {
|
||||
"Fixed": 128
|
||||
},
|
||||
"direction": "Right",
|
||||
"pad_to_multiple_of": null,
|
||||
"pad_id": 0,
|
||||
"pad_type_id": 0,
|
||||
"pad_token": "[PAD]"
|
||||
},
|
||||
"added_tokens": [
|
||||
{
|
||||
"id": 0,
|
||||
|
||||
@@ -1,152 +0,0 @@
|
||||
// Package embedding provides text embedding generation with swappable models.
|
||||
package embedding
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sync"
|
||||
)
|
||||
|
||||
// PoolingStrategy defines how to pool token embeddings into sentence embeddings.
|
||||
type PoolingStrategy string
|
||||
|
||||
const (
|
||||
// PoolingNone means the model already outputs sentence embeddings directly.
|
||||
PoolingNone PoolingStrategy = "none"
|
||||
// PoolingMean averages all token embeddings (weighted by attention mask).
|
||||
PoolingMean PoolingStrategy = "mean"
|
||||
// PoolingCLS uses only the [CLS] token embedding.
|
||||
PoolingCLS PoolingStrategy = "cls"
|
||||
)
|
||||
|
||||
// ONNXConfig describes ONNX-specific model configuration.
|
||||
// This allows different models to specify their tensor names and pooling needs.
|
||||
type ONNXConfig struct {
|
||||
Pooling PoolingStrategy
|
||||
InputNames []string
|
||||
OutputNames []string
|
||||
HiddenSize int
|
||||
}
|
||||
|
||||
// EmbeddingModel represents a text embedding model.
|
||||
type EmbeddingModel interface {
|
||||
// Name returns the human-readable model name (e.g., "bge-small-en-v1.5").
|
||||
Name() string
|
||||
|
||||
// Version returns a short version string for storage (e.g., "bge-v1.5").
|
||||
Version() string
|
||||
|
||||
// Dimensions returns the embedding vector size.
|
||||
Dimensions() int
|
||||
|
||||
// Embed generates an embedding for a single text.
|
||||
Embed(text string) ([]float32, error)
|
||||
|
||||
// EmbedBatch generates embeddings for multiple texts.
|
||||
EmbedBatch(texts []string) ([][]float32, error)
|
||||
|
||||
// Close releases model resources.
|
||||
Close() error
|
||||
}
|
||||
|
||||
// ONNXConfigurer is an optional interface that models can implement
|
||||
// to expose their ONNX configuration for introspection.
|
||||
type ONNXConfigurer interface {
|
||||
// ONNXConfig returns the model's ONNX configuration.
|
||||
ONNXConfig() ONNXConfig
|
||||
}
|
||||
|
||||
// ModelMetadata describes an embedding model for UI/config.
|
||||
type ModelMetadata struct {
|
||||
Name string `json:"name"`
|
||||
Version string `json:"version"`
|
||||
Description string `json:"description"`
|
||||
Dimensions int `json:"dimensions"`
|
||||
Default bool `json:"default"`
|
||||
}
|
||||
|
||||
// ModelFactory creates a new instance of an embedding model.
|
||||
type ModelFactory func() (EmbeddingModel, error)
|
||||
|
||||
// ModelRegistry provides model lookup by version.
|
||||
type ModelRegistry struct {
|
||||
models map[string]ModelFactory
|
||||
metadata map[string]ModelMetadata
|
||||
defaultModel string
|
||||
mu sync.RWMutex
|
||||
}
|
||||
|
||||
// NewModelRegistry creates a new model registry.
|
||||
func NewModelRegistry() *ModelRegistry {
|
||||
return &ModelRegistry{
|
||||
models: make(map[string]ModelFactory),
|
||||
metadata: make(map[string]ModelMetadata),
|
||||
}
|
||||
}
|
||||
|
||||
// Register adds a model factory to the registry.
|
||||
func (r *ModelRegistry) Register(meta ModelMetadata, factory ModelFactory) {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
|
||||
r.models[meta.Version] = factory
|
||||
r.metadata[meta.Version] = meta
|
||||
|
||||
if meta.Default {
|
||||
r.defaultModel = meta.Version
|
||||
}
|
||||
}
|
||||
|
||||
// Get creates a new instance of the model with the given version.
|
||||
func (r *ModelRegistry) Get(version string) (EmbeddingModel, error) {
|
||||
r.mu.RLock()
|
||||
factory, ok := r.models[version]
|
||||
r.mu.RUnlock()
|
||||
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("unknown model version: %s", version)
|
||||
}
|
||||
|
||||
return factory()
|
||||
}
|
||||
|
||||
// Default returns the default model version.
|
||||
func (r *ModelRegistry) Default() string {
|
||||
r.mu.RLock()
|
||||
defer r.mu.RUnlock()
|
||||
return r.defaultModel
|
||||
}
|
||||
|
||||
// List returns metadata for all registered models.
|
||||
func (r *ModelRegistry) List() []ModelMetadata {
|
||||
r.mu.RLock()
|
||||
defer r.mu.RUnlock()
|
||||
|
||||
result := make([]ModelMetadata, 0, len(r.metadata))
|
||||
for _, meta := range r.metadata {
|
||||
result = append(result, meta)
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
// DefaultRegistry is the global model registry with all available models.
|
||||
var DefaultRegistry = NewModelRegistry()
|
||||
|
||||
// RegisterModel adds a model to the default registry.
|
||||
func RegisterModel(meta ModelMetadata, factory ModelFactory) {
|
||||
DefaultRegistry.Register(meta, factory)
|
||||
}
|
||||
|
||||
// GetModel creates a model instance from the default registry.
|
||||
func GetModel(version string) (EmbeddingModel, error) {
|
||||
return DefaultRegistry.Get(version)
|
||||
}
|
||||
|
||||
// GetDefaultModel returns the default model version from the default registry.
|
||||
func GetDefaultModel() string {
|
||||
return DefaultRegistry.Default()
|
||||
}
|
||||
|
||||
// ListModels returns metadata for all models in the default registry.
|
||||
func ListModels() []ModelMetadata {
|
||||
return DefaultRegistry.List()
|
||||
}
|
||||
+60
-281
@@ -1,4 +1,4 @@
|
||||
// Package embedding provides text embedding generation with swappable models.
|
||||
// Package embedding provides text embedding generation using all-MiniLM-L6-v2.
|
||||
package embedding
|
||||
|
||||
import (
|
||||
@@ -15,50 +15,19 @@ import (
|
||||
ort "github.com/yalue/onnxruntime_go"
|
||||
)
|
||||
|
||||
// EmbeddingDim is the dimension of embeddings produced by the current model.
|
||||
// Both all-MiniLM-L6-v2 and bge-small-en-v1.5 produce 384-dimensional embeddings.
|
||||
// EmbeddingDim is the dimension of embeddings produced by all-MiniLM-L6-v2.
|
||||
const EmbeddingDim = 384
|
||||
|
||||
// Model version constants
|
||||
const (
|
||||
// BGEModelVersion is the version string for bge-small-en-v1.5
|
||||
BGEModelVersion = "bge-v1.5"
|
||||
// BGEModelName is the human-readable name for bge-small-en-v1.5
|
||||
BGEModelName = "bge-small-en-v1.5"
|
||||
// DefaultModelVersion is the default model to use
|
||||
DefaultModelVersion = BGEModelVersion
|
||||
)
|
||||
|
||||
// MaxSequenceLength is the maximum token sequence length for the model.
|
||||
const MaxSequenceLength = 512
|
||||
|
||||
// bgeONNXConfig defines the ONNX configuration for BGE models.
|
||||
// BGE outputs last_hidden_state and requires mean pooling.
|
||||
var bgeONNXConfig = ONNXConfig{
|
||||
InputNames: []string{"input_ids", "attention_mask", "token_type_ids"},
|
||||
OutputNames: []string{"last_hidden_state"},
|
||||
Pooling: PoolingMean,
|
||||
HiddenSize: EmbeddingDim,
|
||||
}
|
||||
|
||||
// bgeModel is the ONNX-based embedding model implementation.
|
||||
// Currently supports bge-small-en-v1.5 (previously all-MiniLM-L6-v2).
|
||||
type bgeModel struct {
|
||||
// Service provides thread-safe text embedding generation.
|
||||
type Service struct {
|
||||
tk *tokenizer.Tokenizer
|
||||
session *ort.DynamicAdvancedSession
|
||||
libDir string
|
||||
config ONNXConfig
|
||||
mu sync.Mutex
|
||||
libDir string // temp directory containing extracted libraries
|
||||
}
|
||||
|
||||
// Compile-time check that bgeModel implements EmbeddingModel
|
||||
var _ EmbeddingModel = (*bgeModel)(nil)
|
||||
|
||||
// Compile-time check that bgeModel implements ONNXConfigurer
|
||||
var _ ONNXConfigurer = (*bgeModel)(nil)
|
||||
|
||||
// newBGEModel creates a new BGE embedding model using bundled ONNX runtime and model.
|
||||
func newBGEModel() (EmbeddingModel, error) {
|
||||
// NewService creates a new embedding service using bundled ONNX runtime and model.
|
||||
func NewService() (*Service, error) {
|
||||
// Extract ONNX runtime library to temp directory
|
||||
libDir, err := extractONNXLibrary()
|
||||
if err != nil {
|
||||
@@ -70,8 +39,8 @@ func newBGEModel() (EmbeddingModel, error) {
|
||||
ort.SetSharedLibraryPath(libPath)
|
||||
|
||||
// Initialize ONNX runtime
|
||||
if initErr := ort.InitializeEnvironment(); initErr != nil {
|
||||
return nil, fmt.Errorf("initialize ONNX runtime: %w", initErr)
|
||||
if err := ort.InitializeEnvironment(); err != nil {
|
||||
return nil, fmt.Errorf("initialize ONNX runtime: %w", err)
|
||||
}
|
||||
|
||||
// Load tokenizer from embedded data
|
||||
@@ -80,41 +49,22 @@ func newBGEModel() (EmbeddingModel, error) {
|
||||
return nil, fmt.Errorf("load tokenizer: %w", err)
|
||||
}
|
||||
|
||||
// Create ONNX session using model-specific configuration
|
||||
config := bgeONNXConfig
|
||||
session, err := ort.NewDynamicAdvancedSessionWithONNXData(modelData, config.InputNames, config.OutputNames, nil)
|
||||
// Create ONNX session with embedded model
|
||||
inputNames := []string{"input_ids", "attention_mask", "token_type_ids"}
|
||||
outputNames := []string{"sentence_embedding"}
|
||||
|
||||
session, err := ort.NewDynamicAdvancedSessionWithONNXData(modelData, inputNames, outputNames, nil)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create ONNX session: %w", err)
|
||||
}
|
||||
|
||||
return &bgeModel{
|
||||
return &Service{
|
||||
tk: tk,
|
||||
session: session,
|
||||
libDir: libDir,
|
||||
config: config,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// ONNXConfig returns the model's ONNX configuration.
|
||||
func (m *bgeModel) ONNXConfig() ONNXConfig {
|
||||
return m.config
|
||||
}
|
||||
|
||||
// Name returns the human-readable model name.
|
||||
func (m *bgeModel) Name() string {
|
||||
return BGEModelName
|
||||
}
|
||||
|
||||
// Version returns the short version string for storage.
|
||||
func (m *bgeModel) Version() string {
|
||||
return BGEModelVersion
|
||||
}
|
||||
|
||||
// Dimensions returns the embedding vector size.
|
||||
func (m *bgeModel) Dimensions() int {
|
||||
return EmbeddingDim
|
||||
}
|
||||
|
||||
// extractONNXLibrary extracts the embedded ONNX runtime library to a temp directory.
|
||||
// Uses content hash to avoid re-extracting if already present.
|
||||
func extractONNXLibrary() (string, error) {
|
||||
@@ -157,15 +107,15 @@ func extractONNXLibrary() (string, error) {
|
||||
|
||||
// Embed generates an embedding for a single text.
|
||||
// Returns a 384-dimensional float32 vector.
|
||||
func (m *bgeModel) Embed(text string) ([]float32, error) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
func (s *Service) Embed(text string) ([]float32, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
|
||||
if text == "" {
|
||||
return make([]float32, EmbeddingDim), nil
|
||||
}
|
||||
|
||||
results, err := m.computeBatch([]string{text})
|
||||
results, err := s.computeBatch([]string{text})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -177,13 +127,13 @@ func (m *bgeModel) Embed(text string) ([]float32, error) {
|
||||
|
||||
// EmbedBatch generates embeddings for multiple texts.
|
||||
// Returns slice of 384-dimensional float32 vectors.
|
||||
func (m *bgeModel) EmbedBatch(texts []string) ([][]float32, error) {
|
||||
func (s *Service) EmbedBatch(texts []string) ([][]float32, error) {
|
||||
if len(texts) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
|
||||
// Filter out empty texts and track indices
|
||||
nonEmpty := make([]string, 0, len(texts))
|
||||
@@ -205,7 +155,7 @@ func (m *bgeModel) EmbedBatch(texts []string) ([][]float32, error) {
|
||||
}
|
||||
|
||||
// Compute embeddings for non-empty texts
|
||||
embeddings, err := m.computeBatch(nonEmpty)
|
||||
embeddings, err := s.computeBatch(nonEmpty)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("compute batch embeddings: %w", err)
|
||||
}
|
||||
@@ -223,7 +173,7 @@ func (m *bgeModel) EmbedBatch(texts []string) ([][]float32, error) {
|
||||
}
|
||||
|
||||
// computeBatch runs inference on a batch of texts. Must be called with lock held.
|
||||
func (m *bgeModel) computeBatch(sentences []string) ([][]float32, error) {
|
||||
func (s *Service) computeBatch(sentences []string) ([][]float32, error) {
|
||||
if len(sentences) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
@@ -234,57 +184,31 @@ func (m *bgeModel) computeBatch(sentences []string) ([][]float32, error) {
|
||||
inputBatch[i] = tokenizer.NewSingleEncodeInput(tokenizer.NewRawInputSequence(sent))
|
||||
}
|
||||
|
||||
encodings, err := m.tk.EncodeBatch(inputBatch, true)
|
||||
encodings, err := s.tk.EncodeBatch(inputBatch, true)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("tokenize: %w", err)
|
||||
}
|
||||
|
||||
batchSize := len(encodings)
|
||||
hiddenSize := m.config.HiddenSize
|
||||
|
||||
// Find max sequence length across all encodings (tokenizer may not pad uniformly)
|
||||
// Also enforce MaxSequenceLength to prevent model errors
|
||||
seqLength := 0
|
||||
for _, enc := range encodings {
|
||||
if len(enc.Ids) > seqLength {
|
||||
seqLength = len(enc.Ids)
|
||||
}
|
||||
}
|
||||
// Truncate to max model sequence length
|
||||
if seqLength > MaxSequenceLength {
|
||||
seqLength = MaxSequenceLength
|
||||
}
|
||||
seqLength := len(encodings[0].Ids)
|
||||
hiddenSize := EmbeddingDim
|
||||
|
||||
inputShape := ort.NewShape(int64(batchSize), int64(seqLength))
|
||||
|
||||
// Create input tensors (pre-filled with zeros for padding)
|
||||
// Create input tensors
|
||||
inputIdsData := make([]int64, batchSize*seqLength)
|
||||
attentionMaskData := make([]int64, batchSize*seqLength)
|
||||
tokenTypeIdsData := make([]int64, batchSize*seqLength)
|
||||
|
||||
for b := 0; b < batchSize; b++ {
|
||||
// Copy actual token data (rest remains 0 as padding)
|
||||
// Truncate to seqLength to handle long inputs
|
||||
copyLen := len(encodings[b].Ids)
|
||||
if copyLen > seqLength {
|
||||
copyLen = seqLength
|
||||
for i, id := range encodings[b].Ids {
|
||||
inputIdsData[b*seqLength+i] = int64(id)
|
||||
}
|
||||
for i := 0; i < copyLen; i++ {
|
||||
inputIdsData[b*seqLength+i] = int64(encodings[b].Ids[i])
|
||||
for i, mask := range encodings[b].AttentionMask {
|
||||
attentionMaskData[b*seqLength+i] = int64(mask)
|
||||
}
|
||||
copyLen = len(encodings[b].AttentionMask)
|
||||
if copyLen > seqLength {
|
||||
copyLen = seqLength
|
||||
}
|
||||
for i := 0; i < copyLen; i++ {
|
||||
attentionMaskData[b*seqLength+i] = int64(encodings[b].AttentionMask[i])
|
||||
}
|
||||
copyLen = len(encodings[b].TypeIds)
|
||||
if copyLen > seqLength {
|
||||
copyLen = seqLength
|
||||
}
|
||||
for i := 0; i < copyLen; i++ {
|
||||
tokenTypeIdsData[b*seqLength+i] = int64(encodings[b].TypeIds[i])
|
||||
for i, typeId := range encodings[b].TypeIds {
|
||||
tokenTypeIdsData[b*seqLength+i] = int64(typeId)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -292,145 +216,65 @@ func (m *bgeModel) computeBatch(sentences []string) ([][]float32, error) {
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create input_ids tensor: %w", err)
|
||||
}
|
||||
defer func() { _ = inputIdsTensor.Destroy() }()
|
||||
defer inputIdsTensor.Destroy()
|
||||
|
||||
attentionMaskTensor, err := ort.NewTensor(inputShape, attentionMaskData)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create attention_mask tensor: %w", err)
|
||||
}
|
||||
defer func() { _ = attentionMaskTensor.Destroy() }()
|
||||
defer attentionMaskTensor.Destroy()
|
||||
|
||||
tokenTypeIdsTensor, err := ort.NewTensor(inputShape, tokenTypeIdsData)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create token_type_ids tensor: %w", err)
|
||||
}
|
||||
defer func() { _ = tokenTypeIdsTensor.Destroy() }()
|
||||
defer tokenTypeIdsTensor.Destroy()
|
||||
|
||||
// Create output tensor based on pooling strategy
|
||||
var outputShape ort.Shape
|
||||
|
||||
switch m.config.Pooling {
|
||||
case PoolingNone:
|
||||
// Direct sentence embedding output: [batch, hidden]
|
||||
outputShape = ort.NewShape(int64(batchSize), int64(hiddenSize))
|
||||
case PoolingMean, PoolingCLS:
|
||||
// Token-level output: [batch, seq_len, hidden]
|
||||
outputShape = ort.NewShape(int64(batchSize), int64(seqLength), int64(hiddenSize))
|
||||
default:
|
||||
outputShape = ort.NewShape(int64(batchSize), int64(hiddenSize))
|
||||
}
|
||||
|
||||
outputTensor, err := ort.NewEmptyTensor[float32](outputShape)
|
||||
sentenceOutputShape := ort.NewShape(int64(batchSize), int64(hiddenSize))
|
||||
sentenceOutputTensor, err := ort.NewEmptyTensor[float32](sentenceOutputShape)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create output tensor: %w", err)
|
||||
}
|
||||
defer func() { _ = outputTensor.Destroy() }()
|
||||
defer sentenceOutputTensor.Destroy()
|
||||
|
||||
// Run inference
|
||||
inputTensors := []ort.Value{inputIdsTensor, attentionMaskTensor, tokenTypeIdsTensor}
|
||||
outputTensors := []ort.Value{outputTensor}
|
||||
outputTensors := []ort.Value{sentenceOutputTensor}
|
||||
|
||||
if err := m.session.Run(inputTensors, outputTensors); err != nil {
|
||||
if err := s.session.Run(inputTensors, outputTensors); err != nil {
|
||||
return nil, fmt.Errorf("run inference: %w", err)
|
||||
}
|
||||
|
||||
// Extract and pool results based on strategy
|
||||
flatOutput := outputTensor.GetData()
|
||||
|
||||
switch m.config.Pooling {
|
||||
case PoolingNone:
|
||||
// Direct output, no pooling needed
|
||||
expectedSize := batchSize * hiddenSize
|
||||
if len(flatOutput) != expectedSize {
|
||||
return nil, fmt.Errorf("unexpected output size: got %d, expected %d", len(flatOutput), expectedSize)
|
||||
}
|
||||
results := make([][]float32, batchSize)
|
||||
for i := 0; i < batchSize; i++ {
|
||||
start := i * hiddenSize
|
||||
end := start + hiddenSize
|
||||
results[i] = make([]float32, hiddenSize)
|
||||
copy(results[i], flatOutput[start:end])
|
||||
}
|
||||
return results, nil
|
||||
|
||||
case PoolingMean:
|
||||
// Mean pooling over tokens (weighted by attention mask)
|
||||
return meanPooling(flatOutput, attentionMaskData, batchSize, seqLength, hiddenSize), nil
|
||||
|
||||
case PoolingCLS:
|
||||
// CLS token pooling (first token of each sequence)
|
||||
return clsPooling(flatOutput, batchSize, seqLength, hiddenSize), nil
|
||||
|
||||
default:
|
||||
return nil, fmt.Errorf("unknown pooling strategy: %s", m.config.Pooling)
|
||||
// Extract results
|
||||
flatOutput := sentenceOutputTensor.GetData()
|
||||
expectedSize := batchSize * hiddenSize
|
||||
if len(flatOutput) != expectedSize {
|
||||
return nil, fmt.Errorf("unexpected output size: got %d, expected %d", len(flatOutput), expectedSize)
|
||||
}
|
||||
}
|
||||
|
||||
// meanPooling applies mean pooling over token embeddings, weighted by attention mask.
|
||||
// Input shape: [batch, seq_len, hidden], attention mask: [batch, seq_len]
|
||||
// Output shape: [batch, hidden]
|
||||
func meanPooling(embeddings []float32, attentionMask []int64, batchSize, seqLen, hiddenSize int) [][]float32 {
|
||||
results := make([][]float32, batchSize)
|
||||
|
||||
for b := 0; b < batchSize; b++ {
|
||||
result := make([]float32, hiddenSize)
|
||||
var maskSum float32
|
||||
|
||||
// Sum embeddings weighted by attention mask
|
||||
for s := 0; s < seqLen; s++ {
|
||||
maskVal := float32(attentionMask[b*seqLen+s])
|
||||
maskSum += maskVal
|
||||
|
||||
if maskVal > 0 {
|
||||
embOffset := (b*seqLen + s) * hiddenSize
|
||||
for h := 0; h < hiddenSize; h++ {
|
||||
result[h] += embeddings[embOffset+h] * maskVal
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Normalize by mask sum (avoid division by zero)
|
||||
if maskSum > 0 {
|
||||
for h := 0; h < hiddenSize; h++ {
|
||||
result[h] /= maskSum
|
||||
}
|
||||
}
|
||||
|
||||
results[b] = result
|
||||
for i := 0; i < batchSize; i++ {
|
||||
start := i * hiddenSize
|
||||
end := start + hiddenSize
|
||||
results[i] = make([]float32, hiddenSize)
|
||||
copy(results[i], flatOutput[start:end])
|
||||
}
|
||||
|
||||
return results
|
||||
}
|
||||
|
||||
// clsPooling extracts the [CLS] token embedding (first token).
|
||||
// Input shape: [batch, seq_len, hidden]
|
||||
// Output shape: [batch, hidden]
|
||||
func clsPooling(embeddings []float32, batchSize, seqLen, hiddenSize int) [][]float32 {
|
||||
results := make([][]float32, batchSize)
|
||||
|
||||
for b := 0; b < batchSize; b++ {
|
||||
result := make([]float32, hiddenSize)
|
||||
// CLS token is at position 0
|
||||
embOffset := b * seqLen * hiddenSize
|
||||
copy(result, embeddings[embOffset:embOffset+hiddenSize])
|
||||
results[b] = result
|
||||
}
|
||||
|
||||
return results
|
||||
return results, nil
|
||||
}
|
||||
|
||||
// Close releases model resources.
|
||||
func (m *bgeModel) Close() error {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
func (s *Service) Close() error {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
|
||||
var errs []error
|
||||
|
||||
if m.session != nil {
|
||||
if err := m.session.Destroy(); err != nil {
|
||||
if s.session != nil {
|
||||
if err := s.session.Destroy(); err != nil {
|
||||
errs = append(errs, fmt.Errorf("destroy session: %w", err))
|
||||
}
|
||||
m.session = nil
|
||||
s.session = nil
|
||||
}
|
||||
|
||||
if err := ort.DestroyEnvironment(); err != nil {
|
||||
@@ -438,75 +282,10 @@ func (m *bgeModel) Close() error {
|
||||
}
|
||||
|
||||
// Optionally clean up extracted library (leave for caching)
|
||||
// os.RemoveAll(m.libDir)
|
||||
// os.RemoveAll(s.libDir)
|
||||
|
||||
if len(errs) > 0 {
|
||||
return errs[0]
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Register the BGE model with the default registry at init time
|
||||
func init() {
|
||||
RegisterModel(ModelMetadata{
|
||||
Name: BGEModelName,
|
||||
Version: BGEModelVersion,
|
||||
Dimensions: EmbeddingDim,
|
||||
Description: "High-quality semantic search model",
|
||||
Default: true,
|
||||
}, newBGEModel)
|
||||
}
|
||||
|
||||
// Service provides thread-safe text embedding generation with model abstraction.
|
||||
type Service struct {
|
||||
model EmbeddingModel
|
||||
}
|
||||
|
||||
// NewService creates a new embedding service using the default model.
|
||||
func NewService() (*Service, error) {
|
||||
return NewServiceWithModel(DefaultModelVersion)
|
||||
}
|
||||
|
||||
// NewServiceWithModel creates a new embedding service using the specified model.
|
||||
func NewServiceWithModel(version string) (*Service, error) {
|
||||
if version == "" {
|
||||
version = DefaultModelVersion
|
||||
}
|
||||
|
||||
model, err := GetModel(version)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("get model %s: %w", version, err)
|
||||
}
|
||||
|
||||
return &Service{model: model}, nil
|
||||
}
|
||||
|
||||
// Name returns the human-readable model name.
|
||||
func (s *Service) Name() string {
|
||||
return s.model.Name()
|
||||
}
|
||||
|
||||
// Version returns the short version string for storage.
|
||||
func (s *Service) Version() string {
|
||||
return s.model.Version()
|
||||
}
|
||||
|
||||
// Dimensions returns the embedding vector size.
|
||||
func (s *Service) Dimensions() int {
|
||||
return s.model.Dimensions()
|
||||
}
|
||||
|
||||
// Embed generates an embedding for a single text.
|
||||
func (s *Service) Embed(text string) ([]float32, error) {
|
||||
return s.model.Embed(text)
|
||||
}
|
||||
|
||||
// EmbedBatch generates embeddings for multiple texts.
|
||||
func (s *Service) EmbedBatch(texts []string) ([][]float32, error) {
|
||||
return s.model.EmbedBatch(texts)
|
||||
}
|
||||
|
||||
// Close releases model resources.
|
||||
func (s *Service) Close() error {
|
||||
return s.model.Close()
|
||||
}
|
||||
|
||||
@@ -1,334 +0,0 @@
|
||||
package embedding
|
||||
|
||||
import (
|
||||
"math"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// TestEmbeddingDim verifies the embedding dimension constant.
|
||||
func TestEmbeddingDim(t *testing.T) {
|
||||
assert.Equal(t, 384, EmbeddingDim)
|
||||
}
|
||||
|
||||
// TestNewService tests creating a new embedding service.
|
||||
func TestNewService(t *testing.T) {
|
||||
svc, err := NewService()
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, svc)
|
||||
|
||||
defer svc.Close()
|
||||
|
||||
// Verify the service is properly initialized via public methods
|
||||
assert.NotEmpty(t, svc.Name())
|
||||
assert.NotEmpty(t, svc.Version())
|
||||
assert.Equal(t, EmbeddingDim, svc.Dimensions())
|
||||
}
|
||||
|
||||
// TestEmbed_SingleText tests embedding a single text.
|
||||
func TestEmbed_SingleText(t *testing.T) {
|
||||
svc, err := NewService()
|
||||
require.NoError(t, err)
|
||||
defer svc.Close()
|
||||
|
||||
embedding, err := svc.Embed("Hello, world!")
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.Len(t, embedding, EmbeddingDim)
|
||||
|
||||
// Verify non-zero embedding
|
||||
var sum float32
|
||||
for _, v := range embedding {
|
||||
sum += v * v
|
||||
}
|
||||
assert.Greater(t, sum, float32(0), "Embedding should not be all zeros")
|
||||
}
|
||||
|
||||
// TestEmbed_EmptyText tests embedding an empty string.
|
||||
func TestEmbed_EmptyText(t *testing.T) {
|
||||
svc, err := NewService()
|
||||
require.NoError(t, err)
|
||||
defer svc.Close()
|
||||
|
||||
embedding, err := svc.Embed("")
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.Len(t, embedding, EmbeddingDim)
|
||||
|
||||
// Empty text should return zero vector
|
||||
for _, v := range embedding {
|
||||
assert.Equal(t, float32(0), v)
|
||||
}
|
||||
}
|
||||
|
||||
// TestEmbed_SimilarTexts tests that similar texts produce similar embeddings.
|
||||
func TestEmbed_SimilarTexts(t *testing.T) {
|
||||
svc, err := NewService()
|
||||
require.NoError(t, err)
|
||||
defer svc.Close()
|
||||
|
||||
emb1, err := svc.Embed("The quick brown fox jumps over the lazy dog.")
|
||||
require.NoError(t, err)
|
||||
|
||||
emb2, err := svc.Embed("A fast brown fox leaps over a sleepy dog.")
|
||||
require.NoError(t, err)
|
||||
|
||||
emb3, err := svc.Embed("Go programming language concurrency patterns.")
|
||||
require.NoError(t, err)
|
||||
|
||||
// Calculate cosine similarity
|
||||
sim12 := cosineSimilarity(emb1, emb2)
|
||||
sim13 := cosineSimilarity(emb1, emb3)
|
||||
|
||||
// Similar texts should have higher similarity
|
||||
assert.Greater(t, sim12, sim13, "Similar sentences should have higher similarity than dissimilar ones")
|
||||
assert.Greater(t, sim12, float64(0.7), "Similar sentences should have high similarity")
|
||||
}
|
||||
|
||||
// TestEmbedBatch_MultipleTexts tests batch embedding.
|
||||
func TestEmbedBatch_MultipleTexts(t *testing.T) {
|
||||
svc, err := NewService()
|
||||
require.NoError(t, err)
|
||||
defer svc.Close()
|
||||
|
||||
texts := []string{
|
||||
"First text about programming.",
|
||||
"Second text about databases.",
|
||||
"Third text about machine learning.",
|
||||
}
|
||||
|
||||
embeddings, err := svc.EmbedBatch(texts)
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.Len(t, embeddings, len(texts))
|
||||
for i, emb := range embeddings {
|
||||
assert.Len(t, emb, EmbeddingDim, "Embedding %d should have correct dimension", i)
|
||||
}
|
||||
}
|
||||
|
||||
// TestEmbedBatch_EmptySlice tests batch embedding with empty slice.
|
||||
func TestEmbedBatch_EmptySlice(t *testing.T) {
|
||||
svc, err := NewService()
|
||||
require.NoError(t, err)
|
||||
defer svc.Close()
|
||||
|
||||
embeddings, err := svc.EmbedBatch([]string{})
|
||||
require.NoError(t, err)
|
||||
assert.Nil(t, embeddings)
|
||||
}
|
||||
|
||||
// TestEmbedBatch_WithEmptyTexts tests batch embedding with some empty texts.
|
||||
func TestEmbedBatch_WithEmptyTexts(t *testing.T) {
|
||||
svc, err := NewService()
|
||||
require.NoError(t, err)
|
||||
defer svc.Close()
|
||||
|
||||
texts := []string{
|
||||
"Valid text one.",
|
||||
"",
|
||||
"Valid text two.",
|
||||
"",
|
||||
}
|
||||
|
||||
embeddings, err := svc.EmbedBatch(texts)
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.Len(t, embeddings, 4)
|
||||
|
||||
// Non-empty texts should have non-zero embeddings
|
||||
var sum0 float32
|
||||
for _, v := range embeddings[0] {
|
||||
sum0 += v * v
|
||||
}
|
||||
assert.Greater(t, sum0, float32(0))
|
||||
|
||||
// Empty texts should have zero embeddings
|
||||
for _, v := range embeddings[1] {
|
||||
assert.Equal(t, float32(0), v)
|
||||
}
|
||||
|
||||
var sum2 float32
|
||||
for _, v := range embeddings[2] {
|
||||
sum2 += v * v
|
||||
}
|
||||
assert.Greater(t, sum2, float32(0))
|
||||
|
||||
for _, v := range embeddings[3] {
|
||||
assert.Equal(t, float32(0), v)
|
||||
}
|
||||
}
|
||||
|
||||
// TestEmbedBatch_AllEmpty tests batch embedding with all empty texts.
|
||||
func TestEmbedBatch_AllEmpty(t *testing.T) {
|
||||
svc, err := NewService()
|
||||
require.NoError(t, err)
|
||||
defer svc.Close()
|
||||
|
||||
texts := []string{"", "", ""}
|
||||
|
||||
embeddings, err := svc.EmbedBatch(texts)
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.Len(t, embeddings, 3)
|
||||
for i, emb := range embeddings {
|
||||
assert.Len(t, emb, EmbeddingDim, "Embedding %d should have correct dimension", i)
|
||||
for j, v := range emb {
|
||||
assert.Equal(t, float32(0), v, "Embedding %d[%d] should be zero", i, j)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestEmbed_Concurrent tests concurrent embedding calls.
|
||||
func TestEmbed_Concurrent(t *testing.T) {
|
||||
svc, err := NewService()
|
||||
require.NoError(t, err)
|
||||
defer svc.Close()
|
||||
|
||||
var wg sync.WaitGroup
|
||||
numGoroutines := 10
|
||||
|
||||
errors := make(chan error, numGoroutines)
|
||||
embeddings := make([][]float32, numGoroutines)
|
||||
|
||||
for i := 0; i < numGoroutines; i++ {
|
||||
wg.Add(1)
|
||||
go func(idx int) {
|
||||
defer wg.Done()
|
||||
|
||||
text := "Test text for concurrent embedding test"
|
||||
emb, err := svc.Embed(text)
|
||||
if err != nil {
|
||||
errors <- err
|
||||
return
|
||||
}
|
||||
embeddings[idx] = emb
|
||||
}(i)
|
||||
}
|
||||
|
||||
wg.Wait()
|
||||
close(errors)
|
||||
|
||||
for err := range errors {
|
||||
t.Errorf("Concurrent embedding error: %v", err)
|
||||
}
|
||||
|
||||
// All embeddings should be valid
|
||||
for i, emb := range embeddings {
|
||||
if emb != nil {
|
||||
assert.Len(t, emb, EmbeddingDim, "Embedding %d should have correct dimension", i)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestEmbed_SpecialCharacters tests embedding text with special characters.
|
||||
func TestEmbed_SpecialCharacters(t *testing.T) {
|
||||
svc, err := NewService()
|
||||
require.NoError(t, err)
|
||||
defer svc.Close()
|
||||
|
||||
texts := []string{
|
||||
"Text with unicode: 你好世界 🎉",
|
||||
"Text with newlines:\nLine 1\nLine 2",
|
||||
"Text with tabs:\tColumn1\tColumn2",
|
||||
"Text with quotes: \"quoted\" and 'single'",
|
||||
"Text with code: func main() { fmt.Println(\"hello\") }",
|
||||
}
|
||||
|
||||
for _, text := range texts {
|
||||
t.Run(text[:20], func(t *testing.T) {
|
||||
emb, err := svc.Embed(text)
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, emb, EmbeddingDim)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestEmbed_LongText tests embedding long text.
|
||||
func TestEmbed_LongText(t *testing.T) {
|
||||
svc, err := NewService()
|
||||
require.NoError(t, err)
|
||||
defer svc.Close()
|
||||
|
||||
// Create a long text (tokenizer should truncate appropriately)
|
||||
longText := ""
|
||||
for i := 0; i < 100; i++ {
|
||||
longText += "This is a sentence to make the text very long. "
|
||||
}
|
||||
|
||||
emb, err := svc.Embed(longText)
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, emb, EmbeddingDim)
|
||||
}
|
||||
|
||||
// TestClose tests closing the service.
|
||||
func TestClose(t *testing.T) {
|
||||
svc, err := NewService()
|
||||
require.NoError(t, err)
|
||||
|
||||
err = svc.Close()
|
||||
require.NoError(t, err)
|
||||
|
||||
// After close, embedding should fail (model resources released)
|
||||
// Note: This behavior is model-specific; some models may still work after close
|
||||
}
|
||||
|
||||
// TestEmbedBatch_SingleItem tests batch embedding with single item.
|
||||
func TestEmbedBatch_SingleItem(t *testing.T) {
|
||||
svc, err := NewService()
|
||||
require.NoError(t, err)
|
||||
defer svc.Close()
|
||||
|
||||
texts := []string{"Single text for batch embedding."}
|
||||
|
||||
embeddings, err := svc.EmbedBatch(texts)
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.Len(t, embeddings, 1)
|
||||
assert.Len(t, embeddings[0], EmbeddingDim)
|
||||
}
|
||||
|
||||
// TestEmbed_Deterministic tests that embedding is deterministic.
|
||||
func TestEmbed_Deterministic(t *testing.T) {
|
||||
svc, err := NewService()
|
||||
require.NoError(t, err)
|
||||
defer svc.Close()
|
||||
|
||||
text := "Test text for deterministic embedding."
|
||||
|
||||
emb1, err := svc.Embed(text)
|
||||
require.NoError(t, err)
|
||||
|
||||
emb2, err := svc.Embed(text)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Same text should produce same embedding
|
||||
for i := 0; i < EmbeddingDim; i++ {
|
||||
assert.Equal(t, emb1[i], emb2[i], "Embedding should be deterministic at index %d", i)
|
||||
}
|
||||
}
|
||||
|
||||
// Helper function to calculate cosine similarity
|
||||
func cosineSimilarity(a, b []float32) float64 {
|
||||
if len(a) != len(b) {
|
||||
return 0
|
||||
}
|
||||
|
||||
var dotProduct float64
|
||||
var normA float64
|
||||
var normB float64
|
||||
|
||||
for i := range a {
|
||||
dotProduct += float64(a[i]) * float64(b[i])
|
||||
normA += float64(a[i]) * float64(a[i])
|
||||
normB += float64(b[i]) * float64(b[i])
|
||||
}
|
||||
|
||||
if normA == 0 || normB == 0 {
|
||||
return 0
|
||||
}
|
||||
|
||||
return dotProduct / (math.Sqrt(normA) * math.Sqrt(normB))
|
||||
}
|
||||
@@ -1,417 +0,0 @@
|
||||
package graph
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"math"
|
||||
|
||||
"github.com/lukaszraczylo/claude-mnemonic/pkg/models"
|
||||
"github.com/rs/zerolog/log"
|
||||
)
|
||||
|
||||
const (
|
||||
// SemanticSimilarityThreshold for creating semantic edges
|
||||
SemanticSimilarityThreshold = 0.85
|
||||
|
||||
// MinFileOverlapForEdge minimum file overlap ratio to create edge
|
||||
MinFileOverlapForEdge = 0.3
|
||||
|
||||
// MaxEdgesPerNode prevents creating too many edges
|
||||
MaxEdgesPerNode = 20
|
||||
)
|
||||
|
||||
// DetectEdges identifies relationships between observations
|
||||
func DetectEdges(ctx context.Context, observations []*models.Observation) ([]Edge, error) {
|
||||
if len(observations) < 2 {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
edges := make([]Edge, 0)
|
||||
|
||||
// Build lookup maps for efficient detection
|
||||
sessionMap := buildSessionMap(observations)
|
||||
conceptMap := buildConceptMap(observations)
|
||||
fileMap := buildFileMap(observations)
|
||||
|
||||
log.Info().
|
||||
Int("observations", len(observations)).
|
||||
Int("sessions", len(sessionMap)).
|
||||
Int("concepts", len(conceptMap)).
|
||||
Msg("Starting edge detection")
|
||||
|
||||
// Detect temporal edges (same session)
|
||||
temporalEdges := detectTemporalEdges(sessionMap)
|
||||
edges = append(edges, temporalEdges...)
|
||||
|
||||
// Detect concept edges (shared tags)
|
||||
conceptEdges := detectConceptEdges(conceptMap)
|
||||
edges = append(edges, conceptEdges...)
|
||||
|
||||
// Detect file overlap edges
|
||||
fileEdges := detectFileOverlapEdges(fileMap, observations)
|
||||
edges = append(edges, fileEdges...)
|
||||
|
||||
// Prune excessive edges per node
|
||||
edges = pruneEdges(edges, MaxEdgesPerNode)
|
||||
|
||||
log.Info().
|
||||
Int("temporal_edges", len(temporalEdges)).
|
||||
Int("concept_edges", len(conceptEdges)).
|
||||
Int("file_edges", len(fileEdges)).
|
||||
Int("total_edges", len(edges)).
|
||||
Msg("Edge detection complete")
|
||||
|
||||
return edges, nil
|
||||
}
|
||||
|
||||
// buildSessionMap groups observations by SDK session
|
||||
func buildSessionMap(observations []*models.Observation) map[string][]int64 {
|
||||
sessionMap := make(map[string][]int64)
|
||||
|
||||
for _, obs := range observations {
|
||||
if obs.SDKSessionID != "" {
|
||||
sessionMap[obs.SDKSessionID] = append(sessionMap[obs.SDKSessionID], obs.ID)
|
||||
}
|
||||
}
|
||||
|
||||
return sessionMap
|
||||
}
|
||||
|
||||
// buildConceptMap groups observations by concept tags
|
||||
func buildConceptMap(observations []*models.Observation) map[string][]int64 {
|
||||
conceptMap := make(map[string][]int64)
|
||||
|
||||
for _, obs := range observations {
|
||||
for _, concept := range obs.Concepts {
|
||||
conceptMap[concept] = append(conceptMap[concept], obs.ID)
|
||||
}
|
||||
}
|
||||
|
||||
return conceptMap
|
||||
}
|
||||
|
||||
// buildFileMap maps files to observations (from both FilesRead and FilesModified)
|
||||
func buildFileMap(observations []*models.Observation) map[string][]int64 {
|
||||
fileMap := make(map[string][]int64)
|
||||
|
||||
for _, obs := range observations {
|
||||
// Add files from FilesRead
|
||||
for _, file := range obs.FilesRead {
|
||||
fileMap[file] = append(fileMap[file], obs.ID)
|
||||
}
|
||||
// Add files from FilesModified
|
||||
for _, file := range obs.FilesModified {
|
||||
fileMap[file] = append(fileMap[file], obs.ID)
|
||||
}
|
||||
}
|
||||
|
||||
return fileMap
|
||||
}
|
||||
|
||||
// detectTemporalEdges creates edges between observations in the same session
|
||||
func detectTemporalEdges(sessionMap map[string][]int64) []Edge {
|
||||
edges := make([]Edge, 0)
|
||||
|
||||
for _, obsIDs := range sessionMap {
|
||||
if len(obsIDs) < 2 {
|
||||
continue
|
||||
}
|
||||
|
||||
// Create edges between consecutive observations in session
|
||||
for i := 0; i < len(obsIDs)-1; i++ {
|
||||
edges = append(edges, Edge{
|
||||
FromID: obsIDs[i],
|
||||
ToID: obsIDs[i+1],
|
||||
Relation: RelationTemporal,
|
||||
Weight: 0.8, // High weight for temporal proximity
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
return edges
|
||||
}
|
||||
|
||||
// detectConceptEdges creates edges between observations sharing concepts
|
||||
func detectConceptEdges(conceptMap map[string][]int64) []Edge {
|
||||
edges := make([]Edge, 0)
|
||||
seen := make(map[string]bool)
|
||||
|
||||
for concept, obsIDs := range conceptMap {
|
||||
if len(obsIDs) < 2 {
|
||||
continue
|
||||
}
|
||||
|
||||
// Create edges between all observations sharing this concept
|
||||
for i := 0; i < len(obsIDs); i++ {
|
||||
for j := i + 1; j < len(obsIDs); j++ {
|
||||
// Use sorted pair as key to avoid duplicates
|
||||
pairKey := edgeKey(obsIDs[i], obsIDs[j])
|
||||
if seen[pairKey] {
|
||||
continue
|
||||
}
|
||||
seen[pairKey] = true
|
||||
|
||||
// Weight based on concept specificity (longer = more specific)
|
||||
weight := float32(0.5 + 0.3*math.Min(1.0, float64(len(concept))/20.0))
|
||||
|
||||
edges = append(edges, Edge{
|
||||
FromID: obsIDs[i],
|
||||
ToID: obsIDs[j],
|
||||
Relation: RelationConcept,
|
||||
Weight: weight,
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return edges
|
||||
}
|
||||
|
||||
// detectFileOverlapEdges creates edges based on file references
|
||||
func detectFileOverlapEdges(fileMap map[string][]int64, observations []*models.Observation) []Edge {
|
||||
edges := make([]Edge, 0)
|
||||
seen := make(map[string]bool)
|
||||
|
||||
// Build observation ID to observation map for quick lookup
|
||||
obsMap := make(map[int64]*models.Observation)
|
||||
for _, obs := range observations {
|
||||
obsMap[obs.ID] = obs
|
||||
}
|
||||
|
||||
for _, obsIDs := range fileMap {
|
||||
if len(obsIDs) < 2 {
|
||||
continue
|
||||
}
|
||||
|
||||
// Create edges between observations referencing same files
|
||||
for i := 0; i < len(obsIDs); i++ {
|
||||
for j := i + 1; j < len(obsIDs); j++ {
|
||||
pairKey := edgeKey(obsIDs[i], obsIDs[j])
|
||||
if seen[pairKey] {
|
||||
continue
|
||||
}
|
||||
seen[pairKey] = true
|
||||
|
||||
// Calculate file overlap ratio
|
||||
obs1, ok1 := obsMap[obsIDs[i]]
|
||||
obs2, ok2 := obsMap[obsIDs[j]]
|
||||
|
||||
if !ok1 || !ok2 {
|
||||
continue
|
||||
}
|
||||
|
||||
// Merge FilesRead and FilesModified for both observations
|
||||
files1 := append([]string{}, obs1.FilesRead...)
|
||||
files1 = append(files1, obs1.FilesModified...)
|
||||
files2 := append([]string{}, obs2.FilesRead...)
|
||||
files2 = append(files2, obs2.FilesModified...)
|
||||
|
||||
overlap := calculateFileOverlap(files1, files2)
|
||||
if overlap < MinFileOverlapForEdge {
|
||||
continue
|
||||
}
|
||||
|
||||
edges = append(edges, Edge{
|
||||
FromID: obsIDs[i],
|
||||
ToID: obsIDs[j],
|
||||
Relation: RelationFileOverlap,
|
||||
Weight: overlap,
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return edges
|
||||
}
|
||||
|
||||
// calculateFileOverlap computes Jaccard similarity of file sets
|
||||
func calculateFileOverlap(files1, files2 []string) float32 {
|
||||
if len(files1) == 0 || len(files2) == 0 {
|
||||
return 0.0
|
||||
}
|
||||
|
||||
// Convert to sets
|
||||
set1 := make(map[string]bool)
|
||||
for _, f := range files1 {
|
||||
set1[f] = true
|
||||
}
|
||||
|
||||
set2 := make(map[string]bool)
|
||||
for _, f := range files2 {
|
||||
set2[f] = true
|
||||
}
|
||||
|
||||
// Count intersection
|
||||
intersection := 0
|
||||
for f := range set1 {
|
||||
if set2[f] {
|
||||
intersection++
|
||||
}
|
||||
}
|
||||
|
||||
// Jaccard similarity = intersection / union
|
||||
union := len(set1) + len(set2) - intersection
|
||||
if union == 0 {
|
||||
return 0.0
|
||||
}
|
||||
|
||||
return float32(intersection) / float32(union)
|
||||
}
|
||||
|
||||
// pruneEdges limits edges per node to prevent graph explosion
|
||||
func pruneEdges(edges []Edge, maxPerNode int) []Edge {
|
||||
if maxPerNode <= 0 {
|
||||
return edges
|
||||
}
|
||||
|
||||
// Count edges per node
|
||||
outEdges := make(map[int64][]Edge)
|
||||
inEdges := make(map[int64][]Edge)
|
||||
|
||||
for _, edge := range edges {
|
||||
outEdges[edge.FromID] = append(outEdges[edge.FromID], edge)
|
||||
inEdges[edge.ToID] = append(inEdges[edge.ToID], edge)
|
||||
}
|
||||
|
||||
// Prune low-weight edges if node has too many
|
||||
pruned := make([]Edge, 0, len(edges))
|
||||
processed := make(map[string]bool)
|
||||
|
||||
for _, edge := range edges {
|
||||
pairKey := edgeKey(edge.FromID, edge.ToID)
|
||||
if processed[pairKey] {
|
||||
continue
|
||||
}
|
||||
processed[pairKey] = true
|
||||
|
||||
// Check if either node has too many edges
|
||||
fromCount := len(outEdges[edge.FromID])
|
||||
toCount := len(inEdges[edge.ToID])
|
||||
|
||||
if fromCount <= maxPerNode && toCount <= maxPerNode {
|
||||
pruned = append(pruned, edge)
|
||||
continue
|
||||
}
|
||||
|
||||
// Keep edge if it's high-weight (top edges for this node)
|
||||
if shouldKeepEdge(edge, outEdges[edge.FromID], maxPerNode) {
|
||||
pruned = append(pruned, edge)
|
||||
}
|
||||
}
|
||||
|
||||
if len(pruned) < len(edges) {
|
||||
log.Debug().
|
||||
Int("original", len(edges)).
|
||||
Int("pruned", len(pruned)).
|
||||
Int("removed", len(edges)-len(pruned)).
|
||||
Msg("Pruned excessive edges")
|
||||
}
|
||||
|
||||
return pruned
|
||||
}
|
||||
|
||||
// shouldKeepEdge determines if edge should be kept during pruning
|
||||
func shouldKeepEdge(edge Edge, nodeEdges []Edge, maxPerNode int) bool {
|
||||
// Sort node's edges by weight descending
|
||||
sortedEdges := make([]Edge, len(nodeEdges))
|
||||
copy(sortedEdges, nodeEdges)
|
||||
|
||||
sortEdgesByWeight(sortedEdges)
|
||||
|
||||
// Keep edge if it's in top maxPerNode
|
||||
for i := 0; i < maxPerNode && i < len(sortedEdges); i++ {
|
||||
if sortedEdges[i].FromID == edge.FromID && sortedEdges[i].ToID == edge.ToID {
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
// sortEdgesByWeight sorts edges by weight descending
|
||||
func sortEdgesByWeight(edges []Edge) {
|
||||
// Simple bubble sort (edges are typically small per node)
|
||||
n := len(edges)
|
||||
for i := 0; i < n-1; i++ {
|
||||
for j := 0; j < n-i-1; j++ {
|
||||
if edges[j].Weight < edges[j+1].Weight {
|
||||
edges[j], edges[j+1] = edges[j+1], edges[j]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// edgeKey creates a unique key for an edge pair (sorted)
|
||||
func edgeKey(id1, id2 int64) string {
|
||||
if id1 < id2 {
|
||||
return fmt.Sprintf("%d-%d", id1, id2)
|
||||
}
|
||||
return fmt.Sprintf("%d-%d", id2, id1)
|
||||
}
|
||||
|
||||
// DetectSemanticEdges creates edges based on semantic similarity
|
||||
// This requires embeddings and is called separately when available
|
||||
func DetectSemanticEdges(ctx context.Context, observations []*models.Observation, embeddings map[int64][]float32) []Edge {
|
||||
edges := make([]Edge, 0)
|
||||
seen := make(map[string]bool)
|
||||
|
||||
// Compare all pairs (expensive, but necessary for semantic similarity)
|
||||
for i := 0; i < len(observations); i++ {
|
||||
emb1, ok1 := embeddings[observations[i].ID]
|
||||
if !ok1 {
|
||||
continue
|
||||
}
|
||||
|
||||
for j := i + 1; j < len(observations); j++ {
|
||||
emb2, ok2 := embeddings[observations[j].ID]
|
||||
if !ok2 {
|
||||
continue
|
||||
}
|
||||
|
||||
similarity := cosineSimilarity(emb1, emb2)
|
||||
if similarity < SemanticSimilarityThreshold {
|
||||
continue
|
||||
}
|
||||
|
||||
pairKey := edgeKey(observations[i].ID, observations[j].ID)
|
||||
if seen[pairKey] {
|
||||
continue
|
||||
}
|
||||
seen[pairKey] = true
|
||||
|
||||
edges = append(edges, Edge{
|
||||
FromID: observations[i].ID,
|
||||
ToID: observations[j].ID,
|
||||
Relation: RelationSemantic,
|
||||
Weight: similarity,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
log.Info().
|
||||
Int("semantic_edges", len(edges)).
|
||||
Float32("threshold", SemanticSimilarityThreshold).
|
||||
Msg("Detected semantic edges")
|
||||
|
||||
return edges
|
||||
}
|
||||
|
||||
// cosineSimilarity computes cosine similarity between two vectors
|
||||
func cosineSimilarity(a, b []float32) float32 {
|
||||
if len(a) != len(b) {
|
||||
return 0.0
|
||||
}
|
||||
|
||||
var dotProduct, normA, normB float32
|
||||
for i := range a {
|
||||
dotProduct += a[i] * b[i]
|
||||
normA += a[i] * a[i]
|
||||
normB += b[i] * b[i]
|
||||
}
|
||||
|
||||
if normA == 0 || normB == 0 {
|
||||
return 0.0
|
||||
}
|
||||
|
||||
return dotProduct / float32(math.Sqrt(float64(normA))*math.Sqrt(float64(normB)))
|
||||
}
|
||||
@@ -1,423 +0,0 @@
|
||||
// Package graph provides observation relationship graphs for LEANN Phase 2.
|
||||
//
|
||||
// This package implements graph-based selective recomputation where observation
|
||||
// relationships (file overlap, semantic similarity, temporal proximity) form a
|
||||
// graph structure. Hub nodes (high-degree observations) store embeddings, while
|
||||
// leaf nodes recompute on-demand.
|
||||
package graph
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"math"
|
||||
"sort"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/lukaszraczylo/claude-mnemonic/pkg/models"
|
||||
"github.com/rs/zerolog/log"
|
||||
)
|
||||
|
||||
// RelationType defines the type of relationship between observations
|
||||
type RelationType int
|
||||
|
||||
const (
|
||||
// RelationFileOverlap indicates observations reference overlapping files
|
||||
RelationFileOverlap RelationType = iota
|
||||
// RelationSemantic indicates high semantic similarity (cosine > 0.85)
|
||||
RelationSemantic
|
||||
// RelationTemporal indicates observations from same session
|
||||
RelationTemporal
|
||||
// RelationConcept indicates shared concept tags
|
||||
RelationConcept
|
||||
)
|
||||
|
||||
// Edge represents a relationship between two observations
|
||||
type Edge struct {
|
||||
FromID int64
|
||||
ToID int64
|
||||
Relation RelationType
|
||||
Weight float32 // 0.0-1.0, higher = stronger relationship
|
||||
}
|
||||
|
||||
// Node represents an observation in the graph
|
||||
type Node struct {
|
||||
Metadata NodeMetadata
|
||||
LastAccess time.Time
|
||||
StoredEmb []float32 // Nil if recomputed on-demand
|
||||
ID int64
|
||||
Degree int // Number of edges (hub detection)
|
||||
AccessCount int
|
||||
}
|
||||
|
||||
// NodeMetadata contains observation metadata
|
||||
type NodeMetadata struct {
|
||||
CreatedAt time.Time
|
||||
Project string
|
||||
Type string
|
||||
Title string
|
||||
IsSuperseded bool
|
||||
}
|
||||
|
||||
// CSRGraph represents a graph in Compressed Sparse Row format for memory efficiency
|
||||
type CSRGraph struct {
|
||||
RowPtr []int32 // Node adjacency list pointers
|
||||
ColIdx []int32 // Edge destination IDs
|
||||
Weights []float32 // Edge weights
|
||||
mu sync.RWMutex
|
||||
}
|
||||
|
||||
// ObservationGraph manages the observation relationship graph
|
||||
type ObservationGraph struct {
|
||||
nodes map[int64]*Node
|
||||
csr *CSRGraph
|
||||
edges []Edge
|
||||
nodesMu sync.RWMutex
|
||||
edgesMu sync.RWMutex
|
||||
}
|
||||
|
||||
// NewObservationGraph creates a new empty observation graph
|
||||
func NewObservationGraph() *ObservationGraph {
|
||||
return &ObservationGraph{
|
||||
nodes: make(map[int64]*Node),
|
||||
edges: make([]Edge, 0),
|
||||
csr: &CSRGraph{},
|
||||
}
|
||||
}
|
||||
|
||||
// AddNode adds or updates a node in the graph
|
||||
func (g *ObservationGraph) AddNode(node *Node) {
|
||||
g.nodesMu.Lock()
|
||||
defer g.nodesMu.Unlock()
|
||||
|
||||
g.nodes[node.ID] = node
|
||||
}
|
||||
|
||||
// AddEdge adds an edge to the graph
|
||||
func (g *ObservationGraph) AddEdge(edge Edge) {
|
||||
g.edgesMu.Lock()
|
||||
defer g.edgesMu.Unlock()
|
||||
|
||||
g.edges = append(g.edges, edge)
|
||||
|
||||
// Update degree counts
|
||||
g.nodesMu.Lock()
|
||||
if fromNode, ok := g.nodes[edge.FromID]; ok {
|
||||
fromNode.Degree++
|
||||
}
|
||||
if toNode, ok := g.nodes[edge.ToID]; ok {
|
||||
toNode.Degree++
|
||||
}
|
||||
g.nodesMu.Unlock()
|
||||
}
|
||||
|
||||
// BuildCSR converts edge list to CSR format for efficient traversal
|
||||
func (g *ObservationGraph) BuildCSR() error {
|
||||
g.edgesMu.RLock()
|
||||
g.nodesMu.RLock()
|
||||
defer g.edgesMu.RUnlock()
|
||||
defer g.nodesMu.RUnlock()
|
||||
|
||||
if len(g.nodes) == 0 {
|
||||
return fmt.Errorf("no nodes in graph")
|
||||
}
|
||||
|
||||
// Create node ID to index mapping
|
||||
nodeIDs := make([]int64, 0, len(g.nodes))
|
||||
for id := range g.nodes {
|
||||
nodeIDs = append(nodeIDs, id)
|
||||
}
|
||||
sort.Slice(nodeIDs, func(i, j int) bool {
|
||||
return nodeIDs[i] < nodeIDs[j]
|
||||
})
|
||||
|
||||
idToIdx := make(map[int64]int32)
|
||||
for idx, id := range nodeIDs {
|
||||
// #nosec G115 - observation count will never exceed int32 max (2.1B) in practice
|
||||
idToIdx[id] = int32(idx)
|
||||
}
|
||||
|
||||
// Count edges per node
|
||||
edgeCounts := make([]int, len(nodeIDs))
|
||||
for _, edge := range g.edges {
|
||||
if fromIdx, ok := idToIdx[edge.FromID]; ok {
|
||||
edgeCounts[fromIdx]++
|
||||
}
|
||||
}
|
||||
|
||||
// Build row pointers
|
||||
rowPtr := make([]int32, len(nodeIDs)+1)
|
||||
rowPtr[0] = 0
|
||||
for i := 0; i < len(nodeIDs); i++ {
|
||||
// #nosec G115 - edge counts per node will not exceed int32 max
|
||||
rowPtr[i+1] = rowPtr[i] + int32(edgeCounts[i])
|
||||
}
|
||||
|
||||
// Build column indices and weights
|
||||
totalEdges := rowPtr[len(nodeIDs)]
|
||||
colIdx := make([]int32, totalEdges)
|
||||
weights := make([]float32, totalEdges)
|
||||
|
||||
// Temporary counter for filling CSR
|
||||
currentPos := make([]int32, len(nodeIDs))
|
||||
copy(currentPos, rowPtr[:len(nodeIDs)])
|
||||
|
||||
for _, edge := range g.edges {
|
||||
fromIdx, fromOk := idToIdx[edge.FromID]
|
||||
toIdx, toOk := idToIdx[edge.ToID]
|
||||
|
||||
if fromOk && toOk {
|
||||
pos := currentPos[fromIdx]
|
||||
colIdx[pos] = toIdx
|
||||
weights[pos] = edge.Weight
|
||||
currentPos[fromIdx]++
|
||||
}
|
||||
}
|
||||
|
||||
g.csr.mu.Lock()
|
||||
g.csr.RowPtr = rowPtr
|
||||
g.csr.ColIdx = colIdx
|
||||
g.csr.Weights = weights
|
||||
g.csr.mu.Unlock()
|
||||
|
||||
log.Info().
|
||||
Int("nodes", len(nodeIDs)).
|
||||
Int("edges", int(totalEdges)).
|
||||
Msg("Built CSR graph representation")
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetNeighbors returns neighboring nodes and their edge weights
|
||||
func (g *ObservationGraph) GetNeighbors(nodeID int64) ([]int64, []float32, error) {
|
||||
g.csr.mu.RLock()
|
||||
defer g.csr.mu.RUnlock()
|
||||
|
||||
// Find node index in CSR
|
||||
g.nodesMu.RLock()
|
||||
nodeIDs := make([]int64, 0, len(g.nodes))
|
||||
for id := range g.nodes {
|
||||
nodeIDs = append(nodeIDs, id)
|
||||
}
|
||||
g.nodesMu.RUnlock()
|
||||
|
||||
sort.Slice(nodeIDs, func(i, j int) bool {
|
||||
return nodeIDs[i] < nodeIDs[j]
|
||||
})
|
||||
|
||||
nodeIdx := sort.Search(len(nodeIDs), func(i int) bool {
|
||||
return nodeIDs[i] >= nodeID
|
||||
})
|
||||
|
||||
if nodeIdx >= len(nodeIDs) || nodeIDs[nodeIdx] != nodeID {
|
||||
return nil, nil, fmt.Errorf("node %d not found", nodeID)
|
||||
}
|
||||
|
||||
// Extract neighbors from CSR
|
||||
startIdx := g.csr.RowPtr[nodeIdx]
|
||||
endIdx := g.csr.RowPtr[nodeIdx+1]
|
||||
|
||||
neighborCount := endIdx - startIdx
|
||||
neighbors := make([]int64, neighborCount)
|
||||
weights := make([]float32, neighborCount)
|
||||
|
||||
for i := int32(0); i < neighborCount; i++ {
|
||||
neighborIdx := g.csr.ColIdx[startIdx+i]
|
||||
neighbors[i] = nodeIDs[neighborIdx]
|
||||
weights[i] = g.csr.Weights[startIdx+i]
|
||||
}
|
||||
|
||||
return neighbors, weights, nil
|
||||
}
|
||||
|
||||
// GetNode retrieves a node by ID
|
||||
func (g *ObservationGraph) GetNode(nodeID int64) (*Node, error) {
|
||||
g.nodesMu.RLock()
|
||||
defer g.nodesMu.RUnlock()
|
||||
|
||||
node, ok := g.nodes[nodeID]
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("node %d not found", nodeID)
|
||||
}
|
||||
|
||||
return node, nil
|
||||
}
|
||||
|
||||
// FindHubs identifies hub nodes (high degree) in the graph
|
||||
func (g *ObservationGraph) FindHubs(percentile float64) []int64 {
|
||||
g.nodesMu.RLock()
|
||||
defer g.nodesMu.RUnlock()
|
||||
|
||||
if len(g.nodes) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Collect all degrees
|
||||
degrees := make([]int, 0, len(g.nodes))
|
||||
nodeIDs := make([]int64, 0, len(g.nodes))
|
||||
|
||||
for id, node := range g.nodes {
|
||||
degrees = append(degrees, node.Degree)
|
||||
nodeIDs = append(nodeIDs, id)
|
||||
}
|
||||
|
||||
// Sort by degree
|
||||
type nodeDegree struct {
|
||||
ID int64
|
||||
Degree int
|
||||
}
|
||||
|
||||
nodeDegrees := make([]nodeDegree, len(nodeIDs))
|
||||
for i := range nodeIDs {
|
||||
nodeDegrees[i] = nodeDegree{
|
||||
ID: nodeIDs[i],
|
||||
Degree: degrees[i],
|
||||
}
|
||||
}
|
||||
|
||||
sort.Slice(nodeDegrees, func(i, j int) bool {
|
||||
return nodeDegrees[i].Degree > nodeDegrees[j].Degree
|
||||
})
|
||||
|
||||
// Return top percentile
|
||||
cutoff := int(math.Ceil(float64(len(nodeDegrees)) * (1.0 - percentile)))
|
||||
if cutoff > len(nodeDegrees) {
|
||||
cutoff = len(nodeDegrees)
|
||||
}
|
||||
|
||||
hubs := make([]int64, cutoff)
|
||||
for i := 0; i < cutoff; i++ {
|
||||
hubs[i] = nodeDegrees[i].ID
|
||||
}
|
||||
|
||||
log.Info().
|
||||
Int("total_nodes", len(g.nodes)).
|
||||
Int("hubs", len(hubs)).
|
||||
Float64("percentile", percentile).
|
||||
Msg("Identified hub nodes")
|
||||
|
||||
return hubs
|
||||
}
|
||||
|
||||
// Stats returns graph statistics
|
||||
func (g *ObservationGraph) Stats() GraphStats {
|
||||
g.nodesMu.RLock()
|
||||
g.edgesMu.RLock()
|
||||
defer g.nodesMu.RUnlock()
|
||||
defer g.edgesMu.RUnlock()
|
||||
|
||||
stats := GraphStats{
|
||||
NodeCount: len(g.nodes),
|
||||
EdgeCount: len(g.edges),
|
||||
}
|
||||
|
||||
if len(g.nodes) > 0 {
|
||||
degrees := make([]int, 0, len(g.nodes))
|
||||
for _, node := range g.nodes {
|
||||
degrees = append(degrees, node.Degree)
|
||||
}
|
||||
|
||||
sort.Ints(degrees)
|
||||
stats.AvgDegree = float64(sum(degrees)) / float64(len(degrees))
|
||||
stats.MaxDegree = degrees[len(degrees)-1]
|
||||
stats.MinDegree = degrees[0]
|
||||
|
||||
// Median
|
||||
mid := len(degrees) / 2
|
||||
if len(degrees)%2 == 0 {
|
||||
stats.MedianDegree = float64(degrees[mid-1]+degrees[mid]) / 2.0
|
||||
} else {
|
||||
stats.MedianDegree = float64(degrees[mid])
|
||||
}
|
||||
}
|
||||
|
||||
// Count edge types
|
||||
stats.EdgeTypes = make(map[RelationType]int)
|
||||
for _, edge := range g.edges {
|
||||
stats.EdgeTypes[edge.Relation]++
|
||||
}
|
||||
|
||||
return stats
|
||||
}
|
||||
|
||||
// GraphStats contains graph statistics
|
||||
type GraphStats struct {
|
||||
EdgeTypes map[RelationType]int
|
||||
AvgDegree float64
|
||||
MedianDegree float64
|
||||
NodeCount int
|
||||
EdgeCount int
|
||||
MaxDegree int
|
||||
MinDegree int
|
||||
}
|
||||
|
||||
// BuildFromObservations constructs a graph from a list of observations
|
||||
func BuildFromObservations(ctx context.Context, observations []*models.Observation) (*ObservationGraph, error) {
|
||||
graph := NewObservationGraph()
|
||||
|
||||
// Add nodes
|
||||
for _, obs := range observations {
|
||||
// Extract title from sql.NullString
|
||||
title := ""
|
||||
if obs.Title.Valid {
|
||||
title = obs.Title.String
|
||||
}
|
||||
|
||||
node := &Node{
|
||||
ID: obs.ID,
|
||||
Degree: 0,
|
||||
Metadata: NodeMetadata{
|
||||
Project: obs.Project,
|
||||
Type: string(obs.Type),
|
||||
Title: title,
|
||||
CreatedAt: time.UnixMilli(obs.CreatedAtEpoch),
|
||||
IsSuperseded: obs.IsSuperseded,
|
||||
},
|
||||
LastAccess: time.Now(),
|
||||
AccessCount: 0,
|
||||
}
|
||||
graph.AddNode(node)
|
||||
}
|
||||
|
||||
// Detect edges (will be implemented in edge_detector.go)
|
||||
edges, err := DetectEdges(ctx, observations)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("detect edges: %w", err)
|
||||
}
|
||||
|
||||
for _, edge := range edges {
|
||||
graph.AddEdge(edge)
|
||||
}
|
||||
|
||||
// Build CSR representation
|
||||
if err := graph.BuildCSR(); err != nil {
|
||||
return nil, fmt.Errorf("build CSR: %w", err)
|
||||
}
|
||||
|
||||
return graph, nil
|
||||
}
|
||||
|
||||
// Helper function to sum integers
|
||||
func sum(values []int) int {
|
||||
total := 0
|
||||
for _, v := range values {
|
||||
total += v
|
||||
}
|
||||
return total
|
||||
}
|
||||
|
||||
// String returns a human-readable representation of RelationType
|
||||
func (r RelationType) String() string {
|
||||
switch r {
|
||||
case RelationFileOverlap:
|
||||
return "file_overlap"
|
||||
case RelationSemantic:
|
||||
return "semantic"
|
||||
case RelationTemporal:
|
||||
return "temporal"
|
||||
case RelationConcept:
|
||||
return "concept"
|
||||
default:
|
||||
return "unknown"
|
||||
}
|
||||
}
|
||||
+10
-10
@@ -15,10 +15,10 @@ import (
|
||||
|
||||
// Server is the MCP server that exposes search tools.
|
||||
type Server struct {
|
||||
stdin io.Reader
|
||||
stdout io.Writer
|
||||
searchMgr *search.Manager
|
||||
version string
|
||||
stdin io.Reader
|
||||
stdout io.Writer
|
||||
}
|
||||
|
||||
// NewServer creates a new MCP server.
|
||||
@@ -41,17 +41,17 @@ type Request struct {
|
||||
|
||||
// Response represents a JSON-RPC response.
|
||||
type Response struct {
|
||||
JSONRPC string `json:"jsonrpc"`
|
||||
ID any `json:"id"`
|
||||
Result any `json:"result,omitempty"`
|
||||
Error *Error `json:"error,omitempty"`
|
||||
JSONRPC string `json:"jsonrpc"`
|
||||
}
|
||||
|
||||
// Error represents a JSON-RPC error.
|
||||
type Error struct {
|
||||
Data any `json:"data,omitempty"`
|
||||
Message string `json:"message"`
|
||||
Code int `json:"code"`
|
||||
Message string `json:"message"`
|
||||
Data any `json:"data,omitempty"`
|
||||
}
|
||||
|
||||
// ToolCallParams represents parameters for tools/call method.
|
||||
@@ -62,9 +62,9 @@ type ToolCallParams struct {
|
||||
|
||||
// Tool represents an MCP tool definition.
|
||||
type Tool struct {
|
||||
InputSchema map[string]any `json:"inputSchema"`
|
||||
Name string `json:"name"`
|
||||
Description string `json:"description"`
|
||||
InputSchema map[string]any `json:"inputSchema"`
|
||||
}
|
||||
|
||||
// Run starts the MCP server loop.
|
||||
@@ -440,17 +440,17 @@ func (s *Server) callTool(ctx context.Context, name string, args json.RawMessage
|
||||
|
||||
// TimelineParams represents parameters for timeline operations.
|
||||
type TimelineParams struct {
|
||||
AnchorID int64 `json:"anchor_id"`
|
||||
Query string `json:"query"`
|
||||
Before int `json:"before"`
|
||||
After int `json:"after"`
|
||||
Project string `json:"project"`
|
||||
ObsType string `json:"obs_type"`
|
||||
Concepts string `json:"concepts"`
|
||||
Files string `json:"files"`
|
||||
Format string `json:"format"`
|
||||
AnchorID int64 `json:"anchor_id"`
|
||||
Before int `json:"before"`
|
||||
After int `json:"after"`
|
||||
DateStart int64 `json:"dateStart"`
|
||||
DateEnd int64 `json:"dateEnd"`
|
||||
Format string `json:"format"`
|
||||
}
|
||||
|
||||
// handleTimeline handles timeline requests.
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,432 +0,0 @@
|
||||
// Package pattern provides pattern detection and recognition functionality.
|
||||
package pattern
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/lukaszraczylo/claude-mnemonic/internal/db/sqlite"
|
||||
"github.com/lukaszraczylo/claude-mnemonic/pkg/models"
|
||||
"github.com/rs/zerolog/log"
|
||||
)
|
||||
|
||||
// DetectorConfig contains configuration for the pattern detector.
|
||||
type DetectorConfig struct {
|
||||
// MinMatchScore is the minimum similarity score to consider a match (0.0-1.0).
|
||||
MinMatchScore float64
|
||||
// MinFrequencyForPattern is the minimum occurrences before creating a pattern.
|
||||
MinFrequencyForPattern int
|
||||
// AnalysisInterval is how often to run background pattern analysis.
|
||||
AnalysisInterval time.Duration
|
||||
// MaxPatternsToTrack is the maximum number of active patterns.
|
||||
MaxPatternsToTrack int
|
||||
}
|
||||
|
||||
// DefaultConfig returns the default detector configuration.
|
||||
func DefaultConfig() DetectorConfig {
|
||||
return DetectorConfig{
|
||||
MinMatchScore: 0.3, // 30% similarity threshold
|
||||
MinFrequencyForPattern: 2, // At least 2 occurrences to form a pattern
|
||||
AnalysisInterval: 5 * time.Minute,
|
||||
MaxPatternsToTrack: 1000,
|
||||
}
|
||||
}
|
||||
|
||||
// PatternSyncFunc is a callback for syncing patterns to vector store.
|
||||
type PatternSyncFunc func(pattern *models.Pattern)
|
||||
|
||||
// Detector detects and tracks recurring patterns across observations.
|
||||
type Detector struct {
|
||||
ctx context.Context
|
||||
patternStore *sqlite.PatternStore
|
||||
observationStore *sqlite.ObservationStore
|
||||
syncFunc PatternSyncFunc
|
||||
candidates map[string]*candidatePattern
|
||||
cancel context.CancelFunc
|
||||
config DetectorConfig
|
||||
wg sync.WaitGroup
|
||||
candidatesMu sync.RWMutex
|
||||
}
|
||||
|
||||
// SetSyncFunc sets the callback for syncing patterns to vector store.
|
||||
func (d *Detector) SetSyncFunc(fn PatternSyncFunc) {
|
||||
d.syncFunc = fn
|
||||
}
|
||||
|
||||
// candidatePattern tracks a potential pattern before it reaches frequency threshold.
|
||||
type candidatePattern struct {
|
||||
patternType models.PatternType
|
||||
title string
|
||||
signature []string
|
||||
observationIDs []int64
|
||||
projects []string
|
||||
lastSeenEpoch int64
|
||||
}
|
||||
|
||||
// NewDetector creates a new pattern detector.
|
||||
func NewDetector(patternStore *sqlite.PatternStore, observationStore *sqlite.ObservationStore, config DetectorConfig) *Detector {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
return &Detector{
|
||||
config: config,
|
||||
patternStore: patternStore,
|
||||
observationStore: observationStore,
|
||||
candidates: make(map[string]*candidatePattern),
|
||||
ctx: ctx,
|
||||
cancel: cancel,
|
||||
}
|
||||
}
|
||||
|
||||
// Start begins background pattern analysis.
|
||||
func (d *Detector) Start() {
|
||||
d.wg.Add(1)
|
||||
go d.backgroundAnalysis()
|
||||
log.Info().Dur("interval", d.config.AnalysisInterval).Msg("Pattern detector started")
|
||||
}
|
||||
|
||||
// Stop stops background pattern analysis.
|
||||
func (d *Detector) Stop() {
|
||||
d.cancel()
|
||||
d.wg.Wait()
|
||||
log.Info().Msg("Pattern detector stopped")
|
||||
}
|
||||
|
||||
// backgroundAnalysis runs periodic pattern analysis.
|
||||
func (d *Detector) backgroundAnalysis() {
|
||||
defer d.wg.Done()
|
||||
|
||||
ticker := time.NewTicker(d.config.AnalysisInterval)
|
||||
defer ticker.Stop()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-d.ctx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
if err := d.AnalyzeRecentObservations(d.ctx); err != nil {
|
||||
log.Warn().Err(err).Msg("Background pattern analysis failed")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// AnalyzeObservation processes a new observation for pattern detection.
|
||||
// This is called synchronously when a new observation is stored.
|
||||
func (d *Detector) AnalyzeObservation(ctx context.Context, obs *models.Observation) (*DetectionResult, error) {
|
||||
result := &DetectionResult{}
|
||||
|
||||
// Extract signature from observation
|
||||
signature := models.ExtractSignature(
|
||||
obs.Concepts,
|
||||
obs.Title.String,
|
||||
obs.Narrative.String,
|
||||
)
|
||||
if len(signature) == 0 {
|
||||
return result, nil // Nothing to detect
|
||||
}
|
||||
|
||||
// Check against existing patterns
|
||||
matches, err := d.patternStore.FindMatchingPatterns(ctx, signature, d.config.MinMatchScore)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if len(matches) > 0 {
|
||||
// Found existing pattern match
|
||||
bestMatch := matches[0]
|
||||
for _, m := range matches[1:] {
|
||||
if models.CalculateMatchScore(signature, m.Signature) > models.CalculateMatchScore(signature, bestMatch.Signature) {
|
||||
bestMatch = m
|
||||
}
|
||||
}
|
||||
|
||||
// Update the pattern with new occurrence
|
||||
if err := d.patternStore.IncrementPatternFrequency(ctx, bestMatch.ID, obs.Project, obs.ID); err != nil {
|
||||
log.Warn().Err(err).Int64("pattern_id", bestMatch.ID).Msg("Failed to update pattern frequency")
|
||||
}
|
||||
|
||||
result.MatchedPattern = bestMatch
|
||||
result.MatchScore = models.CalculateMatchScore(signature, bestMatch.Signature)
|
||||
result.IsNewPattern = false
|
||||
|
||||
log.Debug().
|
||||
Int64("pattern_id", bestMatch.ID).
|
||||
Str("pattern_name", bestMatch.Name).
|
||||
Float64("score", result.MatchScore).
|
||||
Msg("Observation matched existing pattern")
|
||||
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// No existing pattern match - check candidates
|
||||
candidateKey := generateCandidateKey(signature)
|
||||
d.candidatesMu.Lock()
|
||||
defer d.candidatesMu.Unlock()
|
||||
|
||||
if candidate, exists := d.candidates[candidateKey]; exists {
|
||||
// Update existing candidate
|
||||
candidate.observationIDs = append(candidate.observationIDs, obs.ID)
|
||||
|
||||
// Add project if not already present
|
||||
found := false
|
||||
for _, p := range candidate.projects {
|
||||
if p == obs.Project {
|
||||
found = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
candidate.projects = append(candidate.projects, obs.Project)
|
||||
}
|
||||
candidate.lastSeenEpoch = time.Now().UnixMilli()
|
||||
|
||||
// Check if candidate should be promoted to pattern
|
||||
if len(candidate.observationIDs) >= d.config.MinFrequencyForPattern {
|
||||
pattern, err := d.promoteCandidate(ctx, candidateKey, candidate)
|
||||
if err != nil {
|
||||
log.Warn().Err(err).Msg("Failed to promote candidate to pattern")
|
||||
} else {
|
||||
result.MatchedPattern = pattern
|
||||
result.IsNewPattern = true
|
||||
log.Info().
|
||||
Str("pattern_name", pattern.Name).
|
||||
Int("frequency", pattern.Frequency).
|
||||
Msg("New pattern detected and stored")
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// Create new candidate
|
||||
patternType := models.DetectPatternType(obs.Concepts, obs.Title.String, obs.Narrative.String)
|
||||
d.candidates[candidateKey] = &candidatePattern{
|
||||
signature: signature,
|
||||
observationIDs: []int64{obs.ID},
|
||||
projects: []string{obs.Project},
|
||||
patternType: patternType,
|
||||
title: obs.Title.String,
|
||||
lastSeenEpoch: time.Now().UnixMilli(),
|
||||
}
|
||||
log.Debug().
|
||||
Str("candidate_key", candidateKey).
|
||||
Strs("signature", signature).
|
||||
Msg("New pattern candidate created")
|
||||
}
|
||||
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// promoteCandidate converts a candidate to a stored pattern.
|
||||
func (d *Detector) promoteCandidate(ctx context.Context, key string, candidate *candidatePattern) (*models.Pattern, error) {
|
||||
// Generate pattern name from signature
|
||||
name := generatePatternName(candidate.patternType, candidate.signature, candidate.title)
|
||||
|
||||
// Create base pattern using NewPattern with first observation
|
||||
firstProject := ""
|
||||
if len(candidate.projects) > 0 {
|
||||
firstProject = candidate.projects[0]
|
||||
}
|
||||
var firstObsID int64
|
||||
if len(candidate.observationIDs) > 0 {
|
||||
firstObsID = candidate.observationIDs[0]
|
||||
}
|
||||
pattern := models.NewPattern(
|
||||
name,
|
||||
candidate.patternType,
|
||||
"Automatically detected pattern from recurring observations",
|
||||
candidate.signature,
|
||||
firstProject,
|
||||
firstObsID,
|
||||
)
|
||||
|
||||
// Add remaining projects and observations
|
||||
for i := 1; i < len(candidate.projects); i++ {
|
||||
pattern.Projects = append(pattern.Projects, candidate.projects[i])
|
||||
}
|
||||
for i := 1; i < len(candidate.observationIDs); i++ {
|
||||
pattern.ObservationIDs = append(pattern.ObservationIDs, candidate.observationIDs[i])
|
||||
}
|
||||
pattern.Frequency = len(candidate.observationIDs)
|
||||
|
||||
id, err := d.patternStore.StorePattern(ctx, pattern)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
pattern.ID = id
|
||||
|
||||
// Sync to vector store if callback is set
|
||||
if d.syncFunc != nil {
|
||||
d.syncFunc(pattern)
|
||||
}
|
||||
|
||||
// Remove from candidates
|
||||
delete(d.candidates, key)
|
||||
|
||||
return pattern, nil
|
||||
}
|
||||
|
||||
// AnalyzeRecentObservations analyzes recent observations for pattern detection.
|
||||
// This is used for background batch analysis.
|
||||
func (d *Detector) AnalyzeRecentObservations(ctx context.Context) error {
|
||||
// Get observations from the last 24 hours that haven't been analyzed
|
||||
observations, err := d.observationStore.GetRecentObservations(ctx, "", 100)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
analyzed := 0
|
||||
patternsFound := 0
|
||||
for _, obs := range observations {
|
||||
result, err := d.AnalyzeObservation(ctx, obs)
|
||||
if err != nil {
|
||||
log.Warn().Err(err).Int64("obs_id", obs.ID).Msg("Failed to analyze observation")
|
||||
continue
|
||||
}
|
||||
analyzed++
|
||||
if result.MatchedPattern != nil {
|
||||
patternsFound++
|
||||
}
|
||||
}
|
||||
|
||||
if analyzed > 0 {
|
||||
log.Info().
|
||||
Int("analyzed", analyzed).
|
||||
Int("patterns_found", patternsFound).
|
||||
Msg("Background pattern analysis completed")
|
||||
}
|
||||
|
||||
// Clean up old candidates (older than 7 days)
|
||||
d.cleanupOldCandidates()
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// cleanupOldCandidates removes candidates that haven't been seen recently.
|
||||
func (d *Detector) cleanupOldCandidates() {
|
||||
d.candidatesMu.Lock()
|
||||
defer d.candidatesMu.Unlock()
|
||||
|
||||
threshold := time.Now().Add(-7 * 24 * time.Hour).UnixMilli()
|
||||
for key, candidate := range d.candidates {
|
||||
if candidate.lastSeenEpoch < threshold {
|
||||
delete(d.candidates, key)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// GetPatternInsight returns a formatted insight string for a pattern.
|
||||
func (d *Detector) GetPatternInsight(ctx context.Context, patternID int64) (string, error) {
|
||||
pattern, err := d.patternStore.GetPatternByID(ctx, patternID)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
return formatPatternInsight(pattern), nil
|
||||
}
|
||||
|
||||
// DetectionResult contains the result of pattern detection.
|
||||
type DetectionResult struct {
|
||||
MatchedPattern *models.Pattern
|
||||
MatchScore float64
|
||||
IsNewPattern bool
|
||||
}
|
||||
|
||||
// generateCandidateKey creates a unique key for a signature.
|
||||
func generateCandidateKey(signature []string) string {
|
||||
if len(signature) == 0 {
|
||||
return ""
|
||||
}
|
||||
key := ""
|
||||
for _, s := range signature {
|
||||
key += s + "|"
|
||||
}
|
||||
return key
|
||||
}
|
||||
|
||||
// generatePatternName creates a human-readable name for a pattern.
|
||||
func generatePatternName(patternType models.PatternType, signature []string, title string) string {
|
||||
// Use title if available and meaningful
|
||||
if title != "" && len(title) < 60 {
|
||||
return title
|
||||
}
|
||||
|
||||
// Otherwise generate from type and signature
|
||||
prefix := ""
|
||||
switch patternType {
|
||||
case models.PatternTypeBug:
|
||||
prefix = "Bug Pattern: "
|
||||
case models.PatternTypeRefactor:
|
||||
prefix = "Refactor Pattern: "
|
||||
case models.PatternTypeArchitecture:
|
||||
prefix = "Architecture Pattern: "
|
||||
case models.PatternTypeAntiPattern:
|
||||
prefix = "Anti-Pattern: "
|
||||
case models.PatternTypeBestPractice:
|
||||
prefix = "Best Practice: "
|
||||
}
|
||||
|
||||
// Use first few signature elements
|
||||
if len(signature) > 0 {
|
||||
name := prefix
|
||||
for i, s := range signature {
|
||||
if i >= 3 {
|
||||
break
|
||||
}
|
||||
if i > 0 {
|
||||
name += ", "
|
||||
}
|
||||
name += s
|
||||
}
|
||||
return name
|
||||
}
|
||||
|
||||
return prefix + "Unnamed"
|
||||
}
|
||||
|
||||
// formatPatternInsight creates a human-readable insight from a pattern.
|
||||
func formatPatternInsight(pattern *models.Pattern) string {
|
||||
insight := "I've encountered this pattern " +
|
||||
itoa(pattern.Frequency) + " times"
|
||||
|
||||
if len(pattern.Projects) > 1 {
|
||||
insight += " across " + itoa(len(pattern.Projects)) + " projects"
|
||||
}
|
||||
|
||||
insight += ". "
|
||||
|
||||
if pattern.Recommendation.Valid && pattern.Recommendation.String != "" {
|
||||
insight += "What works: " + pattern.Recommendation.String
|
||||
} else {
|
||||
switch pattern.Type {
|
||||
case models.PatternTypeBug:
|
||||
insight += "This appears to be a recurring bug pattern."
|
||||
case models.PatternTypeAntiPattern:
|
||||
insight += "This is an identified anti-pattern to avoid."
|
||||
case models.PatternTypeBestPractice:
|
||||
insight += "This is a validated best practice."
|
||||
default:
|
||||
insight += "This is a recognized pattern in the codebase."
|
||||
}
|
||||
}
|
||||
|
||||
return insight
|
||||
}
|
||||
|
||||
// itoa converts int to string without importing strconv.
|
||||
func itoa(n int) string {
|
||||
if n == 0 {
|
||||
return "0"
|
||||
}
|
||||
negative := false
|
||||
if n < 0 {
|
||||
negative = true
|
||||
n = -n
|
||||
}
|
||||
var digits []byte
|
||||
for n > 0 {
|
||||
digits = append([]byte{byte('0' + n%10)}, digits...)
|
||||
n /= 10
|
||||
}
|
||||
if negative {
|
||||
digits = append([]byte{'-'}, digits...)
|
||||
}
|
||||
return string(digits)
|
||||
}
|
||||
@@ -1,450 +0,0 @@
|
||||
package pattern
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/lukaszraczylo/claude-mnemonic/internal/db/sqlite"
|
||||
"github.com/lukaszraczylo/claude-mnemonic/pkg/models"
|
||||
)
|
||||
|
||||
func TestNewDetector(t *testing.T) {
|
||||
store := setupTestStore(t)
|
||||
defer store.Close()
|
||||
|
||||
patternStore := sqlite.NewPatternStore(store)
|
||||
observationStore := sqlite.NewObservationStore(store)
|
||||
config := DefaultConfig()
|
||||
|
||||
detector := NewDetector(patternStore, observationStore, config)
|
||||
if detector == nil {
|
||||
t.Fatal("Expected non-nil detector")
|
||||
}
|
||||
|
||||
if detector.config.MinMatchScore != config.MinMatchScore {
|
||||
t.Errorf("Expected MinMatchScore %f, got %f",
|
||||
config.MinMatchScore, detector.config.MinMatchScore)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDetector_StartStop(t *testing.T) {
|
||||
store := setupTestStore(t)
|
||||
defer store.Close()
|
||||
|
||||
patternStore := sqlite.NewPatternStore(store)
|
||||
observationStore := sqlite.NewObservationStore(store)
|
||||
config := DefaultConfig()
|
||||
config.AnalysisInterval = 100 * time.Millisecond // Short interval for testing
|
||||
|
||||
detector := NewDetector(patternStore, observationStore, config)
|
||||
|
||||
// Start
|
||||
detector.Start()
|
||||
|
||||
// Wait a bit to ensure background goroutine is running
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
|
||||
// Stop
|
||||
detector.Stop()
|
||||
|
||||
// Verify we can stop without hanging
|
||||
// (if this test hangs, the Stop logic is broken)
|
||||
}
|
||||
|
||||
func TestDetector_AnalyzeObservation_NewCandidate(t *testing.T) {
|
||||
store := setupTestStore(t)
|
||||
defer store.Close()
|
||||
|
||||
patternStore := sqlite.NewPatternStore(store)
|
||||
observationStore := sqlite.NewObservationStore(store)
|
||||
config := DefaultConfig()
|
||||
config.MinFrequencyForPattern = 2
|
||||
|
||||
detector := NewDetector(patternStore, observationStore, config)
|
||||
ctx := context.Background()
|
||||
|
||||
// Create an observation
|
||||
obs := createTestObservation(1, "nil-handling", []string{"nil", "error-handling"})
|
||||
|
||||
// Analyze first observation
|
||||
result, err := detector.AnalyzeObservation(ctx, obs)
|
||||
if err != nil {
|
||||
t.Fatalf("AnalyzeObservation() error = %v", err)
|
||||
}
|
||||
|
||||
// First observation should create a candidate, not a pattern
|
||||
if result.MatchedPattern != nil {
|
||||
t.Errorf("Expected no pattern match for first observation")
|
||||
}
|
||||
if result.IsNewPattern {
|
||||
t.Errorf("Expected IsNewPattern to be false for first observation")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDetector_AnalyzeObservation_PromoteToPattern(t *testing.T) {
|
||||
store := setupTestStore(t)
|
||||
defer store.Close()
|
||||
|
||||
patternStore := sqlite.NewPatternStore(store)
|
||||
observationStore := sqlite.NewObservationStore(store)
|
||||
config := DefaultConfig()
|
||||
config.MinFrequencyForPattern = 2
|
||||
|
||||
detector := NewDetector(patternStore, observationStore, config)
|
||||
ctx := context.Background()
|
||||
|
||||
// Create two similar observations
|
||||
obs1 := createTestObservation(1, "Nil pointer handling", []string{"nil", "error-handling"})
|
||||
obs2 := createTestObservation(2, "Nil pointer handling", []string{"nil", "error-handling"})
|
||||
|
||||
// Analyze first observation
|
||||
_, err := detector.AnalyzeObservation(ctx, obs1)
|
||||
if err != nil {
|
||||
t.Fatalf("AnalyzeObservation(obs1) error = %v", err)
|
||||
}
|
||||
|
||||
// Analyze second observation - should promote to pattern
|
||||
result, err := detector.AnalyzeObservation(ctx, obs2)
|
||||
if err != nil {
|
||||
t.Fatalf("AnalyzeObservation(obs2) error = %v", err)
|
||||
}
|
||||
|
||||
if result.MatchedPattern == nil {
|
||||
t.Fatal("Expected pattern to be created after second occurrence")
|
||||
}
|
||||
if !result.IsNewPattern {
|
||||
t.Errorf("Expected IsNewPattern to be true for newly promoted pattern")
|
||||
}
|
||||
if result.MatchedPattern.Frequency != 2 {
|
||||
t.Errorf("Expected frequency 2, got %d", result.MatchedPattern.Frequency)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDetector_AnalyzeObservation_MatchExisting(t *testing.T) {
|
||||
store := setupTestStore(t)
|
||||
defer store.Close()
|
||||
|
||||
patternStore := sqlite.NewPatternStore(store)
|
||||
observationStore := sqlite.NewObservationStore(store)
|
||||
config := DefaultConfig()
|
||||
|
||||
detector := NewDetector(patternStore, observationStore, config)
|
||||
ctx := context.Background()
|
||||
|
||||
// Create an existing pattern
|
||||
pattern := &models.Pattern{
|
||||
Name: "Existing Pattern",
|
||||
Type: models.PatternTypeBug,
|
||||
Signature: []string{"nil", "error-handling", "pointer"},
|
||||
Frequency: 5,
|
||||
Projects: []string{"proj1"},
|
||||
ObservationIDs: []int64{1, 2, 3, 4, 5},
|
||||
Status: models.PatternStatusActive,
|
||||
Confidence: 0.7,
|
||||
LastSeenAt: time.Now().Format(time.RFC3339),
|
||||
LastSeenEpoch: time.Now().UnixMilli(),
|
||||
CreatedAt: time.Now().Format(time.RFC3339),
|
||||
CreatedAtEpoch: time.Now().UnixMilli(),
|
||||
}
|
||||
patternStore.StorePattern(ctx, pattern)
|
||||
|
||||
// Create observation with similar signature
|
||||
obs := createTestObservation(10, "Nil check", []string{"nil", "error-handling"})
|
||||
|
||||
// Analyze - should match existing pattern
|
||||
result, err := detector.AnalyzeObservation(ctx, obs)
|
||||
if err != nil {
|
||||
t.Fatalf("AnalyzeObservation() error = %v", err)
|
||||
}
|
||||
|
||||
if result.MatchedPattern == nil {
|
||||
t.Fatal("Expected to match existing pattern")
|
||||
}
|
||||
if result.IsNewPattern {
|
||||
t.Errorf("Expected IsNewPattern to be false for existing pattern")
|
||||
}
|
||||
if result.MatchScore < 0.3 {
|
||||
t.Errorf("Expected match score >= 0.3, got %f", result.MatchScore)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDetector_AnalyzeObservation_NoMatch(t *testing.T) {
|
||||
store := setupTestStore(t)
|
||||
defer store.Close()
|
||||
|
||||
patternStore := sqlite.NewPatternStore(store)
|
||||
observationStore := sqlite.NewObservationStore(store)
|
||||
config := DefaultConfig()
|
||||
config.MinMatchScore = 0.5 // Higher threshold
|
||||
|
||||
detector := NewDetector(patternStore, observationStore, config)
|
||||
ctx := context.Background()
|
||||
|
||||
// Create an existing pattern with specific signature
|
||||
pattern := &models.Pattern{
|
||||
Name: "Specific Pattern",
|
||||
Type: models.PatternTypeBug,
|
||||
Signature: []string{"database", "connection", "pool"},
|
||||
Frequency: 3,
|
||||
Projects: []string{"proj1"},
|
||||
ObservationIDs: []int64{1, 2, 3},
|
||||
Status: models.PatternStatusActive,
|
||||
Confidence: 0.6,
|
||||
LastSeenAt: time.Now().Format(time.RFC3339),
|
||||
LastSeenEpoch: time.Now().UnixMilli(),
|
||||
CreatedAt: time.Now().Format(time.RFC3339),
|
||||
CreatedAtEpoch: time.Now().UnixMilli(),
|
||||
}
|
||||
patternStore.StorePattern(ctx, pattern)
|
||||
|
||||
// Create observation with completely different signature
|
||||
obs := createTestObservation(10, "UI Component", []string{"frontend", "react", "component"})
|
||||
|
||||
// Analyze - should not match
|
||||
result, err := detector.AnalyzeObservation(ctx, obs)
|
||||
if err != nil {
|
||||
t.Fatalf("AnalyzeObservation() error = %v", err)
|
||||
}
|
||||
|
||||
if result.MatchedPattern != nil {
|
||||
t.Errorf("Expected no match for unrelated observation")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDetector_CandidateCleanup(t *testing.T) {
|
||||
store := setupTestStore(t)
|
||||
defer store.Close()
|
||||
|
||||
patternStore := sqlite.NewPatternStore(store)
|
||||
observationStore := sqlite.NewObservationStore(store)
|
||||
config := DefaultConfig()
|
||||
config.MinFrequencyForPattern = 3 // Higher threshold
|
||||
|
||||
detector := NewDetector(patternStore, observationStore, config)
|
||||
|
||||
// Add an old candidate manually
|
||||
oldKey := "old|candidate|"
|
||||
detector.candidates[oldKey] = &candidatePattern{
|
||||
signature: []string{"old", "candidate"},
|
||||
observationIDs: []int64{1},
|
||||
projects: []string{"proj1"},
|
||||
patternType: models.PatternTypeBug,
|
||||
title: "Old Candidate",
|
||||
lastSeenEpoch: time.Now().Add(-8 * 24 * time.Hour).UnixMilli(), // 8 days ago
|
||||
}
|
||||
|
||||
// Add a recent candidate
|
||||
recentKey := "recent|candidate|"
|
||||
detector.candidates[recentKey] = &candidatePattern{
|
||||
signature: []string{"recent", "candidate"},
|
||||
observationIDs: []int64{2},
|
||||
projects: []string{"proj1"},
|
||||
patternType: models.PatternTypeBug,
|
||||
title: "Recent Candidate",
|
||||
lastSeenEpoch: time.Now().UnixMilli(),
|
||||
}
|
||||
|
||||
// Run cleanup
|
||||
detector.cleanupOldCandidates()
|
||||
|
||||
// Old candidate should be removed
|
||||
if _, exists := detector.candidates[oldKey]; exists {
|
||||
t.Errorf("Expected old candidate to be cleaned up")
|
||||
}
|
||||
|
||||
// Recent candidate should remain
|
||||
if _, exists := detector.candidates[recentKey]; !exists {
|
||||
t.Errorf("Expected recent candidate to remain")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDetector_GetPatternInsight(t *testing.T) {
|
||||
store := setupTestStore(t)
|
||||
defer store.Close()
|
||||
|
||||
patternStore := sqlite.NewPatternStore(store)
|
||||
observationStore := sqlite.NewObservationStore(store)
|
||||
config := DefaultConfig()
|
||||
|
||||
detector := NewDetector(patternStore, observationStore, config)
|
||||
ctx := context.Background()
|
||||
|
||||
// Create pattern with recommendation
|
||||
pattern := &models.Pattern{
|
||||
Name: "Insight Test Pattern",
|
||||
Type: models.PatternTypeBestPractice,
|
||||
Signature: []string{"test"},
|
||||
Recommendation: sql.NullString{String: "Always write tests first", Valid: true},
|
||||
Frequency: 12,
|
||||
Projects: []string{"proj1", "proj2", "proj3"},
|
||||
ObservationIDs: []int64{1},
|
||||
Status: models.PatternStatusActive,
|
||||
Confidence: 0.8,
|
||||
LastSeenAt: time.Now().Format(time.RFC3339),
|
||||
LastSeenEpoch: time.Now().UnixMilli(),
|
||||
CreatedAt: time.Now().Format(time.RFC3339),
|
||||
CreatedAtEpoch: time.Now().UnixMilli(),
|
||||
}
|
||||
id, _ := patternStore.StorePattern(ctx, pattern)
|
||||
|
||||
// Get insight
|
||||
insight, err := detector.GetPatternInsight(ctx, id)
|
||||
if err != nil {
|
||||
t.Fatalf("GetPatternInsight() error = %v", err)
|
||||
}
|
||||
|
||||
// Verify insight contains expected elements
|
||||
if insight == "" {
|
||||
t.Error("Expected non-empty insight")
|
||||
}
|
||||
if !containsString(insight, "12 times") {
|
||||
t.Errorf("Expected insight to contain frequency, got: %s", insight)
|
||||
}
|
||||
if !containsString(insight, "3 projects") {
|
||||
t.Errorf("Expected insight to contain project count, got: %s", insight)
|
||||
}
|
||||
if !containsString(insight, "Always write tests first") {
|
||||
t.Errorf("Expected insight to contain recommendation, got: %s", insight)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDefaultConfig(t *testing.T) {
|
||||
config := DefaultConfig()
|
||||
|
||||
if config.MinMatchScore <= 0 || config.MinMatchScore > 1 {
|
||||
t.Errorf("Invalid MinMatchScore: %f", config.MinMatchScore)
|
||||
}
|
||||
if config.MinFrequencyForPattern < 1 {
|
||||
t.Errorf("Invalid MinFrequencyForPattern: %d", config.MinFrequencyForPattern)
|
||||
}
|
||||
if config.AnalysisInterval <= 0 {
|
||||
t.Errorf("Invalid AnalysisInterval: %v", config.AnalysisInterval)
|
||||
}
|
||||
if config.MaxPatternsToTrack <= 0 {
|
||||
t.Errorf("Invalid MaxPatternsToTrack: %d", config.MaxPatternsToTrack)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGeneratePatternName(t *testing.T) {
|
||||
tests := []struct {
|
||||
patternType models.PatternType
|
||||
title string
|
||||
wantPrefix string
|
||||
signature []string
|
||||
}{
|
||||
{patternType: models.PatternTypeBug, signature: []string{"nil", "error"}, title: "", wantPrefix: "Bug Pattern:"},
|
||||
{patternType: models.PatternTypeRefactor, signature: []string{"extract"}, title: "", wantPrefix: "Refactor Pattern:"},
|
||||
{patternType: models.PatternTypeArchitecture, signature: []string{"service"}, title: "", wantPrefix: "Architecture Pattern:"},
|
||||
{patternType: models.PatternTypeAntiPattern, signature: []string{"god-class"}, title: "", wantPrefix: "Anti-Pattern:"},
|
||||
{patternType: models.PatternTypeBestPractice, signature: []string{"testing"}, title: "", wantPrefix: "Best Practice:"},
|
||||
{patternType: models.PatternTypeBug, signature: []string{}, title: "Short Title", wantPrefix: "Short Title"}, // Use title directly
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
name := generatePatternName(tt.patternType, tt.signature, tt.title)
|
||||
if !hasPrefix(name, tt.wantPrefix) {
|
||||
t.Errorf("generatePatternName(%v, %v, %q) = %q, want prefix %q",
|
||||
tt.patternType, tt.signature, tt.title, name, tt.wantPrefix)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestFormatPatternInsight(t *testing.T) {
|
||||
// Pattern without recommendation
|
||||
pattern1 := &models.Pattern{
|
||||
Type: models.PatternTypeBug,
|
||||
Frequency: 5,
|
||||
Projects: []string{"proj1"},
|
||||
}
|
||||
insight1 := formatPatternInsight(pattern1)
|
||||
if !containsString(insight1, "5 times") {
|
||||
t.Errorf("Expected insight to contain frequency")
|
||||
}
|
||||
if !containsString(insight1, "recurring bug pattern") {
|
||||
t.Errorf("Expected bug pattern description")
|
||||
}
|
||||
|
||||
// Pattern with recommendation
|
||||
pattern2 := &models.Pattern{
|
||||
Type: models.PatternTypeBestPractice,
|
||||
Frequency: 10,
|
||||
Projects: []string{"proj1", "proj2"},
|
||||
Recommendation: sql.NullString{String: "Do this", Valid: true},
|
||||
}
|
||||
insight2 := formatPatternInsight(pattern2)
|
||||
if !containsString(insight2, "10 times") {
|
||||
t.Errorf("Expected insight to contain frequency")
|
||||
}
|
||||
if !containsString(insight2, "2 projects") {
|
||||
t.Errorf("Expected insight to contain project count")
|
||||
}
|
||||
if !containsString(insight2, "Do this") {
|
||||
t.Errorf("Expected insight to contain recommendation")
|
||||
}
|
||||
}
|
||||
|
||||
// Helper functions
|
||||
|
||||
func setupTestStore(t *testing.T) *sqlite.Store {
|
||||
t.Helper()
|
||||
|
||||
// Create temp database file
|
||||
tmpFile, err := os.CreateTemp("", "pattern_test_*.db")
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to create temp file: %v", err)
|
||||
}
|
||||
tmpFile.Close()
|
||||
|
||||
t.Cleanup(func() {
|
||||
os.Remove(tmpFile.Name())
|
||||
})
|
||||
|
||||
store, err := sqlite.NewStore(sqlite.StoreConfig{
|
||||
Path: tmpFile.Name(),
|
||||
MaxConns: 1,
|
||||
WALMode: true,
|
||||
})
|
||||
if err != nil {
|
||||
// Check if this is an FTS5 related error
|
||||
if containsString(err.Error(), "fts5") || containsString(err.Error(), "no such module") {
|
||||
t.Skip("FTS5 not available in this SQLite build")
|
||||
}
|
||||
t.Fatalf("Failed to create store: %v", err)
|
||||
}
|
||||
|
||||
return store
|
||||
}
|
||||
|
||||
func createTestObservation(id int64, title string, concepts []string) *models.Observation {
|
||||
return &models.Observation{
|
||||
ID: id,
|
||||
SDKSessionID: "test-session",
|
||||
Project: "test-project",
|
||||
Scope: models.ScopeProject,
|
||||
Type: models.ObsTypeBugfix,
|
||||
Title: sql.NullString{String: title, Valid: true},
|
||||
Narrative: sql.NullString{String: "Test narrative", Valid: true},
|
||||
Concepts: concepts,
|
||||
CreatedAt: time.Now().Format(time.RFC3339),
|
||||
CreatedAtEpoch: time.Now().UnixMilli(),
|
||||
}
|
||||
}
|
||||
|
||||
func containsString(s, substr string) bool {
|
||||
for i := 0; i <= len(s)-len(substr); i++ {
|
||||
if s[i:i+len(substr)] == substr {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func hasPrefix(s, prefix string) bool {
|
||||
if len(s) < len(prefix) {
|
||||
return false
|
||||
}
|
||||
return s[:len(prefix)] == prefix
|
||||
}
|
||||
@@ -1,14 +0,0 @@
|
||||
// Package reranking provides cross-encoder reranking for search results.
|
||||
package reranking
|
||||
|
||||
import (
|
||||
_ "embed"
|
||||
)
|
||||
|
||||
// Cross-encoder model and tokenizer files - embedded for all platforms
|
||||
//
|
||||
//go:embed assets/model.onnx
|
||||
var crossEncoderModelData []byte
|
||||
|
||||
//go:embed assets/tokenizer.json
|
||||
var crossEncoderTokenizerData []byte
|
||||
Binary file not shown.
File diff suppressed because it is too large
Load Diff
@@ -1,57 +0,0 @@
|
||||
{
|
||||
"added_tokens_decoder": {
|
||||
"0": {
|
||||
"content": "[PAD]",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"100": {
|
||||
"content": "[UNK]",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"101": {
|
||||
"content": "[CLS]",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"102": {
|
||||
"content": "[SEP]",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"103": {
|
||||
"content": "[MASK]",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
}
|
||||
},
|
||||
"clean_up_tokenization_spaces": true,
|
||||
"cls_token": "[CLS]",
|
||||
"do_basic_tokenize": true,
|
||||
"do_lower_case": true,
|
||||
"mask_token": "[MASK]",
|
||||
"model_max_length": 512,
|
||||
"never_split": null,
|
||||
"pad_token": "[PAD]",
|
||||
"sep_token": "[SEP]",
|
||||
"strip_accents": null,
|
||||
"tokenize_chinese_chars": true,
|
||||
"tokenizer_class": "BertTokenizer",
|
||||
"unk_token": "[UNK]"
|
||||
}
|
||||
@@ -1,380 +0,0 @@
|
||||
// Package reranking provides cross-encoder reranking for search results.
|
||||
// Uses MS-MARCO MiniLM L6 v2 cross-encoder model for relevance scoring.
|
||||
package reranking
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"math"
|
||||
"sort"
|
||||
"sync"
|
||||
|
||||
"github.com/rs/zerolog/log"
|
||||
"github.com/sugarme/tokenizer"
|
||||
"github.com/sugarme/tokenizer/pretrained"
|
||||
ort "github.com/yalue/onnxruntime_go"
|
||||
)
|
||||
|
||||
const (
|
||||
// ModelName is the human-readable name for the cross-encoder model
|
||||
ModelName = "ms-marco-MiniLM-L6-v2"
|
||||
// ModelVersion is the short version string for identification
|
||||
ModelVersion = "msmarco-v2"
|
||||
// MaxSequenceLength is the maximum combined query+document token length
|
||||
MaxSequenceLength = 512
|
||||
// DefaultCandidateLimit is the default number of candidates to rerank
|
||||
DefaultCandidateLimit = 100
|
||||
// DefaultResultLimit is the default number of results to return after reranking
|
||||
DefaultResultLimit = 10
|
||||
)
|
||||
|
||||
// Candidate represents a search result candidate for reranking.
|
||||
type Candidate struct {
|
||||
Metadata map[string]any
|
||||
RerankInfo map[string]float64
|
||||
ID string
|
||||
Content string
|
||||
Score float64
|
||||
}
|
||||
|
||||
// RerankResult represents a reranked search result.
|
||||
type RerankResult struct {
|
||||
Metadata map[string]any
|
||||
ID string
|
||||
Content string
|
||||
OriginalScore float64
|
||||
RerankScore float64
|
||||
CombinedScore float64
|
||||
OriginalRank int
|
||||
RerankRank int
|
||||
RankImprovement int
|
||||
}
|
||||
|
||||
// Service provides cross-encoder reranking functionality.
|
||||
type Service struct {
|
||||
tk *tokenizer.Tokenizer
|
||||
session *ort.DynamicAdvancedSession
|
||||
mu sync.Mutex
|
||||
|
||||
// Weight for combining scores: combined = alpha*rerank + (1-alpha)*original
|
||||
// Default 0.7 favors cross-encoder score
|
||||
Alpha float64
|
||||
}
|
||||
|
||||
// Config holds configuration for the reranking service.
|
||||
type Config struct {
|
||||
// Alpha is the weight for combining scores (0.0-1.0)
|
||||
// Higher values favor cross-encoder scores, lower values favor bi-encoder scores
|
||||
Alpha float64
|
||||
}
|
||||
|
||||
// DefaultConfig returns sensible defaults for reranking.
|
||||
func DefaultConfig() Config {
|
||||
return Config{
|
||||
Alpha: 0.7, // Favor cross-encoder by default
|
||||
}
|
||||
}
|
||||
|
||||
// NewService creates a new cross-encoder reranking service.
|
||||
// Note: ONNX runtime must be initialized before calling this (via embedding.NewService).
|
||||
func NewService(cfg Config) (*Service, error) {
|
||||
// Load tokenizer from embedded data
|
||||
tk, err := pretrained.FromReader(bytes.NewReader(crossEncoderTokenizerData))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("load cross-encoder tokenizer: %w", err)
|
||||
}
|
||||
|
||||
// Configure tokenizer for sequence classification (pairs)
|
||||
tk.WithTruncation(&tokenizer.TruncationParams{
|
||||
MaxLength: MaxSequenceLength,
|
||||
Strategy: tokenizer.LongestFirst,
|
||||
Stride: 0,
|
||||
})
|
||||
|
||||
// Cross-encoder outputs a single logit for relevance scoring
|
||||
inputNames := []string{"input_ids", "attention_mask", "token_type_ids"}
|
||||
outputNames := []string{"logits"}
|
||||
|
||||
session, err := ort.NewDynamicAdvancedSessionWithONNXData(
|
||||
crossEncoderModelData,
|
||||
inputNames,
|
||||
outputNames,
|
||||
nil,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create cross-encoder ONNX session: %w", err)
|
||||
}
|
||||
|
||||
alpha := cfg.Alpha
|
||||
if alpha <= 0 || alpha > 1 {
|
||||
alpha = 0.7
|
||||
}
|
||||
|
||||
return &Service{
|
||||
tk: tk,
|
||||
session: session,
|
||||
Alpha: alpha,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Rerank reranks candidates using the cross-encoder model.
|
||||
// Takes a query and list of candidates, returns reranked results.
|
||||
func (s *Service) Rerank(query string, candidates []Candidate, limit int) ([]RerankResult, error) {
|
||||
if len(candidates) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
if limit <= 0 {
|
||||
limit = DefaultResultLimit
|
||||
}
|
||||
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
|
||||
// Score all query-document pairs
|
||||
scores, err := s.scoreAll(query, candidates)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("score candidates: %w", err)
|
||||
}
|
||||
|
||||
// Build results with combined scores
|
||||
results := make([]RerankResult, len(candidates))
|
||||
for i, c := range candidates {
|
||||
// Normalize cross-encoder score to 0-1 range using sigmoid
|
||||
normalizedRerank := sigmoid(scores[i])
|
||||
|
||||
results[i] = RerankResult{
|
||||
ID: c.ID,
|
||||
Content: c.Content,
|
||||
OriginalScore: c.Score,
|
||||
RerankScore: normalizedRerank,
|
||||
CombinedScore: s.Alpha*normalizedRerank + (1-s.Alpha)*c.Score,
|
||||
Metadata: c.Metadata,
|
||||
OriginalRank: i + 1,
|
||||
}
|
||||
}
|
||||
|
||||
// Sort by combined score (descending)
|
||||
sort.Slice(results, func(i, j int) bool {
|
||||
return results[i].CombinedScore > results[j].CombinedScore
|
||||
})
|
||||
|
||||
// Assign rerank positions and calculate improvement
|
||||
for i := range results {
|
||||
results[i].RerankRank = i + 1
|
||||
results[i].RankImprovement = results[i].OriginalRank - results[i].RerankRank
|
||||
}
|
||||
|
||||
// Apply limit
|
||||
if len(results) > limit {
|
||||
results = results[:limit]
|
||||
}
|
||||
|
||||
log.Debug().
|
||||
Int("candidates", len(candidates)).
|
||||
Int("returned", len(results)).
|
||||
Float64("alpha", s.Alpha).
|
||||
Msg("Cross-encoder reranking completed")
|
||||
|
||||
return results, nil
|
||||
}
|
||||
|
||||
// RerankByScore reranks candidates and returns sorted by pure cross-encoder score.
|
||||
// Useful when you want to completely replace bi-encoder ranking.
|
||||
func (s *Service) RerankByScore(query string, candidates []Candidate, limit int) ([]RerankResult, error) {
|
||||
if len(candidates) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
if limit <= 0 {
|
||||
limit = DefaultResultLimit
|
||||
}
|
||||
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
|
||||
scores, err := s.scoreAll(query, candidates)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("score candidates: %w", err)
|
||||
}
|
||||
|
||||
results := make([]RerankResult, len(candidates))
|
||||
for i, c := range candidates {
|
||||
normalizedRerank := sigmoid(scores[i])
|
||||
results[i] = RerankResult{
|
||||
ID: c.ID,
|
||||
Content: c.Content,
|
||||
OriginalScore: c.Score,
|
||||
RerankScore: normalizedRerank,
|
||||
CombinedScore: normalizedRerank, // Use pure rerank score
|
||||
Metadata: c.Metadata,
|
||||
OriginalRank: i + 1,
|
||||
}
|
||||
}
|
||||
|
||||
// Sort by rerank score only
|
||||
sort.Slice(results, func(i, j int) bool {
|
||||
return results[i].RerankScore > results[j].RerankScore
|
||||
})
|
||||
|
||||
for i := range results {
|
||||
results[i].RerankRank = i + 1
|
||||
results[i].RankImprovement = results[i].OriginalRank - results[i].RerankRank
|
||||
}
|
||||
|
||||
if len(results) > limit {
|
||||
results = results[:limit]
|
||||
}
|
||||
|
||||
return results, nil
|
||||
}
|
||||
|
||||
// scoreAll scores all query-document pairs using the cross-encoder.
|
||||
// Returns raw logits (before sigmoid normalization).
|
||||
func (s *Service) scoreAll(query string, candidates []Candidate) ([]float64, error) {
|
||||
batchSize := len(candidates)
|
||||
|
||||
// Tokenize all query-document pairs
|
||||
pairs := make([]tokenizer.EncodeInput, batchSize)
|
||||
for i, c := range candidates {
|
||||
// Cross-encoder takes query and document as a pair
|
||||
pairs[i] = tokenizer.NewDualEncodeInput(
|
||||
tokenizer.NewRawInputSequence(query),
|
||||
tokenizer.NewRawInputSequence(c.Content),
|
||||
)
|
||||
}
|
||||
|
||||
encodings, err := s.tk.EncodeBatch(pairs, true)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("tokenize pairs: %w", err)
|
||||
}
|
||||
|
||||
// Find max sequence length
|
||||
seqLength := 0
|
||||
for _, enc := range encodings {
|
||||
if len(enc.Ids) > seqLength {
|
||||
seqLength = len(enc.Ids)
|
||||
}
|
||||
}
|
||||
if seqLength > MaxSequenceLength {
|
||||
seqLength = MaxSequenceLength
|
||||
}
|
||||
|
||||
inputShape := ort.NewShape(int64(batchSize), int64(seqLength))
|
||||
|
||||
// Create input tensors
|
||||
inputIdsData := make([]int64, batchSize*seqLength)
|
||||
attentionMaskData := make([]int64, batchSize*seqLength)
|
||||
tokenTypeIdsData := make([]int64, batchSize*seqLength)
|
||||
|
||||
for b := 0; b < batchSize; b++ {
|
||||
copyLen := len(encodings[b].Ids)
|
||||
if copyLen > seqLength {
|
||||
copyLen = seqLength
|
||||
}
|
||||
for i := 0; i < copyLen; i++ {
|
||||
inputIdsData[b*seqLength+i] = int64(encodings[b].Ids[i])
|
||||
}
|
||||
|
||||
copyLen = len(encodings[b].AttentionMask)
|
||||
if copyLen > seqLength {
|
||||
copyLen = seqLength
|
||||
}
|
||||
for i := 0; i < copyLen; i++ {
|
||||
attentionMaskData[b*seqLength+i] = int64(encodings[b].AttentionMask[i])
|
||||
}
|
||||
|
||||
copyLen = len(encodings[b].TypeIds)
|
||||
if copyLen > seqLength {
|
||||
copyLen = seqLength
|
||||
}
|
||||
for i := 0; i < copyLen; i++ {
|
||||
tokenTypeIdsData[b*seqLength+i] = int64(encodings[b].TypeIds[i])
|
||||
}
|
||||
}
|
||||
|
||||
inputIdsTensor, err := ort.NewTensor(inputShape, inputIdsData)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create input_ids tensor: %w", err)
|
||||
}
|
||||
defer func() { _ = inputIdsTensor.Destroy() }()
|
||||
|
||||
attentionMaskTensor, err := ort.NewTensor(inputShape, attentionMaskData)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create attention_mask tensor: %w", err)
|
||||
}
|
||||
defer func() { _ = attentionMaskTensor.Destroy() }()
|
||||
|
||||
tokenTypeIdsTensor, err := ort.NewTensor(inputShape, tokenTypeIdsData)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create token_type_ids tensor: %w", err)
|
||||
}
|
||||
defer func() { _ = tokenTypeIdsTensor.Destroy() }()
|
||||
|
||||
// Cross-encoder outputs [batch, 1] logits
|
||||
outputShape := ort.NewShape(int64(batchSize), 1)
|
||||
outputTensor, err := ort.NewEmptyTensor[float32](outputShape)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create output tensor: %w", err)
|
||||
}
|
||||
defer func() { _ = outputTensor.Destroy() }()
|
||||
|
||||
// Run inference
|
||||
inputTensors := []ort.Value{inputIdsTensor, attentionMaskTensor, tokenTypeIdsTensor}
|
||||
outputTensors := []ort.Value{outputTensor}
|
||||
|
||||
if err := s.session.Run(inputTensors, outputTensors); err != nil {
|
||||
return nil, fmt.Errorf("run cross-encoder inference: %w", err)
|
||||
}
|
||||
|
||||
// Extract scores
|
||||
flatOutput := outputTensor.GetData()
|
||||
scores := make([]float64, batchSize)
|
||||
for i := 0; i < batchSize; i++ {
|
||||
scores[i] = float64(flatOutput[i])
|
||||
}
|
||||
|
||||
return scores, nil
|
||||
}
|
||||
|
||||
// Score scores a single query-document pair.
|
||||
// Returns the raw cross-encoder logit and normalized score.
|
||||
func (s *Service) Score(query, document string) (rawScore, normalizedScore float64, err error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
|
||||
scores, err := s.scoreAll(query, []Candidate{{Content: document}})
|
||||
if err != nil {
|
||||
return 0, 0, err
|
||||
}
|
||||
|
||||
rawScore = scores[0]
|
||||
normalizedScore = sigmoid(rawScore)
|
||||
return rawScore, normalizedScore, nil
|
||||
}
|
||||
|
||||
// Close releases model resources.
|
||||
func (s *Service) Close() error {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
|
||||
if s.session != nil {
|
||||
if err := s.session.Destroy(); err != nil {
|
||||
return fmt.Errorf("destroy cross-encoder session: %w", err)
|
||||
}
|
||||
s.session = nil
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// sigmoid applies the sigmoid function to normalize scores to 0-1 range.
|
||||
func sigmoid(x float64) float64 {
|
||||
if x > 20 {
|
||||
return 1.0
|
||||
}
|
||||
if x < -20 {
|
||||
return 0.0
|
||||
}
|
||||
return 1.0 / (1.0 + math.Exp(-x))
|
||||
}
|
||||
@@ -1,448 +0,0 @@
|
||||
package reranking
|
||||
|
||||
import (
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"github.com/lukaszraczylo/claude-mnemonic/internal/embedding"
|
||||
)
|
||||
|
||||
// initONNX initializes the ONNX runtime via the embedding service.
|
||||
// Must be called before creating reranking service.
|
||||
func initONNX(t *testing.T) func() {
|
||||
t.Helper()
|
||||
embSvc, err := embedding.NewService()
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to initialize ONNX via embedding service: %v", err)
|
||||
}
|
||||
return func() {
|
||||
embSvc.Close()
|
||||
}
|
||||
}
|
||||
|
||||
// TestSigmoid tests the sigmoid normalization function.
|
||||
func TestSigmoid(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
input float64
|
||||
wantMin float64
|
||||
wantMax float64
|
||||
}{
|
||||
{"positive large", 10, 0.9999, 1.0},
|
||||
{"positive small", 1, 0.7, 0.8},
|
||||
{"zero", 0, 0.4999, 0.5001},
|
||||
{"negative small", -1, 0.2, 0.3},
|
||||
{"negative large", -10, 0, 0.0001},
|
||||
{"very positive", 25, 0.999999, 1.0},
|
||||
{"very negative", -25, 0, 0.000001},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := sigmoid(tt.input)
|
||||
if got < tt.wantMin || got > tt.wantMax {
|
||||
t.Errorf("sigmoid(%v) = %v, want in range [%v, %v]",
|
||||
tt.input, got, tt.wantMin, tt.wantMax)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestDefaultConfig tests the default configuration values.
|
||||
func TestDefaultConfig(t *testing.T) {
|
||||
cfg := DefaultConfig()
|
||||
|
||||
if cfg.Alpha < 0 || cfg.Alpha > 1 {
|
||||
t.Errorf("DefaultConfig().Alpha = %v, want in range [0, 1]", cfg.Alpha)
|
||||
}
|
||||
if cfg.Alpha != 0.7 {
|
||||
t.Errorf("DefaultConfig().Alpha = %v, want 0.7", cfg.Alpha)
|
||||
}
|
||||
}
|
||||
|
||||
// TestNewService tests service creation.
|
||||
func TestNewService(t *testing.T) {
|
||||
cleanup := initONNX(t)
|
||||
defer cleanup()
|
||||
|
||||
cfg := DefaultConfig()
|
||||
svc, err := NewService(cfg)
|
||||
if err != nil {
|
||||
t.Fatalf("NewService() error = %v", err)
|
||||
}
|
||||
defer svc.Close()
|
||||
|
||||
if svc == nil {
|
||||
t.Fatal("NewService() returned nil")
|
||||
}
|
||||
|
||||
if svc.Alpha != cfg.Alpha {
|
||||
t.Errorf("Service.Alpha = %v, want %v", svc.Alpha, cfg.Alpha)
|
||||
}
|
||||
}
|
||||
|
||||
// TestScore tests single pair scoring.
|
||||
func TestScore(t *testing.T) {
|
||||
cleanup := initONNX(t)
|
||||
defer cleanup()
|
||||
|
||||
cfg := DefaultConfig()
|
||||
svc, err := NewService(cfg)
|
||||
if err != nil {
|
||||
t.Fatalf("NewService() error = %v", err)
|
||||
}
|
||||
defer svc.Close()
|
||||
|
||||
query := "What is the capital of France?"
|
||||
relevant := "Paris is the capital and largest city of France."
|
||||
irrelevant := "Dogs are popular pets known for their loyalty."
|
||||
|
||||
// Score relevant document
|
||||
_, relevantNorm, err := svc.Score(query, relevant)
|
||||
if err != nil {
|
||||
t.Fatalf("Score(relevant) error = %v", err)
|
||||
}
|
||||
|
||||
// Score irrelevant document
|
||||
_, irrelevantNorm, err := svc.Score(query, irrelevant)
|
||||
if err != nil {
|
||||
t.Fatalf("Score(irrelevant) error = %v", err)
|
||||
}
|
||||
|
||||
// Relevant document should score higher
|
||||
if relevantNorm <= irrelevantNorm {
|
||||
t.Errorf("Expected relevant (%v) > irrelevant (%v)",
|
||||
relevantNorm, irrelevantNorm)
|
||||
}
|
||||
}
|
||||
|
||||
// TestRerank tests the reranking functionality.
|
||||
func TestRerank(t *testing.T) {
|
||||
cleanup := initONNX(t)
|
||||
defer cleanup()
|
||||
|
||||
cfg := DefaultConfig()
|
||||
svc, err := NewService(cfg)
|
||||
if err != nil {
|
||||
t.Fatalf("NewService() error = %v", err)
|
||||
}
|
||||
defer svc.Close()
|
||||
|
||||
query := "How to handle errors in Go?"
|
||||
candidates := []Candidate{
|
||||
{
|
||||
ID: "1",
|
||||
Content: "Python exception handling with try/except blocks.",
|
||||
Score: 0.8, // High bi-encoder score but irrelevant
|
||||
},
|
||||
{
|
||||
ID: "2",
|
||||
Content: "Go error handling uses explicit return values. Functions return error as the last value.",
|
||||
Score: 0.6, // Lower bi-encoder score but relevant
|
||||
},
|
||||
{
|
||||
ID: "3",
|
||||
Content: "JavaScript uses Promise.catch for async error handling.",
|
||||
Score: 0.7,
|
||||
},
|
||||
}
|
||||
|
||||
results, err := svc.Rerank(query, candidates, 3)
|
||||
if err != nil {
|
||||
t.Fatalf("Rerank() error = %v", err)
|
||||
}
|
||||
|
||||
if len(results) != 3 {
|
||||
t.Fatalf("Rerank() returned %d results, want 3", len(results))
|
||||
}
|
||||
|
||||
// The Go error handling document should rank higher after reranking
|
||||
var goRank int
|
||||
for i, r := range results {
|
||||
if r.ID == "2" {
|
||||
goRank = i + 1
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if goRank == 0 {
|
||||
t.Error("Go document not found in results")
|
||||
}
|
||||
|
||||
// Verify all results have required fields populated
|
||||
for i, r := range results {
|
||||
if r.ID == "" {
|
||||
t.Errorf("Result %d has empty ID", i)
|
||||
}
|
||||
if r.Content == "" {
|
||||
t.Errorf("Result %d has empty Content", i)
|
||||
}
|
||||
if r.RerankRank != i+1 {
|
||||
t.Errorf("Result %d has RerankRank %d, want %d", i, r.RerankRank, i+1)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestRerankEmpty tests reranking with empty input.
|
||||
func TestRerankEmpty(t *testing.T) {
|
||||
cleanup := initONNX(t)
|
||||
defer cleanup()
|
||||
|
||||
cfg := DefaultConfig()
|
||||
svc, err := NewService(cfg)
|
||||
if err != nil {
|
||||
t.Fatalf("NewService() error = %v", err)
|
||||
}
|
||||
defer svc.Close()
|
||||
|
||||
results, err := svc.Rerank("test query", nil, 10)
|
||||
if err != nil {
|
||||
t.Fatalf("Rerank(nil) error = %v", err)
|
||||
}
|
||||
|
||||
if results != nil {
|
||||
t.Errorf("Rerank(nil) = %v, want nil", results)
|
||||
}
|
||||
|
||||
results, err = svc.Rerank("test query", []Candidate{}, 10)
|
||||
if err != nil {
|
||||
t.Fatalf("Rerank([]) error = %v", err)
|
||||
}
|
||||
|
||||
if results != nil {
|
||||
t.Errorf("Rerank([]) = %v, want nil", results)
|
||||
}
|
||||
}
|
||||
|
||||
// TestRerankLimit tests that limit is respected.
|
||||
func TestRerankLimit(t *testing.T) {
|
||||
cleanup := initONNX(t)
|
||||
defer cleanup()
|
||||
|
||||
cfg := DefaultConfig()
|
||||
svc, err := NewService(cfg)
|
||||
if err != nil {
|
||||
t.Fatalf("NewService() error = %v", err)
|
||||
}
|
||||
defer svc.Close()
|
||||
|
||||
candidates := make([]Candidate, 20)
|
||||
for i := range candidates {
|
||||
candidates[i] = Candidate{
|
||||
ID: string(rune('A' + i)),
|
||||
Content: "Test document content for ranking.",
|
||||
Score: 0.5,
|
||||
}
|
||||
}
|
||||
|
||||
results, err := svc.Rerank("test query", candidates, 5)
|
||||
if err != nil {
|
||||
t.Fatalf("Rerank() error = %v", err)
|
||||
}
|
||||
|
||||
if len(results) != 5 {
|
||||
t.Errorf("Rerank() returned %d results, want 5", len(results))
|
||||
}
|
||||
}
|
||||
|
||||
// TestRerankByScore tests pure cross-encoder ranking.
|
||||
func TestRerankByScore(t *testing.T) {
|
||||
cleanup := initONNX(t)
|
||||
defer cleanup()
|
||||
|
||||
cfg := DefaultConfig()
|
||||
svc, err := NewService(cfg)
|
||||
if err != nil {
|
||||
t.Fatalf("NewService() error = %v", err)
|
||||
}
|
||||
defer svc.Close()
|
||||
|
||||
query := "machine learning algorithms"
|
||||
candidates := []Candidate{
|
||||
{
|
||||
ID: "1",
|
||||
Content: "Cooking recipes for Italian pasta dishes.",
|
||||
Score: 0.9, // High original score
|
||||
},
|
||||
{
|
||||
ID: "2",
|
||||
Content: "Neural networks are a type of machine learning algorithm.",
|
||||
Score: 0.3, // Low original score
|
||||
},
|
||||
}
|
||||
|
||||
results, err := svc.RerankByScore(query, candidates, 2)
|
||||
if err != nil {
|
||||
t.Fatalf("RerankByScore() error = %v", err)
|
||||
}
|
||||
|
||||
// Document 2 should rank first since it's about ML
|
||||
if results[0].ID != "2" {
|
||||
t.Errorf("Expected ML document to rank first, got %v", results[0].ID)
|
||||
}
|
||||
|
||||
// CombinedScore should equal RerankScore when using RerankByScore
|
||||
for _, r := range results {
|
||||
if r.CombinedScore != r.RerankScore {
|
||||
t.Errorf("RerankByScore: CombinedScore (%v) != RerankScore (%v)",
|
||||
r.CombinedScore, r.RerankScore)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestRankImprovement tests that rank improvement is calculated correctly.
|
||||
func TestRankImprovement(t *testing.T) {
|
||||
cleanup := initONNX(t)
|
||||
defer cleanup()
|
||||
|
||||
cfg := DefaultConfig()
|
||||
svc, err := NewService(cfg)
|
||||
if err != nil {
|
||||
t.Fatalf("NewService() error = %v", err)
|
||||
}
|
||||
defer svc.Close()
|
||||
|
||||
// Create candidates where we know the expected reranking
|
||||
candidates := []Candidate{
|
||||
{ID: "A", Content: "Unrelated content about weather forecasting.", Score: 0.9},
|
||||
{ID: "B", Content: "How to fix memory leaks in Go programs.", Score: 0.8},
|
||||
{ID: "C", Content: "More unrelated content about gardening tips.", Score: 0.7},
|
||||
}
|
||||
|
||||
results, err := svc.Rerank("debugging memory issues in Go", candidates, 3)
|
||||
if err != nil {
|
||||
t.Fatalf("Rerank() error = %v", err)
|
||||
}
|
||||
|
||||
for _, r := range results {
|
||||
// RankImprovement = OriginalRank - RerankRank
|
||||
// Positive means moved up, negative means moved down
|
||||
expectedImprovement := r.OriginalRank - r.RerankRank
|
||||
if r.RankImprovement != expectedImprovement {
|
||||
t.Errorf("ID %s: RankImprovement = %d, want %d (orig=%d, new=%d)",
|
||||
r.ID, r.RankImprovement, expectedImprovement,
|
||||
r.OriginalRank, r.RerankRank)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestConcurrentRerank tests concurrent reranking calls.
|
||||
func TestConcurrentRerank(t *testing.T) {
|
||||
cleanup := initONNX(t)
|
||||
defer cleanup()
|
||||
|
||||
cfg := DefaultConfig()
|
||||
svc, err := NewService(cfg)
|
||||
if err != nil {
|
||||
t.Fatalf("NewService() error = %v", err)
|
||||
}
|
||||
defer svc.Close()
|
||||
|
||||
candidates := []Candidate{
|
||||
{ID: "1", Content: "Test document one.", Score: 0.5},
|
||||
{ID: "2", Content: "Test document two.", Score: 0.5},
|
||||
}
|
||||
|
||||
var wg sync.WaitGroup
|
||||
errors := make(chan error, 10)
|
||||
|
||||
for i := 0; i < 10; i++ {
|
||||
wg.Add(1)
|
||||
go func(idx int) {
|
||||
defer wg.Done()
|
||||
_, err := svc.Rerank("concurrent test query", candidates, 2)
|
||||
if err != nil {
|
||||
errors <- err
|
||||
}
|
||||
}(i)
|
||||
}
|
||||
|
||||
wg.Wait()
|
||||
close(errors)
|
||||
|
||||
for err := range errors {
|
||||
t.Errorf("Concurrent Rerank error: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestClose tests service cleanup.
|
||||
func TestClose(t *testing.T) {
|
||||
cleanup := initONNX(t)
|
||||
defer cleanup()
|
||||
|
||||
cfg := DefaultConfig()
|
||||
svc, err := NewService(cfg)
|
||||
if err != nil {
|
||||
t.Fatalf("NewService() error = %v", err)
|
||||
}
|
||||
|
||||
err = svc.Close()
|
||||
if err != nil {
|
||||
t.Errorf("Close() error = %v", err)
|
||||
}
|
||||
|
||||
// Double close should not panic
|
||||
err = svc.Close()
|
||||
if err != nil {
|
||||
t.Errorf("Close() on closed service error = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestMetadataPreserved tests that metadata is preserved through reranking.
|
||||
func TestMetadataPreserved(t *testing.T) {
|
||||
cleanup := initONNX(t)
|
||||
defer cleanup()
|
||||
|
||||
cfg := DefaultConfig()
|
||||
svc, err := NewService(cfg)
|
||||
if err != nil {
|
||||
t.Fatalf("NewService() error = %v", err)
|
||||
}
|
||||
defer svc.Close()
|
||||
|
||||
candidates := []Candidate{
|
||||
{
|
||||
ID: "1",
|
||||
Content: "Test content.",
|
||||
Score: 0.5,
|
||||
Metadata: map[string]any{"custom": "value1", "num": 42},
|
||||
},
|
||||
{
|
||||
ID: "2",
|
||||
Content: "Another test.",
|
||||
Score: 0.5,
|
||||
Metadata: map[string]any{"custom": "value2"},
|
||||
},
|
||||
}
|
||||
|
||||
results, err := svc.Rerank("query", candidates, 2)
|
||||
if err != nil {
|
||||
t.Fatalf("Rerank() error = %v", err)
|
||||
}
|
||||
|
||||
for _, r := range results {
|
||||
if r.Metadata == nil {
|
||||
t.Errorf("Result %s has nil metadata", r.ID)
|
||||
continue
|
||||
}
|
||||
|
||||
// Find original candidate
|
||||
var original *Candidate
|
||||
for i := range candidates {
|
||||
if candidates[i].ID == r.ID {
|
||||
original = &candidates[i]
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if original == nil {
|
||||
t.Errorf("Could not find original for result %s", r.ID)
|
||||
continue
|
||||
}
|
||||
|
||||
// Check metadata preserved
|
||||
if original.Metadata["custom"] != r.Metadata["custom"] {
|
||||
t.Errorf("Metadata not preserved for %s", r.ID)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,168 +0,0 @@
|
||||
// Package scoring provides importance score calculation for observations.
|
||||
package scoring
|
||||
|
||||
import (
|
||||
"math"
|
||||
"time"
|
||||
|
||||
"github.com/lukaszraczylo/claude-mnemonic/pkg/models"
|
||||
)
|
||||
|
||||
// Calculator computes importance scores for observations.
|
||||
type Calculator struct {
|
||||
config *models.ScoringConfig
|
||||
}
|
||||
|
||||
// NewCalculator creates a new scoring calculator.
|
||||
// If config is nil, uses the default configuration.
|
||||
func NewCalculator(config *models.ScoringConfig) *Calculator {
|
||||
if config == nil {
|
||||
config = models.DefaultScoringConfig()
|
||||
}
|
||||
return &Calculator{config: config}
|
||||
}
|
||||
|
||||
// Calculate computes the importance score for an observation at the given time.
|
||||
//
|
||||
// The scoring formula:
|
||||
//
|
||||
// FinalScore = (BaseScore × TypeWeight × RecencyDecay) + FeedbackContrib + ConceptContrib + RetrievalContrib
|
||||
//
|
||||
// Where:
|
||||
// - BaseScore = 1.0
|
||||
// - TypeWeight = observation type multiplier (e.g., bugfix=1.3, change=0.9)
|
||||
// - RecencyDecay = 0.5^(age_days / half_life_days) - halves every 7 days by default
|
||||
// - FeedbackContrib = user_feedback × feedback_weight
|
||||
// - ConceptContrib = sum(concept_weights) × concept_weight_factor
|
||||
// - RetrievalContrib = log2(retrieval_count + 1) × 0.1 × retrieval_weight
|
||||
func (c *Calculator) Calculate(obs *models.Observation, now time.Time) float64 {
|
||||
// 1. Get base type weight
|
||||
typeWeight := models.TypeBaseScore(obs.Type)
|
||||
|
||||
// 2. Calculate recency decay: 0.5^(age_days / half_life_days)
|
||||
ageDays := now.Sub(time.UnixMilli(obs.CreatedAtEpoch)).Hours() / 24.0
|
||||
if ageDays < 0 {
|
||||
ageDays = 0 // Handle future timestamps gracefully
|
||||
}
|
||||
recencyDecay := math.Pow(0.5, ageDays/c.config.RecencyHalfLifeDays)
|
||||
|
||||
// Core score = 1.0 × type_weight × recency_decay
|
||||
coreScore := 1.0 * typeWeight * recencyDecay
|
||||
|
||||
// 3. User feedback contribution: feedback × weight
|
||||
feedbackContrib := float64(obs.UserFeedback) * c.config.FeedbackWeight
|
||||
|
||||
// 4. Concept boost contribution: sum of matching concept weights × factor
|
||||
conceptBoost := 0.0
|
||||
for _, concept := range obs.Concepts {
|
||||
if weight, ok := c.config.ConceptWeights[concept]; ok {
|
||||
conceptBoost += weight
|
||||
}
|
||||
}
|
||||
conceptContrib := conceptBoost * c.config.ConceptWeight
|
||||
|
||||
// 5. Retrieval boost: log2(count + 1) × 0.1 × weight (diminishing returns)
|
||||
retrievalContrib := 0.0
|
||||
if obs.RetrievalCount > 0 {
|
||||
// log2(count + 1) gives diminishing returns: 1→1, 3→2, 7→3, 15→4, etc.
|
||||
retrievalBoost := math.Log2(float64(obs.RetrievalCount)+1) * 0.1
|
||||
retrievalContrib = retrievalBoost * c.config.RetrievalWeight
|
||||
}
|
||||
|
||||
// Final score with minimum threshold
|
||||
finalScore := coreScore + feedbackContrib + conceptContrib + retrievalContrib
|
||||
if finalScore < c.config.MinScore {
|
||||
finalScore = c.config.MinScore
|
||||
}
|
||||
|
||||
return finalScore
|
||||
}
|
||||
|
||||
// CalculateComponents returns the individual components of the importance score.
|
||||
// Useful for debugging and explaining scores to users.
|
||||
func (c *Calculator) CalculateComponents(obs *models.Observation, now time.Time) ScoreComponents {
|
||||
typeWeight := models.TypeBaseScore(obs.Type)
|
||||
|
||||
ageDays := now.Sub(time.UnixMilli(obs.CreatedAtEpoch)).Hours() / 24.0
|
||||
if ageDays < 0 {
|
||||
ageDays = 0
|
||||
}
|
||||
recencyDecay := math.Pow(0.5, ageDays/c.config.RecencyHalfLifeDays)
|
||||
|
||||
coreScore := 1.0 * typeWeight * recencyDecay
|
||||
feedbackContrib := float64(obs.UserFeedback) * c.config.FeedbackWeight
|
||||
|
||||
conceptBoost := 0.0
|
||||
for _, concept := range obs.Concepts {
|
||||
if weight, ok := c.config.ConceptWeights[concept]; ok {
|
||||
conceptBoost += weight
|
||||
}
|
||||
}
|
||||
conceptContrib := conceptBoost * c.config.ConceptWeight
|
||||
|
||||
retrievalContrib := 0.0
|
||||
if obs.RetrievalCount > 0 {
|
||||
retrievalBoost := math.Log2(float64(obs.RetrievalCount)+1) * 0.1
|
||||
retrievalContrib = retrievalBoost * c.config.RetrievalWeight
|
||||
}
|
||||
|
||||
finalScore := coreScore + feedbackContrib + conceptContrib + retrievalContrib
|
||||
if finalScore < c.config.MinScore {
|
||||
finalScore = c.config.MinScore
|
||||
}
|
||||
|
||||
return ScoreComponents{
|
||||
TypeWeight: typeWeight,
|
||||
RecencyDecay: recencyDecay,
|
||||
CoreScore: coreScore,
|
||||
FeedbackContrib: feedbackContrib,
|
||||
ConceptContrib: conceptContrib,
|
||||
RetrievalContrib: retrievalContrib,
|
||||
FinalScore: finalScore,
|
||||
AgeDays: ageDays,
|
||||
}
|
||||
}
|
||||
|
||||
// ScoreComponents contains the breakdown of an importance score calculation.
|
||||
type ScoreComponents struct {
|
||||
TypeWeight float64 `json:"type_weight"`
|
||||
RecencyDecay float64 `json:"recency_decay"`
|
||||
CoreScore float64 `json:"core_score"`
|
||||
FeedbackContrib float64 `json:"feedback_contrib"`
|
||||
ConceptContrib float64 `json:"concept_contrib"`
|
||||
RetrievalContrib float64 `json:"retrieval_contrib"`
|
||||
FinalScore float64 `json:"final_score"`
|
||||
AgeDays float64 `json:"age_days"`
|
||||
}
|
||||
|
||||
// BatchCalculate computes scores for multiple observations.
|
||||
// Returns a map of observation ID to calculated score.
|
||||
func (c *Calculator) BatchCalculate(observations []*models.Observation, now time.Time) map[int64]float64 {
|
||||
scores := make(map[int64]float64, len(observations))
|
||||
for _, obs := range observations {
|
||||
scores[obs.ID] = c.Calculate(obs, now)
|
||||
}
|
||||
return scores
|
||||
}
|
||||
|
||||
// RecalculateThreshold returns the minimum duration before an observation
|
||||
// should have its score recalculated. This prevents excessive recalculation
|
||||
// while ensuring scores stay reasonably fresh.
|
||||
func (c *Calculator) RecalculateThreshold() time.Duration {
|
||||
// Recalculate at most every 6 hours
|
||||
// This balances freshness with performance
|
||||
return 6 * time.Hour
|
||||
}
|
||||
|
||||
// UpdateConfig updates the calculator's scoring configuration.
|
||||
// This allows runtime tuning of scoring parameters.
|
||||
func (c *Calculator) UpdateConfig(config *models.ScoringConfig) {
|
||||
if config != nil {
|
||||
c.config = config
|
||||
}
|
||||
}
|
||||
|
||||
// GetConfig returns the current scoring configuration.
|
||||
func (c *Calculator) GetConfig() *models.ScoringConfig {
|
||||
return c.config
|
||||
}
|
||||
@@ -1,638 +0,0 @@
|
||||
// Package scoring provides importance score calculation for observations.
|
||||
package scoring
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/lukaszraczylo/claude-mnemonic/pkg/models"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"github.com/stretchr/testify/suite"
|
||||
)
|
||||
|
||||
// CalculatorSuite is a test suite for the Calculator.
|
||||
type CalculatorSuite struct {
|
||||
suite.Suite
|
||||
calc *Calculator
|
||||
config *models.ScoringConfig
|
||||
now time.Time
|
||||
}
|
||||
|
||||
func (s *CalculatorSuite) SetupTest() {
|
||||
s.config = models.DefaultScoringConfig()
|
||||
s.calc = NewCalculator(s.config)
|
||||
s.now = time.Date(2025, 1, 15, 12, 0, 0, 0, time.UTC)
|
||||
}
|
||||
|
||||
func TestCalculatorSuite(t *testing.T) {
|
||||
suite.Run(t, new(CalculatorSuite))
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
// GOOD SCENARIOS - Expected normal operations
|
||||
// =============================================================================
|
||||
|
||||
func (s *CalculatorSuite) TestCalculate_GoodScenarios_NewObservation() {
|
||||
// A brand new observation should have score close to type weight
|
||||
obs := &models.Observation{
|
||||
ID: 1,
|
||||
Type: models.ObsTypeBugfix,
|
||||
CreatedAtEpoch: s.now.UnixMilli(),
|
||||
}
|
||||
|
||||
score := s.calc.Calculate(obs, s.now)
|
||||
|
||||
// Expected: 1.0 × 1.3 (bugfix weight) × 1.0 (no decay) = 1.3
|
||||
s.InDelta(1.3, score, 0.01, "new bugfix should score ~1.3")
|
||||
}
|
||||
|
||||
func (s *CalculatorSuite) TestCalculate_GoodScenarios_OneWeekOld() {
|
||||
// One week old observation should have half the recency score
|
||||
obs := &models.Observation{
|
||||
ID: 1,
|
||||
Type: models.ObsTypeDiscovery,
|
||||
CreatedAtEpoch: s.now.Add(-7 * 24 * time.Hour).UnixMilli(),
|
||||
}
|
||||
|
||||
score := s.calc.Calculate(obs, s.now)
|
||||
|
||||
// Expected: 1.0 × 1.1 (discovery) × 0.5 (7 days half-life) = 0.55
|
||||
s.InDelta(0.55, score, 0.05, "7-day old discovery should score ~0.55")
|
||||
}
|
||||
|
||||
func (s *CalculatorSuite) TestCalculate_GoodScenarios_TwoWeeksOld() {
|
||||
// Two weeks old should have 1/4 recency score
|
||||
obs := &models.Observation{
|
||||
ID: 1,
|
||||
Type: models.ObsTypeFeature,
|
||||
CreatedAtEpoch: s.now.Add(-14 * 24 * time.Hour).UnixMilli(),
|
||||
}
|
||||
|
||||
score := s.calc.Calculate(obs, s.now)
|
||||
|
||||
// Expected: 1.0 × 1.2 (feature) × 0.25 (14 days = 2 half-lives) = 0.30
|
||||
s.InDelta(0.30, score, 0.05, "14-day old feature should score ~0.30")
|
||||
}
|
||||
|
||||
func (s *CalculatorSuite) TestCalculate_GoodScenarios_PositiveFeedback() {
|
||||
// Positive feedback should boost score
|
||||
obs := &models.Observation{
|
||||
ID: 1,
|
||||
Type: models.ObsTypeChange,
|
||||
CreatedAtEpoch: s.now.UnixMilli(),
|
||||
UserFeedback: 1, // thumbs up
|
||||
}
|
||||
|
||||
score := s.calc.Calculate(obs, s.now)
|
||||
|
||||
// Expected: (1.0 × 0.9) + 0.30 (feedback) = 1.20
|
||||
s.InDelta(1.20, score, 0.01, "thumbs up should boost score by 0.30")
|
||||
}
|
||||
|
||||
func (s *CalculatorSuite) TestCalculate_GoodScenarios_NegativeFeedback() {
|
||||
// Negative feedback should reduce score
|
||||
obs := &models.Observation{
|
||||
ID: 1,
|
||||
Type: models.ObsTypeChange,
|
||||
CreatedAtEpoch: s.now.UnixMilli(),
|
||||
UserFeedback: -1, // thumbs down
|
||||
}
|
||||
|
||||
score := s.calc.Calculate(obs, s.now)
|
||||
|
||||
// Expected: (1.0 × 0.9) - 0.30 (feedback) = 0.60
|
||||
s.InDelta(0.60, score, 0.01, "thumbs down should reduce score by 0.30")
|
||||
}
|
||||
|
||||
func (s *CalculatorSuite) TestCalculate_GoodScenarios_WithConcepts() {
|
||||
// Observation with valuable concepts should get boost
|
||||
obs := &models.Observation{
|
||||
ID: 1,
|
||||
Type: models.ObsTypeBugfix,
|
||||
CreatedAtEpoch: s.now.UnixMilli(),
|
||||
Concepts: []string{"security", "gotcha"},
|
||||
}
|
||||
|
||||
score := s.calc.Calculate(obs, s.now)
|
||||
|
||||
// Concept boost: (0.30 + 0.25) × 0.20 = 0.11
|
||||
// Expected: 1.3 + 0.11 = 1.41
|
||||
s.InDelta(1.41, score, 0.05, "security+gotcha concepts should boost score")
|
||||
}
|
||||
|
||||
func (s *CalculatorSuite) TestCalculate_GoodScenarios_WithRetrievals() {
|
||||
// Popular observations should get retrieval boost
|
||||
obs := &models.Observation{
|
||||
ID: 1,
|
||||
Type: models.ObsTypeDiscovery,
|
||||
CreatedAtEpoch: s.now.UnixMilli(),
|
||||
RetrievalCount: 7, // log2(8) = 3
|
||||
}
|
||||
|
||||
score := s.calc.Calculate(obs, s.now)
|
||||
|
||||
// Retrieval boost: log2(7+1) × 0.1 × 0.15 = 3 × 0.1 × 0.15 = 0.045
|
||||
// Expected: 1.1 + 0.045 ≈ 1.145
|
||||
s.InDelta(1.145, score, 0.05, "7 retrievals should add small boost")
|
||||
}
|
||||
|
||||
func (s *CalculatorSuite) TestCalculate_GoodScenarios_CombinedFactors() {
|
||||
// Test with all factors combined
|
||||
obs := &models.Observation{
|
||||
ID: 1,
|
||||
Type: models.ObsTypeBugfix,
|
||||
CreatedAtEpoch: s.now.Add(-7 * 24 * time.Hour).UnixMilli(), // 7 days old
|
||||
UserFeedback: 1,
|
||||
Concepts: []string{"security"},
|
||||
RetrievalCount: 3,
|
||||
}
|
||||
|
||||
score := s.calc.Calculate(obs, s.now)
|
||||
|
||||
// Core: 1.0 × 1.3 × 0.5 = 0.65
|
||||
// Feedback: 0.30
|
||||
// Concept: 0.30 × 0.20 = 0.06
|
||||
// Retrieval: log2(4) × 0.1 × 0.15 = 2 × 0.1 × 0.15 = 0.03
|
||||
// Total ≈ 1.04
|
||||
s.InDelta(1.04, score, 0.1, "combined factors should result in ~1.04")
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
// WORSE SCENARIOS - Degraded but acceptable operations
|
||||
// =============================================================================
|
||||
|
||||
func (s *CalculatorSuite) TestCalculate_WorseScenarios_VeryOldObservation() {
|
||||
// Very old observation should have low but non-zero score
|
||||
obs := &models.Observation{
|
||||
ID: 1,
|
||||
Type: models.ObsTypeChange,
|
||||
CreatedAtEpoch: s.now.Add(-90 * 24 * time.Hour).UnixMilli(), // 90 days old
|
||||
}
|
||||
|
||||
score := s.calc.Calculate(obs, s.now)
|
||||
|
||||
// 90 days = ~12.86 half-lives → decay ≈ 0.00014
|
||||
// Core: 1.0 × 0.9 × 0.00014 = 0.000126
|
||||
// But minimum score is 0.01
|
||||
s.GreaterOrEqual(score, 0.01, "very old observation should still meet minimum")
|
||||
s.Less(score, 0.1, "very old observation should be low scoring")
|
||||
}
|
||||
|
||||
func (s *CalculatorSuite) TestCalculate_WorseScenarios_NegativeFeedbackOld() {
|
||||
// Old observation with negative feedback should still have minimum score
|
||||
obs := &models.Observation{
|
||||
ID: 1,
|
||||
Type: models.ObsTypeChange,
|
||||
CreatedAtEpoch: s.now.Add(-60 * 24 * time.Hour).UnixMilli(),
|
||||
UserFeedback: -1,
|
||||
}
|
||||
|
||||
score := s.calc.Calculate(obs, s.now)
|
||||
|
||||
s.GreaterOrEqual(score, s.config.MinScore, "should never go below minimum score")
|
||||
}
|
||||
|
||||
func (s *CalculatorSuite) TestCalculate_WorseScenarios_UnknownConcepts() {
|
||||
// Unknown concepts should not affect score negatively
|
||||
obs := &models.Observation{
|
||||
ID: 1,
|
||||
Type: models.ObsTypeDiscovery,
|
||||
CreatedAtEpoch: s.now.UnixMilli(),
|
||||
Concepts: []string{"unknown-concept", "another-unknown"},
|
||||
}
|
||||
|
||||
score := s.calc.Calculate(obs, s.now)
|
||||
|
||||
// Should just be the base score without concept boost
|
||||
s.InDelta(1.1, score, 0.01, "unknown concepts should not affect score")
|
||||
}
|
||||
|
||||
func (s *CalculatorSuite) TestCalculate_WorseScenarios_MixedConcepts() {
|
||||
// Mix of known and unknown concepts
|
||||
obs := &models.Observation{
|
||||
ID: 1,
|
||||
Type: models.ObsTypeDiscovery,
|
||||
CreatedAtEpoch: s.now.UnixMilli(),
|
||||
Concepts: []string{"security", "unknown-concept"},
|
||||
}
|
||||
|
||||
score := s.calc.Calculate(obs, s.now)
|
||||
|
||||
// Only security should contribute
|
||||
// Expected: 1.1 + (0.30 × 0.20) = 1.16
|
||||
s.InDelta(1.16, score, 0.05, "only known concepts should boost score")
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
// BAD SCENARIOS - Edge cases and error conditions
|
||||
// =============================================================================
|
||||
|
||||
func (s *CalculatorSuite) TestCalculate_BadScenarios_FutureTimestamp() {
|
||||
// Observation created in the future (clock skew)
|
||||
obs := &models.Observation{
|
||||
ID: 1,
|
||||
Type: models.ObsTypeBugfix,
|
||||
CreatedAtEpoch: s.now.Add(24 * time.Hour).UnixMilli(), // 1 day in future
|
||||
}
|
||||
|
||||
score := s.calc.Calculate(obs, s.now)
|
||||
|
||||
// Should handle gracefully - age should be 0
|
||||
s.InDelta(1.3, score, 0.01, "future timestamp should be treated as now")
|
||||
}
|
||||
|
||||
func (s *CalculatorSuite) TestCalculate_BadScenarios_ZeroEpoch() {
|
||||
// Missing creation timestamp
|
||||
obs := &models.Observation{
|
||||
ID: 1,
|
||||
Type: models.ObsTypeDiscovery,
|
||||
CreatedAtEpoch: 0, // Missing timestamp
|
||||
}
|
||||
|
||||
score := s.calc.Calculate(obs, s.now)
|
||||
|
||||
// This will be treated as very old (1970)
|
||||
s.GreaterOrEqual(score, s.config.MinScore, "should still meet minimum")
|
||||
}
|
||||
|
||||
func (s *CalculatorSuite) TestCalculate_BadScenarios_EmptyObservation() {
|
||||
// Minimal observation with defaults
|
||||
obs := &models.Observation{
|
||||
ID: 1,
|
||||
Type: "", // Empty type
|
||||
CreatedAtEpoch: s.now.UnixMilli(),
|
||||
}
|
||||
|
||||
score := s.calc.Calculate(obs, s.now)
|
||||
|
||||
// Unknown type should default to 1.0 weight
|
||||
s.InDelta(1.0, score, 0.01, "empty type should use default weight 1.0")
|
||||
}
|
||||
|
||||
func (s *CalculatorSuite) TestCalculate_BadScenarios_ExtremeRetrievalCount() {
|
||||
// Very high retrieval count
|
||||
obs := &models.Observation{
|
||||
ID: 1,
|
||||
Type: models.ObsTypeDiscovery,
|
||||
CreatedAtEpoch: s.now.UnixMilli(),
|
||||
RetrievalCount: 1000000, // Extreme value
|
||||
}
|
||||
|
||||
score := s.calc.Calculate(obs, s.now)
|
||||
|
||||
// log2(1000001) ≈ 19.93, so boost = 19.93 × 0.1 × 0.15 ≈ 0.30
|
||||
// Score should be reasonable, not exploding
|
||||
s.Less(score, 2.0, "extreme retrieval count should not explode score")
|
||||
}
|
||||
|
||||
func (s *CalculatorSuite) TestCalculate_BadScenarios_NegativeRetrievalCount() {
|
||||
// Negative retrieval count (should not happen but test defensively)
|
||||
obs := &models.Observation{
|
||||
ID: 1,
|
||||
Type: models.ObsTypeDiscovery,
|
||||
CreatedAtEpoch: s.now.UnixMilli(),
|
||||
RetrievalCount: -5,
|
||||
}
|
||||
|
||||
score := s.calc.Calculate(obs, s.now)
|
||||
|
||||
// Should not panic and should give base score
|
||||
s.InDelta(1.1, score, 0.01, "negative retrieval should be ignored")
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
// EDGE CASES - Boundary conditions
|
||||
// =============================================================================
|
||||
|
||||
func (s *CalculatorSuite) TestCalculate_EdgeCases_ExactlyOneHalfLife() {
|
||||
obs := &models.Observation{
|
||||
ID: 1,
|
||||
Type: models.ObsTypeChange, // 0.9 weight
|
||||
CreatedAtEpoch: s.now.Add(-7 * 24 * time.Hour).UnixMilli(),
|
||||
}
|
||||
|
||||
score := s.calc.Calculate(obs, s.now)
|
||||
s.InDelta(0.45, score, 0.01, "exactly 7 days should give 0.5 decay")
|
||||
}
|
||||
|
||||
func (s *CalculatorSuite) TestCalculate_EdgeCases_ExactlyTwoHalfLives() {
|
||||
obs := &models.Observation{
|
||||
ID: 1,
|
||||
Type: models.ObsTypeChange,
|
||||
CreatedAtEpoch: s.now.Add(-14 * 24 * time.Hour).UnixMilli(),
|
||||
}
|
||||
|
||||
score := s.calc.Calculate(obs, s.now)
|
||||
s.InDelta(0.225, score, 0.01, "exactly 14 days should give 0.25 decay")
|
||||
}
|
||||
|
||||
func (s *CalculatorSuite) TestCalculate_EdgeCases_AllTypeWeights() {
|
||||
types := []struct {
|
||||
t models.ObservationType
|
||||
weight float64
|
||||
}{
|
||||
{models.ObsTypeBugfix, 1.3},
|
||||
{models.ObsTypeFeature, 1.2},
|
||||
{models.ObsTypeDiscovery, 1.1},
|
||||
{models.ObsTypeDecision, 1.1},
|
||||
{models.ObsTypeRefactor, 1.0},
|
||||
{models.ObsTypeChange, 0.9},
|
||||
}
|
||||
|
||||
for _, tt := range types {
|
||||
s.Run(string(tt.t), func() {
|
||||
obs := &models.Observation{
|
||||
ID: 1,
|
||||
Type: tt.t,
|
||||
CreatedAtEpoch: s.now.UnixMilli(),
|
||||
}
|
||||
score := s.calc.Calculate(obs, s.now)
|
||||
s.InDelta(tt.weight, score, 0.01)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func (s *CalculatorSuite) TestCalculate_EdgeCases_MinimumScoreEnforced() {
|
||||
// Create worst case scenario
|
||||
obs := &models.Observation{
|
||||
ID: 1,
|
||||
Type: models.ObsTypeChange, // Lowest weight 0.9
|
||||
CreatedAtEpoch: time.Date(2000, 1, 1, 0, 0, 0, 0, time.UTC).UnixMilli(), // Very old
|
||||
UserFeedback: -1, // Negative feedback
|
||||
}
|
||||
|
||||
score := s.calc.Calculate(obs, s.now)
|
||||
|
||||
s.Equal(s.config.MinScore, score, "should be exactly minimum score")
|
||||
}
|
||||
|
||||
func (s *CalculatorSuite) TestCalculate_EdgeCases_AllConceptsMaxWeight() {
|
||||
// Observation with all high-value concepts
|
||||
obs := &models.Observation{
|
||||
ID: 1,
|
||||
Type: models.ObsTypeBugfix,
|
||||
CreatedAtEpoch: s.now.UnixMilli(),
|
||||
Concepts: []string{"security", "gotcha", "best-practice", "anti-pattern"},
|
||||
}
|
||||
|
||||
score := s.calc.Calculate(obs, s.now)
|
||||
|
||||
// security=0.30, gotcha=0.25, best-practice=0.20, anti-pattern=0.20 = 0.95
|
||||
// Concept contrib: 0.95 × 0.20 = 0.19
|
||||
// Total: 1.3 + 0.19 = 1.49
|
||||
s.InDelta(1.49, score, 0.05, "all high-value concepts should boost significantly")
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
// CALCULATE COMPONENTS TESTS
|
||||
// =============================================================================
|
||||
|
||||
func (s *CalculatorSuite) TestCalculateComponents_ReturnsAllComponents() {
|
||||
obs := &models.Observation{
|
||||
ID: 1,
|
||||
Type: models.ObsTypeBugfix,
|
||||
CreatedAtEpoch: s.now.Add(-7 * 24 * time.Hour).UnixMilli(),
|
||||
UserFeedback: 1,
|
||||
Concepts: []string{"security"},
|
||||
RetrievalCount: 7,
|
||||
}
|
||||
|
||||
components := s.calc.CalculateComponents(obs, s.now)
|
||||
|
||||
s.InDelta(1.3, components.TypeWeight, 0.01)
|
||||
s.InDelta(0.5, components.RecencyDecay, 0.01)
|
||||
s.InDelta(0.65, components.CoreScore, 0.05)
|
||||
s.InDelta(0.30, components.FeedbackContrib, 0.01)
|
||||
s.InDelta(0.06, components.ConceptContrib, 0.02)
|
||||
s.Greater(components.RetrievalContrib, 0.0)
|
||||
s.InDelta(7.0, components.AgeDays, 0.1)
|
||||
s.Greater(components.FinalScore, 0.0)
|
||||
}
|
||||
|
||||
func (s *CalculatorSuite) TestCalculateComponents_MatchesCalculate() {
|
||||
obs := &models.Observation{
|
||||
ID: 1,
|
||||
Type: models.ObsTypeFeature,
|
||||
CreatedAtEpoch: s.now.Add(-3 * 24 * time.Hour).UnixMilli(),
|
||||
UserFeedback: -1,
|
||||
Concepts: []string{"performance", "architecture"},
|
||||
RetrievalCount: 15,
|
||||
}
|
||||
|
||||
score := s.calc.Calculate(obs, s.now)
|
||||
components := s.calc.CalculateComponents(obs, s.now)
|
||||
|
||||
s.InDelta(score, components.FinalScore, 0.001, "Calculate and CalculateComponents should match")
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
// BATCH CALCULATE TESTS
|
||||
// =============================================================================
|
||||
|
||||
func (s *CalculatorSuite) TestBatchCalculate_Empty() {
|
||||
scores := s.calc.BatchCalculate(nil, s.now)
|
||||
s.Empty(scores)
|
||||
|
||||
scores = s.calc.BatchCalculate([]*models.Observation{}, s.now)
|
||||
s.Empty(scores)
|
||||
}
|
||||
|
||||
func (s *CalculatorSuite) TestBatchCalculate_Multiple() {
|
||||
obs := []*models.Observation{
|
||||
{ID: 1, Type: models.ObsTypeBugfix, CreatedAtEpoch: s.now.UnixMilli()},
|
||||
{ID: 2, Type: models.ObsTypeFeature, CreatedAtEpoch: s.now.Add(-7 * 24 * time.Hour).UnixMilli()},
|
||||
{ID: 3, Type: models.ObsTypeChange, CreatedAtEpoch: s.now.Add(-14 * 24 * time.Hour).UnixMilli()},
|
||||
}
|
||||
|
||||
scores := s.calc.BatchCalculate(obs, s.now)
|
||||
|
||||
s.Len(scores, 3)
|
||||
s.Contains(scores, int64(1))
|
||||
s.Contains(scores, int64(2))
|
||||
s.Contains(scores, int64(3))
|
||||
|
||||
s.InDelta(1.3, scores[1], 0.01) // New bugfix
|
||||
s.InDelta(0.6, scores[2], 0.1) // 7-day feature
|
||||
s.InDelta(0.225, scores[3], 0.05) // 14-day change
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
// CONFIGURATION TESTS
|
||||
// =============================================================================
|
||||
|
||||
func (s *CalculatorSuite) TestNewCalculator_NilConfig() {
|
||||
calc := NewCalculator(nil)
|
||||
s.NotNil(calc)
|
||||
s.NotNil(calc.config)
|
||||
s.Equal(7.0, calc.config.RecencyHalfLifeDays)
|
||||
}
|
||||
|
||||
func (s *CalculatorSuite) TestUpdateConfig() {
|
||||
newConfig := &models.ScoringConfig{
|
||||
RecencyHalfLifeDays: 14.0, // Changed from 7
|
||||
FeedbackWeight: 0.50,
|
||||
ConceptWeight: 0.10,
|
||||
RetrievalWeight: 0.05,
|
||||
MinScore: 0.001,
|
||||
ConceptWeights: map[string]float64{"test": 0.5},
|
||||
}
|
||||
|
||||
s.calc.UpdateConfig(newConfig)
|
||||
|
||||
obs := &models.Observation{
|
||||
ID: 1,
|
||||
Type: models.ObsTypeChange,
|
||||
CreatedAtEpoch: s.now.Add(-14 * 24 * time.Hour).UnixMilli(),
|
||||
}
|
||||
|
||||
score := s.calc.Calculate(obs, s.now)
|
||||
|
||||
// With 14-day half-life, 14 days = exactly one half-life
|
||||
// Expected: 1.0 × 0.9 × 0.5 = 0.45
|
||||
s.InDelta(0.45, score, 0.01)
|
||||
}
|
||||
|
||||
func (s *CalculatorSuite) TestUpdateConfig_NilIgnored() {
|
||||
originalConfig := s.calc.GetConfig()
|
||||
s.calc.UpdateConfig(nil)
|
||||
s.Equal(originalConfig, s.calc.GetConfig())
|
||||
}
|
||||
|
||||
func (s *CalculatorSuite) TestGetConfig() {
|
||||
config := s.calc.GetConfig()
|
||||
s.NotNil(config)
|
||||
s.Equal(7.0, config.RecencyHalfLifeDays)
|
||||
}
|
||||
|
||||
func (s *CalculatorSuite) TestRecalculateThreshold() {
|
||||
threshold := s.calc.RecalculateThreshold()
|
||||
s.Equal(6*time.Hour, threshold)
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
// STANDALONE TESTS (non-suite)
|
||||
// =============================================================================
|
||||
|
||||
func TestNewCalculator_DefaultConfig(t *testing.T) {
|
||||
calc := NewCalculator(nil)
|
||||
require.NotNil(t, calc)
|
||||
assert.Equal(t, 7.0, calc.config.RecencyHalfLifeDays)
|
||||
assert.Equal(t, 0.30, calc.config.FeedbackWeight)
|
||||
assert.Equal(t, 0.01, calc.config.MinScore)
|
||||
}
|
||||
|
||||
func TestCalculator_ConcurrentAccess(t *testing.T) {
|
||||
calc := NewCalculator(nil)
|
||||
now := time.Now()
|
||||
|
||||
// Test that calculator is safe for concurrent reads
|
||||
done := make(chan bool, 10)
|
||||
for i := 0; i < 10; i++ {
|
||||
go func(id int64) {
|
||||
obs := &models.Observation{
|
||||
ID: id,
|
||||
Type: models.ObsTypeBugfix,
|
||||
CreatedAtEpoch: now.UnixMilli(),
|
||||
}
|
||||
score := calc.Calculate(obs, now)
|
||||
assert.Greater(t, score, 0.0)
|
||||
done <- true
|
||||
}(int64(i))
|
||||
}
|
||||
|
||||
for i := 0; i < 10; i++ {
|
||||
<-done
|
||||
}
|
||||
}
|
||||
|
||||
func TestCalculator_DecayPrecision(t *testing.T) {
|
||||
calc := NewCalculator(nil)
|
||||
now := time.Now()
|
||||
|
||||
// Test that decay is mathematically correct
|
||||
testCases := []struct {
|
||||
days int
|
||||
expectedDecay float64
|
||||
}{
|
||||
{0, 1.0},
|
||||
{7, 0.5},
|
||||
{14, 0.25},
|
||||
{21, 0.125},
|
||||
{28, 0.0625},
|
||||
}
|
||||
|
||||
for _, tc := range testCases {
|
||||
t.Run(string(rune('0'+tc.days/7))+"_half_lives", func(t *testing.T) {
|
||||
obs := &models.Observation{
|
||||
ID: 1,
|
||||
Type: models.ObsTypeRefactor, // 1.0 weight
|
||||
CreatedAtEpoch: now.Add(-time.Duration(tc.days) * 24 * time.Hour).UnixMilli(),
|
||||
}
|
||||
components := calc.CalculateComponents(obs, now)
|
||||
assert.InDelta(t, tc.expectedDecay, components.RecencyDecay, 0.001)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestTypeBaseScore_UnknownType(t *testing.T) {
|
||||
score := models.TypeBaseScore("unknown-type")
|
||||
assert.Equal(t, 1.0, score, "unknown type should default to 1.0")
|
||||
}
|
||||
|
||||
func TestTypeBaseScore_AllKnownTypes(t *testing.T) {
|
||||
expected := map[models.ObservationType]float64{
|
||||
models.ObsTypeBugfix: 1.3,
|
||||
models.ObsTypeFeature: 1.2,
|
||||
models.ObsTypeDiscovery: 1.1,
|
||||
models.ObsTypeDecision: 1.1,
|
||||
models.ObsTypeRefactor: 1.0,
|
||||
models.ObsTypeChange: 0.9,
|
||||
}
|
||||
|
||||
for obsType, expectedScore := range expected {
|
||||
t.Run(string(obsType), func(t *testing.T) {
|
||||
score := models.TypeBaseScore(obsType)
|
||||
assert.Equal(t, expectedScore, score)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestCalculator_RetrievalBoostDiminishingReturns(t *testing.T) {
|
||||
calc := NewCalculator(nil)
|
||||
now := time.Now()
|
||||
|
||||
// Test that retrieval boost has diminishing returns
|
||||
// When retrieval count doubles, the boost should NOT double (log2 gives diminishing returns)
|
||||
|
||||
// Collect boosts for different counts
|
||||
boosts := make([]float64, 0)
|
||||
retrievalCounts := []int{1, 3, 7, 15, 31, 63, 127}
|
||||
|
||||
for _, count := range retrievalCounts {
|
||||
obs := &models.Observation{
|
||||
ID: 1,
|
||||
Type: models.ObsTypeRefactor,
|
||||
CreatedAtEpoch: now.UnixMilli(),
|
||||
RetrievalCount: count,
|
||||
}
|
||||
components := calc.CalculateComponents(obs, now)
|
||||
boosts = append(boosts, components.RetrievalContrib)
|
||||
}
|
||||
|
||||
// Verify boost increases but at a decreasing rate
|
||||
for i := 1; i < len(boosts); i++ {
|
||||
// Each boost should be higher than the previous
|
||||
assert.Greater(t, boosts[i], boosts[i-1],
|
||||
"boost should increase with more retrievals")
|
||||
|
||||
// But not proportionally - calculate the ratios
|
||||
if i >= 2 {
|
||||
ratio1 := boosts[i-1] / boosts[i-2]
|
||||
ratio2 := boosts[i] / boosts[i-1]
|
||||
// The growth ratio should be decreasing (diminishing returns)
|
||||
assert.Less(t, ratio2, ratio1+0.01, // Allow small floating point tolerance
|
||||
"growth rate should decrease (diminishing returns)")
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,186 +0,0 @@
|
||||
// Package scoring provides importance score calculation for observations.
|
||||
package scoring
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/rs/zerolog"
|
||||
|
||||
"github.com/lukaszraczylo/claude-mnemonic/internal/db/sqlite"
|
||||
"github.com/lukaszraczylo/claude-mnemonic/pkg/models"
|
||||
)
|
||||
|
||||
// ObservationStore defines the interface for observation storage operations needed by the recalculator.
|
||||
type ObservationStore interface {
|
||||
GetObservationsNeedingScoreUpdate(ctx context.Context, threshold time.Duration, limit int) ([]*models.Observation, error)
|
||||
UpdateImportanceScores(ctx context.Context, scores map[int64]float64) error
|
||||
GetConceptWeights(ctx context.Context) (map[string]float64, error)
|
||||
}
|
||||
|
||||
// Recalculator periodically recalculates importance scores for observations.
|
||||
type Recalculator struct {
|
||||
log zerolog.Logger
|
||||
store ObservationStore
|
||||
calculator *Calculator
|
||||
stopCh chan struct{}
|
||||
doneCh chan struct{}
|
||||
interval time.Duration
|
||||
batchSize int
|
||||
mu sync.Mutex
|
||||
running bool
|
||||
}
|
||||
|
||||
// NewRecalculator creates a new background recalculator.
|
||||
func NewRecalculator(store ObservationStore, calc *Calculator, log zerolog.Logger) *Recalculator {
|
||||
return &Recalculator{
|
||||
store: store,
|
||||
calculator: calc,
|
||||
log: log.With().Str("component", "recalculator").Logger(),
|
||||
interval: 1 * time.Hour, // Run every hour
|
||||
batchSize: 500, // Process 500 observations at a time
|
||||
stopCh: make(chan struct{}),
|
||||
doneCh: make(chan struct{}),
|
||||
}
|
||||
}
|
||||
|
||||
// Start begins the background recalculation loop.
|
||||
// This should be called in a goroutine.
|
||||
func (r *Recalculator) Start(ctx context.Context) {
|
||||
r.mu.Lock()
|
||||
if r.running {
|
||||
r.mu.Unlock()
|
||||
return
|
||||
}
|
||||
r.running = true
|
||||
r.mu.Unlock()
|
||||
|
||||
defer func() {
|
||||
r.mu.Lock()
|
||||
r.running = false
|
||||
r.mu.Unlock()
|
||||
close(r.doneCh)
|
||||
}()
|
||||
|
||||
// Initial run
|
||||
r.recalculate(ctx)
|
||||
|
||||
r.mu.Lock()
|
||||
interval := r.interval
|
||||
r.mu.Unlock()
|
||||
|
||||
ticker := time.NewTicker(interval)
|
||||
defer ticker.Stop()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
r.log.Info().Msg("recalculator shutting down due to context cancellation")
|
||||
return
|
||||
case <-r.stopCh:
|
||||
r.log.Info().Msg("recalculator stopping")
|
||||
return
|
||||
case <-ticker.C:
|
||||
r.recalculate(ctx)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Stop stops the background recalculation loop.
|
||||
func (r *Recalculator) Stop() {
|
||||
r.mu.Lock()
|
||||
if !r.running {
|
||||
r.mu.Unlock()
|
||||
return
|
||||
}
|
||||
r.mu.Unlock()
|
||||
|
||||
close(r.stopCh)
|
||||
<-r.doneCh
|
||||
}
|
||||
|
||||
// recalculate performs a single recalculation batch.
|
||||
func (r *Recalculator) recalculate(ctx context.Context) {
|
||||
now := time.Now()
|
||||
threshold := r.calculator.RecalculateThreshold()
|
||||
|
||||
r.mu.Lock()
|
||||
batchSize := r.batchSize
|
||||
r.mu.Unlock()
|
||||
|
||||
observations, err := r.store.GetObservationsNeedingScoreUpdate(ctx, threshold, batchSize)
|
||||
if err != nil {
|
||||
r.log.Error().Err(err).Msg("failed to get observations for score update")
|
||||
return
|
||||
}
|
||||
|
||||
if len(observations) == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
scores := r.calculator.BatchCalculate(observations, now)
|
||||
|
||||
if err := r.store.UpdateImportanceScores(ctx, scores); err != nil {
|
||||
r.log.Error().Err(err).Msg("failed to update importance scores")
|
||||
return
|
||||
}
|
||||
|
||||
r.log.Info().
|
||||
Int("count", len(scores)).
|
||||
Dur("elapsed", time.Since(now)).
|
||||
Msg("recalculated importance scores")
|
||||
}
|
||||
|
||||
// RecalculateNow triggers an immediate recalculation.
|
||||
// This is useful for testing or when scores need to be updated urgently.
|
||||
func (r *Recalculator) RecalculateNow(ctx context.Context) error {
|
||||
r.recalculate(ctx)
|
||||
return nil
|
||||
}
|
||||
|
||||
// RefreshConceptWeights reloads concept weights from the database.
|
||||
// Call this after updating concept weights to apply changes.
|
||||
func (r *Recalculator) RefreshConceptWeights(ctx context.Context) error {
|
||||
weights, err := r.store.GetConceptWeights(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
config := r.calculator.GetConfig()
|
||||
config.ConceptWeights = weights
|
||||
r.calculator.UpdateConfig(config)
|
||||
|
||||
r.log.Info().Int("count", len(weights)).Msg("refreshed concept weights")
|
||||
return nil
|
||||
}
|
||||
|
||||
// Stats returns statistics about the recalculator.
|
||||
type Stats struct {
|
||||
Running bool `json:"running"`
|
||||
Interval time.Duration `json:"interval"`
|
||||
BatchSize int `json:"batch_size"`
|
||||
HalfLife float64 `json:"half_life_days"`
|
||||
MinScore float64 `json:"min_score"`
|
||||
ConceptsLen int `json:"concepts_count"`
|
||||
}
|
||||
|
||||
// GetStats returns current recalculator statistics.
|
||||
func (r *Recalculator) GetStats() Stats {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
|
||||
config := r.calculator.GetConfig()
|
||||
|
||||
return Stats{
|
||||
Running: r.running,
|
||||
Interval: r.interval,
|
||||
BatchSize: r.batchSize,
|
||||
HalfLife: config.RecencyHalfLifeDays,
|
||||
MinScore: config.MinScore,
|
||||
ConceptsLen: len(config.ConceptWeights),
|
||||
}
|
||||
}
|
||||
|
||||
// Ensure ObservationStore satisfies the interface
|
||||
var _ ObservationStore = (*sqlite.ObservationStore)(nil)
|
||||
@@ -1,447 +0,0 @@
|
||||
// Package scoring provides importance score calculation for observations.
|
||||
package scoring
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/rs/zerolog"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/lukaszraczylo/claude-mnemonic/pkg/models"
|
||||
)
|
||||
|
||||
// MockObservationStore is a mock implementation of ObservationStore for testing.
|
||||
type MockObservationStore struct {
|
||||
updateErr error
|
||||
getErr error
|
||||
getConceptsErr error
|
||||
scores map[int64]float64
|
||||
conceptWeights map[string]float64
|
||||
observations []*models.Observation
|
||||
updateScoresCalls int
|
||||
mu sync.Mutex
|
||||
}
|
||||
|
||||
func NewMockObservationStore() *MockObservationStore {
|
||||
return &MockObservationStore{
|
||||
observations: []*models.Observation{},
|
||||
scores: make(map[int64]float64),
|
||||
conceptWeights: make(map[string]float64),
|
||||
}
|
||||
}
|
||||
|
||||
func (m *MockObservationStore) GetObservationsNeedingScoreUpdate(ctx context.Context, threshold time.Duration, limit int) ([]*models.Observation, error) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
|
||||
if m.getErr != nil {
|
||||
return nil, m.getErr
|
||||
}
|
||||
|
||||
// Return observations that haven't been updated within threshold
|
||||
now := time.Now()
|
||||
var result []*models.Observation
|
||||
for _, obs := range m.observations {
|
||||
if !obs.ScoreUpdatedAt.Valid || now.Sub(time.Unix(obs.ScoreUpdatedAt.Int64, 0)) > threshold {
|
||||
result = append(result, obs)
|
||||
if len(result) >= limit {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (m *MockObservationStore) UpdateImportanceScores(ctx context.Context, scores map[int64]float64) error {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
|
||||
m.updateScoresCalls++
|
||||
if m.updateErr != nil {
|
||||
return m.updateErr
|
||||
}
|
||||
|
||||
for id, score := range scores {
|
||||
m.scores[id] = score
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *MockObservationStore) GetConceptWeights(ctx context.Context) (map[string]float64, error) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
|
||||
if m.getConceptsErr != nil {
|
||||
return nil, m.getConceptsErr
|
||||
}
|
||||
return m.conceptWeights, nil
|
||||
}
|
||||
|
||||
func (m *MockObservationStore) AddObservation(obs *models.Observation) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
m.observations = append(m.observations, obs)
|
||||
}
|
||||
|
||||
func (m *MockObservationStore) SetConceptWeights(weights map[string]float64) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
m.conceptWeights = weights
|
||||
}
|
||||
|
||||
func (m *MockObservationStore) GetScore(id int64) (float64, bool) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
score, ok := m.scores[id]
|
||||
return score, ok
|
||||
}
|
||||
|
||||
func (m *MockObservationStore) GetUpdateScoresCalls() int {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
return m.updateScoresCalls
|
||||
}
|
||||
|
||||
func (m *MockObservationStore) SetUpdateError(err error) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
m.updateErr = err
|
||||
}
|
||||
|
||||
func (m *MockObservationStore) SetGetError(err error) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
m.getErr = err
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
// RECALCULATOR TESTS
|
||||
// =============================================================================
|
||||
|
||||
func TestNewRecalculator(t *testing.T) {
|
||||
store := NewMockObservationStore()
|
||||
calc := NewCalculator(nil)
|
||||
log := zerolog.Nop()
|
||||
|
||||
recalc := NewRecalculator(store, calc, log)
|
||||
|
||||
require.NotNil(t, recalc)
|
||||
assert.NotNil(t, recalc.store)
|
||||
assert.NotNil(t, recalc.calculator)
|
||||
assert.Equal(t, 1*time.Hour, recalc.interval)
|
||||
assert.Equal(t, 500, recalc.batchSize)
|
||||
assert.False(t, recalc.running)
|
||||
}
|
||||
|
||||
func TestRecalculator_RecalculateNow(t *testing.T) {
|
||||
store := NewMockObservationStore()
|
||||
calc := NewCalculator(nil)
|
||||
log := zerolog.Nop()
|
||||
recalc := NewRecalculator(store, calc, log)
|
||||
|
||||
// Add observations
|
||||
now := time.Now()
|
||||
store.AddObservation(&models.Observation{
|
||||
ID: 1,
|
||||
Type: models.ObsTypeBugfix,
|
||||
CreatedAtEpoch: now.UnixMilli(),
|
||||
})
|
||||
store.AddObservation(&models.Observation{
|
||||
ID: 2,
|
||||
Type: models.ObsTypeFeature,
|
||||
CreatedAtEpoch: now.Add(-7 * 24 * time.Hour).UnixMilli(),
|
||||
})
|
||||
|
||||
ctx := context.Background()
|
||||
err := recalc.RecalculateNow(ctx)
|
||||
|
||||
require.NoError(t, err)
|
||||
|
||||
// Verify scores were calculated
|
||||
score1, ok := store.GetScore(1)
|
||||
assert.True(t, ok)
|
||||
assert.Greater(t, score1, 0.0)
|
||||
|
||||
score2, ok := store.GetScore(2)
|
||||
assert.True(t, ok)
|
||||
assert.Greater(t, score2, 0.0)
|
||||
assert.Less(t, score2, score1, "older observation should have lower score")
|
||||
}
|
||||
|
||||
func TestRecalculator_RefreshConceptWeights(t *testing.T) {
|
||||
store := NewMockObservationStore()
|
||||
calc := NewCalculator(nil)
|
||||
log := zerolog.Nop()
|
||||
recalc := NewRecalculator(store, calc, log)
|
||||
|
||||
// Set up custom weights in store
|
||||
customWeights := map[string]float64{
|
||||
"security": 0.50,
|
||||
"performance": 0.25,
|
||||
}
|
||||
store.SetConceptWeights(customWeights)
|
||||
|
||||
ctx := context.Background()
|
||||
err := recalc.RefreshConceptWeights(ctx)
|
||||
|
||||
require.NoError(t, err)
|
||||
|
||||
// Verify weights were updated in calculator config
|
||||
config := calc.GetConfig()
|
||||
assert.Equal(t, 0.50, config.ConceptWeights["security"])
|
||||
assert.Equal(t, 0.25, config.ConceptWeights["performance"])
|
||||
}
|
||||
|
||||
func TestRecalculator_RefreshConceptWeights_Error(t *testing.T) {
|
||||
store := NewMockObservationStore()
|
||||
calc := NewCalculator(nil)
|
||||
log := zerolog.Nop()
|
||||
recalc := NewRecalculator(store, calc, log)
|
||||
|
||||
store.getConceptsErr = assert.AnError
|
||||
|
||||
ctx := context.Background()
|
||||
err := recalc.RefreshConceptWeights(ctx)
|
||||
|
||||
require.Error(t, err)
|
||||
}
|
||||
|
||||
func TestRecalculator_GetStats(t *testing.T) {
|
||||
store := NewMockObservationStore()
|
||||
calc := NewCalculator(nil)
|
||||
log := zerolog.Nop()
|
||||
recalc := NewRecalculator(store, calc, log)
|
||||
|
||||
// Set fields directly for testing
|
||||
recalc.interval = 2 * time.Hour
|
||||
recalc.batchSize = 250
|
||||
|
||||
stats := recalc.GetStats()
|
||||
|
||||
assert.False(t, stats.Running)
|
||||
assert.Equal(t, 2*time.Hour, stats.Interval)
|
||||
assert.Equal(t, 250, stats.BatchSize)
|
||||
assert.Equal(t, 7.0, stats.HalfLife)
|
||||
assert.Equal(t, 0.01, stats.MinScore)
|
||||
}
|
||||
|
||||
func TestRecalculator_StartStop(t *testing.T) {
|
||||
store := NewMockObservationStore()
|
||||
calc := NewCalculator(nil)
|
||||
log := zerolog.Nop()
|
||||
recalc := NewRecalculator(store, calc, log)
|
||||
|
||||
// Use a short interval for testing
|
||||
recalc.interval = 50 * time.Millisecond
|
||||
|
||||
// Add an observation
|
||||
store.AddObservation(&models.Observation{
|
||||
ID: 1,
|
||||
Type: models.ObsTypeBugfix,
|
||||
CreatedAtEpoch: time.Now().UnixMilli(),
|
||||
})
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
|
||||
// Start in goroutine
|
||||
go recalc.Start(ctx)
|
||||
|
||||
// Wait for initial run and at least one tick
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
|
||||
// Verify it ran
|
||||
calls := store.GetUpdateScoresCalls()
|
||||
assert.GreaterOrEqual(t, calls, 1, "should have run at least once")
|
||||
|
||||
// Stop via context cancellation
|
||||
cancel()
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
|
||||
// Verify stopped
|
||||
stats := recalc.GetStats()
|
||||
assert.False(t, stats.Running)
|
||||
}
|
||||
|
||||
func TestRecalculator_StartTwice(t *testing.T) {
|
||||
store := NewMockObservationStore()
|
||||
calc := NewCalculator(nil)
|
||||
log := zerolog.Nop()
|
||||
recalc := NewRecalculator(store, calc, log)
|
||||
|
||||
recalc.interval = 1 * time.Hour // Long interval so it doesn't tick during test
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
|
||||
// Start first time
|
||||
go recalc.Start(ctx)
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
|
||||
// Try to start second time (should return immediately)
|
||||
go recalc.Start(ctx)
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
|
||||
// Should still be running (only once)
|
||||
stats := recalc.GetStats()
|
||||
assert.True(t, stats.Running)
|
||||
|
||||
cancel()
|
||||
}
|
||||
|
||||
func TestRecalculator_StopWhenNotRunning(t *testing.T) {
|
||||
store := NewMockObservationStore()
|
||||
calc := NewCalculator(nil)
|
||||
log := zerolog.Nop()
|
||||
recalc := NewRecalculator(store, calc, log)
|
||||
|
||||
// Stop without starting - should not panic
|
||||
recalc.Stop()
|
||||
|
||||
stats := recalc.GetStats()
|
||||
assert.False(t, stats.Running)
|
||||
}
|
||||
|
||||
func TestRecalculator_EmptyStore(t *testing.T) {
|
||||
store := NewMockObservationStore()
|
||||
calc := NewCalculator(nil)
|
||||
log := zerolog.Nop()
|
||||
recalc := NewRecalculator(store, calc, log)
|
||||
|
||||
ctx := context.Background()
|
||||
err := recalc.RecalculateNow(ctx)
|
||||
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 0, store.GetUpdateScoresCalls(), "should not call update with no observations")
|
||||
}
|
||||
|
||||
func TestRecalculator_GetObservationsError(t *testing.T) {
|
||||
store := NewMockObservationStore()
|
||||
calc := NewCalculator(nil)
|
||||
log := zerolog.Nop()
|
||||
recalc := NewRecalculator(store, calc, log)
|
||||
|
||||
store.SetGetError(assert.AnError)
|
||||
|
||||
ctx := context.Background()
|
||||
err := recalc.RecalculateNow(ctx)
|
||||
|
||||
// Should not return error (logs it instead)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 0, store.GetUpdateScoresCalls())
|
||||
}
|
||||
|
||||
func TestRecalculator_UpdateScoresError(t *testing.T) {
|
||||
store := NewMockObservationStore()
|
||||
calc := NewCalculator(nil)
|
||||
log := zerolog.Nop()
|
||||
recalc := NewRecalculator(store, calc, log)
|
||||
|
||||
store.AddObservation(&models.Observation{
|
||||
ID: 1,
|
||||
Type: models.ObsTypeBugfix,
|
||||
CreatedAtEpoch: time.Now().UnixMilli(),
|
||||
})
|
||||
|
||||
store.SetUpdateError(assert.AnError)
|
||||
|
||||
ctx := context.Background()
|
||||
err := recalc.RecalculateNow(ctx)
|
||||
|
||||
// Should not return error (logs it instead)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 1, store.GetUpdateScoresCalls())
|
||||
}
|
||||
|
||||
func TestRecalculator_BatchProcessing(t *testing.T) {
|
||||
store := NewMockObservationStore()
|
||||
calc := NewCalculator(nil)
|
||||
log := zerolog.Nop()
|
||||
recalc := NewRecalculator(store, calc, log)
|
||||
|
||||
// Set small batch size
|
||||
recalc.batchSize = 3
|
||||
|
||||
// Add 5 observations
|
||||
now := time.Now()
|
||||
for i := 1; i <= 5; i++ {
|
||||
store.AddObservation(&models.Observation{
|
||||
ID: int64(i),
|
||||
Type: models.ObsTypeBugfix,
|
||||
CreatedAtEpoch: now.UnixMilli(),
|
||||
})
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
err := recalc.RecalculateNow(ctx)
|
||||
|
||||
require.NoError(t, err)
|
||||
|
||||
// Should only process batch size (3)
|
||||
scores := 0
|
||||
for i := 1; i <= 5; i++ {
|
||||
if _, ok := store.GetScore(int64(i)); ok {
|
||||
scores++
|
||||
}
|
||||
}
|
||||
assert.Equal(t, 3, scores, "should only process batch size observations")
|
||||
}
|
||||
|
||||
func TestRecalculator_ConcurrentAccess(t *testing.T) {
|
||||
store := NewMockObservationStore()
|
||||
calc := NewCalculator(nil)
|
||||
log := zerolog.Nop()
|
||||
recalc := NewRecalculator(store, calc, log)
|
||||
|
||||
// Add observations
|
||||
now := time.Now()
|
||||
for i := 1; i <= 10; i++ {
|
||||
store.AddObservation(&models.Observation{
|
||||
ID: int64(i),
|
||||
Type: models.ObsTypeBugfix,
|
||||
CreatedAtEpoch: now.UnixMilli(),
|
||||
})
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
var wg sync.WaitGroup
|
||||
|
||||
// Run multiple recalculations concurrently
|
||||
for i := 0; i < 5; i++ {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
_ = recalc.RecalculateNow(ctx)
|
||||
}()
|
||||
}
|
||||
|
||||
wg.Wait()
|
||||
|
||||
// Should complete without race conditions
|
||||
// (use -race flag to verify)
|
||||
assert.GreaterOrEqual(t, store.GetUpdateScoresCalls(), 1)
|
||||
}
|
||||
|
||||
func TestRecalculator_StatsThreadSafe(t *testing.T) {
|
||||
store := NewMockObservationStore()
|
||||
calc := NewCalculator(nil)
|
||||
log := zerolog.Nop()
|
||||
recalc := NewRecalculator(store, calc, log)
|
||||
|
||||
var wg sync.WaitGroup
|
||||
|
||||
// Concurrent reads (use -race flag to verify)
|
||||
for i := 0; i < 10; i++ {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
_ = recalc.GetStats()
|
||||
}()
|
||||
}
|
||||
|
||||
wg.Wait()
|
||||
}
|
||||
@@ -1,448 +0,0 @@
|
||||
// Package expansion provides context-aware query expansion for improved search recall.
|
||||
package expansion
|
||||
|
||||
import (
|
||||
"context"
|
||||
"regexp"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"github.com/lukaszraczylo/claude-mnemonic/internal/embedding"
|
||||
"github.com/rs/zerolog/log"
|
||||
)
|
||||
|
||||
// QueryIntent represents the detected intent of a query.
|
||||
type QueryIntent string
|
||||
|
||||
const (
|
||||
// IntentQuestion indicates a question-type query (how, why, what, etc.)
|
||||
IntentQuestion QueryIntent = "question"
|
||||
// IntentError indicates an error/debugging query
|
||||
IntentError QueryIntent = "error"
|
||||
// IntentImplementation indicates an implementation/coding query
|
||||
IntentImplementation QueryIntent = "implementation"
|
||||
// IntentArchitecture indicates an architecture/design query
|
||||
IntentArchitecture QueryIntent = "architecture"
|
||||
// IntentGeneral indicates a general lookup query
|
||||
IntentGeneral QueryIntent = "general"
|
||||
)
|
||||
|
||||
// ExpandedQuery represents a query variant with metadata.
|
||||
type ExpandedQuery struct {
|
||||
Query string `json:"query"`
|
||||
Source string `json:"source"`
|
||||
Intent QueryIntent `json:"intent"`
|
||||
Weight float64 `json:"weight"`
|
||||
}
|
||||
|
||||
// Expander provides context-aware query expansion.
|
||||
type Expander struct {
|
||||
embedSvc *embedding.Service
|
||||
intentPatterns map[QueryIntent][]*regexp.Regexp
|
||||
vocabulary []VocabEntry
|
||||
vocabVectors [][]float32
|
||||
vocabMu sync.RWMutex
|
||||
}
|
||||
|
||||
// VocabEntry represents a vocabulary term from observations.
|
||||
type VocabEntry struct {
|
||||
Term string
|
||||
Source string
|
||||
Weight float64
|
||||
}
|
||||
|
||||
// Config holds expander configuration.
|
||||
type Config struct {
|
||||
// MaxExpansions limits the number of expanded queries returned
|
||||
MaxExpansions int
|
||||
// MinSimilarity is the minimum similarity score for vocabulary expansion
|
||||
MinSimilarity float64
|
||||
// EnableVocabularyExpansion enables finding related terms from observations
|
||||
EnableVocabularyExpansion bool
|
||||
}
|
||||
|
||||
// DefaultConfig returns sensible default configuration.
|
||||
func DefaultConfig() Config {
|
||||
return Config{
|
||||
MaxExpansions: 4,
|
||||
MinSimilarity: 0.5,
|
||||
EnableVocabularyExpansion: true,
|
||||
}
|
||||
}
|
||||
|
||||
// NewExpander creates a new query expander.
|
||||
func NewExpander(embedSvc *embedding.Service) *Expander {
|
||||
e := &Expander{
|
||||
embedSvc: embedSvc,
|
||||
intentPatterns: buildIntentPatterns(),
|
||||
}
|
||||
return e
|
||||
}
|
||||
|
||||
// buildIntentPatterns creates regex patterns for intent detection.
|
||||
func buildIntentPatterns() map[QueryIntent][]*regexp.Regexp {
|
||||
patterns := make(map[QueryIntent][]*regexp.Regexp)
|
||||
|
||||
// Question patterns
|
||||
patterns[IntentQuestion] = []*regexp.Regexp{
|
||||
regexp.MustCompile(`(?i)^(how|why|what|when|where|which|who)\b`),
|
||||
regexp.MustCompile(`(?i)\?$`),
|
||||
regexp.MustCompile(`(?i)\b(explain|describe|understand)\b`),
|
||||
}
|
||||
|
||||
// Error/debugging patterns
|
||||
patterns[IntentError] = []*regexp.Regexp{
|
||||
regexp.MustCompile(`(?i)\b(error|bug|issue|problem|fail|crash|exception|panic)\b`),
|
||||
regexp.MustCompile(`(?i)\b(fix|debug|troubleshoot|resolve)\b`),
|
||||
regexp.MustCompile(`(?i)\b(doesn't work|not working|broken)\b`),
|
||||
}
|
||||
|
||||
// Implementation patterns
|
||||
patterns[IntentImplementation] = []*regexp.Regexp{
|
||||
regexp.MustCompile(`(?i)\b(implement|add|create|build|write|code)\b`),
|
||||
regexp.MustCompile(`(?i)\b(function|method|handler|endpoint|api)\b`),
|
||||
regexp.MustCompile(`(?i)\b(feature|functionality)\b`),
|
||||
}
|
||||
|
||||
// Architecture patterns
|
||||
patterns[IntentArchitecture] = []*regexp.Regexp{
|
||||
regexp.MustCompile(`(?i)\b(architecture|design|pattern|structure)\b`),
|
||||
regexp.MustCompile(`(?i)\b(component|module|layer|service)\b`),
|
||||
regexp.MustCompile(`(?i)\b(flow|pipeline|workflow)\b`),
|
||||
}
|
||||
|
||||
return patterns
|
||||
}
|
||||
|
||||
// DetectIntent analyzes a query to determine its intent.
|
||||
func (e *Expander) DetectIntent(query string) QueryIntent {
|
||||
query = strings.TrimSpace(query)
|
||||
if query == "" {
|
||||
return IntentGeneral
|
||||
}
|
||||
|
||||
// Check patterns in priority order
|
||||
intentOrder := []QueryIntent{IntentError, IntentQuestion, IntentImplementation, IntentArchitecture}
|
||||
|
||||
for _, intent := range intentOrder {
|
||||
patterns := e.intentPatterns[intent]
|
||||
for _, pattern := range patterns {
|
||||
if pattern.MatchString(query) {
|
||||
return intent
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return IntentGeneral
|
||||
}
|
||||
|
||||
// Expand generates expanded query variants based on the original query.
|
||||
func (e *Expander) Expand(ctx context.Context, query string, cfg Config) []ExpandedQuery {
|
||||
query = strings.TrimSpace(query)
|
||||
if query == "" {
|
||||
return nil
|
||||
}
|
||||
|
||||
intent := e.DetectIntent(query)
|
||||
expansions := make([]ExpandedQuery, 0, cfg.MaxExpansions)
|
||||
|
||||
// Always include the original query with highest weight
|
||||
expansions = append(expansions, ExpandedQuery{
|
||||
Query: query,
|
||||
Weight: 1.0,
|
||||
Source: "original",
|
||||
Intent: intent,
|
||||
})
|
||||
|
||||
// Generate intent-based expansions
|
||||
intentExpansions := e.expandByIntent(query, intent)
|
||||
expansions = append(expansions, intentExpansions...)
|
||||
|
||||
// Generate vocabulary-based expansions if enabled and we have vocabulary
|
||||
if cfg.EnableVocabularyExpansion && e.embedSvc != nil && len(e.vocabulary) > 0 {
|
||||
vocabExpansions := e.expandByVocabulary(ctx, query, cfg.MinSimilarity)
|
||||
expansions = append(expansions, vocabExpansions...)
|
||||
}
|
||||
|
||||
// Deduplicate and limit
|
||||
expansions = deduplicateExpansions(expansions)
|
||||
if len(expansions) > cfg.MaxExpansions {
|
||||
expansions = expansions[:cfg.MaxExpansions]
|
||||
}
|
||||
|
||||
log.Debug().
|
||||
Str("query", truncate(query, 50)).
|
||||
Str("intent", string(intent)).
|
||||
Int("expansions", len(expansions)).
|
||||
Msg("Query expanded")
|
||||
|
||||
return expansions
|
||||
}
|
||||
|
||||
// expandByIntent generates expansions based on detected query intent.
|
||||
func (e *Expander) expandByIntent(query string, intent QueryIntent) []ExpandedQuery {
|
||||
var expansions []ExpandedQuery
|
||||
|
||||
// Extract key terms from query for context-aware expansion
|
||||
keyTerms := extractKeyTerms(query)
|
||||
|
||||
switch intent {
|
||||
case IntentQuestion:
|
||||
// For questions, create a declarative variant
|
||||
declarative := makeDeclarative(query)
|
||||
if declarative != query {
|
||||
expansions = append(expansions, ExpandedQuery{
|
||||
Query: declarative,
|
||||
Weight: 0.85,
|
||||
Source: "intent:declarative",
|
||||
Intent: intent,
|
||||
})
|
||||
}
|
||||
|
||||
case IntentError:
|
||||
// For errors, expand with solution-oriented terms
|
||||
if len(keyTerms) > 0 {
|
||||
solutionQuery := strings.Join(keyTerms, " ") + " solution fix"
|
||||
expansions = append(expansions, ExpandedQuery{
|
||||
Query: solutionQuery,
|
||||
Weight: 0.8,
|
||||
Source: "intent:solution",
|
||||
Intent: intent,
|
||||
})
|
||||
}
|
||||
|
||||
case IntentImplementation:
|
||||
// For implementation queries, focus on the what/how
|
||||
if len(keyTerms) > 0 {
|
||||
howQuery := "how " + strings.Join(keyTerms, " ")
|
||||
expansions = append(expansions, ExpandedQuery{
|
||||
Query: howQuery,
|
||||
Weight: 0.75,
|
||||
Source: "intent:how",
|
||||
Intent: intent,
|
||||
})
|
||||
}
|
||||
|
||||
case IntentArchitecture:
|
||||
// For architecture queries, expand with design context
|
||||
if len(keyTerms) > 0 {
|
||||
designQuery := strings.Join(keyTerms, " ") + " design structure"
|
||||
expansions = append(expansions, ExpandedQuery{
|
||||
Query: designQuery,
|
||||
Weight: 0.75,
|
||||
Source: "intent:design",
|
||||
Intent: intent,
|
||||
})
|
||||
}
|
||||
|
||||
case IntentGeneral:
|
||||
// For general queries, try noun phrase extraction
|
||||
// No additional expansion - rely on vocabulary expansion
|
||||
}
|
||||
|
||||
return expansions
|
||||
}
|
||||
|
||||
// expandByVocabulary finds similar terms from the observation vocabulary.
|
||||
func (e *Expander) expandByVocabulary(ctx context.Context, query string, minSimilarity float64) []ExpandedQuery {
|
||||
e.vocabMu.RLock()
|
||||
defer e.vocabMu.RUnlock()
|
||||
|
||||
if len(e.vocabulary) == 0 || e.embedSvc == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Embed the query
|
||||
queryEmb, err := e.embedSvc.Embed(query)
|
||||
if err != nil {
|
||||
log.Warn().Err(err).Msg("Failed to embed query for vocabulary expansion")
|
||||
return nil
|
||||
}
|
||||
|
||||
// Find similar vocabulary terms
|
||||
type scoredTerm struct {
|
||||
entry VocabEntry
|
||||
score float64
|
||||
}
|
||||
|
||||
var similar []scoredTerm
|
||||
for i, entry := range e.vocabulary {
|
||||
if i >= len(e.vocabVectors) {
|
||||
break
|
||||
}
|
||||
|
||||
score := cosineSimilarity(queryEmb, e.vocabVectors[i])
|
||||
if score >= minSimilarity {
|
||||
similar = append(similar, scoredTerm{entry: entry, score: score})
|
||||
}
|
||||
}
|
||||
|
||||
if len(similar) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Sort by score (descending) using bubble sort
|
||||
for i := 0; i < len(similar)-1; i++ {
|
||||
for j := i + 1; j < len(similar); j++ {
|
||||
if similar[j].score > similar[i].score {
|
||||
similar[i], similar[j] = similar[j], similar[i]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Create expansion by combining top similar terms with query
|
||||
var expansions []ExpandedQuery
|
||||
if len(similar) > 0 {
|
||||
// Take top 2 similar terms and combine with original key terms
|
||||
keyTerms := extractKeyTerms(query)
|
||||
for i := 0; i < min(2, len(similar)); i++ {
|
||||
term := similar[i].entry.Term
|
||||
// Don't add if term is already in query
|
||||
if strings.Contains(strings.ToLower(query), strings.ToLower(term)) {
|
||||
continue
|
||||
}
|
||||
|
||||
combinedQuery := strings.Join(keyTerms, " ") + " " + term
|
||||
expansions = append(expansions, ExpandedQuery{
|
||||
Query: combinedQuery,
|
||||
Weight: 0.7 * similar[i].score * similar[i].entry.Weight,
|
||||
Source: "vocabulary:" + term,
|
||||
Intent: IntentGeneral,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
return expansions
|
||||
}
|
||||
|
||||
// Helper functions
|
||||
|
||||
// extractKeyTerms extracts meaningful terms from a query.
|
||||
func extractKeyTerms(query string) []string {
|
||||
// Common stop words to filter out
|
||||
stopWords := map[string]bool{
|
||||
"a": true, "an": true, "the": true, "is": true, "are": true,
|
||||
"was": true, "were": true, "be": true, "been": true, "being": true,
|
||||
"have": true, "has": true, "had": true, "do": true, "does": true,
|
||||
"did": true, "will": true, "would": true, "could": true, "should": true,
|
||||
"may": true, "might": true, "must": true, "can": true,
|
||||
"i": true, "me": true, "my": true, "we": true, "our": true,
|
||||
"you": true, "your": true, "it": true, "its": true,
|
||||
"this": true, "that": true, "these": true, "those": true,
|
||||
"what": true, "which": true, "who": true, "whom": true,
|
||||
"how": true, "why": true, "when": true, "where": true,
|
||||
"to": true, "for": true, "with": true, "about": true, "from": true,
|
||||
"in": true, "on": true, "at": true, "by": true, "of": true,
|
||||
"and": true, "or": true, "but": true, "if": true, "then": true,
|
||||
}
|
||||
|
||||
// Split and filter
|
||||
words := strings.Fields(strings.ToLower(query))
|
||||
var terms []string
|
||||
|
||||
for _, word := range words {
|
||||
// Remove punctuation
|
||||
word = strings.Trim(word, ".,?!;:'\"()[]{}")
|
||||
if len(word) < 2 {
|
||||
continue
|
||||
}
|
||||
if stopWords[word] {
|
||||
continue
|
||||
}
|
||||
terms = append(terms, word)
|
||||
}
|
||||
|
||||
return terms
|
||||
}
|
||||
|
||||
// makeDeclarative converts a question to a declarative statement.
|
||||
func makeDeclarative(query string) string {
|
||||
query = strings.TrimSpace(query)
|
||||
|
||||
// Remove question mark
|
||||
query = strings.TrimSuffix(query, "?")
|
||||
|
||||
// Handle common question patterns
|
||||
patterns := []struct {
|
||||
prefix string
|
||||
replacement string
|
||||
}{
|
||||
{"how do i ", ""},
|
||||
{"how to ", ""},
|
||||
{"how does ", ""},
|
||||
{"how is ", ""},
|
||||
{"what is ", ""},
|
||||
{"what are ", ""},
|
||||
{"why does ", ""},
|
||||
{"why is ", ""},
|
||||
{"where is ", ""},
|
||||
{"where are ", ""},
|
||||
{"when does ", ""},
|
||||
{"when is ", ""},
|
||||
}
|
||||
|
||||
lower := strings.ToLower(query)
|
||||
for _, p := range patterns {
|
||||
if strings.HasPrefix(lower, p.prefix) {
|
||||
return strings.TrimSpace(query[len(p.prefix):])
|
||||
}
|
||||
}
|
||||
|
||||
return query
|
||||
}
|
||||
|
||||
// deduplicateExpansions removes duplicate queries while preserving order.
|
||||
func deduplicateExpansions(expansions []ExpandedQuery) []ExpandedQuery {
|
||||
seen := make(map[string]bool)
|
||||
result := make([]ExpandedQuery, 0, len(expansions))
|
||||
|
||||
for _, exp := range expansions {
|
||||
normalized := strings.ToLower(strings.TrimSpace(exp.Query))
|
||||
if !seen[normalized] {
|
||||
seen[normalized] = true
|
||||
result = append(result, exp)
|
||||
}
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
// cosineSimilarity computes cosine similarity between two vectors.
|
||||
func cosineSimilarity(a, b []float32) float64 {
|
||||
if len(a) != len(b) || len(a) == 0 {
|
||||
return 0
|
||||
}
|
||||
|
||||
var dot, normA, normB float64
|
||||
for i := range a {
|
||||
dot += float64(a[i]) * float64(b[i])
|
||||
normA += float64(a[i]) * float64(a[i])
|
||||
normB += float64(b[i]) * float64(b[i])
|
||||
}
|
||||
|
||||
if normA == 0 || normB == 0 {
|
||||
return 0
|
||||
}
|
||||
|
||||
return dot / (sqrt(normA) * sqrt(normB))
|
||||
}
|
||||
|
||||
// sqrt is a simple square root implementation.
|
||||
func sqrt(x float64) float64 {
|
||||
if x <= 0 {
|
||||
return 0
|
||||
}
|
||||
z := x
|
||||
for i := 0; i < 10; i++ {
|
||||
z = (z + x/z) / 2
|
||||
}
|
||||
return z
|
||||
}
|
||||
|
||||
// truncate truncates a string to maxLen characters.
|
||||
func truncate(s string, maxLen int) string {
|
||||
if len(s) <= maxLen {
|
||||
return s
|
||||
}
|
||||
return s[:maxLen] + "..."
|
||||
}
|
||||
@@ -1,518 +0,0 @@
|
||||
// Package expansion provides context-aware query expansion for improved search recall.
|
||||
package expansion
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/suite"
|
||||
)
|
||||
|
||||
// ExpanderSuite tests the Expander functionality.
|
||||
type ExpanderSuite struct {
|
||||
suite.Suite
|
||||
expander *Expander
|
||||
}
|
||||
|
||||
func TestExpanderSuite(t *testing.T) {
|
||||
suite.Run(t, new(ExpanderSuite))
|
||||
}
|
||||
|
||||
func (s *ExpanderSuite) SetupTest() {
|
||||
// Create expander without embedding service for basic tests
|
||||
s.expander = NewExpander(nil)
|
||||
}
|
||||
|
||||
// TestNewExpander tests expander creation.
|
||||
func (s *ExpanderSuite) TestNewExpander() {
|
||||
e := NewExpander(nil)
|
||||
s.NotNil(e)
|
||||
s.NotNil(e.intentPatterns)
|
||||
s.Nil(e.embedSvc)
|
||||
}
|
||||
|
||||
// TestDetectIntent tests intent detection.
|
||||
func (s *ExpanderSuite) TestDetectIntent() {
|
||||
tests := []struct {
|
||||
name string
|
||||
query string
|
||||
expected QueryIntent
|
||||
}{
|
||||
// Question intents
|
||||
{"how_question", "how do I implement auth?", IntentQuestion},
|
||||
{"why_question", "why does this fail?", IntentError}, // "fail" triggers error first
|
||||
{"what_question", "what is the purpose of this function?", IntentQuestion},
|
||||
{"question_mark", "the handler for auth?", IntentQuestion},
|
||||
{"explain", "explain the architecture", IntentQuestion},
|
||||
|
||||
// Error intents
|
||||
{"error_word", "authentication error in login", IntentError},
|
||||
{"bug_word", "bug in user registration", IntentError},
|
||||
{"fix_word", "fix the memory leak", IntentError},
|
||||
{"not_working", "login not working", IntentError},
|
||||
{"crash", "application crash on startup", IntentError},
|
||||
|
||||
// Implementation intents
|
||||
{"implement", "implement user authentication", IntentImplementation},
|
||||
{"add_feature", "add new endpoint for users", IntentImplementation},
|
||||
{"create", "create a handler for uploads", IntentImplementation},
|
||||
{"function", "function to validate input", IntentImplementation},
|
||||
|
||||
// Architecture intents
|
||||
{"architecture", "architecture of the system", IntentArchitecture},
|
||||
{"design", "design pattern for observers", IntentArchitecture},
|
||||
{"component", "component structure", IntentArchitecture},
|
||||
{"flow", "data flow in the pipeline", IntentArchitecture},
|
||||
|
||||
// General intents
|
||||
{"general", "user authentication", IntentGeneral},
|
||||
{"empty", "", IntentGeneral},
|
||||
{"simple", "database", IntentGeneral},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
s.Run(tt.name, func() {
|
||||
result := s.expander.DetectIntent(tt.query)
|
||||
s.Equal(tt.expected, result, "Query: %s", tt.query)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestExpand tests basic query expansion.
|
||||
func (s *ExpanderSuite) TestExpand() {
|
||||
ctx := context.Background()
|
||||
cfg := DefaultConfig()
|
||||
cfg.EnableVocabularyExpansion = false // Disable for unit test
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
query string
|
||||
expectedIntent QueryIntent
|
||||
minExpansions int
|
||||
hasOriginal bool
|
||||
}{
|
||||
{name: "question", query: "how do I implement auth", expectedIntent: IntentQuestion, minExpansions: 1, hasOriginal: true},
|
||||
{name: "error", query: "fix the bug in login", expectedIntent: IntentError, minExpansions: 1, hasOriginal: true},
|
||||
{name: "implementation", query: "implement user handler", expectedIntent: IntentImplementation, minExpansions: 1, hasOriginal: true},
|
||||
{name: "architecture", query: "architecture design", expectedIntent: IntentArchitecture, minExpansions: 1, hasOriginal: true},
|
||||
{name: "general", query: "database connection", expectedIntent: IntentGeneral, minExpansions: 1, hasOriginal: true},
|
||||
{name: "empty", query: "", expectedIntent: IntentGeneral, minExpansions: 0, hasOriginal: false},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
s.Run(tt.name, func() {
|
||||
expansions := s.expander.Expand(ctx, tt.query, cfg)
|
||||
|
||||
if tt.minExpansions == 0 {
|
||||
s.Empty(expansions)
|
||||
return
|
||||
}
|
||||
|
||||
s.GreaterOrEqual(len(expansions), tt.minExpansions)
|
||||
|
||||
if tt.hasOriginal {
|
||||
// First expansion should be the original
|
||||
s.Equal(tt.query, expansions[0].Query)
|
||||
s.Equal(1.0, expansions[0].Weight)
|
||||
s.Equal("original", expansions[0].Source)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestExpandWithConfig tests expansion with custom config.
|
||||
func (s *ExpanderSuite) TestExpandWithConfig() {
|
||||
ctx := context.Background()
|
||||
|
||||
cfg := Config{
|
||||
MaxExpansions: 2,
|
||||
MinSimilarity: 0.7,
|
||||
EnableVocabularyExpansion: false,
|
||||
}
|
||||
|
||||
expansions := s.expander.Expand(ctx, "how to implement authentication", cfg)
|
||||
s.LessOrEqual(len(expansions), cfg.MaxExpansions)
|
||||
}
|
||||
|
||||
// TestExpandDeduplication tests that duplicates are removed.
|
||||
func (s *ExpanderSuite) TestExpandDeduplication() {
|
||||
ctx := context.Background()
|
||||
cfg := DefaultConfig()
|
||||
cfg.EnableVocabularyExpansion = false
|
||||
|
||||
// Query that might generate duplicate expansions
|
||||
query := "how to fix authentication"
|
||||
expansions := s.expander.Expand(ctx, query, cfg)
|
||||
|
||||
// Check for duplicates
|
||||
seen := make(map[string]bool)
|
||||
for _, exp := range expansions {
|
||||
normalized := exp.Query
|
||||
s.False(seen[normalized], "Duplicate expansion found: %s", exp.Query)
|
||||
seen[normalized] = true
|
||||
}
|
||||
}
|
||||
|
||||
// TestExtractKeyTerms tests key term extraction.
|
||||
func TestExtractKeyTerms(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
query string
|
||||
expected []string
|
||||
}{
|
||||
{
|
||||
name: "simple",
|
||||
query: "user authentication handler",
|
||||
expected: []string{"user", "authentication", "handler"},
|
||||
},
|
||||
{
|
||||
name: "with_stop_words",
|
||||
query: "how to implement the user login",
|
||||
expected: []string{"implement", "user", "login"},
|
||||
},
|
||||
{
|
||||
name: "with_punctuation",
|
||||
query: "fix the bug, please!",
|
||||
expected: []string{"fix", "bug", "please"},
|
||||
},
|
||||
{
|
||||
name: "empty",
|
||||
query: "",
|
||||
expected: nil,
|
||||
},
|
||||
{
|
||||
name: "only_stop_words",
|
||||
query: "the a an is are",
|
||||
expected: nil,
|
||||
},
|
||||
{
|
||||
name: "short_words_filtered",
|
||||
query: "a b c auth",
|
||||
expected: []string{"auth"},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
result := extractKeyTerms(tt.query)
|
||||
if tt.expected == nil {
|
||||
assert.Empty(t, result)
|
||||
} else {
|
||||
assert.Equal(t, tt.expected, result)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestMakeDeclarative tests question to declarative conversion.
|
||||
func TestMakeDeclarative(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
query string
|
||||
expected string
|
||||
}{
|
||||
{
|
||||
name: "how_do_i",
|
||||
query: "how do I implement auth?",
|
||||
expected: "implement auth",
|
||||
},
|
||||
{
|
||||
name: "how_to",
|
||||
query: "how to fix the bug",
|
||||
expected: "fix the bug",
|
||||
},
|
||||
{
|
||||
name: "what_is",
|
||||
query: "what is the purpose of this?",
|
||||
expected: "the purpose of this",
|
||||
},
|
||||
{
|
||||
name: "why_does",
|
||||
query: "why does this fail?",
|
||||
expected: "this fail",
|
||||
},
|
||||
{
|
||||
name: "already_declarative",
|
||||
query: "user authentication",
|
||||
expected: "user authentication",
|
||||
},
|
||||
{
|
||||
name: "question_mark_only",
|
||||
query: "authentication?",
|
||||
expected: "authentication",
|
||||
},
|
||||
{
|
||||
name: "case_insensitive",
|
||||
query: "How To Fix Auth?",
|
||||
expected: "Fix Auth",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
result := makeDeclarative(tt.query)
|
||||
assert.Equal(t, tt.expected, result)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestDeduplicateExpansions tests deduplication.
|
||||
func TestDeduplicateExpansions(t *testing.T) {
|
||||
expansions := []ExpandedQuery{
|
||||
{Query: "auth handler", Weight: 1.0},
|
||||
{Query: "AUTH HANDLER", Weight: 0.8}, // Duplicate (case insensitive)
|
||||
{Query: "auth handler ", Weight: 0.7}, // Duplicate (whitespace)
|
||||
{Query: "user auth", Weight: 0.6},
|
||||
}
|
||||
|
||||
result := deduplicateExpansions(expansions)
|
||||
assert.Len(t, result, 2) // "auth handler" and "user auth"
|
||||
assert.Equal(t, "auth handler", result[0].Query)
|
||||
assert.Equal(t, 1.0, result[0].Weight) // First one preserved
|
||||
assert.Equal(t, "user auth", result[1].Query)
|
||||
}
|
||||
|
||||
// TestCosineSimilarity tests cosine similarity calculation.
|
||||
func TestCosineSimilarity(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
a []float32
|
||||
b []float32
|
||||
expected float64
|
||||
delta float64
|
||||
}{
|
||||
{
|
||||
name: "identical_vectors",
|
||||
a: []float32{1, 0, 0},
|
||||
b: []float32{1, 0, 0},
|
||||
expected: 1.0,
|
||||
delta: 0.001,
|
||||
},
|
||||
{
|
||||
name: "orthogonal_vectors",
|
||||
a: []float32{1, 0, 0},
|
||||
b: []float32{0, 1, 0},
|
||||
expected: 0.0,
|
||||
delta: 0.001,
|
||||
},
|
||||
{
|
||||
name: "opposite_vectors",
|
||||
a: []float32{1, 0, 0},
|
||||
b: []float32{-1, 0, 0},
|
||||
expected: -1.0,
|
||||
delta: 0.001,
|
||||
},
|
||||
{
|
||||
name: "similar_vectors",
|
||||
a: []float32{1, 1, 0},
|
||||
b: []float32{1, 0, 0},
|
||||
expected: 0.707,
|
||||
delta: 0.01,
|
||||
},
|
||||
{
|
||||
name: "empty_vectors",
|
||||
a: []float32{},
|
||||
b: []float32{},
|
||||
expected: 0.0,
|
||||
delta: 0.001,
|
||||
},
|
||||
{
|
||||
name: "different_lengths",
|
||||
a: []float32{1, 0},
|
||||
b: []float32{1, 0, 0},
|
||||
expected: 0.0,
|
||||
delta: 0.001,
|
||||
},
|
||||
{
|
||||
name: "zero_vector",
|
||||
a: []float32{0, 0, 0},
|
||||
b: []float32{1, 1, 1},
|
||||
expected: 0.0,
|
||||
delta: 0.001,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
result := cosineSimilarity(tt.a, tt.b)
|
||||
assert.InDelta(t, tt.expected, result, tt.delta)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestDefaultConfig tests default configuration.
|
||||
func TestDefaultConfig(t *testing.T) {
|
||||
cfg := DefaultConfig()
|
||||
|
||||
assert.Equal(t, 4, cfg.MaxExpansions)
|
||||
assert.Equal(t, 0.5, cfg.MinSimilarity)
|
||||
assert.True(t, cfg.EnableVocabularyExpansion)
|
||||
}
|
||||
|
||||
// TestExpandedQueryStruct tests ExpandedQuery struct.
|
||||
func TestExpandedQueryStruct(t *testing.T) {
|
||||
eq := ExpandedQuery{
|
||||
Query: "test query",
|
||||
Weight: 0.85,
|
||||
Source: "vocabulary:auth",
|
||||
Intent: IntentQuestion,
|
||||
}
|
||||
|
||||
assert.Equal(t, "test query", eq.Query)
|
||||
assert.Equal(t, 0.85, eq.Weight)
|
||||
assert.Equal(t, "vocabulary:auth", eq.Source)
|
||||
assert.Equal(t, IntentQuestion, eq.Intent)
|
||||
}
|
||||
|
||||
// TestVocabEntry tests VocabEntry struct.
|
||||
func TestVocabEntry(t *testing.T) {
|
||||
ve := VocabEntry{
|
||||
Term: "authentication",
|
||||
Weight: 0.9,
|
||||
Source: "concept",
|
||||
}
|
||||
|
||||
assert.Equal(t, "authentication", ve.Term)
|
||||
assert.Equal(t, 0.9, ve.Weight)
|
||||
assert.Equal(t, "concept", ve.Source)
|
||||
}
|
||||
|
||||
// TestIntentConstants tests intent constant values.
|
||||
func TestIntentConstants(t *testing.T) {
|
||||
assert.Equal(t, QueryIntent("question"), IntentQuestion)
|
||||
assert.Equal(t, QueryIntent("error"), IntentError)
|
||||
assert.Equal(t, QueryIntent("implementation"), IntentImplementation)
|
||||
assert.Equal(t, QueryIntent("architecture"), IntentArchitecture)
|
||||
assert.Equal(t, QueryIntent("general"), IntentGeneral)
|
||||
}
|
||||
|
||||
// TestTruncate tests the truncate helper.
|
||||
func TestTruncate(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
input string
|
||||
expected string
|
||||
maxLen int
|
||||
}{
|
||||
{name: "short", input: "hello", maxLen: 10, expected: "hello"},
|
||||
{name: "exact", input: "hello", maxLen: 5, expected: "hello"},
|
||||
{name: "long", input: "hello world", maxLen: 5, expected: "hello..."},
|
||||
{name: "empty", input: "", maxLen: 10, expected: ""},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
result := truncate(tt.input, tt.maxLen)
|
||||
assert.Equal(t, tt.expected, result)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestSqrt tests the sqrt helper.
|
||||
func TestSqrt(t *testing.T) {
|
||||
tests := []struct {
|
||||
input float64
|
||||
expected float64
|
||||
delta float64
|
||||
}{
|
||||
{4.0, 2.0, 0.001},
|
||||
{9.0, 3.0, 0.001},
|
||||
{16.0, 4.0, 0.001},
|
||||
{2.0, 1.414, 0.01},
|
||||
{0.0, 0.0, 0.001},
|
||||
{-1.0, 0.0, 0.001},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run("", func(t *testing.T) {
|
||||
result := sqrt(tt.input)
|
||||
assert.InDelta(t, tt.expected, result, tt.delta)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestExpandByIntentError tests error intent expansion.
|
||||
func (s *ExpanderSuite) TestExpandByIntentError() {
|
||||
expansions := s.expander.expandByIntent("fix authentication bug", IntentError)
|
||||
s.NotEmpty(expansions)
|
||||
|
||||
// Should have solution-oriented expansion
|
||||
hasSolution := false
|
||||
for _, exp := range expansions {
|
||||
if exp.Source == "intent:solution" {
|
||||
hasSolution = true
|
||||
break
|
||||
}
|
||||
}
|
||||
s.True(hasSolution)
|
||||
}
|
||||
|
||||
// TestExpandByIntentQuestion tests question intent expansion.
|
||||
func (s *ExpanderSuite) TestExpandByIntentQuestion() {
|
||||
expansions := s.expander.expandByIntent("how do I implement auth", IntentQuestion)
|
||||
s.NotEmpty(expansions)
|
||||
|
||||
// Should have declarative expansion
|
||||
hasDeclarative := false
|
||||
for _, exp := range expansions {
|
||||
if exp.Source == "intent:declarative" {
|
||||
hasDeclarative = true
|
||||
break
|
||||
}
|
||||
}
|
||||
s.True(hasDeclarative)
|
||||
}
|
||||
|
||||
// TestExpandByIntentImplementation tests implementation intent expansion.
|
||||
func (s *ExpanderSuite) TestExpandByIntentImplementation() {
|
||||
expansions := s.expander.expandByIntent("implement user handler", IntentImplementation)
|
||||
s.NotEmpty(expansions)
|
||||
|
||||
// Should have how expansion
|
||||
hasHow := false
|
||||
for _, exp := range expansions {
|
||||
if exp.Source == "intent:how" {
|
||||
hasHow = true
|
||||
break
|
||||
}
|
||||
}
|
||||
s.True(hasHow)
|
||||
}
|
||||
|
||||
// TestExpandByIntentArchitecture tests architecture intent expansion.
|
||||
func (s *ExpanderSuite) TestExpandByIntentArchitecture() {
|
||||
expansions := s.expander.expandByIntent("system architecture design", IntentArchitecture)
|
||||
s.NotEmpty(expansions)
|
||||
|
||||
// Should have design expansion
|
||||
hasDesign := false
|
||||
for _, exp := range expansions {
|
||||
if exp.Source == "intent:design" {
|
||||
hasDesign = true
|
||||
break
|
||||
}
|
||||
}
|
||||
s.True(hasDesign)
|
||||
}
|
||||
|
||||
// TestExpandByIntentGeneral tests general intent returns no expansions.
|
||||
func (s *ExpanderSuite) TestExpandByIntentGeneral() {
|
||||
expansions := s.expander.expandByIntent("database", IntentGeneral)
|
||||
s.Empty(expansions) // General intent doesn't add intent-based expansions
|
||||
}
|
||||
|
||||
// TestEmptyVocabulary tests expansion with empty vocabulary.
|
||||
func (s *ExpanderSuite) TestEmptyVocabulary() {
|
||||
ctx := context.Background()
|
||||
expansions := s.expander.expandByVocabulary(ctx, "test query", 0.5)
|
||||
s.Empty(expansions)
|
||||
}
|
||||
|
||||
// TestIntentPatternsExist tests that all intents have patterns.
|
||||
func (s *ExpanderSuite) TestIntentPatternsExist() {
|
||||
s.NotEmpty(s.expander.intentPatterns[IntentQuestion])
|
||||
s.NotEmpty(s.expander.intentPatterns[IntentError])
|
||||
s.NotEmpty(s.expander.intentPatterns[IntentImplementation])
|
||||
s.NotEmpty(s.expander.intentPatterns[IntentArchitecture])
|
||||
}
|
||||
@@ -1,646 +0,0 @@
|
||||
// Package search provides unified search capabilities for claude-mnemonic.
|
||||
package search
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"github.com/lukaszraczylo/claude-mnemonic/internal/db/sqlite"
|
||||
"github.com/lukaszraczylo/claude-mnemonic/pkg/models"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"github.com/stretchr/testify/suite"
|
||||
|
||||
// Import sqlite driver
|
||||
_ "github.com/mattn/go-sqlite3"
|
||||
)
|
||||
|
||||
// Ensure context is used (for later tests)
|
||||
var _ = context.Background
|
||||
|
||||
// hasFTS5 checks if FTS5 is available in the SQLite build.
|
||||
func hasFTS5(t *testing.T) bool {
|
||||
t.Helper()
|
||||
|
||||
tmpDir, err := os.MkdirTemp("", "fts5-check-*")
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
defer func() { _ = os.RemoveAll(tmpDir) }()
|
||||
|
||||
dbPath := tmpDir + "/check.db"
|
||||
db, err := sql.Open("sqlite3", dbPath)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
defer func() { _ = db.Close() }()
|
||||
|
||||
_, err = db.Exec("CREATE VIRTUAL TABLE IF NOT EXISTS fts5_test USING fts5(content)")
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
_, _ = db.Exec("DROP TABLE IF EXISTS fts5_test")
|
||||
return true
|
||||
}
|
||||
|
||||
// testStore creates a sqlite.Store with a temporary database for testing.
|
||||
func testStore(t *testing.T) (*sqlite.Store, func()) {
|
||||
t.Helper()
|
||||
|
||||
if !hasFTS5(t) {
|
||||
t.Skip("FTS5 not available in this SQLite build")
|
||||
}
|
||||
|
||||
tmpDir, err := os.MkdirTemp("", "search-integration-test-*")
|
||||
require.NoError(t, err)
|
||||
|
||||
dbPath := tmpDir + "/test.db"
|
||||
|
||||
store, err := sqlite.NewStore(sqlite.StoreConfig{
|
||||
Path: dbPath,
|
||||
MaxConns: 1,
|
||||
WALMode: true,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
cleanup := func() {
|
||||
_ = store.Close()
|
||||
_ = os.RemoveAll(tmpDir)
|
||||
}
|
||||
|
||||
return store, cleanup
|
||||
}
|
||||
|
||||
// SearchIntegrationSuite tests search with real SQLite stores.
|
||||
type SearchIntegrationSuite struct {
|
||||
suite.Suite
|
||||
store *sqlite.Store
|
||||
cleanup func()
|
||||
manager *Manager
|
||||
obsStore *sqlite.ObservationStore
|
||||
sumStore *sqlite.SummaryStore
|
||||
prmStore *sqlite.PromptStore
|
||||
}
|
||||
|
||||
func (s *SearchIntegrationSuite) SetupTest() {
|
||||
if !hasFTS5(s.T()) {
|
||||
s.T().Skip("FTS5 not available in this SQLite build")
|
||||
}
|
||||
|
||||
s.store, s.cleanup = testStore(s.T())
|
||||
|
||||
// Create real stores backed by SQLite
|
||||
s.obsStore = sqlite.NewObservationStore(s.store)
|
||||
s.sumStore = sqlite.NewSummaryStore(s.store)
|
||||
s.prmStore = sqlite.NewPromptStore(s.store)
|
||||
|
||||
// Create search manager with real stores (no vector client for now)
|
||||
s.manager = NewManager(s.obsStore, s.sumStore, s.prmStore, nil)
|
||||
}
|
||||
|
||||
func (s *SearchIntegrationSuite) TearDownTest() {
|
||||
if s.cleanup != nil {
|
||||
s.cleanup()
|
||||
}
|
||||
}
|
||||
|
||||
func TestSearchIntegrationSuite(t *testing.T) {
|
||||
suite.Run(t, new(SearchIntegrationSuite))
|
||||
}
|
||||
|
||||
// seedObservations inserts test observations into the database.
|
||||
func (s *SearchIntegrationSuite) seedObservations(ctx context.Context) []int64 {
|
||||
var ids []int64
|
||||
|
||||
// Observation 1: Authentication bug fix
|
||||
obs1 := &models.ParsedObservation{
|
||||
Type: models.ObsTypeBugfix,
|
||||
Scope: models.ScopeProject,
|
||||
Title: "Fixed Authentication Bug",
|
||||
Narrative: "Resolved JWT token validation issue that caused intermittent login failures",
|
||||
Concepts: []string{"authentication", "jwt", "security"},
|
||||
FilesRead: []string{"auth/handler.go", "auth/jwt.go"},
|
||||
}
|
||||
id1, _, err := s.obsStore.StoreObservation(ctx, "sdk-sess-1", "test-project", obs1, 1, 100)
|
||||
s.Require().NoError(err)
|
||||
ids = append(ids, id1)
|
||||
|
||||
// Observation 2: Database optimization decision
|
||||
obs2 := &models.ParsedObservation{
|
||||
Type: models.ObsTypeDecision,
|
||||
Scope: models.ScopeProject,
|
||||
Title: "Database Query Optimization Decision",
|
||||
Narrative: "Decided to add indexes on user_id and created_at columns for better performance",
|
||||
Concepts: []string{"database", "performance", "decision"},
|
||||
FilesRead: []string{"db/migrations/001.sql"},
|
||||
}
|
||||
id2, _, err := s.obsStore.StoreObservation(ctx, "sdk-sess-1", "test-project", obs2, 2, 150)
|
||||
s.Require().NoError(err)
|
||||
ids = append(ids, id2)
|
||||
|
||||
// Observation 3: Global best practice
|
||||
obs3 := &models.ParsedObservation{
|
||||
Type: models.ObsTypeDiscovery,
|
||||
Scope: models.ScopeGlobal,
|
||||
Title: "Error Handling Best Practice",
|
||||
Narrative: "Use wrapped errors with context for better debugging: errors.Wrap(err, context)",
|
||||
Concepts: []string{"best-practice", "errors", "patterns"},
|
||||
FilesRead: []string{"pkg/errors/errors.go"},
|
||||
}
|
||||
id3, _, err := s.obsStore.StoreObservation(ctx, "sdk-sess-2", "other-project", obs3, 1, 80)
|
||||
s.Require().NoError(err)
|
||||
ids = append(ids, id3)
|
||||
|
||||
// Observation 4: Code change/refactoring
|
||||
obs4 := &models.ParsedObservation{
|
||||
Type: models.ObsTypeChange,
|
||||
Scope: models.ScopeProject,
|
||||
Title: "Refactored User Service",
|
||||
Narrative: "Changed user service to use repository pattern, modified interfaces for better testability",
|
||||
Concepts: []string{"refactoring", "architecture"},
|
||||
FilesModified: []string{"services/user.go", "services/user_test.go"},
|
||||
}
|
||||
id4, _, err := s.obsStore.StoreObservation(ctx, "sdk-sess-1", "test-project", obs4, 3, 200)
|
||||
s.Require().NoError(err)
|
||||
ids = append(ids, id4)
|
||||
|
||||
return ids
|
||||
}
|
||||
|
||||
// seedSummaries inserts test session summaries into the database.
|
||||
func (s *SearchIntegrationSuite) seedSummaries(ctx context.Context) []int64 {
|
||||
var ids []int64
|
||||
|
||||
// Summary 1
|
||||
sum1 := &models.ParsedSummary{
|
||||
Request: "Fix authentication bug",
|
||||
Investigated: "JWT token validation and session handling",
|
||||
Learned: "JWT validation requires algorithm check to prevent alg:none attacks",
|
||||
Completed: "Fixed JWT validation, added tests",
|
||||
NextSteps: "Review other security endpoints",
|
||||
}
|
||||
id1, _, err := s.sumStore.StoreSummary(ctx, "sdk-sess-1", "test-project", sum1, 1, 100)
|
||||
s.Require().NoError(err)
|
||||
ids = append(ids, id1)
|
||||
|
||||
// Summary 2
|
||||
sum2 := &models.ParsedSummary{
|
||||
Request: "Optimize database queries",
|
||||
Investigated: "Query execution plans and index usage",
|
||||
Learned: "Composite indexes work better for range queries",
|
||||
Completed: "Added indexes, verified performance improvement",
|
||||
NextSteps: "Monitor query times in production",
|
||||
}
|
||||
id2, _, err := s.sumStore.StoreSummary(ctx, "sdk-sess-1", "test-project", sum2, 2, 150)
|
||||
s.Require().NoError(err)
|
||||
ids = append(ids, id2)
|
||||
|
||||
return ids
|
||||
}
|
||||
|
||||
// TestFilterSearch_WithRealStores tests filterSearch with seeded data.
|
||||
func (s *SearchIntegrationSuite) TestFilterSearch_WithRealStores() {
|
||||
ctx := context.Background()
|
||||
|
||||
// Seed test data
|
||||
obsIDs := s.seedObservations(ctx)
|
||||
sumIDs := s.seedSummaries(ctx)
|
||||
s.Require().Len(obsIDs, 4)
|
||||
s.Require().Len(sumIDs, 2)
|
||||
|
||||
// Test filter search for observations only
|
||||
result, err := s.manager.filterSearch(ctx, SearchParams{
|
||||
Project: "test-project",
|
||||
Type: "observations",
|
||||
Limit: 10,
|
||||
Format: "full",
|
||||
})
|
||||
s.Require().NoError(err)
|
||||
s.NotNil(result)
|
||||
|
||||
// Should return project observations + global observation (4 total: 3 project + 1 global)
|
||||
s.GreaterOrEqual(len(result.Results), 3)
|
||||
|
||||
// Verify result types
|
||||
for _, r := range result.Results {
|
||||
s.Equal("observation", r.Type)
|
||||
}
|
||||
}
|
||||
|
||||
// TestFilterSearch_SessionsOnly tests filterSearch for sessions.
|
||||
func (s *SearchIntegrationSuite) TestFilterSearch_SessionsOnly() {
|
||||
ctx := context.Background()
|
||||
|
||||
// Seed test data
|
||||
_ = s.seedObservations(ctx)
|
||||
sumIDs := s.seedSummaries(ctx)
|
||||
s.Require().Len(sumIDs, 2)
|
||||
|
||||
// Test filter search for sessions only
|
||||
result, err := s.manager.filterSearch(ctx, SearchParams{
|
||||
Project: "test-project",
|
||||
Type: "sessions",
|
||||
Limit: 10,
|
||||
Format: "full",
|
||||
})
|
||||
s.Require().NoError(err)
|
||||
s.NotNil(result)
|
||||
|
||||
// Should return 2 summaries
|
||||
s.Len(result.Results, 2)
|
||||
|
||||
// Verify result types
|
||||
for _, r := range result.Results {
|
||||
s.Equal("session", r.Type)
|
||||
s.NotEmpty(r.Title) // Title should be populated from Request
|
||||
}
|
||||
}
|
||||
|
||||
// TestFilterSearch_AllTypes tests filterSearch for all types.
|
||||
func (s *SearchIntegrationSuite) TestFilterSearch_AllTypes() {
|
||||
ctx := context.Background()
|
||||
|
||||
// Seed test data
|
||||
obsIDs := s.seedObservations(ctx)
|
||||
sumIDs := s.seedSummaries(ctx)
|
||||
s.Require().Len(obsIDs, 4)
|
||||
s.Require().Len(sumIDs, 2)
|
||||
|
||||
// Test filter search for all types (Type = "")
|
||||
result, err := s.manager.filterSearch(ctx, SearchParams{
|
||||
Project: "test-project",
|
||||
Type: "", // All types
|
||||
Limit: 20,
|
||||
Format: "full",
|
||||
})
|
||||
s.Require().NoError(err)
|
||||
s.NotNil(result)
|
||||
|
||||
// Should return both observations and sessions
|
||||
hasObservations := false
|
||||
hasSessions := false
|
||||
for _, r := range result.Results {
|
||||
if r.Type == "observation" {
|
||||
hasObservations = true
|
||||
}
|
||||
if r.Type == "session" {
|
||||
hasSessions = true
|
||||
}
|
||||
}
|
||||
s.True(hasObservations, "Should have observation results")
|
||||
s.True(hasSessions, "Should have session results")
|
||||
}
|
||||
|
||||
// TestUnifiedSearch_DefaultLimit tests UnifiedSearch with default limit.
|
||||
func (s *SearchIntegrationSuite) TestUnifiedSearch_DefaultLimit() {
|
||||
ctx := context.Background()
|
||||
|
||||
// Seed test data
|
||||
s.seedObservations(ctx)
|
||||
s.seedSummaries(ctx)
|
||||
|
||||
// Test with no limit specified (should default to 20)
|
||||
result, err := s.manager.UnifiedSearch(ctx, SearchParams{
|
||||
Project: "test-project",
|
||||
})
|
||||
s.Require().NoError(err)
|
||||
s.NotNil(result)
|
||||
s.LessOrEqual(len(result.Results), 20)
|
||||
}
|
||||
|
||||
// TestUnifiedSearch_LimitCapping tests UnifiedSearch limit capping.
|
||||
func (s *SearchIntegrationSuite) TestUnifiedSearch_LimitCapping() {
|
||||
ctx := context.Background()
|
||||
|
||||
// Seed test data
|
||||
s.seedObservations(ctx)
|
||||
s.seedSummaries(ctx)
|
||||
|
||||
// Test with limit > 100 (should be capped to 100)
|
||||
result, err := s.manager.UnifiedSearch(ctx, SearchParams{
|
||||
Project: "test-project",
|
||||
Limit: 500,
|
||||
})
|
||||
s.Require().NoError(err)
|
||||
s.NotNil(result)
|
||||
s.LessOrEqual(len(result.Results), 100)
|
||||
}
|
||||
|
||||
// TestDecisions_WithRealStores tests the Decisions method falls back to filterSearch.
|
||||
func (s *SearchIntegrationSuite) TestDecisions_WithRealStores() {
|
||||
ctx := context.Background()
|
||||
|
||||
// Seed test data
|
||||
s.seedObservations(ctx)
|
||||
|
||||
// Test Decisions search (without vector client, falls back to filterSearch)
|
||||
result, err := s.manager.Decisions(ctx, SearchParams{
|
||||
Project: "test-project",
|
||||
Query: "database",
|
||||
Limit: 10,
|
||||
})
|
||||
s.Require().NoError(err)
|
||||
s.NotNil(result)
|
||||
|
||||
// Without vector client, falls back to filterSearch which returns observations
|
||||
// All results should be observations (type is forced to "observations" in Decisions)
|
||||
for _, r := range result.Results {
|
||||
s.Equal("observation", r.Type)
|
||||
}
|
||||
}
|
||||
|
||||
// TestChanges_WithRealStores tests the Changes method falls back to filterSearch.
|
||||
func (s *SearchIntegrationSuite) TestChanges_WithRealStores() {
|
||||
ctx := context.Background()
|
||||
|
||||
// Seed test data
|
||||
s.seedObservations(ctx)
|
||||
|
||||
// Test Changes search (without vector client, falls back to filterSearch)
|
||||
result, err := s.manager.Changes(ctx, SearchParams{
|
||||
Project: "test-project",
|
||||
Query: "user service",
|
||||
Limit: 10,
|
||||
})
|
||||
s.Require().NoError(err)
|
||||
s.NotNil(result)
|
||||
|
||||
// Without vector client, falls back to filterSearch which returns observations
|
||||
// All results should be observations (type is forced to "observations" in Changes)
|
||||
for _, r := range result.Results {
|
||||
s.Equal("observation", r.Type)
|
||||
}
|
||||
}
|
||||
|
||||
// TestHowItWorks_WithRealStores tests the HowItWorks method falls back to filterSearch.
|
||||
func (s *SearchIntegrationSuite) TestHowItWorks_WithRealStores() {
|
||||
ctx := context.Background()
|
||||
|
||||
// Seed test data
|
||||
s.seedObservations(ctx)
|
||||
|
||||
// Test HowItWorks search (without vector client, falls back to filterSearch)
|
||||
result, err := s.manager.HowItWorks(ctx, SearchParams{
|
||||
Project: "test-project",
|
||||
Query: "authentication",
|
||||
Limit: 10,
|
||||
})
|
||||
s.Require().NoError(err)
|
||||
s.NotNil(result)
|
||||
|
||||
// Without vector client, falls back to filterSearch which returns observations
|
||||
// All results should be observations (type is forced to "observations" in HowItWorks)
|
||||
for _, r := range result.Results {
|
||||
s.Equal("observation", r.Type)
|
||||
}
|
||||
}
|
||||
|
||||
// TestObservationToResult tests observation to result conversion with full format.
|
||||
func (s *SearchIntegrationSuite) TestObservationToResult_FullFormat() {
|
||||
ctx := context.Background()
|
||||
|
||||
// Insert single observation
|
||||
obs := &models.ParsedObservation{
|
||||
Type: models.ObsTypeDiscovery,
|
||||
Scope: models.ScopeProject,
|
||||
Title: "Test Title",
|
||||
Narrative: "Detailed narrative content for testing",
|
||||
Concepts: []string{"testing", "content"},
|
||||
}
|
||||
id, _, err := s.obsStore.StoreObservation(ctx, "sdk-test", "test-project", obs, 1, 50)
|
||||
s.Require().NoError(err)
|
||||
|
||||
// Retrieve and convert
|
||||
retrieved, err := s.obsStore.GetObservationByID(ctx, id)
|
||||
s.Require().NoError(err)
|
||||
s.Require().NotNil(retrieved)
|
||||
|
||||
result := s.manager.observationToResult(retrieved, "full")
|
||||
|
||||
s.Equal("observation", result.Type)
|
||||
s.Equal(id, result.ID)
|
||||
s.Equal("Test Title", result.Title)
|
||||
s.Equal("Detailed narrative content for testing", result.Content)
|
||||
s.Equal("test-project", result.Project)
|
||||
s.Equal("project", result.Scope)
|
||||
s.NotNil(result.Metadata)
|
||||
s.Equal("discovery", result.Metadata["obs_type"])
|
||||
}
|
||||
|
||||
// TestObservationToResult_IndexFormat tests index format (no content).
|
||||
func (s *SearchIntegrationSuite) TestObservationToResult_IndexFormat() {
|
||||
ctx := context.Background()
|
||||
|
||||
obs := &models.ParsedObservation{
|
||||
Type: models.ObsTypeBugfix,
|
||||
Scope: models.ScopeGlobal,
|
||||
Title: "Bug Fix Title",
|
||||
Narrative: "This should not appear in index format",
|
||||
}
|
||||
id, _, err := s.obsStore.StoreObservation(ctx, "sdk-test", "test-project", obs, 1, 50)
|
||||
s.Require().NoError(err)
|
||||
|
||||
retrieved, err := s.obsStore.GetObservationByID(ctx, id)
|
||||
s.Require().NoError(err)
|
||||
|
||||
result := s.manager.observationToResult(retrieved, "index")
|
||||
|
||||
s.Equal("observation", result.Type)
|
||||
s.Equal("Bug Fix Title", result.Title)
|
||||
s.Empty(result.Content, "Index format should not include content")
|
||||
s.Equal("global", result.Scope)
|
||||
}
|
||||
|
||||
// TestSummaryToResult_FullFormat tests summary to result conversion.
|
||||
func (s *SearchIntegrationSuite) TestSummaryToResult_FullFormat() {
|
||||
ctx := context.Background()
|
||||
|
||||
sum := &models.ParsedSummary{
|
||||
Request: "Implement new feature",
|
||||
Learned: "Learned important lessons about testing",
|
||||
}
|
||||
id, _, err := s.sumStore.StoreSummary(ctx, "sdk-test", "test-project", sum, 1, 50)
|
||||
s.Require().NoError(err)
|
||||
|
||||
// Retrieve via GetRecentSummaries since there's no GetByID
|
||||
summaries, err := s.sumStore.GetRecentSummaries(ctx, "test-project", 10)
|
||||
s.Require().NoError(err)
|
||||
s.Require().NotEmpty(summaries)
|
||||
|
||||
var retrieved *models.SessionSummary
|
||||
for _, s := range summaries {
|
||||
if s.ID == id {
|
||||
retrieved = s
|
||||
break
|
||||
}
|
||||
}
|
||||
s.Require().NotNil(retrieved)
|
||||
|
||||
result := s.manager.summaryToResult(retrieved, "full")
|
||||
|
||||
s.Equal("session", result.Type)
|
||||
s.Equal(id, result.ID)
|
||||
s.Contains(result.Title, "Implement new feature")
|
||||
s.Equal("Learned important lessons about testing", result.Content)
|
||||
s.Equal("test-project", result.Project)
|
||||
}
|
||||
|
||||
// TestPromptToResult_FullFormat tests prompt to result conversion.
|
||||
func (s *SearchIntegrationSuite) TestPromptToResult_FullFormat() {
|
||||
// First create a session
|
||||
ctx := context.Background()
|
||||
sessionStore := sqlite.NewSessionStore(s.store)
|
||||
_, err := sessionStore.CreateSDKSession(ctx, "sdk-prompt-test", "test-project", "initial prompt")
|
||||
s.Require().NoError(err)
|
||||
|
||||
// Save a user prompt
|
||||
promptID, err := s.prmStore.SaveUserPromptWithMatches(ctx, "sdk-prompt-test", 1, "Help me fix this authentication bug", 3)
|
||||
s.Require().NoError(err)
|
||||
|
||||
// Retrieve prompts
|
||||
prompts, err := s.prmStore.GetPromptsByIDs(ctx, []int64{promptID}, "date_desc", 10)
|
||||
s.Require().NoError(err)
|
||||
s.Require().NotEmpty(prompts)
|
||||
|
||||
result := s.manager.promptToResult(prompts[0], "full")
|
||||
|
||||
s.Equal("prompt", result.Type)
|
||||
s.Equal(promptID, result.ID)
|
||||
s.Contains(result.Title, "Help me fix")
|
||||
s.Equal("Help me fix this authentication bug", result.Content)
|
||||
}
|
||||
|
||||
// TestTruncate_TableDriven tests truncation with various inputs.
|
||||
func TestTruncate_TableDriven(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
input string
|
||||
expected string
|
||||
maxLen int
|
||||
}{
|
||||
{name: "short_string", input: "hello", maxLen: 10, expected: "hello"},
|
||||
{name: "exact_length", input: "hello", maxLen: 5, expected: "hello"},
|
||||
{name: "long_string", input: "hello world", maxLen: 5, expected: "hello..."},
|
||||
{name: "empty_string", input: "", maxLen: 10, expected: ""},
|
||||
{name: "whitespace_only", input: " ", maxLen: 10, expected: ""},
|
||||
{name: "with_leading_space", input: " hello ", maxLen: 10, expected: "hello"},
|
||||
{name: "very_long", input: "this is a very long string that should be truncated", maxLen: 20, expected: "this is a very long ..."},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
result := truncate(tt.input, tt.maxLen)
|
||||
assert.Equal(t, tt.expected, result)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestManagerWithNilStores tests that Manager handles nil stores gracefully.
|
||||
func TestManagerWithNilStores(t *testing.T) {
|
||||
m := NewManager(nil, nil, nil, nil)
|
||||
assert.NotNil(t, m)
|
||||
assert.Nil(t, m.observationStore)
|
||||
assert.Nil(t, m.summaryStore)
|
||||
assert.Nil(t, m.promptStore)
|
||||
assert.Nil(t, m.vectorClient)
|
||||
}
|
||||
|
||||
// TestSearchResultMetadataFields tests all metadata fields with real data.
|
||||
func (s *SearchIntegrationSuite) TestSearchResultMetadataFields() {
|
||||
ctx := context.Background()
|
||||
|
||||
obs := &models.ParsedObservation{
|
||||
Type: models.ObsTypeDecision,
|
||||
Scope: models.ScopeGlobal,
|
||||
Title: "Architecture Decision",
|
||||
Concepts: []string{"auth", "security"},
|
||||
FilesRead: []string{"handler.go", "auth.go"},
|
||||
}
|
||||
id, _, err := s.obsStore.StoreObservation(ctx, "sdk-meta-test", "test-project", obs, 1, 50)
|
||||
s.Require().NoError(err)
|
||||
|
||||
retrieved, err := s.obsStore.GetObservationByID(ctx, id)
|
||||
s.Require().NoError(err)
|
||||
|
||||
result := s.manager.observationToResult(retrieved, "full")
|
||||
|
||||
// Check metadata fields
|
||||
s.NotNil(result.Metadata)
|
||||
s.Equal("decision", result.Metadata["obs_type"])
|
||||
s.Equal("global", result.Metadata["scope"])
|
||||
s.Equal("global", result.Scope)
|
||||
}
|
||||
|
||||
// TestObservationToResult_AllFormats tests different format options.
|
||||
func TestObservationToResult_AllFormats(t *testing.T) {
|
||||
m := NewManager(nil, nil, nil, nil)
|
||||
|
||||
obs := &models.Observation{
|
||||
ID: 1,
|
||||
Project: "test",
|
||||
Type: models.ObsTypeBugfix,
|
||||
Scope: models.ScopeProject,
|
||||
Title: sql.NullString{String: "Bug Fix Title", Valid: true},
|
||||
Narrative: sql.NullString{String: "Detailed bug fix narrative", Valid: true},
|
||||
CreatedAtEpoch: 1704067200000,
|
||||
}
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
format string
|
||||
expectContent bool
|
||||
}{
|
||||
{"full_format", "full", true},
|
||||
{"index_format", "index", false},
|
||||
{"empty_format", "", false},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
result := m.observationToResult(obs, tt.format)
|
||||
assert.Equal(t, "observation", result.Type)
|
||||
assert.Equal(t, int64(1), result.ID)
|
||||
if tt.expectContent {
|
||||
assert.NotEmpty(t, result.Content)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestSummaryToResult_AllFormats tests different format options for summaries.
|
||||
func TestSummaryToResult_AllFormats(t *testing.T) {
|
||||
m := NewManager(nil, nil, nil, nil)
|
||||
|
||||
summary := &models.SessionSummary{
|
||||
ID: 1,
|
||||
Project: "test",
|
||||
Request: sql.NullString{String: "Test request", Valid: true},
|
||||
Learned: sql.NullString{String: "Test learned", Valid: true},
|
||||
Completed: sql.NullString{String: "Test completed", Valid: true},
|
||||
NextSteps: sql.NullString{String: "Test next steps", Valid: true},
|
||||
CreatedAtEpoch: 1704067200000,
|
||||
}
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
format string
|
||||
expectContent bool
|
||||
}{
|
||||
{"full_format", "full", true},
|
||||
{"index_format", "index", false},
|
||||
{"empty_format", "", false},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
result := m.summaryToResult(summary, tt.format)
|
||||
assert.Equal(t, "session", result.Type)
|
||||
assert.Equal(t, int64(1), result.ID)
|
||||
if tt.expectContent {
|
||||
assert.NotEmpty(t, result.Content)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
+20
-34
@@ -35,41 +35,40 @@ func NewManager(
|
||||
|
||||
// SearchParams contains parameters for unified search.
|
||||
type SearchParams struct {
|
||||
Format string
|
||||
Type string
|
||||
Project string
|
||||
ObsType string
|
||||
Concepts string
|
||||
Files string
|
||||
Query string
|
||||
Scope string
|
||||
OrderBy string
|
||||
DateStart int64
|
||||
Offset int
|
||||
Limit int
|
||||
DateEnd int64
|
||||
IncludeGlobal bool
|
||||
ExcludeSuperseded bool
|
||||
Query string
|
||||
Type string // "observations", "sessions", "prompts", or empty for all
|
||||
Project string
|
||||
ObsType string // Observation type filter
|
||||
Concepts string
|
||||
Files string
|
||||
DateStart int64
|
||||
DateEnd int64
|
||||
OrderBy string // "relevance", "date_desc", "date_asc"
|
||||
Limit int
|
||||
Offset int
|
||||
Format string // "index" or "full"
|
||||
Scope string // "project", "global", or empty for project+global
|
||||
IncludeGlobal bool // If true, include global observations along with project-scoped
|
||||
}
|
||||
|
||||
// SearchResult represents a unified search result.
|
||||
type SearchResult struct {
|
||||
Metadata map[string]interface{} `json:"metadata,omitempty"`
|
||||
Type string `json:"type"`
|
||||
Type string `json:"type"` // "observation", "session", "prompt"
|
||||
ID int64 `json:"id"`
|
||||
Title string `json:"title,omitempty"`
|
||||
Content string `json:"content,omitempty"`
|
||||
Project string `json:"project"`
|
||||
Scope string `json:"scope,omitempty"`
|
||||
ID int64 `json:"id"`
|
||||
Scope string `json:"scope,omitempty"` // "project" or "global"
|
||||
CreatedAt int64 `json:"created_at_epoch"`
|
||||
Score float64 `json:"score,omitempty"`
|
||||
Metadata map[string]interface{} `json:"metadata,omitempty"`
|
||||
}
|
||||
|
||||
// UnifiedSearchResult contains the combined search results.
|
||||
type UnifiedSearchResult struct {
|
||||
Query string `json:"query,omitempty"`
|
||||
Results []SearchResult `json:"results"`
|
||||
TotalCount int `json:"total_count"`
|
||||
Query string `json:"query,omitempty"`
|
||||
}
|
||||
|
||||
// UnifiedSearch performs a unified search across all document types.
|
||||
@@ -127,10 +126,6 @@ func (m *Manager) vectorSearch(ctx context.Context, params SearchParams) (*Unifi
|
||||
obs, err := m.observationStore.GetObservationsByIDs(ctx, obsIDs, params.OrderBy, 0)
|
||||
if err == nil {
|
||||
for _, o := range obs {
|
||||
// Skip superseded observations when requested
|
||||
if params.ExcludeSuperseded && o.IsSuperseded {
|
||||
continue
|
||||
}
|
||||
results = append(results, m.observationToResult(o, params.Format))
|
||||
}
|
||||
}
|
||||
@@ -172,16 +167,7 @@ func (m *Manager) filterSearch(ctx context.Context, params SearchParams) (*Unifi
|
||||
|
||||
// Search observations
|
||||
if params.Type == "" || params.Type == "observations" {
|
||||
var obs []*models.Observation
|
||||
var err error
|
||||
|
||||
// Use active observations (excluding superseded) when requested
|
||||
if params.ExcludeSuperseded {
|
||||
obs, err = m.observationStore.GetActiveObservations(ctx, params.Project, params.Limit)
|
||||
} else {
|
||||
obs, err = m.observationStore.GetRecentObservations(ctx, params.Project, params.Limit)
|
||||
}
|
||||
|
||||
obs, err := m.observationStore.GetRecentObservations(ctx, params.Project, params.Limit)
|
||||
if err == nil {
|
||||
for _, o := range obs {
|
||||
results = append(results, m.observationToResult(o, params.Format))
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
+62
-122
@@ -27,17 +27,17 @@ const (
|
||||
GitHubRepo = "lukaszraczylo/claude-mnemonic"
|
||||
ReleasesAPI = "https://api.github.com/repos/" + GitHubRepo + "/releases/latest"
|
||||
CheckInterval = 24 * time.Hour
|
||||
MaxExtractedSize = 250 * 1024 * 1024 // 250MB max per extracted file
|
||||
MaxExtractedSize = 100 * 1024 * 1024 // 100MB max per extracted file
|
||||
RestartDelay = 500 * time.Millisecond
|
||||
)
|
||||
|
||||
// Release represents a GitHub release.
|
||||
type Release struct {
|
||||
PublishedAt time.Time `json:"published_at"`
|
||||
TagName string `json:"tag_name"`
|
||||
Name string `json:"name"`
|
||||
Body string `json:"body"`
|
||||
PublishedAt time.Time `json:"published_at"`
|
||||
Assets []Asset `json:"assets"`
|
||||
Body string `json:"body"`
|
||||
}
|
||||
|
||||
// Asset represents a release asset.
|
||||
@@ -49,47 +49,34 @@ type Asset struct {
|
||||
|
||||
// UpdateInfo contains information about an available update.
|
||||
type UpdateInfo struct {
|
||||
PublishedAt time.Time `json:"published_at,omitempty"`
|
||||
CurrentVersion string `json:"current_version"`
|
||||
LatestVersion string `json:"latest_version"`
|
||||
ReleaseNotes string `json:"release_notes,omitempty"`
|
||||
DownloadURL string `json:"download_url,omitempty"`
|
||||
ChecksumsURL string `json:"checksums_url,omitempty"`
|
||||
BundleURL string `json:"bundle_url,omitempty"`
|
||||
ManualUpdateCommand string `json:"manual_update_command,omitempty"`
|
||||
Available bool `json:"available"`
|
||||
}
|
||||
|
||||
// InstallScriptURL is the URL to the remote installation script.
|
||||
const InstallScriptURL = "https://raw.githubusercontent.com/" + GitHubRepo + "/main/scripts/install.sh"
|
||||
|
||||
// GetManualUpdateCommand returns the curl command for manual update.
|
||||
// If version is empty, it installs the latest version.
|
||||
func GetManualUpdateCommand(version string) string {
|
||||
if version == "" {
|
||||
return fmt.Sprintf("curl -sSL %s | bash", InstallScriptURL)
|
||||
}
|
||||
return fmt.Sprintf("curl -sSL %s | bash -s -- %s", InstallScriptURL, version)
|
||||
Available bool `json:"available"`
|
||||
CurrentVersion string `json:"current_version"`
|
||||
LatestVersion string `json:"latest_version"`
|
||||
ReleaseNotes string `json:"release_notes,omitempty"`
|
||||
PublishedAt time.Time `json:"published_at,omitempty"`
|
||||
DownloadURL string `json:"download_url,omitempty"`
|
||||
ChecksumsURL string `json:"checksums_url,omitempty"`
|
||||
BundleURL string `json:"bundle_url,omitempty"` // Sigstore bundle (.sigstore.json)
|
||||
}
|
||||
|
||||
// UpdateStatus represents the current update status.
|
||||
type UpdateStatus struct {
|
||||
State string `json:"state"`
|
||||
Message string `json:"message"`
|
||||
Error string `json:"error,omitempty"`
|
||||
ManualUpdateCommand string `json:"manual_update_command,omitempty"`
|
||||
Progress float64 `json:"progress"`
|
||||
State string `json:"state"` // "idle", "checking", "downloading", "verifying", "applying", "done", "error"
|
||||
Progress float64 `json:"progress"`
|
||||
Message string `json:"message"`
|
||||
Error string `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
// Updater handles self-updates.
|
||||
type Updater struct { //nolint:govet
|
||||
httpClient *http.Client
|
||||
cachedUpdate *UpdateInfo
|
||||
lastCheck time.Time
|
||||
status UpdateStatus
|
||||
type Updater struct {
|
||||
currentVersion string
|
||||
installDir string
|
||||
mu sync.RWMutex
|
||||
httpClient *http.Client
|
||||
|
||||
mu sync.RWMutex
|
||||
status UpdateStatus
|
||||
lastCheck time.Time
|
||||
cachedUpdate *UpdateInfo
|
||||
}
|
||||
|
||||
// New creates a new Updater.
|
||||
@@ -125,10 +112,9 @@ func (u *Updater) setError(err error) {
|
||||
u.mu.Lock()
|
||||
defer u.mu.Unlock()
|
||||
u.status = UpdateStatus{
|
||||
State: "error",
|
||||
Message: "Update failed",
|
||||
Error: err.Error(),
|
||||
ManualUpdateCommand: GetManualUpdateCommand(""), // Always provide fallback command
|
||||
State: "error",
|
||||
Message: "Update failed",
|
||||
Error: err.Error(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -183,9 +169,6 @@ func (u *Updater) CheckForUpdate(ctx context.Context) (*UpdateInfo, error) {
|
||||
// Compare versions
|
||||
info.Available = isNewerVersion(info.LatestVersion, u.currentVersion)
|
||||
|
||||
// Always include manual update command as an alternative option
|
||||
info.ManualUpdateCommand = GetManualUpdateCommand("v" + info.LatestVersion)
|
||||
|
||||
if info.Available {
|
||||
// Find download URLs for current platform
|
||||
platform := getPlatform()
|
||||
@@ -468,100 +451,57 @@ func (u *Updater) extractTarGz(archivePath, destDir string) error {
|
||||
}
|
||||
|
||||
func (u *Updater) replaceBinaries(extractDir string) error {
|
||||
// Binary files to replace
|
||||
binaryFiles := []string{
|
||||
"worker",
|
||||
"mcp-server",
|
||||
"hooks/session-start",
|
||||
"hooks/user-prompt",
|
||||
"hooks/post-tool-use",
|
||||
"hooks/stop",
|
||||
"hooks/subagent-stop",
|
||||
"hooks/statusline",
|
||||
// Files to replace
|
||||
binaries := []struct {
|
||||
src string
|
||||
dest string
|
||||
}{
|
||||
{"worker", filepath.Join(u.installDir, "worker")},
|
||||
{"mcp-server", filepath.Join(u.installDir, "mcp-server")},
|
||||
{"hooks/session-start", filepath.Join(u.installDir, "hooks", "session-start")},
|
||||
{"hooks/user-prompt", filepath.Join(u.installDir, "hooks", "user-prompt")},
|
||||
{"hooks/post-tool-use", filepath.Join(u.installDir, "hooks", "post-tool-use")},
|
||||
{"hooks/stop", filepath.Join(u.installDir, "hooks", "stop")},
|
||||
{"hooks/subagent-stop", filepath.Join(u.installDir, "hooks", "subagent-stop")},
|
||||
}
|
||||
|
||||
// Get all install directories (marketplaces and cache directories)
|
||||
installDirs := u.getInstallDirectories()
|
||||
for _, b := range binaries {
|
||||
src := filepath.Join(extractDir, b.src)
|
||||
if _, err := os.Stat(src); os.IsNotExist(err) {
|
||||
continue // Skip if not in archive
|
||||
}
|
||||
|
||||
for _, installDir := range installDirs {
|
||||
log.Debug().Str("dir", installDir).Msg("Installing binaries to directory")
|
||||
|
||||
for _, binaryFile := range binaryFiles {
|
||||
src := filepath.Join(extractDir, binaryFile)
|
||||
if _, err := os.Stat(src); os.IsNotExist(err) {
|
||||
continue // Skip if not in archive
|
||||
// Backup existing binary
|
||||
if _, err := os.Stat(b.dest); err == nil {
|
||||
backup := b.dest + ".bak"
|
||||
if err := os.Rename(b.dest, backup); err != nil {
|
||||
return fmt.Errorf("failed to backup %s: %w", b.dest, err)
|
||||
}
|
||||
|
||||
dest := filepath.Join(installDir, binaryFile)
|
||||
|
||||
// Backup existing binary
|
||||
if _, err := os.Stat(dest); err == nil {
|
||||
backup := dest + ".bak"
|
||||
if err := os.Rename(dest, backup); err != nil {
|
||||
log.Warn().Err(err).Str("file", dest).Msg("Failed to backup, continuing anyway")
|
||||
defer func(backup, dest string) {
|
||||
// Clean up backup on success, restore on failure
|
||||
if _, err := os.Stat(dest); err == nil {
|
||||
_ = os.Remove(backup)
|
||||
} else {
|
||||
defer func(backup, dest string) {
|
||||
// Clean up backup on success, restore on failure
|
||||
if _, err := os.Stat(dest); err == nil {
|
||||
_ = os.Remove(backup)
|
||||
} else {
|
||||
_ = os.Rename(backup, dest)
|
||||
}
|
||||
}(backup, dest)
|
||||
_ = os.Rename(backup, dest)
|
||||
}
|
||||
}
|
||||
}(backup, b.dest)
|
||||
}
|
||||
|
||||
// Copy new binary
|
||||
if err := copyFile(src, dest); err != nil {
|
||||
return fmt.Errorf("failed to install %s: %w", dest, err)
|
||||
}
|
||||
// Copy new binary
|
||||
if err := copyFile(src, b.dest); err != nil {
|
||||
return fmt.Errorf("failed to install %s: %w", b.dest, err)
|
||||
}
|
||||
|
||||
// Make executable
|
||||
// #nosec G302 -- executables require 0755 permissions
|
||||
if err := os.Chmod(dest, 0755); err != nil {
|
||||
return fmt.Errorf("failed to chmod %s: %w", dest, err)
|
||||
}
|
||||
// Make executable
|
||||
// #nosec G302 -- executables require 0755 permissions
|
||||
if err := os.Chmod(b.dest, 0755); err != nil {
|
||||
return fmt.Errorf("failed to chmod %s: %w", b.dest, err)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// getInstallDirectories returns all directories where binaries should be installed.
|
||||
// This includes the marketplaces directory and any cache directories.
|
||||
func (u *Updater) getInstallDirectories() []string {
|
||||
dirs := []string{u.installDir}
|
||||
|
||||
// Also check cache directories where Claude Code looks for plugins
|
||||
home, err := os.UserHomeDir()
|
||||
if err != nil {
|
||||
return dirs
|
||||
}
|
||||
|
||||
// Look for cache directories under ~/.claude/plugins/cache/claude-mnemonic/claude-mnemonic/
|
||||
cacheBase := filepath.Join(home, ".claude/plugins/cache/claude-mnemonic/claude-mnemonic")
|
||||
entries, err := os.ReadDir(cacheBase)
|
||||
if err != nil {
|
||||
return dirs
|
||||
}
|
||||
|
||||
for _, entry := range entries {
|
||||
if entry.IsDir() {
|
||||
cacheDir := filepath.Join(cacheBase, entry.Name())
|
||||
// Only add if it's different from installDir and contains a worker binary
|
||||
if cacheDir != u.installDir {
|
||||
workerPath := filepath.Join(cacheDir, "worker")
|
||||
if _, err := os.Stat(workerPath); err == nil {
|
||||
dirs = append(dirs, cacheDir)
|
||||
log.Debug().Str("dir", cacheDir).Msg("Found cache directory to update")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return dirs
|
||||
}
|
||||
|
||||
func copyFile(src, dst string) error {
|
||||
// Ensure parent directory exists
|
||||
if err := os.MkdirAll(filepath.Dir(dst), 0750); err != nil {
|
||||
|
||||
@@ -1,309 +0,0 @@
|
||||
package hybrid
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/lukaszraczylo/claude-mnemonic/internal/vector/sqlitevec"
|
||||
"github.com/rs/zerolog/log"
|
||||
)
|
||||
|
||||
// AutoTuner dynamically adjusts hub threshold based on query performance
|
||||
type AutoTuner struct {
|
||||
ctx context.Context
|
||||
client *Client
|
||||
cancel context.CancelFunc
|
||||
latencies []time.Duration
|
||||
wg sync.WaitGroup
|
||||
queries int64
|
||||
targetLatency time.Duration
|
||||
adjustPeriod time.Duration
|
||||
minThreshold int
|
||||
maxThreshold int
|
||||
adjustments int
|
||||
latenciesMu sync.Mutex
|
||||
}
|
||||
|
||||
// AutoTunerConfig configures the auto-tuner
|
||||
type AutoTunerConfig struct {
|
||||
TargetLatency time.Duration // Target p95 latency (default: 50ms)
|
||||
MinThreshold int // Min hub threshold (default: 2)
|
||||
MaxThreshold int // Max hub threshold (default: 20)
|
||||
AdjustPeriod time.Duration // Adjustment frequency (default: 5min)
|
||||
}
|
||||
|
||||
// DefaultAutoTunerConfig returns sensible defaults
|
||||
func DefaultAutoTunerConfig() AutoTunerConfig {
|
||||
return AutoTunerConfig{
|
||||
TargetLatency: 50 * time.Millisecond,
|
||||
MinThreshold: 2,
|
||||
MaxThreshold: 20,
|
||||
AdjustPeriod: 5 * time.Minute,
|
||||
}
|
||||
}
|
||||
|
||||
// NewAutoTuner creates a new auto-tuner for the hybrid client
|
||||
func NewAutoTuner(client *Client, cfg AutoTunerConfig) *AutoTuner {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
|
||||
tuner := &AutoTuner{
|
||||
client: client,
|
||||
targetLatency: cfg.TargetLatency,
|
||||
minThreshold: cfg.MinThreshold,
|
||||
maxThreshold: cfg.MaxThreshold,
|
||||
adjustPeriod: cfg.AdjustPeriod,
|
||||
latencies: make([]time.Duration, 0, 1000),
|
||||
ctx: ctx,
|
||||
cancel: cancel,
|
||||
}
|
||||
|
||||
return tuner
|
||||
}
|
||||
|
||||
// Start begins auto-tuning in the background
|
||||
func (a *AutoTuner) Start() {
|
||||
a.wg.Add(1)
|
||||
go a.tuningLoop()
|
||||
|
||||
log.Info().
|
||||
Dur("target_latency", a.targetLatency).
|
||||
Int("min_threshold", a.minThreshold).
|
||||
Int("max_threshold", a.maxThreshold).
|
||||
Dur("adjust_period", a.adjustPeriod).
|
||||
Msg("Auto-tuner started")
|
||||
}
|
||||
|
||||
// Stop stops the auto-tuner
|
||||
func (a *AutoTuner) Stop() {
|
||||
a.cancel()
|
||||
a.wg.Wait()
|
||||
log.Info().Msg("Auto-tuner stopped")
|
||||
}
|
||||
|
||||
// RecordQuery records a query latency for analysis
|
||||
func (a *AutoTuner) RecordQuery(latency time.Duration) {
|
||||
a.latenciesMu.Lock()
|
||||
defer a.latenciesMu.Unlock()
|
||||
|
||||
a.queries++
|
||||
a.latencies = append(a.latencies, latency)
|
||||
|
||||
// Keep only recent queries (last 1000)
|
||||
if len(a.latencies) > 1000 {
|
||||
a.latencies = a.latencies[len(a.latencies)-1000:]
|
||||
}
|
||||
}
|
||||
|
||||
// tuningLoop periodically adjusts hub threshold
|
||||
func (a *AutoTuner) tuningLoop() {
|
||||
defer a.wg.Done()
|
||||
|
||||
ticker := time.NewTicker(a.adjustPeriod)
|
||||
defer ticker.Stop()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-a.ctx.Done():
|
||||
return
|
||||
|
||||
case <-ticker.C:
|
||||
a.adjustThreshold()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// adjustThreshold analyzes recent queries and adjusts hub threshold
|
||||
func (a *AutoTuner) adjustThreshold() {
|
||||
a.latenciesMu.Lock()
|
||||
defer a.latenciesMu.Unlock()
|
||||
|
||||
if len(a.latencies) < 10 {
|
||||
// Not enough data yet
|
||||
return
|
||||
}
|
||||
|
||||
// Calculate p95 latency
|
||||
p95 := calculateP95(a.latencies)
|
||||
|
||||
currentThreshold := a.client.hubThreshold
|
||||
|
||||
log.Debug().
|
||||
Dur("p95_latency", p95).
|
||||
Dur("target_latency", a.targetLatency).
|
||||
Int("current_threshold", currentThreshold).
|
||||
Int("queries", len(a.latencies)).
|
||||
Msg("Auto-tuner evaluating performance")
|
||||
|
||||
// Determine adjustment direction
|
||||
var newThreshold int
|
||||
|
||||
if p95 > a.targetLatency {
|
||||
// Too slow - lower threshold (more hubs = faster queries)
|
||||
adjustment := calculateAdjustment(p95, a.targetLatency)
|
||||
newThreshold = currentThreshold - adjustment
|
||||
|
||||
if newThreshold < a.minThreshold {
|
||||
newThreshold = a.minThreshold
|
||||
}
|
||||
|
||||
log.Info().
|
||||
Dur("p95", p95).
|
||||
Int("old_threshold", currentThreshold).
|
||||
Int("new_threshold", newThreshold).
|
||||
Msg("Auto-tuner: Lowering hub threshold (too slow)")
|
||||
|
||||
} else if p95 < a.targetLatency*8/10 {
|
||||
// Too fast - raise threshold (fewer hubs = more savings)
|
||||
// Only adjust if significantly faster (20% margin)
|
||||
adjustment := calculateAdjustment(a.targetLatency, p95)
|
||||
newThreshold = currentThreshold + adjustment
|
||||
|
||||
if newThreshold > a.maxThreshold {
|
||||
newThreshold = a.maxThreshold
|
||||
}
|
||||
|
||||
log.Info().
|
||||
Dur("p95", p95).
|
||||
Int("old_threshold", currentThreshold).
|
||||
Int("new_threshold", newThreshold).
|
||||
Msg("Auto-tuner: Raising hub threshold (room for savings)")
|
||||
|
||||
} else {
|
||||
// Within acceptable range, no adjustment needed
|
||||
log.Debug().
|
||||
Dur("p95", p95).
|
||||
Int("threshold", currentThreshold).
|
||||
Msg("Auto-tuner: Performance acceptable, no adjustment")
|
||||
return
|
||||
}
|
||||
|
||||
// Apply adjustment
|
||||
if newThreshold != currentThreshold {
|
||||
a.client.hubThreshold = newThreshold
|
||||
a.adjustments++
|
||||
|
||||
// Clear latency history after adjustment
|
||||
a.latencies = make([]time.Duration, 0, 1000)
|
||||
|
||||
log.Info().
|
||||
Int("threshold", newThreshold).
|
||||
Int("total_adjustments", a.adjustments).
|
||||
Msg("Hub threshold adjusted by auto-tuner")
|
||||
}
|
||||
}
|
||||
|
||||
// calculateP95 computes the 95th percentile latency
|
||||
func calculateP95(latencies []time.Duration) time.Duration {
|
||||
if len(latencies) == 0 {
|
||||
return 0
|
||||
}
|
||||
|
||||
// Sort latencies
|
||||
sorted := make([]time.Duration, len(latencies))
|
||||
copy(sorted, latencies)
|
||||
|
||||
// Simple bubble sort (small dataset)
|
||||
n := len(sorted)
|
||||
for i := 0; i < n-1; i++ {
|
||||
for j := 0; j < n-i-1; j++ {
|
||||
if sorted[j] > sorted[j+1] {
|
||||
sorted[j], sorted[j+1] = sorted[j+1], sorted[j]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Return 95th percentile
|
||||
idx := int(float64(len(sorted)) * 0.95)
|
||||
if idx >= len(sorted) {
|
||||
idx = len(sorted) - 1
|
||||
}
|
||||
|
||||
return sorted[idx]
|
||||
}
|
||||
|
||||
// calculateAdjustment determines how much to adjust threshold
|
||||
func calculateAdjustment(actual, target time.Duration) int {
|
||||
// Calculate percentage difference
|
||||
diff := float64(actual-target) / float64(target)
|
||||
|
||||
// Adjust more aggressively for larger differences
|
||||
if diff > 0.5 || diff < -0.5 {
|
||||
return 3 // Large adjustment
|
||||
} else if diff > 0.2 || diff < -0.2 {
|
||||
return 2 // Medium adjustment
|
||||
}
|
||||
|
||||
return 1 // Small adjustment
|
||||
}
|
||||
|
||||
// GetStats returns auto-tuner statistics
|
||||
func (a *AutoTuner) GetStats() AutoTunerStats {
|
||||
a.latenciesMu.Lock()
|
||||
defer a.latenciesMu.Unlock()
|
||||
|
||||
stats := AutoTunerStats{
|
||||
CurrentThreshold: a.client.hubThreshold,
|
||||
TargetLatency: a.targetLatency,
|
||||
TotalQueries: a.queries,
|
||||
TotalAdjustments: a.adjustments,
|
||||
RecentQueries: len(a.latencies),
|
||||
}
|
||||
|
||||
if len(a.latencies) > 0 {
|
||||
stats.P95Latency = calculateP95(a.latencies)
|
||||
|
||||
// Calculate average
|
||||
var total time.Duration
|
||||
for _, lat := range a.latencies {
|
||||
total += lat
|
||||
}
|
||||
stats.AvgLatency = total / time.Duration(len(a.latencies))
|
||||
}
|
||||
|
||||
return stats
|
||||
}
|
||||
|
||||
// AutoTunerStats contains auto-tuner statistics
|
||||
type AutoTunerStats struct {
|
||||
CurrentThreshold int
|
||||
TargetLatency time.Duration
|
||||
P95Latency time.Duration
|
||||
AvgLatency time.Duration
|
||||
TotalQueries int64
|
||||
TotalAdjustments int
|
||||
RecentQueries int
|
||||
}
|
||||
|
||||
// AutoTunedClient wraps Client with automatic performance tuning
|
||||
type AutoTunedClient struct {
|
||||
*Client
|
||||
tuner *AutoTuner
|
||||
}
|
||||
|
||||
// Query wraps the underlying Query call with latency tracking
|
||||
func (a *AutoTunedClient) Query(ctx context.Context, query string, limit int, where map[string]any) ([]sqlitevec.QueryResult, error) {
|
||||
start := time.Now()
|
||||
results, err := a.Client.Query(ctx, query, limit, where)
|
||||
latency := time.Since(start)
|
||||
|
||||
a.tuner.RecordQuery(latency)
|
||||
|
||||
return results, err
|
||||
}
|
||||
|
||||
// WithAutoTuning wraps a hybrid client with auto-tuning enabled
|
||||
func WithAutoTuning(client *Client, cfg AutoTunerConfig) *AutoTunedClient {
|
||||
tuner := NewAutoTuner(client, cfg)
|
||||
tuner.Start()
|
||||
|
||||
return &AutoTunedClient{
|
||||
Client: client,
|
||||
tuner: tuner,
|
||||
}
|
||||
}
|
||||
|
||||
// Stop stops the auto-tuner
|
||||
func (a *AutoTunedClient) StopTuning() {
|
||||
a.tuner.Stop()
|
||||
}
|
||||
@@ -1,515 +0,0 @@
|
||||
// Package hybrid provides LEANN-inspired selective vector storage for claude-mnemonic.
|
||||
//
|
||||
// This package implements a hybrid storage strategy where frequently-accessed
|
||||
// observations ("hubs") have their embeddings stored, while infrequently-accessed
|
||||
// observations have their embeddings recomputed on-demand during search.
|
||||
//
|
||||
// This approach reduces storage by 60-80% with minimal impact on search latency (<50ms).
|
||||
package hybrid
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"math"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/lukaszraczylo/claude-mnemonic/internal/embedding"
|
||||
"github.com/lukaszraczylo/claude-mnemonic/internal/vector/sqlitevec"
|
||||
"github.com/rs/zerolog/log"
|
||||
)
|
||||
|
||||
// VectorStorageStrategy defines how embeddings are stored/computed
|
||||
type VectorStorageStrategy int
|
||||
|
||||
const (
|
||||
// StorageAlways stores all embeddings (current behavior, backwards compatible)
|
||||
StorageAlways VectorStorageStrategy = iota
|
||||
// StorageHub stores only frequently-accessed "hub" embeddings (recommended)
|
||||
StorageHub
|
||||
// StorageOnDemand recomputes all embeddings during search (maximum savings)
|
||||
StorageOnDemand
|
||||
)
|
||||
|
||||
// Client wraps sqlitevec.Client with selective storage logic
|
||||
type Client struct {
|
||||
base *sqlitevec.Client
|
||||
db *sql.DB
|
||||
embedSvc *embedding.Service
|
||||
accessCount map[string]int
|
||||
lastAccess map[string]time.Time
|
||||
contentCache map[string]string
|
||||
strategy VectorStorageStrategy
|
||||
hubThreshold int
|
||||
mu sync.RWMutex
|
||||
cacheMu sync.RWMutex
|
||||
}
|
||||
|
||||
// Config for hybrid client
|
||||
type Config struct {
|
||||
BaseClient *sqlitevec.Client
|
||||
DB *sql.DB
|
||||
EmbedSvc *embedding.Service
|
||||
Strategy VectorStorageStrategy
|
||||
HubThreshold int // Default: 5 accesses
|
||||
}
|
||||
|
||||
// NewClient creates a new hybrid vector client
|
||||
func NewClient(cfg Config) *Client {
|
||||
if cfg.HubThreshold <= 0 {
|
||||
cfg.HubThreshold = 5
|
||||
}
|
||||
|
||||
log.Info().
|
||||
Str("strategy", strategyToString(cfg.Strategy)).
|
||||
Int("hub_threshold", cfg.HubThreshold).
|
||||
Msg("Initializing LEANN hybrid vector client")
|
||||
|
||||
return &Client{
|
||||
base: cfg.BaseClient,
|
||||
db: cfg.DB,
|
||||
embedSvc: cfg.EmbedSvc,
|
||||
strategy: cfg.Strategy,
|
||||
hubThreshold: cfg.HubThreshold,
|
||||
accessCount: make(map[string]int),
|
||||
lastAccess: make(map[string]time.Time),
|
||||
contentCache: make(map[string]string),
|
||||
}
|
||||
}
|
||||
|
||||
// AddDocuments implements selective storage based on strategy
|
||||
func (c *Client) AddDocuments(ctx context.Context, docs []sqlitevec.Document) error {
|
||||
if len(docs) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
switch c.strategy {
|
||||
case StorageAlways:
|
||||
// Use existing implementation - store all embeddings
|
||||
return c.base.AddDocuments(ctx, docs)
|
||||
|
||||
case StorageHub:
|
||||
// Store only hub candidates
|
||||
return c.addDocumentsSelective(ctx, docs)
|
||||
|
||||
case StorageOnDemand:
|
||||
// Don't store embeddings, only cache content
|
||||
return c.cacheDocuments(ctx, docs)
|
||||
|
||||
default:
|
||||
return c.base.AddDocuments(ctx, docs)
|
||||
}
|
||||
}
|
||||
|
||||
// addDocumentsSelective stores embeddings only for hub-qualified documents
|
||||
func (c *Client) addDocumentsSelective(ctx context.Context, docs []sqlitevec.Document) error {
|
||||
// Always cache content for potential recomputation
|
||||
if err := c.cacheDocuments(ctx, docs); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Filter to hub documents
|
||||
hubDocs := make([]sqlitevec.Document, 0, len(docs))
|
||||
for _, doc := range docs {
|
||||
if c.isHub(doc.ID) {
|
||||
hubDocs = append(hubDocs, doc)
|
||||
}
|
||||
}
|
||||
|
||||
// Store only hub embeddings
|
||||
if len(hubDocs) > 0 {
|
||||
log.Debug().
|
||||
Int("total", len(docs)).
|
||||
Int("hubs", len(hubDocs)).
|
||||
Msg("Storing selective embeddings")
|
||||
return c.base.AddDocuments(ctx, hubDocs)
|
||||
}
|
||||
|
||||
log.Debug().Int("total", len(docs)).Msg("All documents cached, no hubs to store")
|
||||
return nil
|
||||
}
|
||||
|
||||
// cacheDocuments stores content for later recomputation
|
||||
func (c *Client) cacheDocuments(ctx context.Context, docs []sqlitevec.Document) error {
|
||||
c.cacheMu.Lock()
|
||||
defer c.cacheMu.Unlock()
|
||||
|
||||
for _, doc := range docs {
|
||||
c.contentCache[doc.ID] = doc.Content
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// DeleteDocuments removes documents by their IDs
|
||||
func (c *Client) DeleteDocuments(ctx context.Context, ids []string) error {
|
||||
// Remove from base storage
|
||||
if err := c.base.DeleteDocuments(ctx, ids); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Clean up caches
|
||||
c.mu.Lock()
|
||||
for _, id := range ids {
|
||||
delete(c.accessCount, id)
|
||||
delete(c.lastAccess, id)
|
||||
}
|
||||
c.mu.Unlock()
|
||||
|
||||
c.cacheMu.Lock()
|
||||
for _, id := range ids {
|
||||
delete(c.contentCache, id)
|
||||
}
|
||||
c.cacheMu.Unlock()
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Query performs search with dynamic recomputation
|
||||
func (c *Client) Query(ctx context.Context, query string, limit int, where map[string]any) ([]sqlitevec.QueryResult, error) {
|
||||
switch c.strategy {
|
||||
case StorageAlways:
|
||||
// Use existing implementation
|
||||
return c.queryAndTrack(ctx, query, limit, where)
|
||||
|
||||
case StorageHub:
|
||||
// Search hubs, then expand with recomputation
|
||||
return c.queryHybrid(ctx, query, limit, where)
|
||||
|
||||
case StorageOnDemand:
|
||||
// Fully dynamic search
|
||||
return c.queryDynamic(ctx, query, limit, where)
|
||||
|
||||
default:
|
||||
return c.queryAndTrack(ctx, query, limit, where)
|
||||
}
|
||||
}
|
||||
|
||||
// queryAndTrack wraps base Query with access tracking
|
||||
func (c *Client) queryAndTrack(ctx context.Context, query string, limit int, where map[string]any) ([]sqlitevec.QueryResult, error) {
|
||||
results, err := c.base.Query(ctx, query, limit, where)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Track access for hub detection
|
||||
c.trackAccess(results)
|
||||
|
||||
return results, nil
|
||||
}
|
||||
|
||||
// queryHybrid searches stored hubs and recomputes non-hubs
|
||||
func (c *Client) queryHybrid(ctx context.Context, query string, limit int, where map[string]any) ([]sqlitevec.QueryResult, error) {
|
||||
startTime := time.Now()
|
||||
|
||||
// 1. Query stored hub embeddings (limit * 2 for expansion)
|
||||
hubResults, err := c.base.Query(ctx, query, limit*2, where)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// 2. Track access
|
||||
c.trackAccess(hubResults)
|
||||
|
||||
// 3. Get candidate non-hub IDs (from content cache)
|
||||
candidates := c.getCandidateNonHubs(where, limit*2)
|
||||
|
||||
// 4. Recompute embeddings for candidates if we have any
|
||||
var recomputedResults []sqlitevec.QueryResult
|
||||
if len(candidates) > 0 {
|
||||
recomputedResults, err = c.recomputeAndScore(ctx, query, candidates)
|
||||
if err != nil {
|
||||
// Log but don't fail - use hub results only
|
||||
log.Warn().Err(err).Msg("Failed to recompute embeddings, using hub results only")
|
||||
recomputedResults = nil
|
||||
}
|
||||
}
|
||||
|
||||
// 5. Merge and rank
|
||||
allResults := append(hubResults, recomputedResults...)
|
||||
sortBySimilarity(allResults)
|
||||
|
||||
// 6. Return top K
|
||||
if len(allResults) > limit {
|
||||
allResults = allResults[:limit]
|
||||
}
|
||||
|
||||
duration := time.Since(startTime)
|
||||
log.Debug().
|
||||
Dur("duration_ms", duration).
|
||||
Int("hubs", len(hubResults)).
|
||||
Int("recomputed", len(recomputedResults)).
|
||||
Int("results", len(allResults)).
|
||||
Msg("Hybrid search completed")
|
||||
|
||||
return allResults, nil
|
||||
}
|
||||
|
||||
// queryDynamic recomputes all embeddings on-the-fly
|
||||
func (c *Client) queryDynamic(ctx context.Context, query string, limit int, where map[string]any) ([]sqlitevec.QueryResult, error) {
|
||||
startTime := time.Now()
|
||||
|
||||
// Get all candidate IDs from content cache
|
||||
candidates := c.getCandidateNonHubs(where, limit*5)
|
||||
|
||||
// Recompute and score all
|
||||
results, err := c.recomputeAndScore(ctx, query, candidates)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Track access
|
||||
c.trackAccess(results)
|
||||
|
||||
// Return top K
|
||||
if len(results) > limit {
|
||||
results = results[:limit]
|
||||
}
|
||||
|
||||
duration := time.Since(startTime)
|
||||
log.Debug().
|
||||
Dur("duration_ms", duration).
|
||||
Int("recomputed", len(candidates)).
|
||||
Int("results", len(results)).
|
||||
Msg("Dynamic search completed")
|
||||
|
||||
return results, nil
|
||||
}
|
||||
|
||||
// recomputeAndScore generates embeddings and computes similarities
|
||||
func (c *Client) recomputeAndScore(ctx context.Context, query string, candidateIDs []string) ([]sqlitevec.QueryResult, error) {
|
||||
if len(candidateIDs) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// Generate query embedding
|
||||
queryEmb, err := c.embedSvc.Embed(query)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("embed query: %w", err)
|
||||
}
|
||||
|
||||
// Get content for candidates
|
||||
c.cacheMu.RLock()
|
||||
texts := make([]string, 0, len(candidateIDs))
|
||||
validIDs := make([]string, 0, len(candidateIDs))
|
||||
for _, id := range candidateIDs {
|
||||
if content, ok := c.contentCache[id]; ok && content != "" {
|
||||
texts = append(texts, content)
|
||||
validIDs = append(validIDs, id)
|
||||
}
|
||||
}
|
||||
c.cacheMu.RUnlock()
|
||||
|
||||
if len(texts) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// Batch generate embeddings
|
||||
embeddings, err := c.embedSvc.EmbedBatch(texts)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("batch embed: %w", err)
|
||||
}
|
||||
|
||||
// Compute similarities
|
||||
results := make([]sqlitevec.QueryResult, len(embeddings))
|
||||
for i, emb := range embeddings {
|
||||
similarity := cosineSimilarity(queryEmb, emb)
|
||||
distance := 1.0 - similarity // Convert to distance
|
||||
|
||||
results[i] = sqlitevec.QueryResult{
|
||||
ID: validIDs[i],
|
||||
Distance: float64(distance),
|
||||
Similarity: float64(similarity),
|
||||
Metadata: make(map[string]any),
|
||||
}
|
||||
}
|
||||
|
||||
return results, nil
|
||||
}
|
||||
|
||||
// trackAccess records document access for hub detection
|
||||
func (c *Client) trackAccess(results []sqlitevec.QueryResult) {
|
||||
if len(results) == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
|
||||
now := time.Now()
|
||||
for _, r := range results {
|
||||
c.accessCount[r.ID]++
|
||||
c.lastAccess[r.ID] = now
|
||||
}
|
||||
}
|
||||
|
||||
// isHub checks if a document qualifies as a hub
|
||||
func (c *Client) isHub(docID string) bool {
|
||||
c.mu.RLock()
|
||||
defer c.mu.RUnlock()
|
||||
|
||||
count := c.accessCount[docID]
|
||||
return count >= c.hubThreshold
|
||||
}
|
||||
|
||||
// getCandidateNonHubs returns IDs of non-hub documents matching filter
|
||||
func (c *Client) getCandidateNonHubs(where map[string]any, limit int) []string {
|
||||
c.cacheMu.RLock()
|
||||
defer c.cacheMu.RUnlock()
|
||||
|
||||
candidates := make([]string, 0, limit)
|
||||
for id := range c.contentCache {
|
||||
if !c.isHub(id) {
|
||||
candidates = append(candidates, id)
|
||||
if len(candidates) >= limit {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return candidates
|
||||
}
|
||||
|
||||
// IsConnected always returns true (wraps base client)
|
||||
func (c *Client) IsConnected() bool {
|
||||
return c.base.IsConnected()
|
||||
}
|
||||
|
||||
// Close releases resources
|
||||
func (c *Client) Close() error {
|
||||
return c.base.Close()
|
||||
}
|
||||
|
||||
// Count returns the total number of vectors in the store
|
||||
func (c *Client) Count(ctx context.Context) (int64, error) {
|
||||
return c.base.Count(ctx)
|
||||
}
|
||||
|
||||
// ModelVersion returns the current embedding model version
|
||||
func (c *Client) ModelVersion() string {
|
||||
return c.base.ModelVersion()
|
||||
}
|
||||
|
||||
// NeedsRebuild checks if vectors need to be rebuilt due to model version change
|
||||
func (c *Client) NeedsRebuild(ctx context.Context) (bool, string) {
|
||||
return c.base.NeedsRebuild(ctx)
|
||||
}
|
||||
|
||||
// GetStaleVectors returns doc_ids of vectors with mismatched or null model versions
|
||||
func (c *Client) GetStaleVectors(ctx context.Context) ([]sqlitevec.StaleVectorInfo, error) {
|
||||
return c.base.GetStaleVectors(ctx)
|
||||
}
|
||||
|
||||
// DeleteVectorsByDocIDs removes vectors by their doc_ids
|
||||
func (c *Client) DeleteVectorsByDocIDs(ctx context.Context, docIDs []string) error {
|
||||
return c.base.DeleteVectorsByDocIDs(ctx, docIDs)
|
||||
}
|
||||
|
||||
// GetStorageStats returns storage efficiency metrics
|
||||
func (c *Client) GetStorageStats(ctx context.Context) (StorageStats, error) {
|
||||
c.mu.RLock()
|
||||
c.cacheMu.RLock()
|
||||
defer c.mu.RUnlock()
|
||||
defer c.cacheMu.RUnlock()
|
||||
|
||||
totalDocs := len(c.contentCache)
|
||||
hubCount := 0
|
||||
for id := range c.contentCache {
|
||||
if c.accessCount[id] >= c.hubThreshold {
|
||||
hubCount++
|
||||
}
|
||||
}
|
||||
|
||||
storedCount := hubCount
|
||||
if c.strategy == StorageAlways {
|
||||
// Get actual count from database
|
||||
if count, err := c.base.Count(ctx); err == nil {
|
||||
storedCount = int(count)
|
||||
}
|
||||
} else if c.strategy == StorageOnDemand {
|
||||
storedCount = 0
|
||||
}
|
||||
|
||||
embeddingSize := 384 * 4 // 384 dims × 4 bytes (float32)
|
||||
storedBytes := storedCount * embeddingSize
|
||||
potentialBytes := totalDocs * embeddingSize
|
||||
|
||||
savingsPercent := 0.0
|
||||
if potentialBytes > 0 {
|
||||
savingsPercent = (1.0 - float64(storedBytes)/float64(potentialBytes)) * 100
|
||||
}
|
||||
|
||||
return StorageStats{
|
||||
TotalDocuments: totalDocs,
|
||||
HubDocuments: hubCount,
|
||||
StoredEmbeddings: storedCount,
|
||||
StorageBytes: storedBytes,
|
||||
SavingsPercent: savingsPercent,
|
||||
Strategy: c.strategy,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// StorageStats contains storage efficiency metrics
|
||||
type StorageStats struct {
|
||||
TotalDocuments int
|
||||
HubDocuments int
|
||||
StoredEmbeddings int
|
||||
StorageBytes int
|
||||
SavingsPercent float64
|
||||
Strategy VectorStorageStrategy
|
||||
}
|
||||
|
||||
// Helper functions
|
||||
|
||||
func cosineSimilarity(a, b []float32) float32 {
|
||||
var dotProduct, normA, normB float32
|
||||
for i := range a {
|
||||
dotProduct += a[i] * b[i]
|
||||
normA += a[i] * a[i]
|
||||
normB += b[i] * b[i]
|
||||
}
|
||||
if normA == 0 || normB == 0 {
|
||||
return 0
|
||||
}
|
||||
return dotProduct / float32(math.Sqrt(float64(normA))*math.Sqrt(float64(normB)))
|
||||
}
|
||||
|
||||
func sortBySimilarity(results []sqlitevec.QueryResult) {
|
||||
// Use a simple but efficient sorting algorithm
|
||||
n := len(results)
|
||||
for i := 0; i < n-1; i++ {
|
||||
for j := 0; j < n-i-1; j++ {
|
||||
if results[j].Similarity < results[j+1].Similarity {
|
||||
results[j], results[j+1] = results[j+1], results[j]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func strategyToString(s VectorStorageStrategy) string {
|
||||
switch s {
|
||||
case StorageAlways:
|
||||
return "always"
|
||||
case StorageHub:
|
||||
return "hub"
|
||||
case StorageOnDemand:
|
||||
return "on_demand"
|
||||
default:
|
||||
return "unknown"
|
||||
}
|
||||
}
|
||||
|
||||
// ParseStrategy converts a string to VectorStorageStrategy
|
||||
func ParseStrategy(s string) VectorStorageStrategy {
|
||||
switch s {
|
||||
case "hub":
|
||||
return StorageHub
|
||||
case "on_demand":
|
||||
return StorageOnDemand
|
||||
case "always":
|
||||
return StorageAlways
|
||||
default:
|
||||
return StorageHub // Default to hub strategy
|
||||
}
|
||||
}
|
||||
@@ -1,186 +0,0 @@
|
||||
package hybrid
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/lukaszraczylo/claude-mnemonic/internal/vector/sqlitevec"
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestParseStrategy(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
input string
|
||||
expected VectorStorageStrategy
|
||||
}{
|
||||
{"hub_strategy", "hub", StorageHub},
|
||||
{"on_demand_strategy", "on_demand", StorageOnDemand},
|
||||
{"always_strategy", "always", StorageAlways},
|
||||
{"invalid_defaults_to_hub", "invalid", StorageHub},
|
||||
{"empty_defaults_to_hub", "", StorageHub},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
result := ParseStrategy(tt.input)
|
||||
assert.Equal(t, tt.expected, result)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestStrategyToString(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
expected string
|
||||
input VectorStorageStrategy
|
||||
}{
|
||||
{"hub_to_string", "hub", StorageHub},
|
||||
{"on_demand_to_string", "on_demand", StorageOnDemand},
|
||||
{"always_to_string", "always", StorageAlways},
|
||||
{"invalid_to_unknown", "unknown", VectorStorageStrategy(99)},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
result := strategyToString(tt.input)
|
||||
assert.Equal(t, tt.expected, result)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestCosineSimilarity(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
a []float32
|
||||
b []float32
|
||||
expected float32
|
||||
}{
|
||||
{
|
||||
name: "identical_vectors",
|
||||
a: []float32{1, 0, 0},
|
||||
b: []float32{1, 0, 0},
|
||||
expected: 1.0,
|
||||
},
|
||||
{
|
||||
name: "orthogonal_vectors",
|
||||
a: []float32{1, 0, 0},
|
||||
b: []float32{0, 1, 0},
|
||||
expected: 0.0,
|
||||
},
|
||||
{
|
||||
name: "opposite_vectors",
|
||||
a: []float32{1, 0, 0},
|
||||
b: []float32{-1, 0, 0},
|
||||
expected: -1.0,
|
||||
},
|
||||
{
|
||||
name: "zero_vector",
|
||||
a: []float32{0, 0, 0},
|
||||
b: []float32{1, 1, 1},
|
||||
expected: 0.0,
|
||||
},
|
||||
{
|
||||
name: "parallel_vectors",
|
||||
a: []float32{2, 0, 0},
|
||||
b: []float32{4, 0, 0},
|
||||
expected: 1.0,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
result := cosineSimilarity(tt.a, tt.b)
|
||||
assert.InDelta(t, tt.expected, result, 0.001)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestSortBySimilarity(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
input []sqlitevec.QueryResult
|
||||
expected []string // Expected order of IDs
|
||||
}{
|
||||
{
|
||||
name: "already_sorted",
|
||||
input: []sqlitevec.QueryResult{
|
||||
{ID: "doc1", Similarity: 0.9},
|
||||
{ID: "doc2", Similarity: 0.7},
|
||||
{ID: "doc3", Similarity: 0.5},
|
||||
},
|
||||
expected: []string{"doc1", "doc2", "doc3"},
|
||||
},
|
||||
{
|
||||
name: "reverse_sorted",
|
||||
input: []sqlitevec.QueryResult{
|
||||
{ID: "doc1", Similarity: 0.3},
|
||||
{ID: "doc2", Similarity: 0.7},
|
||||
{ID: "doc3", Similarity: 0.9},
|
||||
},
|
||||
expected: []string{"doc3", "doc2", "doc1"},
|
||||
},
|
||||
{
|
||||
name: "random_order",
|
||||
input: []sqlitevec.QueryResult{
|
||||
{ID: "doc1", Similarity: 0.5},
|
||||
{ID: "doc2", Similarity: 0.9},
|
||||
{ID: "doc3", Similarity: 0.3},
|
||||
{ID: "doc4", Similarity: 0.7},
|
||||
},
|
||||
expected: []string{"doc2", "doc4", "doc1", "doc3"},
|
||||
},
|
||||
{
|
||||
name: "identical_similarities",
|
||||
input: []sqlitevec.QueryResult{
|
||||
{ID: "doc1", Similarity: 0.5},
|
||||
{ID: "doc2", Similarity: 0.5},
|
||||
{ID: "doc3", Similarity: 0.5},
|
||||
},
|
||||
expected: []string{"doc1", "doc2", "doc3"},
|
||||
},
|
||||
{
|
||||
name: "empty_list",
|
||||
input: []sqlitevec.QueryResult{},
|
||||
expected: []string{},
|
||||
},
|
||||
{
|
||||
name: "single_element",
|
||||
input: []sqlitevec.QueryResult{
|
||||
{ID: "doc1", Similarity: 0.5},
|
||||
},
|
||||
expected: []string{"doc1"},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
sortBySimilarity(tt.input)
|
||||
|
||||
actual := make([]string, len(tt.input))
|
||||
for i, r := range tt.input {
|
||||
actual[i] = r.ID
|
||||
}
|
||||
|
||||
assert.Equal(t, tt.expected, actual)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestSortBySimilarity_PreserveOtherFields(t *testing.T) {
|
||||
input := []sqlitevec.QueryResult{
|
||||
{ID: "doc1", Similarity: 0.3, Distance: 0.7, Metadata: map[string]any{"key": "val1"}},
|
||||
{ID: "doc2", Similarity: 0.9, Distance: 0.1, Metadata: map[string]any{"key": "val2"}},
|
||||
}
|
||||
|
||||
sortBySimilarity(input)
|
||||
|
||||
assert.Equal(t, "doc2", input[0].ID)
|
||||
assert.InDelta(t, 0.9, input[0].Similarity, 0.001)
|
||||
assert.InDelta(t, 0.1, input[0].Distance, 0.001)
|
||||
assert.Equal(t, "val2", input[0].Metadata["key"])
|
||||
|
||||
assert.Equal(t, "doc1", input[1].ID)
|
||||
assert.InDelta(t, 0.3, input[1].Similarity, 0.001)
|
||||
assert.InDelta(t, 0.7, input[1].Distance, 0.001)
|
||||
assert.Equal(t, "val1", input[1].Metadata["key"])
|
||||
}
|
||||
@@ -1,62 +0,0 @@
|
||||
package hybrid
|
||||
|
||||
import (
|
||||
"os"
|
||||
"strconv"
|
||||
|
||||
"github.com/rs/zerolog/log"
|
||||
)
|
||||
|
||||
// GetStrategyFromEnv reads CLAUDE_MNEMONIC_VECTOR_STRATEGY from environment
|
||||
func GetStrategyFromEnv() VectorStorageStrategy {
|
||||
strategyStr := os.Getenv("CLAUDE_MNEMONIC_VECTOR_STRATEGY")
|
||||
if strategyStr == "" {
|
||||
// Default to hub strategy for optimal balance
|
||||
return StorageHub
|
||||
}
|
||||
|
||||
strategy := ParseStrategy(strategyStr)
|
||||
log.Info().
|
||||
Str("env_value", strategyStr).
|
||||
Str("strategy", strategyToString(strategy)).
|
||||
Msg("Vector storage strategy from environment")
|
||||
|
||||
return strategy
|
||||
}
|
||||
|
||||
// GetHubThresholdFromEnv reads CLAUDE_MNEMONIC_HUB_THRESHOLD from environment
|
||||
func GetHubThresholdFromEnv() int {
|
||||
thresholdStr := os.Getenv("CLAUDE_MNEMONIC_HUB_THRESHOLD")
|
||||
if thresholdStr == "" {
|
||||
return 5 // Default threshold
|
||||
}
|
||||
|
||||
threshold, err := strconv.Atoi(thresholdStr)
|
||||
if err != nil {
|
||||
log.Warn().
|
||||
Err(err).
|
||||
Str("env_value", thresholdStr).
|
||||
Msg("Invalid hub threshold in environment, using default")
|
||||
return 5
|
||||
}
|
||||
|
||||
if threshold < 1 {
|
||||
log.Warn().
|
||||
Int("env_value", threshold).
|
||||
Msg("Hub threshold too low, using minimum of 1")
|
||||
return 1
|
||||
}
|
||||
|
||||
log.Info().
|
||||
Int("threshold", threshold).
|
||||
Msg("Hub threshold from environment")
|
||||
|
||||
return threshold
|
||||
}
|
||||
|
||||
// IsHybridEnabled checks if hybrid storage should be used
|
||||
// Returns false if CLAUDE_MNEMONIC_VECTOR_STRATEGY=always (backwards compat)
|
||||
func IsHybridEnabled() bool {
|
||||
strategy := GetStrategyFromEnv()
|
||||
return strategy != StorageAlways
|
||||
}
|
||||
@@ -1,308 +0,0 @@
|
||||
package hybrid
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sort"
|
||||
"time"
|
||||
|
||||
"github.com/lukaszraczylo/claude-mnemonic/internal/graph"
|
||||
"github.com/lukaszraczylo/claude-mnemonic/internal/vector/sqlitevec"
|
||||
"github.com/lukaszraczylo/claude-mnemonic/pkg/models"
|
||||
"github.com/rs/zerolog/log"
|
||||
)
|
||||
|
||||
// GraphConfig configures graph-aware search
|
||||
type GraphConfig struct {
|
||||
Enabled bool
|
||||
MaxHops int // Maximum graph traversal depth (default: 2)
|
||||
BranchFactor int // Number of neighbors to expand per node (default: 5)
|
||||
EdgeWeight float64 // Minimum edge weight to follow (default: 0.3)
|
||||
}
|
||||
|
||||
// DefaultGraphConfig returns sensible defaults for graph search
|
||||
func DefaultGraphConfig() GraphConfig {
|
||||
return GraphConfig{
|
||||
Enabled: true,
|
||||
MaxHops: 2,
|
||||
BranchFactor: 5,
|
||||
EdgeWeight: 0.3,
|
||||
}
|
||||
}
|
||||
|
||||
// GraphSearchClient wraps hybrid.Client with graph-aware search
|
||||
type GraphSearchClient struct {
|
||||
*Client
|
||||
graph *graph.ObservationGraph
|
||||
graphConfig GraphConfig
|
||||
}
|
||||
|
||||
// NewGraphSearchClient creates a graph-enhanced hybrid client
|
||||
func NewGraphSearchClient(baseClient *Client, observationGraph *graph.ObservationGraph, cfg GraphConfig) *GraphSearchClient {
|
||||
return &GraphSearchClient{
|
||||
Client: baseClient,
|
||||
graph: observationGraph,
|
||||
graphConfig: cfg,
|
||||
}
|
||||
}
|
||||
|
||||
// Query performs graph-aware vector search with two-level traversal
|
||||
func (g *GraphSearchClient) Query(ctx context.Context, query string, limit int, where map[string]any) ([]sqlitevec.QueryResult, error) {
|
||||
if !g.graphConfig.Enabled || g.graph == nil {
|
||||
// Fall back to standard hybrid search
|
||||
return g.Client.Query(ctx, query, limit, where)
|
||||
}
|
||||
|
||||
startTime := time.Now()
|
||||
|
||||
// 1. Generate query embedding
|
||||
queryEmb, err := g.embedSvc.Embed(query)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("embed query: %w", err)
|
||||
}
|
||||
|
||||
// 2. Search hub nodes (stored embeddings)
|
||||
hubResults, err := g.base.Query(ctx, query, limit*2, where)
|
||||
if err != nil {
|
||||
// Fall back to standard search on error
|
||||
log.Warn().Err(err).Msg("Hub search failed, falling back to hybrid search")
|
||||
return g.Client.Query(ctx, query, limit, where)
|
||||
}
|
||||
|
||||
// 3. Track hub access
|
||||
g.trackAccess(hubResults)
|
||||
|
||||
// 4. Expand via graph traversal
|
||||
expandedIDs := g.expandFromHubs(hubResults, limit*4)
|
||||
|
||||
// 5. Filter to non-hubs that need recomputation
|
||||
nonHubIDs := make([]string, 0)
|
||||
for _, id := range expandedIDs {
|
||||
if !g.isHub(id) {
|
||||
nonHubIDs = append(nonHubIDs, id)
|
||||
}
|
||||
}
|
||||
|
||||
// 6. Batch recompute non-hub embeddings
|
||||
recomputedResults, err := g.recomputeAndScore(ctx, query, nonHubIDs)
|
||||
if err != nil {
|
||||
log.Warn().Err(err).Msg("Recomputation failed, using hub results only")
|
||||
recomputedResults = nil
|
||||
}
|
||||
|
||||
// 7. Apply graph-based ranking boost
|
||||
allResults := g.mergeAndRankWithGraph(hubResults, recomputedResults, queryEmb)
|
||||
|
||||
// 8. Return top K
|
||||
if len(allResults) > limit {
|
||||
allResults = allResults[:limit]
|
||||
}
|
||||
|
||||
duration := time.Since(startTime)
|
||||
log.Debug().
|
||||
Dur("duration_ms", duration).
|
||||
Int("hubs", len(hubResults)).
|
||||
Int("expanded", len(expandedIDs)).
|
||||
Int("recomputed", len(recomputedResults)).
|
||||
Int("results", len(allResults)).
|
||||
Msg("Graph search completed")
|
||||
|
||||
return allResults, nil
|
||||
}
|
||||
|
||||
// expandFromHubs traverses graph from hub nodes to find promising candidates
|
||||
func (g *GraphSearchClient) expandFromHubs(hubResults []sqlitevec.QueryResult, maxCandidates int) []string {
|
||||
if g.graph == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
expanded := make(map[string]float64) // doc_id -> relevance score
|
||||
visited := make(map[int64]bool)
|
||||
|
||||
// Start from top hub results
|
||||
for i, result := range hubResults {
|
||||
if i >= g.graphConfig.BranchFactor*2 {
|
||||
break // Limit starting points
|
||||
}
|
||||
|
||||
// Parse observation ID from doc_id
|
||||
obsID := parseObservationID(result.ID)
|
||||
if obsID == 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
// Mark as visited with high relevance (direct match)
|
||||
visited[obsID] = true
|
||||
expanded[result.ID] = result.Similarity
|
||||
|
||||
// Traverse graph from this hub
|
||||
g.traverseGraph(obsID, result.Similarity, 0, expanded, visited)
|
||||
}
|
||||
|
||||
// Convert to sorted list
|
||||
type candidate struct {
|
||||
ID string
|
||||
Relevance float64
|
||||
}
|
||||
|
||||
candidates := make([]candidate, 0, len(expanded))
|
||||
for id, rel := range expanded {
|
||||
candidates = append(candidates, candidate{ID: id, Relevance: rel})
|
||||
}
|
||||
|
||||
// Sort by relevance descending
|
||||
sort.Slice(candidates, func(i, j int) bool {
|
||||
return candidates[i].Relevance > candidates[j].Relevance
|
||||
})
|
||||
|
||||
// Return top candidates
|
||||
if len(candidates) > maxCandidates {
|
||||
candidates = candidates[:maxCandidates]
|
||||
}
|
||||
|
||||
result := make([]string, len(candidates))
|
||||
for i, c := range candidates {
|
||||
result[i] = c.ID
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
// traverseGraph performs depth-limited graph traversal
|
||||
func (g *GraphSearchClient) traverseGraph(nodeID int64, baseRelevance float64, depth int, expanded map[string]float64, visited map[int64]bool) {
|
||||
if depth >= g.graphConfig.MaxHops {
|
||||
return // Max depth reached
|
||||
}
|
||||
|
||||
// Get neighbors from graph
|
||||
neighbors, weights, err := g.graph.GetNeighbors(nodeID)
|
||||
if err != nil {
|
||||
return // No neighbors or error
|
||||
}
|
||||
|
||||
// Traverse top neighbors by weight
|
||||
type neighborWeight struct {
|
||||
ID int64
|
||||
Weight float32
|
||||
}
|
||||
|
||||
neighborList := make([]neighborWeight, len(neighbors))
|
||||
for i := range neighbors {
|
||||
neighborList[i] = neighborWeight{
|
||||
ID: neighbors[i],
|
||||
Weight: weights[i],
|
||||
}
|
||||
}
|
||||
|
||||
// Sort by weight descending
|
||||
sort.Slice(neighborList, func(i, j int) bool {
|
||||
return neighborList[i].Weight > neighborList[j].Weight
|
||||
})
|
||||
|
||||
// Expand top branch_factor neighbors
|
||||
expanded_count := 0
|
||||
for _, nw := range neighborList {
|
||||
if expanded_count >= g.graphConfig.BranchFactor {
|
||||
break
|
||||
}
|
||||
|
||||
// Skip if edge weight too low
|
||||
if float64(nw.Weight) < g.graphConfig.EdgeWeight {
|
||||
continue
|
||||
}
|
||||
|
||||
// Skip if already visited
|
||||
if visited[nw.ID] {
|
||||
continue
|
||||
}
|
||||
visited[nw.ID] = true
|
||||
|
||||
// Calculate propagated relevance (decays with distance)
|
||||
decay := 0.7 // 30% decay per hop
|
||||
propagatedRelevance := baseRelevance * float64(nw.Weight) * decay
|
||||
|
||||
// Add to expanded set
|
||||
docID := formatObservationDocID(nw.ID)
|
||||
if existing, ok := expanded[docID]; !ok || propagatedRelevance > existing {
|
||||
expanded[docID] = propagatedRelevance
|
||||
}
|
||||
|
||||
// Recursively traverse
|
||||
g.traverseGraph(nw.ID, propagatedRelevance, depth+1, expanded, visited)
|
||||
expanded_count++
|
||||
}
|
||||
}
|
||||
|
||||
// mergeAndRankWithGraph combines hub and recomputed results with graph-based ranking
|
||||
func (g *GraphSearchClient) mergeAndRankWithGraph(hubResults, recomputedResults []sqlitevec.QueryResult, queryEmb []float32) []sqlitevec.QueryResult {
|
||||
// Merge results
|
||||
allResults := append(hubResults, recomputedResults...)
|
||||
|
||||
// Apply graph-based re-ranking
|
||||
if g.graph != nil {
|
||||
for i := range allResults {
|
||||
obsID := parseObservationID(allResults[i].ID)
|
||||
if obsID == 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
// Boost score based on node degree (hubs are more important)
|
||||
node, err := g.graph.GetNode(obsID)
|
||||
if err == nil && node.Degree > 0 {
|
||||
// Degree boost: up to 10% increase for high-degree nodes
|
||||
degreeBoost := 1.0 + (0.1 * float64(node.Degree) / 20.0)
|
||||
if degreeBoost > 1.1 {
|
||||
degreeBoost = 1.1
|
||||
}
|
||||
allResults[i].Similarity *= degreeBoost
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Sort by adjusted similarity
|
||||
sortBySimilarity(allResults)
|
||||
|
||||
return allResults
|
||||
}
|
||||
|
||||
// parseObservationID extracts observation ID from doc_id
|
||||
// Format: "obs-{id}-{field}"
|
||||
func parseObservationID(docID string) int64 {
|
||||
var obsID int64
|
||||
// Ignore error - returns 0 on parse failure, which callers handle
|
||||
_, _ = fmt.Sscanf(docID, "obs-%d-", &obsID)
|
||||
return obsID
|
||||
}
|
||||
|
||||
// formatObservationDocID creates a doc_id for an observation
|
||||
func formatObservationDocID(obsID int64) string {
|
||||
return fmt.Sprintf("obs-%d-combined", obsID)
|
||||
}
|
||||
|
||||
// GetGraphStats returns statistics about the observation graph
|
||||
func (g *GraphSearchClient) GetGraphStats() graph.GraphStats {
|
||||
if g.graph == nil {
|
||||
return graph.GraphStats{}
|
||||
}
|
||||
return g.graph.Stats()
|
||||
}
|
||||
|
||||
// RebuildGraph rebuilds the observation graph from current observations
|
||||
// This should be called periodically or when observations change significantly
|
||||
func (g *GraphSearchClient) RebuildGraph(ctx context.Context, observations []*models.Observation) error {
|
||||
log.Info().Int("observations", len(observations)).Msg("Rebuilding observation graph")
|
||||
|
||||
newGraph, err := graph.BuildFromObservations(ctx, observations)
|
||||
if err != nil {
|
||||
return fmt.Errorf("build graph: %w", err)
|
||||
}
|
||||
|
||||
g.graph = newGraph
|
||||
|
||||
log.Info().
|
||||
Int("nodes", newGraph.Stats().NodeCount).
|
||||
Int("edges", newGraph.Stats().EdgeCount).
|
||||
Msg("Graph rebuilt successfully")
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -1,16 +0,0 @@
|
||||
package hybrid
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/lukaszraczylo/claude-mnemonic/internal/vector"
|
||||
)
|
||||
|
||||
// TestInterfaceImplementation verifies that hybrid clients implement vector.Client interface
|
||||
func TestInterfaceImplementation(t *testing.T) {
|
||||
// Compile-time check that Client implements vector.Client
|
||||
var _ vector.Client = (*Client)(nil)
|
||||
|
||||
// Compile-time check that GraphSearchClient implements vector.Client
|
||||
var _ vector.Client = (*GraphSearchClient)(nil)
|
||||
}
|
||||
@@ -1,272 +0,0 @@
|
||||
package hybrid
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Metrics tracks performance and usage statistics for hybrid vector storage
|
||||
type Metrics struct {
|
||||
startTime time.Time
|
||||
recentLatencies []time.Duration
|
||||
latenciesMu sync.Mutex
|
||||
totalQueries atomic.Int64
|
||||
hubOnlyQueries atomic.Int64
|
||||
hybridQueries atomic.Int64
|
||||
onDemandQueries atomic.Int64
|
||||
graphQueries atomic.Int64
|
||||
totalLatency atomic.Int64 // Sum in microseconds
|
||||
hubLatency atomic.Int64
|
||||
recomputeLatency atomic.Int64
|
||||
totalDocuments atomic.Int64
|
||||
hubDocuments atomic.Int64
|
||||
storedEmbeddings atomic.Int64
|
||||
recomputedCount atomic.Int64
|
||||
cacheHits atomic.Int64
|
||||
cacheMisses atomic.Int64
|
||||
graphTraversals atomic.Int64
|
||||
avgTraversalDepth atomic.Int64
|
||||
}
|
||||
|
||||
// NewMetrics creates a new metrics tracker
|
||||
func NewMetrics() *Metrics {
|
||||
return &Metrics{
|
||||
recentLatencies: make([]time.Duration, 0, 1000),
|
||||
startTime: time.Now(),
|
||||
}
|
||||
}
|
||||
|
||||
// RecordQuery records a query execution
|
||||
func (m *Metrics) RecordQuery(queryType string, latency time.Duration, recomputed int) {
|
||||
m.totalQueries.Add(1)
|
||||
m.totalLatency.Add(latency.Microseconds())
|
||||
|
||||
switch queryType {
|
||||
case "hub_only":
|
||||
m.hubOnlyQueries.Add(1)
|
||||
case "hybrid":
|
||||
m.hybridQueries.Add(1)
|
||||
case "on_demand":
|
||||
m.onDemandQueries.Add(1)
|
||||
case "graph":
|
||||
m.graphQueries.Add(1)
|
||||
}
|
||||
|
||||
if recomputed > 0 {
|
||||
m.recomputedCount.Add(int64(recomputed))
|
||||
}
|
||||
|
||||
// Track recent latencies
|
||||
m.latenciesMu.Lock()
|
||||
m.recentLatencies = append(m.recentLatencies, latency)
|
||||
if len(m.recentLatencies) > 1000 {
|
||||
m.recentLatencies = m.recentLatencies[len(m.recentLatencies)-1000:]
|
||||
}
|
||||
m.latenciesMu.Unlock()
|
||||
}
|
||||
|
||||
// RecordHubLatency records time spent in hub search
|
||||
func (m *Metrics) RecordHubLatency(latency time.Duration) {
|
||||
m.hubLatency.Add(latency.Microseconds())
|
||||
}
|
||||
|
||||
// RecordRecomputeLatency records time spent recomputing embeddings
|
||||
func (m *Metrics) RecordRecomputeLatency(latency time.Duration) {
|
||||
m.recomputeLatency.Add(latency.Microseconds())
|
||||
}
|
||||
|
||||
// RecordCacheHit records a content cache hit
|
||||
func (m *Metrics) RecordCacheHit() {
|
||||
m.cacheHits.Add(1)
|
||||
}
|
||||
|
||||
// RecordCacheMiss records a content cache miss
|
||||
func (m *Metrics) RecordCacheMiss() {
|
||||
m.cacheMisses.Add(1)
|
||||
}
|
||||
|
||||
// RecordGraphTraversal records a graph traversal operation
|
||||
func (m *Metrics) RecordGraphTraversal(depth int) {
|
||||
m.graphTraversals.Add(1)
|
||||
m.avgTraversalDepth.Add(int64(depth))
|
||||
}
|
||||
|
||||
// UpdateStorageStats updates current storage statistics
|
||||
func (m *Metrics) UpdateStorageStats(total, hubs, stored int) {
|
||||
m.totalDocuments.Store(int64(total))
|
||||
m.hubDocuments.Store(int64(hubs))
|
||||
m.storedEmbeddings.Store(int64(stored))
|
||||
}
|
||||
|
||||
// GetSnapshot returns current metrics snapshot
|
||||
func (m *Metrics) GetSnapshot() MetricsSnapshot {
|
||||
m.latenciesMu.Lock()
|
||||
defer m.latenciesMu.Unlock()
|
||||
|
||||
totalQueries := m.totalQueries.Load()
|
||||
|
||||
snapshot := MetricsSnapshot{
|
||||
// Query counts
|
||||
TotalQueries: totalQueries,
|
||||
HubOnlyQueries: m.hubOnlyQueries.Load(),
|
||||
HybridQueries: m.hybridQueries.Load(),
|
||||
OnDemandQueries: m.onDemandQueries.Load(),
|
||||
GraphQueries: m.graphQueries.Load(),
|
||||
|
||||
// Storage
|
||||
TotalDocuments: int(m.totalDocuments.Load()),
|
||||
HubDocuments: int(m.hubDocuments.Load()),
|
||||
StoredEmbeddings: int(m.storedEmbeddings.Load()),
|
||||
RecomputedTotal: m.recomputedCount.Load(),
|
||||
|
||||
// Cache
|
||||
CacheHits: m.cacheHits.Load(),
|
||||
CacheMisses: m.cacheMisses.Load(),
|
||||
|
||||
// Graph
|
||||
GraphTraversals: m.graphTraversals.Load(),
|
||||
|
||||
// Runtime
|
||||
Uptime: time.Since(m.startTime),
|
||||
}
|
||||
|
||||
// Calculate latencies
|
||||
if totalQueries > 0 {
|
||||
snapshot.AvgLatency = time.Duration(m.totalLatency.Load()/totalQueries) * time.Microsecond
|
||||
snapshot.AvgHubLatency = time.Duration(m.hubLatency.Load()/totalQueries) * time.Microsecond
|
||||
}
|
||||
|
||||
if m.recomputedCount.Load() > 0 {
|
||||
snapshot.AvgRecomputeLatency = time.Duration(m.recomputeLatency.Load()/m.recomputedCount.Load()) * time.Microsecond
|
||||
}
|
||||
|
||||
// Calculate percentiles
|
||||
if len(m.recentLatencies) > 0 {
|
||||
sorted := make([]time.Duration, len(m.recentLatencies))
|
||||
copy(sorted, m.recentLatencies)
|
||||
sortDurations(sorted)
|
||||
|
||||
snapshot.P50Latency = percentile(sorted, 0.50)
|
||||
snapshot.P95Latency = percentile(sorted, 0.95)
|
||||
snapshot.P99Latency = percentile(sorted, 0.99)
|
||||
}
|
||||
|
||||
// Calculate cache hit rate
|
||||
totalCacheOps := snapshot.CacheHits + snapshot.CacheMisses
|
||||
if totalCacheOps > 0 {
|
||||
snapshot.CacheHitRate = float64(snapshot.CacheHits) / float64(totalCacheOps)
|
||||
}
|
||||
|
||||
// Calculate storage savings
|
||||
if snapshot.TotalDocuments > 0 {
|
||||
embeddingSize := 384 * 4 // 384 dims × 4 bytes
|
||||
fullStorage := snapshot.TotalDocuments * embeddingSize
|
||||
actualStorage := snapshot.StoredEmbeddings * embeddingSize
|
||||
|
||||
if fullStorage > 0 {
|
||||
snapshot.StorageSavingsPercent = (1.0 - float64(actualStorage)/float64(fullStorage)) * 100
|
||||
}
|
||||
}
|
||||
|
||||
// Calculate avg traversal depth
|
||||
if snapshot.GraphTraversals > 0 {
|
||||
snapshot.AvgTraversalDepth = float64(m.avgTraversalDepth.Load()) / float64(snapshot.GraphTraversals)
|
||||
}
|
||||
|
||||
return snapshot
|
||||
}
|
||||
|
||||
// MetricsSnapshot represents a point-in-time metrics snapshot
|
||||
type MetricsSnapshot struct {
|
||||
// Query metrics
|
||||
TotalQueries int64
|
||||
HubOnlyQueries int64
|
||||
HybridQueries int64
|
||||
OnDemandQueries int64
|
||||
GraphQueries int64
|
||||
|
||||
// Latency metrics
|
||||
AvgLatency time.Duration
|
||||
P50Latency time.Duration
|
||||
P95Latency time.Duration
|
||||
P99Latency time.Duration
|
||||
AvgHubLatency time.Duration
|
||||
AvgRecomputeLatency time.Duration
|
||||
|
||||
// Storage metrics
|
||||
TotalDocuments int
|
||||
HubDocuments int
|
||||
StoredEmbeddings int
|
||||
StorageSavingsPercent float64
|
||||
RecomputedTotal int64
|
||||
|
||||
// Cache metrics
|
||||
CacheHits int64
|
||||
CacheMisses int64
|
||||
CacheHitRate float64
|
||||
|
||||
// Graph metrics
|
||||
GraphTraversals int64
|
||||
AvgTraversalDepth float64
|
||||
|
||||
// Runtime
|
||||
Uptime time.Duration
|
||||
}
|
||||
|
||||
// sortDurations sorts a slice of durations in ascending order
|
||||
func sortDurations(durations []time.Duration) {
|
||||
n := len(durations)
|
||||
for i := 0; i < n-1; i++ {
|
||||
for j := 0; j < n-i-1; j++ {
|
||||
if durations[j] > durations[j+1] {
|
||||
durations[j], durations[j+1] = durations[j+1], durations[j]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// percentile calculates the Nth percentile from a sorted slice
|
||||
func percentile(sorted []time.Duration, p float64) time.Duration {
|
||||
if len(sorted) == 0 {
|
||||
return 0
|
||||
}
|
||||
|
||||
idx := int(float64(len(sorted)) * p)
|
||||
if idx >= len(sorted) {
|
||||
idx = len(sorted) - 1
|
||||
}
|
||||
|
||||
return sorted[idx]
|
||||
}
|
||||
|
||||
// String returns a human-readable representation of metrics
|
||||
func (s MetricsSnapshot) String() string {
|
||||
return fmt.Sprintf(`Hybrid Vector Storage Metrics:
|
||||
Queries:
|
||||
Total: %d (Hub: %d, Hybrid: %d, OnDemand: %d, Graph: %d)
|
||||
Avg Latency: %v (p50: %v, p95: %v, p99: %v)
|
||||
Hub Latency: %v, Recompute Latency: %v
|
||||
Storage:
|
||||
Documents: %d (Hubs: %d, %.1f%%)
|
||||
Stored Embeddings: %d
|
||||
Savings: %.1f%%
|
||||
Total Recomputed: %d
|
||||
Cache:
|
||||
Hits: %d, Misses: %d (Hit Rate: %.1f%%)
|
||||
Graph:
|
||||
Traversals: %d (Avg Depth: %.2f)
|
||||
Runtime: %v`,
|
||||
s.TotalQueries, s.HubOnlyQueries, s.HybridQueries, s.OnDemandQueries, s.GraphQueries,
|
||||
s.AvgLatency, s.P50Latency, s.P95Latency, s.P99Latency,
|
||||
s.AvgHubLatency, s.AvgRecomputeLatency,
|
||||
s.TotalDocuments, s.HubDocuments, float64(s.HubDocuments)/float64(s.TotalDocuments)*100,
|
||||
s.StoredEmbeddings,
|
||||
s.StorageSavingsPercent,
|
||||
s.RecomputedTotal,
|
||||
s.CacheHits, s.CacheMisses, s.CacheHitRate*100,
|
||||
s.GraphTraversals, s.AvgTraversalDepth,
|
||||
s.Uptime,
|
||||
)
|
||||
}
|
||||
@@ -1,42 +0,0 @@
|
||||
// Package vector provides common interfaces for vector storage implementations
|
||||
package vector
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/lukaszraczylo/claude-mnemonic/internal/vector/sqlitevec"
|
||||
)
|
||||
|
||||
// Client defines the interface for vector storage operations.
|
||||
// Both sqlitevec.Client and hybrid.Client implement this interface.
|
||||
type Client interface {
|
||||
// AddDocuments adds documents with their embeddings to the vector store
|
||||
AddDocuments(ctx context.Context, docs []sqlitevec.Document) error
|
||||
|
||||
// DeleteDocuments removes documents by their IDs
|
||||
DeleteDocuments(ctx context.Context, ids []string) error
|
||||
|
||||
// Query performs a vector similarity search
|
||||
Query(ctx context.Context, query string, limit int, where map[string]any) ([]sqlitevec.QueryResult, error)
|
||||
|
||||
// IsConnected checks if the vector store is available
|
||||
IsConnected() bool
|
||||
|
||||
// Close releases resources
|
||||
Close() error
|
||||
|
||||
// Count returns the total number of vectors in the store
|
||||
Count(ctx context.Context) (int64, error)
|
||||
|
||||
// ModelVersion returns the current embedding model version
|
||||
ModelVersion() string
|
||||
|
||||
// NeedsRebuild checks if vectors need to be rebuilt due to model version change
|
||||
NeedsRebuild(ctx context.Context) (bool, string)
|
||||
|
||||
// GetStaleVectors returns doc_ids of vectors with mismatched or null model versions
|
||||
GetStaleVectors(ctx context.Context) ([]sqlitevec.StaleVectorInfo, error)
|
||||
|
||||
// DeleteVectorsByDocIDs removes vectors by their doc_ids
|
||||
DeleteVectorsByDocIDs(ctx context.Context, docIDs []string) error
|
||||
}
|
||||
@@ -60,15 +60,12 @@ func (c *Client) AddDocuments(ctx context.Context, docs []Document) error {
|
||||
return fmt.Errorf("generate embeddings: %w", err)
|
||||
}
|
||||
|
||||
// Insert into vectors table with model version tracking
|
||||
// Insert into vectors table
|
||||
const insertQuery = `
|
||||
INSERT OR REPLACE INTO vectors (doc_id, embedding, sqlite_id, doc_type, field_type, project, scope, model_version)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
||||
INSERT OR REPLACE INTO vectors (doc_id, embedding, sqlite_id, doc_type, field_type, project, scope)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?)
|
||||
`
|
||||
|
||||
// Get current model version for tracking
|
||||
modelVersion := c.embedSvc.Version()
|
||||
|
||||
tx, err := c.db.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return fmt.Errorf("begin transaction: %w", err)
|
||||
@@ -87,9 +84,9 @@ func (c *Client) AddDocuments(ctx context.Context, docs []Document) error {
|
||||
|
||||
for i, doc := range docs {
|
||||
// Serialize embedding to blob format
|
||||
embBlob, serErr := sqlite_vec.SerializeFloat32(embeddings[i])
|
||||
if serErr != nil {
|
||||
return fmt.Errorf("serialize embedding for %s: %w", doc.ID, serErr)
|
||||
embBlob, err := sqlite_vec.SerializeFloat32(embeddings[i])
|
||||
if err != nil {
|
||||
return fmt.Errorf("serialize embedding for %s: %w", doc.ID, err)
|
||||
}
|
||||
|
||||
// Extract metadata
|
||||
@@ -107,7 +104,6 @@ func (c *Client) AddDocuments(ctx context.Context, docs []Document) error {
|
||||
fieldType,
|
||||
project,
|
||||
scope,
|
||||
modelVersion,
|
||||
)
|
||||
if err != nil {
|
||||
return fmt.Errorf("insert document %s: %w", doc.ID, err)
|
||||
@@ -118,7 +114,7 @@ func (c *Client) AddDocuments(ctx context.Context, docs []Document) error {
|
||||
return fmt.Errorf("commit transaction: %w", err)
|
||||
}
|
||||
|
||||
log.Debug().Int("count", len(docs)).Str("model", modelVersion).Msg("Added documents to sqlite-vec")
|
||||
log.Debug().Int("count", len(docs)).Msg("Added documents to sqlite-vec")
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -212,11 +208,10 @@ func (c *Client) Query(ctx context.Context, query string, limit int, where map[s
|
||||
var sqliteID int64
|
||||
var docType, fieldType, project, scope sql.NullString
|
||||
|
||||
if scanErr := rows.Scan(&r.ID, &r.Distance, &sqliteID, &docType, &fieldType, &project, &scope); scanErr != nil {
|
||||
return nil, fmt.Errorf("scan row: %w", scanErr)
|
||||
if err := rows.Scan(&r.ID, &r.Distance, &sqliteID, &docType, &fieldType, &project, &scope); err != nil {
|
||||
return nil, fmt.Errorf("scan row: %w", err)
|
||||
}
|
||||
|
||||
r.Similarity = DistanceToSimilarity(r.Distance)
|
||||
r.Metadata = map[string]any{
|
||||
"sqlite_id": float64(sqliteID), // Keep as float64 for compatibility
|
||||
"doc_type": docType.String,
|
||||
@@ -257,148 +252,3 @@ func truncateString(s string, maxLen int) string {
|
||||
}
|
||||
return s[:maxLen] + "..."
|
||||
}
|
||||
|
||||
// Count returns the total number of vectors in the store.
|
||||
func (c *Client) Count(ctx context.Context) (int64, error) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
|
||||
var count int64
|
||||
err := c.db.QueryRowContext(ctx, "SELECT COUNT(*) FROM vectors").Scan(&count)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("count vectors: %w", err)
|
||||
}
|
||||
return count, nil
|
||||
}
|
||||
|
||||
// ModelVersion returns the current embedding model version.
|
||||
func (c *Client) ModelVersion() string {
|
||||
return c.embedSvc.Version()
|
||||
}
|
||||
|
||||
// NeedsRebuild checks if vectors need to be rebuilt due to model version change.
|
||||
// Returns true if:
|
||||
// - The vectors table is empty
|
||||
// - Any vectors have a different model_version than the current model
|
||||
func (c *Client) NeedsRebuild(ctx context.Context) (bool, string) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
|
||||
currentModel := c.embedSvc.Version()
|
||||
|
||||
// Check total count
|
||||
var totalCount int64
|
||||
err := c.db.QueryRowContext(ctx, "SELECT COUNT(*) FROM vectors").Scan(&totalCount)
|
||||
if err != nil {
|
||||
log.Warn().Err(err).Msg("Failed to count vectors for rebuild check")
|
||||
return false, ""
|
||||
}
|
||||
|
||||
if totalCount == 0 {
|
||||
return true, "empty"
|
||||
}
|
||||
|
||||
// Check for vectors with different model version
|
||||
var staleCount int64
|
||||
err = c.db.QueryRowContext(ctx,
|
||||
"SELECT COUNT(*) FROM vectors WHERE model_version != ? OR model_version IS NULL",
|
||||
currentModel,
|
||||
).Scan(&staleCount)
|
||||
if err != nil {
|
||||
log.Warn().Err(err).Msg("Failed to count stale vectors")
|
||||
return false, ""
|
||||
}
|
||||
|
||||
if staleCount > 0 {
|
||||
return true, fmt.Sprintf("model_mismatch:%d", staleCount)
|
||||
}
|
||||
|
||||
return false, ""
|
||||
}
|
||||
|
||||
// StaleVectorInfo contains information about a vector that needs rebuilding.
|
||||
type StaleVectorInfo struct {
|
||||
DocID string
|
||||
DocType string
|
||||
FieldType string
|
||||
Project string
|
||||
Scope string
|
||||
SQLiteID int64
|
||||
}
|
||||
|
||||
// GetStaleVectors returns doc_ids of vectors with mismatched or null model versions.
|
||||
// This enables granular rebuild - only re-embedding documents that need updating.
|
||||
func (c *Client) GetStaleVectors(ctx context.Context) ([]StaleVectorInfo, error) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
|
||||
currentModel := c.embedSvc.Version()
|
||||
|
||||
query := `
|
||||
SELECT doc_id, sqlite_id, doc_type, field_type, project, scope
|
||||
FROM vectors
|
||||
WHERE model_version != ? OR model_version IS NULL
|
||||
`
|
||||
|
||||
rows, err := c.db.QueryContext(ctx, query, currentModel)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("query stale vectors: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var results []StaleVectorInfo
|
||||
for rows.Next() {
|
||||
var info StaleVectorInfo
|
||||
var sqliteID sql.NullInt64
|
||||
var docType, fieldType, project, scope sql.NullString
|
||||
|
||||
if scanErr := rows.Scan(&info.DocID, &sqliteID, &docType, &fieldType, &project, &scope); scanErr != nil {
|
||||
return nil, fmt.Errorf("scan row: %w", scanErr)
|
||||
}
|
||||
|
||||
info.SQLiteID = sqliteID.Int64
|
||||
info.DocType = docType.String
|
||||
info.FieldType = fieldType.String
|
||||
info.Project = project.String
|
||||
info.Scope = scope.String
|
||||
|
||||
results = append(results, info)
|
||||
}
|
||||
|
||||
if err = rows.Err(); err != nil {
|
||||
return nil, fmt.Errorf("iterate rows: %w", err)
|
||||
}
|
||||
|
||||
return results, nil
|
||||
}
|
||||
|
||||
// DeleteVectorsByDocIDs removes vectors by their doc_ids.
|
||||
// Used for granular rebuild - delete stale vectors before re-adding.
|
||||
func (c *Client) DeleteVectorsByDocIDs(ctx context.Context, docIDs []string) error {
|
||||
if len(docIDs) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
|
||||
// Build placeholder string
|
||||
placeholders := make([]string, len(docIDs))
|
||||
args := make([]interface{}, len(docIDs))
|
||||
for i, id := range docIDs {
|
||||
placeholders[i] = "?"
|
||||
args[i] = id
|
||||
}
|
||||
|
||||
// #nosec G201 -- Placeholders are "?" strings, actual values are parameterized via args
|
||||
query := fmt.Sprintf("DELETE FROM vectors WHERE doc_id IN (%s)",
|
||||
strings.Join(placeholders, ","))
|
||||
|
||||
_, err := c.db.ExecContext(ctx, query, args...)
|
||||
if err != nil {
|
||||
return fmt.Errorf("delete vectors by doc_ids: %w", err)
|
||||
}
|
||||
|
||||
log.Debug().Int("count", len(docIDs)).Msg("Deleted stale vectors by doc_id")
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -1,503 +0,0 @@
|
||||
package sqlitevec
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
sqlite_vec "github.com/asg017/sqlite-vec-go-bindings/cgo"
|
||||
"github.com/lukaszraczylo/claude-mnemonic/internal/embedding"
|
||||
_ "github.com/mattn/go-sqlite3"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// testDB creates a test SQLite database with the vectors table.
|
||||
func testDB(t *testing.T) (*sql.DB, func()) {
|
||||
t.Helper()
|
||||
|
||||
// Create temp directory
|
||||
tmpDir, err := os.MkdirTemp("", "sqlitevec-test-*")
|
||||
require.NoError(t, err)
|
||||
|
||||
dbPath := filepath.Join(tmpDir, "test.db")
|
||||
db, err := sql.Open("sqlite3", dbPath)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Enable sqlite-vec
|
||||
sqlite_vec.Auto()
|
||||
|
||||
// Create vectors table (matches production schema)
|
||||
_, err = db.Exec(`
|
||||
CREATE VIRTUAL TABLE IF NOT EXISTS vectors USING vec0(
|
||||
doc_id TEXT PRIMARY KEY,
|
||||
embedding float[384],
|
||||
sqlite_id INTEGER,
|
||||
doc_type TEXT,
|
||||
field_type TEXT,
|
||||
project TEXT,
|
||||
scope TEXT,
|
||||
model_version TEXT
|
||||
)
|
||||
`)
|
||||
require.NoError(t, err)
|
||||
|
||||
cleanup := func() {
|
||||
db.Close()
|
||||
os.RemoveAll(tmpDir)
|
||||
}
|
||||
|
||||
return db, cleanup
|
||||
}
|
||||
|
||||
// testEmbeddingService creates a test embedding service.
|
||||
func testEmbeddingService(t *testing.T) (*embedding.Service, func()) {
|
||||
t.Helper()
|
||||
|
||||
svc, err := embedding.NewService()
|
||||
require.NoError(t, err)
|
||||
|
||||
cleanup := func() {
|
||||
svc.Close()
|
||||
}
|
||||
|
||||
return svc, cleanup
|
||||
}
|
||||
|
||||
func TestNewClient_Success(t *testing.T) {
|
||||
db, dbCleanup := testDB(t)
|
||||
defer dbCleanup()
|
||||
|
||||
embedSvc, embedCleanup := testEmbeddingService(t)
|
||||
defer embedCleanup()
|
||||
|
||||
client, err := NewClient(Config{DB: db}, embedSvc)
|
||||
require.NoError(t, err)
|
||||
assert.NotNil(t, client)
|
||||
}
|
||||
|
||||
func TestNewClient_NilDB(t *testing.T) {
|
||||
embedSvc, embedCleanup := testEmbeddingService(t)
|
||||
defer embedCleanup()
|
||||
|
||||
client, err := NewClient(Config{DB: nil}, embedSvc)
|
||||
assert.Error(t, err)
|
||||
assert.Nil(t, client)
|
||||
assert.Contains(t, err.Error(), "database connection required")
|
||||
}
|
||||
|
||||
func TestNewClient_NilEmbedding(t *testing.T) {
|
||||
db, dbCleanup := testDB(t)
|
||||
defer dbCleanup()
|
||||
|
||||
client, err := NewClient(Config{DB: db}, nil)
|
||||
assert.Error(t, err)
|
||||
assert.Nil(t, client)
|
||||
assert.Contains(t, err.Error(), "embedding service required")
|
||||
}
|
||||
|
||||
func TestClient_AddDocuments_Empty(t *testing.T) {
|
||||
db, dbCleanup := testDB(t)
|
||||
defer dbCleanup()
|
||||
|
||||
embedSvc, embedCleanup := testEmbeddingService(t)
|
||||
defer embedCleanup()
|
||||
|
||||
client, err := NewClient(Config{DB: db}, embedSvc)
|
||||
require.NoError(t, err)
|
||||
|
||||
err = client.AddDocuments(context.Background(), []Document{})
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
func TestClient_AddDocuments_Single(t *testing.T) {
|
||||
db, dbCleanup := testDB(t)
|
||||
defer dbCleanup()
|
||||
|
||||
embedSvc, embedCleanup := testEmbeddingService(t)
|
||||
defer embedCleanup()
|
||||
|
||||
client, err := NewClient(Config{DB: db}, embedSvc)
|
||||
require.NoError(t, err)
|
||||
|
||||
docs := []Document{
|
||||
{
|
||||
ID: "obs-1-title",
|
||||
Content: "This is a test observation about authentication.",
|
||||
Metadata: map[string]any{
|
||||
"sqlite_id": int64(1),
|
||||
"doc_type": "observation",
|
||||
"field_type": "title",
|
||||
"project": "test-project",
|
||||
"scope": "project",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
err = client.AddDocuments(context.Background(), docs)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Verify document was inserted
|
||||
var count int
|
||||
err = db.QueryRow("SELECT COUNT(*) FROM vectors WHERE doc_id = ?", "obs-1-title").Scan(&count)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 1, count)
|
||||
}
|
||||
|
||||
func TestClient_AddDocuments_Multiple(t *testing.T) {
|
||||
db, dbCleanup := testDB(t)
|
||||
defer dbCleanup()
|
||||
|
||||
embedSvc, embedCleanup := testEmbeddingService(t)
|
||||
defer embedCleanup()
|
||||
|
||||
client, err := NewClient(Config{DB: db}, embedSvc)
|
||||
require.NoError(t, err)
|
||||
|
||||
docs := []Document{
|
||||
{
|
||||
ID: "obs-1-title",
|
||||
Content: "Authentication flow implementation.",
|
||||
Metadata: map[string]any{
|
||||
"sqlite_id": int64(1),
|
||||
"doc_type": "observation",
|
||||
"field_type": "title",
|
||||
"project": "test-project",
|
||||
"scope": "project",
|
||||
},
|
||||
},
|
||||
{
|
||||
ID: "obs-1-narrative",
|
||||
Content: "We implemented JWT-based authentication.",
|
||||
Metadata: map[string]any{
|
||||
"sqlite_id": int64(1),
|
||||
"doc_type": "observation",
|
||||
"field_type": "narrative",
|
||||
"project": "test-project",
|
||||
"scope": "project",
|
||||
},
|
||||
},
|
||||
{
|
||||
ID: "obs-2-title",
|
||||
Content: "Database optimization.",
|
||||
Metadata: map[string]any{
|
||||
"sqlite_id": int64(2),
|
||||
"doc_type": "observation",
|
||||
"field_type": "title",
|
||||
"project": "test-project",
|
||||
"scope": "global",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
err = client.AddDocuments(context.Background(), docs)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Verify all documents were inserted
|
||||
var count int
|
||||
err = db.QueryRow("SELECT COUNT(*) FROM vectors").Scan(&count)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 3, count)
|
||||
}
|
||||
|
||||
func TestClient_DeleteDocuments_Empty(t *testing.T) {
|
||||
db, dbCleanup := testDB(t)
|
||||
defer dbCleanup()
|
||||
|
||||
embedSvc, embedCleanup := testEmbeddingService(t)
|
||||
defer embedCleanup()
|
||||
|
||||
client, err := NewClient(Config{DB: db}, embedSvc)
|
||||
require.NoError(t, err)
|
||||
|
||||
err = client.DeleteDocuments(context.Background(), []string{})
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
func TestClient_DeleteDocuments_Existing(t *testing.T) {
|
||||
db, dbCleanup := testDB(t)
|
||||
defer dbCleanup()
|
||||
|
||||
embedSvc, embedCleanup := testEmbeddingService(t)
|
||||
defer embedCleanup()
|
||||
|
||||
client, err := NewClient(Config{DB: db}, embedSvc)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Add documents first
|
||||
docs := []Document{
|
||||
{
|
||||
ID: "doc-1",
|
||||
Content: "First document.",
|
||||
Metadata: map[string]any{
|
||||
"sqlite_id": int64(1),
|
||||
"doc_type": "observation",
|
||||
},
|
||||
},
|
||||
{
|
||||
ID: "doc-2",
|
||||
Content: "Second document.",
|
||||
Metadata: map[string]any{
|
||||
"sqlite_id": int64(2),
|
||||
"doc_type": "observation",
|
||||
},
|
||||
},
|
||||
}
|
||||
err = client.AddDocuments(context.Background(), docs)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Delete one document
|
||||
err = client.DeleteDocuments(context.Background(), []string{"doc-1"})
|
||||
require.NoError(t, err)
|
||||
|
||||
// Verify only one remains
|
||||
var count int
|
||||
err = db.QueryRow("SELECT COUNT(*) FROM vectors").Scan(&count)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 1, count)
|
||||
}
|
||||
|
||||
func TestClient_Query_Basic(t *testing.T) {
|
||||
db, dbCleanup := testDB(t)
|
||||
defer dbCleanup()
|
||||
|
||||
embedSvc, embedCleanup := testEmbeddingService(t)
|
||||
defer embedCleanup()
|
||||
|
||||
client, err := NewClient(Config{DB: db}, embedSvc)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Add some test documents
|
||||
docs := []Document{
|
||||
{
|
||||
ID: "obs-1",
|
||||
Content: "Authentication and login security implementation.",
|
||||
Metadata: map[string]any{
|
||||
"sqlite_id": int64(1),
|
||||
"doc_type": "observation",
|
||||
"project": "test-project",
|
||||
"scope": "project",
|
||||
},
|
||||
},
|
||||
{
|
||||
ID: "obs-2",
|
||||
Content: "Database query optimization techniques.",
|
||||
Metadata: map[string]any{
|
||||
"sqlite_id": int64(2),
|
||||
"doc_type": "observation",
|
||||
"project": "test-project",
|
||||
"scope": "project",
|
||||
},
|
||||
},
|
||||
}
|
||||
err = client.AddDocuments(context.Background(), docs)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Query for authentication-related content
|
||||
results, err := client.Query(context.Background(), "login authentication", 10, nil)
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.NotEmpty(t, results)
|
||||
assert.LessOrEqual(t, len(results), 10)
|
||||
|
||||
// First result should be the authentication document (higher similarity)
|
||||
assert.Equal(t, "obs-1", results[0].ID)
|
||||
}
|
||||
|
||||
func TestClient_Query_WithDocTypeFilter(t *testing.T) {
|
||||
db, dbCleanup := testDB(t)
|
||||
defer dbCleanup()
|
||||
|
||||
embedSvc, embedCleanup := testEmbeddingService(t)
|
||||
defer embedCleanup()
|
||||
|
||||
client, err := NewClient(Config{DB: db}, embedSvc)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Add documents of different types
|
||||
docs := []Document{
|
||||
{
|
||||
ID: "obs-1",
|
||||
Content: "Test content for observation.",
|
||||
Metadata: map[string]any{
|
||||
"sqlite_id": int64(1),
|
||||
"doc_type": "observation",
|
||||
"project": "test-project",
|
||||
},
|
||||
},
|
||||
{
|
||||
ID: "summary-1",
|
||||
Content: "Test content for summary.",
|
||||
Metadata: map[string]any{
|
||||
"sqlite_id": int64(10),
|
||||
"doc_type": "session_summary",
|
||||
"project": "test-project",
|
||||
},
|
||||
},
|
||||
}
|
||||
err = client.AddDocuments(context.Background(), docs)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Query with doc_type filter
|
||||
where := map[string]any{"doc_type": "observation"}
|
||||
results, err := client.Query(context.Background(), "test content", 10, where)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Should only return observation documents
|
||||
for _, r := range results {
|
||||
docType, _ := r.Metadata["doc_type"].(string)
|
||||
assert.Equal(t, "observation", docType)
|
||||
}
|
||||
}
|
||||
|
||||
func TestClient_Query_WithProjectFilter(t *testing.T) {
|
||||
db, dbCleanup := testDB(t)
|
||||
defer dbCleanup()
|
||||
|
||||
embedSvc, embedCleanup := testEmbeddingService(t)
|
||||
defer embedCleanup()
|
||||
|
||||
client, err := NewClient(Config{DB: db}, embedSvc)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Add documents from different projects
|
||||
docs := []Document{
|
||||
{
|
||||
ID: "obs-1",
|
||||
Content: "Project A authentication content.",
|
||||
Metadata: map[string]any{
|
||||
"sqlite_id": int64(1),
|
||||
"doc_type": "observation",
|
||||
"project": "project-a",
|
||||
"scope": "project",
|
||||
},
|
||||
},
|
||||
{
|
||||
ID: "obs-2",
|
||||
Content: "Project B database content.",
|
||||
Metadata: map[string]any{
|
||||
"sqlite_id": int64(2),
|
||||
"doc_type": "observation",
|
||||
"project": "project-b",
|
||||
"scope": "project",
|
||||
},
|
||||
},
|
||||
{
|
||||
ID: "obs-3",
|
||||
Content: "Global security best practices.",
|
||||
Metadata: map[string]any{
|
||||
"sqlite_id": int64(3),
|
||||
"doc_type": "observation",
|
||||
"project": "project-b",
|
||||
"scope": "global",
|
||||
},
|
||||
},
|
||||
}
|
||||
err = client.AddDocuments(context.Background(), docs)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Query without project filter to verify all docs are there
|
||||
results, err := client.Query(context.Background(), "authentication security", 10, nil)
|
||||
require.NoError(t, err)
|
||||
assert.NotEmpty(t, results, "Should find some results")
|
||||
}
|
||||
|
||||
func TestClient_IsConnected(t *testing.T) {
|
||||
db, dbCleanup := testDB(t)
|
||||
defer dbCleanup()
|
||||
|
||||
embedSvc, embedCleanup := testEmbeddingService(t)
|
||||
defer embedCleanup()
|
||||
|
||||
client, err := NewClient(Config{DB: db}, embedSvc)
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.True(t, client.IsConnected())
|
||||
}
|
||||
|
||||
func TestClient_Close(t *testing.T) {
|
||||
db, dbCleanup := testDB(t)
|
||||
defer dbCleanup()
|
||||
|
||||
embedSvc, embedCleanup := testEmbeddingService(t)
|
||||
defer embedCleanup()
|
||||
|
||||
client, err := NewClient(Config{DB: db}, embedSvc)
|
||||
require.NoError(t, err)
|
||||
|
||||
err = client.Close()
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
func TestConfig_Fields(t *testing.T) {
|
||||
db, dbCleanup := testDB(t)
|
||||
defer dbCleanup()
|
||||
|
||||
cfg := Config{DB: db}
|
||||
assert.Equal(t, db, cfg.DB)
|
||||
}
|
||||
|
||||
func TestClient_UpdateDocument_DeleteThenAdd(t *testing.T) {
|
||||
db, dbCleanup := testDB(t)
|
||||
defer dbCleanup()
|
||||
|
||||
embedSvc, embedCleanup := testEmbeddingService(t)
|
||||
defer embedCleanup()
|
||||
|
||||
client, err := NewClient(Config{DB: db}, embedSvc)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Add document
|
||||
docs1 := []Document{
|
||||
{
|
||||
ID: "doc-1",
|
||||
Content: "Original content.",
|
||||
Metadata: map[string]any{
|
||||
"sqlite_id": int64(1),
|
||||
"doc_type": "observation",
|
||||
},
|
||||
},
|
||||
}
|
||||
err = client.AddDocuments(context.Background(), docs1)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Delete then add with new content (proper update pattern)
|
||||
err = client.DeleteDocuments(context.Background(), []string{"doc-1"})
|
||||
require.NoError(t, err)
|
||||
|
||||
docs2 := []Document{
|
||||
{
|
||||
ID: "doc-1",
|
||||
Content: "Updated content.",
|
||||
Metadata: map[string]any{
|
||||
"sqlite_id": int64(1),
|
||||
"doc_type": "observation",
|
||||
},
|
||||
},
|
||||
}
|
||||
err = client.AddDocuments(context.Background(), docs2)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Should have exactly 1 document
|
||||
var count int
|
||||
err = db.QueryRow("SELECT COUNT(*) FROM vectors WHERE doc_id = ?", "doc-1").Scan(&count)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 1, count)
|
||||
}
|
||||
|
||||
func TestClient_DeleteDocuments_NonExistent(t *testing.T) {
|
||||
db, dbCleanup := testDB(t)
|
||||
defer dbCleanup()
|
||||
|
||||
embedSvc, embedCleanup := testEmbeddingService(t)
|
||||
defer embedCleanup()
|
||||
|
||||
client, err := NewClient(Config{DB: db}, embedSvc)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Deleting non-existent document should not error
|
||||
err = client.DeleteDocuments(context.Background(), []string{"non-existent-id"})
|
||||
require.NoError(t, err)
|
||||
}
|
||||
@@ -8,44 +8,20 @@ const (
|
||||
DocTypeObservation DocType = "observation"
|
||||
DocTypeSessionSummary DocType = "session_summary"
|
||||
DocTypeUserPrompt DocType = "user_prompt"
|
||||
DocTypeCodeChunk DocType = "code_chunk"
|
||||
)
|
||||
|
||||
// Document represents a document to store with vector embedding.
|
||||
type Document struct {
|
||||
Metadata map[string]any
|
||||
ID string
|
||||
Content string
|
||||
Metadata map[string]any
|
||||
}
|
||||
|
||||
// QueryResult represents a search result from vector search.
|
||||
type QueryResult struct {
|
||||
Metadata map[string]any
|
||||
ID string
|
||||
Distance float64
|
||||
Similarity float64
|
||||
}
|
||||
|
||||
// DistanceToSimilarity converts sqlite-vec cosine distance to similarity score.
|
||||
// Cosine distance: 0 = identical, 2 = opposite
|
||||
// Similarity: 1.0 = identical, 0.0 = opposite
|
||||
func DistanceToSimilarity(distance float64) float64 {
|
||||
return 1.0 - (distance / 2.0)
|
||||
}
|
||||
|
||||
// FilterByThreshold filters results to only include those above the similarity threshold.
|
||||
// If maxResults > 0, also caps the number of results.
|
||||
func FilterByThreshold(results []QueryResult, threshold float64, maxResults int) []QueryResult {
|
||||
var filtered []QueryResult
|
||||
for _, r := range results {
|
||||
if r.Similarity >= threshold {
|
||||
filtered = append(filtered, r)
|
||||
if maxResults > 0 && len(filtered) >= maxResults {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
return filtered
|
||||
ID string
|
||||
Distance float64
|
||||
Metadata map[string]any
|
||||
}
|
||||
|
||||
// ExtractedIDs contains SQLite IDs extracted from query results, grouped by document type.
|
||||
|
||||
@@ -1,574 +0,0 @@
|
||||
package sqlitevec
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestDocTypes(t *testing.T) {
|
||||
assert.Equal(t, DocType("observation"), DocTypeObservation)
|
||||
assert.Equal(t, DocType("session_summary"), DocTypeSessionSummary)
|
||||
assert.Equal(t, DocType("user_prompt"), DocTypeUserPrompt)
|
||||
}
|
||||
|
||||
func TestDocument_Fields(t *testing.T) {
|
||||
doc := Document{
|
||||
ID: "doc-123",
|
||||
Content: "test content",
|
||||
Metadata: map[string]any{
|
||||
"key": "value",
|
||||
},
|
||||
}
|
||||
|
||||
assert.Equal(t, "doc-123", doc.ID)
|
||||
assert.Equal(t, "test content", doc.Content)
|
||||
assert.Equal(t, "value", doc.Metadata["key"])
|
||||
}
|
||||
|
||||
func TestQueryResult_Fields(t *testing.T) {
|
||||
result := QueryResult{
|
||||
ID: "result-123",
|
||||
Distance: 0.5,
|
||||
Metadata: map[string]any{
|
||||
"sqlite_id": float64(42),
|
||||
},
|
||||
}
|
||||
|
||||
assert.Equal(t, "result-123", result.ID)
|
||||
assert.Equal(t, 0.5, result.Distance)
|
||||
assert.Equal(t, float64(42), result.Metadata["sqlite_id"])
|
||||
}
|
||||
|
||||
func TestBuildWhereFilter(t *testing.T) {
|
||||
tests := []struct {
|
||||
expected map[string]interface{}
|
||||
name string
|
||||
docType DocType
|
||||
project string
|
||||
}{
|
||||
{
|
||||
name: "empty_filters",
|
||||
docType: "",
|
||||
project: "",
|
||||
expected: map[string]interface{}{},
|
||||
},
|
||||
{
|
||||
name: "doc_type_only",
|
||||
docType: DocTypeObservation,
|
||||
project: "",
|
||||
expected: map[string]interface{}{
|
||||
"doc_type": "observation",
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "project_only",
|
||||
docType: "",
|
||||
project: "my-project",
|
||||
expected: map[string]interface{}{
|
||||
"project": "my-project",
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "both_filters",
|
||||
docType: DocTypeSessionSummary,
|
||||
project: "test-project",
|
||||
expected: map[string]interface{}{
|
||||
"doc_type": "session_summary",
|
||||
"project": "test-project",
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "user_prompt_type",
|
||||
docType: DocTypeUserPrompt,
|
||||
project: "prompt-project",
|
||||
expected: map[string]interface{}{
|
||||
"doc_type": "user_prompt",
|
||||
"project": "prompt-project",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
result := BuildWhereFilter(tt.docType, tt.project)
|
||||
assert.Equal(t, tt.expected, result)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractIDsByDocType_Empty(t *testing.T) {
|
||||
results := []QueryResult{}
|
||||
ids := ExtractIDsByDocType(results)
|
||||
|
||||
assert.Empty(t, ids.ObservationIDs)
|
||||
assert.Empty(t, ids.SummaryIDs)
|
||||
assert.Empty(t, ids.PromptIDs)
|
||||
}
|
||||
|
||||
func TestExtractIDsByDocType_AllTypes(t *testing.T) {
|
||||
results := []QueryResult{
|
||||
{
|
||||
ID: "obs-1",
|
||||
Distance: 0.1,
|
||||
Metadata: map[string]any{
|
||||
"sqlite_id": float64(1),
|
||||
"doc_type": "observation",
|
||||
},
|
||||
},
|
||||
{
|
||||
ID: "obs-2",
|
||||
Distance: 0.2,
|
||||
Metadata: map[string]any{
|
||||
"sqlite_id": float64(2),
|
||||
"doc_type": "observation",
|
||||
},
|
||||
},
|
||||
{
|
||||
ID: "summary-1",
|
||||
Distance: 0.3,
|
||||
Metadata: map[string]any{
|
||||
"sqlite_id": float64(10),
|
||||
"doc_type": "session_summary",
|
||||
},
|
||||
},
|
||||
{
|
||||
ID: "prompt-1",
|
||||
Distance: 0.4,
|
||||
Metadata: map[string]any{
|
||||
"sqlite_id": float64(20),
|
||||
"doc_type": "user_prompt",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
ids := ExtractIDsByDocType(results)
|
||||
|
||||
assert.Equal(t, []int64{1, 2}, ids.ObservationIDs)
|
||||
assert.Equal(t, []int64{10}, ids.SummaryIDs)
|
||||
assert.Equal(t, []int64{20}, ids.PromptIDs)
|
||||
}
|
||||
|
||||
func TestExtractIDsByDocType_Deduplication(t *testing.T) {
|
||||
results := []QueryResult{
|
||||
{
|
||||
ID: "obs-1-field1",
|
||||
Distance: 0.1,
|
||||
Metadata: map[string]any{
|
||||
"sqlite_id": float64(1),
|
||||
"doc_type": "observation",
|
||||
},
|
||||
},
|
||||
{
|
||||
ID: "obs-1-field2",
|
||||
Distance: 0.2,
|
||||
Metadata: map[string]any{
|
||||
"sqlite_id": float64(1), // Same ID, different field
|
||||
"doc_type": "observation",
|
||||
},
|
||||
},
|
||||
{
|
||||
ID: "obs-2",
|
||||
Distance: 0.3,
|
||||
Metadata: map[string]any{
|
||||
"sqlite_id": float64(2),
|
||||
"doc_type": "observation",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
ids := ExtractIDsByDocType(results)
|
||||
|
||||
assert.Equal(t, []int64{1, 2}, ids.ObservationIDs) // Should be deduplicated
|
||||
}
|
||||
|
||||
func TestExtractIDsByDocType_Int64Fallback(t *testing.T) {
|
||||
results := []QueryResult{
|
||||
{
|
||||
ID: "obs-1",
|
||||
Distance: 0.1,
|
||||
Metadata: map[string]any{
|
||||
"sqlite_id": int64(42), // int64 instead of float64
|
||||
"doc_type": "observation",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
ids := ExtractIDsByDocType(results)
|
||||
|
||||
assert.Equal(t, []int64{42}, ids.ObservationIDs)
|
||||
}
|
||||
|
||||
func TestExtractIDsByDocType_MissingSqliteID(t *testing.T) {
|
||||
results := []QueryResult{
|
||||
{
|
||||
ID: "obs-1",
|
||||
Distance: 0.1,
|
||||
Metadata: map[string]any{
|
||||
"doc_type": "observation",
|
||||
// Missing sqlite_id
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
ids := ExtractIDsByDocType(results)
|
||||
|
||||
assert.Empty(t, ids.ObservationIDs)
|
||||
}
|
||||
|
||||
func TestExtractIDsByDocType_UnknownType(t *testing.T) {
|
||||
results := []QueryResult{
|
||||
{
|
||||
ID: "unknown-1",
|
||||
Distance: 0.1,
|
||||
Metadata: map[string]any{
|
||||
"sqlite_id": float64(1),
|
||||
"doc_type": "unknown_type",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
ids := ExtractIDsByDocType(results)
|
||||
|
||||
// Should not be added to any category
|
||||
assert.Empty(t, ids.ObservationIDs)
|
||||
assert.Empty(t, ids.SummaryIDs)
|
||||
assert.Empty(t, ids.PromptIDs)
|
||||
}
|
||||
|
||||
func TestExtractObservationIDs_NoFilter(t *testing.T) {
|
||||
results := []QueryResult{
|
||||
{
|
||||
ID: "obs-1",
|
||||
Metadata: map[string]any{
|
||||
"sqlite_id": float64(1),
|
||||
"doc_type": "observation",
|
||||
"project": "proj-a",
|
||||
},
|
||||
},
|
||||
{
|
||||
ID: "obs-2",
|
||||
Metadata: map[string]any{
|
||||
"sqlite_id": float64(2),
|
||||
"doc_type": "observation",
|
||||
"project": "proj-b",
|
||||
},
|
||||
},
|
||||
{
|
||||
ID: "summary-1",
|
||||
Metadata: map[string]any{
|
||||
"sqlite_id": float64(10),
|
||||
"doc_type": "session_summary",
|
||||
"project": "proj-a",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
ids := ExtractObservationIDs(results, "")
|
||||
|
||||
assert.Equal(t, []int64{1, 2}, ids)
|
||||
}
|
||||
|
||||
func TestExtractObservationIDs_WithProjectFilter(t *testing.T) {
|
||||
results := []QueryResult{
|
||||
{
|
||||
ID: "obs-1",
|
||||
Metadata: map[string]any{
|
||||
"sqlite_id": float64(1),
|
||||
"doc_type": "observation",
|
||||
"project": "proj-a",
|
||||
"scope": "project",
|
||||
},
|
||||
},
|
||||
{
|
||||
ID: "obs-2",
|
||||
Metadata: map[string]any{
|
||||
"sqlite_id": float64(2),
|
||||
"doc_type": "observation",
|
||||
"project": "proj-b",
|
||||
"scope": "project",
|
||||
},
|
||||
},
|
||||
{
|
||||
ID: "obs-global",
|
||||
Metadata: map[string]any{
|
||||
"sqlite_id": float64(3),
|
||||
"doc_type": "observation",
|
||||
"project": "proj-b",
|
||||
"scope": "global",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
ids := ExtractObservationIDs(results, "proj-a")
|
||||
|
||||
// Should include proj-a and global scope observations
|
||||
assert.Equal(t, []int64{1, 3}, ids)
|
||||
}
|
||||
|
||||
func TestExtractObservationIDs_Deduplication(t *testing.T) {
|
||||
results := []QueryResult{
|
||||
{
|
||||
ID: "obs-1-field1",
|
||||
Metadata: map[string]any{
|
||||
"sqlite_id": float64(1),
|
||||
"doc_type": "observation",
|
||||
},
|
||||
},
|
||||
{
|
||||
ID: "obs-1-field2",
|
||||
Metadata: map[string]any{
|
||||
"sqlite_id": float64(1), // Same ID
|
||||
"doc_type": "observation",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
ids := ExtractObservationIDs(results, "")
|
||||
|
||||
assert.Equal(t, []int64{1}, ids)
|
||||
}
|
||||
|
||||
func TestExtractSummaryIDs_NoFilter(t *testing.T) {
|
||||
results := []QueryResult{
|
||||
{
|
||||
ID: "summary-1",
|
||||
Metadata: map[string]any{
|
||||
"sqlite_id": float64(10),
|
||||
"doc_type": "session_summary",
|
||||
"project": "proj-a",
|
||||
},
|
||||
},
|
||||
{
|
||||
ID: "summary-2",
|
||||
Metadata: map[string]any{
|
||||
"sqlite_id": float64(20),
|
||||
"doc_type": "session_summary",
|
||||
"project": "proj-b",
|
||||
},
|
||||
},
|
||||
{
|
||||
ID: "obs-1",
|
||||
Metadata: map[string]any{
|
||||
"sqlite_id": float64(1),
|
||||
"doc_type": "observation",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
ids := ExtractSummaryIDs(results, "")
|
||||
|
||||
assert.Equal(t, []int64{10, 20}, ids)
|
||||
}
|
||||
|
||||
func TestExtractSummaryIDs_WithProjectFilter(t *testing.T) {
|
||||
results := []QueryResult{
|
||||
{
|
||||
ID: "summary-1",
|
||||
Metadata: map[string]any{
|
||||
"sqlite_id": float64(10),
|
||||
"doc_type": "session_summary",
|
||||
"project": "proj-a",
|
||||
},
|
||||
},
|
||||
{
|
||||
ID: "summary-2",
|
||||
Metadata: map[string]any{
|
||||
"sqlite_id": float64(20),
|
||||
"doc_type": "session_summary",
|
||||
"project": "proj-b",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
ids := ExtractSummaryIDs(results, "proj-a")
|
||||
|
||||
assert.Equal(t, []int64{10}, ids)
|
||||
}
|
||||
|
||||
func TestExtractPromptIDs_NoFilter(t *testing.T) {
|
||||
results := []QueryResult{
|
||||
{
|
||||
ID: "prompt-1",
|
||||
Metadata: map[string]any{
|
||||
"sqlite_id": float64(100),
|
||||
"doc_type": "user_prompt",
|
||||
"project": "proj-a",
|
||||
},
|
||||
},
|
||||
{
|
||||
ID: "prompt-2",
|
||||
Metadata: map[string]any{
|
||||
"sqlite_id": float64(200),
|
||||
"doc_type": "user_prompt",
|
||||
"project": "proj-b",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
ids := ExtractPromptIDs(results, "")
|
||||
|
||||
assert.Equal(t, []int64{100, 200}, ids)
|
||||
}
|
||||
|
||||
func TestExtractPromptIDs_WithProjectFilter(t *testing.T) {
|
||||
results := []QueryResult{
|
||||
{
|
||||
ID: "prompt-1",
|
||||
Metadata: map[string]any{
|
||||
"sqlite_id": float64(100),
|
||||
"doc_type": "user_prompt",
|
||||
"project": "proj-a",
|
||||
},
|
||||
},
|
||||
{
|
||||
ID: "prompt-2",
|
||||
Metadata: map[string]any{
|
||||
"sqlite_id": float64(200),
|
||||
"doc_type": "user_prompt",
|
||||
"project": "proj-b",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
ids := ExtractPromptIDs(results, "proj-b")
|
||||
|
||||
assert.Equal(t, []int64{200}, ids)
|
||||
}
|
||||
|
||||
func TestCopyMetadata(t *testing.T) {
|
||||
base := map[string]any{
|
||||
"key1": "value1",
|
||||
"key2": 42,
|
||||
}
|
||||
|
||||
result := copyMetadata(base, "key3", "value3")
|
||||
|
||||
// Original should be unchanged
|
||||
assert.Len(t, base, 2)
|
||||
|
||||
// Result should have new key
|
||||
assert.Len(t, result, 3)
|
||||
assert.Equal(t, "value1", result["key1"])
|
||||
assert.Equal(t, 42, result["key2"])
|
||||
assert.Equal(t, "value3", result["key3"])
|
||||
}
|
||||
|
||||
func TestCopyMetadataMulti(t *testing.T) {
|
||||
base := map[string]any{
|
||||
"key1": "value1",
|
||||
}
|
||||
extra := map[string]any{
|
||||
"key2": "value2",
|
||||
"key3": "value3",
|
||||
}
|
||||
|
||||
result := copyMetadataMulti(base, extra)
|
||||
|
||||
assert.Len(t, result, 3)
|
||||
assert.Equal(t, "value1", result["key1"])
|
||||
assert.Equal(t, "value2", result["key2"])
|
||||
assert.Equal(t, "value3", result["key3"])
|
||||
}
|
||||
|
||||
func TestJoinStrings(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
sep string
|
||||
expected string
|
||||
strs []string
|
||||
}{
|
||||
{
|
||||
name: "empty_slice",
|
||||
strs: []string{},
|
||||
sep: ", ",
|
||||
expected: "",
|
||||
},
|
||||
{
|
||||
name: "single_element",
|
||||
strs: []string{"one"},
|
||||
sep: ", ",
|
||||
expected: "one",
|
||||
},
|
||||
{
|
||||
name: "multiple_elements",
|
||||
strs: []string{"one", "two", "three"},
|
||||
sep: ", ",
|
||||
expected: "one, two, three",
|
||||
},
|
||||
{
|
||||
name: "different_separator",
|
||||
strs: []string{"a", "b", "c"},
|
||||
sep: "-",
|
||||
expected: "a-b-c",
|
||||
},
|
||||
{
|
||||
name: "empty_separator",
|
||||
strs: []string{"a", "b", "c"},
|
||||
sep: "",
|
||||
expected: "abc",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
result := joinStrings(tt.strs, tt.sep)
|
||||
assert.Equal(t, tt.expected, result)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestTruncateString(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
input string
|
||||
expected string
|
||||
maxLen int
|
||||
}{
|
||||
{
|
||||
name: "shorter_than_max",
|
||||
input: "hello",
|
||||
maxLen: 10,
|
||||
expected: "hello",
|
||||
},
|
||||
{
|
||||
name: "equal_to_max",
|
||||
input: "hello",
|
||||
maxLen: 5,
|
||||
expected: "hello",
|
||||
},
|
||||
{
|
||||
name: "longer_than_max",
|
||||
input: "hello world",
|
||||
maxLen: 5,
|
||||
expected: "hello...",
|
||||
},
|
||||
{
|
||||
name: "empty_string",
|
||||
input: "",
|
||||
maxLen: 5,
|
||||
expected: "",
|
||||
},
|
||||
{
|
||||
name: "zero_max_length",
|
||||
input: "hello",
|
||||
maxLen: 0,
|
||||
expected: "...",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
result := truncateString(tt.input, tt.maxLen)
|
||||
assert.Equal(t, tt.expected, result)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractedIDs_Empty(t *testing.T) {
|
||||
ids := &ExtractedIDs{}
|
||||
|
||||
assert.Nil(t, ids.ObservationIDs)
|
||||
assert.Nil(t, ids.SummaryIDs)
|
||||
assert.Nil(t, ids.PromptIDs)
|
||||
}
|
||||
@@ -5,15 +5,13 @@ import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"github.com/lukaszraczylo/claude-mnemonic/internal/chunking"
|
||||
"github.com/lukaszraczylo/claude-mnemonic/pkg/models"
|
||||
"github.com/rs/zerolog/log"
|
||||
)
|
||||
|
||||
// Sync provides synchronization between SQLite data and vector embeddings.
|
||||
type Sync struct {
|
||||
client *Client
|
||||
chunkingManager *chunking.Manager
|
||||
client *Client
|
||||
}
|
||||
|
||||
// NewSync creates a new sync service.
|
||||
@@ -21,23 +19,9 @@ func NewSync(client *Client) *Sync {
|
||||
return &Sync{client: client}
|
||||
}
|
||||
|
||||
// SetChunkingManager sets the code chunking manager (optional).
|
||||
// If set, observations will include code chunks from tracked files.
|
||||
func (s *Sync) SetChunkingManager(manager *chunking.Manager) {
|
||||
s.chunkingManager = manager
|
||||
}
|
||||
|
||||
// SyncObservation syncs a single observation to the vector store.
|
||||
// If a chunking manager is configured, also chunks tracked code files.
|
||||
func (s *Sync) SyncObservation(ctx context.Context, obs *models.Observation) error {
|
||||
docs := s.formatObservationDocs(obs)
|
||||
|
||||
// Add code chunks from tracked files if chunking manager is available
|
||||
if s.chunkingManager != nil {
|
||||
codeChunkDocs := s.formatCodeChunkDocs(ctx, obs)
|
||||
docs = append(docs, codeChunkDocs...)
|
||||
}
|
||||
|
||||
if len(docs) == 0 {
|
||||
return nil
|
||||
}
|
||||
@@ -115,98 +99,6 @@ func (s *Sync) formatObservationDocs(obs *models.Observation) []Document {
|
||||
return docs
|
||||
}
|
||||
|
||||
// formatCodeChunkDocs formats code chunks from tracked files into vector documents.
|
||||
// Uses AST-aware chunking to extract semantic code units (functions, classes, methods).
|
||||
func (s *Sync) formatCodeChunkDocs(ctx context.Context, obs *models.Observation) []Document {
|
||||
if s.chunkingManager == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Determine scope for metadata
|
||||
scope := string(obs.Scope)
|
||||
if scope == "" {
|
||||
scope = "project"
|
||||
}
|
||||
|
||||
// Collect all tracked files (read + modified)
|
||||
allFiles := make([]string, 0, len(obs.FilesRead)+len(obs.FilesModified))
|
||||
allFiles = append(allFiles, obs.FilesRead...)
|
||||
allFiles = append(allFiles, obs.FilesModified...)
|
||||
|
||||
// Filter to only files supported by chunking manager
|
||||
var supportedFiles []string
|
||||
for _, file := range allFiles {
|
||||
if s.chunkingManager.SupportsFile(file) {
|
||||
supportedFiles = append(supportedFiles, file)
|
||||
}
|
||||
}
|
||||
|
||||
if len(supportedFiles) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Chunk all supported files
|
||||
results, errs := s.chunkingManager.ChunkFiles(ctx, supportedFiles)
|
||||
if len(errs) > 0 {
|
||||
// Log errors but don't fail the entire sync
|
||||
for _, err := range errs {
|
||||
log.Warn().Err(err).Msg("Failed to chunk file")
|
||||
}
|
||||
}
|
||||
|
||||
// Convert chunks to vector documents
|
||||
docs := make([]Document, 0)
|
||||
chunkIndex := 0
|
||||
|
||||
for filePath, chunks := range results {
|
||||
for _, chunk := range chunks {
|
||||
doc := Document{
|
||||
ID: fmt.Sprintf("obs_%d_chunk_%d", obs.ID, chunkIndex),
|
||||
Content: chunk.SearchableContent(),
|
||||
Metadata: map[string]any{
|
||||
"sqlite_id": obs.ID,
|
||||
"doc_type": string(DocTypeCodeChunk),
|
||||
"field_type": "code_chunk",
|
||||
"sdk_session_id": obs.SDKSessionID,
|
||||
"project": obs.Project,
|
||||
"scope": scope,
|
||||
"created_at_epoch": obs.CreatedAtEpoch,
|
||||
// Code chunk specific metadata
|
||||
"file_path": filePath,
|
||||
"language": string(chunk.Language),
|
||||
"chunk_type": string(chunk.Type),
|
||||
"symbol_name": chunk.Name,
|
||||
"start_line": chunk.StartLine,
|
||||
"end_line": chunk.EndLine,
|
||||
},
|
||||
}
|
||||
|
||||
// Add parent name if this is a method
|
||||
if chunk.ParentName != "" {
|
||||
doc.Metadata["parent_name"] = chunk.ParentName
|
||||
}
|
||||
|
||||
// Add signature if available
|
||||
if chunk.Signature != "" {
|
||||
doc.Metadata["signature"] = chunk.Signature
|
||||
}
|
||||
|
||||
docs = append(docs, doc)
|
||||
chunkIndex++
|
||||
}
|
||||
}
|
||||
|
||||
if len(docs) > 0 {
|
||||
log.Debug().
|
||||
Int64("observationId", obs.ID).
|
||||
Int("codeChunks", len(docs)).
|
||||
Int("files", len(results)).
|
||||
Msg("Generated code chunk documents")
|
||||
}
|
||||
|
||||
return docs
|
||||
}
|
||||
|
||||
// SyncSummary syncs a single session summary to the vector store.
|
||||
func (s *Sync) SyncSummary(ctx context.Context, summary *models.SessionSummary) error {
|
||||
docs := s.formatSummaryDocs(summary)
|
||||
@@ -299,27 +191,21 @@ func (s *Sync) SyncUserPrompt(ctx context.Context, prompt *models.UserPromptWith
|
||||
}
|
||||
|
||||
// DeleteObservations removes observation documents from the vector store.
|
||||
// Includes both observation fields (narrative, facts) and code chunks.
|
||||
func (s *Sync) DeleteObservations(ctx context.Context, observationIDs []int64) error {
|
||||
if len(observationIDs) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Generate all possible document IDs for these observations
|
||||
// Pattern: obs_{id}_narrative, obs_{id}_fact_{0..n}, obs_{id}_chunk_{0..n}
|
||||
// Pattern: obs_{id}_narrative, obs_{id}_fact_{0..n}
|
||||
const maxFactsPerObs = 20
|
||||
const maxChunksPerObs = 100 // Reasonable upper bound for code chunks
|
||||
ids := make([]string, 0, len(observationIDs)*(maxFactsPerObs+maxChunksPerObs+1))
|
||||
ids := make([]string, 0, len(observationIDs)*(maxFactsPerObs+1))
|
||||
|
||||
for _, obsID := range observationIDs {
|
||||
ids = append(ids, fmt.Sprintf("obs_%d_narrative", obsID))
|
||||
for i := 0; i < maxFactsPerObs; i++ {
|
||||
ids = append(ids, fmt.Sprintf("obs_%d_fact_%d", obsID, i))
|
||||
}
|
||||
// Include code chunk IDs
|
||||
for i := 0; i < maxChunksPerObs; i++ {
|
||||
ids = append(ids, fmt.Sprintf("obs_%d_chunk_%d", obsID, i))
|
||||
}
|
||||
}
|
||||
|
||||
if err := s.client.DeleteDocuments(ctx, ids); err != nil {
|
||||
@@ -354,101 +240,3 @@ func (s *Sync) DeleteUserPrompts(ctx context.Context, promptIDs []int64) error {
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// SyncPattern syncs a single pattern to the vector store.
|
||||
func (s *Sync) SyncPattern(ctx context.Context, pattern *models.Pattern) error {
|
||||
docs := s.formatPatternDocs(pattern)
|
||||
if len(docs) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
if err := s.client.AddDocuments(ctx, docs); err != nil {
|
||||
return fmt.Errorf("add pattern docs: %w", err)
|
||||
}
|
||||
|
||||
log.Debug().
|
||||
Int64("patternId", pattern.ID).
|
||||
Int("docCount", len(docs)).
|
||||
Msg("Synced pattern to sqlite-vec")
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// formatPatternDocs formats a pattern into vector documents.
|
||||
func (s *Sync) formatPatternDocs(pattern *models.Pattern) []Document {
|
||||
docs := make([]Document, 0, 3)
|
||||
|
||||
baseMetadata := map[string]any{
|
||||
"sqlite_id": pattern.ID,
|
||||
"doc_type": "pattern",
|
||||
"pattern_type": string(pattern.Type),
|
||||
"status": string(pattern.Status),
|
||||
"scope": "global", // Patterns are always global
|
||||
"frequency": pattern.Frequency,
|
||||
"confidence": pattern.Confidence,
|
||||
"created_at_epoch": pattern.CreatedAtEpoch,
|
||||
}
|
||||
|
||||
if len(pattern.Signature) > 0 {
|
||||
baseMetadata["signature"] = joinStrings(pattern.Signature, ",")
|
||||
}
|
||||
if len(pattern.Projects) > 0 {
|
||||
baseMetadata["projects"] = joinStrings(pattern.Projects, ",")
|
||||
}
|
||||
|
||||
// Pattern name as document
|
||||
if pattern.Name != "" {
|
||||
docs = append(docs, Document{
|
||||
ID: fmt.Sprintf("pattern_%d_name", pattern.ID),
|
||||
Content: pattern.Name,
|
||||
Metadata: copyMetadata(baseMetadata, "field_type", "name"),
|
||||
})
|
||||
}
|
||||
|
||||
// Pattern description as document
|
||||
if pattern.Description.Valid && pattern.Description.String != "" {
|
||||
docs = append(docs, Document{
|
||||
ID: fmt.Sprintf("pattern_%d_description", pattern.ID),
|
||||
Content: pattern.Description.String,
|
||||
Metadata: copyMetadata(baseMetadata, "field_type", "description"),
|
||||
})
|
||||
}
|
||||
|
||||
// Pattern recommendation as document
|
||||
if pattern.Recommendation.Valid && pattern.Recommendation.String != "" {
|
||||
docs = append(docs, Document{
|
||||
ID: fmt.Sprintf("pattern_%d_recommendation", pattern.ID),
|
||||
Content: pattern.Recommendation.String,
|
||||
Metadata: copyMetadata(baseMetadata, "field_type", "recommendation"),
|
||||
})
|
||||
}
|
||||
|
||||
return docs
|
||||
}
|
||||
|
||||
// DeletePatterns removes pattern documents from the vector store.
|
||||
func (s *Sync) DeletePatterns(ctx context.Context, patternIDs []int64) error {
|
||||
if len(patternIDs) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Generate all possible document IDs for these patterns
|
||||
// Pattern: pattern_{id}_name, pattern_{id}_description, pattern_{id}_recommendation
|
||||
ids := make([]string, 0, len(patternIDs)*3)
|
||||
|
||||
for _, patternID := range patternIDs {
|
||||
ids = append(ids, fmt.Sprintf("pattern_%d_name", patternID))
|
||||
ids = append(ids, fmt.Sprintf("pattern_%d_description", patternID))
|
||||
ids = append(ids, fmt.Sprintf("pattern_%d_recommendation", patternID))
|
||||
}
|
||||
|
||||
if err := s.client.DeleteDocuments(ctx, ids); err != nil {
|
||||
return fmt.Errorf("delete pattern docs: %w", err)
|
||||
}
|
||||
|
||||
log.Debug().
|
||||
Int("patternCount", len(patternIDs)).
|
||||
Msg("Deleted patterns from sqlite-vec")
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -1,348 +0,0 @@
|
||||
package sqlitevec
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"testing"
|
||||
|
||||
"github.com/lukaszraczylo/claude-mnemonic/pkg/models"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// testClient creates a test client for sync tests.
|
||||
func testClient(t *testing.T) (*Client, func()) {
|
||||
t.Helper()
|
||||
|
||||
db, dbCleanup := testDB(t)
|
||||
embedSvc, embedCleanup := testEmbeddingService(t)
|
||||
|
||||
client, err := NewClient(Config{DB: db}, embedSvc)
|
||||
require.NoError(t, err)
|
||||
|
||||
cleanup := func() {
|
||||
embedCleanup()
|
||||
dbCleanup()
|
||||
}
|
||||
|
||||
return client, cleanup
|
||||
}
|
||||
|
||||
func TestNewSync(t *testing.T) {
|
||||
client, cleanup := testClient(t)
|
||||
defer cleanup()
|
||||
|
||||
sync := NewSync(client)
|
||||
assert.NotNil(t, sync)
|
||||
}
|
||||
|
||||
func TestSync_SyncObservation_Empty(t *testing.T) {
|
||||
client, cleanup := testClient(t)
|
||||
defer cleanup()
|
||||
|
||||
sync := NewSync(client)
|
||||
|
||||
// Observation with no content should be handled gracefully
|
||||
obs := &models.Observation{
|
||||
ID: 1,
|
||||
SDKSessionID: "test-session",
|
||||
Project: "test-project",
|
||||
Type: models.ObsTypeDiscovery,
|
||||
}
|
||||
|
||||
err := sync.SyncObservation(context.Background(), obs)
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
func TestSync_SyncObservation_WithContent(t *testing.T) {
|
||||
client, cleanup := testClient(t)
|
||||
defer cleanup()
|
||||
|
||||
sync := NewSync(client)
|
||||
|
||||
obs := &models.Observation{
|
||||
ID: 1,
|
||||
SDKSessionID: "test-session",
|
||||
Project: "test-project",
|
||||
Type: models.ObsTypeDiscovery,
|
||||
Scope: models.ScopeProject,
|
||||
Title: sql.NullString{String: "Authentication bug fix", Valid: true},
|
||||
Subtitle: sql.NullString{String: "Fixed JWT validation", Valid: true},
|
||||
Narrative: sql.NullString{String: "Fixed the JWT token validation to handle expired tokens correctly.", Valid: true},
|
||||
Facts: []string{"JWT tokens expire after 24 hours", "Refresh tokens are used for renewal"},
|
||||
Concepts: []string{"authentication", "security"},
|
||||
FilesRead: []string{"auth.go"},
|
||||
FilesModified: []string{"handler.go"},
|
||||
}
|
||||
|
||||
err := sync.SyncObservation(context.Background(), obs)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Verify documents were added
|
||||
results, err := client.Query(context.Background(), "authentication", 10, nil)
|
||||
require.NoError(t, err)
|
||||
assert.GreaterOrEqual(t, len(results), 1)
|
||||
}
|
||||
|
||||
func TestSync_SyncObservation_DefaultScope(t *testing.T) {
|
||||
client, cleanup := testClient(t)
|
||||
defer cleanup()
|
||||
|
||||
sync := NewSync(client)
|
||||
|
||||
// Observation without explicit scope
|
||||
obs := &models.Observation{
|
||||
ID: 2,
|
||||
SDKSessionID: "test-session",
|
||||
Project: "test-project",
|
||||
Type: models.ObsTypeBugfix,
|
||||
Narrative: sql.NullString{String: "Fixed a null pointer exception.", Valid: true},
|
||||
}
|
||||
|
||||
err := sync.SyncObservation(context.Background(), obs)
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
func TestSync_SyncSummary_Empty(t *testing.T) {
|
||||
client, cleanup := testClient(t)
|
||||
defer cleanup()
|
||||
|
||||
sync := NewSync(client)
|
||||
|
||||
// Summary with no content
|
||||
summary := &models.SessionSummary{
|
||||
ID: 1,
|
||||
SDKSessionID: "test-session",
|
||||
Project: "test-project",
|
||||
}
|
||||
|
||||
err := sync.SyncSummary(context.Background(), summary)
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
func TestSync_SyncSummary_WithContent(t *testing.T) {
|
||||
client, cleanup := testClient(t)
|
||||
defer cleanup()
|
||||
|
||||
sync := NewSync(client)
|
||||
|
||||
summary := &models.SessionSummary{
|
||||
ID: 1,
|
||||
SDKSessionID: "test-session",
|
||||
Project: "test-project",
|
||||
Request: sql.NullString{String: "Help me fix the authentication bug", Valid: true},
|
||||
Investigated: sql.NullString{String: "Looked at auth.go and handler.go", Valid: true},
|
||||
Learned: sql.NullString{String: "JWT tokens were not being validated properly", Valid: true},
|
||||
Completed: sql.NullString{String: "Fixed the JWT validation logic", Valid: true},
|
||||
NextSteps: sql.NullString{String: "Add tests for edge cases", Valid: true},
|
||||
Notes: sql.NullString{String: "Consider using a library for JWT handling", Valid: true},
|
||||
PromptNumber: sql.NullInt64{Int64: 1, Valid: true},
|
||||
}
|
||||
|
||||
err := sync.SyncSummary(context.Background(), summary)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Verify documents were added
|
||||
results, err := client.Query(context.Background(), "authentication", 10, nil)
|
||||
require.NoError(t, err)
|
||||
assert.GreaterOrEqual(t, len(results), 1)
|
||||
}
|
||||
|
||||
func TestSync_SyncUserPrompt(t *testing.T) {
|
||||
client, cleanup := testClient(t)
|
||||
defer cleanup()
|
||||
|
||||
sync := NewSync(client)
|
||||
|
||||
prompt := &models.UserPromptWithSession{
|
||||
UserPrompt: models.UserPrompt{
|
||||
ID: 1,
|
||||
PromptNumber: 1,
|
||||
PromptText: "Help me fix the authentication bug in the login handler",
|
||||
CreatedAtEpoch: 1234567890,
|
||||
},
|
||||
SDKSessionID: "test-session",
|
||||
Project: "test-project",
|
||||
}
|
||||
|
||||
err := sync.SyncUserPrompt(context.Background(), prompt)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Verify document was added
|
||||
results, err := client.Query(context.Background(), "authentication", 10, nil)
|
||||
require.NoError(t, err)
|
||||
assert.GreaterOrEqual(t, len(results), 1)
|
||||
}
|
||||
|
||||
func TestSync_DeleteObservations_Empty(t *testing.T) {
|
||||
client, cleanup := testClient(t)
|
||||
defer cleanup()
|
||||
|
||||
sync := NewSync(client)
|
||||
|
||||
// Should handle empty list
|
||||
err := sync.DeleteObservations(context.Background(), []int64{})
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
func TestSync_DeleteObservations_WithData(t *testing.T) {
|
||||
client, cleanup := testClient(t)
|
||||
defer cleanup()
|
||||
|
||||
sync := NewSync(client)
|
||||
|
||||
// First add an observation
|
||||
obs := &models.Observation{
|
||||
ID: 10,
|
||||
SDKSessionID: "test-session",
|
||||
Project: "test-project",
|
||||
Type: models.ObsTypeDiscovery,
|
||||
Narrative: sql.NullString{String: "This observation should be deleted.", Valid: true},
|
||||
Facts: []string{"Fact 1", "Fact 2"},
|
||||
}
|
||||
|
||||
err := sync.SyncObservation(context.Background(), obs)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Then delete it
|
||||
err = sync.DeleteObservations(context.Background(), []int64{10})
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
func TestSync_DeleteUserPrompts_Empty(t *testing.T) {
|
||||
client, cleanup := testClient(t)
|
||||
defer cleanup()
|
||||
|
||||
sync := NewSync(client)
|
||||
|
||||
// Should handle empty list
|
||||
err := sync.DeleteUserPrompts(context.Background(), []int64{})
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
func TestSync_DeleteUserPrompts_WithData(t *testing.T) {
|
||||
client, cleanup := testClient(t)
|
||||
defer cleanup()
|
||||
|
||||
sync := NewSync(client)
|
||||
|
||||
// First add a prompt
|
||||
prompt := &models.UserPromptWithSession{
|
||||
UserPrompt: models.UserPrompt{
|
||||
ID: 20,
|
||||
PromptNumber: 1,
|
||||
PromptText: "This prompt should be deleted.",
|
||||
CreatedAtEpoch: 1234567890,
|
||||
},
|
||||
SDKSessionID: "test-session",
|
||||
Project: "test-project",
|
||||
}
|
||||
|
||||
err := sync.SyncUserPrompt(context.Background(), prompt)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Then delete it
|
||||
err = sync.DeleteUserPrompts(context.Background(), []int64{20})
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
func TestSync_FormatObservationDocs_AllFields(t *testing.T) {
|
||||
client, cleanup := testClient(t)
|
||||
defer cleanup()
|
||||
|
||||
sync := NewSync(client)
|
||||
|
||||
obs := &models.Observation{
|
||||
ID: 100,
|
||||
SDKSessionID: "sdk-123",
|
||||
Project: "my-project",
|
||||
Type: models.ObsTypeFeature,
|
||||
Scope: models.ScopeGlobal,
|
||||
Title: sql.NullString{String: "Feature Title", Valid: true},
|
||||
Subtitle: sql.NullString{String: "Feature Subtitle", Valid: true},
|
||||
Narrative: sql.NullString{String: "Feature narrative content", Valid: true},
|
||||
Facts: []string{"Fact A", "Fact B", "Fact C"},
|
||||
Concepts: []string{"api", "performance"},
|
||||
FilesRead: []string{"file1.go", "file2.go"},
|
||||
FilesModified: []string{"file3.go"},
|
||||
}
|
||||
|
||||
docs := sync.formatObservationDocs(obs)
|
||||
|
||||
// Should have 1 narrative + 3 facts = 4 docs
|
||||
assert.Len(t, docs, 4)
|
||||
|
||||
// Check narrative doc
|
||||
var narrativeDoc *Document
|
||||
for i := range docs {
|
||||
if docs[i].ID == "obs_100_narrative" {
|
||||
narrativeDoc = &docs[i]
|
||||
break
|
||||
}
|
||||
}
|
||||
require.NotNil(t, narrativeDoc)
|
||||
assert.Equal(t, "Feature narrative content", narrativeDoc.Content)
|
||||
assert.Equal(t, int64(100), narrativeDoc.Metadata["sqlite_id"])
|
||||
assert.Equal(t, "observation", narrativeDoc.Metadata["doc_type"])
|
||||
assert.Equal(t, "global", narrativeDoc.Metadata["scope"])
|
||||
assert.Equal(t, "narrative", narrativeDoc.Metadata["field_type"])
|
||||
}
|
||||
|
||||
func TestSync_FormatSummaryDocs_AllFields(t *testing.T) {
|
||||
client, cleanup := testClient(t)
|
||||
defer cleanup()
|
||||
|
||||
sync := NewSync(client)
|
||||
|
||||
summary := &models.SessionSummary{
|
||||
ID: 200,
|
||||
SDKSessionID: "sdk-456",
|
||||
Project: "summary-project",
|
||||
Request: sql.NullString{String: "Request content", Valid: true},
|
||||
Investigated: sql.NullString{String: "Investigated content", Valid: true},
|
||||
Learned: sql.NullString{String: "Learned content", Valid: true},
|
||||
Completed: sql.NullString{String: "Completed content", Valid: true},
|
||||
NextSteps: sql.NullString{String: "Next steps content", Valid: true},
|
||||
Notes: sql.NullString{String: "Notes content", Valid: true},
|
||||
PromptNumber: sql.NullInt64{Int64: 5, Valid: true},
|
||||
}
|
||||
|
||||
docs := sync.formatSummaryDocs(summary)
|
||||
|
||||
// Should have 6 docs (one for each field)
|
||||
assert.Len(t, docs, 6)
|
||||
|
||||
// Check request doc
|
||||
var requestDoc *Document
|
||||
for i := range docs {
|
||||
if docs[i].ID == "summary_200_request" {
|
||||
requestDoc = &docs[i]
|
||||
break
|
||||
}
|
||||
}
|
||||
require.NotNil(t, requestDoc)
|
||||
assert.Equal(t, "Request content", requestDoc.Content)
|
||||
assert.Equal(t, int64(200), requestDoc.Metadata["sqlite_id"])
|
||||
assert.Equal(t, "session_summary", requestDoc.Metadata["doc_type"])
|
||||
assert.Equal(t, int64(5), requestDoc.Metadata["prompt_number"])
|
||||
}
|
||||
|
||||
func TestSync_FormatObservationDocs_EmptyScope(t *testing.T) {
|
||||
client, cleanup := testClient(t)
|
||||
defer cleanup()
|
||||
|
||||
sync := NewSync(client)
|
||||
|
||||
obs := &models.Observation{
|
||||
ID: 300,
|
||||
SDKSessionID: "sdk-789",
|
||||
Project: "scope-test",
|
||||
Type: models.ObsTypeDecision,
|
||||
Narrative: sql.NullString{String: "Test narrative", Valid: true},
|
||||
// Scope intentionally left empty
|
||||
}
|
||||
|
||||
docs := sync.formatObservationDocs(obs)
|
||||
assert.Len(t, docs, 1)
|
||||
assert.Equal(t, "project", docs[0].Metadata["scope"])
|
||||
}
|
||||
@@ -16,15 +16,15 @@ import (
|
||||
// Watcher monitors a file or directory for deletion and calls onDelete when removed.
|
||||
// It watches the parent directory since fsnotify cannot watch non-existent files.
|
||||
type Watcher struct {
|
||||
ctx context.Context
|
||||
onDelete func()
|
||||
targetPath string // The file/directory to watch for deletion
|
||||
parentPath string // Parent directory (what we actually watch)
|
||||
onDelete func() // Callback when target is deleted
|
||||
watcher *fsnotify.Watcher
|
||||
ctx context.Context
|
||||
cancel context.CancelFunc
|
||||
targetPath string
|
||||
parentPath string
|
||||
debounce time.Duration
|
||||
mu sync.Mutex
|
||||
running bool
|
||||
debounce time.Duration
|
||||
}
|
||||
|
||||
// New creates a new Watcher for the given target path.
|
||||
|
||||
+16
-389
@@ -4,18 +4,13 @@ package worker
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"sort"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
"github.com/lukaszraczylo/claude-mnemonic/internal/db/sqlite"
|
||||
"github.com/lukaszraczylo/claude-mnemonic/internal/embedding"
|
||||
"github.com/lukaszraczylo/claude-mnemonic/internal/privacy"
|
||||
"github.com/lukaszraczylo/claude-mnemonic/internal/reranking"
|
||||
"github.com/lukaszraczylo/claude-mnemonic/internal/search/expansion"
|
||||
"github.com/lukaszraczylo/claude-mnemonic/internal/vector/sqlitevec"
|
||||
"github.com/lukaszraczylo/claude-mnemonic/internal/worker/sdk"
|
||||
"github.com/lukaszraczylo/claude-mnemonic/internal/worker/session"
|
||||
@@ -158,19 +153,13 @@ type SessionInitRequest struct {
|
||||
|
||||
// SessionInitResponse is the response for session initialization.
|
||||
type SessionInitResponse struct {
|
||||
Reason string `json:"reason,omitempty"`
|
||||
SessionDBID int64 `json:"sessionDbId"`
|
||||
PromptNumber int `json:"promptNumber"`
|
||||
Skipped bool `json:"skipped,omitempty"`
|
||||
Reason string `json:"reason,omitempty"`
|
||||
}
|
||||
|
||||
// DuplicatePromptWindowSeconds is the time window for detecting duplicate prompt submissions.
|
||||
// If the same prompt text is seen within this window, it's considered a duplicate hook invocation.
|
||||
const DuplicatePromptWindowSeconds = 10
|
||||
|
||||
// handleSessionInit handles session initialization from user-prompt hook.
|
||||
// This handler is idempotent - duplicate requests within a short time window
|
||||
// return the existing prompt data without creating duplicates.
|
||||
func (s *Service) handleSessionInit(w http.ResponseWriter, r *http.Request) {
|
||||
var req SessionInitRequest
|
||||
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
||||
@@ -193,31 +182,8 @@ func (s *Service) handleSessionInit(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
|
||||
// Clean prompt
|
||||
// Clean prompt and create session
|
||||
cleanedPrompt := privacy.Clean(req.Prompt)
|
||||
|
||||
// DUPLICATE DETECTION: Check if this exact prompt was already saved recently.
|
||||
// This prevents the bug where the hook fires multiple times for the same user action,
|
||||
// creating many duplicate prompts with incrementing numbers.
|
||||
if existingID, existingNum, found := s.promptStore.FindRecentPromptByText(r.Context(), req.ClaudeSessionID, cleanedPrompt, DuplicatePromptWindowSeconds); found {
|
||||
// Get or create session (idempotent)
|
||||
sessionID, _ := s.sessionStore.CreateSDKSession(r.Context(), req.ClaudeSessionID, req.Project, cleanedPrompt)
|
||||
|
||||
log.Debug().
|
||||
Int64("sessionId", sessionID).
|
||||
Int("promptNumber", existingNum).
|
||||
Int64("promptId", existingID).
|
||||
Msg("Duplicate prompt detected - returning existing")
|
||||
|
||||
// Return existing prompt data without incrementing or saving again
|
||||
writeJSON(w, SessionInitResponse{
|
||||
SessionDBID: sessionID,
|
||||
PromptNumber: existingNum,
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
// Create session (idempotent)
|
||||
sessionID, err := s.sessionStore.CreateSDKSession(r.Context(), req.ClaudeSessionID, req.Project, cleanedPrompt)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
@@ -296,8 +262,8 @@ func (s *Service) handleSessionStart(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
|
||||
var req SessionStartRequest
|
||||
if decodeErr := json.NewDecoder(r.Body).Decode(&req); decodeErr != nil {
|
||||
http.Error(w, decodeErr.Error(), http.StatusBadRequest)
|
||||
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
||||
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
@@ -512,8 +478,8 @@ func (s *Service) handleGetObservations(w http.ResponseWriter, r *http.Request)
|
||||
// Fall back to SQLite if vector search not used
|
||||
if !usedVector {
|
||||
if project != "" {
|
||||
// Strict project filtering for dashboard - only observations from this project
|
||||
observations, err = s.observationStore.GetObservationsByProjectStrict(r.Context(), project, limit)
|
||||
// Filter by project - includes project-scoped and global observations
|
||||
observations, err = s.observationStore.GetRecentObservations(r.Context(), project, limit)
|
||||
} else {
|
||||
// All projects
|
||||
observations, err = s.observationStore.GetAllRecentObservations(r.Context(), limit)
|
||||
@@ -646,22 +612,9 @@ func (s *Service) handleGetTypes(w http.ResponseWriter, r *http.Request) {
|
||||
})
|
||||
}
|
||||
|
||||
// handleGetModels returns available embedding models.
|
||||
func (s *Service) handleGetModels(w http.ResponseWriter, _ *http.Request) {
|
||||
models := embedding.ListModels()
|
||||
defaultModel := embedding.GetDefaultModel()
|
||||
|
||||
writeJSON(w, map[string]interface{}{
|
||||
"models": models,
|
||||
"default": defaultModel,
|
||||
"current": s.embedSvc.Version(),
|
||||
})
|
||||
}
|
||||
|
||||
// handleGetStats returns worker statistics.
|
||||
func (s *Service) handleGetStats(w http.ResponseWriter, r *http.Request) {
|
||||
project := r.URL.Query().Get("project")
|
||||
retrievalStats := s.GetRetrievalStats(project)
|
||||
retrievalStats := s.GetRetrievalStats()
|
||||
sessionsToday, _ := s.sessionStore.GetSessionsToday(r.Context())
|
||||
|
||||
response := map[string]interface{}{
|
||||
@@ -675,24 +628,8 @@ func (s *Service) handleGetStats(w http.ResponseWriter, r *http.Request) {
|
||||
"ready": s.ready.Load(),
|
||||
}
|
||||
|
||||
// Add embedding model info
|
||||
if s.embedSvc != nil {
|
||||
response["embeddingModel"] = map[string]interface{}{
|
||||
"name": s.embedSvc.Name(),
|
||||
"version": s.embedSvc.Version(),
|
||||
"dimensions": s.embedSvc.Dimensions(),
|
||||
}
|
||||
}
|
||||
|
||||
// Add vector count
|
||||
if s.vectorClient != nil {
|
||||
if count, err := s.vectorClient.Count(r.Context()); err == nil {
|
||||
response["vectorCount"] = count
|
||||
}
|
||||
}
|
||||
|
||||
// Include project-specific observation count if project is specified
|
||||
if project != "" {
|
||||
if project := r.URL.Query().Get("project"); project != "" {
|
||||
count, err := s.observationStore.GetObservationCount(r.Context(), project)
|
||||
if err == nil {
|
||||
response["projectObservations"] = count
|
||||
@@ -705,8 +642,7 @@ func (s *Service) handleGetStats(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
// handleGetRetrievalStats returns detailed retrieval statistics.
|
||||
func (s *Service) handleGetRetrievalStats(w http.ResponseWriter, r *http.Request) {
|
||||
project := r.URL.Query().Get("project")
|
||||
stats := s.GetRetrievalStats(project)
|
||||
stats := s.GetRetrievalStats()
|
||||
writeJSON(w, stats)
|
||||
}
|
||||
|
||||
@@ -748,69 +684,15 @@ func (s *Service) handleSearchByPrompt(w http.ResponseWriter, r *http.Request) {
|
||||
var observations []*models.Observation
|
||||
var err error
|
||||
var usedVector bool
|
||||
similarityScores := make(map[int64]float64) // Track similarity per observation
|
||||
|
||||
// Get threshold settings from config
|
||||
threshold := s.config.ContextRelevanceThreshold
|
||||
maxResults := s.config.ContextMaxPromptResults
|
||||
|
||||
// Generate expanded queries if query expander is available
|
||||
var expandedQueries []expansion.ExpandedQuery
|
||||
var detectedIntent string
|
||||
if s.queryExpander != nil {
|
||||
cfg := expansion.DefaultConfig()
|
||||
cfg.EnableVocabularyExpansion = false // Vocabulary expansion is optional
|
||||
expandedQueries = s.queryExpander.Expand(r.Context(), query, cfg)
|
||||
if len(expandedQueries) > 0 {
|
||||
detectedIntent = string(expandedQueries[0].Intent)
|
||||
}
|
||||
}
|
||||
if len(expandedQueries) == 0 {
|
||||
// Fallback to just the original query
|
||||
expandedQueries = []expansion.ExpandedQuery{
|
||||
{Query: query, Weight: 1.0, Source: "original"},
|
||||
}
|
||||
}
|
||||
|
||||
// Try vector search first if available
|
||||
if s.vectorClient != nil && s.vectorClient.IsConnected() {
|
||||
where := sqlitevec.BuildWhereFilter(sqlitevec.DocTypeObservation, "")
|
||||
|
||||
// Search with each expanded query and merge results
|
||||
allVectorResults := make([]sqlitevec.QueryResult, 0)
|
||||
queryWeights := make(map[string]float64) // Track weights for score merging
|
||||
|
||||
for _, eq := range expandedQueries {
|
||||
vectorResults, vecErr := s.vectorClient.Query(r.Context(), eq.Query, limit*2, where)
|
||||
if vecErr == nil && len(vectorResults) > 0 {
|
||||
// Apply weight to similarity scores before merging
|
||||
for i := range vectorResults {
|
||||
vectorResults[i].Similarity *= eq.Weight
|
||||
}
|
||||
allVectorResults = append(allVectorResults, vectorResults...)
|
||||
queryWeights[eq.Query] = eq.Weight
|
||||
}
|
||||
}
|
||||
|
||||
if len(allVectorResults) > 0 {
|
||||
// Filter by relevance threshold before extracting IDs
|
||||
// Use a slightly lower threshold for expanded queries
|
||||
effectiveThreshold := threshold * 0.9 // Allow slightly lower scores for expanded queries
|
||||
filteredResults := sqlitevec.FilterByThreshold(allVectorResults, effectiveThreshold, 0)
|
||||
|
||||
// Build similarity map for filtered results (keeping highest weighted score per observation)
|
||||
for _, vr := range filteredResults {
|
||||
if sqliteID, ok := vr.Metadata["sqlite_id"].(float64); ok {
|
||||
id := int64(sqliteID)
|
||||
// Keep the highest score for each observation
|
||||
if existing, exists := similarityScores[id]; !exists || vr.Similarity > existing {
|
||||
similarityScores[id] = vr.Similarity
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
vectorResults, vecErr := s.vectorClient.Query(r.Context(), query, limit*2, where)
|
||||
if vecErr == nil && len(vectorResults) > 0 {
|
||||
// Extract observation IDs with project/scope filtering using shared helper
|
||||
obsIDs := sqlitevec.ExtractObservationIDs(filteredResults, project)
|
||||
obsIDs := sqlitevec.ExtractObservationIDs(vectorResults, project)
|
||||
|
||||
if len(obsIDs) > 0 {
|
||||
// Fetch full observations from SQLite
|
||||
@@ -860,132 +742,23 @@ func (s *Service) handleSearchByPrompt(w http.ResponseWriter, r *http.Request) {
|
||||
freshObservations = append(freshObservations, obs)
|
||||
}
|
||||
|
||||
// Apply cross-encoder reranking if available
|
||||
var reranked bool
|
||||
if s.reranker != nil && len(freshObservations) > 0 && usedVector {
|
||||
// Build candidates from observations with their bi-encoder scores
|
||||
candidates := make([]reranking.Candidate, len(freshObservations))
|
||||
for i, obs := range freshObservations {
|
||||
content := obs.Title.String
|
||||
if obs.Narrative.Valid && obs.Narrative.String != "" {
|
||||
content = content + " " + obs.Narrative.String
|
||||
}
|
||||
candidates[i] = reranking.Candidate{
|
||||
ID: fmt.Sprintf("%d", obs.ID),
|
||||
Content: content,
|
||||
Score: similarityScores[obs.ID],
|
||||
Metadata: map[string]any{"obs_idx": i},
|
||||
}
|
||||
}
|
||||
|
||||
// Rerank using cross-encoder - use pure mode or combined scores
|
||||
var rerankResults []reranking.RerankResult
|
||||
var rerankErr error
|
||||
if s.config.RerankingPureMode {
|
||||
rerankResults, rerankErr = s.reranker.RerankByScore(query, candidates, s.config.RerankingResults)
|
||||
} else {
|
||||
rerankResults, rerankErr = s.reranker.Rerank(query, candidates, s.config.RerankingResults)
|
||||
}
|
||||
if rerankErr != nil {
|
||||
log.Warn().Err(rerankErr).Msg("Cross-encoder reranking failed, using original order")
|
||||
} else if len(rerankResults) > 0 {
|
||||
// Update similarity scores with reranked scores
|
||||
for _, rr := range rerankResults {
|
||||
if id, err := strconv.ParseInt(rr.ID, 10, 64); err == nil {
|
||||
similarityScores[id] = rr.CombinedScore
|
||||
}
|
||||
}
|
||||
|
||||
// Reorder observations based on rerank results
|
||||
reorderedObs := make([]*models.Observation, 0, len(rerankResults))
|
||||
obsMap := make(map[int64]*models.Observation)
|
||||
for _, obs := range freshObservations {
|
||||
obsMap[obs.ID] = obs
|
||||
}
|
||||
for _, rr := range rerankResults {
|
||||
if id, err := strconv.ParseInt(rr.ID, 10, 64); err == nil {
|
||||
if obs, ok := obsMap[id]; ok {
|
||||
reorderedObs = append(reorderedObs, obs)
|
||||
}
|
||||
}
|
||||
}
|
||||
freshObservations = reorderedObs
|
||||
reranked = true
|
||||
|
||||
log.Debug().
|
||||
Int("candidates", len(candidates)).
|
||||
Int("returned", len(rerankResults)).
|
||||
Msg("Cross-encoder reranking complete")
|
||||
}
|
||||
}
|
||||
|
||||
// Cluster similar observations to remove duplicates
|
||||
clusteredObservations := clusterObservations(freshObservations, 0.4)
|
||||
|
||||
// Sort by similarity score (highest first) if we have scores and didn't rerank
|
||||
if len(similarityScores) > 0 && len(clusteredObservations) > 0 && !reranked {
|
||||
sort.Slice(clusteredObservations, func(i, j int) bool {
|
||||
scoreI := similarityScores[clusteredObservations[i].ID]
|
||||
scoreJ := similarityScores[clusteredObservations[j].ID]
|
||||
return scoreI > scoreJ
|
||||
})
|
||||
}
|
||||
|
||||
// Apply max results cap if configured
|
||||
if maxResults > 0 && len(clusteredObservations) > maxResults {
|
||||
clusteredObservations = clusteredObservations[:maxResults]
|
||||
}
|
||||
|
||||
// Record retrieval stats (no verification done, so verified=0, deleted=0)
|
||||
s.recordRetrievalStats(project, int64(len(clusteredObservations)), 0, 0, true)
|
||||
|
||||
// Increment retrieval counts for scoring (async, non-blocking)
|
||||
if len(clusteredObservations) > 0 {
|
||||
ids := make([]int64, len(clusteredObservations))
|
||||
for i, obs := range clusteredObservations {
|
||||
ids[i] = obs.ID
|
||||
}
|
||||
s.incrementRetrievalCounts(ids)
|
||||
}
|
||||
s.recordRetrievalStats(int64(len(clusteredObservations)), 0, 0, true)
|
||||
|
||||
log.Info().
|
||||
Str("project", project).
|
||||
Str("query", query).
|
||||
Str("intent", detectedIntent).
|
||||
Int("expansions", len(expandedQueries)).
|
||||
Int("found", len(clusteredObservations)).
|
||||
Int("stale_excluded", staleCount).
|
||||
Float64("threshold", threshold).
|
||||
Msg("Prompt-based observation search")
|
||||
|
||||
// Build response with similarity scores
|
||||
obsWithScores := make([]map[string]interface{}, len(clusteredObservations))
|
||||
for i, obs := range clusteredObservations {
|
||||
obsMap := obs.ToMap()
|
||||
if score, ok := similarityScores[obs.ID]; ok {
|
||||
obsMap["similarity"] = score
|
||||
}
|
||||
obsWithScores[i] = obsMap
|
||||
}
|
||||
|
||||
// Build expansion info for response
|
||||
expansionInfo := make([]map[string]interface{}, len(expandedQueries))
|
||||
for i, eq := range expandedQueries {
|
||||
expansionInfo[i] = map[string]interface{}{
|
||||
"query": eq.Query,
|
||||
"weight": eq.Weight,
|
||||
"source": eq.Source,
|
||||
}
|
||||
}
|
||||
|
||||
writeJSON(w, map[string]interface{}{
|
||||
"project": project,
|
||||
"query": query,
|
||||
"intent": detectedIntent,
|
||||
"expansions": expansionInfo,
|
||||
"observations": obsWithScores,
|
||||
"threshold": threshold,
|
||||
"max_results": maxResults,
|
||||
"observations": clusteredObservations,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -1051,16 +824,7 @@ func (s *Service) handleContextInject(w http.ResponseWriter, r *http.Request) {
|
||||
duplicatesRemoved := len(freshObservations) - len(clusteredObservations)
|
||||
|
||||
// Record retrieval stats (no verification done)
|
||||
s.recordRetrievalStats(project, int64(len(clusteredObservations)), 0, 0, false)
|
||||
|
||||
// Increment retrieval counts for scoring (async, non-blocking)
|
||||
if len(clusteredObservations) > 0 {
|
||||
ids := make([]int64, len(clusteredObservations))
|
||||
for i, obs := range clusteredObservations {
|
||||
ids[i] = obs.ID
|
||||
}
|
||||
s.incrementRetrievalCounts(ids)
|
||||
}
|
||||
s.recordRetrievalStats(int64(len(clusteredObservations)), 0, 0, false)
|
||||
|
||||
log.Info().
|
||||
Str("project", project).
|
||||
@@ -1220,35 +984,6 @@ func (s *Service) handleSelfCheck(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
components = append(components, sseStatus)
|
||||
|
||||
// Check Cross-Encoder Reranker
|
||||
rerankerStatus := ComponentHealth{Name: "Cross-Encoder Reranker", Status: "healthy"}
|
||||
if !s.config.RerankingEnabled {
|
||||
rerankerStatus.Status = "degraded"
|
||||
rerankerStatus.Message = "Disabled in config"
|
||||
if overall == "healthy" {
|
||||
overall = "degraded"
|
||||
}
|
||||
} else if s.reranker == nil {
|
||||
rerankerStatus.Status = "degraded"
|
||||
rerankerStatus.Message = "Not initialized"
|
||||
if overall == "healthy" {
|
||||
overall = "degraded"
|
||||
}
|
||||
} else {
|
||||
// Verify reranker is functional using Score
|
||||
_, normalizedScore, err := s.reranker.Score("test query", "test document")
|
||||
if err != nil {
|
||||
rerankerStatus.Status = "unhealthy"
|
||||
rerankerStatus.Message = fmt.Sprintf("Score check failed: %v", err)
|
||||
if overall == "healthy" {
|
||||
overall = "degraded"
|
||||
}
|
||||
} else {
|
||||
rerankerStatus.Message = fmt.Sprintf("Score check passed (%.4f)", normalizedScore)
|
||||
}
|
||||
}
|
||||
components = append(components, rerankerStatus)
|
||||
|
||||
// Calculate uptime
|
||||
uptime := time.Since(s.startTime).Round(time.Second).String()
|
||||
|
||||
@@ -1260,7 +995,7 @@ func (s *Service) handleSelfCheck(w http.ResponseWriter, r *http.Request) {
|
||||
})
|
||||
}
|
||||
|
||||
// handleUpdateRestart restarts the worker with the new binary (after update).
|
||||
// handleUpdateRestart restarts the worker with the new binary.
|
||||
func (s *Service) handleUpdateRestart(w http.ResponseWriter, r *http.Request) {
|
||||
status := s.updater.GetStatus()
|
||||
if status.State != "done" {
|
||||
@@ -1286,111 +1021,3 @@ func (s *Service) handleUpdateRestart(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
// handleRestart restarts the worker process (general restart, not tied to update).
|
||||
func (s *Service) handleRestart(w http.ResponseWriter, r *http.Request) {
|
||||
log.Info().Msg("Manual restart requested via API")
|
||||
|
||||
// Send response before restarting
|
||||
writeJSON(w, map[string]interface{}{
|
||||
"success": true,
|
||||
"message": "Restarting worker...",
|
||||
"version": s.version,
|
||||
})
|
||||
|
||||
// Flush the response
|
||||
if f, ok := w.(http.Flusher); ok {
|
||||
f.Flush()
|
||||
}
|
||||
|
||||
// Restart in background after response is sent
|
||||
go func() {
|
||||
// Small delay to ensure response is sent
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
if err := s.updater.Restart(); err != nil {
|
||||
log.Error().Err(err).Msg("Failed to restart worker")
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
// handleGetGraphStats returns observation graph statistics.
|
||||
func (s *Service) handleGetGraphStats(w http.ResponseWriter, r *http.Request) {
|
||||
if s.graphSearchClient == nil {
|
||||
writeJSON(w, map[string]interface{}{
|
||||
"enabled": false,
|
||||
"message": "Graph search not enabled",
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
stats := s.graphSearchClient.GetGraphStats()
|
||||
|
||||
response := map[string]interface{}{
|
||||
"enabled": s.config.GraphEnabled,
|
||||
"nodeCount": stats.NodeCount,
|
||||
"edgeCount": stats.EdgeCount,
|
||||
"avgDegree": stats.AvgDegree,
|
||||
"maxDegree": stats.MaxDegree,
|
||||
"minDegree": stats.MinDegree,
|
||||
"medianDegree": stats.MedianDegree,
|
||||
"edgeTypes": stats.EdgeTypes,
|
||||
"config": map[string]interface{}{
|
||||
"maxHops": s.config.GraphMaxHops,
|
||||
"branchFactor": s.config.GraphBranchFactor,
|
||||
"edgeWeight": s.config.GraphEdgeWeight,
|
||||
"rebuildIntervalMin": s.config.GraphRebuildIntervalMin,
|
||||
},
|
||||
}
|
||||
|
||||
writeJSON(w, response)
|
||||
}
|
||||
|
||||
// handleGetVectorMetrics returns hybrid vector storage metrics.
|
||||
func (s *Service) handleGetVectorMetrics(w http.ResponseWriter, r *http.Request) {
|
||||
if s.hybridMetrics == nil {
|
||||
writeJSON(w, map[string]interface{}{
|
||||
"enabled": false,
|
||||
"message": "Vector metrics not available",
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
snapshot := s.hybridMetrics.GetSnapshot()
|
||||
|
||||
response := map[string]interface{}{
|
||||
"queries": map[string]interface{}{
|
||||
"total": snapshot.TotalQueries,
|
||||
"hubOnly": snapshot.HubOnlyQueries,
|
||||
"hybrid": snapshot.HybridQueries,
|
||||
"onDemand": snapshot.OnDemandQueries,
|
||||
"graph": snapshot.GraphQueries,
|
||||
},
|
||||
"latency": map[string]interface{}{
|
||||
"avg": snapshot.AvgLatency.String(),
|
||||
"p50": snapshot.P50Latency.String(),
|
||||
"p95": snapshot.P95Latency.String(),
|
||||
"p99": snapshot.P99Latency.String(),
|
||||
"avgHub": snapshot.AvgHubLatency.String(),
|
||||
"avgRecompute": snapshot.AvgRecomputeLatency.String(),
|
||||
},
|
||||
"storage": map[string]interface{}{
|
||||
"totalDocuments": snapshot.TotalDocuments,
|
||||
"hubDocuments": snapshot.HubDocuments,
|
||||
"storedEmbeddings": snapshot.StoredEmbeddings,
|
||||
"savingsPercent": snapshot.StorageSavingsPercent,
|
||||
"recomputedTotal": snapshot.RecomputedTotal,
|
||||
},
|
||||
"cache": map[string]interface{}{
|
||||
"hits": snapshot.CacheHits,
|
||||
"misses": snapshot.CacheMisses,
|
||||
"hitRate": snapshot.CacheHitRate,
|
||||
},
|
||||
"graph": map[string]interface{}{
|
||||
"traversals": snapshot.GraphTraversals,
|
||||
"avgDepth": snapshot.AvgTraversalDepth,
|
||||
},
|
||||
"uptime": snapshot.Uptime.String(),
|
||||
}
|
||||
|
||||
writeJSON(w, response)
|
||||
}
|
||||
|
||||
@@ -1,292 +0,0 @@
|
||||
// Package worker provides the main worker service for claude-mnemonic.
|
||||
package worker
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"strconv"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
"github.com/lukaszraczylo/claude-mnemonic/pkg/models"
|
||||
)
|
||||
|
||||
// DefaultPatternsLimit is the default number of patterns to return.
|
||||
const DefaultPatternsLimit = 100
|
||||
|
||||
// handleGetPatterns returns all active patterns, optionally filtered by type or project.
|
||||
func (s *Service) handleGetPatterns(w http.ResponseWriter, r *http.Request) {
|
||||
s.initMu.RLock()
|
||||
store := s.patternStore
|
||||
s.initMu.RUnlock()
|
||||
|
||||
if store == nil {
|
||||
http.Error(w, "pattern store not initialized", http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
|
||||
// Parse query parameters
|
||||
limit := DefaultPatternsLimit
|
||||
if l := r.URL.Query().Get("limit"); l != "" {
|
||||
if parsed, err := strconv.Atoi(l); err == nil && parsed > 0 {
|
||||
limit = parsed
|
||||
}
|
||||
}
|
||||
|
||||
patternType := r.URL.Query().Get("type")
|
||||
project := r.URL.Query().Get("project")
|
||||
|
||||
var patterns []*models.Pattern
|
||||
var err error
|
||||
|
||||
if patternType != "" {
|
||||
// Filter by type
|
||||
patterns, err = store.GetPatternsByType(r.Context(), models.PatternType(patternType), limit)
|
||||
} else if project != "" {
|
||||
// Filter by project
|
||||
patterns, err = store.GetPatternsByProject(r.Context(), project, limit)
|
||||
} else {
|
||||
// Get all active patterns
|
||||
patterns, err = store.GetActivePatterns(r.Context(), limit)
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
writeJSON(w, patterns)
|
||||
}
|
||||
|
||||
// handleGetPatternStats returns aggregate statistics about patterns.
|
||||
func (s *Service) handleGetPatternStats(w http.ResponseWriter, r *http.Request) {
|
||||
s.initMu.RLock()
|
||||
store := s.patternStore
|
||||
s.initMu.RUnlock()
|
||||
|
||||
if store == nil {
|
||||
http.Error(w, "pattern store not initialized", http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
|
||||
stats, err := store.GetPatternStats(r.Context())
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
writeJSON(w, stats)
|
||||
}
|
||||
|
||||
// handleGetPatternByID returns a single pattern by ID.
|
||||
func (s *Service) handleGetPatternByID(w http.ResponseWriter, r *http.Request) {
|
||||
s.initMu.RLock()
|
||||
store := s.patternStore
|
||||
s.initMu.RUnlock()
|
||||
|
||||
if store == nil {
|
||||
http.Error(w, "pattern store not initialized", http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
|
||||
idStr := chi.URLParam(r, "id")
|
||||
id, err := strconv.ParseInt(idStr, 10, 64)
|
||||
if err != nil {
|
||||
http.Error(w, "invalid pattern ID", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
pattern, err := store.GetPatternByID(r.Context(), id)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
if pattern == nil {
|
||||
http.Error(w, "pattern not found", http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
|
||||
writeJSON(w, pattern)
|
||||
}
|
||||
|
||||
// handleGetPatternInsight returns a formatted insight string for a pattern.
|
||||
func (s *Service) handleGetPatternInsight(w http.ResponseWriter, r *http.Request) {
|
||||
s.initMu.RLock()
|
||||
detector := s.patternDetector
|
||||
s.initMu.RUnlock()
|
||||
|
||||
if detector == nil {
|
||||
http.Error(w, "pattern detector not initialized", http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
|
||||
idStr := chi.URLParam(r, "id")
|
||||
id, err := strconv.ParseInt(idStr, 10, 64)
|
||||
if err != nil {
|
||||
http.Error(w, "invalid pattern ID", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
insight, err := detector.GetPatternInsight(r.Context(), id)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
writeJSON(w, map[string]string{"insight": insight})
|
||||
}
|
||||
|
||||
// handleDeletePattern deletes a pattern by ID.
|
||||
func (s *Service) handleDeletePattern(w http.ResponseWriter, r *http.Request) {
|
||||
s.initMu.RLock()
|
||||
store := s.patternStore
|
||||
s.initMu.RUnlock()
|
||||
|
||||
if store == nil {
|
||||
http.Error(w, "pattern store not initialized", http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
|
||||
idStr := chi.URLParam(r, "id")
|
||||
id, err := strconv.ParseInt(idStr, 10, 64)
|
||||
if err != nil {
|
||||
http.Error(w, "invalid pattern ID", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
if err := store.DeletePattern(r.Context(), id); err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
writeJSON(w, map[string]string{"status": "deleted"})
|
||||
}
|
||||
|
||||
// handleDeprecatePattern marks a pattern as deprecated.
|
||||
func (s *Service) handleDeprecatePattern(w http.ResponseWriter, r *http.Request) {
|
||||
s.initMu.RLock()
|
||||
store := s.patternStore
|
||||
s.initMu.RUnlock()
|
||||
|
||||
if store == nil {
|
||||
http.Error(w, "pattern store not initialized", http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
|
||||
idStr := chi.URLParam(r, "id")
|
||||
id, err := strconv.ParseInt(idStr, 10, 64)
|
||||
if err != nil {
|
||||
http.Error(w, "invalid pattern ID", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
if err := store.MarkPatternDeprecated(r.Context(), id); err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
writeJSON(w, map[string]string{"status": "deprecated"})
|
||||
}
|
||||
|
||||
// MergePatternsRequest is the request body for merging patterns.
|
||||
type MergePatternsRequest struct {
|
||||
SourceID int64 `json:"source_id"`
|
||||
TargetID int64 `json:"target_id"`
|
||||
}
|
||||
|
||||
// handleSearchPatterns performs full-text search on patterns.
|
||||
func (s *Service) handleSearchPatterns(w http.ResponseWriter, r *http.Request) {
|
||||
s.initMu.RLock()
|
||||
store := s.patternStore
|
||||
s.initMu.RUnlock()
|
||||
|
||||
if store == nil {
|
||||
http.Error(w, "pattern store not initialized", http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
|
||||
query := r.URL.Query().Get("q")
|
||||
if query == "" {
|
||||
http.Error(w, "query parameter 'q' is required", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
limit := DefaultPatternsLimit
|
||||
if l := r.URL.Query().Get("limit"); l != "" {
|
||||
if parsed, err := strconv.Atoi(l); err == nil && parsed > 0 {
|
||||
limit = parsed
|
||||
}
|
||||
}
|
||||
|
||||
patterns, err := store.SearchPatternsFTS(r.Context(), query, limit)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
writeJSON(w, patterns)
|
||||
}
|
||||
|
||||
// handleGetPatternByName returns a pattern by its name.
|
||||
func (s *Service) handleGetPatternByName(w http.ResponseWriter, r *http.Request) {
|
||||
s.initMu.RLock()
|
||||
store := s.patternStore
|
||||
s.initMu.RUnlock()
|
||||
|
||||
if store == nil {
|
||||
http.Error(w, "pattern store not initialized", http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
|
||||
name := r.URL.Query().Get("name")
|
||||
if name == "" {
|
||||
http.Error(w, "query parameter 'name' is required", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
pattern, err := store.GetPatternByName(r.Context(), name)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
if pattern == nil {
|
||||
http.Error(w, "pattern not found", http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
|
||||
writeJSON(w, pattern)
|
||||
}
|
||||
|
||||
// handleMergePatterns merges a source pattern into a target pattern.
|
||||
func (s *Service) handleMergePatterns(w http.ResponseWriter, r *http.Request) {
|
||||
s.initMu.RLock()
|
||||
store := s.patternStore
|
||||
s.initMu.RUnlock()
|
||||
|
||||
if store == nil {
|
||||
http.Error(w, "pattern store not initialized", http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
|
||||
var req MergePatternsRequest
|
||||
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
||||
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
if req.SourceID == 0 || req.TargetID == 0 {
|
||||
http.Error(w, "source_id and target_id are required", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
if req.SourceID == req.TargetID {
|
||||
http.Error(w, "source_id and target_id cannot be the same", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
if err := store.MergePatterns(r.Context(), req.SourceID, req.TargetID); err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
writeJSON(w, map[string]string{"status": "merged"})
|
||||
}
|
||||
@@ -1,174 +0,0 @@
|
||||
// Package worker provides the main worker service for claude-mnemonic.
|
||||
package worker
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"strconv"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
"github.com/lukaszraczylo/claude-mnemonic/pkg/models"
|
||||
)
|
||||
|
||||
// DefaultRelationsLimit is the default number of relations to return.
|
||||
const DefaultRelationsLimit = 50
|
||||
|
||||
// handleGetRelations returns relations for an observation.
|
||||
func (s *Service) handleGetRelations(w http.ResponseWriter, r *http.Request) {
|
||||
idStr := chi.URLParam(r, "id")
|
||||
id, err := strconv.ParseInt(idStr, 10, 64)
|
||||
if err != nil {
|
||||
http.Error(w, "invalid observation id", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
relations, err := s.relationStore.GetRelationsWithDetails(r.Context(), id)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
if relations == nil {
|
||||
relations = []*models.RelationWithDetails{}
|
||||
}
|
||||
|
||||
writeJSON(w, relations)
|
||||
}
|
||||
|
||||
// handleGetRelationGraph returns the relation graph for an observation.
|
||||
func (s *Service) handleGetRelationGraph(w http.ResponseWriter, r *http.Request) {
|
||||
idStr := chi.URLParam(r, "id")
|
||||
id, err := strconv.ParseInt(idStr, 10, 64)
|
||||
if err != nil {
|
||||
http.Error(w, "invalid observation id", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
// Get depth parameter (default 2)
|
||||
depth := 2
|
||||
if depthStr := r.URL.Query().Get("depth"); depthStr != "" {
|
||||
if d, atoiErr := strconv.Atoi(depthStr); atoiErr == nil && d > 0 && d <= 5 {
|
||||
depth = d
|
||||
}
|
||||
}
|
||||
|
||||
graph, err := s.relationStore.GetRelationGraph(r.Context(), id, depth)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
writeJSON(w, graph)
|
||||
}
|
||||
|
||||
// handleGetRelatedObservations returns observations related to a given one.
|
||||
func (s *Service) handleGetRelatedObservations(w http.ResponseWriter, r *http.Request) {
|
||||
idStr := chi.URLParam(r, "id")
|
||||
id, err := strconv.ParseInt(idStr, 10, 64)
|
||||
if err != nil {
|
||||
http.Error(w, "invalid observation id", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
// Get minimum confidence parameter (default 0.4)
|
||||
minConfidence := 0.4
|
||||
if confStr := r.URL.Query().Get("min_confidence"); confStr != "" {
|
||||
if c, parseErr := strconv.ParseFloat(confStr, 64); parseErr == nil && c >= 0 && c <= 1 {
|
||||
minConfidence = c
|
||||
}
|
||||
}
|
||||
|
||||
// Get related observation IDs
|
||||
relatedIDs, err := s.relationStore.GetRelatedObservationIDs(r.Context(), id, minConfidence)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
if len(relatedIDs) == 0 {
|
||||
writeJSON(w, []*models.Observation{})
|
||||
return
|
||||
}
|
||||
|
||||
// Fetch full observations
|
||||
observations, err := s.observationStore.GetObservationsByIDs(r.Context(), relatedIDs, "importance", 50)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
if observations == nil {
|
||||
observations = []*models.Observation{}
|
||||
}
|
||||
|
||||
writeJSON(w, observations)
|
||||
}
|
||||
|
||||
// handleGetRelationsByType returns all relations of a specific type.
|
||||
func (s *Service) handleGetRelationsByType(w http.ResponseWriter, r *http.Request) {
|
||||
relType := chi.URLParam(r, "type")
|
||||
|
||||
// Validate relation type
|
||||
validType := false
|
||||
for _, t := range models.AllRelationTypes {
|
||||
if string(t) == relType {
|
||||
validType = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !validType {
|
||||
http.Error(w, "invalid relation type", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
limit := DefaultRelationsLimit
|
||||
if limitStr := r.URL.Query().Get("limit"); limitStr != "" {
|
||||
if l, err := strconv.Atoi(limitStr); err == nil && l > 0 && l <= 100 {
|
||||
limit = l
|
||||
}
|
||||
}
|
||||
|
||||
relations, err := s.relationStore.GetRelationsByType(r.Context(), models.RelationType(relType), limit)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
if relations == nil {
|
||||
relations = []*models.ObservationRelation{}
|
||||
}
|
||||
|
||||
writeJSON(w, relations)
|
||||
}
|
||||
|
||||
// handleGetRelationStats returns statistics about relations.
|
||||
func (s *Service) handleGetRelationStats(w http.ResponseWriter, r *http.Request) {
|
||||
// Get total relation count
|
||||
totalCount, err := s.relationStore.GetTotalRelationCount(r.Context())
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
// Get high confidence relations count
|
||||
highConfRelations, err := s.relationStore.GetHighConfidenceRelations(r.Context(), 0.7, 1000)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
// Count by relation type
|
||||
typeCounts := make(map[string]int)
|
||||
for _, t := range models.AllRelationTypes {
|
||||
relations, err := s.relationStore.GetRelationsByType(r.Context(), t, 1000)
|
||||
if err == nil {
|
||||
typeCounts[string(t)] = len(relations)
|
||||
}
|
||||
}
|
||||
|
||||
writeJSON(w, map[string]interface{}{
|
||||
"total_count": totalCount,
|
||||
"high_confidence": len(highConfRelations),
|
||||
"by_type": typeCounts,
|
||||
"min_confidence_used": 0.4,
|
||||
})
|
||||
}
|
||||
@@ -1,359 +0,0 @@
|
||||
// Package worker provides the main worker service for claude-mnemonic.
|
||||
package worker
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
"github.com/lukaszraczylo/claude-mnemonic/pkg/models"
|
||||
"github.com/rs/zerolog/log"
|
||||
)
|
||||
|
||||
// FeedbackRequest represents a user feedback submission.
|
||||
type FeedbackRequest struct {
|
||||
Feedback int `json:"feedback"` // -1 (thumbs down), 0 (neutral), 1 (thumbs up)
|
||||
}
|
||||
|
||||
// handleObservationFeedback handles user feedback (thumbs up/down) for an observation.
|
||||
// POST /api/observations/{id}/feedback
|
||||
func (s *Service) handleObservationFeedback(w http.ResponseWriter, r *http.Request) {
|
||||
// Parse observation ID
|
||||
idStr := chi.URLParam(r, "id")
|
||||
id, err := strconv.ParseInt(idStr, 10, 64)
|
||||
if err != nil {
|
||||
http.Error(w, "invalid observation id", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
// Parse request body
|
||||
var req FeedbackRequest
|
||||
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
||||
http.Error(w, "invalid request body", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
// Validate feedback value
|
||||
if req.Feedback < -1 || req.Feedback > 1 {
|
||||
http.Error(w, "feedback must be -1, 0, or 1", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
// Get required components
|
||||
s.initMu.RLock()
|
||||
observationStore := s.observationStore
|
||||
scoreCalculator := s.scoreCalculator
|
||||
s.initMu.RUnlock()
|
||||
|
||||
if observationStore == nil {
|
||||
http.Error(w, "service not ready", http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
|
||||
// Update feedback in database
|
||||
if err := observationStore.UpdateObservationFeedback(r.Context(), id, req.Feedback); err != nil {
|
||||
http.Error(w, "failed to update feedback", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
// Recalculate score immediately if calculator is available
|
||||
var newScore float64
|
||||
if scoreCalculator != nil {
|
||||
obs, err := observationStore.GetObservationByID(r.Context(), id)
|
||||
if err == nil && obs != nil {
|
||||
obs.UserFeedback = req.Feedback // Apply the new feedback
|
||||
newScore = scoreCalculator.Calculate(obs, time.Now())
|
||||
if scoreErr := observationStore.UpdateImportanceScore(r.Context(), id, newScore); scoreErr != nil {
|
||||
// Log but don't fail - feedback was recorded
|
||||
// Score will be updated on next recalculation cycle
|
||||
log.Warn().Err(scoreErr).Int64("id", id).Msg("Failed to update importance score after feedback")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Broadcast update via SSE
|
||||
s.sseBroadcaster.Broadcast(map[string]interface{}{
|
||||
"type": "observation_feedback",
|
||||
"id": id,
|
||||
"feedback": req.Feedback,
|
||||
"score": newScore,
|
||||
})
|
||||
|
||||
writeJSON(w, map[string]interface{}{
|
||||
"status": "ok",
|
||||
"id": id,
|
||||
"feedback": req.Feedback,
|
||||
"score": newScore,
|
||||
})
|
||||
}
|
||||
|
||||
// handleGetScoringStats returns scoring statistics and configuration.
|
||||
// GET /api/scoring/stats
|
||||
func (s *Service) handleGetScoringStats(w http.ResponseWriter, r *http.Request) {
|
||||
project := r.URL.Query().Get("project")
|
||||
|
||||
s.initMu.RLock()
|
||||
observationStore := s.observationStore
|
||||
recalculator := s.recalculator
|
||||
s.initMu.RUnlock()
|
||||
|
||||
if observationStore == nil {
|
||||
http.Error(w, "service not ready", http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
|
||||
// Get feedback statistics
|
||||
feedbackStats, err := observationStore.GetObservationFeedbackStats(r.Context(), project)
|
||||
if err != nil {
|
||||
http.Error(w, "failed to get feedback stats", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
response := map[string]interface{}{
|
||||
"feedback": feedbackStats,
|
||||
}
|
||||
|
||||
// Add recalculator stats if available
|
||||
if recalculator != nil {
|
||||
response["recalculator"] = recalculator.GetStats()
|
||||
}
|
||||
|
||||
writeJSON(w, response)
|
||||
}
|
||||
|
||||
// handleGetTopObservations returns the highest-scoring observations.
|
||||
// GET /api/observations/top
|
||||
func (s *Service) handleGetTopObservations(w http.ResponseWriter, r *http.Request) {
|
||||
limit := parseIntParam(r, "limit", 10)
|
||||
project := r.URL.Query().Get("project")
|
||||
|
||||
s.initMu.RLock()
|
||||
observationStore := s.observationStore
|
||||
s.initMu.RUnlock()
|
||||
|
||||
if observationStore == nil {
|
||||
http.Error(w, "service not ready", http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
|
||||
observations, err := observationStore.GetTopScoringObservations(r.Context(), project, limit)
|
||||
if err != nil {
|
||||
http.Error(w, "failed to get top observations", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
if observations == nil {
|
||||
observations = []*models.Observation{}
|
||||
}
|
||||
|
||||
writeJSON(w, observations)
|
||||
}
|
||||
|
||||
// handleGetMostRetrieved returns the most frequently retrieved observations.
|
||||
// GET /api/observations/most-retrieved
|
||||
func (s *Service) handleGetMostRetrieved(w http.ResponseWriter, r *http.Request) {
|
||||
limit := parseIntParam(r, "limit", 10)
|
||||
project := r.URL.Query().Get("project")
|
||||
|
||||
s.initMu.RLock()
|
||||
observationStore := s.observationStore
|
||||
s.initMu.RUnlock()
|
||||
|
||||
if observationStore == nil {
|
||||
http.Error(w, "service not ready", http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
|
||||
observations, err := observationStore.GetMostRetrievedObservations(r.Context(), project, limit)
|
||||
if err != nil {
|
||||
http.Error(w, "failed to get most retrieved observations", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
if observations == nil {
|
||||
observations = []*models.Observation{}
|
||||
}
|
||||
|
||||
writeJSON(w, observations)
|
||||
}
|
||||
|
||||
// handleExplainScore returns a breakdown of how an observation's score was calculated.
|
||||
// GET /api/observations/{id}/score
|
||||
func (s *Service) handleExplainScore(w http.ResponseWriter, r *http.Request) {
|
||||
// Parse observation ID
|
||||
idStr := chi.URLParam(r, "id")
|
||||
id, err := strconv.ParseInt(idStr, 10, 64)
|
||||
if err != nil {
|
||||
http.Error(w, "invalid observation id", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
s.initMu.RLock()
|
||||
observationStore := s.observationStore
|
||||
scoreCalculator := s.scoreCalculator
|
||||
s.initMu.RUnlock()
|
||||
|
||||
if observationStore == nil || scoreCalculator == nil {
|
||||
http.Error(w, "service not ready", http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
|
||||
// Get observation
|
||||
obs, err := observationStore.GetObservationByID(r.Context(), id)
|
||||
if err != nil {
|
||||
http.Error(w, "failed to get observation", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
if obs == nil {
|
||||
http.Error(w, "observation not found", http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
|
||||
// Calculate score components
|
||||
components := scoreCalculator.CalculateComponents(obs, time.Now())
|
||||
|
||||
writeJSON(w, map[string]interface{}{
|
||||
"id": id,
|
||||
"components": components,
|
||||
"config": scoreCalculator.GetConfig(),
|
||||
})
|
||||
}
|
||||
|
||||
// handleUpdateConceptWeight updates a concept weight.
|
||||
// PUT /api/scoring/concepts/{concept}
|
||||
func (s *Service) handleUpdateConceptWeight(w http.ResponseWriter, r *http.Request) {
|
||||
concept := chi.URLParam(r, "concept")
|
||||
if concept == "" {
|
||||
http.Error(w, "concept is required", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
var req struct {
|
||||
Weight float64 `json:"weight"`
|
||||
}
|
||||
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
||||
http.Error(w, "invalid request body", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
// Validate weight
|
||||
if req.Weight < 0 || req.Weight > 1 {
|
||||
http.Error(w, "weight must be between 0 and 1", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
s.initMu.RLock()
|
||||
observationStore := s.observationStore
|
||||
recalculator := s.recalculator
|
||||
s.initMu.RUnlock()
|
||||
|
||||
if observationStore == nil {
|
||||
http.Error(w, "service not ready", http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
|
||||
// Update in database
|
||||
if err := observationStore.UpdateConceptWeight(r.Context(), concept, req.Weight); err != nil {
|
||||
http.Error(w, "failed to update concept weight", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
// Refresh concept weights in recalculator
|
||||
if recalculator != nil {
|
||||
if refreshErr := recalculator.RefreshConceptWeights(r.Context()); refreshErr != nil {
|
||||
// Log but don't fail - weight was saved
|
||||
log.Warn().Err(refreshErr).Str("concept", concept).Msg("Failed to refresh concept weights in recalculator")
|
||||
}
|
||||
}
|
||||
|
||||
writeJSON(w, map[string]interface{}{
|
||||
"status": "ok",
|
||||
"concept": concept,
|
||||
"weight": req.Weight,
|
||||
})
|
||||
}
|
||||
|
||||
// handleGetConceptWeights returns all concept weights.
|
||||
// GET /api/scoring/concepts
|
||||
func (s *Service) handleGetConceptWeights(w http.ResponseWriter, r *http.Request) {
|
||||
s.initMu.RLock()
|
||||
observationStore := s.observationStore
|
||||
s.initMu.RUnlock()
|
||||
|
||||
if observationStore == nil {
|
||||
http.Error(w, "service not ready", http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
|
||||
weights, err := observationStore.GetConceptWeights(r.Context())
|
||||
if err != nil {
|
||||
http.Error(w, "failed to get concept weights", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
writeJSON(w, weights)
|
||||
}
|
||||
|
||||
// handleTriggerRecalculation triggers an immediate score recalculation.
|
||||
// POST /api/scoring/recalculate
|
||||
func (s *Service) handleTriggerRecalculation(w http.ResponseWriter, r *http.Request) {
|
||||
s.initMu.RLock()
|
||||
recalculator := s.recalculator
|
||||
s.initMu.RUnlock()
|
||||
|
||||
if recalculator == nil {
|
||||
http.Error(w, "recalculator not available", http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
|
||||
// Run recalculation in background
|
||||
go func() {
|
||||
if recalcErr := recalculator.RecalculateNow(r.Context()); recalcErr != nil {
|
||||
// Log error but don't block response
|
||||
log.Warn().Err(recalcErr).Msg("Failed to trigger score recalculation")
|
||||
}
|
||||
}()
|
||||
|
||||
writeJSON(w, map[string]string{"status": "recalculation triggered"})
|
||||
}
|
||||
|
||||
// parseIntParam parses an integer query parameter with a default value.
|
||||
func parseIntParam(r *http.Request, name string, defaultVal int) int {
|
||||
if val := r.URL.Query().Get(name); val != "" {
|
||||
if parsed, err := strconv.Atoi(val); err == nil && parsed > 0 {
|
||||
return parsed
|
||||
}
|
||||
}
|
||||
return defaultVal
|
||||
}
|
||||
|
||||
// incrementRetrievalCounts increments retrieval counts for observations.
|
||||
// Called after search results are returned to track popularity.
|
||||
func (s *Service) incrementRetrievalCounts(ids []int64) {
|
||||
if len(ids) == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
s.initMu.RLock()
|
||||
store := s.observationStore
|
||||
s.initMu.RUnlock()
|
||||
|
||||
if store == nil {
|
||||
return
|
||||
}
|
||||
|
||||
// Increment in background to not block response
|
||||
go func() {
|
||||
// Create a new context with timeout for the background operation
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
|
||||
defer cancel()
|
||||
|
||||
if incrErr := store.IncrementRetrievalCount(ctx, ids); incrErr != nil {
|
||||
// Log but don't fail - this is a background operation
|
||||
log.Warn().Err(incrErr).Msg("Failed to increment retrieval count in background")
|
||||
}
|
||||
}()
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,537 +0,0 @@
|
||||
package sdk
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/lukaszraczylo/claude-mnemonic/pkg/models"
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestParseObservations_SingleObservation(t *testing.T) {
|
||||
text := `Some text before
|
||||
<observation>
|
||||
<type>bugfix</type>
|
||||
<title>Fixed null pointer error</title>
|
||||
<subtitle>In user service</subtitle>
|
||||
<narrative>The service was crashing when user ID was nil</narrative>
|
||||
<facts>
|
||||
<fact>Added nil check</fact>
|
||||
<fact>Added unit test</fact>
|
||||
</facts>
|
||||
<concepts>
|
||||
<concept>error-handling</concept>
|
||||
<concept>debugging</concept>
|
||||
</concepts>
|
||||
<files_read>
|
||||
<file>user_service.go</file>
|
||||
</files_read>
|
||||
<files_modified>
|
||||
<file>user_service.go</file>
|
||||
<file>user_service_test.go</file>
|
||||
</files_modified>
|
||||
</observation>
|
||||
Some text after`
|
||||
|
||||
observations := ParseObservations(text, "test-correlation-id")
|
||||
|
||||
assert.Len(t, observations, 1)
|
||||
obs := observations[0]
|
||||
assert.Equal(t, models.ObservationType("bugfix"), obs.Type)
|
||||
assert.Equal(t, "Fixed null pointer error", obs.Title)
|
||||
assert.Equal(t, "In user service", obs.Subtitle)
|
||||
assert.Equal(t, "The service was crashing when user ID was nil", obs.Narrative)
|
||||
assert.Equal(t, []string{"Added nil check", "Added unit test"}, obs.Facts)
|
||||
assert.Equal(t, []string{"error-handling", "debugging"}, obs.Concepts)
|
||||
assert.Equal(t, []string{"user_service.go"}, obs.FilesRead)
|
||||
assert.Equal(t, []string{"user_service.go", "user_service_test.go"}, obs.FilesModified)
|
||||
}
|
||||
|
||||
func TestParseObservations_MultipleObservations(t *testing.T) {
|
||||
text := `
|
||||
<observation>
|
||||
<type>feature</type>
|
||||
<title>Added caching</title>
|
||||
<narrative>Implemented Redis caching</narrative>
|
||||
<facts><fact>Added cache layer</fact></facts>
|
||||
<concepts><concept>caching</concept></concepts>
|
||||
</observation>
|
||||
<observation>
|
||||
<type>refactor</type>
|
||||
<title>Cleaned up code</title>
|
||||
<narrative>Removed dead code</narrative>
|
||||
<facts><fact>Removed unused functions</fact></facts>
|
||||
<concepts><concept>refactoring</concept></concepts>
|
||||
</observation>
|
||||
`
|
||||
|
||||
observations := ParseObservations(text, "test-id")
|
||||
|
||||
assert.Len(t, observations, 2)
|
||||
assert.Equal(t, models.ObservationType("feature"), observations[0].Type)
|
||||
assert.Equal(t, "Added caching", observations[0].Title)
|
||||
assert.Equal(t, models.ObservationType("refactor"), observations[1].Type)
|
||||
assert.Equal(t, "Cleaned up code", observations[1].Title)
|
||||
}
|
||||
|
||||
func TestParseObservations_TableDriven(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
input string
|
||||
expectedType models.ObservationType
|
||||
expectedTitle string
|
||||
checkConcepts []string
|
||||
expectedCount int
|
||||
}{
|
||||
{
|
||||
name: "valid_bugfix_observation",
|
||||
input: `<observation>
|
||||
<type>bugfix</type>
|
||||
<title>Fixed bug</title>
|
||||
<narrative>Details</narrative>
|
||||
</observation>`,
|
||||
expectedCount: 1,
|
||||
expectedType: models.ObsTypeBugfix,
|
||||
expectedTitle: "Fixed bug",
|
||||
},
|
||||
{
|
||||
name: "valid_feature_observation",
|
||||
input: `<observation>
|
||||
<type>feature</type>
|
||||
<title>New feature</title>
|
||||
<narrative>Added new stuff</narrative>
|
||||
</observation>`,
|
||||
expectedCount: 1,
|
||||
expectedType: models.ObsTypeFeature,
|
||||
expectedTitle: "New feature",
|
||||
},
|
||||
{
|
||||
name: "valid_refactor_observation",
|
||||
input: `<observation>
|
||||
<type>refactor</type>
|
||||
<title>Code cleanup</title>
|
||||
<narrative>Refactored module</narrative>
|
||||
</observation>`,
|
||||
expectedCount: 1,
|
||||
expectedType: models.ObsTypeRefactor,
|
||||
expectedTitle: "Code cleanup",
|
||||
},
|
||||
{
|
||||
name: "valid_change_observation",
|
||||
input: `<observation>
|
||||
<type>change</type>
|
||||
<title>Config update</title>
|
||||
<narrative>Changed settings</narrative>
|
||||
</observation>`,
|
||||
expectedCount: 1,
|
||||
expectedType: models.ObsTypeChange,
|
||||
expectedTitle: "Config update",
|
||||
},
|
||||
{
|
||||
name: "valid_discovery_observation",
|
||||
input: `<observation>
|
||||
<type>discovery</type>
|
||||
<title>Found pattern</title>
|
||||
<narrative>Discovered new pattern</narrative>
|
||||
</observation>`,
|
||||
expectedCount: 1,
|
||||
expectedType: models.ObsTypeDiscovery,
|
||||
expectedTitle: "Found pattern",
|
||||
},
|
||||
{
|
||||
name: "valid_decision_observation",
|
||||
input: `<observation>
|
||||
<type>decision</type>
|
||||
<title>Architecture decision</title>
|
||||
<narrative>Chose microservices</narrative>
|
||||
</observation>`,
|
||||
expectedCount: 1,
|
||||
expectedType: models.ObsTypeDecision,
|
||||
expectedTitle: "Architecture decision",
|
||||
},
|
||||
{
|
||||
name: "invalid_type_defaults_to_change",
|
||||
input: `<observation>
|
||||
<type>invalid_type</type>
|
||||
<title>Some title</title>
|
||||
<narrative>Details</narrative>
|
||||
</observation>`,
|
||||
expectedCount: 1,
|
||||
expectedType: models.ObsTypeChange,
|
||||
expectedTitle: "Some title",
|
||||
},
|
||||
{
|
||||
name: "missing_type_defaults_to_change",
|
||||
input: `<observation>
|
||||
<title>No type specified</title>
|
||||
<narrative>Details</narrative>
|
||||
</observation>`,
|
||||
expectedCount: 1,
|
||||
expectedType: models.ObsTypeChange,
|
||||
expectedTitle: "No type specified",
|
||||
},
|
||||
{
|
||||
name: "empty_input",
|
||||
input: "",
|
||||
expectedCount: 0,
|
||||
},
|
||||
{
|
||||
name: "no_observation_tags",
|
||||
input: "Just regular text without any observation",
|
||||
expectedCount: 0,
|
||||
},
|
||||
{
|
||||
name: "valid_concepts_filtered",
|
||||
input: `<observation>
|
||||
<type>bugfix</type>
|
||||
<title>Test</title>
|
||||
<narrative>Test</narrative>
|
||||
<concepts>
|
||||
<concept>best-practice</concept>
|
||||
<concept>invalid-concept</concept>
|
||||
<concept>security</concept>
|
||||
</concepts>
|
||||
</observation>`,
|
||||
expectedCount: 1,
|
||||
expectedType: models.ObsTypeBugfix,
|
||||
checkConcepts: []string{"best-practice", "security"},
|
||||
},
|
||||
{
|
||||
name: "type_in_concepts_filtered_out",
|
||||
input: `<observation>
|
||||
<type>bugfix</type>
|
||||
<title>Test</title>
|
||||
<narrative>Test</narrative>
|
||||
<concepts>
|
||||
<concept>bugfix</concept>
|
||||
<concept>security</concept>
|
||||
</concepts>
|
||||
</observation>`,
|
||||
expectedCount: 1,
|
||||
expectedType: models.ObsTypeBugfix,
|
||||
checkConcepts: []string{"security"},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
observations := ParseObservations(tt.input, "test-correlation-id")
|
||||
|
||||
assert.Len(t, observations, tt.expectedCount)
|
||||
if tt.expectedCount > 0 {
|
||||
obs := observations[0]
|
||||
assert.Equal(t, tt.expectedType, obs.Type)
|
||||
if tt.expectedTitle != "" {
|
||||
assert.Equal(t, tt.expectedTitle, obs.Title)
|
||||
}
|
||||
if tt.checkConcepts != nil {
|
||||
assert.Equal(t, tt.checkConcepts, obs.Concepts)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseObservations_AllValidConcepts(t *testing.T) {
|
||||
// Test all valid concepts are accepted
|
||||
validConcepts := []string{
|
||||
"how-it-works", "why-it-exists", "what-changed", "problem-solution", "gotcha", "pattern", "trade-off",
|
||||
"best-practice", "anti-pattern", "architecture", "security", "performance", "testing", "debugging", "workflow", "tooling",
|
||||
"refactoring", "api", "database", "configuration", "error-handling", "caching", "logging", "auth", "validation",
|
||||
}
|
||||
|
||||
for _, concept := range validConcepts {
|
||||
t.Run("concept_"+concept, func(t *testing.T) {
|
||||
input := `<observation>
|
||||
<type>discovery</type>
|
||||
<title>Test</title>
|
||||
<narrative>Test</narrative>
|
||||
<concepts><concept>` + concept + `</concept></concepts>
|
||||
</observation>`
|
||||
|
||||
observations := ParseObservations(input, "test-id")
|
||||
assert.Len(t, observations, 1)
|
||||
assert.Contains(t, observations[0].Concepts, concept)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseObservations_ConceptCaseInsensitive(t *testing.T) {
|
||||
input := `<observation>
|
||||
<type>discovery</type>
|
||||
<title>Test</title>
|
||||
<narrative>Test</narrative>
|
||||
<concepts>
|
||||
<concept>SECURITY</concept>
|
||||
<concept>Best-Practice</concept>
|
||||
<concept> caching </concept>
|
||||
</concepts>
|
||||
</observation>`
|
||||
|
||||
observations := ParseObservations(input, "test-id")
|
||||
|
||||
assert.Len(t, observations, 1)
|
||||
assert.Equal(t, []string{"security", "best-practice", "caching"}, observations[0].Concepts)
|
||||
}
|
||||
|
||||
func TestParseSummary_ValidSummary(t *testing.T) {
|
||||
text := `Some text before
|
||||
<summary>
|
||||
<request>User asked to fix the bug</request>
|
||||
<investigated>Looked at error logs and stack traces</investigated>
|
||||
<learned>The issue was a race condition</learned>
|
||||
<completed>Fixed the race condition with mutex</completed>
|
||||
<next_steps>Add more tests for concurrent access</next_steps>
|
||||
<notes>May need to review similar code elsewhere</notes>
|
||||
</summary>
|
||||
Some text after`
|
||||
|
||||
summary := ParseSummary(text, 123)
|
||||
|
||||
assert.NotNil(t, summary)
|
||||
assert.Equal(t, "User asked to fix the bug", summary.Request)
|
||||
assert.Equal(t, "Looked at error logs and stack traces", summary.Investigated)
|
||||
assert.Equal(t, "The issue was a race condition", summary.Learned)
|
||||
assert.Equal(t, "Fixed the race condition with mutex", summary.Completed)
|
||||
assert.Equal(t, "Add more tests for concurrent access", summary.NextSteps)
|
||||
assert.Equal(t, "May need to review similar code elsewhere", summary.Notes)
|
||||
}
|
||||
|
||||
func TestParseSummary_TableDriven(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
input string
|
||||
expectedRequest string
|
||||
sessionID int64
|
||||
expectNil bool
|
||||
}{
|
||||
{
|
||||
name: "empty_input",
|
||||
input: "",
|
||||
sessionID: 1,
|
||||
expectNil: true,
|
||||
},
|
||||
{
|
||||
name: "no_summary_tag",
|
||||
input: "Just some text without summary",
|
||||
sessionID: 1,
|
||||
expectNil: true,
|
||||
},
|
||||
{
|
||||
name: "skip_summary_tag",
|
||||
input: `<skip_summary reason="No significant changes made"/>`,
|
||||
sessionID: 1,
|
||||
expectNil: true,
|
||||
},
|
||||
{
|
||||
name: "skip_summary_with_different_reason",
|
||||
input: `<skip_summary reason="Only read files"/>`,
|
||||
sessionID: 2,
|
||||
expectNil: true,
|
||||
},
|
||||
{
|
||||
name: "valid_summary_minimal",
|
||||
input: `<summary>
|
||||
<request>Test request</request>
|
||||
</summary>`,
|
||||
sessionID: 3,
|
||||
expectNil: false,
|
||||
expectedRequest: "Test request",
|
||||
},
|
||||
{
|
||||
name: "valid_summary_all_fields",
|
||||
input: `<summary>
|
||||
<request>Full request</request>
|
||||
<investigated>Full investigated</investigated>
|
||||
<learned>Full learned</learned>
|
||||
<completed>Full completed</completed>
|
||||
<next_steps>Full next steps</next_steps>
|
||||
<notes>Full notes</notes>
|
||||
</summary>`,
|
||||
sessionID: 4,
|
||||
expectNil: false,
|
||||
expectedRequest: "Full request",
|
||||
},
|
||||
{
|
||||
name: "summary_with_empty_fields",
|
||||
input: `<summary>
|
||||
<request></request>
|
||||
<investigated></investigated>
|
||||
</summary>`,
|
||||
sessionID: 5,
|
||||
expectNil: false,
|
||||
expectedRequest: "",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
summary := ParseSummary(tt.input, tt.sessionID)
|
||||
|
||||
if tt.expectNil {
|
||||
assert.Nil(t, summary)
|
||||
} else {
|
||||
assert.NotNil(t, summary)
|
||||
assert.Equal(t, tt.expectedRequest, summary.Request)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseSummary_SkipSummaryPriority(t *testing.T) {
|
||||
// skip_summary should take priority over summary block
|
||||
text := `<skip_summary reason="No changes"/>
|
||||
<summary>
|
||||
<request>This should be ignored</request>
|
||||
</summary>`
|
||||
|
||||
summary := ParseSummary(text, 1)
|
||||
assert.Nil(t, summary)
|
||||
}
|
||||
|
||||
func TestExtractField_TableDriven(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
content string
|
||||
fieldName string
|
||||
expected string
|
||||
}{
|
||||
{
|
||||
name: "simple_field",
|
||||
content: "<title>Test Title</title>",
|
||||
fieldName: "title",
|
||||
expected: "Test Title",
|
||||
},
|
||||
{
|
||||
name: "field_with_whitespace",
|
||||
content: "<title> Test Title </title>",
|
||||
fieldName: "title",
|
||||
expected: "Test Title",
|
||||
},
|
||||
{
|
||||
name: "field_not_found",
|
||||
content: "<other>Value</other>",
|
||||
fieldName: "title",
|
||||
expected: "",
|
||||
},
|
||||
{
|
||||
name: "empty_field",
|
||||
content: "<title></title>",
|
||||
fieldName: "title",
|
||||
expected: "",
|
||||
},
|
||||
{
|
||||
name: "nested_content",
|
||||
content: "<wrapper><title>Nested</title></wrapper>",
|
||||
fieldName: "title",
|
||||
expected: "Nested",
|
||||
},
|
||||
{
|
||||
name: "field_among_others",
|
||||
content: "<a>A</a><title>Target</title><b>B</b>",
|
||||
fieldName: "title",
|
||||
expected: "Target",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
result := extractField(tt.content, tt.fieldName)
|
||||
assert.Equal(t, tt.expected, result)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractArrayElements_TableDriven(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
content string
|
||||
arrayName string
|
||||
elementName string
|
||||
expected []string
|
||||
}{
|
||||
{
|
||||
name: "simple_array",
|
||||
content: "<facts><fact>One</fact><fact>Two</fact></facts>",
|
||||
arrayName: "facts",
|
||||
elementName: "fact",
|
||||
expected: []string{"One", "Two"},
|
||||
},
|
||||
{
|
||||
name: "empty_array",
|
||||
content: "<facts></facts>",
|
||||
arrayName: "facts",
|
||||
elementName: "fact",
|
||||
expected: nil,
|
||||
},
|
||||
{
|
||||
name: "array_not_found",
|
||||
content: "<other><item>Value</item></other>",
|
||||
arrayName: "facts",
|
||||
elementName: "fact",
|
||||
expected: nil,
|
||||
},
|
||||
{
|
||||
name: "single_element",
|
||||
content: "<concepts><concept>security</concept></concepts>",
|
||||
arrayName: "concepts",
|
||||
elementName: "concept",
|
||||
expected: []string{"security"},
|
||||
},
|
||||
{
|
||||
name: "multiline_array",
|
||||
content: `<files>
|
||||
<file>file1.go</file>
|
||||
<file>file2.go</file>
|
||||
<file>file3.go</file>
|
||||
</files>`,
|
||||
arrayName: "files",
|
||||
elementName: "file",
|
||||
expected: []string{"file1.go", "file2.go", "file3.go"},
|
||||
},
|
||||
{
|
||||
name: "whitespace_trimmed",
|
||||
content: "<items><item> trimmed </item></items>",
|
||||
arrayName: "items",
|
||||
elementName: "item",
|
||||
expected: []string{"trimmed"},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
result := extractArrayElements(tt.content, tt.arrayName, tt.elementName)
|
||||
assert.Equal(t, tt.expected, result)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidObsTypes(t *testing.T) {
|
||||
expected := map[string]bool{
|
||||
"bugfix": true,
|
||||
"feature": true,
|
||||
"refactor": true,
|
||||
"change": true,
|
||||
"discovery": true,
|
||||
"decision": true,
|
||||
}
|
||||
assert.Equal(t, expected, validObsTypes)
|
||||
}
|
||||
|
||||
func TestValidConcepts(t *testing.T) {
|
||||
// Verify expected concepts are valid
|
||||
expectedValid := []string{
|
||||
"how-it-works", "why-it-exists", "what-changed", "problem-solution", "gotcha", "pattern", "trade-off",
|
||||
"best-practice", "anti-pattern", "architecture", "security", "performance", "testing", "debugging", "workflow", "tooling",
|
||||
"refactoring", "api", "database", "configuration", "error-handling", "caching", "logging", "auth", "validation",
|
||||
}
|
||||
|
||||
for _, concept := range expectedValid {
|
||||
assert.True(t, validConcepts[concept], "Expected %s to be valid", concept)
|
||||
}
|
||||
|
||||
// Verify invalid concepts
|
||||
invalidConcepts := []string{"random", "invalid", "not-a-concept", "foo", "bar"}
|
||||
for _, concept := range invalidConcepts {
|
||||
assert.False(t, validConcepts[concept], "Expected %s to be invalid", concept)
|
||||
}
|
||||
}
|
||||
@@ -31,14 +31,15 @@ type SyncSummaryFunc func(summary *models.SessionSummary)
|
||||
|
||||
// Processor handles SDK agent processing of observations and summaries using Claude Code CLI.
|
||||
type Processor struct {
|
||||
claudePath string
|
||||
model string
|
||||
observationStore *sqlite.ObservationStore
|
||||
summaryStore *sqlite.SummaryStore
|
||||
broadcastFunc BroadcastFunc
|
||||
syncObservationFunc SyncObservationFunc
|
||||
syncSummaryFunc SyncSummaryFunc
|
||||
sem chan struct{}
|
||||
claudePath string
|
||||
model string
|
||||
// Semaphore to limit concurrent Claude CLI calls (prevents API overload)
|
||||
sem chan struct{}
|
||||
}
|
||||
|
||||
// SetBroadcastFunc sets the broadcast callback for SSE events.
|
||||
@@ -233,19 +234,11 @@ func (p *Processor) ProcessObservation(ctx context.Context, sdkSessionID, projec
|
||||
|
||||
// ProcessSummary processes a session summary request.
|
||||
func (p *Processor) ProcessSummary(ctx context.Context, sessionDBID int64, sdkSessionID, project, userPrompt, lastUserMsg, lastAssistantMsg string) error {
|
||||
// Debug: log what we received
|
||||
log.Debug().
|
||||
Int64("sessionId", sessionDBID).
|
||||
Int("lastAssistantMsgLen", len(lastAssistantMsg)).
|
||||
Str("lastAssistantMsgPreview", truncateForLog(lastAssistantMsg, 200)).
|
||||
Msg("ProcessSummary called")
|
||||
|
||||
// Skip summary generation if there's no meaningful assistant response
|
||||
// This prevents generic "initial session setup" summaries
|
||||
if !hasMeaningfulContent(lastAssistantMsg) {
|
||||
log.Info().
|
||||
Int64("sessionId", sessionDBID).
|
||||
Int("msgLen", len(lastAssistantMsg)).
|
||||
Msg("Skipping summary - no meaningful assistant response")
|
||||
return nil
|
||||
}
|
||||
@@ -641,57 +634,25 @@ Your response:`,
|
||||
// rather than actual user work. These summaries should be filtered out.
|
||||
func isSelfReferentialSummary(summary *models.ParsedSummary) bool {
|
||||
// Combine all summary fields for checking
|
||||
content := strings.ToLower(summary.Request + " " + summary.Completed + " " + summary.Learned + " " + summary.NextSteps + " " + summary.Investigated + " " + summary.Notes)
|
||||
content := strings.ToLower(summary.Request + " " + summary.Completed + " " + summary.Learned + " " + summary.NextSteps)
|
||||
|
||||
// Indicators that the summary is about the memory agent, not user work
|
||||
selfReferentialPhrases := []string{
|
||||
// Agent references
|
||||
"memory extraction",
|
||||
"memory agent",
|
||||
"extraction agent",
|
||||
"hook execution",
|
||||
"hook mechanism",
|
||||
// Session meta-state
|
||||
"session initialization",
|
||||
"session setup",
|
||||
"session has just started",
|
||||
"session just started",
|
||||
"agent initialization",
|
||||
"no technical learnings",
|
||||
"no code or project work",
|
||||
// Waiting states
|
||||
"waiting for the user",
|
||||
"waiting for user",
|
||||
"awaiting actual",
|
||||
"awaiting claude code",
|
||||
"awaiting tool",
|
||||
"awaiting user",
|
||||
// Meta checkpoint references
|
||||
"progress checkpoint",
|
||||
"checkpoint request",
|
||||
// Common no-work phrases
|
||||
"no work has been completed",
|
||||
"no work completed",
|
||||
"no work done",
|
||||
"no actual work",
|
||||
"nothing has been completed",
|
||||
"nothing completed",
|
||||
// Role/guideline parroting
|
||||
"role definition",
|
||||
"operational guidelines",
|
||||
"providing role",
|
||||
"providing guidelines",
|
||||
// System prompt echoes
|
||||
"extract meaningful observations",
|
||||
"meaningful learnings",
|
||||
"analyze tool executions",
|
||||
"observations for future sessions",
|
||||
// Empty session indicators
|
||||
"empty session",
|
||||
"no substantive work",
|
||||
"no meaningful work",
|
||||
"just beginning",
|
||||
"just begun",
|
||||
}
|
||||
|
||||
matchCount := 0
|
||||
@@ -716,33 +677,20 @@ func hasMeaningfulContent(assistantMsg string) bool {
|
||||
|
||||
lowerMsg := strings.ToLower(assistantMsg)
|
||||
|
||||
// Skip messages that are primarily about system/hook status or meta-instructions
|
||||
// Skip messages that are primarily about system/hook status
|
||||
skipIndicators := []string{
|
||||
// System/hook markers
|
||||
"hook success",
|
||||
"callback hook",
|
||||
"session start",
|
||||
"sessionstart",
|
||||
"system-reminder",
|
||||
// Agent self-references
|
||||
"memory extraction agent",
|
||||
"memory agent",
|
||||
"extraction agent",
|
||||
// No-work indicators
|
||||
"no technical learnings",
|
||||
"waiting for",
|
||||
"waiting to",
|
||||
"no code or project work",
|
||||
"no substantive",
|
||||
"no work has been completed",
|
||||
"no work done",
|
||||
"awaiting tool",
|
||||
"awaiting user",
|
||||
// Meta-instruction echoes
|
||||
"role definition",
|
||||
"operational guidelines",
|
||||
"analyze tool executions",
|
||||
"extract meaningful observations",
|
||||
}
|
||||
|
||||
skipCount := 0
|
||||
@@ -779,14 +727,6 @@ func hasMeaningfulContent(assistantMsg string) bool {
|
||||
return matchCount >= 2
|
||||
}
|
||||
|
||||
// truncateForLog truncates a string for logging purposes.
|
||||
func truncateForLog(s string, maxLen int) string {
|
||||
if len(s) <= maxLen {
|
||||
return s
|
||||
}
|
||||
return s[:maxLen] + "..."
|
||||
}
|
||||
|
||||
const systemPrompt = `You are a memory extraction agent for Claude Code sessions. Your job is to analyze tool executions and extract meaningful observations that would be useful for future sessions.
|
||||
|
||||
GUIDELINES:
|
||||
|
||||
@@ -1,976 +0,0 @@
|
||||
package sdk
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/lukaszraczylo/claude-mnemonic/pkg/models"
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestIsSelfReferentialSummary(t *testing.T) {
|
||||
tests := []struct {
|
||||
summary *models.ParsedSummary
|
||||
name string
|
||||
expected bool
|
||||
}{
|
||||
{
|
||||
name: "meta summary about memory agent role",
|
||||
summary: &models.ParsedSummary{
|
||||
Request: "Memory extraction agent role - analyze tool executions and extract meaningful observations for future sessions",
|
||||
Completed: "No work has been completed yet. The session has just started with the user providing role definition and operational guidelines.",
|
||||
Learned: "The system expects observations to be created from meaningful learnings during Claude Code sessions, with focus on decisions, bugs fixed, patterns discovered, project structure changes, and code modifications.",
|
||||
NextSteps: "Awaiting tool executions or user requests that contain actual work performed in a Claude Code session.",
|
||||
},
|
||||
expected: true,
|
||||
},
|
||||
{
|
||||
name: "legitimate summary about code changes",
|
||||
summary: &models.ParsedSummary{
|
||||
Request: "Fix authentication bug in login handler",
|
||||
Completed: "Updated the auth middleware to properly validate JWT tokens and fixed the session expiry check.",
|
||||
Learned: "The JWT library requires explicit algorithm validation to prevent token substitution attacks.",
|
||||
NextSteps: "Add unit tests for the authentication flow.",
|
||||
},
|
||||
expected: false,
|
||||
},
|
||||
{
|
||||
name: "awaiting user summary",
|
||||
summary: &models.ParsedSummary{
|
||||
Request: "Session initialization",
|
||||
Completed: "No work completed yet.",
|
||||
Learned: "Awaiting user input to begin work.",
|
||||
NextSteps: "Waiting for the user to provide instructions.",
|
||||
},
|
||||
expected: true,
|
||||
},
|
||||
{
|
||||
name: "summary about refactoring",
|
||||
summary: &models.ParsedSummary{
|
||||
Request: "Refactor database connection pooling",
|
||||
Completed: "Implemented connection pooling using pgxpool with max 10 connections.",
|
||||
Learned: "pgxpool automatically handles connection reuse and health checks.",
|
||||
NextSteps: "Run benchmarks to verify performance improvement.",
|
||||
},
|
||||
expected: false,
|
||||
},
|
||||
{
|
||||
name: "meta summary with extraction agent mention",
|
||||
summary: &models.ParsedSummary{
|
||||
Request: "Extraction agent initialization",
|
||||
Completed: "No substantive work has been done.",
|
||||
Learned: "The memory extraction agent analyzes tool executions.",
|
||||
NextSteps: "Awaiting tool results to extract observations.",
|
||||
},
|
||||
expected: true,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
result := isSelfReferentialSummary(tt.summary)
|
||||
if result != tt.expected {
|
||||
t.Errorf("isSelfReferentialSummary() = %v, want %v", result, tt.expected)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestHasMeaningfulContent(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
content string
|
||||
expected bool
|
||||
}{
|
||||
{
|
||||
name: "empty content",
|
||||
content: "",
|
||||
expected: false,
|
||||
},
|
||||
{
|
||||
name: "too short content",
|
||||
content: "Hello world",
|
||||
expected: false,
|
||||
},
|
||||
{
|
||||
name: "meta content about memory agent",
|
||||
content: `This is the memory extraction agent role definition.
|
||||
The system expects you to analyze tool executions and extract meaningful observations.
|
||||
No work has been completed yet. Awaiting tool results from the user's session.`,
|
||||
expected: false,
|
||||
},
|
||||
{
|
||||
name: "legitimate code discussion",
|
||||
content: `I've updated the handler.go file to fix the authentication bug.
|
||||
The function validateToken() was not checking token expiry correctly.
|
||||
I've added a check for exp claim and implemented proper error handling.
|
||||
The changes have been tested and the build passes.`,
|
||||
expected: true,
|
||||
},
|
||||
{
|
||||
name: "hook status messages",
|
||||
content: `SessionStart:Callback hook success: Success
|
||||
The memory agent is waiting for user input.
|
||||
System-reminder about available tools.
|
||||
No substantive work performed yet.`,
|
||||
expected: false,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
result := hasMeaningfulContent(tt.content)
|
||||
if result != tt.expected {
|
||||
t.Errorf("hasMeaningfulContent() = %v, want %v", result, tt.expected)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestShouldSkipTool(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
toolName string
|
||||
expected bool
|
||||
}{
|
||||
// Tools that should be skipped
|
||||
{"TodoWrite", "TodoWrite", true},
|
||||
{"Task", "Task", true},
|
||||
{"TaskOutput", "TaskOutput", true},
|
||||
{"Glob", "Glob", true},
|
||||
{"ListDir", "ListDir", true},
|
||||
{"LS", "LS", true},
|
||||
{"KillShell", "KillShell", true},
|
||||
{"AskUserQuestion", "AskUserQuestion", true},
|
||||
{"EnterPlanMode", "EnterPlanMode", true},
|
||||
{"ExitPlanMode", "ExitPlanMode", true},
|
||||
{"Skill", "Skill", true},
|
||||
{"SlashCommand", "SlashCommand", true},
|
||||
|
||||
// Tools that should NOT be skipped
|
||||
{"Read", "Read", false},
|
||||
{"Edit", "Edit", false},
|
||||
{"Write", "Write", false},
|
||||
{"Grep", "Grep", false},
|
||||
{"Bash", "Bash", false},
|
||||
{"WebFetch", "WebFetch", false},
|
||||
{"WebSearch", "WebSearch", false},
|
||||
{"NotebookEdit", "NotebookEdit", false},
|
||||
|
||||
// Unknown tool (should not be skipped)
|
||||
{"UnknownTool", "SomeUnknownTool", false},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
result := shouldSkipTool(tt.toolName)
|
||||
assert.Equal(t, tt.expected, result)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestShouldSkipTrivialOperation(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
toolName string
|
||||
inputStr string
|
||||
outputStr string
|
||||
expected bool
|
||||
}{
|
||||
// Short output (should be skipped)
|
||||
{
|
||||
name: "output_too_short",
|
||||
toolName: "Read",
|
||||
inputStr: `{"file_path": "/some/file.go"}`,
|
||||
outputStr: "short",
|
||||
expected: true,
|
||||
},
|
||||
// Trivial outputs
|
||||
{
|
||||
name: "no_matches_found",
|
||||
toolName: "Grep",
|
||||
inputStr: `{"pattern": "foo"}`,
|
||||
outputStr: "No matches found in the codebase",
|
||||
expected: true,
|
||||
},
|
||||
{
|
||||
name: "file_not_found",
|
||||
toolName: "Read",
|
||||
inputStr: `{"file_path": "/nonexistent.go"}`,
|
||||
outputStr: "Error: File not found at specified path",
|
||||
expected: true,
|
||||
},
|
||||
{
|
||||
name: "empty_array",
|
||||
toolName: "Grep",
|
||||
inputStr: `{"pattern": "foo"}`,
|
||||
outputStr: "[]",
|
||||
expected: true,
|
||||
},
|
||||
// Boring files
|
||||
{
|
||||
name: "package_lock_json",
|
||||
toolName: "Read",
|
||||
inputStr: `{"file_path": "/project/package-lock.json"}`,
|
||||
outputStr: "This is a very long package-lock.json content that has more than 50 characters",
|
||||
expected: true,
|
||||
},
|
||||
{
|
||||
name: "go_sum",
|
||||
toolName: "Read",
|
||||
inputStr: `{"file_path": "/project/go.sum"}`,
|
||||
outputStr: "This is a very long go.sum file content that has more than 50 characters",
|
||||
expected: true,
|
||||
},
|
||||
// Grep with too many matches
|
||||
{
|
||||
name: "grep_too_many_matches",
|
||||
toolName: "Grep",
|
||||
inputStr: `{"pattern": "import"}`,
|
||||
outputStr: func() string {
|
||||
s := ""
|
||||
for i := 0; i < 55; i++ {
|
||||
s += "match line\n"
|
||||
}
|
||||
return s
|
||||
}(),
|
||||
expected: true,
|
||||
},
|
||||
// Boring Bash commands
|
||||
{
|
||||
name: "git_status",
|
||||
toolName: "Bash",
|
||||
inputStr: `{"command": "git status"}`,
|
||||
outputStr: "On branch main\nYour branch is up to date with 'origin/main'.\nnothing to commit, working tree clean",
|
||||
expected: true,
|
||||
},
|
||||
{
|
||||
name: "ls_command",
|
||||
toolName: "Bash",
|
||||
inputStr: `{"command": "ls -la /some/directory"}`,
|
||||
outputStr: "total 123\ndrwxr-xr-x some long listing that is at least 50 chars",
|
||||
expected: true,
|
||||
},
|
||||
// Valid operations that should NOT be skipped
|
||||
{
|
||||
name: "valid_read",
|
||||
toolName: "Read",
|
||||
inputStr: `{"file_path": "/project/main.go"}`,
|
||||
outputStr: "package main\n\nimport \"fmt\"\n\nfunc main() {\n\tfmt.Println(\"Hello World\")\n}",
|
||||
expected: false,
|
||||
},
|
||||
{
|
||||
name: "valid_edit",
|
||||
toolName: "Edit",
|
||||
inputStr: `{"file_path": "/project/handler.go", "old_string": "foo", "new_string": "bar"}`,
|
||||
outputStr: "Edit applied successfully. File /project/handler.go has been modified with the requested changes.",
|
||||
expected: false,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
result := shouldSkipTrivialOperation(tt.toolName, tt.inputStr, tt.outputStr)
|
||||
assert.Equal(t, tt.expected, result)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestTruncateForLog(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
input string
|
||||
expected string
|
||||
maxLen int
|
||||
}{
|
||||
{
|
||||
name: "shorter_than_max",
|
||||
input: "hello",
|
||||
maxLen: 10,
|
||||
expected: "hello",
|
||||
},
|
||||
{
|
||||
name: "equal_to_max",
|
||||
input: "hello",
|
||||
maxLen: 5,
|
||||
expected: "hello",
|
||||
},
|
||||
{
|
||||
name: "longer_than_max",
|
||||
input: "hello world",
|
||||
maxLen: 5,
|
||||
expected: "hello...",
|
||||
},
|
||||
{
|
||||
name: "empty_string",
|
||||
input: "",
|
||||
maxLen: 5,
|
||||
expected: "",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
result := truncateForLog(tt.input, tt.maxLen)
|
||||
assert.Equal(t, tt.expected, result)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestToJSONString(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
input interface{}
|
||||
expected string
|
||||
}{
|
||||
{
|
||||
name: "nil_value",
|
||||
input: nil,
|
||||
expected: "",
|
||||
},
|
||||
{
|
||||
name: "string_value",
|
||||
input: "hello",
|
||||
expected: "hello",
|
||||
},
|
||||
{
|
||||
name: "int_value",
|
||||
input: 42,
|
||||
expected: "42",
|
||||
},
|
||||
{
|
||||
name: "map_value",
|
||||
input: map[string]string{"key": "value"},
|
||||
expected: `{"key":"value"}`,
|
||||
},
|
||||
{
|
||||
name: "slice_value",
|
||||
input: []string{"a", "b", "c"},
|
||||
expected: `["a","b","c"]`,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
result := toJSONString(tt.input)
|
||||
assert.Equal(t, tt.expected, result)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestCaptureFileMtimes(t *testing.T) {
|
||||
// Create a temp directory with test files
|
||||
tmpDir, err := os.MkdirTemp("", "mtime-test-*")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer os.RemoveAll(tmpDir)
|
||||
|
||||
// Create test files
|
||||
file1 := filepath.Join(tmpDir, "file1.txt")
|
||||
file2 := filepath.Join(tmpDir, "file2.txt")
|
||||
|
||||
err = os.WriteFile(file1, []byte("content1"), 0600)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
err = os.WriteFile(file2, []byte("content2"), 0600)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
t.Run("captures_mtimes_for_existing_files", func(t *testing.T) {
|
||||
mtimes := captureFileMtimes([]string{file1}, []string{file2}, "")
|
||||
assert.Len(t, mtimes, 2)
|
||||
assert.Contains(t, mtimes, file1)
|
||||
assert.Contains(t, mtimes, file2)
|
||||
assert.Greater(t, mtimes[file1], int64(0))
|
||||
assert.Greater(t, mtimes[file2], int64(0))
|
||||
})
|
||||
|
||||
t.Run("handles_nonexistent_files", func(t *testing.T) {
|
||||
mtimes := captureFileMtimes([]string{"/nonexistent/file.txt"}, nil, "")
|
||||
assert.Empty(t, mtimes)
|
||||
})
|
||||
|
||||
t.Run("handles_relative_paths_with_cwd", func(t *testing.T) {
|
||||
mtimes := captureFileMtimes([]string{"file1.txt"}, nil, tmpDir)
|
||||
assert.Len(t, mtimes, 1)
|
||||
assert.Contains(t, mtimes, "file1.txt")
|
||||
})
|
||||
|
||||
t.Run("empty_inputs", func(t *testing.T) {
|
||||
mtimes := captureFileMtimes(nil, nil, "")
|
||||
assert.Empty(t, mtimes)
|
||||
})
|
||||
}
|
||||
|
||||
func TestGetFileMtimes(t *testing.T) {
|
||||
// Create a temp file
|
||||
tmpDir, err := os.MkdirTemp("", "getmtime-test-*")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer os.RemoveAll(tmpDir)
|
||||
|
||||
testFile := filepath.Join(tmpDir, "test.txt")
|
||||
err = os.WriteFile(testFile, []byte("content"), 0600)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
mtimes := GetFileMtimes([]string{testFile}, "")
|
||||
assert.Len(t, mtimes, 1)
|
||||
assert.Contains(t, mtimes, testFile)
|
||||
assert.Greater(t, mtimes[testFile], int64(0))
|
||||
}
|
||||
|
||||
func TestGetFileContent(t *testing.T) {
|
||||
// Create a temp directory with test files
|
||||
tmpDir, err := os.MkdirTemp("", "content-test-*")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer os.RemoveAll(tmpDir)
|
||||
|
||||
t.Run("reads_existing_file", func(t *testing.T) {
|
||||
testFile := filepath.Join(tmpDir, "test.txt")
|
||||
content := "test content"
|
||||
err := os.WriteFile(testFile, []byte(content), 0600)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
result, ok := GetFileContent(testFile, "")
|
||||
assert.True(t, ok)
|
||||
assert.Equal(t, content, result)
|
||||
})
|
||||
|
||||
t.Run("returns_false_for_nonexistent_file", func(t *testing.T) {
|
||||
result, ok := GetFileContent("/nonexistent/file.txt", "")
|
||||
assert.False(t, ok)
|
||||
assert.Empty(t, result)
|
||||
})
|
||||
|
||||
t.Run("truncates_long_content", func(t *testing.T) {
|
||||
testFile := filepath.Join(tmpDir, "long.txt")
|
||||
longContent := ""
|
||||
for i := 0; i < 3000; i++ {
|
||||
longContent += "x"
|
||||
}
|
||||
err := os.WriteFile(testFile, []byte(longContent), 0600)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
result, ok := GetFileContent(testFile, "")
|
||||
assert.True(t, ok)
|
||||
assert.Contains(t, result, "[truncated]")
|
||||
assert.LessOrEqual(t, len(result), 2100)
|
||||
})
|
||||
|
||||
t.Run("resolves_relative_path_with_cwd", func(t *testing.T) {
|
||||
testFile := filepath.Join(tmpDir, "relative.txt")
|
||||
content := "relative content"
|
||||
err := os.WriteFile(testFile, []byte(content), 0600)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
result, ok := GetFileContent("relative.txt", tmpDir)
|
||||
assert.True(t, ok)
|
||||
assert.Equal(t, content, result)
|
||||
})
|
||||
}
|
||||
|
||||
func TestMaxConcurrentCLICalls(t *testing.T) {
|
||||
assert.Equal(t, 4, MaxConcurrentCLICalls)
|
||||
}
|
||||
|
||||
func TestObservationTypes(t *testing.T) {
|
||||
expected := []string{"bugfix", "feature", "refactor", "change", "discovery", "decision"}
|
||||
assert.Equal(t, expected, ObservationTypes)
|
||||
}
|
||||
|
||||
func TestObservationConcepts(t *testing.T) {
|
||||
expectedConcepts := []string{
|
||||
"how-it-works",
|
||||
"why-it-exists",
|
||||
"what-changed",
|
||||
"problem-solution",
|
||||
"gotcha",
|
||||
"pattern",
|
||||
"trade-off",
|
||||
}
|
||||
assert.Equal(t, expectedConcepts, ObservationConcepts)
|
||||
}
|
||||
|
||||
// TestProcessorStruct tests processor struct initialization and methods.
|
||||
func TestProcessorStruct(t *testing.T) {
|
||||
p := &Processor{
|
||||
claudePath: "/path/to/claude",
|
||||
model: "haiku",
|
||||
sem: make(chan struct{}, MaxConcurrentCLICalls),
|
||||
}
|
||||
|
||||
assert.Equal(t, "/path/to/claude", p.claudePath)
|
||||
assert.Equal(t, "haiku", p.model)
|
||||
assert.NotNil(t, p.sem)
|
||||
}
|
||||
|
||||
// TestSetBroadcastFunc tests the broadcast callback setter.
|
||||
func TestSetBroadcastFunc(t *testing.T) {
|
||||
p := &Processor{}
|
||||
|
||||
assert.Nil(t, p.broadcastFunc)
|
||||
|
||||
var called bool
|
||||
var receivedEvent map[string]interface{}
|
||||
fn := func(event map[string]interface{}) {
|
||||
called = true
|
||||
receivedEvent = event
|
||||
}
|
||||
|
||||
p.SetBroadcastFunc(fn)
|
||||
assert.NotNil(t, p.broadcastFunc)
|
||||
|
||||
// Test broadcast
|
||||
p.broadcast(map[string]interface{}{"type": "test"})
|
||||
assert.True(t, called)
|
||||
assert.Equal(t, "test", receivedEvent["type"])
|
||||
}
|
||||
|
||||
// TestSetSyncObservationFunc tests the sync observation callback setter.
|
||||
func TestSetSyncObservationFunc(t *testing.T) {
|
||||
p := &Processor{}
|
||||
|
||||
assert.Nil(t, p.syncObservationFunc)
|
||||
|
||||
var called bool
|
||||
fn := func(obs *models.Observation) {
|
||||
called = true
|
||||
}
|
||||
|
||||
p.SetSyncObservationFunc(fn)
|
||||
assert.NotNil(t, p.syncObservationFunc)
|
||||
|
||||
// Verify it was set
|
||||
p.syncObservationFunc(&models.Observation{})
|
||||
assert.True(t, called)
|
||||
}
|
||||
|
||||
// TestSetSyncSummaryFunc tests the sync summary callback setter.
|
||||
func TestSetSyncSummaryFunc(t *testing.T) {
|
||||
p := &Processor{}
|
||||
|
||||
assert.Nil(t, p.syncSummaryFunc)
|
||||
|
||||
var called bool
|
||||
fn := func(summary *models.SessionSummary) {
|
||||
called = true
|
||||
}
|
||||
|
||||
p.SetSyncSummaryFunc(fn)
|
||||
assert.NotNil(t, p.syncSummaryFunc)
|
||||
|
||||
// Verify it was set
|
||||
p.syncSummaryFunc(&models.SessionSummary{})
|
||||
assert.True(t, called)
|
||||
}
|
||||
|
||||
// TestBroadcast_NilFunc tests broadcast with nil callback.
|
||||
func TestBroadcast_NilFunc(t *testing.T) {
|
||||
p := &Processor{}
|
||||
|
||||
// Should not panic
|
||||
p.broadcast(map[string]interface{}{"type": "test"})
|
||||
}
|
||||
|
||||
// TestIsAvailable_NonexistentPath tests IsAvailable with non-existent path.
|
||||
func TestIsAvailable_NonexistentPath(t *testing.T) {
|
||||
p := &Processor{
|
||||
claudePath: "/nonexistent/path/to/claude",
|
||||
}
|
||||
|
||||
assert.False(t, p.IsAvailable())
|
||||
}
|
||||
|
||||
// TestIsAvailable_ExistingPath tests IsAvailable with existing path.
|
||||
func TestIsAvailable_ExistingPath(t *testing.T) {
|
||||
// Create a temp file to simulate claude binary
|
||||
tmpFile, err := os.CreateTemp("", "claude-test-*")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer os.Remove(tmpFile.Name())
|
||||
tmpFile.Close()
|
||||
|
||||
p := &Processor{
|
||||
claudePath: tmpFile.Name(),
|
||||
}
|
||||
|
||||
assert.True(t, p.IsAvailable())
|
||||
}
|
||||
|
||||
// TestShouldSkipTrivialOperation_EdgeCases tests edge cases for trivial operation detection.
|
||||
func TestShouldSkipTrivialOperation_EdgeCases(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
toolName string
|
||||
inputStr string
|
||||
outputStr string
|
||||
expected bool
|
||||
}{
|
||||
{
|
||||
name: "gitignore_file",
|
||||
toolName: "Read",
|
||||
inputStr: `{"file_path": "/project/.gitignore"}`,
|
||||
outputStr: "This is a gitignore file content that has more than 50 characters long",
|
||||
expected: true,
|
||||
},
|
||||
{
|
||||
name: "eslintignore_file",
|
||||
toolName: "Read",
|
||||
inputStr: `{"file_path": "/project/.eslintignore"}`,
|
||||
outputStr: "This is an eslintignore file content that has more than 50 characters long",
|
||||
expected: true,
|
||||
},
|
||||
{
|
||||
name: "tsconfig_file",
|
||||
toolName: "Read",
|
||||
inputStr: `{"file_path": "/project/tsconfig.json"}`,
|
||||
outputStr: "This is a tsconfig.json file content that has more than 50 characters long",
|
||||
expected: true,
|
||||
},
|
||||
{
|
||||
name: "tailwind_config",
|
||||
toolName: "Read",
|
||||
inputStr: `{"file_path": "/project/tailwind.config.js"}`,
|
||||
outputStr: "This is a tailwind.config file content that has more than 50 characters long",
|
||||
expected: true,
|
||||
},
|
||||
{
|
||||
name: "pwd_command",
|
||||
toolName: "Bash",
|
||||
inputStr: `{"command": "pwd"}`,
|
||||
outputStr: "/home/user/project/some/long/path/that/is/more/than/fifty/chars",
|
||||
expected: true,
|
||||
},
|
||||
{
|
||||
name: "echo_command",
|
||||
toolName: "Bash",
|
||||
inputStr: `{"command": "echo Hello World"}`,
|
||||
outputStr: "Hello World output that is long enough to pass the length check here",
|
||||
expected: true,
|
||||
},
|
||||
{
|
||||
name: "npm_audit_command",
|
||||
toolName: "Bash",
|
||||
inputStr: `{"command": "npm audit"}`,
|
||||
outputStr: "found 0 vulnerabilities in 500 packages which is more than fifty characters",
|
||||
expected: true,
|
||||
},
|
||||
{
|
||||
name: "permission_denied",
|
||||
toolName: "Read",
|
||||
inputStr: `{"file_path": "/root/secret"}`,
|
||||
outputStr: "Error: Permission denied accessing the file at specified path",
|
||||
expected: true,
|
||||
},
|
||||
{
|
||||
name: "is_a_directory",
|
||||
toolName: "Read",
|
||||
inputStr: `{"file_path": "/some/dir"}`,
|
||||
outputStr: "Error: /some/dir is a directory, not a file that can be read",
|
||||
expected: true,
|
||||
},
|
||||
{
|
||||
name: "empty_object",
|
||||
toolName: "Grep",
|
||||
inputStr: `{"pattern": "nonexistent"}`,
|
||||
outputStr: "{}",
|
||||
expected: true,
|
||||
},
|
||||
{
|
||||
name: "valid_grep_result",
|
||||
toolName: "Grep",
|
||||
inputStr: `{"pattern": "func main"}`,
|
||||
outputStr: "main.go:10:func main() {\nmain.go:11: fmt.Println(\"Hello\")\n}",
|
||||
expected: false,
|
||||
},
|
||||
{
|
||||
name: "valid_bash_build",
|
||||
toolName: "Bash",
|
||||
inputStr: `{"command": "go build ./..."}`,
|
||||
outputStr: "Build completed successfully. Binary output at ./bin/myapp with size 10MB.",
|
||||
expected: false,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
result := shouldSkipTrivialOperation(tt.toolName, tt.inputStr, tt.outputStr)
|
||||
assert.Equal(t, tt.expected, result)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestIsSelfReferentialSummary_MoreCases tests additional self-referential detection cases.
|
||||
func TestIsSelfReferentialSummary_MoreCases(t *testing.T) {
|
||||
tests := []struct {
|
||||
summary *models.ParsedSummary
|
||||
name string
|
||||
expected bool
|
||||
}{
|
||||
{
|
||||
name: "progress_checkpoint",
|
||||
summary: &models.ParsedSummary{
|
||||
Request: "Progress checkpoint for current session",
|
||||
Completed: "Responding to progress checkpoint request",
|
||||
Learned: "No technical learnings yet",
|
||||
},
|
||||
expected: true,
|
||||
},
|
||||
{
|
||||
name: "empty_session",
|
||||
summary: &models.ParsedSummary{
|
||||
Request: "Empty session",
|
||||
Completed: "Just beginning the session",
|
||||
Learned: "Nothing has been completed yet",
|
||||
},
|
||||
expected: true,
|
||||
},
|
||||
{
|
||||
name: "hook_mechanism",
|
||||
summary: &models.ParsedSummary{
|
||||
Request: "Hook execution for session start",
|
||||
Completed: "Hook mechanism triggered successfully",
|
||||
Learned: "System hooks are working",
|
||||
},
|
||||
expected: true,
|
||||
},
|
||||
{
|
||||
name: "api_implementation",
|
||||
summary: &models.ParsedSummary{
|
||||
Request: "Implement REST API endpoints",
|
||||
Completed: "Created /users and /posts endpoints with CRUD operations",
|
||||
Learned: "chi router handles middleware chaining elegantly",
|
||||
NextSteps: "Add authentication middleware",
|
||||
},
|
||||
expected: false,
|
||||
},
|
||||
{
|
||||
name: "database_migration",
|
||||
summary: &models.ParsedSummary{
|
||||
Request: "Add database migration for new user fields",
|
||||
Completed: "Created migration 003_add_user_profile.sql with new columns",
|
||||
Learned: "SQLite ALTER TABLE has limited capabilities, need to recreate table",
|
||||
NextSteps: "Test migration rollback",
|
||||
},
|
||||
expected: false,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
result := isSelfReferentialSummary(tt.summary)
|
||||
assert.Equal(t, tt.expected, result)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestHasMeaningfulContent_MoreCases tests additional meaningful content detection.
|
||||
func TestHasMeaningfulContent_MoreCases(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
content string
|
||||
expected bool
|
||||
}{
|
||||
{
|
||||
name: "code_with_functions",
|
||||
content: `I've created a new handler function in handlers.go.
|
||||
The function validateRequest() checks the incoming JSON payload.
|
||||
Here's the implementation:
|
||||
` + "```go\nfunc validateRequest(r *http.Request) error {\n\treturn nil\n}\n```",
|
||||
expected: true,
|
||||
},
|
||||
{
|
||||
name: "python_code_discussion",
|
||||
content: `Updated the data processing module in processor.py.
|
||||
Changed the filter function to use list comprehension.
|
||||
def process_data(items):
|
||||
return [item for item in items if item.valid]
|
||||
This improved performance by 30%.`,
|
||||
expected: true,
|
||||
},
|
||||
{
|
||||
name: "typescript_changes",
|
||||
content: `I've modified the React component in UserProfile.tsx.
|
||||
Added a new functional component with proper TypeScript type annotations.
|
||||
Here's the updated implementation:
|
||||
` + "```tsx\nconst UserProfile: FC<Props> = ({ user }) => {\n return <div>{user.name}</div>;\n};\n```" + `
|
||||
The type annotations ensure type safety across the application.
|
||||
The component has been updated with proper error handling and loading states.`,
|
||||
expected: true,
|
||||
},
|
||||
{
|
||||
name: "yaml_config_update",
|
||||
content: `I've updated the kubernetes deployment config in deploy.yaml.
|
||||
Changed replicas from 2 to 4 and added resource limits for memory and CPU.
|
||||
The deployment.yaml file now includes the following struct configuration:
|
||||
` + "```yaml\nreplicas: 4\nresources:\n limits:\n memory: 512Mi\n```" + `
|
||||
The changes have been implemented and will be applied on next deploy.`,
|
||||
expected: true,
|
||||
},
|
||||
{
|
||||
name: "just_system_messages",
|
||||
content: `SessionStart:Callback hook success
|
||||
System-reminder about tools
|
||||
The session is starting
|
||||
Waiting for user instructions`,
|
||||
expected: false,
|
||||
},
|
||||
{
|
||||
name: "borderline_short",
|
||||
content: "Fixed bug. Updated file. Added test. Committed changes to repository.",
|
||||
expected: false, // Too short (< 200 chars)
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
result := hasMeaningfulContent(tt.content)
|
||||
assert.Equal(t, tt.expected, result)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestToJSONString_ComplexTypes tests JSON conversion for complex types.
|
||||
func TestToJSONString_ComplexTypes(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
input interface{}
|
||||
contains string
|
||||
}{
|
||||
{
|
||||
name: "nested_map",
|
||||
input: map[string]interface{}{
|
||||
"outer": map[string]string{"inner": "value"},
|
||||
},
|
||||
contains: "inner",
|
||||
},
|
||||
{
|
||||
name: "bool_true",
|
||||
input: true,
|
||||
contains: "true",
|
||||
},
|
||||
{
|
||||
name: "bool_false",
|
||||
input: false,
|
||||
contains: "false",
|
||||
},
|
||||
{
|
||||
name: "float_value",
|
||||
input: 3.14,
|
||||
contains: "3.14",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
result := toJSONString(tt.input)
|
||||
assert.Contains(t, result, tt.contains)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestSystemPrompt tests that the system prompt is defined.
|
||||
func TestSystemPrompt(t *testing.T) {
|
||||
assert.NotEmpty(t, systemPrompt)
|
||||
assert.Contains(t, systemPrompt, "memory extraction agent")
|
||||
assert.Contains(t, systemPrompt, "observation")
|
||||
assert.Contains(t, systemPrompt, "GUIDELINES")
|
||||
}
|
||||
|
||||
// TestProcessorSemaphore tests the semaphore behavior.
|
||||
func TestProcessorSemaphore(t *testing.T) {
|
||||
p := &Processor{
|
||||
sem: make(chan struct{}, 2),
|
||||
}
|
||||
|
||||
// Acquire 2 slots
|
||||
p.sem <- struct{}{}
|
||||
p.sem <- struct{}{}
|
||||
|
||||
// Third should block (we can test with select)
|
||||
select {
|
||||
case p.sem <- struct{}{}:
|
||||
t.Error("Semaphore should be full")
|
||||
default:
|
||||
// Expected - semaphore is full
|
||||
}
|
||||
|
||||
// Release one
|
||||
<-p.sem
|
||||
|
||||
// Now should be able to acquire
|
||||
select {
|
||||
case p.sem <- struct{}{}:
|
||||
// Expected
|
||||
default:
|
||||
t.Error("Should be able to acquire after release")
|
||||
}
|
||||
}
|
||||
|
||||
// TestCaptureFileMtimes_DuplicatePaths tests mtime capture with overlapping paths.
|
||||
func TestCaptureFileMtimes_DuplicatePaths(t *testing.T) {
|
||||
tmpDir, err := os.MkdirTemp("", "mtime-dup-test-*")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer os.RemoveAll(tmpDir)
|
||||
|
||||
testFile := filepath.Join(tmpDir, "shared.txt")
|
||||
err = os.WriteFile(testFile, []byte("content"), 0600)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Same file in both read and modified lists
|
||||
mtimes := captureFileMtimes([]string{testFile}, []string{testFile}, "")
|
||||
|
||||
// Should only have one entry (no duplicates)
|
||||
assert.Len(t, mtimes, 1)
|
||||
assert.Contains(t, mtimes, testFile)
|
||||
}
|
||||
|
||||
// TestTruncateForLog_ZeroLength tests truncation with zero length.
|
||||
func TestTruncateForLog_ZeroLength(t *testing.T) {
|
||||
result := truncateForLog("hello", 0)
|
||||
assert.Equal(t, "...", result)
|
||||
}
|
||||
|
||||
// TestBroadcastFuncType tests the BroadcastFunc type.
|
||||
func TestBroadcastFuncType(t *testing.T) {
|
||||
var fn BroadcastFunc = func(event map[string]interface{}) {
|
||||
// Do nothing
|
||||
}
|
||||
assert.NotNil(t, fn)
|
||||
}
|
||||
|
||||
// TestSyncObservationFuncType tests the SyncObservationFunc type.
|
||||
func TestSyncObservationFuncType(t *testing.T) {
|
||||
var fn SyncObservationFunc = func(obs *models.Observation) {
|
||||
// Do nothing
|
||||
}
|
||||
assert.NotNil(t, fn)
|
||||
}
|
||||
|
||||
// TestSyncSummaryFuncType tests the SyncSummaryFunc type.
|
||||
func TestSyncSummaryFuncType(t *testing.T) {
|
||||
var fn SyncSummaryFunc = func(summary *models.SessionSummary) {
|
||||
// Do nothing
|
||||
}
|
||||
assert.NotNil(t, fn)
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user