Move from chroma to sqlitevec with local embedding

This commit is contained in:
2025-12-16 11:28:26 +00:00
parent a208d447dd
commit a68bc36102
25 changed files with 31652 additions and 1161 deletions
+18
View File
@@ -254,6 +254,24 @@ var Migrations = []Migration{
ALTER TABLE user_prompts ADD COLUMN matched_observations INTEGER DEFAULT 0;
`,
},
{
Version: 17,
Name: "sqlite_vec_vectors",
SQL: `
-- Vector embeddings table using sqlite-vec
-- Each document (narrative, fact, summary field, prompt) gets one vector
-- Uses all-MiniLM-L6-v2 embeddings (384 dimensions)
CREATE VIRTUAL TABLE IF NOT EXISTS vectors USING vec0(
doc_id TEXT PRIMARY KEY,
embedding float[384],
sqlite_id INTEGER,
doc_type TEXT,
field_type TEXT,
project TEXT,
scope TEXT
);
`,
},
}
// MigrationManager handles database schema migrations.
+10
View File
@@ -7,6 +7,7 @@ import (
"fmt"
"sync"
sqlite_vec "github.com/asg017/sqlite-vec-go-bindings/cgo"
_ "github.com/mattn/go-sqlite3"
)
@@ -26,6 +27,9 @@ type StoreConfig struct {
// NewStore creates a new database store with the given configuration.
func NewStore(cfg StoreConfig) (*Store, error) {
// Register sqlite-vec extension for vector operations
sqlite_vec.Auto()
// Build connection string with pragmas
connStr := cfg.Path + "?_journal_mode=WAL&_synchronous=NORMAL&_foreign_keys=ON"
@@ -137,3 +141,9 @@ func (s *Store) QueryRowContext(ctx context.Context, query string, args ...inter
func (s *Store) Ping() error {
return s.db.Ping()
}
// DB returns the underlying database connection for direct access.
// Use this sparingly - prefer the store methods for most operations.
func (s *Store) DB() *sql.DB {
return s.db
}