// Package gorm provides GORM-based database operations for claude-mnemonic. package gorm import ( "database/sql" "fmt" "time" "github.com/go-gormigrate/gormigrate/v2" "gorm.io/gorm" "gorm.io/gorm/clause" ) // runMigrations runs all database migrations using gormigrate. func runMigrations(db *gorm.DB, sqlDB *sql.DB) error { m := gormigrate.New(db, gormigrate.DefaultOptions, []*gormigrate.Migration{ // Migration 001: Core tables (SDKSession, Observation, SessionSummary) { ID: "001_core_tables", Migrate: func(tx *gorm.DB) error { // AutoMigrate creates tables with all indexes from struct tags if err := tx.AutoMigrate(&SDKSession{}); err != nil { return err } if err := tx.AutoMigrate(&Observation{}); err != nil { return err } return tx.AutoMigrate(&SessionSummary{}) }, Rollback: func(tx *gorm.DB) error { return tx.Migrator().DropTable("sdk_sessions", "observations", "session_summaries") }, }, // Migration 002: User prompts table { ID: "002_user_prompts", Migrate: func(tx *gorm.DB) error { return tx.AutoMigrate(&UserPrompt{}) }, Rollback: func(tx *gorm.DB) error { return tx.Migrator().DropTable("user_prompts") }, }, // Migration 003: FTS5 virtual table for user prompts { ID: "003_user_prompts_fts", Migrate: func(tx *gorm.DB) error { sqls := []string{ `CREATE VIRTUAL TABLE IF NOT EXISTS user_prompts_fts USING fts5( prompt_text, content='user_prompts', content_rowid='id' )`, `CREATE TRIGGER IF NOT EXISTS user_prompts_ai AFTER INSERT ON user_prompts BEGIN INSERT INTO user_prompts_fts(rowid, prompt_text) VALUES (new.id, new.prompt_text); END`, `CREATE TRIGGER IF NOT EXISTS user_prompts_ad AFTER DELETE ON user_prompts BEGIN INSERT INTO user_prompts_fts(user_prompts_fts, rowid, prompt_text) VALUES('delete', old.id, old.prompt_text); END`, `CREATE TRIGGER IF NOT EXISTS user_prompts_au AFTER UPDATE ON user_prompts BEGIN INSERT INTO user_prompts_fts(user_prompts_fts, rowid, prompt_text) VALUES('delete', old.id, old.prompt_text); INSERT INTO user_prompts_fts(rowid, prompt_text) VALUES (new.id, new.prompt_text); END`, } for _, s := range sqls { if err := tx.Exec(s).Error; err != nil { return err } } return nil }, Rollback: func(tx *gorm.DB) error { sqls := []string{ "DROP TRIGGER IF EXISTS user_prompts_au", "DROP TRIGGER IF EXISTS user_prompts_ad", "DROP TRIGGER IF EXISTS user_prompts_ai", "DROP TABLE IF EXISTS user_prompts_fts", } for _, s := range sqls { if err := tx.Exec(s).Error; err != nil { return err } } return nil }, }, // Migration 004: FTS5 virtual table for observations { ID: "004_observations_fts", Migrate: func(tx *gorm.DB) error { sqls := []string{ `CREATE VIRTUAL TABLE IF NOT EXISTS observations_fts USING fts5( title, subtitle, narrative, content='observations', content_rowid='id' )`, `CREATE TRIGGER IF NOT EXISTS observations_ai AFTER INSERT ON observations BEGIN INSERT INTO observations_fts(rowid, title, subtitle, narrative) VALUES (new.id, new.title, new.subtitle, new.narrative); END`, `CREATE TRIGGER IF NOT EXISTS observations_ad AFTER DELETE ON observations BEGIN INSERT INTO observations_fts(observations_fts, rowid, title, subtitle, narrative) VALUES('delete', old.id, old.title, old.subtitle, old.narrative); END`, `CREATE TRIGGER IF NOT EXISTS observations_au AFTER UPDATE ON observations BEGIN INSERT INTO observations_fts(observations_fts, rowid, title, subtitle, narrative) VALUES('delete', old.id, old.title, old.subtitle, old.narrative); INSERT INTO observations_fts(rowid, title, subtitle, narrative) VALUES (new.id, new.title, new.subtitle, new.narrative); END`, } for _, s := range sqls { if err := tx.Exec(s).Error; err != nil { return err } } return nil }, Rollback: func(tx *gorm.DB) error { sqls := []string{ "DROP TRIGGER IF EXISTS observations_au", "DROP TRIGGER IF EXISTS observations_ad", "DROP TRIGGER IF EXISTS observations_ai", "DROP TABLE IF EXISTS observations_fts", } for _, s := range sqls { if err := tx.Exec(s).Error; err != nil { return err } } return nil }, }, // Migration 005: FTS5 virtual table for session summaries { ID: "005_session_summaries_fts", Migrate: func(tx *gorm.DB) error { sqls := []string{ `CREATE VIRTUAL TABLE IF NOT EXISTS session_summaries_fts USING fts5( request, investigated, learned, completed, next_steps, notes, content='session_summaries', content_rowid='id' )`, `CREATE TRIGGER IF NOT EXISTS session_summaries_ai AFTER INSERT ON session_summaries BEGIN INSERT INTO session_summaries_fts(rowid, request, investigated, learned, completed, next_steps, notes) VALUES (new.id, new.request, new.investigated, new.learned, new.completed, new.next_steps, new.notes); END`, `CREATE TRIGGER IF NOT EXISTS session_summaries_ad AFTER DELETE ON session_summaries BEGIN INSERT INTO session_summaries_fts(session_summaries_fts, rowid, request, investigated, learned, completed, next_steps, notes) VALUES('delete', old.id, old.request, old.investigated, old.learned, old.completed, old.next_steps, old.notes); END`, `CREATE TRIGGER IF NOT EXISTS session_summaries_au AFTER UPDATE ON session_summaries BEGIN INSERT INTO session_summaries_fts(session_summaries_fts, rowid, request, investigated, learned, completed, next_steps, notes) VALUES('delete', old.id, old.request, old.investigated, old.learned, old.completed, old.next_steps, old.notes); INSERT INTO session_summaries_fts(rowid, request, investigated, learned, completed, next_steps, notes) VALUES (new.id, new.request, new.investigated, new.learned, new.completed, new.next_steps, new.notes); END`, } for _, s := range sqls { if err := tx.Exec(s).Error; err != nil { return err } } return nil }, Rollback: func(tx *gorm.DB) error { sqls := []string{ "DROP TRIGGER IF EXISTS session_summaries_au", "DROP TRIGGER IF EXISTS session_summaries_ad", "DROP TRIGGER IF EXISTS session_summaries_ai", "DROP TABLE IF EXISTS session_summaries_fts", } for _, s := range sqls { if err := tx.Exec(s).Error; err != nil { return err } } return nil }, }, // Migration 006: sqlite-vec vectors table { ID: "006_sqlite_vec_vectors", Migrate: func(tx *gorm.DB) error { // Note: Uses bge-small-en-v1.5 embeddings (384 dimensions) with model_version sql := `CREATE VIRTUAL TABLE IF NOT EXISTS vectors USING vec0( doc_id TEXT PRIMARY KEY, embedding float[384], sqlite_id INTEGER, doc_type TEXT, field_type TEXT, project TEXT, scope TEXT, model_version TEXT )` return tx.Exec(sql).Error }, Rollback: func(tx *gorm.DB) error { return tx.Exec("DROP TABLE IF EXISTS vectors").Error }, }, // Migration 007: Concept weights table with seed data { ID: "007_concept_weights", Migrate: func(tx *gorm.DB) error { if err := tx.AutoMigrate(&ConceptWeight{}); err != nil { return err } // Seed default concept weights now := time.Now().Format(time.RFC3339) weights := []ConceptWeight{ {Concept: "security", Weight: 0.30, UpdatedAt: now}, {Concept: "gotcha", Weight: 0.25, UpdatedAt: now}, {Concept: "best-practice", Weight: 0.20, UpdatedAt: now}, {Concept: "anti-pattern", Weight: 0.20, UpdatedAt: now}, {Concept: "architecture", Weight: 0.15, UpdatedAt: now}, {Concept: "performance", Weight: 0.15, UpdatedAt: now}, {Concept: "error-handling", Weight: 0.15, UpdatedAt: now}, {Concept: "pattern", Weight: 0.10, UpdatedAt: now}, {Concept: "testing", Weight: 0.10, UpdatedAt: now}, {Concept: "debugging", Weight: 0.10, UpdatedAt: now}, {Concept: "workflow", Weight: 0.05, UpdatedAt: now}, {Concept: "tooling", Weight: 0.05, UpdatedAt: now}, } // INSERT OR IGNORE equivalent in GORM return tx.Clauses(clause.OnConflict{DoNothing: true}).Create(&weights).Error }, Rollback: func(tx *gorm.DB) error { return tx.Migrator().DropTable("concept_weights") }, }, // Migration 008: Observation conflicts table { ID: "008_observation_conflicts", Migrate: func(tx *gorm.DB) error { return tx.AutoMigrate(&ObservationConflict{}) }, Rollback: func(tx *gorm.DB) error { return tx.Migrator().DropTable("observation_conflicts") }, }, // Migration 009: Patterns table { ID: "009_patterns", Migrate: func(tx *gorm.DB) error { return tx.AutoMigrate(&Pattern{}) }, Rollback: func(tx *gorm.DB) error { return tx.Migrator().DropTable("patterns") }, }, // Migration 010: FTS5 virtual table for patterns { ID: "010_patterns_fts", Migrate: func(tx *gorm.DB) error { sqls := []string{ `CREATE VIRTUAL TABLE IF NOT EXISTS patterns_fts USING fts5( name, description, recommendation, content='patterns', content_rowid='id' )`, `CREATE TRIGGER IF NOT EXISTS patterns_ai AFTER INSERT ON patterns BEGIN INSERT INTO patterns_fts(rowid, name, description, recommendation) VALUES (new.id, new.name, new.description, new.recommendation); END`, `CREATE TRIGGER IF NOT EXISTS patterns_ad AFTER DELETE ON patterns BEGIN INSERT INTO patterns_fts(patterns_fts, rowid, name, description, recommendation) VALUES('delete', old.id, old.name, old.description, old.recommendation); END`, `CREATE TRIGGER IF NOT EXISTS patterns_au AFTER UPDATE ON patterns BEGIN INSERT INTO patterns_fts(patterns_fts, rowid, name, description, recommendation) VALUES('delete', old.id, old.name, old.description, old.recommendation); INSERT INTO patterns_fts(rowid, name, description, recommendation) VALUES (new.id, new.name, new.description, new.recommendation); END`, } for _, s := range sqls { if err := tx.Exec(s).Error; err != nil { return err } } return nil }, Rollback: func(tx *gorm.DB) error { sqls := []string{ "DROP TRIGGER IF EXISTS patterns_au", "DROP TRIGGER IF EXISTS patterns_ad", "DROP TRIGGER IF EXISTS patterns_ai", "DROP TABLE IF EXISTS patterns_fts", } for _, s := range sqls { if err := tx.Exec(s).Error; err != nil { return err } } return nil }, }, // Migration 011: Observation relations table { ID: "011_observation_relations", Migrate: func(tx *gorm.DB) error { return tx.AutoMigrate(&ObservationRelation{}) }, Rollback: func(tx *gorm.DB) error { return tx.Migrator().DropTable("observation_relations") }, }, // Migration 012: Query optimization indexes // Adds covering and composite indexes for common query patterns { ID: "012_query_optimization_indexes", Migrate: func(tx *gorm.DB) error { sqls := []string{ // Composite index for observation queries by project + scope + importance // Covers the common pattern: WHERE project = ? OR scope = 'global' ORDER BY importance_score DESC `CREATE INDEX IF NOT EXISTS idx_observations_project_scope_importance ON observations(project, scope, importance_score DESC, created_at_epoch DESC)`, // Covering index for observation retrieval (includes most used columns) // Allows index-only scans for listing queries `CREATE INDEX IF NOT EXISTS idx_observations_project_covering ON observations(project, scope, is_superseded, importance_score DESC) WHERE is_superseded = 0 OR is_superseded IS NULL`, // Index for session summary lookups `CREATE INDEX IF NOT EXISTS idx_summaries_project_importance ON session_summaries(project, importance_score DESC, created_at_epoch DESC)`, // Index for prompt retrieval by session `CREATE INDEX IF NOT EXISTS idx_prompts_session_number ON user_prompts(claude_session_id, prompt_number)`, // Index for pattern queries by frequency `CREATE INDEX IF NOT EXISTS idx_patterns_frequency ON patterns(frequency DESC, last_seen_at_epoch DESC) WHERE is_deprecated = 0`, } for _, s := range sqls { if err := tx.Exec(s).Error; err != nil { // Non-fatal: index may already exist or fail for benign reasons continue } } return nil }, Rollback: func(tx *gorm.DB) error { sqls := []string{ "DROP INDEX IF EXISTS idx_observations_project_scope_importance", "DROP INDEX IF EXISTS idx_observations_project_covering", "DROP INDEX IF EXISTS idx_summaries_project_importance", "DROP INDEX IF EXISTS idx_prompts_session_number", "DROP INDEX IF EXISTS idx_patterns_frequency", } for _, s := range sqls { if err := tx.Exec(s).Error; err != nil { continue } } return nil }, }, // Migration 013: Add archival columns to observations { ID: "013_observation_archival", Migrate: func(tx *gorm.DB) error { sqls := []string{ // Add archival columns `ALTER TABLE observations ADD COLUMN is_archived INTEGER DEFAULT 0`, `ALTER TABLE observations ADD COLUMN archived_at_epoch INTEGER`, `ALTER TABLE observations ADD COLUMN archived_reason TEXT`, // Index for archived observations `CREATE INDEX IF NOT EXISTS idx_observations_archived ON observations(is_archived)`, // Composite index for filtering active (non-archived) observations `CREATE INDEX IF NOT EXISTS idx_observations_active ON observations(project, is_archived, is_superseded, importance_score DESC) WHERE (is_archived = 0 OR is_archived IS NULL)`, } for _, s := range sqls { if err := tx.Exec(s).Error; err != nil { // Non-fatal: column may already exist continue } } return nil }, Rollback: func(tx *gorm.DB) error { // SQLite doesn't support DROP COLUMN in older versions // but for newer versions we can try sqls := []string{ "DROP INDEX IF EXISTS idx_observations_active", "DROP INDEX IF EXISTS idx_observations_archived", } for _, s := range sqls { _ = tx.Exec(s).Error } return nil }, }, // Migration 014: Add performance-critical indexes for common query patterns { ID: "014_performance_indexes", Migrate: func(tx *gorm.DB) error { sqls := []string{ // Index for batch ID lookups (IN queries) `CREATE INDEX IF NOT EXISTS idx_observations_id_covering ON observations(id, project, scope, importance_score)`, // Index for vector search result fetching `CREATE INDEX IF NOT EXISTS idx_vectors_doc_type_project ON vectors(doc_type, project, scope)`, // Index for session summaries by project `CREATE INDEX IF NOT EXISTS idx_summaries_project_created ON session_summaries(project, created_at_epoch DESC)`, // Index for user prompts by session `CREATE INDEX IF NOT EXISTS idx_prompts_session_created ON user_prompts(claude_session_id, created_at_epoch DESC)`, // Index for patterns by type and project `CREATE INDEX IF NOT EXISTS idx_patterns_type_project ON patterns(type, project, frequency DESC) WHERE is_deprecated = 0`, // Index for observation relations `CREATE INDEX IF NOT EXISTS idx_relations_source_type ON observation_relations(source_observation_id, relation_type)`, `CREATE INDEX IF NOT EXISTS idx_relations_target_type ON observation_relations(target_observation_id, relation_type)`, } for _, s := range sqls { if err := tx.Exec(s).Error; err != nil { // Non-fatal: index may already exist continue } } return nil }, Rollback: func(tx *gorm.DB) error { sqls := []string{ "DROP INDEX IF EXISTS idx_observations_id_covering", "DROP INDEX IF EXISTS idx_vectors_doc_type_project", "DROP INDEX IF EXISTS idx_summaries_project_created", "DROP INDEX IF EXISTS idx_prompts_session_created", "DROP INDEX IF EXISTS idx_patterns_type_project", "DROP INDEX IF EXISTS idx_relations_source_type", "DROP INDEX IF EXISTS idx_relations_target_type", } for _, s := range sqls { _ = tx.Exec(s).Error } return nil }, }, // Migration 015: Add optimized composite indexes for common query patterns { ID: "015_optimized_composite_indexes", Migrate: func(tx *gorm.DB) error { sqls := []string{ // Composite index for GetRecentObservations with project+scope filtering // Covers: WHERE (project = ? OR scope = 'global') ORDER BY importance_score DESC, created_at_epoch DESC `CREATE INDEX IF NOT EXISTS idx_observations_project_scope_created ON observations(project, scope, created_at_epoch DESC, importance_score DESC)`, // Index for scope='global' queries (common pattern in search) `CREATE INDEX IF NOT EXISTS idx_observations_global_scope ON observations(scope, importance_score DESC, created_at_epoch DESC) WHERE scope = 'global'`, // Index for vector search result deduplication by observation `CREATE INDEX IF NOT EXISTS idx_vectors_observation_lookup ON vectors(doc_type, sqlite_id, project) WHERE doc_type = 'observation'`, // Index for FTS search result ordering `CREATE INDEX IF NOT EXISTS idx_observations_fts_ordering ON observations(project, importance_score DESC) WHERE (is_archived = 0 OR is_archived IS NULL) AND (is_superseded = 0 OR is_superseded IS NULL)`, } for _, s := range sqls { if err := tx.Exec(s).Error; err != nil { // Non-fatal: index may already exist continue } } return nil }, Rollback: func(tx *gorm.DB) error { sqls := []string{ "DROP INDEX IF EXISTS idx_observations_project_scope_created", "DROP INDEX IF EXISTS idx_observations_global_scope", "DROP INDEX IF EXISTS idx_vectors_observation_lookup", "DROP INDEX IF EXISTS idx_observations_fts_ordering", } for _, s := range sqls { _ = tx.Exec(s).Error } return nil }, }, // Migration 016: Add covering indexes for relation joins and active observations { ID: "016_relation_and_active_indexes", Migrate: func(tx *gorm.DB) error { sqls := []string{ // Covering index for observation relation joins (common JOIN patterns) // Speeds up queries like: JOIN observation_relations ON source_id = obs.id WHERE relation_type = ? `CREATE INDEX IF NOT EXISTS idx_relations_source_type_target ON observation_relations(source_observation_id, relation_type, target_observation_id)`, // Covering index for reverse relation lookups `CREATE INDEX IF NOT EXISTS idx_relations_target_type_source ON observation_relations(target_observation_id, relation_type, source_observation_id)`, // Partial index for active (non-archived, non-superseded) observations // Optimizes activeObservationFilter queries `CREATE INDEX IF NOT EXISTS idx_observations_active ON observations(project, importance_score DESC, created_at_epoch DESC) WHERE (is_archived = 0 OR is_archived IS NULL) AND (is_superseded = 0 OR is_superseded IS NULL)`, } for _, s := range sqls { if err := tx.Exec(s).Error; err != nil { // Non-fatal: index may already exist continue } } return nil }, Rollback: func(tx *gorm.DB) error { sqls := []string{ "DROP INDEX IF EXISTS idx_relations_source_type_target", "DROP INDEX IF EXISTS idx_relations_target_type_source", "DROP INDEX IF EXISTS idx_observations_active", } for _, s := range sqls { _ = tx.Exec(s).Error } return nil }, }, }) if err := m.Migrate(); err != nil { return fmt.Errorf("run gormigrate migrations: %w", err) } return nil }