fix: bound SQLite WAL growth and prevent worker hangs (#49)

The worker's SQLite WAL could grow unbounded (observed 19MB) and wedge the DB, hanging Claude Code on every prompt. No checkpoint ever truncated the WAL (only PASSIVE auto-checkpoint, which cannot reclaim the file), the connection-scoped pragmas were set via a single Exec so only one pooled connection received them (e.g. busy_timeout=0 on the rest), and the maintenance service that would optimize/checkpoint was never wired up. - Register a sqlite3 ConnectHook driver so all pragmas (busy_timeout, journal_mode, synchronous, cache_size, foreign_keys, journal_size_limit) apply to every pooled connection; enable safe connection recycling. - Add Store.Checkpoint (TRUNCATE), checkpoint-on-Close, and a periodic size-gated checkpoint loop with configurable interval/threshold. - Wire up the previously-dead maintenance service; make trigger_maintenance actually run DB maintenance instead of only recalculating scores. - Harden the user-prompt hook to honor its deadline and fail open so a slow worker can never stall a prompt. - Add regression tests for WAL truncation, checkpoint-on-close, and per-connection pragmas.
2026-10-12 04:55:18 +00:00 · 2026-06-01 16:40:27 +01:00
parent f78370a531
commit b7b82ce22f
10 changed files with 957 additions and 93 deletions
@@ -39,44 +39,46 @@ var CriticalConcepts = []string{
 // Config holds the application configuration.
 // Field order optimized for memory alignment (fieldalignment).
 type Config struct {
-	ContextFullField          string   `json:"context_full_field"`
-	DBPath                    string   `json:"db_path"`
-	Model                     string   `json:"model"`
-	ClaudeCodePath            string   `json:"claude_code_path"`
-	EmbeddingModel            string   `json:"embedding_model"`
-	VectorStorageStrategy     string   `json:"vector_storage_strategy"`
-	ContextObsConcepts        []string `json:"context_obs_concepts"`
-	ContextObsTypes           []string `json:"context_obs_types"`
-	ContextFullCount          int      `json:"context_full_count"`
-	GraphBranchFactor         int      `json:"graph_branch_factor"`
-	GraphEdgeWeight           float64  `json:"graph_edge_weight"`
-	ContextRelevanceThreshold float64  `json:"context_relevance_threshold"`
-	RerankingCandidates       int      `json:"reranking_candidates"`
-	WorkerPort                int      `json:"worker_port"`
-	DeduplicationThreshold    float64  `json:"deduplication_threshold"`
-	RerankingMinImprovement   float64  `json:"reranking_min_improvement"`
-	ContextObservations       int      `json:"context_observations"`
-	ContextMaxPromptResults   int      `json:"context_max_prompt_results"`
-	ContextSessionCount       int      `json:"context_session_count"`
-	MaxConns                  int      `json:"max_conns"`
-	RerankingAlpha            float64  `json:"reranking_alpha"`
-	GraphMaxHops              int      `json:"graph_max_hops"`
-	RerankingResults          int      `json:"reranking_results"`
-	GraphRebuildIntervalMin   int      `json:"graph_rebuild_interval_min"`
-	HubThreshold              int      `json:"hub_threshold"`
-	ObservationRetentionDays  int      `json:"observation_retention_days"`
-	MaintenanceIntervalHours  int      `json:"maintenance_interval_hours"`
-	ContextMaxTokensStartup   int      `json:"context_max_tokens_startup"`
-	ContextMaxTokensPrompt    int      `json:"context_max_tokens_prompt"`
-	ContextShowWorkTokens     bool     `json:"context_show_work_tokens"`
-	ContextShowReadTokens     bool     `json:"context_show_read_tokens"`
-	RerankingPureMode         bool     `json:"reranking_pure_mode"`
-	GraphEnabled              bool     `json:"graph_enabled"`
-	DeduplicationEnabled      bool     `json:"deduplication_enabled"`
-	MaintenanceEnabled        bool     `json:"maintenance_enabled"`
-	RerankingEnabled          bool     `json:"reranking_enabled"`
-	ContextShowLastSummary    bool     `json:"context_show_last_summary"`
-	CleanupStaleObservations  bool     `json:"cleanup_stale_observations"`
+	ContextFullField             string   `json:"context_full_field"`
+	DBPath                       string   `json:"db_path"`
+	Model                        string   `json:"model"`
+	ClaudeCodePath               string   `json:"claude_code_path"`
+	EmbeddingModel               string   `json:"embedding_model"`
+	VectorStorageStrategy        string   `json:"vector_storage_strategy"`
+	ContextObsConcepts           []string `json:"context_obs_concepts"`
+	ContextObsTypes              []string `json:"context_obs_types"`
+	ContextFullCount             int      `json:"context_full_count"`
+	GraphBranchFactor            int      `json:"graph_branch_factor"`
+	GraphEdgeWeight              float64  `json:"graph_edge_weight"`
+	ContextRelevanceThreshold    float64  `json:"context_relevance_threshold"`
+	RerankingCandidates          int      `json:"reranking_candidates"`
+	WorkerPort                   int      `json:"worker_port"`
+	DeduplicationThreshold       float64  `json:"deduplication_threshold"`
+	RerankingMinImprovement      float64  `json:"reranking_min_improvement"`
+	ContextObservations          int      `json:"context_observations"`
+	ContextMaxPromptResults      int      `json:"context_max_prompt_results"`
+	ContextSessionCount          int      `json:"context_session_count"`
+	MaxConns                     int      `json:"max_conns"`
+	RerankingAlpha               float64  `json:"reranking_alpha"`
+	GraphMaxHops                 int      `json:"graph_max_hops"`
+	RerankingResults             int      `json:"reranking_results"`
+	GraphRebuildIntervalMin      int      `json:"graph_rebuild_interval_min"`
+	HubThreshold                 int      `json:"hub_threshold"`
+	ObservationRetentionDays     int      `json:"observation_retention_days"`
+	MaintenanceIntervalHours     int      `json:"maintenance_interval_hours"`
+	WALCheckpointIntervalSeconds int      `json:"wal_checkpoint_interval_seconds"`
+	WALCheckpointThresholdBytes  int64    `json:"wal_checkpoint_threshold_bytes"`
+	ContextMaxTokensStartup      int      `json:"context_max_tokens_startup"`
+	ContextMaxTokensPrompt       int      `json:"context_max_tokens_prompt"`
+	ContextShowWorkTokens        bool     `json:"context_show_work_tokens"`
+	ContextShowReadTokens        bool     `json:"context_show_read_tokens"`
+	RerankingPureMode            bool     `json:"reranking_pure_mode"`
+	GraphEnabled                 bool     `json:"graph_enabled"`
+	DeduplicationEnabled         bool     `json:"deduplication_enabled"`
+	MaintenanceEnabled           bool     `json:"maintenance_enabled"`
+	RerankingEnabled             bool     `json:"reranking_enabled"`
+	ContextShowLastSummary       bool     `json:"context_show_last_summary"`
+	CleanupStaleObservations     bool     `json:"cleanup_stale_observations"`
 }

 var (
@@ -181,6 +183,10 @@ func Default() *Config {
 		MaintenanceIntervalHours:  6,     // Run every 6 hours
 		ObservationRetentionDays:  0,     // 0 = no age-based deletion (keep all)
 		CleanupStaleObservations:  false, // Don't auto-cleanup stale observations
+		// WAL checkpoint loop tunables (issue #49). Defaults mirror the worker constants:
+		// check the WAL every 60s and TRUNCATE-checkpoint once it reaches 4 MiB.
+		WALCheckpointIntervalSeconds: 60,
+		WALCheckpointThresholdBytes:  4 << 20, // 4 MiB
 	}
 }

@@ -284,6 +290,13 @@ func Load() (*Config, error) {
 	if v, ok := settings["CLAUDE_MNEMONIC_CONTEXT_MAX_TOKENS_PROMPT"].(float64); ok && v > 0 {
 		cfg.ContextMaxTokensPrompt = int(v)
 	}
+	// WAL checkpoint loop tunables (issue #49)
+	if v, ok := settings["CLAUDE_MNEMONIC_WAL_CHECKPOINT_INTERVAL_SECONDS"].(float64); ok && v > 0 {
+		cfg.WALCheckpointIntervalSeconds = int(v)
+	}
+	if v, ok := settings["CLAUDE_MNEMONIC_WAL_CHECKPOINT_THRESHOLD_BYTES"].(float64); ok && v > 0 {
+		cfg.WALCheckpointThresholdBytes = int64(v)
+	}
 	// Deduplication settings
 	if v, ok := settings["CLAUDE_MNEMONIC_DEDUP_ENABLED"].(bool); ok {
 		cfg.DeduplicationEnabled = v