chore(schema): migrate to GORM V2 with multi-database support

- [x] Implement GORM V2 metadata store with SQLite, PostgreSQL, and MySQL support
- [x] Add database migration system using gormigrate for schema versioning
- [x] Create migration CLI tool with support for migrate, rollback, and status commands
- [x] Add Docker support for migration container (Dockerfile.migrate)
- [x] Implement automatic partition management for PostgreSQL time-series tables
- [x] Add background aggregation worker for download statistics
- [x] Support connection pooling configuration (max_open_conns, max_idle_conns, conn_max_lifetime)
- [x] Add blocking mechanism based on vulnerability thresholds in stats and handlers
- [x] Update Helm charts with migration init containers and multi-database configuration
- [x] Replace deprecated SQLite store with optimized GORM implementation
- [x] Add comprehensive integration tests for MySQL and PostgreSQL
- [x] Update frontend to display blocked packages and storage utilization
- [x] Add goreleaser configuration for migrate binary and container image
- [x] Update configuration examples with database backend options and recommendations
This commit is contained in:
2026-01-03 20:44:23 +00:00
parent b129279fb8
commit c0061b99e3
37 changed files with 5711 additions and 1222 deletions
+27 -6
View File
@@ -142,6 +142,13 @@ func (a *App) handleListPackages(c *fiber.Ctx) error {
severityCounts[strings.ToUpper(vuln.Severity)]++
}
// Check if package should be blocked based on thresholds
isBlocked := false
if a.scanManager != nil {
blocked, _, _ := a.scanManager.CheckVulnerabilities(ctx, pkg.Registry, entry.originalName, pkg.Version)
isBlocked = blocked
}
pkgMap["vulnerabilities"] = map[string]interface{}{
"scanned": true,
"status": scanResult.Status,
@@ -152,18 +159,21 @@ func (a *App) handleListPackages(c *fiber.Ctx) error {
"moderate": severityCounts["MODERATE"],
"low": severityCounts["LOW"],
},
"total": scanResult.VulnerabilityCount,
"total": scanResult.VulnerabilityCount,
"isBlocked": isBlocked,
}
} else {
pkgMap["vulnerabilities"] = map[string]interface{}{
"scanned": false,
"status": "pending",
"scanned": false,
"status": "pending",
"isBlocked": false,
}
}
} else {
pkgMap["vulnerabilities"] = map[string]interface{}{
"scanned": false,
"status": "not_scanned",
"scanned": false,
"status": "not_scanned",
"isBlocked": false,
}
}
@@ -351,8 +361,9 @@ func (a *App) handleStats(c *fiber.Ctx) error {
packages = []*metadata.Package{}
}
// Calculate per-registry breakdown (exclude metadata entries like "list", "latest")
// Calculate per-registry breakdown and blocked packages count
registryStats := make(map[string]map[string]interface{})
blockedCount := int64(0)
for _, pkg := range packages {
// Skip metadata entries (npm metadata pages, pypi pages, etc.)
@@ -371,6 +382,14 @@ func (a *App) handleStats(c *fiber.Ctx) error {
registryStats[pkg.Registry]["count"] = registryStats[pkg.Registry]["count"].(int) + 1
registryStats[pkg.Registry]["size"] = registryStats[pkg.Registry]["size"].(int64) + pkg.Size
registryStats[pkg.Registry]["downloads"] = registryStats[pkg.Registry]["downloads"].(int64) + int64(pkg.DownloadCount)
// Check if package is blocked (only if security scanning is enabled and package is scanned)
if a.config.Security.Enabled && a.scanManager != nil && pkg.SecurityScanned {
blocked, _, _ := a.scanManager.CheckVulnerabilities(ctx, pkg.Registry, pkg.Name, pkg.Version)
if blocked {
blockedCount++
}
}
}
// Combine statistics using database stats for accuracy
@@ -378,12 +397,14 @@ func (a *App) handleStats(c *fiber.Ctx) error {
"total_packages": cacheStats.TotalPackages,
"total_downloads": cacheStats.TotalDownloads,
"total_size": cacheStats.TotalSize,
"max_cache_size": a.config.Cache.MaxSizeBytes,
"cache_hits": cacheStats.TotalDownloads,
"cache_misses": 0, // TODO: Track cache misses
"cache_evictions": 0, // TODO: Track evictions
"cache_size": cacheStats.TotalSize,
"scanned_packages": cacheStats.ScannedPackages,
"vulnerable_packages": cacheStats.VulnerablePackages,
"blocked_packages": blockedCount,
}
// Convert registry stats to interface map