chore(schema): migrate to GORM V2 with multi-database support

- [x] Implement GORM V2 metadata store with SQLite, PostgreSQL, and MySQL support
- [x] Add database migration system using gormigrate for schema versioning
- [x] Create migration CLI tool with support for migrate, rollback, and status commands
- [x] Add Docker support for migration container (Dockerfile.migrate)
- [x] Implement automatic partition management for PostgreSQL time-series tables
- [x] Add background aggregation worker for download statistics
- [x] Support connection pooling configuration (max_open_conns, max_idle_conns, conn_max_lifetime)
- [x] Add blocking mechanism based on vulnerability thresholds in stats and handlers
- [x] Update Helm charts with migration init containers and multi-database configuration
- [x] Replace deprecated SQLite store with optimized GORM implementation
- [x] Add comprehensive integration tests for MySQL and PostgreSQL
- [x] Update frontend to display blocked packages and storage utilization
- [x] Add goreleaser configuration for migrate binary and container image
- [x] Update configuration examples with database backend options and recommendations
This commit is contained in:
2026-01-03 20:44:23 +00:00
parent b129279fb8
commit c0061b99e3
37 changed files with 5711 additions and 1222 deletions
+16 -3
View File
@@ -148,6 +148,17 @@ func (h *Handler) handlePackagePage(ctx context.Context, w http.ResponseWriter,
func (h *Handler) handlePackageFile(ctx context.Context, w http.ResponseWriter, r *http.Request, path string) {
packageName, version := extractPackageFileInfo(path)
// Make version unique by appending file type to avoid cache collisions
// between .whl and .metadata files with same version
cacheVersion := version
if strings.HasSuffix(path, ".metadata") {
cacheVersion = version + ".metadata"
} else if strings.HasSuffix(path, ".whl") {
cacheVersion = version + ".whl"
} else if strings.HasSuffix(path, ".tar.gz") {
cacheVersion = version + ".tar.gz"
}
// Extract credentials from request
credentials := h.credExtractor.Extract(r)
credHash := h.credHasher.Hash(credentials)
@@ -170,12 +181,13 @@ func (h *Handler) handlePackageFile(ctx context.Context, w http.ResponseWriter,
Str("path", path).
Str("package", packageName).
Str("version", version).
Str("cache_version", cacheVersion).
Str("url", originalURL).
Str("cred_hash", credHash).
Bool("has_credentials", credentials != "").
Msg("Handling PyPI package file request")
entry, err := h.cache.Get(ctx, "pypi", packageName, version, func(ctx context.Context) (io.ReadCloser, string, error) {
entry, err := h.cache.Get(ctx, "pypi", packageName, cacheVersion, func(ctx context.Context) (io.ReadCloser, string, error) {
// Prepare headers for upstream request
headers := make(map[string]string)
if credentials != "" {
@@ -281,11 +293,12 @@ func isPackagePage(path string) bool {
// isPackageFile checks if the request is for a package file
func isPackageFile(path string) bool {
// Package files (not including .metadata files which need special handling)
// Package files including .metadata files for PEP 658 support
return strings.HasSuffix(path, ".whl") ||
strings.HasSuffix(path, ".tar.gz") ||
strings.HasSuffix(path, ".zip") ||
strings.HasSuffix(path, ".egg")
strings.HasSuffix(path, ".egg") ||
strings.HasSuffix(path, ".metadata")
}
// extractPackageName extracts package name from path