From c0061b99e3a084459f447cbd580ddbde801a6f8b Mon Sep 17 00:00:00 2001 From: Lukasz Raczylo Date: Sat, 3 Jan 2026 20:44:23 +0000 Subject: [PATCH] chore(schema): migrate to GORM V2 with multi-database support - [x] Implement GORM V2 metadata store with SQLite, PostgreSQL, and MySQL support - [x] Add database migration system using gormigrate for schema versioning - [x] Create migration CLI tool with support for migrate, rollback, and status commands - [x] Add Docker support for migration container (Dockerfile.migrate) - [x] Implement automatic partition management for PostgreSQL time-series tables - [x] Add background aggregation worker for download statistics - [x] Support connection pooling configuration (max_open_conns, max_idle_conns, conn_max_lifetime) - [x] Add blocking mechanism based on vulnerability thresholds in stats and handlers - [x] Update Helm charts with migration init containers and multi-database configuration - [x] Replace deprecated SQLite store with optimized GORM implementation - [x] Add comprehensive integration tests for MySQL and PostgreSQL - [x] Update frontend to display blocked packages and storage utilization - [x] Add goreleaser configuration for migrate binary and container image - [x] Update configuration examples with database backend options and recommendations --- .goreleaser.yaml | 40 + Dockerfile.migrate | 30 + Makefile | 19 +- cmd/migrate/main.go | 258 ++++ config.yaml.example | 41 +- frontend/src/components/PackageList.vue | 1 + frontend/src/components/Stats.vue | 63 +- .../src/components/VulnerabilityBadge.vue | 12 + .../src/components/VulnerablePackages.vue | 26 +- frontend/src/router/index.ts | 5 + frontend/src/stores/packages.ts | 1 + go.mod | 72 +- go.sum | 219 +++- helm/gohoarder/templates/_helpers.tpl | 2 +- .../templates/deployment-scanner.yaml | 117 ++ .../templates/deployment-server.yaml | 94 ++ helm/gohoarder/templates/secret.yaml | 13 + helm/gohoarder/values.yaml | 60 +- migrate | Bin 0 -> 22530882 bytes migrations/001_create_schema_v2_mysql.sql | 367 ++++++ migrations/001_create_schema_v2_postgres.sql | 470 +++++++ pkg/app/app.go | 79 +- pkg/app/handlers.go | 33 +- pkg/cache/cache.go | 21 +- pkg/config/config.go | 33 +- pkg/metadata/gormstore/aggregation_worker.go | 357 ++++++ pkg/metadata/gormstore/config.go | 78 ++ pkg/metadata/gormstore/gormstore_v2.go | 1000 +++++++++++++++ .../gormstore/gormstore_v2_mysql_test.go | 279 +++++ .../gormstore/gormstore_v2_postgres_test.go | 202 +++ pkg/metadata/gormstore/gormstore_v2_test.go | 871 +++++++++++++ pkg/metadata/gormstore/migrations.go | 228 ++++ pkg/metadata/gormstore/models_v2.go | 328 +++++ pkg/metadata/gormstore/partition_manager.go | 380 ++++++ pkg/metadata/interface.go | 19 +- pkg/metadata/sqlite/sqlite.go | 1096 ----------------- pkg/proxy/pypi/pypi.go | 19 +- 37 files changed, 5711 insertions(+), 1222 deletions(-) create mode 100644 Dockerfile.migrate create mode 100644 cmd/migrate/main.go create mode 100755 migrate create mode 100644 migrations/001_create_schema_v2_mysql.sql create mode 100644 migrations/001_create_schema_v2_postgres.sql create mode 100644 pkg/metadata/gormstore/aggregation_worker.go create mode 100644 pkg/metadata/gormstore/config.go create mode 100644 pkg/metadata/gormstore/gormstore_v2.go create mode 100644 pkg/metadata/gormstore/gormstore_v2_mysql_test.go create mode 100644 pkg/metadata/gormstore/gormstore_v2_postgres_test.go create mode 100644 pkg/metadata/gormstore/gormstore_v2_test.go create mode 100644 pkg/metadata/gormstore/migrations.go create mode 100644 pkg/metadata/gormstore/models_v2.go create mode 100644 pkg/metadata/gormstore/partition_manager.go delete mode 100644 pkg/metadata/sqlite/sqlite.go diff --git a/.goreleaser.yaml b/.goreleaser.yaml index 99a8d22..e8a7332 100644 --- a/.goreleaser.yaml +++ b/.goreleaser.yaml @@ -32,6 +32,24 @@ builds: - -X github.com/lukaszraczylo/gohoarder/internal/version.GitCommit={{.ShortCommit}} - -X github.com/lukaszraczylo/gohoarder/internal/version.BuildTime={{.Date}} + - id: migrate + main: ./cmd/migrate + binary: migrate + env: + - CGO_ENABLED=0 + goos: + - linux + - darwin + - windows + goarch: + - amd64 + - arm64 + ldflags: + - -s -w + - -X main.Version={{.Version}} + - -X main.GitCommit={{.ShortCommit}} + - -X main.BuildTime={{.Date}} + # Archives for releases archives: - id: default @@ -182,6 +200,28 @@ dockers_v2: org.opencontainers.image.created: "{{ .Date }}" org.opencontainers.image.revision: "{{ .FullCommit }}" + # 5. Migration Engine - Database migration tool + - id: gohoarder-migrate + ids: + - migrate + images: + - ghcr.io/lukaszraczylo/gohoarder-migrate + tags: + - "{{ .Version }}" + - latest + platforms: + - linux/amd64 + - linux/arm64 + dockerfile: Dockerfile.migrate + labels: + org.opencontainers.image.title: GoHoarder Migrate + org.opencontainers.image.description: Database migration tool for GoHoarder V2 schema + org.opencontainers.image.url: https://github.com/lukaszraczylo/gohoarder + org.opencontainers.image.source: https://github.com/lukaszraczylo/gohoarder + org.opencontainers.image.version: "{{ .Version }}" + org.opencontainers.image.created: "{{ .Date }}" + org.opencontainers.image.revision: "{{ .FullCommit }}" + # Artifact signing with cosign signs: - cmd: cosign diff --git a/Dockerfile.migrate b/Dockerfile.migrate new file mode 100644 index 0000000..b5a394e --- /dev/null +++ b/Dockerfile.migrate @@ -0,0 +1,30 @@ +# Migration Engine - Database Migration Tool +ARG TARGETOS +ARG TARGETARCH + +FROM alpine:latest + +# Install runtime dependencies +RUN apk add --no-cache \ + ca-certificates \ + postgresql-client \ + mysql-client \ + busybox-extras \ + && update-ca-certificates + +# Create non-root user +RUN addgroup -g 1000 gohoarder && \ + adduser -D -u 1000 -G gohoarder gohoarder + +# Copy binary (from platform-specific path) +ARG TARGETOS +ARG TARGETARCH +COPY ${TARGETOS}/${TARGETARCH}/migrate /usr/local/bin/migrate +RUN chmod +x /usr/local/bin/migrate + +WORKDIR /app +USER gohoarder + +# Run migrations +ENTRYPOINT ["/usr/local/bin/migrate"] +CMD ["--action=migrate"] diff --git a/Makefile b/Makefile index 19c9b16..88e3587 100644 --- a/Makefile +++ b/Makefile @@ -78,17 +78,24 @@ clean: ## Clean build artifacts @rm -f *.db *.db-shm *.db-wal @echo "Clean complete" -clean-db: ## Clean all local cache and database files (from config.yaml paths) +clean-db: ## Clean all local cache and database files (requires confirmation) @echo "WARNING: This will delete all cached packages and scan results!" - @echo "Paths from config.yaml:" + @echo "Paths to be cleaned:" @echo " - ./data/storage (package cache)" - @echo " - ./data/gohoarder.db (metadata database)" + @echo " - ./data/gohoarder.db and gohoarder.db (metadata database)" @echo " - /tmp/trivy (Trivy cache)" @echo "" - @read -p "Are you sure you want to continue? [y/N] " confirm && [ "$$confirm" = "y" ] || exit 1 + @printf "Are you sure you want to continue? [y/N] " && read confirm && [ "$$confirm" = "y" ] || (echo "Cancelled." && exit 1) @echo "Cleaning database and cache..." - @rm -rf ./data/storage - @rm -f ./data/gohoarder.db ./data/gohoarder.db-shm ./data/gohoarder.db-wal + @rm -rf ./data/storage ./data + @rm -f gohoarder.db gohoarder.db-shm gohoarder.db-wal + @rm -rf /tmp/trivy + @echo "Database and cache cleaned successfully" + +clean-db-force: ## Clean all local cache and database files (no confirmation) + @echo "Cleaning database and cache..." + @rm -rf ./data/storage ./data + @rm -f gohoarder.db gohoarder.db-shm gohoarder.db-wal @rm -rf /tmp/trivy @echo "Database and cache cleaned successfully" diff --git a/cmd/migrate/main.go b/cmd/migrate/main.go new file mode 100644 index 0000000..f45ea7b --- /dev/null +++ b/cmd/migrate/main.go @@ -0,0 +1,258 @@ +package main + +import ( + "context" + "database/sql" + "flag" + "fmt" + stdlog "log" + "os" + "time" + + "github.com/go-gormigrate/gormigrate/v2" + _ "github.com/go-sql-driver/mysql" + _ "github.com/lib/pq" + "github.com/lukaszraczylo/gohoarder/pkg/metadata/gormstore" + "github.com/rs/zerolog" + "github.com/rs/zerolog/log" + "gorm.io/driver/mysql" + "gorm.io/driver/postgres" + "gorm.io/driver/sqlite" + "gorm.io/gorm" + "gorm.io/gorm/logger" +) + +type MigrationConfig struct { + Driver string + DSN string + Timeout time.Duration + Action string // migrate, rollback, rollback-to, list + TargetID string // For rollback-to + LogLevel string +} + +func main() { + cfg := MigrationConfig{} + + flag.StringVar(&cfg.Driver, "driver", os.Getenv("DB_DRIVER"), "Database driver (postgres, mysql, sqlite)") + flag.StringVar(&cfg.DSN, "dsn", os.Getenv("DATABASE_URL"), "Database connection string") + flag.DurationVar(&cfg.Timeout, "timeout", 10*time.Minute, "Migration timeout") + flag.StringVar(&cfg.Action, "action", "migrate", "Action: migrate, rollback, rollback-to, list") + flag.StringVar(&cfg.TargetID, "target", "", "Target migration ID (for rollback-to)") + flag.StringVar(&cfg.LogLevel, "log-level", "info", "Log level: debug, info, warn, error") + flag.Parse() + + // Setup logging + setupLogging(cfg.LogLevel) + + log.Info(). + Str("driver", cfg.Driver). + Str("action", cfg.Action). + Msg("Starting database migration") + + if err := RunMigration(cfg); err != nil { + log.Fatal().Err(err).Msg("Migration failed") + } + + log.Info().Msg("Migration completed successfully") +} + +func setupLogging(level string) { + zerolog.TimeFieldFormat = zerolog.TimeFormatUnix + log.Logger = log.Output(zerolog.ConsoleWriter{Out: os.Stdout, TimeFormat: time.RFC3339}) + + switch level { + case "debug": + zerolog.SetGlobalLevel(zerolog.DebugLevel) + case "info": + zerolog.SetGlobalLevel(zerolog.InfoLevel) + case "warn": + zerolog.SetGlobalLevel(zerolog.WarnLevel) + case "error": + zerolog.SetGlobalLevel(zerolog.ErrorLevel) + default: + zerolog.SetGlobalLevel(zerolog.InfoLevel) + } +} + +func RunMigration(cfg MigrationConfig) error { + // Validate config + if cfg.Driver == "" { + return fmt.Errorf("driver is required (set DB_DRIVER or --driver)") + } + if cfg.DSN == "" { + return fmt.Errorf("DSN is required (set DATABASE_URL or --dsn)") + } + + ctx, cancel := context.WithTimeout(context.Background(), cfg.Timeout) + defer cancel() + + // Connect to database + db, err := connectToDatabase(cfg.Driver, cfg.DSN) + if err != nil { + return fmt.Errorf("failed to connect to database: %w", err) + } + + sqlDB, err := db.DB() + if err != nil { + return fmt.Errorf("failed to get sql.DB: %w", err) + } + defer sqlDB.Close() + + // Wait for database to be ready + if err := waitForDB(ctx, sqlDB, 60*time.Second); err != nil { + return fmt.Errorf("database not ready: %w", err) + } + + // Initialize gormigrate with custom options + opts := gormigrate.DefaultOptions + opts.TableName = "gohoarder_migrations" + m := gormigrate.New(db, opts, gormstore.GetMigrations()) + + log.Info(). + Str("table", "gohoarder_migrations"). + Msg("Migration tracking table initialized") + + // Execute action + switch cfg.Action { + case "migrate": + return runMigrate(m) + case "rollback": + return runRollback(m) + case "rollback-to": + if cfg.TargetID == "" { + return fmt.Errorf("target migration ID required for rollback-to") + } + return runRollbackTo(m, cfg.TargetID) + case "list": + return listMigrations(db) + default: + return fmt.Errorf("unknown action: %s (use: migrate, rollback, rollback-to, list)", cfg.Action) + } +} + +func connectToDatabase(driver, dsn string) (*gorm.DB, error) { + // Configure GORM logger using standard library log + gormLogger := logger.New( + stdlog.New(os.Stdout, "\r\n", stdlog.LstdFlags), + logger.Config{ + SlowThreshold: 200 * time.Millisecond, + LogLevel: logger.Info, + IgnoreRecordNotFoundError: true, + Colorful: true, + }, + ) + + var dialector gorm.Dialector + switch driver { + case "sqlite": + dialector = sqlite.Open(dsn) + case "postgres", "postgresql": + dialector = postgres.Open(dsn) + case "mysql": + dialector = mysql.Open(dsn) + default: + return nil, fmt.Errorf("unsupported driver: %s", driver) + } + + db, err := gorm.Open(dialector, &gorm.Config{ + Logger: gormLogger, + SkipDefaultTransaction: false, // Migrations should be transactional + PrepareStmt: true, + }) + + if err != nil { + return nil, err + } + + return db, nil +} + +func waitForDB(ctx context.Context, db *sql.DB, timeout time.Duration) error { + deadline := time.Now().Add(timeout) + attempt := 0 + + for { + attempt++ + if time.Now().After(deadline) { + return fmt.Errorf("timeout waiting for database after %d attempts", attempt) + } + + if err := db.PingContext(ctx); err == nil { + log.Info(). + Int("attempts", attempt). + Msg("Database is ready") + return nil + } + + log.Debug(). + Int("attempt", attempt). + Msg("Waiting for database...") + time.Sleep(2 * time.Second) + } +} + +func runMigrate(m *gormigrate.Gormigrate) error { + log.Info().Msg("Running migrations...") + + if err := m.Migrate(); err != nil { + return fmt.Errorf("migration failed: %w", err) + } + + log.Info().Msg("✓ All migrations applied successfully") + return nil +} + +func runRollback(m *gormigrate.Gormigrate) error { + log.Warn().Msg("Rolling back last migration...") + + if err := m.RollbackLast(); err != nil { + return fmt.Errorf("rollback failed: %w", err) + } + + log.Info().Msg("✓ Rollback completed") + return nil +} + +func runRollbackTo(m *gormigrate.Gormigrate, targetID string) error { + log.Warn(). + Str("target_id", targetID). + Msg("Rolling back to migration...") + + if err := m.RollbackTo(targetID); err != nil { + return fmt.Errorf("rollback to %s failed: %w", targetID, err) + } + + log.Info(). + Str("target_id", targetID). + Msg("✓ Rollback completed") + return nil +} + +func listMigrations(db *gorm.DB) error { + log.Info().Msg("Applied migrations:") + + type Migration struct { + ID string + } + + var migrations []Migration + if err := db.Table("gohoarder_migrations").Find(&migrations).Error; err != nil { + return fmt.Errorf("failed to list migrations: %w", err) + } + + if len(migrations) == 0 { + log.Info().Msg(" (no migrations applied yet)") + return nil + } + + for _, m := range migrations { + log.Info().Str("id", m.ID).Msg(" ✓") + } + + log.Info(). + Int("total", len(migrations)). + Msg("Applied migrations") + + return nil +} diff --git a/config.yaml.example b/config.yaml.example index 49a5f75..fd5f4bb 100644 --- a/config.yaml.example +++ b/config.yaml.example @@ -40,20 +40,53 @@ storage: domain: "" metadata: - backend: "sqlite" # sqlite, postgresql, file - connection: "file:gohoarder.db?cache=shared&mode=rwc" + # Backend: sqlite, postgresql, mysql, mariadb, file + # + # Choose based on your deployment: + # - sqlite: Single instance, local storage (NOT for network filesystems like SMB/NFS!) + # - postgresql: Production, multiple replicas, works with any storage including SMB/NFS + # - mysql: Production alternative to PostgreSQL + # - file: Simple file-based metadata (limited features) + # + # IMPORTANT: SQLite + SMB/NFS = Database locked errors! + # For network storage (SMB, NFS), use PostgreSQL or MySQL. + backend: "sqlite" + connection: "file:gohoarder.db?cache=shared&mode=rwc" # Legacy, not used with GORM + # SQLite configuration (for local storage only) + # Use with local storage classes (local-path, hostPath, or RWX like longhorn) + # DO NOT use with SMB/NFS network storage! sqlite: path: "gohoarder.db" - wal_mode: true + wal_mode: true # Set to false for network filesystems if you must use SQLite + # PostgreSQL configuration (recommended for production) + # Works with any storage including SMB/NFS + # Supports multiple replicas and high availability postgresql: host: "localhost" port: 5432 database: "gohoarder" user: "gohoarder" password: "" - ssl_mode: "disable" + ssl_mode: "disable" # disable, require, verify-ca, verify-full + + # MySQL/MariaDB configuration (alternative to PostgreSQL) + # Works with any storage including SMB/NFS + mysql: + host: "localhost" + port: 3306 + database: "gohoarder" + user: "gohoarder" + password: "" + charset: "utf8mb4" + parse_time: true + + # GORM connection pool settings (applies to all database backends) + max_open_conns: 25 # Maximum number of open connections to the database + max_idle_conns: 5 # Maximum number of idle connections in the pool + conn_max_lifetime: 3600 # Maximum lifetime of a connection in seconds (1 hour) + log_level: "warn" # GORM log level: silent, error, warn, info cache: default_ttl: "168h" # 7 days diff --git a/frontend/src/components/PackageList.vue b/frontend/src/components/PackageList.vue index 6c49a9a..6593cde 100644 --- a/frontend/src/components/PackageList.vue +++ b/frontend/src/components/PackageList.vue @@ -128,6 +128,7 @@ :counts="version.vulnerabilities.counts" :total="version.vulnerabilities.total" :scannedAt="version.vulnerabilities.scannedAt" + :isBlocked="version.vulnerabilities.isBlocked" @click="showVulnerabilityDetails(group.registry, group.name, version.version)" /> diff --git a/frontend/src/components/Stats.vue b/frontend/src/components/Stats.vue index 5dc5723..d60acf8 100644 --- a/frontend/src/components/Stats.vue +++ b/frontend/src/components/Stats.vue @@ -29,11 +29,26 @@

Total Packages

-
-

- {{ formatBytes(stats?.total_size || 0) }} -

-

Total Storage Used

+
+
+

+ {{ formatBytes(stats?.total_size || 0) }} / {{ formatBytes(stats?.max_cache_size || 0) }} +

+

Storage Used

+
+
+
+
+

{{ storagePercentage.toFixed(1) }}% used

@@ -51,7 +66,7 @@

Security Scanning

-
+

@@ -63,11 +78,11 @@

-

+

{{ formatNumber(stats?.vulnerable_packages || 0) }}

@@ -75,7 +90,23 @@ (click to view)

- + +
+
+
+

+ {{ formatNumber(stats?.blocked_packages || 0) }} +

+

+ Blocked Packages + (click to view) +

+
+
@@ -141,6 +172,14 @@ function showVulnerablePackages() { router.push('/vulnerable-packages') } +function showBlockedPackages() { + if ((stats.value?.blocked_packages || 0) === 0) { + return + } + + router.push('/blocked-packages') +} + // Registry configuration for icons and colors const registryConfig: Record = { npm: { @@ -180,6 +219,12 @@ const registries = computed(() => { }) }) +const storagePercentage = computed(() => { + const totalSize = stats.value?.total_size || 0 + const maxSize = stats.value?.max_cache_size || 1 + return (totalSize / maxSize) * 100 +}) + function formatNumber(num: number): string { return new Intl.NumberFormat().format(num) } diff --git a/frontend/src/components/VulnerabilityBadge.vue b/frontend/src/components/VulnerabilityBadge.vue index 085d9c8..05201a5 100644 --- a/frontend/src/components/VulnerabilityBadge.vue +++ b/frontend/src/components/VulnerabilityBadge.vue @@ -1,5 +1,15 @@