lukaszraczylo/gohoarder
1
0
Fork 0
mirror of https://github.com/lukaszraczylo/gohoarder.git synced 2026-06-05 22:53:53 +00:00
Code Issues Packages Projects Releases Wiki Activity

chore(schema): migrate to GORM V2 with multi-database support

Browse Source 
- [x] Implement GORM V2 metadata store with SQLite, PostgreSQL, and MySQL support
- [x] Add database migration system using gormigrate for schema versioning
- [x] Create migration CLI tool with support for migrate, rollback, and status commands
- [x] Add Docker support for migration container (Dockerfile.migrate)
- [x] Implement automatic partition management for PostgreSQL time-series tables
- [x] Add background aggregation worker for download statistics
- [x] Support connection pooling configuration (max_open_conns, max_idle_conns, conn_max_lifetime)
- [x] Add blocking mechanism based on vulnerability thresholds in stats and handlers
- [x] Update Helm charts with migration init containers and multi-database configuration
- [x] Replace deprecated SQLite store with optimized GORM implementation
- [x] Add comprehensive integration tests for MySQL and PostgreSQL
- [x] Update frontend to display blocked packages and storage utilization
- [x] Add goreleaser configuration for migrate binary and container image
- [x] Update configuration examples with database backend options and recommendations
This commit is contained in:
Lukasz Raczylo
2026-01-03 20:44:23 +00:00
parent b129279fb8
commit c0061b99e3
37 changed files with 5711 additions and 1222 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.goreleaser.yaml
+40
View File
@@ -32,6 +32,24 @@ builds:
- -X github.com/lukaszraczylo/gohoarder/internal/version.GitCommit={{.ShortCommit}}
- -X github.com/lukaszraczylo/gohoarder/internal/version.BuildTime={{.Date}}
- id: migrate
main: ./cmd/migrate
binary: migrate
env:
- CGO_ENABLED=0
goos:
- linux
- darwin
- windows
goarch:
- amd64
- arm64
ldflags:
- -s -w
- -X main.Version={{.Version}}
- -X main.GitCommit={{.ShortCommit}}
- -X main.BuildTime={{.Date}}
# Archives for releases
archives:
- id: default
@@ -182,6 +200,28 @@ dockers_v2:
org.opencontainers.image.created: "{{ .Date }}"
org.opencontainers.image.revision: "{{ .FullCommit }}"
# 5. Migration Engine - Database migration tool
- id: gohoarder-migrate
ids:
- migrate
images:
- ghcr.io/lukaszraczylo/gohoarder-migrate
tags:
- "{{ .Version }}"
- latest
platforms:
- linux/amd64
- linux/arm64
dockerfile: Dockerfile.migrate
labels:
org.opencontainers.image.title: GoHoarder Migrate
org.opencontainers.image.description: Database migration tool for GoHoarder V2 schema
org.opencontainers.image.url: https://github.com/lukaszraczylo/gohoarder
org.opencontainers.image.source: https://github.com/lukaszraczylo/gohoarder
org.opencontainers.image.version: "{{ .Version }}"
org.opencontainers.image.created: "{{ .Date }}"
org.opencontainers.image.revision: "{{ .FullCommit }}"
# Artifact signing with cosign
signs:
- cmd: cosign
Dockerfile.migrate
+30
View File
@@ -0,0 +1,30 @@
# Migration Engine - Database Migration Tool
ARG TARGETOS
ARG TARGETARCH
FROM alpine:latest
# Install runtime dependencies
RUN apk add --no-cache \
ca-certificates \
postgresql-client \
mysql-client \
busybox-extras \
&& update-ca-certificates
# Create non-root user
RUN addgroup -g 1000 gohoarder && \
adduser -D -u 1000 -G gohoarder gohoarder
# Copy binary (from platform-specific path)
ARG TARGETOS
ARG TARGETARCH
COPY ${TARGETOS}/${TARGETARCH}/migrate /usr/local/bin/migrate
RUN chmod +x /usr/local/bin/migrate
WORKDIR /app
USER gohoarder
# Run migrations
ENTRYPOINT ["/usr/local/bin/migrate"]
CMD ["--action=migrate"]
Makefile
+13 -6
View File
@@ -78,17 +78,24 @@ clean: ## Clean build artifacts
@rm -f *.db *.db-shm *.db-wal
@echo "Clean complete"
clean-db: ## Clean all local cache and database files (from config.yaml paths)
clean-db: ## Clean all local cache and database files (requires confirmation)
@echo "WARNING: This will delete all cached packages and scan results!"
@echo "Paths from config.yaml:"
@echo "Paths to be cleaned:"
@echo " - ./data/storage (package cache)"
@echo " - ./data/gohoarder.db (metadata database)"
@echo " - ./data/gohoarder.db and gohoarder.db (metadata database)"
@echo " - /tmp/trivy (Trivy cache)"
@echo ""
@read -p "Are you sure you want to continue? [y/N] " confirm && [ "$$confirm" = "y" ] || exit 1
@printf "Are you sure you want to continue? [y/N] " && read confirm && [ "$$confirm" = "y" ] || (echo "Cancelled." && exit 1)
@echo "Cleaning database and cache..."
@rm -rf ./data/storage
@rm -f ./data/gohoarder.db ./data/gohoarder.db-shm ./data/gohoarder.db-wal
@rm -rf ./data/storage ./data
@rm -f gohoarder.db gohoarder.db-shm gohoarder.db-wal
@rm -rf /tmp/trivy
@echo "Database and cache cleaned successfully"
clean-db-force: ## Clean all local cache and database files (no confirmation)
@echo "Cleaning database and cache..."
@rm -rf ./data/storage ./data
@rm -f gohoarder.db gohoarder.db-shm gohoarder.db-wal
@rm -rf /tmp/trivy
@echo "Database and cache cleaned successfully"
cmd/migrate/main.go
+258
View File
@@ -0,0 +1,258 @@
package main
import (
"context"
"database/sql"
"flag"
"fmt"
stdlog "log"
"os"
"time"
"github.com/go-gormigrate/gormigrate/v2"
_ "github.com/go-sql-driver/mysql"
_ "github.com/lib/pq"
"github.com/lukaszraczylo/gohoarder/pkg/metadata/gormstore"
"github.com/rs/zerolog"
"github.com/rs/zerolog/log"
"gorm.io/driver/mysql"
"gorm.io/driver/postgres"
"gorm.io/driver/sqlite"
"gorm.io/gorm"
"gorm.io/gorm/logger"
)
type MigrationConfig struct {
Driver string
DSN string
Timeout time.Duration
Action string // migrate, rollback, rollback-to, list
TargetID string // For rollback-to
LogLevel string
}
func main() {
cfg := MigrationConfig{}
flag.StringVar(&cfg.Driver, "driver", os.Getenv("DB_DRIVER"), "Database driver (postgres, mysql, sqlite)")
flag.StringVar(&cfg.DSN, "dsn", os.Getenv("DATABASE_URL"), "Database connection string")
flag.DurationVar(&cfg.Timeout, "timeout", 10*time.Minute, "Migration timeout")
flag.StringVar(&cfg.Action, "action", "migrate", "Action: migrate, rollback, rollback-to, list")
flag.StringVar(&cfg.TargetID, "target", "", "Target migration ID (for rollback-to)")
flag.StringVar(&cfg.LogLevel, "log-level", "info", "Log level: debug, info, warn, error")
flag.Parse()
// Setup logging
setupLogging(cfg.LogLevel)
log.Info().
Str("driver", cfg.Driver).
Str("action", cfg.Action).
Msg("Starting database migration")
if err := RunMigration(cfg); err != nil {
log.Fatal().Err(err).Msg("Migration failed")
}
log.Info().Msg("Migration completed successfully")
}
func setupLogging(level string) {
zerolog.TimeFieldFormat = zerolog.TimeFormatUnix
log.Logger = log.Output(zerolog.ConsoleWriter{Out: os.Stdout, TimeFormat: time.RFC3339})
switch level {
case "debug":
zerolog.SetGlobalLevel(zerolog.DebugLevel)
case "info":
zerolog.SetGlobalLevel(zerolog.InfoLevel)
case "warn":
zerolog.SetGlobalLevel(zerolog.WarnLevel)
case "error":
zerolog.SetGlobalLevel(zerolog.ErrorLevel)
default:
zerolog.SetGlobalLevel(zerolog.InfoLevel)
}
}
func RunMigration(cfg MigrationConfig) error {
// Validate config
if cfg.Driver == "" {
return fmt.Errorf("driver is required (set DB_DRIVER or --driver)")
}
if cfg.DSN == "" {
return fmt.Errorf("DSN is required (set DATABASE_URL or --dsn)")
}
ctx, cancel := context.WithTimeout(context.Background(), cfg.Timeout)
defer cancel()
// Connect to database
db, err := connectToDatabase(cfg.Driver, cfg.DSN)
if err != nil {
return fmt.Errorf("failed to connect to database: %w", err)
}
sqlDB, err := db.DB()
if err != nil {
return fmt.Errorf("failed to get sql.DB: %w", err)
}
defer sqlDB.Close()
// Wait for database to be ready
if err := waitForDB(ctx, sqlDB, 60*time.Second); err != nil {
return fmt.Errorf("database not ready: %w", err)
}
// Initialize gormigrate with custom options
opts := gormigrate.DefaultOptions
opts.TableName = "gohoarder_migrations"
m := gormigrate.New(db, opts, gormstore.GetMigrations())
log.Info().
Str("table", "gohoarder_migrations").
Msg("Migration tracking table initialized")
// Execute action
switch cfg.Action {
case "migrate":
return runMigrate(m)
case "rollback":
return runRollback(m)
case "rollback-to":
if cfg.TargetID == "" {
return fmt.Errorf("target migration ID required for rollback-to")
}
return runRollbackTo(m, cfg.TargetID)
case "list":
return listMigrations(db)
default:
return fmt.Errorf("unknown action: %s (use: migrate, rollback, rollback-to, list)", cfg.Action)
}
}
func connectToDatabase(driver, dsn string) (*gorm.DB, error) {
// Configure GORM logger using standard library log
gormLogger := logger.New(
stdlog.New(os.Stdout, "\r\n", stdlog.LstdFlags),
logger.Config{
SlowThreshold: 200 * time.Millisecond,
LogLevel: logger.Info,
IgnoreRecordNotFoundError: true,
Colorful: true,
},
)
var dialector gorm.Dialector
switch driver {
case "sqlite":
dialector = sqlite.Open(dsn)
case "postgres", "postgresql":
dialector = postgres.Open(dsn)
case "mysql":
dialector = mysql.Open(dsn)
default:
return nil, fmt.Errorf("unsupported driver: %s", driver)
}
db, err := gorm.Open(dialector, &gorm.Config{
Logger: gormLogger,
SkipDefaultTransaction: false, // Migrations should be transactional
PrepareStmt: true,
})
if err != nil {
return nil, err
}
return db, nil
}
func waitForDB(ctx context.Context, db *sql.DB, timeout time.Duration) error {
deadline := time.Now().Add(timeout)
attempt := 0
for {
attempt++
if time.Now().After(deadline) {
return fmt.Errorf("timeout waiting for database after %d attempts", attempt)
}
if err := db.PingContext(ctx); err == nil {
log.Info().
Int("attempts", attempt).
Msg("Database is ready")
return nil
}
log.Debug().
Int("attempt", attempt).
Msg("Waiting for database...")
time.Sleep(2 * time.Second)
}
}
func runMigrate(m *gormigrate.Gormigrate) error {
log.Info().Msg("Running migrations...")
if err := m.Migrate(); err != nil {
return fmt.Errorf("migration failed: %w", err)
}
log.Info().Msg("✓ All migrations applied successfully")
return nil
}
func runRollback(m *gormigrate.Gormigrate) error {
log.Warn().Msg("Rolling back last migration...")
if err := m.RollbackLast(); err != nil {
return fmt.Errorf("rollback failed: %w", err)
}
log.Info().Msg("✓ Rollback completed")
return nil
}
func runRollbackTo(m *gormigrate.Gormigrate, targetID string) error {
log.Warn().
Str("target_id", targetID).
Msg("Rolling back to migration...")
if err := m.RollbackTo(targetID); err != nil {
return fmt.Errorf("rollback to %s failed: %w", targetID, err)
}
log.Info().
Str("target_id", targetID).
Msg("✓ Rollback completed")
return nil
}
func listMigrations(db *gorm.DB) error {
log.Info().Msg("Applied migrations:")
type Migration struct {
ID string
}
var migrations []Migration
if err := db.Table("gohoarder_migrations").Find(&migrations).Error; err != nil {
return fmt.Errorf("failed to list migrations: %w", err)
}
if len(migrations) == 0 {
log.Info().Msg(" (no migrations applied yet)")
return nil
}
for _, m := range migrations {
log.Info().Str("id", m.ID).Msg(" ✓")
}
log.Info().
Int("total", len(migrations)).
Msg("Applied migrations")
return nil
}
config.yaml.example
+37 -4
View File
@@ -40,20 +40,53 @@ storage:
domain: ""
metadata:
backend: "sqlite" # sqlite, postgresql, file
connection: "file:gohoarder.db?cache=shared&mode=rwc"
# Backend: sqlite, postgresql, mysql, mariadb, file
#
# Choose based on your deployment:
# - sqlite: Single instance, local storage (NOT for network filesystems like SMB/NFS!)
# - postgresql: Production, multiple replicas, works with any storage including SMB/NFS
# - mysql: Production alternative to PostgreSQL
# - file: Simple file-based metadata (limited features)
#
# IMPORTANT: SQLite + SMB/NFS = Database locked errors!
# For network storage (SMB, NFS), use PostgreSQL or MySQL.
backend: "sqlite"
connection: "file:gohoarder.db?cache=shared&mode=rwc" # Legacy, not used with GORM
# SQLite configuration (for local storage only)
# Use with local storage classes (local-path, hostPath, or RWX like longhorn)
# DO NOT use with SMB/NFS network storage!
sqlite:
path: "gohoarder.db"
wal_mode: true
wal_mode: true # Set to false for network filesystems if you must use SQLite
# PostgreSQL configuration (recommended for production)
# Works with any storage including SMB/NFS
# Supports multiple replicas and high availability
postgresql:
host: "localhost"
port: 5432
database: "gohoarder"
user: "gohoarder"
password: ""
ssl_mode: "disable"
ssl_mode: "disable" # disable, require, verify-ca, verify-full
# MySQL/MariaDB configuration (alternative to PostgreSQL)
# Works with any storage including SMB/NFS
mysql:
host: "localhost"
port: 3306
database: "gohoarder"
user: "gohoarder"
password: ""
charset: "utf8mb4"
parse_time: true
# GORM connection pool settings (applies to all database backends)
max_open_conns: 25 # Maximum number of open connections to the database
max_idle_conns: 5 # Maximum number of idle connections in the pool
conn_max_lifetime: 3600 # Maximum lifetime of a connection in seconds (1 hour)
log_level: "warn" # GORM log level: silent, error, warn, info
cache:
default_ttl: "168h" # 7 days
frontend/src/components/PackageList.vue
+1
View File
@@ -128,6 +128,7 @@
:counts="version.vulnerabilities.counts"
:total="version.vulnerabilities.total"
:scannedAt="version.vulnerabilities.scannedAt"
:isBlocked="version.vulnerabilities.isBlocked"
@click="showVulnerabilityDetails(group.registry, group.name, version.version)"
/>
</div>
frontend/src/components/Stats.vue
+53 -8
View File
@@ -29,11 +29,26 @@
</p>
<p class="text-sm text-gray-600">Total Packages</p>
</div>
<div class="text-center p-6 bg-gray-50 rounded-lg">
<p class="text-4xl font-bold text-blue-600 mb-2">
{{ formatBytes(stats?.total_size || 0) }}
<div class="p-6 bg-gray-50 rounded-lg">
<div class="text-center mb-3">
<p class="text-2xl font-bold text-blue-600">
{{ formatBytes(stats?.total_size || 0) }} / {{ formatBytes(stats?.max_cache_size || 0) }}
</p>
<p class="text-sm text-gray-600">Total Storage Used</p>
<p class="text-sm text-gray-600 mt-1">Storage Used</p>
</div>
<div class="w-full bg-gray-200 rounded-full h-3 overflow-hidden">
<div
class="h-full bg-blue-600 rounded-full transition-all duration-300"
:style="{ width: storagePercentage + '%' }"
:class="{
'bg-green-600': storagePercentage < 50,
'bg-yellow-600': storagePercentage >= 50 && storagePercentage < 80,
'bg-orange-600': storagePercentage >= 80 && storagePercentage < 90,
'bg-red-600': storagePercentage >= 90
}"
></div>
</div>
<p class="text-xs text-gray-500 text-center mt-1">{{ storagePercentage.toFixed(1) }}% used</p>
</div>
<div class="text-center p-6 bg-gray-50 rounded-lg">
<p class="text-4xl font-bold text-green-600 mb-2">
@@ -51,7 +66,7 @@
<h3 class="text-xl font-semibold text-gray-900 mb-6">
<i class="fas fa-shield-alt mr-2"></i>Security Scanning
</h3>
<div class="grid grid-cols-1 md:grid-cols-2 gap-6">
<div class="grid grid-cols-1 md:grid-cols-3 gap-6">
<div class="flex items-center justify-between p-6 bg-green-50 rounded-lg border border-green-200">
<div>
<p class="text-3xl font-bold text-green-600">
@@ -63,11 +78,11 @@
</div>
<div
@click="showVulnerablePackages"
class="flex items-center justify-between p-6 bg-red-50 rounded-lg border border-red-200 cursor-pointer hover:bg-red-100 transition-colors"
class="flex items-center justify-between p-6 bg-orange-50 rounded-lg border border-orange-200 cursor-pointer hover:bg-orange-100 transition-colors"
:class="{ 'opacity-50': (stats?.vulnerable_packages || 0) === 0 }"
>
<div>
<p class="text-3xl font-bold text-red-600">
<p class="text-3xl font-bold text-orange-600">
{{ formatNumber(stats?.vulnerable_packages || 0) }}
</p>
<p class="text-sm text-gray-600 mt-1">
@@ -75,7 +90,23 @@
<span v-if="(stats?.vulnerable_packages || 0) > 0" class="text-xs ml-1">(click to view)</span>
</p>
</div>
<i class="fas fa-exclamation-triangle text-5xl text-red-400"></i>
<i class="fas fa-exclamation-triangle text-5xl text-orange-400"></i>
</div>
<div
@click="showBlockedPackages"
class="flex items-center justify-between p-6 bg-red-50 rounded-lg border border-red-200 cursor-pointer hover:bg-red-100 transition-colors"
:class="{ 'opacity-50': (stats?.blocked_packages || 0) === 0 }"
>
<div>
<p class="text-3xl font-bold text-red-600">
{{ formatNumber(stats?.blocked_packages || 0) }}
</p>
<p class="text-sm text-gray-600 mt-1">
Blocked Packages
<span v-if="(stats?.blocked_packages || 0) > 0" class="text-xs ml-1">(click to view)</span>
</p>
</div>
<i class="fas fa-hand text-5xl text-red-400"></i>
</div>
</div>
</CardContent>
@@ -141,6 +172,14 @@ function showVulnerablePackages() {
router.push('/vulnerable-packages')
}
function showBlockedPackages() {
if ((stats.value?.blocked_packages || 0) === 0) {
return
}
router.push('/blocked-packages')
}
// Registry configuration for icons and colors
const registryConfig: Record<string, {label: string, icon: string, color: string}> = {
npm: {
@@ -180,6 +219,12 @@ const registries = computed(() => {
})
})
const storagePercentage = computed(() => {
const totalSize = stats.value?.total_size || 0
const maxSize = stats.value?.max_cache_size || 1
return (totalSize / maxSize) * 100
})
function formatNumber(num: number): string {
return new Intl.NumberFormat().format(num)
}
frontend/src/components/VulnerabilityBadge.vue
+12
View File
@@ -1,5 +1,15 @@
<template>
<div class="flex items-center gap-2">
<!-- Blocked Icon (if package exceeds thresholds) -->
<span
v-if="isBlocked"
class="inline-flex items-center px-2.5 py-1 rounded-md text-xs font-medium bg-red-600 text-white border border-red-700"
title="Download blocked - exceeds vulnerability thresholds"
>
<i class="fas fa-hand mr-1"></i>
BLOCKED
</span>
<!-- Critical Vulnerabilities -->
<button
v-if="counts.critical > 0"
@@ -89,12 +99,14 @@ interface Props {
counts?: VulnerabilityCounts
total?: number
scannedAt?: string // ISO 8601 timestamp
isBlocked?: boolean // Whether download is blocked due to vulnerabilities
}
const props = withDefaults(defineProps<Props>(), {
scanned: false,
status: 'not_scanned',
total: 0,
isBlocked: false,
})
const emit = defineEmits<{
frontend/src/components/VulnerablePackages.vue
+20 -6
View File
@@ -7,11 +7,16 @@
Back to Stats
</Button>
<div class="flex items-center gap-3">
<i class="fas fa-exclamation-triangle text-3xl text-red-600"></i>
<i :class="showOnlyBlocked ? 'fas fa-hand' : 'fas fa-exclamation-triangle'" class="text-3xl text-red-600"></i>
<div>
<h1 class="text-3xl font-bold text-gray-900">Vulnerable Packages</h1>
<h1 class="text-3xl font-bold text-gray-900">
{{ showOnlyBlocked ? 'Blocked Packages' : 'Vulnerable Packages' }}
</h1>
<p class="text-gray-600 mt-1">
Packages with known security vulnerabilities, sorted by risk
{{ showOnlyBlocked
? 'Packages blocked from download due to exceeding vulnerability thresholds'
: 'Packages with known security vulnerabilities, sorted by risk'
}}
</p>
</div>
</div>
@@ -132,6 +137,7 @@
:counts="version.vulnerabilities.counts"
:total="version.vulnerabilities.total"
:scannedAt="version.vulnerabilities.scannedAt"
:isBlocked="version.vulnerabilities.isBlocked"
@click.stop="navigateToPackage(version)"
/>
</div>
@@ -170,11 +176,15 @@ import { getRegistryBadgeClass } from '@/composables/useBadgeStyles'
const store = usePackageStore()
const router = useRouter()
const route = useRoute()
const loading = ref(false)
const error = ref<string | null>(null)
const vulnerablePackages = ref<Package[]>([])
// Check if we should filter to show only blocked packages
const showOnlyBlocked = computed(() => route.path === '/blocked-packages')
onMounted(async () => {
await fetchVulnerablePackages()
})
@@ -185,9 +195,13 @@ async function fetchVulnerablePackages() {
try {
await store.fetchPackages()
vulnerablePackages.value = store.packages.filter(
pkg => pkg.vulnerabilities?.status === 'vulnerable'
)
vulnerablePackages.value = store.packages.filter(pkg => {
const isVulnerable = pkg.vulnerabilities?.status === 'vulnerable'
if (showOnlyBlocked.value) {
return isVulnerable && pkg.vulnerabilities?.isBlocked === true
}
return isVulnerable
})
} catch (err: any) {
console.error('Failed to load vulnerable packages:', err)
error.value = err.message || 'Failed to load vulnerable packages'
frontend/src/router/index.ts
+5
View File
@@ -37,6 +37,11 @@ const router = createRouter({
name: 'vulnerable-packages',
component: VulnerablePackages,
},
{
path: '/blocked-packages',
name: 'blocked-packages',
component: VulnerablePackages,
},
{
path: '/admin/bypasses',
name: 'bypasses',
frontend/src/stores/packages.ts
+1
View File
@@ -15,6 +15,7 @@ export interface VulnerabilityInfo {
counts?: VulnerabilityCounts
total?: number
scannedAt?: string // ISO 8601 timestamp
isBlocked?: boolean // Whether download is blocked due to vulnerability thresholds
}
export interface Package {
go.mod
+62 -10
View File
@@ -7,23 +7,35 @@ require (
github.com/aws/aws-sdk-go-v2/config v1.32.6
github.com/aws/aws-sdk-go-v2/credentials v1.19.6
github.com/aws/aws-sdk-go-v2/service/s3 v1.95.0
github.com/go-gormigrate/gormigrate/v2 v2.1.5
github.com/go-sql-driver/mysql v1.9.3
github.com/goccy/go-json v0.10.5
github.com/gofiber/fiber/v2 v2.52.10
github.com/gorilla/websocket v1.5.3
github.com/hirochachacha/go-smb2 v1.1.0
github.com/lib/pq v1.10.9
github.com/prometheus/client_golang v1.23.2
github.com/redis/go-redis/v9 v9.17.2
github.com/rs/zerolog v1.34.0
github.com/spf13/cobra v1.10.2
github.com/spf13/viper v1.21.0
github.com/stretchr/testify v1.11.1
github.com/testcontainers/testcontainers-go v0.40.0
github.com/testcontainers/testcontainers-go/modules/mysql v0.40.0
github.com/testcontainers/testcontainers-go/modules/postgres v0.40.0
golang.org/x/crypto v0.46.0
golang.org/x/sync v0.19.0
golang.org/x/time v0.14.0
modernc.org/sqlite v1.42.2
gorm.io/driver/mysql v1.6.0
gorm.io/driver/postgres v1.6.0
gorm.io/driver/sqlite v1.6.0
gorm.io/gorm v1.31.1
)
require (
dario.cat/mergo v1.0.2 // indirect
filippo.io/edwards25519 v1.1.0 // indirect
github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c // indirect
github.com/Microsoft/go-winio v0.6.2 // indirect
github.com/andybalholm/brotli v1.2.0 // indirect
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4 // indirect
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.16 // indirect
@@ -41,45 +53,85 @@ require (
github.com/aws/aws-sdk-go-v2/service/sts v1.41.5 // indirect
github.com/aws/smithy-go v1.24.0 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/cenkalti/backoff/v4 v4.3.0 // indirect
github.com/cespare/xxhash/v2 v2.3.0 // indirect
github.com/clipperhouse/stringish v0.1.1 // indirect
github.com/clipperhouse/uax29/v2 v2.3.0 // indirect
github.com/containerd/errdefs v1.0.0 // indirect
github.com/containerd/errdefs/pkg v0.3.0 // indirect
github.com/containerd/log v0.1.0 // indirect
github.com/containerd/platforms v0.2.1 // indirect
github.com/cpuguy83/dockercfg v0.3.2 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
github.com/dustin/go-humanize v1.0.1 // indirect
github.com/distribution/reference v0.6.0 // indirect
github.com/docker/docker v28.5.2+incompatible // indirect
github.com/docker/go-connections v0.6.0 // indirect
github.com/docker/go-units v0.5.0 // indirect
github.com/ebitengine/purego v0.9.1 // indirect
github.com/felixge/httpsnoop v1.0.4 // indirect
github.com/fsnotify/fsnotify v1.9.0 // indirect
github.com/geoffgarside/ber v1.2.0 // indirect
github.com/go-logr/logr v1.4.3 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/go-ole/go-ole v1.3.0 // indirect
github.com/go-viper/mapstructure/v2 v2.4.0 // indirect
github.com/google/uuid v1.6.0 // indirect
github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.4 // indirect
github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/jackc/pgpassfile v1.0.0 // indirect
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
github.com/jackc/pgx/v5 v5.8.0 // indirect
github.com/jackc/puddle/v2 v2.2.2 // indirect
github.com/jinzhu/inflection v1.0.0 // indirect
github.com/jinzhu/now v1.1.5 // indirect
github.com/klauspost/compress v1.18.2 // indirect
github.com/lufia/plan9stats v0.0.0-20251013123823-9fd1530e3ec3 // indirect
github.com/magiconair/properties v1.8.10 // indirect
github.com/mattn/go-colorable v0.1.14 // indirect
github.com/mattn/go-isatty v0.0.20 // indirect
github.com/mattn/go-runewidth v0.0.19 // indirect
github.com/mattn/go-sqlite3 v1.14.33 // indirect
github.com/moby/docker-image-spec v1.3.1 // indirect
github.com/moby/go-archive v0.2.0 // indirect
github.com/moby/patternmatcher v0.6.0 // indirect
github.com/moby/sys/sequential v0.6.0 // indirect
github.com/moby/sys/user v0.4.0 // indirect
github.com/moby/sys/userns v0.1.0 // indirect
github.com/moby/term v0.5.2 // indirect
github.com/morikuni/aec v1.1.0 // indirect
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
github.com/ncruces/go-strftime v1.0.0 // indirect
github.com/opencontainers/go-digest v1.0.0 // indirect
github.com/opencontainers/image-spec v1.1.1 // indirect
github.com/pelletier/go-toml/v2 v2.2.4 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect
github.com/prometheus/client_model v0.6.2 // indirect
github.com/prometheus/common v0.67.4 // indirect
github.com/prometheus/procfs v0.19.2 // indirect
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
github.com/sagikazarmark/locafero v0.12.0 // indirect
github.com/shirou/gopsutil/v4 v4.25.12 // indirect
github.com/sirupsen/logrus v1.9.3 // indirect
github.com/spf13/afero v1.15.0 // indirect
github.com/spf13/cast v1.10.0 // indirect
github.com/spf13/pflag v1.0.10 // indirect
github.com/stretchr/objx v0.5.2 // indirect
github.com/subosito/gotenv v1.6.0 // indirect
github.com/tklauser/go-sysconf v0.3.16 // indirect
github.com/tklauser/numcpus v0.11.0 // indirect
github.com/valyala/bytebufferpool v1.0.0 // indirect
github.com/valyala/fasthttp v1.68.0 // indirect
github.com/yusufpapurcu/wmi v1.2.4 // indirect
go.opentelemetry.io/auto/sdk v1.2.1 // indirect
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.64.0 // indirect
go.opentelemetry.io/otel v1.39.0 // indirect
go.opentelemetry.io/otel/metric v1.39.0 // indirect
go.opentelemetry.io/otel/trace v1.39.0 // indirect
go.yaml.in/yaml/v2 v2.4.3 // indirect
go.yaml.in/yaml/v3 v3.0.4 // indirect
golang.org/x/exp v0.0.0-20251219203646-944ab1f22d93 // indirect
golang.org/x/sys v0.39.0 // indirect
golang.org/x/text v0.32.0 // indirect
google.golang.org/grpc v1.78.0 // indirect
google.golang.org/protobuf v1.36.11 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
modernc.org/libc v1.67.4 // indirect
modernc.org/mathutil v1.7.1 // indirect
modernc.org/memory v1.11.0 // indirect
)
go.sum
+165 -54
View File
@@ -1,3 +1,13 @@
dario.cat/mergo v1.0.2 h1:85+piFYR1tMbRrLcDwR18y4UKJ3aH1Tbzi24VRW1TK8=
dario.cat/mergo v1.0.2/go.mod h1:E/hbnu0NxMFBjpMIE34DRGLWqDy0g5FuKDhCb31ngxA=
filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA=
filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4=
github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6 h1:He8afgbRMd7mFxO99hRNu+6tazq8nFF9lIwo9JFroBk=
github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6/go.mod h1:8o94RPi1/7XTJvwPpRSzSUedZrtlirdB3r9Z20bi2f8=
github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c h1:udKWzYgxTojEKWjV8V+WSxDXJ4NFATAsZjh8iIbsQIg=
github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E=
github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY=
github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU=
github.com/andybalholm/brotli v1.2.0 h1:ukwgCxwYrmACq68yiUqwIWnGY0cTPox/M94sVwToPjQ=
github.com/andybalholm/brotli v1.2.0/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY=
github.com/aws/aws-sdk-go-v2 v1.41.0 h1:tNvqh1s+v0vFYdA1xq0aOJH+Y5cRyZ5upu6roPgPKd4=
@@ -40,24 +50,43 @@ github.com/aws/smithy-go v1.24.0 h1:LpilSUItNPFr1eY85RYgTIg5eIEPtvFbskaFcmmIUnk=
github.com/aws/smithy-go v1.24.0/go.mod h1:LEj2LM3rBRQJxPZTB4KuzZkaZYnZPnvgIhb4pu07mx0=
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs=
github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c=
github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA=
github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0=
github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8=
github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE=
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/clipperhouse/stringish v0.1.1 h1:+NSqMOr3GR6k1FdRhhnXrLfztGzuG+VuFDfatpWHKCs=
github.com/clipperhouse/stringish v0.1.1/go.mod h1:v/WhFtE1q0ovMta2+m+UbpZ+2/HEXNWYXQgCt4hdOzA=
github.com/clipperhouse/uax29/v2 v2.3.0 h1:SNdx9DVUqMoBuBoW3iLOj4FQv3dN5mDtuqwuhIGpJy4=
github.com/clipperhouse/uax29/v2 v2.3.0/go.mod h1:Wn1g7MK6OoeDT0vL+Q0SQLDz/KpfsVRgg6W7ihQeh4g=
github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI=
github.com/containerd/errdefs v1.0.0/go.mod h1:+YBYIdtsnF4Iw6nWZhJcqGSg/dwvV7tyJ/kCkyJ2k+M=
github.com/containerd/errdefs/pkg v0.3.0 h1:9IKJ06FvyNlexW690DXuQNx2KA2cUJXx151Xdx3ZPPE=
github.com/containerd/errdefs/pkg v0.3.0/go.mod h1:NJw6s9HwNuRhnjJhM7pylWwMyAkmCQvQ4GpJHEqRLVk=
github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I=
github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo=
github.com/containerd/platforms v0.2.1 h1:zvwtM3rz2YHPQsF2CHYM8+KtB5dvhISiXh5ZpSBQv6A=
github.com/containerd/platforms v0.2.1/go.mod h1:XHCb+2/hzowdiut9rkudds9bE5yJ7npe7dG/wG+uFPw=
github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
github.com/cpuguy83/dockercfg v0.3.2 h1:DlJTyZGBDlXqUZ2Dk2Q3xHs/FtnooJJVaad2S9GKorA=
github.com/cpuguy83/dockercfg v0.3.2/go.mod h1:sugsbF4//dDlL/i+S+rtpIWp+5h0BHJHfjj5/jFyUJc=
github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY=
github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk=
github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E=
github.com/docker/docker v28.5.2+incompatible h1:DBX0Y0zAjZbSrm1uzOkdr1onVghKaftjlSWt4AFexzM=
github.com/docker/docker v28.5.2+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
github.com/docker/go-connections v0.6.0 h1:LlMG9azAe1TqfR7sO+NJttz1gy6KO7VJBh+pMmjSD94=
github.com/docker/go-connections v0.6.0/go.mod h1:AahvXYshr6JgfUJGdDCs2b5EZG/vmaMAntpSFH5BFKE=
github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4=
github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
github.com/ebitengine/purego v0.9.1 h1:a/k2f2HQU3Pi399RPW1MOaZyhKJL9w/xFpKAg4q1s0A=
github.com/ebitengine/purego v0.9.1/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ=
github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg=
github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8=
github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k=
@@ -65,6 +94,18 @@ github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8
github.com/geoffgarside/ber v1.1.0/go.mod h1:jVPKeCbj6MvQZhwLYsGwaGI52oUorHoHKNecGT85ZCc=
github.com/geoffgarside/ber v1.2.0 h1:/loowoRcs/MWLYmGX9QtIAbA+V/FrnVLsMMPhwiRm64=
github.com/geoffgarside/ber v1.2.0/go.mod h1:jVPKeCbj6MvQZhwLYsGwaGI52oUorHoHKNecGT85ZCc=
github.com/go-gormigrate/gormigrate/v2 v2.1.5 h1:1OyorA5LtdQw12cyJDEHuTrEV3GiXiIhS4/QTTa/SM8=
github.com/go-gormigrate/gormigrate/v2 v2.1.5/go.mod h1:mj9ekk/7CPF3VjopaFvWKN2v7fN3D9d3eEOAXRhi/+M=
github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI=
github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE=
github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78=
github.com/go-sql-driver/mysql v1.9.3 h1:U/N249h2WzJ3Ukj8SowVFjdtZKfu9vlLZxjPXV1aweo=
github.com/go-sql-driver/mysql v1.9.3/go.mod h1:qn46aNg1333BRMNU69Lq93t8du/dwxI64Gl8i5p1WMU=
github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9LvH92wZUgs=
github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM=
github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4=
@@ -74,18 +115,28 @@ github.com/gofiber/fiber/v2 v2.52.10 h1:jRHROi2BuNti6NYXmZ6gbNSfT3zj/8c0xy94GOU5
github.com/gofiber/fiber/v2 v2.52.10/go.mod h1:YEcBbO/FB+5M1IZNBP9FO3J9281zgPAreiI1oqg8nDw=
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e h1:ijClszYn+mADRFY17kjQEVQ1XRhq2/JR1M3sGqeJoxs=
github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e/go.mod h1:boTsfXsheKC2y+lKOCMpSfarhxDeIzfZG1jqGcPl3cA=
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg=
github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k=
github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.4 h1:kEISI/Gx67NzH3nJxAmY/dGac80kKZgZt134u7Y/k1s=
github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.4/go.mod h1:6Nz966r3vQYCqIzWsuEl9d7cf7mRhtDmm++sOxlnfxI=
github.com/hirochachacha/go-smb2 v1.1.0 h1:b6hs9qKIql9eVXAiN0M2wSFY5xnhbHAQoCwRKbaRTZI=
github.com/hirochachacha/go-smb2 v1.1.0/go.mod h1:8F1A4d5EZzrGu5R7PU163UcMRDJQl4FtcxjBfsY8TZE=
github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=
github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo=
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM=
github.com/jackc/pgx/v5 v5.8.0 h1:TYPDoleBBme0xGSAX3/+NujXXtpZn9HBONkQC7IEZSo=
github.com/jackc/pgx/v5 v5.8.0/go.mod h1:QVeDInX2m9VyzvNeiCJVjCkNFqzsNb43204HshNSZKw=
github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo=
github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E=
github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ=
github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
github.com/klauspost/compress v1.18.2 h1:iiPHWW0YrcFgpBYhsA6D1+fqHssJscY/Tm/y2Uqnapk=
github.com/klauspost/compress v1.18.2/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4=
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
@@ -94,6 +145,12 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw=
github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
github.com/lufia/plan9stats v0.0.0-20251013123823-9fd1530e3ec3 h1:PwQumkgq4/acIiZhtifTV5OUqqiP82UAl0h87xj/l9k=
github.com/lufia/plan9stats v0.0.0-20251013123823-9fd1530e3ec3/go.mod h1:autxFIvghDt3jPTLoqZ9OZ7s9qTGNAWmYCjVFWPX/zg=
github.com/magiconair/properties v1.8.10 h1:s31yESBquKXCV9a/ScB3ESkOjUYYv+X0rg8SYxI99mE=
github.com/magiconair/properties v1.8.10/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0=
github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE=
github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8=
@@ -103,15 +160,42 @@ github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWE
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
github.com/mattn/go-runewidth v0.0.19 h1:v++JhqYnZuu5jSKrk9RbgF5v4CGUjqRfBm05byFGLdw=
github.com/mattn/go-runewidth v0.0.19/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhgLpndooCuJAs=
github.com/mattn/go-sqlite3 v1.14.33 h1:A5blZ5ulQo2AtayQ9/limgHEkFreKj1Dv226a1K73s0=
github.com/mattn/go-sqlite3 v1.14.33/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
github.com/mdelapenya/tlscert v0.2.0 h1:7H81W6Z/4weDvZBNOfQte5GpIMo0lGYEeWbkGp5LJHI=
github.com/mdelapenya/tlscert v0.2.0/go.mod h1:O4njj3ELLnJjGdkN7M/vIVCpZ+Cf0L6muqOG4tLSl8o=
github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0=
github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo=
github.com/moby/go-archive v0.2.0 h1:zg5QDUM2mi0JIM9fdQZWC7U8+2ZfixfTYoHL7rWUcP8=
github.com/moby/go-archive v0.2.0/go.mod h1:mNeivT14o8xU+5q1YnNrkQVpK+dnNe/K6fHqnTg4qPU=
github.com/moby/patternmatcher v0.6.0 h1:GmP9lR19aU5GqSSFko+5pRqHi+Ohk1O69aFiKkVGiPk=
github.com/moby/patternmatcher v0.6.0/go.mod h1:hDPoyOpDY7OrrMDLaYoY3hf52gNCR/YOUYxkhApJIxc=
github.com/moby/sys/atomicwriter v0.1.0 h1:kw5D/EqkBwsBFi0ss9v1VG3wIkVhzGvLklJ+w3A14Sw=
github.com/moby/sys/atomicwriter v0.1.0/go.mod h1:Ul8oqv2ZMNHOceF643P6FKPXeCmYtlQMvpizfsSoaWs=
github.com/moby/sys/sequential v0.6.0 h1:qrx7XFUd/5DxtqcoH1h438hF5TmOvzC/lspjy7zgvCU=
github.com/moby/sys/sequential v0.6.0/go.mod h1:uyv8EUTrca5PnDsdMGXhZe6CCe8U/UiTWd+lL+7b/Ko=
github.com/moby/sys/user v0.4.0 h1:jhcMKit7SA80hivmFJcbB1vqmw//wU61Zdui2eQXuMs=
github.com/moby/sys/user v0.4.0/go.mod h1:bG+tYYYJgaMtRKgEmuueC0hJEAZWwtIbZTB+85uoHjs=
github.com/moby/sys/userns v0.1.0 h1:tVLXkFOxVu9A64/yh59slHVv9ahO9UIev4JZusOLG/g=
github.com/moby/sys/userns v0.1.0/go.mod h1:IHUYgu/kao6N8YZlp9Cf444ySSvCmDlmzUcYfDHOl28=
github.com/moby/term v0.5.2 h1:6qk3FJAFDs6i/q3W/pQ97SX192qKfZgGjCQqfCJkgzQ=
github.com/moby/term v0.5.2/go.mod h1:d3djjFCrjnB+fl8NJux+EJzu0msscUP+f8it8hPkFLc=
github.com/morikuni/aec v1.1.0 h1:vBBl0pUnvi/Je71dsRrhMBtreIqNMYErSAbEeb8jrXQ=
github.com/morikuni/aec v1.1.0/go.mod h1:xDRgiq/iw5l+zkao76YTKzKttOp2cwPEne25HDkJnBw=
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w=
github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls=
github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040=
github.com/opencontainers/image-spec v1.1.1/go.mod h1:qpqAh3Dmcf36wStyyWU+kCeDgrGnAve2nCC8+7h8Q0M=
github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4=
github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 h1:o4JXh1EVt9k/+g42oCprj/FisM4qX9L3sZB3upGN2ZU=
github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE=
github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o=
github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg=
github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk=
@@ -120,18 +204,18 @@ github.com/prometheus/common v0.67.4 h1:yR3NqWO1/UyO1w2PhUvXlGQs/PtFmoveVO0KZ4+L
github.com/prometheus/common v0.67.4/go.mod h1:gP0fq6YjjNCLssJCQp0yk4M8W6ikLURwkdd/YKtTbyI=
github.com/prometheus/procfs v0.19.2 h1:zUMhqEW66Ex7OXIiDkll3tl9a1ZdilUOd/F6ZXw4Vws=
github.com/prometheus/procfs v0.19.2/go.mod h1:M0aotyiemPhBCM0z5w87kL22CxfcH05ZpYlu+b4J7mw=
github.com/redis/go-redis/v9 v9.17.2 h1:P2EGsA4qVIM3Pp+aPocCJ7DguDHhqrXNhVcEp4ViluI=
github.com/redis/go-redis/v9 v9.17.2/go.mod h1:u410H11HMLoB+TP67dz8rL9s6QW2j76l0//kSOd3370=
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ=
github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog=
github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ=
github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc=
github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0=
github.com/rs/zerolog v1.34.0 h1:k43nTLIwcTVQAncfCw4KZ2VY6ukYoZaBPNOE8txlOeY=
github.com/rs/zerolog v1.34.0/go.mod h1:bJsvje4Z08ROH4Nhs5iH600c3IkWhwp44iRc54W6wYQ=
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/sagikazarmark/locafero v0.12.0 h1:/NQhBAkUb4+fH1jivKHWusDYFjMOOKU88eegjfxfHb4=
github.com/sagikazarmark/locafero v0.12.0/go.mod h1:sZh36u/YSZ918v0Io+U9ogLYQJ9tLLBmM4eneO6WwsI=
github.com/shirou/gopsutil/v4 v4.25.12 h1:e7PvW/0RmJ8p8vPGJH4jvNkOyLmbkXgXW4m6ZPic6CY=
github.com/shirou/gopsutil/v4 v4.25.12/go.mod h1:EivAfP5x2EhLp2ovdpKSozecVXn1TmuG7SMzs/Wh4PU=
github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
github.com/spf13/afero v1.15.0 h1:b/YBCLWAJdFWJTN9cLhiXXcD7mzKn9Dm86dNnfyQw1I=
github.com/spf13/afero v1.15.0/go.mod h1:NC2ByUVxtQs4b3sIUphxK0NioZnmxgyCrfzeuq8lxMg=
github.com/spf13/cast v1.10.0 h1:h2x0u2shc1QuLHfxi+cTJvs30+ZAHOGRic8uyGTDWxY=
@@ -143,18 +227,53 @@ github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk=
github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
github.com/spf13/viper v1.21.0 h1:x5S+0EU27Lbphp4UKm1C+1oQO+rKx36vfCoaVebLFSU=
github.com/spf13/viper v1.21.0/go.mod h1:P0lhsswPGWD/1lZJ9ny3fYnVqxiegrlNrEmgLjbTCAY=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY=
github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8=
github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU=
github.com/testcontainers/testcontainers-go v0.40.0 h1:pSdJYLOVgLE8YdUY2FHQ1Fxu+aMnb6JfVz1mxk7OeMU=
github.com/testcontainers/testcontainers-go v0.40.0/go.mod h1:FSXV5KQtX2HAMlm7U3APNyLkkap35zNLxukw9oBi/MY=
github.com/testcontainers/testcontainers-go/modules/mysql v0.40.0 h1:P9Txfy5Jothx2wFdcus0QoSmX/PKSIXZxrTbZPVJswA=
github.com/testcontainers/testcontainers-go/modules/mysql v0.40.0/go.mod h1:oZPHHqJqXG7FD8OB/yWH7gLnDvZUlFHAVJNrGftL+eg=
github.com/testcontainers/testcontainers-go/modules/postgres v0.40.0 h1:s2bIayFXlbDFexo96y+htn7FzuhpXLYJNnIuglNKqOk=
github.com/testcontainers/testcontainers-go/modules/postgres v0.40.0/go.mod h1:h+u/2KoREGTnTl9UwrQ/g+XhasAT8E6dClclAADeXoQ=
github.com/tklauser/go-sysconf v0.3.16 h1:frioLaCQSsF5Cy1jgRBrzr6t502KIIwQ0MArYICU0nA=
github.com/tklauser/go-sysconf v0.3.16/go.mod h1:/qNL9xxDhc7tx3HSRsLWNnuzbVfh3e7gh/BmM179nYI=
github.com/tklauser/numcpus v0.11.0 h1:nSTwhKH5e1dMNsCdVBukSZrURJRoHbSEQjdEbY+9RXw=
github.com/tklauser/numcpus v0.11.0/go.mod h1:z+LwcLq54uWZTX0u/bGobaV34u6V7KNlTZejzM6/3MQ=
github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
github.com/valyala/fasthttp v1.68.0 h1:v12Nx16iepr8r9ySOwqI+5RBJ/DqTxhOy1HrHoDFnok=
github.com/valyala/fasthttp v1.68.0/go.mod h1:5EXiRfYQAoiO/khu4oU9VISC/eVY6JqmSpPJoHCKsz4=
github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU=
github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E=
github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0=
github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64=
go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y=
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.64.0 h1:ssfIgGNANqpVFCndZvcuyKbl0g+UAVcbBcqGkG28H0Y=
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.64.0/go.mod h1:GQ/474YrbE4Jx8gZ4q5I4hrhUzM6UPzyrqJYV2AqPoQ=
go.opentelemetry.io/otel v1.39.0 h1:8yPrr/S0ND9QEfTfdP9V+SiwT4E0G7Y5MO7p85nis48=
go.opentelemetry.io/otel v1.39.0/go.mod h1:kLlFTywNWrFyEdH0oj2xK0bFYZtHRYUdv1NklR/tgc8=
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.19.0 h1:Mne5On7VWdx7omSrSSZvM4Kw7cS7NQkOOmLcgscI51U=
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.19.0/go.mod h1:IPtUMKL4O3tH5y+iXVyAXqpAwMuzC1IrxVS81rummfE=
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.19.0 h1:IeMeyr1aBvBiPVYihXIaeIZba6b8E1bYp7lbdxK8CQg=
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.19.0/go.mod h1:oVdCUtjq9MK9BlS7TtucsQwUcXcymNiEDjgDD2jMtZU=
go.opentelemetry.io/otel/metric v1.39.0 h1:d1UzonvEZriVfpNKEVmHXbdf909uGTOQjA0HF0Ls5Q0=
go.opentelemetry.io/otel/metric v1.39.0/go.mod h1:jrZSWL33sD7bBxg1xjrqyDjnuzTUB0x1nBERXd7Ftcs=
go.opentelemetry.io/otel/sdk v1.39.0 h1:nMLYcjVsvdui1B/4FRkwjzoRVsMK8uL/cj0OyhKzt18=
go.opentelemetry.io/otel/sdk v1.39.0/go.mod h1:vDojkC4/jsTJsE+kh+LXYQlbL8CgrEcwmt1ENZszdJE=
go.opentelemetry.io/otel/sdk/metric v1.39.0 h1:cXMVVFVgsIf2YL6QkRF4Urbr/aMInf+2WKg+sEJTtB8=
go.opentelemetry.io/otel/sdk/metric v1.39.0/go.mod h1:xq9HEVH7qeX69/JnwEfp6fVq5wosJsY1mt4lLfYdVew=
go.opentelemetry.io/otel/trace v1.39.0 h1:2d2vfpEDmCJ5zVYz7ijaJdOF59xLomrvj7bjt6/qCJI=
go.opentelemetry.io/otel/trace v1.39.0/go.mod h1:88w4/PnZSazkGzz/w84VHpQafiU4EtqqlVdxWy+rNOA=
go.opentelemetry.io/proto/otlp v1.0.0 h1:T0TX0tmXU8a3CbNXzEKGeU5mIVOdf0oykP+u2lIVU/I=
go.opentelemetry.io/proto/otlp v1.0.0/go.mod h1:Sy6pihPLfYHkr3NkUbEhGHFhINUSI/v80hjKIs5JXpM=
go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
go.yaml.in/yaml/v2 v2.4.3 h1:6gvOSjQoTB3vt1l+CU+tSyi/HOjfOjRLJ4YwYZGwRO0=
@@ -165,59 +284,51 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk
golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.46.0 h1:cKRW/pmt1pKAfetfu+RCEvjvZkA9RimPbh7bhFjGVBU=
golang.org/x/crypto v0.46.0/go.mod h1:Evb/oLKmMraqjZ2iQTwDwvCtJkczlDuTmdJXoZVzqU0=
golang.org/x/exp v0.0.0-20251219203646-944ab1f22d93 h1:fQsdNF2N+/YewlRZiricy4P1iimyPKZ/xwniHj8Q2a0=
golang.org/x/exp v0.0.0-20251219203646-944ab1f22d93/go.mod h1:EPRbTFwzwjXj9NpYyyrvenVh9Y+GFeEvMNh7Xuz7xgU=
golang.org/x/mod v0.31.0 h1:HaW9xtz0+kOcWKwli0ZXy79Ix+UW/vOfmWI5QVd2tgI=
golang.org/x/mod v0.31.0/go.mod h1:43JraMp9cGx1Rx3AqioxrbrhNsLl2l/iNAvuBkrezpg=
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY=
golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU=
golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4=
golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.39.0 h1:CvCKL8MeisomCi6qNZ+wbb0DN9E5AATixKsvNtMoMFk=
golang.org/x/sys v0.39.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
golang.org/x/term v0.38.0 h1:PQ5pkm/rLO6HnxFR7N2lJHOZX6Kez5Y1gDSJla6jo7Q=
golang.org/x/term v0.38.0/go.mod h1:bSEAKrOT1W+VSu9TSCMtoGEOUcKxOKgl3LE5QEF/xVg=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.32.0 h1:ZD01bjUt1FQ9WJ0ClOL5vxgxOI/sVCNgX1YtKwcY0mU=
golang.org/x/text v0.32.0/go.mod h1:o/rUWzghvpD5TXrTIBuJU77MTaN0ljMWE47kxGJQ7jY=
golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI=
golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4=
golang.org/x/tools v0.40.0 h1:yLkxfA+Qnul4cs9QA3KnlFu0lVmd8JJfoq+E41uSutA=
golang.org/x/tools v0.40.0/go.mod h1:Ik/tzLRlbscWpqqMRjyWYDisX8bG13FrdXp3o4Sr9lc=
google.golang.org/genproto/googleapis/api v0.0.0-20251222181119-0a764e51fe1b h1:uA40e2M6fYRBf0+8uN5mLlqUtV192iiksiICIBkYJ1E=
google.golang.org/genproto/googleapis/api v0.0.0-20251222181119-0a764e51fe1b/go.mod h1:Xa7le7qx2vmqB/SzWUBa7KdMjpdpAHlh5QCSnjessQk=
google.golang.org/genproto/googleapis/rpc v0.0.0-20251222181119-0a764e51fe1b h1:Mv8VFug0MP9e5vUxfBcE3vUkV6CImK3cMNMIDFjmzxU=
google.golang.org/genproto/googleapis/rpc v0.0.0-20251222181119-0a764e51fe1b/go.mod h1:j9x/tPzZkyxcgEFkiKEEGxfvyumM01BEtsW8xzOahRQ=
google.golang.org/grpc v1.78.0 h1:K1XZG/yGDJnzMdd/uZHAkVqJE+xIDOcmdSFZkBUicNc=
google.golang.org/grpc v1.78.0/go.mod h1:I47qjTo4OKbMkjA/aOOwxDIiPSBofUtQUI5EfpWvW7U=
google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE=
google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
modernc.org/cc/v4 v4.27.1 h1:9W30zRlYrefrDV2JE2O8VDtJ1yPGownxciz5rrbQZis=
modernc.org/cc/v4 v4.27.1/go.mod h1:uVtb5OGqUKpoLWhqwNQo/8LwvoiEBLvZXIQ/SmO6mL0=
modernc.org/ccgo/v4 v4.30.1 h1:4r4U1J6Fhj98NKfSjnPUN7Ze2c6MnAdL0hWw6+LrJpc=
modernc.org/ccgo/v4 v4.30.1/go.mod h1:bIOeI1JL54Utlxn+LwrFyjCx2n2RDiYEaJVSrgdrRfM=
modernc.org/fileutil v1.3.40 h1:ZGMswMNc9JOCrcrakF1HrvmergNLAmxOPjizirpfqBA=
modernc.org/fileutil v1.3.40/go.mod h1:HxmghZSZVAz/LXcMNwZPA/DRrQZEVP9VX0V4LQGQFOc=
modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI=
modernc.org/gc/v2 v2.6.5/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito=
modernc.org/gc/v3 v3.1.1 h1:k8T3gkXWY9sEiytKhcgyiZ2L0DTyCQ/nvX+LoCljoRE=
modernc.org/gc/v3 v3.1.1/go.mod h1:HFK/6AGESC7Ex+EZJhJ2Gni6cTaYpSMmU/cT9RmlfYY=
modernc.org/goabi0 v0.2.0 h1:HvEowk7LxcPd0eq6mVOAEMai46V+i7Jrj13t4AzuNks=
modernc.org/goabi0 v0.2.0/go.mod h1:CEFRnnJhKvWT1c1JTI3Avm+tgOWbkOu5oPA8eH8LnMI=
modernc.org/libc v1.67.4 h1:zZGmCMUVPORtKv95c2ReQN5VDjvkoRm9GWPTEPuvlWg=
modernc.org/libc v1.67.4/go.mod h1:QvvnnJ5P7aitu0ReNpVIEyesuhmDLQ8kaEoyMjIFZJA=
modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU=
modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg=
modernc.org/memory v1.11.0 h1:o4QC8aMQzmcwCK3t3Ux/ZHmwFPzE6hf2Y5LbkRs+hbI=
modernc.org/memory v1.11.0/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw=
modernc.org/opt v0.1.4 h1:2kNGMRiUjrp4LcaPuLY2PzUfqM/w9N23quVwhKt5Qm8=
modernc.org/opt v0.1.4/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns=
modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w=
modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE=
modernc.org/sqlite v1.42.2 h1:7hkZUNJvJFN2PgfUdjni9Kbvd4ef4mNLOu0B9FGxM74=
modernc.org/sqlite v1.42.2/go.mod h1:+VkC6v3pLOAE0A0uVucQEcbVW0I5nHCeDaBf+DpsQT8=
modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0=
modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A=
modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y=
modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM=
gorm.io/driver/mysql v1.6.0 h1:eNbLmNTpPpTOVZi8MMxCi2aaIm0ZpInbORNXDwyLGvg=
gorm.io/driver/mysql v1.6.0/go.mod h1:D/oCC2GWK3M/dqoLxnOlaNKmXz8WNTfcS9y5ovaSqKo=
gorm.io/driver/postgres v1.6.0 h1:2dxzU8xJ+ivvqTRph34QX+WrRaJlmfyPqXmoGVjMBa4=
gorm.io/driver/postgres v1.6.0/go.mod h1:vUw0mrGgrTK+uPHEhAdV4sfFELrByKVGnaVRkXDhtWo=
gorm.io/driver/sqlite v1.6.0 h1:WHRRrIiulaPiPFmDcod6prc4l2VGVWHz80KspNsxSfQ=
gorm.io/driver/sqlite v1.6.0/go.mod h1:AO9V1qIQddBESngQUKWL9yoH93HIeA1X6V633rBwyT8=
gorm.io/gorm v1.31.1 h1:7CA8FTFz/gRfgqgpeKIBcervUn3xSyPUmr6B2WXJ7kg=
gorm.io/gorm v1.31.1/go.mod h1:XyQVbO2k6YkOis7C2437jSit3SsDK72s7n7rsSHd+Gs=
gotest.tools/v3 v3.5.2 h1:7koQfIKdy+I8UTetycgUqXWSDwpgv193Ka+qRsmBY8Q=
gotest.tools/v3 v3.5.2/go.mod h1:LtdLGcnqToBH83WByAAi/wiwSFCArdFIUV/xxN4pcjA=
helm/gohoarder/templates/_helpers.tpl
+1 -1
View File
@@ -181,7 +181,7 @@ Validate SQLite configuration - SQLite cannot be used with SMB/NFS network stora
{{- if .Values.metadata.sqlite.persistence.enabled }}
{{- $storageClass := .Values.metadata.sqlite.persistence.storageClass | default .Values.storage.storageClass }}
{{- if or (contains "smb" ($storageClass | lower)) (contains "cifs" ($storageClass | lower)) (contains "nfs" ($storageClass | lower)) }}
{{- fail "\n\n❌ ERROR: SQLite cannot be used with SMB/CIFS/NFS network storage!\n\nSQLite requires POSIX file locking which is not reliably supported over network filesystems.\nThis will cause 'database is locked' errors and data corruption.\n\nPlease choose ONE of the following solutions:\n\n1. Use PostgreSQL for network storage (RECOMMENDED for production):\n metadata:\n backend: postgresql\n postgresql:\n host: your-postgres-host\n ...\n\n2. Use local storage for SQLite (OK for development):\n metadata:\n sqlite:\n persistence:\n enabled: true\n storageClass: local-path # or another local storage class\n\n3. Disable persistence (data will be lost on pod restart):\n metadata:\n sqlite:\n persistence:\n enabled: false\n\nFor more information, see: https://www.sqlite.org/lockingv3.html\n" }}
{{- fail "\n\n❌ ERROR: SQLite cannot be used with SMB/CIFS/NFS network storage!\n\nSQLite requires POSIX file locking which is not reliably supported over network filesystems.\nThis will cause 'database is locked' errors and data corruption.\n\nPlease choose ONE of the following solutions:\n\n1. Use PostgreSQL for network storage (RECOMMENDED for production):\n metadata:\n backend: postgresql\n postgresql:\n host: your-postgres-host\n ...\n\n2. Use MySQL/MariaDB for network storage (alternative to PostgreSQL):\n metadata:\n backend: mysql\n mysql:\n host: your-mysql-host\n ...\n\n3. Use local storage for SQLite (OK for development):\n metadata:\n sqlite:\n persistence:\n enabled: true\n storageClass: local-path # or another local storage class\n\n4. Disable persistence (data will be lost on pod restart):\n metadata:\n sqlite:\n persistence:\n enabled: false\n\nFor more information, see: https://www.sqlite.org/lockingv3.html\n" }}
{{- end }}
{{- end }}
{{- end }}
helm/gohoarder/templates/deployment-scanner.yaml
+117
View File
@@ -29,6 +29,77 @@ spec:
serviceAccountName: {{ include "gohoarder.serviceAccountName" . }}
securityContext:
{{- toYaml .Values.podSecurityContext | nindent 8 }}
{{- if .Values.migration.enabled }}
initContainers:
# Wait for database to be ready
- name: wait-for-db
image: busybox:1.36
command:
- sh
- -c
- |
echo "Waiting for database..."
{{- if eq .Values.metadata.backend "postgresql" }}
until nc -z {{ .Values.metadata.postgresql.host }} {{ .Values.metadata.postgresql.port }}; do
echo " PostgreSQL not ready, retrying in 2s..."
sleep 2
done
echo "✓ PostgreSQL is ready"
{{- else if eq .Values.metadata.backend "mysql" }}
until nc -z {{ .Values.metadata.mysql.host }} {{ .Values.metadata.mysql.port }}; do
echo " MySQL not ready, retrying in 2s..."
sleep 2
done
echo "✓ MySQL is ready"
{{- else }}
echo "✓ SQLite (no wait needed)"
{{- end }}
securityContext:
allowPrivilegeEscalation: false
runAsNonRoot: true
runAsUser: 1000
resources:
limits:
cpu: 100m
memory: 64Mi
requests:
cpu: 10m
memory: 32Mi
# Run database migrations
- name: migrate
image: "{{ .Values.migration.image.repository }}:{{ .Values.migration.image.tag | default .Chart.AppVersion }}"
imagePullPolicy: {{ .Values.migration.image.pullPolicy }}
env:
- name: DB_DRIVER
value: {{ .Values.metadata.backend | quote }}
{{- if eq .Values.metadata.backend "postgresql" }}
- name: DATABASE_URL
value: "postgresql://{{ .Values.metadata.postgresql.username }}:{{ .Values.metadata.postgresql.password }}@{{ .Values.metadata.postgresql.host }}:{{ .Values.metadata.postgresql.port }}/{{ .Values.metadata.postgresql.database }}?sslmode={{ .Values.metadata.postgresql.sslMode }}"
{{- else if eq .Values.metadata.backend "mysql" }}
- name: DATABASE_URL
value: "{{ .Values.metadata.mysql.username }}:{{ .Values.metadata.mysql.password }}@tcp({{ .Values.metadata.mysql.host }}:{{ .Values.metadata.mysql.port }})/{{ .Values.metadata.mysql.database }}?charset={{ .Values.metadata.mysql.charset }}&parseTime={{ .Values.metadata.mysql.parseTime }}"
{{- else }}
- name: DATABASE_URL
value: "/var/lib/gohoarder/metadata/gohoarder.db"
{{- end }}
args:
- --driver=$(DB_DRIVER)
- --dsn=$(DATABASE_URL)
- --action=migrate
- --log-level={{ .Values.migration.logLevel | default "info" }}
- --timeout={{ .Values.migration.timeout | default "5m" }}
securityContext:
allowPrivilegeEscalation: false
runAsNonRoot: true
runAsUser: 1000
resources:
{{- toYaml .Values.migration.resources | nindent 10 }}
{{- if eq .Values.metadata.backend "sqlite" }}
volumeMounts:
- name: metadata
mountPath: /var/lib/gohoarder/metadata
{{- end }}
{{- end }}
containers:
- name: scanner
securityContext:
@@ -38,6 +109,52 @@ spec:
env:
- name: CONFIG_FILE
value: /etc/gohoarder/config.yaml
{{- if and (eq .Values.metadata.backend "postgresql") .Values.metadata.postgresql.existingSecret }}
- name: POSTGRES_USER
valueFrom:
secretKeyRef:
name: {{ .Values.metadata.postgresql.existingSecret }}
key: username
- name: POSTGRES_PASSWORD
valueFrom:
secretKeyRef:
name: {{ .Values.metadata.postgresql.existingSecret }}
key: password
{{- else if and (eq .Values.metadata.backend "postgresql") .Values.metadata.postgresql.username }}
- name: POSTGRES_USER
valueFrom:
secretKeyRef:
name: {{ include "gohoarder.fullname" . }}-postgresql
key: username
- name: POSTGRES_PASSWORD
valueFrom:
secretKeyRef:
name: {{ include "gohoarder.fullname" . }}-postgresql
key: password
{{- end }}
{{- if and (or (eq .Values.metadata.backend "mysql") (eq .Values.metadata.backend "mariadb")) .Values.metadata.mysql.existingSecret }}
- name: MYSQL_USER
valueFrom:
secretKeyRef:
name: {{ .Values.metadata.mysql.existingSecret }}
key: username
- name: MYSQL_PASSWORD
valueFrom:
secretKeyRef:
name: {{ .Values.metadata.mysql.existingSecret }}
key: password
{{- else if and (or (eq .Values.metadata.backend "mysql") (eq .Values.metadata.backend "mariadb")) .Values.metadata.mysql.username }}
- name: MYSQL_USER
valueFrom:
secretKeyRef:
name: {{ include "gohoarder.fullname" . }}-mysql
key: username
- name: MYSQL_PASSWORD
valueFrom:
secretKeyRef:
name: {{ include "gohoarder.fullname" . }}-mysql
key: password
{{- end }}
{{- if and .Values.security.scanners.ghsa.enabled .Values.security.scanners.ghsa.existingSecret }}
- name: GHSA_TOKEN
valueFrom:
helm/gohoarder/templates/deployment-server.yaml
+94
View File
@@ -30,6 +30,77 @@ spec:
serviceAccountName: {{ include "gohoarder.serviceAccountName" . }}
securityContext:
{{- toYaml .Values.podSecurityContext | nindent 8 }}
{{- if .Values.migration.enabled }}
initContainers:
# Wait for database to be ready
- name: wait-for-db
image: busybox:1.36
command:
- sh
- -c
- |
echo "Waiting for database..."
{{- if eq .Values.metadata.backend "postgresql" }}
until nc -z {{ .Values.metadata.postgresql.host }} {{ .Values.metadata.postgresql.port }}; do
echo " PostgreSQL not ready, retrying in 2s..."
sleep 2
done
echo "✓ PostgreSQL is ready"
{{- else if eq .Values.metadata.backend "mysql" }}
until nc -z {{ .Values.metadata.mysql.host }} {{ .Values.metadata.mysql.port }}; do
echo " MySQL not ready, retrying in 2s..."
sleep 2
done
echo "✓ MySQL is ready"
{{- else }}
echo "✓ SQLite (no wait needed)"
{{- end }}
securityContext:
allowPrivilegeEscalation: false
runAsNonRoot: true
runAsUser: 1000
resources:
limits:
cpu: 100m
memory: 64Mi
requests:
cpu: 10m
memory: 32Mi
# Run database migrations
- name: migrate
image: "{{ .Values.migration.image.repository }}:{{ .Values.migration.image.tag | default .Chart.AppVersion }}"
imagePullPolicy: {{ .Values.migration.image.pullPolicy }}
env:
- name: DB_DRIVER
value: {{ .Values.metadata.backend | quote }}
{{- if eq .Values.metadata.backend "postgresql" }}
- name: DATABASE_URL
value: "postgresql://{{ .Values.metadata.postgresql.username }}:{{ .Values.metadata.postgresql.password }}@{{ .Values.metadata.postgresql.host }}:{{ .Values.metadata.postgresql.port }}/{{ .Values.metadata.postgresql.database }}?sslmode={{ .Values.metadata.postgresql.sslMode }}"
{{- else if eq .Values.metadata.backend "mysql" }}
- name: DATABASE_URL
value: "{{ .Values.metadata.mysql.username }}:{{ .Values.metadata.mysql.password }}@tcp({{ .Values.metadata.mysql.host }}:{{ .Values.metadata.mysql.port }})/{{ .Values.metadata.mysql.database }}?charset={{ .Values.metadata.mysql.charset }}&parseTime={{ .Values.metadata.mysql.parseTime }}"
{{- else }}
- name: DATABASE_URL
value: "/var/lib/gohoarder/metadata/gohoarder.db"
{{- end }}
args:
- --driver=$(DB_DRIVER)
- --dsn=$(DATABASE_URL)
- --action=migrate
- --log-level={{ .Values.migration.logLevel | default "info" }}
- --timeout={{ .Values.migration.timeout | default "5m" }}
securityContext:
allowPrivilegeEscalation: false
runAsNonRoot: true
runAsUser: 1000
resources:
{{- toYaml .Values.migration.resources | nindent 10 }}
{{- if eq .Values.metadata.backend "sqlite" }}
volumeMounts:
- name: metadata
mountPath: /var/lib/gohoarder/metadata
{{- end }}
{{- end }}
containers:
- name: server
securityContext:
@@ -125,6 +196,29 @@ spec:
name: {{ include "gohoarder.fullname" . }}-postgresql
key: password
{{- end }}
{{- if and (or (eq .Values.metadata.backend "mysql") (eq .Values.metadata.backend "mariadb")) .Values.metadata.mysql.existingSecret }}
- name: MYSQL_USER
valueFrom:
secretKeyRef:
name: {{ .Values.metadata.mysql.existingSecret }}
key: username
- name: MYSQL_PASSWORD
valueFrom:
secretKeyRef:
name: {{ .Values.metadata.mysql.existingSecret }}
key: password
{{- else if and (or (eq .Values.metadata.backend "mysql") (eq .Values.metadata.backend "mariadb")) .Values.metadata.mysql.username }}
- name: MYSQL_USER
valueFrom:
secretKeyRef:
name: {{ include "gohoarder.fullname" . }}-mysql
key: username
- name: MYSQL_PASSWORD
valueFrom:
secretKeyRef:
name: {{ include "gohoarder.fullname" . }}-mysql
key: password
{{- end }}
{{- if and .Values.security.scanners.ghsa.enabled .Values.security.scanners.ghsa.existingSecret }}
- name: GHSA_TOKEN
valueFrom:
helm/gohoarder/templates/secret.yaml
+13
View File
@@ -53,6 +53,19 @@ data:
password: {{ .Values.metadata.postgresql.password | b64enc | quote }}
{{- end }}
---
{{- if and (or (eq .Values.metadata.backend "mysql") (eq .Values.metadata.backend "mariadb")) (not .Values.metadata.mysql.existingSecret) .Values.metadata.mysql.username }}
apiVersion: v1
kind: Secret
metadata:
name: {{ include "gohoarder.fullname" . }}-mysql
labels:
{{- include "gohoarder.labels" . | nindent 4 }}
type: Opaque
data:
username: {{ .Values.metadata.mysql.username | b64enc | quote }}
password: {{ .Values.metadata.mysql.password | b64enc | quote }}
{{- end }}
---
{{- if and .Values.security.scanners.ghsa.enabled (not .Values.security.scanners.ghsa.existingSecret) .Values.security.scanners.ghsa.token }}
apiVersion: v1
kind: Secret
helm/gohoarder/values.yaml
+58 -2
View File
@@ -272,7 +272,7 @@ storage:
# Metadata storage configuration
metadata:
# Backend: sqlite, postgresql
# Backend: sqlite, postgresql, mysql
#
# IMPORTANT: SQLite CANNOT be used with SMB/CIFS/NFS network storage!
# SQLite requires POSIX file locking which causes "database is locked" errors on network filesystems.
@@ -286,6 +286,13 @@ metadata:
# 2. PostgreSQL with any storage (RECOMMENDED for production)
# - Set backend: postgresql
# - Configure postgresql settings below
# - Works with any storage including SMB/NFS
# - Supports multiple replicas and high availability
#
# 3. MySQL/MariaDB with any storage (alternative to PostgreSQL)
# - Set backend: mysql
# - Configure mysql settings below
# - Works with any storage including SMB/NFS
#
backend: "sqlite"
@@ -305,6 +312,8 @@ metadata:
walMode: false
# PostgreSQL configuration
# Works with any storage including SMB/NFS
# Recommended for production deployments
postgresql:
# Use bundled PostgreSQL (sets up postgresql subchart)
enabled: false
@@ -313,10 +322,57 @@ metadata:
database: "gohoarder"
username: "gohoarder"
password: ""
sslMode: "disable"
sslMode: "disable" # disable, require, verify-ca, verify-full
# Use existing secret for PostgreSQL credentials
existingSecret: ""
# MySQL/MariaDB configuration
# Works with any storage including SMB/NFS
# Alternative to PostgreSQL for production deployments
mysql:
host: "localhost"
port: 3306
database: "gohoarder"
username: "gohoarder"
password: ""
charset: "utf8mb4"
parseTime: true
# Use existing secret for MySQL credentials
existingSecret: ""
# GORM connection pool settings (applies to all database backends)
# These settings control database connection pooling and performance
maxOpenConns: 25 # Maximum number of open connections to the database
maxIdleConns: 5 # Maximum number of idle connections in the pool
connMaxLifetime: 3600 # Maximum lifetime of a connection in seconds (1 hour)
logLevel: "warn" # GORM log level: silent, error, warn, info
# Database migration configuration
migration:
# Enable automatic database migrations via init containers
# When enabled, each pod will run migrations before starting the main container
# Gormigrate handles concurrency automatically - safe for multiple pods
enabled: true
# Migration image configuration
image:
repository: ghcr.io/lukaszraczylo/gohoarder-migrate
pullPolicy: IfNotPresent
tag: "latest" # Should match the application version
# Migration settings
logLevel: "info" # debug, info, warn, error
timeout: "5m" # Maximum time for migrations to complete
# Resource limits for migration init container
resources:
limits:
cpu: 500m
memory: 256Mi
requests:
cpu: 100m
memory: 128Mi
# Cache configuration
cache:
defaultTTL: "168h" # 7 days
migrate
Executable
BIN
View File
Binary file not shown.
migrations/001_create_schema_v2_mysql.sql
+367
View File
@@ -0,0 +1,367 @@
-- GoHoarder Database Schema V2 - MySQL/MariaDB
-- Optimized for multi-user production deployments
-- Created: 2026-01-03
-- Requires: MySQL 8.0+ or MariaDB 10.5+
-- Set charset and collation
SET NAMES utf8mb4;
SET CHARACTER SET utf8mb4;
-- ============================================================================
-- TABLE: registries
-- Purpose: Normalized registry data (eliminates repeated strings)
-- ============================================================================
CREATE TABLE IF NOT EXISTS registries (
id INT UNSIGNED AUTO_INCREMENT PRIMARY KEY,
name VARCHAR(50) UNIQUE NOT NULL,
display_name VARCHAR(100) NOT NULL,
upstream_url VARCHAR(512) NOT NULL,
enabled BOOLEAN NOT NULL DEFAULT TRUE,
scan_by_default BOOLEAN NOT NULL DEFAULT TRUE,
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
deleted_at TIMESTAMP NULL,
INDEX idx_registry_name (name),
INDEX idx_registry_enabled (enabled, deleted_at)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
-- ============================================================================
-- TABLE: packages
-- Purpose: Core package metadata with denormalized counts for performance
-- ============================================================================
CREATE TABLE IF NOT EXISTS packages (
id BIGINT UNSIGNED AUTO_INCREMENT PRIMARY KEY,
registry_id INT UNSIGNED NOT NULL,
name VARCHAR(255) NOT NULL,
version VARCHAR(100) NOT NULL,
-- Storage information
storage_key VARCHAR(512) UNIQUE NOT NULL,
size BIGINT NOT NULL,
checksum_md5 CHAR(32),
checksum_sha256 CHAR(64),
upstream_url VARCHAR(1024),
-- Cache management
cached_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
last_accessed TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
expires_at TIMESTAMP NULL,
access_count BIGINT NOT NULL DEFAULT 0,
-- Security (denormalized for performance)
security_scanned BOOLEAN NOT NULL DEFAULT FALSE,
last_scanned_at TIMESTAMP NULL,
vulnerability_count INT NOT NULL DEFAULT 0,
highest_severity VARCHAR(20),
-- Authentication
requires_auth BOOLEAN NOT NULL DEFAULT FALSE,
auth_provider VARCHAR(50),
-- Audit trail
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
deleted_at TIMESTAMP NULL,
FOREIGN KEY (registry_id) REFERENCES registries(id) ON DELETE RESTRICT ON UPDATE CASCADE,
UNIQUE INDEX idx_package_registry_name_version (registry_id, name, version, deleted_at),
INDEX idx_package_storage_key (storage_key),
INDEX idx_package_name (name(50)),
INDEX idx_package_last_accessed (last_accessed DESC),
INDEX idx_package_expires_at (expires_at),
INDEX idx_package_access_count (access_count DESC),
INDEX idx_package_size (size DESC),
INDEX idx_package_vuln_count (vulnerability_count),
INDEX idx_package_severity (highest_severity),
INDEX idx_package_security_scanned (security_scanned, deleted_at)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
-- ============================================================================
-- TABLE: package_metadata
-- Purpose: Structured metadata (1:1 with packages)
-- ============================================================================
CREATE TABLE IF NOT EXISTS package_metadata (
package_id BIGINT UNSIGNED PRIMARY KEY,
author VARCHAR(255),
license VARCHAR(100),
homepage VARCHAR(512),
repository VARCHAR(512),
description TEXT,
keywords JSON, -- JSON array for MySQL 8.0+
raw_metadata JSON, -- Full metadata as JSON
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
deleted_at TIMESTAMP NULL,
FOREIGN KEY (package_id) REFERENCES packages(id) ON DELETE CASCADE ON UPDATE CASCADE,
INDEX idx_metadata_author (author(100)),
INDEX idx_metadata_license (license)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
-- ============================================================================
-- TABLE: vulnerabilities
-- Purpose: Normalized vulnerability data (each CVE stored once)
-- ============================================================================
CREATE TABLE IF NOT EXISTS vulnerabilities (
id BIGINT UNSIGNED AUTO_INCREMENT PRIMARY KEY,
cve_id VARCHAR(50) UNIQUE NOT NULL,
title VARCHAR(512) NOT NULL,
description TEXT,
severity VARCHAR(20) NOT NULL,
cvss FLOAT,
published_at TIMESTAMP NOT NULL,
fixed_version VARCHAR(100),
references JSON,
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
deleted_at TIMESTAMP NULL,
UNIQUE INDEX idx_vuln_cve_id (cve_id),
INDEX idx_vuln_severity (severity),
INDEX idx_vuln_cvss (cvss DESC),
INDEX idx_vuln_published (published_at DESC)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
-- ============================================================================
-- TABLE: package_vulnerabilities
-- Purpose: Many-to-many relationship between packages and vulnerabilities
-- ============================================================================
CREATE TABLE IF NOT EXISTS package_vulnerabilities (
id BIGINT UNSIGNED AUTO_INCREMENT PRIMARY KEY,
package_id BIGINT UNSIGNED NOT NULL,
vulnerability_id BIGINT UNSIGNED NOT NULL,
scanner VARCHAR(50) NOT NULL,
detected_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
bypassed BOOLEAN NOT NULL DEFAULT FALSE,
bypass_id BIGINT UNSIGNED,
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
deleted_at TIMESTAMP NULL,
FOREIGN KEY (package_id) REFERENCES packages(id) ON DELETE CASCADE ON UPDATE CASCADE,
FOREIGN KEY (vulnerability_id) REFERENCES vulnerabilities(id) ON DELETE CASCADE ON UPDATE CASCADE,
INDEX idx_pkg_vuln_package (package_id, deleted_at),
INDEX idx_pkg_vuln_vuln (vulnerability_id, deleted_at),
INDEX idx_pkg_vuln_composite (package_id, vulnerability_id, deleted_at),
INDEX idx_pkg_vuln_scanner (scanner),
INDEX idx_pkg_vuln_bypassed (bypassed)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
-- ============================================================================
-- TABLE: scan_results
-- Purpose: Security scan results with severity breakdown
-- ============================================================================
CREATE TABLE IF NOT EXISTS scan_results (
id BIGINT UNSIGNED AUTO_INCREMENT PRIMARY KEY,
package_id BIGINT UNSIGNED NOT NULL,
scanner VARCHAR(50) NOT NULL,
scanned_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
status VARCHAR(20) NOT NULL,
vuln_count INT NOT NULL DEFAULT 0,
critical_count INT NOT NULL DEFAULT 0,
high_count INT NOT NULL DEFAULT 0,
medium_count INT NOT NULL DEFAULT 0,
low_count INT NOT NULL DEFAULT 0,
scan_duration INT NOT NULL DEFAULT 0,
details JSON,
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
deleted_at TIMESTAMP NULL,
FOREIGN KEY (package_id) REFERENCES packages(id) ON DELETE CASCADE ON UPDATE CASCADE,
INDEX idx_scan_package_scanner (package_id, scanner, deleted_at),
INDEX idx_scan_scanned_at (scanned_at DESC),
INDEX idx_scan_status (status),
INDEX idx_scan_vuln_count (vuln_count)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
-- ============================================================================
-- TABLE: cve_bypasses
-- Purpose: CVE bypass rules with usage tracking
-- ============================================================================
CREATE TABLE IF NOT EXISTS cve_bypasses (
id BIGINT UNSIGNED AUTO_INCREMENT PRIMARY KEY,
type VARCHAR(20) NOT NULL,
target VARCHAR(512) NOT NULL,
reason TEXT NOT NULL,
created_by VARCHAR(255) NOT NULL,
expires_at TIMESTAMP NOT NULL,
notify_on_expiry BOOLEAN NOT NULL DEFAULT FALSE,
active BOOLEAN NOT NULL DEFAULT TRUE,
usage_count BIGINT NOT NULL DEFAULT 0,
last_used_at TIMESTAMP NULL,
registry_id INT UNSIGNED,
package_id BIGINT UNSIGNED,
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
deleted_at TIMESTAMP NULL,
FOREIGN KEY (registry_id) REFERENCES registries(id) ON DELETE SET NULL ON UPDATE CASCADE,
FOREIGN KEY (package_id) REFERENCES packages(id) ON DELETE SET NULL ON UPDATE CASCADE,
INDEX idx_bypass_type (type),
INDEX idx_bypass_target (target(100)),
INDEX idx_bypass_active (active, deleted_at),
INDEX idx_bypass_expires_at (expires_at, active),
INDEX idx_bypass_created_by (created_by(100))
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
-- ============================================================================
-- TABLE: download_events
-- Purpose: High-volume time-series data
-- Note: MySQL doesn't support native partitioning as elegantly as PostgreSQL
-- Consider manual partitioning or TimescaleDB if needed
-- ============================================================================
CREATE TABLE IF NOT EXISTS download_events (
id BIGINT UNSIGNED AUTO_INCREMENT PRIMARY KEY,
package_id BIGINT UNSIGNED NOT NULL,
registry_id INT UNSIGNED NOT NULL,
downloaded_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
user_agent VARCHAR(512),
ip_address VARCHAR(45),
authenticated BOOLEAN NOT NULL DEFAULT FALSE,
username VARCHAR(255),
INDEX idx_download_events_package (package_id, downloaded_at),
INDEX idx_download_events_registry (registry_id),
INDEX idx_download_events_time (downloaded_at)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
-- ============================================================================
-- TABLE: download_stats_hourly
-- Purpose: Pre-aggregated hourly statistics
-- ============================================================================
CREATE TABLE IF NOT EXISTS download_stats_hourly (
id BIGINT UNSIGNED AUTO_INCREMENT PRIMARY KEY,
registry_id INT UNSIGNED NOT NULL,
package_id BIGINT UNSIGNED,
time_bucket TIMESTAMP NOT NULL,
download_count BIGINT NOT NULL DEFAULT 0,
unique_ips BIGINT NOT NULL DEFAULT 0,
auth_downloads BIGINT NOT NULL DEFAULT 0,
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
FOREIGN KEY (registry_id) REFERENCES registries(id) ON DELETE CASCADE ON UPDATE CASCADE,
FOREIGN KEY (package_id) REFERENCES packages(id) ON DELETE CASCADE ON UPDATE CASCADE,
UNIQUE INDEX idx_stats_hourly_composite (registry_id, IFNULL(package_id, 0), time_bucket),
INDEX idx_stats_hourly_time (time_bucket DESC)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
-- ============================================================================
-- TABLE: download_stats_daily
-- Purpose: Pre-aggregated daily statistics
-- ============================================================================
CREATE TABLE IF NOT EXISTS download_stats_daily (
id BIGINT UNSIGNED AUTO_INCREMENT PRIMARY KEY,
registry_id INT UNSIGNED NOT NULL,
package_id BIGINT UNSIGNED,
time_bucket TIMESTAMP NOT NULL,
download_count BIGINT NOT NULL DEFAULT 0,
unique_ips BIGINT NOT NULL DEFAULT 0,
auth_downloads BIGINT NOT NULL DEFAULT 0,
top_user_agents JSON,
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
FOREIGN KEY (registry_id) REFERENCES registries(id) ON DELETE CASCADE ON UPDATE CASCADE,
FOREIGN KEY (package_id) REFERENCES packages(id) ON DELETE CASCADE ON UPDATE CASCADE,
UNIQUE INDEX idx_stats_daily_composite (registry_id, IFNULL(package_id, 0), time_bucket),
INDEX idx_stats_daily_time (time_bucket DESC)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
-- ============================================================================
-- TABLE: audit_log
-- Purpose: Audit trail for compliance
-- ============================================================================
CREATE TABLE IF NOT EXISTS audit_log (
id BIGINT UNSIGNED AUTO_INCREMENT PRIMARY KEY,
entity_type VARCHAR(50) NOT NULL,
entity_id BIGINT NOT NULL,
action VARCHAR(20) NOT NULL,
username VARCHAR(255) NOT NULL,
timestamp TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
changes JSON,
ip_address VARCHAR(45),
user_agent VARCHAR(512),
INDEX idx_audit_log_entity (entity_type, entity_id),
INDEX idx_audit_log_username (username(100)),
INDEX idx_audit_log_timestamp (timestamp DESC)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
-- ============================================================================
-- SEED DATA: Default registries
-- ============================================================================
INSERT INTO registries (name, display_name, upstream_url, enabled, scan_by_default) VALUES
('npm', 'NPM Registry', 'https://registry.npmjs.org', TRUE, TRUE),
('pypi', 'PyPI', 'https://pypi.org', TRUE, TRUE),
('go', 'Go Modules', 'https://proxy.golang.org', TRUE, TRUE)
ON DUPLICATE KEY UPDATE
display_name = VALUES(display_name),
upstream_url = VALUES(upstream_url);
-- ============================================================================
-- VIEWS: Convenience views for common queries
-- ============================================================================
CREATE OR REPLACE VIEW v_vulnerable_packages AS
SELECT
r.name AS registry,
p.name,
p.version,
p.vulnerability_count,
p.highest_severity,
p.last_scanned_at
FROM packages p
JOIN registries r ON p.registry_id = r.id
WHERE p.vulnerability_count > 0 AND p.deleted_at IS NULL
ORDER BY
CASE p.highest_severity
WHEN 'critical' THEN 1
WHEN 'high' THEN 2
WHEN 'medium' THEN 3
WHEN 'low' THEN 4
ELSE 5
END,
p.vulnerability_count DESC;
-- ============================================================================
-- PERFORMANCE TUNING RECOMMENDATIONS
-- ============================================================================
-- Set InnoDB buffer pool size to 50-70% of RAM
-- SET GLOBAL innodb_buffer_pool_size = 4294967296; -- 4GB
-- Enable query cache (MySQL 5.7 and earlier)
-- SET GLOBAL query_cache_type = 1;
-- SET GLOBAL query_cache_size = 67108864; -- 64MB
-- Optimize for SSD
-- SET GLOBAL innodb_flush_log_at_trx_commit = 2;
-- SET GLOBAL innodb_io_capacity = 2000;
-- ============================================================================
-- COMPLETE
-- ============================================================================
SELECT 'Schema V2 created successfully for MySQL/MariaDB!' AS status;
migrations/001_create_schema_v2_postgres.sql
+470
View File
@@ -0,0 +1,470 @@
-- GoHoarder Database Schema V2 - PostgreSQL
-- Optimized for multi-user production deployments
-- Created: 2026-01-03
-- Enable required extensions
CREATE EXTENSION IF NOT EXISTS "uuid-ossp";
-- ============================================================================
-- TABLE: registries
-- Purpose: Normalized registry data (eliminates repeated strings)
-- ============================================================================
CREATE TABLE IF NOT EXISTS registries (
id SERIAL PRIMARY KEY,
name VARCHAR(50) UNIQUE NOT NULL,
display_name VARCHAR(100) NOT NULL,
upstream_url VARCHAR(512) NOT NULL,
enabled BOOLEAN NOT NULL DEFAULT TRUE,
scan_by_default BOOLEAN NOT NULL DEFAULT TRUE,
created_at TIMESTAMP NOT NULL DEFAULT NOW(),
updated_at TIMESTAMP NOT NULL DEFAULT NOW(),
deleted_at TIMESTAMP
);
CREATE INDEX idx_registry_name ON registries(name) WHERE deleted_at IS NULL;
CREATE INDEX idx_registry_enabled ON registries(enabled) WHERE enabled = TRUE AND deleted_at IS NULL;
COMMENT ON TABLE registries IS 'Normalized registry data (npm, pypi, go)';
COMMENT ON COLUMN registries.name IS 'Short name: npm, pypi, go';
COMMENT ON COLUMN registries.display_name IS 'Human-readable name: NPM Registry, PyPI';
-- ============================================================================
-- TABLE: packages
-- Purpose: Core package metadata with denormalized counts for performance
-- ============================================================================
CREATE TABLE IF NOT EXISTS packages (
id BIGSERIAL PRIMARY KEY,
registry_id INTEGER NOT NULL REFERENCES registries(id) ON DELETE RESTRICT,
name VARCHAR(255) NOT NULL,
version VARCHAR(100) NOT NULL,
-- Storage information
storage_key VARCHAR(512) UNIQUE NOT NULL,
size BIGINT NOT NULL,
checksum_md5 VARCHAR(32),
checksum_sha256 VARCHAR(64),
upstream_url VARCHAR(1024),
-- Cache management
cached_at TIMESTAMP NOT NULL DEFAULT NOW(),
last_accessed TIMESTAMP NOT NULL DEFAULT NOW(),
expires_at TIMESTAMP,
access_count BIGINT NOT NULL DEFAULT 0,
-- Security (denormalized for performance)
security_scanned BOOLEAN NOT NULL DEFAULT FALSE,
last_scanned_at TIMESTAMP,
vulnerability_count INTEGER NOT NULL DEFAULT 0,
highest_severity VARCHAR(20), -- critical, high, medium, low, none
-- Authentication
requires_auth BOOLEAN NOT NULL DEFAULT FALSE,
auth_provider VARCHAR(50),
-- Audit trail
created_at TIMESTAMP NOT NULL DEFAULT NOW(),
updated_at TIMESTAMP NOT NULL DEFAULT NOW(),
deleted_at TIMESTAMP
);
-- Composite indexes for common queries
CREATE UNIQUE INDEX idx_package_registry_name_version
ON packages(registry_id, name, version) WHERE deleted_at IS NULL;
CREATE INDEX idx_package_storage_key ON packages(storage_key);
CREATE INDEX idx_package_name ON packages(name text_pattern_ops) WHERE deleted_at IS NULL;
CREATE INDEX idx_package_last_accessed ON packages(last_accessed DESC) WHERE deleted_at IS NULL;
CREATE INDEX idx_package_expires_at ON packages(expires_at) WHERE expires_at IS NOT NULL AND deleted_at IS NULL;
CREATE INDEX idx_package_access_count ON packages(access_count DESC) WHERE deleted_at IS NULL;
CREATE INDEX idx_package_size ON packages(size DESC);
-- Partial indexes for security queries
CREATE INDEX idx_package_vuln_count ON packages(vulnerability_count) WHERE vulnerability_count > 0 AND deleted_at IS NULL;
CREATE INDEX idx_package_severity ON packages(highest_severity) WHERE highest_severity IN ('critical', 'high') AND deleted_at IS NULL;
CREATE INDEX idx_package_security_scanned ON packages(security_scanned) WHERE deleted_at IS NULL;
COMMENT ON TABLE packages IS 'Core package metadata (optimized V2 schema)';
COMMENT ON COLUMN packages.access_count IS 'Total downloads (denormalized from stats)';
COMMENT ON COLUMN packages.vulnerability_count IS 'Number of vulnerabilities (denormalized)';
-- ============================================================================
-- TABLE: package_metadata
-- Purpose: Structured metadata (1:1 with packages, reduces main table size)
-- ============================================================================
CREATE TABLE IF NOT EXISTS package_metadata (
package_id BIGINT PRIMARY KEY REFERENCES packages(id) ON DELETE CASCADE,
author VARCHAR(255),
license VARCHAR(100),
homepage VARCHAR(512),
repository VARCHAR(512),
description TEXT,
keywords JSONB, -- Array of keywords
raw_metadata JSONB, -- Full metadata
created_at TIMESTAMP NOT NULL DEFAULT NOW(),
updated_at TIMESTAMP NOT NULL DEFAULT NOW(),
deleted_at TIMESTAMP
);
CREATE INDEX idx_metadata_author ON package_metadata(author);
CREATE INDEX idx_metadata_license ON package_metadata(license);
CREATE INDEX idx_metadata_keywords ON package_metadata USING GIN(keywords);
CREATE INDEX idx_metadata_raw ON package_metadata USING GIN(raw_metadata);
COMMENT ON TABLE package_metadata IS 'Structured package metadata (separated for performance)';
-- ============================================================================
-- TABLE: vulnerabilities
-- Purpose: Normalized vulnerability data (each CVE stored once)
-- ============================================================================
CREATE TABLE IF NOT EXISTS vulnerabilities (
id BIGSERIAL PRIMARY KEY,
cve_id VARCHAR(50) UNIQUE NOT NULL,
title VARCHAR(512) NOT NULL,
description TEXT,
severity VARCHAR(20) NOT NULL, -- critical, high, medium, low
cvss REAL,
published_at TIMESTAMP NOT NULL,
fixed_version VARCHAR(100),
references JSONB, -- Array of URLs
created_at TIMESTAMP NOT NULL DEFAULT NOW(),
updated_at TIMESTAMP NOT NULL DEFAULT NOW(),
deleted_at TIMESTAMP
);
CREATE UNIQUE INDEX idx_vuln_cve_id ON vulnerabilities(cve_id);
CREATE INDEX idx_vuln_severity ON vulnerabilities(severity);
CREATE INDEX idx_vuln_cvss ON vulnerabilities(cvss DESC NULLS LAST);
CREATE INDEX idx_vuln_published ON vulnerabilities(published_at DESC);
COMMENT ON TABLE vulnerabilities IS 'Normalized vulnerability data (99% storage reduction)';
-- ============================================================================
-- TABLE: package_vulnerabilities
-- Purpose: Many-to-many relationship between packages and vulnerabilities
-- ============================================================================
CREATE TABLE IF NOT EXISTS package_vulnerabilities (
id BIGSERIAL PRIMARY KEY,
package_id BIGINT NOT NULL REFERENCES packages(id) ON DELETE CASCADE,
vulnerability_id BIGINT NOT NULL REFERENCES vulnerabilities(id) ON DELETE CASCADE,
scanner VARCHAR(50) NOT NULL,
detected_at TIMESTAMP NOT NULL DEFAULT NOW(),
bypassed BOOLEAN NOT NULL DEFAULT FALSE,
bypass_id BIGINT, -- References cve_bypasses.id (soft reference)
created_at TIMESTAMP NOT NULL DEFAULT NOW(),
updated_at TIMESTAMP NOT NULL DEFAULT NOW(),
deleted_at TIMESTAMP
);
CREATE INDEX idx_pkg_vuln_package ON package_vulnerabilities(package_id) WHERE deleted_at IS NULL;
CREATE INDEX idx_pkg_vuln_vuln ON package_vulnerabilities(vulnerability_id) WHERE deleted_at IS NULL;
CREATE INDEX idx_pkg_vuln_composite ON package_vulnerabilities(package_id, vulnerability_id) WHERE deleted_at IS NULL;
CREATE INDEX idx_pkg_vuln_scanner ON package_vulnerabilities(scanner);
CREATE INDEX idx_pkg_vuln_bypassed ON package_vulnerabilities(bypassed) WHERE bypassed = FALSE;
-- ============================================================================
-- TABLE: scan_results
-- Purpose: Security scan results with severity breakdown
-- ============================================================================
CREATE TABLE IF NOT EXISTS scan_results (
id BIGSERIAL PRIMARY KEY,
package_id BIGINT NOT NULL REFERENCES packages(id) ON DELETE CASCADE,
scanner VARCHAR(50) NOT NULL,
scanned_at TIMESTAMP NOT NULL DEFAULT NOW(),
status VARCHAR(20) NOT NULL, -- success, failed, pending
vuln_count INTEGER NOT NULL DEFAULT 0,
critical_count INTEGER NOT NULL DEFAULT 0,
high_count INTEGER NOT NULL DEFAULT 0,
medium_count INTEGER NOT NULL DEFAULT 0,
low_count INTEGER NOT NULL DEFAULT 0,
scan_duration INTEGER NOT NULL DEFAULT 0, -- milliseconds
details JSONB,
created_at TIMESTAMP NOT NULL DEFAULT NOW(),
updated_at TIMESTAMP NOT NULL DEFAULT NOW(),
deleted_at TIMESTAMP
);
CREATE INDEX idx_scan_package_scanner ON scan_results(package_id, scanner) WHERE deleted_at IS NULL;
CREATE INDEX idx_scan_scanned_at ON scan_results(scanned_at DESC);
CREATE INDEX idx_scan_status ON scan_results(status);
CREATE INDEX idx_scan_vuln_count ON scan_results(vuln_count) WHERE vuln_count > 0;
COMMENT ON TABLE scan_results IS 'Security scan results (optimized V2)';
-- ============================================================================
-- TABLE: cve_bypasses
-- Purpose: CVE bypass rules with usage tracking
-- ============================================================================
CREATE TABLE IF NOT EXISTS cve_bypasses (
id BIGSERIAL PRIMARY KEY,
type VARCHAR(20) NOT NULL, -- cve, package, registry
target VARCHAR(512) NOT NULL,
reason TEXT NOT NULL,
created_by VARCHAR(255) NOT NULL,
expires_at TIMESTAMP NOT NULL,
notify_on_expiry BOOLEAN NOT NULL DEFAULT FALSE,
active BOOLEAN NOT NULL DEFAULT TRUE,
usage_count BIGINT NOT NULL DEFAULT 0,
last_used_at TIMESTAMP,
registry_id INTEGER REFERENCES registries(id),
package_id BIGINT REFERENCES packages(id),
created_at TIMESTAMP NOT NULL DEFAULT NOW(),
updated_at TIMESTAMP NOT NULL DEFAULT NOW(),
deleted_at TIMESTAMP
);
CREATE INDEX idx_bypass_type ON cve_bypasses(type);
CREATE INDEX idx_bypass_target ON cve_bypasses(target);
CREATE INDEX idx_bypass_active ON cve_bypasses(active) WHERE active = TRUE AND deleted_at IS NULL;
CREATE INDEX idx_bypass_expires_at ON cve_bypasses(expires_at) WHERE active = TRUE;
CREATE INDEX idx_bypass_created_by ON cve_bypasses(created_by);
COMMENT ON TABLE cve_bypasses IS 'CVE bypass rules with scope limiting';
-- ============================================================================
-- PARTITIONED TABLE: download_events
-- Purpose: High-volume time-series data (partitioned by month)
-- ============================================================================
CREATE TABLE IF NOT EXISTS download_events (
id BIGSERIAL,
package_id BIGINT NOT NULL,
registry_id INTEGER NOT NULL,
downloaded_at TIMESTAMP NOT NULL,
user_agent VARCHAR(512),
ip_address VARCHAR(45),
authenticated BOOLEAN NOT NULL DEFAULT FALSE,
username VARCHAR(255)
) PARTITION BY RANGE (downloaded_at);
CREATE INDEX idx_download_events_package ON download_events(package_id, downloaded_at);
CREATE INDEX idx_download_events_registry ON download_events(registry_id);
CREATE INDEX idx_download_events_time ON download_events(downloaded_at);
COMMENT ON TABLE download_events IS 'Download events (partitioned by month for performance)';
-- Create partitions for current month ± 2 months
DO $$
DECLARE
start_date DATE;
end_date DATE;
partition_name TEXT;
i INTEGER;
BEGIN
FOR i IN -2..2 LOOP
start_date := date_trunc('month', NOW() + (i || ' months')::INTERVAL)::DATE;
end_date := (start_date + INTERVAL '1 month')::DATE;
partition_name := 'download_events_' || to_char(start_date, 'YYYY_MM');
EXECUTE format('CREATE TABLE IF NOT EXISTS %I PARTITION OF download_events FOR VALUES FROM (%L) TO (%L)',
partition_name, start_date, end_date);
EXECUTE format('CREATE INDEX IF NOT EXISTS %I ON %I(package_id, downloaded_at)',
partition_name || '_package_idx', partition_name);
EXECUTE format('CREATE INDEX IF NOT EXISTS %I ON %I(registry_id)',
partition_name || '_registry_idx', partition_name);
END LOOP;
END $$;
-- ============================================================================
-- TABLE: download_stats_hourly
-- Purpose: Pre-aggregated hourly statistics (1000x faster queries)
-- ============================================================================
CREATE TABLE IF NOT EXISTS download_stats_hourly (
id BIGSERIAL PRIMARY KEY,
registry_id INTEGER NOT NULL REFERENCES registries(id),
package_id BIGINT REFERENCES packages(id), -- NULL = all packages in registry
time_bucket TIMESTAMP NOT NULL,
download_count BIGINT NOT NULL DEFAULT 0,
unique_ips BIGINT NOT NULL DEFAULT 0,
auth_downloads BIGINT NOT NULL DEFAULT 0,
created_at TIMESTAMP NOT NULL DEFAULT NOW(),
updated_at TIMESTAMP NOT NULL DEFAULT NOW()
);
CREATE UNIQUE INDEX idx_stats_hourly_composite
ON download_stats_hourly(registry_id, COALESCE(package_id, 0), time_bucket);
CREATE INDEX idx_stats_hourly_time ON download_stats_hourly(time_bucket DESC);
COMMENT ON TABLE download_stats_hourly IS 'Hourly aggregated stats (pre-computed)';
-- ============================================================================
-- TABLE: download_stats_daily
-- Purpose: Pre-aggregated daily statistics with analytics
-- ============================================================================
CREATE TABLE IF NOT EXISTS download_stats_daily (
id BIGSERIAL PRIMARY KEY,
registry_id INTEGER NOT NULL REFERENCES registries(id),
package_id BIGINT REFERENCES packages(id),
time_bucket TIMESTAMP NOT NULL,
download_count BIGINT NOT NULL DEFAULT 0,
unique_ips BIGINT NOT NULL DEFAULT 0,
auth_downloads BIGINT NOT NULL DEFAULT 0,
top_user_agents JSONB,
created_at TIMESTAMP NOT NULL DEFAULT NOW(),
updated_at TIMESTAMP NOT NULL DEFAULT NOW()
);
CREATE UNIQUE INDEX idx_stats_daily_composite
ON download_stats_daily(registry_id, COALESCE(package_id, 0), time_bucket);
CREATE INDEX idx_stats_daily_time ON download_stats_daily(time_bucket DESC);
COMMENT ON TABLE download_stats_daily IS 'Daily aggregated stats with analytics';
-- ============================================================================
-- PARTITIONED TABLE: audit_log
-- Purpose: Audit trail for compliance (partitioned by month)
-- ============================================================================
CREATE TABLE IF NOT EXISTS audit_log (
id BIGSERIAL,
entity_type VARCHAR(50) NOT NULL,
entity_id BIGINT NOT NULL,
action VARCHAR(20) NOT NULL, -- create, update, delete
username VARCHAR(255) NOT NULL,
timestamp TIMESTAMP NOT NULL DEFAULT NOW(),
changes JSONB,
ip_address VARCHAR(45),
user_agent VARCHAR(512)
) PARTITION BY RANGE (timestamp);
CREATE INDEX idx_audit_log_entity ON audit_log(entity_type, entity_id);
CREATE INDEX idx_audit_log_username ON audit_log(username);
CREATE INDEX idx_audit_log_timestamp ON audit_log(timestamp DESC);
COMMENT ON TABLE audit_log IS 'Audit trail for compliance and debugging';
-- Create audit_log partitions
DO $$
DECLARE
start_date DATE;
end_date DATE;
partition_name TEXT;
i INTEGER;
BEGIN
FOR i IN -1..2 LOOP
start_date := date_trunc('month', NOW() + (i || ' months')::INTERVAL)::DATE;
end_date := (start_date + INTERVAL '1 month')::DATE;
partition_name := 'audit_log_' || to_char(start_date, 'YYYY_MM');
EXECUTE format('CREATE TABLE IF NOT EXISTS %I PARTITION OF audit_log FOR VALUES FROM (%L) TO (%L)',
partition_name, start_date, end_date);
EXECUTE format('CREATE INDEX IF NOT EXISTS %I ON %I(entity_type, entity_id)',
partition_name || '_entity_idx', partition_name);
EXECUTE format('CREATE INDEX IF NOT EXISTS %I ON %I(username)',
partition_name || '_user_idx', partition_name);
END LOOP;
END $$;
-- ============================================================================
-- FUNCTIONS: Automatic partition creation
-- ============================================================================
CREATE OR REPLACE FUNCTION create_next_month_partitions()
RETURNS void AS $$
DECLARE
next_month DATE := date_trunc('month', NOW() + INTERVAL '2 months');
partition_name TEXT;
start_date TEXT;
end_date TEXT;
BEGIN
-- Download events partition
partition_name := 'download_events_' || to_char(next_month, 'YYYY_MM');
start_date := to_char(next_month, 'YYYY-MM-DD');
end_date := to_char(next_month + INTERVAL '1 month', 'YYYY-MM-DD');
EXECUTE format('CREATE TABLE IF NOT EXISTS %I PARTITION OF download_events FOR VALUES FROM (%L) TO (%L)',
partition_name, start_date, end_date);
EXECUTE format('CREATE INDEX IF NOT EXISTS %I ON %I(package_id, downloaded_at)',
partition_name || '_package_idx', partition_name);
EXECUTE format('CREATE INDEX IF NOT EXISTS %I ON %I(registry_id)',
partition_name || '_registry_idx', partition_name);
-- Audit log partition
partition_name := 'audit_log_' || to_char(next_month, 'YYYY_MM');
EXECUTE format('CREATE TABLE IF NOT EXISTS %I PARTITION OF audit_log FOR VALUES FROM (%L) TO (%L)',
partition_name, start_date, end_date);
EXECUTE format('CREATE INDEX IF NOT EXISTS %I ON %I(entity_type, entity_id)',
partition_name || '_entity_idx', partition_name);
RAISE NOTICE 'Created partitions for %', to_char(next_month, 'YYYY-MM');
END;
$$ LANGUAGE plpgsql;
COMMENT ON FUNCTION create_next_month_partitions() IS 'Auto-create partitions for next month';
-- ============================================================================
-- TRIGGERS: Updated_at timestamp
-- ============================================================================
CREATE OR REPLACE FUNCTION update_updated_at_column()
RETURNS TRIGGER AS $$
BEGIN
NEW.updated_at = NOW();
RETURN NEW;
END;
$$ LANGUAGE plpgsql;
CREATE TRIGGER update_registries_updated_at BEFORE UPDATE ON registries
FOR EACH ROW EXECUTE FUNCTION update_updated_at_column();
CREATE TRIGGER update_packages_updated_at BEFORE UPDATE ON packages
FOR EACH ROW EXECUTE FUNCTION update_updated_at_column();
CREATE TRIGGER update_package_metadata_updated_at BEFORE UPDATE ON package_metadata
FOR EACH ROW EXECUTE FUNCTION update_updated_at_column();
-- ============================================================================
-- SEED DATA: Default registries
-- ============================================================================
INSERT INTO registries (name, display_name, upstream_url, enabled, scan_by_default) VALUES
('npm', 'NPM Registry', 'https://registry.npmjs.org', TRUE, TRUE),
('pypi', 'PyPI', 'https://pypi.org', TRUE, TRUE),
('go', 'Go Modules', 'https://proxy.golang.org', TRUE, TRUE)
ON CONFLICT (name) DO NOTHING;
-- ============================================================================
-- VIEWS: Convenience views for common queries
-- ============================================================================
CREATE OR REPLACE VIEW v_vulnerable_packages AS
SELECT
r.name AS registry,
p.name,
p.version,
p.vulnerability_count,
p.highest_severity,
p.last_scanned_at
FROM packages p
JOIN registries r ON p.registry_id = r.id
WHERE p.vulnerability_count > 0 AND p.deleted_at IS NULL
ORDER BY
CASE p.highest_severity
WHEN 'critical' THEN 1
WHEN 'high' THEN 2
WHEN 'medium' THEN 3
WHEN 'low' THEN 4
ELSE 5
END,
p.vulnerability_count DESC;
COMMENT ON VIEW v_vulnerable_packages IS 'All packages with vulnerabilities (sorted by severity)';
-- ============================================================================
-- COMPLETE
-- ============================================================================
SELECT 'Schema V2 created successfully!' AS status;
pkg/app/app.go
+72 -7
View File
@@ -19,7 +19,7 @@ import (
"github.com/lukaszraczylo/gohoarder/pkg/health"
"github.com/lukaszraczylo/gohoarder/pkg/metadata"
metafile "github.com/lukaszraczylo/gohoarder/pkg/metadata/file"
metasqlite "github.com/lukaszraczylo/gohoarder/pkg/metadata/sqlite"
metagorm "github.com/lukaszraczylo/gohoarder/pkg/metadata/gormstore"
"github.com/lukaszraczylo/gohoarder/pkg/metrics"
"github.com/lukaszraczylo/gohoarder/pkg/network"
"github.com/lukaszraczylo/gohoarder/pkg/prewarming"
@@ -119,18 +119,67 @@ func (a *App) initializeComponents() error {
log.Info().Str("backend", a.config.Metadata.Backend).Msg("Initializing metadata store")
switch a.config.Metadata.Backend {
case "sqlite":
a.metadata, err = metasqlite.New(metasqlite.Config{
Path: a.config.Metadata.Connection,
WALMode: a.config.Metadata.SQLite.WALMode,
// Use GORM for SQLite
a.metadata, err = metagorm.NewV2(metagorm.Config{
Driver: "sqlite",
DSN: metagorm.BuildSQLiteDSN(a.config.Metadata.SQLite.Path, a.config.Metadata.SQLite.WALMode),
MaxOpenConns: getOrDefault(a.config.Metadata.MaxOpenConns, 25),
MaxIdleConns: getOrDefault(a.config.Metadata.MaxIdleConns, 5),
ConnMaxLifetime: time.Duration(getOrDefault(a.config.Metadata.ConnMaxLifetime, 3600)) * time.Second,
LogLevel: getOrDefaultStr(a.config.Metadata.LogLevel, "warn"),
})
case "postgresql", "postgres":
// Use GORM for PostgreSQL
dsn := metagorm.BuildPostgresDSN(
a.config.Metadata.PostgreSQL.Host,
a.config.Metadata.PostgreSQL.Port,
a.config.Metadata.PostgreSQL.User,
a.config.Metadata.PostgreSQL.Password,
a.config.Metadata.PostgreSQL.Database,
getOrDefaultStr(a.config.Metadata.PostgreSQL.SSLMode, "disable"),
)
a.metadata, err = metagorm.NewV2(metagorm.Config{
Driver: "postgres",
DSN: dsn,
MaxOpenConns: getOrDefault(a.config.Metadata.MaxOpenConns, 25),
MaxIdleConns: getOrDefault(a.config.Metadata.MaxIdleConns, 5),
ConnMaxLifetime: time.Duration(getOrDefault(a.config.Metadata.ConnMaxLifetime, 3600)) * time.Second,
LogLevel: getOrDefaultStr(a.config.Metadata.LogLevel, "warn"),
})
case "mysql", "mariadb":
// Use GORM for MySQL/MariaDB
dsn := metagorm.BuildMySQLDSN(
a.config.Metadata.MySQL.Host,
a.config.Metadata.MySQL.Port,
a.config.Metadata.MySQL.User,
a.config.Metadata.MySQL.Password,
a.config.Metadata.MySQL.Database,
getOrDefaultStr(a.config.Metadata.MySQL.Charset, "utf8mb4"),
)
a.metadata, err = metagorm.NewV2(metagorm.Config{
Driver: "mysql",
DSN: dsn,
MaxOpenConns: getOrDefault(a.config.Metadata.MaxOpenConns, 25),
MaxIdleConns: getOrDefault(a.config.Metadata.MaxIdleConns, 5),
ConnMaxLifetime: time.Duration(getOrDefault(a.config.Metadata.ConnMaxLifetime, 3600)) * time.Second,
LogLevel: getOrDefaultStr(a.config.Metadata.LogLevel, "warn"),
})
case "file":
// Keep file backend as-is for file-based metadata
a.metadata, err = metafile.New(metafile.Config{
Path: a.config.Metadata.Connection,
})
default:
a.metadata, err = metasqlite.New(metasqlite.Config{
Path: "gohoarder.db",
WALMode: false, // Default to DELETE mode for compatibility
// Default to SQLite with GORM
log.Warn().Str("backend", a.config.Metadata.Backend).Msg("Unknown metadata backend, defaulting to SQLite with GORM")
a.metadata, err = metagorm.NewV2(metagorm.Config{
Driver: "sqlite",
DSN: metagorm.BuildSQLiteDSN("gohoarder.db", false),
LogLevel: "warn",
})
}
if err != nil {
@@ -479,3 +528,19 @@ func (a *App) startAggregationWorker(ctx context.Context) {
}
}
}
// getOrDefault returns the value if it's non-zero, otherwise returns the default
func getOrDefault(value, defaultValue int) int {
if value == 0 {
return defaultValue
}
return value
}
// getOrDefaultStr returns the value if it's non-empty, otherwise returns the default
func getOrDefaultStr(value, defaultValue string) string {
if value == "" {
return defaultValue
}
return value
}
pkg/app/handlers.go
+22 -1
View File
@@ -142,6 +142,13 @@ func (a *App) handleListPackages(c *fiber.Ctx) error {
severityCounts[strings.ToUpper(vuln.Severity)]++
}
// Check if package should be blocked based on thresholds
isBlocked := false
if a.scanManager != nil {
blocked, _, _ := a.scanManager.CheckVulnerabilities(ctx, pkg.Registry, entry.originalName, pkg.Version)
isBlocked = blocked
}
pkgMap["vulnerabilities"] = map[string]interface{}{
"scanned": true,
"status": scanResult.Status,
@@ -153,17 +160,20 @@ func (a *App) handleListPackages(c *fiber.Ctx) error {
"low": severityCounts["LOW"],
},
"total": scanResult.VulnerabilityCount,
"isBlocked": isBlocked,
}
} else {
pkgMap["vulnerabilities"] = map[string]interface{}{
"scanned": false,
"status": "pending",
"isBlocked": false,
}
}
} else {
pkgMap["vulnerabilities"] = map[string]interface{}{
"scanned": false,
"status": "not_scanned",
"isBlocked": false,
}
}
@@ -351,8 +361,9 @@ func (a *App) handleStats(c *fiber.Ctx) error {
packages = []*metadata.Package{}
}
// Calculate per-registry breakdown (exclude metadata entries like "list", "latest")
// Calculate per-registry breakdown and blocked packages count
registryStats := make(map[string]map[string]interface{})
blockedCount := int64(0)
for _, pkg := range packages {
// Skip metadata entries (npm metadata pages, pypi pages, etc.)
@@ -371,6 +382,14 @@ func (a *App) handleStats(c *fiber.Ctx) error {
registryStats[pkg.Registry]["count"] = registryStats[pkg.Registry]["count"].(int) + 1
registryStats[pkg.Registry]["size"] = registryStats[pkg.Registry]["size"].(int64) + pkg.Size
registryStats[pkg.Registry]["downloads"] = registryStats[pkg.Registry]["downloads"].(int64) + int64(pkg.DownloadCount)
// Check if package is blocked (only if security scanning is enabled and package is scanned)
if a.config.Security.Enabled && a.scanManager != nil && pkg.SecurityScanned {
blocked, _, _ := a.scanManager.CheckVulnerabilities(ctx, pkg.Registry, pkg.Name, pkg.Version)
if blocked {
blockedCount++
}
}
}
// Combine statistics using database stats for accuracy
@@ -378,12 +397,14 @@ func (a *App) handleStats(c *fiber.Ctx) error {
"total_packages": cacheStats.TotalPackages,
"total_downloads": cacheStats.TotalDownloads,
"total_size": cacheStats.TotalSize,
"max_cache_size": a.config.Cache.MaxSizeBytes,
"cache_hits": cacheStats.TotalDownloads,
"cache_misses": 0, // TODO: Track cache misses
"cache_evictions": 0, // TODO: Track evictions
"cache_size": cacheStats.TotalSize,
"scanned_packages": cacheStats.ScannedPackages,
"vulnerable_packages": cacheStats.VulnerablePackages,
"blocked_packages": blockedCount,
}
// Convert registry stats to interface map
pkg/cache/cache.go
Vendored
+10 -3
View File
@@ -203,9 +203,12 @@ func (m *Manager) getOrFetch(ctx context.Context, registry, name, version string
return nil, err
}
// Skip security scan wait for metadata entries (index pages, lists, etc.)
isMetadataEntry := version == "list" || version == "page" || version == "latest" || version == "metadata"
// Wait briefly for initial scan to complete if scanner is enabled
// This prevents serving vulnerable packages on first request
if m.scanner != nil {
if m.scanner != nil && !isMetadataEntry {
// Wait up to 30 seconds for scan to complete
scanCtx, cancel := context.WithTimeout(ctx, 30*time.Second)
defer cancel()
@@ -360,15 +363,19 @@ func (m *Manager) store(ctx context.Context, registry, name, version string, dat
Metadata: make(map[string]string),
}
// Save metadata
// Save metadata (skip metadata entries like index pages, lists, etc.)
isMetadataEntry := version == "list" || version == "page" || version == "latest" || version == "metadata"
if !isMetadataEntry {
if err := m.metadata.SavePackage(ctx, pkg); err != nil {
// Clean up storage if metadata save fails
_ = m.storage.Delete(ctx, storageKey) // #nosec G104 -- Cleanup, error logged
return nil, err
}
}
// Scan package if scanner is enabled (run in background to not block cache operations)
if m.scanner != nil {
// Skip scanning metadata entries (index pages, lists, etc.)
if m.scanner != nil && !isMetadataEntry {
go func() {
scanCtx := context.Background()
var filePath string
pkg/config/config.go
+28 -3
View File
@@ -73,10 +73,17 @@ type SMBConfig struct {
// MetadataConfig contains metadata store configuration
type MetadataConfig struct {
PostgreSQL PostgreSQLConfig `mapstructure:"postgresql" json:"postgresql"`
Backend string `mapstructure:"backend" json:"backend"`
Connection string `mapstructure:"connection" json:"connection"`
SQLite SQLiteConfig `mapstructure:"sqlite" json:"sqlite"`
PostgreSQL PostgreSQLConfig `mapstructure:"postgresql" json:"postgresql"`
MySQL MySQLConfig `mapstructure:"mysql" json:"mysql"`
// GORM-specific settings
MaxOpenConns int `mapstructure:"max_open_conns" json:"max_open_conns"`
MaxIdleConns int `mapstructure:"max_idle_conns" json:"max_idle_conns"`
ConnMaxLifetime int `mapstructure:"conn_max_lifetime" json:"conn_max_lifetime"` // seconds
LogLevel string `mapstructure:"log_level" json:"log_level"` // "silent", "error", "warn", "info"
}
// SQLiteConfig contains SQLite-specific configuration
@@ -88,11 +95,22 @@ type SQLiteConfig struct {
// PostgreSQLConfig contains PostgreSQL-specific configuration
type PostgreSQLConfig struct {
Host string `mapstructure:"host" json:"host"`
Port int `mapstructure:"port" json:"port"`
Database string `mapstructure:"database" json:"database"`
User string `mapstructure:"user" json:"user"`
Password string `mapstructure:"password" json:"-"`
SSLMode string `mapstructure:"ssl_mode" json:"ssl_mode"`
}
// MySQLConfig contains MySQL/MariaDB-specific configuration
type MySQLConfig struct {
Host string `mapstructure:"host" json:"host"`
Port int `mapstructure:"port" json:"port"`
Database string `mapstructure:"database" json:"database"`
User string `mapstructure:"user" json:"user"`
Password string `mapstructure:"password" json:"-"` // Don't serialize
Charset string `mapstructure:"charset" json:"charset"`
ParseTime bool `mapstructure:"parse_time" json:"parse_time"`
}
// CacheConfig contains cache management configuration
@@ -415,9 +433,16 @@ func (c *Config) Validate() error {
}
// Validate metadata backend
validMetadataBackends := map[string]bool{"sqlite": true, "postgresql": true, "file": true}
validMetadataBackends := map[string]bool{
"sqlite": true,
"postgresql": true,
"postgres": true,
"mysql": true,
"mariadb": true,
"file": true,
}
if !validMetadataBackends[c.Metadata.Backend] {
return fmt.Errorf("metadata.backend must be one of: sqlite, postgresql, file; got %s", c.Metadata.Backend)
return fmt.Errorf("metadata.backend must be one of: sqlite, postgresql, mysql, file; got %s", c.Metadata.Backend)
}
// Validate cache
pkg/metadata/gormstore/aggregation_worker.go
+357
View File
@@ -0,0 +1,357 @@
package gormstore
import (
"time"
"github.com/rs/zerolog/log"
"gorm.io/gorm"
)
// AggregationWorker handles background aggregation of download statistics
type AggregationWorker struct {
db *gorm.DB
stopChan chan struct{}
ticker *time.Ticker
}
// NewAggregationWorker creates a new aggregation worker
func NewAggregationWorker(db *gorm.DB) *AggregationWorker {
return &AggregationWorker{
db: db,
stopChan: make(chan struct{}),
ticker: time.NewTicker(1 * time.Hour), // Run every hour
}
}
// Start begins the aggregation worker
func (w *AggregationWorker) Start() {
log.Info().Msg("Starting aggregation worker")
// Run immediately on start
if err := w.AggregateHourly(); err != nil {
log.Error().Err(err).Msg("Failed to run initial hourly aggregation")
}
for {
select {
case <-w.ticker.C:
if err := w.AggregateHourly(); err != nil {
log.Error().Err(err).Msg("Failed to aggregate hourly stats")
}
// Check if it's time for daily aggregation (run at midnight)
now := time.Now()
if now.Hour() == 0 {
if err := w.AggregateDaily(); err != nil {
log.Error().Err(err).Msg("Failed to aggregate daily stats")
}
}
case <-w.stopChan:
log.Info().Msg("Stopping aggregation worker")
w.ticker.Stop()
return
}
}
}
// Stop stops the aggregation worker
func (w *AggregationWorker) Stop() {
close(w.stopChan)
}
// AggregateHourly aggregates download events into hourly stats
func (w *AggregationWorker) AggregateHourly() error {
startTime := time.Now()
log.Debug().Msg("Starting hourly aggregation")
// Get dialect name
dialectName := w.db.Dialector.Name()
// Calculate cutoff time (aggregate events older than 5 minutes to avoid partial data)
cutoff := time.Now().Add(-5 * time.Minute).Truncate(time.Hour)
return w.db.Transaction(func(tx *gorm.DB) error {
var aggregateSQL string
switch dialectName {
case "postgres":
// PostgreSQL: Use date_trunc for time bucketing
aggregateSQL = `
INSERT INTO download_stats_hourly (registry_id, package_id, time_bucket, download_count, unique_ips, auth_downloads, created_at, updated_at)
SELECT
de.registry_id,
de.package_id,
date_trunc('hour', de.downloaded_at) AS time_bucket,
COUNT(*) AS download_count,
COUNT(DISTINCT de.ip_address) AS unique_ips,
COUNT(*) FILTER (WHERE de.authenticated = true) AS auth_downloads,
NOW() AS created_at,
NOW() AS updated_at
FROM download_events de
WHERE de.downloaded_at < ?
GROUP BY de.registry_id, de.package_id, time_bucket
ON CONFLICT (registry_id, COALESCE(package_id, 0), time_bucket)
DO UPDATE SET
download_count = download_stats_hourly.download_count + EXCLUDED.download_count,
unique_ips = GREATEST(download_stats_hourly.unique_ips, EXCLUDED.unique_ips),
auth_downloads = download_stats_hourly.auth_downloads + EXCLUDED.auth_downloads,
updated_at = NOW()
`
case "mysql":
// MySQL: Use DATE_FORMAT for time bucketing
aggregateSQL = `
INSERT INTO download_stats_hourly (registry_id, package_id, time_bucket, download_count, unique_ips, auth_downloads, created_at, updated_at)
SELECT
de.registry_id,
de.package_id,
DATE_FORMAT(de.downloaded_at, '%Y-%m-%d %H:00:00') AS time_bucket,
COUNT(*) AS download_count,
COUNT(DISTINCT de.ip_address) AS unique_ips,
SUM(CASE WHEN de.authenticated = true THEN 1 ELSE 0 END) AS auth_downloads,
NOW() AS created_at,
NOW() AS updated_at
FROM download_events de
WHERE de.downloaded_at < ?
GROUP BY de.registry_id, de.package_id, time_bucket
ON DUPLICATE KEY UPDATE
download_count = download_stats_hourly.download_count + VALUES(download_count),
unique_ips = GREATEST(download_stats_hourly.unique_ips, VALUES(unique_ips)),
auth_downloads = download_stats_hourly.auth_downloads + VALUES(auth_downloads),
updated_at = NOW()
`
default: // SQLite
// SQLite: Use strftime for time bucketing
// Note: SQLite doesn't support UPSERT as elegantly, need to handle separately
aggregateSQL = `
INSERT OR REPLACE INTO download_stats_hourly (registry_id, package_id, time_bucket, download_count, unique_ips, auth_downloads, created_at, updated_at)
SELECT
de.registry_id,
de.package_id,
strftime('%Y-%m-%d %H:00:00', de.downloaded_at) AS time_bucket,
COUNT(*) AS download_count,
COUNT(DISTINCT de.ip_address) AS unique_ips,
SUM(CASE WHEN de.authenticated = 1 THEN 1 ELSE 0 END) AS auth_downloads,
datetime('now') AS created_at,
datetime('now') AS updated_at
FROM download_events de
WHERE de.downloaded_at < ?
GROUP BY de.registry_id, de.package_id, time_bucket
`
}
// Execute aggregation
if err := tx.Exec(aggregateSQL, cutoff).Error; err != nil {
return err
}
// Delete aggregated events (older than 24 hours to keep recent data for debugging)
deleteOlder := time.Now().Add(-24 * time.Hour)
deleteResult := tx.Exec("DELETE FROM download_events WHERE downloaded_at < ?", deleteOlder)
if deleteResult.Error != nil {
return deleteResult.Error
}
// Also update package-level stats (NULL package_id = registry totals)
var registryAggSQL string
if dialectName == "postgres" {
registryAggSQL = `
INSERT INTO download_stats_hourly (registry_id, package_id, time_bucket, download_count, unique_ips, auth_downloads, created_at, updated_at)
SELECT
registry_id,
NULL as package_id,
time_bucket,
SUM(download_count) as download_count,
SUM(unique_ips) as unique_ips,
SUM(auth_downloads) as auth_downloads,
NOW() as created_at,
NOW() as updated_at
FROM download_stats_hourly
WHERE package_id IS NOT NULL
GROUP BY registry_id, time_bucket
ON CONFLICT (registry_id, COALESCE(package_id, 0), time_bucket)
DO UPDATE SET
download_count = EXCLUDED.download_count,
unique_ips = EXCLUDED.unique_ips,
auth_downloads = EXCLUDED.auth_downloads,
updated_at = NOW()
`
} else if dialectName == "mysql" {
registryAggSQL = `
INSERT INTO download_stats_hourly (registry_id, package_id, time_bucket, download_count, unique_ips, auth_downloads, created_at, updated_at)
SELECT
registry_id,
NULL as package_id,
time_bucket,
SUM(download_count) as download_count,
SUM(unique_ips) as unique_ips,
SUM(auth_downloads) as auth_downloads,
NOW() as created_at,
NOW() as updated_at
FROM download_stats_hourly
WHERE package_id IS NOT NULL
GROUP BY registry_id, time_bucket
ON DUPLICATE KEY UPDATE
download_count = VALUES(download_count),
unique_ips = VALUES(unique_ips),
auth_downloads = VALUES(auth_downloads),
updated_at = NOW()
`
} else {
// SQLite
registryAggSQL = `
INSERT OR REPLACE INTO download_stats_hourly (registry_id, package_id, time_bucket, download_count, unique_ips, auth_downloads, created_at, updated_at)
SELECT
registry_id,
NULL as package_id,
time_bucket,
SUM(download_count) as download_count,
SUM(unique_ips) as unique_ips,
SUM(auth_downloads) as auth_downloads,
datetime('now') as created_at,
datetime('now') as updated_at
FROM download_stats_hourly
WHERE package_id IS NOT NULL
GROUP BY registry_id, time_bucket
`
}
if err := tx.Exec(registryAggSQL).Error; err != nil {
log.Warn().Err(err).Msg("Failed to aggregate registry totals (continuing anyway)")
}
elapsed := time.Since(startTime)
log.Info().
Int64("deleted_events", deleteResult.RowsAffected).
Dur("duration", elapsed).
Msg("Completed hourly aggregation")
return nil
})
}
// AggregateDaily aggregates hourly stats into daily stats
func (w *AggregationWorker) AggregateDaily() error {
startTime := time.Now()
log.Debug().Msg("Starting daily aggregation")
dialectName := w.db.Dialector.Name()
// Aggregate yesterday's data
yesterday := time.Now().AddDate(0, 0, -1).Truncate(24 * time.Hour)
dayEnd := yesterday.Add(24 * time.Hour)
return w.db.Transaction(func(tx *gorm.DB) error {
var aggregateSQL string
switch dialectName {
case "postgres":
aggregateSQL = `
INSERT INTO download_stats_daily (registry_id, package_id, time_bucket, download_count, unique_ips, auth_downloads, top_user_agents, created_at, updated_at)
SELECT
registry_id,
package_id,
date_trunc('day', time_bucket) AS time_bucket,
SUM(download_count) AS download_count,
MAX(unique_ips) AS unique_ips,
SUM(auth_downloads) AS auth_downloads,
'{}' AS top_user_agents,
NOW() AS created_at,
NOW() AS updated_at
FROM download_stats_hourly
WHERE time_bucket >= ? AND time_bucket < ?
GROUP BY registry_id, package_id, date_trunc('day', time_bucket)
ON CONFLICT (registry_id, COALESCE(package_id, 0), time_bucket)
DO UPDATE SET
download_count = EXCLUDED.download_count,
unique_ips = EXCLUDED.unique_ips,
auth_downloads = EXCLUDED.auth_downloads,
updated_at = NOW()
`
case "mysql":
aggregateSQL = `
INSERT INTO download_stats_daily (registry_id, package_id, time_bucket, download_count, unique_ips, auth_downloads, top_user_agents, created_at, updated_at)
SELECT
registry_id,
package_id,
DATE_FORMAT(time_bucket, '%Y-%m-%d 00:00:00') AS time_bucket,
SUM(download_count) AS download_count,
MAX(unique_ips) AS unique_ips,
SUM(auth_downloads) AS auth_downloads,
'{}' AS top_user_agents,
NOW() AS created_at,
NOW() AS updated_at
FROM download_stats_hourly
WHERE time_bucket >= ? AND time_bucket < ?
GROUP BY registry_id, package_id, DATE_FORMAT(time_bucket, '%Y-%m-%d 00:00:00')
ON DUPLICATE KEY UPDATE
download_count = VALUES(download_count),
unique_ips = VALUES(unique_ips),
auth_downloads = VALUES(auth_downloads),
updated_at = NOW()
`
default: // SQLite
aggregateSQL = `
INSERT OR REPLACE INTO download_stats_daily (registry_id, package_id, time_bucket, download_count, unique_ips, auth_downloads, top_user_agents, created_at, updated_at)
SELECT
registry_id,
package_id,
date(time_bucket) AS time_bucket,
SUM(download_count) AS download_count,
MAX(unique_ips) AS unique_ips,
SUM(auth_downloads) AS auth_downloads,
'{}' AS top_user_agents,
datetime('now') AS created_at,
datetime('now') AS updated_at
FROM download_stats_hourly
WHERE time_bucket >= ? AND time_bucket < ?
GROUP BY registry_id, package_id, date(time_bucket)
`
}
if err := tx.Exec(aggregateSQL, yesterday, dayEnd).Error; err != nil {
return err
}
// Delete old hourly stats (keep last 7 days)
deleteOlder := time.Now().AddDate(0, 0, -7)
deleteResult := tx.Exec("DELETE FROM download_stats_hourly WHERE time_bucket < ?", deleteOlder)
if deleteResult.Error != nil {
return deleteResult.Error
}
elapsed := time.Since(startTime)
log.Info().
Int64("deleted_hourly_stats", deleteResult.RowsAffected).
Dur("duration", elapsed).
Msg("Completed daily aggregation")
return nil
})
}
// UpdatePackageAccessCounts synchronizes package access_count from download stats
func (w *AggregationWorker) UpdatePackageAccessCounts() error {
log.Debug().Msg("Updating package access counts")
// Update from download_stats_hourly (sum all-time downloads per package)
updateSQL := `
UPDATE packages p
SET access_count = COALESCE((
SELECT SUM(download_count)
FROM download_stats_hourly dsh
WHERE dsh.package_id = p.id
), 0)
`
if err := w.db.Exec(updateSQL).Error; err != nil {
return err
}
log.Info().Msg("Updated package access counts")
return nil
}
pkg/metadata/gormstore/config.go
+78
View File
@@ -0,0 +1,78 @@
package gormstore
import (
"fmt"
"time"
"github.com/lukaszraczylo/gohoarder/pkg/errors"
)
// Config holds GORM store configuration
type Config struct {
// Database connection
Driver string // "sqlite", "postgres", "mysql"
DSN string // Data Source Name
// Connection pool
MaxOpenConns int
MaxIdleConns int
ConnMaxLifetime time.Duration
// GORM settings
LogLevel string // "silent", "error", "warn", "info"
}
// Validate validates the configuration
func (c *Config) Validate() error {
if c.Driver == "" {
return errors.New(errors.ErrCodeInvalidConfig, "driver is required")
}
if c.DSN == "" {
return errors.New(errors.ErrCodeInvalidConfig, "DSN is required")
}
// Set defaults
if c.MaxOpenConns == 0 {
c.MaxOpenConns = 25
}
if c.MaxIdleConns == 0 {
c.MaxIdleConns = 5
}
if c.ConnMaxLifetime == 0 {
c.ConnMaxLifetime = time.Hour
}
if c.LogLevel == "" {
c.LogLevel = "warn"
}
return nil
}
// BuildPostgresDSN builds PostgreSQL DSN from structured config
func BuildPostgresDSN(host string, port int, user, password, database, sslmode string) string {
if sslmode == "" {
sslmode = "disable"
}
return fmt.Sprintf("host=%s port=%d user=%s password=%s dbname=%s sslmode=%s",
host, port, user, password, database, sslmode)
}
// BuildMySQLDSN builds MySQL/MariaDB DSN from structured config
func BuildMySQLDSN(host string, port int, user, password, database, charset string) string {
if charset == "" {
charset = "utf8mb4"
}
return fmt.Sprintf("%s:%s@tcp(%s:%d)/%s?charset=%s&parseTime=True&loc=Local",
user, password, host, port, database, charset)
}
// BuildSQLiteDSN builds SQLite DSN with pragmas
func BuildSQLiteDSN(path string, walMode bool) string {
if path == "" {
path = "gohoarder.db"
}
if walMode {
return fmt.Sprintf("%s?_journal_mode=WAL&_busy_timeout=5000&_synchronous=NORMAL&_cache_size=2000", path)
}
return fmt.Sprintf("%s?_journal_mode=DELETE&_busy_timeout=5000&_synchronous=NORMAL", path)
}
pkg/metadata/gormstore/gormstore_v2.go
+1000
View File
File diff suppressed because it is too large Load Diff
pkg/metadata/gormstore/gormstore_v2_mysql_test.go
+279
View File
@@ -0,0 +1,279 @@
//go:build integration
// +build integration
package gormstore
import (
"context"
"testing"
"time"
"github.com/lukaszraczylo/gohoarder/pkg/metadata"
"github.com/stretchr/testify/suite"
"github.com/testcontainers/testcontainers-go"
"github.com/testcontainers/testcontainers-go/modules/mysql"
"github.com/testcontainers/testcontainers-go/wait"
)
// MySQLV2IntegrationTestSuite embeds the V2 test suite with MySQL container
type MySQLV2IntegrationTestSuite struct {
GORMStoreV2TestSuite
container *mysql.MySQLContainer
}
// SetupSuite runs once before all tests
func (s *MySQLV2IntegrationTestSuite) SetupSuite() {
ctx := context.Background()
// Start MySQL container
container, err := mysql.RunContainer(ctx,
testcontainers.WithImage("mysql:8.0"),
mysql.WithDatabase("testdb"),
mysql.WithUsername("testuser"),
mysql.WithPassword("testpass"),
testcontainers.WithWaitStrategy(
wait.ForLog("port: 3306 MySQL Community Server").
WithOccurrence(1).
WithStartupTimeout(60*time.Second),
),
)
s.Require().NoError(err)
s.container = container
}
// TearDownSuite runs once after all tests
func (s *MySQLV2IntegrationTestSuite) TearDownSuite() {
if s.container != nil {
ctx := context.Background()
err := s.container.Terminate(ctx)
s.NoError(err)
}
}
// SetupTest runs before each test
func (s *MySQLV2IntegrationTestSuite) SetupTest() {
s.ctx = context.Background()
// Get connection string from container
connStr, err := s.container.ConnectionString(s.ctx)
s.Require().NoError(err)
// Create GORM store with MySQL
cfg := Config{
Driver: "mysql",
DSN: connStr,
MaxOpenConns: 10,
MaxIdleConns: 5,
ConnMaxLifetime: 3600 * time.Second,
LogLevel: "silent",
}
s.store, err = NewV2(cfg)
s.Require().NoError(err)
s.Require().NotNil(s.store)
}
// TearDownTest runs after each test
func (s *MySQLV2IntegrationTestSuite) TearDownTest() {
if s.store != nil {
// Clean up all tables for next test
tables := []string{
"download_events",
"download_stats_hourly",
"download_stats_daily",
"audit_log",
"cve_bypasses",
"scan_results",
"package_vulnerabilities",
"vulnerabilities",
"package_metadata",
"packages",
"registries",
}
for _, table := range tables {
s.store.db.Exec("TRUNCATE TABLE " + table)
}
// Re-seed default registries after truncate
defaultRegistries := []RegistryModel{
{Name: "npm", DisplayName: "NPM Registry", UpstreamURL: "https://registry.npmjs.org", Enabled: true, ScanByDefault: true},
{Name: "pypi", DisplayName: "PyPI", UpstreamURL: "https://pypi.org", Enabled: true, ScanByDefault: true},
{Name: "go", DisplayName: "Go Modules", UpstreamURL: "https://proxy.golang.org", Enabled: true, ScanByDefault: true},
}
for _, reg := range defaultRegistries {
s.store.db.Create(&reg)
}
// Rebuild registry cache
s.store.rebuildRegistryCache()
s.store.Close()
}
}
// TestMySQLV2IntegrationTestSuite runs the integration test suite with MySQL
func TestMySQLV2IntegrationTestSuite(t *testing.T) {
if testing.Short() {
t.Skip("Skipping integration tests in short mode")
}
suite.Run(t, new(MySQLV2IntegrationTestSuite))
}
// Test_MySQLV2_SpecificFeatures tests MySQL-specific features
func (s *MySQLV2IntegrationTestSuite) Test_MySQLV2_SpecificFeatures() {
// Test that we're actually using MySQL
var version string
err := s.store.db.Raw("SELECT VERSION()").Scan(&version).Error
s.NoError(err)
s.Contains(version, "MySQL")
}
// Test_MySQLV2_NoPartitioning tests that partition manager is nil for MySQL
func (s *MySQLV2IntegrationTestSuite) Test_MySQLV2_NoPartitioning() {
// MySQL doesn't use our partition manager (uses native partitioning differently)
s.Nil(s.store.partitionManager)
}
// Test_MySQLV2_HighConcurrency tests MySQL's concurrent write support
func (s *MySQLV2IntegrationTestSuite) Test_MySQLV2_HighConcurrency() {
pkg := &metadata.Package{
Registry: "npm",
Name: "concurrent-test",
Version: "1.0.0",
StorageKey: "npm/concurrent-test/1.0.0.tgz",
Size: 1000,
CachedAt: time.Now(),
LastAccessed: time.Now(),
}
err := s.store.SavePackage(s.ctx, pkg)
s.NoError(err)
// MySQL can handle concurrent writes (with InnoDB row-level locking)
concurrency := 15
done := make(chan bool, concurrency)
for i := 0; i < concurrency; i++ {
go func() {
err := s.store.UpdateDownloadCount(s.ctx, "npm", "concurrent-test", "1.0.0")
s.NoError(err)
done <- true
}()
}
// Wait for all goroutines
for i := 0; i < concurrency; i++ {
<-done
}
// Verify all updates succeeded
retrieved, err := s.store.GetPackage(s.ctx, "npm", "concurrent-test", "1.0.0")
s.NoError(err)
s.Equal(int64(concurrency), retrieved.DownloadCount)
}
// Test_MySQLV2_JSON tests MySQL JSON functionality
func (s *MySQLV2IntegrationTestSuite) Test_MySQLV2_JSON() {
metadata := map[string]interface{}{
"author": "Test Author",
"license": "MIT",
"description": "Test package",
"keywords": []interface{}{"test", "mysql", "json"},
}
pkg := &metadata.Package{
Registry: "npm",
Name: "json-test",
Version: "1.0.0",
StorageKey: "npm/json-test/1.0.0.tgz",
Size: 1000,
CachedAt: time.Now(),
LastAccessed: time.Now(),
Metadata: metadata,
}
err := s.store.SavePackage(s.ctx, pkg)
s.NoError(err)
// Retrieve and verify JSON data
retrieved, err := s.store.GetPackage(s.ctx, "npm", "json-test", "1.0.0")
s.NoError(err)
s.NotNil(retrieved.Metadata)
s.Equal("MIT", retrieved.Metadata["license"])
s.Equal("Test Author", retrieved.Metadata["author"])
}
// Test_MySQLV2_TransactionRollback tests MySQL transaction rollback
func (s *MySQLV2IntegrationTestSuite) Test_MySQLV2_TransactionRollback() {
pkg := &metadata.Package{
Registry: "npm",
Name: "tx-test",
Version: "1.0.0",
StorageKey: "npm/tx-test/1.0.0.tgz",
Size: 1000,
CachedAt: time.Now(),
LastAccessed: time.Now(),
}
err := s.store.SavePackage(s.ctx, pkg)
s.NoError(err)
// Try to update with invalid data that should trigger rollback
err = s.store.db.Transaction(func(tx *gorm.DB) error {
// First update succeeds
result := tx.Model(&PackageModel{}).
Where("registry_id = ? AND name = ? AND version = ?",
s.store.registryCache["npm"], "tx-test", "1.0.0").
Update("access_count", gorm.Expr("access_count + ?", 1))
if result.Error != nil {
return result.Error
}
// Second operation fails (invalid foreign key)
invalidModel := &PackageModel{
RegistryID: 9999, // Non-existent registry
Name: "invalid",
Version: "1.0.0",
StorageKey: "invalid",
Size: 100,
CachedAt: time.Now(),
LastAccessed: time.Now(),
}
return tx.Create(invalidModel).Error
})
// Transaction should fail
s.Error(err)
// Verify first update was rolled back
retrieved, err := s.store.GetPackage(s.ctx, "npm", "tx-test", "1.0.0")
s.NoError(err)
s.Equal(int64(0), retrieved.DownloadCount) // Should still be 0, not 1
}
// Test_MySQLV2_CharacterSet tests MySQL UTF-8 support
func (s *MySQLV2IntegrationTestSuite) Test_MySQLV2_CharacterSet() {
// Test package with Unicode characters
pkg := &metadata.Package{
Registry: "npm",
Name: "unicode-test-世界-🚀",
Version: "1.0.0",
StorageKey: "npm/unicode-test/1.0.0.tgz",
Size: 1000,
CachedAt: time.Now(),
LastAccessed: time.Now(),
Metadata: map[string]interface{}{
"description": "Test with emoji 🎉 and Chinese 中文",
},
}
err := s.store.SavePackage(s.ctx, pkg)
s.NoError(err)
// Retrieve and verify Unicode data preserved
retrieved, err := s.store.GetPackage(s.ctx, "npm", "unicode-test-世界-🚀", "1.0.0")
s.NoError(err)
s.Equal("unicode-test-世界-🚀", retrieved.Name)
s.Contains(retrieved.Metadata["description"], "🎉")
s.Contains(retrieved.Metadata["description"], "中文")
}
pkg/metadata/gormstore/gormstore_v2_postgres_test.go
+202
View File
@@ -0,0 +1,202 @@
//go:build integration
// +build integration
package gormstore
import (
"context"
"testing"
"time"
"github.com/lukaszraczylo/gohoarder/pkg/metadata"
"github.com/stretchr/testify/suite"
"github.com/testcontainers/testcontainers-go"
"github.com/testcontainers/testcontainers-go/modules/postgres"
"github.com/testcontainers/testcontainers-go/wait"
)
// PostgresV2IntegrationTestSuite embeds the V2 test suite with PostgreSQL container
type PostgresV2IntegrationTestSuite struct {
GORMStoreV2TestSuite
container *postgres.PostgresContainer
}
// SetupSuite runs once before all tests
func (s *PostgresV2IntegrationTestSuite) SetupSuite() {
ctx := context.Background()
// Start PostgreSQL container
container, err := postgres.RunContainer(ctx,
testcontainers.WithImage("postgres:16-alpine"),
postgres.WithDatabase("testdb"),
postgres.WithUsername("testuser"),
postgres.WithPassword("testpass"),
testcontainers.WithWaitStrategy(
wait.ForLog("database system is ready to accept connections").
WithOccurrence(2).
WithStartupTimeout(60*time.Second),
),
)
s.Require().NoError(err)
s.container = container
}
// TearDownSuite runs once after all tests
func (s *PostgresV2IntegrationTestSuite) TearDownSuite() {
if s.container != nil {
ctx := context.Background()
err := s.container.Terminate(ctx)
s.NoError(err)
}
}
// SetupTest runs before each test
func (s *PostgresV2IntegrationTestSuite) SetupTest() {
s.ctx = context.Background()
// Get connection string from container
connStr, err := s.container.ConnectionString(s.ctx)
s.Require().NoError(err)
// Create GORM store with PostgreSQL
cfg := Config{
Driver: "postgres",
DSN: connStr + "sslmode=disable",
MaxOpenConns: 10,
MaxIdleConns: 5,
ConnMaxLifetime: 3600 * time.Second,
LogLevel: "silent",
}
s.store, err = NewV2(cfg)
s.Require().NoError(err)
s.Require().NotNil(s.store)
}
// TearDownTest runs after each test
func (s *PostgresV2IntegrationTestSuite) TearDownTest() {
if s.store != nil {
// Clean up all tables for next test
tables := []string{
"download_events",
"download_stats_hourly",
"download_stats_daily",
"audit_log",
"cve_bypasses",
"scan_results",
"package_vulnerabilities",
"vulnerabilities",
"package_metadata",
"packages",
}
for _, table := range tables {
s.store.db.Exec("TRUNCATE TABLE " + table + " CASCADE")
}
s.store.Close()
}
}
// TestPostgresV2IntegrationTestSuite runs the integration test suite with PostgreSQL
func TestPostgresV2IntegrationTestSuite(t *testing.T) {
if testing.Short() {
t.Skip("Skipping integration tests in short mode")
}
suite.Run(t, new(PostgresV2IntegrationTestSuite))
}
// Test_PostgresV2_SpecificFeatures tests PostgreSQL-specific features
func (s *PostgresV2IntegrationTestSuite) Test_PostgresV2_SpecificFeatures() {
// Test that we're actually using PostgreSQL
var version string
err := s.store.db.Raw("SELECT version()").Scan(&version).Error
s.NoError(err)
s.Contains(version, "PostgreSQL")
}
// Test_PostgresV2_Partitioning tests partition manager
func (s *PostgresV2IntegrationTestSuite) Test_PostgresV2_Partitioning() {
s.NotNil(s.store.partitionManager)
// Get partition info
info, err := s.store.partitionManager.GetPartitionInfo()
s.NoError(err)
s.NotNil(info)
// Should have created partitions
downloadPartitions := info["download_events_partitions"].(int64)
s.Greater(downloadPartitions, int64(0))
auditPartitions := info["audit_log_partitions"].(int64)
s.Greater(auditPartitions, int64(0))
}
// Test_PostgresV2_HighConcurrency tests PostgreSQL's excellent concurrent write support
func (s *PostgresV2IntegrationTestSuite) Test_PostgresV2_HighConcurrency() {
pkg := &metadata.Package{
Registry: "npm",
Name: "concurrent-test",
Version: "1.0.0",
StorageKey: "npm/concurrent-test/1.0.0.tgz",
Size: 1000,
CachedAt: time.Now(),
LastAccessed: time.Now(),
}
err := s.store.SavePackage(s.ctx, pkg)
s.NoError(err)
// PostgreSQL can handle many concurrent writes
concurrency := 20
done := make(chan bool, concurrency)
for i := 0; i < concurrency; i++ {
go func() {
err := s.store.UpdateDownloadCount(s.ctx, "npm", "concurrent-test", "1.0.0")
s.NoError(err)
done <- true
}()
}
// Wait for all goroutines
for i := 0; i < concurrency; i++ {
<-done
}
// Verify all updates succeeded
retrieved, err := s.store.GetPackage(s.ctx, "npm", "concurrent-test", "1.0.0")
s.NoError(err)
s.Equal(int64(concurrency), retrieved.DownloadCount)
}
// Test_PostgresV2_JSONB tests PostgreSQL JSONB functionality
func (s *PostgresV2IntegrationTestSuite) Test_PostgresV2_JSONB() {
metadata := map[string]interface{}{
"author": "Test Author",
"license": "MIT",
"description": "Test package",
"keywords": []interface{}{"test", "postgres", "jsonb"},
}
pkg := &metadata.Package{
Registry: "npm",
Name: "jsonb-test",
Version: "1.0.0",
StorageKey: "npm/jsonb-test/1.0.0.tgz",
Size: 1000,
CachedAt: time.Now(),
LastAccessed: time.Now(),
Metadata: metadata,
}
err := s.store.SavePackage(s.ctx, pkg)
s.NoError(err)
// Retrieve and verify JSONB data
retrieved, err := s.store.GetPackage(s.ctx, "npm", "jsonb-test", "1.0.0")
s.NoError(err)
s.NotNil(retrieved.Metadata)
s.Equal("MIT", retrieved.Metadata["license"])
s.Equal("Test Author", retrieved.Metadata["author"])
}
pkg/metadata/gormstore/gormstore_v2_test.go
+871
View File
@@ -0,0 +1,871 @@
package gormstore
import (
"context"
"fmt"
"testing"
"time"
"github.com/lukaszraczylo/gohoarder/pkg/metadata"
"github.com/stretchr/testify/suite"
"gorm.io/driver/sqlite"
"gorm.io/gorm"
"gorm.io/gorm/logger"
)
// GORMStoreV2TestSuite is the test suite for V2 GORM implementation
type GORMStoreV2TestSuite struct {
suite.Suite
db *gorm.DB
store *GORMStoreV2
ctx context.Context
}
// SetupSuite runs once before all tests
func (s *GORMStoreV2TestSuite) SetupSuite() {
// Use in-memory SQLite for fast tests
db, err := gorm.Open(sqlite.Open("file::memory:?cache=shared"), &gorm.Config{
Logger: logger.Default.LogMode(logger.Silent),
})
s.Require().NoError(err)
s.db = db
}
// SetupTest runs before each test
func (s *GORMStoreV2TestSuite) SetupTest() {
s.ctx = context.Background()
// Create fresh store with V2 schema
cfg := Config{
Driver: "sqlite",
DSN: "file::memory:?cache=shared",
MaxOpenConns: 10,
MaxIdleConns: 5,
ConnMaxLifetime: 3600 * time.Second,
LogLevel: "silent",
}
store, err := NewV2(cfg)
s.Require().NoError(err)
s.Require().NotNil(store)
s.store = store
}
// TearDownTest runs after each test
func (s *GORMStoreV2TestSuite) TearDownTest() {
if s.store != nil {
// Clean up all tables for next test
tables := []string{
"audit_log",
"download_stats_daily",
"download_stats_hourly",
"download_events",
"cve_bypasses",
"scan_results",
"package_vulnerabilities",
"vulnerabilities",
"package_metadata",
"packages",
}
for _, table := range tables {
s.store.db.Exec(fmt.Sprintf("DELETE FROM %s", table))
}
s.store.Close()
}
}
// TestGORMStoreV2TestSuite runs the test suite
func TestGORMStoreV2TestSuite(t *testing.T) {
suite.Run(t, new(GORMStoreV2TestSuite))
}
// Test_V2_SavePackage_Success tests saving a package
func (s *GORMStoreV2TestSuite) Test_V2_SavePackage_Success() {
pkg := &metadata.Package{
Registry: "npm",
Name: "test-package",
Version: "1.0.0",
StorageKey: "npm/test-package/1.0.0.tgz",
Size: 12345,
ChecksumMD5: "abc123",
ChecksumSHA256: "def456",
UpstreamURL: "https://registry.npmjs.org/test-package",
CachedAt: time.Now(),
LastAccessed: time.Now(),
}
err := s.store.SavePackage(s.ctx, pkg)
s.NoError(err)
// Verify package was saved
retrieved, err := s.store.GetPackage(s.ctx, "npm", "test-package", "1.0.0")
s.NoError(err)
s.NotNil(retrieved)
s.Equal("npm", retrieved.Registry)
s.Equal("test-package", retrieved.Name)
s.Equal("1.0.0", retrieved.Version)
s.Equal(int64(12345), retrieved.Size)
}
// Test_V2_SavePackage_WithMetadata tests saving package with metadata
func (s *GORMStoreV2TestSuite) Test_V2_SavePackage_WithMetadata() {
metadataMap := map[string]string{
"author": "Test Author",
"license": "MIT",
"homepage": "https://example.com",
"description": "Test package description",
}
pkg := &metadata.Package{
Registry: "npm",
Name: "meta-package",
Version: "2.0.0",
StorageKey: "npm/meta-package/2.0.0.tgz",
Size: 5000,
CachedAt: time.Now(),
LastAccessed: time.Now(),
Metadata: metadataMap,
}
err := s.store.SavePackage(s.ctx, pkg)
s.NoError(err)
// Verify metadata was saved in separate table
var pkgMetadata PackageMetadataModel
err = s.store.db.Where("package_id = (SELECT id FROM packages WHERE name = ?)", "meta-package").
First(&pkgMetadata).Error
s.NoError(err)
s.Equal("Test Author", pkgMetadata.Author)
s.Equal("MIT", pkgMetadata.License)
s.Equal("https://example.com", pkgMetadata.Homepage)
}
// Test_V2_SavePackage_Upsert tests update on conflict
func (s *GORMStoreV2TestSuite) Test_V2_SavePackage_Upsert() {
// Save initial package
pkg := &metadata.Package{
Registry: "npm",
Name: "upsert-test",
Version: "1.0.0",
StorageKey: "npm/upsert-test/1.0.0.tgz",
Size: 1000,
CachedAt: time.Now(),
LastAccessed: time.Now(),
}
err := s.store.SavePackage(s.ctx, pkg)
s.NoError(err)
// Update same package
pkg.Size = 2000
pkg.ChecksumMD5 = "updated"
err = s.store.SavePackage(s.ctx, pkg)
s.NoError(err)
// Verify updated
retrieved, err := s.store.GetPackage(s.ctx, "npm", "upsert-test", "1.0.0")
s.NoError(err)
s.Equal(int64(2000), retrieved.Size)
s.Equal("updated", retrieved.ChecksumMD5)
}
// Test_V2_GetPackage_NotFound tests getting non-existent package
func (s *GORMStoreV2TestSuite) Test_V2_GetPackage_NotFound() {
_, err := s.store.GetPackage(s.ctx, "npm", "nonexistent", "1.0.0")
s.Error(err)
s.Contains(err.Error(), "not found")
}
// Test_V2_DeletePackage_Success tests soft delete
func (s *GORMStoreV2TestSuite) Test_V2_DeletePackage_Success() {
// Save package
pkg := &metadata.Package{
Registry: "npm",
Name: "delete-test",
Version: "1.0.0",
StorageKey: "npm/delete-test/1.0.0.tgz",
Size: 1000,
CachedAt: time.Now(),
LastAccessed: time.Now(),
}
err := s.store.SavePackage(s.ctx, pkg)
s.NoError(err)
// Delete package (soft delete)
err = s.store.DeletePackage(s.ctx, "npm", "delete-test", "1.0.0")
s.NoError(err)
// Verify deleted (should not be found)
_, err = s.store.GetPackage(s.ctx, "npm", "delete-test", "1.0.0")
s.Error(err)
// Verify soft delete (deleted_at set)
var count int64
s.store.db.Unscoped().Model(&PackageModel{}).
Where("name = ?", "delete-test").
Count(&count)
s.Equal(int64(1), count) // Still in DB, just soft deleted
}
// Test_V2_ListPackages_All tests listing all packages
func (s *GORMStoreV2TestSuite) Test_V2_ListPackages_All() {
// Create multiple packages
for i := 0; i < 5; i++ {
pkg := &metadata.Package{
Registry: "npm",
Name: fmt.Sprintf("package-%d", i),
Version: "1.0.0",
StorageKey: fmt.Sprintf("npm/package-%d/1.0.0.tgz", i),
Size: int64(i * 1000),
CachedAt: time.Now(),
LastAccessed: time.Now(),
}
err := s.store.SavePackage(s.ctx, pkg)
s.NoError(err)
}
// List all packages
packages, err := s.store.ListPackages(s.ctx, &metadata.ListOptions{})
s.NoError(err)
s.Len(packages, 5)
}
// Test_V2_ListPackages_FilterByRegistry tests filtering by registry
func (s *GORMStoreV2TestSuite) Test_V2_ListPackages_FilterByRegistry() {
// Create packages in different registries
registries := []string{"npm", "pypi", "go"}
for _, reg := range registries {
pkg := &metadata.Package{
Registry: reg,
Name: "test-package",
Version: "1.0.0",
StorageKey: fmt.Sprintf("%s/test-package/1.0.0", reg),
Size: 1000,
CachedAt: time.Now(),
LastAccessed: time.Now(),
}
err := s.store.SavePackage(s.ctx, pkg)
s.NoError(err)
}
// Filter by npm registry
packages, err := s.store.ListPackages(s.ctx, &metadata.ListOptions{
Registry: "npm",
})
s.NoError(err)
s.Len(packages, 1)
s.Equal("npm", packages[0].Registry)
}
// Test_V2_ListPackages_Pagination tests pagination
func (s *GORMStoreV2TestSuite) Test_V2_ListPackages_Pagination() {
// Create 10 packages
for i := 0; i < 10; i++ {
pkg := &metadata.Package{
Registry: "npm",
Name: fmt.Sprintf("package-%d", i),
Version: "1.0.0",
StorageKey: fmt.Sprintf("npm/package-%d/1.0.0.tgz", i),
Size: int64(i * 1000),
CachedAt: time.Now(),
LastAccessed: time.Now(),
}
err := s.store.SavePackage(s.ctx, pkg)
s.NoError(err)
}
// Get first page (5 items)
page1, err := s.store.ListPackages(s.ctx, &metadata.ListOptions{
Limit: 5,
Offset: 0,
})
s.NoError(err)
s.Len(page1, 5)
// Get second page (5 items)
page2, err := s.store.ListPackages(s.ctx, &metadata.ListOptions{
Limit: 5,
Offset: 5,
})
s.NoError(err)
s.Len(page2, 5)
// Verify different packages
s.NotEqual(page1[0].Name, page2[0].Name)
}
// Test_V2_UpdateDownloadCount_Success tests incrementing download count
func (s *GORMStoreV2TestSuite) Test_V2_UpdateDownloadCount_Success() {
// Create package
pkg := &metadata.Package{
Registry: "npm",
Name: "download-test",
Version: "1.0.0",
StorageKey: "npm/download-test/1.0.0.tgz",
Size: 1000,
CachedAt: time.Now(),
LastAccessed: time.Now(),
}
err := s.store.SavePackage(s.ctx, pkg)
s.NoError(err)
// Update download count
err = s.store.UpdateDownloadCount(s.ctx, "npm", "download-test", "1.0.0")
s.NoError(err)
// Verify count incremented
retrieved, err := s.store.GetPackage(s.ctx, "npm", "download-test", "1.0.0")
s.NoError(err)
s.Equal(int64(1), retrieved.DownloadCount)
// Update again
err = s.store.UpdateDownloadCount(s.ctx, "npm", "download-test", "1.0.0")
s.NoError(err)
retrieved, err = s.store.GetPackage(s.ctx, "npm", "download-test", "1.0.0")
s.NoError(err)
s.Equal(int64(2), retrieved.DownloadCount)
// Verify download event was recorded
var eventCount int64
s.store.db.Model(&DownloadEventModel{}).Count(&eventCount)
s.Equal(int64(2), eventCount)
}
// Test_V2_Count tests counting packages
func (s *GORMStoreV2TestSuite) Test_V2_Count() {
// Initially zero
count, err := s.store.Count(s.ctx)
s.NoError(err)
s.Equal(0, count)
// Create 3 packages
for i := 0; i < 3; i++ {
pkg := &metadata.Package{
Registry: "npm",
Name: fmt.Sprintf("count-test-%d", i),
Version: "1.0.0",
StorageKey: fmt.Sprintf("npm/count-test-%d/1.0.0.tgz", i),
Size: 1000,
CachedAt: time.Now(),
LastAccessed: time.Now(),
}
err := s.store.SavePackage(s.ctx, pkg)
s.NoError(err)
}
count, err = s.store.Count(s.ctx)
s.NoError(err)
s.Equal(3, count)
}
// Test_V2_GetStats tests aggregated statistics
func (s *GORMStoreV2TestSuite) Test_V2_GetStats() {
// Create packages in different registries
packages := []*metadata.Package{
{Registry: "npm", Name: "pkg1", Version: "1.0.0", StorageKey: "npm/pkg1/1.0.0.tgz", Size: 1000, CachedAt: time.Now(), LastAccessed: time.Now()},
{Registry: "npm", Name: "pkg2", Version: "1.0.0", StorageKey: "npm/pkg2/1.0.0.tgz", Size: 2000, CachedAt: time.Now(), LastAccessed: time.Now()},
{Registry: "pypi", Name: "pkg3", Version: "1.0.0", StorageKey: "pypi/pkg3/1.0.0.tar.gz", Size: 3000, CachedAt: time.Now(), LastAccessed: time.Now()},
}
for _, pkg := range packages {
err := s.store.SavePackage(s.ctx, pkg)
s.NoError(err)
}
// Update download counts
s.store.UpdateDownloadCount(s.ctx, "npm", "pkg1", "1.0.0")
s.store.UpdateDownloadCount(s.ctx, "npm", "pkg1", "1.0.0")
s.store.UpdateDownloadCount(s.ctx, "npm", "pkg2", "1.0.0")
// Get stats for all registries
statsAll, err := s.store.GetStats(s.ctx, "")
s.NoError(err)
s.Equal(int64(3), statsAll.TotalPackages)
s.Equal(int64(6000), statsAll.TotalSize)
s.Equal(int64(3), statsAll.TotalDownloads)
// Get stats for npm registry
statsNpm, err := s.store.GetStats(s.ctx, "npm")
s.NoError(err)
s.Equal("npm", statsNpm.Registry)
s.Equal(int64(2), statsNpm.TotalPackages)
s.Equal(int64(3000), statsNpm.TotalSize)
s.Equal(int64(3), statsNpm.TotalDownloads)
// Get stats for pypi registry
statsPypi, err := s.store.GetStats(s.ctx, "pypi")
s.NoError(err)
s.Equal("pypi", statsPypi.Registry)
s.Equal(int64(1), statsPypi.TotalPackages)
s.Equal(int64(3000), statsPypi.TotalSize)
}
// Test_V2_Health tests database health check
func (s *GORMStoreV2TestSuite) Test_V2_Health() {
err := s.store.Health(s.ctx)
s.NoError(err)
}
// Test_V2_RegistryCache tests registry caching
func (s *GORMStoreV2TestSuite) Test_V2_RegistryCache() {
// Default registries should be cached
s.Contains(s.store.registryCache, "npm")
s.Contains(s.store.registryCache, "pypi")
s.Contains(s.store.registryCache, "go")
// Get registry ID from cache
npmID, err := s.store.getRegistryID("npm")
s.NoError(err)
s.Greater(npmID, int32(0))
// Second call should use cache (no DB query)
npmID2, err := s.store.getRegistryID("npm")
s.NoError(err)
s.Equal(npmID, npmID2)
// Non-existent registry
_, err = s.store.getRegistryID("nonexistent")
s.Error(err)
s.Contains(err.Error(), "not found")
}
// Test_V2_SoftDelete tests soft delete behavior
func (s *GORMStoreV2TestSuite) Test_V2_SoftDelete() {
// Create package
pkg := &metadata.Package{
Registry: "npm",
Name: "soft-delete",
Version: "1.0.0",
StorageKey: "npm/soft-delete/1.0.0.tgz",
Size: 1000,
CachedAt: time.Now(),
LastAccessed: time.Now(),
}
err := s.store.SavePackage(s.ctx, pkg)
s.NoError(err)
// Delete
err = s.store.DeletePackage(s.ctx, "npm", "soft-delete", "1.0.0")
s.NoError(err)
// Count should not include deleted
count, err := s.store.Count(s.ctx)
s.NoError(err)
s.Equal(0, count)
// But record still exists with deleted_at set
var pkgModel PackageModel
err = s.store.db.Unscoped().Where("name = ?", "soft-delete").First(&pkgModel).Error
s.NoError(err)
s.NotNil(pkgModel.DeletedAt)
}
// Test_V2_AggregationWorker tests that aggregation worker is initialized
func (s *GORMStoreV2TestSuite) Test_V2_AggregationWorker() {
s.NotNil(s.store.aggregationWorker)
}
// Test_V2_ConcurrentUpdates tests concurrent download count updates
func (s *GORMStoreV2TestSuite) Test_V2_ConcurrentUpdates() {
// Create package
pkg := &metadata.Package{
Registry: "npm",
Name: "concurrent-test",
Version: "1.0.0",
StorageKey: "npm/concurrent-test/1.0.0.tgz",
Size: 1000,
CachedAt: time.Now(),
LastAccessed: time.Now(),
}
err := s.store.SavePackage(s.ctx, pkg)
s.NoError(err)
// SQLite: Sequential updates only (write lock prevents concurrent writes)
updateCount := 5
for i := 0; i < updateCount; i++ {
err := s.store.UpdateDownloadCount(s.ctx, "npm", "concurrent-test", "1.0.0")
s.NoError(err)
}
// Verify all updates succeeded
retrieved, err := s.store.GetPackage(s.ctx, "npm", "concurrent-test", "1.0.0")
s.NoError(err)
s.Equal(int64(updateCount), retrieved.DownloadCount)
}
// Test_V2_SaveScanResult tests saving a scan result
func (s *GORMStoreV2TestSuite) Test_V2_SaveScanResult() {
// Create a package first
pkg := &metadata.Package{
Registry: "npm",
Name: "test-package",
Version: "1.0.0",
StorageKey: "/cache/npm/test-package-1.0.0.tgz",
Size: 1024,
UpstreamURL: "https://registry.npmjs.org/test-package",
CachedAt: time.Now(),
LastAccessed: time.Now(),
}
err := s.store.SavePackage(s.ctx, pkg)
s.NoError(err)
// Create and save a scan result
scanResult := &metadata.ScanResult{
Registry: "npm",
PackageName: "test-package",
PackageVersion: "1.0.0",
Scanner: "trivy",
Status: metadata.ScanStatusVulnerable,
ScannedAt: time.Now(),
Vulnerabilities: []metadata.Vulnerability{
{
ID: "CVE-2024-0001",
Severity: "HIGH",
Title: "Test vulnerability",
Description: "Test description",
References: []string{"https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2024-0001"},
},
{
ID: "CVE-2024-0002",
Severity: "CRITICAL",
Title: "Critical vulnerability",
Description: "Critical test description",
References: []string{"https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2024-0002"},
},
},
VulnerabilityCount: 2,
Details: map[string]interface{}{
"scan_duration": 42,
"scanner_version": "1.0.0",
},
}
err = s.store.SaveScanResult(s.ctx, scanResult)
s.NoError(err)
// Verify the scan result was saved and package was updated
retrievedPkg, err := s.store.GetPackage(s.ctx, "npm", "test-package", "1.0.0")
s.NoError(err)
s.True(retrievedPkg.SecurityScanned)
}
// Test_V2_GetScanResult tests retrieving a scan result
func (s *GORMStoreV2TestSuite) Test_V2_GetScanResult() {
// Create a package
pkg := &metadata.Package{
Registry: "npm",
Name: "scan-test",
Version: "2.0.0",
StorageKey: "/cache/npm/scan-test-2.0.0.tgz",
Size: 2048,
UpstreamURL: "https://registry.npmjs.org/scan-test",
CachedAt: time.Now(),
LastAccessed: time.Now(),
}
err := s.store.SavePackage(s.ctx, pkg)
s.NoError(err)
// Save a scan result with vulnerabilities
scanResult := &metadata.ScanResult{
Registry: "npm",
PackageName: "scan-test",
PackageVersion: "2.0.0",
Scanner: "grype",
Status: metadata.ScanStatusVulnerable,
ScannedAt: time.Now(),
Vulnerabilities: []metadata.Vulnerability{
{
ID: "CVE-2024-1234",
Severity: "HIGH",
Title: "Test High Severity",
Description: "High severity test",
References: []string{"https://example.com/cve-2024-1234"},
FixedIn: "2.1.0",
},
{
ID: "CVE-2024-5678",
Severity: "MODERATE",
Title: "Test Moderate Severity",
Description: "Moderate severity test",
References: []string{"https://example.com/cve-2024-5678"},
},
},
VulnerabilityCount: 2,
}
err = s.store.SaveScanResult(s.ctx, scanResult)
s.NoError(err)
// Retrieve the scan result
retrieved, err := s.store.GetScanResult(s.ctx, "npm", "scan-test", "2.0.0")
s.NoError(err)
s.NotNil(retrieved)
s.Equal("grype", retrieved.Scanner)
s.Equal(metadata.ScanStatusVulnerable, retrieved.Status)
s.Equal(2, retrieved.VulnerabilityCount)
s.Len(retrieved.Vulnerabilities, 2)
// Verify vulnerability details are retrieved correctly
s.Equal("CVE-2024-1234", retrieved.Vulnerabilities[0].ID)
s.Equal("HIGH", retrieved.Vulnerabilities[0].Severity)
s.Equal("Test High Severity", retrieved.Vulnerabilities[0].Title)
s.Equal("2.1.0", retrieved.Vulnerabilities[0].FixedIn)
s.Len(retrieved.Vulnerabilities[0].References, 1)
}
// Test_V2_GetScanResult_NotFound tests retrieving a non-existent scan result
func (s *GORMStoreV2TestSuite) Test_V2_GetScanResult_NotFound() {
_, err := s.store.GetScanResult(s.ctx, "npm", "nonexistent", "1.0.0")
s.Error(err)
}
// Test_V2_SaveCVEBypass tests saving a CVE bypass
func (s *GORMStoreV2TestSuite) Test_V2_SaveCVEBypass() {
bypass := &metadata.CVEBypass{
Type: metadata.BypassTypeCVE,
Target: "CVE-2024-0001",
Reason: "False positive - not applicable to our use case",
CreatedBy: "admin@example.com",
ExpiresAt: time.Now().Add(30 * 24 * time.Hour), // 30 days
NotifyOnExpiry: true,
Active: true,
}
err := s.store.SaveCVEBypass(s.ctx, bypass)
s.NoError(err)
s.NotEmpty(bypass.ID)
s.NotZero(bypass.CreatedAt)
}
// Test_V2_SaveCVEBypass_Update tests updating an existing CVE bypass
func (s *GORMStoreV2TestSuite) Test_V2_SaveCVEBypass_Update() {
// Create initial bypass
bypass := &metadata.CVEBypass{
Type: metadata.BypassTypeCVE,
Target: "CVE-2024-0002",
Reason: "Initial reason",
CreatedBy: "admin@example.com",
ExpiresAt: time.Now().Add(30 * 24 * time.Hour),
NotifyOnExpiry: false,
Active: true,
}
err := s.store.SaveCVEBypass(s.ctx, bypass)
s.NoError(err)
s.NotEmpty(bypass.ID)
// Update the bypass
bypass.Reason = "Updated reason"
bypass.NotifyOnExpiry = true
err = s.store.SaveCVEBypass(s.ctx, bypass)
s.NoError(err)
}
// Test_V2_GetActiveCVEBypasses tests retrieving active CVE bypasses
func (s *GORMStoreV2TestSuite) Test_V2_GetActiveCVEBypasses() {
// Create active bypass with unique target
uniqueTarget := fmt.Sprintf("CVE-2024-TEST-%d", time.Now().UnixNano())
activeBypass := &metadata.CVEBypass{
Type: metadata.BypassTypeCVE,
Target: uniqueTarget,
Reason: "Active bypass",
CreatedBy: "admin@example.com",
ExpiresAt: time.Now().Add(30 * 24 * time.Hour),
Active: true,
}
err := s.store.SaveCVEBypass(s.ctx, activeBypass)
s.NoError(err)
// Create expired bypass
expiredBypass := &metadata.CVEBypass{
Type: metadata.BypassTypeCVE,
Target: "CVE-2024-0004",
Reason: "Expired bypass",
CreatedBy: "admin@example.com",
ExpiresAt: time.Now().Add(-24 * time.Hour), // Expired yesterday
Active: true,
}
err = s.store.SaveCVEBypass(s.ctx, expiredBypass)
s.NoError(err)
// Create inactive bypass
inactiveBypass := &metadata.CVEBypass{
Type: metadata.BypassTypeCVE,
Target: "CVE-2024-0005",
Reason: "Inactive bypass",
CreatedBy: "admin@example.com",
ExpiresAt: time.Now().Add(30 * 24 * time.Hour),
Active: false,
}
err = s.store.SaveCVEBypass(s.ctx, inactiveBypass)
s.NoError(err)
// Retrieve active bypasses
bypasses, err := s.store.GetActiveCVEBypasses(s.ctx)
s.NoError(err)
// Should contain our active bypass, but may contain others from parallel tests
found := false
for _, b := range bypasses {
if b.Target == uniqueTarget {
found = true
break
}
// All bypasses should be active and non-expired
s.True(b.Active)
s.True(b.ExpiresAt.After(time.Now()))
}
s.True(found, "Should find our unique active bypass")
}
// Test_V2_ListCVEBypasses tests listing CVE bypasses with filters
func (s *GORMStoreV2TestSuite) Test_V2_ListCVEBypasses() {
// Create multiple bypasses with unique targets
nano := time.Now().UnixNano()
bypasses := []*metadata.CVEBypass{
{
Type: metadata.BypassTypeCVE,
Target: fmt.Sprintf("CVE-2024-LIST-%d-1", nano),
Reason: "Test 1",
CreatedBy: "admin@example.com",
ExpiresAt: time.Now().Add(30 * 24 * time.Hour),
Active: true,
},
{
Type: metadata.BypassTypePackage,
Target: fmt.Sprintf("npm/vulnerable-package@%d", nano),
Reason: "Test 2",
CreatedBy: "admin@example.com",
ExpiresAt: time.Now().Add(15 * 24 * time.Hour),
Active: true,
},
{
Type: metadata.BypassTypeCVE,
Target: fmt.Sprintf("CVE-2024-LIST-%d-2", nano),
Reason: "Test 3",
CreatedBy: "admin@example.com",
ExpiresAt: time.Now().Add(-1 * time.Hour), // Expired
Active: true,
},
}
for _, b := range bypasses {
err := s.store.SaveCVEBypass(s.ctx, b)
s.NoError(err)
}
// List only CVE type
opts := &metadata.BypassListOptions{
Type: metadata.BypassTypeCVE,
}
cveOnly, err := s.store.ListCVEBypasses(s.ctx, opts)
s.NoError(err)
for _, b := range cveOnly {
s.Equal(metadata.BypassTypeCVE, b.Type)
}
// List only non-expired
opts = &metadata.BypassListOptions{
IncludeExpired: false,
}
nonExpired, err := s.store.ListCVEBypasses(s.ctx, opts)
s.NoError(err)
for _, b := range nonExpired {
s.True(b.ExpiresAt.After(time.Now()))
}
// Test pagination
opts = &metadata.BypassListOptions{
Limit: 1,
Offset: 0,
}
page1, err := s.store.ListCVEBypasses(s.ctx, opts)
s.NoError(err)
s.LessOrEqual(len(page1), 1) // Should be at most 1
}
// Test_V2_DeleteCVEBypass tests deleting a CVE bypass
func (s *GORMStoreV2TestSuite) Test_V2_DeleteCVEBypass() {
// Create a bypass
bypass := &metadata.CVEBypass{
Type: metadata.BypassTypeCVE,
Target: "CVE-2024-0008",
Reason: "To be deleted",
CreatedBy: "admin@example.com",
ExpiresAt: time.Now().Add(30 * 24 * time.Hour),
Active: true,
}
err := s.store.SaveCVEBypass(s.ctx, bypass)
s.NoError(err)
s.NotEmpty(bypass.ID)
// Delete the bypass
err = s.store.DeleteCVEBypass(s.ctx, bypass.ID)
s.NoError(err)
// Verify it's no longer in active bypasses
active, err := s.store.GetActiveCVEBypasses(s.ctx)
s.NoError(err)
for _, b := range active {
s.NotEqual(bypass.ID, b.ID)
}
}
// Test_V2_DeleteCVEBypass_NotFound tests deleting a non-existent bypass
func (s *GORMStoreV2TestSuite) Test_V2_DeleteCVEBypass_NotFound() {
err := s.store.DeleteCVEBypass(s.ctx, "99999999")
s.Error(err)
}
// Test_V2_DeleteCVEBypass_InvalidID tests deleting with invalid ID
func (s *GORMStoreV2TestSuite) Test_V2_DeleteCVEBypass_InvalidID() {
err := s.store.DeleteCVEBypass(s.ctx, "invalid-id")
s.Error(err)
}
// Test_V2_CleanupExpiredBypasses tests cleaning up expired bypasses
func (s *GORMStoreV2TestSuite) Test_V2_CleanupExpiredBypasses() {
// Create expired bypasses
for i := 0; i < 3; i++ {
bypass := &metadata.CVEBypass{
Type: metadata.BypassTypeCVE,
Target: fmt.Sprintf("CVE-2024-00%d", 10+i),
Reason: "Expired bypass",
CreatedBy: "admin@example.com",
ExpiresAt: time.Now().Add(-24 * time.Hour), // Expired
Active: true,
}
err := s.store.SaveCVEBypass(s.ctx, bypass)
s.NoError(err)
}
// Create active bypass (should not be deleted)
activeBypass := &metadata.CVEBypass{
Type: metadata.BypassTypeCVE,
Target: "CVE-2024-0999",
Reason: "Active bypass",
CreatedBy: "admin@example.com",
ExpiresAt: time.Now().Add(30 * 24 * time.Hour),
Active: true,
}
err := s.store.SaveCVEBypass(s.ctx, activeBypass)
s.NoError(err)
// Cleanup expired bypasses
count, err := s.store.CleanupExpiredBypasses(s.ctx)
s.NoError(err)
s.GreaterOrEqual(count, 3) // At least the 3 we just created
// Verify active bypass is still there
active, err := s.store.GetActiveCVEBypasses(s.ctx)
s.NoError(err)
found := false
for _, b := range active {
if b.Target == "CVE-2024-0999" {
found = true
break
}
}
s.True(found, "Active bypass should still exist")
}
pkg/metadata/gormstore/migrations.go
+228
View File
@@ -0,0 +1,228 @@
package gormstore
import (
"github.com/go-gormigrate/gormigrate/v2"
"gorm.io/gorm"
)
// GetMigrations returns all database migrations for gormigrate
func GetMigrations() []*gormigrate.Migration {
return []*gormigrate.Migration{
{
ID: "202601030001",
Migrate: func(tx *gorm.DB) error {
// Migration: Create V2 schema
return migrateToV2(tx)
},
Rollback: func(tx *gorm.DB) error {
// Rollback: Drop V2 schema (careful!)
return rollbackFromV2(tx)
},
},
// Future migrations go here
// {
// ID: "202601040001",
// Migrate: func(tx *gorm.DB) error {
// // Add new column, index, etc.
// return tx.Exec("ALTER TABLE packages ADD COLUMN new_field VARCHAR(255)").Error
// },
// Rollback: func(tx *gorm.DB) error {
// return tx.Exec("ALTER TABLE packages DROP COLUMN new_field").Error
// },
// },
}
}
// migrateToV2 creates the complete V2 schema
func migrateToV2(tx *gorm.DB) error {
// Get dialect name for database-specific features
dialectName := tx.Dialector.Name()
// Step 1: Create all tables using GORM AutoMigrate
// This handles cross-database compatibility automatically
if err := tx.AutoMigrate(GetAllModels()...); err != nil {
return err
}
// Step 2: Seed default registries
registries := []RegistryModel{
{Name: "npm", DisplayName: "NPM Registry", UpstreamURL: "https://registry.npmjs.org", Enabled: true, ScanByDefault: true},
{Name: "pypi", DisplayName: "PyPI", UpstreamURL: "https://pypi.org", Enabled: true, ScanByDefault: true},
{Name: "go", DisplayName: "Go Modules", UpstreamURL: "https://proxy.golang.org", Enabled: true, ScanByDefault: true},
}
for _, reg := range registries {
// Upsert: create if not exists
if err := tx.Where("name = ?", reg.Name).FirstOrCreate(&reg).Error; err != nil {
return err
}
}
// Step 3: Create database-specific optimizations
if dialectName == "postgres" {
if err := createPostgreSQLOptimizations(tx); err != nil {
return err
}
} else if dialectName == "mysql" {
if err := createMySQLOptimizations(tx); err != nil {
return err
}
}
return nil
}
// createPostgreSQLOptimizations adds PostgreSQL-specific features
func createPostgreSQLOptimizations(tx *gorm.DB) error {
optimizations := []string{
// Create GIN indexes for JSONB columns
`CREATE INDEX IF NOT EXISTS idx_package_metadata_keywords_gin
ON package_metadata USING GIN(keywords)`,
`CREATE INDEX IF NOT EXISTS idx_package_metadata_raw_gin
ON package_metadata USING GIN(raw_metadata)`,
`CREATE INDEX IF NOT EXISTS idx_vulnerabilities_references_gin
ON vulnerabilities USING GIN(references)`,
// Create partial indexes (only non-deleted records)
`CREATE INDEX IF NOT EXISTS idx_packages_active
ON packages(registry_id, name, version) WHERE deleted_at IS NULL`,
`CREATE INDEX IF NOT EXISTS idx_packages_vulnerable
ON packages(vulnerability_count, highest_severity)
WHERE vulnerability_count > 0 AND deleted_at IS NULL`,
// Create view for vulnerable packages
`CREATE OR REPLACE VIEW v_vulnerable_packages AS
SELECT
r.name AS registry,
p.name,
p.version,
p.vulnerability_count,
p.highest_severity,
p.last_scanned_at
FROM packages p
JOIN registries r ON p.registry_id = r.id
WHERE p.vulnerability_count > 0 AND p.deleted_at IS NULL
ORDER BY
CASE p.highest_severity
WHEN 'critical' THEN 1
WHEN 'high' THEN 2
WHEN 'medium' THEN 3
WHEN 'low' THEN 4
ELSE 5
END,
p.vulnerability_count DESC`,
// Create function for automatic partition creation
`CREATE OR REPLACE FUNCTION create_next_month_partitions()
RETURNS void AS $$
DECLARE
next_month DATE := date_trunc('month', NOW() + INTERVAL '2 months');
partition_name TEXT;
start_date TEXT;
end_date TEXT;
BEGIN
-- Download events partition
partition_name := 'download_events_' || to_char(next_month, 'YYYY_MM');
start_date := to_char(next_month, 'YYYY-MM-DD');
end_date := to_char(next_month + INTERVAL '1 month', 'YYYY-MM-DD');
EXECUTE format('CREATE TABLE IF NOT EXISTS %I PARTITION OF download_events FOR VALUES FROM (%L) TO (%L)',
partition_name, start_date, end_date);
-- Audit log partition
partition_name := 'audit_log_' || to_char(next_month, 'YYYY_MM');
EXECUTE format('CREATE TABLE IF NOT EXISTS %I PARTITION OF audit_log FOR VALUES FROM (%L) TO (%L)',
partition_name, start_date, end_date);
RAISE NOTICE 'Created partitions for %', to_char(next_month, 'YYYY-MM');
END;
$$ LANGUAGE plpgsql`,
}
for _, sql := range optimizations {
if err := tx.Exec(sql).Error; err != nil {
// Log warning but don't fail migration
// Some optimizations might already exist
continue
}
}
return nil
}
// createMySQLOptimizations adds MySQL-specific features
func createMySQLOptimizations(tx *gorm.DB) error {
optimizations := []string{
// Create view for vulnerable packages
`CREATE OR REPLACE VIEW v_vulnerable_packages AS
SELECT
r.name AS registry,
p.name,
p.version,
p.vulnerability_count,
p.highest_severity,
p.last_scanned_at
FROM packages p
JOIN registries r ON p.registry_id = r.id
WHERE p.vulnerability_count > 0 AND p.deleted_at IS NULL
ORDER BY
CASE p.highest_severity
WHEN 'critical' THEN 1
WHEN 'high' THEN 2
WHEN 'medium' THEN 3
WHEN 'low' THEN 4
ELSE 5
END,
p.vulnerability_count DESC`,
}
for _, sql := range optimizations {
if err := tx.Exec(sql).Error; err != nil {
continue
}
}
return nil
}
// rollbackFromV2 drops all V2 tables (USE WITH CAUTION!)
func rollbackFromV2(tx *gorm.DB) error {
// Drop in reverse order to respect foreign keys
tables := []string{
"audit_log",
"download_stats_daily",
"download_stats_hourly",
"download_events",
"cve_bypasses",
"scan_results",
"package_vulnerabilities",
"vulnerabilities",
"package_metadata",
"packages",
"registries",
}
// Drop PostgreSQL-specific objects
if tx.Dialector.Name() == "postgres" {
tx.Exec("DROP VIEW IF EXISTS v_vulnerable_packages")
tx.Exec("DROP FUNCTION IF EXISTS create_next_month_partitions()")
}
// Drop MySQL-specific objects
if tx.Dialector.Name() == "mysql" {
tx.Exec("DROP VIEW IF EXISTS v_vulnerable_packages")
}
// Drop all tables
for _, table := range tables {
if err := tx.Migrator().DropTable(table); err != nil {
// Continue even if table doesn't exist
continue
}
}
return nil
}
pkg/metadata/gormstore/models_v2.go
+328
View File
@@ -0,0 +1,328 @@
package gormstore
import (
"database/sql/driver"
"encoding/json"
"time"
"gorm.io/gorm"
)
// BaseModel provides common fields for all models with audit trail
type BaseModel struct {
CreatedAt time.Time `gorm:"not null"`
UpdatedAt time.Time `gorm:"not null"`
DeletedAt gorm.DeletedAt `gorm:"index"` // Soft delete support (auto-generated index name per table)
}
// RegistryModel represents package registries (normalized)
// This eliminates repetition of "npm", "pypi", "go" across millions of rows
type RegistryModel struct {
ID int32 `gorm:"primaryKey;autoIncrement"`
Name string `gorm:"uniqueIndex:idx_registry_name;not null;size:50"` // npm, pypi, go
DisplayName string `gorm:"not null;size:100"` // NPM Registry, PyPI, Go Modules
UpstreamURL string `gorm:"not null;size:512"` // https://registry.npmjs.org
Enabled bool `gorm:"not null;default:true;index:idx_registry_enabled"`
ScanByDefault bool `gorm:"not null;default:true"`
BaseModel
}
func (RegistryModel) TableName() string {
return "registries"
}
// PackageModel represents the core package data (optimized)
type PackageModel struct {
ID int64 `gorm:"primaryKey;autoIncrement"`
RegistryID int32 `gorm:"not null;index:idx_package_registry_name_version,priority:1"` // Foreign key to registries
Name string `gorm:"not null;size:255;index:idx_package_name;index:idx_package_registry_name_version,priority:2"`
Version string `gorm:"not null;size:100;index:idx_package_registry_name_version,priority:3"`
// Storage information
StorageKey string `gorm:"not null;uniqueIndex:idx_package_storage_key;size:512"`
Size int64 `gorm:"not null;index:idx_package_size"` // For storage quota queries
ChecksumMD5 string `gorm:"size:32;index:idx_package_md5"`
ChecksumSHA256 string `gorm:"size:64;index:idx_package_sha256"`
UpstreamURL string `gorm:"size:1024"`
// Cache management
CachedAt time.Time `gorm:"not null;index:idx_package_cached_at"`
LastAccessed time.Time `gorm:"not null;index:idx_package_last_accessed"` // For LRU eviction
ExpiresAt *time.Time `gorm:"index:idx_package_expires_at"` // For cache invalidation
AccessCount int64 `gorm:"not null;default:0;index:idx_package_access_count"` // Total access count (denormalized for performance)
// Security
SecurityScanned bool `gorm:"not null;default:false;index:idx_package_security_scanned"`
LastScannedAt *time.Time `gorm:"index:idx_package_last_scanned"`
VulnerabilityCount int `gorm:"not null;default:0;index:idx_package_vuln_count"` // Denormalized for fast filtering
HighestSeverity string `gorm:"size:20;index:idx_package_severity"` // critical, high, medium, low, none
CriticalCount int `gorm:"not null;default:0"` // Count of critical vulnerabilities
HighCount int `gorm:"not null;default:0"` // Count of high vulnerabilities
ModerateCount int `gorm:"not null;default:0"` // Count of moderate vulnerabilities
LowCount int `gorm:"not null;default:0"` // Count of low vulnerabilities
// Authentication
RequiresAuth bool `gorm:"not null;default:false;index:idx_package_requires_auth"`
AuthProvider string `gorm:"size:50;index:idx_package_auth_provider"` // github, gitlab, custom
BaseModel
// Relationships
Registry RegistryModel `gorm:"foreignKey:RegistryID;constraint:OnUpdate:CASCADE,OnDelete:RESTRICT"`
Metadata *PackageMetadataModel `gorm:"foreignKey:PackageID;constraint:OnUpdate:CASCADE,OnDelete:CASCADE"`
ScanResults []ScanResultModel `gorm:"foreignKey:PackageID;constraint:OnUpdate:CASCADE,OnDelete:CASCADE"`
Vulnerabilities []PackageVulnerabilityModel `gorm:"foreignKey:PackageID;constraint:OnUpdate:CASCADE,OnDelete:CASCADE"`
}
func (PackageModel) TableName() string {
return "packages"
}
// BeforeCreate hook to set access count
func (p *PackageModel) BeforeCreate(tx *gorm.DB) error {
if p.AccessCount == 0 {
p.AccessCount = 0
}
return nil
}
// PackageMetadataModel stores structured package metadata (1:1 with packages)
// Separated from main table to reduce row size and improve query performance
type PackageMetadataModel struct {
PackageID int64 `gorm:"primaryKey;not null"` // 1:1 relationship
Author string `gorm:"size:255;index:idx_metadata_author"`
License string `gorm:"size:100;index:idx_metadata_license"`
Homepage string `gorm:"size:512"`
Repository string `gorm:"size:512"`
Description string `gorm:"type:text"`
Keywords PostgresArray `gorm:"type:text"` // JSONB array for PostgreSQL, JSON for MySQL/SQLite
RawMetadata JSONBField `gorm:"type:jsonb"` // Full metadata as JSONB (PostgreSQL) or JSON
BaseModel
}
func (PackageMetadataModel) TableName() string {
return "package_metadata"
}
// ScanResultModel represents security scan results (optimized)
type ScanResultModel struct {
ID int64 `gorm:"primaryKey;autoIncrement"`
PackageID int64 `gorm:"not null;index:idx_scan_package_scanner,priority:1"` // Foreign key
Scanner string `gorm:"not null;size:50;index:idx_scan_scanner;index:idx_scan_package_scanner,priority:2"`
ScannedAt time.Time `gorm:"not null;index:idx_scan_scanned_at"`
Status string `gorm:"not null;size:20;index:idx_scan_status"` // success, failed, pending
VulnCount int `gorm:"not null;default:0;index:idx_scan_vuln_count"`
CriticalCount int `gorm:"not null;default:0"`
HighCount int `gorm:"not null;default:0"`
MediumCount int `gorm:"not null;default:0"`
LowCount int `gorm:"not null;default:0"`
ScanDuration int `gorm:"not null;default:0"` // milliseconds
Details JSONBField `gorm:"type:jsonb"` // Scanner-specific details
BaseModel
Package PackageModel `gorm:"foreignKey:PackageID;constraint:OnUpdate:CASCADE,OnDelete:CASCADE"`
}
func (ScanResultModel) TableName() string {
return "scan_results"
}
// VulnerabilityModel represents unique vulnerabilities (normalized)
type VulnerabilityModel struct {
ID int64 `gorm:"primaryKey;autoIncrement"`
CVEID string `gorm:"uniqueIndex:idx_vuln_cve_id;not null;size:50"` // CVE-2021-12345
Title string `gorm:"not null;size:512"`
Description string `gorm:"type:text"`
Severity string `gorm:"not null;size:20;index:idx_vuln_severity"` // critical, high, medium, low
CVSS float32 `gorm:"index:idx_vuln_cvss"` // CVSS score for sorting
PublishedAt time.Time `gorm:"not null;index:idx_vuln_published"`
FixedVersion string `gorm:"size:100"` // First version where it's fixed
References PostgresArray `gorm:"type:text"` // URLs to advisories
BaseModel
}
func (VulnerabilityModel) TableName() string {
return "vulnerabilities"
}
// PackageVulnerabilityModel is a many-to-many relationship between packages and vulnerabilities
type PackageVulnerabilityModel struct {
ID int64 `gorm:"primaryKey;autoIncrement"`
PackageID int64 `gorm:"not null;index:idx_pkg_vuln_package,priority:1;index:idx_pkg_vuln_composite,priority:1"`
VulnerabilityID int64 `gorm:"not null;index:idx_pkg_vuln_vuln,priority:1;index:idx_pkg_vuln_composite,priority:2"`
Scanner string `gorm:"not null;size:50;index:idx_pkg_vuln_scanner"`
DetectedAt time.Time `gorm:"not null;index:idx_pkg_vuln_detected"`
Bypassed bool `gorm:"not null;default:false;index:idx_pkg_vuln_bypassed"`
BypassID *int64 `gorm:"index:idx_pkg_vuln_bypass_id"` // Reference to bypass if applicable
BaseModel
Package PackageModel `gorm:"foreignKey:PackageID;constraint:OnUpdate:CASCADE,OnDelete:CASCADE"`
Vulnerability VulnerabilityModel `gorm:"foreignKey:VulnerabilityID;constraint:OnUpdate:CASCADE,OnDelete:CASCADE"`
}
func (PackageVulnerabilityModel) TableName() string {
return "package_vulnerabilities"
}
// CVEBypassModel represents CVE bypass rules (improved)
type CVEBypassModel struct {
ID int64 `gorm:"primaryKey;autoIncrement"`
Type string `gorm:"not null;size:20;index:idx_bypass_type"` // cve, package, registry
Target string `gorm:"not null;size:512;index:idx_bypass_target"` // CVE-ID, package name, etc.
Reason string `gorm:"not null;type:text"`
CreatedBy string `gorm:"not null;size:255;index:idx_bypass_created_by"`
ExpiresAt time.Time `gorm:"not null;index:idx_bypass_expires_at"`
NotifyOnExpiry bool `gorm:"not null;default:false"`
Active bool `gorm:"not null;default:true;index:idx_bypass_active"`
UsageCount int64 `gorm:"not null;default:0"` // How many times this bypass has been used
LastUsedAt *time.Time `gorm:"index:idx_bypass_last_used"`
// Scope limiting (optional)
RegistryID *int32 `gorm:"index:idx_bypass_registry"` // NULL = all registries
PackageID *int64 `gorm:"index:idx_bypass_package"` // NULL = all packages
BaseModel
}
func (CVEBypassModel) TableName() string {
return "cve_bypasses"
}
// DownloadEventModel represents raw download events (partitioned by month)
// This table should use PostgreSQL partitioning or time-series DB features
type DownloadEventModel struct {
ID int64 `gorm:"primaryKey;autoIncrement"`
PackageID int64 `gorm:"not null;index:idx_download_package,priority:1"`
RegistryID int32 `gorm:"not null;index:idx_download_registry"`
DownloadedAt time.Time `gorm:"not null;index:idx_download_time;index:idx_download_package,priority:2"` // Partition key
UserAgent string `gorm:"size:512"` // For analytics
IPAddress string `gorm:"size:45;index:idx_download_ip"` // IPv6 support
Authenticated bool `gorm:"not null;default:false"`
Username string `gorm:"size:255;index:idx_download_user"`
// No BaseModel - this is append-only, no updates/deletes on individual rows
// Partitioned tables handle cleanup via DROP PARTITION
}
func (DownloadEventModel) TableName() string {
return "download_events"
}
// DownloadStatsHourlyModel represents pre-aggregated hourly statistics (partitioned)
type DownloadStatsHourlyModel struct {
ID int64 `gorm:"primaryKey;autoIncrement"`
RegistryID int32 `gorm:"not null;index:idx_stats_hourly_composite,priority:1"`
PackageID *int64 `gorm:"index:idx_stats_hourly_package"` // NULL = all packages in registry
TimeBucket time.Time `gorm:"not null;index:idx_stats_hourly_composite,priority:2"` // Truncated to hour
DownloadCount int64 `gorm:"not null;default:0"`
UniqueIPs int64 `gorm:"not null;default:0"` // Unique downloaders
AuthDownloads int64 `gorm:"not null;default:0"` // Authenticated downloads
BaseModel
}
func (DownloadStatsHourlyModel) TableName() string {
return "download_stats_hourly"
}
// DownloadStatsDailyModel represents pre-aggregated daily statistics
type DownloadStatsDailyModel struct {
ID int64 `gorm:"primaryKey;autoIncrement"`
RegistryID int32 `gorm:"not null;index:idx_stats_daily_composite,priority:1"`
PackageID *int64 `gorm:"index:idx_stats_daily_package"` // NULL = all packages in registry
TimeBucket time.Time `gorm:"not null;index:idx_stats_daily_composite,priority:2"` // Truncated to day
DownloadCount int64 `gorm:"not null;default:0"`
UniqueIPs int64 `gorm:"not null;default:0"`
AuthDownloads int64 `gorm:"not null;default:0"`
TopUserAgents JSONBField `gorm:"type:jsonb"` // Top 10 user agents
BaseModel
}
func (DownloadStatsDailyModel) TableName() string {
return "download_stats_daily"
}
// AuditLogModel tracks all important changes (optional, for compliance)
type AuditLogModel struct {
ID int64 `gorm:"primaryKey;autoIncrement"`
EntityType string `gorm:"not null;size:50;index:idx_audit_entity_type"` // package, bypass, registry
EntityID int64 `gorm:"not null;index:idx_audit_entity_id"`
Action string `gorm:"not null;size:20;index:idx_audit_action"` // create, update, delete
Username string `gorm:"not null;size:255;index:idx_audit_username"`
Timestamp time.Time `gorm:"not null;index:idx_audit_timestamp"`
Changes JSONBField `gorm:"type:jsonb"` // Before/after values
IPAddress string `gorm:"size:45"`
UserAgent string `gorm:"size:512"`
// No BaseModel - append-only audit log
}
func (AuditLogModel) TableName() string {
return "audit_log"
}
// JSONBField is a custom type for JSONB (PostgreSQL) / JSON (MySQL/SQLite)
type JSONBField map[string]interface{}
func (j JSONBField) Value() (driver.Value, error) {
if j == nil {
return nil, nil
}
return json.Marshal(j)
}
func (j *JSONBField) Scan(value interface{}) error {
if value == nil {
*j = nil
return nil
}
bytes, ok := value.([]byte)
if !ok {
return nil
}
return json.Unmarshal(bytes, j)
}
// PostgresArray is a custom type for PostgreSQL arrays stored as JSON
type PostgresArray []string
func (a PostgresArray) Value() (driver.Value, error) {
if a == nil {
return nil, nil
}
return json.Marshal(a)
}
func (a *PostgresArray) Scan(value interface{}) error {
if value == nil {
*a = nil
return nil
}
bytes, ok := value.([]byte)
if !ok {
return nil
}
return json.Unmarshal(bytes, a)
}
// GetAllModels returns all models for GORM auto-migration
func GetAllModels() []interface{} {
return []interface{}{
&RegistryModel{},
&PackageModel{},
&PackageMetadataModel{},
&ScanResultModel{},
&VulnerabilityModel{},
&PackageVulnerabilityModel{},
&CVEBypassModel{},
&DownloadEventModel{},
&DownloadStatsHourlyModel{},
&DownloadStatsDailyModel{},
&AuditLogModel{},
}
}
pkg/metadata/gormstore/partition_manager.go
+380
View File
@@ -0,0 +1,380 @@
package gormstore
import (
"fmt"
"time"
"github.com/rs/zerolog/log"
"gorm.io/gorm"
)
// PartitionManager handles automatic partition creation and cleanup for PostgreSQL
type PartitionManager struct {
db *gorm.DB
}
// NewPartitionManager creates a new partition manager
func NewPartitionManager(db *gorm.DB) *PartitionManager {
return &PartitionManager{db: db}
}
// EnsurePartitions ensures required partitions exist for current and future months
func (pm *PartitionManager) EnsurePartitions() error {
// Check if we're using PostgreSQL
if pm.db.Dialector.Name() != "postgres" {
log.Debug().Msg("Partitioning only supported on PostgreSQL, skipping")
return nil
}
log.Info().Msg("Ensuring partitions exist")
// Create partitions for download_events
if err := pm.ensureDownloadEventPartitions(); err != nil {
return err
}
// Create partitions for audit_log
if err := pm.ensureAuditLogPartitions(); err != nil {
return err
}
// Set up automatic partition creation
if err := pm.createPartitionFunction(); err != nil {
log.Warn().Err(err).Msg("Failed to create partition function (may already exist)")
}
return nil
}
// ensureDownloadEventPartitions creates download_events partitions
func (pm *PartitionManager) ensureDownloadEventPartitions() error {
// Check if table is already partitioned
var isPartitioned bool
err := pm.db.Raw(`
SELECT EXISTS (
SELECT 1 FROM pg_partitioned_table
WHERE partrelid = 'download_events'::regclass
)
`).Scan(&isPartitioned).Error
if err != nil {
return err
}
if !isPartitioned {
log.Info().Msg("Converting download_events to partitioned table")
// Rename existing table
if err := pm.db.Exec("ALTER TABLE IF EXISTS download_events RENAME TO download_events_old").Error; err != nil {
log.Warn().Err(err).Msg("Could not rename old table (may not exist)")
}
// Create partitioned table
createTableSQL := `
CREATE TABLE IF NOT EXISTS download_events (
id BIGSERIAL,
package_id BIGINT NOT NULL,
registry_id INTEGER NOT NULL,
downloaded_at TIMESTAMP NOT NULL,
user_agent VARCHAR(512),
ip_address VARCHAR(45),
authenticated BOOLEAN NOT NULL DEFAULT FALSE,
username VARCHAR(255)
) PARTITION BY RANGE (downloaded_at)
`
if err := pm.db.Exec(createTableSQL).Error; err != nil {
return fmt.Errorf("failed to create partitioned table: %w", err)
}
log.Info().Msg("Created partitioned download_events table")
}
// Create partitions for past 3 months, current month, and next 3 months
now := time.Now()
for i := -3; i <= 3; i++ {
month := now.AddDate(0, i, 0)
if err := pm.createDownloadEventPartition(month); err != nil {
log.Error().Err(err).Time("month", month).Msg("Failed to create partition")
}
}
return nil
}
// createDownloadEventPartition creates a partition for a specific month
func (pm *PartitionManager) createDownloadEventPartition(month time.Time) error {
// Truncate to start of month
startOfMonth := time.Date(month.Year(), month.Month(), 1, 0, 0, 0, 0, time.UTC)
endOfMonth := startOfMonth.AddDate(0, 1, 0)
partitionName := fmt.Sprintf("download_events_%d_%02d", month.Year(), month.Month())
// Check if partition already exists
var exists bool
err := pm.db.Raw("SELECT EXISTS (SELECT 1 FROM pg_tables WHERE tablename = ?)", partitionName).Scan(&exists).Error
if err != nil {
return err
}
if exists {
log.Debug().Str("partition", partitionName).Msg("Partition already exists")
return nil
}
// Create partition
createPartitionSQL := fmt.Sprintf(`
CREATE TABLE %s PARTITION OF download_events
FOR VALUES FROM ('%s') TO ('%s')
`, partitionName, startOfMonth.Format("2006-01-02"), endOfMonth.Format("2006-01-02"))
if err := pm.db.Exec(createPartitionSQL).Error; err != nil {
return fmt.Errorf("failed to create partition %s: %w", partitionName, err)
}
// Create indexes on partition
indexSQL := []string{
fmt.Sprintf("CREATE INDEX IF NOT EXISTS %s_package_idx ON %s(package_id, downloaded_at)", partitionName, partitionName),
fmt.Sprintf("CREATE INDEX IF NOT EXISTS %s_registry_idx ON %s(registry_id)", partitionName, partitionName),
fmt.Sprintf("CREATE INDEX IF NOT EXISTS %s_time_idx ON %s(downloaded_at)", partitionName, partitionName),
}
for _, sql := range indexSQL {
if err := pm.db.Exec(sql).Error; err != nil {
log.Warn().Err(err).Str("sql", sql).Msg("Failed to create index")
}
}
log.Info().Str("partition", partitionName).Msg("Created partition")
return nil
}
// ensureAuditLogPartitions creates audit_log partitions
func (pm *PartitionManager) ensureAuditLogPartitions() error {
// Check if table exists
var exists bool
err := pm.db.Raw("SELECT EXISTS (SELECT 1 FROM pg_tables WHERE tablename = 'audit_log')").Scan(&exists).Error
if err != nil {
return err
}
if !exists {
// Create partitioned table
createTableSQL := `
CREATE TABLE IF NOT EXISTS audit_log (
id BIGSERIAL,
entity_type VARCHAR(50) NOT NULL,
entity_id BIGINT NOT NULL,
action VARCHAR(20) NOT NULL,
username VARCHAR(255) NOT NULL,
timestamp TIMESTAMP NOT NULL,
changes JSONB,
ip_address VARCHAR(45),
user_agent VARCHAR(512)
) PARTITION BY RANGE (timestamp)
`
if err := pm.db.Exec(createTableSQL).Error; err != nil {
return fmt.Errorf("failed to create audit_log table: %w", err)
}
log.Info().Msg("Created partitioned audit_log table")
}
// Create partitions for past month, current month, and next 2 months
now := time.Now()
for i := -1; i <= 2; i++ {
month := now.AddDate(0, i, 0)
if err := pm.createAuditLogPartition(month); err != nil {
log.Error().Err(err).Time("month", month).Msg("Failed to create audit partition")
}
}
return nil
}
// createAuditLogPartition creates a partition for a specific month
func (pm *PartitionManager) createAuditLogPartition(month time.Time) error {
startOfMonth := time.Date(month.Year(), month.Month(), 1, 0, 0, 0, 0, time.UTC)
endOfMonth := startOfMonth.AddDate(0, 1, 0)
partitionName := fmt.Sprintf("audit_log_%d_%02d", month.Year(), month.Month())
// Check if partition already exists
var exists bool
err := pm.db.Raw("SELECT EXISTS (SELECT 1 FROM pg_tables WHERE tablename = ?)", partitionName).Scan(&exists).Error
if err != nil {
return err
}
if exists {
return nil
}
// Create partition
createPartitionSQL := fmt.Sprintf(`
CREATE TABLE %s PARTITION OF audit_log
FOR VALUES FROM ('%s') TO ('%s')
`, partitionName, startOfMonth.Format("2006-01-02"), endOfMonth.Format("2006-01-02"))
if err := pm.db.Exec(createPartitionSQL).Error; err != nil {
return fmt.Errorf("failed to create partition %s: %w", partitionName, err)
}
// Create indexes
indexSQL := []string{
fmt.Sprintf("CREATE INDEX IF NOT EXISTS %s_entity_idx ON %s(entity_type, entity_id)", partitionName, partitionName),
fmt.Sprintf("CREATE INDEX IF NOT EXISTS %s_user_idx ON %s(username)", partitionName, partitionName),
fmt.Sprintf("CREATE INDEX IF NOT EXISTS %s_time_idx ON %s(timestamp)", partitionName, partitionName),
}
for _, sql := range indexSQL {
if err := pm.db.Exec(sql).Error; err != nil {
log.Warn().Err(err).Msg("Failed to create audit index")
}
}
log.Info().Str("partition", partitionName).Msg("Created audit partition")
return nil
}
// createPartitionFunction creates a PostgreSQL function for automatic partition creation
func (pm *PartitionManager) createPartitionFunction() error {
functionSQL := `
CREATE OR REPLACE FUNCTION create_next_month_partitions()
RETURNS void AS $$
DECLARE
next_month DATE := date_trunc('month', NOW() + INTERVAL '2 months');
partition_name TEXT;
start_date TEXT;
end_date TEXT;
BEGIN
-- Create download_events partition
partition_name := 'download_events_' || to_char(next_month, 'YYYY_MM');
start_date := to_char(next_month, 'YYYY-MM-DD');
end_date := to_char(next_month + INTERVAL '1 month', 'YYYY-MM-DD');
EXECUTE format('CREATE TABLE IF NOT EXISTS %I PARTITION OF download_events FOR VALUES FROM (%L) TO (%L)',
partition_name, start_date, end_date);
EXECUTE format('CREATE INDEX IF NOT EXISTS %I ON %I(package_id, downloaded_at)',
partition_name || '_package_idx', partition_name);
EXECUTE format('CREATE INDEX IF NOT EXISTS %I ON %I(registry_id)',
partition_name || '_registry_idx', partition_name);
-- Create audit_log partition
partition_name := 'audit_log_' || to_char(next_month, 'YYYY_MM');
EXECUTE format('CREATE TABLE IF NOT EXISTS %I PARTITION OF audit_log FOR VALUES FROM (%L) TO (%L)',
partition_name, start_date, end_date);
EXECUTE format('CREATE INDEX IF NOT EXISTS %I ON %I(entity_type, entity_id)',
partition_name || '_entity_idx', partition_name);
RAISE NOTICE 'Created partitions for %', to_char(next_month, 'YYYY-MM');
END;
$$ LANGUAGE plpgsql;
`
if err := pm.db.Exec(functionSQL).Error; err != nil {
return err
}
log.Info().Msg("Created partition management function")
return nil
}
// CleanupOldPartitions drops partitions older than the retention period
func (pm *PartitionManager) CleanupOldPartitions(retentionMonths int) error {
if pm.db.Dialector.Name() != "postgres" {
return nil
}
cutoffDate := time.Now().AddDate(0, -retentionMonths, 0)
cutoffPartition := fmt.Sprintf("%d_%02d", cutoffDate.Year(), cutoffDate.Month())
log.Info().
Str("cutoff", cutoffPartition).
Int("retention_months", retentionMonths).
Msg("Cleaning up old partitions")
// Find and drop old download_events partitions
var downloadPartitions []string
err := pm.db.Raw(`
SELECT tablename FROM pg_tables
WHERE tablename LIKE 'download_events_%'
AND tablename < 'download_events_' || ?
`, cutoffPartition).Scan(&downloadPartitions).Error
if err != nil {
return err
}
for _, partition := range downloadPartitions {
log.Info().Str("partition", partition).Msg("Dropping old partition")
if err := pm.db.Exec(fmt.Sprintf("DROP TABLE IF EXISTS %s", partition)).Error; err != nil {
log.Error().Err(err).Str("partition", partition).Msg("Failed to drop partition")
}
}
// Find and drop old audit_log partitions
var auditPartitions []string
err = pm.db.Raw(`
SELECT tablename FROM pg_tables
WHERE tablename LIKE 'audit_log_%'
AND tablename < 'audit_log_' || ?
`, cutoffPartition).Scan(&auditPartitions).Error
if err != nil {
return err
}
for _, partition := range auditPartitions {
log.Info().Str("partition", partition).Msg("Dropping old audit partition")
if err := pm.db.Exec(fmt.Sprintf("DROP TABLE IF EXISTS %s", partition)).Error; err != nil {
log.Error().Err(err).Str("partition", partition).Msg("Failed to drop audit partition")
}
}
return nil
}
// GetPartitionInfo returns information about current partitions
func (pm *PartitionManager) GetPartitionInfo() (map[string]interface{}, error) {
if pm.db.Dialector.Name() != "postgres" {
return map[string]interface{}{"status": "not_applicable"}, nil
}
info := make(map[string]interface{})
// Count download_events partitions
var downloadCount int64
pm.db.Raw("SELECT COUNT(*) FROM pg_tables WHERE tablename LIKE 'download_events_%'").Scan(&downloadCount)
info["download_events_partitions"] = downloadCount
// Count audit_log partitions
var auditCount int64
pm.db.Raw("SELECT COUNT(*) FROM pg_tables WHERE tablename LIKE 'audit_log_%'").Scan(&auditCount)
info["audit_log_partitions"] = auditCount
// Get partition sizes
type PartitionSize struct {
TableName string
SizeMB float64
}
var partitionSizes []PartitionSize
pm.db.Raw(`
SELECT
tablename AS table_name,
pg_total_relation_size(tablename::regclass) / 1024.0 / 1024.0 AS size_mb
FROM pg_tables
WHERE tablename LIKE 'download_events_%' OR tablename LIKE 'audit_log_%'
ORDER BY size_mb DESC
LIMIT 10
`).Scan(&partitionSizes)
info["largest_partitions"] = partitionSizes
return info, nil
}
pkg/metadata/interface.go
+5
View File
@@ -150,6 +150,11 @@ type Stats struct {
TotalDownloads int64 `json:"total_downloads"`
ScannedPackages int64 `json:"scanned_packages"`
VulnerablePackages int64 `json:"vulnerable_packages"`
BlockedPackages int64 `json:"blocked_packages"`
CriticalVulnerabilities int64 `json:"critical_vulnerabilities"`
HighVulnerabilities int64 `json:"high_vulnerabilities"`
ModerateVulnerabilities int64 `json:"moderate_vulnerabilities"`
LowVulnerabilities int64 `json:"low_vulnerabilities"`
}
// TimeSeriesDataPoint represents a single data point in time-series
pkg/metadata/sqlite/sqlite.go
-1096
View File
File diff suppressed because it is too large Load Diff
pkg/proxy/pypi/pypi.go
+16 -3
View File
@@ -148,6 +148,17 @@ func (h *Handler) handlePackagePage(ctx context.Context, w http.ResponseWriter,
func (h *Handler) handlePackageFile(ctx context.Context, w http.ResponseWriter, r *http.Request, path string) {
packageName, version := extractPackageFileInfo(path)
// Make version unique by appending file type to avoid cache collisions
// between .whl and .metadata files with same version
cacheVersion := version
if strings.HasSuffix(path, ".metadata") {
cacheVersion = version + ".metadata"
} else if strings.HasSuffix(path, ".whl") {
cacheVersion = version + ".whl"
} else if strings.HasSuffix(path, ".tar.gz") {
cacheVersion = version + ".tar.gz"
}
// Extract credentials from request
credentials := h.credExtractor.Extract(r)
credHash := h.credHasher.Hash(credentials)
@@ -170,12 +181,13 @@ func (h *Handler) handlePackageFile(ctx context.Context, w http.ResponseWriter,
Str("path", path).
Str("package", packageName).
Str("version", version).
Str("cache_version", cacheVersion).
Str("url", originalURL).
Str("cred_hash", credHash).
Bool("has_credentials", credentials != "").
Msg("Handling PyPI package file request")
entry, err := h.cache.Get(ctx, "pypi", packageName, version, func(ctx context.Context) (io.ReadCloser, string, error) {
entry, err := h.cache.Get(ctx, "pypi", packageName, cacheVersion, func(ctx context.Context) (io.ReadCloser, string, error) {
// Prepare headers for upstream request
headers := make(map[string]string)
if credentials != "" {
@@ -281,11 +293,12 @@ func isPackagePage(path string) bool {
// isPackageFile checks if the request is for a package file
func isPackageFile(path string) bool {
// Package files (not including .metadata files which need special handling)
// Package files including .metadata files for PEP 658 support
return strings.HasSuffix(path, ".whl") ||
strings.HasSuffix(path, ".tar.gz") ||
strings.HasSuffix(path, ".zip") ||
strings.HasSuffix(path, ".egg")
strings.HasSuffix(path, ".egg") ||
strings.HasSuffix(path, ".metadata")
}
// extractPackageName extracts package name from path