lukaszraczylo/gohoarder
1
0
Fork 0
mirror of https://github.com/lukaszraczylo/gohoarder.git synced 2026-07-05 03:54:53 +00:00
Code Issues Packages Projects Releases Wiki Activity

chore(schema): migrate to GORM V2 with multi-database support

Browse Source 
- [x] Implement GORM V2 metadata store with SQLite, PostgreSQL, and MySQL support
- [x] Add database migration system using gormigrate for schema versioning
- [x] Create migration CLI tool with support for migrate, rollback, and status commands
- [x] Add Docker support for migration container (Dockerfile.migrate)
- [x] Implement automatic partition management for PostgreSQL time-series tables
- [x] Add background aggregation worker for download statistics
- [x] Support connection pooling configuration (max_open_conns, max_idle_conns, conn_max_lifetime)
- [x] Add blocking mechanism based on vulnerability thresholds in stats and handlers
- [x] Update Helm charts with migration init containers and multi-database configuration
- [x] Replace deprecated SQLite store with optimized GORM implementation
- [x] Add comprehensive integration tests for MySQL and PostgreSQL
- [x] Update frontend to display blocked packages and storage utilization
- [x] Add goreleaser configuration for migrate binary and container image
- [x] Update configuration examples with database backend options and recommendations
This commit is contained in:
Lukasz Raczylo
2026-01-03 20:44:23 +00:00
parent b129279fb8
commit c0061b99e3
37 changed files with 5711 additions and 1222 deletions
Download Patch File Download Diff File Expand all files Collapse all files
helm/gohoarder/templates/_helpers.tpl
+1 -1
View File
@@ -181,7 +181,7 @@ Validate SQLite configuration - SQLite cannot be used with SMB/NFS network stora
{{- if .Values.metadata.sqlite.persistence.enabled }}
{{- $storageClass := .Values.metadata.sqlite.persistence.storageClass | default .Values.storage.storageClass }}
{{- if or (contains "smb" ($storageClass | lower)) (contains "cifs" ($storageClass | lower)) (contains "nfs" ($storageClass | lower)) }}
{{- fail "\n\n❌ ERROR: SQLite cannot be used with SMB/CIFS/NFS network storage!\n\nSQLite requires POSIX file locking which is not reliably supported over network filesystems.\nThis will cause 'database is locked' errors and data corruption.\n\nPlease choose ONE of the following solutions:\n\n1. Use PostgreSQL for network storage (RECOMMENDED for production):\n metadata:\n backend: postgresql\n postgresql:\n host: your-postgres-host\n ...\n\n2. Use local storage for SQLite (OK for development):\n metadata:\n sqlite:\n persistence:\n enabled: true\n storageClass: local-path # or another local storage class\n\n3. Disable persistence (data will be lost on pod restart):\n metadata:\n sqlite:\n persistence:\n enabled: false\n\nFor more information, see: https://www.sqlite.org/lockingv3.html\n" }}
{{- fail "\n\n❌ ERROR: SQLite cannot be used with SMB/CIFS/NFS network storage!\n\nSQLite requires POSIX file locking which is not reliably supported over network filesystems.\nThis will cause 'database is locked' errors and data corruption.\n\nPlease choose ONE of the following solutions:\n\n1. Use PostgreSQL for network storage (RECOMMENDED for production):\n metadata:\n backend: postgresql\n postgresql:\n host: your-postgres-host\n ...\n\n2. Use MySQL/MariaDB for network storage (alternative to PostgreSQL):\n metadata:\n backend: mysql\n mysql:\n host: your-mysql-host\n ...\n\n3. Use local storage for SQLite (OK for development):\n metadata:\n sqlite:\n persistence:\n enabled: true\n storageClass: local-path # or another local storage class\n\n4. Disable persistence (data will be lost on pod restart):\n metadata:\n sqlite:\n persistence:\n enabled: false\n\nFor more information, see: https://www.sqlite.org/lockingv3.html\n" }}
{{- end }}
{{- end }}
{{- end }}
helm/gohoarder/templates/deployment-scanner.yaml
+117
View File
@@ -29,6 +29,77 @@ spec:
serviceAccountName: {{ include "gohoarder.serviceAccountName" . }}
securityContext:
{{- toYaml .Values.podSecurityContext | nindent 8 }}
{{- if .Values.migration.enabled }}
initContainers:
# Wait for database to be ready
- name: wait-for-db
image: busybox:1.36
command:
- sh
- -c
- |
echo "Waiting for database..."
{{- if eq .Values.metadata.backend "postgresql" }}
until nc -z {{ .Values.metadata.postgresql.host }} {{ .Values.metadata.postgresql.port }}; do
echo " PostgreSQL not ready, retrying in 2s..."
sleep 2
done
echo "✓ PostgreSQL is ready"
{{- else if eq .Values.metadata.backend "mysql" }}
until nc -z {{ .Values.metadata.mysql.host }} {{ .Values.metadata.mysql.port }}; do
echo " MySQL not ready, retrying in 2s..."
sleep 2
done
echo "✓ MySQL is ready"
{{- else }}
echo "✓ SQLite (no wait needed)"
{{- end }}
securityContext:
allowPrivilegeEscalation: false
runAsNonRoot: true
runAsUser: 1000
resources:
limits:
cpu: 100m
memory: 64Mi
requests:
cpu: 10m
memory: 32Mi
# Run database migrations
- name: migrate
image: "{{ .Values.migration.image.repository }}:{{ .Values.migration.image.tag | default .Chart.AppVersion }}"
imagePullPolicy: {{ .Values.migration.image.pullPolicy }}
env:
- name: DB_DRIVER
value: {{ .Values.metadata.backend | quote }}
{{- if eq .Values.metadata.backend "postgresql" }}
- name: DATABASE_URL
value: "postgresql://{{ .Values.metadata.postgresql.username }}:{{ .Values.metadata.postgresql.password }}@{{ .Values.metadata.postgresql.host }}:{{ .Values.metadata.postgresql.port }}/{{ .Values.metadata.postgresql.database }}?sslmode={{ .Values.metadata.postgresql.sslMode }}"
{{- else if eq .Values.metadata.backend "mysql" }}
- name: DATABASE_URL
value: "{{ .Values.metadata.mysql.username }}:{{ .Values.metadata.mysql.password }}@tcp({{ .Values.metadata.mysql.host }}:{{ .Values.metadata.mysql.port }})/{{ .Values.metadata.mysql.database }}?charset={{ .Values.metadata.mysql.charset }}&parseTime={{ .Values.metadata.mysql.parseTime }}"
{{- else }}
- name: DATABASE_URL
value: "/var/lib/gohoarder/metadata/gohoarder.db"
{{- end }}
args:
- --driver=$(DB_DRIVER)
- --dsn=$(DATABASE_URL)
- --action=migrate
- --log-level={{ .Values.migration.logLevel | default "info" }}
- --timeout={{ .Values.migration.timeout | default "5m" }}
securityContext:
allowPrivilegeEscalation: false
runAsNonRoot: true
runAsUser: 1000
resources:
{{- toYaml .Values.migration.resources | nindent 10 }}
{{- if eq .Values.metadata.backend "sqlite" }}
volumeMounts:
- name: metadata
mountPath: /var/lib/gohoarder/metadata
{{- end }}
{{- end }}
containers:
- name: scanner
securityContext:
@@ -38,6 +109,52 @@ spec:
env:
- name: CONFIG_FILE
value: /etc/gohoarder/config.yaml
{{- if and (eq .Values.metadata.backend "postgresql") .Values.metadata.postgresql.existingSecret }}
- name: POSTGRES_USER
valueFrom:
secretKeyRef:
name: {{ .Values.metadata.postgresql.existingSecret }}
key: username
- name: POSTGRES_PASSWORD
valueFrom:
secretKeyRef:
name: {{ .Values.metadata.postgresql.existingSecret }}
key: password
{{- else if and (eq .Values.metadata.backend "postgresql") .Values.metadata.postgresql.username }}
- name: POSTGRES_USER
valueFrom:
secretKeyRef:
name: {{ include "gohoarder.fullname" . }}-postgresql
key: username
- name: POSTGRES_PASSWORD
valueFrom:
secretKeyRef:
name: {{ include "gohoarder.fullname" . }}-postgresql
key: password
{{- end }}
{{- if and (or (eq .Values.metadata.backend "mysql") (eq .Values.metadata.backend "mariadb")) .Values.metadata.mysql.existingSecret }}
- name: MYSQL_USER
valueFrom:
secretKeyRef:
name: {{ .Values.metadata.mysql.existingSecret }}
key: username
- name: MYSQL_PASSWORD
valueFrom:
secretKeyRef:
name: {{ .Values.metadata.mysql.existingSecret }}
key: password
{{- else if and (or (eq .Values.metadata.backend "mysql") (eq .Values.metadata.backend "mariadb")) .Values.metadata.mysql.username }}
- name: MYSQL_USER
valueFrom:
secretKeyRef:
name: {{ include "gohoarder.fullname" . }}-mysql
key: username
- name: MYSQL_PASSWORD
valueFrom:
secretKeyRef:
name: {{ include "gohoarder.fullname" . }}-mysql
key: password
{{- end }}
{{- if and .Values.security.scanners.ghsa.enabled .Values.security.scanners.ghsa.existingSecret }}
- name: GHSA_TOKEN
valueFrom:
helm/gohoarder/templates/deployment-server.yaml
+94
View File
@@ -30,6 +30,77 @@ spec:
serviceAccountName: {{ include "gohoarder.serviceAccountName" . }}
securityContext:
{{- toYaml .Values.podSecurityContext | nindent 8 }}
{{- if .Values.migration.enabled }}
initContainers:
# Wait for database to be ready
- name: wait-for-db
image: busybox:1.36
command:
- sh
- -c
- |
echo "Waiting for database..."
{{- if eq .Values.metadata.backend "postgresql" }}
until nc -z {{ .Values.metadata.postgresql.host }} {{ .Values.metadata.postgresql.port }}; do
echo " PostgreSQL not ready, retrying in 2s..."
sleep 2
done
echo "✓ PostgreSQL is ready"
{{- else if eq .Values.metadata.backend "mysql" }}
until nc -z {{ .Values.metadata.mysql.host }} {{ .Values.metadata.mysql.port }}; do
echo " MySQL not ready, retrying in 2s..."
sleep 2
done
echo "✓ MySQL is ready"
{{- else }}
echo "✓ SQLite (no wait needed)"
{{- end }}
securityContext:
allowPrivilegeEscalation: false
runAsNonRoot: true
runAsUser: 1000
resources:
limits:
cpu: 100m
memory: 64Mi
requests:
cpu: 10m
memory: 32Mi
# Run database migrations
- name: migrate
image: "{{ .Values.migration.image.repository }}:{{ .Values.migration.image.tag | default .Chart.AppVersion }}"
imagePullPolicy: {{ .Values.migration.image.pullPolicy }}
env:
- name: DB_DRIVER
value: {{ .Values.metadata.backend | quote }}
{{- if eq .Values.metadata.backend "postgresql" }}
- name: DATABASE_URL
value: "postgresql://{{ .Values.metadata.postgresql.username }}:{{ .Values.metadata.postgresql.password }}@{{ .Values.metadata.postgresql.host }}:{{ .Values.metadata.postgresql.port }}/{{ .Values.metadata.postgresql.database }}?sslmode={{ .Values.metadata.postgresql.sslMode }}"
{{- else if eq .Values.metadata.backend "mysql" }}
- name: DATABASE_URL
value: "{{ .Values.metadata.mysql.username }}:{{ .Values.metadata.mysql.password }}@tcp({{ .Values.metadata.mysql.host }}:{{ .Values.metadata.mysql.port }})/{{ .Values.metadata.mysql.database }}?charset={{ .Values.metadata.mysql.charset }}&parseTime={{ .Values.metadata.mysql.parseTime }}"
{{- else }}
- name: DATABASE_URL
value: "/var/lib/gohoarder/metadata/gohoarder.db"
{{- end }}
args:
- --driver=$(DB_DRIVER)
- --dsn=$(DATABASE_URL)
- --action=migrate
- --log-level={{ .Values.migration.logLevel | default "info" }}
- --timeout={{ .Values.migration.timeout | default "5m" }}
securityContext:
allowPrivilegeEscalation: false
runAsNonRoot: true
runAsUser: 1000
resources:
{{- toYaml .Values.migration.resources | nindent 10 }}
{{- if eq .Values.metadata.backend "sqlite" }}
volumeMounts:
- name: metadata
mountPath: /var/lib/gohoarder/metadata
{{- end }}
{{- end }}
containers:
- name: server
securityContext:
@@ -125,6 +196,29 @@ spec:
name: {{ include "gohoarder.fullname" . }}-postgresql
key: password
{{- end }}
{{- if and (or (eq .Values.metadata.backend "mysql") (eq .Values.metadata.backend "mariadb")) .Values.metadata.mysql.existingSecret }}
- name: MYSQL_USER
valueFrom:
secretKeyRef:
name: {{ .Values.metadata.mysql.existingSecret }}
key: username
- name: MYSQL_PASSWORD
valueFrom:
secretKeyRef:
name: {{ .Values.metadata.mysql.existingSecret }}
key: password
{{- else if and (or (eq .Values.metadata.backend "mysql") (eq .Values.metadata.backend "mariadb")) .Values.metadata.mysql.username }}
- name: MYSQL_USER
valueFrom:
secretKeyRef:
name: {{ include "gohoarder.fullname" . }}-mysql
key: username
- name: MYSQL_PASSWORD
valueFrom:
secretKeyRef:
name: {{ include "gohoarder.fullname" . }}-mysql
key: password
{{- end }}
{{- if and .Values.security.scanners.ghsa.enabled .Values.security.scanners.ghsa.existingSecret }}
- name: GHSA_TOKEN
valueFrom:
helm/gohoarder/templates/secret.yaml
+13
View File
@@ -53,6 +53,19 @@ data:
password: {{ .Values.metadata.postgresql.password | b64enc | quote }}
{{- end }}
---
{{- if and (or (eq .Values.metadata.backend "mysql") (eq .Values.metadata.backend "mariadb")) (not .Values.metadata.mysql.existingSecret) .Values.metadata.mysql.username }}
apiVersion: v1
kind: Secret
metadata:
name: {{ include "gohoarder.fullname" . }}-mysql
labels:
{{- include "gohoarder.labels" . | nindent 4 }}
type: Opaque
data:
username: {{ .Values.metadata.mysql.username | b64enc | quote }}
password: {{ .Values.metadata.mysql.password | b64enc | quote }}
{{- end }}
---
{{- if and .Values.security.scanners.ghsa.enabled (not .Values.security.scanners.ghsa.existingSecret) .Values.security.scanners.ghsa.token }}
apiVersion: v1
kind: Secret
helm/gohoarder/values.yaml
+58 -2
View File
@@ -272,7 +272,7 @@ storage:
# Metadata storage configuration
metadata:
# Backend: sqlite, postgresql
# Backend: sqlite, postgresql, mysql
#
# IMPORTANT: SQLite CANNOT be used with SMB/CIFS/NFS network storage!
# SQLite requires POSIX file locking which causes "database is locked" errors on network filesystems.
@@ -286,6 +286,13 @@ metadata:
# 2. PostgreSQL with any storage (RECOMMENDED for production)
# - Set backend: postgresql
# - Configure postgresql settings below
# - Works with any storage including SMB/NFS
# - Supports multiple replicas and high availability
#
# 3. MySQL/MariaDB with any storage (alternative to PostgreSQL)
# - Set backend: mysql
# - Configure mysql settings below
# - Works with any storage including SMB/NFS
#
backend: "sqlite"
@@ -305,6 +312,8 @@ metadata:
walMode: false
# PostgreSQL configuration
# Works with any storage including SMB/NFS
# Recommended for production deployments
postgresql:
# Use bundled PostgreSQL (sets up postgresql subchart)
enabled: false
@@ -313,10 +322,57 @@ metadata:
database: "gohoarder"
username: "gohoarder"
password: ""
sslMode: "disable"
sslMode: "disable" # disable, require, verify-ca, verify-full
# Use existing secret for PostgreSQL credentials
existingSecret: ""
# MySQL/MariaDB configuration
# Works with any storage including SMB/NFS
# Alternative to PostgreSQL for production deployments
mysql:
host: "localhost"
port: 3306
database: "gohoarder"
username: "gohoarder"
password: ""
charset: "utf8mb4"
parseTime: true
# Use existing secret for MySQL credentials
existingSecret: ""
# GORM connection pool settings (applies to all database backends)
# These settings control database connection pooling and performance
maxOpenConns: 25 # Maximum number of open connections to the database
maxIdleConns: 5 # Maximum number of idle connections in the pool
connMaxLifetime: 3600 # Maximum lifetime of a connection in seconds (1 hour)
logLevel: "warn" # GORM log level: silent, error, warn, info
# Database migration configuration
migration:
# Enable automatic database migrations via init containers
# When enabled, each pod will run migrations before starting the main container
# Gormigrate handles concurrency automatically - safe for multiple pods
enabled: true
# Migration image configuration
image:
repository: ghcr.io/lukaszraczylo/gohoarder-migrate
pullPolicy: IfNotPresent
tag: "latest" # Should match the application version
# Migration settings
logLevel: "info" # debug, info, warn, error
timeout: "5m" # Maximum time for migrations to complete
# Resource limits for migration init container
resources:
limits:
cpu: 500m
memory: 256Mi
requests:
cpu: 100m
memory: 128Mi
# Cache configuration
cache:
defaultTTL: "168h" # 7 days