mirror of
https://github.com/lukaszraczylo/gohoarder.git
synced 2026-07-22 06:20:09 +00:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
b129279fb8 | ||
|
|
bcc0185924 | ||
|
|
89bf86a6e1 | ||
|
|
6f2be85b9e | ||
|
|
72143ecd3d | ||
|
|
e6fe925dcb | ||
|
|
f03a288326 |
+16
-7
@@ -24,9 +24,18 @@ RUN curl -sSfL https://raw.githubusercontent.com/anchore/grype/main/install.sh |
|
||||
RUN addgroup -g 1000 scanner && \
|
||||
adduser -D -u 1000 -G scanner scanner
|
||||
|
||||
# Create necessary directories
|
||||
RUN mkdir -p /data/cache /data/scans && \
|
||||
chown -R scanner:scanner /data
|
||||
# Create necessary directories with proper permissions
|
||||
RUN mkdir -p /var/cache/gohoarder \
|
||||
/var/lib/gohoarder/metadata \
|
||||
/var/lib/trivy \
|
||||
/tmp/gohoarder && \
|
||||
chown -R scanner:scanner /var/cache/gohoarder \
|
||||
/var/lib/gohoarder \
|
||||
/var/lib/trivy \
|
||||
/tmp/gohoarder && \
|
||||
chmod -R 750 /var/cache/gohoarder \
|
||||
/var/lib/gohoarder \
|
||||
/var/lib/trivy
|
||||
|
||||
# Copy binary (from platform-specific path)
|
||||
ARG TARGETOS
|
||||
@@ -37,7 +46,7 @@ RUN chmod +x /usr/local/bin/gohoarder
|
||||
# Copy example config
|
||||
COPY config.yaml.example /etc/gohoarder/config.yaml.example
|
||||
|
||||
WORKDIR /data
|
||||
WORKDIR /var/cache/gohoarder
|
||||
USER scanner
|
||||
|
||||
# Expose metrics port
|
||||
@@ -53,7 +62,7 @@ ENV SCANNER_MODE=true \
|
||||
SCANNER_INTERVAL=300
|
||||
|
||||
# Run the scanner in background mode
|
||||
# Note: You may need to add a scanner-specific command to your CLI
|
||||
# For now, this assumes the serve command can run in scanner mode
|
||||
# The scanner runs the same serve command but uses SCANNER_MODE env var
|
||||
# and configuration to determine its role
|
||||
ENTRYPOINT ["/usr/local/bin/gohoarder"]
|
||||
CMD ["serve", "--scanner-only"]
|
||||
CMD ["serve"]
|
||||
|
||||
+10
-4
@@ -14,9 +14,15 @@ RUN apk add --no-cache \
|
||||
RUN addgroup -g 1000 gohoarder && \
|
||||
adduser -D -u 1000 -G gohoarder gohoarder
|
||||
|
||||
# Create necessary directories
|
||||
RUN mkdir -p /data/cache /data/metadata && \
|
||||
chown -R gohoarder:gohoarder /data
|
||||
# Create necessary directories with proper permissions
|
||||
RUN mkdir -p /var/cache/gohoarder \
|
||||
/var/lib/gohoarder/metadata \
|
||||
/tmp/gohoarder && \
|
||||
chown -R gohoarder:gohoarder /var/cache/gohoarder \
|
||||
/var/lib/gohoarder \
|
||||
/tmp/gohoarder && \
|
||||
chmod -R 750 /var/cache/gohoarder \
|
||||
/var/lib/gohoarder
|
||||
|
||||
# Copy binary (from platform-specific path)
|
||||
ARG TARGETOS
|
||||
@@ -27,7 +33,7 @@ RUN chmod +x /usr/local/bin/gohoarder
|
||||
# Copy example config
|
||||
COPY config.yaml.example /etc/gohoarder/config.yaml.example
|
||||
|
||||
WORKDIR /data
|
||||
WORKDIR /var/cache/gohoarder
|
||||
USER gohoarder
|
||||
|
||||
# Expose ports
|
||||
|
||||
@@ -172,3 +172,17 @@ Trivy cache volume configuration
|
||||
emptyDir: {}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
||||
{{/*
|
||||
Validate SQLite configuration - SQLite cannot be used with SMB/NFS network storage
|
||||
*/}}
|
||||
{{- define "gohoarder.validateSQLiteConfig" -}}
|
||||
{{- if eq .Values.metadata.backend "sqlite" }}
|
||||
{{- if .Values.metadata.sqlite.persistence.enabled }}
|
||||
{{- $storageClass := .Values.metadata.sqlite.persistence.storageClass | default .Values.storage.storageClass }}
|
||||
{{- if or (contains "smb" ($storageClass | lower)) (contains "cifs" ($storageClass | lower)) (contains "nfs" ($storageClass | lower)) }}
|
||||
{{- fail "\n\n❌ ERROR: SQLite cannot be used with SMB/CIFS/NFS network storage!\n\nSQLite requires POSIX file locking which is not reliably supported over network filesystems.\nThis will cause 'database is locked' errors and data corruption.\n\nPlease choose ONE of the following solutions:\n\n1. Use PostgreSQL for network storage (RECOMMENDED for production):\n metadata:\n backend: postgresql\n postgresql:\n host: your-postgres-host\n ...\n\n2. Use local storage for SQLite (OK for development):\n metadata:\n sqlite:\n persistence:\n enabled: true\n storageClass: local-path # or another local storage class\n\n3. Disable persistence (data will be lost on pod restart):\n metadata:\n sqlite:\n persistence:\n enabled: false\n\nFor more information, see: https://www.sqlite.org/lockingv3.html\n" }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
{{- if .Values.security.enabled }}
|
||||
{{- include "gohoarder.validateSQLiteConfig" . }}
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
@@ -28,32 +29,6 @@ spec:
|
||||
serviceAccountName: {{ include "gohoarder.serviceAccountName" . }}
|
||||
securityContext:
|
||||
{{- toYaml .Values.podSecurityContext | nindent 8 }}
|
||||
initContainers:
|
||||
- name: init-permissions
|
||||
image: busybox:latest
|
||||
command: ['sh', '-c']
|
||||
args:
|
||||
- |
|
||||
mkdir -p /var/cache/gohoarder /var/lib/gohoarder/metadata /tmp/gohoarder
|
||||
{{- if .Values.security.scanners.trivy.enabled }}
|
||||
mkdir -p {{ .Values.security.scanners.trivy.cacheDb }}
|
||||
chown -R 1000:1000 {{ .Values.security.scanners.trivy.cacheDb }}
|
||||
{{- end }}
|
||||
chown -R 1000:1000 /var/cache/gohoarder /var/lib/gohoarder /tmp/gohoarder
|
||||
chmod 750 /var/cache/gohoarder /var/lib/gohoarder
|
||||
volumeMounts:
|
||||
- name: storage
|
||||
mountPath: /var/cache/gohoarder
|
||||
- name: metadata
|
||||
mountPath: /var/lib/gohoarder/metadata
|
||||
{{- if .Values.security.scanners.trivy.enabled }}
|
||||
- name: trivy-cache
|
||||
mountPath: {{ .Values.security.scanners.trivy.cacheDb }}
|
||||
{{- end }}
|
||||
- name: tmp
|
||||
mountPath: /tmp/gohoarder
|
||||
securityContext:
|
||||
runAsUser: 0
|
||||
containers:
|
||||
- name: scanner
|
||||
securityContext:
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
{{- include "gohoarder.validateSQLiteConfig" . }}
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
@@ -29,24 +30,6 @@ spec:
|
||||
serviceAccountName: {{ include "gohoarder.serviceAccountName" . }}
|
||||
securityContext:
|
||||
{{- toYaml .Values.podSecurityContext | nindent 8 }}
|
||||
initContainers:
|
||||
- name: init-permissions
|
||||
image: busybox:latest
|
||||
command: ['sh', '-c']
|
||||
args:
|
||||
- |
|
||||
mkdir -p /var/cache/gohoarder /var/lib/gohoarder/metadata /tmp/gohoarder
|
||||
chown -R 1000:1000 /var/cache/gohoarder /var/lib/gohoarder /tmp/gohoarder
|
||||
chmod 750 /var/cache/gohoarder /var/lib/gohoarder
|
||||
volumeMounts:
|
||||
- name: storage
|
||||
mountPath: /var/cache/gohoarder
|
||||
- name: metadata
|
||||
mountPath: /var/lib/gohoarder/metadata
|
||||
- name: tmp
|
||||
mountPath: /tmp/gohoarder
|
||||
securityContext:
|
||||
runAsUser: 0
|
||||
containers:
|
||||
- name: server
|
||||
securityContext:
|
||||
|
||||
@@ -273,19 +273,36 @@ storage:
|
||||
# Metadata storage configuration
|
||||
metadata:
|
||||
# Backend: sqlite, postgresql
|
||||
# For multiple server replicas: postgresql is recommended (sqlite has concurrency limitations)
|
||||
#
|
||||
# IMPORTANT: SQLite CANNOT be used with SMB/CIFS/NFS network storage!
|
||||
# SQLite requires POSIX file locking which causes "database is locked" errors on network filesystems.
|
||||
#
|
||||
# Choose your configuration:
|
||||
# 1. SQLite with local storage (development/single-node only)
|
||||
# - Set backend: sqlite
|
||||
# - Set sqlite.persistence.storageClass to a LOCAL storage class (e.g., "local-path")
|
||||
# - OR set sqlite.persistence.enabled: false to use emptyDir (data lost on pod restart)
|
||||
#
|
||||
# 2. PostgreSQL with any storage (RECOMMENDED for production)
|
||||
# - Set backend: postgresql
|
||||
# - Configure postgresql settings below
|
||||
#
|
||||
backend: "sqlite"
|
||||
|
||||
# SQLite configuration
|
||||
# WARNING: Do NOT use SMB/CIFS/NFS storage classes with SQLite!
|
||||
sqlite:
|
||||
# Use PVC for SQLite database
|
||||
# IMPORTANT: storageClass must be LOCAL storage, NOT network storage (smb/nfs)
|
||||
persistence:
|
||||
enabled: true
|
||||
storageClass: ""
|
||||
enabled: false # Changed to false by default - use emptyDir unless you have local storage
|
||||
storageClass: "" # Must be local-path or similar LOCAL storage class if enabled
|
||||
size: "10Gi"
|
||||
accessMode: "ReadWriteOnce"
|
||||
existingClaim: ""
|
||||
walMode: true
|
||||
# WAL mode provides better concurrency but doesn't work on network filesystems (SMB, NFS)
|
||||
# Set to false when using network storage for the metadata volume
|
||||
walMode: false
|
||||
|
||||
# PostgreSQL configuration
|
||||
postgresql:
|
||||
|
||||
+4
-2
@@ -120,7 +120,8 @@ func (a *App) initializeComponents() error {
|
||||
switch a.config.Metadata.Backend {
|
||||
case "sqlite":
|
||||
a.metadata, err = metasqlite.New(metasqlite.Config{
|
||||
Path: a.config.Metadata.Connection,
|
||||
Path: a.config.Metadata.Connection,
|
||||
WALMode: a.config.Metadata.SQLite.WALMode,
|
||||
})
|
||||
case "file":
|
||||
a.metadata, err = metafile.New(metafile.Config{
|
||||
@@ -128,7 +129,8 @@ func (a *App) initializeComponents() error {
|
||||
})
|
||||
default:
|
||||
a.metadata, err = metasqlite.New(metasqlite.Config{
|
||||
Path: "gohoarder.db",
|
||||
Path: "gohoarder.db",
|
||||
WALMode: false, // Default to DELETE mode for compatibility
|
||||
})
|
||||
}
|
||||
if err != nil {
|
||||
|
||||
@@ -114,7 +114,7 @@ func (s *AuthHandlersTestSuite) TestHandleGenerateAPIKey() {
|
||||
req := httptest.NewRequest("POST", "/api/admin/keys", bytes.NewReader(bodyBytes))
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
|
||||
resp, err := s.app.Test(req)
|
||||
resp, err := s.app.Test(req, 5000) // 5 second timeout for CI environments
|
||||
s.Require().NoError(err)
|
||||
s.Equal(tt.expectedStatus, resp.StatusCode)
|
||||
|
||||
@@ -144,7 +144,7 @@ func (s *AuthHandlersTestSuite) TestHandleListAPIKeys() {
|
||||
s.authManager.GenerateAPIKey("test-key-3", auth.RoleAdmin, nil)
|
||||
|
||||
req := httptest.NewRequest("GET", "/api/admin/keys", nil)
|
||||
resp, err := s.app.Test(req)
|
||||
resp, err := s.app.Test(req, 5000) // 5 second timeout for CI environments
|
||||
s.Require().NoError(err)
|
||||
s.Equal(200, resp.StatusCode)
|
||||
|
||||
@@ -189,7 +189,7 @@ func (s *AuthHandlersTestSuite) TestHandleRevokeAPIKey() {
|
||||
for _, tt := range tests {
|
||||
s.Run(tt.name, func() {
|
||||
req := httptest.NewRequest("DELETE", "/api/admin/keys/"+tt.keyID, nil)
|
||||
resp, err := s.app.Test(req)
|
||||
resp, err := s.app.Test(req, 5000) // 5 second timeout for CI environments
|
||||
s.Require().NoError(err)
|
||||
s.Equal(tt.expectedStatus, resp.StatusCode)
|
||||
|
||||
@@ -207,7 +207,7 @@ func (s *AuthHandlersTestSuite) TestHandleGenerateAPIKeyInvalidJSON() {
|
||||
req := httptest.NewRequest("POST", "/api/admin/keys", bytes.NewReader([]byte("invalid json")))
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
|
||||
resp, err := s.app.Test(req)
|
||||
resp, err := s.app.Test(req, 5000) // 5 second timeout for CI environments
|
||||
s.Require().NoError(err)
|
||||
s.Equal(400, resp.StatusCode)
|
||||
}
|
||||
@@ -221,7 +221,7 @@ func (s *AuthHandlersTestSuite) TestGenerateAndRevokeKeyFlow() {
|
||||
|
||||
req1 := httptest.NewRequest("POST", "/api/admin/keys", bytes.NewReader(bodyBytes))
|
||||
req1.Header.Set("Content-Type", "application/json")
|
||||
resp1, err := s.app.Test(req1)
|
||||
resp1, err := s.app.Test(req1, 5000) // 5 second timeout for CI environments
|
||||
s.Require().NoError(err)
|
||||
s.Equal(201, resp1.StatusCode)
|
||||
|
||||
@@ -235,7 +235,7 @@ func (s *AuthHandlersTestSuite) TestGenerateAndRevokeKeyFlow() {
|
||||
|
||||
// List keys - should include our new key
|
||||
req2 := httptest.NewRequest("GET", "/api/admin/keys", nil)
|
||||
resp2, err := s.app.Test(req2)
|
||||
resp2, err := s.app.Test(req2, 5000) // 5 second timeout for CI environments
|
||||
s.Require().NoError(err)
|
||||
s.Equal(200, resp2.StatusCode)
|
||||
|
||||
@@ -257,13 +257,13 @@ func (s *AuthHandlersTestSuite) TestGenerateAndRevokeKeyFlow() {
|
||||
|
||||
// Revoke the key
|
||||
req3 := httptest.NewRequest("DELETE", "/api/admin/keys/"+keyID, nil)
|
||||
resp3, err := s.app.Test(req3)
|
||||
resp3, err := s.app.Test(req3, 5000) // 5 second timeout for CI environments
|
||||
s.Require().NoError(err)
|
||||
s.Equal(200, resp3.StatusCode)
|
||||
|
||||
// List keys again - should not include the revoked key
|
||||
req4 := httptest.NewRequest("GET", "/api/admin/keys", nil)
|
||||
resp4, err := s.app.Test(req4)
|
||||
resp4, err := s.app.Test(req4, 5000) // 5 second timeout for CI environments
|
||||
s.Require().NoError(err)
|
||||
s.Equal(200, resp4.StatusCode)
|
||||
|
||||
|
||||
@@ -27,6 +27,7 @@ type Config struct {
|
||||
Path string // Database file path
|
||||
MaxOpenConns int // Maximum open connections
|
||||
MaxIdleConns int // Maximum idle connections
|
||||
WALMode bool // Enable WAL mode (should be false for network filesystems)
|
||||
}
|
||||
|
||||
const schema = `
|
||||
@@ -134,8 +135,14 @@ func New(cfg Config) (*SQLiteStore, error) {
|
||||
cfg.MaxIdleConns = 5
|
||||
}
|
||||
|
||||
// Open database with WAL mode for better concurrency
|
||||
dsn := fmt.Sprintf("%s?_journal_mode=WAL&_busy_timeout=5000&_synchronous=NORMAL&_cache_size=2000", cfg.Path)
|
||||
// Build DSN with journal mode based on configuration
|
||||
// WAL mode is better for concurrency but doesn't work on network filesystems (SMB, NFS)
|
||||
// Use DELETE mode for network filesystems for compatibility
|
||||
journalMode := "DELETE"
|
||||
if cfg.WALMode {
|
||||
journalMode = "WAL"
|
||||
}
|
||||
dsn := fmt.Sprintf("%s?_journal_mode=%s&_busy_timeout=5000&_synchronous=NORMAL&_cache_size=2000", cfg.Path, journalMode)
|
||||
db, err := sql.Open("sqlite", dsn)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, errors.ErrCodeStorageFailure, "failed to open SQLite database")
|
||||
|
||||
@@ -0,0 +1,4 @@
|
||||
version: 1
|
||||
force:
|
||||
existing: true
|
||||
minor: 1
|
||||
|
||||
Reference in New Issue
Block a user