mirror of
https://github.com/lukaszraczylo/gohoarder.git
synced 2026-07-22 06:20:09 +00:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
875fd9025a | ||
|
|
1fc61b1ec3 | ||
|
|
c38209d8d1 | ||
|
|
4ffb17757c | ||
|
|
5dcf1cf6b0 | ||
|
|
923651d11a | ||
|
|
fedb40c826 | ||
|
|
b9cd641db3 | ||
|
|
ddad5883fe | ||
|
|
b25dfcd263 | ||
|
|
8c85532c86 | ||
|
|
7e9b0067f1 | ||
|
|
7fb2ed6d8d | ||
|
|
65446ddbe7 | ||
|
|
205a8c0ee5 | ||
|
|
9462c66edf | ||
|
|
7c727df4f8 | ||
|
|
26fd95cea9 | ||
|
|
79c46f3b1e | ||
|
|
33f371967f | ||
|
|
7cc93d768d | ||
|
|
a1b7f871bf | ||
|
|
9ee7df5f50 | ||
|
|
180254581c | ||
|
|
28752e7602 | ||
|
|
d589cc0ffa | ||
|
|
97a1954a1b | ||
|
|
f98ce363dd | ||
|
|
f17f0b3310 | ||
|
|
140f19ed6f | ||
|
|
1e9ac2a9f2 | ||
|
|
aeb67f7dd9 | ||
|
|
afc0ece8d0 | ||
|
|
170d5a9f08 | ||
|
|
6e351f1a25 | ||
|
|
d67cc0bf8e | ||
|
|
239b7e1f75 | ||
|
|
9f1d134d7d | ||
|
|
2dfad4f4e3 | ||
|
|
defa86573e | ||
|
|
0880cc45f7 | ||
|
|
0c2dc0795a | ||
|
|
af81f6c99b | ||
|
|
a46af4b645 | ||
|
|
fe3c0ebb63 | ||
|
|
b3e15180af | ||
|
|
6735ed2071 | ||
|
|
2f379efc83 | ||
|
|
5f8e3b4dee | ||
|
|
0704a3b7c8 | ||
|
|
b22129fcba | ||
|
|
210602566e | ||
|
|
1e39f9f2f4 | ||
|
|
a98ee60d8b | ||
|
|
1293ca4f9d | ||
|
|
2e95518d48 | ||
|
|
0388d99d9e | ||
|
|
113564b497 | ||
|
|
b78e6e79e3 | ||
|
|
3b26752e3c | ||
|
|
ec3312d836 | ||
|
|
f1913087e4 | ||
|
|
eae3e33e7e | ||
|
|
2e8baad645 | ||
|
|
e6f127cf9e | ||
|
|
a1e16c989b | ||
|
|
955250650e | ||
|
|
85c76150f5 | ||
|
|
1baf0993de | ||
|
|
434a15076e | ||
|
|
4e7350363d | ||
|
|
38edd735b6 | ||
|
|
c207aa72e9 | ||
|
|
bf0925a4fc | ||
|
|
bc854aa183 | ||
|
|
c4bb2f6e3a | ||
|
|
8848656193 | ||
|
|
3ecff61114 | ||
|
|
f86943b884 | ||
|
|
448bb70ac8 | ||
|
|
d63ae21133 | ||
|
|
f71414bbb1 | ||
|
|
bdcacfc8db | ||
|
|
a36080c6de | ||
|
|
0e6510636d | ||
|
|
276d406fb6 | ||
|
|
fb79d82072 | ||
|
|
bbcbeeaf88 | ||
|
|
9f4d8fc777 | ||
|
|
1f781f6620 | ||
|
|
8a9d786b1a | ||
|
|
e1a02a6d69 | ||
|
|
5c8565367c | ||
|
|
9db929ed8b | ||
|
|
0a2c5f6c2c | ||
|
|
ae632a3dd7 | ||
|
|
c212077478 | ||
|
|
af5e08a864 | ||
|
|
adc7388ee1 | ||
|
|
116ba9b006 | ||
|
|
03860dcb49 | ||
|
|
a1ec05b210 | ||
|
|
dc1d507a20 | ||
|
|
72f284f987 | ||
|
|
ef11972274 | ||
|
|
96f9f4a36c | ||
|
|
311e4d13f6 | ||
|
|
f936dfa359 | ||
|
|
c1103630f0 | ||
|
|
64f6f5cda4 | ||
|
|
0be529f7be | ||
|
|
c0061b99e3 | ||
|
|
b129279fb8 | ||
|
|
bcc0185924 | ||
|
|
89bf86a6e1 | ||
|
|
6f2be85b9e | ||
|
|
72143ecd3d | ||
|
|
e6fe925dcb | ||
|
|
f03a288326 | ||
|
|
6afa55b5f5 | ||
|
|
6b037a92b4 |
@@ -20,10 +20,15 @@ permissions:
|
|||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
release:
|
release:
|
||||||
uses: lukaszraczylo/shared-actions/.github/workflows/go-release.yaml@main
|
uses: lukaszraczylo/shared-actions/.github/workflows/go-release-cgo.yaml@main
|
||||||
with:
|
with:
|
||||||
go-version: "1.25"
|
go-version: "1.25"
|
||||||
docker-enabled: true
|
# Enable frontend build
|
||||||
|
node-enabled: true
|
||||||
|
node-version: "20"
|
||||||
|
node-build-script: "cd frontend && npm install -g pnpm && pnpm install --frozen-lockfile && pnpm run build"
|
||||||
|
node-output-path: "frontend/dist"
|
||||||
|
node-cache-dependency-path: "frontend/pnpm-lock.yaml"
|
||||||
secrets: inherit
|
secrets: inherit
|
||||||
|
|
||||||
benchmark:
|
benchmark:
|
||||||
|
|||||||
+127
-52
@@ -7,30 +7,91 @@ project_name: gohoarder
|
|||||||
before:
|
before:
|
||||||
hooks:
|
hooks:
|
||||||
- go mod tidy
|
- go mod tidy
|
||||||
# Generate semantic version if not provided via git tag
|
# Download and setup Zig for cross-compilation (Linux only)
|
||||||
# This script can be used by CI/CD to inject custom versions
|
- bash -c 'if [ "$(uname -s)" = "Linux" ]; then wget https://ziglang.org/download/0.15.2/zig-x86_64-linux-0.15.2.tar.xz -O /tmp/zig.tar.xz && ls -lh /tmp/zig.tar.xz && tar -xvf /tmp/zig.tar.xz -C /tmp && echo "/tmp/zig-x86_64-linux-0.15.2" >> $GITHUB_PATH; fi'
|
||||||
# Usage: export GORELEASER_CURRENT_TAG=$(./script/generate-version.sh)
|
|
||||||
# - ./script/generate-version.sh
|
|
||||||
|
|
||||||
# Build configuration
|
# Build configuration
|
||||||
|
# All binaries built using Zig for consistent cross-compilation
|
||||||
|
# Zig handles CGO cross-compilation without platform-specific toolchains
|
||||||
|
# Binaries are injected into Docker images (no Docker compilation)
|
||||||
builds:
|
builds:
|
||||||
- id: gohoarder
|
- id: gohoarder
|
||||||
main: ./cmd/gohoarder
|
main: ./cmd/gohoarder
|
||||||
binary: gohoarder
|
binary: gohoarder
|
||||||
env:
|
env:
|
||||||
- CGO_ENABLED=0
|
- CGO_ENABLED=1
|
||||||
goos:
|
tags:
|
||||||
- linux
|
- fts5
|
||||||
- darwin
|
flags:
|
||||||
- windows
|
- -trimpath
|
||||||
goarch:
|
|
||||||
- amd64
|
|
||||||
- arm64
|
|
||||||
ldflags:
|
ldflags:
|
||||||
- -s -w
|
- -s -w
|
||||||
- -X github.com/lukaszraczylo/gohoarder/internal/version.Version={{.Version}}
|
- -X github.com/lukaszraczylo/gohoarder/internal/version.Version={{.Version}}
|
||||||
- -X github.com/lukaszraczylo/gohoarder/internal/version.GitCommit={{.ShortCommit}}
|
- -X github.com/lukaszraczylo/gohoarder/internal/version.GitCommit={{.ShortCommit}}
|
||||||
- -X github.com/lukaszraczylo/gohoarder/internal/version.BuildTime={{.Date}}
|
- -X github.com/lukaszraczylo/gohoarder/internal/version.BuildTime={{.Date}}
|
||||||
|
goos:
|
||||||
|
- linux
|
||||||
|
- darwin
|
||||||
|
goarch:
|
||||||
|
- amd64
|
||||||
|
- arm64
|
||||||
|
ignore:
|
||||||
|
- goos: darwin
|
||||||
|
goarch: amd64
|
||||||
|
overrides:
|
||||||
|
# Use Zig only for Linux cross-compilation
|
||||||
|
- goos: linux
|
||||||
|
goarch: amd64
|
||||||
|
env:
|
||||||
|
- CGO_ENABLED=1
|
||||||
|
- CC=/tmp/zig-x86_64-linux-0.15.2/zig cc -target x86_64-linux-musl
|
||||||
|
- CXX=/tmp/zig-x86_64-linux-0.15.2/zig c++ -target x86_64-linux-musl
|
||||||
|
- goos: linux
|
||||||
|
goarch: arm64
|
||||||
|
env:
|
||||||
|
- CGO_ENABLED=1
|
||||||
|
- CC=/tmp/zig-x86_64-linux-0.15.2/zig cc -target aarch64-linux-musl
|
||||||
|
- CXX=/tmp/zig-x86_64-linux-0.15.2/zig c++ -target aarch64-linux-musl
|
||||||
|
# darwin/arm64 builds natively on macOS runner (no Zig)
|
||||||
|
|
||||||
|
- id: migrate
|
||||||
|
main: ./cmd/migrate
|
||||||
|
binary: migrate
|
||||||
|
env:
|
||||||
|
- CGO_ENABLED=1
|
||||||
|
tags:
|
||||||
|
- fts5
|
||||||
|
flags:
|
||||||
|
- -trimpath
|
||||||
|
ldflags:
|
||||||
|
- -s -w
|
||||||
|
- -X main.Version={{.Version}}
|
||||||
|
- -X main.GitCommit={{.ShortCommit}}
|
||||||
|
- -X main.BuildTime={{.Date}}
|
||||||
|
goos:
|
||||||
|
- linux
|
||||||
|
- darwin
|
||||||
|
goarch:
|
||||||
|
- amd64
|
||||||
|
- arm64
|
||||||
|
ignore:
|
||||||
|
- goos: darwin
|
||||||
|
goarch: amd64
|
||||||
|
overrides:
|
||||||
|
# Use Zig only for Linux cross-compilation
|
||||||
|
- goos: linux
|
||||||
|
goarch: amd64
|
||||||
|
env:
|
||||||
|
- CGO_ENABLED=1
|
||||||
|
- CC=/tmp/zig-x86_64-linux-0.15.2/zig cc -target x86_64-linux-musl
|
||||||
|
- CXX=/tmp/zig-x86_64-linux-0.15.2/zig c++ -target x86_64-linux-musl
|
||||||
|
- goos: linux
|
||||||
|
goarch: arm64
|
||||||
|
env:
|
||||||
|
- CGO_ENABLED=1
|
||||||
|
- CC=/tmp/zig-x86_64-linux-0.15.2/zig cc -target aarch64-linux-musl
|
||||||
|
- CXX=/tmp/zig-x86_64-linux-0.15.2/zig c++ -target aarch64-linux-musl
|
||||||
|
# darwin/arm64 builds natively on macOS runner (no Zig)
|
||||||
|
|
||||||
# Archives for releases
|
# Archives for releases
|
||||||
archives:
|
archives:
|
||||||
@@ -87,8 +148,13 @@ release:
|
|||||||
prerelease: auto
|
prerelease: auto
|
||||||
|
|
||||||
# Docker images (v2 - modern syntax)
|
# Docker images (v2 - modern syntax)
|
||||||
|
# Uses PRE-BUILT binaries from native builds (no Docker compilation - much faster!)
|
||||||
|
# GoReleaser injects the platform-specific binary into each Docker image automatically
|
||||||
|
# This avoids slow QEMU emulation for cross-architecture builds
|
||||||
dockers_v2:
|
dockers_v2:
|
||||||
# 1. Application Engine - Main GoHoarder server
|
# 1. Application Engine - Main GoHoarder server
|
||||||
|
# Uses pre-built binary from 'gohoarder' build (no Docker compilation)
|
||||||
|
# linux/arm64 binary is cross-compiled using Zig (fast, no QEMU!)
|
||||||
- id: gohoarder-server
|
- id: gohoarder-server
|
||||||
ids:
|
ids:
|
||||||
- gohoarder
|
- gohoarder
|
||||||
@@ -101,21 +167,21 @@ dockers_v2:
|
|||||||
- linux/amd64
|
- linux/amd64
|
||||||
- linux/arm64
|
- linux/arm64
|
||||||
dockerfile: Dockerfile.server
|
dockerfile: Dockerfile.server
|
||||||
labels:
|
flags:
|
||||||
org.opencontainers.image.title: GoHoarder Server
|
- "--pull"
|
||||||
org.opencontainers.image.description: Universal package cache proxy server
|
- "--label=org.opencontainers.image.title=GoHoarder Server"
|
||||||
org.opencontainers.image.url: https://github.com/lukaszraczylo/gohoarder
|
- "--label=org.opencontainers.image.description=Universal package cache proxy server"
|
||||||
org.opencontainers.image.source: https://github.com/lukaszraczylo/gohoarder
|
- "--label=org.opencontainers.image.url=https://github.com/lukaszraczylo/gohoarder"
|
||||||
org.opencontainers.image.version: "{{ .Version }}"
|
- "--label=org.opencontainers.image.source=https://github.com/lukaszraczylo/gohoarder"
|
||||||
org.opencontainers.image.created: "{{ .Date }}"
|
- "--label=org.opencontainers.image.version={{ .Version }}"
|
||||||
org.opencontainers.image.revision: "{{ .FullCommit }}"
|
- "--label=org.opencontainers.image.created={{ .Date }}"
|
||||||
|
- "--label=org.opencontainers.image.revision={{ .FullCommit }}"
|
||||||
extra_files:
|
extra_files:
|
||||||
- config.yaml.example
|
- config.yaml.example
|
||||||
|
|
||||||
# 2. Website - Frontend Dashboard
|
# 2. Website - Frontend Dashboard
|
||||||
|
# Note: Frontend is pre-built on CI runner and injected via frontend/dist
|
||||||
- id: gohoarder-frontend
|
- id: gohoarder-frontend
|
||||||
ids:
|
|
||||||
- gohoarder
|
|
||||||
images:
|
images:
|
||||||
- ghcr.io/lukaszraczylo/gohoarder-frontend
|
- ghcr.io/lukaszraczylo/gohoarder-frontend
|
||||||
tags:
|
tags:
|
||||||
@@ -125,18 +191,21 @@ dockers_v2:
|
|||||||
- linux/amd64
|
- linux/amd64
|
||||||
- linux/arm64
|
- linux/arm64
|
||||||
dockerfile: Dockerfile.frontend
|
dockerfile: Dockerfile.frontend
|
||||||
labels:
|
flags:
|
||||||
org.opencontainers.image.title: GoHoarder Frontend
|
- "--pull"
|
||||||
org.opencontainers.image.description: GoHoarder web dashboard
|
- "--label=org.opencontainers.image.title=GoHoarder Frontend"
|
||||||
org.opencontainers.image.url: https://github.com/lukaszraczylo/gohoarder
|
- "--label=org.opencontainers.image.description=GoHoarder web dashboard"
|
||||||
org.opencontainers.image.source: https://github.com/lukaszraczylo/gohoarder
|
- "--label=org.opencontainers.image.url=https://github.com/lukaszraczylo/gohoarder"
|
||||||
org.opencontainers.image.version: "{{ .Version }}"
|
- "--label=org.opencontainers.image.source=https://github.com/lukaszraczylo/gohoarder"
|
||||||
org.opencontainers.image.created: "{{ .Date }}"
|
- "--label=org.opencontainers.image.version={{ .Version }}"
|
||||||
org.opencontainers.image.revision: "{{ .FullCommit }}"
|
- "--label=org.opencontainers.image.created={{ .Date }}"
|
||||||
|
- "--label=org.opencontainers.image.revision={{ .FullCommit }}"
|
||||||
extra_files:
|
extra_files:
|
||||||
- frontend
|
- frontend/dist
|
||||||
|
|
||||||
# 3. Scanning Engine - Background scanner worker
|
# 3. Scanning Engine - Background scanner worker
|
||||||
|
# Uses pre-built binary from 'gohoarder' build (no Docker compilation)
|
||||||
|
# linux/arm64 binary is cross-compiled using Zig (fast, no QEMU!)
|
||||||
- id: gohoarder-scanner
|
- id: gohoarder-scanner
|
||||||
ids:
|
ids:
|
||||||
- gohoarder
|
- gohoarder
|
||||||
@@ -149,38 +218,44 @@ dockers_v2:
|
|||||||
- linux/amd64
|
- linux/amd64
|
||||||
- linux/arm64
|
- linux/arm64
|
||||||
dockerfile: Dockerfile.scanner
|
dockerfile: Dockerfile.scanner
|
||||||
labels:
|
flags:
|
||||||
org.opencontainers.image.title: GoHoarder Scanner
|
- "--pull"
|
||||||
org.opencontainers.image.description: GoHoarder vulnerability scanning engine
|
- "--label=org.opencontainers.image.title=GoHoarder Scanner"
|
||||||
org.opencontainers.image.url: https://github.com/lukaszraczylo/gohoarder
|
- "--label=org.opencontainers.image.description=GoHoarder vulnerability scanning engine"
|
||||||
org.opencontainers.image.source: https://github.com/lukaszraczylo/gohoarder
|
- "--label=org.opencontainers.image.url=https://github.com/lukaszraczylo/gohoarder"
|
||||||
org.opencontainers.image.version: "{{ .Version }}"
|
- "--label=org.opencontainers.image.source=https://github.com/lukaszraczylo/gohoarder"
|
||||||
org.opencontainers.image.created: "{{ .Date }}"
|
- "--label=org.opencontainers.image.version={{ .Version }}"
|
||||||
org.opencontainers.image.revision: "{{ .FullCommit }}"
|
- "--label=org.opencontainers.image.created={{ .Date }}"
|
||||||
|
- "--label=org.opencontainers.image.revision={{ .FullCommit }}"
|
||||||
extra_files:
|
extra_files:
|
||||||
- config.yaml.example
|
- config.yaml.example
|
||||||
|
|
||||||
# 4. Gateway - Nginx reverse proxy for unified deployment
|
# 4. Migration Engine - Database migration tool
|
||||||
- id: gohoarder-gateway
|
# Uses pre-built binary from 'migrate' build (no Docker compilation)
|
||||||
|
# linux/arm64 binary is cross-compiled using Zig (fast, no QEMU!)
|
||||||
|
- id: gohoarder-migrate
|
||||||
ids:
|
ids:
|
||||||
- gohoarder
|
- migrate
|
||||||
images:
|
images:
|
||||||
- ghcr.io/lukaszraczylo/gohoarder-gateway
|
- ghcr.io/lukaszraczylo/gohoarder-migrate
|
||||||
tags:
|
tags:
|
||||||
- "{{ .Version }}"
|
- "{{ .Version }}"
|
||||||
- latest
|
- latest
|
||||||
platforms:
|
platforms:
|
||||||
- linux/amd64
|
- linux/amd64
|
||||||
- linux/arm64
|
- linux/arm64
|
||||||
dockerfile: Dockerfile.gateway
|
dockerfile: Dockerfile.migrate
|
||||||
labels:
|
flags:
|
||||||
org.opencontainers.image.title: GoHoarder Gateway
|
- "--pull"
|
||||||
org.opencontainers.image.description: Nginx reverse proxy for unified GoHoarder deployment
|
- "--label=org.opencontainers.image.title=GoHoarder Migrate"
|
||||||
org.opencontainers.image.url: https://github.com/lukaszraczylo/gohoarder
|
- "--label=org.opencontainers.image.description=Database migration tool for GoHoarder V2 schema"
|
||||||
org.opencontainers.image.source: https://github.com/lukaszraczylo/gohoarder
|
- "--label=org.opencontainers.image.url=https://github.com/lukaszraczylo/gohoarder"
|
||||||
org.opencontainers.image.version: "{{ .Version }}"
|
- "--label=org.opencontainers.image.source=https://github.com/lukaszraczylo/gohoarder"
|
||||||
org.opencontainers.image.created: "{{ .Date }}"
|
- "--label=org.opencontainers.image.version={{ .Version }}"
|
||||||
org.opencontainers.image.revision: "{{ .FullCommit }}"
|
- "--label=org.opencontainers.image.created={{ .Date }}"
|
||||||
|
- "--label=org.opencontainers.image.revision={{ .FullCommit }}"
|
||||||
|
extra_files:
|
||||||
|
- migrations
|
||||||
|
|
||||||
# Artifact signing with cosign
|
# Artifact signing with cosign
|
||||||
signs:
|
signs:
|
||||||
|
|||||||
+127
-32
@@ -1,28 +1,15 @@
|
|||||||
# Website - Frontend Dashboard
|
# Website - Frontend Dashboard with integrated reverse proxy
|
||||||
# Build stage
|
# Combines frontend serving and backend proxy (previously separate gateway)
|
||||||
FROM node:20-alpine AS builder
|
# EXPECTS: Pre-built frontend files in frontend/dist/ directory
|
||||||
|
|
||||||
WORKDIR /build
|
|
||||||
|
|
||||||
# Copy frontend source
|
|
||||||
COPY frontend/package.json frontend/pnpm-lock.yaml ./
|
|
||||||
COPY frontend/ ./
|
|
||||||
|
|
||||||
# Install pnpm and dependencies
|
|
||||||
RUN npm install -g pnpm && \
|
|
||||||
pnpm install --frozen-lockfile
|
|
||||||
|
|
||||||
# Build the frontend
|
|
||||||
RUN pnpm run build
|
|
||||||
|
|
||||||
# Production stage
|
|
||||||
FROM nginx:alpine
|
FROM nginx:alpine
|
||||||
|
|
||||||
# Install envsubst for runtime configuration
|
# Install envsubst for runtime configuration
|
||||||
RUN apk add --no-cache gettext
|
RUN apk add --no-cache gettext
|
||||||
|
|
||||||
# Copy built frontend
|
# Copy pre-built frontend files
|
||||||
COPY --from=builder /build/dist /usr/share/nginx/html
|
# These are built on the CI runner and injected via extra_files
|
||||||
|
COPY frontend/dist /usr/share/nginx/html
|
||||||
|
|
||||||
# Create runtime config injection script
|
# Create runtime config injection script
|
||||||
RUN cat > /docker-entrypoint.d/40-inject-config.sh <<'EOF'
|
RUN cat > /docker-entrypoint.d/40-inject-config.sh <<'EOF'
|
||||||
@@ -48,21 +35,121 @@ EOF
|
|||||||
|
|
||||||
RUN chmod +x /docker-entrypoint.d/40-inject-config.sh
|
RUN chmod +x /docker-entrypoint.d/40-inject-config.sh
|
||||||
|
|
||||||
# Copy nginx configuration
|
# Create nginx templates directory and configuration template with backend proxy support
|
||||||
RUN cat > /etc/nginx/conf.d/default.conf <<'EOF'
|
RUN mkdir -p /etc/nginx/templates && \
|
||||||
|
cat > /etc/nginx/templates/default.conf.template <<'EOF'
|
||||||
|
# Upstream backend server
|
||||||
|
upstream backend {
|
||||||
|
server ${BACKEND_HOST}:${BACKEND_PORT};
|
||||||
|
keepalive 32;
|
||||||
|
}
|
||||||
|
|
||||||
|
# Cache configuration
|
||||||
|
proxy_cache_path /var/cache/nginx/static levels=1:2 keys_zone=static_cache:10m max_size=100m inactive=60m use_temp_path=off;
|
||||||
|
|
||||||
server {
|
server {
|
||||||
listen 80;
|
listen 80;
|
||||||
server_name _;
|
server_name ${SERVER_NAME};
|
||||||
root /usr/share/nginx/html;
|
root /usr/share/nginx/html;
|
||||||
index index.html;
|
index index.html;
|
||||||
|
|
||||||
|
# Security headers
|
||||||
|
add_header X-Frame-Options "SAMEORIGIN" always;
|
||||||
|
add_header X-Content-Type-Options "nosniff" always;
|
||||||
|
add_header X-XSS-Protection "1; mode=block" always;
|
||||||
|
add_header Referrer-Policy "strict-origin-when-cross-origin" always;
|
||||||
|
|
||||||
|
# Client body size for package uploads
|
||||||
|
client_max_body_size 500M;
|
||||||
|
client_body_timeout 300s;
|
||||||
|
|
||||||
|
# Logging
|
||||||
|
access_log /var/log/nginx/access.log combined;
|
||||||
|
error_log /var/log/nginx/error.log warn;
|
||||||
|
|
||||||
# Compression
|
# Compression
|
||||||
gzip on;
|
gzip on;
|
||||||
gzip_types text/plain text/css application/json application/javascript text/xml application/xml text/javascript;
|
gzip_types text/plain text/css application/json application/javascript text/xml application/xml text/javascript;
|
||||||
|
|
||||||
# SPA routing
|
# API endpoints - proxy to backend
|
||||||
location / {
|
location /api/ {
|
||||||
try_files $uri $uri/ /index.html;
|
# Proxy settings
|
||||||
|
proxy_pass http://backend;
|
||||||
|
proxy_http_version 1.1;
|
||||||
|
|
||||||
|
# Headers
|
||||||
|
proxy_set_header Host $host;
|
||||||
|
proxy_set_header X-Real-IP $remote_addr;
|
||||||
|
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||||
|
proxy_set_header X-Forwarded-Proto $scheme;
|
||||||
|
proxy_set_header X-Forwarded-Host $host;
|
||||||
|
proxy_set_header X-Forwarded-Port $server_port;
|
||||||
|
proxy_set_header Connection "";
|
||||||
|
|
||||||
|
# Timeouts for long-running operations
|
||||||
|
proxy_connect_timeout 60s;
|
||||||
|
proxy_send_timeout 300s;
|
||||||
|
proxy_read_timeout 300s;
|
||||||
|
|
||||||
|
# Buffer settings
|
||||||
|
proxy_buffering on;
|
||||||
|
proxy_buffer_size 4k;
|
||||||
|
proxy_buffers 8 4k;
|
||||||
|
proxy_busy_buffers_size 8k;
|
||||||
|
}
|
||||||
|
|
||||||
|
# Health check endpoint
|
||||||
|
location /health {
|
||||||
|
proxy_pass http://backend/health;
|
||||||
|
proxy_http_version 1.1;
|
||||||
|
proxy_set_header Connection "";
|
||||||
|
access_log off;
|
||||||
|
}
|
||||||
|
|
||||||
|
# Metrics endpoint
|
||||||
|
location /metrics {
|
||||||
|
proxy_pass http://backend/metrics;
|
||||||
|
proxy_http_version 1.1;
|
||||||
|
proxy_set_header Connection "";
|
||||||
|
}
|
||||||
|
|
||||||
|
# Package download endpoints
|
||||||
|
location ~ ^/(npm|pypi|go)/ {
|
||||||
|
proxy_pass http://backend;
|
||||||
|
proxy_http_version 1.1;
|
||||||
|
proxy_set_header Host $host;
|
||||||
|
proxy_set_header X-Real-IP $remote_addr;
|
||||||
|
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||||
|
proxy_set_header X-Forwarded-Proto $scheme;
|
||||||
|
proxy_set_header Connection "";
|
||||||
|
|
||||||
|
# Extended timeouts for package downloads
|
||||||
|
proxy_connect_timeout 60s;
|
||||||
|
proxy_send_timeout 600s;
|
||||||
|
proxy_read_timeout 600s;
|
||||||
|
|
||||||
|
# Large buffer for package downloads
|
||||||
|
proxy_buffering on;
|
||||||
|
proxy_buffer_size 128k;
|
||||||
|
proxy_buffers 4 256k;
|
||||||
|
proxy_busy_buffers_size 256k;
|
||||||
|
}
|
||||||
|
|
||||||
|
# WebSocket support
|
||||||
|
location /ws/ {
|
||||||
|
proxy_pass http://backend/ws/;
|
||||||
|
proxy_http_version 1.1;
|
||||||
|
proxy_set_header Upgrade $http_upgrade;
|
||||||
|
proxy_set_header Connection "upgrade";
|
||||||
|
proxy_set_header Host $host;
|
||||||
|
proxy_set_header X-Real-IP $remote_addr;
|
||||||
|
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||||
|
proxy_set_header X-Forwarded-Proto $scheme;
|
||||||
|
|
||||||
|
# WebSocket timeouts
|
||||||
|
proxy_connect_timeout 7d;
|
||||||
|
proxy_send_timeout 7d;
|
||||||
|
proxy_read_timeout 7d;
|
||||||
}
|
}
|
||||||
|
|
||||||
# Runtime configuration endpoint
|
# Runtime configuration endpoint
|
||||||
@@ -72,19 +159,23 @@ server {
|
|||||||
add_header Expires "0";
|
add_header Expires "0";
|
||||||
}
|
}
|
||||||
|
|
||||||
# Security headers
|
|
||||||
add_header X-Frame-Options "SAMEORIGIN" always;
|
|
||||||
add_header X-Content-Type-Options "nosniff" always;
|
|
||||||
add_header X-XSS-Protection "1; mode=block" always;
|
|
||||||
|
|
||||||
# Cache static assets
|
# Cache static assets
|
||||||
location ~* \.(js|css|png|jpg|jpeg|gif|ico|svg|woff|woff2|ttf|eot)$ {
|
location ~* \.(js|css|png|jpg|jpeg|gif|ico|svg|woff|woff2|ttf|eot)$ {
|
||||||
expires 1y;
|
expires 1y;
|
||||||
add_header Cache-Control "public, immutable";
|
add_header Cache-Control "public, immutable";
|
||||||
}
|
}
|
||||||
|
|
||||||
|
# Frontend SPA - serve index.html for all other routes
|
||||||
|
location / {
|
||||||
|
try_files $uri $uri/ /index.html;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
EOF
|
EOF
|
||||||
|
|
||||||
|
# Create cache directory
|
||||||
|
RUN mkdir -p /var/cache/nginx/static && \
|
||||||
|
chown -R nginx:nginx /var/cache/nginx
|
||||||
|
|
||||||
# Expose port
|
# Expose port
|
||||||
EXPOSE 80
|
EXPOSE 80
|
||||||
|
|
||||||
@@ -95,6 +186,10 @@ HEALTHCHECK --interval=30s --timeout=10s --start-period=5s --retries=3 \
|
|||||||
# Environment variables with defaults
|
# Environment variables with defaults
|
||||||
ENV API_BASE_URL=/api \
|
ENV API_BASE_URL=/api \
|
||||||
APP_VERSION=unknown \
|
APP_VERSION=unknown \
|
||||||
APP_NAME=GoHoarder
|
APP_NAME=GoHoarder \
|
||||||
|
BACKEND_HOST=gohoarder-server \
|
||||||
|
BACKEND_PORT=8080 \
|
||||||
|
SERVER_NAME=_
|
||||||
|
|
||||||
CMD ["nginx", "-g", "daemon off;"]
|
# Use nginx template substitution and start nginx
|
||||||
|
CMD ["/bin/sh", "-c", "envsubst '$$BACKEND_HOST $$BACKEND_PORT $$SERVER_NAME' < /etc/nginx/templates/default.conf.template > /etc/nginx/conf.d/default.conf && nginx -g 'daemon off;'"]
|
||||||
|
|||||||
@@ -1,197 +0,0 @@
|
|||||||
# Gateway - Nginx reverse proxy for unified deployment
|
|
||||||
# Routes traffic between frontend and backend under single vhost
|
|
||||||
FROM nginx:alpine
|
|
||||||
|
|
||||||
# Install envsubst for runtime configuration
|
|
||||||
RUN apk add --no-cache gettext
|
|
||||||
|
|
||||||
# Copy nginx configuration template
|
|
||||||
COPY <<'EOF' /etc/nginx/templates/default.conf.template
|
|
||||||
# Upstream servers
|
|
||||||
upstream backend {
|
|
||||||
server ${BACKEND_HOST}:${BACKEND_PORT};
|
|
||||||
keepalive 32;
|
|
||||||
}
|
|
||||||
|
|
||||||
upstream frontend {
|
|
||||||
server ${FRONTEND_HOST}:${FRONTEND_PORT};
|
|
||||||
keepalive 32;
|
|
||||||
}
|
|
||||||
|
|
||||||
# Rate limiting zones
|
|
||||||
limit_req_zone $binary_remote_addr zone=api_limit:10m rate=10r/s;
|
|
||||||
limit_req_zone $binary_remote_addr zone=download_limit:10m rate=5r/s;
|
|
||||||
|
|
||||||
# Cache configuration
|
|
||||||
proxy_cache_path /var/cache/nginx/static levels=1:2 keys_zone=static_cache:10m max_size=100m inactive=60m use_temp_path=off;
|
|
||||||
|
|
||||||
server {
|
|
||||||
listen 80;
|
|
||||||
server_name ${SERVER_NAME};
|
|
||||||
|
|
||||||
# Security headers
|
|
||||||
add_header X-Frame-Options "SAMEORIGIN" always;
|
|
||||||
add_header X-Content-Type-Options "nosniff" always;
|
|
||||||
add_header X-XSS-Protection "1; mode=block" always;
|
|
||||||
add_header Referrer-Policy "strict-origin-when-cross-origin" always;
|
|
||||||
|
|
||||||
# Client body size for package uploads
|
|
||||||
client_max_body_size 500M;
|
|
||||||
client_body_timeout 300s;
|
|
||||||
|
|
||||||
# Logging
|
|
||||||
access_log /var/log/nginx/access.log combined;
|
|
||||||
error_log /var/log/nginx/error.log warn;
|
|
||||||
|
|
||||||
# API endpoints - proxy to backend
|
|
||||||
location /api/ {
|
|
||||||
# Rate limiting
|
|
||||||
limit_req zone=api_limit burst=20 nodelay;
|
|
||||||
|
|
||||||
# Proxy settings
|
|
||||||
proxy_pass http://backend/;
|
|
||||||
proxy_http_version 1.1;
|
|
||||||
|
|
||||||
# Headers
|
|
||||||
proxy_set_header Host $host;
|
|
||||||
proxy_set_header X-Real-IP $remote_addr;
|
|
||||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
|
||||||
proxy_set_header X-Forwarded-Proto $scheme;
|
|
||||||
proxy_set_header X-Forwarded-Host $host;
|
|
||||||
proxy_set_header X-Forwarded-Port $server_port;
|
|
||||||
|
|
||||||
# Connection reuse
|
|
||||||
proxy_set_header Connection "";
|
|
||||||
|
|
||||||
# Timeouts for long-running operations
|
|
||||||
proxy_connect_timeout 60s;
|
|
||||||
proxy_send_timeout 300s;
|
|
||||||
proxy_read_timeout 300s;
|
|
||||||
|
|
||||||
# Buffer settings
|
|
||||||
proxy_buffering on;
|
|
||||||
proxy_buffer_size 4k;
|
|
||||||
proxy_buffers 8 4k;
|
|
||||||
proxy_busy_buffers_size 8k;
|
|
||||||
}
|
|
||||||
|
|
||||||
# Health check endpoint
|
|
||||||
location /health {
|
|
||||||
proxy_pass http://backend/health;
|
|
||||||
proxy_http_version 1.1;
|
|
||||||
proxy_set_header Connection "";
|
|
||||||
access_log off;
|
|
||||||
}
|
|
||||||
|
|
||||||
# Metrics endpoint (optional - may want to restrict access)
|
|
||||||
location /metrics {
|
|
||||||
# Uncomment to restrict to internal networks
|
|
||||||
# allow 10.0.0.0/8;
|
|
||||||
# allow 172.16.0.0/12;
|
|
||||||
# allow 192.168.0.0/16;
|
|
||||||
# deny all;
|
|
||||||
|
|
||||||
proxy_pass http://backend/metrics;
|
|
||||||
proxy_http_version 1.1;
|
|
||||||
proxy_set_header Connection "";
|
|
||||||
}
|
|
||||||
|
|
||||||
# Package download endpoints with rate limiting
|
|
||||||
location ~ ^/(npm|pypi|go)/ {
|
|
||||||
limit_req zone=download_limit burst=10 nodelay;
|
|
||||||
|
|
||||||
proxy_pass http://backend;
|
|
||||||
proxy_http_version 1.1;
|
|
||||||
proxy_set_header Host $host;
|
|
||||||
proxy_set_header X-Real-IP $remote_addr;
|
|
||||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
|
||||||
proxy_set_header X-Forwarded-Proto $scheme;
|
|
||||||
proxy_set_header Connection "";
|
|
||||||
|
|
||||||
# Extended timeouts for package downloads
|
|
||||||
proxy_connect_timeout 60s;
|
|
||||||
proxy_send_timeout 600s;
|
|
||||||
proxy_read_timeout 600s;
|
|
||||||
|
|
||||||
# Large buffer for package downloads
|
|
||||||
proxy_buffering on;
|
|
||||||
proxy_buffer_size 128k;
|
|
||||||
proxy_buffers 4 256k;
|
|
||||||
proxy_busy_buffers_size 256k;
|
|
||||||
}
|
|
||||||
|
|
||||||
# Frontend - serve SPA
|
|
||||||
location / {
|
|
||||||
proxy_pass http://frontend;
|
|
||||||
proxy_http_version 1.1;
|
|
||||||
proxy_set_header Host $host;
|
|
||||||
proxy_set_header X-Real-IP $remote_addr;
|
|
||||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
|
||||||
proxy_set_header X-Forwarded-Proto $scheme;
|
|
||||||
proxy_set_header Connection "";
|
|
||||||
|
|
||||||
# Cache static assets
|
|
||||||
proxy_cache static_cache;
|
|
||||||
proxy_cache_valid 200 1h;
|
|
||||||
proxy_cache_bypass $http_cache_control;
|
|
||||||
add_header X-Cache-Status $upstream_cache_status;
|
|
||||||
}
|
|
||||||
|
|
||||||
# WebSocket support (if needed for future features)
|
|
||||||
location /ws/ {
|
|
||||||
proxy_pass http://backend/ws/;
|
|
||||||
proxy_http_version 1.1;
|
|
||||||
proxy_set_header Upgrade $http_upgrade;
|
|
||||||
proxy_set_header Connection "upgrade";
|
|
||||||
proxy_set_header Host $host;
|
|
||||||
proxy_set_header X-Real-IP $remote_addr;
|
|
||||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
|
||||||
proxy_set_header X-Forwarded-Proto $scheme;
|
|
||||||
|
|
||||||
# WebSocket timeouts
|
|
||||||
proxy_connect_timeout 7d;
|
|
||||||
proxy_send_timeout 7d;
|
|
||||||
proxy_read_timeout 7d;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
# HTTPS server (uncomment and configure SSL certificates)
|
|
||||||
# server {
|
|
||||||
# listen 443 ssl http2;
|
|
||||||
# server_name ${SERVER_NAME};
|
|
||||||
#
|
|
||||||
# ssl_certificate /etc/nginx/ssl/cert.pem;
|
|
||||||
# ssl_certificate_key /etc/nginx/ssl/key.pem;
|
|
||||||
#
|
|
||||||
# # SSL configuration
|
|
||||||
# ssl_protocols TLSv1.2 TLSv1.3;
|
|
||||||
# ssl_ciphers HIGH:!aNULL:!MD5;
|
|
||||||
# ssl_prefer_server_ciphers on;
|
|
||||||
# ssl_session_cache shared:SSL:10m;
|
|
||||||
# ssl_session_timeout 10m;
|
|
||||||
#
|
|
||||||
# # Include all location blocks from above
|
|
||||||
# # ... (copy from HTTP server block)
|
|
||||||
# }
|
|
||||||
EOF
|
|
||||||
|
|
||||||
# Create cache directory
|
|
||||||
RUN mkdir -p /var/cache/nginx/static && \
|
|
||||||
chown -R nginx:nginx /var/cache/nginx
|
|
||||||
|
|
||||||
# Expose port
|
|
||||||
EXPOSE 80 443
|
|
||||||
|
|
||||||
# Health check
|
|
||||||
HEALTHCHECK --interval=30s --timeout=10s --start-period=5s --retries=3 \
|
|
||||||
CMD wget --quiet --tries=1 --spider http://localhost/health || exit 1
|
|
||||||
|
|
||||||
# Environment variables with defaults
|
|
||||||
ENV BACKEND_HOST=gohoarder-server \
|
|
||||||
BACKEND_PORT=8080 \
|
|
||||||
FRONTEND_HOST=gohoarder-frontend \
|
|
||||||
FRONTEND_PORT=80 \
|
|
||||||
SERVER_NAME=_
|
|
||||||
|
|
||||||
# Use nginx with template substitution
|
|
||||||
CMD ["/bin/sh", "-c", "envsubst '$$BACKEND_HOST $$BACKEND_PORT $$FRONTEND_HOST $$FRONTEND_PORT $$SERVER_NAME' < /etc/nginx/templates/default.conf.template > /etc/nginx/conf.d/default.conf && nginx -g 'daemon off;'"]
|
|
||||||
@@ -0,0 +1,37 @@
|
|||||||
|
# Migration Engine - Database Migration Tool
|
||||||
|
# This Dockerfile expects a PRE-BUILT binary from GoReleaser (no compilation)
|
||||||
|
# GoReleaser injects the platform-specific binary automatically
|
||||||
|
|
||||||
|
FROM --platform=$TARGETPLATFORM alpine:latest
|
||||||
|
|
||||||
|
ARG TARGETARCH
|
||||||
|
|
||||||
|
# Install runtime dependencies (including CGO/SQLite dependencies)
|
||||||
|
RUN apk add --no-cache \
|
||||||
|
ca-certificates \
|
||||||
|
postgresql-client \
|
||||||
|
mysql-client \
|
||||||
|
busybox-extras \
|
||||||
|
sqlite-libs \
|
||||||
|
musl \
|
||||||
|
&& update-ca-certificates
|
||||||
|
|
||||||
|
# Create non-root user
|
||||||
|
RUN addgroup -g 1000 gohoarder && \
|
||||||
|
adduser -D -u 1000 -G gohoarder gohoarder
|
||||||
|
|
||||||
|
# Copy pre-built binary from GoReleaser
|
||||||
|
# GoReleaser will automatically inject the correct binary for the target platform
|
||||||
|
# In split/merge mode, binaries are in linux/${TARGETARCH}/ subdirectories
|
||||||
|
COPY linux/${TARGETARCH}/migrate /usr/local/bin/migrate
|
||||||
|
RUN chmod +x /usr/local/bin/migrate
|
||||||
|
|
||||||
|
# Copy migration SQL files
|
||||||
|
COPY migrations /migrations
|
||||||
|
|
||||||
|
WORKDIR /app
|
||||||
|
USER gohoarder
|
||||||
|
|
||||||
|
# Run migrations
|
||||||
|
ENTRYPOINT ["/usr/local/bin/migrate"]
|
||||||
|
CMD ["--action=migrate"]
|
||||||
+39
-15
@@ -1,10 +1,12 @@
|
|||||||
# Scanning Engine - Background Scanner Worker
|
# Scanning Engine - Background Scanner Worker
|
||||||
ARG TARGETOS
|
# This Dockerfile expects a PRE-BUILT binary from GoReleaser (no compilation)
|
||||||
|
# GoReleaser injects the platform-specific binary automatically
|
||||||
|
|
||||||
|
FROM --platform=$TARGETPLATFORM alpine:latest
|
||||||
|
|
||||||
ARG TARGETARCH
|
ARG TARGETARCH
|
||||||
|
|
||||||
FROM alpine:latest
|
# Install scanning tools and runtime dependencies (including CGO/SQLite dependencies)
|
||||||
|
|
||||||
# Install scanning tools and runtime dependencies
|
|
||||||
RUN apk add --no-cache \
|
RUN apk add --no-cache \
|
||||||
ca-certificates \
|
ca-certificates \
|
||||||
tzdata \
|
tzdata \
|
||||||
@@ -12,6 +14,12 @@ RUN apk add --no-cache \
|
|||||||
curl \
|
curl \
|
||||||
wget \
|
wget \
|
||||||
bash \
|
bash \
|
||||||
|
sqlite-libs \
|
||||||
|
musl \
|
||||||
|
python3 \
|
||||||
|
py3-pip \
|
||||||
|
npm \
|
||||||
|
go \
|
||||||
&& update-ca-certificates
|
&& update-ca-certificates
|
||||||
|
|
||||||
# Install Trivy for container scanning
|
# Install Trivy for container scanning
|
||||||
@@ -20,24 +28,40 @@ RUN curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/
|
|||||||
# Install Grype for vulnerability scanning
|
# Install Grype for vulnerability scanning
|
||||||
RUN curl -sSfL https://raw.githubusercontent.com/anchore/grype/main/install.sh | sh -s -- -b /usr/local/bin
|
RUN curl -sSfL https://raw.githubusercontent.com/anchore/grype/main/install.sh | sh -s -- -b /usr/local/bin
|
||||||
|
|
||||||
|
# Install govulncheck for Go vulnerability scanning
|
||||||
|
RUN go install golang.org/x/vuln/cmd/govulncheck@latest && \
|
||||||
|
mv /root/go/bin/govulncheck /usr/local/bin/
|
||||||
|
|
||||||
|
# Install pip-audit for Python package vulnerability scanning
|
||||||
|
RUN pip3 install --no-cache-dir pip-audit --break-system-packages
|
||||||
|
|
||||||
# Create non-root user
|
# Create non-root user
|
||||||
RUN addgroup -g 1000 scanner && \
|
RUN addgroup -g 1000 scanner && \
|
||||||
adduser -D -u 1000 -G scanner scanner
|
adduser -D -u 1000 -G scanner scanner
|
||||||
|
|
||||||
# Create necessary directories
|
# Create necessary directories with proper permissions
|
||||||
RUN mkdir -p /data/cache /data/scans && \
|
RUN mkdir -p /var/cache/gohoarder \
|
||||||
chown -R scanner:scanner /data
|
/var/lib/gohoarder/metadata \
|
||||||
|
/var/lib/trivy \
|
||||||
|
/tmp/gohoarder && \
|
||||||
|
chown -R scanner:scanner /var/cache/gohoarder \
|
||||||
|
/var/lib/gohoarder \
|
||||||
|
/var/lib/trivy \
|
||||||
|
/tmp/gohoarder && \
|
||||||
|
chmod -R 750 /var/cache/gohoarder \
|
||||||
|
/var/lib/gohoarder \
|
||||||
|
/var/lib/trivy
|
||||||
|
|
||||||
# Copy binary (from platform-specific path)
|
# Copy pre-built binary from GoReleaser
|
||||||
ARG TARGETOS
|
# GoReleaser will automatically inject the correct binary for the target platform
|
||||||
ARG TARGETARCH
|
# In split/merge mode, binaries are in linux/${TARGETARCH}/ subdirectories
|
||||||
COPY ${TARGETOS}/${TARGETARCH}/gohoarder /usr/local/bin/gohoarder
|
COPY linux/${TARGETARCH}/gohoarder /usr/local/bin/gohoarder
|
||||||
RUN chmod +x /usr/local/bin/gohoarder
|
RUN chmod +x /usr/local/bin/gohoarder
|
||||||
|
|
||||||
# Copy example config
|
# Copy example config
|
||||||
COPY config.yaml.example /etc/gohoarder/config.yaml.example
|
COPY config.yaml.example /etc/gohoarder/config.yaml.example
|
||||||
|
|
||||||
WORKDIR /data
|
WORKDIR /var/cache/gohoarder
|
||||||
USER scanner
|
USER scanner
|
||||||
|
|
||||||
# Expose metrics port
|
# Expose metrics port
|
||||||
@@ -53,7 +77,7 @@ ENV SCANNER_MODE=true \
|
|||||||
SCANNER_INTERVAL=300
|
SCANNER_INTERVAL=300
|
||||||
|
|
||||||
# Run the scanner in background mode
|
# Run the scanner in background mode
|
||||||
# Note: You may need to add a scanner-specific command to your CLI
|
# The scanner runs the same serve command but uses SCANNER_MODE env var
|
||||||
# For now, this assumes the serve command can run in scanner mode
|
# and configuration to determine its role
|
||||||
ENTRYPOINT ["/usr/local/bin/gohoarder"]
|
ENTRYPOINT ["/usr/local/bin/gohoarder"]
|
||||||
CMD ["serve", "--scanner-only"]
|
CMD ["serve"]
|
||||||
|
|||||||
+23
-13
@@ -1,33 +1,43 @@
|
|||||||
# Application Engine - GoHoarder Server
|
# Application Engine - Main GoHoarder Server
|
||||||
ARG TARGETOS
|
# This Dockerfile expects a PRE-BUILT binary from GoReleaser (no compilation)
|
||||||
|
# GoReleaser injects the platform-specific binary automatically
|
||||||
|
|
||||||
|
FROM --platform=$TARGETPLATFORM alpine:latest
|
||||||
|
|
||||||
ARG TARGETARCH
|
ARG TARGETARCH
|
||||||
|
|
||||||
FROM alpine:latest
|
# Install runtime dependencies (CGO/SQLite requires these)
|
||||||
|
|
||||||
# Install runtime dependencies
|
|
||||||
RUN apk add --no-cache \
|
RUN apk add --no-cache \
|
||||||
ca-certificates \
|
ca-certificates \
|
||||||
tzdata \
|
tzdata \
|
||||||
|
sqlite-libs \
|
||||||
|
musl \
|
||||||
&& update-ca-certificates
|
&& update-ca-certificates
|
||||||
|
|
||||||
# Create non-root user
|
# Create non-root user
|
||||||
RUN addgroup -g 1000 gohoarder && \
|
RUN addgroup -g 1000 gohoarder && \
|
||||||
adduser -D -u 1000 -G gohoarder gohoarder
|
adduser -D -u 1000 -G gohoarder gohoarder
|
||||||
|
|
||||||
# Create necessary directories
|
# Create necessary directories with proper permissions
|
||||||
RUN mkdir -p /data/cache /data/metadata && \
|
RUN mkdir -p /var/cache/gohoarder \
|
||||||
chown -R gohoarder:gohoarder /data
|
/var/lib/gohoarder/metadata \
|
||||||
|
/tmp/gohoarder && \
|
||||||
|
chown -R gohoarder:gohoarder /var/cache/gohoarder \
|
||||||
|
/var/lib/gohoarder \
|
||||||
|
/tmp/gohoarder && \
|
||||||
|
chmod -R 750 /var/cache/gohoarder \
|
||||||
|
/var/lib/gohoarder
|
||||||
|
|
||||||
# Copy binary (from platform-specific path)
|
# Copy pre-built binary from GoReleaser
|
||||||
ARG TARGETOS
|
# GoReleaser will automatically inject the correct binary for the target platform
|
||||||
ARG TARGETARCH
|
# In split/merge mode, binaries are in linux/${TARGETARCH}/ subdirectories
|
||||||
COPY ${TARGETOS}/${TARGETARCH}/gohoarder /usr/local/bin/gohoarder
|
COPY linux/${TARGETARCH}/gohoarder /usr/local/bin/gohoarder
|
||||||
RUN chmod +x /usr/local/bin/gohoarder
|
RUN chmod +x /usr/local/bin/gohoarder
|
||||||
|
|
||||||
# Copy example config
|
# Copy example config
|
||||||
COPY config.yaml.example /etc/gohoarder/config.yaml.example
|
COPY config.yaml.example /etc/gohoarder/config.yaml.example
|
||||||
|
|
||||||
WORKDIR /data
|
WORKDIR /var/cache/gohoarder
|
||||||
USER gohoarder
|
USER gohoarder
|
||||||
|
|
||||||
# Expose ports
|
# Expose ports
|
||||||
|
|||||||
@@ -78,17 +78,24 @@ clean: ## Clean build artifacts
|
|||||||
@rm -f *.db *.db-shm *.db-wal
|
@rm -f *.db *.db-shm *.db-wal
|
||||||
@echo "Clean complete"
|
@echo "Clean complete"
|
||||||
|
|
||||||
clean-db: ## Clean all local cache and database files (from config.yaml paths)
|
clean-db: ## Clean all local cache and database files (requires confirmation)
|
||||||
@echo "WARNING: This will delete all cached packages and scan results!"
|
@echo "WARNING: This will delete all cached packages and scan results!"
|
||||||
@echo "Paths from config.yaml:"
|
@echo "Paths to be cleaned:"
|
||||||
@echo " - ./data/storage (package cache)"
|
@echo " - ./data/storage (package cache)"
|
||||||
@echo " - ./data/gohoarder.db (metadata database)"
|
@echo " - ./data/gohoarder.db and gohoarder.db (metadata database)"
|
||||||
@echo " - /tmp/trivy (Trivy cache)"
|
@echo " - /tmp/trivy (Trivy cache)"
|
||||||
@echo ""
|
@echo ""
|
||||||
@read -p "Are you sure you want to continue? [y/N] " confirm && [ "$$confirm" = "y" ] || exit 1
|
@printf "Are you sure you want to continue? [y/N] " && read confirm && [ "$$confirm" = "y" ] || (echo "Cancelled." && exit 1)
|
||||||
@echo "Cleaning database and cache..."
|
@echo "Cleaning database and cache..."
|
||||||
@rm -rf ./data/storage
|
@rm -rf ./data/storage ./data
|
||||||
@rm -f ./data/gohoarder.db ./data/gohoarder.db-shm ./data/gohoarder.db-wal
|
@rm -f gohoarder.db gohoarder.db-shm gohoarder.db-wal
|
||||||
|
@rm -rf /tmp/trivy
|
||||||
|
@echo "Database and cache cleaned successfully"
|
||||||
|
|
||||||
|
clean-db-force: ## Clean all local cache and database files (no confirmation)
|
||||||
|
@echo "Cleaning database and cache..."
|
||||||
|
@rm -rf ./data/storage ./data
|
||||||
|
@rm -f gohoarder.db gohoarder.db-shm gohoarder.db-wal
|
||||||
@rm -rf /tmp/trivy
|
@rm -rf /tmp/trivy
|
||||||
@echo "Database and cache cleaned successfully"
|
@echo "Database and cache cleaned successfully"
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,258 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"database/sql"
|
||||||
|
"flag"
|
||||||
|
"fmt"
|
||||||
|
stdlog "log"
|
||||||
|
"os"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/go-gormigrate/gormigrate/v2"
|
||||||
|
_ "github.com/go-sql-driver/mysql"
|
||||||
|
_ "github.com/lib/pq"
|
||||||
|
"github.com/lukaszraczylo/gohoarder/pkg/metadata/gormstore"
|
||||||
|
"github.com/rs/zerolog"
|
||||||
|
"github.com/rs/zerolog/log"
|
||||||
|
"gorm.io/driver/mysql"
|
||||||
|
"gorm.io/driver/postgres"
|
||||||
|
"gorm.io/driver/sqlite"
|
||||||
|
"gorm.io/gorm"
|
||||||
|
"gorm.io/gorm/logger"
|
||||||
|
)
|
||||||
|
|
||||||
|
type MigrationConfig struct {
|
||||||
|
Driver string
|
||||||
|
DSN string
|
||||||
|
Timeout time.Duration
|
||||||
|
Action string // migrate, rollback, rollback-to, list
|
||||||
|
TargetID string // For rollback-to
|
||||||
|
LogLevel string
|
||||||
|
}
|
||||||
|
|
||||||
|
func main() {
|
||||||
|
cfg := MigrationConfig{}
|
||||||
|
|
||||||
|
flag.StringVar(&cfg.Driver, "driver", os.Getenv("DB_DRIVER"), "Database driver (postgres, mysql, sqlite)")
|
||||||
|
flag.StringVar(&cfg.DSN, "dsn", os.Getenv("DATABASE_URL"), "Database connection string")
|
||||||
|
flag.DurationVar(&cfg.Timeout, "timeout", 10*time.Minute, "Migration timeout")
|
||||||
|
flag.StringVar(&cfg.Action, "action", "migrate", "Action: migrate, rollback, rollback-to, list")
|
||||||
|
flag.StringVar(&cfg.TargetID, "target", "", "Target migration ID (for rollback-to)")
|
||||||
|
flag.StringVar(&cfg.LogLevel, "log-level", "info", "Log level: debug, info, warn, error")
|
||||||
|
flag.Parse()
|
||||||
|
|
||||||
|
// Setup logging
|
||||||
|
setupLogging(cfg.LogLevel)
|
||||||
|
|
||||||
|
log.Info().
|
||||||
|
Str("driver", cfg.Driver).
|
||||||
|
Str("action", cfg.Action).
|
||||||
|
Msg("Starting database migration")
|
||||||
|
|
||||||
|
if err := RunMigration(cfg); err != nil {
|
||||||
|
log.Fatal().Err(err).Msg("Migration failed")
|
||||||
|
}
|
||||||
|
|
||||||
|
log.Info().Msg("Migration completed successfully")
|
||||||
|
}
|
||||||
|
|
||||||
|
func setupLogging(level string) {
|
||||||
|
zerolog.TimeFieldFormat = zerolog.TimeFormatUnix
|
||||||
|
log.Logger = log.Output(zerolog.ConsoleWriter{Out: os.Stdout, TimeFormat: time.RFC3339})
|
||||||
|
|
||||||
|
switch level {
|
||||||
|
case "debug":
|
||||||
|
zerolog.SetGlobalLevel(zerolog.DebugLevel)
|
||||||
|
case "info":
|
||||||
|
zerolog.SetGlobalLevel(zerolog.InfoLevel)
|
||||||
|
case "warn":
|
||||||
|
zerolog.SetGlobalLevel(zerolog.WarnLevel)
|
||||||
|
case "error":
|
||||||
|
zerolog.SetGlobalLevel(zerolog.ErrorLevel)
|
||||||
|
default:
|
||||||
|
zerolog.SetGlobalLevel(zerolog.InfoLevel)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func RunMigration(cfg MigrationConfig) error {
|
||||||
|
// Validate config
|
||||||
|
if cfg.Driver == "" {
|
||||||
|
return fmt.Errorf("driver is required (set DB_DRIVER or --driver)")
|
||||||
|
}
|
||||||
|
if cfg.DSN == "" {
|
||||||
|
return fmt.Errorf("DSN is required (set DATABASE_URL or --dsn)")
|
||||||
|
}
|
||||||
|
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), cfg.Timeout)
|
||||||
|
defer cancel()
|
||||||
|
|
||||||
|
// Connect to database
|
||||||
|
db, err := connectToDatabase(cfg.Driver, cfg.DSN)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to connect to database: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
sqlDB, err := db.DB()
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to get sql.DB: %w", err)
|
||||||
|
}
|
||||||
|
defer sqlDB.Close()
|
||||||
|
|
||||||
|
// Wait for database to be ready
|
||||||
|
if err := waitForDB(ctx, sqlDB, 60*time.Second); err != nil {
|
||||||
|
return fmt.Errorf("database not ready: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Initialize gormigrate with custom options
|
||||||
|
opts := gormigrate.DefaultOptions
|
||||||
|
opts.TableName = "gohoarder_migrations"
|
||||||
|
m := gormigrate.New(db, opts, gormstore.GetMigrations())
|
||||||
|
|
||||||
|
log.Info().
|
||||||
|
Str("table", "gohoarder_migrations").
|
||||||
|
Msg("Migration tracking table initialized")
|
||||||
|
|
||||||
|
// Execute action
|
||||||
|
switch cfg.Action {
|
||||||
|
case "migrate":
|
||||||
|
return runMigrate(m)
|
||||||
|
case "rollback":
|
||||||
|
return runRollback(m)
|
||||||
|
case "rollback-to":
|
||||||
|
if cfg.TargetID == "" {
|
||||||
|
return fmt.Errorf("target migration ID required for rollback-to")
|
||||||
|
}
|
||||||
|
return runRollbackTo(m, cfg.TargetID)
|
||||||
|
case "list":
|
||||||
|
return listMigrations(db)
|
||||||
|
default:
|
||||||
|
return fmt.Errorf("unknown action: %s (use: migrate, rollback, rollback-to, list)", cfg.Action)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func connectToDatabase(driver, dsn string) (*gorm.DB, error) {
|
||||||
|
// Configure GORM logger using standard library log
|
||||||
|
gormLogger := logger.New(
|
||||||
|
stdlog.New(os.Stdout, "\r\n", stdlog.LstdFlags),
|
||||||
|
logger.Config{
|
||||||
|
SlowThreshold: 200 * time.Millisecond,
|
||||||
|
LogLevel: logger.Info,
|
||||||
|
IgnoreRecordNotFoundError: true,
|
||||||
|
Colorful: true,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
var dialector gorm.Dialector
|
||||||
|
switch driver {
|
||||||
|
case "sqlite":
|
||||||
|
dialector = sqlite.Open(dsn)
|
||||||
|
case "postgres", "postgresql":
|
||||||
|
dialector = postgres.Open(dsn)
|
||||||
|
case "mysql":
|
||||||
|
dialector = mysql.Open(dsn)
|
||||||
|
default:
|
||||||
|
return nil, fmt.Errorf("unsupported driver: %s", driver)
|
||||||
|
}
|
||||||
|
|
||||||
|
db, err := gorm.Open(dialector, &gorm.Config{
|
||||||
|
Logger: gormLogger,
|
||||||
|
SkipDefaultTransaction: false, // Migrations should be transactional
|
||||||
|
PrepareStmt: true,
|
||||||
|
})
|
||||||
|
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
return db, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func waitForDB(ctx context.Context, db *sql.DB, timeout time.Duration) error {
|
||||||
|
deadline := time.Now().Add(timeout)
|
||||||
|
attempt := 0
|
||||||
|
|
||||||
|
for {
|
||||||
|
attempt++
|
||||||
|
if time.Now().After(deadline) {
|
||||||
|
return fmt.Errorf("timeout waiting for database after %d attempts", attempt)
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := db.PingContext(ctx); err == nil {
|
||||||
|
log.Info().
|
||||||
|
Int("attempts", attempt).
|
||||||
|
Msg("Database is ready")
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
log.Debug().
|
||||||
|
Int("attempt", attempt).
|
||||||
|
Msg("Waiting for database...")
|
||||||
|
time.Sleep(2 * time.Second)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func runMigrate(m *gormigrate.Gormigrate) error {
|
||||||
|
log.Info().Msg("Running migrations...")
|
||||||
|
|
||||||
|
if err := m.Migrate(); err != nil {
|
||||||
|
return fmt.Errorf("migration failed: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
log.Info().Msg("✓ All migrations applied successfully")
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func runRollback(m *gormigrate.Gormigrate) error {
|
||||||
|
log.Warn().Msg("Rolling back last migration...")
|
||||||
|
|
||||||
|
if err := m.RollbackLast(); err != nil {
|
||||||
|
return fmt.Errorf("rollback failed: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
log.Info().Msg("✓ Rollback completed")
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func runRollbackTo(m *gormigrate.Gormigrate, targetID string) error {
|
||||||
|
log.Warn().
|
||||||
|
Str("target_id", targetID).
|
||||||
|
Msg("Rolling back to migration...")
|
||||||
|
|
||||||
|
if err := m.RollbackTo(targetID); err != nil {
|
||||||
|
return fmt.Errorf("rollback to %s failed: %w", targetID, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
log.Info().
|
||||||
|
Str("target_id", targetID).
|
||||||
|
Msg("✓ Rollback completed")
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func listMigrations(db *gorm.DB) error {
|
||||||
|
log.Info().Msg("Applied migrations:")
|
||||||
|
|
||||||
|
type Migration struct {
|
||||||
|
ID string
|
||||||
|
}
|
||||||
|
|
||||||
|
var migrations []Migration
|
||||||
|
if err := db.Table("gohoarder_migrations").Find(&migrations).Error; err != nil {
|
||||||
|
return fmt.Errorf("failed to list migrations: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(migrations) == 0 {
|
||||||
|
log.Info().Msg(" (no migrations applied yet)")
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, m := range migrations {
|
||||||
|
log.Info().Str("id", m.ID).Msg(" ✓")
|
||||||
|
}
|
||||||
|
|
||||||
|
log.Info().
|
||||||
|
Int("total", len(migrations)).
|
||||||
|
Msg("Applied migrations")
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
+37
-4
@@ -40,20 +40,53 @@ storage:
|
|||||||
domain: ""
|
domain: ""
|
||||||
|
|
||||||
metadata:
|
metadata:
|
||||||
backend: "sqlite" # sqlite, postgresql, file
|
# Backend: sqlite, postgresql, mysql, mariadb, file
|
||||||
connection: "file:gohoarder.db?cache=shared&mode=rwc"
|
#
|
||||||
|
# Choose based on your deployment:
|
||||||
|
# - sqlite: Single instance, local storage (NOT for network filesystems like SMB/NFS!)
|
||||||
|
# - postgresql: Production, multiple replicas, works with any storage including SMB/NFS
|
||||||
|
# - mysql: Production alternative to PostgreSQL
|
||||||
|
# - file: Simple file-based metadata (limited features)
|
||||||
|
#
|
||||||
|
# IMPORTANT: SQLite + SMB/NFS = Database locked errors!
|
||||||
|
# For network storage (SMB, NFS), use PostgreSQL or MySQL.
|
||||||
|
backend: "sqlite"
|
||||||
|
connection: "file:gohoarder.db?cache=shared&mode=rwc" # Legacy, not used with GORM
|
||||||
|
|
||||||
|
# SQLite configuration (for local storage only)
|
||||||
|
# Use with local storage classes (local-path, hostPath, or RWX like longhorn)
|
||||||
|
# DO NOT use with SMB/NFS network storage!
|
||||||
sqlite:
|
sqlite:
|
||||||
path: "gohoarder.db"
|
path: "gohoarder.db"
|
||||||
wal_mode: true
|
wal_mode: true # Set to false for network filesystems if you must use SQLite
|
||||||
|
|
||||||
|
# PostgreSQL configuration (recommended for production)
|
||||||
|
# Works with any storage including SMB/NFS
|
||||||
|
# Supports multiple replicas and high availability
|
||||||
postgresql:
|
postgresql:
|
||||||
host: "localhost"
|
host: "localhost"
|
||||||
port: 5432
|
port: 5432
|
||||||
database: "gohoarder"
|
database: "gohoarder"
|
||||||
user: "gohoarder"
|
user: "gohoarder"
|
||||||
password: ""
|
password: ""
|
||||||
ssl_mode: "disable"
|
ssl_mode: "disable" # disable, require, verify-ca, verify-full
|
||||||
|
|
||||||
|
# MySQL/MariaDB configuration (alternative to PostgreSQL)
|
||||||
|
# Works with any storage including SMB/NFS
|
||||||
|
mysql:
|
||||||
|
host: "localhost"
|
||||||
|
port: 3306
|
||||||
|
database: "gohoarder"
|
||||||
|
user: "gohoarder"
|
||||||
|
password: ""
|
||||||
|
charset: "utf8mb4"
|
||||||
|
parse_time: true
|
||||||
|
|
||||||
|
# GORM connection pool settings (applies to all database backends)
|
||||||
|
max_open_conns: 25 # Maximum number of open connections to the database
|
||||||
|
max_idle_conns: 5 # Maximum number of idle connections in the pool
|
||||||
|
conn_max_lifetime: 3600 # Maximum lifetime of a connection in seconds (1 hour)
|
||||||
|
log_level: "warn" # GORM log level: silent, error, warn, info
|
||||||
|
|
||||||
cache:
|
cache:
|
||||||
default_ttl: "168h" # 7 days
|
default_ttl: "168h" # 7 days
|
||||||
|
|||||||
@@ -1,5 +1,6 @@
|
|||||||
# GoHoarder - Kubernetes Deployment (All-in-One)
|
# GoHoarder - Kubernetes Deployment (All-in-One)
|
||||||
# This manifest deploys all GoHoarder services under a single ingress
|
# This manifest deploys all GoHoarder services under a single ingress
|
||||||
|
# The frontend includes integrated nginx reverse proxy to the backend
|
||||||
#
|
#
|
||||||
# Usage:
|
# Usage:
|
||||||
# kubectl create namespace gohoarder
|
# kubectl create namespace gohoarder
|
||||||
@@ -222,6 +223,13 @@ spec:
|
|||||||
value: "1.0.0"
|
value: "1.0.0"
|
||||||
- name: APP_NAME
|
- name: APP_NAME
|
||||||
value: GoHoarder
|
value: GoHoarder
|
||||||
|
# Backend proxy configuration (frontend includes nginx reverse proxy)
|
||||||
|
- name: BACKEND_HOST
|
||||||
|
value: gohoarder-server
|
||||||
|
- name: BACKEND_PORT
|
||||||
|
value: "8080"
|
||||||
|
- name: SERVER_NAME
|
||||||
|
value: hoarder.i.raczylo.com
|
||||||
livenessProbe:
|
livenessProbe:
|
||||||
httpGet:
|
httpGet:
|
||||||
path: /
|
path: /
|
||||||
@@ -325,88 +333,6 @@ spec:
|
|||||||
configMap:
|
configMap:
|
||||||
name: gohoarder-config
|
name: gohoarder-config
|
||||||
|
|
||||||
---
|
|
||||||
# Deployment - Gateway (Nginx Reverse Proxy)
|
|
||||||
apiVersion: apps/v1
|
|
||||||
kind: Deployment
|
|
||||||
metadata:
|
|
||||||
name: gohoarder-gateway
|
|
||||||
namespace: gohoarder
|
|
||||||
labels:
|
|
||||||
app.kubernetes.io/name: gohoarder
|
|
||||||
app.kubernetes.io/component: gateway
|
|
||||||
spec:
|
|
||||||
replicas: 2
|
|
||||||
selector:
|
|
||||||
matchLabels:
|
|
||||||
app.kubernetes.io/name: gohoarder
|
|
||||||
app.kubernetes.io/component: gateway
|
|
||||||
template:
|
|
||||||
metadata:
|
|
||||||
labels:
|
|
||||||
app.kubernetes.io/name: gohoarder
|
|
||||||
app.kubernetes.io/component: gateway
|
|
||||||
spec:
|
|
||||||
containers:
|
|
||||||
- name: gateway
|
|
||||||
image: ghcr.io/lukaszraczylo/gohoarder-gateway:latest
|
|
||||||
imagePullPolicy: Always
|
|
||||||
ports:
|
|
||||||
- name: http
|
|
||||||
containerPort: 80
|
|
||||||
protocol: TCP
|
|
||||||
env:
|
|
||||||
- name: BACKEND_HOST
|
|
||||||
value: gohoarder-server
|
|
||||||
- name: BACKEND_PORT
|
|
||||||
value: "8080"
|
|
||||||
- name: FRONTEND_HOST
|
|
||||||
value: gohoarder-frontend
|
|
||||||
- name: FRONTEND_PORT
|
|
||||||
value: "80"
|
|
||||||
- name: SERVER_NAME
|
|
||||||
value: hoarder.i.raczylo.com
|
|
||||||
livenessProbe:
|
|
||||||
httpGet:
|
|
||||||
path: /health
|
|
||||||
port: 80
|
|
||||||
initialDelaySeconds: 5
|
|
||||||
periodSeconds: 30
|
|
||||||
readinessProbe:
|
|
||||||
httpGet:
|
|
||||||
path: /health
|
|
||||||
port: 80
|
|
||||||
initialDelaySeconds: 5
|
|
||||||
periodSeconds: 10
|
|
||||||
resources:
|
|
||||||
requests:
|
|
||||||
cpu: 100m
|
|
||||||
memory: 128Mi
|
|
||||||
limits:
|
|
||||||
cpu: 500m
|
|
||||||
memory: 256Mi
|
|
||||||
|
|
||||||
---
|
|
||||||
# Service - Gateway
|
|
||||||
apiVersion: v1
|
|
||||||
kind: Service
|
|
||||||
metadata:
|
|
||||||
name: gohoarder-gateway
|
|
||||||
namespace: gohoarder
|
|
||||||
labels:
|
|
||||||
app.kubernetes.io/name: gohoarder
|
|
||||||
app.kubernetes.io/component: gateway
|
|
||||||
spec:
|
|
||||||
type: ClusterIP
|
|
||||||
ports:
|
|
||||||
- name: http
|
|
||||||
port: 80
|
|
||||||
targetPort: http
|
|
||||||
protocol: TCP
|
|
||||||
selector:
|
|
||||||
app.kubernetes.io/name: gohoarder
|
|
||||||
app.kubernetes.io/component: gateway
|
|
||||||
|
|
||||||
---
|
---
|
||||||
# Ingress - Expose via domain
|
# Ingress - Expose via domain
|
||||||
apiVersion: networking.k8s.io/v1
|
apiVersion: networking.k8s.io/v1
|
||||||
@@ -436,7 +362,7 @@ spec:
|
|||||||
pathType: Prefix
|
pathType: Prefix
|
||||||
backend:
|
backend:
|
||||||
service:
|
service:
|
||||||
name: gohoarder-gateway
|
name: gohoarder-frontend
|
||||||
port:
|
port:
|
||||||
number: 80
|
number: 80
|
||||||
# Uncomment for HTTPS/TLS
|
# Uncomment for HTTPS/TLS
|
||||||
@@ -477,20 +403,20 @@ spec:
|
|||||||
averageUtilization: 80
|
averageUtilization: 80
|
||||||
|
|
||||||
---
|
---
|
||||||
# HorizontalPodAutoscaler - Gateway
|
# HorizontalPodAutoscaler - Frontend
|
||||||
apiVersion: autoscaling/v2
|
apiVersion: autoscaling/v2
|
||||||
kind: HorizontalPodAutoscaler
|
kind: HorizontalPodAutoscaler
|
||||||
metadata:
|
metadata:
|
||||||
name: gohoarder-gateway
|
name: gohoarder-frontend
|
||||||
namespace: gohoarder
|
namespace: gohoarder
|
||||||
labels:
|
labels:
|
||||||
app.kubernetes.io/name: gohoarder
|
app.kubernetes.io/name: gohoarder
|
||||||
app.kubernetes.io/component: gateway
|
app.kubernetes.io/component: frontend
|
||||||
spec:
|
spec:
|
||||||
scaleTargetRef:
|
scaleTargetRef:
|
||||||
apiVersion: apps/v1
|
apiVersion: apps/v1
|
||||||
kind: Deployment
|
kind: Deployment
|
||||||
name: gohoarder-gateway
|
name: gohoarder-frontend
|
||||||
minReplicas: 2
|
minReplicas: 2
|
||||||
maxReplicas: 10
|
maxReplicas: 10
|
||||||
metrics:
|
metrics:
|
||||||
|
|||||||
+14
-38
@@ -2,7 +2,7 @@ version: '3.8'
|
|||||||
|
|
||||||
# GoHoarder - Unified Deployment Example
|
# GoHoarder - Unified Deployment Example
|
||||||
# This docker-compose file demonstrates deploying all GoHoarder services
|
# This docker-compose file demonstrates deploying all GoHoarder services
|
||||||
# under a single domain using the gateway reverse proxy
|
# The frontend includes integrated reverse proxy to the backend
|
||||||
|
|
||||||
services:
|
services:
|
||||||
# Backend - Main application server
|
# Backend - Main application server
|
||||||
@@ -39,7 +39,7 @@ services:
|
|||||||
retries: 3
|
retries: 3
|
||||||
start_period: 5s
|
start_period: 5s
|
||||||
|
|
||||||
# Frontend - Web dashboard
|
# Frontend - Web dashboard with integrated reverse proxy
|
||||||
gohoarder-frontend:
|
gohoarder-frontend:
|
||||||
image: ghcr.io/lukaszraczylo/gohoarder-frontend:latest
|
image: ghcr.io/lukaszraczylo/gohoarder-frontend:latest
|
||||||
container_name: gohoarder-frontend
|
container_name: gohoarder-frontend
|
||||||
@@ -49,8 +49,19 @@ services:
|
|||||||
- API_BASE_URL=/api
|
- API_BASE_URL=/api
|
||||||
- APP_VERSION=1.0.0
|
- APP_VERSION=1.0.0
|
||||||
- APP_NAME=GoHoarder
|
- APP_NAME=GoHoarder
|
||||||
|
# Backend proxy configuration (frontend includes nginx reverse proxy)
|
||||||
|
- BACKEND_HOST=gohoarder-server
|
||||||
|
- BACKEND_PORT=8080
|
||||||
|
- SERVER_NAME=hoarder.i.raczylo.com
|
||||||
|
ports:
|
||||||
|
# Map to host port 80 (HTTP)
|
||||||
|
- "80:80"
|
||||||
|
# Map to host port 443 (HTTPS) - uncomment if using SSL
|
||||||
|
# - "443:443"
|
||||||
networks:
|
networks:
|
||||||
- gohoarder-internal
|
- gohoarder-internal
|
||||||
|
depends_on:
|
||||||
|
- gohoarder-server
|
||||||
healthcheck:
|
healthcheck:
|
||||||
test: ["CMD", "wget", "--quiet", "--tries=1", "--spider", "http://localhost/"]
|
test: ["CMD", "wget", "--quiet", "--tries=1", "--spider", "http://localhost/"]
|
||||||
interval: 30s
|
interval: 30s
|
||||||
@@ -82,41 +93,6 @@ services:
|
|||||||
# profiles:
|
# profiles:
|
||||||
# - scanner
|
# - scanner
|
||||||
|
|
||||||
# Gateway - Nginx reverse proxy
|
|
||||||
gohoarder-gateway:
|
|
||||||
image: ghcr.io/lukaszraczylo/gohoarder-gateway:latest
|
|
||||||
container_name: gohoarder-gateway
|
|
||||||
restart: unless-stopped
|
|
||||||
environment:
|
|
||||||
# Backend service connection
|
|
||||||
- BACKEND_HOST=gohoarder-server
|
|
||||||
- BACKEND_PORT=8080
|
|
||||||
# Frontend service connection
|
|
||||||
- FRONTEND_HOST=gohoarder-frontend
|
|
||||||
- FRONTEND_PORT=80
|
|
||||||
# Server configuration
|
|
||||||
- SERVER_NAME=hoarder.i.raczylo.com
|
|
||||||
ports:
|
|
||||||
# Map to host port 80 (HTTP)
|
|
||||||
- "80:80"
|
|
||||||
# Map to host port 443 (HTTPS) - uncomment if using SSL
|
|
||||||
# - "443:443"
|
|
||||||
networks:
|
|
||||||
- gohoarder-internal
|
|
||||||
depends_on:
|
|
||||||
- gohoarder-server
|
|
||||||
- gohoarder-frontend
|
|
||||||
# Uncomment if using custom SSL certificates
|
|
||||||
# volumes:
|
|
||||||
# - ./ssl/cert.pem:/etc/nginx/ssl/cert.pem:ro
|
|
||||||
# - ./ssl/key.pem:/etc/nginx/ssl/key.pem:ro
|
|
||||||
healthcheck:
|
|
||||||
test: ["CMD", "wget", "--quiet", "--tries=1", "--spider", "http://localhost/health"]
|
|
||||||
interval: 30s
|
|
||||||
timeout: 10s
|
|
||||||
retries: 3
|
|
||||||
start_period: 5s
|
|
||||||
|
|
||||||
networks:
|
networks:
|
||||||
gohoarder-internal:
|
gohoarder-internal:
|
||||||
driver: bridge
|
driver: bridge
|
||||||
@@ -144,7 +120,7 @@ volumes:
|
|||||||
# - Metrics: http://localhost/metrics
|
# - Metrics: http://localhost/metrics
|
||||||
#
|
#
|
||||||
# For production:
|
# For production:
|
||||||
# - Enable HTTPS in the gateway container
|
# - Enable HTTPS in the frontend container (add SSL certificates to nginx)
|
||||||
# - Set up proper SSL certificates
|
# - Set up proper SSL certificates
|
||||||
# - Configure firewall rules
|
# - Configure firewall rules
|
||||||
# - Set appropriate resource limits
|
# - Set appropriate resource limits
|
||||||
|
|||||||
@@ -0,0 +1,8 @@
|
|||||||
|
{
|
||||||
|
"hash": "eb707743",
|
||||||
|
"configHash": "e17a1322",
|
||||||
|
"lockfileHash": "c2e83396",
|
||||||
|
"browserHash": "ab796ace",
|
||||||
|
"optimized": {},
|
||||||
|
"chunks": {}
|
||||||
|
}
|
||||||
@@ -0,0 +1,3 @@
|
|||||||
|
{
|
||||||
|
"type": "module"
|
||||||
|
}
|
||||||
@@ -47,9 +47,11 @@ describe('Dashboard.vue', () => {
|
|||||||
registry: '',
|
registry: '',
|
||||||
total_packages: 2,
|
total_packages: 2,
|
||||||
total_size: 3072,
|
total_size: 3072,
|
||||||
|
max_cache_size: 10737418240,
|
||||||
total_downloads: 30,
|
total_downloads: 30,
|
||||||
scanned_packages: 2,
|
scanned_packages: 2,
|
||||||
vulnerable_packages: 0,
|
vulnerable_packages: 0,
|
||||||
|
blocked_packages: 0,
|
||||||
}
|
}
|
||||||
await wrapper.vm.$nextTick()
|
await wrapper.vm.$nextTick()
|
||||||
|
|
||||||
@@ -67,9 +69,11 @@ describe('Dashboard.vue', () => {
|
|||||||
registry: '',
|
registry: '',
|
||||||
total_packages: 100,
|
total_packages: 100,
|
||||||
total_size: 0,
|
total_size: 0,
|
||||||
|
max_cache_size: 10737418240,
|
||||||
total_downloads: 0,
|
total_downloads: 0,
|
||||||
scanned_packages: 0,
|
scanned_packages: 0,
|
||||||
vulnerable_packages: 0,
|
vulnerable_packages: 0,
|
||||||
|
blocked_packages: 0,
|
||||||
}
|
}
|
||||||
await wrapper.vm.$nextTick()
|
await wrapper.vm.$nextTick()
|
||||||
|
|
||||||
@@ -85,9 +89,11 @@ describe('Dashboard.vue', () => {
|
|||||||
registry: '',
|
registry: '',
|
||||||
total_packages: 0,
|
total_packages: 0,
|
||||||
total_size: 0,
|
total_size: 0,
|
||||||
|
max_cache_size: 10737418240,
|
||||||
total_downloads: 500,
|
total_downloads: 500,
|
||||||
scanned_packages: 0,
|
scanned_packages: 0,
|
||||||
vulnerable_packages: 0,
|
vulnerable_packages: 0,
|
||||||
|
blocked_packages: 0,
|
||||||
}
|
}
|
||||||
await wrapper.vm.$nextTick()
|
await wrapper.vm.$nextTick()
|
||||||
|
|
||||||
@@ -103,9 +109,11 @@ describe('Dashboard.vue', () => {
|
|||||||
registry: '',
|
registry: '',
|
||||||
total_packages: 0,
|
total_packages: 0,
|
||||||
total_size: 1048576, // 1 MB
|
total_size: 1048576, // 1 MB
|
||||||
|
max_cache_size: 10737418240,
|
||||||
total_downloads: 0,
|
total_downloads: 0,
|
||||||
scanned_packages: 0,
|
scanned_packages: 0,
|
||||||
vulnerable_packages: 0,
|
vulnerable_packages: 0,
|
||||||
|
blocked_packages: 0,
|
||||||
}
|
}
|
||||||
await wrapper.vm.$nextTick()
|
await wrapper.vm.$nextTick()
|
||||||
|
|
||||||
|
|||||||
@@ -128,6 +128,7 @@
|
|||||||
:counts="version.vulnerabilities.counts"
|
:counts="version.vulnerabilities.counts"
|
||||||
:total="version.vulnerabilities.total"
|
:total="version.vulnerabilities.total"
|
||||||
:scannedAt="version.vulnerabilities.scannedAt"
|
:scannedAt="version.vulnerabilities.scannedAt"
|
||||||
|
:isBlocked="version.vulnerabilities.isBlocked"
|
||||||
@click="showVulnerabilityDetails(group.registry, group.name, version.version)"
|
@click="showVulnerabilityDetails(group.registry, group.name, version.version)"
|
||||||
/>
|
/>
|
||||||
</div>
|
</div>
|
||||||
|
|||||||
@@ -43,9 +43,11 @@ describe('Stats.vue', () => {
|
|||||||
registry: '',
|
registry: '',
|
||||||
total_packages: 100,
|
total_packages: 100,
|
||||||
total_size: 1073741824, // 1 GB
|
total_size: 1073741824, // 1 GB
|
||||||
|
max_cache_size: 10737418240,
|
||||||
total_downloads: 500,
|
total_downloads: 500,
|
||||||
scanned_packages: 90,
|
scanned_packages: 90,
|
||||||
vulnerable_packages: 5,
|
vulnerable_packages: 5,
|
||||||
|
blocked_packages: 0,
|
||||||
}
|
}
|
||||||
await wrapper.vm.$nextTick()
|
await wrapper.vm.$nextTick()
|
||||||
|
|
||||||
@@ -63,9 +65,11 @@ describe('Stats.vue', () => {
|
|||||||
registry: '',
|
registry: '',
|
||||||
total_packages: 100,
|
total_packages: 100,
|
||||||
total_size: 1073741824,
|
total_size: 1073741824,
|
||||||
|
max_cache_size: 10737418240,
|
||||||
total_downloads: 500,
|
total_downloads: 500,
|
||||||
scanned_packages: 90,
|
scanned_packages: 90,
|
||||||
vulnerable_packages: 5,
|
vulnerable_packages: 5,
|
||||||
|
blocked_packages: 0,
|
||||||
}
|
}
|
||||||
await wrapper.vm.$nextTick()
|
await wrapper.vm.$nextTick()
|
||||||
|
|
||||||
@@ -83,9 +87,11 @@ describe('Stats.vue', () => {
|
|||||||
registry: '',
|
registry: '',
|
||||||
total_packages: 100,
|
total_packages: 100,
|
||||||
total_size: 1073741824,
|
total_size: 1073741824,
|
||||||
|
max_cache_size: 10737418240,
|
||||||
total_downloads: 500,
|
total_downloads: 500,
|
||||||
scanned_packages: 90,
|
scanned_packages: 90,
|
||||||
vulnerable_packages: 5,
|
vulnerable_packages: 5,
|
||||||
|
blocked_packages: 0,
|
||||||
}
|
}
|
||||||
store.registries = {
|
store.registries = {
|
||||||
npm: {
|
npm: {
|
||||||
@@ -124,9 +130,11 @@ describe('Stats.vue', () => {
|
|||||||
registry: '',
|
registry: '',
|
||||||
total_packages: 100,
|
total_packages: 100,
|
||||||
total_size: 1073741824, // 1 GB
|
total_size: 1073741824, // 1 GB
|
||||||
|
max_cache_size: 10737418240,
|
||||||
total_downloads: 500,
|
total_downloads: 500,
|
||||||
scanned_packages: 90,
|
scanned_packages: 90,
|
||||||
vulnerable_packages: 5,
|
vulnerable_packages: 5,
|
||||||
|
blocked_packages: 0,
|
||||||
}
|
}
|
||||||
await wrapper.vm.$nextTick()
|
await wrapper.vm.$nextTick()
|
||||||
|
|
||||||
@@ -151,9 +159,11 @@ describe('Stats.vue', () => {
|
|||||||
registry: '',
|
registry: '',
|
||||||
total_packages: 3,
|
total_packages: 3,
|
||||||
total_size: 0,
|
total_size: 0,
|
||||||
|
max_cache_size: 10737418240,
|
||||||
total_downloads: 0,
|
total_downloads: 0,
|
||||||
scanned_packages: 0,
|
scanned_packages: 0,
|
||||||
vulnerable_packages: 0,
|
vulnerable_packages: 0,
|
||||||
|
blocked_packages: 0,
|
||||||
}
|
}
|
||||||
store.registries = {
|
store.registries = {
|
||||||
npm: { count: 1, size: 0, downloads: 0 },
|
npm: { count: 1, size: 0, downloads: 0 },
|
||||||
@@ -177,9 +187,11 @@ describe('Stats.vue', () => {
|
|||||||
registry: '',
|
registry: '',
|
||||||
total_packages: 0,
|
total_packages: 0,
|
||||||
total_size: 0,
|
total_size: 0,
|
||||||
|
max_cache_size: 10737418240,
|
||||||
total_downloads: 0,
|
total_downloads: 0,
|
||||||
scanned_packages: 0,
|
scanned_packages: 0,
|
||||||
vulnerable_packages: 0,
|
vulnerable_packages: 0,
|
||||||
|
blocked_packages: 0,
|
||||||
}
|
}
|
||||||
store.registries = {}
|
store.registries = {}
|
||||||
await wrapper.vm.$nextTick()
|
await wrapper.vm.$nextTick()
|
||||||
|
|||||||
@@ -29,11 +29,26 @@
|
|||||||
</p>
|
</p>
|
||||||
<p class="text-sm text-gray-600">Total Packages</p>
|
<p class="text-sm text-gray-600">Total Packages</p>
|
||||||
</div>
|
</div>
|
||||||
<div class="text-center p-6 bg-gray-50 rounded-lg">
|
<div class="p-6 bg-gray-50 rounded-lg">
|
||||||
<p class="text-4xl font-bold text-blue-600 mb-2">
|
<div class="text-center mb-3">
|
||||||
{{ formatBytes(stats?.total_size || 0) }}
|
<p class="text-2xl font-bold text-blue-600">
|
||||||
</p>
|
{{ formatBytes(stats?.total_size || 0) }} / {{ formatBytes(stats?.max_cache_size || 0) }}
|
||||||
<p class="text-sm text-gray-600">Total Storage Used</p>
|
</p>
|
||||||
|
<p class="text-sm text-gray-600 mt-1">Storage Used</p>
|
||||||
|
</div>
|
||||||
|
<div class="w-full bg-gray-200 rounded-full h-3 overflow-hidden">
|
||||||
|
<div
|
||||||
|
class="h-full bg-blue-600 rounded-full transition-all duration-300"
|
||||||
|
:style="{ width: storagePercentage + '%' }"
|
||||||
|
:class="{
|
||||||
|
'bg-green-600': storagePercentage < 50,
|
||||||
|
'bg-yellow-600': storagePercentage >= 50 && storagePercentage < 80,
|
||||||
|
'bg-orange-600': storagePercentage >= 80 && storagePercentage < 90,
|
||||||
|
'bg-red-600': storagePercentage >= 90
|
||||||
|
}"
|
||||||
|
></div>
|
||||||
|
</div>
|
||||||
|
<p class="text-xs text-gray-500 text-center mt-1">{{ storagePercentage.toFixed(1) }}% used</p>
|
||||||
</div>
|
</div>
|
||||||
<div class="text-center p-6 bg-gray-50 rounded-lg">
|
<div class="text-center p-6 bg-gray-50 rounded-lg">
|
||||||
<p class="text-4xl font-bold text-green-600 mb-2">
|
<p class="text-4xl font-bold text-green-600 mb-2">
|
||||||
@@ -51,7 +66,7 @@
|
|||||||
<h3 class="text-xl font-semibold text-gray-900 mb-6">
|
<h3 class="text-xl font-semibold text-gray-900 mb-6">
|
||||||
<i class="fas fa-shield-alt mr-2"></i>Security Scanning
|
<i class="fas fa-shield-alt mr-2"></i>Security Scanning
|
||||||
</h3>
|
</h3>
|
||||||
<div class="grid grid-cols-1 md:grid-cols-2 gap-6">
|
<div class="grid grid-cols-1 md:grid-cols-3 gap-6">
|
||||||
<div class="flex items-center justify-between p-6 bg-green-50 rounded-lg border border-green-200">
|
<div class="flex items-center justify-between p-6 bg-green-50 rounded-lg border border-green-200">
|
||||||
<div>
|
<div>
|
||||||
<p class="text-3xl font-bold text-green-600">
|
<p class="text-3xl font-bold text-green-600">
|
||||||
@@ -63,11 +78,11 @@
|
|||||||
</div>
|
</div>
|
||||||
<div
|
<div
|
||||||
@click="showVulnerablePackages"
|
@click="showVulnerablePackages"
|
||||||
class="flex items-center justify-between p-6 bg-red-50 rounded-lg border border-red-200 cursor-pointer hover:bg-red-100 transition-colors"
|
class="flex items-center justify-between p-6 bg-orange-50 rounded-lg border border-orange-200 cursor-pointer hover:bg-orange-100 transition-colors"
|
||||||
:class="{ 'opacity-50': (stats?.vulnerable_packages || 0) === 0 }"
|
:class="{ 'opacity-50': (stats?.vulnerable_packages || 0) === 0 }"
|
||||||
>
|
>
|
||||||
<div>
|
<div>
|
||||||
<p class="text-3xl font-bold text-red-600">
|
<p class="text-3xl font-bold text-orange-600">
|
||||||
{{ formatNumber(stats?.vulnerable_packages || 0) }}
|
{{ formatNumber(stats?.vulnerable_packages || 0) }}
|
||||||
</p>
|
</p>
|
||||||
<p class="text-sm text-gray-600 mt-1">
|
<p class="text-sm text-gray-600 mt-1">
|
||||||
@@ -75,7 +90,23 @@
|
|||||||
<span v-if="(stats?.vulnerable_packages || 0) > 0" class="text-xs ml-1">(click to view)</span>
|
<span v-if="(stats?.vulnerable_packages || 0) > 0" class="text-xs ml-1">(click to view)</span>
|
||||||
</p>
|
</p>
|
||||||
</div>
|
</div>
|
||||||
<i class="fas fa-exclamation-triangle text-5xl text-red-400"></i>
|
<i class="fas fa-exclamation-triangle text-5xl text-orange-400"></i>
|
||||||
|
</div>
|
||||||
|
<div
|
||||||
|
@click="showBlockedPackages"
|
||||||
|
class="flex items-center justify-between p-6 bg-red-50 rounded-lg border border-red-200 cursor-pointer hover:bg-red-100 transition-colors"
|
||||||
|
:class="{ 'opacity-50': (stats?.blocked_packages || 0) === 0 }"
|
||||||
|
>
|
||||||
|
<div>
|
||||||
|
<p class="text-3xl font-bold text-red-600">
|
||||||
|
{{ formatNumber(stats?.blocked_packages || 0) }}
|
||||||
|
</p>
|
||||||
|
<p class="text-sm text-gray-600 mt-1">
|
||||||
|
Blocked Packages
|
||||||
|
<span v-if="(stats?.blocked_packages || 0) > 0" class="text-xs ml-1">(click to view)</span>
|
||||||
|
</p>
|
||||||
|
</div>
|
||||||
|
<i class="fas fa-hand text-5xl text-red-400"></i>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
</CardContent>
|
</CardContent>
|
||||||
@@ -141,6 +172,14 @@ function showVulnerablePackages() {
|
|||||||
router.push('/vulnerable-packages')
|
router.push('/vulnerable-packages')
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function showBlockedPackages() {
|
||||||
|
if ((stats.value?.blocked_packages || 0) === 0) {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
router.push('/blocked-packages')
|
||||||
|
}
|
||||||
|
|
||||||
// Registry configuration for icons and colors
|
// Registry configuration for icons and colors
|
||||||
const registryConfig: Record<string, {label: string, icon: string, color: string}> = {
|
const registryConfig: Record<string, {label: string, icon: string, color: string}> = {
|
||||||
npm: {
|
npm: {
|
||||||
@@ -180,6 +219,12 @@ const registries = computed(() => {
|
|||||||
})
|
})
|
||||||
})
|
})
|
||||||
|
|
||||||
|
const storagePercentage = computed(() => {
|
||||||
|
const totalSize = stats.value?.total_size || 0
|
||||||
|
const maxSize = stats.value?.max_cache_size || 1
|
||||||
|
return (totalSize / maxSize) * 100
|
||||||
|
})
|
||||||
|
|
||||||
function formatNumber(num: number): string {
|
function formatNumber(num: number): string {
|
||||||
return new Intl.NumberFormat().format(num)
|
return new Intl.NumberFormat().format(num)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,5 +1,15 @@
|
|||||||
<template>
|
<template>
|
||||||
<div class="flex items-center gap-2">
|
<div class="flex items-center gap-2">
|
||||||
|
<!-- Blocked Icon (if package exceeds thresholds) -->
|
||||||
|
<span
|
||||||
|
v-if="isBlocked"
|
||||||
|
class="inline-flex items-center px-2.5 py-1 rounded-md text-xs font-medium bg-red-600 text-white border border-red-700"
|
||||||
|
title="Download blocked - exceeds vulnerability thresholds"
|
||||||
|
>
|
||||||
|
<i class="fas fa-hand mr-1"></i>
|
||||||
|
BLOCKED
|
||||||
|
</span>
|
||||||
|
|
||||||
<!-- Critical Vulnerabilities -->
|
<!-- Critical Vulnerabilities -->
|
||||||
<button
|
<button
|
||||||
v-if="counts.critical > 0"
|
v-if="counts.critical > 0"
|
||||||
@@ -89,12 +99,14 @@ interface Props {
|
|||||||
counts?: VulnerabilityCounts
|
counts?: VulnerabilityCounts
|
||||||
total?: number
|
total?: number
|
||||||
scannedAt?: string // ISO 8601 timestamp
|
scannedAt?: string // ISO 8601 timestamp
|
||||||
|
isBlocked?: boolean // Whether download is blocked due to vulnerabilities
|
||||||
}
|
}
|
||||||
|
|
||||||
const props = withDefaults(defineProps<Props>(), {
|
const props = withDefaults(defineProps<Props>(), {
|
||||||
scanned: false,
|
scanned: false,
|
||||||
status: 'not_scanned',
|
status: 'not_scanned',
|
||||||
total: 0,
|
total: 0,
|
||||||
|
isBlocked: false,
|
||||||
})
|
})
|
||||||
|
|
||||||
const emit = defineEmits<{
|
const emit = defineEmits<{
|
||||||
|
|||||||
@@ -7,11 +7,16 @@
|
|||||||
Back to Stats
|
Back to Stats
|
||||||
</Button>
|
</Button>
|
||||||
<div class="flex items-center gap-3">
|
<div class="flex items-center gap-3">
|
||||||
<i class="fas fa-exclamation-triangle text-3xl text-red-600"></i>
|
<i :class="showOnlyBlocked ? 'fas fa-hand' : 'fas fa-exclamation-triangle'" class="text-3xl text-red-600"></i>
|
||||||
<div>
|
<div>
|
||||||
<h1 class="text-3xl font-bold text-gray-900">Vulnerable Packages</h1>
|
<h1 class="text-3xl font-bold text-gray-900">
|
||||||
|
{{ showOnlyBlocked ? 'Blocked Packages' : 'Vulnerable Packages' }}
|
||||||
|
</h1>
|
||||||
<p class="text-gray-600 mt-1">
|
<p class="text-gray-600 mt-1">
|
||||||
Packages with known security vulnerabilities, sorted by risk
|
{{ showOnlyBlocked
|
||||||
|
? 'Packages blocked from download due to exceeding vulnerability thresholds'
|
||||||
|
: 'Packages with known security vulnerabilities, sorted by risk'
|
||||||
|
}}
|
||||||
</p>
|
</p>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
@@ -132,6 +137,7 @@
|
|||||||
:counts="version.vulnerabilities.counts"
|
:counts="version.vulnerabilities.counts"
|
||||||
:total="version.vulnerabilities.total"
|
:total="version.vulnerabilities.total"
|
||||||
:scannedAt="version.vulnerabilities.scannedAt"
|
:scannedAt="version.vulnerabilities.scannedAt"
|
||||||
|
:isBlocked="version.vulnerabilities.isBlocked"
|
||||||
@click.stop="navigateToPackage(version)"
|
@click.stop="navigateToPackage(version)"
|
||||||
/>
|
/>
|
||||||
</div>
|
</div>
|
||||||
@@ -153,7 +159,7 @@
|
|||||||
|
|
||||||
<script setup lang="ts">
|
<script setup lang="ts">
|
||||||
import { computed, onMounted, ref } from 'vue'
|
import { computed, onMounted, ref } from 'vue'
|
||||||
import { useRouter } from 'vue-router'
|
import { useRouter, useRoute } from 'vue-router'
|
||||||
import { usePackageStore, type Package } from '../stores/packages'
|
import { usePackageStore, type Package } from '../stores/packages'
|
||||||
import { Alert, AlertDescription } from '@/components/ui/alert'
|
import { Alert, AlertDescription } from '@/components/ui/alert'
|
||||||
import { Card, CardContent } from '@/components/ui/card'
|
import { Card, CardContent } from '@/components/ui/card'
|
||||||
@@ -170,11 +176,15 @@ import { getRegistryBadgeClass } from '@/composables/useBadgeStyles'
|
|||||||
|
|
||||||
const store = usePackageStore()
|
const store = usePackageStore()
|
||||||
const router = useRouter()
|
const router = useRouter()
|
||||||
|
const route = useRoute()
|
||||||
|
|
||||||
const loading = ref(false)
|
const loading = ref(false)
|
||||||
const error = ref<string | null>(null)
|
const error = ref<string | null>(null)
|
||||||
const vulnerablePackages = ref<Package[]>([])
|
const vulnerablePackages = ref<Package[]>([])
|
||||||
|
|
||||||
|
// Check if we should filter to show only blocked packages
|
||||||
|
const showOnlyBlocked = computed(() => route.path === '/blocked-packages')
|
||||||
|
|
||||||
onMounted(async () => {
|
onMounted(async () => {
|
||||||
await fetchVulnerablePackages()
|
await fetchVulnerablePackages()
|
||||||
})
|
})
|
||||||
@@ -185,9 +195,13 @@ async function fetchVulnerablePackages() {
|
|||||||
|
|
||||||
try {
|
try {
|
||||||
await store.fetchPackages()
|
await store.fetchPackages()
|
||||||
vulnerablePackages.value = store.packages.filter(
|
vulnerablePackages.value = store.packages.filter(pkg => {
|
||||||
pkg => pkg.vulnerabilities?.status === 'vulnerable'
|
const isVulnerable = pkg.vulnerabilities?.status === 'vulnerable'
|
||||||
)
|
if (showOnlyBlocked.value) {
|
||||||
|
return isVulnerable && pkg.vulnerabilities?.isBlocked === true
|
||||||
|
}
|
||||||
|
return isVulnerable
|
||||||
|
})
|
||||||
} catch (err: any) {
|
} catch (err: any) {
|
||||||
console.error('Failed to load vulnerable packages:', err)
|
console.error('Failed to load vulnerable packages:', err)
|
||||||
error.value = err.message || 'Failed to load vulnerable packages'
|
error.value = err.message || 'Failed to load vulnerable packages'
|
||||||
|
|||||||
@@ -37,6 +37,11 @@ const router = createRouter({
|
|||||||
name: 'vulnerable-packages',
|
name: 'vulnerable-packages',
|
||||||
component: VulnerablePackages,
|
component: VulnerablePackages,
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
path: '/blocked-packages',
|
||||||
|
name: 'blocked-packages',
|
||||||
|
component: VulnerablePackages,
|
||||||
|
},
|
||||||
{
|
{
|
||||||
path: '/admin/bypasses',
|
path: '/admin/bypasses',
|
||||||
name: 'bypasses',
|
name: 'bypasses',
|
||||||
|
|||||||
@@ -15,6 +15,7 @@ export interface VulnerabilityInfo {
|
|||||||
counts?: VulnerabilityCounts
|
counts?: VulnerabilityCounts
|
||||||
total?: number
|
total?: number
|
||||||
scannedAt?: string // ISO 8601 timestamp
|
scannedAt?: string // ISO 8601 timestamp
|
||||||
|
isBlocked?: boolean // Whether download is blocked due to vulnerability thresholds
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface Package {
|
export interface Package {
|
||||||
@@ -33,9 +34,11 @@ export interface Stats {
|
|||||||
registry: string
|
registry: string
|
||||||
total_packages: number
|
total_packages: number
|
||||||
total_size: number
|
total_size: number
|
||||||
|
max_cache_size: number
|
||||||
total_downloads: number
|
total_downloads: number
|
||||||
scanned_packages: number
|
scanned_packages: number
|
||||||
vulnerable_packages: number
|
vulnerable_packages: number
|
||||||
|
blocked_packages: number
|
||||||
}
|
}
|
||||||
|
|
||||||
export const usePackageStore = defineStore('packages', () => {
|
export const usePackageStore = defineStore('packages', () => {
|
||||||
|
|||||||
@@ -3,83 +3,133 @@ module github.com/lukaszraczylo/gohoarder
|
|||||||
go 1.25.5
|
go 1.25.5
|
||||||
|
|
||||||
require (
|
require (
|
||||||
github.com/aws/aws-sdk-go-v2 v1.41.0
|
github.com/aws/aws-sdk-go-v2 v1.41.7
|
||||||
github.com/aws/aws-sdk-go-v2/config v1.32.6
|
github.com/aws/aws-sdk-go-v2/config v1.32.18
|
||||||
github.com/aws/aws-sdk-go-v2/credentials v1.19.6
|
github.com/aws/aws-sdk-go-v2/credentials v1.19.17
|
||||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.95.0
|
github.com/aws/aws-sdk-go-v2/service/s3 v1.101.0
|
||||||
github.com/goccy/go-json v0.10.5
|
github.com/go-gormigrate/gormigrate/v2 v2.1.5
|
||||||
github.com/gofiber/fiber/v2 v2.52.10
|
github.com/go-sql-driver/mysql v1.10.0
|
||||||
|
github.com/goccy/go-json v0.10.6
|
||||||
|
github.com/gofiber/fiber/v2 v2.52.13
|
||||||
github.com/gorilla/websocket v1.5.3
|
github.com/gorilla/websocket v1.5.3
|
||||||
github.com/hirochachacha/go-smb2 v1.1.0
|
github.com/hirochachacha/go-smb2 v1.1.0
|
||||||
|
github.com/lib/pq v1.12.3
|
||||||
github.com/prometheus/client_golang v1.23.2
|
github.com/prometheus/client_golang v1.23.2
|
||||||
github.com/redis/go-redis/v9 v9.17.2
|
github.com/rs/zerolog v1.35.1
|
||||||
github.com/rs/zerolog v1.34.0
|
|
||||||
github.com/spf13/cobra v1.10.2
|
github.com/spf13/cobra v1.10.2
|
||||||
github.com/spf13/viper v1.21.0
|
github.com/spf13/viper v1.21.0
|
||||||
github.com/stretchr/testify v1.11.1
|
github.com/stretchr/testify v1.11.1
|
||||||
golang.org/x/crypto v0.46.0
|
github.com/testcontainers/testcontainers-go v0.40.0
|
||||||
golang.org/x/sync v0.19.0
|
github.com/testcontainers/testcontainers-go/modules/mysql v0.40.0
|
||||||
golang.org/x/time v0.14.0
|
github.com/testcontainers/testcontainers-go/modules/postgres v0.40.0
|
||||||
modernc.org/sqlite v1.42.2
|
golang.org/x/crypto v0.52.0
|
||||||
|
golang.org/x/sync v0.20.0
|
||||||
|
golang.org/x/time v0.15.0
|
||||||
|
gorm.io/driver/mysql v1.6.0
|
||||||
|
gorm.io/driver/postgres v1.6.0
|
||||||
|
gorm.io/driver/sqlite v1.6.0
|
||||||
|
gorm.io/gorm v1.31.1
|
||||||
)
|
)
|
||||||
|
|
||||||
require (
|
require (
|
||||||
github.com/andybalholm/brotli v1.2.0 // indirect
|
dario.cat/mergo v1.0.2 // indirect
|
||||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4 // indirect
|
filippo.io/edwards25519 v1.2.0 // indirect
|
||||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.16 // indirect
|
github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c // indirect
|
||||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.16 // indirect
|
github.com/Microsoft/go-winio v0.6.2 // indirect
|
||||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.16 // indirect
|
github.com/andybalholm/brotli v1.2.1 // indirect
|
||||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 // indirect
|
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10 // indirect
|
||||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.16 // indirect
|
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.23 // indirect
|
||||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.4 // indirect
|
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23 // indirect
|
||||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.7 // indirect
|
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23 // indirect
|
||||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.16 // indirect
|
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.24 // indirect
|
||||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.16 // indirect
|
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.9 // indirect
|
||||||
github.com/aws/aws-sdk-go-v2/service/signin v1.0.4 // indirect
|
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.15 // indirect
|
||||||
github.com/aws/aws-sdk-go-v2/service/sso v1.30.8 // indirect
|
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.23 // indirect
|
||||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.12 // indirect
|
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.23 // indirect
|
||||||
github.com/aws/aws-sdk-go-v2/service/sts v1.41.5 // indirect
|
github.com/aws/aws-sdk-go-v2/service/signin v1.0.11 // indirect
|
||||||
github.com/aws/smithy-go v1.24.0 // indirect
|
github.com/aws/aws-sdk-go-v2/service/sso v1.30.17 // indirect
|
||||||
|
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.36.0 // indirect
|
||||||
|
github.com/aws/aws-sdk-go-v2/service/sts v1.42.1 // indirect
|
||||||
|
github.com/aws/smithy-go v1.25.1 // indirect
|
||||||
github.com/beorn7/perks v1.0.1 // indirect
|
github.com/beorn7/perks v1.0.1 // indirect
|
||||||
|
github.com/cenkalti/backoff/v4 v4.3.0 // indirect
|
||||||
github.com/cespare/xxhash/v2 v2.3.0 // indirect
|
github.com/cespare/xxhash/v2 v2.3.0 // indirect
|
||||||
github.com/clipperhouse/stringish v0.1.1 // indirect
|
github.com/clipperhouse/uax29/v2 v2.7.0 // indirect
|
||||||
github.com/clipperhouse/uax29/v2 v2.3.0 // indirect
|
github.com/containerd/errdefs v1.0.0 // indirect
|
||||||
|
github.com/containerd/errdefs/pkg v0.3.0 // indirect
|
||||||
|
github.com/containerd/log v0.1.0 // indirect
|
||||||
|
github.com/containerd/platforms v0.2.1 // indirect
|
||||||
|
github.com/cpuguy83/dockercfg v0.3.2 // indirect
|
||||||
github.com/davecgh/go-spew v1.1.1 // indirect
|
github.com/davecgh/go-spew v1.1.1 // indirect
|
||||||
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
|
github.com/distribution/reference v0.6.0 // indirect
|
||||||
github.com/dustin/go-humanize v1.0.1 // indirect
|
github.com/docker/docker v28.5.2+incompatible // indirect
|
||||||
github.com/fsnotify/fsnotify v1.9.0 // indirect
|
github.com/docker/go-connections v0.6.0 // indirect
|
||||||
|
github.com/docker/go-units v0.5.0 // indirect
|
||||||
|
github.com/ebitengine/purego v0.9.1 // indirect
|
||||||
|
github.com/felixge/httpsnoop v1.0.4 // indirect
|
||||||
|
github.com/fsnotify/fsnotify v1.10.1 // indirect
|
||||||
github.com/geoffgarside/ber v1.2.0 // indirect
|
github.com/geoffgarside/ber v1.2.0 // indirect
|
||||||
github.com/go-viper/mapstructure/v2 v2.4.0 // indirect
|
github.com/go-logr/logr v1.4.3 // indirect
|
||||||
|
github.com/go-logr/stdr v1.2.2 // indirect
|
||||||
|
github.com/go-ole/go-ole v1.3.0 // indirect
|
||||||
|
github.com/go-viper/mapstructure/v2 v2.5.0 // indirect
|
||||||
github.com/google/uuid v1.6.0 // indirect
|
github.com/google/uuid v1.6.0 // indirect
|
||||||
|
github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.4 // indirect
|
||||||
github.com/inconshreveable/mousetrap v1.1.0 // indirect
|
github.com/inconshreveable/mousetrap v1.1.0 // indirect
|
||||||
github.com/klauspost/compress v1.18.2 // indirect
|
github.com/jackc/pgpassfile v1.0.0 // indirect
|
||||||
|
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
|
||||||
|
github.com/jackc/pgx/v5 v5.9.2 // indirect
|
||||||
|
github.com/jackc/puddle/v2 v2.2.2 // indirect
|
||||||
|
github.com/jinzhu/inflection v1.0.0 // indirect
|
||||||
|
github.com/jinzhu/now v1.1.5 // indirect
|
||||||
|
github.com/klauspost/compress v1.18.6 // indirect
|
||||||
|
github.com/lufia/plan9stats v0.0.0-20251013123823-9fd1530e3ec3 // indirect
|
||||||
|
github.com/magiconair/properties v1.8.10 // indirect
|
||||||
github.com/mattn/go-colorable v0.1.14 // indirect
|
github.com/mattn/go-colorable v0.1.14 // indirect
|
||||||
github.com/mattn/go-isatty v0.0.20 // indirect
|
github.com/mattn/go-isatty v0.0.22 // indirect
|
||||||
github.com/mattn/go-runewidth v0.0.19 // indirect
|
github.com/mattn/go-runewidth v0.0.23 // indirect
|
||||||
|
github.com/mattn/go-sqlite3 v1.14.44 // indirect
|
||||||
|
github.com/moby/docker-image-spec v1.3.1 // indirect
|
||||||
|
github.com/moby/go-archive v0.2.0 // indirect
|
||||||
|
github.com/moby/patternmatcher v0.6.0 // indirect
|
||||||
|
github.com/moby/sys/sequential v0.6.0 // indirect
|
||||||
|
github.com/moby/sys/user v0.4.0 // indirect
|
||||||
|
github.com/moby/sys/userns v0.1.0 // indirect
|
||||||
|
github.com/moby/term v0.5.2 // indirect
|
||||||
|
github.com/morikuni/aec v1.1.0 // indirect
|
||||||
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
|
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
|
||||||
github.com/ncruces/go-strftime v1.0.0 // indirect
|
github.com/opencontainers/go-digest v1.0.0 // indirect
|
||||||
github.com/pelletier/go-toml/v2 v2.2.4 // indirect
|
github.com/opencontainers/image-spec v1.1.1 // indirect
|
||||||
|
github.com/pelletier/go-toml/v2 v2.3.1 // indirect
|
||||||
|
github.com/pkg/errors v0.9.1 // indirect
|
||||||
github.com/pmezard/go-difflib v1.0.0 // indirect
|
github.com/pmezard/go-difflib v1.0.0 // indirect
|
||||||
|
github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect
|
||||||
github.com/prometheus/client_model v0.6.2 // indirect
|
github.com/prometheus/client_model v0.6.2 // indirect
|
||||||
github.com/prometheus/common v0.67.4 // indirect
|
github.com/prometheus/common v0.67.5 // indirect
|
||||||
github.com/prometheus/procfs v0.19.2 // indirect
|
github.com/prometheus/procfs v0.20.1 // indirect
|
||||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
|
|
||||||
github.com/sagikazarmark/locafero v0.12.0 // indirect
|
github.com/sagikazarmark/locafero v0.12.0 // indirect
|
||||||
|
github.com/shirou/gopsutil/v4 v4.25.12 // indirect
|
||||||
|
github.com/sirupsen/logrus v1.9.3 // indirect
|
||||||
github.com/spf13/afero v1.15.0 // indirect
|
github.com/spf13/afero v1.15.0 // indirect
|
||||||
github.com/spf13/cast v1.10.0 // indirect
|
github.com/spf13/cast v1.10.0 // indirect
|
||||||
github.com/spf13/pflag v1.0.10 // indirect
|
github.com/spf13/pflag v1.0.10 // indirect
|
||||||
github.com/stretchr/objx v0.5.2 // indirect
|
github.com/stretchr/objx v0.5.2 // indirect
|
||||||
github.com/subosito/gotenv v1.6.0 // indirect
|
github.com/subosito/gotenv v1.6.0 // indirect
|
||||||
|
github.com/tklauser/go-sysconf v0.3.16 // indirect
|
||||||
|
github.com/tklauser/numcpus v0.11.0 // indirect
|
||||||
github.com/valyala/bytebufferpool v1.0.0 // indirect
|
github.com/valyala/bytebufferpool v1.0.0 // indirect
|
||||||
github.com/valyala/fasthttp v1.68.0 // indirect
|
github.com/valyala/fasthttp v1.71.0 // indirect
|
||||||
go.yaml.in/yaml/v2 v2.4.3 // indirect
|
github.com/yusufpapurcu/wmi v1.2.4 // indirect
|
||||||
|
go.opentelemetry.io/auto/sdk v1.2.1 // indirect
|
||||||
|
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.64.0 // indirect
|
||||||
|
go.opentelemetry.io/otel v1.39.0 // indirect
|
||||||
|
go.opentelemetry.io/otel/metric v1.39.0 // indirect
|
||||||
|
go.opentelemetry.io/otel/trace v1.39.0 // indirect
|
||||||
|
go.yaml.in/yaml/v2 v2.4.4 // indirect
|
||||||
go.yaml.in/yaml/v3 v3.0.4 // indirect
|
go.yaml.in/yaml/v3 v3.0.4 // indirect
|
||||||
golang.org/x/exp v0.0.0-20251219203646-944ab1f22d93 // indirect
|
golang.org/x/sys v0.45.0 // indirect
|
||||||
golang.org/x/sys v0.39.0 // indirect
|
golang.org/x/text v0.37.0 // indirect
|
||||||
golang.org/x/text v0.32.0 // indirect
|
google.golang.org/grpc v1.78.0 // indirect
|
||||||
google.golang.org/protobuf v1.36.11 // indirect
|
google.golang.org/protobuf v1.36.11 // indirect
|
||||||
gopkg.in/yaml.v3 v3.0.1 // indirect
|
gopkg.in/yaml.v3 v3.0.1 // indirect
|
||||||
modernc.org/libc v1.67.4 // indirect
|
|
||||||
modernc.org/mathutil v1.7.1 // indirect
|
|
||||||
modernc.org/memory v1.11.0 // indirect
|
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -1,137 +1,211 @@
|
|||||||
github.com/andybalholm/brotli v1.2.0 h1:ukwgCxwYrmACq68yiUqwIWnGY0cTPox/M94sVwToPjQ=
|
dario.cat/mergo v1.0.2 h1:85+piFYR1tMbRrLcDwR18y4UKJ3aH1Tbzi24VRW1TK8=
|
||||||
github.com/andybalholm/brotli v1.2.0/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY=
|
dario.cat/mergo v1.0.2/go.mod h1:E/hbnu0NxMFBjpMIE34DRGLWqDy0g5FuKDhCb31ngxA=
|
||||||
github.com/aws/aws-sdk-go-v2 v1.41.0 h1:tNvqh1s+v0vFYdA1xq0aOJH+Y5cRyZ5upu6roPgPKd4=
|
filippo.io/edwards25519 v1.2.0 h1:crnVqOiS4jqYleHd9vaKZ+HKtHfllngJIiOpNpoJsjo=
|
||||||
github.com/aws/aws-sdk-go-v2 v1.41.0/go.mod h1:MayyLB8y+buD9hZqkCW3kX1AKq07Y5pXxtgB+rRFhz0=
|
filippo.io/edwards25519 v1.2.0/go.mod h1:xzAOLCNug/yB62zG1bQ8uziwrIqIuxhctzJT18Q77mc=
|
||||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4 h1:489krEF9xIGkOaaX3CE/Be2uWjiXrkCH6gUX+bZA/BU=
|
github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6 h1:He8afgbRMd7mFxO99hRNu+6tazq8nFF9lIwo9JFroBk=
|
||||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4/go.mod h1:IOAPF6oT9KCsceNTvvYMNHy0+kMF8akOjeDvPENWxp4=
|
github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6/go.mod h1:8o94RPi1/7XTJvwPpRSzSUedZrtlirdB3r9Z20bi2f8=
|
||||||
github.com/aws/aws-sdk-go-v2/config v1.32.6 h1:hFLBGUKjmLAekvi1evLi5hVvFQtSo3GYwi+Bx4lpJf8=
|
github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c h1:udKWzYgxTojEKWjV8V+WSxDXJ4NFATAsZjh8iIbsQIg=
|
||||||
github.com/aws/aws-sdk-go-v2/config v1.32.6/go.mod h1:lcUL/gcd8WyjCrMnxez5OXkO3/rwcNmvfno62tnXNcI=
|
github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E=
|
||||||
github.com/aws/aws-sdk-go-v2/credentials v1.19.6 h1:F9vWao2TwjV2MyiyVS+duza0NIRtAslgLUM0vTA1ZaE=
|
github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY=
|
||||||
github.com/aws/aws-sdk-go-v2/credentials v1.19.6/go.mod h1:SgHzKjEVsdQr6Opor0ihgWtkWdfRAIwxYzSJ8O85VHY=
|
github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU=
|
||||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.16 h1:80+uETIWS1BqjnN9uJ0dBUaETh+P1XwFy5vwHwK5r9k=
|
github.com/andybalholm/brotli v1.2.1 h1:R+f5xP285VArJDRgowrfb9DqL18yVK0gKAW/F+eTWro=
|
||||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.16/go.mod h1:wOOsYuxYuB/7FlnVtzeBYRcjSRtQpAW0hCP7tIULMwo=
|
github.com/andybalholm/brotli v1.2.1/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY=
|
||||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.16 h1:rgGwPzb82iBYSvHMHXc8h9mRoOUBZIGFgKb9qniaZZc=
|
github.com/aws/aws-sdk-go-v2 v1.41.7 h1:DWpAJt66FmnnaRIOT/8ASTucrvuDPZASqhhLey6tLY8=
|
||||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.16/go.mod h1:L/UxsGeKpGoIj6DxfhOWHWQ/kGKcd4I1VncE4++IyKA=
|
github.com/aws/aws-sdk-go-v2 v1.41.7/go.mod h1:4LAfZOPHNVNQEckOACQx60Y8pSRjIkNZQz1w92xpMJc=
|
||||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.16 h1:1jtGzuV7c82xnqOVfx2F0xmJcOw5374L7N6juGW6x6U=
|
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10 h1:gx1AwW1Iyk9Z9dD9F4akX5gnN3QZwUB20GGKH/I+Rho=
|
||||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.16/go.mod h1:M2E5OQf+XLe+SZGmmpaI2yy+J326aFf6/+54PoxSANc=
|
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10/go.mod h1:qqY157uZoqm5OXq/amuaBJyC9hgBCBQnsaWnPe905GY=
|
||||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 h1:WKuaxf++XKWlHWu9ECbMlha8WOEGm0OUEZqm4K/Gcfk=
|
github.com/aws/aws-sdk-go-v2/config v1.32.18 h1:Hcia46bxhGgF3BaSnG8nSNCWmqTK6bj9xN9/FJ3WK6Q=
|
||||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4/go.mod h1:ZWy7j6v1vWGmPReu0iSGvRiise4YI5SkR3OHKTZ6Wuc=
|
github.com/aws/aws-sdk-go-v2/config v1.32.18/go.mod h1:zEjCAYmxqDadH1WX8CdBvmLKhUEUVFgKRQG38zjDmrY=
|
||||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.16 h1:CjMzUs78RDDv4ROu3JnJn/Ig1r6ZD7/T2DXLLRpejic=
|
github.com/aws/aws-sdk-go-v2/credentials v1.19.17 h1:gP2nkGsS+KMvF/jfFz2Vv2qiiOqWKyPACSzPsqHgoW8=
|
||||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.16/go.mod h1:uVW4OLBqbJXSHJYA9svT9BluSvvwbzLQ2Crf6UPzR3c=
|
github.com/aws/aws-sdk-go-v2/credentials v1.19.17/go.mod h1:Bsew3S/moG5iT77giPj1q8wb/s0RE5/QfH+ASjYtuQc=
|
||||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.4 h1:0ryTNEdJbzUCEWkVXEXoqlXV72J5keC1GvILMOuD00E=
|
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.23 h1:UuSfcORqNSz/ey3VPRS8TcVH2Ikf0/sC+Hdj400QI6U=
|
||||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.4/go.mod h1:HQ4qwNZh32C3CBeO6iJLQlgtMzqeG17ziAA/3KDJFow=
|
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.23/go.mod h1:+G/OSGiOFnSOkYloKj/9M35s74LgVAdJBSD5lsFfqKg=
|
||||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.7 h1:DIBqIrJ7hv+e4CmIk2z3pyKT+3B6qVMgRsawHiR3qso=
|
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23 h1:GpT/TrnBYuE5gan2cZbTtvP+JlHsutdmlV2YfEyNde0=
|
||||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.7/go.mod h1:vLm00xmBke75UmpNvOcZQ/Q30ZFjbczeLFqGx5urmGo=
|
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23/go.mod h1:xYWD6BS9ywC5bS3sz9Xh04whO/hzK2plt2Zkyrp4JuA=
|
||||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.16 h1:oHjJHeUy0ImIV0bsrX0X91GkV5nJAyv1l1CC9lnO0TI=
|
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23 h1:bpd8vxhlQi2r1hiueOw02f/duEPTMK59Q4QMAoTTtTo=
|
||||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.16/go.mod h1:iRSNGgOYmiYwSCXxXaKb9HfOEj40+oTKn8pTxMlYkRM=
|
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23/go.mod h1:15DfR2nw+CRHIk0tqNyifu3G1YdAOy68RftkhMDDwYk=
|
||||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.16 h1:NSbvS17MlI2lurYgXnCOLvCFX38sBW4eiVER7+kkgsU=
|
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.24 h1:OQqn11BtaYv1WLUowvcA30MpzIu8Ti4pcLPIIyoKZrA=
|
||||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.16/go.mod h1:SwT8Tmqd4sA6G1qaGdzWCJN99bUmPGHfRwwq3G5Qb+A=
|
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.24/go.mod h1:X5ZJyfwVrWA96GzPmUCWFQaEARPR7gCrpq2E92PJwAE=
|
||||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.95.0 h1:MIWra+MSq53CFaXXAywB2qg9YvVZifkk6vEGl/1Qor0=
|
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.9 h1:FLudkZLt5ci0ozzgkVo8BJGwvqNaZbTWb3UcucAateA=
|
||||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.95.0/go.mod h1:79S2BdqCJpScXZA2y+cpZuocWsjGjJINyXnOsf5DTz8=
|
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.9/go.mod h1:w7wZ/s9qK7c8g4al+UyoF1Sp/Z45UwMGcqIzLWVQHWk=
|
||||||
github.com/aws/aws-sdk-go-v2/service/signin v1.0.4 h1:HpI7aMmJ+mm1wkSHIA2t5EaFFv5EFYXePW30p1EIrbQ=
|
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.15 h1:ieLCO1JxUWuxTZ1cRd0GAaeX7O6cIxnwk7tc1LsQhC4=
|
||||||
github.com/aws/aws-sdk-go-v2/service/signin v1.0.4/go.mod h1:C5RdGMYGlfM0gYq/tifqgn4EbyX99V15P2V3R+VHbQU=
|
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.15/go.mod h1:e3IzZvQ3kAWNykvE0Tr0RDZCMFInMvhku3qNpcIQXhM=
|
||||||
github.com/aws/aws-sdk-go-v2/service/sso v1.30.8 h1:aM/Q24rIlS3bRAhTyFurowU8A0SMyGDtEOY/l/s/1Uw=
|
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.23 h1:pbrxO/kuIwgEsOPLkaHu0O+m4fNgLU8B3vxQ+72jTPw=
|
||||||
github.com/aws/aws-sdk-go-v2/service/sso v1.30.8/go.mod h1:+fWt2UHSb4kS7Pu8y+BMBvJF0EWx+4H0hzNwtDNRTrg=
|
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.23/go.mod h1:/CMNUqoj46HpS3MNRDEDIwcgEnrtZlKRaHNaHxIFpNA=
|
||||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.12 h1:AHDr0DaHIAo8c9t1emrzAlVDFp+iMMKnPdYy6XO4MCE=
|
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.23 h1:03xatSQO4+AM1lTAbnRg5OK528EUg744nW7F73U8DKw=
|
||||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.12/go.mod h1:GQ73XawFFiWxyWXMHWfhiomvP3tXtdNar/fi8z18sx0=
|
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.23/go.mod h1:M8l3mwgx5ToK7wot2sBBce/ojzgnPzZXUV445gTSyE8=
|
||||||
github.com/aws/aws-sdk-go-v2/service/sts v1.41.5 h1:SciGFVNZ4mHdm7gpD1dgZYnCuVdX1s+lFTg4+4DOy70=
|
github.com/aws/aws-sdk-go-v2/service/s3 v1.101.0 h1:etqBTKY581iwLL/H/S2sVgk3C9lAsTJFeXWFDsDcWOU=
|
||||||
github.com/aws/aws-sdk-go-v2/service/sts v1.41.5/go.mod h1:iW40X4QBmUxdP+fZNOpfmkdMZqsovezbAeO+Ubiv2pk=
|
github.com/aws/aws-sdk-go-v2/service/s3 v1.101.0/go.mod h1:L2dcoOgS2VSgbPLvpak2NyUPsO1TBN7M45Z4H7DlRc4=
|
||||||
github.com/aws/smithy-go v1.24.0 h1:LpilSUItNPFr1eY85RYgTIg5eIEPtvFbskaFcmmIUnk=
|
github.com/aws/aws-sdk-go-v2/service/signin v1.0.11 h1:TdJ+HdzOBhU8+iVAOGUTU63VXopcumCOF1paFulHWZc=
|
||||||
github.com/aws/smithy-go v1.24.0/go.mod h1:LEj2LM3rBRQJxPZTB4KuzZkaZYnZPnvgIhb4pu07mx0=
|
github.com/aws/aws-sdk-go-v2/service/signin v1.0.11/go.mod h1:R82ZRExE/nheo0N+T8zHPcLRTcH8MGsnR3BiVGX0TwI=
|
||||||
|
github.com/aws/aws-sdk-go-v2/service/sso v1.30.17 h1:7byT8HUWrgoRp6sXjxtZwgOKfhss5fW6SkLBtqzgRoE=
|
||||||
|
github.com/aws/aws-sdk-go-v2/service/sso v1.30.17/go.mod h1:xNWknVi4Ezm1vg1QsB/5EWpAJURq22uqd38U8qKvOJc=
|
||||||
|
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.36.0 h1:nDARhv/oF55bcxF7rCI/4PDxOKnVXVWwDuDwCs2I2SQ=
|
||||||
|
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.36.0/go.mod h1:4vIRDq+CJB2xFAXZ+YgGUTiEft7oAQlhIs71xcSeuVg=
|
||||||
|
github.com/aws/aws-sdk-go-v2/service/sts v1.42.1 h1:F/M5Y9I3nwr2IEpshZgh1GeHpOItExNM9L1euNuh/fk=
|
||||||
|
github.com/aws/aws-sdk-go-v2/service/sts v1.42.1/go.mod h1:mTNxImtovCOEEuD65mKW7DCsL+2gjEH+RPEAexAzAio=
|
||||||
|
github.com/aws/smithy-go v1.25.1 h1:J8ERsGSU7d+aCmdQur5Txg6bVoYelvQJgtZehD12GkI=
|
||||||
|
github.com/aws/smithy-go v1.25.1/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc=
|
||||||
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
|
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
|
||||||
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
|
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
|
||||||
github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs=
|
github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8=
|
||||||
github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c=
|
github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE=
|
||||||
github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA=
|
|
||||||
github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0=
|
|
||||||
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
|
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
|
||||||
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
||||||
github.com/clipperhouse/stringish v0.1.1 h1:+NSqMOr3GR6k1FdRhhnXrLfztGzuG+VuFDfatpWHKCs=
|
github.com/clipperhouse/uax29/v2 v2.7.0 h1:+gs4oBZ2gPfVrKPthwbMzWZDaAFPGYK72F0NJv2v7Vk=
|
||||||
github.com/clipperhouse/stringish v0.1.1/go.mod h1:v/WhFtE1q0ovMta2+m+UbpZ+2/HEXNWYXQgCt4hdOzA=
|
github.com/clipperhouse/uax29/v2 v2.7.0/go.mod h1:EFJ2TJMRUaplDxHKj1qAEhCtQPW2tJSwu5BF98AuoVM=
|
||||||
github.com/clipperhouse/uax29/v2 v2.3.0 h1:SNdx9DVUqMoBuBoW3iLOj4FQv3dN5mDtuqwuhIGpJy4=
|
github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI=
|
||||||
github.com/clipperhouse/uax29/v2 v2.3.0/go.mod h1:Wn1g7MK6OoeDT0vL+Q0SQLDz/KpfsVRgg6W7ihQeh4g=
|
github.com/containerd/errdefs v1.0.0/go.mod h1:+YBYIdtsnF4Iw6nWZhJcqGSg/dwvV7tyJ/kCkyJ2k+M=
|
||||||
github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
|
github.com/containerd/errdefs/pkg v0.3.0 h1:9IKJ06FvyNlexW690DXuQNx2KA2cUJXx151Xdx3ZPPE=
|
||||||
|
github.com/containerd/errdefs/pkg v0.3.0/go.mod h1:NJw6s9HwNuRhnjJhM7pylWwMyAkmCQvQ4GpJHEqRLVk=
|
||||||
|
github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I=
|
||||||
|
github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo=
|
||||||
|
github.com/containerd/platforms v0.2.1 h1:zvwtM3rz2YHPQsF2CHYM8+KtB5dvhISiXh5ZpSBQv6A=
|
||||||
|
github.com/containerd/platforms v0.2.1/go.mod h1:XHCb+2/hzowdiut9rkudds9bE5yJ7npe7dG/wG+uFPw=
|
||||||
|
github.com/cpuguy83/dockercfg v0.3.2 h1:DlJTyZGBDlXqUZ2Dk2Q3xHs/FtnooJJVaad2S9GKorA=
|
||||||
|
github.com/cpuguy83/dockercfg v0.3.2/go.mod h1:sugsbF4//dDlL/i+S+rtpIWp+5h0BHJHfjj5/jFyUJc=
|
||||||
github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
|
github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
|
||||||
|
github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY=
|
||||||
|
github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4=
|
||||||
|
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||||
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=
|
github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk=
|
||||||
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
|
github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E=
|
||||||
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
|
github.com/docker/docker v28.5.2+incompatible h1:DBX0Y0zAjZbSrm1uzOkdr1onVghKaftjlSWt4AFexzM=
|
||||||
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
|
github.com/docker/docker v28.5.2+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
|
||||||
|
github.com/docker/go-connections v0.6.0 h1:LlMG9azAe1TqfR7sO+NJttz1gy6KO7VJBh+pMmjSD94=
|
||||||
|
github.com/docker/go-connections v0.6.0/go.mod h1:AahvXYshr6JgfUJGdDCs2b5EZG/vmaMAntpSFH5BFKE=
|
||||||
|
github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4=
|
||||||
|
github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
|
||||||
|
github.com/ebitengine/purego v0.9.1 h1:a/k2f2HQU3Pi399RPW1MOaZyhKJL9w/xFpKAg4q1s0A=
|
||||||
|
github.com/ebitengine/purego v0.9.1/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ=
|
||||||
|
github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg=
|
||||||
|
github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
|
||||||
github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8=
|
github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8=
|
||||||
github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
|
github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
|
||||||
github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k=
|
github.com/fsnotify/fsnotify v1.10.1 h1:b0/UzAf9yR5rhf3RPm9gf3ehBPpf0oZKIjtpKrx59Ho=
|
||||||
github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0=
|
github.com/fsnotify/fsnotify v1.10.1/go.mod h1:TLheqan6HD6GBK6PrDWyDPBaEV8LspOxvPSjC+bVfgo=
|
||||||
github.com/geoffgarside/ber v1.1.0/go.mod h1:jVPKeCbj6MvQZhwLYsGwaGI52oUorHoHKNecGT85ZCc=
|
github.com/geoffgarside/ber v1.1.0/go.mod h1:jVPKeCbj6MvQZhwLYsGwaGI52oUorHoHKNecGT85ZCc=
|
||||||
github.com/geoffgarside/ber v1.2.0 h1:/loowoRcs/MWLYmGX9QtIAbA+V/FrnVLsMMPhwiRm64=
|
github.com/geoffgarside/ber v1.2.0 h1:/loowoRcs/MWLYmGX9QtIAbA+V/FrnVLsMMPhwiRm64=
|
||||||
github.com/geoffgarside/ber v1.2.0/go.mod h1:jVPKeCbj6MvQZhwLYsGwaGI52oUorHoHKNecGT85ZCc=
|
github.com/geoffgarside/ber v1.2.0/go.mod h1:jVPKeCbj6MvQZhwLYsGwaGI52oUorHoHKNecGT85ZCc=
|
||||||
github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9LvH92wZUgs=
|
github.com/go-gormigrate/gormigrate/v2 v2.1.5 h1:1OyorA5LtdQw12cyJDEHuTrEV3GiXiIhS4/QTTa/SM8=
|
||||||
github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM=
|
github.com/go-gormigrate/gormigrate/v2 v2.1.5/go.mod h1:mj9ekk/7CPF3VjopaFvWKN2v7fN3D9d3eEOAXRhi/+M=
|
||||||
github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4=
|
github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
|
||||||
github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M=
|
github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI=
|
||||||
github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
|
github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
|
||||||
github.com/gofiber/fiber/v2 v2.52.10 h1:jRHROi2BuNti6NYXmZ6gbNSfT3zj/8c0xy94GOU5elY=
|
github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
|
||||||
github.com/gofiber/fiber/v2 v2.52.10/go.mod h1:YEcBbO/FB+5M1IZNBP9FO3J9281zgPAreiI1oqg8nDw=
|
github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
|
||||||
|
github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
|
||||||
|
github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE=
|
||||||
|
github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78=
|
||||||
|
github.com/go-sql-driver/mysql v1.10.0 h1:Q+1LV8DkHJvSYAdR83XzuhDaTykuDx0l6fkXxoWCWfw=
|
||||||
|
github.com/go-sql-driver/mysql v1.10.0/go.mod h1:M+cqaI7+xxXGG9swrdeUIoPG3Y3KCkF0pZej+SK+nWk=
|
||||||
|
github.com/go-viper/mapstructure/v2 v2.5.0 h1:vM5IJoUAy3d7zRSVtIwQgBj7BiWtMPfmPEgAXnvj1Ro=
|
||||||
|
github.com/go-viper/mapstructure/v2 v2.5.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM=
|
||||||
|
github.com/goccy/go-json v0.10.6 h1:p8HrPJzOakx/mn/bQtjgNjdTcN+/S6FcG2CTtQOrHVU=
|
||||||
|
github.com/goccy/go-json v0.10.6/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M=
|
||||||
|
github.com/gofiber/fiber/v2 v2.52.13 h1:TOKP64iqC9b5P49VrBW5tHhUOvDyrtJ0xePEfzJbCbk=
|
||||||
|
github.com/gofiber/fiber/v2 v2.52.13/go.mod h1:YEcBbO/FB+5M1IZNBP9FO3J9281zgPAreiI1oqg8nDw=
|
||||||
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
|
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
|
||||||
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
|
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
|
||||||
github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e h1:ijClszYn+mADRFY17kjQEVQ1XRhq2/JR1M3sGqeJoxs=
|
|
||||||
github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e/go.mod h1:boTsfXsheKC2y+lKOCMpSfarhxDeIzfZG1jqGcPl3cA=
|
|
||||||
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
||||||
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||||
github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg=
|
github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg=
|
||||||
github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
|
github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
|
||||||
github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k=
|
github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.4 h1:kEISI/Gx67NzH3nJxAmY/dGac80kKZgZt134u7Y/k1s=
|
||||||
github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
|
github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.4/go.mod h1:6Nz966r3vQYCqIzWsuEl9d7cf7mRhtDmm++sOxlnfxI=
|
||||||
github.com/hirochachacha/go-smb2 v1.1.0 h1:b6hs9qKIql9eVXAiN0M2wSFY5xnhbHAQoCwRKbaRTZI=
|
github.com/hirochachacha/go-smb2 v1.1.0 h1:b6hs9qKIql9eVXAiN0M2wSFY5xnhbHAQoCwRKbaRTZI=
|
||||||
github.com/hirochachacha/go-smb2 v1.1.0/go.mod h1:8F1A4d5EZzrGu5R7PU163UcMRDJQl4FtcxjBfsY8TZE=
|
github.com/hirochachacha/go-smb2 v1.1.0/go.mod h1:8F1A4d5EZzrGu5R7PU163UcMRDJQl4FtcxjBfsY8TZE=
|
||||||
github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
|
github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
|
||||||
github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
|
github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
|
||||||
github.com/klauspost/compress v1.18.2 h1:iiPHWW0YrcFgpBYhsA6D1+fqHssJscY/Tm/y2Uqnapk=
|
github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=
|
||||||
github.com/klauspost/compress v1.18.2/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4=
|
github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
|
||||||
|
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo=
|
||||||
|
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM=
|
||||||
|
github.com/jackc/pgx/v5 v5.9.2 h1:3ZhOzMWnR4yJ+RW1XImIPsD1aNSz4T4fyP7zlQb56hw=
|
||||||
|
github.com/jackc/pgx/v5 v5.9.2/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4=
|
||||||
|
github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo=
|
||||||
|
github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
|
||||||
|
github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E=
|
||||||
|
github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
|
||||||
|
github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ=
|
||||||
|
github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
|
||||||
|
github.com/klauspost/compress v1.18.6 h1:2jupLlAwFm95+YDR+NwD2MEfFO9d4z4Prjl1XXDjuao=
|
||||||
|
github.com/klauspost/compress v1.18.6/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
|
||||||
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
|
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
|
||||||
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
|
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
|
||||||
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
|
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
|
||||||
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
|
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
|
||||||
github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
|
github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
|
||||||
github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
|
github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
|
||||||
github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
|
github.com/lib/pq v1.12.3 h1:tTWxr2YLKwIvK90ZXEw8GP7UFHtcbTtty8zsI+YjrfQ=
|
||||||
|
github.com/lib/pq v1.12.3/go.mod h1:/p+8NSbOcwzAEI7wiMXFlgydTwcgTr3OSKMsD2BitpA=
|
||||||
|
github.com/lufia/plan9stats v0.0.0-20251013123823-9fd1530e3ec3 h1:PwQumkgq4/acIiZhtifTV5OUqqiP82UAl0h87xj/l9k=
|
||||||
|
github.com/lufia/plan9stats v0.0.0-20251013123823-9fd1530e3ec3/go.mod h1:autxFIvghDt3jPTLoqZ9OZ7s9qTGNAWmYCjVFWPX/zg=
|
||||||
|
github.com/magiconair/properties v1.8.10 h1:s31yESBquKXCV9a/ScB3ESkOjUYYv+X0rg8SYxI99mE=
|
||||||
|
github.com/magiconair/properties v1.8.10/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0=
|
||||||
github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE=
|
github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE=
|
||||||
github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8=
|
github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8=
|
||||||
github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
|
github.com/mattn/go-isatty v0.0.22 h1:j8l17JJ9i6VGPUFUYoTUKPSgKe/83EYU2zBC7YNKMw4=
|
||||||
github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
|
github.com/mattn/go-isatty v0.0.22/go.mod h1:ZXfXG4SQHsB/w3ZeOYbR0PrPwLy+n6xiMrJlRFqopa4=
|
||||||
github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
|
github.com/mattn/go-runewidth v0.0.23 h1:7ykA0T0jkPpzSvMS5i9uoNn2Xy3R383f9HDx3RybWcw=
|
||||||
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
|
github.com/mattn/go-runewidth v0.0.23/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhgLpndooCuJAs=
|
||||||
github.com/mattn/go-runewidth v0.0.19 h1:v++JhqYnZuu5jSKrk9RbgF5v4CGUjqRfBm05byFGLdw=
|
github.com/mattn/go-sqlite3 v1.14.44 h1:3VSe+xafpbzsLbdr2AWlAZk9yRHiBhTBakioXaCKTF8=
|
||||||
github.com/mattn/go-runewidth v0.0.19/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhgLpndooCuJAs=
|
github.com/mattn/go-sqlite3 v1.14.44/go.mod h1:pjEuOr8IwzLJP2MfGeTb0A35jauH+C2kbHKBr7yXKVQ=
|
||||||
|
github.com/mdelapenya/tlscert v0.2.0 h1:7H81W6Z/4weDvZBNOfQte5GpIMo0lGYEeWbkGp5LJHI=
|
||||||
|
github.com/mdelapenya/tlscert v0.2.0/go.mod h1:O4njj3ELLnJjGdkN7M/vIVCpZ+Cf0L6muqOG4tLSl8o=
|
||||||
|
github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0=
|
||||||
|
github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo=
|
||||||
|
github.com/moby/go-archive v0.2.0 h1:zg5QDUM2mi0JIM9fdQZWC7U8+2ZfixfTYoHL7rWUcP8=
|
||||||
|
github.com/moby/go-archive v0.2.0/go.mod h1:mNeivT14o8xU+5q1YnNrkQVpK+dnNe/K6fHqnTg4qPU=
|
||||||
|
github.com/moby/patternmatcher v0.6.0 h1:GmP9lR19aU5GqSSFko+5pRqHi+Ohk1O69aFiKkVGiPk=
|
||||||
|
github.com/moby/patternmatcher v0.6.0/go.mod h1:hDPoyOpDY7OrrMDLaYoY3hf52gNCR/YOUYxkhApJIxc=
|
||||||
|
github.com/moby/sys/atomicwriter v0.1.0 h1:kw5D/EqkBwsBFi0ss9v1VG3wIkVhzGvLklJ+w3A14Sw=
|
||||||
|
github.com/moby/sys/atomicwriter v0.1.0/go.mod h1:Ul8oqv2ZMNHOceF643P6FKPXeCmYtlQMvpizfsSoaWs=
|
||||||
|
github.com/moby/sys/sequential v0.6.0 h1:qrx7XFUd/5DxtqcoH1h438hF5TmOvzC/lspjy7zgvCU=
|
||||||
|
github.com/moby/sys/sequential v0.6.0/go.mod h1:uyv8EUTrca5PnDsdMGXhZe6CCe8U/UiTWd+lL+7b/Ko=
|
||||||
|
github.com/moby/sys/user v0.4.0 h1:jhcMKit7SA80hivmFJcbB1vqmw//wU61Zdui2eQXuMs=
|
||||||
|
github.com/moby/sys/user v0.4.0/go.mod h1:bG+tYYYJgaMtRKgEmuueC0hJEAZWwtIbZTB+85uoHjs=
|
||||||
|
github.com/moby/sys/userns v0.1.0 h1:tVLXkFOxVu9A64/yh59slHVv9ahO9UIev4JZusOLG/g=
|
||||||
|
github.com/moby/sys/userns v0.1.0/go.mod h1:IHUYgu/kao6N8YZlp9Cf444ySSvCmDlmzUcYfDHOl28=
|
||||||
|
github.com/moby/term v0.5.2 h1:6qk3FJAFDs6i/q3W/pQ97SX192qKfZgGjCQqfCJkgzQ=
|
||||||
|
github.com/moby/term v0.5.2/go.mod h1:d3djjFCrjnB+fl8NJux+EJzu0msscUP+f8it8hPkFLc=
|
||||||
|
github.com/morikuni/aec v1.1.0 h1:vBBl0pUnvi/Je71dsRrhMBtreIqNMYErSAbEeb8jrXQ=
|
||||||
|
github.com/morikuni/aec v1.1.0/go.mod h1:xDRgiq/iw5l+zkao76YTKzKttOp2cwPEne25HDkJnBw=
|
||||||
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
|
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
|
||||||
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
|
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
|
||||||
github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w=
|
github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
|
||||||
github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls=
|
github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
|
||||||
github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4=
|
github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040=
|
||||||
github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY=
|
github.com/opencontainers/image-spec v1.1.1/go.mod h1:qpqAh3Dmcf36wStyyWU+kCeDgrGnAve2nCC8+7h8Q0M=
|
||||||
|
github.com/pelletier/go-toml/v2 v2.3.1 h1:MYEvvGnQjeNkRF1qUuGolNtNExTDwct51yp7olPtrEc=
|
||||||
|
github.com/pelletier/go-toml/v2 v2.3.1/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY=
|
||||||
|
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
|
||||||
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||||
|
github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 h1:o4JXh1EVt9k/+g42oCprj/FisM4qX9L3sZB3upGN2ZU=
|
||||||
|
github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE=
|
||||||
github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o=
|
github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o=
|
||||||
github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg=
|
github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg=
|
||||||
github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk=
|
github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk=
|
||||||
github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE=
|
github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE=
|
||||||
github.com/prometheus/common v0.67.4 h1:yR3NqWO1/UyO1w2PhUvXlGQs/PtFmoveVO0KZ4+Lvsc=
|
github.com/prometheus/common v0.67.5 h1:pIgK94WWlQt1WLwAC5j2ynLaBRDiinoAb86HZHTUGI4=
|
||||||
github.com/prometheus/common v0.67.4/go.mod h1:gP0fq6YjjNCLssJCQp0yk4M8W6ikLURwkdd/YKtTbyI=
|
github.com/prometheus/common v0.67.5/go.mod h1:SjE/0MzDEEAyrdr5Gqc6G+sXI67maCxzaT3A2+HqjUw=
|
||||||
github.com/prometheus/procfs v0.19.2 h1:zUMhqEW66Ex7OXIiDkll3tl9a1ZdilUOd/F6ZXw4Vws=
|
github.com/prometheus/procfs v0.20.1 h1:XwbrGOIplXW/AU3YhIhLODXMJYyC1isLFfYCsTEycfc=
|
||||||
github.com/prometheus/procfs v0.19.2/go.mod h1:M0aotyiemPhBCM0z5w87kL22CxfcH05ZpYlu+b4J7mw=
|
github.com/prometheus/procfs v0.20.1/go.mod h1:o9EMBZGRyvDrSPH1RqdxhojkuXstoe4UlK79eF5TGGo=
|
||||||
github.com/redis/go-redis/v9 v9.17.2 h1:P2EGsA4qVIM3Pp+aPocCJ7DguDHhqrXNhVcEp4ViluI=
|
github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ=
|
||||||
github.com/redis/go-redis/v9 v9.17.2/go.mod h1:u410H11HMLoB+TP67dz8rL9s6QW2j76l0//kSOd3370=
|
github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc=
|
||||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
|
github.com/rs/zerolog v1.35.1 h1:m7xQeoiLIiV0BCEY4Hs+j2NG4Gp2o2KPKmhnnLiazKI=
|
||||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
|
github.com/rs/zerolog v1.35.1/go.mod h1:EjML9kdfa/RMA7h/6z6pYmq1ykOuA8/mjWaEvGI+jcw=
|
||||||
github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ=
|
|
||||||
github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog=
|
|
||||||
github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0=
|
|
||||||
github.com/rs/zerolog v1.34.0 h1:k43nTLIwcTVQAncfCw4KZ2VY6ukYoZaBPNOE8txlOeY=
|
|
||||||
github.com/rs/zerolog v1.34.0/go.mod h1:bJsvje4Z08ROH4Nhs5iH600c3IkWhwp44iRc54W6wYQ=
|
|
||||||
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
|
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
|
||||||
github.com/sagikazarmark/locafero v0.12.0 h1:/NQhBAkUb4+fH1jivKHWusDYFjMOOKU88eegjfxfHb4=
|
github.com/sagikazarmark/locafero v0.12.0 h1:/NQhBAkUb4+fH1jivKHWusDYFjMOOKU88eegjfxfHb4=
|
||||||
github.com/sagikazarmark/locafero v0.12.0/go.mod h1:sZh36u/YSZ918v0Io+U9ogLYQJ9tLLBmM4eneO6WwsI=
|
github.com/sagikazarmark/locafero v0.12.0/go.mod h1:sZh36u/YSZ918v0Io+U9ogLYQJ9tLLBmM4eneO6WwsI=
|
||||||
|
github.com/shirou/gopsutil/v4 v4.25.12 h1:e7PvW/0RmJ8p8vPGJH4jvNkOyLmbkXgXW4m6ZPic6CY=
|
||||||
|
github.com/shirou/gopsutil/v4 v4.25.12/go.mod h1:EivAfP5x2EhLp2ovdpKSozecVXn1TmuG7SMzs/Wh4PU=
|
||||||
|
github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
|
||||||
|
github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
|
||||||
github.com/spf13/afero v1.15.0 h1:b/YBCLWAJdFWJTN9cLhiXXcD7mzKn9Dm86dNnfyQw1I=
|
github.com/spf13/afero v1.15.0 h1:b/YBCLWAJdFWJTN9cLhiXXcD7mzKn9Dm86dNnfyQw1I=
|
||||||
github.com/spf13/afero v1.15.0/go.mod h1:NC2ByUVxtQs4b3sIUphxK0NioZnmxgyCrfzeuq8lxMg=
|
github.com/spf13/afero v1.15.0/go.mod h1:NC2ByUVxtQs4b3sIUphxK0NioZnmxgyCrfzeuq8lxMg=
|
||||||
github.com/spf13/cast v1.10.0 h1:h2x0u2shc1QuLHfxi+cTJvs30+ZAHOGRic8uyGTDWxY=
|
github.com/spf13/cast v1.10.0 h1:h2x0u2shc1QuLHfxi+cTJvs30+ZAHOGRic8uyGTDWxY=
|
||||||
@@ -143,81 +217,105 @@ github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk=
|
|||||||
github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
|
github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
|
||||||
github.com/spf13/viper v1.21.0 h1:x5S+0EU27Lbphp4UKm1C+1oQO+rKx36vfCoaVebLFSU=
|
github.com/spf13/viper v1.21.0 h1:x5S+0EU27Lbphp4UKm1C+1oQO+rKx36vfCoaVebLFSU=
|
||||||
github.com/spf13/viper v1.21.0/go.mod h1:P0lhsswPGWD/1lZJ9ny3fYnVqxiegrlNrEmgLjbTCAY=
|
github.com/spf13/viper v1.21.0/go.mod h1:P0lhsswPGWD/1lZJ9ny3fYnVqxiegrlNrEmgLjbTCAY=
|
||||||
|
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||||
github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY=
|
github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY=
|
||||||
github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
|
github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
|
||||||
|
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
||||||
|
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||||
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
|
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
|
||||||
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
|
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
|
||||||
github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8=
|
github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8=
|
||||||
github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU=
|
github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU=
|
||||||
|
github.com/testcontainers/testcontainers-go v0.40.0 h1:pSdJYLOVgLE8YdUY2FHQ1Fxu+aMnb6JfVz1mxk7OeMU=
|
||||||
|
github.com/testcontainers/testcontainers-go v0.40.0/go.mod h1:FSXV5KQtX2HAMlm7U3APNyLkkap35zNLxukw9oBi/MY=
|
||||||
|
github.com/testcontainers/testcontainers-go/modules/mysql v0.40.0 h1:P9Txfy5Jothx2wFdcus0QoSmX/PKSIXZxrTbZPVJswA=
|
||||||
|
github.com/testcontainers/testcontainers-go/modules/mysql v0.40.0/go.mod h1:oZPHHqJqXG7FD8OB/yWH7gLnDvZUlFHAVJNrGftL+eg=
|
||||||
|
github.com/testcontainers/testcontainers-go/modules/postgres v0.40.0 h1:s2bIayFXlbDFexo96y+htn7FzuhpXLYJNnIuglNKqOk=
|
||||||
|
github.com/testcontainers/testcontainers-go/modules/postgres v0.40.0/go.mod h1:h+u/2KoREGTnTl9UwrQ/g+XhasAT8E6dClclAADeXoQ=
|
||||||
|
github.com/tklauser/go-sysconf v0.3.16 h1:frioLaCQSsF5Cy1jgRBrzr6t502KIIwQ0MArYICU0nA=
|
||||||
|
github.com/tklauser/go-sysconf v0.3.16/go.mod h1:/qNL9xxDhc7tx3HSRsLWNnuzbVfh3e7gh/BmM179nYI=
|
||||||
|
github.com/tklauser/numcpus v0.11.0 h1:nSTwhKH5e1dMNsCdVBukSZrURJRoHbSEQjdEbY+9RXw=
|
||||||
|
github.com/tklauser/numcpus v0.11.0/go.mod h1:z+LwcLq54uWZTX0u/bGobaV34u6V7KNlTZejzM6/3MQ=
|
||||||
github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
|
github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
|
||||||
github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
|
github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
|
||||||
github.com/valyala/fasthttp v1.68.0 h1:v12Nx16iepr8r9ySOwqI+5RBJ/DqTxhOy1HrHoDFnok=
|
github.com/valyala/fasthttp v1.71.0 h1:tepR7H+Guh9VUqxxcPggYi8R3lGUu2Rsdh+z7/FCY3k=
|
||||||
github.com/valyala/fasthttp v1.68.0/go.mod h1:5EXiRfYQAoiO/khu4oU9VISC/eVY6JqmSpPJoHCKsz4=
|
github.com/valyala/fasthttp v1.71.0/go.mod h1:z1sDUvOShhXq/C9mwH/fSm1Vb71tUJwmQdgkBrBNwnA=
|
||||||
github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU=
|
github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU=
|
||||||
github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E=
|
github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E=
|
||||||
|
github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0=
|
||||||
|
github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
|
||||||
|
go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64=
|
||||||
|
go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y=
|
||||||
|
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.64.0 h1:ssfIgGNANqpVFCndZvcuyKbl0g+UAVcbBcqGkG28H0Y=
|
||||||
|
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.64.0/go.mod h1:GQ/474YrbE4Jx8gZ4q5I4hrhUzM6UPzyrqJYV2AqPoQ=
|
||||||
|
go.opentelemetry.io/otel v1.39.0 h1:8yPrr/S0ND9QEfTfdP9V+SiwT4E0G7Y5MO7p85nis48=
|
||||||
|
go.opentelemetry.io/otel v1.39.0/go.mod h1:kLlFTywNWrFyEdH0oj2xK0bFYZtHRYUdv1NklR/tgc8=
|
||||||
|
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.19.0 h1:Mne5On7VWdx7omSrSSZvM4Kw7cS7NQkOOmLcgscI51U=
|
||||||
|
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.19.0/go.mod h1:IPtUMKL4O3tH5y+iXVyAXqpAwMuzC1IrxVS81rummfE=
|
||||||
|
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.19.0 h1:IeMeyr1aBvBiPVYihXIaeIZba6b8E1bYp7lbdxK8CQg=
|
||||||
|
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.19.0/go.mod h1:oVdCUtjq9MK9BlS7TtucsQwUcXcymNiEDjgDD2jMtZU=
|
||||||
|
go.opentelemetry.io/otel/metric v1.39.0 h1:d1UzonvEZriVfpNKEVmHXbdf909uGTOQjA0HF0Ls5Q0=
|
||||||
|
go.opentelemetry.io/otel/metric v1.39.0/go.mod h1:jrZSWL33sD7bBxg1xjrqyDjnuzTUB0x1nBERXd7Ftcs=
|
||||||
|
go.opentelemetry.io/otel/sdk v1.39.0 h1:nMLYcjVsvdui1B/4FRkwjzoRVsMK8uL/cj0OyhKzt18=
|
||||||
|
go.opentelemetry.io/otel/sdk v1.39.0/go.mod h1:vDojkC4/jsTJsE+kh+LXYQlbL8CgrEcwmt1ENZszdJE=
|
||||||
|
go.opentelemetry.io/otel/sdk/metric v1.39.0 h1:cXMVVFVgsIf2YL6QkRF4Urbr/aMInf+2WKg+sEJTtB8=
|
||||||
|
go.opentelemetry.io/otel/sdk/metric v1.39.0/go.mod h1:xq9HEVH7qeX69/JnwEfp6fVq5wosJsY1mt4lLfYdVew=
|
||||||
|
go.opentelemetry.io/otel/trace v1.39.0 h1:2d2vfpEDmCJ5zVYz7ijaJdOF59xLomrvj7bjt6/qCJI=
|
||||||
|
go.opentelemetry.io/otel/trace v1.39.0/go.mod h1:88w4/PnZSazkGzz/w84VHpQafiU4EtqqlVdxWy+rNOA=
|
||||||
|
go.opentelemetry.io/proto/otlp v1.0.0 h1:T0TX0tmXU8a3CbNXzEKGeU5mIVOdf0oykP+u2lIVU/I=
|
||||||
|
go.opentelemetry.io/proto/otlp v1.0.0/go.mod h1:Sy6pihPLfYHkr3NkUbEhGHFhINUSI/v80hjKIs5JXpM=
|
||||||
go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
|
go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
|
||||||
go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
|
go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
|
||||||
go.yaml.in/yaml/v2 v2.4.3 h1:6gvOSjQoTB3vt1l+CU+tSyi/HOjfOjRLJ4YwYZGwRO0=
|
go.yaml.in/yaml/v2 v2.4.4 h1:tuyd0P+2Ont/d6e2rl3be67goVK4R6deVxCUX5vyPaQ=
|
||||||
go.yaml.in/yaml/v2 v2.4.3/go.mod h1:zSxWcmIDjOzPXpjlTTbAsKokqkDNAVtZO0WOMiT90s8=
|
go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ=
|
||||||
go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc=
|
go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc=
|
||||||
go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
|
go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
|
||||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||||
golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
|
golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
|
||||||
golang.org/x/crypto v0.46.0 h1:cKRW/pmt1pKAfetfu+RCEvjvZkA9RimPbh7bhFjGVBU=
|
golang.org/x/crypto v0.52.0 h1:RMs7fP2rXdep0CftQlK8Uf+kibLm7qkCcradZWYz988=
|
||||||
golang.org/x/crypto v0.46.0/go.mod h1:Evb/oLKmMraqjZ2iQTwDwvCtJkczlDuTmdJXoZVzqU0=
|
golang.org/x/crypto v0.52.0/go.mod h1:1QgfPxDqh0T2M/elOJtp9RvuR95kVjir0e6/BvEmGbc=
|
||||||
golang.org/x/exp v0.0.0-20251219203646-944ab1f22d93 h1:fQsdNF2N+/YewlRZiricy4P1iimyPKZ/xwniHj8Q2a0=
|
|
||||||
golang.org/x/exp v0.0.0-20251219203646-944ab1f22d93/go.mod h1:EPRbTFwzwjXj9NpYyyrvenVh9Y+GFeEvMNh7Xuz7xgU=
|
|
||||||
golang.org/x/mod v0.31.0 h1:HaW9xtz0+kOcWKwli0ZXy79Ix+UW/vOfmWI5QVd2tgI=
|
|
||||||
golang.org/x/mod v0.31.0/go.mod h1:43JraMp9cGx1Rx3AqioxrbrhNsLl2l/iNAvuBkrezpg=
|
|
||||||
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||||
golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4=
|
golang.org/x/net v0.54.0 h1:2zJIZAxAHV/OHCDTCOHAYehQzLfSXuf/5SoL/Dv6w/w=
|
||||||
golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
|
golang.org/x/net v0.54.0/go.mod h1:Sj4oj8jK6XmHpBZU/zWHw3BV3abl4Kvi+Ut7cQcY+cQ=
|
||||||
|
golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4=
|
||||||
|
golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
|
||||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||||
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||||
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||||
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||||
golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||||
golang.org/x/sys v0.39.0 h1:CvCKL8MeisomCi6qNZ+wbb0DN9E5AATixKsvNtMoMFk=
|
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||||
golang.org/x/sys v0.39.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
|
golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||||
|
golang.org/x/sys v0.45.0 h1:dO4czNzziLiiXplLQgBCEpCvXQ3dnkn0SdaZSYdQ+FY=
|
||||||
|
golang.org/x/sys v0.45.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
|
||||||
|
golang.org/x/term v0.43.0 h1:S4RLU2sB31O/NCl+zFN9Aru9A/Cq2aqKpTZJ6B+DwT4=
|
||||||
|
golang.org/x/term v0.43.0/go.mod h1:lrhlHNdQJHO+1qVYiHfFKVuVioJIheAc3fBSMFYEIsk=
|
||||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||||
golang.org/x/text v0.32.0 h1:ZD01bjUt1FQ9WJ0ClOL5vxgxOI/sVCNgX1YtKwcY0mU=
|
golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc=
|
||||||
golang.org/x/text v0.32.0/go.mod h1:o/rUWzghvpD5TXrTIBuJU77MTaN0ljMWE47kxGJQ7jY=
|
golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38=
|
||||||
golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI=
|
golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U=
|
||||||
golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4=
|
golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno=
|
||||||
golang.org/x/tools v0.40.0 h1:yLkxfA+Qnul4cs9QA3KnlFu0lVmd8JJfoq+E41uSutA=
|
google.golang.org/genproto/googleapis/api v0.0.0-20251222181119-0a764e51fe1b h1:uA40e2M6fYRBf0+8uN5mLlqUtV192iiksiICIBkYJ1E=
|
||||||
golang.org/x/tools v0.40.0/go.mod h1:Ik/tzLRlbscWpqqMRjyWYDisX8bG13FrdXp3o4Sr9lc=
|
google.golang.org/genproto/googleapis/api v0.0.0-20251222181119-0a764e51fe1b/go.mod h1:Xa7le7qx2vmqB/SzWUBa7KdMjpdpAHlh5QCSnjessQk=
|
||||||
|
google.golang.org/genproto/googleapis/rpc v0.0.0-20251222181119-0a764e51fe1b h1:Mv8VFug0MP9e5vUxfBcE3vUkV6CImK3cMNMIDFjmzxU=
|
||||||
|
google.golang.org/genproto/googleapis/rpc v0.0.0-20251222181119-0a764e51fe1b/go.mod h1:j9x/tPzZkyxcgEFkiKEEGxfvyumM01BEtsW8xzOahRQ=
|
||||||
|
google.golang.org/grpc v1.78.0 h1:K1XZG/yGDJnzMdd/uZHAkVqJE+xIDOcmdSFZkBUicNc=
|
||||||
|
google.golang.org/grpc v1.78.0/go.mod h1:I47qjTo4OKbMkjA/aOOwxDIiPSBofUtQUI5EfpWvW7U=
|
||||||
google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE=
|
google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE=
|
||||||
google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
|
google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
|
||||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
|
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
|
||||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
|
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
|
||||||
|
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||||
modernc.org/cc/v4 v4.27.1 h1:9W30zRlYrefrDV2JE2O8VDtJ1yPGownxciz5rrbQZis=
|
gorm.io/driver/mysql v1.6.0 h1:eNbLmNTpPpTOVZi8MMxCi2aaIm0ZpInbORNXDwyLGvg=
|
||||||
modernc.org/cc/v4 v4.27.1/go.mod h1:uVtb5OGqUKpoLWhqwNQo/8LwvoiEBLvZXIQ/SmO6mL0=
|
gorm.io/driver/mysql v1.6.0/go.mod h1:D/oCC2GWK3M/dqoLxnOlaNKmXz8WNTfcS9y5ovaSqKo=
|
||||||
modernc.org/ccgo/v4 v4.30.1 h1:4r4U1J6Fhj98NKfSjnPUN7Ze2c6MnAdL0hWw6+LrJpc=
|
gorm.io/driver/postgres v1.6.0 h1:2dxzU8xJ+ivvqTRph34QX+WrRaJlmfyPqXmoGVjMBa4=
|
||||||
modernc.org/ccgo/v4 v4.30.1/go.mod h1:bIOeI1JL54Utlxn+LwrFyjCx2n2RDiYEaJVSrgdrRfM=
|
gorm.io/driver/postgres v1.6.0/go.mod h1:vUw0mrGgrTK+uPHEhAdV4sfFELrByKVGnaVRkXDhtWo=
|
||||||
modernc.org/fileutil v1.3.40 h1:ZGMswMNc9JOCrcrakF1HrvmergNLAmxOPjizirpfqBA=
|
gorm.io/driver/sqlite v1.6.0 h1:WHRRrIiulaPiPFmDcod6prc4l2VGVWHz80KspNsxSfQ=
|
||||||
modernc.org/fileutil v1.3.40/go.mod h1:HxmghZSZVAz/LXcMNwZPA/DRrQZEVP9VX0V4LQGQFOc=
|
gorm.io/driver/sqlite v1.6.0/go.mod h1:AO9V1qIQddBESngQUKWL9yoH93HIeA1X6V633rBwyT8=
|
||||||
modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI=
|
gorm.io/gorm v1.31.1 h1:7CA8FTFz/gRfgqgpeKIBcervUn3xSyPUmr6B2WXJ7kg=
|
||||||
modernc.org/gc/v2 v2.6.5/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito=
|
gorm.io/gorm v1.31.1/go.mod h1:XyQVbO2k6YkOis7C2437jSit3SsDK72s7n7rsSHd+Gs=
|
||||||
modernc.org/gc/v3 v3.1.1 h1:k8T3gkXWY9sEiytKhcgyiZ2L0DTyCQ/nvX+LoCljoRE=
|
gotest.tools/v3 v3.5.2 h1:7koQfIKdy+I8UTetycgUqXWSDwpgv193Ka+qRsmBY8Q=
|
||||||
modernc.org/gc/v3 v3.1.1/go.mod h1:HFK/6AGESC7Ex+EZJhJ2Gni6cTaYpSMmU/cT9RmlfYY=
|
gotest.tools/v3 v3.5.2/go.mod h1:LtdLGcnqToBH83WByAAi/wiwSFCArdFIUV/xxN4pcjA=
|
||||||
modernc.org/goabi0 v0.2.0 h1:HvEowk7LxcPd0eq6mVOAEMai46V+i7Jrj13t4AzuNks=
|
|
||||||
modernc.org/goabi0 v0.2.0/go.mod h1:CEFRnnJhKvWT1c1JTI3Avm+tgOWbkOu5oPA8eH8LnMI=
|
|
||||||
modernc.org/libc v1.67.4 h1:zZGmCMUVPORtKv95c2ReQN5VDjvkoRm9GWPTEPuvlWg=
|
|
||||||
modernc.org/libc v1.67.4/go.mod h1:QvvnnJ5P7aitu0ReNpVIEyesuhmDLQ8kaEoyMjIFZJA=
|
|
||||||
modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU=
|
|
||||||
modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg=
|
|
||||||
modernc.org/memory v1.11.0 h1:o4QC8aMQzmcwCK3t3Ux/ZHmwFPzE6hf2Y5LbkRs+hbI=
|
|
||||||
modernc.org/memory v1.11.0/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw=
|
|
||||||
modernc.org/opt v0.1.4 h1:2kNGMRiUjrp4LcaPuLY2PzUfqM/w9N23quVwhKt5Qm8=
|
|
||||||
modernc.org/opt v0.1.4/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns=
|
|
||||||
modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w=
|
|
||||||
modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE=
|
|
||||||
modernc.org/sqlite v1.42.2 h1:7hkZUNJvJFN2PgfUdjni9Kbvd4ef4mNLOu0B9FGxM74=
|
|
||||||
modernc.org/sqlite v1.42.2/go.mod h1:+VkC6v3pLOAE0A0uVucQEcbVW0I5nHCeDaBf+DpsQT8=
|
|
||||||
modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0=
|
|
||||||
modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A=
|
|
||||||
modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y=
|
|
||||||
modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM=
|
|
||||||
|
|||||||
@@ -19,4 +19,4 @@ sources:
|
|||||||
maintainers:
|
maintainers:
|
||||||
- name: Lukasz Raczylo
|
- name: Lukasz Raczylo
|
||||||
email: [email protected]
|
email: [email protected]
|
||||||
icon: https://raw.githubusercontent.com/lukaszraczylo/gohoarder/main/docs/logo.png
|
# icon: https://raw.githubusercontent.com/lukaszraczylo/gohoarder/main/docs/logo.png # TODO: Add logo
|
||||||
|
|||||||
@@ -172,3 +172,17 @@ Trivy cache volume configuration
|
|||||||
emptyDir: {}
|
emptyDir: {}
|
||||||
{{- end }}
|
{{- end }}
|
||||||
{{- end }}
|
{{- end }}
|
||||||
|
|
||||||
|
{{/*
|
||||||
|
Validate SQLite configuration - SQLite cannot be used with SMB/NFS network storage
|
||||||
|
*/}}
|
||||||
|
{{- define "gohoarder.validateSQLiteConfig" -}}
|
||||||
|
{{- if eq .Values.metadata.backend "sqlite" }}
|
||||||
|
{{- if .Values.metadata.sqlite.persistence.enabled }}
|
||||||
|
{{- $storageClass := .Values.metadata.sqlite.persistence.storageClass | default .Values.storage.storageClass }}
|
||||||
|
{{- if or (contains "smb" ($storageClass | lower)) (contains "cifs" ($storageClass | lower)) (contains "nfs" ($storageClass | lower)) }}
|
||||||
|
{{- fail "\n\n❌ ERROR: SQLite cannot be used with SMB/CIFS/NFS network storage!\n\nSQLite requires POSIX file locking which is not reliably supported over network filesystems.\nThis will cause 'database is locked' errors and data corruption.\n\nPlease choose ONE of the following solutions:\n\n1. Use PostgreSQL for network storage (RECOMMENDED for production):\n metadata:\n backend: postgresql\n postgresql:\n host: your-postgres-host\n ...\n\n2. Use MySQL/MariaDB for network storage (alternative to PostgreSQL):\n metadata:\n backend: mysql\n mysql:\n host: your-mysql-host\n ...\n\n3. Use local storage for SQLite (OK for development):\n metadata:\n sqlite:\n persistence:\n enabled: true\n storageClass: local-path # or another local storage class\n\n4. Disable persistence (data will be lost on pod restart):\n metadata:\n sqlite:\n persistence:\n enabled: false\n\nFor more information, see: https://www.sqlite.org/lockingv3.html\n" }}
|
||||||
|
{{- end }}
|
||||||
|
{{- end }}
|
||||||
|
{{- end }}
|
||||||
|
{{- end }}
|
||||||
|
|||||||
@@ -93,24 +93,30 @@ data:
|
|||||||
low: {{ .Values.security.blockThresholds.low }}
|
low: {{ .Values.security.blockThresholds.low }}
|
||||||
scanners:
|
scanners:
|
||||||
trivy:
|
trivy:
|
||||||
enabled: {{ .Values.security.scanners.trivy.enabled }}
|
# Disabled in server config (no trivy binary), enabled via env var in scanner pod
|
||||||
|
enabled: false
|
||||||
timeout: {{ .Values.security.scanners.trivy.timeout | quote }}
|
timeout: {{ .Values.security.scanners.trivy.timeout | quote }}
|
||||||
cache_db: {{ .Values.security.scanners.trivy.cacheDb | quote }}
|
cache_db: {{ .Values.security.scanners.trivy.cacheDb | quote }}
|
||||||
osv:
|
osv:
|
||||||
|
# API-based scanner - works in both server and scanner pods
|
||||||
enabled: {{ .Values.security.scanners.osv.enabled }}
|
enabled: {{ .Values.security.scanners.osv.enabled }}
|
||||||
api_url: {{ .Values.security.scanners.osv.apiUrl | quote }}
|
api_url: {{ .Values.security.scanners.osv.apiUrl | quote }}
|
||||||
timeout: {{ .Values.security.scanners.osv.timeout | quote }}
|
timeout: {{ .Values.security.scanners.osv.timeout | quote }}
|
||||||
grype:
|
grype:
|
||||||
enabled: {{ .Values.security.scanners.grype.enabled }}
|
# Disabled in server config (no grype binary), enabled via env var in scanner pod
|
||||||
|
enabled: false
|
||||||
timeout: {{ .Values.security.scanners.grype.timeout | quote }}
|
timeout: {{ .Values.security.scanners.grype.timeout | quote }}
|
||||||
govulncheck:
|
govulncheck:
|
||||||
enabled: {{ .Values.security.scanners.govulncheck.enabled }}
|
# Disabled in server config (no go/govulncheck binary), enabled via env var in scanner pod
|
||||||
|
enabled: false
|
||||||
timeout: {{ .Values.security.scanners.govulncheck.timeout | quote }}
|
timeout: {{ .Values.security.scanners.govulncheck.timeout | quote }}
|
||||||
npm_audit:
|
npm_audit:
|
||||||
enabled: {{ .Values.security.scanners.npmAudit.enabled }}
|
# Disabled in server config (no npm binary), enabled via env var in scanner pod
|
||||||
|
enabled: false
|
||||||
timeout: {{ .Values.security.scanners.npmAudit.timeout | quote }}
|
timeout: {{ .Values.security.scanners.npmAudit.timeout | quote }}
|
||||||
pip_audit:
|
pip_audit:
|
||||||
enabled: {{ .Values.security.scanners.pipAudit.enabled }}
|
# Disabled in server config (no pip binary), enabled via env var in scanner pod
|
||||||
|
enabled: false
|
||||||
timeout: {{ .Values.security.scanners.pipAudit.timeout | quote }}
|
timeout: {{ .Values.security.scanners.pipAudit.timeout | quote }}
|
||||||
ghsa:
|
ghsa:
|
||||||
enabled: {{ .Values.security.scanners.ghsa.enabled }}
|
enabled: {{ .Values.security.scanners.ghsa.enabled }}
|
||||||
|
|||||||
@@ -26,7 +26,30 @@ spec:
|
|||||||
{{- end }}
|
{{- end }}
|
||||||
serviceAccountName: {{ include "gohoarder.serviceAccountName" . }}
|
serviceAccountName: {{ include "gohoarder.serviceAccountName" . }}
|
||||||
securityContext:
|
securityContext:
|
||||||
{{- toYaml .Values.podSecurityContext | nindent 8 }}
|
fsGroup: 101
|
||||||
|
runAsNonRoot: true
|
||||||
|
runAsUser: 101
|
||||||
|
initContainers:
|
||||||
|
- name: copy-static-files
|
||||||
|
image: "{{ .Values.image.frontend.repository }}:{{ .Values.image.frontend.tag | default .Chart.AppVersion }}"
|
||||||
|
command: ['sh', '-c']
|
||||||
|
args:
|
||||||
|
- |
|
||||||
|
# Copy built frontend files to writable volume
|
||||||
|
cp -rp /usr/share/nginx/html/* /html/
|
||||||
|
# Copy nginx config to writable volume
|
||||||
|
cp -rp /etc/nginx/conf.d/* /conf/
|
||||||
|
volumeMounts:
|
||||||
|
- name: nginx-html
|
||||||
|
mountPath: /html
|
||||||
|
- name: nginx-conf
|
||||||
|
mountPath: /conf
|
||||||
|
securityContext:
|
||||||
|
runAsUser: 101
|
||||||
|
allowPrivilegeEscalation: false
|
||||||
|
capabilities:
|
||||||
|
drop:
|
||||||
|
- ALL
|
||||||
containers:
|
containers:
|
||||||
- name: frontend
|
- name: frontend
|
||||||
securityContext:
|
securityContext:
|
||||||
@@ -35,6 +58,7 @@ spec:
|
|||||||
drop:
|
drop:
|
||||||
- ALL
|
- ALL
|
||||||
readOnlyRootFilesystem: false
|
readOnlyRootFilesystem: false
|
||||||
|
runAsUser: 101
|
||||||
image: "{{ .Values.image.frontend.repository }}:{{ .Values.image.frontend.tag | default .Chart.AppVersion }}"
|
image: "{{ .Values.image.frontend.repository }}:{{ .Values.image.frontend.tag | default .Chart.AppVersion }}"
|
||||||
imagePullPolicy: {{ .Values.image.frontend.pullPolicy }}
|
imagePullPolicy: {{ .Values.image.frontend.pullPolicy }}
|
||||||
ports:
|
ports:
|
||||||
@@ -43,11 +67,18 @@ spec:
|
|||||||
protocol: TCP
|
protocol: TCP
|
||||||
env:
|
env:
|
||||||
- name: API_BASE_URL
|
- name: API_BASE_URL
|
||||||
value: {{ .Values.frontend.backendUrl | default (printf "http://%s-server:%d" (include "gohoarder.fullname" .) (.Values.server.service.port | int)) | quote }}
|
value: {{ .Values.frontend.backendUrl | default "/api" | quote }}
|
||||||
- name: APP_VERSION
|
- name: APP_VERSION
|
||||||
value: {{ .Chart.AppVersion | quote }}
|
value: {{ .Chart.AppVersion | quote }}
|
||||||
- name: APP_NAME
|
- name: APP_NAME
|
||||||
value: "GoHoarder"
|
value: "GoHoarder"
|
||||||
|
# Backend proxy configuration (frontend now includes reverse proxy)
|
||||||
|
- name: BACKEND_HOST
|
||||||
|
value: {{ include "gohoarder.fullname" . }}-server
|
||||||
|
- name: BACKEND_PORT
|
||||||
|
value: {{ .Values.server.service.port | quote }}
|
||||||
|
- name: SERVER_NAME
|
||||||
|
value: {{ .Values.frontend.serverName | default "_" | quote }}
|
||||||
{{- with .Values.frontend.env }}
|
{{- with .Values.frontend.env }}
|
||||||
{{- toYaml . | nindent 8 }}
|
{{- toYaml . | nindent 8 }}
|
||||||
{{- end }}
|
{{- end }}
|
||||||
@@ -64,6 +95,10 @@ spec:
|
|||||||
mountPath: /var/cache/nginx
|
mountPath: /var/cache/nginx
|
||||||
- name: nginx-run
|
- name: nginx-run
|
||||||
mountPath: /var/run
|
mountPath: /var/run
|
||||||
|
- name: nginx-html
|
||||||
|
mountPath: /usr/share/nginx/html
|
||||||
|
- name: nginx-conf
|
||||||
|
mountPath: /etc/nginx/conf.d
|
||||||
volumes:
|
volumes:
|
||||||
- name: tmp
|
- name: tmp
|
||||||
emptyDir: {}
|
emptyDir: {}
|
||||||
@@ -71,6 +106,10 @@ spec:
|
|||||||
emptyDir: {}
|
emptyDir: {}
|
||||||
- name: nginx-run
|
- name: nginx-run
|
||||||
emptyDir: {}
|
emptyDir: {}
|
||||||
|
- name: nginx-html
|
||||||
|
emptyDir: {}
|
||||||
|
- name: nginx-conf
|
||||||
|
emptyDir: {}
|
||||||
{{- with .Values.frontend.nodeSelector }}
|
{{- with .Values.frontend.nodeSelector }}
|
||||||
nodeSelector:
|
nodeSelector:
|
||||||
{{- toYaml . | nindent 8 }}
|
{{- toYaml . | nindent 8 }}
|
||||||
|
|||||||
@@ -1,4 +1,5 @@
|
|||||||
{{- if .Values.security.enabled }}
|
{{- if .Values.security.enabled }}
|
||||||
|
{{- include "gohoarder.validateSQLiteConfig" . }}
|
||||||
apiVersion: apps/v1
|
apiVersion: apps/v1
|
||||||
kind: Deployment
|
kind: Deployment
|
||||||
metadata:
|
metadata:
|
||||||
@@ -28,27 +29,77 @@ spec:
|
|||||||
serviceAccountName: {{ include "gohoarder.serviceAccountName" . }}
|
serviceAccountName: {{ include "gohoarder.serviceAccountName" . }}
|
||||||
securityContext:
|
securityContext:
|
||||||
{{- toYaml .Values.podSecurityContext | nindent 8 }}
|
{{- toYaml .Values.podSecurityContext | nindent 8 }}
|
||||||
|
{{- if .Values.migration.enabled }}
|
||||||
initContainers:
|
initContainers:
|
||||||
- name: init-permissions
|
# Wait for database to be ready
|
||||||
image: busybox:latest
|
- name: wait-for-db
|
||||||
command: ['sh', '-c']
|
image: busybox:1.36
|
||||||
args:
|
command:
|
||||||
|
- sh
|
||||||
|
- -c
|
||||||
- |
|
- |
|
||||||
mkdir -p /var/cache/gohoarder /var/lib/gohoarder/metadata /tmp/gohoarder
|
echo "Waiting for database..."
|
||||||
{{- if .Values.security.scanners.trivy.enabled }}
|
{{- if eq .Values.metadata.backend "postgresql" }}
|
||||||
mkdir -p {{ .Values.security.scanners.trivy.cacheDb }}
|
until nc -z {{ .Values.metadata.postgresql.host }} {{ .Values.metadata.postgresql.port }}; do
|
||||||
chown -R 1000:1000 {{ .Values.security.scanners.trivy.cacheDb }}
|
echo " PostgreSQL not ready, retrying in 2s..."
|
||||||
|
sleep 2
|
||||||
|
done
|
||||||
|
echo "✓ PostgreSQL is ready"
|
||||||
|
{{- else if eq .Values.metadata.backend "mysql" }}
|
||||||
|
until nc -z {{ .Values.metadata.mysql.host }} {{ .Values.metadata.mysql.port }}; do
|
||||||
|
echo " MySQL not ready, retrying in 2s..."
|
||||||
|
sleep 2
|
||||||
|
done
|
||||||
|
echo "✓ MySQL is ready"
|
||||||
|
{{- else }}
|
||||||
|
echo "✓ SQLite (no wait needed)"
|
||||||
{{- end }}
|
{{- end }}
|
||||||
chown -R 1000:1000 /var/cache/gohoarder /var/lib/gohoarder /tmp/gohoarder
|
|
||||||
chmod 750 /var/cache/gohoarder /var/lib/gohoarder
|
|
||||||
volumeMounts:
|
|
||||||
{{- include "gohoarder.storageVolume" . | nindent 8 }}
|
|
||||||
{{- include "gohoarder.metadataVolume" . | nindent 8 }}
|
|
||||||
{{- include "gohoarder.trivyCacheVolume" . | nindent 8 }}
|
|
||||||
- name: tmp
|
|
||||||
mountPath: /tmp/gohoarder
|
|
||||||
securityContext:
|
securityContext:
|
||||||
runAsUser: 0
|
allowPrivilegeEscalation: false
|
||||||
|
runAsNonRoot: true
|
||||||
|
runAsUser: 1000
|
||||||
|
resources:
|
||||||
|
limits:
|
||||||
|
cpu: 100m
|
||||||
|
memory: 64Mi
|
||||||
|
requests:
|
||||||
|
cpu: 10m
|
||||||
|
memory: 32Mi
|
||||||
|
# Run database migrations
|
||||||
|
- name: migrate
|
||||||
|
image: "{{ .Values.migration.image.repository }}:{{ .Values.migration.image.tag | default .Chart.AppVersion }}"
|
||||||
|
imagePullPolicy: {{ .Values.migration.image.pullPolicy }}
|
||||||
|
env:
|
||||||
|
- name: DB_DRIVER
|
||||||
|
value: {{ .Values.metadata.backend | quote }}
|
||||||
|
{{- if eq .Values.metadata.backend "postgresql" }}
|
||||||
|
- name: DATABASE_URL
|
||||||
|
value: "postgresql://{{ .Values.metadata.postgresql.username }}:{{ .Values.metadata.postgresql.password }}@{{ .Values.metadata.postgresql.host }}:{{ .Values.metadata.postgresql.port }}/{{ .Values.metadata.postgresql.database }}?sslmode={{ .Values.metadata.postgresql.sslMode }}"
|
||||||
|
{{- else if eq .Values.metadata.backend "mysql" }}
|
||||||
|
- name: DATABASE_URL
|
||||||
|
value: "{{ .Values.metadata.mysql.username }}:{{ .Values.metadata.mysql.password }}@tcp({{ .Values.metadata.mysql.host }}:{{ .Values.metadata.mysql.port }})/{{ .Values.metadata.mysql.database }}?charset={{ .Values.metadata.mysql.charset }}&parseTime={{ .Values.metadata.mysql.parseTime }}"
|
||||||
|
{{- else }}
|
||||||
|
- name: DATABASE_URL
|
||||||
|
value: "/var/lib/gohoarder/metadata/gohoarder.db"
|
||||||
|
{{- end }}
|
||||||
|
args:
|
||||||
|
- --driver=$(DB_DRIVER)
|
||||||
|
- --dsn=$(DATABASE_URL)
|
||||||
|
- --action=migrate
|
||||||
|
- --log-level={{ .Values.migration.logLevel | default "info" }}
|
||||||
|
- --timeout={{ .Values.migration.timeout | default "5m" }}
|
||||||
|
securityContext:
|
||||||
|
allowPrivilegeEscalation: false
|
||||||
|
runAsNonRoot: true
|
||||||
|
runAsUser: 1000
|
||||||
|
resources:
|
||||||
|
{{- toYaml .Values.migration.resources | nindent 10 }}
|
||||||
|
{{- if eq .Values.metadata.backend "sqlite" }}
|
||||||
|
volumeMounts:
|
||||||
|
- name: metadata
|
||||||
|
mountPath: /var/lib/gohoarder/metadata
|
||||||
|
{{- end }}
|
||||||
|
{{- end }}
|
||||||
containers:
|
containers:
|
||||||
- name: scanner
|
- name: scanner
|
||||||
securityContext:
|
securityContext:
|
||||||
@@ -58,6 +109,63 @@ spec:
|
|||||||
env:
|
env:
|
||||||
- name: CONFIG_FILE
|
- name: CONFIG_FILE
|
||||||
value: /etc/gohoarder/config.yaml
|
value: /etc/gohoarder/config.yaml
|
||||||
|
# Enable tool-based scanners only in scanner pod (server doesn't have the tools)
|
||||||
|
- name: GOHOARDER_SECURITY_SCANNERS_TRIVY_ENABLED
|
||||||
|
value: "{{ .Values.security.scanners.trivy.enabled }}"
|
||||||
|
- name: GOHOARDER_SECURITY_SCANNERS_GRYPE_ENABLED
|
||||||
|
value: "{{ .Values.security.scanners.grype.enabled }}"
|
||||||
|
- name: GOHOARDER_SECURITY_SCANNERS_GOVULNCHECK_ENABLED
|
||||||
|
value: "{{ .Values.security.scanners.govulncheck.enabled }}"
|
||||||
|
- name: GOHOARDER_SECURITY_SCANNERS_NPM_AUDIT_ENABLED
|
||||||
|
value: "{{ .Values.security.scanners.npmAudit.enabled }}"
|
||||||
|
- name: GOHOARDER_SECURITY_SCANNERS_PIP_AUDIT_ENABLED
|
||||||
|
value: "{{ .Values.security.scanners.pipAudit.enabled }}"
|
||||||
|
{{- if and (eq .Values.metadata.backend "postgresql") .Values.metadata.postgresql.existingSecret }}
|
||||||
|
- name: POSTGRES_USER
|
||||||
|
valueFrom:
|
||||||
|
secretKeyRef:
|
||||||
|
name: {{ .Values.metadata.postgresql.existingSecret }}
|
||||||
|
key: username
|
||||||
|
- name: POSTGRES_PASSWORD
|
||||||
|
valueFrom:
|
||||||
|
secretKeyRef:
|
||||||
|
name: {{ .Values.metadata.postgresql.existingSecret }}
|
||||||
|
key: password
|
||||||
|
{{- else if and (eq .Values.metadata.backend "postgresql") .Values.metadata.postgresql.username }}
|
||||||
|
- name: POSTGRES_USER
|
||||||
|
valueFrom:
|
||||||
|
secretKeyRef:
|
||||||
|
name: {{ include "gohoarder.fullname" . }}-postgresql
|
||||||
|
key: username
|
||||||
|
- name: POSTGRES_PASSWORD
|
||||||
|
valueFrom:
|
||||||
|
secretKeyRef:
|
||||||
|
name: {{ include "gohoarder.fullname" . }}-postgresql
|
||||||
|
key: password
|
||||||
|
{{- end }}
|
||||||
|
{{- if and (or (eq .Values.metadata.backend "mysql") (eq .Values.metadata.backend "mariadb")) .Values.metadata.mysql.existingSecret }}
|
||||||
|
- name: MYSQL_USER
|
||||||
|
valueFrom:
|
||||||
|
secretKeyRef:
|
||||||
|
name: {{ .Values.metadata.mysql.existingSecret }}
|
||||||
|
key: username
|
||||||
|
- name: MYSQL_PASSWORD
|
||||||
|
valueFrom:
|
||||||
|
secretKeyRef:
|
||||||
|
name: {{ .Values.metadata.mysql.existingSecret }}
|
||||||
|
key: password
|
||||||
|
{{- else if and (or (eq .Values.metadata.backend "mysql") (eq .Values.metadata.backend "mariadb")) .Values.metadata.mysql.username }}
|
||||||
|
- name: MYSQL_USER
|
||||||
|
valueFrom:
|
||||||
|
secretKeyRef:
|
||||||
|
name: {{ include "gohoarder.fullname" . }}-mysql
|
||||||
|
key: username
|
||||||
|
- name: MYSQL_PASSWORD
|
||||||
|
valueFrom:
|
||||||
|
secretKeyRef:
|
||||||
|
name: {{ include "gohoarder.fullname" . }}-mysql
|
||||||
|
key: password
|
||||||
|
{{- end }}
|
||||||
{{- if and .Values.security.scanners.ghsa.enabled .Values.security.scanners.ghsa.existingSecret }}
|
{{- if and .Values.security.scanners.ghsa.enabled .Values.security.scanners.ghsa.existingSecret }}
|
||||||
- name: GHSA_TOKEN
|
- name: GHSA_TOKEN
|
||||||
valueFrom:
|
valueFrom:
|
||||||
|
|||||||
@@ -1,3 +1,4 @@
|
|||||||
|
{{- include "gohoarder.validateSQLiteConfig" . }}
|
||||||
apiVersion: apps/v1
|
apiVersion: apps/v1
|
||||||
kind: Deployment
|
kind: Deployment
|
||||||
metadata:
|
metadata:
|
||||||
@@ -29,22 +30,77 @@ spec:
|
|||||||
serviceAccountName: {{ include "gohoarder.serviceAccountName" . }}
|
serviceAccountName: {{ include "gohoarder.serviceAccountName" . }}
|
||||||
securityContext:
|
securityContext:
|
||||||
{{- toYaml .Values.podSecurityContext | nindent 8 }}
|
{{- toYaml .Values.podSecurityContext | nindent 8 }}
|
||||||
|
{{- if .Values.migration.enabled }}
|
||||||
initContainers:
|
initContainers:
|
||||||
- name: init-permissions
|
# Wait for database to be ready
|
||||||
image: busybox:latest
|
- name: wait-for-db
|
||||||
command: ['sh', '-c']
|
image: busybox:1.36
|
||||||
args:
|
command:
|
||||||
|
- sh
|
||||||
|
- -c
|
||||||
- |
|
- |
|
||||||
mkdir -p /var/cache/gohoarder /var/lib/gohoarder/metadata /tmp/gohoarder
|
echo "Waiting for database..."
|
||||||
chown -R 1000:1000 /var/cache/gohoarder /var/lib/gohoarder /tmp/gohoarder
|
{{- if eq .Values.metadata.backend "postgresql" }}
|
||||||
chmod 750 /var/cache/gohoarder /var/lib/gohoarder
|
until nc -z {{ .Values.metadata.postgresql.host }} {{ .Values.metadata.postgresql.port }}; do
|
||||||
volumeMounts:
|
echo " PostgreSQL not ready, retrying in 2s..."
|
||||||
{{- include "gohoarder.storageVolume" . | nindent 8 }}
|
sleep 2
|
||||||
{{- include "gohoarder.metadataVolume" . | nindent 8 }}
|
done
|
||||||
- name: tmp
|
echo "✓ PostgreSQL is ready"
|
||||||
mountPath: /tmp/gohoarder
|
{{- else if eq .Values.metadata.backend "mysql" }}
|
||||||
|
until nc -z {{ .Values.metadata.mysql.host }} {{ .Values.metadata.mysql.port }}; do
|
||||||
|
echo " MySQL not ready, retrying in 2s..."
|
||||||
|
sleep 2
|
||||||
|
done
|
||||||
|
echo "✓ MySQL is ready"
|
||||||
|
{{- else }}
|
||||||
|
echo "✓ SQLite (no wait needed)"
|
||||||
|
{{- end }}
|
||||||
securityContext:
|
securityContext:
|
||||||
runAsUser: 0
|
allowPrivilegeEscalation: false
|
||||||
|
runAsNonRoot: true
|
||||||
|
runAsUser: 1000
|
||||||
|
resources:
|
||||||
|
limits:
|
||||||
|
cpu: 100m
|
||||||
|
memory: 64Mi
|
||||||
|
requests:
|
||||||
|
cpu: 10m
|
||||||
|
memory: 32Mi
|
||||||
|
# Run database migrations
|
||||||
|
- name: migrate
|
||||||
|
image: "{{ .Values.migration.image.repository }}:{{ .Values.migration.image.tag | default .Chart.AppVersion }}"
|
||||||
|
imagePullPolicy: {{ .Values.migration.image.pullPolicy }}
|
||||||
|
env:
|
||||||
|
- name: DB_DRIVER
|
||||||
|
value: {{ .Values.metadata.backend | quote }}
|
||||||
|
{{- if eq .Values.metadata.backend "postgresql" }}
|
||||||
|
- name: DATABASE_URL
|
||||||
|
value: "postgresql://{{ .Values.metadata.postgresql.username }}:{{ .Values.metadata.postgresql.password }}@{{ .Values.metadata.postgresql.host }}:{{ .Values.metadata.postgresql.port }}/{{ .Values.metadata.postgresql.database }}?sslmode={{ .Values.metadata.postgresql.sslMode }}"
|
||||||
|
{{- else if eq .Values.metadata.backend "mysql" }}
|
||||||
|
- name: DATABASE_URL
|
||||||
|
value: "{{ .Values.metadata.mysql.username }}:{{ .Values.metadata.mysql.password }}@tcp({{ .Values.metadata.mysql.host }}:{{ .Values.metadata.mysql.port }})/{{ .Values.metadata.mysql.database }}?charset={{ .Values.metadata.mysql.charset }}&parseTime={{ .Values.metadata.mysql.parseTime }}"
|
||||||
|
{{- else }}
|
||||||
|
- name: DATABASE_URL
|
||||||
|
value: "/var/lib/gohoarder/metadata/gohoarder.db"
|
||||||
|
{{- end }}
|
||||||
|
args:
|
||||||
|
- --driver=$(DB_DRIVER)
|
||||||
|
- --dsn=$(DATABASE_URL)
|
||||||
|
- --action=migrate
|
||||||
|
- --log-level={{ .Values.migration.logLevel | default "info" }}
|
||||||
|
- --timeout={{ .Values.migration.timeout | default "5m" }}
|
||||||
|
securityContext:
|
||||||
|
allowPrivilegeEscalation: false
|
||||||
|
runAsNonRoot: true
|
||||||
|
runAsUser: 1000
|
||||||
|
resources:
|
||||||
|
{{- toYaml .Values.migration.resources | nindent 10 }}
|
||||||
|
{{- if eq .Values.metadata.backend "sqlite" }}
|
||||||
|
volumeMounts:
|
||||||
|
- name: metadata
|
||||||
|
mountPath: /var/lib/gohoarder/metadata
|
||||||
|
{{- end }}
|
||||||
|
{{- end }}
|
||||||
containers:
|
containers:
|
||||||
- name: server
|
- name: server
|
||||||
securityContext:
|
securityContext:
|
||||||
@@ -140,6 +196,29 @@ spec:
|
|||||||
name: {{ include "gohoarder.fullname" . }}-postgresql
|
name: {{ include "gohoarder.fullname" . }}-postgresql
|
||||||
key: password
|
key: password
|
||||||
{{- end }}
|
{{- end }}
|
||||||
|
{{- if and (or (eq .Values.metadata.backend "mysql") (eq .Values.metadata.backend "mariadb")) .Values.metadata.mysql.existingSecret }}
|
||||||
|
- name: MYSQL_USER
|
||||||
|
valueFrom:
|
||||||
|
secretKeyRef:
|
||||||
|
name: {{ .Values.metadata.mysql.existingSecret }}
|
||||||
|
key: username
|
||||||
|
- name: MYSQL_PASSWORD
|
||||||
|
valueFrom:
|
||||||
|
secretKeyRef:
|
||||||
|
name: {{ .Values.metadata.mysql.existingSecret }}
|
||||||
|
key: password
|
||||||
|
{{- else if and (or (eq .Values.metadata.backend "mysql") (eq .Values.metadata.backend "mariadb")) .Values.metadata.mysql.username }}
|
||||||
|
- name: MYSQL_USER
|
||||||
|
valueFrom:
|
||||||
|
secretKeyRef:
|
||||||
|
name: {{ include "gohoarder.fullname" . }}-mysql
|
||||||
|
key: username
|
||||||
|
- name: MYSQL_PASSWORD
|
||||||
|
valueFrom:
|
||||||
|
secretKeyRef:
|
||||||
|
name: {{ include "gohoarder.fullname" . }}-mysql
|
||||||
|
key: password
|
||||||
|
{{- end }}
|
||||||
{{- if and .Values.security.scanners.ghsa.enabled .Values.security.scanners.ghsa.existingSecret }}
|
{{- if and .Values.security.scanners.ghsa.enabled .Values.security.scanners.ghsa.existingSecret }}
|
||||||
- name: GHSA_TOKEN
|
- name: GHSA_TOKEN
|
||||||
valueFrom:
|
valueFrom:
|
||||||
|
|||||||
@@ -1,11 +1,10 @@
|
|||||||
{{- if .Values.ingress.enabled -}}
|
{{- if .Values.ingress.enabled -}}
|
||||||
{{- if .Values.ingress.frontend.enabled -}}
|
|
||||||
apiVersion: networking.k8s.io/v1
|
apiVersion: networking.k8s.io/v1
|
||||||
kind: Ingress
|
kind: Ingress
|
||||||
metadata:
|
metadata:
|
||||||
name: {{ include "gohoarder.fullname" . }}-frontend
|
name: {{ include "gohoarder.fullname" . }}
|
||||||
labels:
|
labels:
|
||||||
{{- include "gohoarder.frontend.labels" . | nindent 4 }}
|
{{- include "gohoarder.labels" . | nindent 4 }}
|
||||||
{{- with .Values.ingress.annotations }}
|
{{- with .Values.ingress.annotations }}
|
||||||
annotations:
|
annotations:
|
||||||
{{- toYaml . | nindent 4 }}
|
{{- toYaml . | nindent 4 }}
|
||||||
@@ -14,65 +13,17 @@ spec:
|
|||||||
{{- if .Values.ingress.className }}
|
{{- if .Values.ingress.className }}
|
||||||
ingressClassName: {{ .Values.ingress.className }}
|
ingressClassName: {{ .Values.ingress.className }}
|
||||||
{{- end }}
|
{{- end }}
|
||||||
{{- if .Values.ingress.frontend.tls.enabled }}
|
{{- if .Values.ingress.tls.enabled }}
|
||||||
tls:
|
tls:
|
||||||
- hosts:
|
- hosts:
|
||||||
- {{ .Values.ingress.frontend.host | default (printf "%s.%s" "gohoarder" .Values.global.domain) | quote }}
|
- {{ .Values.ingress.host | default (printf "%s.%s" "gohoarder" .Values.global.domain) | quote }}
|
||||||
secretName: {{ .Values.ingress.frontend.tls.secretName }}
|
secretName: {{ .Values.ingress.tls.secretName }}
|
||||||
{{- end }}
|
{{- end }}
|
||||||
rules:
|
rules:
|
||||||
- host: {{ .Values.ingress.frontend.host | default (printf "%s.%s" "gohoarder" .Values.global.domain) | quote }}
|
- host: {{ .Values.ingress.host | default (printf "%s.%s" "gohoarder" .Values.global.domain) | quote }}
|
||||||
http:
|
http:
|
||||||
paths:
|
paths:
|
||||||
- path: /npm
|
# Route all traffic to frontend (which now includes reverse proxy to backend)
|
||||||
pathType: Prefix
|
|
||||||
backend:
|
|
||||||
service:
|
|
||||||
name: {{ include "gohoarder.fullname" . }}-server
|
|
||||||
port:
|
|
||||||
number: {{ .Values.server.service.port }}
|
|
||||||
- path: /pypi
|
|
||||||
pathType: Prefix
|
|
||||||
backend:
|
|
||||||
service:
|
|
||||||
name: {{ include "gohoarder.fullname" . }}-server
|
|
||||||
port:
|
|
||||||
number: {{ .Values.server.service.port }}
|
|
||||||
- path: /go
|
|
||||||
pathType: Prefix
|
|
||||||
backend:
|
|
||||||
service:
|
|
||||||
name: {{ include "gohoarder.fullname" . }}-server
|
|
||||||
port:
|
|
||||||
number: {{ .Values.server.service.port }}
|
|
||||||
- path: /api
|
|
||||||
pathType: Prefix
|
|
||||||
backend:
|
|
||||||
service:
|
|
||||||
name: {{ include "gohoarder.fullname" . }}-server
|
|
||||||
port:
|
|
||||||
number: {{ .Values.server.service.port }}
|
|
||||||
- path: /ws
|
|
||||||
pathType: Prefix
|
|
||||||
backend:
|
|
||||||
service:
|
|
||||||
name: {{ include "gohoarder.fullname" . }}-server
|
|
||||||
port:
|
|
||||||
number: {{ .Values.server.service.port }}
|
|
||||||
- path: /health
|
|
||||||
pathType: Prefix
|
|
||||||
backend:
|
|
||||||
service:
|
|
||||||
name: {{ include "gohoarder.fullname" . }}-server
|
|
||||||
port:
|
|
||||||
number: {{ .Values.server.service.port }}
|
|
||||||
- path: /metrics
|
|
||||||
pathType: Prefix
|
|
||||||
backend:
|
|
||||||
service:
|
|
||||||
name: {{ include "gohoarder.fullname" . }}-server
|
|
||||||
port:
|
|
||||||
number: {{ .Values.server.service.port }}
|
|
||||||
- path: /
|
- path: /
|
||||||
pathType: Prefix
|
pathType: Prefix
|
||||||
backend:
|
backend:
|
||||||
@@ -81,38 +32,3 @@ spec:
|
|||||||
port:
|
port:
|
||||||
number: {{ .Values.frontend.service.port }}
|
number: {{ .Values.frontend.service.port }}
|
||||||
{{- end }}
|
{{- end }}
|
||||||
---
|
|
||||||
{{- if .Values.ingress.api.enabled }}
|
|
||||||
apiVersion: networking.k8s.io/v1
|
|
||||||
kind: Ingress
|
|
||||||
metadata:
|
|
||||||
name: {{ include "gohoarder.fullname" . }}-api
|
|
||||||
labels:
|
|
||||||
{{- include "gohoarder.server.labels" . | nindent 4 }}
|
|
||||||
{{- with .Values.ingress.annotations }}
|
|
||||||
annotations:
|
|
||||||
{{- toYaml . | nindent 4 }}
|
|
||||||
{{- end }}
|
|
||||||
spec:
|
|
||||||
{{- if .Values.ingress.className }}
|
|
||||||
ingressClassName: {{ .Values.ingress.className }}
|
|
||||||
{{- end }}
|
|
||||||
{{- if .Values.ingress.api.tls.enabled }}
|
|
||||||
tls:
|
|
||||||
- hosts:
|
|
||||||
- {{ .Values.ingress.api.host | default (printf "api.%s.%s" "gohoarder" .Values.global.domain) | quote }}
|
|
||||||
secretName: {{ .Values.ingress.api.tls.secretName }}
|
|
||||||
{{- end }}
|
|
||||||
rules:
|
|
||||||
- host: {{ .Values.ingress.api.host | default (printf "api.%s.%s" "gohoarder" .Values.global.domain) | quote }}
|
|
||||||
http:
|
|
||||||
paths:
|
|
||||||
- path: /
|
|
||||||
pathType: Prefix
|
|
||||||
backend:
|
|
||||||
service:
|
|
||||||
name: {{ include "gohoarder.fullname" . }}-server
|
|
||||||
port:
|
|
||||||
number: {{ .Values.server.service.port }}
|
|
||||||
{{- end }}
|
|
||||||
{{- end }}
|
|
||||||
|
|||||||
@@ -53,6 +53,19 @@ data:
|
|||||||
password: {{ .Values.metadata.postgresql.password | b64enc | quote }}
|
password: {{ .Values.metadata.postgresql.password | b64enc | quote }}
|
||||||
{{- end }}
|
{{- end }}
|
||||||
---
|
---
|
||||||
|
{{- if and (or (eq .Values.metadata.backend "mysql") (eq .Values.metadata.backend "mariadb")) (not .Values.metadata.mysql.existingSecret) .Values.metadata.mysql.username }}
|
||||||
|
apiVersion: v1
|
||||||
|
kind: Secret
|
||||||
|
metadata:
|
||||||
|
name: {{ include "gohoarder.fullname" . }}-mysql
|
||||||
|
labels:
|
||||||
|
{{- include "gohoarder.labels" . | nindent 4 }}
|
||||||
|
type: Opaque
|
||||||
|
data:
|
||||||
|
username: {{ .Values.metadata.mysql.username | b64enc | quote }}
|
||||||
|
password: {{ .Values.metadata.mysql.password | b64enc | quote }}
|
||||||
|
{{- end }}
|
||||||
|
---
|
||||||
{{- if and .Values.security.scanners.ghsa.enabled (not .Values.security.scanners.ghsa.existingSecret) .Values.security.scanners.ghsa.token }}
|
{{- if and .Values.security.scanners.ghsa.enabled (not .Values.security.scanners.ghsa.existingSecret) .Values.security.scanners.ghsa.token }}
|
||||||
apiVersion: v1
|
apiVersion: v1
|
||||||
kind: Secret
|
kind: Secret
|
||||||
|
|||||||
+84
-20
@@ -272,22 +272,48 @@ storage:
|
|||||||
|
|
||||||
# Metadata storage configuration
|
# Metadata storage configuration
|
||||||
metadata:
|
metadata:
|
||||||
# Backend: sqlite, postgresql
|
# Backend: sqlite, postgresql, mysql
|
||||||
# For multiple server replicas: postgresql is recommended (sqlite has concurrency limitations)
|
#
|
||||||
|
# IMPORTANT: SQLite CANNOT be used with SMB/CIFS/NFS network storage!
|
||||||
|
# SQLite requires POSIX file locking which causes "database is locked" errors on network filesystems.
|
||||||
|
#
|
||||||
|
# Choose your configuration:
|
||||||
|
# 1. SQLite with local storage (development/single-node only)
|
||||||
|
# - Set backend: sqlite
|
||||||
|
# - Set sqlite.persistence.storageClass to a LOCAL storage class (e.g., "local-path")
|
||||||
|
# - OR set sqlite.persistence.enabled: false to use emptyDir (data lost on pod restart)
|
||||||
|
#
|
||||||
|
# 2. PostgreSQL with any storage (RECOMMENDED for production)
|
||||||
|
# - Set backend: postgresql
|
||||||
|
# - Configure postgresql settings below
|
||||||
|
# - Works with any storage including SMB/NFS
|
||||||
|
# - Supports multiple replicas and high availability
|
||||||
|
#
|
||||||
|
# 3. MySQL/MariaDB with any storage (alternative to PostgreSQL)
|
||||||
|
# - Set backend: mysql
|
||||||
|
# - Configure mysql settings below
|
||||||
|
# - Works with any storage including SMB/NFS
|
||||||
|
#
|
||||||
backend: "sqlite"
|
backend: "sqlite"
|
||||||
|
|
||||||
# SQLite configuration
|
# SQLite configuration
|
||||||
|
# WARNING: Do NOT use SMB/CIFS/NFS storage classes with SQLite!
|
||||||
sqlite:
|
sqlite:
|
||||||
# Use PVC for SQLite database
|
# Use PVC for SQLite database
|
||||||
|
# IMPORTANT: storageClass must be LOCAL storage, NOT network storage (smb/nfs)
|
||||||
persistence:
|
persistence:
|
||||||
enabled: true
|
enabled: false # Changed to false by default - use emptyDir unless you have local storage
|
||||||
storageClass: ""
|
storageClass: "" # Must be local-path or similar LOCAL storage class if enabled
|
||||||
size: "10Gi"
|
size: "10Gi"
|
||||||
accessMode: "ReadWriteOnce"
|
accessMode: "ReadWriteOnce"
|
||||||
existingClaim: ""
|
existingClaim: ""
|
||||||
walMode: true
|
# WAL mode provides better concurrency but doesn't work on network filesystems (SMB, NFS)
|
||||||
|
# Set to false when using network storage for the metadata volume
|
||||||
|
walMode: false
|
||||||
|
|
||||||
# PostgreSQL configuration
|
# PostgreSQL configuration
|
||||||
|
# Works with any storage including SMB/NFS
|
||||||
|
# Recommended for production deployments
|
||||||
postgresql:
|
postgresql:
|
||||||
# Use bundled PostgreSQL (sets up postgresql subchart)
|
# Use bundled PostgreSQL (sets up postgresql subchart)
|
||||||
enabled: false
|
enabled: false
|
||||||
@@ -296,10 +322,57 @@ metadata:
|
|||||||
database: "gohoarder"
|
database: "gohoarder"
|
||||||
username: "gohoarder"
|
username: "gohoarder"
|
||||||
password: ""
|
password: ""
|
||||||
sslMode: "disable"
|
sslMode: "disable" # disable, require, verify-ca, verify-full
|
||||||
# Use existing secret for PostgreSQL credentials
|
# Use existing secret for PostgreSQL credentials
|
||||||
existingSecret: ""
|
existingSecret: ""
|
||||||
|
|
||||||
|
# MySQL/MariaDB configuration
|
||||||
|
# Works with any storage including SMB/NFS
|
||||||
|
# Alternative to PostgreSQL for production deployments
|
||||||
|
mysql:
|
||||||
|
host: "localhost"
|
||||||
|
port: 3306
|
||||||
|
database: "gohoarder"
|
||||||
|
username: "gohoarder"
|
||||||
|
password: ""
|
||||||
|
charset: "utf8mb4"
|
||||||
|
parseTime: true
|
||||||
|
# Use existing secret for MySQL credentials
|
||||||
|
existingSecret: ""
|
||||||
|
|
||||||
|
# GORM connection pool settings (applies to all database backends)
|
||||||
|
# These settings control database connection pooling and performance
|
||||||
|
maxOpenConns: 25 # Maximum number of open connections to the database
|
||||||
|
maxIdleConns: 5 # Maximum number of idle connections in the pool
|
||||||
|
connMaxLifetime: 3600 # Maximum lifetime of a connection in seconds (1 hour)
|
||||||
|
logLevel: "warn" # GORM log level: silent, error, warn, info
|
||||||
|
|
||||||
|
# Database migration configuration
|
||||||
|
migration:
|
||||||
|
# Enable automatic database migrations via init containers
|
||||||
|
# When enabled, each pod will run migrations before starting the main container
|
||||||
|
# Gormigrate handles concurrency automatically - safe for multiple pods
|
||||||
|
enabled: true
|
||||||
|
|
||||||
|
# Migration image configuration
|
||||||
|
image:
|
||||||
|
repository: ghcr.io/lukaszraczylo/gohoarder-migrate
|
||||||
|
pullPolicy: IfNotPresent
|
||||||
|
tag: "latest" # Should match the application version
|
||||||
|
|
||||||
|
# Migration settings
|
||||||
|
logLevel: "info" # debug, info, warn, error
|
||||||
|
timeout: "5m" # Maximum time for migrations to complete
|
||||||
|
|
||||||
|
# Resource limits for migration init container
|
||||||
|
resources:
|
||||||
|
limits:
|
||||||
|
cpu: 500m
|
||||||
|
memory: 256Mi
|
||||||
|
requests:
|
||||||
|
cpu: 100m
|
||||||
|
memory: 128Mi
|
||||||
|
|
||||||
# Cache configuration
|
# Cache configuration
|
||||||
cache:
|
cache:
|
||||||
defaultTTL: "168h" # 7 days
|
defaultTTL: "168h" # 7 days
|
||||||
@@ -434,21 +507,12 @@ ingress:
|
|||||||
nginx.ingress.kubernetes.io/proxy-read-timeout: "300"
|
nginx.ingress.kubernetes.io/proxy-read-timeout: "300"
|
||||||
nginx.ingress.kubernetes.io/proxy-send-timeout: "300"
|
nginx.ingress.kubernetes.io/proxy-send-timeout: "300"
|
||||||
|
|
||||||
# Ingress for frontend
|
# Single ingress routes all traffic to frontend
|
||||||
frontend:
|
# Frontend now includes reverse proxy to backend (merged gateway functionality)
|
||||||
enabled: true
|
host: "gohoarder.local"
|
||||||
host: "gohoarder.local"
|
tls:
|
||||||
tls:
|
|
||||||
enabled: false
|
|
||||||
secretName: "gohoarder-frontend-tls"
|
|
||||||
|
|
||||||
# Ingress for API (if you want separate ingress)
|
|
||||||
api:
|
|
||||||
enabled: false
|
enabled: false
|
||||||
host: "api.gohoarder.local"
|
secretName: "gohoarder-tls"
|
||||||
tls:
|
|
||||||
enabled: false
|
|
||||||
secretName: "gohoarder-api-tls"
|
|
||||||
|
|
||||||
# Autoscaling configuration
|
# Autoscaling configuration
|
||||||
autoscaling:
|
autoscaling:
|
||||||
|
|||||||
@@ -0,0 +1,367 @@
|
|||||||
|
-- GoHoarder Database Schema V2 - MySQL/MariaDB
|
||||||
|
-- Optimized for multi-user production deployments
|
||||||
|
-- Created: 2026-01-03
|
||||||
|
-- Requires: MySQL 8.0+ or MariaDB 10.5+
|
||||||
|
|
||||||
|
-- Set charset and collation
|
||||||
|
SET NAMES utf8mb4;
|
||||||
|
SET CHARACTER SET utf8mb4;
|
||||||
|
|
||||||
|
-- ============================================================================
|
||||||
|
-- TABLE: registries
|
||||||
|
-- Purpose: Normalized registry data (eliminates repeated strings)
|
||||||
|
-- ============================================================================
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS registries (
|
||||||
|
id INT UNSIGNED AUTO_INCREMENT PRIMARY KEY,
|
||||||
|
name VARCHAR(50) UNIQUE NOT NULL,
|
||||||
|
display_name VARCHAR(100) NOT NULL,
|
||||||
|
upstream_url VARCHAR(512) NOT NULL,
|
||||||
|
enabled BOOLEAN NOT NULL DEFAULT TRUE,
|
||||||
|
scan_by_default BOOLEAN NOT NULL DEFAULT TRUE,
|
||||||
|
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||||
|
updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
|
||||||
|
deleted_at TIMESTAMP NULL,
|
||||||
|
|
||||||
|
INDEX idx_registry_name (name),
|
||||||
|
INDEX idx_registry_enabled (enabled, deleted_at)
|
||||||
|
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
|
||||||
|
|
||||||
|
-- ============================================================================
|
||||||
|
-- TABLE: packages
|
||||||
|
-- Purpose: Core package metadata with denormalized counts for performance
|
||||||
|
-- ============================================================================
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS packages (
|
||||||
|
id BIGINT UNSIGNED AUTO_INCREMENT PRIMARY KEY,
|
||||||
|
registry_id INT UNSIGNED NOT NULL,
|
||||||
|
name VARCHAR(255) NOT NULL,
|
||||||
|
version VARCHAR(100) NOT NULL,
|
||||||
|
|
||||||
|
-- Storage information
|
||||||
|
storage_key VARCHAR(512) UNIQUE NOT NULL,
|
||||||
|
size BIGINT NOT NULL,
|
||||||
|
checksum_md5 CHAR(32),
|
||||||
|
checksum_sha256 CHAR(64),
|
||||||
|
upstream_url VARCHAR(1024),
|
||||||
|
|
||||||
|
-- Cache management
|
||||||
|
cached_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||||
|
last_accessed TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||||
|
expires_at TIMESTAMP NULL,
|
||||||
|
access_count BIGINT NOT NULL DEFAULT 0,
|
||||||
|
|
||||||
|
-- Security (denormalized for performance)
|
||||||
|
security_scanned BOOLEAN NOT NULL DEFAULT FALSE,
|
||||||
|
last_scanned_at TIMESTAMP NULL,
|
||||||
|
vulnerability_count INT NOT NULL DEFAULT 0,
|
||||||
|
highest_severity VARCHAR(20),
|
||||||
|
|
||||||
|
-- Authentication
|
||||||
|
requires_auth BOOLEAN NOT NULL DEFAULT FALSE,
|
||||||
|
auth_provider VARCHAR(50),
|
||||||
|
|
||||||
|
-- Audit trail
|
||||||
|
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||||
|
updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
|
||||||
|
deleted_at TIMESTAMP NULL,
|
||||||
|
|
||||||
|
FOREIGN KEY (registry_id) REFERENCES registries(id) ON DELETE RESTRICT ON UPDATE CASCADE,
|
||||||
|
|
||||||
|
UNIQUE INDEX idx_package_registry_name_version (registry_id, name, version, deleted_at),
|
||||||
|
INDEX idx_package_storage_key (storage_key),
|
||||||
|
INDEX idx_package_name (name(50)),
|
||||||
|
INDEX idx_package_last_accessed (last_accessed DESC),
|
||||||
|
INDEX idx_package_expires_at (expires_at),
|
||||||
|
INDEX idx_package_access_count (access_count DESC),
|
||||||
|
INDEX idx_package_size (size DESC),
|
||||||
|
INDEX idx_package_vuln_count (vulnerability_count),
|
||||||
|
INDEX idx_package_severity (highest_severity),
|
||||||
|
INDEX idx_package_security_scanned (security_scanned, deleted_at)
|
||||||
|
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
|
||||||
|
|
||||||
|
-- ============================================================================
|
||||||
|
-- TABLE: package_metadata
|
||||||
|
-- Purpose: Structured metadata (1:1 with packages)
|
||||||
|
-- ============================================================================
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS package_metadata (
|
||||||
|
package_id BIGINT UNSIGNED PRIMARY KEY,
|
||||||
|
author VARCHAR(255),
|
||||||
|
license VARCHAR(100),
|
||||||
|
homepage VARCHAR(512),
|
||||||
|
repository VARCHAR(512),
|
||||||
|
description TEXT,
|
||||||
|
keywords JSON, -- JSON array for MySQL 8.0+
|
||||||
|
raw_metadata JSON, -- Full metadata as JSON
|
||||||
|
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||||
|
updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
|
||||||
|
deleted_at TIMESTAMP NULL,
|
||||||
|
|
||||||
|
FOREIGN KEY (package_id) REFERENCES packages(id) ON DELETE CASCADE ON UPDATE CASCADE,
|
||||||
|
|
||||||
|
INDEX idx_metadata_author (author(100)),
|
||||||
|
INDEX idx_metadata_license (license)
|
||||||
|
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
|
||||||
|
|
||||||
|
-- ============================================================================
|
||||||
|
-- TABLE: vulnerabilities
|
||||||
|
-- Purpose: Normalized vulnerability data (each CVE stored once)
|
||||||
|
-- ============================================================================
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS vulnerabilities (
|
||||||
|
id BIGINT UNSIGNED AUTO_INCREMENT PRIMARY KEY,
|
||||||
|
cve_id VARCHAR(50) UNIQUE NOT NULL,
|
||||||
|
title VARCHAR(512) NOT NULL,
|
||||||
|
description TEXT,
|
||||||
|
severity VARCHAR(20) NOT NULL,
|
||||||
|
cvss FLOAT,
|
||||||
|
published_at TIMESTAMP NOT NULL,
|
||||||
|
fixed_version VARCHAR(100),
|
||||||
|
references JSON,
|
||||||
|
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||||
|
updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
|
||||||
|
deleted_at TIMESTAMP NULL,
|
||||||
|
|
||||||
|
UNIQUE INDEX idx_vuln_cve_id (cve_id),
|
||||||
|
INDEX idx_vuln_severity (severity),
|
||||||
|
INDEX idx_vuln_cvss (cvss DESC),
|
||||||
|
INDEX idx_vuln_published (published_at DESC)
|
||||||
|
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
|
||||||
|
|
||||||
|
-- ============================================================================
|
||||||
|
-- TABLE: package_vulnerabilities
|
||||||
|
-- Purpose: Many-to-many relationship between packages and vulnerabilities
|
||||||
|
-- ============================================================================
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS package_vulnerabilities (
|
||||||
|
id BIGINT UNSIGNED AUTO_INCREMENT PRIMARY KEY,
|
||||||
|
package_id BIGINT UNSIGNED NOT NULL,
|
||||||
|
vulnerability_id BIGINT UNSIGNED NOT NULL,
|
||||||
|
scanner VARCHAR(50) NOT NULL,
|
||||||
|
detected_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||||
|
bypassed BOOLEAN NOT NULL DEFAULT FALSE,
|
||||||
|
bypass_id BIGINT UNSIGNED,
|
||||||
|
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||||
|
updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
|
||||||
|
deleted_at TIMESTAMP NULL,
|
||||||
|
|
||||||
|
FOREIGN KEY (package_id) REFERENCES packages(id) ON DELETE CASCADE ON UPDATE CASCADE,
|
||||||
|
FOREIGN KEY (vulnerability_id) REFERENCES vulnerabilities(id) ON DELETE CASCADE ON UPDATE CASCADE,
|
||||||
|
|
||||||
|
INDEX idx_pkg_vuln_package (package_id, deleted_at),
|
||||||
|
INDEX idx_pkg_vuln_vuln (vulnerability_id, deleted_at),
|
||||||
|
INDEX idx_pkg_vuln_composite (package_id, vulnerability_id, deleted_at),
|
||||||
|
INDEX idx_pkg_vuln_scanner (scanner),
|
||||||
|
INDEX idx_pkg_vuln_bypassed (bypassed)
|
||||||
|
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
|
||||||
|
|
||||||
|
-- ============================================================================
|
||||||
|
-- TABLE: scan_results
|
||||||
|
-- Purpose: Security scan results with severity breakdown
|
||||||
|
-- ============================================================================
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS scan_results (
|
||||||
|
id BIGINT UNSIGNED AUTO_INCREMENT PRIMARY KEY,
|
||||||
|
package_id BIGINT UNSIGNED NOT NULL,
|
||||||
|
scanner VARCHAR(50) NOT NULL,
|
||||||
|
scanned_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||||
|
status VARCHAR(20) NOT NULL,
|
||||||
|
vuln_count INT NOT NULL DEFAULT 0,
|
||||||
|
critical_count INT NOT NULL DEFAULT 0,
|
||||||
|
high_count INT NOT NULL DEFAULT 0,
|
||||||
|
medium_count INT NOT NULL DEFAULT 0,
|
||||||
|
low_count INT NOT NULL DEFAULT 0,
|
||||||
|
scan_duration INT NOT NULL DEFAULT 0,
|
||||||
|
details JSON,
|
||||||
|
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||||
|
updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
|
||||||
|
deleted_at TIMESTAMP NULL,
|
||||||
|
|
||||||
|
FOREIGN KEY (package_id) REFERENCES packages(id) ON DELETE CASCADE ON UPDATE CASCADE,
|
||||||
|
|
||||||
|
INDEX idx_scan_package_scanner (package_id, scanner, deleted_at),
|
||||||
|
INDEX idx_scan_scanned_at (scanned_at DESC),
|
||||||
|
INDEX idx_scan_status (status),
|
||||||
|
INDEX idx_scan_vuln_count (vuln_count)
|
||||||
|
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
|
||||||
|
|
||||||
|
-- ============================================================================
|
||||||
|
-- TABLE: cve_bypasses
|
||||||
|
-- Purpose: CVE bypass rules with usage tracking
|
||||||
|
-- ============================================================================
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS cve_bypasses (
|
||||||
|
id BIGINT UNSIGNED AUTO_INCREMENT PRIMARY KEY,
|
||||||
|
type VARCHAR(20) NOT NULL,
|
||||||
|
target VARCHAR(512) NOT NULL,
|
||||||
|
reason TEXT NOT NULL,
|
||||||
|
created_by VARCHAR(255) NOT NULL,
|
||||||
|
expires_at TIMESTAMP NOT NULL,
|
||||||
|
notify_on_expiry BOOLEAN NOT NULL DEFAULT FALSE,
|
||||||
|
active BOOLEAN NOT NULL DEFAULT TRUE,
|
||||||
|
usage_count BIGINT NOT NULL DEFAULT 0,
|
||||||
|
last_used_at TIMESTAMP NULL,
|
||||||
|
registry_id INT UNSIGNED,
|
||||||
|
package_id BIGINT UNSIGNED,
|
||||||
|
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||||
|
updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
|
||||||
|
deleted_at TIMESTAMP NULL,
|
||||||
|
|
||||||
|
FOREIGN KEY (registry_id) REFERENCES registries(id) ON DELETE SET NULL ON UPDATE CASCADE,
|
||||||
|
FOREIGN KEY (package_id) REFERENCES packages(id) ON DELETE SET NULL ON UPDATE CASCADE,
|
||||||
|
|
||||||
|
INDEX idx_bypass_type (type),
|
||||||
|
INDEX idx_bypass_target (target(100)),
|
||||||
|
INDEX idx_bypass_active (active, deleted_at),
|
||||||
|
INDEX idx_bypass_expires_at (expires_at, active),
|
||||||
|
INDEX idx_bypass_created_by (created_by(100))
|
||||||
|
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
|
||||||
|
|
||||||
|
-- ============================================================================
|
||||||
|
-- TABLE: download_events
|
||||||
|
-- Purpose: High-volume time-series data
|
||||||
|
-- Note: MySQL doesn't support native partitioning as elegantly as PostgreSQL
|
||||||
|
-- Consider manual partitioning or TimescaleDB if needed
|
||||||
|
-- ============================================================================
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS download_events (
|
||||||
|
id BIGINT UNSIGNED AUTO_INCREMENT PRIMARY KEY,
|
||||||
|
package_id BIGINT UNSIGNED NOT NULL,
|
||||||
|
registry_id INT UNSIGNED NOT NULL,
|
||||||
|
downloaded_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||||
|
user_agent VARCHAR(512),
|
||||||
|
ip_address VARCHAR(45),
|
||||||
|
authenticated BOOLEAN NOT NULL DEFAULT FALSE,
|
||||||
|
username VARCHAR(255),
|
||||||
|
|
||||||
|
INDEX idx_download_events_package (package_id, downloaded_at),
|
||||||
|
INDEX idx_download_events_registry (registry_id),
|
||||||
|
INDEX idx_download_events_time (downloaded_at)
|
||||||
|
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
|
||||||
|
|
||||||
|
-- ============================================================================
|
||||||
|
-- TABLE: download_stats_hourly
|
||||||
|
-- Purpose: Pre-aggregated hourly statistics
|
||||||
|
-- ============================================================================
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS download_stats_hourly (
|
||||||
|
id BIGINT UNSIGNED AUTO_INCREMENT PRIMARY KEY,
|
||||||
|
registry_id INT UNSIGNED NOT NULL,
|
||||||
|
package_id BIGINT UNSIGNED,
|
||||||
|
time_bucket TIMESTAMP NOT NULL,
|
||||||
|
download_count BIGINT NOT NULL DEFAULT 0,
|
||||||
|
unique_ips BIGINT NOT NULL DEFAULT 0,
|
||||||
|
auth_downloads BIGINT NOT NULL DEFAULT 0,
|
||||||
|
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||||
|
updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
|
||||||
|
|
||||||
|
FOREIGN KEY (registry_id) REFERENCES registries(id) ON DELETE CASCADE ON UPDATE CASCADE,
|
||||||
|
FOREIGN KEY (package_id) REFERENCES packages(id) ON DELETE CASCADE ON UPDATE CASCADE,
|
||||||
|
|
||||||
|
UNIQUE INDEX idx_stats_hourly_composite (registry_id, IFNULL(package_id, 0), time_bucket),
|
||||||
|
INDEX idx_stats_hourly_time (time_bucket DESC)
|
||||||
|
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
|
||||||
|
|
||||||
|
-- ============================================================================
|
||||||
|
-- TABLE: download_stats_daily
|
||||||
|
-- Purpose: Pre-aggregated daily statistics
|
||||||
|
-- ============================================================================
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS download_stats_daily (
|
||||||
|
id BIGINT UNSIGNED AUTO_INCREMENT PRIMARY KEY,
|
||||||
|
registry_id INT UNSIGNED NOT NULL,
|
||||||
|
package_id BIGINT UNSIGNED,
|
||||||
|
time_bucket TIMESTAMP NOT NULL,
|
||||||
|
download_count BIGINT NOT NULL DEFAULT 0,
|
||||||
|
unique_ips BIGINT NOT NULL DEFAULT 0,
|
||||||
|
auth_downloads BIGINT NOT NULL DEFAULT 0,
|
||||||
|
top_user_agents JSON,
|
||||||
|
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||||
|
updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
|
||||||
|
|
||||||
|
FOREIGN KEY (registry_id) REFERENCES registries(id) ON DELETE CASCADE ON UPDATE CASCADE,
|
||||||
|
FOREIGN KEY (package_id) REFERENCES packages(id) ON DELETE CASCADE ON UPDATE CASCADE,
|
||||||
|
|
||||||
|
UNIQUE INDEX idx_stats_daily_composite (registry_id, IFNULL(package_id, 0), time_bucket),
|
||||||
|
INDEX idx_stats_daily_time (time_bucket DESC)
|
||||||
|
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
|
||||||
|
|
||||||
|
-- ============================================================================
|
||||||
|
-- TABLE: audit_log
|
||||||
|
-- Purpose: Audit trail for compliance
|
||||||
|
-- ============================================================================
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS audit_log (
|
||||||
|
id BIGINT UNSIGNED AUTO_INCREMENT PRIMARY KEY,
|
||||||
|
entity_type VARCHAR(50) NOT NULL,
|
||||||
|
entity_id BIGINT NOT NULL,
|
||||||
|
action VARCHAR(20) NOT NULL,
|
||||||
|
username VARCHAR(255) NOT NULL,
|
||||||
|
timestamp TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||||
|
changes JSON,
|
||||||
|
ip_address VARCHAR(45),
|
||||||
|
user_agent VARCHAR(512),
|
||||||
|
|
||||||
|
INDEX idx_audit_log_entity (entity_type, entity_id),
|
||||||
|
INDEX idx_audit_log_username (username(100)),
|
||||||
|
INDEX idx_audit_log_timestamp (timestamp DESC)
|
||||||
|
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
|
||||||
|
|
||||||
|
-- ============================================================================
|
||||||
|
-- SEED DATA: Default registries
|
||||||
|
-- ============================================================================
|
||||||
|
|
||||||
|
INSERT INTO registries (name, display_name, upstream_url, enabled, scan_by_default) VALUES
|
||||||
|
('npm', 'NPM Registry', 'https://registry.npmjs.org', TRUE, TRUE),
|
||||||
|
('pypi', 'PyPI', 'https://pypi.org', TRUE, TRUE),
|
||||||
|
('go', 'Go Modules', 'https://proxy.golang.org', TRUE, TRUE)
|
||||||
|
ON DUPLICATE KEY UPDATE
|
||||||
|
display_name = VALUES(display_name),
|
||||||
|
upstream_url = VALUES(upstream_url);
|
||||||
|
|
||||||
|
-- ============================================================================
|
||||||
|
-- VIEWS: Convenience views for common queries
|
||||||
|
-- ============================================================================
|
||||||
|
|
||||||
|
CREATE OR REPLACE VIEW v_vulnerable_packages AS
|
||||||
|
SELECT
|
||||||
|
r.name AS registry,
|
||||||
|
p.name,
|
||||||
|
p.version,
|
||||||
|
p.vulnerability_count,
|
||||||
|
p.highest_severity,
|
||||||
|
p.last_scanned_at
|
||||||
|
FROM packages p
|
||||||
|
JOIN registries r ON p.registry_id = r.id
|
||||||
|
WHERE p.vulnerability_count > 0 AND p.deleted_at IS NULL
|
||||||
|
ORDER BY
|
||||||
|
CASE p.highest_severity
|
||||||
|
WHEN 'critical' THEN 1
|
||||||
|
WHEN 'high' THEN 2
|
||||||
|
WHEN 'medium' THEN 3
|
||||||
|
WHEN 'low' THEN 4
|
||||||
|
ELSE 5
|
||||||
|
END,
|
||||||
|
p.vulnerability_count DESC;
|
||||||
|
|
||||||
|
-- ============================================================================
|
||||||
|
-- PERFORMANCE TUNING RECOMMENDATIONS
|
||||||
|
-- ============================================================================
|
||||||
|
|
||||||
|
-- Set InnoDB buffer pool size to 50-70% of RAM
|
||||||
|
-- SET GLOBAL innodb_buffer_pool_size = 4294967296; -- 4GB
|
||||||
|
|
||||||
|
-- Enable query cache (MySQL 5.7 and earlier)
|
||||||
|
-- SET GLOBAL query_cache_type = 1;
|
||||||
|
-- SET GLOBAL query_cache_size = 67108864; -- 64MB
|
||||||
|
|
||||||
|
-- Optimize for SSD
|
||||||
|
-- SET GLOBAL innodb_flush_log_at_trx_commit = 2;
|
||||||
|
-- SET GLOBAL innodb_io_capacity = 2000;
|
||||||
|
|
||||||
|
-- ============================================================================
|
||||||
|
-- COMPLETE
|
||||||
|
-- ============================================================================
|
||||||
|
|
||||||
|
SELECT 'Schema V2 created successfully for MySQL/MariaDB!' AS status;
|
||||||
@@ -0,0 +1,470 @@
|
|||||||
|
-- GoHoarder Database Schema V2 - PostgreSQL
|
||||||
|
-- Optimized for multi-user production deployments
|
||||||
|
-- Created: 2026-01-03
|
||||||
|
|
||||||
|
-- Enable required extensions
|
||||||
|
CREATE EXTENSION IF NOT EXISTS "uuid-ossp";
|
||||||
|
|
||||||
|
-- ============================================================================
|
||||||
|
-- TABLE: registries
|
||||||
|
-- Purpose: Normalized registry data (eliminates repeated strings)
|
||||||
|
-- ============================================================================
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS registries (
|
||||||
|
id SERIAL PRIMARY KEY,
|
||||||
|
name VARCHAR(50) UNIQUE NOT NULL,
|
||||||
|
display_name VARCHAR(100) NOT NULL,
|
||||||
|
upstream_url VARCHAR(512) NOT NULL,
|
||||||
|
enabled BOOLEAN NOT NULL DEFAULT TRUE,
|
||||||
|
scan_by_default BOOLEAN NOT NULL DEFAULT TRUE,
|
||||||
|
created_at TIMESTAMP NOT NULL DEFAULT NOW(),
|
||||||
|
updated_at TIMESTAMP NOT NULL DEFAULT NOW(),
|
||||||
|
deleted_at TIMESTAMP
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE INDEX idx_registry_name ON registries(name) WHERE deleted_at IS NULL;
|
||||||
|
CREATE INDEX idx_registry_enabled ON registries(enabled) WHERE enabled = TRUE AND deleted_at IS NULL;
|
||||||
|
|
||||||
|
COMMENT ON TABLE registries IS 'Normalized registry data (npm, pypi, go)';
|
||||||
|
COMMENT ON COLUMN registries.name IS 'Short name: npm, pypi, go';
|
||||||
|
COMMENT ON COLUMN registries.display_name IS 'Human-readable name: NPM Registry, PyPI';
|
||||||
|
|
||||||
|
-- ============================================================================
|
||||||
|
-- TABLE: packages
|
||||||
|
-- Purpose: Core package metadata with denormalized counts for performance
|
||||||
|
-- ============================================================================
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS packages (
|
||||||
|
id BIGSERIAL PRIMARY KEY,
|
||||||
|
registry_id INTEGER NOT NULL REFERENCES registries(id) ON DELETE RESTRICT,
|
||||||
|
name VARCHAR(255) NOT NULL,
|
||||||
|
version VARCHAR(100) NOT NULL,
|
||||||
|
|
||||||
|
-- Storage information
|
||||||
|
storage_key VARCHAR(512) UNIQUE NOT NULL,
|
||||||
|
size BIGINT NOT NULL,
|
||||||
|
checksum_md5 VARCHAR(32),
|
||||||
|
checksum_sha256 VARCHAR(64),
|
||||||
|
upstream_url VARCHAR(1024),
|
||||||
|
|
||||||
|
-- Cache management
|
||||||
|
cached_at TIMESTAMP NOT NULL DEFAULT NOW(),
|
||||||
|
last_accessed TIMESTAMP NOT NULL DEFAULT NOW(),
|
||||||
|
expires_at TIMESTAMP,
|
||||||
|
access_count BIGINT NOT NULL DEFAULT 0,
|
||||||
|
|
||||||
|
-- Security (denormalized for performance)
|
||||||
|
security_scanned BOOLEAN NOT NULL DEFAULT FALSE,
|
||||||
|
last_scanned_at TIMESTAMP,
|
||||||
|
vulnerability_count INTEGER NOT NULL DEFAULT 0,
|
||||||
|
highest_severity VARCHAR(20), -- critical, high, medium, low, none
|
||||||
|
|
||||||
|
-- Authentication
|
||||||
|
requires_auth BOOLEAN NOT NULL DEFAULT FALSE,
|
||||||
|
auth_provider VARCHAR(50),
|
||||||
|
|
||||||
|
-- Audit trail
|
||||||
|
created_at TIMESTAMP NOT NULL DEFAULT NOW(),
|
||||||
|
updated_at TIMESTAMP NOT NULL DEFAULT NOW(),
|
||||||
|
deleted_at TIMESTAMP
|
||||||
|
);
|
||||||
|
|
||||||
|
-- Composite indexes for common queries
|
||||||
|
CREATE UNIQUE INDEX idx_package_registry_name_version
|
||||||
|
ON packages(registry_id, name, version) WHERE deleted_at IS NULL;
|
||||||
|
|
||||||
|
CREATE INDEX idx_package_storage_key ON packages(storage_key);
|
||||||
|
CREATE INDEX idx_package_name ON packages(name text_pattern_ops) WHERE deleted_at IS NULL;
|
||||||
|
CREATE INDEX idx_package_last_accessed ON packages(last_accessed DESC) WHERE deleted_at IS NULL;
|
||||||
|
CREATE INDEX idx_package_expires_at ON packages(expires_at) WHERE expires_at IS NOT NULL AND deleted_at IS NULL;
|
||||||
|
CREATE INDEX idx_package_access_count ON packages(access_count DESC) WHERE deleted_at IS NULL;
|
||||||
|
CREATE INDEX idx_package_size ON packages(size DESC);
|
||||||
|
|
||||||
|
-- Partial indexes for security queries
|
||||||
|
CREATE INDEX idx_package_vuln_count ON packages(vulnerability_count) WHERE vulnerability_count > 0 AND deleted_at IS NULL;
|
||||||
|
CREATE INDEX idx_package_severity ON packages(highest_severity) WHERE highest_severity IN ('critical', 'high') AND deleted_at IS NULL;
|
||||||
|
CREATE INDEX idx_package_security_scanned ON packages(security_scanned) WHERE deleted_at IS NULL;
|
||||||
|
|
||||||
|
COMMENT ON TABLE packages IS 'Core package metadata (optimized V2 schema)';
|
||||||
|
COMMENT ON COLUMN packages.access_count IS 'Total downloads (denormalized from stats)';
|
||||||
|
COMMENT ON COLUMN packages.vulnerability_count IS 'Number of vulnerabilities (denormalized)';
|
||||||
|
|
||||||
|
-- ============================================================================
|
||||||
|
-- TABLE: package_metadata
|
||||||
|
-- Purpose: Structured metadata (1:1 with packages, reduces main table size)
|
||||||
|
-- ============================================================================
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS package_metadata (
|
||||||
|
package_id BIGINT PRIMARY KEY REFERENCES packages(id) ON DELETE CASCADE,
|
||||||
|
author VARCHAR(255),
|
||||||
|
license VARCHAR(100),
|
||||||
|
homepage VARCHAR(512),
|
||||||
|
repository VARCHAR(512),
|
||||||
|
description TEXT,
|
||||||
|
keywords JSONB, -- Array of keywords
|
||||||
|
raw_metadata JSONB, -- Full metadata
|
||||||
|
created_at TIMESTAMP NOT NULL DEFAULT NOW(),
|
||||||
|
updated_at TIMESTAMP NOT NULL DEFAULT NOW(),
|
||||||
|
deleted_at TIMESTAMP
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE INDEX idx_metadata_author ON package_metadata(author);
|
||||||
|
CREATE INDEX idx_metadata_license ON package_metadata(license);
|
||||||
|
CREATE INDEX idx_metadata_keywords ON package_metadata USING GIN(keywords);
|
||||||
|
CREATE INDEX idx_metadata_raw ON package_metadata USING GIN(raw_metadata);
|
||||||
|
|
||||||
|
COMMENT ON TABLE package_metadata IS 'Structured package metadata (separated for performance)';
|
||||||
|
|
||||||
|
-- ============================================================================
|
||||||
|
-- TABLE: vulnerabilities
|
||||||
|
-- Purpose: Normalized vulnerability data (each CVE stored once)
|
||||||
|
-- ============================================================================
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS vulnerabilities (
|
||||||
|
id BIGSERIAL PRIMARY KEY,
|
||||||
|
cve_id VARCHAR(50) UNIQUE NOT NULL,
|
||||||
|
title VARCHAR(512) NOT NULL,
|
||||||
|
description TEXT,
|
||||||
|
severity VARCHAR(20) NOT NULL, -- critical, high, medium, low
|
||||||
|
cvss REAL,
|
||||||
|
published_at TIMESTAMP NOT NULL,
|
||||||
|
fixed_version VARCHAR(100),
|
||||||
|
references JSONB, -- Array of URLs
|
||||||
|
created_at TIMESTAMP NOT NULL DEFAULT NOW(),
|
||||||
|
updated_at TIMESTAMP NOT NULL DEFAULT NOW(),
|
||||||
|
deleted_at TIMESTAMP
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE UNIQUE INDEX idx_vuln_cve_id ON vulnerabilities(cve_id);
|
||||||
|
CREATE INDEX idx_vuln_severity ON vulnerabilities(severity);
|
||||||
|
CREATE INDEX idx_vuln_cvss ON vulnerabilities(cvss DESC NULLS LAST);
|
||||||
|
CREATE INDEX idx_vuln_published ON vulnerabilities(published_at DESC);
|
||||||
|
|
||||||
|
COMMENT ON TABLE vulnerabilities IS 'Normalized vulnerability data (99% storage reduction)';
|
||||||
|
|
||||||
|
-- ============================================================================
|
||||||
|
-- TABLE: package_vulnerabilities
|
||||||
|
-- Purpose: Many-to-many relationship between packages and vulnerabilities
|
||||||
|
-- ============================================================================
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS package_vulnerabilities (
|
||||||
|
id BIGSERIAL PRIMARY KEY,
|
||||||
|
package_id BIGINT NOT NULL REFERENCES packages(id) ON DELETE CASCADE,
|
||||||
|
vulnerability_id BIGINT NOT NULL REFERENCES vulnerabilities(id) ON DELETE CASCADE,
|
||||||
|
scanner VARCHAR(50) NOT NULL,
|
||||||
|
detected_at TIMESTAMP NOT NULL DEFAULT NOW(),
|
||||||
|
bypassed BOOLEAN NOT NULL DEFAULT FALSE,
|
||||||
|
bypass_id BIGINT, -- References cve_bypasses.id (soft reference)
|
||||||
|
created_at TIMESTAMP NOT NULL DEFAULT NOW(),
|
||||||
|
updated_at TIMESTAMP NOT NULL DEFAULT NOW(),
|
||||||
|
deleted_at TIMESTAMP
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE INDEX idx_pkg_vuln_package ON package_vulnerabilities(package_id) WHERE deleted_at IS NULL;
|
||||||
|
CREATE INDEX idx_pkg_vuln_vuln ON package_vulnerabilities(vulnerability_id) WHERE deleted_at IS NULL;
|
||||||
|
CREATE INDEX idx_pkg_vuln_composite ON package_vulnerabilities(package_id, vulnerability_id) WHERE deleted_at IS NULL;
|
||||||
|
CREATE INDEX idx_pkg_vuln_scanner ON package_vulnerabilities(scanner);
|
||||||
|
CREATE INDEX idx_pkg_vuln_bypassed ON package_vulnerabilities(bypassed) WHERE bypassed = FALSE;
|
||||||
|
|
||||||
|
-- ============================================================================
|
||||||
|
-- TABLE: scan_results
|
||||||
|
-- Purpose: Security scan results with severity breakdown
|
||||||
|
-- ============================================================================
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS scan_results (
|
||||||
|
id BIGSERIAL PRIMARY KEY,
|
||||||
|
package_id BIGINT NOT NULL REFERENCES packages(id) ON DELETE CASCADE,
|
||||||
|
scanner VARCHAR(50) NOT NULL,
|
||||||
|
scanned_at TIMESTAMP NOT NULL DEFAULT NOW(),
|
||||||
|
status VARCHAR(20) NOT NULL, -- success, failed, pending
|
||||||
|
vuln_count INTEGER NOT NULL DEFAULT 0,
|
||||||
|
critical_count INTEGER NOT NULL DEFAULT 0,
|
||||||
|
high_count INTEGER NOT NULL DEFAULT 0,
|
||||||
|
medium_count INTEGER NOT NULL DEFAULT 0,
|
||||||
|
low_count INTEGER NOT NULL DEFAULT 0,
|
||||||
|
scan_duration INTEGER NOT NULL DEFAULT 0, -- milliseconds
|
||||||
|
details JSONB,
|
||||||
|
created_at TIMESTAMP NOT NULL DEFAULT NOW(),
|
||||||
|
updated_at TIMESTAMP NOT NULL DEFAULT NOW(),
|
||||||
|
deleted_at TIMESTAMP
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE INDEX idx_scan_package_scanner ON scan_results(package_id, scanner) WHERE deleted_at IS NULL;
|
||||||
|
CREATE INDEX idx_scan_scanned_at ON scan_results(scanned_at DESC);
|
||||||
|
CREATE INDEX idx_scan_status ON scan_results(status);
|
||||||
|
CREATE INDEX idx_scan_vuln_count ON scan_results(vuln_count) WHERE vuln_count > 0;
|
||||||
|
|
||||||
|
COMMENT ON TABLE scan_results IS 'Security scan results (optimized V2)';
|
||||||
|
|
||||||
|
-- ============================================================================
|
||||||
|
-- TABLE: cve_bypasses
|
||||||
|
-- Purpose: CVE bypass rules with usage tracking
|
||||||
|
-- ============================================================================
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS cve_bypasses (
|
||||||
|
id BIGSERIAL PRIMARY KEY,
|
||||||
|
type VARCHAR(20) NOT NULL, -- cve, package, registry
|
||||||
|
target VARCHAR(512) NOT NULL,
|
||||||
|
reason TEXT NOT NULL,
|
||||||
|
created_by VARCHAR(255) NOT NULL,
|
||||||
|
expires_at TIMESTAMP NOT NULL,
|
||||||
|
notify_on_expiry BOOLEAN NOT NULL DEFAULT FALSE,
|
||||||
|
active BOOLEAN NOT NULL DEFAULT TRUE,
|
||||||
|
usage_count BIGINT NOT NULL DEFAULT 0,
|
||||||
|
last_used_at TIMESTAMP,
|
||||||
|
registry_id INTEGER REFERENCES registries(id),
|
||||||
|
package_id BIGINT REFERENCES packages(id),
|
||||||
|
created_at TIMESTAMP NOT NULL DEFAULT NOW(),
|
||||||
|
updated_at TIMESTAMP NOT NULL DEFAULT NOW(),
|
||||||
|
deleted_at TIMESTAMP
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE INDEX idx_bypass_type ON cve_bypasses(type);
|
||||||
|
CREATE INDEX idx_bypass_target ON cve_bypasses(target);
|
||||||
|
CREATE INDEX idx_bypass_active ON cve_bypasses(active) WHERE active = TRUE AND deleted_at IS NULL;
|
||||||
|
CREATE INDEX idx_bypass_expires_at ON cve_bypasses(expires_at) WHERE active = TRUE;
|
||||||
|
CREATE INDEX idx_bypass_created_by ON cve_bypasses(created_by);
|
||||||
|
|
||||||
|
COMMENT ON TABLE cve_bypasses IS 'CVE bypass rules with scope limiting';
|
||||||
|
|
||||||
|
-- ============================================================================
|
||||||
|
-- PARTITIONED TABLE: download_events
|
||||||
|
-- Purpose: High-volume time-series data (partitioned by month)
|
||||||
|
-- ============================================================================
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS download_events (
|
||||||
|
id BIGSERIAL,
|
||||||
|
package_id BIGINT NOT NULL,
|
||||||
|
registry_id INTEGER NOT NULL,
|
||||||
|
downloaded_at TIMESTAMP NOT NULL,
|
||||||
|
user_agent VARCHAR(512),
|
||||||
|
ip_address VARCHAR(45),
|
||||||
|
authenticated BOOLEAN NOT NULL DEFAULT FALSE,
|
||||||
|
username VARCHAR(255)
|
||||||
|
) PARTITION BY RANGE (downloaded_at);
|
||||||
|
|
||||||
|
CREATE INDEX idx_download_events_package ON download_events(package_id, downloaded_at);
|
||||||
|
CREATE INDEX idx_download_events_registry ON download_events(registry_id);
|
||||||
|
CREATE INDEX idx_download_events_time ON download_events(downloaded_at);
|
||||||
|
|
||||||
|
COMMENT ON TABLE download_events IS 'Download events (partitioned by month for performance)';
|
||||||
|
|
||||||
|
-- Create partitions for current month ± 2 months
|
||||||
|
DO $$
|
||||||
|
DECLARE
|
||||||
|
start_date DATE;
|
||||||
|
end_date DATE;
|
||||||
|
partition_name TEXT;
|
||||||
|
i INTEGER;
|
||||||
|
BEGIN
|
||||||
|
FOR i IN -2..2 LOOP
|
||||||
|
start_date := date_trunc('month', NOW() + (i || ' months')::INTERVAL)::DATE;
|
||||||
|
end_date := (start_date + INTERVAL '1 month')::DATE;
|
||||||
|
partition_name := 'download_events_' || to_char(start_date, 'YYYY_MM');
|
||||||
|
|
||||||
|
EXECUTE format('CREATE TABLE IF NOT EXISTS %I PARTITION OF download_events FOR VALUES FROM (%L) TO (%L)',
|
||||||
|
partition_name, start_date, end_date);
|
||||||
|
|
||||||
|
EXECUTE format('CREATE INDEX IF NOT EXISTS %I ON %I(package_id, downloaded_at)',
|
||||||
|
partition_name || '_package_idx', partition_name);
|
||||||
|
EXECUTE format('CREATE INDEX IF NOT EXISTS %I ON %I(registry_id)',
|
||||||
|
partition_name || '_registry_idx', partition_name);
|
||||||
|
END LOOP;
|
||||||
|
END $$;
|
||||||
|
|
||||||
|
-- ============================================================================
|
||||||
|
-- TABLE: download_stats_hourly
|
||||||
|
-- Purpose: Pre-aggregated hourly statistics (1000x faster queries)
|
||||||
|
-- ============================================================================
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS download_stats_hourly (
|
||||||
|
id BIGSERIAL PRIMARY KEY,
|
||||||
|
registry_id INTEGER NOT NULL REFERENCES registries(id),
|
||||||
|
package_id BIGINT REFERENCES packages(id), -- NULL = all packages in registry
|
||||||
|
time_bucket TIMESTAMP NOT NULL,
|
||||||
|
download_count BIGINT NOT NULL DEFAULT 0,
|
||||||
|
unique_ips BIGINT NOT NULL DEFAULT 0,
|
||||||
|
auth_downloads BIGINT NOT NULL DEFAULT 0,
|
||||||
|
created_at TIMESTAMP NOT NULL DEFAULT NOW(),
|
||||||
|
updated_at TIMESTAMP NOT NULL DEFAULT NOW()
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE UNIQUE INDEX idx_stats_hourly_composite
|
||||||
|
ON download_stats_hourly(registry_id, COALESCE(package_id, 0), time_bucket);
|
||||||
|
CREATE INDEX idx_stats_hourly_time ON download_stats_hourly(time_bucket DESC);
|
||||||
|
|
||||||
|
COMMENT ON TABLE download_stats_hourly IS 'Hourly aggregated stats (pre-computed)';
|
||||||
|
|
||||||
|
-- ============================================================================
|
||||||
|
-- TABLE: download_stats_daily
|
||||||
|
-- Purpose: Pre-aggregated daily statistics with analytics
|
||||||
|
-- ============================================================================
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS download_stats_daily (
|
||||||
|
id BIGSERIAL PRIMARY KEY,
|
||||||
|
registry_id INTEGER NOT NULL REFERENCES registries(id),
|
||||||
|
package_id BIGINT REFERENCES packages(id),
|
||||||
|
time_bucket TIMESTAMP NOT NULL,
|
||||||
|
download_count BIGINT NOT NULL DEFAULT 0,
|
||||||
|
unique_ips BIGINT NOT NULL DEFAULT 0,
|
||||||
|
auth_downloads BIGINT NOT NULL DEFAULT 0,
|
||||||
|
top_user_agents JSONB,
|
||||||
|
created_at TIMESTAMP NOT NULL DEFAULT NOW(),
|
||||||
|
updated_at TIMESTAMP NOT NULL DEFAULT NOW()
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE UNIQUE INDEX idx_stats_daily_composite
|
||||||
|
ON download_stats_daily(registry_id, COALESCE(package_id, 0), time_bucket);
|
||||||
|
CREATE INDEX idx_stats_daily_time ON download_stats_daily(time_bucket DESC);
|
||||||
|
|
||||||
|
COMMENT ON TABLE download_stats_daily IS 'Daily aggregated stats with analytics';
|
||||||
|
|
||||||
|
-- ============================================================================
|
||||||
|
-- PARTITIONED TABLE: audit_log
|
||||||
|
-- Purpose: Audit trail for compliance (partitioned by month)
|
||||||
|
-- ============================================================================
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS audit_log (
|
||||||
|
id BIGSERIAL,
|
||||||
|
entity_type VARCHAR(50) NOT NULL,
|
||||||
|
entity_id BIGINT NOT NULL,
|
||||||
|
action VARCHAR(20) NOT NULL, -- create, update, delete
|
||||||
|
username VARCHAR(255) NOT NULL,
|
||||||
|
timestamp TIMESTAMP NOT NULL DEFAULT NOW(),
|
||||||
|
changes JSONB,
|
||||||
|
ip_address VARCHAR(45),
|
||||||
|
user_agent VARCHAR(512)
|
||||||
|
) PARTITION BY RANGE (timestamp);
|
||||||
|
|
||||||
|
CREATE INDEX idx_audit_log_entity ON audit_log(entity_type, entity_id);
|
||||||
|
CREATE INDEX idx_audit_log_username ON audit_log(username);
|
||||||
|
CREATE INDEX idx_audit_log_timestamp ON audit_log(timestamp DESC);
|
||||||
|
|
||||||
|
COMMENT ON TABLE audit_log IS 'Audit trail for compliance and debugging';
|
||||||
|
|
||||||
|
-- Create audit_log partitions
|
||||||
|
DO $$
|
||||||
|
DECLARE
|
||||||
|
start_date DATE;
|
||||||
|
end_date DATE;
|
||||||
|
partition_name TEXT;
|
||||||
|
i INTEGER;
|
||||||
|
BEGIN
|
||||||
|
FOR i IN -1..2 LOOP
|
||||||
|
start_date := date_trunc('month', NOW() + (i || ' months')::INTERVAL)::DATE;
|
||||||
|
end_date := (start_date + INTERVAL '1 month')::DATE;
|
||||||
|
partition_name := 'audit_log_' || to_char(start_date, 'YYYY_MM');
|
||||||
|
|
||||||
|
EXECUTE format('CREATE TABLE IF NOT EXISTS %I PARTITION OF audit_log FOR VALUES FROM (%L) TO (%L)',
|
||||||
|
partition_name, start_date, end_date);
|
||||||
|
|
||||||
|
EXECUTE format('CREATE INDEX IF NOT EXISTS %I ON %I(entity_type, entity_id)',
|
||||||
|
partition_name || '_entity_idx', partition_name);
|
||||||
|
EXECUTE format('CREATE INDEX IF NOT EXISTS %I ON %I(username)',
|
||||||
|
partition_name || '_user_idx', partition_name);
|
||||||
|
END LOOP;
|
||||||
|
END $$;
|
||||||
|
|
||||||
|
-- ============================================================================
|
||||||
|
-- FUNCTIONS: Automatic partition creation
|
||||||
|
-- ============================================================================
|
||||||
|
|
||||||
|
CREATE OR REPLACE FUNCTION create_next_month_partitions()
|
||||||
|
RETURNS void AS $$
|
||||||
|
DECLARE
|
||||||
|
next_month DATE := date_trunc('month', NOW() + INTERVAL '2 months');
|
||||||
|
partition_name TEXT;
|
||||||
|
start_date TEXT;
|
||||||
|
end_date TEXT;
|
||||||
|
BEGIN
|
||||||
|
-- Download events partition
|
||||||
|
partition_name := 'download_events_' || to_char(next_month, 'YYYY_MM');
|
||||||
|
start_date := to_char(next_month, 'YYYY-MM-DD');
|
||||||
|
end_date := to_char(next_month + INTERVAL '1 month', 'YYYY-MM-DD');
|
||||||
|
|
||||||
|
EXECUTE format('CREATE TABLE IF NOT EXISTS %I PARTITION OF download_events FOR VALUES FROM (%L) TO (%L)',
|
||||||
|
partition_name, start_date, end_date);
|
||||||
|
|
||||||
|
EXECUTE format('CREATE INDEX IF NOT EXISTS %I ON %I(package_id, downloaded_at)',
|
||||||
|
partition_name || '_package_idx', partition_name);
|
||||||
|
EXECUTE format('CREATE INDEX IF NOT EXISTS %I ON %I(registry_id)',
|
||||||
|
partition_name || '_registry_idx', partition_name);
|
||||||
|
|
||||||
|
-- Audit log partition
|
||||||
|
partition_name := 'audit_log_' || to_char(next_month, 'YYYY_MM');
|
||||||
|
|
||||||
|
EXECUTE format('CREATE TABLE IF NOT EXISTS %I PARTITION OF audit_log FOR VALUES FROM (%L) TO (%L)',
|
||||||
|
partition_name, start_date, end_date);
|
||||||
|
|
||||||
|
EXECUTE format('CREATE INDEX IF NOT EXISTS %I ON %I(entity_type, entity_id)',
|
||||||
|
partition_name || '_entity_idx', partition_name);
|
||||||
|
|
||||||
|
RAISE NOTICE 'Created partitions for %', to_char(next_month, 'YYYY-MM');
|
||||||
|
END;
|
||||||
|
$$ LANGUAGE plpgsql;
|
||||||
|
|
||||||
|
COMMENT ON FUNCTION create_next_month_partitions() IS 'Auto-create partitions for next month';
|
||||||
|
|
||||||
|
-- ============================================================================
|
||||||
|
-- TRIGGERS: Updated_at timestamp
|
||||||
|
-- ============================================================================
|
||||||
|
|
||||||
|
CREATE OR REPLACE FUNCTION update_updated_at_column()
|
||||||
|
RETURNS TRIGGER AS $$
|
||||||
|
BEGIN
|
||||||
|
NEW.updated_at = NOW();
|
||||||
|
RETURN NEW;
|
||||||
|
END;
|
||||||
|
$$ LANGUAGE plpgsql;
|
||||||
|
|
||||||
|
CREATE TRIGGER update_registries_updated_at BEFORE UPDATE ON registries
|
||||||
|
FOR EACH ROW EXECUTE FUNCTION update_updated_at_column();
|
||||||
|
|
||||||
|
CREATE TRIGGER update_packages_updated_at BEFORE UPDATE ON packages
|
||||||
|
FOR EACH ROW EXECUTE FUNCTION update_updated_at_column();
|
||||||
|
|
||||||
|
CREATE TRIGGER update_package_metadata_updated_at BEFORE UPDATE ON package_metadata
|
||||||
|
FOR EACH ROW EXECUTE FUNCTION update_updated_at_column();
|
||||||
|
|
||||||
|
-- ============================================================================
|
||||||
|
-- SEED DATA: Default registries
|
||||||
|
-- ============================================================================
|
||||||
|
|
||||||
|
INSERT INTO registries (name, display_name, upstream_url, enabled, scan_by_default) VALUES
|
||||||
|
('npm', 'NPM Registry', 'https://registry.npmjs.org', TRUE, TRUE),
|
||||||
|
('pypi', 'PyPI', 'https://pypi.org', TRUE, TRUE),
|
||||||
|
('go', 'Go Modules', 'https://proxy.golang.org', TRUE, TRUE)
|
||||||
|
ON CONFLICT (name) DO NOTHING;
|
||||||
|
|
||||||
|
-- ============================================================================
|
||||||
|
-- VIEWS: Convenience views for common queries
|
||||||
|
-- ============================================================================
|
||||||
|
|
||||||
|
CREATE OR REPLACE VIEW v_vulnerable_packages AS
|
||||||
|
SELECT
|
||||||
|
r.name AS registry,
|
||||||
|
p.name,
|
||||||
|
p.version,
|
||||||
|
p.vulnerability_count,
|
||||||
|
p.highest_severity,
|
||||||
|
p.last_scanned_at
|
||||||
|
FROM packages p
|
||||||
|
JOIN registries r ON p.registry_id = r.id
|
||||||
|
WHERE p.vulnerability_count > 0 AND p.deleted_at IS NULL
|
||||||
|
ORDER BY
|
||||||
|
CASE p.highest_severity
|
||||||
|
WHEN 'critical' THEN 1
|
||||||
|
WHEN 'high' THEN 2
|
||||||
|
WHEN 'medium' THEN 3
|
||||||
|
WHEN 'low' THEN 4
|
||||||
|
ELSE 5
|
||||||
|
END,
|
||||||
|
p.vulnerability_count DESC;
|
||||||
|
|
||||||
|
COMMENT ON VIEW v_vulnerable_packages IS 'All packages with vulnerabilities (sorted by severity)';
|
||||||
|
|
||||||
|
-- ============================================================================
|
||||||
|
-- COMPLETE
|
||||||
|
-- ============================================================================
|
||||||
|
|
||||||
|
SELECT 'Schema V2 created successfully!' AS status;
|
||||||
@@ -10,23 +10,23 @@ import (
|
|||||||
|
|
||||||
// PackageDownload represents a package download event
|
// PackageDownload represents a package download event
|
||||||
type PackageDownload struct {
|
type PackageDownload struct {
|
||||||
|
Timestamp time.Time
|
||||||
Registry string
|
Registry string
|
||||||
Name string
|
Name string
|
||||||
Version string
|
Version string
|
||||||
Timestamp time.Time
|
|
||||||
BytesSize int64
|
|
||||||
ClientIP string
|
ClientIP string
|
||||||
UserAgent string
|
UserAgent string
|
||||||
|
BytesSize int64
|
||||||
}
|
}
|
||||||
|
|
||||||
// PackageStats holds statistics for a package
|
// PackageStats holds statistics for a package
|
||||||
type PackageStats struct {
|
type PackageStats struct {
|
||||||
|
LastDownload time.Time
|
||||||
|
FirstSeen time.Time
|
||||||
Registry string
|
Registry string
|
||||||
Name string
|
Name string
|
||||||
TotalDownloads int64
|
TotalDownloads int64
|
||||||
UniqueVersions int
|
UniqueVersions int
|
||||||
LastDownload time.Time
|
|
||||||
FirstSeen time.Time
|
|
||||||
BytesServed int64
|
BytesServed int64
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -48,13 +48,13 @@ type PopularPackage struct {
|
|||||||
|
|
||||||
// Engine tracks and analyzes package downloads
|
// Engine tracks and analyzes package downloads
|
||||||
type Engine struct {
|
type Engine struct {
|
||||||
downloads []PackageDownload
|
stats map[string]*PackageStats
|
||||||
downloadsMu sync.RWMutex
|
|
||||||
stats map[string]*PackageStats // key: registry:name
|
|
||||||
statsMu sync.RWMutex
|
|
||||||
maxEvents int
|
|
||||||
flushTicker *time.Ticker
|
flushTicker *time.Ticker
|
||||||
stopChan chan struct{}
|
stopChan chan struct{}
|
||||||
|
downloads []PackageDownload
|
||||||
|
maxEvents int
|
||||||
|
downloadsMu sync.RWMutex
|
||||||
|
statsMu sync.RWMutex
|
||||||
}
|
}
|
||||||
|
|
||||||
// Config holds analytics engine configuration
|
// Config holds analytics engine configuration
|
||||||
|
|||||||
+140
-27
@@ -17,10 +17,9 @@ import (
|
|||||||
"github.com/lukaszraczylo/gohoarder/pkg/cdn"
|
"github.com/lukaszraczylo/gohoarder/pkg/cdn"
|
||||||
"github.com/lukaszraczylo/gohoarder/pkg/config"
|
"github.com/lukaszraczylo/gohoarder/pkg/config"
|
||||||
"github.com/lukaszraczylo/gohoarder/pkg/health"
|
"github.com/lukaszraczylo/gohoarder/pkg/health"
|
||||||
"github.com/lukaszraczylo/gohoarder/pkg/lock"
|
|
||||||
"github.com/lukaszraczylo/gohoarder/pkg/metadata"
|
"github.com/lukaszraczylo/gohoarder/pkg/metadata"
|
||||||
metafile "github.com/lukaszraczylo/gohoarder/pkg/metadata/file"
|
metafile "github.com/lukaszraczylo/gohoarder/pkg/metadata/file"
|
||||||
metasqlite "github.com/lukaszraczylo/gohoarder/pkg/metadata/sqlite"
|
metagorm "github.com/lukaszraczylo/gohoarder/pkg/metadata/gormstore"
|
||||||
"github.com/lukaszraczylo/gohoarder/pkg/metrics"
|
"github.com/lukaszraczylo/gohoarder/pkg/metrics"
|
||||||
"github.com/lukaszraczylo/gohoarder/pkg/network"
|
"github.com/lukaszraczylo/gohoarder/pkg/network"
|
||||||
"github.com/lukaszraczylo/gohoarder/pkg/prewarming"
|
"github.com/lukaszraczylo/gohoarder/pkg/prewarming"
|
||||||
@@ -30,6 +29,8 @@ import (
|
|||||||
"github.com/lukaszraczylo/gohoarder/pkg/scanner"
|
"github.com/lukaszraczylo/gohoarder/pkg/scanner"
|
||||||
"github.com/lukaszraczylo/gohoarder/pkg/storage"
|
"github.com/lukaszraczylo/gohoarder/pkg/storage"
|
||||||
"github.com/lukaszraczylo/gohoarder/pkg/storage/filesystem"
|
"github.com/lukaszraczylo/gohoarder/pkg/storage/filesystem"
|
||||||
|
"github.com/lukaszraczylo/gohoarder/pkg/storage/s3"
|
||||||
|
"github.com/lukaszraczylo/gohoarder/pkg/storage/smb"
|
||||||
"github.com/lukaszraczylo/gohoarder/pkg/vcs"
|
"github.com/lukaszraczylo/gohoarder/pkg/vcs"
|
||||||
"github.com/lukaszraczylo/gohoarder/pkg/websocket"
|
"github.com/lukaszraczylo/gohoarder/pkg/websocket"
|
||||||
"github.com/rs/zerolog/log"
|
"github.com/rs/zerolog/log"
|
||||||
@@ -50,7 +51,6 @@ type App struct {
|
|||||||
analyticsEngine *analytics.Engine
|
analyticsEngine *analytics.Engine
|
||||||
wsServer *websocket.Server
|
wsServer *websocket.Server
|
||||||
prewarmWorker *prewarming.Worker
|
prewarmWorker *prewarming.Worker
|
||||||
lockManager *lock.Manager
|
|
||||||
cdnMiddleware *cdn.Middleware
|
cdnMiddleware *cdn.Middleware
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -82,7 +82,33 @@ func (a *App) initializeComponents() error {
|
|||||||
switch a.config.Storage.Backend {
|
switch a.config.Storage.Backend {
|
||||||
case "filesystem":
|
case "filesystem":
|
||||||
a.storage, err = filesystem.New(a.config.Storage.Path, a.config.Cache.MaxSizeBytes)
|
a.storage, err = filesystem.New(a.config.Storage.Path, a.config.Cache.MaxSizeBytes)
|
||||||
|
case "s3":
|
||||||
|
a.storage, err = s3.New(s3.Config{
|
||||||
|
Region: a.config.Storage.S3.Region,
|
||||||
|
Bucket: a.config.Storage.S3.Bucket,
|
||||||
|
Prefix: a.config.Storage.S3.Prefix,
|
||||||
|
AccessKeyID: a.config.Storage.S3.AccessKeyID,
|
||||||
|
SecretAccessKey: a.config.Storage.S3.SecretAccessKey,
|
||||||
|
Endpoint: a.config.Storage.S3.Endpoint,
|
||||||
|
ForcePathStyle: a.config.Storage.S3.ForcePathStyle,
|
||||||
|
MaxSizeBytes: a.config.Cache.MaxSizeBytes,
|
||||||
|
})
|
||||||
|
case "smb":
|
||||||
|
a.storage, err = smb.New(smb.Config{
|
||||||
|
Host: a.config.Storage.SMB.Host,
|
||||||
|
Port: 445, // Default SMB port
|
||||||
|
Share: a.config.Storage.SMB.Share,
|
||||||
|
Path: a.config.Storage.Path,
|
||||||
|
Username: a.config.Storage.SMB.Username,
|
||||||
|
Password: a.config.Storage.SMB.Password,
|
||||||
|
Domain: a.config.Storage.SMB.Domain,
|
||||||
|
MaxSizeBytes: a.config.Cache.MaxSizeBytes,
|
||||||
|
PoolSize: 5, // Default connection pool size
|
||||||
|
})
|
||||||
default:
|
default:
|
||||||
|
log.Warn().
|
||||||
|
Str("backend", a.config.Storage.Backend).
|
||||||
|
Msg("Unknown storage backend, defaulting to filesystem")
|
||||||
a.storage, err = filesystem.New(a.config.Storage.Path, a.config.Cache.MaxSizeBytes)
|
a.storage, err = filesystem.New(a.config.Storage.Path, a.config.Cache.MaxSizeBytes)
|
||||||
}
|
}
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -93,16 +119,67 @@ func (a *App) initializeComponents() error {
|
|||||||
log.Info().Str("backend", a.config.Metadata.Backend).Msg("Initializing metadata store")
|
log.Info().Str("backend", a.config.Metadata.Backend).Msg("Initializing metadata store")
|
||||||
switch a.config.Metadata.Backend {
|
switch a.config.Metadata.Backend {
|
||||||
case "sqlite":
|
case "sqlite":
|
||||||
a.metadata, err = metasqlite.New(metasqlite.Config{
|
// Use GORM for SQLite
|
||||||
Path: a.config.Metadata.Connection,
|
a.metadata, err = metagorm.NewV2(metagorm.Config{
|
||||||
|
Driver: "sqlite",
|
||||||
|
DSN: metagorm.BuildSQLiteDSN(a.config.Metadata.SQLite.Path, a.config.Metadata.SQLite.WALMode),
|
||||||
|
MaxOpenConns: getOrDefault(a.config.Metadata.MaxOpenConns, 25),
|
||||||
|
MaxIdleConns: getOrDefault(a.config.Metadata.MaxIdleConns, 5),
|
||||||
|
ConnMaxLifetime: time.Duration(getOrDefault(a.config.Metadata.ConnMaxLifetime, 3600)) * time.Second,
|
||||||
|
LogLevel: getOrDefaultStr(a.config.Metadata.LogLevel, "warn"),
|
||||||
})
|
})
|
||||||
|
|
||||||
|
case "postgresql", "postgres":
|
||||||
|
// Use GORM for PostgreSQL
|
||||||
|
dsn := metagorm.BuildPostgresDSN(
|
||||||
|
a.config.Metadata.PostgreSQL.Host,
|
||||||
|
a.config.Metadata.PostgreSQL.Port,
|
||||||
|
a.config.Metadata.PostgreSQL.User,
|
||||||
|
a.config.Metadata.PostgreSQL.Password,
|
||||||
|
a.config.Metadata.PostgreSQL.Database,
|
||||||
|
getOrDefaultStr(a.config.Metadata.PostgreSQL.SSLMode, "disable"),
|
||||||
|
)
|
||||||
|
a.metadata, err = metagorm.NewV2(metagorm.Config{
|
||||||
|
Driver: "postgres",
|
||||||
|
DSN: dsn,
|
||||||
|
MaxOpenConns: getOrDefault(a.config.Metadata.MaxOpenConns, 25),
|
||||||
|
MaxIdleConns: getOrDefault(a.config.Metadata.MaxIdleConns, 5),
|
||||||
|
ConnMaxLifetime: time.Duration(getOrDefault(a.config.Metadata.ConnMaxLifetime, 3600)) * time.Second,
|
||||||
|
LogLevel: getOrDefaultStr(a.config.Metadata.LogLevel, "warn"),
|
||||||
|
})
|
||||||
|
|
||||||
|
case "mysql", "mariadb":
|
||||||
|
// Use GORM for MySQL/MariaDB
|
||||||
|
dsn := metagorm.BuildMySQLDSN(
|
||||||
|
a.config.Metadata.MySQL.Host,
|
||||||
|
a.config.Metadata.MySQL.Port,
|
||||||
|
a.config.Metadata.MySQL.User,
|
||||||
|
a.config.Metadata.MySQL.Password,
|
||||||
|
a.config.Metadata.MySQL.Database,
|
||||||
|
getOrDefaultStr(a.config.Metadata.MySQL.Charset, "utf8mb4"),
|
||||||
|
)
|
||||||
|
a.metadata, err = metagorm.NewV2(metagorm.Config{
|
||||||
|
Driver: "mysql",
|
||||||
|
DSN: dsn,
|
||||||
|
MaxOpenConns: getOrDefault(a.config.Metadata.MaxOpenConns, 25),
|
||||||
|
MaxIdleConns: getOrDefault(a.config.Metadata.MaxIdleConns, 5),
|
||||||
|
ConnMaxLifetime: time.Duration(getOrDefault(a.config.Metadata.ConnMaxLifetime, 3600)) * time.Second,
|
||||||
|
LogLevel: getOrDefaultStr(a.config.Metadata.LogLevel, "warn"),
|
||||||
|
})
|
||||||
|
|
||||||
case "file":
|
case "file":
|
||||||
|
// Keep file backend as-is for file-based metadata
|
||||||
a.metadata, err = metafile.New(metafile.Config{
|
a.metadata, err = metafile.New(metafile.Config{
|
||||||
Path: a.config.Metadata.Connection,
|
Path: a.config.Metadata.Connection,
|
||||||
})
|
})
|
||||||
|
|
||||||
default:
|
default:
|
||||||
a.metadata, err = metasqlite.New(metasqlite.Config{
|
// Default to SQLite with GORM
|
||||||
Path: "gohoarder.db",
|
log.Warn().Str("backend", a.config.Metadata.Backend).Msg("Unknown metadata backend, defaulting to SQLite with GORM")
|
||||||
|
a.metadata, err = metagorm.NewV2(metagorm.Config{
|
||||||
|
Driver: "sqlite",
|
||||||
|
DSN: metagorm.BuildSQLiteDSN("gohoarder.db", false),
|
||||||
|
LogLevel: "warn",
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -116,9 +193,16 @@ func (a *App) initializeComponents() error {
|
|||||||
return fmt.Errorf("failed to initialize scanner: %w", err)
|
return fmt.Errorf("failed to initialize scanner: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Initialize cache manager with scanner
|
// Initialize analytics engine first (needed by cache)
|
||||||
|
log.Info().Msg("Initializing analytics engine")
|
||||||
|
a.analyticsEngine = analytics.NewEngine(analytics.Config{
|
||||||
|
MaxEvents: 10000,
|
||||||
|
FlushInterval: 5 * time.Minute,
|
||||||
|
})
|
||||||
|
|
||||||
|
// Initialize cache manager with scanner and analytics
|
||||||
log.Info().Msg("Initializing cache manager")
|
log.Info().Msg("Initializing cache manager")
|
||||||
a.cache, err = cache.New(a.storage, a.metadata, a.scanManager, cache.Config{
|
a.cache, err = cache.New(a.storage, a.metadata, a.scanManager, a.analyticsEngine, cache.Config{
|
||||||
DefaultTTL: a.config.Cache.DefaultTTL,
|
DefaultTTL: a.config.Cache.DefaultTTL,
|
||||||
CleanupInterval: 5 * time.Minute,
|
CleanupInterval: 5 * time.Minute,
|
||||||
})
|
})
|
||||||
@@ -153,13 +237,6 @@ func (a *App) initializeComponents() error {
|
|||||||
a.rescanWorker = scanner.NewRescanWorker(a.scanManager, a.metadata, a.storage, a.config.Security.RescanInterval)
|
a.rescanWorker = scanner.NewRescanWorker(a.scanManager, a.metadata, a.storage, a.config.Security.RescanInterval)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Initialize analytics engine
|
|
||||||
log.Info().Msg("Initializing analytics engine")
|
|
||||||
a.analyticsEngine = analytics.NewEngine(analytics.Config{
|
|
||||||
MaxEvents: 10000,
|
|
||||||
FlushInterval: 5 * time.Minute,
|
|
||||||
})
|
|
||||||
|
|
||||||
// Initialize WebSocket server
|
// Initialize WebSocket server
|
||||||
log.Info().Msg("Initializing WebSocket server")
|
log.Info().Msg("Initializing WebSocket server")
|
||||||
a.wsServer = websocket.NewServer(websocket.Config{
|
a.wsServer = websocket.NewServer(websocket.Config{
|
||||||
@@ -213,7 +290,9 @@ func (a *App) initializeComponents() error {
|
|||||||
a.healthChecker.AddCheck("scanner", func(ctx context.Context) (health.Status, string) {
|
a.healthChecker.AddCheck("scanner", func(ctx context.Context) (health.Status, string) {
|
||||||
if a.config.Security.Enabled {
|
if a.config.Security.Enabled {
|
||||||
if err := a.scanManager.Health(ctx); err != nil {
|
if err := a.scanManager.Health(ctx); err != nil {
|
||||||
return health.StatusUnhealthy, err.Error()
|
// Scanner failures (e.g., API rate limits) shouldn't mark server as unhealthy
|
||||||
|
// Server can still serve cached packages, just can't scan new ones
|
||||||
|
return health.StatusDegraded, err.Error()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return health.StatusHealthy, ""
|
return health.StatusHealthy, ""
|
||||||
@@ -248,9 +327,28 @@ func (a *App) setupServer() error {
|
|||||||
a.app.Get("/api/stats/timeseries", a.handleTimeSeriesStats)
|
a.app.Get("/api/stats/timeseries", a.handleTimeSeriesStats)
|
||||||
a.app.Get("/api/info", a.handleInfo)
|
a.app.Get("/api/info", a.handleInfo)
|
||||||
|
|
||||||
|
// Analytics endpoints
|
||||||
|
a.app.Get("/api/analytics/top", a.handleAnalyticsTopPackages)
|
||||||
|
a.app.Get("/api/analytics/trending", a.handleAnalyticsTrendingPackages)
|
||||||
|
a.app.Get("/api/analytics/trends", a.handleAnalyticsTrends)
|
||||||
|
a.app.Get("/api/analytics/total", a.handleAnalyticsTotalStats)
|
||||||
|
a.app.Get("/api/analytics/registry/:registry", a.handleAnalyticsRegistryStats)
|
||||||
|
a.app.Get("/api/analytics/package/:registry/:name", a.handleAnalyticsPackageStats)
|
||||||
|
a.app.Get("/api/analytics/search", a.handleAnalyticsSearch)
|
||||||
|
|
||||||
// Admin endpoints (bypass management)
|
// Admin endpoints (bypass management)
|
||||||
a.app.All("/api/admin/bypasses/:id?", a.requireAdmin, a.handleAdminBypasses)
|
a.app.All("/api/admin/bypasses/:id?", a.requireAdmin, a.handleAdminBypasses)
|
||||||
|
|
||||||
|
// Admin endpoints (pre-warming)
|
||||||
|
a.app.Get("/api/admin/prewarming/status", a.requireAdmin, a.handlePrewarmingStatus)
|
||||||
|
a.app.Post("/api/admin/prewarming/trigger", a.requireAdmin, a.handlePrewarmingTrigger)
|
||||||
|
a.app.Post("/api/admin/prewarming/package", a.requireAdmin, a.handlePrewarmingPackage)
|
||||||
|
|
||||||
|
// Admin endpoints (API key management)
|
||||||
|
a.app.Post("/api/admin/keys", a.requireAdmin, a.handleGenerateAPIKey)
|
||||||
|
a.app.Get("/api/admin/keys", a.requireAdmin, a.handleListAPIKeys)
|
||||||
|
a.app.Delete("/api/admin/keys/:key_id", a.requireAdmin, a.handleRevokeAPIKey)
|
||||||
|
|
||||||
// Proxy handlers (adapted from net/http)
|
// Proxy handlers (adapted from net/http)
|
||||||
// Load git credentials if configured
|
// Load git credentials if configured
|
||||||
var credStore *vcs.CredentialStore
|
var credStore *vcs.CredentialStore
|
||||||
@@ -270,22 +368,28 @@ func (a *App) setupServer() error {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Go proxy with CDN caching
|
||||||
goProxyHandler := goproxy.New(a.cache, a.networkClient, goproxy.Config{
|
goProxyHandler := goproxy.New(a.cache, a.networkClient, goproxy.Config{
|
||||||
Upstream: "https://proxy.golang.org",
|
Upstream: "https://proxy.golang.org",
|
||||||
SumDBURL: "https://sum.golang.org",
|
SumDBURL: "https://sum.golang.org",
|
||||||
CredStore: credStore,
|
CredStore: credStore,
|
||||||
})
|
})
|
||||||
a.app.All("/go/*", adaptor.HTTPHandler(http.StripPrefix("/go", goProxyHandler)))
|
goProxyWithCDN := a.cdnMiddleware.Handler(http.StripPrefix("/go", goProxyHandler))
|
||||||
|
a.app.All("/go/*", adaptor.HTTPHandler(goProxyWithCDN))
|
||||||
|
|
||||||
|
// NPM proxy with CDN caching
|
||||||
npmProxyHandler := npm.New(a.cache, a.networkClient, npm.Config{
|
npmProxyHandler := npm.New(a.cache, a.networkClient, npm.Config{
|
||||||
Upstream: "https://registry.npmjs.org",
|
Upstream: "https://registry.npmjs.org",
|
||||||
})
|
})
|
||||||
a.app.All("/npm/*", adaptor.HTTPHandler(http.StripPrefix("/npm", npmProxyHandler)))
|
npmProxyWithCDN := a.cdnMiddleware.Handler(http.StripPrefix("/npm", npmProxyHandler))
|
||||||
|
a.app.All("/npm/*", adaptor.HTTPHandler(npmProxyWithCDN))
|
||||||
|
|
||||||
|
// PyPI proxy with CDN caching
|
||||||
pypiProxyHandler := pypi.New(a.cache, a.networkClient, pypi.Config{
|
pypiProxyHandler := pypi.New(a.cache, a.networkClient, pypi.Config{
|
||||||
Upstream: "https://pypi.org/simple",
|
Upstream: "https://pypi.org/simple",
|
||||||
})
|
})
|
||||||
a.app.All("/pypi/*", adaptor.HTTPHandler(http.StripPrefix("/pypi", pypiProxyHandler)))
|
pypiProxyWithCDN := a.cdnMiddleware.Handler(http.StripPrefix("/pypi", pypiProxyHandler))
|
||||||
|
a.app.All("/pypi/*", adaptor.HTTPHandler(pypiProxyWithCDN))
|
||||||
|
|
||||||
// Serve frontend static files
|
// Serve frontend static files
|
||||||
frontendDir := "frontend/dist"
|
frontendDir := "frontend/dist"
|
||||||
@@ -397,13 +501,6 @@ func (a *App) Shutdown() error {
|
|||||||
log.Error().Err(err).Msg("Error closing metadata store")
|
log.Error().Err(err).Msg("Error closing metadata store")
|
||||||
}
|
}
|
||||||
|
|
||||||
// Close lock manager if initialized
|
|
||||||
if a.lockManager != nil {
|
|
||||||
if err := a.lockManager.Close(); err != nil {
|
|
||||||
log.Error().Err(err).Msg("Error closing lock manager")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
log.Info().Msg("Shutdown complete")
|
log.Info().Msg("Shutdown complete")
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
@@ -433,3 +530,19 @@ func (a *App) startAggregationWorker(ctx context.Context) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// getOrDefault returns the value if it's non-zero, otherwise returns the default
|
||||||
|
func getOrDefault(value, defaultValue int) int {
|
||||||
|
if value == 0 {
|
||||||
|
return defaultValue
|
||||||
|
}
|
||||||
|
return value
|
||||||
|
}
|
||||||
|
|
||||||
|
// getOrDefaultStr returns the value if it's non-empty, otherwise returns the default
|
||||||
|
func getOrDefaultStr(value, defaultValue string) string {
|
||||||
|
if value == "" {
|
||||||
|
return defaultValue
|
||||||
|
}
|
||||||
|
return value
|
||||||
|
}
|
||||||
|
|||||||
+31
-11
@@ -142,6 +142,13 @@ func (a *App) handleListPackages(c *fiber.Ctx) error {
|
|||||||
severityCounts[strings.ToUpper(vuln.Severity)]++
|
severityCounts[strings.ToUpper(vuln.Severity)]++
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Check if package should be blocked based on thresholds
|
||||||
|
isBlocked := false
|
||||||
|
if a.scanManager != nil {
|
||||||
|
blocked, _, _ := a.scanManager.CheckVulnerabilities(ctx, pkg.Registry, entry.originalName, pkg.Version)
|
||||||
|
isBlocked = blocked
|
||||||
|
}
|
||||||
|
|
||||||
pkgMap["vulnerabilities"] = map[string]interface{}{
|
pkgMap["vulnerabilities"] = map[string]interface{}{
|
||||||
"scanned": true,
|
"scanned": true,
|
||||||
"status": scanResult.Status,
|
"status": scanResult.Status,
|
||||||
@@ -152,18 +159,21 @@ func (a *App) handleListPackages(c *fiber.Ctx) error {
|
|||||||
"moderate": severityCounts["MODERATE"],
|
"moderate": severityCounts["MODERATE"],
|
||||||
"low": severityCounts["LOW"],
|
"low": severityCounts["LOW"],
|
||||||
},
|
},
|
||||||
"total": scanResult.VulnerabilityCount,
|
"total": scanResult.VulnerabilityCount,
|
||||||
|
"isBlocked": isBlocked,
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
pkgMap["vulnerabilities"] = map[string]interface{}{
|
pkgMap["vulnerabilities"] = map[string]interface{}{
|
||||||
"scanned": false,
|
"scanned": false,
|
||||||
"status": "pending",
|
"status": "pending",
|
||||||
|
"isBlocked": false,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
pkgMap["vulnerabilities"] = map[string]interface{}{
|
pkgMap["vulnerabilities"] = map[string]interface{}{
|
||||||
"scanned": false,
|
"scanned": false,
|
||||||
"status": "not_scanned",
|
"status": "not_scanned",
|
||||||
|
"isBlocked": false,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -351,8 +361,9 @@ func (a *App) handleStats(c *fiber.Ctx) error {
|
|||||||
packages = []*metadata.Package{}
|
packages = []*metadata.Package{}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Calculate per-registry breakdown (exclude metadata entries like "list", "latest")
|
// Calculate per-registry breakdown and blocked packages count
|
||||||
registryStats := make(map[string]map[string]interface{})
|
registryStats := make(map[string]map[string]interface{})
|
||||||
|
blockedCount := int64(0)
|
||||||
|
|
||||||
for _, pkg := range packages {
|
for _, pkg := range packages {
|
||||||
// Skip metadata entries (npm metadata pages, pypi pages, etc.)
|
// Skip metadata entries (npm metadata pages, pypi pages, etc.)
|
||||||
@@ -371,6 +382,14 @@ func (a *App) handleStats(c *fiber.Ctx) error {
|
|||||||
registryStats[pkg.Registry]["count"] = registryStats[pkg.Registry]["count"].(int) + 1
|
registryStats[pkg.Registry]["count"] = registryStats[pkg.Registry]["count"].(int) + 1
|
||||||
registryStats[pkg.Registry]["size"] = registryStats[pkg.Registry]["size"].(int64) + pkg.Size
|
registryStats[pkg.Registry]["size"] = registryStats[pkg.Registry]["size"].(int64) + pkg.Size
|
||||||
registryStats[pkg.Registry]["downloads"] = registryStats[pkg.Registry]["downloads"].(int64) + int64(pkg.DownloadCount)
|
registryStats[pkg.Registry]["downloads"] = registryStats[pkg.Registry]["downloads"].(int64) + int64(pkg.DownloadCount)
|
||||||
|
|
||||||
|
// Check if package is blocked (only if security scanning is enabled and package is scanned)
|
||||||
|
if a.config.Security.Enabled && a.scanManager != nil && pkg.SecurityScanned {
|
||||||
|
blocked, _, _ := a.scanManager.CheckVulnerabilities(ctx, pkg.Registry, pkg.Name, pkg.Version)
|
||||||
|
if blocked {
|
||||||
|
blockedCount++
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Combine statistics using database stats for accuracy
|
// Combine statistics using database stats for accuracy
|
||||||
@@ -378,12 +397,14 @@ func (a *App) handleStats(c *fiber.Ctx) error {
|
|||||||
"total_packages": cacheStats.TotalPackages,
|
"total_packages": cacheStats.TotalPackages,
|
||||||
"total_downloads": cacheStats.TotalDownloads,
|
"total_downloads": cacheStats.TotalDownloads,
|
||||||
"total_size": cacheStats.TotalSize,
|
"total_size": cacheStats.TotalSize,
|
||||||
|
"max_cache_size": a.config.Cache.MaxSizeBytes,
|
||||||
"cache_hits": cacheStats.TotalDownloads,
|
"cache_hits": cacheStats.TotalDownloads,
|
||||||
"cache_misses": 0, // TODO: Track cache misses
|
"cache_misses": 0, // TODO: Track cache misses
|
||||||
"cache_evictions": 0, // TODO: Track evictions
|
"cache_evictions": 0, // TODO: Track evictions
|
||||||
"cache_size": cacheStats.TotalSize,
|
"cache_size": cacheStats.TotalSize,
|
||||||
"scanned_packages": cacheStats.ScannedPackages,
|
"scanned_packages": cacheStats.ScannedPackages,
|
||||||
"vulnerable_packages": cacheStats.VulnerablePackages,
|
"vulnerable_packages": cacheStats.VulnerablePackages,
|
||||||
|
"blocked_packages": blockedCount,
|
||||||
}
|
}
|
||||||
|
|
||||||
// Convert registry stats to interface map
|
// Convert registry stats to interface map
|
||||||
@@ -500,11 +521,10 @@ func (a *App) handleInfo(c *fiber.Ctx) error {
|
|||||||
"max_cache_size": a.config.Cache.MaxSizeBytes,
|
"max_cache_size": a.config.Cache.MaxSizeBytes,
|
||||||
},
|
},
|
||||||
"features": map[string]bool{
|
"features": map[string]bool{
|
||||||
"distributed_locking": a.lockManager != nil,
|
"security_scanning": a.config.Security.Enabled,
|
||||||
"security_scanning": a.config.Security.Enabled,
|
"pre_warming": a.prewarmWorker != nil,
|
||||||
"pre_warming": a.prewarmWorker != nil,
|
"websockets": true,
|
||||||
"websockets": true,
|
"analytics": true,
|
||||||
"analytics": true,
|
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -110,13 +110,13 @@ func (a *App) handleListBypasses(c *fiber.Ctx) error {
|
|||||||
|
|
||||||
// CreateBypassRequest represents the request body for creating a bypass
|
// CreateBypassRequest represents the request body for creating a bypass
|
||||||
type CreateBypassRequest struct {
|
type CreateBypassRequest struct {
|
||||||
Type metadata.BypassType `json:"type"` // "cve" or "package"
|
Type metadata.BypassType `json:"type"`
|
||||||
Target string `json:"target"` // CVE ID or package name
|
Target string `json:"target"`
|
||||||
Reason string `json:"reason"` // Why this bypass is needed
|
Reason string `json:"reason"`
|
||||||
CreatedBy string `json:"created_by"` // Admin username
|
CreatedBy string `json:"created_by"`
|
||||||
ExpiresInHours int `json:"expires_in_hours"` // How many hours until expiration
|
AppliesTo string `json:"applies_to,omitempty"`
|
||||||
AppliesTo string `json:"applies_to,omitempty"` // Optional: limit CVE bypass to specific package
|
ExpiresInHours int `json:"expires_in_hours"`
|
||||||
NotifyOnExpiry bool `json:"notify_on_expiry"` // Send notification when expired
|
NotifyOnExpiry bool `json:"notify_on_expiry"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// handleCreateBypass creates a new CVE bypass
|
// handleCreateBypass creates a new CVE bypass
|
||||||
|
|||||||
@@ -0,0 +1,149 @@
|
|||||||
|
package app
|
||||||
|
|
||||||
|
import (
|
||||||
|
"strconv"
|
||||||
|
|
||||||
|
"github.com/gofiber/fiber/v2"
|
||||||
|
"github.com/rs/zerolog/log"
|
||||||
|
)
|
||||||
|
|
||||||
|
// handleAnalyticsTopPackages returns the most downloaded packages
|
||||||
|
func (a *App) handleAnalyticsTopPackages(c *fiber.Ctx) error {
|
||||||
|
c.Set("Content-Type", "application/json")
|
||||||
|
c.Set("Access-Control-Allow-Origin", "*")
|
||||||
|
|
||||||
|
// Get limit from query params (default: 10)
|
||||||
|
limit := 10
|
||||||
|
if limitStr := c.Query("limit"); limitStr != "" {
|
||||||
|
if parsedLimit, err := strconv.Atoi(limitStr); err == nil && parsedLimit > 0 {
|
||||||
|
limit = parsedLimit
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
packages := a.analyticsEngine.GetTopPackages(limit)
|
||||||
|
|
||||||
|
return c.Status(fiber.StatusOK).JSON(fiber.Map{
|
||||||
|
"packages": packages,
|
||||||
|
"total": len(packages),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// handleAnalyticsTrendingPackages returns trending packages
|
||||||
|
func (a *App) handleAnalyticsTrendingPackages(c *fiber.Ctx) error {
|
||||||
|
c.Set("Content-Type", "application/json")
|
||||||
|
c.Set("Access-Control-Allow-Origin", "*")
|
||||||
|
|
||||||
|
// Get limit from query params (default: 10)
|
||||||
|
limit := 10
|
||||||
|
if limitStr := c.Query("limit"); limitStr != "" {
|
||||||
|
if parsedLimit, err := strconv.Atoi(limitStr); err == nil && parsedLimit > 0 {
|
||||||
|
limit = parsedLimit
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
packages := a.analyticsEngine.GetTrendingPackages(limit)
|
||||||
|
|
||||||
|
return c.Status(fiber.StatusOK).JSON(fiber.Map{
|
||||||
|
"packages": packages,
|
||||||
|
"total": len(packages),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// handleAnalyticsTrends returns download trends over time
|
||||||
|
func (a *App) handleAnalyticsTrends(c *fiber.Ctx) error {
|
||||||
|
c.Set("Content-Type", "application/json")
|
||||||
|
c.Set("Access-Control-Allow-Origin", "*")
|
||||||
|
|
||||||
|
trends := a.analyticsEngine.GetTrends()
|
||||||
|
|
||||||
|
return c.Status(fiber.StatusOK).JSON(fiber.Map{
|
||||||
|
"trends": trends,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// handleAnalyticsTotalStats returns overall statistics
|
||||||
|
func (a *App) handleAnalyticsTotalStats(c *fiber.Ctx) error {
|
||||||
|
c.Set("Content-Type", "application/json")
|
||||||
|
c.Set("Access-Control-Allow-Origin", "*")
|
||||||
|
|
||||||
|
stats := a.analyticsEngine.GetTotalStats()
|
||||||
|
|
||||||
|
return c.Status(fiber.StatusOK).JSON(stats)
|
||||||
|
}
|
||||||
|
|
||||||
|
// handleAnalyticsRegistryStats returns per-registry statistics
|
||||||
|
func (a *App) handleAnalyticsRegistryStats(c *fiber.Ctx) error {
|
||||||
|
c.Set("Content-Type", "application/json")
|
||||||
|
c.Set("Access-Control-Allow-Origin", "*")
|
||||||
|
|
||||||
|
registry := c.Params("registry")
|
||||||
|
if registry == "" {
|
||||||
|
return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
|
||||||
|
"error": "registry parameter is required",
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
stats := a.analyticsEngine.GetRegistryStats(registry)
|
||||||
|
|
||||||
|
return c.Status(fiber.StatusOK).JSON(stats)
|
||||||
|
}
|
||||||
|
|
||||||
|
// handleAnalyticsPackageStats returns statistics for a specific package
|
||||||
|
func (a *App) handleAnalyticsPackageStats(c *fiber.Ctx) error {
|
||||||
|
c.Set("Content-Type", "application/json")
|
||||||
|
c.Set("Access-Control-Allow-Origin", "*")
|
||||||
|
|
||||||
|
registry := c.Params("registry")
|
||||||
|
name := c.Params("name")
|
||||||
|
|
||||||
|
if registry == "" || name == "" {
|
||||||
|
return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
|
||||||
|
"error": "registry and name parameters are required",
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
stats, exists := a.analyticsEngine.GetPackageStats(registry, name)
|
||||||
|
if !exists {
|
||||||
|
return c.Status(fiber.StatusNotFound).JSON(fiber.Map{
|
||||||
|
"error": "package not found in analytics",
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
return c.Status(fiber.StatusOK).JSON(fiber.Map{
|
||||||
|
"package": stats,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// handleAnalyticsSearch searches for packages matching a query
|
||||||
|
func (a *App) handleAnalyticsSearch(c *fiber.Ctx) error {
|
||||||
|
c.Set("Content-Type", "application/json")
|
||||||
|
c.Set("Access-Control-Allow-Origin", "*")
|
||||||
|
|
||||||
|
query := c.Query("q")
|
||||||
|
if query == "" {
|
||||||
|
return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
|
||||||
|
"error": "query parameter 'q' is required",
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// Get limit from query params (default: 20)
|
||||||
|
limit := 20
|
||||||
|
if limitStr := c.Query("limit"); limitStr != "" {
|
||||||
|
if parsedLimit, err := strconv.Atoi(limitStr); err == nil && parsedLimit > 0 {
|
||||||
|
limit = parsedLimit
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
results := a.analyticsEngine.SearchPackages(query, limit)
|
||||||
|
|
||||||
|
log.Debug().
|
||||||
|
Str("query", query).
|
||||||
|
Int("results", len(results)).
|
||||||
|
Msg("Analytics search completed")
|
||||||
|
|
||||||
|
return c.Status(fiber.StatusOK).JSON(fiber.Map{
|
||||||
|
"results": results,
|
||||||
|
"total": len(results),
|
||||||
|
"query": query,
|
||||||
|
})
|
||||||
|
}
|
||||||
@@ -0,0 +1,298 @@
|
|||||||
|
package app
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"net/http/httptest"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/gofiber/fiber/v2"
|
||||||
|
"github.com/lukaszraczylo/gohoarder/pkg/analytics"
|
||||||
|
"github.com/stretchr/testify/suite"
|
||||||
|
)
|
||||||
|
|
||||||
|
type AnalyticsHandlersTestSuite struct {
|
||||||
|
suite.Suite
|
||||||
|
app *fiber.App
|
||||||
|
appInst *App
|
||||||
|
engine *analytics.Engine
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *AnalyticsHandlersTestSuite) SetupTest() {
|
||||||
|
// Create analytics engine
|
||||||
|
s.engine = analytics.NewEngine(analytics.Config{
|
||||||
|
MaxEvents: 10000,
|
||||||
|
FlushInterval: 5 * time.Minute,
|
||||||
|
})
|
||||||
|
|
||||||
|
// Seed some test data
|
||||||
|
s.engine.TrackDownload(analytics.PackageDownload{
|
||||||
|
Registry: "npm",
|
||||||
|
Name: "lodash",
|
||||||
|
Version: "4.17.21",
|
||||||
|
Timestamp: time.Now(),
|
||||||
|
BytesSize: 1024,
|
||||||
|
})
|
||||||
|
s.engine.TrackDownload(analytics.PackageDownload{
|
||||||
|
Registry: "npm",
|
||||||
|
Name: "react",
|
||||||
|
Version: "18.0.0",
|
||||||
|
Timestamp: time.Now(),
|
||||||
|
BytesSize: 2048,
|
||||||
|
})
|
||||||
|
s.engine.TrackDownload(analytics.PackageDownload{
|
||||||
|
Registry: "pypi",
|
||||||
|
Name: "requests",
|
||||||
|
Version: "2.28.0",
|
||||||
|
Timestamp: time.Now(),
|
||||||
|
BytesSize: 512,
|
||||||
|
})
|
||||||
|
|
||||||
|
// Create app instance
|
||||||
|
s.appInst = &App{
|
||||||
|
analyticsEngine: s.engine,
|
||||||
|
}
|
||||||
|
|
||||||
|
// Create Fiber app
|
||||||
|
s.app = fiber.New()
|
||||||
|
|
||||||
|
// Register routes
|
||||||
|
s.app.Get("/api/analytics/top", s.appInst.handleAnalyticsTopPackages)
|
||||||
|
s.app.Get("/api/analytics/trending", s.appInst.handleAnalyticsTrendingPackages)
|
||||||
|
s.app.Get("/api/analytics/trends", s.appInst.handleAnalyticsTrends)
|
||||||
|
s.app.Get("/api/analytics/total", s.appInst.handleAnalyticsTotalStats)
|
||||||
|
s.app.Get("/api/analytics/registry/:registry", s.appInst.handleAnalyticsRegistryStats)
|
||||||
|
s.app.Get("/api/analytics/package/:registry/:name", s.appInst.handleAnalyticsPackageStats)
|
||||||
|
s.app.Get("/api/analytics/search", s.appInst.handleAnalyticsSearch)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *AnalyticsHandlersTestSuite) TearDownTest() {
|
||||||
|
if s.engine != nil {
|
||||||
|
s.engine.Close()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAnalyticsHandlersTestSuite(t *testing.T) {
|
||||||
|
suite.Run(t, new(AnalyticsHandlersTestSuite))
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *AnalyticsHandlersTestSuite) TestHandleAnalyticsTopPackages() {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
queryParams string
|
||||||
|
expectedStatus int
|
||||||
|
expectError bool
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "get top packages default",
|
||||||
|
queryParams: "",
|
||||||
|
expectedStatus: 200,
|
||||||
|
expectError: false,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "get top packages with limit",
|
||||||
|
queryParams: "?limit=5",
|
||||||
|
expectedStatus: 200,
|
||||||
|
expectError: false,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "get top packages with registry filter",
|
||||||
|
queryParams: "?registry=npm",
|
||||||
|
expectedStatus: 200,
|
||||||
|
expectError: false,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
s.Run(tt.name, func() {
|
||||||
|
req := httptest.NewRequest("GET", "/api/analytics/top"+tt.queryParams, nil)
|
||||||
|
resp, err := s.app.Test(req)
|
||||||
|
s.Require().NoError(err)
|
||||||
|
s.Equal(tt.expectedStatus, resp.StatusCode)
|
||||||
|
|
||||||
|
if !tt.expectError {
|
||||||
|
var result struct {
|
||||||
|
Packages []analytics.PackageStats `json:"packages"`
|
||||||
|
Total int `json:"total"`
|
||||||
|
}
|
||||||
|
err = json.NewDecoder(resp.Body).Decode(&result)
|
||||||
|
s.NoError(err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *AnalyticsHandlersTestSuite) TestHandleAnalyticsTrendingPackages() {
|
||||||
|
req := httptest.NewRequest("GET", "/api/analytics/trending", nil)
|
||||||
|
resp, err := s.app.Test(req)
|
||||||
|
s.Require().NoError(err)
|
||||||
|
s.Equal(200, resp.StatusCode)
|
||||||
|
|
||||||
|
var result struct {
|
||||||
|
Packages []analytics.PackageStats `json:"packages"`
|
||||||
|
Total int `json:"total"`
|
||||||
|
}
|
||||||
|
err = json.NewDecoder(resp.Body).Decode(&result)
|
||||||
|
s.NoError(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *AnalyticsHandlersTestSuite) TestHandleAnalyticsTrends() {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
queryParams string
|
||||||
|
expectedStatus int
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "get trends default timeframe",
|
||||||
|
queryParams: "",
|
||||||
|
expectedStatus: 200,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "get trends with registry filter",
|
||||||
|
queryParams: "?registry=npm",
|
||||||
|
expectedStatus: 200,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
s.Run(tt.name, func() {
|
||||||
|
req := httptest.NewRequest("GET", "/api/analytics/trends"+tt.queryParams, nil)
|
||||||
|
resp, err := s.app.Test(req)
|
||||||
|
s.Require().NoError(err)
|
||||||
|
s.Equal(tt.expectedStatus, resp.StatusCode)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *AnalyticsHandlersTestSuite) TestHandleAnalyticsTotalStats() {
|
||||||
|
req := httptest.NewRequest("GET", "/api/analytics/total", nil)
|
||||||
|
resp, err := s.app.Test(req)
|
||||||
|
s.Require().NoError(err)
|
||||||
|
s.Equal(200, resp.StatusCode)
|
||||||
|
|
||||||
|
var result struct {
|
||||||
|
TotalDownloads int64 `json:"total_downloads"`
|
||||||
|
TotalBytes int64 `json:"total_bytes"`
|
||||||
|
UniquePackages int `json:"unique_packages"`
|
||||||
|
}
|
||||||
|
err = json.NewDecoder(resp.Body).Decode(&result)
|
||||||
|
s.NoError(err)
|
||||||
|
s.Greater(result.TotalDownloads, int64(0))
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *AnalyticsHandlersTestSuite) TestHandleAnalyticsRegistryStats() {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
registry string
|
||||||
|
expectedStatus int
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "npm registry stats",
|
||||||
|
registry: "npm",
|
||||||
|
expectedStatus: 200,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "pypi registry stats",
|
||||||
|
registry: "pypi",
|
||||||
|
expectedStatus: 200,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "go registry stats",
|
||||||
|
registry: "go",
|
||||||
|
expectedStatus: 200,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
s.Run(tt.name, func() {
|
||||||
|
req := httptest.NewRequest("GET", "/api/analytics/registry/"+tt.registry, nil)
|
||||||
|
resp, err := s.app.Test(req)
|
||||||
|
s.Require().NoError(err)
|
||||||
|
s.Equal(tt.expectedStatus, resp.StatusCode)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *AnalyticsHandlersTestSuite) TestHandleAnalyticsPackageStats() {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
registry string
|
||||||
|
packageName string
|
||||||
|
expectedStatus int
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "lodash package stats",
|
||||||
|
registry: "npm",
|
||||||
|
packageName: "lodash",
|
||||||
|
expectedStatus: 200,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "react package stats",
|
||||||
|
registry: "npm",
|
||||||
|
packageName: "react",
|
||||||
|
expectedStatus: 200,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "requests package stats",
|
||||||
|
registry: "pypi",
|
||||||
|
packageName: "requests",
|
||||||
|
expectedStatus: 200,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
s.Run(tt.name, func() {
|
||||||
|
req := httptest.NewRequest("GET", "/api/analytics/package/"+tt.registry+"/"+tt.packageName, nil)
|
||||||
|
resp, err := s.app.Test(req)
|
||||||
|
s.Require().NoError(err)
|
||||||
|
s.Equal(tt.expectedStatus, resp.StatusCode)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *AnalyticsHandlersTestSuite) TestHandleAnalyticsSearch() {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
queryParams string
|
||||||
|
expectedStatus int
|
||||||
|
expectError bool
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "search for lodash",
|
||||||
|
queryParams: "?q=lodash",
|
||||||
|
expectedStatus: 200,
|
||||||
|
expectError: false,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "search for react",
|
||||||
|
queryParams: "?q=react",
|
||||||
|
expectedStatus: 200,
|
||||||
|
expectError: false,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "search with no query",
|
||||||
|
queryParams: "",
|
||||||
|
expectedStatus: 400, // Query parameter is required
|
||||||
|
expectError: true,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
s.Run(tt.name, func() {
|
||||||
|
req := httptest.NewRequest("GET", "/api/analytics/search"+tt.queryParams, nil)
|
||||||
|
resp, err := s.app.Test(req)
|
||||||
|
s.Require().NoError(err)
|
||||||
|
s.Equal(tt.expectedStatus, resp.StatusCode)
|
||||||
|
|
||||||
|
if !tt.expectError {
|
||||||
|
var result struct {
|
||||||
|
Results []analytics.PackageStats `json:"results"`
|
||||||
|
Total int `json:"total"`
|
||||||
|
Query string `json:"query"`
|
||||||
|
}
|
||||||
|
err = json.NewDecoder(resp.Body).Decode(&result)
|
||||||
|
s.NoError(err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,135 @@
|
|||||||
|
package app
|
||||||
|
|
||||||
|
import (
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/gofiber/fiber/v2"
|
||||||
|
"github.com/lukaszraczylo/gohoarder/pkg/auth"
|
||||||
|
"github.com/rs/zerolog/log"
|
||||||
|
)
|
||||||
|
|
||||||
|
// GenerateAPIKeyRequest represents a request to generate a new API key
|
||||||
|
type GenerateAPIKeyRequest struct {
|
||||||
|
ExpiresInMin *int `json:"expires_in_min"`
|
||||||
|
Name string `json:"name"`
|
||||||
|
Role string `json:"role"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// handleGenerateAPIKey generates a new API key
|
||||||
|
func (a *App) handleGenerateAPIKey(c *fiber.Ctx) error {
|
||||||
|
c.Set("Content-Type", "application/json")
|
||||||
|
|
||||||
|
var req GenerateAPIKeyRequest
|
||||||
|
if err := c.BodyParser(&req); err != nil {
|
||||||
|
return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
|
||||||
|
"error": "invalid JSON in request body",
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// Validate request
|
||||||
|
if req.Name == "" {
|
||||||
|
return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
|
||||||
|
"error": "name is required",
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// Parse role (default to readonly if not specified)
|
||||||
|
var role auth.Role
|
||||||
|
switch req.Role {
|
||||||
|
case "admin":
|
||||||
|
role = auth.RoleAdmin
|
||||||
|
case "readwrite":
|
||||||
|
role = auth.RoleReadWrite
|
||||||
|
case "readonly", "":
|
||||||
|
role = auth.RoleReadOnly
|
||||||
|
default:
|
||||||
|
return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
|
||||||
|
"error": "invalid role, must be 'admin', 'readwrite', or 'readonly'",
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// Calculate expiration
|
||||||
|
var expiresIn *time.Duration
|
||||||
|
if req.ExpiresInMin != nil {
|
||||||
|
duration := time.Duration(*req.ExpiresInMin) * time.Minute
|
||||||
|
expiresIn = &duration
|
||||||
|
}
|
||||||
|
|
||||||
|
// Generate key
|
||||||
|
apiKey, rawKey, err := a.authManager.GenerateAPIKey(req.Name, role, expiresIn)
|
||||||
|
if err != nil {
|
||||||
|
log.Error().Err(err).Str("name", req.Name).Msg("Failed to generate API key")
|
||||||
|
return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
|
||||||
|
"error": "failed to generate API key",
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
log.Info().
|
||||||
|
Str("key_id", apiKey.ID).
|
||||||
|
Str("name", apiKey.Name).
|
||||||
|
Str("role", string(apiKey.Role)).
|
||||||
|
Msg("API key generated")
|
||||||
|
|
||||||
|
// Return the key info and raw key (only time it's shown!)
|
||||||
|
return c.Status(fiber.StatusCreated).JSON(fiber.Map{
|
||||||
|
"key": rawKey, // IMPORTANT: This is the only time the raw key is shown
|
||||||
|
"key_id": apiKey.ID,
|
||||||
|
"name": apiKey.Name,
|
||||||
|
"role": apiKey.Role,
|
||||||
|
"expires": apiKey.ExpiresAt,
|
||||||
|
"message": "Save this key now! It will not be shown again.",
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// handleListAPIKeys lists all API keys
|
||||||
|
func (a *App) handleListAPIKeys(c *fiber.Ctx) error {
|
||||||
|
c.Set("Content-Type", "application/json")
|
||||||
|
|
||||||
|
keys := a.authManager.ListAPIKeys()
|
||||||
|
|
||||||
|
// Convert to response format (excluding hashed keys)
|
||||||
|
response := make([]fiber.Map, len(keys))
|
||||||
|
for i, key := range keys {
|
||||||
|
response[i] = fiber.Map{
|
||||||
|
"id": key.ID,
|
||||||
|
"name": key.Name,
|
||||||
|
"role": key.Role,
|
||||||
|
"created_at": key.CreatedAt,
|
||||||
|
"expires_at": key.ExpiresAt,
|
||||||
|
"last_used_at": key.LastUsedAt,
|
||||||
|
"permissions": key.Permissions,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return c.Status(fiber.StatusOK).JSON(fiber.Map{
|
||||||
|
"keys": response,
|
||||||
|
"total": len(response),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// handleRevokeAPIKey revokes an API key
|
||||||
|
func (a *App) handleRevokeAPIKey(c *fiber.Ctx) error {
|
||||||
|
c.Set("Content-Type", "application/json")
|
||||||
|
|
||||||
|
keyID := c.Params("key_id")
|
||||||
|
if keyID == "" {
|
||||||
|
return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
|
||||||
|
"error": "key_id parameter is required",
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
err := a.authManager.RevokeAPIKey(keyID)
|
||||||
|
if err != nil {
|
||||||
|
log.Warn().Err(err).Str("key_id", keyID).Msg("Failed to revoke API key")
|
||||||
|
return c.Status(fiber.StatusNotFound).JSON(fiber.Map{
|
||||||
|
"error": "API key not found",
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
log.Info().Str("key_id", keyID).Msg("API key revoked")
|
||||||
|
|
||||||
|
return c.Status(fiber.StatusOK).JSON(fiber.Map{
|
||||||
|
"message": "API key revoked successfully",
|
||||||
|
"key_id": keyID,
|
||||||
|
})
|
||||||
|
}
|
||||||
@@ -0,0 +1,285 @@
|
|||||||
|
package app
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"encoding/json"
|
||||||
|
"net/http/httptest"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/gofiber/fiber/v2"
|
||||||
|
"github.com/lukaszraczylo/gohoarder/pkg/auth"
|
||||||
|
"github.com/stretchr/testify/suite"
|
||||||
|
)
|
||||||
|
|
||||||
|
type AuthHandlersTestSuite struct {
|
||||||
|
suite.Suite
|
||||||
|
app *fiber.App
|
||||||
|
appInst *App
|
||||||
|
authManager *auth.Manager
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *AuthHandlersTestSuite) SetupTest() {
|
||||||
|
// Create auth manager
|
||||||
|
s.authManager = auth.New()
|
||||||
|
|
||||||
|
// Create app instance
|
||||||
|
s.appInst = &App{
|
||||||
|
authManager: s.authManager,
|
||||||
|
}
|
||||||
|
|
||||||
|
// Create Fiber app
|
||||||
|
s.app = fiber.New()
|
||||||
|
|
||||||
|
// Register routes
|
||||||
|
s.app.Post("/api/admin/keys", s.appInst.handleGenerateAPIKey)
|
||||||
|
s.app.Get("/api/admin/keys", s.appInst.handleListAPIKeys)
|
||||||
|
s.app.Delete("/api/admin/keys/:key_id", s.appInst.handleRevokeAPIKey)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAuthHandlersTestSuite(t *testing.T) {
|
||||||
|
suite.Run(t, new(AuthHandlersTestSuite))
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *AuthHandlersTestSuite) TestHandleGenerateAPIKey() {
|
||||||
|
tests := []struct {
|
||||||
|
requestBody map[string]string
|
||||||
|
name string
|
||||||
|
expectedStatus int
|
||||||
|
expectedRole string
|
||||||
|
expectKey bool
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "generate read-only key",
|
||||||
|
requestBody: map[string]string{
|
||||||
|
"role": "readonly",
|
||||||
|
"name": "test-readonly-key",
|
||||||
|
},
|
||||||
|
expectedStatus: 201,
|
||||||
|
expectedRole: "readonly",
|
||||||
|
expectKey: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "generate read-write key",
|
||||||
|
requestBody: map[string]string{
|
||||||
|
"role": "readwrite",
|
||||||
|
"name": "test-readwrite-key",
|
||||||
|
},
|
||||||
|
expectedStatus: 201,
|
||||||
|
expectedRole: "readwrite",
|
||||||
|
expectKey: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "generate admin key",
|
||||||
|
requestBody: map[string]string{
|
||||||
|
"role": "admin",
|
||||||
|
"name": "test-admin-key",
|
||||||
|
},
|
||||||
|
expectedStatus: 201,
|
||||||
|
expectedRole: "admin",
|
||||||
|
expectKey: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "invalid role",
|
||||||
|
requestBody: map[string]string{
|
||||||
|
"role": "invalid-role",
|
||||||
|
"name": "test-key",
|
||||||
|
},
|
||||||
|
expectedStatus: 400,
|
||||||
|
expectKey: false,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "missing role defaults to readonly",
|
||||||
|
requestBody: map[string]string{
|
||||||
|
"name": "test-key-default-role",
|
||||||
|
},
|
||||||
|
expectedStatus: 201,
|
||||||
|
expectedRole: "readonly", // Role defaults to readonly when not specified
|
||||||
|
expectKey: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "missing name",
|
||||||
|
requestBody: map[string]string{
|
||||||
|
"role": "read-only",
|
||||||
|
},
|
||||||
|
expectedStatus: 400,
|
||||||
|
expectKey: false,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
s.Run(tt.name, func() {
|
||||||
|
bodyBytes, err := json.Marshal(tt.requestBody)
|
||||||
|
s.Require().NoError(err)
|
||||||
|
|
||||||
|
req := httptest.NewRequest("POST", "/api/admin/keys", bytes.NewReader(bodyBytes))
|
||||||
|
req.Header.Set("Content-Type", "application/json")
|
||||||
|
|
||||||
|
resp, err := s.app.Test(req, 5000) // 5 second timeout for CI environments
|
||||||
|
s.Require().NoError(err)
|
||||||
|
s.Equal(tt.expectedStatus, resp.StatusCode)
|
||||||
|
|
||||||
|
if tt.expectKey {
|
||||||
|
var result struct {
|
||||||
|
Key string `json:"key"`
|
||||||
|
KeyID string `json:"key_id"`
|
||||||
|
Role string `json:"role"`
|
||||||
|
Name string `json:"name"`
|
||||||
|
Message string `json:"message"`
|
||||||
|
}
|
||||||
|
err = json.NewDecoder(resp.Body).Decode(&result)
|
||||||
|
s.NoError(err)
|
||||||
|
s.NotEmpty(result.Key)
|
||||||
|
s.NotEmpty(result.KeyID)
|
||||||
|
s.Equal(tt.expectedRole, result.Role)
|
||||||
|
s.Equal(tt.requestBody["name"], result.Name)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *AuthHandlersTestSuite) TestHandleListAPIKeys() {
|
||||||
|
// Generate some test keys first
|
||||||
|
s.authManager.GenerateAPIKey("test-key-1", auth.RoleReadOnly, nil)
|
||||||
|
s.authManager.GenerateAPIKey("test-key-2", auth.RoleReadWrite, nil)
|
||||||
|
s.authManager.GenerateAPIKey("test-key-3", auth.RoleAdmin, nil)
|
||||||
|
|
||||||
|
req := httptest.NewRequest("GET", "/api/admin/keys", nil)
|
||||||
|
resp, err := s.app.Test(req, 5000) // 5 second timeout for CI environments
|
||||||
|
s.Require().NoError(err)
|
||||||
|
s.Equal(200, resp.StatusCode)
|
||||||
|
|
||||||
|
var result struct {
|
||||||
|
Keys []map[string]interface{} `json:"keys"`
|
||||||
|
Total int `json:"total"`
|
||||||
|
}
|
||||||
|
err = json.NewDecoder(resp.Body).Decode(&result)
|
||||||
|
s.NoError(err)
|
||||||
|
s.GreaterOrEqual(result.Total, 3)
|
||||||
|
|
||||||
|
// Verify keys don't include the actual key value
|
||||||
|
for _, key := range result.Keys {
|
||||||
|
s.NotEmpty(key["id"])
|
||||||
|
s.NotEmpty(key["role"])
|
||||||
|
s.NotEmpty(key["name"])
|
||||||
|
s.NotEmpty(key["created_at"])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *AuthHandlersTestSuite) TestHandleRevokeAPIKey() {
|
||||||
|
// Generate a test key
|
||||||
|
keyInfo, _, _ := s.authManager.GenerateAPIKey("test-revoke-key", auth.RoleReadOnly, nil)
|
||||||
|
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
keyID string
|
||||||
|
expectedStatus int
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "revoke existing key",
|
||||||
|
keyID: keyInfo.ID,
|
||||||
|
expectedStatus: 200,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "revoke non-existent key",
|
||||||
|
keyID: "non-existent-key-id",
|
||||||
|
expectedStatus: 404,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
s.Run(tt.name, func() {
|
||||||
|
req := httptest.NewRequest("DELETE", "/api/admin/keys/"+tt.keyID, nil)
|
||||||
|
resp, err := s.app.Test(req, 5000) // 5 second timeout for CI environments
|
||||||
|
s.Require().NoError(err)
|
||||||
|
s.Equal(tt.expectedStatus, resp.StatusCode)
|
||||||
|
|
||||||
|
if tt.expectedStatus == 200 {
|
||||||
|
var result map[string]string
|
||||||
|
err = json.NewDecoder(resp.Body).Decode(&result)
|
||||||
|
s.NoError(err)
|
||||||
|
s.Contains(result, "message")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *AuthHandlersTestSuite) TestHandleGenerateAPIKeyInvalidJSON() {
|
||||||
|
req := httptest.NewRequest("POST", "/api/admin/keys", bytes.NewReader([]byte("invalid json")))
|
||||||
|
req.Header.Set("Content-Type", "application/json")
|
||||||
|
|
||||||
|
resp, err := s.app.Test(req, 5000) // 5 second timeout for CI environments
|
||||||
|
s.Require().NoError(err)
|
||||||
|
s.Equal(400, resp.StatusCode)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *AuthHandlersTestSuite) TestGenerateAndRevokeKeyFlow() {
|
||||||
|
// Generate a key
|
||||||
|
bodyBytes, _ := json.Marshal(map[string]string{
|
||||||
|
"role": "readonly",
|
||||||
|
"name": "integration-test-key",
|
||||||
|
})
|
||||||
|
|
||||||
|
req1 := httptest.NewRequest("POST", "/api/admin/keys", bytes.NewReader(bodyBytes))
|
||||||
|
req1.Header.Set("Content-Type", "application/json")
|
||||||
|
resp1, err := s.app.Test(req1, 5000) // 5 second timeout for CI environments
|
||||||
|
s.Require().NoError(err)
|
||||||
|
s.Equal(201, resp1.StatusCode)
|
||||||
|
|
||||||
|
var createResult struct {
|
||||||
|
Key string `json:"key"`
|
||||||
|
KeyID string `json:"key_id"`
|
||||||
|
}
|
||||||
|
err = json.NewDecoder(resp1.Body).Decode(&createResult)
|
||||||
|
s.Require().NoError(err)
|
||||||
|
keyID := createResult.KeyID
|
||||||
|
|
||||||
|
// List keys - should include our new key
|
||||||
|
req2 := httptest.NewRequest("GET", "/api/admin/keys", nil)
|
||||||
|
resp2, err := s.app.Test(req2, 5000) // 5 second timeout for CI environments
|
||||||
|
s.Require().NoError(err)
|
||||||
|
s.Equal(200, resp2.StatusCode)
|
||||||
|
|
||||||
|
var listResult struct {
|
||||||
|
Keys []map[string]interface{} `json:"keys"`
|
||||||
|
Total int `json:"total"`
|
||||||
|
}
|
||||||
|
err = json.NewDecoder(resp2.Body).Decode(&listResult)
|
||||||
|
s.Require().NoError(err)
|
||||||
|
|
||||||
|
found := false
|
||||||
|
for _, key := range listResult.Keys {
|
||||||
|
if key["id"].(string) == keyID {
|
||||||
|
found = true
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
s.True(found, "newly created key should be in the list")
|
||||||
|
|
||||||
|
// Revoke the key
|
||||||
|
req3 := httptest.NewRequest("DELETE", "/api/admin/keys/"+keyID, nil)
|
||||||
|
resp3, err := s.app.Test(req3, 5000) // 5 second timeout for CI environments
|
||||||
|
s.Require().NoError(err)
|
||||||
|
s.Equal(200, resp3.StatusCode)
|
||||||
|
|
||||||
|
// List keys again - should not include the revoked key
|
||||||
|
req4 := httptest.NewRequest("GET", "/api/admin/keys", nil)
|
||||||
|
resp4, err := s.app.Test(req4, 5000) // 5 second timeout for CI environments
|
||||||
|
s.Require().NoError(err)
|
||||||
|
s.Equal(200, resp4.StatusCode)
|
||||||
|
|
||||||
|
var listResult2 struct {
|
||||||
|
Keys []map[string]interface{} `json:"keys"`
|
||||||
|
Total int `json:"total"`
|
||||||
|
}
|
||||||
|
err = json.NewDecoder(resp4.Body).Decode(&listResult2)
|
||||||
|
s.Require().NoError(err)
|
||||||
|
|
||||||
|
found = false
|
||||||
|
for _, key := range listResult2.Keys {
|
||||||
|
if key["id"].(string) == keyID {
|
||||||
|
found = true
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
s.False(found, "revoked key should not be in the list")
|
||||||
|
}
|
||||||
@@ -0,0 +1,94 @@
|
|||||||
|
package app
|
||||||
|
|
||||||
|
import (
|
||||||
|
"github.com/gofiber/fiber/v2"
|
||||||
|
"github.com/rs/zerolog/log"
|
||||||
|
)
|
||||||
|
|
||||||
|
// handlePrewarmingStatus returns the status of the pre-warming worker
|
||||||
|
func (a *App) handlePrewarmingStatus(c *fiber.Ctx) error {
|
||||||
|
c.Set("Content-Type", "application/json")
|
||||||
|
|
||||||
|
status := a.prewarmWorker.GetStatus()
|
||||||
|
|
||||||
|
return c.Status(fiber.StatusOK).JSON(status)
|
||||||
|
}
|
||||||
|
|
||||||
|
// handlePrewarmingTrigger manually triggers a pre-warming cycle
|
||||||
|
func (a *App) handlePrewarmingTrigger(c *fiber.Ctx) error {
|
||||||
|
c.Set("Content-Type", "application/json")
|
||||||
|
|
||||||
|
ctx := c.Context()
|
||||||
|
a.prewarmWorker.TriggerPrewarm(ctx)
|
||||||
|
|
||||||
|
log.Info().Msg("Pre-warming manually triggered via API")
|
||||||
|
|
||||||
|
return c.Status(fiber.StatusOK).JSON(fiber.Map{
|
||||||
|
"message": "Pre-warming cycle triggered successfully",
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// PrewarmPackageRequest represents a request to pre-warm a specific package
|
||||||
|
type PrewarmPackageRequest struct {
|
||||||
|
Registry string `json:"registry"`
|
||||||
|
Name string `json:"name"`
|
||||||
|
Version string `json:"version"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// handlePrewarmingPackage pre-warms a specific package
|
||||||
|
func (a *App) handlePrewarmingPackage(c *fiber.Ctx) error {
|
||||||
|
c.Set("Content-Type", "application/json")
|
||||||
|
|
||||||
|
var req PrewarmPackageRequest
|
||||||
|
if err := c.BodyParser(&req); err != nil {
|
||||||
|
return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
|
||||||
|
"error": "invalid JSON in request body",
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// Validate request
|
||||||
|
if req.Registry == "" {
|
||||||
|
return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
|
||||||
|
"error": "registry is required",
|
||||||
|
})
|
||||||
|
}
|
||||||
|
if req.Name == "" {
|
||||||
|
return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
|
||||||
|
"error": "name is required",
|
||||||
|
})
|
||||||
|
}
|
||||||
|
if req.Version == "" {
|
||||||
|
return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
|
||||||
|
"error": "version is required",
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
ctx := c.Context()
|
||||||
|
err := a.prewarmWorker.PrewarmPackage(ctx, req.Registry, req.Name, req.Version)
|
||||||
|
if err != nil {
|
||||||
|
log.Error().
|
||||||
|
Err(err).
|
||||||
|
Str("registry", req.Registry).
|
||||||
|
Str("name", req.Name).
|
||||||
|
Str("version", req.Version).
|
||||||
|
Msg("Failed to pre-warm package")
|
||||||
|
return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
|
||||||
|
"error": "failed to pre-warm package",
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
log.Info().
|
||||||
|
Str("registry", req.Registry).
|
||||||
|
Str("name", req.Name).
|
||||||
|
Str("version", req.Version).
|
||||||
|
Msg("Package pre-warmed via API")
|
||||||
|
|
||||||
|
return c.Status(fiber.StatusOK).JSON(fiber.Map{
|
||||||
|
"message": "Package pre-warmed successfully",
|
||||||
|
"package": fiber.Map{
|
||||||
|
"registry": req.Registry,
|
||||||
|
"name": req.Name,
|
||||||
|
"version": req.Version,
|
||||||
|
},
|
||||||
|
})
|
||||||
|
}
|
||||||
@@ -0,0 +1,162 @@
|
|||||||
|
package app
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"encoding/json"
|
||||||
|
"net/http/httptest"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/gofiber/fiber/v2"
|
||||||
|
"github.com/lukaszraczylo/gohoarder/pkg/prewarming"
|
||||||
|
"github.com/stretchr/testify/suite"
|
||||||
|
)
|
||||||
|
|
||||||
|
type PrewarmingHandlersTestSuite struct {
|
||||||
|
suite.Suite
|
||||||
|
app *fiber.App
|
||||||
|
appInst *App
|
||||||
|
prewarmWorker *prewarming.Worker
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *PrewarmingHandlersTestSuite) SetupTest() {
|
||||||
|
// Create pre-warming worker (disabled by default)
|
||||||
|
s.prewarmWorker = prewarming.NewWorker(prewarming.Config{
|
||||||
|
Enabled: false,
|
||||||
|
MaxConcurrent: 5,
|
||||||
|
})
|
||||||
|
|
||||||
|
// Create app instance
|
||||||
|
s.appInst = &App{
|
||||||
|
prewarmWorker: s.prewarmWorker,
|
||||||
|
}
|
||||||
|
|
||||||
|
// Create Fiber app
|
||||||
|
s.app = fiber.New()
|
||||||
|
|
||||||
|
// Register routes
|
||||||
|
s.app.Get("/api/admin/prewarming/status", s.appInst.handlePrewarmingStatus)
|
||||||
|
s.app.Post("/api/admin/prewarming/trigger", s.appInst.handlePrewarmingTrigger)
|
||||||
|
s.app.Post("/api/admin/prewarming/package", s.appInst.handlePrewarmingPackage)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *PrewarmingHandlersTestSuite) TearDownTest() {
|
||||||
|
if s.prewarmWorker != nil {
|
||||||
|
s.prewarmWorker.Stop()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPrewarmingHandlersTestSuite(t *testing.T) {
|
||||||
|
suite.Run(t, new(PrewarmingHandlersTestSuite))
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *PrewarmingHandlersTestSuite) TestHandlePrewarmingStatus() {
|
||||||
|
req := httptest.NewRequest("GET", "/api/admin/prewarming/status", nil)
|
||||||
|
resp, err := s.app.Test(req)
|
||||||
|
s.Require().NoError(err)
|
||||||
|
s.Equal(200, resp.StatusCode)
|
||||||
|
|
||||||
|
var result struct {
|
||||||
|
Enabled bool `json:"enabled"`
|
||||||
|
Running bool `json:"running"`
|
||||||
|
QueueSize int `json:"queue_size"`
|
||||||
|
ActiveWorkers int `json:"active_workers"`
|
||||||
|
}
|
||||||
|
err = json.NewDecoder(resp.Body).Decode(&result)
|
||||||
|
s.NoError(err)
|
||||||
|
s.False(result.Enabled) // Disabled in test setup
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *PrewarmingHandlersTestSuite) TestHandlePrewarmingTrigger() {
|
||||||
|
req := httptest.NewRequest("POST", "/api/admin/prewarming/trigger", nil)
|
||||||
|
resp, err := s.app.Test(req)
|
||||||
|
s.Require().NoError(err)
|
||||||
|
s.Equal(200, resp.StatusCode)
|
||||||
|
|
||||||
|
var result map[string]interface{}
|
||||||
|
err = json.NewDecoder(resp.Body).Decode(&result)
|
||||||
|
s.NoError(err)
|
||||||
|
s.Contains(result, "message")
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *PrewarmingHandlersTestSuite) TestHandlePrewarmingPackage() {
|
||||||
|
tests := []struct {
|
||||||
|
requestBody map[string]string
|
||||||
|
name string
|
||||||
|
expectedStatus int
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "prewarm npm package",
|
||||||
|
requestBody: map[string]string{
|
||||||
|
"registry": "npm",
|
||||||
|
"name": "lodash",
|
||||||
|
"version": "4.17.21",
|
||||||
|
},
|
||||||
|
expectedStatus: 200,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "prewarm pypi package",
|
||||||
|
requestBody: map[string]string{
|
||||||
|
"registry": "pypi",
|
||||||
|
"name": "requests",
|
||||||
|
"version": "2.28.0",
|
||||||
|
},
|
||||||
|
expectedStatus: 200,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "prewarm go package",
|
||||||
|
requestBody: map[string]string{
|
||||||
|
"registry": "go",
|
||||||
|
"name": "github.com/stretchr/testify",
|
||||||
|
"version": "v1.8.0",
|
||||||
|
},
|
||||||
|
expectedStatus: 200,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "missing registry",
|
||||||
|
requestBody: map[string]string{
|
||||||
|
"name": "lodash",
|
||||||
|
"version": "4.17.21",
|
||||||
|
},
|
||||||
|
expectedStatus: 400,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "missing name",
|
||||||
|
requestBody: map[string]string{
|
||||||
|
"registry": "npm",
|
||||||
|
"version": "4.17.21",
|
||||||
|
},
|
||||||
|
expectedStatus: 400,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "missing version",
|
||||||
|
requestBody: map[string]string{
|
||||||
|
"registry": "npm",
|
||||||
|
"name": "lodash",
|
||||||
|
},
|
||||||
|
expectedStatus: 400,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
s.Run(tt.name, func() {
|
||||||
|
bodyBytes, err := json.Marshal(tt.requestBody)
|
||||||
|
s.Require().NoError(err)
|
||||||
|
|
||||||
|
req := httptest.NewRequest("POST", "/api/admin/prewarming/package", bytes.NewReader(bodyBytes))
|
||||||
|
req.Header.Set("Content-Type", "application/json")
|
||||||
|
|
||||||
|
resp, err := s.app.Test(req)
|
||||||
|
s.Require().NoError(err)
|
||||||
|
s.Equal(tt.expectedStatus, resp.StatusCode)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *PrewarmingHandlersTestSuite) TestHandlePrewarmingPackageInvalidJSON() {
|
||||||
|
req := httptest.NewRequest("POST", "/api/admin/prewarming/package", bytes.NewReader([]byte("invalid json")))
|
||||||
|
req.Header.Set("Content-Type", "application/json")
|
||||||
|
|
||||||
|
resp, err := s.app.Test(req)
|
||||||
|
s.Require().NoError(err)
|
||||||
|
s.Equal(400, resp.StatusCode)
|
||||||
|
}
|
||||||
@@ -7,9 +7,9 @@ import (
|
|||||||
|
|
||||||
// ValidationResult represents a cached credential validation result
|
// ValidationResult represents a cached credential validation result
|
||||||
type ValidationResult struct {
|
type ValidationResult struct {
|
||||||
Allowed bool
|
|
||||||
ExpiresAt time.Time
|
ExpiresAt time.Time
|
||||||
Reason string
|
Reason string
|
||||||
|
Allowed bool
|
||||||
}
|
}
|
||||||
|
|
||||||
// ValidationCache caches credential validation results to reduce upstream checks
|
// ValidationCache caches credential validation results to reduce upstream checks
|
||||||
|
|||||||
Vendored
+108
-22
@@ -11,6 +11,7 @@ import (
|
|||||||
"sync"
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"github.com/lukaszraczylo/gohoarder/pkg/analytics"
|
||||||
"github.com/lukaszraczylo/gohoarder/pkg/errors"
|
"github.com/lukaszraczylo/gohoarder/pkg/errors"
|
||||||
"github.com/lukaszraczylo/gohoarder/pkg/metadata"
|
"github.com/lukaszraczylo/gohoarder/pkg/metadata"
|
||||||
"github.com/lukaszraczylo/gohoarder/pkg/metrics"
|
"github.com/lukaszraczylo/gohoarder/pkg/metrics"
|
||||||
@@ -27,15 +28,21 @@ type ScannerInterface interface {
|
|||||||
CheckVulnerabilities(ctx context.Context, registry, packageName, version string) (blocked bool, reason string, err error)
|
CheckVulnerabilities(ctx context.Context, registry, packageName, version string) (blocked bool, reason string, err error)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// AnalyticsInterface defines the interface for analytics tracking
|
||||||
|
type AnalyticsInterface interface {
|
||||||
|
TrackDownload(download analytics.PackageDownload)
|
||||||
|
}
|
||||||
|
|
||||||
// Manager coordinates caching operations between storage and metadata
|
// Manager coordinates caching operations between storage and metadata
|
||||||
type Manager struct {
|
type Manager struct {
|
||||||
storage storage.StorageBackend
|
storage storage.StorageBackend
|
||||||
metadata metadata.MetadataStore
|
metadata metadata.MetadataStore
|
||||||
scanner ScannerInterface
|
scanner ScannerInterface
|
||||||
config Config
|
analytics AnalyticsInterface
|
||||||
sf singleflight.Group
|
sf singleflight.Group
|
||||||
mu sync.RWMutex
|
config Config
|
||||||
evicting bool
|
mu sync.RWMutex
|
||||||
|
evicting bool
|
||||||
}
|
}
|
||||||
|
|
||||||
// Config holds cache manager configuration
|
// Config holds cache manager configuration
|
||||||
@@ -48,15 +55,15 @@ type Config struct {
|
|||||||
|
|
||||||
// CacheEntry represents a cached package
|
// CacheEntry represents a cached package
|
||||||
type CacheEntry struct {
|
type CacheEntry struct {
|
||||||
Package *metadata.Package
|
|
||||||
Data io.ReadCloser
|
Data io.ReadCloser
|
||||||
FromCache bool
|
Package *metadata.Package
|
||||||
UpstreamURL string
|
UpstreamURL string
|
||||||
CacheControl string
|
CacheControl string
|
||||||
|
FromCache bool
|
||||||
}
|
}
|
||||||
|
|
||||||
// New creates a new cache manager
|
// New creates a new cache manager
|
||||||
func New(storage storage.StorageBackend, metadata metadata.MetadataStore, scanner ScannerInterface, config Config) (*Manager, error) {
|
func New(storage storage.StorageBackend, metadata metadata.MetadataStore, scanner ScannerInterface, analytics AnalyticsInterface, config Config) (*Manager, error) {
|
||||||
if storage == nil {
|
if storage == nil {
|
||||||
return nil, errors.New(errors.ErrCodeInvalidConfig, "storage backend is required")
|
return nil, errors.New(errors.ErrCodeInvalidConfig, "storage backend is required")
|
||||||
}
|
}
|
||||||
@@ -70,6 +77,11 @@ func New(storage storage.StorageBackend, metadata metadata.MetadataStore, scanne
|
|||||||
log.Info().Msg("Cache manager initialized with security scanning enabled")
|
log.Info().Msg("Cache manager initialized with security scanning enabled")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Analytics is optional - can be nil if analytics tracking is disabled
|
||||||
|
if analytics != nil {
|
||||||
|
log.Info().Msg("Cache manager initialized with analytics tracking enabled")
|
||||||
|
}
|
||||||
|
|
||||||
if config.DefaultTTL == 0 {
|
if config.DefaultTTL == 0 {
|
||||||
config.DefaultTTL = 7 * 24 * time.Hour // 7 days default
|
config.DefaultTTL = 7 * 24 * time.Hour // 7 days default
|
||||||
}
|
}
|
||||||
@@ -87,10 +99,11 @@ func New(storage storage.StorageBackend, metadata metadata.MetadataStore, scanne
|
|||||||
}
|
}
|
||||||
|
|
||||||
manager := &Manager{
|
manager := &Manager{
|
||||||
storage: storage,
|
storage: storage,
|
||||||
metadata: metadata,
|
metadata: metadata,
|
||||||
scanner: scanner,
|
scanner: scanner,
|
||||||
config: config,
|
analytics: analytics,
|
||||||
|
config: config,
|
||||||
}
|
}
|
||||||
|
|
||||||
// Start background cleanup worker
|
// Start background cleanup worker
|
||||||
@@ -132,7 +145,42 @@ func (m *Manager) getOrFetch(ctx context.Context, registry, name, version string
|
|||||||
if err == nil {
|
if err == nil {
|
||||||
// Cache hit!
|
// Cache hit!
|
||||||
metrics.RecordCacheHit(registry)
|
metrics.RecordCacheHit(registry)
|
||||||
_ = m.metadata.UpdateDownloadCount(ctx, registry, name, version) // #nosec G104 -- Async update, error logged
|
|
||||||
|
// Update download count (log errors for debugging)
|
||||||
|
if err := m.metadata.UpdateDownloadCount(ctx, registry, name, version); err != nil {
|
||||||
|
log.Warn().
|
||||||
|
Err(err).
|
||||||
|
Str("registry", registry).
|
||||||
|
Str("package", name).
|
||||||
|
Str("version", version).
|
||||||
|
Msg("Failed to update download count - package may not exist in database")
|
||||||
|
|
||||||
|
// Try to save package to database if it doesn't exist
|
||||||
|
// This handles the case where storage has files but database was migrated/reset
|
||||||
|
if saveErr := m.metadata.SavePackage(ctx, pkg); saveErr != nil {
|
||||||
|
log.Error().
|
||||||
|
Err(saveErr).
|
||||||
|
Str("registry", registry).
|
||||||
|
Str("package", name).
|
||||||
|
Str("version", version).
|
||||||
|
Msg("Failed to save package to database")
|
||||||
|
} else {
|
||||||
|
// Retry download count update after saving package
|
||||||
|
if retryErr := m.metadata.UpdateDownloadCount(ctx, registry, name, version); retryErr != nil {
|
||||||
|
log.Error().
|
||||||
|
Err(retryErr).
|
||||||
|
Str("registry", registry).
|
||||||
|
Str("package", name).
|
||||||
|
Str("version", version).
|
||||||
|
Msg("Failed to update download count even after saving package")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Track download in analytics if enabled
|
||||||
|
if m.analytics != nil {
|
||||||
|
m.trackDownload(registry, name, version, pkg.Size)
|
||||||
|
}
|
||||||
|
|
||||||
// Check for vulnerabilities if scanner is enabled
|
// Check for vulnerabilities if scanner is enabled
|
||||||
if m.scanner != nil {
|
if m.scanner != nil {
|
||||||
@@ -185,9 +233,12 @@ func (m *Manager) getOrFetch(ctx context.Context, registry, name, version string
|
|||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Skip security scan wait for metadata entries (index pages, lists, etc.)
|
||||||
|
isMetadataEntry := version == "list" || version == "page" || version == "latest" || version == "metadata"
|
||||||
|
|
||||||
// Wait briefly for initial scan to complete if scanner is enabled
|
// Wait briefly for initial scan to complete if scanner is enabled
|
||||||
// This prevents serving vulnerable packages on first request
|
// This prevents serving vulnerable packages on first request
|
||||||
if m.scanner != nil {
|
if m.scanner != nil && !isMetadataEntry {
|
||||||
// Wait up to 30 seconds for scan to complete
|
// Wait up to 30 seconds for scan to complete
|
||||||
scanCtx, cancel := context.WithTimeout(ctx, 30*time.Second)
|
scanCtx, cancel := context.WithTimeout(ctx, 30*time.Second)
|
||||||
defer cancel()
|
defer cancel()
|
||||||
@@ -269,6 +320,22 @@ servePkg:
|
|||||||
return nil, errors.Wrap(err, errors.ErrCodeStorageFailure, "failed to retrieve just-stored package")
|
return nil, errors.Wrap(err, errors.ErrCodeStorageFailure, "failed to retrieve just-stored package")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Track download count for first-time download (cache miss)
|
||||||
|
// This ensures download count increments regardless of cache hit/miss
|
||||||
|
if err := m.metadata.UpdateDownloadCount(ctx, registry, name, version); err != nil {
|
||||||
|
log.Warn().
|
||||||
|
Err(err).
|
||||||
|
Str("registry", registry).
|
||||||
|
Str("package", name).
|
||||||
|
Str("version", version).
|
||||||
|
Msg("Failed to update download count for newly cached package")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Track download in analytics if enabled
|
||||||
|
if m.analytics != nil {
|
||||||
|
m.trackDownload(registry, name, version, storedPkg.Size)
|
||||||
|
}
|
||||||
|
|
||||||
return &CacheEntry{
|
return &CacheEntry{
|
||||||
Package: storedPkg,
|
Package: storedPkg,
|
||||||
Data: storedData,
|
Data: storedData,
|
||||||
@@ -342,15 +409,19 @@ func (m *Manager) store(ctx context.Context, registry, name, version string, dat
|
|||||||
Metadata: make(map[string]string),
|
Metadata: make(map[string]string),
|
||||||
}
|
}
|
||||||
|
|
||||||
// Save metadata
|
// Save metadata (skip metadata entries like index pages, lists, etc.)
|
||||||
if err := m.metadata.SavePackage(ctx, pkg); err != nil {
|
isMetadataEntry := version == "list" || version == "page" || version == "latest" || version == "metadata"
|
||||||
// Clean up storage if metadata save fails
|
if !isMetadataEntry {
|
||||||
_ = m.storage.Delete(ctx, storageKey) // #nosec G104 -- Cleanup, error logged
|
if err := m.metadata.SavePackage(ctx, pkg); err != nil {
|
||||||
return nil, err
|
// Clean up storage if metadata save fails
|
||||||
|
_ = m.storage.Delete(ctx, storageKey) // #nosec G104 -- Cleanup, error logged
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Scan package if scanner is enabled (run in background to not block cache operations)
|
// Scan package if scanner is enabled (run in background to not block cache operations)
|
||||||
if m.scanner != nil {
|
// Skip scanning metadata entries (index pages, lists, etc.)
|
||||||
|
if m.scanner != nil && !isMetadataEntry {
|
||||||
go func() {
|
go func() {
|
||||||
scanCtx := context.Background()
|
scanCtx := context.Background()
|
||||||
var filePath string
|
var filePath string
|
||||||
@@ -552,6 +623,21 @@ func (m *Manager) Health(ctx context.Context) error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// trackDownload tracks a package download event in analytics
|
||||||
|
func (m *Manager) trackDownload(registry, name, version string, size int64) {
|
||||||
|
download := analytics.PackageDownload{
|
||||||
|
Registry: registry,
|
||||||
|
Name: name,
|
||||||
|
Version: version,
|
||||||
|
Timestamp: time.Now(),
|
||||||
|
BytesSize: size,
|
||||||
|
ClientIP: "", // TODO: Extract from context if available
|
||||||
|
UserAgent: "", // TODO: Extract from context if available
|
||||||
|
}
|
||||||
|
|
||||||
|
m.analytics.TrackDownload(download)
|
||||||
|
}
|
||||||
|
|
||||||
// Close closes the cache manager
|
// Close closes the cache manager
|
||||||
func (m *Manager) Close() error {
|
func (m *Manager) Close() error {
|
||||||
var err error
|
var err error
|
||||||
|
|||||||
Vendored
+25
-22
@@ -197,12 +197,12 @@ func (m *MockMetadataStore) AggregateDownloadData(ctx context.Context) error {
|
|||||||
// TestNew tests cache manager creation
|
// TestNew tests cache manager creation
|
||||||
func TestNew(t *testing.T) {
|
func TestNew(t *testing.T) {
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
name string
|
|
||||||
storage storage.StorageBackend
|
storage storage.StorageBackend
|
||||||
metadata metadata.MetadataStore
|
metadata metadata.MetadataStore
|
||||||
|
name string
|
||||||
|
errContains string
|
||||||
config Config
|
config Config
|
||||||
wantErr bool
|
wantErr bool
|
||||||
errContains string
|
|
||||||
}{
|
}{
|
||||||
// GOOD: Valid configuration
|
// GOOD: Valid configuration
|
||||||
{
|
{
|
||||||
@@ -262,7 +262,7 @@ func TestNew(t *testing.T) {
|
|||||||
|
|
||||||
for _, tt := range tests {
|
for _, tt := range tests {
|
||||||
t.Run(tt.name, func(t *testing.T) {
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
manager, err := New(tt.storage, tt.metadata, nil, tt.config)
|
manager, err := New(tt.storage, tt.metadata, nil, nil, tt.config)
|
||||||
|
|
||||||
if tt.wantErr {
|
if tt.wantErr {
|
||||||
require.Error(t, err)
|
require.Error(t, err)
|
||||||
@@ -295,15 +295,15 @@ func TestNew(t *testing.T) {
|
|||||||
// TestGet tests cache retrieval with various scenarios
|
// TestGet tests cache retrieval with various scenarios
|
||||||
func TestGet(t *testing.T) {
|
func TestGet(t *testing.T) {
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
|
setupMock func(*MockStorageBackend, *MockMetadataStore)
|
||||||
|
fetchFunc func(context.Context) (io.ReadCloser, string, error)
|
||||||
name string
|
name string
|
||||||
registry string
|
registry string
|
||||||
packageName string
|
packageName string
|
||||||
version string
|
version string
|
||||||
setupMock func(*MockStorageBackend, *MockMetadataStore)
|
errContains string
|
||||||
fetchFunc func(context.Context) (io.ReadCloser, string, error)
|
|
||||||
wantFromCache bool
|
wantFromCache bool
|
||||||
wantErr bool
|
wantErr bool
|
||||||
errContains string
|
|
||||||
}{
|
}{
|
||||||
// GOOD: Cache hit
|
// GOOD: Cache hit
|
||||||
{
|
{
|
||||||
@@ -343,6 +343,7 @@ func TestGet(t *testing.T) {
|
|||||||
s.On("Put", mock.Anything, "npm/lodash/4.17.21", mock.Anything, mock.Anything).Return(nil)
|
s.On("Put", mock.Anything, "npm/lodash/4.17.21", mock.Anything, mock.Anything).Return(nil)
|
||||||
m.On("SavePackage", mock.Anything, mock.Anything).Return(nil)
|
m.On("SavePackage", mock.Anything, mock.Anything).Return(nil)
|
||||||
s.On("Get", mock.Anything, "npm/lodash/4.17.21").Return(io.NopCloser(strings.NewReader("upstream data")), nil)
|
s.On("Get", mock.Anything, "npm/lodash/4.17.21").Return(io.NopCloser(strings.NewReader("upstream data")), nil)
|
||||||
|
m.On("UpdateDownloadCount", mock.Anything, "npm", "lodash", "4.17.21").Return(nil)
|
||||||
},
|
},
|
||||||
fetchFunc: func(ctx context.Context) (io.ReadCloser, string, error) {
|
fetchFunc: func(ctx context.Context) (io.ReadCloser, string, error) {
|
||||||
return io.NopCloser(strings.NewReader("upstream data")), "https://registry.npmjs.org/lodash", nil
|
return io.NopCloser(strings.NewReader("upstream data")), "https://registry.npmjs.org/lodash", nil
|
||||||
@@ -374,6 +375,7 @@ func TestGet(t *testing.T) {
|
|||||||
s.On("Put", mock.Anything, "npm/expired-pkg/1.0.0", mock.Anything, mock.Anything).Return(nil)
|
s.On("Put", mock.Anything, "npm/expired-pkg/1.0.0", mock.Anything, mock.Anything).Return(nil)
|
||||||
m.On("SavePackage", mock.Anything, mock.Anything).Return(nil)
|
m.On("SavePackage", mock.Anything, mock.Anything).Return(nil)
|
||||||
s.On("Get", mock.Anything, "npm/expired-pkg/1.0.0").Return(io.NopCloser(strings.NewReader("refreshed data")), nil)
|
s.On("Get", mock.Anything, "npm/expired-pkg/1.0.0").Return(io.NopCloser(strings.NewReader("refreshed data")), nil)
|
||||||
|
m.On("UpdateDownloadCount", mock.Anything, "npm", "expired-pkg", "1.0.0").Return(nil)
|
||||||
},
|
},
|
||||||
fetchFunc: func(ctx context.Context) (io.ReadCloser, string, error) {
|
fetchFunc: func(ctx context.Context) (io.ReadCloser, string, error) {
|
||||||
return io.NopCloser(strings.NewReader("refreshed data")), "https://registry.npmjs.org/expired-pkg", nil
|
return io.NopCloser(strings.NewReader("refreshed data")), "https://registry.npmjs.org/expired-pkg", nil
|
||||||
@@ -435,6 +437,7 @@ func TestGet(t *testing.T) {
|
|||||||
m.On("SavePackage", mock.Anything, mock.Anything).Return(nil)
|
m.On("SavePackage", mock.Anything, mock.Anything).Return(nil)
|
||||||
// Second Get succeeds (after re-storing)
|
// Second Get succeeds (after re-storing)
|
||||||
s.On("Get", mock.Anything, "npm/inconsistent/1.0.0").Return(io.NopCloser(strings.NewReader("recovered data")), nil).Once()
|
s.On("Get", mock.Anything, "npm/inconsistent/1.0.0").Return(io.NopCloser(strings.NewReader("recovered data")), nil).Once()
|
||||||
|
m.On("UpdateDownloadCount", mock.Anything, "npm", "inconsistent", "1.0.0").Return(nil)
|
||||||
},
|
},
|
||||||
fetchFunc: func(ctx context.Context) (io.ReadCloser, string, error) {
|
fetchFunc: func(ctx context.Context) (io.ReadCloser, string, error) {
|
||||||
return io.NopCloser(strings.NewReader("recovered data")), "https://registry.npmjs.org/inconsistent", nil
|
return io.NopCloser(strings.NewReader("recovered data")), "https://registry.npmjs.org/inconsistent", nil
|
||||||
@@ -489,7 +492,7 @@ func TestGet(t *testing.T) {
|
|||||||
tt.setupMock(mockStorage, mockMetadata)
|
tt.setupMock(mockStorage, mockMetadata)
|
||||||
}
|
}
|
||||||
|
|
||||||
manager, err := New(mockStorage, mockMetadata, nil, Config{
|
manager, err := New(mockStorage, mockMetadata, nil, nil, Config{
|
||||||
DefaultTTL: 24 * time.Hour,
|
DefaultTTL: 24 * time.Hour,
|
||||||
CleanupInterval: 1 * time.Hour,
|
CleanupInterval: 1 * time.Hour,
|
||||||
})
|
})
|
||||||
@@ -523,13 +526,13 @@ func TestGet(t *testing.T) {
|
|||||||
// TestDelete tests package deletion
|
// TestDelete tests package deletion
|
||||||
func TestDelete(t *testing.T) {
|
func TestDelete(t *testing.T) {
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
|
setupMock func(*MockStorageBackend, *MockMetadataStore)
|
||||||
name string
|
name string
|
||||||
registry string
|
registry string
|
||||||
packageName string
|
packageName string
|
||||||
version string
|
version string
|
||||||
setupMock func(*MockStorageBackend, *MockMetadataStore)
|
|
||||||
wantErr bool
|
|
||||||
errContains string
|
errContains string
|
||||||
|
wantErr bool
|
||||||
}{
|
}{
|
||||||
// GOOD: Successful deletion
|
// GOOD: Successful deletion
|
||||||
{
|
{
|
||||||
@@ -615,7 +618,7 @@ func TestDelete(t *testing.T) {
|
|||||||
tt.setupMock(mockStorage, mockMetadata)
|
tt.setupMock(mockStorage, mockMetadata)
|
||||||
}
|
}
|
||||||
|
|
||||||
manager, err := New(mockStorage, mockMetadata, nil, Config{})
|
manager, err := New(mockStorage, mockMetadata, nil, nil, Config{})
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
ctx := context.Background()
|
ctx := context.Background()
|
||||||
@@ -639,10 +642,10 @@ func TestDelete(t *testing.T) {
|
|||||||
// TestHealth tests health check functionality
|
// TestHealth tests health check functionality
|
||||||
func TestHealth(t *testing.T) {
|
func TestHealth(t *testing.T) {
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
name string
|
|
||||||
setupMock func(*MockStorageBackend, *MockMetadataStore)
|
setupMock func(*MockStorageBackend, *MockMetadataStore)
|
||||||
wantErr bool
|
name string
|
||||||
errContains string
|
errContains string
|
||||||
|
wantErr bool
|
||||||
}{
|
}{
|
||||||
// GOOD: Both healthy
|
// GOOD: Both healthy
|
||||||
{
|
{
|
||||||
@@ -692,7 +695,7 @@ func TestHealth(t *testing.T) {
|
|||||||
tt.setupMock(mockStorage, mockMetadata)
|
tt.setupMock(mockStorage, mockMetadata)
|
||||||
}
|
}
|
||||||
|
|
||||||
manager, err := New(mockStorage, mockMetadata, nil, Config{})
|
manager, err := New(mockStorage, mockMetadata, nil, nil, Config{})
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
ctx := context.Background()
|
ctx := context.Background()
|
||||||
@@ -727,7 +730,7 @@ func TestGetStats(t *testing.T) {
|
|||||||
|
|
||||||
mockMetadata.On("GetStats", mock.Anything, "npm").Return(expectedStats, nil)
|
mockMetadata.On("GetStats", mock.Anything, "npm").Return(expectedStats, nil)
|
||||||
|
|
||||||
manager, err := New(mockStorage, mockMetadata, nil, Config{})
|
manager, err := New(mockStorage, mockMetadata, nil, nil, Config{})
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
ctx := context.Background()
|
ctx := context.Background()
|
||||||
@@ -741,8 +744,8 @@ func TestGetStats(t *testing.T) {
|
|||||||
// TestClose tests manager cleanup
|
// TestClose tests manager cleanup
|
||||||
func TestClose(t *testing.T) {
|
func TestClose(t *testing.T) {
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
name string
|
|
||||||
setupMock func(*MockStorageBackend, *MockMetadataStore)
|
setupMock func(*MockStorageBackend, *MockMetadataStore)
|
||||||
|
name string
|
||||||
wantErr bool
|
wantErr bool
|
||||||
}{
|
}{
|
||||||
// GOOD: Clean close
|
// GOOD: Clean close
|
||||||
@@ -792,7 +795,7 @@ func TestClose(t *testing.T) {
|
|||||||
tt.setupMock(mockStorage, mockMetadata)
|
tt.setupMock(mockStorage, mockMetadata)
|
||||||
}
|
}
|
||||||
|
|
||||||
manager, err := New(mockStorage, mockMetadata, nil, Config{})
|
manager, err := New(mockStorage, mockMetadata, nil, nil, Config{})
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
err = manager.Close() // #nosec G104 -- Cleanup, error not critical
|
err = manager.Close() // #nosec G104 -- Cleanup, error not critical
|
||||||
@@ -812,11 +815,11 @@ func TestClose(t *testing.T) {
|
|||||||
// TestEvict tests LRU eviction
|
// TestEvict tests LRU eviction
|
||||||
func TestEvict(t *testing.T) {
|
func TestEvict(t *testing.T) {
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
name string
|
|
||||||
needed int64
|
|
||||||
setupMock func(*MockStorageBackend, *MockMetadataStore)
|
setupMock func(*MockStorageBackend, *MockMetadataStore)
|
||||||
wantErr bool
|
name string
|
||||||
errContains string
|
errContains string
|
||||||
|
needed int64
|
||||||
|
wantErr bool
|
||||||
}{
|
}{
|
||||||
// GOOD: Successful eviction
|
// GOOD: Successful eviction
|
||||||
{
|
{
|
||||||
@@ -881,7 +884,7 @@ func TestEvict(t *testing.T) {
|
|||||||
tt.setupMock(mockStorage, mockMetadata)
|
tt.setupMock(mockStorage, mockMetadata)
|
||||||
}
|
}
|
||||||
|
|
||||||
manager, err := New(mockStorage, mockMetadata, nil, Config{})
|
manager, err := New(mockStorage, mockMetadata, nil, nil, Config{})
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
ctx := context.Background()
|
ctx := context.Background()
|
||||||
@@ -907,7 +910,7 @@ func TestGenerateStorageKey(t *testing.T) {
|
|||||||
mockStorage := &MockStorageBackend{}
|
mockStorage := &MockStorageBackend{}
|
||||||
mockMetadata := &MockMetadataStore{}
|
mockMetadata := &MockMetadataStore{}
|
||||||
|
|
||||||
manager, err := New(mockStorage, mockMetadata, nil, Config{})
|
manager, err := New(mockStorage, mockMetadata, nil, nil, Config{})
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
@@ -954,7 +957,7 @@ func TestConcurrentGet(t *testing.T) {
|
|||||||
io.NopCloser(bytes.NewReader([]byte("test data"))), nil).Maybe()
|
io.NopCloser(bytes.NewReader([]byte("test data"))), nil).Maybe()
|
||||||
mockMetadata.On("UpdateDownloadCount", mock.Anything, "npm", "concurrent", "1.0.0").Return(nil).Maybe()
|
mockMetadata.On("UpdateDownloadCount", mock.Anything, "npm", "concurrent", "1.0.0").Return(nil).Maybe()
|
||||||
|
|
||||||
manager, err := New(mockStorage, mockMetadata, nil, Config{})
|
manager, err := New(mockStorage, mockMetadata, nil, nil, Config{})
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
ctx := context.Background()
|
ctx := context.Background()
|
||||||
|
|||||||
+1
-128
@@ -4,10 +4,7 @@ import (
|
|||||||
"crypto/md5" // #nosec G501 -- MD5 used for ETag generation, not cryptographic security
|
"crypto/md5" // #nosec G501 -- MD5 used for ETag generation, not cryptographic security
|
||||||
"encoding/hex"
|
"encoding/hex"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
|
||||||
"net/http"
|
"net/http"
|
||||||
"strconv"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/rs/zerolog/log"
|
"github.com/rs/zerolog/log"
|
||||||
)
|
)
|
||||||
@@ -214,38 +211,11 @@ func (m *Middleware) generateETag(body []byte) string {
|
|||||||
return `"` + hex.EncodeToString(hash[:]) + `"`
|
return `"` + hex.EncodeToString(hash[:]) + `"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// SetLastModified sets the Last-Modified header
|
|
||||||
func SetLastModified(w http.ResponseWriter, t time.Time) {
|
|
||||||
w.Header().Set("Last-Modified", t.UTC().Format(http.TimeFormat))
|
|
||||||
}
|
|
||||||
|
|
||||||
// SetCacheControl sets a custom Cache-Control header
|
|
||||||
func SetCacheControl(w http.ResponseWriter, cc CacheControl) {
|
|
||||||
w.Header().Set("Cache-Control", cc.String())
|
|
||||||
}
|
|
||||||
|
|
||||||
// SetNoCache sets headers to prevent caching
|
|
||||||
func SetNoCache(w http.ResponseWriter) {
|
|
||||||
w.Header().Set("Cache-Control", "no-cache, no-store, must-revalidate")
|
|
||||||
w.Header().Set("Pragma", "no-cache")
|
|
||||||
w.Header().Set("Expires", "0")
|
|
||||||
}
|
|
||||||
|
|
||||||
// SetImmutable sets headers for immutable content (content-addressed files)
|
|
||||||
func SetImmutable(w http.ResponseWriter, maxAge int) {
|
|
||||||
cc := CacheControl{
|
|
||||||
Public: true,
|
|
||||||
MaxAge: maxAge,
|
|
||||||
Immutable: true,
|
|
||||||
}
|
|
||||||
w.Header().Set("Cache-Control", cc.String())
|
|
||||||
}
|
|
||||||
|
|
||||||
// responseWriter wraps http.ResponseWriter to capture response
|
// responseWriter wraps http.ResponseWriter to capture response
|
||||||
type responseWriter struct {
|
type responseWriter struct {
|
||||||
http.ResponseWriter
|
http.ResponseWriter
|
||||||
statusCode int
|
|
||||||
body []byte
|
body []byte
|
||||||
|
statusCode int
|
||||||
}
|
}
|
||||||
|
|
||||||
func (rw *responseWriter) WriteHeader(statusCode int) {
|
func (rw *responseWriter) WriteHeader(statusCode int) {
|
||||||
@@ -261,100 +231,3 @@ func (rw *responseWriter) Write(b []byte) (int, error) {
|
|||||||
rw.body = append(rw.body, b...)
|
rw.body = append(rw.body, b...)
|
||||||
return rw.ResponseWriter.Write(b)
|
return rw.ResponseWriter.Write(b)
|
||||||
}
|
}
|
||||||
|
|
||||||
// HandleRange handles HTTP Range requests for partial content
|
|
||||||
func HandleRange(w http.ResponseWriter, r *http.Request, content io.ReadSeeker, size int64, modTime time.Time) error {
|
|
||||||
// Set Last-Modified header
|
|
||||||
SetLastModified(w, modTime)
|
|
||||||
|
|
||||||
// Check for Range header
|
|
||||||
rangeHeader := r.Header.Get("Range")
|
|
||||||
if rangeHeader == "" {
|
|
||||||
// No range request - serve full content
|
|
||||||
w.Header().Set("Content-Length", strconv.FormatInt(size, 10))
|
|
||||||
w.Header().Set("Accept-Ranges", "bytes")
|
|
||||||
w.WriteHeader(http.StatusOK)
|
|
||||||
_, err := io.Copy(w, content)
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
// Parse range header (simplified - only handles single range)
|
|
||||||
// Format: bytes=start-end
|
|
||||||
var start, end int64
|
|
||||||
n, err := fmt.Sscanf(rangeHeader, "bytes=%d-%d", &start, &end)
|
|
||||||
if err != nil || n != 2 {
|
|
||||||
// Invalid range - serve full content
|
|
||||||
w.Header().Set("Content-Length", strconv.FormatInt(size, 10))
|
|
||||||
w.Header().Set("Accept-Ranges", "bytes")
|
|
||||||
w.WriteHeader(http.StatusOK)
|
|
||||||
_, err := io.Copy(w, content)
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
// Validate range
|
|
||||||
if start < 0 || start >= size || end < start || end >= size {
|
|
||||||
w.Header().Set("Content-Range", fmt.Sprintf("bytes */%d", size))
|
|
||||||
w.WriteHeader(http.StatusRequestedRangeNotSatisfiable)
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// Seek to start position
|
|
||||||
if _, err := content.Seek(start, io.SeekStart); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
// Calculate content length
|
|
||||||
contentLength := end - start + 1
|
|
||||||
|
|
||||||
// Set headers for partial content
|
|
||||||
w.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", start, end, size))
|
|
||||||
w.Header().Set("Content-Length", strconv.FormatInt(contentLength, 10))
|
|
||||||
w.Header().Set("Accept-Ranges", "bytes")
|
|
||||||
w.WriteHeader(http.StatusPartialContent)
|
|
||||||
|
|
||||||
// Copy range to response
|
|
||||||
_, err = io.CopyN(w, content, contentLength)
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
// DefaultCacheControl returns sensible defaults for different content types
|
|
||||||
func DefaultCacheControl(contentType string, versioned bool) CacheControl {
|
|
||||||
if versioned {
|
|
||||||
// Content-addressed or versioned resources can be cached forever
|
|
||||||
return CacheControl{
|
|
||||||
Public: true,
|
|
||||||
MaxAge: 31536000, // 1 year
|
|
||||||
Immutable: true,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Default caching based on content type
|
|
||||||
switch contentType {
|
|
||||||
case "application/json":
|
|
||||||
return CacheControl{
|
|
||||||
Public: true,
|
|
||||||
MaxAge: 3600, // 1 hour
|
|
||||||
SMaxAge: 7200, // 2 hours for shared caches
|
|
||||||
}
|
|
||||||
case "application/octet-stream", "application/x-gzip", "application/zip":
|
|
||||||
// Binary packages
|
|
||||||
return CacheControl{
|
|
||||||
Public: true,
|
|
||||||
MaxAge: 86400, // 1 day
|
|
||||||
SMaxAge: 604800, // 1 week for shared caches
|
|
||||||
}
|
|
||||||
case "text/html":
|
|
||||||
// HTML should revalidate
|
|
||||||
return CacheControl{
|
|
||||||
Public: true,
|
|
||||||
MaxAge: 0,
|
|
||||||
MustRevalidate: true,
|
|
||||||
}
|
|
||||||
default:
|
|
||||||
return CacheControl{
|
|
||||||
Public: true,
|
|
||||||
MaxAge: 3600, // 1 hour default
|
|
||||||
SMaxAge: 7200,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -0,0 +1,299 @@
|
|||||||
|
package cdn
|
||||||
|
|
||||||
|
import (
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/stretchr/testify/suite"
|
||||||
|
)
|
||||||
|
|
||||||
|
type CDNMiddlewareTestSuite struct {
|
||||||
|
suite.Suite
|
||||||
|
middleware *Middleware
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *CDNMiddlewareTestSuite) SetupTest() {
|
||||||
|
s.middleware = NewMiddleware(Config{
|
||||||
|
DefaultCacheControl: CacheControl{
|
||||||
|
Public: true,
|
||||||
|
MaxAge: 3600,
|
||||||
|
SMaxAge: 7200,
|
||||||
|
},
|
||||||
|
EnableETag: true,
|
||||||
|
EnableVary: true,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCDNMiddlewareTestSuite(t *testing.T) {
|
||||||
|
suite.Run(t, new(CDNMiddlewareTestSuite))
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *CDNMiddlewareTestSuite) TestCacheControlHeader() {
|
||||||
|
handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
w.WriteHeader(http.StatusOK)
|
||||||
|
w.Write([]byte("test response"))
|
||||||
|
})
|
||||||
|
|
||||||
|
wrappedHandler := s.middleware.Handler(handler)
|
||||||
|
|
||||||
|
req := httptest.NewRequest("GET", "/test", nil)
|
||||||
|
w := httptest.NewRecorder()
|
||||||
|
|
||||||
|
wrappedHandler.ServeHTTP(w, req)
|
||||||
|
|
||||||
|
s.Equal(http.StatusOK, w.Code)
|
||||||
|
s.Contains(w.Header().Get("Cache-Control"), "public")
|
||||||
|
s.Contains(w.Header().Get("Cache-Control"), "max-age=3600")
|
||||||
|
s.Contains(w.Header().Get("Cache-Control"), "s-maxage=7200")
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *CDNMiddlewareTestSuite) TestETagGeneration() {
|
||||||
|
handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
w.WriteHeader(http.StatusOK)
|
||||||
|
w.Write([]byte("test response content"))
|
||||||
|
})
|
||||||
|
|
||||||
|
wrappedHandler := s.middleware.Handler(handler)
|
||||||
|
|
||||||
|
req := httptest.NewRequest("GET", "/test", nil)
|
||||||
|
w := httptest.NewRecorder()
|
||||||
|
|
||||||
|
wrappedHandler.ServeHTTP(w, req)
|
||||||
|
|
||||||
|
s.Equal(http.StatusOK, w.Code)
|
||||||
|
etag := w.Header().Get("ETag")
|
||||||
|
s.NotEmpty(etag)
|
||||||
|
s.True(len(etag) > 0)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *CDNMiddlewareTestSuite) TestETagConsistencyAcrossRequests() {
|
||||||
|
responseBody := []byte("test response content")
|
||||||
|
handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
w.WriteHeader(http.StatusOK)
|
||||||
|
w.Write(responseBody)
|
||||||
|
})
|
||||||
|
|
||||||
|
wrappedHandler := s.middleware.Handler(handler)
|
||||||
|
|
||||||
|
// First request to get ETag
|
||||||
|
req1 := httptest.NewRequest("GET", "/test", nil)
|
||||||
|
w1 := httptest.NewRecorder()
|
||||||
|
wrappedHandler.ServeHTTP(w1, req1)
|
||||||
|
etag := w1.Header().Get("ETag")
|
||||||
|
s.NotEmpty(etag)
|
||||||
|
s.Equal(http.StatusOK, w1.Code)
|
||||||
|
|
||||||
|
// Verify ETag is consistent for same content
|
||||||
|
req2 := httptest.NewRequest("GET", "/test", nil)
|
||||||
|
w2 := httptest.NewRecorder()
|
||||||
|
wrappedHandler.ServeHTTP(w2, req2)
|
||||||
|
etag2 := w2.Header().Get("ETag")
|
||||||
|
s.Equal(etag, etag2, "ETag should be consistent for same content")
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *CDNMiddlewareTestSuite) TestVaryHeader() {
|
||||||
|
handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
w.WriteHeader(http.StatusOK)
|
||||||
|
w.Write([]byte("test"))
|
||||||
|
})
|
||||||
|
|
||||||
|
wrappedHandler := s.middleware.Handler(handler)
|
||||||
|
|
||||||
|
req := httptest.NewRequest("GET", "/test", nil)
|
||||||
|
req.Header.Set("Accept-Encoding", "gzip")
|
||||||
|
req.Header.Set("Accept", "application/json")
|
||||||
|
req.Header.Set("Authorization", "Bearer token")
|
||||||
|
|
||||||
|
w := httptest.NewRecorder()
|
||||||
|
wrappedHandler.ServeHTTP(w, req)
|
||||||
|
|
||||||
|
vary := w.Header().Get("Vary")
|
||||||
|
s.NotEmpty(vary)
|
||||||
|
s.Contains(vary, "Accept-Encoding")
|
||||||
|
s.Contains(vary, "Authorization")
|
||||||
|
s.Contains(vary, "Accept")
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *CDNMiddlewareTestSuite) TestCacheControlString() {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
expected string
|
||||||
|
cc CacheControl
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "public with max-age",
|
||||||
|
cc: CacheControl{
|
||||||
|
Public: true,
|
||||||
|
MaxAge: 3600,
|
||||||
|
},
|
||||||
|
expected: "public, max-age=3600",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "private with no-cache",
|
||||||
|
cc: CacheControl{
|
||||||
|
Private: true,
|
||||||
|
NoCache: true,
|
||||||
|
},
|
||||||
|
expected: "private, no-cache",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "immutable",
|
||||||
|
cc: CacheControl{
|
||||||
|
Public: true,
|
||||||
|
MaxAge: 31536000,
|
||||||
|
Immutable: true,
|
||||||
|
},
|
||||||
|
expected: "public, immutable, max-age=31536000",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "no-store",
|
||||||
|
cc: CacheControl{
|
||||||
|
NoStore: true,
|
||||||
|
},
|
||||||
|
expected: "no-store",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "must-revalidate",
|
||||||
|
cc: CacheControl{
|
||||||
|
Public: true,
|
||||||
|
MustRevalidate: true,
|
||||||
|
},
|
||||||
|
expected: "public, must-revalidate",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "s-maxage",
|
||||||
|
cc: CacheControl{
|
||||||
|
Public: true,
|
||||||
|
MaxAge: 3600,
|
||||||
|
SMaxAge: 7200,
|
||||||
|
},
|
||||||
|
expected: "public, max-age=3600, s-maxage=7200",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "stale-while-revalidate",
|
||||||
|
cc: CacheControl{
|
||||||
|
Public: true,
|
||||||
|
MaxAge: 3600,
|
||||||
|
StaleWhileRevalidate: 86400,
|
||||||
|
},
|
||||||
|
expected: "public, max-age=3600, stale-while-revalidate=86400",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
s.Run(tt.name, func() {
|
||||||
|
result := tt.cc.String()
|
||||||
|
// Check that all expected parts are in the result
|
||||||
|
for _, part := range splitCacheControl(tt.expected) {
|
||||||
|
s.Contains(result, part)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *CDNMiddlewareTestSuite) TestGenerateETag() {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
body []byte
|
||||||
|
expected bool // true if ETag should be generated
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "non-empty body",
|
||||||
|
body: []byte("test content"),
|
||||||
|
expected: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "empty body",
|
||||||
|
body: []byte{},
|
||||||
|
expected: true, // Empty body still generates ETag (MD5 of empty string)
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "nil body",
|
||||||
|
body: nil,
|
||||||
|
expected: false,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
s.Run(tt.name, func() {
|
||||||
|
etag := s.middleware.generateETag(tt.body)
|
||||||
|
if tt.expected {
|
||||||
|
s.NotEmpty(etag)
|
||||||
|
s.True(len(etag) > 2) // Should be quoted
|
||||||
|
} else {
|
||||||
|
s.Empty(etag)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *CDNMiddlewareTestSuite) TestETagConsistency() {
|
||||||
|
// Same content should produce same ETag
|
||||||
|
body := []byte("consistent content")
|
||||||
|
etag1 := s.middleware.generateETag(body)
|
||||||
|
etag2 := s.middleware.generateETag(body)
|
||||||
|
|
||||||
|
s.Equal(etag1, etag2)
|
||||||
|
|
||||||
|
// Different content should produce different ETag
|
||||||
|
body2 := []byte("different content")
|
||||||
|
etag3 := s.middleware.generateETag(body2)
|
||||||
|
|
||||||
|
s.NotEqual(etag1, etag3)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *CDNMiddlewareTestSuite) TestNoCacheFor4xxErrors() {
|
||||||
|
handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
w.WriteHeader(http.StatusNotFound)
|
||||||
|
w.Write([]byte("not found"))
|
||||||
|
})
|
||||||
|
|
||||||
|
wrappedHandler := s.middleware.Handler(handler)
|
||||||
|
|
||||||
|
req := httptest.NewRequest("GET", "/test", nil)
|
||||||
|
w := httptest.NewRecorder()
|
||||||
|
|
||||||
|
wrappedHandler.ServeHTTP(w, req)
|
||||||
|
|
||||||
|
s.Equal(http.StatusNotFound, w.Code)
|
||||||
|
// 4xx errors should not have cache headers applied
|
||||||
|
// (based on the middleware only applying headers for 2xx status codes)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *CDNMiddlewareTestSuite) TestNoCacheFor5xxErrors() {
|
||||||
|
handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
w.WriteHeader(http.StatusInternalServerError)
|
||||||
|
w.Write([]byte("error"))
|
||||||
|
})
|
||||||
|
|
||||||
|
wrappedHandler := s.middleware.Handler(handler)
|
||||||
|
|
||||||
|
req := httptest.NewRequest("GET", "/test", nil)
|
||||||
|
w := httptest.NewRecorder()
|
||||||
|
|
||||||
|
wrappedHandler.ServeHTTP(w, req)
|
||||||
|
|
||||||
|
s.Equal(http.StatusInternalServerError, w.Code)
|
||||||
|
// 5xx errors should not have cache headers applied
|
||||||
|
}
|
||||||
|
|
||||||
|
// Helper function to split cache-control string
|
||||||
|
func splitCacheControl(s string) []string {
|
||||||
|
var parts []string
|
||||||
|
current := ""
|
||||||
|
for _, char := range s {
|
||||||
|
if char == ',' {
|
||||||
|
if current != "" {
|
||||||
|
parts = append(parts, current)
|
||||||
|
current = ""
|
||||||
|
}
|
||||||
|
} else if char != ' ' {
|
||||||
|
current += string(char)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if current != "" {
|
||||||
|
parts = append(parts, current)
|
||||||
|
}
|
||||||
|
return parts
|
||||||
|
}
|
||||||
+70
-43
@@ -7,42 +7,42 @@ import (
|
|||||||
|
|
||||||
// Config is the main configuration struct
|
// Config is the main configuration struct
|
||||||
type Config struct {
|
type Config struct {
|
||||||
Server ServerConfig `mapstructure:"server" json:"server"`
|
|
||||||
Storage StorageConfig `mapstructure:"storage" json:"storage"`
|
Storage StorageConfig `mapstructure:"storage" json:"storage"`
|
||||||
Metadata MetadataConfig `mapstructure:"metadata" json:"metadata"`
|
|
||||||
Cache CacheConfig `mapstructure:"cache" json:"cache"`
|
Cache CacheConfig `mapstructure:"cache" json:"cache"`
|
||||||
Security SecurityConfig `mapstructure:"security" json:"security"`
|
Security SecurityConfig `mapstructure:"security" json:"security"`
|
||||||
Auth AuthConfig `mapstructure:"auth" json:"auth"`
|
Metadata MetadataConfig `mapstructure:"metadata" json:"metadata"`
|
||||||
Network NetworkConfig `mapstructure:"network" json:"network"`
|
|
||||||
Logging LoggingConfig `mapstructure:"logging" json:"logging"`
|
|
||||||
Handlers HandlersConfig `mapstructure:"handlers" json:"handlers"`
|
Handlers HandlersConfig `mapstructure:"handlers" json:"handlers"`
|
||||||
|
Server ServerConfig `mapstructure:"server" json:"server"`
|
||||||
|
Logging LoggingConfig `mapstructure:"logging" json:"logging"`
|
||||||
|
Network NetworkConfig `mapstructure:"network" json:"network"`
|
||||||
|
Auth AuthConfig `mapstructure:"auth" json:"auth"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// ServerConfig contains HTTP server configuration
|
// ServerConfig contains HTTP server configuration
|
||||||
type ServerConfig struct {
|
type ServerConfig struct {
|
||||||
|
TLS TLSConfig `mapstructure:"tls" json:"tls"`
|
||||||
Host string `mapstructure:"host" json:"host"`
|
Host string `mapstructure:"host" json:"host"`
|
||||||
Port int `mapstructure:"port" json:"port"`
|
Port int `mapstructure:"port" json:"port"`
|
||||||
ReadTimeout time.Duration `mapstructure:"read_timeout" json:"read_timeout"`
|
ReadTimeout time.Duration `mapstructure:"read_timeout" json:"read_timeout"`
|
||||||
WriteTimeout time.Duration `mapstructure:"write_timeout" json:"write_timeout"`
|
WriteTimeout time.Duration `mapstructure:"write_timeout" json:"write_timeout"`
|
||||||
IdleTimeout time.Duration `mapstructure:"idle_timeout" json:"idle_timeout"`
|
IdleTimeout time.Duration `mapstructure:"idle_timeout" json:"idle_timeout"`
|
||||||
TLS TLSConfig `mapstructure:"tls" json:"tls"`
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// TLSConfig contains TLS/HTTPS configuration
|
// TLSConfig contains TLS/HTTPS configuration
|
||||||
type TLSConfig struct {
|
type TLSConfig struct {
|
||||||
Enabled bool `mapstructure:"enabled" json:"enabled"`
|
|
||||||
CertFile string `mapstructure:"cert_file" json:"cert_file"`
|
CertFile string `mapstructure:"cert_file" json:"cert_file"`
|
||||||
KeyFile string `mapstructure:"key_file" json:"key_file"`
|
KeyFile string `mapstructure:"key_file" json:"key_file"`
|
||||||
|
Enabled bool `mapstructure:"enabled" json:"enabled"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// StorageConfig contains storage backend configuration
|
// StorageConfig contains storage backend configuration
|
||||||
type StorageConfig struct {
|
type StorageConfig struct {
|
||||||
Backend string `mapstructure:"backend" json:"backend"` // filesystem, s3, smb, nfs
|
Options map[string]interface{} `mapstructure:"options" json:"options"`
|
||||||
|
SMB SMBConfig `mapstructure:"smb" json:"smb"`
|
||||||
|
Backend string `mapstructure:"backend" json:"backend"`
|
||||||
Path string `mapstructure:"path" json:"path"`
|
Path string `mapstructure:"path" json:"path"`
|
||||||
Filesystem FilesystemConfig `mapstructure:"filesystem" json:"filesystem"`
|
Filesystem FilesystemConfig `mapstructure:"filesystem" json:"filesystem"`
|
||||||
S3 S3Config `mapstructure:"s3" json:"s3"`
|
S3 S3Config `mapstructure:"s3" json:"s3"`
|
||||||
SMB SMBConfig `mapstructure:"smb" json:"smb"`
|
|
||||||
Options map[string]interface{} `mapstructure:"options" json:"options"`
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// FilesystemConfig contains local filesystem storage configuration
|
// FilesystemConfig contains local filesystem storage configuration
|
||||||
@@ -52,12 +52,14 @@ type FilesystemConfig struct {
|
|||||||
|
|
||||||
// S3Config contains S3-compatible storage configuration
|
// S3Config contains S3-compatible storage configuration
|
||||||
type S3Config struct {
|
type S3Config struct {
|
||||||
Endpoint string `mapstructure:"endpoint" json:"endpoint"`
|
Endpoint string `mapstructure:"endpoint" json:"endpoint"` // Optional: for MinIO, etc.
|
||||||
Region string `mapstructure:"region" json:"region"`
|
Region string `mapstructure:"region" json:"region"` // AWS region (e.g., us-east-1)
|
||||||
Bucket string `mapstructure:"bucket" json:"bucket"`
|
Bucket string `mapstructure:"bucket" json:"bucket"` // S3 bucket name
|
||||||
AccessKeyID string `mapstructure:"access_key_id" json:"access_key_id"`
|
Prefix string `mapstructure:"prefix" json:"prefix"` // Optional: key prefix
|
||||||
SecretAccessKey string `mapstructure:"secret_access_key" json:"-"` // Don't serialize secrets
|
AccessKeyID string `mapstructure:"access_key_id" json:"access_key_id"` // AWS access key
|
||||||
UseSSL bool `mapstructure:"use_ssl" json:"use_ssl"`
|
SecretAccessKey string `mapstructure:"secret_access_key" json:"-"` // AWS secret key (not serialized)
|
||||||
|
ForcePathStyle bool `mapstructure:"force_path_style" json:"force_path_style"` // For MinIO compatibility
|
||||||
|
UseSSL bool `mapstructure:"use_ssl" json:"use_ssl"` // Deprecated: use endpoint with https://
|
||||||
}
|
}
|
||||||
|
|
||||||
// SMBConfig contains SMB/CIFS storage configuration
|
// SMBConfig contains SMB/CIFS storage configuration
|
||||||
@@ -71,10 +73,17 @@ type SMBConfig struct {
|
|||||||
|
|
||||||
// MetadataConfig contains metadata store configuration
|
// MetadataConfig contains metadata store configuration
|
||||||
type MetadataConfig struct {
|
type MetadataConfig struct {
|
||||||
Backend string `mapstructure:"backend" json:"backend"` // sqlite, postgresql, file
|
Backend string `mapstructure:"backend" json:"backend"`
|
||||||
Connection string `mapstructure:"connection" json:"connection"`
|
Connection string `mapstructure:"connection" json:"connection"`
|
||||||
SQLite SQLiteConfig `mapstructure:"sqlite" json:"sqlite"`
|
SQLite SQLiteConfig `mapstructure:"sqlite" json:"sqlite"`
|
||||||
PostgreSQL PostgreSQLConfig `mapstructure:"postgresql" json:"postgresql"`
|
PostgreSQL PostgreSQLConfig `mapstructure:"postgresql" json:"postgresql"`
|
||||||
|
MySQL MySQLConfig `mapstructure:"mysql" json:"mysql"`
|
||||||
|
|
||||||
|
// GORM-specific settings
|
||||||
|
MaxOpenConns int `mapstructure:"max_open_conns" json:"max_open_conns"`
|
||||||
|
MaxIdleConns int `mapstructure:"max_idle_conns" json:"max_idle_conns"`
|
||||||
|
ConnMaxLifetime int `mapstructure:"conn_max_lifetime" json:"conn_max_lifetime"` // seconds
|
||||||
|
LogLevel string `mapstructure:"log_level" json:"log_level"` // "silent", "error", "warn", "info"
|
||||||
}
|
}
|
||||||
|
|
||||||
// SQLiteConfig contains SQLite-specific configuration
|
// SQLiteConfig contains SQLite-specific configuration
|
||||||
@@ -89,30 +98,41 @@ type PostgreSQLConfig struct {
|
|||||||
Port int `mapstructure:"port" json:"port"`
|
Port int `mapstructure:"port" json:"port"`
|
||||||
Database string `mapstructure:"database" json:"database"`
|
Database string `mapstructure:"database" json:"database"`
|
||||||
User string `mapstructure:"user" json:"user"`
|
User string `mapstructure:"user" json:"user"`
|
||||||
Password string `mapstructure:"password" json:"-"` // Don't serialize secrets
|
Password string `mapstructure:"password" json:"-"`
|
||||||
SSLMode string `mapstructure:"ssl_mode" json:"ssl_mode"`
|
SSLMode string `mapstructure:"ssl_mode" json:"ssl_mode"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// MySQLConfig contains MySQL/MariaDB-specific configuration
|
||||||
|
type MySQLConfig struct {
|
||||||
|
Host string `mapstructure:"host" json:"host"`
|
||||||
|
Port int `mapstructure:"port" json:"port"`
|
||||||
|
Database string `mapstructure:"database" json:"database"`
|
||||||
|
User string `mapstructure:"user" json:"user"`
|
||||||
|
Password string `mapstructure:"password" json:"-"` // Don't serialize
|
||||||
|
Charset string `mapstructure:"charset" json:"charset"`
|
||||||
|
ParseTime bool `mapstructure:"parse_time" json:"parse_time"`
|
||||||
|
}
|
||||||
|
|
||||||
// CacheConfig contains cache management configuration
|
// CacheConfig contains cache management configuration
|
||||||
type CacheConfig struct {
|
type CacheConfig struct {
|
||||||
|
TTLOverrides map[string]time.Duration `mapstructure:"ttl_overrides" json:"ttl_overrides"`
|
||||||
DefaultTTL time.Duration `mapstructure:"default_ttl" json:"default_ttl"`
|
DefaultTTL time.Duration `mapstructure:"default_ttl" json:"default_ttl"`
|
||||||
CleanupInterval time.Duration `mapstructure:"cleanup_interval" json:"cleanup_interval"`
|
CleanupInterval time.Duration `mapstructure:"cleanup_interval" json:"cleanup_interval"`
|
||||||
MaxSizeBytes int64 `mapstructure:"max_size_bytes" json:"max_size_bytes"`
|
MaxSizeBytes int64 `mapstructure:"max_size_bytes" json:"max_size_bytes"`
|
||||||
PerProjectQuota int64 `mapstructure:"per_project_quota" json:"per_project_quota"`
|
PerProjectQuota int64 `mapstructure:"per_project_quota" json:"per_project_quota"`
|
||||||
TTLOverrides map[string]time.Duration `mapstructure:"ttl_overrides" json:"ttl_overrides"` // Per ecosystem
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// SecurityConfig contains security scanning configuration
|
// SecurityConfig contains security scanning configuration
|
||||||
type SecurityConfig struct {
|
type SecurityConfig struct {
|
||||||
Enabled bool `mapstructure:"enabled" json:"enabled"`
|
|
||||||
ScanOnDownload bool `mapstructure:"scan_on_download" json:"scan_on_download"` // Scan packages on first download
|
|
||||||
RescanInterval time.Duration `mapstructure:"rescan_interval" json:"rescan_interval"` // How often to re-scan (e.g., 24h, 168h for weekly)
|
|
||||||
BlockOnSeverity string `mapstructure:"block_on_severity" json:"block_on_severity"` // none, low, medium, high, critical
|
|
||||||
BlockThresholds VulnerabilityThresholds `mapstructure:"block_thresholds" json:"block_thresholds"` // Max vulns per severity before blocking
|
|
||||||
UpdateDBOnStartup bool `mapstructure:"update_db_on_startup" json:"update_db_on_startup"` // Update vulnerability databases on startup
|
|
||||||
AllowedPackages []string `mapstructure:"allowed_packages" json:"allowed_packages"` // Packages that bypass security checks (format: "registry/name@version" or "registry/name")
|
|
||||||
IgnoredCVEs []string `mapstructure:"ignored_cves" json:"ignored_cves"` // CVE IDs to ignore globally (e.g., "CVE-2021-23337")
|
|
||||||
Scanners ScannersConfig `mapstructure:"scanners" json:"scanners"`
|
Scanners ScannersConfig `mapstructure:"scanners" json:"scanners"`
|
||||||
|
BlockOnSeverity string `mapstructure:"block_on_severity" json:"block_on_severity"`
|
||||||
|
AllowedPackages []string `mapstructure:"allowed_packages" json:"allowed_packages"`
|
||||||
|
IgnoredCVEs []string `mapstructure:"ignored_cves" json:"ignored_cves"`
|
||||||
|
BlockThresholds VulnerabilityThresholds `mapstructure:"block_thresholds" json:"block_thresholds"`
|
||||||
|
RescanInterval time.Duration `mapstructure:"rescan_interval" json:"rescan_interval"`
|
||||||
|
Enabled bool `mapstructure:"enabled" json:"enabled"`
|
||||||
|
ScanOnDownload bool `mapstructure:"scan_on_download" json:"scan_on_download"`
|
||||||
|
UpdateDBOnStartup bool `mapstructure:"update_db_on_startup" json:"update_db_on_startup"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// VulnerabilityThresholds defines max allowed vulnerabilities per severity
|
// VulnerabilityThresholds defines max allowed vulnerabilities per severity
|
||||||
@@ -126,36 +146,36 @@ type VulnerabilityThresholds struct {
|
|||||||
// ScannersConfig contains individual scanner configurations
|
// ScannersConfig contains individual scanner configurations
|
||||||
type ScannersConfig struct {
|
type ScannersConfig struct {
|
||||||
Trivy TrivyConfig `mapstructure:"trivy" json:"trivy"`
|
Trivy TrivyConfig `mapstructure:"trivy" json:"trivy"`
|
||||||
OSV OSVConfig `mapstructure:"osv" json:"osv"`
|
GHSA GHSAConfig `mapstructure:"ghsa" json:"ghsa"`
|
||||||
Static StaticConfig `mapstructure:"static" json:"static"`
|
Static StaticConfig `mapstructure:"static" json:"static"`
|
||||||
|
OSV OSVConfig `mapstructure:"osv" json:"osv"`
|
||||||
Grype GrypeConfig `mapstructure:"grype" json:"grype"`
|
Grype GrypeConfig `mapstructure:"grype" json:"grype"`
|
||||||
Govulncheck GovulncheckConfig `mapstructure:"govulncheck" json:"govulncheck"`
|
Govulncheck GovulncheckConfig `mapstructure:"govulncheck" json:"govulncheck"`
|
||||||
NpmAudit NpmAuditConfig `mapstructure:"npm_audit" json:"npm_audit"`
|
NpmAudit NpmAuditConfig `mapstructure:"npm_audit" json:"npm_audit"`
|
||||||
PipAudit PipAuditConfig `mapstructure:"pip_audit" json:"pip_audit"`
|
PipAudit PipAuditConfig `mapstructure:"pip_audit" json:"pip_audit"`
|
||||||
GHSA GHSAConfig `mapstructure:"ghsa" json:"ghsa"`
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// TrivyConfig contains Trivy scanner configuration
|
// TrivyConfig contains Trivy scanner configuration
|
||||||
type TrivyConfig struct {
|
type TrivyConfig struct {
|
||||||
Enabled bool `mapstructure:"enabled" json:"enabled"`
|
|
||||||
Timeout time.Duration `mapstructure:"timeout" json:"timeout"`
|
|
||||||
CacheDB string `mapstructure:"cache_db" json:"cache_db"`
|
CacheDB string `mapstructure:"cache_db" json:"cache_db"`
|
||||||
|
Timeout time.Duration `mapstructure:"timeout" json:"timeout"`
|
||||||
|
Enabled bool `mapstructure:"enabled" json:"enabled"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// OSVConfig contains OSV scanner configuration
|
// OSVConfig contains OSV scanner configuration
|
||||||
type OSVConfig struct {
|
type OSVConfig struct {
|
||||||
Enabled bool `mapstructure:"enabled" json:"enabled"`
|
|
||||||
APIURL string `mapstructure:"api_url" json:"api_url"`
|
APIURL string `mapstructure:"api_url" json:"api_url"`
|
||||||
Timeout time.Duration `mapstructure:"timeout" json:"timeout"`
|
Timeout time.Duration `mapstructure:"timeout" json:"timeout"`
|
||||||
|
Enabled bool `mapstructure:"enabled" json:"enabled"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// StaticConfig contains static analysis configuration
|
// StaticConfig contains static analysis configuration
|
||||||
type StaticConfig struct {
|
type StaticConfig struct {
|
||||||
Enabled bool `mapstructure:"enabled" json:"enabled"`
|
AllowedLicenses []string `mapstructure:"allowed_licenses" json:"allowed_licenses"`
|
||||||
MaxPackageSize int64 `mapstructure:"max_package_size" json:"max_package_size"`
|
MaxPackageSize int64 `mapstructure:"max_package_size" json:"max_package_size"`
|
||||||
|
Enabled bool `mapstructure:"enabled" json:"enabled"`
|
||||||
CheckChecksums bool `mapstructure:"check_checksums" json:"check_checksums"`
|
CheckChecksums bool `mapstructure:"check_checksums" json:"check_checksums"`
|
||||||
BlockSuspicious bool `mapstructure:"block_suspicious" json:"block_suspicious"`
|
BlockSuspicious bool `mapstructure:"block_suspicious" json:"block_suspicious"`
|
||||||
AllowedLicenses []string `mapstructure:"allowed_licenses" json:"allowed_licenses"`
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// GrypeConfig contains Grype scanner configuration
|
// GrypeConfig contains Grype scanner configuration
|
||||||
@@ -184,16 +204,16 @@ type PipAuditConfig struct {
|
|||||||
|
|
||||||
// GHSAConfig contains GitHub Advisory Database scanner configuration
|
// GHSAConfig contains GitHub Advisory Database scanner configuration
|
||||||
type GHSAConfig struct {
|
type GHSAConfig struct {
|
||||||
Enabled bool `mapstructure:"enabled" json:"enabled"`
|
Token string `mapstructure:"token" json:"-"`
|
||||||
Timeout time.Duration `mapstructure:"timeout" json:"timeout"`
|
Timeout time.Duration `mapstructure:"timeout" json:"timeout"`
|
||||||
Token string `mapstructure:"token" json:"-"` // GitHub token for higher rate limits (don't serialize)
|
Enabled bool `mapstructure:"enabled" json:"enabled"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// AuthConfig contains authentication configuration
|
// AuthConfig contains authentication configuration
|
||||||
type AuthConfig struct {
|
type AuthConfig struct {
|
||||||
Enabled bool `mapstructure:"enabled" json:"enabled"`
|
|
||||||
KeyExpiration time.Duration `mapstructure:"key_expiration" json:"key_expiration"`
|
KeyExpiration time.Duration `mapstructure:"key_expiration" json:"key_expiration"`
|
||||||
BcryptCost int `mapstructure:"bcrypt_cost" json:"bcrypt_cost"`
|
BcryptCost int `mapstructure:"bcrypt_cost" json:"bcrypt_cost"`
|
||||||
|
Enabled bool `mapstructure:"enabled" json:"enabled"`
|
||||||
AuditLog bool `mapstructure:"audit_log" json:"audit_log"`
|
AuditLog bool `mapstructure:"audit_log" json:"audit_log"`
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -245,24 +265,24 @@ type HandlersConfig struct {
|
|||||||
|
|
||||||
// GoHandlerConfig contains Go proxy configuration
|
// GoHandlerConfig contains Go proxy configuration
|
||||||
type GoHandlerConfig struct {
|
type GoHandlerConfig struct {
|
||||||
Enabled bool `mapstructure:"enabled" json:"enabled"`
|
|
||||||
UpstreamProxy string `mapstructure:"upstream_proxy" json:"upstream_proxy"`
|
UpstreamProxy string `mapstructure:"upstream_proxy" json:"upstream_proxy"`
|
||||||
ChecksumDB string `mapstructure:"checksum_db" json:"checksum_db"`
|
ChecksumDB string `mapstructure:"checksum_db" json:"checksum_db"`
|
||||||
|
GitCredentialsFile string `mapstructure:"git_credentials_file" json:"git_credentials_file"`
|
||||||
|
Enabled bool `mapstructure:"enabled" json:"enabled"`
|
||||||
VerifyChecksums bool `mapstructure:"verify_checksums" json:"verify_checksums"`
|
VerifyChecksums bool `mapstructure:"verify_checksums" json:"verify_checksums"`
|
||||||
GitCredentialsFile string `mapstructure:"git_credentials_file" json:"git_credentials_file"` // Path to git credentials JSON file
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// NPMHandlerConfig contains NPM registry configuration
|
// NPMHandlerConfig contains NPM registry configuration
|
||||||
type NPMHandlerConfig struct {
|
type NPMHandlerConfig struct {
|
||||||
Enabled bool `mapstructure:"enabled" json:"enabled"`
|
|
||||||
UpstreamRegistry string `mapstructure:"upstream_registry" json:"upstream_registry"`
|
UpstreamRegistry string `mapstructure:"upstream_registry" json:"upstream_registry"`
|
||||||
|
Enabled bool `mapstructure:"enabled" json:"enabled"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// PyPIHandlerConfig contains PyPI configuration
|
// PyPIHandlerConfig contains PyPI configuration
|
||||||
type PyPIHandlerConfig struct {
|
type PyPIHandlerConfig struct {
|
||||||
Enabled bool `mapstructure:"enabled" json:"enabled"`
|
|
||||||
UpstreamURL string `mapstructure:"upstream_url" json:"upstream_url"`
|
UpstreamURL string `mapstructure:"upstream_url" json:"upstream_url"`
|
||||||
SimpleAPIURL string `mapstructure:"simple_api_url" json:"simple_api_url"`
|
SimpleAPIURL string `mapstructure:"simple_api_url" json:"simple_api_url"`
|
||||||
|
Enabled bool `mapstructure:"enabled" json:"enabled"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// Default returns a configuration with sensible defaults
|
// Default returns a configuration with sensible defaults
|
||||||
@@ -413,9 +433,16 @@ func (c *Config) Validate() error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Validate metadata backend
|
// Validate metadata backend
|
||||||
validMetadataBackends := map[string]bool{"sqlite": true, "postgresql": true, "file": true}
|
validMetadataBackends := map[string]bool{
|
||||||
|
"sqlite": true,
|
||||||
|
"postgresql": true,
|
||||||
|
"postgres": true,
|
||||||
|
"mysql": true,
|
||||||
|
"mariadb": true,
|
||||||
|
"file": true,
|
||||||
|
}
|
||||||
if !validMetadataBackends[c.Metadata.Backend] {
|
if !validMetadataBackends[c.Metadata.Backend] {
|
||||||
return fmt.Errorf("metadata.backend must be one of: sqlite, postgresql, file; got %s", c.Metadata.Backend)
|
return fmt.Errorf("metadata.backend must be one of: sqlite, postgresql, mysql, file; got %s", c.Metadata.Backend)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Validate cache
|
// Validate cache
|
||||||
|
|||||||
@@ -41,10 +41,10 @@ func (s *ConfigTestSuite) TestDefault() {
|
|||||||
|
|
||||||
func (s *ConfigTestSuite) TestValidate() {
|
func (s *ConfigTestSuite) TestValidate() {
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
name string
|
|
||||||
modify func(*Config)
|
modify func(*Config)
|
||||||
expectError bool
|
name string
|
||||||
errorSubstr string
|
errorSubstr string
|
||||||
|
expectError bool
|
||||||
}{
|
}{
|
||||||
{
|
{
|
||||||
name: "valid_config",
|
name: "valid_config",
|
||||||
@@ -175,11 +175,11 @@ func (s *ConfigTestSuite) TestValidate() {
|
|||||||
|
|
||||||
func (s *ConfigTestSuite) TestLoad() {
|
func (s *ConfigTestSuite) TestLoad() {
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
|
envVars map[string]string
|
||||||
|
validate func(*Config)
|
||||||
name string
|
name string
|
||||||
configYAML string
|
configYAML string
|
||||||
envVars map[string]string
|
|
||||||
expectError bool
|
expectError bool
|
||||||
validate func(*Config)
|
|
||||||
}{
|
}{
|
||||||
{
|
{
|
||||||
name: "valid_yaml_config",
|
name: "valid_yaml_config",
|
||||||
@@ -319,13 +319,6 @@ func (s *ConfigTestSuite) TestLoadMissingFile() {
|
|||||||
s.Nil(cfg)
|
s.Nil(cfg)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *ConfigTestSuite) TestLoadWithDefaults() {
|
|
||||||
// Invalid config path should return defaults
|
|
||||||
cfg := LoadWithDefaults("/invalid/path/config.yaml")
|
|
||||||
s.NotNil(cfg)
|
|
||||||
s.Equal(8080, cfg.Server.Port)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Benchmark tests
|
// Benchmark tests
|
||||||
func BenchmarkDefault(b *testing.B) {
|
func BenchmarkDefault(b *testing.B) {
|
||||||
for i := 0; i < b.N; i++ {
|
for i := 0; i < b.N; i++ {
|
||||||
@@ -344,8 +337,8 @@ func BenchmarkValidate(b *testing.B) {
|
|||||||
// Table-driven edge cases
|
// Table-driven edge cases
|
||||||
func TestConfigEdgeCases(t *testing.T) {
|
func TestConfigEdgeCases(t *testing.T) {
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
name string
|
|
||||||
config *Config
|
config *Config
|
||||||
|
name string
|
||||||
valid bool
|
valid bool
|
||||||
}{
|
}{
|
||||||
{
|
{
|
||||||
|
|||||||
@@ -51,12 +51,3 @@ func Load(configPath string) (*Config, error) {
|
|||||||
|
|
||||||
return cfg, nil
|
return cfg, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// LoadWithDefaults loads configuration or returns defaults on error
|
|
||||||
func LoadWithDefaults(configPath string) *Config {
|
|
||||||
cfg, err := Load(configPath)
|
|
||||||
if err != nil {
|
|
||||||
return Default()
|
|
||||||
}
|
|
||||||
return cfg
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -58,11 +58,3 @@ var HTTPStatusCode = map[string]int{
|
|||||||
ErrCodeServiceUnavailable: 503,
|
ErrCodeServiceUnavailable: 503,
|
||||||
ErrCodeCircuitOpen: 503,
|
ErrCodeCircuitOpen: 503,
|
||||||
}
|
}
|
||||||
|
|
||||||
// GetHTTPStatus returns the HTTP status code for an error code
|
|
||||||
func GetHTTPStatus(code string) int {
|
|
||||||
if status, ok := HTTPStatusCode[code]; ok {
|
|
||||||
return status
|
|
||||||
}
|
|
||||||
return 500 // Default to internal server error
|
|
||||||
}
|
|
||||||
|
|||||||
+4
-50
@@ -6,11 +6,11 @@ import (
|
|||||||
|
|
||||||
// Error represents a structured error with code and details
|
// Error represents a structured error with code and details
|
||||||
type Error struct {
|
type Error struct {
|
||||||
|
Details interface{} `json:"details,omitempty"`
|
||||||
|
Cause error `json:"-"`
|
||||||
Code string `json:"code"`
|
Code string `json:"code"`
|
||||||
Message string `json:"message"`
|
Message string `json:"message"`
|
||||||
Details interface{} `json:"details,omitempty"`
|
|
||||||
Trace []string `json:"trace,omitempty"`
|
Trace []string `json:"trace,omitempty"`
|
||||||
Cause error `json:"-"` // Internal cause, not serialized
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Error implements the error interface
|
// Error implements the error interface
|
||||||
@@ -34,26 +34,12 @@ func New(code, message string) *Error {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Newf creates a new error with formatted message
|
|
||||||
func Newf(code, format string, args ...interface{}) *Error {
|
|
||||||
return &Error{
|
|
||||||
Code: code,
|
|
||||||
Message: fmt.Sprintf(format, args...),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// WithDetails adds details to the error
|
// WithDetails adds details to the error
|
||||||
func (e *Error) WithDetails(details interface{}) *Error {
|
func (e *Error) WithDetails(details interface{}) *Error {
|
||||||
e.Details = details
|
e.Details = details
|
||||||
return e
|
return e
|
||||||
}
|
}
|
||||||
|
|
||||||
// WithTrace adds stack trace to the error
|
|
||||||
func (e *Error) WithTrace(trace []string) *Error {
|
|
||||||
e.Trace = trace
|
|
||||||
return e
|
|
||||||
}
|
|
||||||
|
|
||||||
// WithCause adds an underlying cause to the error
|
// WithCause adds an underlying cause to the error
|
||||||
func (e *Error) WithCause(cause error) *Error {
|
func (e *Error) WithCause(cause error) *Error {
|
||||||
e.Cause = cause
|
e.Cause = cause
|
||||||
@@ -69,44 +55,12 @@ func Wrap(err error, code, message string) *Error {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Wrapf wraps an existing error with formatted message
|
// NotFound creates a not found error
|
||||||
func Wrapf(err error, code, format string, args ...interface{}) *Error {
|
|
||||||
return &Error{
|
|
||||||
Code: code,
|
|
||||||
Message: fmt.Sprintf(format, args...),
|
|
||||||
Cause: err,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Common error constructors
|
|
||||||
func BadRequest(message string) *Error {
|
|
||||||
return New(ErrCodeBadRequest, message)
|
|
||||||
}
|
|
||||||
|
|
||||||
func Unauthorized(message string) *Error {
|
|
||||||
return New(ErrCodeUnauthorized, message)
|
|
||||||
}
|
|
||||||
|
|
||||||
func Forbidden(message string) *Error {
|
|
||||||
return New(ErrCodeForbidden, message)
|
|
||||||
}
|
|
||||||
|
|
||||||
func NotFound(message string) *Error {
|
func NotFound(message string) *Error {
|
||||||
return New(ErrCodeNotFound, message)
|
return New(ErrCodeNotFound, message)
|
||||||
}
|
}
|
||||||
|
|
||||||
func InternalServer(message string) *Error {
|
// QuotaExceeded creates a quota exceeded error
|
||||||
return New(ErrCodeInternalServer, message)
|
|
||||||
}
|
|
||||||
|
|
||||||
func PackageNotFound(name, version string) *Error {
|
|
||||||
return New(ErrCodePackageNotFound, fmt.Sprintf("Package %s@%s not found", name, version)).
|
|
||||||
WithDetails(map[string]string{
|
|
||||||
"package": name,
|
|
||||||
"version": version,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
func QuotaExceeded(limit int64) *Error {
|
func QuotaExceeded(limit int64) *Error {
|
||||||
return New(ErrCodeQuotaExceeded, "Storage quota exceeded").
|
return New(ErrCodeQuotaExceeded, "Storage quota exceeded").
|
||||||
WithDetails(map[string]interface{}{
|
WithDetails(map[string]interface{}{
|
||||||
|
|||||||
+1
-131
@@ -4,7 +4,6 @@ import (
|
|||||||
"errors"
|
"errors"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"github.com/stretchr/testify/assert"
|
|
||||||
"github.com/stretchr/testify/suite"
|
"github.com/stretchr/testify/suite"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -46,43 +45,10 @@ func (s *ErrorsTestSuite) TestNew() {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *ErrorsTestSuite) TestNewf() {
|
|
||||||
tests := []struct {
|
|
||||||
name string
|
|
||||||
code string
|
|
||||||
format string
|
|
||||||
args []interface{}
|
|
||||||
expected string
|
|
||||||
}{
|
|
||||||
{
|
|
||||||
name: "formatted_message",
|
|
||||||
code: ErrCodePackageNotFound,
|
|
||||||
format: "Package %s@%s not found",
|
|
||||||
args: []interface{}{"react", "18.2.0"},
|
|
||||||
expected: "Package [email protected] not found",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "no_args",
|
|
||||||
code: ErrCodeInternalServer,
|
|
||||||
format: "Internal error",
|
|
||||||
args: []interface{}{},
|
|
||||||
expected: "Internal error",
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, tt := range tests {
|
|
||||||
s.Run(tt.name, func() {
|
|
||||||
err := Newf(tt.code, tt.format, tt.args...)
|
|
||||||
s.Equal(tt.code, err.Code)
|
|
||||||
s.Equal(tt.expected, err.Message)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *ErrorsTestSuite) TestWithDetails() {
|
func (s *ErrorsTestSuite) TestWithDetails() {
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
name string
|
|
||||||
details interface{}
|
details interface{}
|
||||||
|
name string
|
||||||
}{
|
}{
|
||||||
{
|
{
|
||||||
name: "map_details",
|
name: "map_details",
|
||||||
@@ -106,12 +72,6 @@ func (s *ErrorsTestSuite) TestWithDetails() {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *ErrorsTestSuite) TestWithTrace() {
|
|
||||||
trace := []string{"file1.go:10", "file2.go:20"}
|
|
||||||
err := New(ErrCodeInternalServer, "test").WithTrace(trace)
|
|
||||||
s.Equal(trace, err.Trace)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *ErrorsTestSuite) TestWithCause() {
|
func (s *ErrorsTestSuite) TestWithCause() {
|
||||||
cause := errors.New("underlying error")
|
cause := errors.New("underlying error")
|
||||||
err := New(ErrCodeStorageFailure, "test").WithCause(cause)
|
err := New(ErrCodeStorageFailure, "test").WithCause(cause)
|
||||||
@@ -129,15 +89,6 @@ func (s *ErrorsTestSuite) TestWrap() {
|
|||||||
s.True(errors.Is(wrapped, cause))
|
s.True(errors.Is(wrapped, cause))
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *ErrorsTestSuite) TestWrapf() {
|
|
||||||
cause := errors.New("connection refused")
|
|
||||||
wrapped := Wrapf(cause, ErrCodeUpstreamFailure, "failed to connect to %s", "registry.npmjs.org")
|
|
||||||
|
|
||||||
s.Equal(ErrCodeUpstreamFailure, wrapped.Code)
|
|
||||||
s.Equal("failed to connect to registry.npmjs.org", wrapped.Message)
|
|
||||||
s.Equal(cause, wrapped.Cause)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *ErrorsTestSuite) TestErrorString() {
|
func (s *ErrorsTestSuite) TestErrorString() {
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
name string
|
name string
|
||||||
@@ -163,59 +114,6 @@ func (s *ErrorsTestSuite) TestErrorString() {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *ErrorsTestSuite) TestCommonConstructors() {
|
|
||||||
tests := []struct {
|
|
||||||
name string
|
|
||||||
fn func() *Error
|
|
||||||
wantCode string
|
|
||||||
}{
|
|
||||||
{
|
|
||||||
name: "bad_request",
|
|
||||||
fn: func() *Error { return BadRequest("invalid input") },
|
|
||||||
wantCode: ErrCodeBadRequest,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "unauthorized",
|
|
||||||
fn: func() *Error { return Unauthorized("invalid token") },
|
|
||||||
wantCode: ErrCodeUnauthorized,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "forbidden",
|
|
||||||
fn: func() *Error { return Forbidden("access denied") },
|
|
||||||
wantCode: ErrCodeForbidden,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "not_found",
|
|
||||||
fn: func() *Error { return NotFound("resource missing") },
|
|
||||||
wantCode: ErrCodeNotFound,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "internal_server",
|
|
||||||
fn: func() *Error { return InternalServer("server error") },
|
|
||||||
wantCode: ErrCodeInternalServer,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, tt := range tests {
|
|
||||||
s.Run(tt.name, func() {
|
|
||||||
err := tt.fn()
|
|
||||||
s.Equal(tt.wantCode, err.Code)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *ErrorsTestSuite) TestPackageNotFound() {
|
|
||||||
err := PackageNotFound("lodash", "4.17.21")
|
|
||||||
s.Equal(ErrCodePackageNotFound, err.Code)
|
|
||||||
s.Equal("Package [email protected] not found", err.Message)
|
|
||||||
s.NotNil(err.Details)
|
|
||||||
|
|
||||||
details, ok := err.Details.(map[string]string)
|
|
||||||
s.True(ok)
|
|
||||||
s.Equal("lodash", details["package"])
|
|
||||||
s.Equal("4.17.21", details["version"])
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *ErrorsTestSuite) TestQuotaExceeded() {
|
func (s *ErrorsTestSuite) TestQuotaExceeded() {
|
||||||
limit := int64(1000000)
|
limit := int64(1000000)
|
||||||
err := QuotaExceeded(limit)
|
err := QuotaExceeded(limit)
|
||||||
@@ -275,31 +173,3 @@ func (s *ErrorsTestSuite) TestEdgeCases() {
|
|||||||
s.Equal(largeDetails, err.Details)
|
s.Equal(largeDetails, err.Details)
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
// Table-driven test for error codes
|
|
||||||
func TestGetHTTPStatus(t *testing.T) {
|
|
||||||
tests := []struct {
|
|
||||||
code string
|
|
||||||
expectedStatus int
|
|
||||||
}{
|
|
||||||
{ErrCodeBadRequest, 400},
|
|
||||||
{ErrCodeUnauthorized, 401},
|
|
||||||
{ErrCodeForbidden, 403},
|
|
||||||
{ErrCodeNotFound, 404},
|
|
||||||
{ErrCodeConflict, 409},
|
|
||||||
{ErrCodePayloadTooLarge, 413},
|
|
||||||
{ErrCodeChecksumMismatch, 422},
|
|
||||||
{ErrCodeRateLimited, 429},
|
|
||||||
{ErrCodeInternalServer, 500},
|
|
||||||
{ErrCodeDatabaseFailure, 500},
|
|
||||||
{ErrCodeUpstreamFailure, 502},
|
|
||||||
{ErrCodeServiceUnavailable, 503},
|
|
||||||
{"UNKNOWN_CODE", 500}, // Default
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, tt := range tests {
|
|
||||||
t.Run(tt.code, func(t *testing.T) {
|
|
||||||
assert.Equal(t, tt.expectedStatus, GetHTTPStatus(tt.code))
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -1,90 +0,0 @@
|
|||||||
package errors
|
|
||||||
|
|
||||||
import (
|
|
||||||
"net/http"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
json "github.com/goccy/go-json"
|
|
||||||
)
|
|
||||||
|
|
||||||
// Response is the standard API response envelope
|
|
||||||
type Response struct {
|
|
||||||
Success bool `json:"success"`
|
|
||||||
Data interface{} `json:"data,omitempty"`
|
|
||||||
Error *ErrorResponse `json:"error,omitempty"`
|
|
||||||
Metadata *ResponseMeta `json:"metadata,omitempty"`
|
|
||||||
}
|
|
||||||
|
|
||||||
// ErrorResponse contains error details
|
|
||||||
type ErrorResponse struct {
|
|
||||||
Code string `json:"code"`
|
|
||||||
Message string `json:"message"`
|
|
||||||
Details interface{} `json:"details,omitempty"`
|
|
||||||
Trace []string `json:"trace,omitempty"`
|
|
||||||
}
|
|
||||||
|
|
||||||
// ResponseMeta contains request metadata
|
|
||||||
type ResponseMeta struct {
|
|
||||||
RequestID string `json:"request_id"`
|
|
||||||
Timestamp string `json:"timestamp"`
|
|
||||||
Duration string `json:"duration,omitempty"`
|
|
||||||
Version string `json:"version"`
|
|
||||||
}
|
|
||||||
|
|
||||||
// WriteJSON writes a success response as JSON
|
|
||||||
func WriteJSON(w http.ResponseWriter, statusCode int, data interface{}, meta *ResponseMeta) {
|
|
||||||
response := Response{
|
|
||||||
Success: statusCode < 400,
|
|
||||||
Data: data,
|
|
||||||
Metadata: meta,
|
|
||||||
}
|
|
||||||
|
|
||||||
w.Header().Set("Content-Type", "application/json")
|
|
||||||
w.WriteHeader(statusCode)
|
|
||||||
|
|
||||||
if err := json.NewEncoder(w).Encode(response); err != nil {
|
|
||||||
// Fallback to simple error response
|
|
||||||
http.Error(w, `{"success":false,"error":{"code":"ENCODING_ERROR","message":"Failed to encode response"}}`, http.StatusInternalServerError)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// WriteError writes an error response as JSON
|
|
||||||
func WriteError(w http.ResponseWriter, statusCode int, err *Error, meta *ResponseMeta) {
|
|
||||||
errResp := &ErrorResponse{
|
|
||||||
Code: err.Code,
|
|
||||||
Message: err.Message,
|
|
||||||
Details: err.Details,
|
|
||||||
Trace: err.Trace,
|
|
||||||
}
|
|
||||||
|
|
||||||
response := Response{
|
|
||||||
Success: false,
|
|
||||||
Error: errResp,
|
|
||||||
Metadata: meta,
|
|
||||||
}
|
|
||||||
|
|
||||||
w.Header().Set("Content-Type", "application/json")
|
|
||||||
w.WriteHeader(statusCode)
|
|
||||||
|
|
||||||
if encErr := json.NewEncoder(w).Encode(response); encErr != nil {
|
|
||||||
// Fallback to simple error response
|
|
||||||
http.Error(w, `{"success":false,"error":{"code":"ENCODING_ERROR","message":"Failed to encode error response"}}`, http.StatusInternalServerError)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// WriteErrorSimple writes an error without metadata
|
|
||||||
func WriteErrorSimple(w http.ResponseWriter, err *Error) {
|
|
||||||
statusCode := GetHTTPStatus(err.Code)
|
|
||||||
meta := &ResponseMeta{
|
|
||||||
Timestamp: time.Now().UTC().Format(time.RFC3339),
|
|
||||||
}
|
|
||||||
WriteError(w, statusCode, err, meta)
|
|
||||||
}
|
|
||||||
|
|
||||||
// WriteJSONSimple writes a success response without metadata
|
|
||||||
func WriteJSONSimple(w http.ResponseWriter, statusCode int, data interface{}) {
|
|
||||||
meta := &ResponseMeta{
|
|
||||||
Timestamp: time.Now().UTC().Format(time.RFC3339),
|
|
||||||
}
|
|
||||||
WriteJSON(w, statusCode, data, meta)
|
|
||||||
}
|
|
||||||
@@ -21,25 +21,25 @@ const (
|
|||||||
|
|
||||||
// Check represents a single health check
|
// Check represents a single health check
|
||||||
type Check struct {
|
type Check struct {
|
||||||
|
Fn func(context.Context) (Status, string) `json:"-"`
|
||||||
Name string `json:"name"`
|
Name string `json:"name"`
|
||||||
Status Status `json:"status"`
|
Status Status `json:"status"`
|
||||||
Error string `json:"error,omitempty"`
|
Error string `json:"error,omitempty"`
|
||||||
Fn func(context.Context) (Status, string) `json:"-"`
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Response is the health check response
|
// Response is the health check response
|
||||||
type Response struct {
|
type Response struct {
|
||||||
Success bool `json:"success"`
|
|
||||||
Data *HealthData `json:"data,omitempty"`
|
Data *HealthData `json:"data,omitempty"`
|
||||||
Metadata *Metadata `json:"metadata,omitempty"`
|
Metadata *Metadata `json:"metadata,omitempty"`
|
||||||
|
Success bool `json:"success"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// HealthData contains health check data
|
// HealthData contains health check data
|
||||||
type HealthData struct {
|
type HealthData struct {
|
||||||
|
Components map[string]*Component `json:"components"`
|
||||||
Status Status `json:"status"`
|
Status Status `json:"status"`
|
||||||
Version string `json:"version"`
|
Version string `json:"version"`
|
||||||
Uptime string `json:"uptime"`
|
Uptime string `json:"uptime"`
|
||||||
Components map[string]*Component `json:"components"`
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Component represents a system component
|
// Component represents a system component
|
||||||
@@ -57,8 +57,8 @@ type Metadata struct {
|
|||||||
|
|
||||||
// Checker manages health checks
|
// Checker manages health checks
|
||||||
type Checker struct {
|
type Checker struct {
|
||||||
checks []*Check
|
|
||||||
startTime time.Time
|
startTime time.Time
|
||||||
|
checks []*Check
|
||||||
mu sync.RWMutex
|
mu sync.RWMutex
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -1,275 +0,0 @@
|
|||||||
package lock
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"crypto/rand"
|
|
||||||
"encoding/hex"
|
|
||||||
"errors"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/redis/go-redis/v9"
|
|
||||||
"github.com/rs/zerolog/log"
|
|
||||||
)
|
|
||||||
|
|
||||||
var (
|
|
||||||
ErrLockNotAcquired = errors.New("lock not acquired")
|
|
||||||
ErrLockNotHeld = errors.New("lock not held by this instance")
|
|
||||||
ErrInvalidTTL = errors.New("invalid TTL: must be positive")
|
|
||||||
)
|
|
||||||
|
|
||||||
// Lock represents a distributed lock
|
|
||||||
type Lock struct {
|
|
||||||
client *redis.Client
|
|
||||||
key string
|
|
||||||
value string
|
|
||||||
ttl time.Duration
|
|
||||||
}
|
|
||||||
|
|
||||||
// Manager manages distributed locks using Redis
|
|
||||||
type Manager struct {
|
|
||||||
client *redis.Client
|
|
||||||
}
|
|
||||||
|
|
||||||
// Config holds Redis connection configuration
|
|
||||||
type Config struct {
|
|
||||||
Addr string
|
|
||||||
Password string
|
|
||||||
DB int
|
|
||||||
}
|
|
||||||
|
|
||||||
// NewManager creates a new lock manager
|
|
||||||
func NewManager(cfg Config) (*Manager, error) {
|
|
||||||
client := redis.NewClient(&redis.Options{
|
|
||||||
Addr: cfg.Addr,
|
|
||||||
Password: cfg.Password,
|
|
||||||
DB: cfg.DB,
|
|
||||||
})
|
|
||||||
|
|
||||||
// Test connection
|
|
||||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
|
||||||
defer cancel()
|
|
||||||
|
|
||||||
if err := client.Ping(ctx).Err(); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
log.Info().
|
|
||||||
Str("addr", cfg.Addr).
|
|
||||||
Int("db", cfg.DB).
|
|
||||||
Msg("Connected to Redis for distributed locking")
|
|
||||||
|
|
||||||
return &Manager{
|
|
||||||
client: client,
|
|
||||||
}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// Acquire attempts to acquire a lock with the given key and TTL
|
|
||||||
// Returns a Lock instance if successful, or an error if the lock is already held
|
|
||||||
func (m *Manager) Acquire(ctx context.Context, key string, ttl time.Duration) (*Lock, error) {
|
|
||||||
if ttl <= 0 {
|
|
||||||
return nil, ErrInvalidTTL
|
|
||||||
}
|
|
||||||
|
|
||||||
// Generate unique value for this lock instance
|
|
||||||
value, err := generateLockValue()
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
// Try to acquire lock using SET NX (set if not exists)
|
|
||||||
success, err := m.client.SetNX(ctx, key, value, ttl).Result()
|
|
||||||
if err != nil {
|
|
||||||
log.Error().
|
|
||||||
Err(err).
|
|
||||||
Str("key", key).
|
|
||||||
Msg("Failed to acquire lock")
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
if !success {
|
|
||||||
log.Debug().
|
|
||||||
Str("key", key).
|
|
||||||
Msg("Lock already held by another instance")
|
|
||||||
return nil, ErrLockNotAcquired
|
|
||||||
}
|
|
||||||
|
|
||||||
log.Debug().
|
|
||||||
Str("key", key).
|
|
||||||
Dur("ttl", ttl).
|
|
||||||
Msg("Lock acquired successfully")
|
|
||||||
|
|
||||||
return &Lock{
|
|
||||||
client: m.client,
|
|
||||||
key: key,
|
|
||||||
value: value,
|
|
||||||
ttl: ttl,
|
|
||||||
}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// TryAcquire attempts to acquire a lock, retrying for the specified duration
|
|
||||||
// Returns a Lock instance if successful within the timeout, or an error
|
|
||||||
func (m *Manager) TryAcquire(ctx context.Context, key string, ttl, timeout time.Duration) (*Lock, error) {
|
|
||||||
if ttl <= 0 {
|
|
||||||
return nil, ErrInvalidTTL
|
|
||||||
}
|
|
||||||
|
|
||||||
deadline := time.Now().Add(timeout)
|
|
||||||
ticker := time.NewTicker(50 * time.Millisecond)
|
|
||||||
defer ticker.Stop()
|
|
||||||
|
|
||||||
for {
|
|
||||||
lock, err := m.Acquire(ctx, key, ttl)
|
|
||||||
if err == nil {
|
|
||||||
return lock, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
if err != ErrLockNotAcquired {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
select {
|
|
||||||
case <-ctx.Done():
|
|
||||||
return nil, ctx.Err()
|
|
||||||
case <-ticker.C:
|
|
||||||
if time.Now().After(deadline) {
|
|
||||||
return nil, ErrLockNotAcquired
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Release releases the lock
|
|
||||||
// Returns an error if the lock is not held by this instance
|
|
||||||
func (l *Lock) Release(ctx context.Context) error {
|
|
||||||
// Use Lua script to ensure atomic check-and-delete
|
|
||||||
// Only delete if the value matches (ensures we own the lock)
|
|
||||||
script := redis.NewScript(`
|
|
||||||
if redis.call("get", KEYS[1]) == ARGV[1] then
|
|
||||||
return redis.call("del", KEYS[1])
|
|
||||||
else
|
|
||||||
return 0
|
|
||||||
end
|
|
||||||
`)
|
|
||||||
|
|
||||||
result, err := script.Run(ctx, l.client, []string{l.key}, l.value).Result()
|
|
||||||
if err != nil {
|
|
||||||
log.Error().
|
|
||||||
Err(err).
|
|
||||||
Str("key", l.key).
|
|
||||||
Msg("Failed to release lock")
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
// Result of 0 means the lock was not deleted (not owned by us)
|
|
||||||
if result.(int64) == 0 {
|
|
||||||
log.Warn().
|
|
||||||
Str("key", l.key).
|
|
||||||
Msg("Attempted to release lock not held by this instance")
|
|
||||||
return ErrLockNotHeld
|
|
||||||
}
|
|
||||||
|
|
||||||
log.Debug().
|
|
||||||
Str("key", l.key).
|
|
||||||
Msg("Lock released successfully")
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// Extend extends the lock TTL
|
|
||||||
// Returns an error if the lock is not held by this instance
|
|
||||||
func (l *Lock) Extend(ctx context.Context, additionalTTL time.Duration) error {
|
|
||||||
// Use Lua script to ensure atomic check-and-extend
|
|
||||||
script := redis.NewScript(`
|
|
||||||
if redis.call("get", KEYS[1]) == ARGV[1] then
|
|
||||||
return redis.call("expire", KEYS[1], ARGV[2])
|
|
||||||
else
|
|
||||||
return 0
|
|
||||||
end
|
|
||||||
`)
|
|
||||||
|
|
||||||
newTTL := l.ttl + additionalTTL
|
|
||||||
result, err := script.Run(ctx, l.client, []string{l.key}, l.value, int(newTTL.Seconds())).Result()
|
|
||||||
if err != nil {
|
|
||||||
log.Error().
|
|
||||||
Err(err).
|
|
||||||
Str("key", l.key).
|
|
||||||
Msg("Failed to extend lock")
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
if result.(int64) == 0 {
|
|
||||||
log.Warn().
|
|
||||||
Str("key", l.key).
|
|
||||||
Msg("Attempted to extend lock not held by this instance")
|
|
||||||
return ErrLockNotHeld
|
|
||||||
}
|
|
||||||
|
|
||||||
l.ttl = newTTL
|
|
||||||
log.Debug().
|
|
||||||
Str("key", l.key).
|
|
||||||
Dur("new_ttl", newTTL).
|
|
||||||
Msg("Lock TTL extended")
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// IsHeld checks if the lock is still held by this instance
|
|
||||||
func (l *Lock) IsHeld(ctx context.Context) bool {
|
|
||||||
value, err := l.client.Get(ctx, l.key).Result()
|
|
||||||
if err != nil {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
return value == l.value
|
|
||||||
}
|
|
||||||
|
|
||||||
// Close closes the lock manager and its Redis connection
|
|
||||||
func (m *Manager) Close() error {
|
|
||||||
return m.client.Close() // #nosec G104 -- Cleanup, error not critical
|
|
||||||
}
|
|
||||||
|
|
||||||
// generateLockValue generates a cryptographically random lock value
|
|
||||||
func generateLockValue() (string, error) {
|
|
||||||
bytes := make([]byte, 16)
|
|
||||||
if _, err := rand.Read(bytes); err != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
return hex.EncodeToString(bytes), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// WithLock executes a function while holding a distributed lock
|
|
||||||
// The lock is automatically released when the function returns
|
|
||||||
func (m *Manager) WithLock(ctx context.Context, key string, ttl time.Duration, fn func(context.Context) error) error {
|
|
||||||
lock, err := m.Acquire(ctx, key, ttl)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
defer func() {
|
|
||||||
if err := lock.Release(context.Background()); err != nil {
|
|
||||||
log.Error().
|
|
||||||
Err(err).
|
|
||||||
Str("key", key).
|
|
||||||
Msg("Failed to release lock in defer")
|
|
||||||
}
|
|
||||||
}()
|
|
||||||
|
|
||||||
return fn(ctx)
|
|
||||||
}
|
|
||||||
|
|
||||||
// WithRetryLock executes a function while holding a distributed lock
|
|
||||||
// It retries acquisition for the specified timeout duration
|
|
||||||
func (m *Manager) WithRetryLock(ctx context.Context, key string, ttl, timeout time.Duration, fn func(context.Context) error) error {
|
|
||||||
lock, err := m.TryAcquire(ctx, key, ttl, timeout)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
defer func() {
|
|
||||||
if err := lock.Release(context.Background()); err != nil {
|
|
||||||
log.Error().
|
|
||||||
Err(err).
|
|
||||||
Str("key", key).
|
|
||||||
Msg("Failed to release lock in defer")
|
|
||||||
}
|
|
||||||
}()
|
|
||||||
|
|
||||||
return fn(ctx)
|
|
||||||
}
|
|
||||||
@@ -35,23 +35,3 @@ func Init(cfg Config) error {
|
|||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// Get returns the global logger
|
|
||||||
func Get() *zerolog.Logger {
|
|
||||||
return &log.Logger
|
|
||||||
}
|
|
||||||
|
|
||||||
// WithFields returns a logger with additional fields
|
|
||||||
func WithFields(fields map[string]interface{}) *zerolog.Logger {
|
|
||||||
logger := log.Logger
|
|
||||||
for k, v := range fields {
|
|
||||||
logger = logger.With().Interface(k, v).Logger()
|
|
||||||
}
|
|
||||||
return &logger
|
|
||||||
}
|
|
||||||
|
|
||||||
// WithRequestID returns a logger with request ID
|
|
||||||
func WithRequestID(requestID string) *zerolog.Logger {
|
|
||||||
logger := log.With().Str("request_id", requestID).Logger()
|
|
||||||
return &logger
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -1,65 +0,0 @@
|
|||||||
package logger
|
|
||||||
|
|
||||||
import (
|
|
||||||
"net/http"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/lukaszraczylo/gohoarder/pkg/uuid"
|
|
||||||
"github.com/rs/zerolog/log"
|
|
||||||
)
|
|
||||||
|
|
||||||
// responseWriter wraps http.ResponseWriter to capture status code
|
|
||||||
type responseWriter struct {
|
|
||||||
http.ResponseWriter
|
|
||||||
statusCode int
|
|
||||||
written int64
|
|
||||||
}
|
|
||||||
|
|
||||||
func (rw *responseWriter) WriteHeader(code int) {
|
|
||||||
rw.statusCode = code
|
|
||||||
rw.ResponseWriter.WriteHeader(code)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (rw *responseWriter) Write(b []byte) (int, error) {
|
|
||||||
n, err := rw.ResponseWriter.Write(b)
|
|
||||||
rw.written += int64(n)
|
|
||||||
return n, err
|
|
||||||
}
|
|
||||||
|
|
||||||
// Middleware is HTTP middleware for request logging
|
|
||||||
func Middleware(next http.Handler) http.Handler {
|
|
||||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
start := time.Now()
|
|
||||||
|
|
||||||
// Generate request ID
|
|
||||||
requestID := r.Header.Get("X-Request-ID")
|
|
||||||
if requestID == "" {
|
|
||||||
requestID = "req_" + uuid.New().String()[:8]
|
|
||||||
}
|
|
||||||
|
|
||||||
// Wrap response writer
|
|
||||||
rw := &responseWriter{
|
|
||||||
ResponseWriter: w,
|
|
||||||
statusCode: http.StatusOK,
|
|
||||||
}
|
|
||||||
|
|
||||||
// Set request ID in response header
|
|
||||||
rw.Header().Set("X-Request-ID", requestID)
|
|
||||||
|
|
||||||
// Call next handler
|
|
||||||
next.ServeHTTP(rw, r)
|
|
||||||
|
|
||||||
// Log request
|
|
||||||
duration := time.Since(start)
|
|
||||||
log.Info().
|
|
||||||
Str("request_id", requestID).
|
|
||||||
Str("method", r.Method).
|
|
||||||
Str("path", r.URL.Path).
|
|
||||||
Str("remote_addr", r.RemoteAddr).
|
|
||||||
Str("user_agent", r.UserAgent()).
|
|
||||||
Int("status", rw.statusCode).
|
|
||||||
Int64("bytes", rw.written).
|
|
||||||
Dur("duration_ms", duration).
|
|
||||||
Msg("HTTP request")
|
|
||||||
})
|
|
||||||
}
|
|
||||||
@@ -0,0 +1,357 @@
|
|||||||
|
package gormstore
|
||||||
|
|
||||||
|
import (
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/rs/zerolog/log"
|
||||||
|
"gorm.io/gorm"
|
||||||
|
)
|
||||||
|
|
||||||
|
// AggregationWorker handles background aggregation of download statistics
|
||||||
|
type AggregationWorker struct {
|
||||||
|
db *gorm.DB
|
||||||
|
stopChan chan struct{}
|
||||||
|
ticker *time.Ticker
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewAggregationWorker creates a new aggregation worker
|
||||||
|
func NewAggregationWorker(db *gorm.DB) *AggregationWorker {
|
||||||
|
return &AggregationWorker{
|
||||||
|
db: db,
|
||||||
|
stopChan: make(chan struct{}),
|
||||||
|
ticker: time.NewTicker(1 * time.Hour), // Run every hour
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Start begins the aggregation worker
|
||||||
|
func (w *AggregationWorker) Start() {
|
||||||
|
log.Info().Msg("Starting aggregation worker")
|
||||||
|
|
||||||
|
// Run immediately on start
|
||||||
|
if err := w.AggregateHourly(); err != nil {
|
||||||
|
log.Error().Err(err).Msg("Failed to run initial hourly aggregation")
|
||||||
|
}
|
||||||
|
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-w.ticker.C:
|
||||||
|
if err := w.AggregateHourly(); err != nil {
|
||||||
|
log.Error().Err(err).Msg("Failed to aggregate hourly stats")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check if it's time for daily aggregation (run at midnight)
|
||||||
|
now := time.Now()
|
||||||
|
if now.Hour() == 0 {
|
||||||
|
if err := w.AggregateDaily(); err != nil {
|
||||||
|
log.Error().Err(err).Msg("Failed to aggregate daily stats")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
case <-w.stopChan:
|
||||||
|
log.Info().Msg("Stopping aggregation worker")
|
||||||
|
w.ticker.Stop()
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Stop stops the aggregation worker
|
||||||
|
func (w *AggregationWorker) Stop() {
|
||||||
|
close(w.stopChan)
|
||||||
|
}
|
||||||
|
|
||||||
|
// AggregateHourly aggregates download events into hourly stats
|
||||||
|
func (w *AggregationWorker) AggregateHourly() error {
|
||||||
|
startTime := time.Now()
|
||||||
|
log.Debug().Msg("Starting hourly aggregation")
|
||||||
|
|
||||||
|
// Get dialect name
|
||||||
|
dialectName := w.db.Dialector.Name()
|
||||||
|
|
||||||
|
// Calculate cutoff time (aggregate events older than 5 minutes to avoid partial data)
|
||||||
|
cutoff := time.Now().Add(-5 * time.Minute).Truncate(time.Hour)
|
||||||
|
|
||||||
|
return w.db.Transaction(func(tx *gorm.DB) error {
|
||||||
|
var aggregateSQL string
|
||||||
|
|
||||||
|
switch dialectName {
|
||||||
|
case "postgres":
|
||||||
|
// PostgreSQL: Use date_trunc for time bucketing
|
||||||
|
aggregateSQL = `
|
||||||
|
INSERT INTO download_stats_hourly (registry_id, package_id, time_bucket, download_count, unique_ips, auth_downloads, created_at, updated_at)
|
||||||
|
SELECT
|
||||||
|
de.registry_id,
|
||||||
|
de.package_id,
|
||||||
|
date_trunc('hour', de.downloaded_at) AS time_bucket,
|
||||||
|
COUNT(*) AS download_count,
|
||||||
|
COUNT(DISTINCT de.ip_address) AS unique_ips,
|
||||||
|
COUNT(*) FILTER (WHERE de.authenticated = true) AS auth_downloads,
|
||||||
|
NOW() AS created_at,
|
||||||
|
NOW() AS updated_at
|
||||||
|
FROM download_events de
|
||||||
|
WHERE de.downloaded_at < ?
|
||||||
|
GROUP BY de.registry_id, de.package_id, time_bucket
|
||||||
|
ON CONFLICT (registry_id, COALESCE(package_id, 0), time_bucket)
|
||||||
|
DO UPDATE SET
|
||||||
|
download_count = download_stats_hourly.download_count + EXCLUDED.download_count,
|
||||||
|
unique_ips = GREATEST(download_stats_hourly.unique_ips, EXCLUDED.unique_ips),
|
||||||
|
auth_downloads = download_stats_hourly.auth_downloads + EXCLUDED.auth_downloads,
|
||||||
|
updated_at = NOW()
|
||||||
|
`
|
||||||
|
|
||||||
|
case "mysql":
|
||||||
|
// MySQL: Use DATE_FORMAT for time bucketing
|
||||||
|
aggregateSQL = `
|
||||||
|
INSERT INTO download_stats_hourly (registry_id, package_id, time_bucket, download_count, unique_ips, auth_downloads, created_at, updated_at)
|
||||||
|
SELECT
|
||||||
|
de.registry_id,
|
||||||
|
de.package_id,
|
||||||
|
DATE_FORMAT(de.downloaded_at, '%Y-%m-%d %H:00:00') AS time_bucket,
|
||||||
|
COUNT(*) AS download_count,
|
||||||
|
COUNT(DISTINCT de.ip_address) AS unique_ips,
|
||||||
|
SUM(CASE WHEN de.authenticated = true THEN 1 ELSE 0 END) AS auth_downloads,
|
||||||
|
NOW() AS created_at,
|
||||||
|
NOW() AS updated_at
|
||||||
|
FROM download_events de
|
||||||
|
WHERE de.downloaded_at < ?
|
||||||
|
GROUP BY de.registry_id, de.package_id, time_bucket
|
||||||
|
ON DUPLICATE KEY UPDATE
|
||||||
|
download_count = download_stats_hourly.download_count + VALUES(download_count),
|
||||||
|
unique_ips = GREATEST(download_stats_hourly.unique_ips, VALUES(unique_ips)),
|
||||||
|
auth_downloads = download_stats_hourly.auth_downloads + VALUES(auth_downloads),
|
||||||
|
updated_at = NOW()
|
||||||
|
`
|
||||||
|
|
||||||
|
default: // SQLite
|
||||||
|
// SQLite: Use strftime for time bucketing
|
||||||
|
// Note: SQLite doesn't support UPSERT as elegantly, need to handle separately
|
||||||
|
aggregateSQL = `
|
||||||
|
INSERT OR REPLACE INTO download_stats_hourly (registry_id, package_id, time_bucket, download_count, unique_ips, auth_downloads, created_at, updated_at)
|
||||||
|
SELECT
|
||||||
|
de.registry_id,
|
||||||
|
de.package_id,
|
||||||
|
strftime('%Y-%m-%d %H:00:00', de.downloaded_at) AS time_bucket,
|
||||||
|
COUNT(*) AS download_count,
|
||||||
|
COUNT(DISTINCT de.ip_address) AS unique_ips,
|
||||||
|
SUM(CASE WHEN de.authenticated = 1 THEN 1 ELSE 0 END) AS auth_downloads,
|
||||||
|
datetime('now') AS created_at,
|
||||||
|
datetime('now') AS updated_at
|
||||||
|
FROM download_events de
|
||||||
|
WHERE de.downloaded_at < ?
|
||||||
|
GROUP BY de.registry_id, de.package_id, time_bucket
|
||||||
|
`
|
||||||
|
}
|
||||||
|
|
||||||
|
// Execute aggregation
|
||||||
|
if err := tx.Exec(aggregateSQL, cutoff).Error; err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Delete aggregated events (older than 24 hours to keep recent data for debugging)
|
||||||
|
deleteOlder := time.Now().Add(-24 * time.Hour)
|
||||||
|
deleteResult := tx.Exec("DELETE FROM download_events WHERE downloaded_at < ?", deleteOlder)
|
||||||
|
if deleteResult.Error != nil {
|
||||||
|
return deleteResult.Error
|
||||||
|
}
|
||||||
|
|
||||||
|
// Also update package-level stats (NULL package_id = registry totals)
|
||||||
|
var registryAggSQL string
|
||||||
|
if dialectName == "postgres" {
|
||||||
|
registryAggSQL = `
|
||||||
|
INSERT INTO download_stats_hourly (registry_id, package_id, time_bucket, download_count, unique_ips, auth_downloads, created_at, updated_at)
|
||||||
|
SELECT
|
||||||
|
registry_id,
|
||||||
|
NULL as package_id,
|
||||||
|
time_bucket,
|
||||||
|
SUM(download_count) as download_count,
|
||||||
|
SUM(unique_ips) as unique_ips,
|
||||||
|
SUM(auth_downloads) as auth_downloads,
|
||||||
|
NOW() as created_at,
|
||||||
|
NOW() as updated_at
|
||||||
|
FROM download_stats_hourly
|
||||||
|
WHERE package_id IS NOT NULL
|
||||||
|
GROUP BY registry_id, time_bucket
|
||||||
|
ON CONFLICT (registry_id, COALESCE(package_id, 0), time_bucket)
|
||||||
|
DO UPDATE SET
|
||||||
|
download_count = EXCLUDED.download_count,
|
||||||
|
unique_ips = EXCLUDED.unique_ips,
|
||||||
|
auth_downloads = EXCLUDED.auth_downloads,
|
||||||
|
updated_at = NOW()
|
||||||
|
`
|
||||||
|
} else if dialectName == "mysql" {
|
||||||
|
registryAggSQL = `
|
||||||
|
INSERT INTO download_stats_hourly (registry_id, package_id, time_bucket, download_count, unique_ips, auth_downloads, created_at, updated_at)
|
||||||
|
SELECT
|
||||||
|
registry_id,
|
||||||
|
NULL as package_id,
|
||||||
|
time_bucket,
|
||||||
|
SUM(download_count) as download_count,
|
||||||
|
SUM(unique_ips) as unique_ips,
|
||||||
|
SUM(auth_downloads) as auth_downloads,
|
||||||
|
NOW() as created_at,
|
||||||
|
NOW() as updated_at
|
||||||
|
FROM download_stats_hourly
|
||||||
|
WHERE package_id IS NOT NULL
|
||||||
|
GROUP BY registry_id, time_bucket
|
||||||
|
ON DUPLICATE KEY UPDATE
|
||||||
|
download_count = VALUES(download_count),
|
||||||
|
unique_ips = VALUES(unique_ips),
|
||||||
|
auth_downloads = VALUES(auth_downloads),
|
||||||
|
updated_at = NOW()
|
||||||
|
`
|
||||||
|
} else {
|
||||||
|
// SQLite
|
||||||
|
registryAggSQL = `
|
||||||
|
INSERT OR REPLACE INTO download_stats_hourly (registry_id, package_id, time_bucket, download_count, unique_ips, auth_downloads, created_at, updated_at)
|
||||||
|
SELECT
|
||||||
|
registry_id,
|
||||||
|
NULL as package_id,
|
||||||
|
time_bucket,
|
||||||
|
SUM(download_count) as download_count,
|
||||||
|
SUM(unique_ips) as unique_ips,
|
||||||
|
SUM(auth_downloads) as auth_downloads,
|
||||||
|
datetime('now') as created_at,
|
||||||
|
datetime('now') as updated_at
|
||||||
|
FROM download_stats_hourly
|
||||||
|
WHERE package_id IS NOT NULL
|
||||||
|
GROUP BY registry_id, time_bucket
|
||||||
|
`
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := tx.Exec(registryAggSQL).Error; err != nil {
|
||||||
|
log.Warn().Err(err).Msg("Failed to aggregate registry totals (continuing anyway)")
|
||||||
|
}
|
||||||
|
|
||||||
|
elapsed := time.Since(startTime)
|
||||||
|
log.Info().
|
||||||
|
Int64("deleted_events", deleteResult.RowsAffected).
|
||||||
|
Dur("duration", elapsed).
|
||||||
|
Msg("Completed hourly aggregation")
|
||||||
|
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// AggregateDaily aggregates hourly stats into daily stats
|
||||||
|
func (w *AggregationWorker) AggregateDaily() error {
|
||||||
|
startTime := time.Now()
|
||||||
|
log.Debug().Msg("Starting daily aggregation")
|
||||||
|
|
||||||
|
dialectName := w.db.Dialector.Name()
|
||||||
|
|
||||||
|
// Aggregate yesterday's data
|
||||||
|
yesterday := time.Now().AddDate(0, 0, -1).Truncate(24 * time.Hour)
|
||||||
|
dayEnd := yesterday.Add(24 * time.Hour)
|
||||||
|
|
||||||
|
return w.db.Transaction(func(tx *gorm.DB) error {
|
||||||
|
var aggregateSQL string
|
||||||
|
|
||||||
|
switch dialectName {
|
||||||
|
case "postgres":
|
||||||
|
aggregateSQL = `
|
||||||
|
INSERT INTO download_stats_daily (registry_id, package_id, time_bucket, download_count, unique_ips, auth_downloads, top_user_agents, created_at, updated_at)
|
||||||
|
SELECT
|
||||||
|
registry_id,
|
||||||
|
package_id,
|
||||||
|
date_trunc('day', time_bucket) AS time_bucket,
|
||||||
|
SUM(download_count) AS download_count,
|
||||||
|
MAX(unique_ips) AS unique_ips,
|
||||||
|
SUM(auth_downloads) AS auth_downloads,
|
||||||
|
'{}' AS top_user_agents,
|
||||||
|
NOW() AS created_at,
|
||||||
|
NOW() AS updated_at
|
||||||
|
FROM download_stats_hourly
|
||||||
|
WHERE time_bucket >= ? AND time_bucket < ?
|
||||||
|
GROUP BY registry_id, package_id, date_trunc('day', time_bucket)
|
||||||
|
ON CONFLICT (registry_id, COALESCE(package_id, 0), time_bucket)
|
||||||
|
DO UPDATE SET
|
||||||
|
download_count = EXCLUDED.download_count,
|
||||||
|
unique_ips = EXCLUDED.unique_ips,
|
||||||
|
auth_downloads = EXCLUDED.auth_downloads,
|
||||||
|
updated_at = NOW()
|
||||||
|
`
|
||||||
|
|
||||||
|
case "mysql":
|
||||||
|
aggregateSQL = `
|
||||||
|
INSERT INTO download_stats_daily (registry_id, package_id, time_bucket, download_count, unique_ips, auth_downloads, top_user_agents, created_at, updated_at)
|
||||||
|
SELECT
|
||||||
|
registry_id,
|
||||||
|
package_id,
|
||||||
|
DATE_FORMAT(time_bucket, '%Y-%m-%d 00:00:00') AS time_bucket,
|
||||||
|
SUM(download_count) AS download_count,
|
||||||
|
MAX(unique_ips) AS unique_ips,
|
||||||
|
SUM(auth_downloads) AS auth_downloads,
|
||||||
|
'{}' AS top_user_agents,
|
||||||
|
NOW() AS created_at,
|
||||||
|
NOW() AS updated_at
|
||||||
|
FROM download_stats_hourly
|
||||||
|
WHERE time_bucket >= ? AND time_bucket < ?
|
||||||
|
GROUP BY registry_id, package_id, DATE_FORMAT(time_bucket, '%Y-%m-%d 00:00:00')
|
||||||
|
ON DUPLICATE KEY UPDATE
|
||||||
|
download_count = VALUES(download_count),
|
||||||
|
unique_ips = VALUES(unique_ips),
|
||||||
|
auth_downloads = VALUES(auth_downloads),
|
||||||
|
updated_at = NOW()
|
||||||
|
`
|
||||||
|
|
||||||
|
default: // SQLite
|
||||||
|
aggregateSQL = `
|
||||||
|
INSERT OR REPLACE INTO download_stats_daily (registry_id, package_id, time_bucket, download_count, unique_ips, auth_downloads, top_user_agents, created_at, updated_at)
|
||||||
|
SELECT
|
||||||
|
registry_id,
|
||||||
|
package_id,
|
||||||
|
date(time_bucket) AS time_bucket,
|
||||||
|
SUM(download_count) AS download_count,
|
||||||
|
MAX(unique_ips) AS unique_ips,
|
||||||
|
SUM(auth_downloads) AS auth_downloads,
|
||||||
|
'{}' AS top_user_agents,
|
||||||
|
datetime('now') AS created_at,
|
||||||
|
datetime('now') AS updated_at
|
||||||
|
FROM download_stats_hourly
|
||||||
|
WHERE time_bucket >= ? AND time_bucket < ?
|
||||||
|
GROUP BY registry_id, package_id, date(time_bucket)
|
||||||
|
`
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := tx.Exec(aggregateSQL, yesterday, dayEnd).Error; err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Delete old hourly stats (keep last 7 days)
|
||||||
|
deleteOlder := time.Now().AddDate(0, 0, -7)
|
||||||
|
deleteResult := tx.Exec("DELETE FROM download_stats_hourly WHERE time_bucket < ?", deleteOlder)
|
||||||
|
if deleteResult.Error != nil {
|
||||||
|
return deleteResult.Error
|
||||||
|
}
|
||||||
|
|
||||||
|
elapsed := time.Since(startTime)
|
||||||
|
log.Info().
|
||||||
|
Int64("deleted_hourly_stats", deleteResult.RowsAffected).
|
||||||
|
Dur("duration", elapsed).
|
||||||
|
Msg("Completed daily aggregation")
|
||||||
|
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// UpdatePackageAccessCounts synchronizes package access_count from download stats
|
||||||
|
func (w *AggregationWorker) UpdatePackageAccessCounts() error {
|
||||||
|
log.Debug().Msg("Updating package access counts")
|
||||||
|
|
||||||
|
// Update from download_stats_hourly (sum all-time downloads per package)
|
||||||
|
updateSQL := `
|
||||||
|
UPDATE packages p
|
||||||
|
SET access_count = COALESCE((
|
||||||
|
SELECT SUM(download_count)
|
||||||
|
FROM download_stats_hourly dsh
|
||||||
|
WHERE dsh.package_id = p.id
|
||||||
|
), 0)
|
||||||
|
`
|
||||||
|
|
||||||
|
if err := w.db.Exec(updateSQL).Error; err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
log.Info().Msg("Updated package access counts")
|
||||||
|
return nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,78 @@
|
|||||||
|
package gormstore
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/lukaszraczylo/gohoarder/pkg/errors"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Config holds GORM store configuration
|
||||||
|
type Config struct {
|
||||||
|
// Database connection
|
||||||
|
Driver string // "sqlite", "postgres", "mysql"
|
||||||
|
DSN string // Data Source Name
|
||||||
|
|
||||||
|
// Connection pool
|
||||||
|
MaxOpenConns int
|
||||||
|
MaxIdleConns int
|
||||||
|
ConnMaxLifetime time.Duration
|
||||||
|
|
||||||
|
// GORM settings
|
||||||
|
LogLevel string // "silent", "error", "warn", "info"
|
||||||
|
}
|
||||||
|
|
||||||
|
// Validate validates the configuration
|
||||||
|
func (c *Config) Validate() error {
|
||||||
|
if c.Driver == "" {
|
||||||
|
return errors.New(errors.ErrCodeInvalidConfig, "driver is required")
|
||||||
|
}
|
||||||
|
if c.DSN == "" {
|
||||||
|
return errors.New(errors.ErrCodeInvalidConfig, "DSN is required")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Set defaults
|
||||||
|
if c.MaxOpenConns == 0 {
|
||||||
|
c.MaxOpenConns = 25
|
||||||
|
}
|
||||||
|
if c.MaxIdleConns == 0 {
|
||||||
|
c.MaxIdleConns = 5
|
||||||
|
}
|
||||||
|
if c.ConnMaxLifetime == 0 {
|
||||||
|
c.ConnMaxLifetime = time.Hour
|
||||||
|
}
|
||||||
|
if c.LogLevel == "" {
|
||||||
|
c.LogLevel = "warn"
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// BuildPostgresDSN builds PostgreSQL DSN from structured config
|
||||||
|
func BuildPostgresDSN(host string, port int, user, password, database, sslmode string) string {
|
||||||
|
if sslmode == "" {
|
||||||
|
sslmode = "disable"
|
||||||
|
}
|
||||||
|
return fmt.Sprintf("host=%s port=%d user=%s password=%s dbname=%s sslmode=%s",
|
||||||
|
host, port, user, password, database, sslmode)
|
||||||
|
}
|
||||||
|
|
||||||
|
// BuildMySQLDSN builds MySQL/MariaDB DSN from structured config
|
||||||
|
func BuildMySQLDSN(host string, port int, user, password, database, charset string) string {
|
||||||
|
if charset == "" {
|
||||||
|
charset = "utf8mb4"
|
||||||
|
}
|
||||||
|
return fmt.Sprintf("%s:%s@tcp(%s:%d)/%s?charset=%s&parseTime=True&loc=Local",
|
||||||
|
user, password, host, port, database, charset)
|
||||||
|
}
|
||||||
|
|
||||||
|
// BuildSQLiteDSN builds SQLite DSN with pragmas
|
||||||
|
func BuildSQLiteDSN(path string, walMode bool) string {
|
||||||
|
if path == "" {
|
||||||
|
path = "gohoarder.db"
|
||||||
|
}
|
||||||
|
if walMode {
|
||||||
|
return fmt.Sprintf("%s?_journal_mode=WAL&_busy_timeout=5000&_synchronous=NORMAL&_cache_size=2000", path)
|
||||||
|
}
|
||||||
|
return fmt.Sprintf("%s?_journal_mode=DELETE&_busy_timeout=5000&_synchronous=NORMAL", path)
|
||||||
|
}
|
||||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,279 @@
|
|||||||
|
//go:build integration
|
||||||
|
// +build integration
|
||||||
|
|
||||||
|
package gormstore
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/lukaszraczylo/gohoarder/pkg/metadata"
|
||||||
|
"github.com/stretchr/testify/suite"
|
||||||
|
"github.com/testcontainers/testcontainers-go"
|
||||||
|
"github.com/testcontainers/testcontainers-go/modules/mysql"
|
||||||
|
"github.com/testcontainers/testcontainers-go/wait"
|
||||||
|
)
|
||||||
|
|
||||||
|
// MySQLV2IntegrationTestSuite embeds the V2 test suite with MySQL container
|
||||||
|
type MySQLV2IntegrationTestSuite struct {
|
||||||
|
GORMStoreV2TestSuite
|
||||||
|
container *mysql.MySQLContainer
|
||||||
|
}
|
||||||
|
|
||||||
|
// SetupSuite runs once before all tests
|
||||||
|
func (s *MySQLV2IntegrationTestSuite) SetupSuite() {
|
||||||
|
ctx := context.Background()
|
||||||
|
|
||||||
|
// Start MySQL container
|
||||||
|
container, err := mysql.RunContainer(ctx,
|
||||||
|
testcontainers.WithImage("mysql:8.0"),
|
||||||
|
mysql.WithDatabase("testdb"),
|
||||||
|
mysql.WithUsername("testuser"),
|
||||||
|
mysql.WithPassword("testpass"),
|
||||||
|
testcontainers.WithWaitStrategy(
|
||||||
|
wait.ForLog("port: 3306 MySQL Community Server").
|
||||||
|
WithOccurrence(1).
|
||||||
|
WithStartupTimeout(60*time.Second),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
s.Require().NoError(err)
|
||||||
|
s.container = container
|
||||||
|
}
|
||||||
|
|
||||||
|
// TearDownSuite runs once after all tests
|
||||||
|
func (s *MySQLV2IntegrationTestSuite) TearDownSuite() {
|
||||||
|
if s.container != nil {
|
||||||
|
ctx := context.Background()
|
||||||
|
err := s.container.Terminate(ctx)
|
||||||
|
s.NoError(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// SetupTest runs before each test
|
||||||
|
func (s *MySQLV2IntegrationTestSuite) SetupTest() {
|
||||||
|
s.ctx = context.Background()
|
||||||
|
|
||||||
|
// Get connection string from container
|
||||||
|
connStr, err := s.container.ConnectionString(s.ctx)
|
||||||
|
s.Require().NoError(err)
|
||||||
|
|
||||||
|
// Create GORM store with MySQL
|
||||||
|
cfg := Config{
|
||||||
|
Driver: "mysql",
|
||||||
|
DSN: connStr,
|
||||||
|
MaxOpenConns: 10,
|
||||||
|
MaxIdleConns: 5,
|
||||||
|
ConnMaxLifetime: 3600 * time.Second,
|
||||||
|
LogLevel: "silent",
|
||||||
|
}
|
||||||
|
|
||||||
|
s.store, err = NewV2(cfg)
|
||||||
|
s.Require().NoError(err)
|
||||||
|
s.Require().NotNil(s.store)
|
||||||
|
}
|
||||||
|
|
||||||
|
// TearDownTest runs after each test
|
||||||
|
func (s *MySQLV2IntegrationTestSuite) TearDownTest() {
|
||||||
|
if s.store != nil {
|
||||||
|
// Clean up all tables for next test
|
||||||
|
tables := []string{
|
||||||
|
"download_events",
|
||||||
|
"download_stats_hourly",
|
||||||
|
"download_stats_daily",
|
||||||
|
"audit_log",
|
||||||
|
"cve_bypasses",
|
||||||
|
"scan_results",
|
||||||
|
"package_vulnerabilities",
|
||||||
|
"vulnerabilities",
|
||||||
|
"package_metadata",
|
||||||
|
"packages",
|
||||||
|
"registries",
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, table := range tables {
|
||||||
|
s.store.db.Exec("TRUNCATE TABLE " + table)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Re-seed default registries after truncate
|
||||||
|
defaultRegistries := []RegistryModel{
|
||||||
|
{Name: "npm", DisplayName: "NPM Registry", UpstreamURL: "https://registry.npmjs.org", Enabled: true, ScanByDefault: true},
|
||||||
|
{Name: "pypi", DisplayName: "PyPI", UpstreamURL: "https://pypi.org", Enabled: true, ScanByDefault: true},
|
||||||
|
{Name: "go", DisplayName: "Go Modules", UpstreamURL: "https://proxy.golang.org", Enabled: true, ScanByDefault: true},
|
||||||
|
}
|
||||||
|
for _, reg := range defaultRegistries {
|
||||||
|
s.store.db.Create(®)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Rebuild registry cache
|
||||||
|
s.store.rebuildRegistryCache()
|
||||||
|
|
||||||
|
s.store.Close()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestMySQLV2IntegrationTestSuite runs the integration test suite with MySQL
|
||||||
|
func TestMySQLV2IntegrationTestSuite(t *testing.T) {
|
||||||
|
if testing.Short() {
|
||||||
|
t.Skip("Skipping integration tests in short mode")
|
||||||
|
}
|
||||||
|
|
||||||
|
suite.Run(t, new(MySQLV2IntegrationTestSuite))
|
||||||
|
}
|
||||||
|
|
||||||
|
// Test_MySQLV2_SpecificFeatures tests MySQL-specific features
|
||||||
|
func (s *MySQLV2IntegrationTestSuite) Test_MySQLV2_SpecificFeatures() {
|
||||||
|
// Test that we're actually using MySQL
|
||||||
|
var version string
|
||||||
|
err := s.store.db.Raw("SELECT VERSION()").Scan(&version).Error
|
||||||
|
s.NoError(err)
|
||||||
|
s.Contains(version, "MySQL")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Test_MySQLV2_NoPartitioning tests that partition manager is nil for MySQL
|
||||||
|
func (s *MySQLV2IntegrationTestSuite) Test_MySQLV2_NoPartitioning() {
|
||||||
|
// MySQL doesn't use our partition manager (uses native partitioning differently)
|
||||||
|
s.Nil(s.store.partitionManager)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Test_MySQLV2_HighConcurrency tests MySQL's concurrent write support
|
||||||
|
func (s *MySQLV2IntegrationTestSuite) Test_MySQLV2_HighConcurrency() {
|
||||||
|
pkg := &metadata.Package{
|
||||||
|
Registry: "npm",
|
||||||
|
Name: "concurrent-test",
|
||||||
|
Version: "1.0.0",
|
||||||
|
StorageKey: "npm/concurrent-test/1.0.0.tgz",
|
||||||
|
Size: 1000,
|
||||||
|
CachedAt: time.Now(),
|
||||||
|
LastAccessed: time.Now(),
|
||||||
|
}
|
||||||
|
err := s.store.SavePackage(s.ctx, pkg)
|
||||||
|
s.NoError(err)
|
||||||
|
|
||||||
|
// MySQL can handle concurrent writes (with InnoDB row-level locking)
|
||||||
|
concurrency := 15
|
||||||
|
done := make(chan bool, concurrency)
|
||||||
|
|
||||||
|
for i := 0; i < concurrency; i++ {
|
||||||
|
go func() {
|
||||||
|
err := s.store.UpdateDownloadCount(s.ctx, "npm", "concurrent-test", "1.0.0")
|
||||||
|
s.NoError(err)
|
||||||
|
done <- true
|
||||||
|
}()
|
||||||
|
}
|
||||||
|
|
||||||
|
// Wait for all goroutines
|
||||||
|
for i := 0; i < concurrency; i++ {
|
||||||
|
<-done
|
||||||
|
}
|
||||||
|
|
||||||
|
// Verify all updates succeeded
|
||||||
|
retrieved, err := s.store.GetPackage(s.ctx, "npm", "concurrent-test", "1.0.0")
|
||||||
|
s.NoError(err)
|
||||||
|
s.Equal(int64(concurrency), retrieved.DownloadCount)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Test_MySQLV2_JSON tests MySQL JSON functionality
|
||||||
|
func (s *MySQLV2IntegrationTestSuite) Test_MySQLV2_JSON() {
|
||||||
|
metadata := map[string]interface{}{
|
||||||
|
"author": "Test Author",
|
||||||
|
"license": "MIT",
|
||||||
|
"description": "Test package",
|
||||||
|
"keywords": []interface{}{"test", "mysql", "json"},
|
||||||
|
}
|
||||||
|
|
||||||
|
pkg := &metadata.Package{
|
||||||
|
Registry: "npm",
|
||||||
|
Name: "json-test",
|
||||||
|
Version: "1.0.0",
|
||||||
|
StorageKey: "npm/json-test/1.0.0.tgz",
|
||||||
|
Size: 1000,
|
||||||
|
CachedAt: time.Now(),
|
||||||
|
LastAccessed: time.Now(),
|
||||||
|
Metadata: metadata,
|
||||||
|
}
|
||||||
|
|
||||||
|
err := s.store.SavePackage(s.ctx, pkg)
|
||||||
|
s.NoError(err)
|
||||||
|
|
||||||
|
// Retrieve and verify JSON data
|
||||||
|
retrieved, err := s.store.GetPackage(s.ctx, "npm", "json-test", "1.0.0")
|
||||||
|
s.NoError(err)
|
||||||
|
s.NotNil(retrieved.Metadata)
|
||||||
|
s.Equal("MIT", retrieved.Metadata["license"])
|
||||||
|
s.Equal("Test Author", retrieved.Metadata["author"])
|
||||||
|
}
|
||||||
|
|
||||||
|
// Test_MySQLV2_TransactionRollback tests MySQL transaction rollback
|
||||||
|
func (s *MySQLV2IntegrationTestSuite) Test_MySQLV2_TransactionRollback() {
|
||||||
|
pkg := &metadata.Package{
|
||||||
|
Registry: "npm",
|
||||||
|
Name: "tx-test",
|
||||||
|
Version: "1.0.0",
|
||||||
|
StorageKey: "npm/tx-test/1.0.0.tgz",
|
||||||
|
Size: 1000,
|
||||||
|
CachedAt: time.Now(),
|
||||||
|
LastAccessed: time.Now(),
|
||||||
|
}
|
||||||
|
err := s.store.SavePackage(s.ctx, pkg)
|
||||||
|
s.NoError(err)
|
||||||
|
|
||||||
|
// Try to update with invalid data that should trigger rollback
|
||||||
|
err = s.store.db.Transaction(func(tx *gorm.DB) error {
|
||||||
|
// First update succeeds
|
||||||
|
result := tx.Model(&PackageModel{}).
|
||||||
|
Where("registry_id = ? AND name = ? AND version = ?",
|
||||||
|
s.store.registryCache["npm"], "tx-test", "1.0.0").
|
||||||
|
Update("access_count", gorm.Expr("access_count + ?", 1))
|
||||||
|
if result.Error != nil {
|
||||||
|
return result.Error
|
||||||
|
}
|
||||||
|
|
||||||
|
// Second operation fails (invalid foreign key)
|
||||||
|
invalidModel := &PackageModel{
|
||||||
|
RegistryID: 9999, // Non-existent registry
|
||||||
|
Name: "invalid",
|
||||||
|
Version: "1.0.0",
|
||||||
|
StorageKey: "invalid",
|
||||||
|
Size: 100,
|
||||||
|
CachedAt: time.Now(),
|
||||||
|
LastAccessed: time.Now(),
|
||||||
|
}
|
||||||
|
return tx.Create(invalidModel).Error
|
||||||
|
})
|
||||||
|
|
||||||
|
// Transaction should fail
|
||||||
|
s.Error(err)
|
||||||
|
|
||||||
|
// Verify first update was rolled back
|
||||||
|
retrieved, err := s.store.GetPackage(s.ctx, "npm", "tx-test", "1.0.0")
|
||||||
|
s.NoError(err)
|
||||||
|
s.Equal(int64(0), retrieved.DownloadCount) // Should still be 0, not 1
|
||||||
|
}
|
||||||
|
|
||||||
|
// Test_MySQLV2_CharacterSet tests MySQL UTF-8 support
|
||||||
|
func (s *MySQLV2IntegrationTestSuite) Test_MySQLV2_CharacterSet() {
|
||||||
|
// Test package with Unicode characters
|
||||||
|
pkg := &metadata.Package{
|
||||||
|
Registry: "npm",
|
||||||
|
Name: "unicode-test-世界-🚀",
|
||||||
|
Version: "1.0.0",
|
||||||
|
StorageKey: "npm/unicode-test/1.0.0.tgz",
|
||||||
|
Size: 1000,
|
||||||
|
CachedAt: time.Now(),
|
||||||
|
LastAccessed: time.Now(),
|
||||||
|
Metadata: map[string]interface{}{
|
||||||
|
"description": "Test with emoji 🎉 and Chinese 中文",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
err := s.store.SavePackage(s.ctx, pkg)
|
||||||
|
s.NoError(err)
|
||||||
|
|
||||||
|
// Retrieve and verify Unicode data preserved
|
||||||
|
retrieved, err := s.store.GetPackage(s.ctx, "npm", "unicode-test-世界-🚀", "1.0.0")
|
||||||
|
s.NoError(err)
|
||||||
|
s.Equal("unicode-test-世界-🚀", retrieved.Name)
|
||||||
|
s.Contains(retrieved.Metadata["description"], "🎉")
|
||||||
|
s.Contains(retrieved.Metadata["description"], "中文")
|
||||||
|
}
|
||||||
@@ -0,0 +1,202 @@
|
|||||||
|
//go:build integration
|
||||||
|
// +build integration
|
||||||
|
|
||||||
|
package gormstore
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/lukaszraczylo/gohoarder/pkg/metadata"
|
||||||
|
"github.com/stretchr/testify/suite"
|
||||||
|
"github.com/testcontainers/testcontainers-go"
|
||||||
|
"github.com/testcontainers/testcontainers-go/modules/postgres"
|
||||||
|
"github.com/testcontainers/testcontainers-go/wait"
|
||||||
|
)
|
||||||
|
|
||||||
|
// PostgresV2IntegrationTestSuite embeds the V2 test suite with PostgreSQL container
|
||||||
|
type PostgresV2IntegrationTestSuite struct {
|
||||||
|
GORMStoreV2TestSuite
|
||||||
|
container *postgres.PostgresContainer
|
||||||
|
}
|
||||||
|
|
||||||
|
// SetupSuite runs once before all tests
|
||||||
|
func (s *PostgresV2IntegrationTestSuite) SetupSuite() {
|
||||||
|
ctx := context.Background()
|
||||||
|
|
||||||
|
// Start PostgreSQL container
|
||||||
|
container, err := postgres.RunContainer(ctx,
|
||||||
|
testcontainers.WithImage("postgres:16-alpine"),
|
||||||
|
postgres.WithDatabase("testdb"),
|
||||||
|
postgres.WithUsername("testuser"),
|
||||||
|
postgres.WithPassword("testpass"),
|
||||||
|
testcontainers.WithWaitStrategy(
|
||||||
|
wait.ForLog("database system is ready to accept connections").
|
||||||
|
WithOccurrence(2).
|
||||||
|
WithStartupTimeout(60*time.Second),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
s.Require().NoError(err)
|
||||||
|
s.container = container
|
||||||
|
}
|
||||||
|
|
||||||
|
// TearDownSuite runs once after all tests
|
||||||
|
func (s *PostgresV2IntegrationTestSuite) TearDownSuite() {
|
||||||
|
if s.container != nil {
|
||||||
|
ctx := context.Background()
|
||||||
|
err := s.container.Terminate(ctx)
|
||||||
|
s.NoError(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// SetupTest runs before each test
|
||||||
|
func (s *PostgresV2IntegrationTestSuite) SetupTest() {
|
||||||
|
s.ctx = context.Background()
|
||||||
|
|
||||||
|
// Get connection string from container
|
||||||
|
connStr, err := s.container.ConnectionString(s.ctx)
|
||||||
|
s.Require().NoError(err)
|
||||||
|
|
||||||
|
// Create GORM store with PostgreSQL
|
||||||
|
cfg := Config{
|
||||||
|
Driver: "postgres",
|
||||||
|
DSN: connStr + "sslmode=disable",
|
||||||
|
MaxOpenConns: 10,
|
||||||
|
MaxIdleConns: 5,
|
||||||
|
ConnMaxLifetime: 3600 * time.Second,
|
||||||
|
LogLevel: "silent",
|
||||||
|
}
|
||||||
|
|
||||||
|
s.store, err = NewV2(cfg)
|
||||||
|
s.Require().NoError(err)
|
||||||
|
s.Require().NotNil(s.store)
|
||||||
|
}
|
||||||
|
|
||||||
|
// TearDownTest runs after each test
|
||||||
|
func (s *PostgresV2IntegrationTestSuite) TearDownTest() {
|
||||||
|
if s.store != nil {
|
||||||
|
// Clean up all tables for next test
|
||||||
|
tables := []string{
|
||||||
|
"download_events",
|
||||||
|
"download_stats_hourly",
|
||||||
|
"download_stats_daily",
|
||||||
|
"audit_log",
|
||||||
|
"cve_bypasses",
|
||||||
|
"scan_results",
|
||||||
|
"package_vulnerabilities",
|
||||||
|
"vulnerabilities",
|
||||||
|
"package_metadata",
|
||||||
|
"packages",
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, table := range tables {
|
||||||
|
s.store.db.Exec("TRUNCATE TABLE " + table + " CASCADE")
|
||||||
|
}
|
||||||
|
|
||||||
|
s.store.Close()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestPostgresV2IntegrationTestSuite runs the integration test suite with PostgreSQL
|
||||||
|
func TestPostgresV2IntegrationTestSuite(t *testing.T) {
|
||||||
|
if testing.Short() {
|
||||||
|
t.Skip("Skipping integration tests in short mode")
|
||||||
|
}
|
||||||
|
|
||||||
|
suite.Run(t, new(PostgresV2IntegrationTestSuite))
|
||||||
|
}
|
||||||
|
|
||||||
|
// Test_PostgresV2_SpecificFeatures tests PostgreSQL-specific features
|
||||||
|
func (s *PostgresV2IntegrationTestSuite) Test_PostgresV2_SpecificFeatures() {
|
||||||
|
// Test that we're actually using PostgreSQL
|
||||||
|
var version string
|
||||||
|
err := s.store.db.Raw("SELECT version()").Scan(&version).Error
|
||||||
|
s.NoError(err)
|
||||||
|
s.Contains(version, "PostgreSQL")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Test_PostgresV2_Partitioning tests partition manager
|
||||||
|
func (s *PostgresV2IntegrationTestSuite) Test_PostgresV2_Partitioning() {
|
||||||
|
s.NotNil(s.store.partitionManager)
|
||||||
|
|
||||||
|
// Get partition info
|
||||||
|
info, err := s.store.partitionManager.GetPartitionInfo()
|
||||||
|
s.NoError(err)
|
||||||
|
s.NotNil(info)
|
||||||
|
|
||||||
|
// Should have created partitions
|
||||||
|
downloadPartitions := info["download_events_partitions"].(int64)
|
||||||
|
s.Greater(downloadPartitions, int64(0))
|
||||||
|
|
||||||
|
auditPartitions := info["audit_log_partitions"].(int64)
|
||||||
|
s.Greater(auditPartitions, int64(0))
|
||||||
|
}
|
||||||
|
|
||||||
|
// Test_PostgresV2_HighConcurrency tests PostgreSQL's excellent concurrent write support
|
||||||
|
func (s *PostgresV2IntegrationTestSuite) Test_PostgresV2_HighConcurrency() {
|
||||||
|
pkg := &metadata.Package{
|
||||||
|
Registry: "npm",
|
||||||
|
Name: "concurrent-test",
|
||||||
|
Version: "1.0.0",
|
||||||
|
StorageKey: "npm/concurrent-test/1.0.0.tgz",
|
||||||
|
Size: 1000,
|
||||||
|
CachedAt: time.Now(),
|
||||||
|
LastAccessed: time.Now(),
|
||||||
|
}
|
||||||
|
err := s.store.SavePackage(s.ctx, pkg)
|
||||||
|
s.NoError(err)
|
||||||
|
|
||||||
|
// PostgreSQL can handle many concurrent writes
|
||||||
|
concurrency := 20
|
||||||
|
done := make(chan bool, concurrency)
|
||||||
|
|
||||||
|
for i := 0; i < concurrency; i++ {
|
||||||
|
go func() {
|
||||||
|
err := s.store.UpdateDownloadCount(s.ctx, "npm", "concurrent-test", "1.0.0")
|
||||||
|
s.NoError(err)
|
||||||
|
done <- true
|
||||||
|
}()
|
||||||
|
}
|
||||||
|
|
||||||
|
// Wait for all goroutines
|
||||||
|
for i := 0; i < concurrency; i++ {
|
||||||
|
<-done
|
||||||
|
}
|
||||||
|
|
||||||
|
// Verify all updates succeeded
|
||||||
|
retrieved, err := s.store.GetPackage(s.ctx, "npm", "concurrent-test", "1.0.0")
|
||||||
|
s.NoError(err)
|
||||||
|
s.Equal(int64(concurrency), retrieved.DownloadCount)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Test_PostgresV2_JSONB tests PostgreSQL JSONB functionality
|
||||||
|
func (s *PostgresV2IntegrationTestSuite) Test_PostgresV2_JSONB() {
|
||||||
|
metadata := map[string]interface{}{
|
||||||
|
"author": "Test Author",
|
||||||
|
"license": "MIT",
|
||||||
|
"description": "Test package",
|
||||||
|
"keywords": []interface{}{"test", "postgres", "jsonb"},
|
||||||
|
}
|
||||||
|
|
||||||
|
pkg := &metadata.Package{
|
||||||
|
Registry: "npm",
|
||||||
|
Name: "jsonb-test",
|
||||||
|
Version: "1.0.0",
|
||||||
|
StorageKey: "npm/jsonb-test/1.0.0.tgz",
|
||||||
|
Size: 1000,
|
||||||
|
CachedAt: time.Now(),
|
||||||
|
LastAccessed: time.Now(),
|
||||||
|
Metadata: metadata,
|
||||||
|
}
|
||||||
|
|
||||||
|
err := s.store.SavePackage(s.ctx, pkg)
|
||||||
|
s.NoError(err)
|
||||||
|
|
||||||
|
// Retrieve and verify JSONB data
|
||||||
|
retrieved, err := s.store.GetPackage(s.ctx, "npm", "jsonb-test", "1.0.0")
|
||||||
|
s.NoError(err)
|
||||||
|
s.NotNil(retrieved.Metadata)
|
||||||
|
s.Equal("MIT", retrieved.Metadata["license"])
|
||||||
|
s.Equal("Test Author", retrieved.Metadata["author"])
|
||||||
|
}
|
||||||
@@ -0,0 +1,871 @@
|
|||||||
|
package gormstore
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/lukaszraczylo/gohoarder/pkg/metadata"
|
||||||
|
"github.com/stretchr/testify/suite"
|
||||||
|
"gorm.io/driver/sqlite"
|
||||||
|
"gorm.io/gorm"
|
||||||
|
"gorm.io/gorm/logger"
|
||||||
|
)
|
||||||
|
|
||||||
|
// GORMStoreV2TestSuite is the test suite for V2 GORM implementation
|
||||||
|
type GORMStoreV2TestSuite struct {
|
||||||
|
suite.Suite
|
||||||
|
db *gorm.DB
|
||||||
|
store *GORMStoreV2
|
||||||
|
ctx context.Context
|
||||||
|
}
|
||||||
|
|
||||||
|
// SetupSuite runs once before all tests
|
||||||
|
func (s *GORMStoreV2TestSuite) SetupSuite() {
|
||||||
|
// Use in-memory SQLite for fast tests
|
||||||
|
db, err := gorm.Open(sqlite.Open("file::memory:?cache=shared"), &gorm.Config{
|
||||||
|
Logger: logger.Default.LogMode(logger.Silent),
|
||||||
|
})
|
||||||
|
s.Require().NoError(err)
|
||||||
|
s.db = db
|
||||||
|
}
|
||||||
|
|
||||||
|
// SetupTest runs before each test
|
||||||
|
func (s *GORMStoreV2TestSuite) SetupTest() {
|
||||||
|
s.ctx = context.Background()
|
||||||
|
|
||||||
|
// Create fresh store with V2 schema
|
||||||
|
cfg := Config{
|
||||||
|
Driver: "sqlite",
|
||||||
|
DSN: "file::memory:?cache=shared",
|
||||||
|
MaxOpenConns: 10,
|
||||||
|
MaxIdleConns: 5,
|
||||||
|
ConnMaxLifetime: 3600 * time.Second,
|
||||||
|
LogLevel: "silent",
|
||||||
|
}
|
||||||
|
|
||||||
|
store, err := NewV2(cfg)
|
||||||
|
s.Require().NoError(err)
|
||||||
|
s.Require().NotNil(store)
|
||||||
|
|
||||||
|
s.store = store
|
||||||
|
}
|
||||||
|
|
||||||
|
// TearDownTest runs after each test
|
||||||
|
func (s *GORMStoreV2TestSuite) TearDownTest() {
|
||||||
|
if s.store != nil {
|
||||||
|
// Clean up all tables for next test
|
||||||
|
tables := []string{
|
||||||
|
"audit_log",
|
||||||
|
"download_stats_daily",
|
||||||
|
"download_stats_hourly",
|
||||||
|
"download_events",
|
||||||
|
"cve_bypasses",
|
||||||
|
"scan_results",
|
||||||
|
"package_vulnerabilities",
|
||||||
|
"vulnerabilities",
|
||||||
|
"package_metadata",
|
||||||
|
"packages",
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, table := range tables {
|
||||||
|
s.store.db.Exec(fmt.Sprintf("DELETE FROM %s", table))
|
||||||
|
}
|
||||||
|
|
||||||
|
s.store.Close()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestGORMStoreV2TestSuite runs the test suite
|
||||||
|
func TestGORMStoreV2TestSuite(t *testing.T) {
|
||||||
|
suite.Run(t, new(GORMStoreV2TestSuite))
|
||||||
|
}
|
||||||
|
|
||||||
|
// Test_V2_SavePackage_Success tests saving a package
|
||||||
|
func (s *GORMStoreV2TestSuite) Test_V2_SavePackage_Success() {
|
||||||
|
pkg := &metadata.Package{
|
||||||
|
Registry: "npm",
|
||||||
|
Name: "test-package",
|
||||||
|
Version: "1.0.0",
|
||||||
|
StorageKey: "npm/test-package/1.0.0.tgz",
|
||||||
|
Size: 12345,
|
||||||
|
ChecksumMD5: "abc123",
|
||||||
|
ChecksumSHA256: "def456",
|
||||||
|
UpstreamURL: "https://registry.npmjs.org/test-package",
|
||||||
|
CachedAt: time.Now(),
|
||||||
|
LastAccessed: time.Now(),
|
||||||
|
}
|
||||||
|
|
||||||
|
err := s.store.SavePackage(s.ctx, pkg)
|
||||||
|
s.NoError(err)
|
||||||
|
|
||||||
|
// Verify package was saved
|
||||||
|
retrieved, err := s.store.GetPackage(s.ctx, "npm", "test-package", "1.0.0")
|
||||||
|
s.NoError(err)
|
||||||
|
s.NotNil(retrieved)
|
||||||
|
s.Equal("npm", retrieved.Registry)
|
||||||
|
s.Equal("test-package", retrieved.Name)
|
||||||
|
s.Equal("1.0.0", retrieved.Version)
|
||||||
|
s.Equal(int64(12345), retrieved.Size)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Test_V2_SavePackage_WithMetadata tests saving package with metadata
|
||||||
|
func (s *GORMStoreV2TestSuite) Test_V2_SavePackage_WithMetadata() {
|
||||||
|
metadataMap := map[string]string{
|
||||||
|
"author": "Test Author",
|
||||||
|
"license": "MIT",
|
||||||
|
"homepage": "https://example.com",
|
||||||
|
"description": "Test package description",
|
||||||
|
}
|
||||||
|
|
||||||
|
pkg := &metadata.Package{
|
||||||
|
Registry: "npm",
|
||||||
|
Name: "meta-package",
|
||||||
|
Version: "2.0.0",
|
||||||
|
StorageKey: "npm/meta-package/2.0.0.tgz",
|
||||||
|
Size: 5000,
|
||||||
|
CachedAt: time.Now(),
|
||||||
|
LastAccessed: time.Now(),
|
||||||
|
Metadata: metadataMap,
|
||||||
|
}
|
||||||
|
|
||||||
|
err := s.store.SavePackage(s.ctx, pkg)
|
||||||
|
s.NoError(err)
|
||||||
|
|
||||||
|
// Verify metadata was saved in separate table
|
||||||
|
var pkgMetadata PackageMetadataModel
|
||||||
|
err = s.store.db.Where("package_id = (SELECT id FROM packages WHERE name = ?)", "meta-package").
|
||||||
|
First(&pkgMetadata).Error
|
||||||
|
s.NoError(err)
|
||||||
|
s.Equal("Test Author", pkgMetadata.Author)
|
||||||
|
s.Equal("MIT", pkgMetadata.License)
|
||||||
|
s.Equal("https://example.com", pkgMetadata.Homepage)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Test_V2_SavePackage_Upsert tests update on conflict
|
||||||
|
func (s *GORMStoreV2TestSuite) Test_V2_SavePackage_Upsert() {
|
||||||
|
// Save initial package
|
||||||
|
pkg := &metadata.Package{
|
||||||
|
Registry: "npm",
|
||||||
|
Name: "upsert-test",
|
||||||
|
Version: "1.0.0",
|
||||||
|
StorageKey: "npm/upsert-test/1.0.0.tgz",
|
||||||
|
Size: 1000,
|
||||||
|
CachedAt: time.Now(),
|
||||||
|
LastAccessed: time.Now(),
|
||||||
|
}
|
||||||
|
err := s.store.SavePackage(s.ctx, pkg)
|
||||||
|
s.NoError(err)
|
||||||
|
|
||||||
|
// Update same package
|
||||||
|
pkg.Size = 2000
|
||||||
|
pkg.ChecksumMD5 = "updated"
|
||||||
|
err = s.store.SavePackage(s.ctx, pkg)
|
||||||
|
s.NoError(err)
|
||||||
|
|
||||||
|
// Verify updated
|
||||||
|
retrieved, err := s.store.GetPackage(s.ctx, "npm", "upsert-test", "1.0.0")
|
||||||
|
s.NoError(err)
|
||||||
|
s.Equal(int64(2000), retrieved.Size)
|
||||||
|
s.Equal("updated", retrieved.ChecksumMD5)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Test_V2_GetPackage_NotFound tests getting non-existent package
|
||||||
|
func (s *GORMStoreV2TestSuite) Test_V2_GetPackage_NotFound() {
|
||||||
|
_, err := s.store.GetPackage(s.ctx, "npm", "nonexistent", "1.0.0")
|
||||||
|
s.Error(err)
|
||||||
|
s.Contains(err.Error(), "not found")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Test_V2_DeletePackage_Success tests soft delete
|
||||||
|
func (s *GORMStoreV2TestSuite) Test_V2_DeletePackage_Success() {
|
||||||
|
// Save package
|
||||||
|
pkg := &metadata.Package{
|
||||||
|
Registry: "npm",
|
||||||
|
Name: "delete-test",
|
||||||
|
Version: "1.0.0",
|
||||||
|
StorageKey: "npm/delete-test/1.0.0.tgz",
|
||||||
|
Size: 1000,
|
||||||
|
CachedAt: time.Now(),
|
||||||
|
LastAccessed: time.Now(),
|
||||||
|
}
|
||||||
|
err := s.store.SavePackage(s.ctx, pkg)
|
||||||
|
s.NoError(err)
|
||||||
|
|
||||||
|
// Delete package (soft delete)
|
||||||
|
err = s.store.DeletePackage(s.ctx, "npm", "delete-test", "1.0.0")
|
||||||
|
s.NoError(err)
|
||||||
|
|
||||||
|
// Verify deleted (should not be found)
|
||||||
|
_, err = s.store.GetPackage(s.ctx, "npm", "delete-test", "1.0.0")
|
||||||
|
s.Error(err)
|
||||||
|
|
||||||
|
// Verify soft delete (deleted_at set)
|
||||||
|
var count int64
|
||||||
|
s.store.db.Unscoped().Model(&PackageModel{}).
|
||||||
|
Where("name = ?", "delete-test").
|
||||||
|
Count(&count)
|
||||||
|
s.Equal(int64(1), count) // Still in DB, just soft deleted
|
||||||
|
}
|
||||||
|
|
||||||
|
// Test_V2_ListPackages_All tests listing all packages
|
||||||
|
func (s *GORMStoreV2TestSuite) Test_V2_ListPackages_All() {
|
||||||
|
// Create multiple packages
|
||||||
|
for i := 0; i < 5; i++ {
|
||||||
|
pkg := &metadata.Package{
|
||||||
|
Registry: "npm",
|
||||||
|
Name: fmt.Sprintf("package-%d", i),
|
||||||
|
Version: "1.0.0",
|
||||||
|
StorageKey: fmt.Sprintf("npm/package-%d/1.0.0.tgz", i),
|
||||||
|
Size: int64(i * 1000),
|
||||||
|
CachedAt: time.Now(),
|
||||||
|
LastAccessed: time.Now(),
|
||||||
|
}
|
||||||
|
err := s.store.SavePackage(s.ctx, pkg)
|
||||||
|
s.NoError(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// List all packages
|
||||||
|
packages, err := s.store.ListPackages(s.ctx, &metadata.ListOptions{})
|
||||||
|
s.NoError(err)
|
||||||
|
s.Len(packages, 5)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Test_V2_ListPackages_FilterByRegistry tests filtering by registry
|
||||||
|
func (s *GORMStoreV2TestSuite) Test_V2_ListPackages_FilterByRegistry() {
|
||||||
|
// Create packages in different registries
|
||||||
|
registries := []string{"npm", "pypi", "go"}
|
||||||
|
for _, reg := range registries {
|
||||||
|
pkg := &metadata.Package{
|
||||||
|
Registry: reg,
|
||||||
|
Name: "test-package",
|
||||||
|
Version: "1.0.0",
|
||||||
|
StorageKey: fmt.Sprintf("%s/test-package/1.0.0", reg),
|
||||||
|
Size: 1000,
|
||||||
|
CachedAt: time.Now(),
|
||||||
|
LastAccessed: time.Now(),
|
||||||
|
}
|
||||||
|
err := s.store.SavePackage(s.ctx, pkg)
|
||||||
|
s.NoError(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Filter by npm registry
|
||||||
|
packages, err := s.store.ListPackages(s.ctx, &metadata.ListOptions{
|
||||||
|
Registry: "npm",
|
||||||
|
})
|
||||||
|
s.NoError(err)
|
||||||
|
s.Len(packages, 1)
|
||||||
|
s.Equal("npm", packages[0].Registry)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Test_V2_ListPackages_Pagination tests pagination
|
||||||
|
func (s *GORMStoreV2TestSuite) Test_V2_ListPackages_Pagination() {
|
||||||
|
// Create 10 packages
|
||||||
|
for i := 0; i < 10; i++ {
|
||||||
|
pkg := &metadata.Package{
|
||||||
|
Registry: "npm",
|
||||||
|
Name: fmt.Sprintf("package-%d", i),
|
||||||
|
Version: "1.0.0",
|
||||||
|
StorageKey: fmt.Sprintf("npm/package-%d/1.0.0.tgz", i),
|
||||||
|
Size: int64(i * 1000),
|
||||||
|
CachedAt: time.Now(),
|
||||||
|
LastAccessed: time.Now(),
|
||||||
|
}
|
||||||
|
err := s.store.SavePackage(s.ctx, pkg)
|
||||||
|
s.NoError(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Get first page (5 items)
|
||||||
|
page1, err := s.store.ListPackages(s.ctx, &metadata.ListOptions{
|
||||||
|
Limit: 5,
|
||||||
|
Offset: 0,
|
||||||
|
})
|
||||||
|
s.NoError(err)
|
||||||
|
s.Len(page1, 5)
|
||||||
|
|
||||||
|
// Get second page (5 items)
|
||||||
|
page2, err := s.store.ListPackages(s.ctx, &metadata.ListOptions{
|
||||||
|
Limit: 5,
|
||||||
|
Offset: 5,
|
||||||
|
})
|
||||||
|
s.NoError(err)
|
||||||
|
s.Len(page2, 5)
|
||||||
|
|
||||||
|
// Verify different packages
|
||||||
|
s.NotEqual(page1[0].Name, page2[0].Name)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Test_V2_UpdateDownloadCount_Success tests incrementing download count
|
||||||
|
func (s *GORMStoreV2TestSuite) Test_V2_UpdateDownloadCount_Success() {
|
||||||
|
// Create package
|
||||||
|
pkg := &metadata.Package{
|
||||||
|
Registry: "npm",
|
||||||
|
Name: "download-test",
|
||||||
|
Version: "1.0.0",
|
||||||
|
StorageKey: "npm/download-test/1.0.0.tgz",
|
||||||
|
Size: 1000,
|
||||||
|
CachedAt: time.Now(),
|
||||||
|
LastAccessed: time.Now(),
|
||||||
|
}
|
||||||
|
err := s.store.SavePackage(s.ctx, pkg)
|
||||||
|
s.NoError(err)
|
||||||
|
|
||||||
|
// Update download count
|
||||||
|
err = s.store.UpdateDownloadCount(s.ctx, "npm", "download-test", "1.0.0")
|
||||||
|
s.NoError(err)
|
||||||
|
|
||||||
|
// Verify count incremented
|
||||||
|
retrieved, err := s.store.GetPackage(s.ctx, "npm", "download-test", "1.0.0")
|
||||||
|
s.NoError(err)
|
||||||
|
s.Equal(int64(1), retrieved.DownloadCount)
|
||||||
|
|
||||||
|
// Update again
|
||||||
|
err = s.store.UpdateDownloadCount(s.ctx, "npm", "download-test", "1.0.0")
|
||||||
|
s.NoError(err)
|
||||||
|
|
||||||
|
retrieved, err = s.store.GetPackage(s.ctx, "npm", "download-test", "1.0.0")
|
||||||
|
s.NoError(err)
|
||||||
|
s.Equal(int64(2), retrieved.DownloadCount)
|
||||||
|
|
||||||
|
// Verify download event was recorded
|
||||||
|
var eventCount int64
|
||||||
|
s.store.db.Model(&DownloadEventModel{}).Count(&eventCount)
|
||||||
|
s.Equal(int64(2), eventCount)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Test_V2_Count tests counting packages
|
||||||
|
func (s *GORMStoreV2TestSuite) Test_V2_Count() {
|
||||||
|
// Initially zero
|
||||||
|
count, err := s.store.Count(s.ctx)
|
||||||
|
s.NoError(err)
|
||||||
|
s.Equal(0, count)
|
||||||
|
|
||||||
|
// Create 3 packages
|
||||||
|
for i := 0; i < 3; i++ {
|
||||||
|
pkg := &metadata.Package{
|
||||||
|
Registry: "npm",
|
||||||
|
Name: fmt.Sprintf("count-test-%d", i),
|
||||||
|
Version: "1.0.0",
|
||||||
|
StorageKey: fmt.Sprintf("npm/count-test-%d/1.0.0.tgz", i),
|
||||||
|
Size: 1000,
|
||||||
|
CachedAt: time.Now(),
|
||||||
|
LastAccessed: time.Now(),
|
||||||
|
}
|
||||||
|
err := s.store.SavePackage(s.ctx, pkg)
|
||||||
|
s.NoError(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
count, err = s.store.Count(s.ctx)
|
||||||
|
s.NoError(err)
|
||||||
|
s.Equal(3, count)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Test_V2_GetStats tests aggregated statistics
|
||||||
|
func (s *GORMStoreV2TestSuite) Test_V2_GetStats() {
|
||||||
|
// Create packages in different registries
|
||||||
|
packages := []*metadata.Package{
|
||||||
|
{Registry: "npm", Name: "pkg1", Version: "1.0.0", StorageKey: "npm/pkg1/1.0.0.tgz", Size: 1000, CachedAt: time.Now(), LastAccessed: time.Now()},
|
||||||
|
{Registry: "npm", Name: "pkg2", Version: "1.0.0", StorageKey: "npm/pkg2/1.0.0.tgz", Size: 2000, CachedAt: time.Now(), LastAccessed: time.Now()},
|
||||||
|
{Registry: "pypi", Name: "pkg3", Version: "1.0.0", StorageKey: "pypi/pkg3/1.0.0.tar.gz", Size: 3000, CachedAt: time.Now(), LastAccessed: time.Now()},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, pkg := range packages {
|
||||||
|
err := s.store.SavePackage(s.ctx, pkg)
|
||||||
|
s.NoError(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Update download counts
|
||||||
|
s.store.UpdateDownloadCount(s.ctx, "npm", "pkg1", "1.0.0")
|
||||||
|
s.store.UpdateDownloadCount(s.ctx, "npm", "pkg1", "1.0.0")
|
||||||
|
s.store.UpdateDownloadCount(s.ctx, "npm", "pkg2", "1.0.0")
|
||||||
|
|
||||||
|
// Get stats for all registries
|
||||||
|
statsAll, err := s.store.GetStats(s.ctx, "")
|
||||||
|
s.NoError(err)
|
||||||
|
s.Equal(int64(3), statsAll.TotalPackages)
|
||||||
|
s.Equal(int64(6000), statsAll.TotalSize)
|
||||||
|
s.Equal(int64(3), statsAll.TotalDownloads)
|
||||||
|
|
||||||
|
// Get stats for npm registry
|
||||||
|
statsNpm, err := s.store.GetStats(s.ctx, "npm")
|
||||||
|
s.NoError(err)
|
||||||
|
s.Equal("npm", statsNpm.Registry)
|
||||||
|
s.Equal(int64(2), statsNpm.TotalPackages)
|
||||||
|
s.Equal(int64(3000), statsNpm.TotalSize)
|
||||||
|
s.Equal(int64(3), statsNpm.TotalDownloads)
|
||||||
|
|
||||||
|
// Get stats for pypi registry
|
||||||
|
statsPypi, err := s.store.GetStats(s.ctx, "pypi")
|
||||||
|
s.NoError(err)
|
||||||
|
s.Equal("pypi", statsPypi.Registry)
|
||||||
|
s.Equal(int64(1), statsPypi.TotalPackages)
|
||||||
|
s.Equal(int64(3000), statsPypi.TotalSize)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Test_V2_Health tests database health check
|
||||||
|
func (s *GORMStoreV2TestSuite) Test_V2_Health() {
|
||||||
|
err := s.store.Health(s.ctx)
|
||||||
|
s.NoError(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Test_V2_RegistryCache tests registry caching
|
||||||
|
func (s *GORMStoreV2TestSuite) Test_V2_RegistryCache() {
|
||||||
|
// Default registries should be cached
|
||||||
|
s.Contains(s.store.registryCache, "npm")
|
||||||
|
s.Contains(s.store.registryCache, "pypi")
|
||||||
|
s.Contains(s.store.registryCache, "go")
|
||||||
|
|
||||||
|
// Get registry ID from cache
|
||||||
|
npmID, err := s.store.getRegistryID("npm")
|
||||||
|
s.NoError(err)
|
||||||
|
s.Greater(npmID, int32(0))
|
||||||
|
|
||||||
|
// Second call should use cache (no DB query)
|
||||||
|
npmID2, err := s.store.getRegistryID("npm")
|
||||||
|
s.NoError(err)
|
||||||
|
s.Equal(npmID, npmID2)
|
||||||
|
|
||||||
|
// Non-existent registry
|
||||||
|
_, err = s.store.getRegistryID("nonexistent")
|
||||||
|
s.Error(err)
|
||||||
|
s.Contains(err.Error(), "not found")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Test_V2_SoftDelete tests soft delete behavior
|
||||||
|
func (s *GORMStoreV2TestSuite) Test_V2_SoftDelete() {
|
||||||
|
// Create package
|
||||||
|
pkg := &metadata.Package{
|
||||||
|
Registry: "npm",
|
||||||
|
Name: "soft-delete",
|
||||||
|
Version: "1.0.0",
|
||||||
|
StorageKey: "npm/soft-delete/1.0.0.tgz",
|
||||||
|
Size: 1000,
|
||||||
|
CachedAt: time.Now(),
|
||||||
|
LastAccessed: time.Now(),
|
||||||
|
}
|
||||||
|
err := s.store.SavePackage(s.ctx, pkg)
|
||||||
|
s.NoError(err)
|
||||||
|
|
||||||
|
// Delete
|
||||||
|
err = s.store.DeletePackage(s.ctx, "npm", "soft-delete", "1.0.0")
|
||||||
|
s.NoError(err)
|
||||||
|
|
||||||
|
// Count should not include deleted
|
||||||
|
count, err := s.store.Count(s.ctx)
|
||||||
|
s.NoError(err)
|
||||||
|
s.Equal(0, count)
|
||||||
|
|
||||||
|
// But record still exists with deleted_at set
|
||||||
|
var pkgModel PackageModel
|
||||||
|
err = s.store.db.Unscoped().Where("name = ?", "soft-delete").First(&pkgModel).Error
|
||||||
|
s.NoError(err)
|
||||||
|
s.NotNil(pkgModel.DeletedAt)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Test_V2_AggregationWorker tests that aggregation worker is initialized
|
||||||
|
func (s *GORMStoreV2TestSuite) Test_V2_AggregationWorker() {
|
||||||
|
s.NotNil(s.store.aggregationWorker)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Test_V2_ConcurrentUpdates tests concurrent download count updates
|
||||||
|
func (s *GORMStoreV2TestSuite) Test_V2_ConcurrentUpdates() {
|
||||||
|
// Create package
|
||||||
|
pkg := &metadata.Package{
|
||||||
|
Registry: "npm",
|
||||||
|
Name: "concurrent-test",
|
||||||
|
Version: "1.0.0",
|
||||||
|
StorageKey: "npm/concurrent-test/1.0.0.tgz",
|
||||||
|
Size: 1000,
|
||||||
|
CachedAt: time.Now(),
|
||||||
|
LastAccessed: time.Now(),
|
||||||
|
}
|
||||||
|
err := s.store.SavePackage(s.ctx, pkg)
|
||||||
|
s.NoError(err)
|
||||||
|
|
||||||
|
// SQLite: Sequential updates only (write lock prevents concurrent writes)
|
||||||
|
updateCount := 5
|
||||||
|
for i := 0; i < updateCount; i++ {
|
||||||
|
err := s.store.UpdateDownloadCount(s.ctx, "npm", "concurrent-test", "1.0.0")
|
||||||
|
s.NoError(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Verify all updates succeeded
|
||||||
|
retrieved, err := s.store.GetPackage(s.ctx, "npm", "concurrent-test", "1.0.0")
|
||||||
|
s.NoError(err)
|
||||||
|
s.Equal(int64(updateCount), retrieved.DownloadCount)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Test_V2_SaveScanResult tests saving a scan result
|
||||||
|
func (s *GORMStoreV2TestSuite) Test_V2_SaveScanResult() {
|
||||||
|
// Create a package first
|
||||||
|
pkg := &metadata.Package{
|
||||||
|
Registry: "npm",
|
||||||
|
Name: "test-package",
|
||||||
|
Version: "1.0.0",
|
||||||
|
StorageKey: "/cache/npm/test-package-1.0.0.tgz",
|
||||||
|
Size: 1024,
|
||||||
|
UpstreamURL: "https://registry.npmjs.org/test-package",
|
||||||
|
CachedAt: time.Now(),
|
||||||
|
LastAccessed: time.Now(),
|
||||||
|
}
|
||||||
|
err := s.store.SavePackage(s.ctx, pkg)
|
||||||
|
s.NoError(err)
|
||||||
|
|
||||||
|
// Create and save a scan result
|
||||||
|
scanResult := &metadata.ScanResult{
|
||||||
|
Registry: "npm",
|
||||||
|
PackageName: "test-package",
|
||||||
|
PackageVersion: "1.0.0",
|
||||||
|
Scanner: "trivy",
|
||||||
|
Status: metadata.ScanStatusVulnerable,
|
||||||
|
ScannedAt: time.Now(),
|
||||||
|
Vulnerabilities: []metadata.Vulnerability{
|
||||||
|
{
|
||||||
|
ID: "CVE-2024-0001",
|
||||||
|
Severity: "HIGH",
|
||||||
|
Title: "Test vulnerability",
|
||||||
|
Description: "Test description",
|
||||||
|
References: []string{"https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2024-0001"},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
ID: "CVE-2024-0002",
|
||||||
|
Severity: "CRITICAL",
|
||||||
|
Title: "Critical vulnerability",
|
||||||
|
Description: "Critical test description",
|
||||||
|
References: []string{"https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2024-0002"},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
VulnerabilityCount: 2,
|
||||||
|
Details: map[string]interface{}{
|
||||||
|
"scan_duration": 42,
|
||||||
|
"scanner_version": "1.0.0",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
err = s.store.SaveScanResult(s.ctx, scanResult)
|
||||||
|
s.NoError(err)
|
||||||
|
|
||||||
|
// Verify the scan result was saved and package was updated
|
||||||
|
retrievedPkg, err := s.store.GetPackage(s.ctx, "npm", "test-package", "1.0.0")
|
||||||
|
s.NoError(err)
|
||||||
|
s.True(retrievedPkg.SecurityScanned)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Test_V2_GetScanResult tests retrieving a scan result
|
||||||
|
func (s *GORMStoreV2TestSuite) Test_V2_GetScanResult() {
|
||||||
|
// Create a package
|
||||||
|
pkg := &metadata.Package{
|
||||||
|
Registry: "npm",
|
||||||
|
Name: "scan-test",
|
||||||
|
Version: "2.0.0",
|
||||||
|
StorageKey: "/cache/npm/scan-test-2.0.0.tgz",
|
||||||
|
Size: 2048,
|
||||||
|
UpstreamURL: "https://registry.npmjs.org/scan-test",
|
||||||
|
CachedAt: time.Now(),
|
||||||
|
LastAccessed: time.Now(),
|
||||||
|
}
|
||||||
|
err := s.store.SavePackage(s.ctx, pkg)
|
||||||
|
s.NoError(err)
|
||||||
|
|
||||||
|
// Save a scan result with vulnerabilities
|
||||||
|
scanResult := &metadata.ScanResult{
|
||||||
|
Registry: "npm",
|
||||||
|
PackageName: "scan-test",
|
||||||
|
PackageVersion: "2.0.0",
|
||||||
|
Scanner: "grype",
|
||||||
|
Status: metadata.ScanStatusVulnerable,
|
||||||
|
ScannedAt: time.Now(),
|
||||||
|
Vulnerabilities: []metadata.Vulnerability{
|
||||||
|
{
|
||||||
|
ID: "CVE-2024-1234",
|
||||||
|
Severity: "HIGH",
|
||||||
|
Title: "Test High Severity",
|
||||||
|
Description: "High severity test",
|
||||||
|
References: []string{"https://example.com/cve-2024-1234"},
|
||||||
|
FixedIn: "2.1.0",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
ID: "CVE-2024-5678",
|
||||||
|
Severity: "MODERATE",
|
||||||
|
Title: "Test Moderate Severity",
|
||||||
|
Description: "Moderate severity test",
|
||||||
|
References: []string{"https://example.com/cve-2024-5678"},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
VulnerabilityCount: 2,
|
||||||
|
}
|
||||||
|
err = s.store.SaveScanResult(s.ctx, scanResult)
|
||||||
|
s.NoError(err)
|
||||||
|
|
||||||
|
// Retrieve the scan result
|
||||||
|
retrieved, err := s.store.GetScanResult(s.ctx, "npm", "scan-test", "2.0.0")
|
||||||
|
s.NoError(err)
|
||||||
|
s.NotNil(retrieved)
|
||||||
|
s.Equal("grype", retrieved.Scanner)
|
||||||
|
s.Equal(metadata.ScanStatusVulnerable, retrieved.Status)
|
||||||
|
s.Equal(2, retrieved.VulnerabilityCount)
|
||||||
|
s.Len(retrieved.Vulnerabilities, 2)
|
||||||
|
|
||||||
|
// Verify vulnerability details are retrieved correctly
|
||||||
|
s.Equal("CVE-2024-1234", retrieved.Vulnerabilities[0].ID)
|
||||||
|
s.Equal("HIGH", retrieved.Vulnerabilities[0].Severity)
|
||||||
|
s.Equal("Test High Severity", retrieved.Vulnerabilities[0].Title)
|
||||||
|
s.Equal("2.1.0", retrieved.Vulnerabilities[0].FixedIn)
|
||||||
|
s.Len(retrieved.Vulnerabilities[0].References, 1)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Test_V2_GetScanResult_NotFound tests retrieving a non-existent scan result
|
||||||
|
func (s *GORMStoreV2TestSuite) Test_V2_GetScanResult_NotFound() {
|
||||||
|
_, err := s.store.GetScanResult(s.ctx, "npm", "nonexistent", "1.0.0")
|
||||||
|
s.Error(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Test_V2_SaveCVEBypass tests saving a CVE bypass
|
||||||
|
func (s *GORMStoreV2TestSuite) Test_V2_SaveCVEBypass() {
|
||||||
|
bypass := &metadata.CVEBypass{
|
||||||
|
Type: metadata.BypassTypeCVE,
|
||||||
|
Target: "CVE-2024-0001",
|
||||||
|
Reason: "False positive - not applicable to our use case",
|
||||||
|
CreatedBy: "[email protected]",
|
||||||
|
ExpiresAt: time.Now().Add(30 * 24 * time.Hour), // 30 days
|
||||||
|
NotifyOnExpiry: true,
|
||||||
|
Active: true,
|
||||||
|
}
|
||||||
|
|
||||||
|
err := s.store.SaveCVEBypass(s.ctx, bypass)
|
||||||
|
s.NoError(err)
|
||||||
|
s.NotEmpty(bypass.ID)
|
||||||
|
s.NotZero(bypass.CreatedAt)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Test_V2_SaveCVEBypass_Update tests updating an existing CVE bypass
|
||||||
|
func (s *GORMStoreV2TestSuite) Test_V2_SaveCVEBypass_Update() {
|
||||||
|
// Create initial bypass
|
||||||
|
bypass := &metadata.CVEBypass{
|
||||||
|
Type: metadata.BypassTypeCVE,
|
||||||
|
Target: "CVE-2024-0002",
|
||||||
|
Reason: "Initial reason",
|
||||||
|
CreatedBy: "[email protected]",
|
||||||
|
ExpiresAt: time.Now().Add(30 * 24 * time.Hour),
|
||||||
|
NotifyOnExpiry: false,
|
||||||
|
Active: true,
|
||||||
|
}
|
||||||
|
err := s.store.SaveCVEBypass(s.ctx, bypass)
|
||||||
|
s.NoError(err)
|
||||||
|
s.NotEmpty(bypass.ID)
|
||||||
|
|
||||||
|
// Update the bypass
|
||||||
|
bypass.Reason = "Updated reason"
|
||||||
|
bypass.NotifyOnExpiry = true
|
||||||
|
err = s.store.SaveCVEBypass(s.ctx, bypass)
|
||||||
|
s.NoError(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Test_V2_GetActiveCVEBypasses tests retrieving active CVE bypasses
|
||||||
|
func (s *GORMStoreV2TestSuite) Test_V2_GetActiveCVEBypasses() {
|
||||||
|
// Create active bypass with unique target
|
||||||
|
uniqueTarget := fmt.Sprintf("CVE-2024-TEST-%d", time.Now().UnixNano())
|
||||||
|
activeBypass := &metadata.CVEBypass{
|
||||||
|
Type: metadata.BypassTypeCVE,
|
||||||
|
Target: uniqueTarget,
|
||||||
|
Reason: "Active bypass",
|
||||||
|
CreatedBy: "[email protected]",
|
||||||
|
ExpiresAt: time.Now().Add(30 * 24 * time.Hour),
|
||||||
|
Active: true,
|
||||||
|
}
|
||||||
|
err := s.store.SaveCVEBypass(s.ctx, activeBypass)
|
||||||
|
s.NoError(err)
|
||||||
|
|
||||||
|
// Create expired bypass
|
||||||
|
expiredBypass := &metadata.CVEBypass{
|
||||||
|
Type: metadata.BypassTypeCVE,
|
||||||
|
Target: "CVE-2024-0004",
|
||||||
|
Reason: "Expired bypass",
|
||||||
|
CreatedBy: "[email protected]",
|
||||||
|
ExpiresAt: time.Now().Add(-24 * time.Hour), // Expired yesterday
|
||||||
|
Active: true,
|
||||||
|
}
|
||||||
|
err = s.store.SaveCVEBypass(s.ctx, expiredBypass)
|
||||||
|
s.NoError(err)
|
||||||
|
|
||||||
|
// Create inactive bypass
|
||||||
|
inactiveBypass := &metadata.CVEBypass{
|
||||||
|
Type: metadata.BypassTypeCVE,
|
||||||
|
Target: "CVE-2024-0005",
|
||||||
|
Reason: "Inactive bypass",
|
||||||
|
CreatedBy: "[email protected]",
|
||||||
|
ExpiresAt: time.Now().Add(30 * 24 * time.Hour),
|
||||||
|
Active: false,
|
||||||
|
}
|
||||||
|
err = s.store.SaveCVEBypass(s.ctx, inactiveBypass)
|
||||||
|
s.NoError(err)
|
||||||
|
|
||||||
|
// Retrieve active bypasses
|
||||||
|
bypasses, err := s.store.GetActiveCVEBypasses(s.ctx)
|
||||||
|
s.NoError(err)
|
||||||
|
|
||||||
|
// Should contain our active bypass, but may contain others from parallel tests
|
||||||
|
found := false
|
||||||
|
for _, b := range bypasses {
|
||||||
|
if b.Target == uniqueTarget {
|
||||||
|
found = true
|
||||||
|
break
|
||||||
|
}
|
||||||
|
// All bypasses should be active and non-expired
|
||||||
|
s.True(b.Active)
|
||||||
|
s.True(b.ExpiresAt.After(time.Now()))
|
||||||
|
}
|
||||||
|
s.True(found, "Should find our unique active bypass")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Test_V2_ListCVEBypasses tests listing CVE bypasses with filters
|
||||||
|
func (s *GORMStoreV2TestSuite) Test_V2_ListCVEBypasses() {
|
||||||
|
// Create multiple bypasses with unique targets
|
||||||
|
nano := time.Now().UnixNano()
|
||||||
|
bypasses := []*metadata.CVEBypass{
|
||||||
|
{
|
||||||
|
Type: metadata.BypassTypeCVE,
|
||||||
|
Target: fmt.Sprintf("CVE-2024-LIST-%d-1", nano),
|
||||||
|
Reason: "Test 1",
|
||||||
|
CreatedBy: "[email protected]",
|
||||||
|
ExpiresAt: time.Now().Add(30 * 24 * time.Hour),
|
||||||
|
Active: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Type: metadata.BypassTypePackage,
|
||||||
|
Target: fmt.Sprintf("npm/vulnerable-package@%d", nano),
|
||||||
|
Reason: "Test 2",
|
||||||
|
CreatedBy: "[email protected]",
|
||||||
|
ExpiresAt: time.Now().Add(15 * 24 * time.Hour),
|
||||||
|
Active: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Type: metadata.BypassTypeCVE,
|
||||||
|
Target: fmt.Sprintf("CVE-2024-LIST-%d-2", nano),
|
||||||
|
Reason: "Test 3",
|
||||||
|
CreatedBy: "[email protected]",
|
||||||
|
ExpiresAt: time.Now().Add(-1 * time.Hour), // Expired
|
||||||
|
Active: true,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, b := range bypasses {
|
||||||
|
err := s.store.SaveCVEBypass(s.ctx, b)
|
||||||
|
s.NoError(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// List only CVE type
|
||||||
|
opts := &metadata.BypassListOptions{
|
||||||
|
Type: metadata.BypassTypeCVE,
|
||||||
|
}
|
||||||
|
cveOnly, err := s.store.ListCVEBypasses(s.ctx, opts)
|
||||||
|
s.NoError(err)
|
||||||
|
for _, b := range cveOnly {
|
||||||
|
s.Equal(metadata.BypassTypeCVE, b.Type)
|
||||||
|
}
|
||||||
|
|
||||||
|
// List only non-expired
|
||||||
|
opts = &metadata.BypassListOptions{
|
||||||
|
IncludeExpired: false,
|
||||||
|
}
|
||||||
|
nonExpired, err := s.store.ListCVEBypasses(s.ctx, opts)
|
||||||
|
s.NoError(err)
|
||||||
|
for _, b := range nonExpired {
|
||||||
|
s.True(b.ExpiresAt.After(time.Now()))
|
||||||
|
}
|
||||||
|
|
||||||
|
// Test pagination
|
||||||
|
opts = &metadata.BypassListOptions{
|
||||||
|
Limit: 1,
|
||||||
|
Offset: 0,
|
||||||
|
}
|
||||||
|
page1, err := s.store.ListCVEBypasses(s.ctx, opts)
|
||||||
|
s.NoError(err)
|
||||||
|
s.LessOrEqual(len(page1), 1) // Should be at most 1
|
||||||
|
}
|
||||||
|
|
||||||
|
// Test_V2_DeleteCVEBypass tests deleting a CVE bypass
|
||||||
|
func (s *GORMStoreV2TestSuite) Test_V2_DeleteCVEBypass() {
|
||||||
|
// Create a bypass
|
||||||
|
bypass := &metadata.CVEBypass{
|
||||||
|
Type: metadata.BypassTypeCVE,
|
||||||
|
Target: "CVE-2024-0008",
|
||||||
|
Reason: "To be deleted",
|
||||||
|
CreatedBy: "[email protected]",
|
||||||
|
ExpiresAt: time.Now().Add(30 * 24 * time.Hour),
|
||||||
|
Active: true,
|
||||||
|
}
|
||||||
|
err := s.store.SaveCVEBypass(s.ctx, bypass)
|
||||||
|
s.NoError(err)
|
||||||
|
s.NotEmpty(bypass.ID)
|
||||||
|
|
||||||
|
// Delete the bypass
|
||||||
|
err = s.store.DeleteCVEBypass(s.ctx, bypass.ID)
|
||||||
|
s.NoError(err)
|
||||||
|
|
||||||
|
// Verify it's no longer in active bypasses
|
||||||
|
active, err := s.store.GetActiveCVEBypasses(s.ctx)
|
||||||
|
s.NoError(err)
|
||||||
|
for _, b := range active {
|
||||||
|
s.NotEqual(bypass.ID, b.ID)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Test_V2_DeleteCVEBypass_NotFound tests deleting a non-existent bypass
|
||||||
|
func (s *GORMStoreV2TestSuite) Test_V2_DeleteCVEBypass_NotFound() {
|
||||||
|
err := s.store.DeleteCVEBypass(s.ctx, "99999999")
|
||||||
|
s.Error(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Test_V2_DeleteCVEBypass_InvalidID tests deleting with invalid ID
|
||||||
|
func (s *GORMStoreV2TestSuite) Test_V2_DeleteCVEBypass_InvalidID() {
|
||||||
|
err := s.store.DeleteCVEBypass(s.ctx, "invalid-id")
|
||||||
|
s.Error(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Test_V2_CleanupExpiredBypasses tests cleaning up expired bypasses
|
||||||
|
func (s *GORMStoreV2TestSuite) Test_V2_CleanupExpiredBypasses() {
|
||||||
|
// Create expired bypasses
|
||||||
|
for i := 0; i < 3; i++ {
|
||||||
|
bypass := &metadata.CVEBypass{
|
||||||
|
Type: metadata.BypassTypeCVE,
|
||||||
|
Target: fmt.Sprintf("CVE-2024-00%d", 10+i),
|
||||||
|
Reason: "Expired bypass",
|
||||||
|
CreatedBy: "[email protected]",
|
||||||
|
ExpiresAt: time.Now().Add(-24 * time.Hour), // Expired
|
||||||
|
Active: true,
|
||||||
|
}
|
||||||
|
err := s.store.SaveCVEBypass(s.ctx, bypass)
|
||||||
|
s.NoError(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Create active bypass (should not be deleted)
|
||||||
|
activeBypass := &metadata.CVEBypass{
|
||||||
|
Type: metadata.BypassTypeCVE,
|
||||||
|
Target: "CVE-2024-0999",
|
||||||
|
Reason: "Active bypass",
|
||||||
|
CreatedBy: "[email protected]",
|
||||||
|
ExpiresAt: time.Now().Add(30 * 24 * time.Hour),
|
||||||
|
Active: true,
|
||||||
|
}
|
||||||
|
err := s.store.SaveCVEBypass(s.ctx, activeBypass)
|
||||||
|
s.NoError(err)
|
||||||
|
|
||||||
|
// Cleanup expired bypasses
|
||||||
|
count, err := s.store.CleanupExpiredBypasses(s.ctx)
|
||||||
|
s.NoError(err)
|
||||||
|
s.GreaterOrEqual(count, 3) // At least the 3 we just created
|
||||||
|
|
||||||
|
// Verify active bypass is still there
|
||||||
|
active, err := s.store.GetActiveCVEBypasses(s.ctx)
|
||||||
|
s.NoError(err)
|
||||||
|
found := false
|
||||||
|
for _, b := range active {
|
||||||
|
if b.Target == "CVE-2024-0999" {
|
||||||
|
found = true
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
s.True(found, "Active bypass should still exist")
|
||||||
|
}
|
||||||
@@ -0,0 +1,228 @@
|
|||||||
|
package gormstore
|
||||||
|
|
||||||
|
import (
|
||||||
|
"github.com/go-gormigrate/gormigrate/v2"
|
||||||
|
"gorm.io/gorm"
|
||||||
|
)
|
||||||
|
|
||||||
|
// GetMigrations returns all database migrations for gormigrate
|
||||||
|
func GetMigrations() []*gormigrate.Migration {
|
||||||
|
return []*gormigrate.Migration{
|
||||||
|
{
|
||||||
|
ID: "202601030001",
|
||||||
|
Migrate: func(tx *gorm.DB) error {
|
||||||
|
// Migration: Create V2 schema
|
||||||
|
return migrateToV2(tx)
|
||||||
|
},
|
||||||
|
Rollback: func(tx *gorm.DB) error {
|
||||||
|
// Rollback: Drop V2 schema (careful!)
|
||||||
|
return rollbackFromV2(tx)
|
||||||
|
},
|
||||||
|
},
|
||||||
|
// Future migrations go here
|
||||||
|
// {
|
||||||
|
// ID: "202601040001",
|
||||||
|
// Migrate: func(tx *gorm.DB) error {
|
||||||
|
// // Add new column, index, etc.
|
||||||
|
// return tx.Exec("ALTER TABLE packages ADD COLUMN new_field VARCHAR(255)").Error
|
||||||
|
// },
|
||||||
|
// Rollback: func(tx *gorm.DB) error {
|
||||||
|
// return tx.Exec("ALTER TABLE packages DROP COLUMN new_field").Error
|
||||||
|
// },
|
||||||
|
// },
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// migrateToV2 creates the complete V2 schema
|
||||||
|
func migrateToV2(tx *gorm.DB) error {
|
||||||
|
// Get dialect name for database-specific features
|
||||||
|
dialectName := tx.Dialector.Name()
|
||||||
|
|
||||||
|
// Step 1: Create all tables using GORM AutoMigrate
|
||||||
|
// This handles cross-database compatibility automatically
|
||||||
|
if err := tx.AutoMigrate(GetAllModels()...); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Step 2: Seed default registries
|
||||||
|
registries := []RegistryModel{
|
||||||
|
{Name: "npm", DisplayName: "NPM Registry", UpstreamURL: "https://registry.npmjs.org", Enabled: true, ScanByDefault: true},
|
||||||
|
{Name: "pypi", DisplayName: "PyPI", UpstreamURL: "https://pypi.org", Enabled: true, ScanByDefault: true},
|
||||||
|
{Name: "go", DisplayName: "Go Modules", UpstreamURL: "https://proxy.golang.org", Enabled: true, ScanByDefault: true},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, reg := range registries {
|
||||||
|
// Upsert: create if not exists
|
||||||
|
if err := tx.Where("name = ?", reg.Name).FirstOrCreate(®).Error; err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Step 3: Create database-specific optimizations
|
||||||
|
if dialectName == "postgres" {
|
||||||
|
if err := createPostgreSQLOptimizations(tx); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
} else if dialectName == "mysql" {
|
||||||
|
if err := createMySQLOptimizations(tx); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// createPostgreSQLOptimizations adds PostgreSQL-specific features
|
||||||
|
func createPostgreSQLOptimizations(tx *gorm.DB) error {
|
||||||
|
optimizations := []string{
|
||||||
|
// Create GIN indexes for JSONB columns
|
||||||
|
`CREATE INDEX IF NOT EXISTS idx_package_metadata_keywords_gin
|
||||||
|
ON package_metadata USING GIN(keywords)`,
|
||||||
|
|
||||||
|
`CREATE INDEX IF NOT EXISTS idx_package_metadata_raw_gin
|
||||||
|
ON package_metadata USING GIN(raw_metadata)`,
|
||||||
|
|
||||||
|
`CREATE INDEX IF NOT EXISTS idx_vulnerabilities_references_gin
|
||||||
|
ON vulnerabilities USING GIN(references)`,
|
||||||
|
|
||||||
|
// Create partial indexes (only non-deleted records)
|
||||||
|
`CREATE INDEX IF NOT EXISTS idx_packages_active
|
||||||
|
ON packages(registry_id, name, version) WHERE deleted_at IS NULL`,
|
||||||
|
|
||||||
|
`CREATE INDEX IF NOT EXISTS idx_packages_vulnerable
|
||||||
|
ON packages(vulnerability_count, highest_severity)
|
||||||
|
WHERE vulnerability_count > 0 AND deleted_at IS NULL`,
|
||||||
|
|
||||||
|
// Create view for vulnerable packages
|
||||||
|
`CREATE OR REPLACE VIEW v_vulnerable_packages AS
|
||||||
|
SELECT
|
||||||
|
r.name AS registry,
|
||||||
|
p.name,
|
||||||
|
p.version,
|
||||||
|
p.vulnerability_count,
|
||||||
|
p.highest_severity,
|
||||||
|
p.last_scanned_at
|
||||||
|
FROM packages p
|
||||||
|
JOIN registries r ON p.registry_id = r.id
|
||||||
|
WHERE p.vulnerability_count > 0 AND p.deleted_at IS NULL
|
||||||
|
ORDER BY
|
||||||
|
CASE p.highest_severity
|
||||||
|
WHEN 'critical' THEN 1
|
||||||
|
WHEN 'high' THEN 2
|
||||||
|
WHEN 'medium' THEN 3
|
||||||
|
WHEN 'low' THEN 4
|
||||||
|
ELSE 5
|
||||||
|
END,
|
||||||
|
p.vulnerability_count DESC`,
|
||||||
|
|
||||||
|
// Create function for automatic partition creation
|
||||||
|
`CREATE OR REPLACE FUNCTION create_next_month_partitions()
|
||||||
|
RETURNS void AS $$
|
||||||
|
DECLARE
|
||||||
|
next_month DATE := date_trunc('month', NOW() + INTERVAL '2 months');
|
||||||
|
partition_name TEXT;
|
||||||
|
start_date TEXT;
|
||||||
|
end_date TEXT;
|
||||||
|
BEGIN
|
||||||
|
-- Download events partition
|
||||||
|
partition_name := 'download_events_' || to_char(next_month, 'YYYY_MM');
|
||||||
|
start_date := to_char(next_month, 'YYYY-MM-DD');
|
||||||
|
end_date := to_char(next_month + INTERVAL '1 month', 'YYYY-MM-DD');
|
||||||
|
|
||||||
|
EXECUTE format('CREATE TABLE IF NOT EXISTS %I PARTITION OF download_events FOR VALUES FROM (%L) TO (%L)',
|
||||||
|
partition_name, start_date, end_date);
|
||||||
|
|
||||||
|
-- Audit log partition
|
||||||
|
partition_name := 'audit_log_' || to_char(next_month, 'YYYY_MM');
|
||||||
|
EXECUTE format('CREATE TABLE IF NOT EXISTS %I PARTITION OF audit_log FOR VALUES FROM (%L) TO (%L)',
|
||||||
|
partition_name, start_date, end_date);
|
||||||
|
|
||||||
|
RAISE NOTICE 'Created partitions for %', to_char(next_month, 'YYYY-MM');
|
||||||
|
END;
|
||||||
|
$$ LANGUAGE plpgsql`,
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, sql := range optimizations {
|
||||||
|
if err := tx.Exec(sql).Error; err != nil {
|
||||||
|
// Log warning but don't fail migration
|
||||||
|
// Some optimizations might already exist
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// createMySQLOptimizations adds MySQL-specific features
|
||||||
|
func createMySQLOptimizations(tx *gorm.DB) error {
|
||||||
|
optimizations := []string{
|
||||||
|
// Create view for vulnerable packages
|
||||||
|
`CREATE OR REPLACE VIEW v_vulnerable_packages AS
|
||||||
|
SELECT
|
||||||
|
r.name AS registry,
|
||||||
|
p.name,
|
||||||
|
p.version,
|
||||||
|
p.vulnerability_count,
|
||||||
|
p.highest_severity,
|
||||||
|
p.last_scanned_at
|
||||||
|
FROM packages p
|
||||||
|
JOIN registries r ON p.registry_id = r.id
|
||||||
|
WHERE p.vulnerability_count > 0 AND p.deleted_at IS NULL
|
||||||
|
ORDER BY
|
||||||
|
CASE p.highest_severity
|
||||||
|
WHEN 'critical' THEN 1
|
||||||
|
WHEN 'high' THEN 2
|
||||||
|
WHEN 'medium' THEN 3
|
||||||
|
WHEN 'low' THEN 4
|
||||||
|
ELSE 5
|
||||||
|
END,
|
||||||
|
p.vulnerability_count DESC`,
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, sql := range optimizations {
|
||||||
|
if err := tx.Exec(sql).Error; err != nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// rollbackFromV2 drops all V2 tables (USE WITH CAUTION!)
|
||||||
|
func rollbackFromV2(tx *gorm.DB) error {
|
||||||
|
// Drop in reverse order to respect foreign keys
|
||||||
|
tables := []string{
|
||||||
|
"audit_log",
|
||||||
|
"download_stats_daily",
|
||||||
|
"download_stats_hourly",
|
||||||
|
"download_events",
|
||||||
|
"cve_bypasses",
|
||||||
|
"scan_results",
|
||||||
|
"package_vulnerabilities",
|
||||||
|
"vulnerabilities",
|
||||||
|
"package_metadata",
|
||||||
|
"packages",
|
||||||
|
"registries",
|
||||||
|
}
|
||||||
|
|
||||||
|
// Drop PostgreSQL-specific objects
|
||||||
|
if tx.Dialector.Name() == "postgres" {
|
||||||
|
tx.Exec("DROP VIEW IF EXISTS v_vulnerable_packages")
|
||||||
|
tx.Exec("DROP FUNCTION IF EXISTS create_next_month_partitions()")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Drop MySQL-specific objects
|
||||||
|
if tx.Dialector.Name() == "mysql" {
|
||||||
|
tx.Exec("DROP VIEW IF EXISTS v_vulnerable_packages")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Drop all tables
|
||||||
|
for _, table := range tables {
|
||||||
|
if err := tx.Migrator().DropTable(table); err != nil {
|
||||||
|
// Continue even if table doesn't exist
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,328 @@
|
|||||||
|
package gormstore
|
||||||
|
|
||||||
|
import (
|
||||||
|
"database/sql/driver"
|
||||||
|
"encoding/json"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"gorm.io/gorm"
|
||||||
|
)
|
||||||
|
|
||||||
|
// BaseModel provides common fields for all models with audit trail
|
||||||
|
type BaseModel struct {
|
||||||
|
CreatedAt time.Time `gorm:"not null"`
|
||||||
|
UpdatedAt time.Time `gorm:"not null"`
|
||||||
|
DeletedAt gorm.DeletedAt `gorm:"index"` // Soft delete support (auto-generated index name per table)
|
||||||
|
}
|
||||||
|
|
||||||
|
// RegistryModel represents package registries (normalized)
|
||||||
|
// This eliminates repetition of "npm", "pypi", "go" across millions of rows
|
||||||
|
type RegistryModel struct {
|
||||||
|
ID int32 `gorm:"primaryKey;autoIncrement"`
|
||||||
|
Name string `gorm:"uniqueIndex:idx_registry_name;not null;size:50"` // npm, pypi, go
|
||||||
|
DisplayName string `gorm:"not null;size:100"` // NPM Registry, PyPI, Go Modules
|
||||||
|
UpstreamURL string `gorm:"not null;size:512"` // https://registry.npmjs.org
|
||||||
|
Enabled bool `gorm:"not null;default:true;index:idx_registry_enabled"`
|
||||||
|
ScanByDefault bool `gorm:"not null;default:true"`
|
||||||
|
BaseModel
|
||||||
|
}
|
||||||
|
|
||||||
|
func (RegistryModel) TableName() string {
|
||||||
|
return "registries"
|
||||||
|
}
|
||||||
|
|
||||||
|
// PackageModel represents the core package data (optimized)
|
||||||
|
type PackageModel struct {
|
||||||
|
ID int64 `gorm:"primaryKey;autoIncrement"`
|
||||||
|
RegistryID int32 `gorm:"not null;index:idx_package_registry_name_version,priority:1"` // Foreign key to registries
|
||||||
|
Name string `gorm:"not null;size:255;index:idx_package_name;index:idx_package_registry_name_version,priority:2"`
|
||||||
|
Version string `gorm:"not null;size:100;index:idx_package_registry_name_version,priority:3"`
|
||||||
|
|
||||||
|
// Storage information
|
||||||
|
StorageKey string `gorm:"not null;uniqueIndex:idx_package_storage_key;size:512"`
|
||||||
|
Size int64 `gorm:"not null;index:idx_package_size"` // For storage quota queries
|
||||||
|
ChecksumMD5 string `gorm:"size:32;index:idx_package_md5"`
|
||||||
|
ChecksumSHA256 string `gorm:"size:64;index:idx_package_sha256"`
|
||||||
|
UpstreamURL string `gorm:"size:1024"`
|
||||||
|
|
||||||
|
// Cache management
|
||||||
|
CachedAt time.Time `gorm:"not null;index:idx_package_cached_at"`
|
||||||
|
LastAccessed time.Time `gorm:"not null;index:idx_package_last_accessed"` // For LRU eviction
|
||||||
|
ExpiresAt *time.Time `gorm:"index:idx_package_expires_at"` // For cache invalidation
|
||||||
|
AccessCount int64 `gorm:"not null;default:0;index:idx_package_access_count"` // Total access count (denormalized for performance)
|
||||||
|
|
||||||
|
// Security
|
||||||
|
SecurityScanned bool `gorm:"not null;default:false;index:idx_package_security_scanned"`
|
||||||
|
LastScannedAt *time.Time `gorm:"index:idx_package_last_scanned"`
|
||||||
|
VulnerabilityCount int `gorm:"not null;default:0;index:idx_package_vuln_count"` // Denormalized for fast filtering
|
||||||
|
HighestSeverity string `gorm:"size:20;index:idx_package_severity"` // critical, high, medium, low, none
|
||||||
|
CriticalCount int `gorm:"not null;default:0"` // Count of critical vulnerabilities
|
||||||
|
HighCount int `gorm:"not null;default:0"` // Count of high vulnerabilities
|
||||||
|
ModerateCount int `gorm:"not null;default:0"` // Count of moderate vulnerabilities
|
||||||
|
LowCount int `gorm:"not null;default:0"` // Count of low vulnerabilities
|
||||||
|
|
||||||
|
// Authentication
|
||||||
|
RequiresAuth bool `gorm:"not null;default:false;index:idx_package_requires_auth"`
|
||||||
|
AuthProvider string `gorm:"size:50;index:idx_package_auth_provider"` // github, gitlab, custom
|
||||||
|
|
||||||
|
BaseModel
|
||||||
|
|
||||||
|
// Relationships
|
||||||
|
Registry RegistryModel `gorm:"foreignKey:RegistryID;constraint:OnUpdate:CASCADE,OnDelete:RESTRICT"`
|
||||||
|
Metadata *PackageMetadataModel `gorm:"foreignKey:PackageID;constraint:OnUpdate:CASCADE,OnDelete:CASCADE"`
|
||||||
|
ScanResults []ScanResultModel `gorm:"foreignKey:PackageID;constraint:OnUpdate:CASCADE,OnDelete:CASCADE"`
|
||||||
|
Vulnerabilities []PackageVulnerabilityModel `gorm:"foreignKey:PackageID;constraint:OnUpdate:CASCADE,OnDelete:CASCADE"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func (PackageModel) TableName() string {
|
||||||
|
return "packages"
|
||||||
|
}
|
||||||
|
|
||||||
|
// BeforeCreate hook to set access count
|
||||||
|
func (p *PackageModel) BeforeCreate(tx *gorm.DB) error {
|
||||||
|
if p.AccessCount == 0 {
|
||||||
|
p.AccessCount = 0
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// PackageMetadataModel stores structured package metadata (1:1 with packages)
|
||||||
|
// Separated from main table to reduce row size and improve query performance
|
||||||
|
type PackageMetadataModel struct {
|
||||||
|
PackageID int64 `gorm:"primaryKey;not null"` // 1:1 relationship
|
||||||
|
Author string `gorm:"size:255;index:idx_metadata_author"`
|
||||||
|
License string `gorm:"size:100;index:idx_metadata_license"`
|
||||||
|
Homepage string `gorm:"size:512"`
|
||||||
|
Repository string `gorm:"size:512"`
|
||||||
|
Description string `gorm:"type:text"`
|
||||||
|
Keywords PostgresArray `gorm:"type:text"` // JSONB array for PostgreSQL, JSON for MySQL/SQLite
|
||||||
|
RawMetadata JSONBField `gorm:"type:jsonb"` // Full metadata as JSONB (PostgreSQL) or JSON
|
||||||
|
BaseModel
|
||||||
|
}
|
||||||
|
|
||||||
|
func (PackageMetadataModel) TableName() string {
|
||||||
|
return "package_metadata"
|
||||||
|
}
|
||||||
|
|
||||||
|
// ScanResultModel represents security scan results (optimized)
|
||||||
|
type ScanResultModel struct {
|
||||||
|
ID int64 `gorm:"primaryKey;autoIncrement"`
|
||||||
|
PackageID int64 `gorm:"not null;index:idx_scan_package_scanner,priority:1"` // Foreign key
|
||||||
|
Scanner string `gorm:"not null;size:50;index:idx_scan_scanner;index:idx_scan_package_scanner,priority:2"`
|
||||||
|
ScannedAt time.Time `gorm:"not null;index:idx_scan_scanned_at"`
|
||||||
|
Status string `gorm:"not null;size:20;index:idx_scan_status"` // success, failed, pending
|
||||||
|
VulnCount int `gorm:"not null;default:0;index:idx_scan_vuln_count"`
|
||||||
|
CriticalCount int `gorm:"not null;default:0"`
|
||||||
|
HighCount int `gorm:"not null;default:0"`
|
||||||
|
MediumCount int `gorm:"not null;default:0"`
|
||||||
|
LowCount int `gorm:"not null;default:0"`
|
||||||
|
ScanDuration int `gorm:"not null;default:0"` // milliseconds
|
||||||
|
Details JSONBField `gorm:"type:jsonb"` // Scanner-specific details
|
||||||
|
BaseModel
|
||||||
|
|
||||||
|
Package PackageModel `gorm:"foreignKey:PackageID;constraint:OnUpdate:CASCADE,OnDelete:CASCADE"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func (ScanResultModel) TableName() string {
|
||||||
|
return "scan_results"
|
||||||
|
}
|
||||||
|
|
||||||
|
// VulnerabilityModel represents unique vulnerabilities (normalized)
|
||||||
|
type VulnerabilityModel struct {
|
||||||
|
ID int64 `gorm:"primaryKey;autoIncrement"`
|
||||||
|
CVEID string `gorm:"uniqueIndex:idx_vuln_cve_id;not null;size:50"` // CVE-2021-12345
|
||||||
|
Title string `gorm:"not null;size:512"`
|
||||||
|
Description string `gorm:"type:text"`
|
||||||
|
Severity string `gorm:"not null;size:20;index:idx_vuln_severity"` // critical, high, medium, low
|
||||||
|
CVSS float32 `gorm:"index:idx_vuln_cvss"` // CVSS score for sorting
|
||||||
|
PublishedAt time.Time `gorm:"not null;index:idx_vuln_published"`
|
||||||
|
FixedVersion string `gorm:"size:100"` // First version where it's fixed
|
||||||
|
References PostgresArray `gorm:"type:text"` // URLs to advisories
|
||||||
|
BaseModel
|
||||||
|
}
|
||||||
|
|
||||||
|
func (VulnerabilityModel) TableName() string {
|
||||||
|
return "vulnerabilities"
|
||||||
|
}
|
||||||
|
|
||||||
|
// PackageVulnerabilityModel is a many-to-many relationship between packages and vulnerabilities
|
||||||
|
type PackageVulnerabilityModel struct {
|
||||||
|
ID int64 `gorm:"primaryKey;autoIncrement"`
|
||||||
|
PackageID int64 `gorm:"not null;index:idx_pkg_vuln_package,priority:1;index:idx_pkg_vuln_composite,priority:1"`
|
||||||
|
VulnerabilityID int64 `gorm:"not null;index:idx_pkg_vuln_vuln,priority:1;index:idx_pkg_vuln_composite,priority:2"`
|
||||||
|
Scanner string `gorm:"not null;size:50;index:idx_pkg_vuln_scanner"`
|
||||||
|
DetectedAt time.Time `gorm:"not null;index:idx_pkg_vuln_detected"`
|
||||||
|
Bypassed bool `gorm:"not null;default:false;index:idx_pkg_vuln_bypassed"`
|
||||||
|
BypassID *int64 `gorm:"index:idx_pkg_vuln_bypass_id"` // Reference to bypass if applicable
|
||||||
|
BaseModel
|
||||||
|
|
||||||
|
Package PackageModel `gorm:"foreignKey:PackageID;constraint:OnUpdate:CASCADE,OnDelete:CASCADE"`
|
||||||
|
Vulnerability VulnerabilityModel `gorm:"foreignKey:VulnerabilityID;constraint:OnUpdate:CASCADE,OnDelete:CASCADE"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func (PackageVulnerabilityModel) TableName() string {
|
||||||
|
return "package_vulnerabilities"
|
||||||
|
}
|
||||||
|
|
||||||
|
// CVEBypassModel represents CVE bypass rules (improved)
|
||||||
|
type CVEBypassModel struct {
|
||||||
|
ID int64 `gorm:"primaryKey;autoIncrement"`
|
||||||
|
Type string `gorm:"not null;size:20;index:idx_bypass_type"` // cve, package, registry
|
||||||
|
Target string `gorm:"not null;size:512;index:idx_bypass_target"` // CVE-ID, package name, etc.
|
||||||
|
Reason string `gorm:"not null;type:text"`
|
||||||
|
CreatedBy string `gorm:"not null;size:255;index:idx_bypass_created_by"`
|
||||||
|
ExpiresAt time.Time `gorm:"not null;index:idx_bypass_expires_at"`
|
||||||
|
NotifyOnExpiry bool `gorm:"not null;default:false"`
|
||||||
|
Active bool `gorm:"not null;default:true;index:idx_bypass_active"`
|
||||||
|
UsageCount int64 `gorm:"not null;default:0"` // How many times this bypass has been used
|
||||||
|
LastUsedAt *time.Time `gorm:"index:idx_bypass_last_used"`
|
||||||
|
|
||||||
|
// Scope limiting (optional)
|
||||||
|
RegistryID *int32 `gorm:"index:idx_bypass_registry"` // NULL = all registries
|
||||||
|
PackageID *int64 `gorm:"index:idx_bypass_package"` // NULL = all packages
|
||||||
|
|
||||||
|
BaseModel
|
||||||
|
}
|
||||||
|
|
||||||
|
func (CVEBypassModel) TableName() string {
|
||||||
|
return "cve_bypasses"
|
||||||
|
}
|
||||||
|
|
||||||
|
// DownloadEventModel represents raw download events (partitioned by month)
|
||||||
|
// This table should use PostgreSQL partitioning or time-series DB features
|
||||||
|
type DownloadEventModel struct {
|
||||||
|
ID int64 `gorm:"primaryKey;autoIncrement"`
|
||||||
|
PackageID int64 `gorm:"not null;index:idx_download_package,priority:1"`
|
||||||
|
RegistryID int32 `gorm:"not null;index:idx_download_registry"`
|
||||||
|
DownloadedAt time.Time `gorm:"not null;index:idx_download_time;index:idx_download_package,priority:2"` // Partition key
|
||||||
|
UserAgent string `gorm:"size:512"` // For analytics
|
||||||
|
IPAddress string `gorm:"size:45;index:idx_download_ip"` // IPv6 support
|
||||||
|
Authenticated bool `gorm:"not null;default:false"`
|
||||||
|
Username string `gorm:"size:255;index:idx_download_user"`
|
||||||
|
|
||||||
|
// No BaseModel - this is append-only, no updates/deletes on individual rows
|
||||||
|
// Partitioned tables handle cleanup via DROP PARTITION
|
||||||
|
}
|
||||||
|
|
||||||
|
func (DownloadEventModel) TableName() string {
|
||||||
|
return "download_events"
|
||||||
|
}
|
||||||
|
|
||||||
|
// DownloadStatsHourlyModel represents pre-aggregated hourly statistics (partitioned)
|
||||||
|
type DownloadStatsHourlyModel struct {
|
||||||
|
ID int64 `gorm:"primaryKey;autoIncrement"`
|
||||||
|
RegistryID int32 `gorm:"not null;index:idx_stats_hourly_composite,priority:1"`
|
||||||
|
PackageID *int64 `gorm:"index:idx_stats_hourly_package"` // NULL = all packages in registry
|
||||||
|
TimeBucket time.Time `gorm:"not null;index:idx_stats_hourly_composite,priority:2"` // Truncated to hour
|
||||||
|
DownloadCount int64 `gorm:"not null;default:0"`
|
||||||
|
UniqueIPs int64 `gorm:"not null;default:0"` // Unique downloaders
|
||||||
|
AuthDownloads int64 `gorm:"not null;default:0"` // Authenticated downloads
|
||||||
|
|
||||||
|
BaseModel
|
||||||
|
}
|
||||||
|
|
||||||
|
func (DownloadStatsHourlyModel) TableName() string {
|
||||||
|
return "download_stats_hourly"
|
||||||
|
}
|
||||||
|
|
||||||
|
// DownloadStatsDailyModel represents pre-aggregated daily statistics
|
||||||
|
type DownloadStatsDailyModel struct {
|
||||||
|
ID int64 `gorm:"primaryKey;autoIncrement"`
|
||||||
|
RegistryID int32 `gorm:"not null;index:idx_stats_daily_composite,priority:1"`
|
||||||
|
PackageID *int64 `gorm:"index:idx_stats_daily_package"` // NULL = all packages in registry
|
||||||
|
TimeBucket time.Time `gorm:"not null;index:idx_stats_daily_composite,priority:2"` // Truncated to day
|
||||||
|
DownloadCount int64 `gorm:"not null;default:0"`
|
||||||
|
UniqueIPs int64 `gorm:"not null;default:0"`
|
||||||
|
AuthDownloads int64 `gorm:"not null;default:0"`
|
||||||
|
TopUserAgents JSONBField `gorm:"type:jsonb"` // Top 10 user agents
|
||||||
|
|
||||||
|
BaseModel
|
||||||
|
}
|
||||||
|
|
||||||
|
func (DownloadStatsDailyModel) TableName() string {
|
||||||
|
return "download_stats_daily"
|
||||||
|
}
|
||||||
|
|
||||||
|
// AuditLogModel tracks all important changes (optional, for compliance)
|
||||||
|
type AuditLogModel struct {
|
||||||
|
ID int64 `gorm:"primaryKey;autoIncrement"`
|
||||||
|
EntityType string `gorm:"not null;size:50;index:idx_audit_entity_type"` // package, bypass, registry
|
||||||
|
EntityID int64 `gorm:"not null;index:idx_audit_entity_id"`
|
||||||
|
Action string `gorm:"not null;size:20;index:idx_audit_action"` // create, update, delete
|
||||||
|
Username string `gorm:"not null;size:255;index:idx_audit_username"`
|
||||||
|
Timestamp time.Time `gorm:"not null;index:idx_audit_timestamp"`
|
||||||
|
Changes JSONBField `gorm:"type:jsonb"` // Before/after values
|
||||||
|
IPAddress string `gorm:"size:45"`
|
||||||
|
UserAgent string `gorm:"size:512"`
|
||||||
|
|
||||||
|
// No BaseModel - append-only audit log
|
||||||
|
}
|
||||||
|
|
||||||
|
func (AuditLogModel) TableName() string {
|
||||||
|
return "audit_log"
|
||||||
|
}
|
||||||
|
|
||||||
|
// JSONBField is a custom type for JSONB (PostgreSQL) / JSON (MySQL/SQLite)
|
||||||
|
type JSONBField map[string]interface{}
|
||||||
|
|
||||||
|
func (j JSONBField) Value() (driver.Value, error) {
|
||||||
|
if j == nil {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
return json.Marshal(j)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (j *JSONBField) Scan(value interface{}) error {
|
||||||
|
if value == nil {
|
||||||
|
*j = nil
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
bytes, ok := value.([]byte)
|
||||||
|
if !ok {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
return json.Unmarshal(bytes, j)
|
||||||
|
}
|
||||||
|
|
||||||
|
// PostgresArray is a custom type for PostgreSQL arrays stored as JSON
|
||||||
|
type PostgresArray []string
|
||||||
|
|
||||||
|
func (a PostgresArray) Value() (driver.Value, error) {
|
||||||
|
if a == nil {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
return json.Marshal(a)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (a *PostgresArray) Scan(value interface{}) error {
|
||||||
|
if value == nil {
|
||||||
|
*a = nil
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
bytes, ok := value.([]byte)
|
||||||
|
if !ok {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
return json.Unmarshal(bytes, a)
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetAllModels returns all models for GORM auto-migration
|
||||||
|
func GetAllModels() []interface{} {
|
||||||
|
return []interface{}{
|
||||||
|
&RegistryModel{},
|
||||||
|
&PackageModel{},
|
||||||
|
&PackageMetadataModel{},
|
||||||
|
&ScanResultModel{},
|
||||||
|
&VulnerabilityModel{},
|
||||||
|
&PackageVulnerabilityModel{},
|
||||||
|
&CVEBypassModel{},
|
||||||
|
&DownloadEventModel{},
|
||||||
|
&DownloadStatsHourlyModel{},
|
||||||
|
&DownloadStatsDailyModel{},
|
||||||
|
&AuditLogModel{},
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,380 @@
|
|||||||
|
package gormstore
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/rs/zerolog/log"
|
||||||
|
"gorm.io/gorm"
|
||||||
|
)
|
||||||
|
|
||||||
|
// PartitionManager handles automatic partition creation and cleanup for PostgreSQL
|
||||||
|
type PartitionManager struct {
|
||||||
|
db *gorm.DB
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewPartitionManager creates a new partition manager
|
||||||
|
func NewPartitionManager(db *gorm.DB) *PartitionManager {
|
||||||
|
return &PartitionManager{db: db}
|
||||||
|
}
|
||||||
|
|
||||||
|
// EnsurePartitions ensures required partitions exist for current and future months
|
||||||
|
func (pm *PartitionManager) EnsurePartitions() error {
|
||||||
|
// Check if we're using PostgreSQL
|
||||||
|
if pm.db.Dialector.Name() != "postgres" {
|
||||||
|
log.Debug().Msg("Partitioning only supported on PostgreSQL, skipping")
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
log.Info().Msg("Ensuring partitions exist")
|
||||||
|
|
||||||
|
// Create partitions for download_events
|
||||||
|
if err := pm.ensureDownloadEventPartitions(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Create partitions for audit_log
|
||||||
|
if err := pm.ensureAuditLogPartitions(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Set up automatic partition creation
|
||||||
|
if err := pm.createPartitionFunction(); err != nil {
|
||||||
|
log.Warn().Err(err).Msg("Failed to create partition function (may already exist)")
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// ensureDownloadEventPartitions creates download_events partitions
|
||||||
|
func (pm *PartitionManager) ensureDownloadEventPartitions() error {
|
||||||
|
// Check if table is already partitioned
|
||||||
|
var isPartitioned bool
|
||||||
|
err := pm.db.Raw(`
|
||||||
|
SELECT EXISTS (
|
||||||
|
SELECT 1 FROM pg_partitioned_table
|
||||||
|
WHERE partrelid = 'download_events'::regclass
|
||||||
|
)
|
||||||
|
`).Scan(&isPartitioned).Error
|
||||||
|
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
if !isPartitioned {
|
||||||
|
log.Info().Msg("Converting download_events to partitioned table")
|
||||||
|
|
||||||
|
// Rename existing table
|
||||||
|
if err := pm.db.Exec("ALTER TABLE IF EXISTS download_events RENAME TO download_events_old").Error; err != nil {
|
||||||
|
log.Warn().Err(err).Msg("Could not rename old table (may not exist)")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Create partitioned table
|
||||||
|
createTableSQL := `
|
||||||
|
CREATE TABLE IF NOT EXISTS download_events (
|
||||||
|
id BIGSERIAL,
|
||||||
|
package_id BIGINT NOT NULL,
|
||||||
|
registry_id INTEGER NOT NULL,
|
||||||
|
downloaded_at TIMESTAMP NOT NULL,
|
||||||
|
user_agent VARCHAR(512),
|
||||||
|
ip_address VARCHAR(45),
|
||||||
|
authenticated BOOLEAN NOT NULL DEFAULT FALSE,
|
||||||
|
username VARCHAR(255)
|
||||||
|
) PARTITION BY RANGE (downloaded_at)
|
||||||
|
`
|
||||||
|
|
||||||
|
if err := pm.db.Exec(createTableSQL).Error; err != nil {
|
||||||
|
return fmt.Errorf("failed to create partitioned table: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
log.Info().Msg("Created partitioned download_events table")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Create partitions for past 3 months, current month, and next 3 months
|
||||||
|
now := time.Now()
|
||||||
|
for i := -3; i <= 3; i++ {
|
||||||
|
month := now.AddDate(0, i, 0)
|
||||||
|
if err := pm.createDownloadEventPartition(month); err != nil {
|
||||||
|
log.Error().Err(err).Time("month", month).Msg("Failed to create partition")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// createDownloadEventPartition creates a partition for a specific month
|
||||||
|
func (pm *PartitionManager) createDownloadEventPartition(month time.Time) error {
|
||||||
|
// Truncate to start of month
|
||||||
|
startOfMonth := time.Date(month.Year(), month.Month(), 1, 0, 0, 0, 0, time.UTC)
|
||||||
|
endOfMonth := startOfMonth.AddDate(0, 1, 0)
|
||||||
|
|
||||||
|
partitionName := fmt.Sprintf("download_events_%d_%02d", month.Year(), month.Month())
|
||||||
|
|
||||||
|
// Check if partition already exists
|
||||||
|
var exists bool
|
||||||
|
err := pm.db.Raw("SELECT EXISTS (SELECT 1 FROM pg_tables WHERE tablename = ?)", partitionName).Scan(&exists).Error
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
if exists {
|
||||||
|
log.Debug().Str("partition", partitionName).Msg("Partition already exists")
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Create partition
|
||||||
|
createPartitionSQL := fmt.Sprintf(`
|
||||||
|
CREATE TABLE %s PARTITION OF download_events
|
||||||
|
FOR VALUES FROM ('%s') TO ('%s')
|
||||||
|
`, partitionName, startOfMonth.Format("2006-01-02"), endOfMonth.Format("2006-01-02"))
|
||||||
|
|
||||||
|
if err := pm.db.Exec(createPartitionSQL).Error; err != nil {
|
||||||
|
return fmt.Errorf("failed to create partition %s: %w", partitionName, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Create indexes on partition
|
||||||
|
indexSQL := []string{
|
||||||
|
fmt.Sprintf("CREATE INDEX IF NOT EXISTS %s_package_idx ON %s(package_id, downloaded_at)", partitionName, partitionName),
|
||||||
|
fmt.Sprintf("CREATE INDEX IF NOT EXISTS %s_registry_idx ON %s(registry_id)", partitionName, partitionName),
|
||||||
|
fmt.Sprintf("CREATE INDEX IF NOT EXISTS %s_time_idx ON %s(downloaded_at)", partitionName, partitionName),
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, sql := range indexSQL {
|
||||||
|
if err := pm.db.Exec(sql).Error; err != nil {
|
||||||
|
log.Warn().Err(err).Str("sql", sql).Msg("Failed to create index")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
log.Info().Str("partition", partitionName).Msg("Created partition")
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// ensureAuditLogPartitions creates audit_log partitions
|
||||||
|
func (pm *PartitionManager) ensureAuditLogPartitions() error {
|
||||||
|
// Check if table exists
|
||||||
|
var exists bool
|
||||||
|
err := pm.db.Raw("SELECT EXISTS (SELECT 1 FROM pg_tables WHERE tablename = 'audit_log')").Scan(&exists).Error
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
if !exists {
|
||||||
|
// Create partitioned table
|
||||||
|
createTableSQL := `
|
||||||
|
CREATE TABLE IF NOT EXISTS audit_log (
|
||||||
|
id BIGSERIAL,
|
||||||
|
entity_type VARCHAR(50) NOT NULL,
|
||||||
|
entity_id BIGINT NOT NULL,
|
||||||
|
action VARCHAR(20) NOT NULL,
|
||||||
|
username VARCHAR(255) NOT NULL,
|
||||||
|
timestamp TIMESTAMP NOT NULL,
|
||||||
|
changes JSONB,
|
||||||
|
ip_address VARCHAR(45),
|
||||||
|
user_agent VARCHAR(512)
|
||||||
|
) PARTITION BY RANGE (timestamp)
|
||||||
|
`
|
||||||
|
|
||||||
|
if err := pm.db.Exec(createTableSQL).Error; err != nil {
|
||||||
|
return fmt.Errorf("failed to create audit_log table: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
log.Info().Msg("Created partitioned audit_log table")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Create partitions for past month, current month, and next 2 months
|
||||||
|
now := time.Now()
|
||||||
|
for i := -1; i <= 2; i++ {
|
||||||
|
month := now.AddDate(0, i, 0)
|
||||||
|
if err := pm.createAuditLogPartition(month); err != nil {
|
||||||
|
log.Error().Err(err).Time("month", month).Msg("Failed to create audit partition")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// createAuditLogPartition creates a partition for a specific month
|
||||||
|
func (pm *PartitionManager) createAuditLogPartition(month time.Time) error {
|
||||||
|
startOfMonth := time.Date(month.Year(), month.Month(), 1, 0, 0, 0, 0, time.UTC)
|
||||||
|
endOfMonth := startOfMonth.AddDate(0, 1, 0)
|
||||||
|
|
||||||
|
partitionName := fmt.Sprintf("audit_log_%d_%02d", month.Year(), month.Month())
|
||||||
|
|
||||||
|
// Check if partition already exists
|
||||||
|
var exists bool
|
||||||
|
err := pm.db.Raw("SELECT EXISTS (SELECT 1 FROM pg_tables WHERE tablename = ?)", partitionName).Scan(&exists).Error
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
if exists {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Create partition
|
||||||
|
createPartitionSQL := fmt.Sprintf(`
|
||||||
|
CREATE TABLE %s PARTITION OF audit_log
|
||||||
|
FOR VALUES FROM ('%s') TO ('%s')
|
||||||
|
`, partitionName, startOfMonth.Format("2006-01-02"), endOfMonth.Format("2006-01-02"))
|
||||||
|
|
||||||
|
if err := pm.db.Exec(createPartitionSQL).Error; err != nil {
|
||||||
|
return fmt.Errorf("failed to create partition %s: %w", partitionName, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Create indexes
|
||||||
|
indexSQL := []string{
|
||||||
|
fmt.Sprintf("CREATE INDEX IF NOT EXISTS %s_entity_idx ON %s(entity_type, entity_id)", partitionName, partitionName),
|
||||||
|
fmt.Sprintf("CREATE INDEX IF NOT EXISTS %s_user_idx ON %s(username)", partitionName, partitionName),
|
||||||
|
fmt.Sprintf("CREATE INDEX IF NOT EXISTS %s_time_idx ON %s(timestamp)", partitionName, partitionName),
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, sql := range indexSQL {
|
||||||
|
if err := pm.db.Exec(sql).Error; err != nil {
|
||||||
|
log.Warn().Err(err).Msg("Failed to create audit index")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
log.Info().Str("partition", partitionName).Msg("Created audit partition")
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// createPartitionFunction creates a PostgreSQL function for automatic partition creation
|
||||||
|
func (pm *PartitionManager) createPartitionFunction() error {
|
||||||
|
functionSQL := `
|
||||||
|
CREATE OR REPLACE FUNCTION create_next_month_partitions()
|
||||||
|
RETURNS void AS $$
|
||||||
|
DECLARE
|
||||||
|
next_month DATE := date_trunc('month', NOW() + INTERVAL '2 months');
|
||||||
|
partition_name TEXT;
|
||||||
|
start_date TEXT;
|
||||||
|
end_date TEXT;
|
||||||
|
BEGIN
|
||||||
|
-- Create download_events partition
|
||||||
|
partition_name := 'download_events_' || to_char(next_month, 'YYYY_MM');
|
||||||
|
start_date := to_char(next_month, 'YYYY-MM-DD');
|
||||||
|
end_date := to_char(next_month + INTERVAL '1 month', 'YYYY-MM-DD');
|
||||||
|
|
||||||
|
EXECUTE format('CREATE TABLE IF NOT EXISTS %I PARTITION OF download_events FOR VALUES FROM (%L) TO (%L)',
|
||||||
|
partition_name, start_date, end_date);
|
||||||
|
|
||||||
|
EXECUTE format('CREATE INDEX IF NOT EXISTS %I ON %I(package_id, downloaded_at)',
|
||||||
|
partition_name || '_package_idx', partition_name);
|
||||||
|
EXECUTE format('CREATE INDEX IF NOT EXISTS %I ON %I(registry_id)',
|
||||||
|
partition_name || '_registry_idx', partition_name);
|
||||||
|
|
||||||
|
-- Create audit_log partition
|
||||||
|
partition_name := 'audit_log_' || to_char(next_month, 'YYYY_MM');
|
||||||
|
|
||||||
|
EXECUTE format('CREATE TABLE IF NOT EXISTS %I PARTITION OF audit_log FOR VALUES FROM (%L) TO (%L)',
|
||||||
|
partition_name, start_date, end_date);
|
||||||
|
|
||||||
|
EXECUTE format('CREATE INDEX IF NOT EXISTS %I ON %I(entity_type, entity_id)',
|
||||||
|
partition_name || '_entity_idx', partition_name);
|
||||||
|
|
||||||
|
RAISE NOTICE 'Created partitions for %', to_char(next_month, 'YYYY-MM');
|
||||||
|
END;
|
||||||
|
$$ LANGUAGE plpgsql;
|
||||||
|
`
|
||||||
|
|
||||||
|
if err := pm.db.Exec(functionSQL).Error; err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
log.Info().Msg("Created partition management function")
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// CleanupOldPartitions drops partitions older than the retention period
|
||||||
|
func (pm *PartitionManager) CleanupOldPartitions(retentionMonths int) error {
|
||||||
|
if pm.db.Dialector.Name() != "postgres" {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
cutoffDate := time.Now().AddDate(0, -retentionMonths, 0)
|
||||||
|
cutoffPartition := fmt.Sprintf("%d_%02d", cutoffDate.Year(), cutoffDate.Month())
|
||||||
|
|
||||||
|
log.Info().
|
||||||
|
Str("cutoff", cutoffPartition).
|
||||||
|
Int("retention_months", retentionMonths).
|
||||||
|
Msg("Cleaning up old partitions")
|
||||||
|
|
||||||
|
// Find and drop old download_events partitions
|
||||||
|
var downloadPartitions []string
|
||||||
|
err := pm.db.Raw(`
|
||||||
|
SELECT tablename FROM pg_tables
|
||||||
|
WHERE tablename LIKE 'download_events_%'
|
||||||
|
AND tablename < 'download_events_' || ?
|
||||||
|
`, cutoffPartition).Scan(&downloadPartitions).Error
|
||||||
|
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, partition := range downloadPartitions {
|
||||||
|
log.Info().Str("partition", partition).Msg("Dropping old partition")
|
||||||
|
if err := pm.db.Exec(fmt.Sprintf("DROP TABLE IF EXISTS %s", partition)).Error; err != nil {
|
||||||
|
log.Error().Err(err).Str("partition", partition).Msg("Failed to drop partition")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Find and drop old audit_log partitions
|
||||||
|
var auditPartitions []string
|
||||||
|
err = pm.db.Raw(`
|
||||||
|
SELECT tablename FROM pg_tables
|
||||||
|
WHERE tablename LIKE 'audit_log_%'
|
||||||
|
AND tablename < 'audit_log_' || ?
|
||||||
|
`, cutoffPartition).Scan(&auditPartitions).Error
|
||||||
|
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, partition := range auditPartitions {
|
||||||
|
log.Info().Str("partition", partition).Msg("Dropping old audit partition")
|
||||||
|
if err := pm.db.Exec(fmt.Sprintf("DROP TABLE IF EXISTS %s", partition)).Error; err != nil {
|
||||||
|
log.Error().Err(err).Str("partition", partition).Msg("Failed to drop audit partition")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetPartitionInfo returns information about current partitions
|
||||||
|
func (pm *PartitionManager) GetPartitionInfo() (map[string]interface{}, error) {
|
||||||
|
if pm.db.Dialector.Name() != "postgres" {
|
||||||
|
return map[string]interface{}{"status": "not_applicable"}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
info := make(map[string]interface{})
|
||||||
|
|
||||||
|
// Count download_events partitions
|
||||||
|
var downloadCount int64
|
||||||
|
pm.db.Raw("SELECT COUNT(*) FROM pg_tables WHERE tablename LIKE 'download_events_%'").Scan(&downloadCount)
|
||||||
|
info["download_events_partitions"] = downloadCount
|
||||||
|
|
||||||
|
// Count audit_log partitions
|
||||||
|
var auditCount int64
|
||||||
|
pm.db.Raw("SELECT COUNT(*) FROM pg_tables WHERE tablename LIKE 'audit_log_%'").Scan(&auditCount)
|
||||||
|
info["audit_log_partitions"] = auditCount
|
||||||
|
|
||||||
|
// Get partition sizes
|
||||||
|
type PartitionSize struct {
|
||||||
|
TableName string
|
||||||
|
SizeMB float64
|
||||||
|
}
|
||||||
|
|
||||||
|
var partitionSizes []PartitionSize
|
||||||
|
pm.db.Raw(`
|
||||||
|
SELECT
|
||||||
|
tablename AS table_name,
|
||||||
|
pg_total_relation_size(tablename::regclass) / 1024.0 / 1024.0 AS size_mb
|
||||||
|
FROM pg_tables
|
||||||
|
WHERE tablename LIKE 'download_events_%' OR tablename LIKE 'audit_log_%'
|
||||||
|
ORDER BY size_mb DESC
|
||||||
|
LIMIT 10
|
||||||
|
`).Scan(&partitionSizes)
|
||||||
|
|
||||||
|
info["largest_partitions"] = partitionSizes
|
||||||
|
|
||||||
|
return info, nil
|
||||||
|
}
|
||||||
+43
-38
@@ -68,37 +68,37 @@ type MetadataStore interface {
|
|||||||
|
|
||||||
// Package represents package metadata
|
// Package represents package metadata
|
||||||
type Package struct {
|
type Package struct {
|
||||||
|
CachedAt time.Time `json:"cached_at"`
|
||||||
|
LastAccessed time.Time `json:"last_accessed"`
|
||||||
|
Metadata map[string]string `json:"metadata"`
|
||||||
|
ExpiresAt *time.Time `json:"expires_at"`
|
||||||
|
UpstreamURL string `json:"upstream_url"`
|
||||||
|
ChecksumMD5 string `json:"checksum_md5"`
|
||||||
|
ChecksumSHA256 string `json:"checksum_sha256"`
|
||||||
ID string `json:"id"`
|
ID string `json:"id"`
|
||||||
Registry string `json:"registry"` // npm, pypi, go
|
StorageKey string `json:"storage_key"`
|
||||||
Name string `json:"name"` // Package name
|
Version string `json:"version"`
|
||||||
Version string `json:"version"` // Package version
|
Name string `json:"name"`
|
||||||
StorageKey string `json:"storage_key"` // Key in storage backend
|
Registry string `json:"registry"`
|
||||||
Size int64 `json:"size"` // Package size in bytes
|
AuthProvider string `json:"auth_provider"`
|
||||||
ChecksumMD5 string `json:"checksum_md5"` // MD5 checksum
|
Size int64 `json:"size"`
|
||||||
ChecksumSHA256 string `json:"checksum_sha256"` // SHA256 checksum
|
DownloadCount int64 `json:"download_count"`
|
||||||
UpstreamURL string `json:"upstream_url"` // Original upstream URL
|
SecurityScanned bool `json:"security_scanned"`
|
||||||
CachedAt time.Time `json:"cached_at"` // When cached
|
RequiresAuth bool `json:"requires_auth"`
|
||||||
LastAccessed time.Time `json:"last_accessed"` // Last access time
|
|
||||||
ExpiresAt *time.Time `json:"expires_at"` // Expiration time (nil = never)
|
|
||||||
DownloadCount int64 `json:"download_count"` // Download counter
|
|
||||||
Metadata map[string]string `json:"metadata"` // Additional metadata
|
|
||||||
SecurityScanned bool `json:"security_scanned"` // Has been scanned
|
|
||||||
RequiresAuth bool `json:"requires_auth"` // Package requires authentication
|
|
||||||
AuthProvider string `json:"auth_provider"` // Auth provider (github.com, npm.pkg.github.com, etc.)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// ScanResult represents a security scan result
|
// ScanResult represents a security scan result
|
||||||
type ScanResult struct {
|
type ScanResult struct {
|
||||||
|
ScannedAt time.Time `json:"scanned_at"`
|
||||||
|
Details map[string]interface{} `json:"details"`
|
||||||
ID string `json:"id"`
|
ID string `json:"id"`
|
||||||
Registry string `json:"registry"`
|
Registry string `json:"registry"`
|
||||||
PackageName string `json:"package_name"`
|
PackageName string `json:"package_name"`
|
||||||
PackageVersion string `json:"package_version"`
|
PackageVersion string `json:"package_version"`
|
||||||
Scanner string `json:"scanner"` // trivy, osv, etc.
|
Scanner string `json:"scanner"`
|
||||||
ScannedAt time.Time `json:"scanned_at"`
|
Status ScanStatus `json:"status"`
|
||||||
Status ScanStatus `json:"status"` // clean, vulnerable, error
|
|
||||||
VulnerabilityCount int `json:"vulnerability_count"`
|
|
||||||
Vulnerabilities []Vulnerability `json:"vulnerabilities"`
|
Vulnerabilities []Vulnerability `json:"vulnerabilities"`
|
||||||
Details map[string]interface{} `json:"details"` // Scanner-specific details
|
VulnerabilityCount int `json:"vulnerability_count"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// Vulnerability represents a security vulnerability
|
// Vulnerability represents a security vulnerability
|
||||||
@@ -143,13 +143,18 @@ const (
|
|||||||
|
|
||||||
// Stats represents metadata statistics
|
// Stats represents metadata statistics
|
||||||
type Stats struct {
|
type Stats struct {
|
||||||
Registry string `json:"registry"`
|
LastUpdated time.Time `json:"last_updated"`
|
||||||
TotalPackages int64 `json:"total_packages"`
|
Registry string `json:"registry"`
|
||||||
TotalSize int64 `json:"total_size"`
|
TotalPackages int64 `json:"total_packages"`
|
||||||
TotalDownloads int64 `json:"total_downloads"`
|
TotalSize int64 `json:"total_size"`
|
||||||
ScannedPackages int64 `json:"scanned_packages"`
|
TotalDownloads int64 `json:"total_downloads"`
|
||||||
VulnerablePackages int64 `json:"vulnerable_packages"`
|
ScannedPackages int64 `json:"scanned_packages"`
|
||||||
LastUpdated time.Time `json:"last_updated"`
|
VulnerablePackages int64 `json:"vulnerable_packages"`
|
||||||
|
BlockedPackages int64 `json:"blocked_packages"`
|
||||||
|
CriticalVulnerabilities int64 `json:"critical_vulnerabilities"`
|
||||||
|
HighVulnerabilities int64 `json:"high_vulnerabilities"`
|
||||||
|
ModerateVulnerabilities int64 `json:"moderate_vulnerabilities"`
|
||||||
|
LowVulnerabilities int64 `json:"low_vulnerabilities"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// TimeSeriesDataPoint represents a single data point in time-series
|
// TimeSeriesDataPoint represents a single data point in time-series
|
||||||
@@ -198,14 +203,14 @@ type BypassListOptions struct {
|
|||||||
|
|
||||||
// ListOptions contains options for listing packages
|
// ListOptions contains options for listing packages
|
||||||
type ListOptions struct {
|
type ListOptions struct {
|
||||||
Registry string // Filter by registry
|
SinceDate time.Time
|
||||||
NamePrefix string // Filter by name prefix
|
Registry string
|
||||||
MinSize int64 // Minimum package size
|
NamePrefix string
|
||||||
MaxSize int64 // Maximum package size
|
SortBy string
|
||||||
ScannedOnly bool // Only scanned packages
|
MinSize int64
|
||||||
SinceDate time.Time // Packages cached since date
|
MaxSize int64
|
||||||
Limit int // Max results
|
Limit int
|
||||||
Offset int // Pagination offset
|
Offset int
|
||||||
SortBy string // Sort field (name, size, cached_at, download_count)
|
ScannedOnly bool
|
||||||
SortDesc bool // Sort descending
|
SortDesc bool
|
||||||
}
|
}
|
||||||
|
|||||||
File diff suppressed because it is too large
Load Diff
@@ -137,21 +137,11 @@ func RecordCacheMiss(handler string) {
|
|||||||
CacheRequests.WithLabelValues("miss", handler).Inc()
|
CacheRequests.WithLabelValues("miss", handler).Inc()
|
||||||
}
|
}
|
||||||
|
|
||||||
// RecordCacheError records a cache error
|
|
||||||
func RecordCacheError(handler string) {
|
|
||||||
CacheRequests.WithLabelValues("error", handler).Inc()
|
|
||||||
}
|
|
||||||
|
|
||||||
// UpdateCacheSize updates the cache size metric
|
// UpdateCacheSize updates the cache size metric
|
||||||
func UpdateCacheSize(backend string, bytes int64) {
|
func UpdateCacheSize(backend string, bytes int64) {
|
||||||
CacheSizeBytes.WithLabelValues(backend).Set(float64(bytes))
|
CacheSizeBytes.WithLabelValues(backend).Set(float64(bytes))
|
||||||
}
|
}
|
||||||
|
|
||||||
// UpdateCacheItems updates the cache items metric
|
|
||||||
func UpdateCacheItems(handler string, count int64) {
|
|
||||||
CacheItemsTotal.WithLabelValues(handler).Set(float64(count))
|
|
||||||
}
|
|
||||||
|
|
||||||
// RecordCacheEviction records a cache eviction
|
// RecordCacheEviction records a cache eviction
|
||||||
func RecordCacheEviction(reason string) {
|
func RecordCacheEviction(reason string) {
|
||||||
CacheEvictions.WithLabelValues(reason).Inc()
|
CacheEvictions.WithLabelValues(reason).Inc()
|
||||||
@@ -162,26 +152,11 @@ func RecordStorageOperation(backend, operation, status string) {
|
|||||||
StorageOperations.WithLabelValues(backend, operation, status).Inc()
|
StorageOperations.WithLabelValues(backend, operation, status).Inc()
|
||||||
}
|
}
|
||||||
|
|
||||||
// UpdateStorageQuota updates the storage quota metric
|
|
||||||
func UpdateStorageQuota(project string, bytes int64) {
|
|
||||||
StorageQuotaBytes.WithLabelValues(project).Set(float64(bytes))
|
|
||||||
}
|
|
||||||
|
|
||||||
// RecordUpstreamRequest records an upstream request
|
// RecordUpstreamRequest records an upstream request
|
||||||
func RecordUpstreamRequest(registry, status string) {
|
func RecordUpstreamRequest(registry, status string) {
|
||||||
UpstreamRequests.WithLabelValues(registry, status).Inc()
|
UpstreamRequests.WithLabelValues(registry, status).Inc()
|
||||||
}
|
}
|
||||||
|
|
||||||
// RecordSecurityScan records a security scan
|
|
||||||
func RecordSecurityScan(scanner, result string) {
|
|
||||||
SecurityScans.WithLabelValues(scanner, result).Inc()
|
|
||||||
}
|
|
||||||
|
|
||||||
// RecordVulnerability records a vulnerability finding
|
|
||||||
func RecordVulnerability(severity string) {
|
|
||||||
VulnerabilitiesFound.WithLabelValues(severity).Inc()
|
|
||||||
}
|
|
||||||
|
|
||||||
// UpdateCircuitBreakerState updates the circuit breaker state
|
// UpdateCircuitBreakerState updates the circuit breaker state
|
||||||
func UpdateCircuitBreakerState(name string, state int) {
|
func UpdateCircuitBreakerState(name string, state int) {
|
||||||
CircuitBreakerState.WithLabelValues(name).Set(float64(state))
|
CircuitBreakerState.WithLabelValues(name).Set(float64(state))
|
||||||
|
|||||||
@@ -24,23 +24,23 @@ type Client struct {
|
|||||||
|
|
||||||
// Config holds client configuration
|
// Config holds client configuration
|
||||||
type Config struct {
|
type Config struct {
|
||||||
Timeout time.Duration // Request timeout
|
|
||||||
MaxRetries int // Max retry attempts
|
|
||||||
RetryDelay time.Duration // Initial retry delay
|
|
||||||
RateLimit float64 // Requests per second (0 = unlimited)
|
|
||||||
RateBurst int // Rate limiter burst
|
|
||||||
CircuitBreaker CircuitBreakerConfig
|
|
||||||
UserAgent string
|
UserAgent string
|
||||||
|
CircuitBreaker CircuitBreakerConfig
|
||||||
|
Timeout time.Duration
|
||||||
|
MaxRetries int
|
||||||
|
RetryDelay time.Duration
|
||||||
|
RateLimit float64
|
||||||
|
RateBurst int
|
||||||
MaxConnsPerHost int
|
MaxConnsPerHost int
|
||||||
}
|
}
|
||||||
|
|
||||||
// RetryConfig holds retry configuration
|
// RetryConfig holds retry configuration
|
||||||
type RetryConfig struct {
|
type RetryConfig struct {
|
||||||
|
FixedDelays []time.Duration
|
||||||
MaxAttempts int
|
MaxAttempts int
|
||||||
InitialDelay time.Duration
|
InitialDelay time.Duration
|
||||||
MaxDelay time.Duration
|
MaxDelay time.Duration
|
||||||
Multiplier float64
|
Multiplier float64
|
||||||
FixedDelays []time.Duration // If set, use these delays instead of exponential backoff
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// CircuitBreakerConfig holds circuit breaker configuration
|
// CircuitBreakerConfig holds circuit breaker configuration
|
||||||
@@ -63,11 +63,11 @@ const (
|
|||||||
|
|
||||||
// CircuitBreaker implements the circuit breaker pattern
|
// CircuitBreaker implements the circuit breaker pattern
|
||||||
type CircuitBreaker struct {
|
type CircuitBreaker struct {
|
||||||
|
lastFailureTime time.Time
|
||||||
config CircuitBreakerConfig
|
config CircuitBreakerConfig
|
||||||
state CircuitBreakerState
|
state CircuitBreakerState
|
||||||
failures int
|
failures int
|
||||||
successes int
|
successes int
|
||||||
lastFailureTime time.Time
|
|
||||||
halfOpenCalls int
|
halfOpenCalls int
|
||||||
mu sync.RWMutex
|
mu sync.RWMutex
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -19,14 +19,14 @@ import (
|
|||||||
// TestClientGet tests the HTTP client Get method with various scenarios
|
// TestClientGet tests the HTTP client Get method with various scenarios
|
||||||
func TestClientGet(t *testing.T) {
|
func TestClientGet(t *testing.T) {
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
name string
|
|
||||||
serverBehavior func(*testing.T) *httptest.Server
|
serverBehavior func(*testing.T) *httptest.Server
|
||||||
config network.Config
|
|
||||||
headers map[string]string
|
headers map[string]string
|
||||||
wantErr bool
|
|
||||||
errContains string
|
|
||||||
validateBody func(*testing.T, io.ReadCloser)
|
validateBody func(*testing.T, io.ReadCloser)
|
||||||
validateStatus func(*testing.T, int)
|
validateStatus func(*testing.T, int)
|
||||||
|
name string
|
||||||
|
errContains string
|
||||||
|
config network.Config
|
||||||
|
wantErr bool
|
||||||
}{
|
}{
|
||||||
// GOOD: Successful GET request
|
// GOOD: Successful GET request
|
||||||
{
|
{
|
||||||
|
|||||||
@@ -24,22 +24,22 @@ type Worker struct {
|
|||||||
cache *cache.Manager
|
cache *cache.Manager
|
||||||
analytics *analytics.Engine
|
analytics *analytics.Engine
|
||||||
client *network.Client
|
client *network.Client
|
||||||
|
stopChan chan struct{}
|
||||||
|
wg sync.WaitGroup
|
||||||
interval time.Duration
|
interval time.Duration
|
||||||
maxConcurrent int
|
maxConcurrent int
|
||||||
enabled bool
|
enabled bool
|
||||||
stopChan chan struct{}
|
|
||||||
wg sync.WaitGroup
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Config holds pre-warming worker configuration
|
// Config holds pre-warming worker configuration
|
||||||
type Config struct {
|
type Config struct {
|
||||||
Enabled bool
|
|
||||||
Interval time.Duration
|
|
||||||
MaxConcurrent int
|
|
||||||
TopPackages int
|
|
||||||
CacheManager *cache.Manager
|
CacheManager *cache.Manager
|
||||||
Analytics *analytics.Engine
|
Analytics *analytics.Engine
|
||||||
NetworkClient *network.Client
|
NetworkClient *network.Client
|
||||||
|
Interval time.Duration
|
||||||
|
MaxConcurrent int
|
||||||
|
TopPackages int
|
||||||
|
Enabled bool
|
||||||
}
|
}
|
||||||
|
|
||||||
// NewWorker creates a new pre-warming worker
|
// NewWorker creates a new pre-warming worker
|
||||||
|
|||||||
@@ -1,34 +0,0 @@
|
|||||||
package common
|
|
||||||
|
|
||||||
import (
|
|
||||||
"github.com/lukaszraczylo/gohoarder/pkg/cache"
|
|
||||||
"github.com/lukaszraczylo/gohoarder/pkg/network"
|
|
||||||
)
|
|
||||||
|
|
||||||
// BaseHandler provides common functionality for all proxy handlers
|
|
||||||
type BaseHandler struct {
|
|
||||||
Cache *cache.Manager
|
|
||||||
Client *network.Client
|
|
||||||
Upstream string
|
|
||||||
Registry string
|
|
||||||
}
|
|
||||||
|
|
||||||
// Config holds common proxy configuration
|
|
||||||
type Config struct {
|
|
||||||
Upstream string // Upstream registry URL (e.g., registry.npmjs.org)
|
|
||||||
}
|
|
||||||
|
|
||||||
// GetRegistry returns the registry type
|
|
||||||
func (h *BaseHandler) GetRegistry() string {
|
|
||||||
return h.Registry
|
|
||||||
}
|
|
||||||
|
|
||||||
// NewBaseHandler creates a new base handler with common fields
|
|
||||||
func NewBaseHandler(cache *cache.Manager, client *network.Client, registry, upstream string) *BaseHandler {
|
|
||||||
return &BaseHandler{
|
|
||||||
Cache: cache,
|
|
||||||
Client: client,
|
|
||||||
Upstream: upstream,
|
|
||||||
Registry: registry,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,385 +0,0 @@
|
|||||||
package common
|
|
||||||
|
|
||||||
import (
|
|
||||||
"bytes"
|
|
||||||
"context"
|
|
||||||
"errors"
|
|
||||||
"io"
|
|
||||||
"net/http"
|
|
||||||
"net/http/httptest"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"github.com/lukaszraczylo/gohoarder/pkg/cache"
|
|
||||||
"github.com/stretchr/testify/assert"
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
)
|
|
||||||
|
|
||||||
// TestNewBaseHandler tests base handler creation
|
|
||||||
func TestNewBaseHandler(t *testing.T) {
|
|
||||||
// Use nil for cache and client since we're only testing structure
|
|
||||||
handler := NewBaseHandler(nil, nil, "npm", "https://registry.npmjs.org")
|
|
||||||
|
|
||||||
require.NotNil(t, handler)
|
|
||||||
assert.Equal(t, "npm", handler.Registry)
|
|
||||||
assert.Equal(t, "https://registry.npmjs.org", handler.Upstream)
|
|
||||||
assert.Nil(t, handler.Cache)
|
|
||||||
assert.Nil(t, handler.Client)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestGetRegistry tests registry type retrieval
|
|
||||||
func TestGetRegistry(t *testing.T) {
|
|
||||||
tests := []struct {
|
|
||||||
name string
|
|
||||||
registry string
|
|
||||||
}{
|
|
||||||
{"npm registry", "npm"},
|
|
||||||
{"pypi registry", "pypi"},
|
|
||||||
{"go registry", "go"},
|
|
||||||
{"custom registry", "custom"},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, tt := range tests {
|
|
||||||
t.Run(tt.name, func(t *testing.T) {
|
|
||||||
handler := &BaseHandler{Registry: tt.registry}
|
|
||||||
assert.Equal(t, tt.registry, handler.GetRegistry())
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestHandleUpstreamError tests upstream error handling
|
|
||||||
func TestHandleUpstreamError(t *testing.T) {
|
|
||||||
tests := []struct {
|
|
||||||
name string
|
|
||||||
err error
|
|
||||||
url string
|
|
||||||
context string
|
|
||||||
wantStatus int
|
|
||||||
wantContain string
|
|
||||||
}{
|
|
||||||
// GOOD: Standard error
|
|
||||||
{
|
|
||||||
name: "connection error",
|
|
||||||
err: errors.New("connection refused"),
|
|
||||||
url: "https://registry.npmjs.org/react",
|
|
||||||
context: "package",
|
|
||||||
wantStatus: http.StatusBadGateway,
|
|
||||||
wantContain: "Failed to fetch package",
|
|
||||||
},
|
|
||||||
// WRONG: Timeout error
|
|
||||||
{
|
|
||||||
name: "timeout error",
|
|
||||||
err: context.DeadlineExceeded,
|
|
||||||
url: "https://registry.npmjs.org/lodash",
|
|
||||||
context: "metadata",
|
|
||||||
wantStatus: http.StatusBadGateway,
|
|
||||||
wantContain: "Failed to fetch metadata",
|
|
||||||
},
|
|
||||||
// EDGE: Empty context
|
|
||||||
{
|
|
||||||
name: "empty context",
|
|
||||||
err: errors.New("error"),
|
|
||||||
url: "https://example.com",
|
|
||||||
context: "",
|
|
||||||
wantStatus: http.StatusBadGateway,
|
|
||||||
wantContain: "Failed to fetch",
|
|
||||||
},
|
|
||||||
// EDGE: Long URL
|
|
||||||
{
|
|
||||||
name: "long URL",
|
|
||||||
err: errors.New("error"),
|
|
||||||
url: "https://registry.npmjs.org/@scope/very-long-package-name/versions/1.2.3",
|
|
||||||
context: "package",
|
|
||||||
wantStatus: http.StatusBadGateway,
|
|
||||||
wantContain: "Failed to fetch package",
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, tt := range tests {
|
|
||||||
t.Run(tt.name, func(t *testing.T) {
|
|
||||||
w := httptest.NewRecorder()
|
|
||||||
HandleUpstreamError(w, tt.err, tt.url, tt.context)
|
|
||||||
|
|
||||||
assert.Equal(t, tt.wantStatus, w.Code)
|
|
||||||
assert.Contains(t, w.Body.String(), tt.wantContain)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestCheckUpstreamStatus tests upstream status validation
|
|
||||||
func TestCheckUpstreamStatus(t *testing.T) {
|
|
||||||
tests := []struct {
|
|
||||||
name string
|
|
||||||
statusCode int
|
|
||||||
body io.ReadCloser
|
|
||||||
wantErr bool
|
|
||||||
errContains string
|
|
||||||
bodyClosed bool
|
|
||||||
}{
|
|
||||||
// GOOD: OK status
|
|
||||||
{
|
|
||||||
name: "200 OK",
|
|
||||||
statusCode: http.StatusOK,
|
|
||||||
body: io.NopCloser(strings.NewReader("success")),
|
|
||||||
wantErr: false,
|
|
||||||
},
|
|
||||||
// WRONG: Not found
|
|
||||||
{
|
|
||||||
name: "404 Not Found",
|
|
||||||
statusCode: http.StatusNotFound,
|
|
||||||
body: io.NopCloser(strings.NewReader("not found")),
|
|
||||||
wantErr: true,
|
|
||||||
errContains: "upstream returned status 404",
|
|
||||||
},
|
|
||||||
// WRONG: Server error
|
|
||||||
{
|
|
||||||
name: "500 Internal Server Error",
|
|
||||||
statusCode: http.StatusInternalServerError,
|
|
||||||
body: io.NopCloser(strings.NewReader("error")),
|
|
||||||
wantErr: true,
|
|
||||||
errContains: "upstream returned status 500",
|
|
||||||
},
|
|
||||||
// BAD: Unauthorized
|
|
||||||
{
|
|
||||||
name: "401 Unauthorized",
|
|
||||||
statusCode: http.StatusUnauthorized,
|
|
||||||
body: io.NopCloser(strings.NewReader("unauthorized")),
|
|
||||||
wantErr: true,
|
|
||||||
errContains: "upstream returned status 401",
|
|
||||||
},
|
|
||||||
// EDGE: Nil body
|
|
||||||
{
|
|
||||||
name: "nil body with error",
|
|
||||||
statusCode: http.StatusNotFound,
|
|
||||||
body: nil,
|
|
||||||
wantErr: true,
|
|
||||||
errContains: "upstream returned status 404",
|
|
||||||
},
|
|
||||||
// EDGE: Redirect status
|
|
||||||
{
|
|
||||||
name: "302 Found",
|
|
||||||
statusCode: http.StatusFound,
|
|
||||||
body: io.NopCloser(strings.NewReader("redirect")),
|
|
||||||
wantErr: true,
|
|
||||||
errContains: "upstream returned status 302",
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, tt := range tests {
|
|
||||||
t.Run(tt.name, func(t *testing.T) {
|
|
||||||
err := CheckUpstreamStatus(tt.statusCode, tt.body)
|
|
||||||
|
|
||||||
if tt.wantErr {
|
|
||||||
require.Error(t, err)
|
|
||||||
assert.Contains(t, err.Error(), tt.errContains)
|
|
||||||
} else {
|
|
||||||
require.NoError(t, err)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestHandleInvalidRequest tests invalid request handling
|
|
||||||
func TestHandleInvalidRequest(t *testing.T) {
|
|
||||||
tests := []struct {
|
|
||||||
name string
|
|
||||||
registry string
|
|
||||||
wantStatus int
|
|
||||||
wantContain string
|
|
||||||
}{
|
|
||||||
{
|
|
||||||
name: "npm invalid request",
|
|
||||||
registry: "npm",
|
|
||||||
wantStatus: http.StatusBadRequest,
|
|
||||||
wantContain: "Invalid npm request",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "pypi invalid request",
|
|
||||||
registry: "pypi",
|
|
||||||
wantStatus: http.StatusBadRequest,
|
|
||||||
wantContain: "Invalid pypi request",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "go invalid request",
|
|
||||||
registry: "go",
|
|
||||||
wantStatus: http.StatusBadRequest,
|
|
||||||
wantContain: "Invalid go request",
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, tt := range tests {
|
|
||||||
t.Run(tt.name, func(t *testing.T) {
|
|
||||||
w := httptest.NewRecorder()
|
|
||||||
HandleInvalidRequest(w, tt.registry)
|
|
||||||
|
|
||||||
assert.Equal(t, tt.wantStatus, w.Code)
|
|
||||||
assert.Contains(t, w.Body.String(), tt.wantContain)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestHandleInternalError tests internal error handling
|
|
||||||
func TestHandleInternalError(t *testing.T) {
|
|
||||||
tests := []struct {
|
|
||||||
name string
|
|
||||||
err error
|
|
||||||
context string
|
|
||||||
wantStatus int
|
|
||||||
wantContain string
|
|
||||||
}{
|
|
||||||
{
|
|
||||||
name: "database error",
|
|
||||||
err: errors.New("database connection failed"),
|
|
||||||
context: "database",
|
|
||||||
wantStatus: http.StatusInternalServerError,
|
|
||||||
wantContain: "Internal error: database",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "cache error",
|
|
||||||
err: errors.New("cache write failed"),
|
|
||||||
context: "cache",
|
|
||||||
wantStatus: http.StatusInternalServerError,
|
|
||||||
wantContain: "Internal error: cache",
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, tt := range tests {
|
|
||||||
t.Run(tt.name, func(t *testing.T) {
|
|
||||||
w := httptest.NewRecorder()
|
|
||||||
HandleInternalError(w, tt.err, tt.context)
|
|
||||||
|
|
||||||
assert.Equal(t, tt.wantStatus, w.Code)
|
|
||||||
assert.Contains(t, w.Body.String(), tt.wantContain)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Note: FetchFromUpstream tests would require mocking cache.Manager and network.Client
|
|
||||||
// which requires concrete implementations. Integration tests cover this functionality.
|
|
||||||
|
|
||||||
// TestWriteResponse tests HTTP response writing
|
|
||||||
func TestWriteResponse(t *testing.T) {
|
|
||||||
tests := []struct {
|
|
||||||
name string
|
|
||||||
data string
|
|
||||||
contentType string
|
|
||||||
wantStatus int
|
|
||||||
wantBody string
|
|
||||||
wantErr bool
|
|
||||||
}{
|
|
||||||
// GOOD: Write tarball
|
|
||||||
{
|
|
||||||
name: "write tarball",
|
|
||||||
data: "package data here",
|
|
||||||
contentType: "application/octet-stream",
|
|
||||||
wantStatus: http.StatusOK,
|
|
||||||
wantBody: "package data here",
|
|
||||||
wantErr: false,
|
|
||||||
},
|
|
||||||
// GOOD: Write JSON
|
|
||||||
{
|
|
||||||
name: "write JSON metadata",
|
|
||||||
data: `{"name":"react","version":"18.2.0"}`,
|
|
||||||
contentType: "application/json",
|
|
||||||
wantStatus: http.StatusOK,
|
|
||||||
wantBody: `{"name":"react","version":"18.2.0"}`,
|
|
||||||
wantErr: false,
|
|
||||||
},
|
|
||||||
// EDGE: Empty data
|
|
||||||
{
|
|
||||||
name: "empty data",
|
|
||||||
data: "",
|
|
||||||
contentType: "text/plain",
|
|
||||||
wantStatus: http.StatusOK,
|
|
||||||
wantBody: "",
|
|
||||||
wantErr: false,
|
|
||||||
},
|
|
||||||
// EDGE: Large data
|
|
||||||
{
|
|
||||||
name: "large data",
|
|
||||||
data: strings.Repeat("x", 100000),
|
|
||||||
contentType: "application/octet-stream",
|
|
||||||
wantStatus: http.StatusOK,
|
|
||||||
wantBody: strings.Repeat("x", 100000),
|
|
||||||
wantErr: false,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, tt := range tests {
|
|
||||||
t.Run(tt.name, func(t *testing.T) {
|
|
||||||
w := httptest.NewRecorder()
|
|
||||||
entry := &cache.CacheEntry{
|
|
||||||
Data: io.NopCloser(bytes.NewReader([]byte(tt.data))),
|
|
||||||
}
|
|
||||||
|
|
||||||
err := WriteResponse(w, entry, tt.contentType)
|
|
||||||
|
|
||||||
if tt.wantErr {
|
|
||||||
require.Error(t, err)
|
|
||||||
} else {
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.Equal(t, tt.contentType, w.Header().Get("Content-Type"))
|
|
||||||
assert.Equal(t, tt.wantBody, w.Body.String())
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestBaseHandlerFields tests that BaseHandler fields are properly set
|
|
||||||
func TestBaseHandlerFields(t *testing.T) {
|
|
||||||
handler := NewBaseHandler(nil, nil, "npm", "https://registry.npmjs.org")
|
|
||||||
|
|
||||||
tests := []struct {
|
|
||||||
name string
|
|
||||||
field string
|
|
||||||
expected interface{}
|
|
||||||
}{
|
|
||||||
{"registry field", "registry", "npm"},
|
|
||||||
{"upstream field", "upstream", "https://registry.npmjs.org"},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, tt := range tests {
|
|
||||||
t.Run(tt.name, func(t *testing.T) {
|
|
||||||
switch tt.field {
|
|
||||||
case "registry":
|
|
||||||
assert.Equal(t, tt.expected, handler.Registry)
|
|
||||||
case "upstream":
|
|
||||||
assert.Equal(t, tt.expected, handler.Upstream)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestProxyHandlerInterface tests that BaseHandler can be used as ProxyHandler
|
|
||||||
func TestProxyHandlerInterface(t *testing.T) {
|
|
||||||
handler := NewBaseHandler(nil, nil, "npm", "https://registry.npmjs.org")
|
|
||||||
|
|
||||||
// Verify GetRegistry works
|
|
||||||
registry := handler.GetRegistry()
|
|
||||||
assert.Equal(t, "npm", registry)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestConcurrentWriteResponse tests that WriteResponse is safe for concurrent use
|
|
||||||
func TestConcurrentWriteResponse(t *testing.T) {
|
|
||||||
const numGoroutines = 10
|
|
||||||
|
|
||||||
errs := make(chan error, numGoroutines)
|
|
||||||
for i := 0; i < numGoroutines; i++ {
|
|
||||||
go func(n int) {
|
|
||||||
w := httptest.NewRecorder()
|
|
||||||
data := strings.Repeat("x", 1000)
|
|
||||||
entry := &cache.CacheEntry{
|
|
||||||
Data: io.NopCloser(bytes.NewReader([]byte(data))),
|
|
||||||
}
|
|
||||||
|
|
||||||
err := WriteResponse(w, entry, "text/plain")
|
|
||||||
errs <- err
|
|
||||||
}(i)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Collect results
|
|
||||||
for i := 0; i < numGoroutines; i++ {
|
|
||||||
err := <-errs
|
|
||||||
assert.NoError(t, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,48 +0,0 @@
|
|||||||
package common
|
|
||||||
|
|
||||||
import (
|
|
||||||
"fmt"
|
|
||||||
"io"
|
|
||||||
"net/http"
|
|
||||||
|
|
||||||
"github.com/rs/zerolog/log"
|
|
||||||
)
|
|
||||||
|
|
||||||
// HandleUpstreamError logs an error and sends an HTTP 502 Bad Gateway response
|
|
||||||
// This is the common pattern used across all proxy handlers when upstream fetch fails
|
|
||||||
func HandleUpstreamError(w http.ResponseWriter, err error, url, context string) {
|
|
||||||
log.Error().
|
|
||||||
Err(err).
|
|
||||||
Str("url", url).
|
|
||||||
Str("context", context).
|
|
||||||
Msg("Failed to fetch from upstream")
|
|
||||||
|
|
||||||
http.Error(w, fmt.Sprintf("Failed to fetch %s", context), http.StatusBadGateway)
|
|
||||||
}
|
|
||||||
|
|
||||||
// CheckUpstreamStatus validates HTTP status code from upstream
|
|
||||||
// Returns error if status is not OK, closing body if needed
|
|
||||||
func CheckUpstreamStatus(statusCode int, body io.ReadCloser) error {
|
|
||||||
if statusCode != http.StatusOK {
|
|
||||||
if body != nil {
|
|
||||||
body.Close() // #nosec G104 -- Cleanup, error not critical
|
|
||||||
}
|
|
||||||
return fmt.Errorf("upstream returned status %d", statusCode)
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// HandleInvalidRequest sends a 400 Bad Request response for invalid proxy requests
|
|
||||||
func HandleInvalidRequest(w http.ResponseWriter, registry string) {
|
|
||||||
http.Error(w, fmt.Sprintf("Invalid %s request", registry), http.StatusBadRequest)
|
|
||||||
}
|
|
||||||
|
|
||||||
// HandleInternalError logs an internal error and sends 500 response
|
|
||||||
func HandleInternalError(w http.ResponseWriter, err error, context string) {
|
|
||||||
log.Error().
|
|
||||||
Err(err).
|
|
||||||
Str("context", context).
|
|
||||||
Msg("Internal error processing request")
|
|
||||||
|
|
||||||
http.Error(w, fmt.Sprintf("Internal error: %s", context), http.StatusInternalServerError)
|
|
||||||
}
|
|
||||||
@@ -1,58 +0,0 @@
|
|||||||
package common
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"io"
|
|
||||||
"net/http"
|
|
||||||
|
|
||||||
"github.com/lukaszraczylo/gohoarder/pkg/cache"
|
|
||||||
"github.com/lukaszraczylo/gohoarder/pkg/network"
|
|
||||||
"github.com/rs/zerolog/log"
|
|
||||||
)
|
|
||||||
|
|
||||||
// FetchFromUpstream is a common helper to fetch content from upstream with caching
|
|
||||||
// This encapsulates the common pattern of: cache.Get -> network.Get -> error handling
|
|
||||||
func FetchFromUpstream(
|
|
||||||
ctx context.Context,
|
|
||||||
cacheManager *cache.Manager,
|
|
||||||
client *network.Client,
|
|
||||||
registry, name, version, upstreamURL string,
|
|
||||||
) (*cache.CacheEntry, error) {
|
|
||||||
entry, err := cacheManager.Get(ctx, registry, name, version, func(ctx context.Context) (io.ReadCloser, string, error) {
|
|
||||||
body, statusCode, err := client.Get(ctx, upstreamURL, nil)
|
|
||||||
if err != nil {
|
|
||||||
return nil, "", err
|
|
||||||
}
|
|
||||||
if err := CheckUpstreamStatus(statusCode, body); err != nil {
|
|
||||||
return nil, "", err
|
|
||||||
}
|
|
||||||
return body, upstreamURL, nil
|
|
||||||
})
|
|
||||||
|
|
||||||
if err != nil {
|
|
||||||
log.Error().
|
|
||||||
Err(err).
|
|
||||||
Str("url", upstreamURL).
|
|
||||||
Str("registry", registry).
|
|
||||||
Str("name", name).
|
|
||||||
Str("version", version).
|
|
||||||
Msg("Failed to fetch package from upstream")
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
return entry, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// WriteResponse writes the cache entry data to the HTTP response writer
|
|
||||||
// Sets appropriate content type and handles errors
|
|
||||||
func WriteResponse(w http.ResponseWriter, entry *cache.CacheEntry, contentType string) error {
|
|
||||||
defer entry.Data.Close() // #nosec G104 -- Cleanup, error not critical
|
|
||||||
|
|
||||||
w.Header().Set("Content-Type", contentType)
|
|
||||||
if _, err := io.Copy(w, entry.Data); err != nil {
|
|
||||||
log.Error().Err(err).Msg("Failed to write response")
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
@@ -1,29 +0,0 @@
|
|||||||
package common
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"net/http"
|
|
||||||
"time"
|
|
||||||
)
|
|
||||||
|
|
||||||
// ProxyHandler defines the common interface for all registry proxies
|
|
||||||
type ProxyHandler interface {
|
|
||||||
http.Handler // ServeHTTP(w http.ResponseWriter, r *http.Request)
|
|
||||||
|
|
||||||
// GetRegistry returns the registry type (npm, pypi, go)
|
|
||||||
GetRegistry() string
|
|
||||||
|
|
||||||
// Health checks if the proxy can reach its upstream
|
|
||||||
Health(ctx context.Context) error
|
|
||||||
}
|
|
||||||
|
|
||||||
// Stats represents proxy statistics
|
|
||||||
type Stats struct {
|
|
||||||
Registry string
|
|
||||||
TotalRequests int64
|
|
||||||
CacheHits int64
|
|
||||||
CacheMisses int64
|
|
||||||
UpstreamErrors int64
|
|
||||||
AvgResponseTime time.Duration
|
|
||||||
LastUpdated time.Time
|
|
||||||
}
|
|
||||||
@@ -20,21 +20,21 @@ import (
|
|||||||
type Handler struct {
|
type Handler struct {
|
||||||
cache *cache.Manager
|
cache *cache.Manager
|
||||||
client *network.Client
|
client *network.Client
|
||||||
upstream string
|
|
||||||
sumDBURL string
|
|
||||||
credExtractor *auth.CredentialExtractor
|
credExtractor *auth.CredentialExtractor
|
||||||
credHasher *auth.CredentialHasher
|
credHasher *auth.CredentialHasher
|
||||||
credValidator *auth.GoValidator
|
credValidator *auth.GoValidator
|
||||||
validationCache *auth.ValidationCache
|
validationCache *auth.ValidationCache
|
||||||
gitFetcher *vcs.GitFetcher
|
gitFetcher *vcs.GitFetcher
|
||||||
moduleBuilder *vcs.ModuleBuilder
|
moduleBuilder *vcs.ModuleBuilder
|
||||||
|
upstream string
|
||||||
|
sumDBURL string
|
||||||
}
|
}
|
||||||
|
|
||||||
// Config holds Go proxy configuration
|
// Config holds Go proxy configuration
|
||||||
type Config struct {
|
type Config struct {
|
||||||
Upstream string // Upstream Go proxy (e.g., proxy.golang.org)
|
CredStore *vcs.CredentialStore
|
||||||
SumDBURL string // Checksum database URL
|
Upstream string
|
||||||
CredStore *vcs.CredentialStore // Optional credential store for git access
|
SumDBURL string
|
||||||
}
|
}
|
||||||
|
|
||||||
// New creates a new Go proxy handler
|
// New creates a new Go proxy handler
|
||||||
|
|||||||
@@ -21,11 +21,11 @@ import (
|
|||||||
type Handler struct {
|
type Handler struct {
|
||||||
cache *cache.Manager
|
cache *cache.Manager
|
||||||
client *network.Client
|
client *network.Client
|
||||||
upstream string
|
|
||||||
credExtractor *auth.CredentialExtractor
|
credExtractor *auth.CredentialExtractor
|
||||||
credHasher *auth.CredentialHasher
|
credHasher *auth.CredentialHasher
|
||||||
credValidator *auth.NPMValidator
|
credValidator *auth.NPMValidator
|
||||||
validationCache *auth.ValidationCache
|
validationCache *auth.ValidationCache
|
||||||
|
upstream string
|
||||||
}
|
}
|
||||||
|
|
||||||
// Config holds NPM proxy configuration
|
// Config holds NPM proxy configuration
|
||||||
|
|||||||
+17
-4
@@ -21,11 +21,11 @@ import (
|
|||||||
type Handler struct {
|
type Handler struct {
|
||||||
cache *cache.Manager
|
cache *cache.Manager
|
||||||
client *network.Client
|
client *network.Client
|
||||||
upstream string
|
|
||||||
credExtractor *auth.CredentialExtractor
|
credExtractor *auth.CredentialExtractor
|
||||||
credHasher *auth.CredentialHasher
|
credHasher *auth.CredentialHasher
|
||||||
credValidator *auth.PyPIValidator
|
credValidator *auth.PyPIValidator
|
||||||
validationCache *auth.ValidationCache
|
validationCache *auth.ValidationCache
|
||||||
|
upstream string
|
||||||
}
|
}
|
||||||
|
|
||||||
// Config holds PyPI proxy configuration
|
// Config holds PyPI proxy configuration
|
||||||
@@ -148,6 +148,17 @@ func (h *Handler) handlePackagePage(ctx context.Context, w http.ResponseWriter,
|
|||||||
func (h *Handler) handlePackageFile(ctx context.Context, w http.ResponseWriter, r *http.Request, path string) {
|
func (h *Handler) handlePackageFile(ctx context.Context, w http.ResponseWriter, r *http.Request, path string) {
|
||||||
packageName, version := extractPackageFileInfo(path)
|
packageName, version := extractPackageFileInfo(path)
|
||||||
|
|
||||||
|
// Make version unique by appending file type to avoid cache collisions
|
||||||
|
// between .whl and .metadata files with same version
|
||||||
|
cacheVersion := version
|
||||||
|
if strings.HasSuffix(path, ".metadata") {
|
||||||
|
cacheVersion = version + ".metadata"
|
||||||
|
} else if strings.HasSuffix(path, ".whl") {
|
||||||
|
cacheVersion = version + ".whl"
|
||||||
|
} else if strings.HasSuffix(path, ".tar.gz") {
|
||||||
|
cacheVersion = version + ".tar.gz"
|
||||||
|
}
|
||||||
|
|
||||||
// Extract credentials from request
|
// Extract credentials from request
|
||||||
credentials := h.credExtractor.Extract(r)
|
credentials := h.credExtractor.Extract(r)
|
||||||
credHash := h.credHasher.Hash(credentials)
|
credHash := h.credHasher.Hash(credentials)
|
||||||
@@ -170,12 +181,13 @@ func (h *Handler) handlePackageFile(ctx context.Context, w http.ResponseWriter,
|
|||||||
Str("path", path).
|
Str("path", path).
|
||||||
Str("package", packageName).
|
Str("package", packageName).
|
||||||
Str("version", version).
|
Str("version", version).
|
||||||
|
Str("cache_version", cacheVersion).
|
||||||
Str("url", originalURL).
|
Str("url", originalURL).
|
||||||
Str("cred_hash", credHash).
|
Str("cred_hash", credHash).
|
||||||
Bool("has_credentials", credentials != "").
|
Bool("has_credentials", credentials != "").
|
||||||
Msg("Handling PyPI package file request")
|
Msg("Handling PyPI package file request")
|
||||||
|
|
||||||
entry, err := h.cache.Get(ctx, "pypi", packageName, version, func(ctx context.Context) (io.ReadCloser, string, error) {
|
entry, err := h.cache.Get(ctx, "pypi", packageName, cacheVersion, func(ctx context.Context) (io.ReadCloser, string, error) {
|
||||||
// Prepare headers for upstream request
|
// Prepare headers for upstream request
|
||||||
headers := make(map[string]string)
|
headers := make(map[string]string)
|
||||||
if credentials != "" {
|
if credentials != "" {
|
||||||
@@ -281,11 +293,12 @@ func isPackagePage(path string) bool {
|
|||||||
|
|
||||||
// isPackageFile checks if the request is for a package file
|
// isPackageFile checks if the request is for a package file
|
||||||
func isPackageFile(path string) bool {
|
func isPackageFile(path string) bool {
|
||||||
// Package files (not including .metadata files which need special handling)
|
// Package files including .metadata files for PEP 658 support
|
||||||
return strings.HasSuffix(path, ".whl") ||
|
return strings.HasSuffix(path, ".whl") ||
|
||||||
strings.HasSuffix(path, ".tar.gz") ||
|
strings.HasSuffix(path, ".tar.gz") ||
|
||||||
strings.HasSuffix(path, ".zip") ||
|
strings.HasSuffix(path, ".zip") ||
|
||||||
strings.HasSuffix(path, ".egg")
|
strings.HasSuffix(path, ".egg") ||
|
||||||
|
strings.HasSuffix(path, ".metadata")
|
||||||
}
|
}
|
||||||
|
|
||||||
// extractPackageName extracts package name from path
|
// extractPackageName extracts package name from path
|
||||||
|
|||||||
@@ -20,8 +20,8 @@ const ScannerName = "github-advisory-database"
|
|||||||
|
|
||||||
// Scanner implements the GitHub Advisory Database vulnerability scanner
|
// Scanner implements the GitHub Advisory Database vulnerability scanner
|
||||||
type Scanner struct {
|
type Scanner struct {
|
||||||
config config.GHSAConfig
|
|
||||||
httpClient *http.Client
|
httpClient *http.Client
|
||||||
|
config config.GHSAConfig
|
||||||
}
|
}
|
||||||
|
|
||||||
// New creates a new GitHub Advisory Database scanner
|
// New creates a new GitHub Advisory Database scanner
|
||||||
@@ -107,11 +107,17 @@ func (s *Scanner) Health(ctx context.Context) error {
|
|||||||
}
|
}
|
||||||
defer resp.Body.Close() // #nosec G104 -- Cleanup, error not critical
|
defer resp.Body.Close() // #nosec G104 -- Cleanup, error not critical
|
||||||
|
|
||||||
if resp.StatusCode != http.StatusOK {
|
// Accept any 2xx or 403 (rate limit) as healthy
|
||||||
return fmt.Errorf("github api returned status: %d", resp.StatusCode)
|
// Rate limits are expected without a GitHub token and shouldn't fail health checks
|
||||||
|
if resp.StatusCode >= 200 && resp.StatusCode < 300 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if resp.StatusCode == http.StatusForbidden {
|
||||||
|
log.Debug().Msg("GitHub API rate limited (expected without token)")
|
||||||
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
return nil
|
return fmt.Errorf("github api returned status: %d", resp.StatusCode)
|
||||||
}
|
}
|
||||||
|
|
||||||
// mapRegistryToEcosystem maps our registry names to GitHub ecosystem names
|
// mapRegistryToEcosystem maps our registry names to GitHub ecosystem names
|
||||||
@@ -257,10 +263,10 @@ type GHSAAdvisory struct {
|
|||||||
Description string `json:"description"`
|
Description string `json:"description"`
|
||||||
Severity string `json:"severity"`
|
Severity string `json:"severity"`
|
||||||
HTMLURL string `json:"html_url"`
|
HTMLURL string `json:"html_url"`
|
||||||
References []GHSAReference `json:"references"`
|
|
||||||
Vulnerabilities []GHSAVulnerability `json:"vulnerabilities"`
|
|
||||||
PublishedAt string `json:"published_at"`
|
PublishedAt string `json:"published_at"`
|
||||||
UpdatedAt string `json:"updated_at"`
|
UpdatedAt string `json:"updated_at"`
|
||||||
|
References []GHSAReference `json:"references"`
|
||||||
|
Vulnerabilities []GHSAVulnerability `json:"vulnerabilities"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type GHSAReference struct {
|
type GHSAReference struct {
|
||||||
@@ -268,9 +274,9 @@ type GHSAReference struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
type GHSAVulnerability struct {
|
type GHSAVulnerability struct {
|
||||||
|
FirstPatchedVersion *GHSAPatchVersion `json:"first_patched_version"`
|
||||||
Package GHSAPackage `json:"package"`
|
Package GHSAPackage `json:"package"`
|
||||||
VulnerableVersions string `json:"vulnerable_version_range"`
|
VulnerableVersions string `json:"vulnerable_version_range"`
|
||||||
FirstPatchedVersion *GHSAPatchVersion `json:"first_patched_version"`
|
|
||||||
}
|
}
|
||||||
|
|
||||||
type GHSAPackage struct {
|
type GHSAPackage struct {
|
||||||
|
|||||||
@@ -153,9 +153,9 @@ func (s *Scanner) convertGrypeResult(grypeResult *GrypeResult, registry, package
|
|||||||
|
|
||||||
// GrypeResult represents Grype JSON output structure
|
// GrypeResult represents Grype JSON output structure
|
||||||
type GrypeResult struct {
|
type GrypeResult struct {
|
||||||
Matches []GrypeMatch `json:"matches"`
|
|
||||||
Descriptor GrypeDescriptor `json:"descriptor"`
|
|
||||||
Source GrypeSource `json:"source"`
|
Source GrypeSource `json:"source"`
|
||||||
|
Descriptor GrypeDescriptor `json:"descriptor"`
|
||||||
|
Matches []GrypeMatch `json:"matches"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type GrypeDescriptor struct {
|
type GrypeDescriptor struct {
|
||||||
@@ -164,13 +164,13 @@ type GrypeDescriptor struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
type GrypeSource struct {
|
type GrypeSource struct {
|
||||||
Type string `json:"type"`
|
|
||||||
Target map[string]interface{} `json:"target"`
|
Target map[string]interface{} `json:"target"`
|
||||||
|
Type string `json:"type"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type GrypeMatch struct {
|
type GrypeMatch struct {
|
||||||
Vulnerability GrypeVulnerability `json:"vulnerability"`
|
|
||||||
Artifact GrypeArtifact `json:"artifact"`
|
Artifact GrypeArtifact `json:"artifact"`
|
||||||
|
Vulnerability GrypeVulnerability `json:"vulnerability"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type GrypeVulnerability struct {
|
type GrypeVulnerability struct {
|
||||||
|
|||||||
@@ -199,9 +199,9 @@ func (s *Scanner) convertResult(auditResult *NpmAuditResult, registry, packageNa
|
|||||||
|
|
||||||
// NpmAuditResult represents npm audit JSON output
|
// NpmAuditResult represents npm audit JSON output
|
||||||
type NpmAuditResult struct {
|
type NpmAuditResult struct {
|
||||||
AuditReportVersion int `json:"auditReportVersion"`
|
|
||||||
Vulnerabilities map[string]NpmVulnerability `json:"vulnerabilities"`
|
Vulnerabilities map[string]NpmVulnerability `json:"vulnerabilities"`
|
||||||
Metadata NpmAuditMetadata `json:"metadata"`
|
Metadata NpmAuditMetadata `json:"metadata"`
|
||||||
|
AuditReportVersion int `json:"auditReportVersion"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type NpmVulnerability struct {
|
type NpmVulnerability struct {
|
||||||
|
|||||||
+56
-6
@@ -25,8 +25,8 @@ const (
|
|||||||
|
|
||||||
// Scanner implements the Scanner interface using OSV.dev API
|
// Scanner implements the Scanner interface using OSV.dev API
|
||||||
type Scanner struct {
|
type Scanner struct {
|
||||||
config config.OSVConfig
|
|
||||||
httpClient *http.Client
|
httpClient *http.Client
|
||||||
|
config config.OSVConfig
|
||||||
}
|
}
|
||||||
|
|
||||||
// OSVRequest represents the request structure for OSV API
|
// OSVRequest represents the request structure for OSV API
|
||||||
@@ -48,13 +48,13 @@ type OSVResponse struct {
|
|||||||
|
|
||||||
// OSVVulnerability represents a vulnerability in OSV format
|
// OSVVulnerability represents a vulnerability in OSV format
|
||||||
type OSVVulnerability struct {
|
type OSVVulnerability struct {
|
||||||
|
DatabaseSpecific map[string]interface{} `json:"database_specific,omitempty"`
|
||||||
ID string `json:"id"`
|
ID string `json:"id"`
|
||||||
Summary string `json:"summary"`
|
Summary string `json:"summary"`
|
||||||
Details string `json:"details"`
|
Details string `json:"details"`
|
||||||
Severity []OSVSeverity `json:"severity,omitempty"`
|
Severity []OSVSeverity `json:"severity,omitempty"`
|
||||||
References []OSVReference `json:"references,omitempty"`
|
References []OSVReference `json:"references,omitempty"`
|
||||||
Affected []OSVAffected `json:"affected"`
|
Affected []OSVAffected `json:"affected"`
|
||||||
DatabaseSpecific map[string]interface{} `json:"database_specific,omitempty"`
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// OSVSeverity represents severity information
|
// OSVSeverity represents severity information
|
||||||
@@ -71,11 +71,11 @@ type OSVReference struct {
|
|||||||
|
|
||||||
// OSVAffected represents affected package versions
|
// OSVAffected represents affected package versions
|
||||||
type OSVAffected struct {
|
type OSVAffected struct {
|
||||||
|
DatabaseSpecific map[string]interface{} `json:"database_specific,omitempty"`
|
||||||
|
EcosystemSpecific map[string]interface{} `json:"ecosystem_specific,omitempty"`
|
||||||
Package PackageInfo `json:"package"`
|
Package PackageInfo `json:"package"`
|
||||||
Ranges []OSVRange `json:"ranges,omitempty"`
|
Ranges []OSVRange `json:"ranges,omitempty"`
|
||||||
Versions []string `json:"versions,omitempty"`
|
Versions []string `json:"versions,omitempty"`
|
||||||
DatabaseSpecific map[string]interface{} `json:"database_specific,omitempty"`
|
|
||||||
EcosystemSpecific map[string]interface{} `json:"ecosystem_specific,omitempty"`
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// OSVRange represents version ranges
|
// OSVRange represents version ranges
|
||||||
@@ -121,13 +121,17 @@ func (s *Scanner) Scan(ctx context.Context, registry, packageName, version strin
|
|||||||
// Convert registry to OSV ecosystem
|
// Convert registry to OSV ecosystem
|
||||||
ecosystem := s.registryToEcosystem(registry)
|
ecosystem := s.registryToEcosystem(registry)
|
||||||
|
|
||||||
|
// Clean package name and version for Go modules
|
||||||
|
// Go proxy cache keys include /@v/version.suffix which OSV doesn't understand
|
||||||
|
cleanName, cleanVersion := s.cleanGoModuleName(packageName, version, ecosystem)
|
||||||
|
|
||||||
// Build request
|
// Build request
|
||||||
req := OSVRequest{
|
req := OSVRequest{
|
||||||
Package: PackageInfo{
|
Package: PackageInfo{
|
||||||
Name: packageName,
|
Name: cleanName,
|
||||||
Ecosystem: ecosystem,
|
Ecosystem: ecosystem,
|
||||||
},
|
},
|
||||||
Version: version,
|
Version: cleanVersion,
|
||||||
}
|
}
|
||||||
|
|
||||||
// Marshal request
|
// Marshal request
|
||||||
@@ -199,6 +203,52 @@ func (s *Scanner) registryToEcosystem(registry string) string {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// cleanGoModuleName cleans Go module cache keys to extract the actual module path and version
|
||||||
|
// Go proxy cache keys include /@v/version.suffix patterns that need to be cleaned
|
||||||
|
// Examples:
|
||||||
|
// - "gorm.io/driver/sqlite/@v/v1.6.0.zip" -> "gorm.io/driver/sqlite", "v1.6.0"
|
||||||
|
// - "github.com/pkg/errors/@v/v0.9.1.mod" -> "github.com/pkg/errors", "v0.9.1"
|
||||||
|
// - "regular-package" -> "regular-package", "version" (unchanged for non-Go)
|
||||||
|
func (s *Scanner) cleanGoModuleName(packageName, version, ecosystem string) (string, string) {
|
||||||
|
// Only clean for Go modules
|
||||||
|
if ecosystem != "Go" {
|
||||||
|
return packageName, version
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check if package name contains /@v/ pattern (Go module proxy format)
|
||||||
|
if strings.Contains(packageName, "/@v/") {
|
||||||
|
// Split on /@v/ to get the module path
|
||||||
|
parts := strings.Split(packageName, "/@v/")
|
||||||
|
if len(parts) == 2 {
|
||||||
|
// parts[0] is the clean module path (e.g., "gorm.io/driver/sqlite")
|
||||||
|
// parts[1] might be "v1.6.0.zip" or "v1.6.0.mod" or "v1.6.0.info"
|
||||||
|
cleanName := parts[0]
|
||||||
|
|
||||||
|
// Extract version from the second part if version wasn't already clean
|
||||||
|
// Remove file suffixes like .zip, .mod, .info
|
||||||
|
versionPart := parts[1]
|
||||||
|
versionPart = strings.TrimSuffix(versionPart, ".zip")
|
||||||
|
versionPart = strings.TrimSuffix(versionPart, ".mod")
|
||||||
|
versionPart = strings.TrimSuffix(versionPart, ".info")
|
||||||
|
|
||||||
|
// Use the extracted version if it looks valid, otherwise use the provided version
|
||||||
|
if versionPart != "" && strings.HasPrefix(versionPart, "v") {
|
||||||
|
return cleanName, versionPart
|
||||||
|
}
|
||||||
|
|
||||||
|
return cleanName, version
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Also clean version of any file suffixes
|
||||||
|
cleanVersion := version
|
||||||
|
cleanVersion = strings.TrimSuffix(cleanVersion, ".zip")
|
||||||
|
cleanVersion = strings.TrimSuffix(cleanVersion, ".mod")
|
||||||
|
cleanVersion = strings.TrimSuffix(cleanVersion, ".info")
|
||||||
|
|
||||||
|
return packageName, cleanVersion
|
||||||
|
}
|
||||||
|
|
||||||
// convertOSVResult converts OSV response to metadata.ScanResult
|
// convertOSVResult converts OSV response to metadata.ScanResult
|
||||||
func (s *Scanner) convertOSVResult(osvResp *OSVResponse, registry, packageName, version string) *metadata.ScanResult {
|
func (s *Scanner) convertOSVResult(osvResp *OSVResponse, registry, packageName, version string) *metadata.ScanResult {
|
||||||
vulnerabilities := make([]metadata.Vulnerability, 0, len(osvResp.Vulns))
|
vulnerabilities := make([]metadata.Vulnerability, 0, len(osvResp.Vulns))
|
||||||
|
|||||||
@@ -11,11 +11,11 @@ import (
|
|||||||
|
|
||||||
// RescanWorker handles periodic re-scanning of cached packages
|
// RescanWorker handles periodic re-scanning of cached packages
|
||||||
type RescanWorker struct {
|
type RescanWorker struct {
|
||||||
manager *Manager
|
|
||||||
metadataStore metadata.MetadataStore
|
metadataStore metadata.MetadataStore
|
||||||
storage storage.StorageBackend
|
storage storage.StorageBackend
|
||||||
interval time.Duration
|
manager *Manager
|
||||||
stopCh chan struct{}
|
stopCh chan struct{}
|
||||||
|
interval time.Duration
|
||||||
}
|
}
|
||||||
|
|
||||||
// NewRescanWorker creates a new rescan worker
|
// NewRescanWorker creates a new rescan worker
|
||||||
|
|||||||
@@ -36,10 +36,10 @@ type DatabaseUpdater interface {
|
|||||||
|
|
||||||
// Manager manages multiple security scanners
|
// Manager manages multiple security scanners
|
||||||
type Manager struct {
|
type Manager struct {
|
||||||
|
metadataStore metadata.MetadataStore
|
||||||
|
config config.SecurityConfig
|
||||||
scanners []Scanner
|
scanners []Scanner
|
||||||
enabled bool
|
enabled bool
|
||||||
config config.SecurityConfig
|
|
||||||
metadataStore metadata.MetadataStore
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// New creates a new scanner manager with configured scanners
|
// New creates a new scanner manager with configured scanners
|
||||||
|
|||||||
@@ -25,18 +25,18 @@ type Scanner struct {
|
|||||||
|
|
||||||
// TrivyResult represents Trivy JSON output structure
|
// TrivyResult represents Trivy JSON output structure
|
||||||
type TrivyResult struct {
|
type TrivyResult struct {
|
||||||
SchemaVersion int `json:"SchemaVersion"`
|
Metadata TrivyMetadata `json:"Metadata"`
|
||||||
ArtifactName string `json:"ArtifactName"`
|
ArtifactName string `json:"ArtifactName"`
|
||||||
ArtifactType string `json:"ArtifactType"`
|
ArtifactType string `json:"ArtifactType"`
|
||||||
Metadata TrivyMetadata `json:"Metadata"`
|
|
||||||
Results []TrivyVulnResult `json:"Results"`
|
Results []TrivyVulnResult `json:"Results"`
|
||||||
|
SchemaVersion int `json:"SchemaVersion"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type TrivyMetadata struct {
|
type TrivyMetadata struct {
|
||||||
OS *TrivyOS `json:"OS,omitempty"`
|
OS *TrivyOS `json:"OS,omitempty"`
|
||||||
|
ImageConfig *TrivyImageConfig `json:"ImageConfig,omitempty"`
|
||||||
RepoTags []string `json:"RepoTags,omitempty"`
|
RepoTags []string `json:"RepoTags,omitempty"`
|
||||||
RepoDigests []string `json:"RepoDigests,omitempty"`
|
RepoDigests []string `json:"RepoDigests,omitempty"`
|
||||||
ImageConfig *TrivyImageConfig `json:"ImageConfig,omitempty"`
|
|
||||||
}
|
}
|
||||||
|
|
||||||
type TrivyOS struct {
|
type TrivyOS struct {
|
||||||
@@ -64,8 +64,8 @@ type TrivyVulnerability struct {
|
|||||||
Severity string `json:"Severity"`
|
Severity string `json:"Severity"`
|
||||||
Title string `json:"Title"`
|
Title string `json:"Title"`
|
||||||
Description string `json:"Description"`
|
Description string `json:"Description"`
|
||||||
References []string `json:"References"`
|
|
||||||
PrimaryURL string `json:"PrimaryURL"`
|
PrimaryURL string `json:"PrimaryURL"`
|
||||||
|
References []string `json:"References"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// New creates a new Trivy scanner
|
// New creates a new Trivy scanner
|
||||||
|
|||||||
@@ -1,130 +0,0 @@
|
|||||||
package server
|
|
||||||
|
|
||||||
import (
|
|
||||||
"fmt"
|
|
||||||
"net/http"
|
|
||||||
|
|
||||||
"github.com/lukaszraczylo/gohoarder/pkg/config"
|
|
||||||
"github.com/lukaszraczylo/gohoarder/pkg/health"
|
|
||||||
"github.com/lukaszraczylo/gohoarder/pkg/logger"
|
|
||||||
"github.com/lukaszraczylo/gohoarder/pkg/metrics"
|
|
||||||
)
|
|
||||||
|
|
||||||
// Server wraps http.Server with configuration
|
|
||||||
type Server struct {
|
|
||||||
*http.Server
|
|
||||||
config *config.Config
|
|
||||||
healthChecker *health.Checker
|
|
||||||
}
|
|
||||||
|
|
||||||
// New creates a new HTTP server
|
|
||||||
func New(cfg *config.Config, healthChecker *health.Checker) (*Server, error) {
|
|
||||||
mux := http.NewServeMux()
|
|
||||||
|
|
||||||
// Register routes
|
|
||||||
registerRoutes(mux, cfg, healthChecker)
|
|
||||||
|
|
||||||
srv := &http.Server{
|
|
||||||
Addr: fmt.Sprintf("%s:%d", cfg.Server.Host, cfg.Server.Port),
|
|
||||||
Handler: logger.Middleware(mux),
|
|
||||||
ReadTimeout: cfg.Server.ReadTimeout,
|
|
||||||
WriteTimeout: cfg.Server.WriteTimeout,
|
|
||||||
IdleTimeout: cfg.Server.IdleTimeout,
|
|
||||||
}
|
|
||||||
|
|
||||||
return &Server{
|
|
||||||
Server: srv,
|
|
||||||
config: cfg,
|
|
||||||
healthChecker: healthChecker,
|
|
||||||
}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// registerRoutes registers all HTTP routes
|
|
||||||
func registerRoutes(mux *http.ServeMux, cfg *config.Config, healthChecker *health.Checker) {
|
|
||||||
// Health endpoints
|
|
||||||
mux.HandleFunc("/health", healthChecker.HealthHandler())
|
|
||||||
mux.HandleFunc("/health/ready", healthChecker.ReadyHandler())
|
|
||||||
|
|
||||||
// Metrics endpoint
|
|
||||||
mux.Handle("/metrics", metrics.Handler())
|
|
||||||
|
|
||||||
// API endpoints
|
|
||||||
mux.HandleFunc("/api/v1/info", handleInfo(cfg))
|
|
||||||
|
|
||||||
// Package manager proxy endpoints (placeholders for now)
|
|
||||||
if cfg.Handlers.Go.Enabled {
|
|
||||||
mux.HandleFunc("/go/", handleGoProxy())
|
|
||||||
}
|
|
||||||
if cfg.Handlers.NPM.Enabled {
|
|
||||||
mux.HandleFunc("/npm/", handleNPMProxy())
|
|
||||||
}
|
|
||||||
if cfg.Handlers.PyPI.Enabled {
|
|
||||||
mux.HandleFunc("/pypi/", handlePyPIProxy())
|
|
||||||
}
|
|
||||||
|
|
||||||
// Root endpoint
|
|
||||||
mux.HandleFunc("/", handleRoot())
|
|
||||||
}
|
|
||||||
|
|
||||||
// handleInfo returns server information
|
|
||||||
func handleInfo(cfg *config.Config) http.HandlerFunc {
|
|
||||||
return func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
info := map[string]interface{}{
|
|
||||||
"name": "GoHoarder",
|
|
||||||
"version": "dev",
|
|
||||||
"handlers": map[string]bool{
|
|
||||||
"go": cfg.Handlers.Go.Enabled,
|
|
||||||
"npm": cfg.Handlers.NPM.Enabled,
|
|
||||||
"pypi": cfg.Handlers.PyPI.Enabled,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
w.Header().Set("Content-Type", "application/json")
|
|
||||||
w.WriteHeader(http.StatusOK)
|
|
||||||
fmt.Fprintf(w, `{"success":true,"data":%v}`, toJSON(info))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// handleGoProxy handles Go module proxy requests
|
|
||||||
func handleGoProxy() http.HandlerFunc {
|
|
||||||
return func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
// TODO: Implement Go proxy handler
|
|
||||||
http.Error(w, `{"success":false,"error":{"code":"NOT_IMPLEMENTED","message":"Go proxy not yet implemented"}}`, http.StatusNotImplemented)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// handleNPMProxy handles NPM registry requests
|
|
||||||
func handleNPMProxy() http.HandlerFunc {
|
|
||||||
return func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
// TODO: Implement NPM proxy handler
|
|
||||||
http.Error(w, `{"success":false,"error":{"code":"NOT_IMPLEMENTED","message":"NPM proxy not yet implemented"}}`, http.StatusNotImplemented)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// handlePyPIProxy handles PyPI requests
|
|
||||||
func handlePyPIProxy() http.HandlerFunc {
|
|
||||||
return func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
// TODO: Implement PyPI proxy handler
|
|
||||||
http.Error(w, `{"success":false,"error":{"code":"NOT_IMPLEMENTED","message":"PyPI proxy not yet implemented"}}`, http.StatusNotImplemented)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// handleRoot handles root path
|
|
||||||
func handleRoot() http.HandlerFunc {
|
|
||||||
return func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
if r.URL.Path != "/" {
|
|
||||||
http.NotFound(w, r)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
w.Header().Set("Content-Type", "application/json")
|
|
||||||
w.WriteHeader(http.StatusOK)
|
|
||||||
fmt.Fprint(w, `{"success":true,"data":{"message":"GoHoarder - Universal Package Cache Proxy","docs":"https://github.com/lukaszraczylo/gohoarder"}}`)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// toJSON is a simple JSON encoder (replace with proper implementation)
|
|
||||||
func toJSON(v interface{}) string {
|
|
||||||
// Simplified for now - proper implementation would use goccy/go-json
|
|
||||||
return fmt.Sprintf("%v", v)
|
|
||||||
}
|
|
||||||
@@ -17,8 +17,8 @@ import (
|
|||||||
|
|
||||||
type FilesystemStorageTestSuite struct {
|
type FilesystemStorageTestSuite struct {
|
||||||
suite.Suite
|
suite.Suite
|
||||||
tempDir string
|
|
||||||
fs *FilesystemStorage
|
fs *FilesystemStorage
|
||||||
|
tempDir string
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *FilesystemStorageTestSuite) SetupTest() {
|
func (s *FilesystemStorageTestSuite) SetupTest() {
|
||||||
@@ -46,12 +46,12 @@ func TestFilesystemStorageTestSuite(t *testing.T) {
|
|||||||
// Test Put operation
|
// Test Put operation
|
||||||
func (s *FilesystemStorageTestSuite) TestPut() {
|
func (s *FilesystemStorageTestSuite) TestPut() {
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
|
opts *storage.PutOptions
|
||||||
|
errorCheck func(error) bool
|
||||||
name string
|
name string
|
||||||
key string
|
key string
|
||||||
data string
|
data string
|
||||||
opts *storage.PutOptions
|
|
||||||
expectError bool
|
expectError bool
|
||||||
errorCheck func(error) bool
|
|
||||||
}{
|
}{
|
||||||
{
|
{
|
||||||
name: "successful put",
|
name: "successful put",
|
||||||
@@ -122,8 +122,8 @@ func (s *FilesystemStorageTestSuite) TestGet() {
|
|||||||
tests := []struct {
|
tests := []struct {
|
||||||
name string
|
name string
|
||||||
key string
|
key string
|
||||||
expectError bool
|
|
||||||
expectData string
|
expectData string
|
||||||
|
expectError bool
|
||||||
}{
|
}{
|
||||||
{
|
{
|
||||||
name: "get existing file",
|
name: "get existing file",
|
||||||
@@ -258,11 +258,11 @@ func (s *FilesystemStorageTestSuite) TestList() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
|
opts *storage.ListOptions
|
||||||
name string
|
name string
|
||||||
prefix string
|
prefix string
|
||||||
opts *storage.ListOptions
|
|
||||||
expectedCount int
|
|
||||||
expectedKeys []string
|
expectedKeys []string
|
||||||
|
expectedCount int
|
||||||
}{
|
}{
|
||||||
{
|
{
|
||||||
name: "list all npm packages",
|
name: "list all npm packages",
|
||||||
@@ -422,8 +422,8 @@ func (s *FilesystemStorageTestSuite) TestContextCancellation() {
|
|||||||
cancel()
|
cancel()
|
||||||
|
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
name string
|
|
||||||
fn func() error
|
fn func() error
|
||||||
|
name string
|
||||||
}{
|
}{
|
||||||
{
|
{
|
||||||
name: "Get with cancelled context",
|
name: "Get with cancelled context",
|
||||||
@@ -679,8 +679,8 @@ func (s *FilesystemStorageTestSuite) TestChecksumValidation() {
|
|||||||
correctMD5 := "7dd7323e8ce3e087972f93d3711ef62b"
|
correctMD5 := "7dd7323e8ce3e087972f93d3711ef62b"
|
||||||
|
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
name string
|
|
||||||
opts *storage.PutOptions
|
opts *storage.PutOptions
|
||||||
|
name string
|
||||||
expectError bool
|
expectError bool
|
||||||
}{
|
}{
|
||||||
{
|
{
|
||||||
|
|||||||
@@ -52,21 +52,21 @@ type ListOptions struct {
|
|||||||
|
|
||||||
// StorageObject represents a stored object
|
// StorageObject represents a stored object
|
||||||
type StorageObject struct {
|
type StorageObject struct {
|
||||||
Key string
|
|
||||||
Size int64
|
|
||||||
Modified time.Time
|
Modified time.Time
|
||||||
|
Key string
|
||||||
ETag string
|
ETag string
|
||||||
|
Size int64
|
||||||
}
|
}
|
||||||
|
|
||||||
// StorageInfo contains detailed object information
|
// StorageInfo contains detailed object information
|
||||||
type StorageInfo struct {
|
type StorageInfo struct {
|
||||||
Key string
|
|
||||||
Size int64
|
|
||||||
Modified time.Time
|
Modified time.Time
|
||||||
ETag string
|
|
||||||
ContentType string
|
|
||||||
Metadata map[string]string
|
Metadata map[string]string
|
||||||
Checksums *Checksums
|
Checksums *Checksums
|
||||||
|
Key string
|
||||||
|
ETag string
|
||||||
|
ContentType string
|
||||||
|
Size int64
|
||||||
}
|
}
|
||||||
|
|
||||||
// Checksums contains file checksums
|
// Checksums contains file checksums
|
||||||
|
|||||||
+189
-244
@@ -3,14 +3,10 @@ package s3
|
|||||||
import (
|
import (
|
||||||
"bytes"
|
"bytes"
|
||||||
"context"
|
"context"
|
||||||
"crypto/md5" // #nosec G501 -- MD5 used for S3 Content-MD5 header, not cryptographic security
|
|
||||||
"crypto/sha256"
|
|
||||||
"encoding/hex"
|
|
||||||
stderrors "errors"
|
stderrors "errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"strings"
|
"strings"
|
||||||
"sync"
|
|
||||||
|
|
||||||
"github.com/aws/aws-sdk-go-v2/aws"
|
"github.com/aws/aws-sdk-go-v2/aws"
|
||||||
"github.com/aws/aws-sdk-go-v2/config"
|
"github.com/aws/aws-sdk-go-v2/config"
|
||||||
@@ -18,261 +14,210 @@ import (
|
|||||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||||
"github.com/aws/aws-sdk-go-v2/service/s3/types"
|
"github.com/aws/aws-sdk-go-v2/service/s3/types"
|
||||||
"github.com/lukaszraczylo/gohoarder/pkg/errors"
|
"github.com/lukaszraczylo/gohoarder/pkg/errors"
|
||||||
"github.com/lukaszraczylo/gohoarder/pkg/metrics"
|
|
||||||
"github.com/lukaszraczylo/gohoarder/pkg/storage"
|
"github.com/lukaszraczylo/gohoarder/pkg/storage"
|
||||||
"github.com/rs/zerolog/log"
|
"github.com/rs/zerolog/log"
|
||||||
)
|
)
|
||||||
|
|
||||||
// S3Storage implements storage.StorageBackend for AWS S3
|
// Config holds S3 storage configuration
|
||||||
type S3Storage struct {
|
|
||||||
client *s3.Client
|
|
||||||
bucket string
|
|
||||||
prefix string
|
|
||||||
quota int64
|
|
||||||
mu sync.RWMutex
|
|
||||||
used int64
|
|
||||||
}
|
|
||||||
|
|
||||||
// Config holds S3 configuration
|
|
||||||
type Config struct {
|
type Config struct {
|
||||||
Bucket string
|
|
||||||
Region string
|
Region string
|
||||||
Endpoint string // For S3-compatible services (MinIO, etc.)
|
Bucket string
|
||||||
|
Prefix string
|
||||||
AccessKeyID string
|
AccessKeyID string
|
||||||
SecretAccessKey string
|
SecretAccessKey string
|
||||||
Prefix string // Optional prefix for all keys
|
Endpoint string // Optional: for S3-compatible services like MinIO
|
||||||
Quota int64 // Quota in bytes (0 = unlimited)
|
ForcePathStyle bool // Optional: for S3-compatible services
|
||||||
ForcePathStyle bool // For S3-compatible services
|
MaxSizeBytes int64
|
||||||
|
}
|
||||||
|
|
||||||
|
// S3Storage implements storage.StorageBackend using AWS S3
|
||||||
|
type S3Storage struct {
|
||||||
|
client *s3.Client
|
||||||
|
bucket string
|
||||||
|
prefix string
|
||||||
|
maxSizeBytes int64
|
||||||
}
|
}
|
||||||
|
|
||||||
// New creates a new S3 storage backend
|
// New creates a new S3 storage backend
|
||||||
func New(ctx context.Context, cfg Config) (*S3Storage, error) {
|
func New(cfg Config) (*S3Storage, error) {
|
||||||
if cfg.Bucket == "" {
|
if cfg.Bucket == "" {
|
||||||
return nil, errors.New(errors.ErrCodeInvalidConfig, "S3 bucket is required")
|
return nil, fmt.Errorf("S3 bucket is required")
|
||||||
}
|
}
|
||||||
|
|
||||||
if cfg.Region == "" {
|
if cfg.Region == "" {
|
||||||
return nil, errors.New(errors.ErrCodeInvalidConfig, "S3 region is required")
|
cfg.Region = "us-east-1" // Default region
|
||||||
}
|
}
|
||||||
|
|
||||||
// Build AWS config
|
// Build AWS config
|
||||||
var awsCfg aws.Config
|
var awsConfig aws.Config
|
||||||
var err error
|
var err error
|
||||||
|
|
||||||
|
// Build config options
|
||||||
|
configOpts := []func(*config.LoadOptions) error{
|
||||||
|
config.WithRegion(cfg.Region),
|
||||||
|
}
|
||||||
|
|
||||||
|
// Add credentials if provided
|
||||||
if cfg.AccessKeyID != "" && cfg.SecretAccessKey != "" {
|
if cfg.AccessKeyID != "" && cfg.SecretAccessKey != "" {
|
||||||
// Use static credentials
|
configOpts = append(configOpts, config.WithCredentialsProvider(
|
||||||
awsCfg, err = config.LoadDefaultConfig(ctx,
|
credentials.NewStaticCredentialsProvider(
|
||||||
config.WithRegion(cfg.Region),
|
|
||||||
config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(
|
|
||||||
cfg.AccessKeyID,
|
cfg.AccessKeyID,
|
||||||
cfg.SecretAccessKey,
|
cfg.SecretAccessKey,
|
||||||
"",
|
"",
|
||||||
)),
|
),
|
||||||
)
|
))
|
||||||
} else {
|
|
||||||
// Use default credential chain
|
|
||||||
awsCfg, err = config.LoadDefaultConfig(ctx,
|
|
||||||
config.WithRegion(cfg.Region),
|
|
||||||
)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
awsConfig, err = config.LoadDefaultConfig(context.Background(), configOpts...)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, errors.Wrap(err, errors.ErrCodeStorageFailure, "failed to load AWS config")
|
return nil, fmt.Errorf("failed to load AWS config: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Create S3 client
|
// Create S3 client with service-specific options
|
||||||
var s3Options []func(*s3.Options)
|
client := s3.NewFromConfig(awsConfig, func(o *s3.Options) {
|
||||||
|
// Use custom endpoint if provided (for MinIO, S3-compatible services, etc.)
|
||||||
if cfg.Endpoint != "" {
|
if cfg.Endpoint != "" {
|
||||||
s3Options = append(s3Options, func(o *s3.Options) {
|
|
||||||
o.BaseEndpoint = aws.String(cfg.Endpoint)
|
o.BaseEndpoint = aws.String(cfg.Endpoint)
|
||||||
o.UsePathStyle = cfg.ForcePathStyle
|
}
|
||||||
})
|
if cfg.ForcePathStyle {
|
||||||
|
o.UsePathStyle = true
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
storage := &S3Storage{
|
||||||
|
client: client,
|
||||||
|
bucket: cfg.Bucket,
|
||||||
|
prefix: strings.TrimSuffix(cfg.Prefix, "/"),
|
||||||
|
maxSizeBytes: cfg.MaxSizeBytes,
|
||||||
}
|
}
|
||||||
|
|
||||||
client := s3.NewFromConfig(awsCfg, s3Options...)
|
log.Info().
|
||||||
|
Str("bucket", cfg.Bucket).
|
||||||
|
Str("region", cfg.Region).
|
||||||
|
Str("prefix", cfg.Prefix).
|
||||||
|
Msg("S3 storage initialized")
|
||||||
|
|
||||||
s3Storage := &S3Storage{
|
return storage, nil
|
||||||
client: client,
|
|
||||||
bucket: cfg.Bucket,
|
|
||||||
prefix: strings.TrimSuffix(cfg.Prefix, "/"),
|
|
||||||
quota: cfg.Quota,
|
|
||||||
}
|
|
||||||
|
|
||||||
// Calculate initial usage
|
|
||||||
if err := s3Storage.calculateUsage(ctx); err != nil {
|
|
||||||
log.Warn().Err(err).Msg("Failed to calculate initial S3 storage usage")
|
|
||||||
}
|
|
||||||
|
|
||||||
return s3Storage, nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Get retrieves a file from S3
|
// Get retrieves data from S3
|
||||||
func (s *S3Storage) Get(ctx context.Context, key string) (io.ReadCloser, error) {
|
func (s *S3Storage) Get(ctx context.Context, key string) (io.ReadCloser, error) {
|
||||||
s3Key := s.buildKey(key)
|
fullKey := s.buildKey(key)
|
||||||
|
|
||||||
input := &s3.GetObjectInput{
|
log.Debug().Str("key", fullKey).Msg("Getting object from S3")
|
||||||
|
|
||||||
|
result, err := s.client.GetObject(ctx, &s3.GetObjectInput{
|
||||||
Bucket: aws.String(s.bucket),
|
Bucket: aws.String(s.bucket),
|
||||||
Key: aws.String(s3Key),
|
Key: aws.String(fullKey),
|
||||||
}
|
})
|
||||||
|
|
||||||
result, err := s.client.GetObject(ctx, input)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if isNotFoundError(err) {
|
if isNotFoundError(err) {
|
||||||
metrics.RecordStorageOperation("s3", "get", "not_found")
|
return nil, errors.NotFound(fmt.Sprintf("S3 object not found: %s", key))
|
||||||
return nil, errors.NotFound(fmt.Sprintf("file not found: %s", key))
|
|
||||||
}
|
}
|
||||||
metrics.RecordStorageOperation("s3", "get", "error")
|
|
||||||
return nil, errors.Wrap(err, errors.ErrCodeStorageFailure, "failed to get object from S3")
|
return nil, errors.Wrap(err, errors.ErrCodeStorageFailure, "failed to get object from S3")
|
||||||
}
|
}
|
||||||
|
|
||||||
metrics.RecordStorageOperation("s3", "get", "success")
|
|
||||||
return result.Body, nil
|
return result.Body, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// Put stores a file in S3
|
// Put stores data in S3
|
||||||
func (s *S3Storage) Put(ctx context.Context, key string, data io.Reader, opts *storage.PutOptions) error {
|
func (s *S3Storage) Put(ctx context.Context, key string, data io.Reader, opts *storage.PutOptions) error {
|
||||||
s3Key := s.buildKey(key)
|
fullKey := s.buildKey(key)
|
||||||
|
|
||||||
// Read data into buffer to calculate checksums and size
|
// Read data into buffer to get size
|
||||||
var buf bytes.Buffer
|
buf := new(bytes.Buffer)
|
||||||
md5Hash := md5.New() // #nosec G401 -- MD5 used for S3 integrity check, not cryptographic security
|
size, err := io.Copy(buf, data)
|
||||||
sha256Hash := sha256.New()
|
|
||||||
multiWriter := io.MultiWriter(&buf, md5Hash, sha256Hash)
|
|
||||||
|
|
||||||
written, err := io.Copy(multiWriter, data)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
metrics.RecordStorageOperation("s3", "put", "error")
|
return fmt.Errorf("failed to read data: %w", err)
|
||||||
return errors.Wrap(err, errors.ErrCodeStorageFailure, "failed to read data")
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Check quota before upload
|
log.Debug().
|
||||||
if s.quota > 0 {
|
Str("key", fullKey).
|
||||||
s.mu.RLock()
|
Int64("size", size).
|
||||||
used := s.used
|
Msg("Putting object to S3")
|
||||||
s.mu.RUnlock()
|
|
||||||
|
|
||||||
if used+written > s.quota {
|
// Check quota if set
|
||||||
metrics.RecordStorageOperation("s3", "put", "quota_exceeded")
|
if s.maxSizeBytes > 0 {
|
||||||
return errors.QuotaExceeded(s.quota)
|
currentUsage, err := s.calculateUsage(ctx)
|
||||||
|
if err != nil {
|
||||||
|
log.Warn().Err(err).Msg("Failed to calculate current usage, skipping quota check")
|
||||||
|
} else if currentUsage+size > s.maxSizeBytes {
|
||||||
|
return errors.QuotaExceeded(s.maxSizeBytes)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Verify checksums if provided
|
// Convert metadata to S3 metadata format
|
||||||
if opts != nil {
|
s3Metadata := make(map[string]string)
|
||||||
md5Sum := hex.EncodeToString(md5Hash.Sum(nil))
|
|
||||||
sha256Sum := hex.EncodeToString(sha256Hash.Sum(nil))
|
|
||||||
|
|
||||||
if opts.ChecksumMD5 != "" && opts.ChecksumMD5 != md5Sum {
|
|
||||||
metrics.RecordStorageOperation("s3", "put", "checksum_error")
|
|
||||||
return errors.New(errors.ErrCodeChecksumMismatch, "MD5 checksum mismatch")
|
|
||||||
}
|
|
||||||
|
|
||||||
if opts.ChecksumSHA256 != "" && opts.ChecksumSHA256 != sha256Sum {
|
|
||||||
metrics.RecordStorageOperation("s3", "put", "checksum_error")
|
|
||||||
return errors.New(errors.ErrCodeChecksumMismatch, "SHA256 checksum mismatch")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Prepare metadata
|
|
||||||
metadata := make(map[string]string)
|
|
||||||
if opts != nil && opts.Metadata != nil {
|
if opts != nil && opts.Metadata != nil {
|
||||||
metadata = opts.Metadata
|
for k, v := range opts.Metadata {
|
||||||
}
|
s3Metadata[k] = v
|
||||||
|
}
|
||||||
// Build put input
|
|
||||||
input := &s3.PutObjectInput{
|
|
||||||
Bucket: aws.String(s.bucket),
|
|
||||||
Key: aws.String(s3Key),
|
|
||||||
Body: bytes.NewReader(buf.Bytes()),
|
|
||||||
Metadata: metadata,
|
|
||||||
}
|
|
||||||
|
|
||||||
if opts != nil && opts.ContentType != "" {
|
|
||||||
input.ContentType = aws.String(opts.ContentType)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Upload to S3
|
// Upload to S3
|
||||||
_, err = s.client.PutObject(ctx, input)
|
_, err = s.client.PutObject(ctx, &s3.PutObjectInput{
|
||||||
|
Bucket: aws.String(s.bucket),
|
||||||
|
Key: aws.String(fullKey),
|
||||||
|
Body: bytes.NewReader(buf.Bytes()),
|
||||||
|
Metadata: s3Metadata,
|
||||||
|
})
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
metrics.RecordStorageOperation("s3", "put", "error")
|
return errors.Wrap(err, errors.ErrCodeStorageFailure, "failed to put object to S3")
|
||||||
return errors.Wrap(err, errors.ErrCodeStorageFailure, "failed to upload to S3")
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Update usage
|
|
||||||
s.mu.Lock()
|
|
||||||
s.used += written
|
|
||||||
currentUsed := s.used
|
|
||||||
s.mu.Unlock()
|
|
||||||
|
|
||||||
metrics.RecordStorageOperation("s3", "put", "success")
|
|
||||||
metrics.UpdateCacheSize("s3", currentUsed)
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// Delete removes a file from S3
|
// Delete removes data from S3
|
||||||
func (s *S3Storage) Delete(ctx context.Context, key string) error {
|
func (s *S3Storage) Delete(ctx context.Context, key string) error {
|
||||||
s3Key := s.buildKey(key)
|
fullKey := s.buildKey(key)
|
||||||
|
|
||||||
// Get size before deletion for quota tracking
|
log.Debug().Str("key", fullKey).Msg("Deleting object from S3")
|
||||||
statInfo, err := s.Stat(ctx, key)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
input := &s3.DeleteObjectInput{
|
_, err := s.client.DeleteObject(ctx, &s3.DeleteObjectInput{
|
||||||
Bucket: aws.String(s.bucket),
|
Bucket: aws.String(s.bucket),
|
||||||
Key: aws.String(s3Key),
|
Key: aws.String(fullKey),
|
||||||
}
|
})
|
||||||
|
|
||||||
_, err = s.client.DeleteObject(ctx, input)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
metrics.RecordStorageOperation("s3", "delete", "error")
|
return errors.Wrap(err, errors.ErrCodeStorageFailure, "failed to delete object from S3")
|
||||||
return errors.Wrap(err, errors.ErrCodeStorageFailure, "failed to delete from S3")
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Update usage
|
|
||||||
s.mu.Lock()
|
|
||||||
s.used -= statInfo.Size
|
|
||||||
currentUsed := s.used
|
|
||||||
s.mu.Unlock()
|
|
||||||
|
|
||||||
metrics.RecordStorageOperation("s3", "delete", "success")
|
|
||||||
metrics.UpdateCacheSize("s3", currentUsed)
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// Exists checks if a file exists in S3
|
// Exists checks if data exists in S3
|
||||||
func (s *S3Storage) Exists(ctx context.Context, key string) (bool, error) {
|
func (s *S3Storage) Exists(ctx context.Context, key string) (bool, error) {
|
||||||
s3Key := s.buildKey(key)
|
fullKey := s.buildKey(key)
|
||||||
|
|
||||||
input := &s3.HeadObjectInput{
|
_, err := s.client.HeadObject(ctx, &s3.HeadObjectInput{
|
||||||
Bucket: aws.String(s.bucket),
|
Bucket: aws.String(s.bucket),
|
||||||
Key: aws.String(s3Key),
|
Key: aws.String(fullKey),
|
||||||
}
|
})
|
||||||
|
|
||||||
_, err := s.client.HeadObject(ctx, input)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if isNotFoundError(err) {
|
if isNotFoundError(err) {
|
||||||
return false, nil
|
return false, nil
|
||||||
}
|
}
|
||||||
return false, errors.Wrap(err, errors.ErrCodeStorageFailure, "failed to check existence in S3")
|
return false, errors.Wrap(err, errors.ErrCodeStorageFailure, "failed to check object existence in S3")
|
||||||
}
|
}
|
||||||
|
|
||||||
return true, nil
|
return true, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// List lists files with prefix in S3
|
// List returns a list of objects with the given prefix
|
||||||
func (s *S3Storage) List(ctx context.Context, prefix string, opts *storage.ListOptions) ([]storage.StorageObject, error) {
|
func (s *S3Storage) List(ctx context.Context, prefix string, opts *storage.ListOptions) ([]storage.StorageObject, error) {
|
||||||
s3Prefix := s.buildKey(prefix)
|
fullPrefix := s.buildKey(prefix)
|
||||||
|
|
||||||
input := &s3.ListObjectsV2Input{
|
log.Debug().Str("prefix", fullPrefix).Msg("Listing objects in S3")
|
||||||
Bucket: aws.String(s.bucket),
|
|
||||||
Prefix: aws.String(s3Prefix),
|
|
||||||
}
|
|
||||||
|
|
||||||
var objects []storage.StorageObject
|
var objects []storage.StorageObject
|
||||||
paginator := s3.NewListObjectsV2Paginator(s.client, input)
|
paginator := s3.NewListObjectsV2Paginator(s.client, &s3.ListObjectsV2Input{
|
||||||
|
Bucket: aws.String(s.bucket),
|
||||||
|
Prefix: aws.String(fullPrefix),
|
||||||
|
})
|
||||||
|
|
||||||
for paginator.HasMorePages() {
|
for paginator.HasMorePages() {
|
||||||
page, err := paginator.NextPage(ctx)
|
page, err := paginator.NextPage(ctx)
|
||||||
@@ -281,56 +226,58 @@ func (s *S3Storage) List(ctx context.Context, prefix string, opts *storage.ListO
|
|||||||
}
|
}
|
||||||
|
|
||||||
for _, obj := range page.Contents {
|
for _, obj := range page.Contents {
|
||||||
key := s.stripPrefix(*obj.Key)
|
if obj.Key != nil {
|
||||||
objects = append(objects, storage.StorageObject{
|
// Strip prefix from key
|
||||||
Key: key,
|
key := s.stripPrefix(*obj.Key)
|
||||||
Size: *obj.Size,
|
|
||||||
Modified: *obj.LastModified,
|
|
||||||
ETag: strings.Trim(*obj.ETag, "\""),
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Apply pagination if requested
|
object := storage.StorageObject{
|
||||||
if opts != nil {
|
Key: key,
|
||||||
start := opts.Offset
|
Size: aws.ToInt64(obj.Size),
|
||||||
end := len(objects)
|
}
|
||||||
if opts.MaxResults > 0 && start+opts.MaxResults < end {
|
|
||||||
end = start + opts.MaxResults
|
if obj.LastModified != nil {
|
||||||
}
|
object.Modified = *obj.LastModified
|
||||||
if start < len(objects) {
|
}
|
||||||
objects = objects[start:end]
|
|
||||||
} else {
|
if obj.ETag != nil {
|
||||||
objects = []storage.StorageObject{}
|
object.ETag = *obj.ETag
|
||||||
|
}
|
||||||
|
|
||||||
|
objects = append(objects, object)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return objects, nil
|
return objects, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// Stat gets file metadata from S3
|
// Stat returns metadata about stored data
|
||||||
func (s *S3Storage) Stat(ctx context.Context, key string) (*storage.StorageInfo, error) {
|
func (s *S3Storage) Stat(ctx context.Context, key string) (*storage.StorageInfo, error) {
|
||||||
s3Key := s.buildKey(key)
|
fullKey := s.buildKey(key)
|
||||||
|
|
||||||
input := &s3.HeadObjectInput{
|
result, err := s.client.HeadObject(ctx, &s3.HeadObjectInput{
|
||||||
Bucket: aws.String(s.bucket),
|
Bucket: aws.String(s.bucket),
|
||||||
Key: aws.String(s3Key),
|
Key: aws.String(fullKey),
|
||||||
}
|
})
|
||||||
|
|
||||||
result, err := s.client.HeadObject(ctx, input)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if isNotFoundError(err) {
|
if isNotFoundError(err) {
|
||||||
return nil, errors.NotFound(fmt.Sprintf("file not found: %s", key))
|
return nil, errors.NotFound(fmt.Sprintf("S3 object not found: %s", key))
|
||||||
}
|
}
|
||||||
return nil, errors.Wrap(err, errors.ErrCodeStorageFailure, "failed to stat object in S3")
|
return nil, errors.Wrap(err, errors.ErrCodeStorageFailure, "failed to stat object in S3")
|
||||||
}
|
}
|
||||||
|
|
||||||
info := &storage.StorageInfo{
|
info := &storage.StorageInfo{
|
||||||
Key: key,
|
Key: key,
|
||||||
Size: *result.ContentLength,
|
Size: aws.ToInt64(result.ContentLength),
|
||||||
Modified: *result.LastModified,
|
}
|
||||||
ETag: strings.Trim(*result.ETag, "\""),
|
|
||||||
Metadata: result.Metadata,
|
if result.LastModified != nil {
|
||||||
|
info.Modified = *result.LastModified
|
||||||
|
}
|
||||||
|
|
||||||
|
if result.ETag != nil {
|
||||||
|
info.ETag = *result.ETag
|
||||||
}
|
}
|
||||||
|
|
||||||
if result.ContentType != nil {
|
if result.ContentType != nil {
|
||||||
@@ -340,33 +287,27 @@ func (s *S3Storage) Stat(ctx context.Context, key string) (*storage.StorageInfo,
|
|||||||
return info, nil
|
return info, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// GetQuota returns quota information
|
// GetQuota returns current usage and quota information
|
||||||
func (s *S3Storage) GetQuota(ctx context.Context) (*storage.QuotaInfo, error) {
|
func (s *S3Storage) GetQuota(ctx context.Context) (*storage.QuotaInfo, error) {
|
||||||
s.mu.RLock()
|
usage, err := s.calculateUsage(ctx)
|
||||||
used := s.used
|
if err != nil {
|
||||||
s.mu.RUnlock()
|
return nil, err
|
||||||
|
|
||||||
available := s.quota - used
|
|
||||||
if available < 0 {
|
|
||||||
available = 0
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return &storage.QuotaInfo{
|
return &storage.QuotaInfo{
|
||||||
Used: used,
|
Used: usage,
|
||||||
Available: available,
|
Limit: s.maxSizeBytes,
|
||||||
Limit: s.quota,
|
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// Health checks S3 health
|
// Health checks if the S3 backend is healthy
|
||||||
func (s *S3Storage) Health(ctx context.Context) error {
|
func (s *S3Storage) Health(ctx context.Context) error {
|
||||||
// Try to list bucket to verify connectivity
|
// Try to list objects (lightweight operation)
|
||||||
input := &s3.ListObjectsV2Input{
|
_, err := s.client.ListObjectsV2(ctx, &s3.ListObjectsV2Input{
|
||||||
Bucket: aws.String(s.bucket),
|
Bucket: aws.String(s.bucket),
|
||||||
MaxKeys: aws.Int32(1),
|
MaxKeys: aws.Int32(1),
|
||||||
}
|
})
|
||||||
|
|
||||||
_, err := s.client.ListObjectsV2(ctx, input)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return errors.Wrap(err, errors.ErrCodeStorageFailure, "S3 health check failed")
|
return errors.Wrap(err, errors.ErrCodeStorageFailure, "S3 health check failed")
|
||||||
}
|
}
|
||||||
@@ -374,60 +315,51 @@ func (s *S3Storage) Health(ctx context.Context) error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// Close closes the storage backend
|
// Close closes the S3 storage backend
|
||||||
func (s *S3Storage) Close() error {
|
func (s *S3Storage) Close() error {
|
||||||
// No cleanup needed for S3 client
|
log.Info().Msg("S3 storage closed")
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// buildKey builds the full S3 key with prefix
|
// buildKey constructs the full S3 key with prefix
|
||||||
func (s *S3Storage) buildKey(key string) string {
|
func (s *S3Storage) buildKey(key string) string {
|
||||||
key = strings.TrimPrefix(key, "/")
|
if s.prefix == "" {
|
||||||
if s.prefix != "" {
|
return key
|
||||||
return s.prefix + "/" + key
|
|
||||||
}
|
}
|
||||||
return key
|
return s.prefix + "/" + key
|
||||||
}
|
}
|
||||||
|
|
||||||
// stripPrefix removes the configured prefix from an S3 key
|
// stripPrefix removes the prefix from an S3 key
|
||||||
func (s *S3Storage) stripPrefix(s3Key string) string {
|
func (s *S3Storage) stripPrefix(key string) string {
|
||||||
if s.prefix != "" {
|
if s.prefix == "" {
|
||||||
return strings.TrimPrefix(s3Key, s.prefix+"/")
|
return key
|
||||||
}
|
}
|
||||||
return s3Key
|
return strings.TrimPrefix(key, s.prefix+"/")
|
||||||
}
|
}
|
||||||
|
|
||||||
// calculateUsage calculates current S3 storage usage
|
// calculateUsage calculates total storage usage
|
||||||
func (s *S3Storage) calculateUsage(ctx context.Context) error {
|
func (s *S3Storage) calculateUsage(ctx context.Context) (int64, error) {
|
||||||
var total int64
|
var totalSize int64
|
||||||
|
|
||||||
input := &s3.ListObjectsV2Input{
|
paginator := s3.NewListObjectsV2Paginator(s.client, &s3.ListObjectsV2Input{
|
||||||
Bucket: aws.String(s.bucket),
|
Bucket: aws.String(s.bucket),
|
||||||
}
|
Prefix: aws.String(s.prefix),
|
||||||
|
})
|
||||||
if s.prefix != "" {
|
|
||||||
input.Prefix = aws.String(s.prefix + "/")
|
|
||||||
}
|
|
||||||
|
|
||||||
paginator := s3.NewListObjectsV2Paginator(s.client, input)
|
|
||||||
|
|
||||||
for paginator.HasMorePages() {
|
for paginator.HasMorePages() {
|
||||||
page, err := paginator.NextPage(ctx)
|
page, err := paginator.NextPage(ctx)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return 0, fmt.Errorf("failed to calculate usage: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, obj := range page.Contents {
|
for _, obj := range page.Contents {
|
||||||
total += *obj.Size
|
if obj.Size != nil {
|
||||||
|
totalSize += aws.ToInt64(obj.Size)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
s.mu.Lock()
|
return totalSize, nil
|
||||||
s.used = total
|
|
||||||
s.mu.Unlock()
|
|
||||||
|
|
||||||
metrics.UpdateCacheSize("s3", total)
|
|
||||||
return nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// isNotFoundError checks if an error is a "not found" error
|
// isNotFoundError checks if an error is a "not found" error
|
||||||
@@ -436,8 +368,21 @@ func isNotFoundError(err error) bool {
|
|||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Check for specific S3 error types
|
||||||
var notFound *types.NotFound
|
var notFound *types.NotFound
|
||||||
var noSuchKey *types.NoSuchKey
|
var noSuchKey *types.NoSuchKey
|
||||||
|
|
||||||
return stderrors.As(err, ¬Found) || stderrors.As(err, &noSuchKey)
|
// Use errors.As to check for wrapped errors
|
||||||
|
if ok := stderrors.As(err, ¬Found); ok {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
if ok := stderrors.As(err, &noSuchKey); ok {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check error message as fallback
|
||||||
|
errMsg := err.Error()
|
||||||
|
return strings.Contains(errMsg, "NoSuchKey") ||
|
||||||
|
strings.Contains(errMsg, "NotFound") ||
|
||||||
|
strings.Contains(errMsg, "404")
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,271 @@
|
|||||||
|
package s3
|
||||||
|
|
||||||
|
import (
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/stretchr/testify/suite"
|
||||||
|
)
|
||||||
|
|
||||||
|
type S3StorageTestSuite struct {
|
||||||
|
suite.Suite
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestS3StorageTestSuite(t *testing.T) {
|
||||||
|
suite.Run(t, new(S3StorageTestSuite))
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *S3StorageTestSuite) TestNewS3Storage() {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
config Config
|
||||||
|
expectError bool
|
||||||
|
errorMsg string
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "valid config with credentials",
|
||||||
|
config: Config{
|
||||||
|
Region: "us-east-1",
|
||||||
|
Bucket: "test-bucket",
|
||||||
|
Prefix: "packages/",
|
||||||
|
AccessKeyID: "AKIAIOSFODNN7EXAMPLE",
|
||||||
|
SecretAccessKey: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY",
|
||||||
|
MaxSizeBytes: 1024 * 1024,
|
||||||
|
},
|
||||||
|
expectError: false,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "valid config with custom endpoint",
|
||||||
|
config: Config{
|
||||||
|
Region: "us-east-1",
|
||||||
|
Bucket: "test-bucket",
|
||||||
|
Endpoint: "https://minio.example.com",
|
||||||
|
AccessKeyID: "minioadmin",
|
||||||
|
SecretAccessKey: "minioadmin",
|
||||||
|
ForcePathStyle: true,
|
||||||
|
},
|
||||||
|
expectError: false,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "valid config with default region",
|
||||||
|
config: Config{
|
||||||
|
Bucket: "test-bucket",
|
||||||
|
AccessKeyID: "test",
|
||||||
|
SecretAccessKey: "test",
|
||||||
|
},
|
||||||
|
expectError: false,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "missing bucket",
|
||||||
|
config: Config{
|
||||||
|
Region: "us-east-1",
|
||||||
|
AccessKeyID: "test",
|
||||||
|
SecretAccessKey: "test",
|
||||||
|
},
|
||||||
|
expectError: true,
|
||||||
|
errorMsg: "bucket is required",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
s.Run(tt.name, func() {
|
||||||
|
storage, err := New(tt.config)
|
||||||
|
|
||||||
|
if tt.expectError {
|
||||||
|
s.Error(err)
|
||||||
|
if tt.errorMsg != "" {
|
||||||
|
s.Contains(err.Error(), tt.errorMsg)
|
||||||
|
}
|
||||||
|
s.Nil(storage)
|
||||||
|
} else {
|
||||||
|
s.NoError(err)
|
||||||
|
s.NotNil(storage)
|
||||||
|
s.Equal(tt.config.Bucket, storage.bucket)
|
||||||
|
s.Equal(tt.config.MaxSizeBytes, storage.maxSizeBytes)
|
||||||
|
|
||||||
|
// Test prefix normalization
|
||||||
|
if tt.config.Prefix != "" {
|
||||||
|
s.NotContains(storage.prefix, "/", "prefix should not end with /")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *S3StorageTestSuite) TestBuildKey() {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
prefix string
|
||||||
|
key string
|
||||||
|
expected string
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "with prefix",
|
||||||
|
prefix: "packages",
|
||||||
|
key: "test/file.txt",
|
||||||
|
expected: "packages/test/file.txt",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "without prefix",
|
||||||
|
prefix: "",
|
||||||
|
key: "test/file.txt",
|
||||||
|
expected: "test/file.txt",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "with trailing slash in prefix",
|
||||||
|
prefix: "packages/",
|
||||||
|
key: "test/file.txt",
|
||||||
|
expected: "packages/test/file.txt",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
s.Run(tt.name, func() {
|
||||||
|
storage := &S3Storage{
|
||||||
|
prefix: tt.prefix,
|
||||||
|
}
|
||||||
|
// Normalize prefix like in New()
|
||||||
|
if storage.prefix != "" && storage.prefix[len(storage.prefix)-1] == '/' {
|
||||||
|
storage.prefix = storage.prefix[:len(storage.prefix)-1]
|
||||||
|
}
|
||||||
|
|
||||||
|
result := storage.buildKey(tt.key)
|
||||||
|
s.Equal(tt.expected, result)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *S3StorageTestSuite) TestStripPrefix() {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
prefix string
|
||||||
|
key string
|
||||||
|
expected string
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "with prefix",
|
||||||
|
prefix: "packages",
|
||||||
|
key: "packages/test/file.txt",
|
||||||
|
expected: "test/file.txt",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "without prefix",
|
||||||
|
prefix: "",
|
||||||
|
key: "test/file.txt",
|
||||||
|
expected: "test/file.txt",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "key without prefix but prefix set",
|
||||||
|
prefix: "packages",
|
||||||
|
key: "test/file.txt",
|
||||||
|
expected: "test/file.txt",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
s.Run(tt.name, func() {
|
||||||
|
storage := &S3Storage{
|
||||||
|
prefix: tt.prefix,
|
||||||
|
}
|
||||||
|
|
||||||
|
result := storage.stripPrefix(tt.key)
|
||||||
|
s.Equal(tt.expected, result)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *S3StorageTestSuite) TestIsNotFoundError() {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
err error
|
||||||
|
expected bool
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "nil error",
|
||||||
|
err: nil,
|
||||||
|
expected: false,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
s.Run(tt.name, func() {
|
||||||
|
result := isNotFoundError(tt.err)
|
||||||
|
s.Equal(tt.expected, result)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *S3StorageTestSuite) TestConfigDefaults() {
|
||||||
|
config := Config{
|
||||||
|
Bucket: "test-bucket",
|
||||||
|
AccessKeyID: "test",
|
||||||
|
SecretAccessKey: "test",
|
||||||
|
}
|
||||||
|
|
||||||
|
storage, err := New(config)
|
||||||
|
s.Require().NoError(err)
|
||||||
|
s.NotNil(storage)
|
||||||
|
|
||||||
|
// Verify defaults
|
||||||
|
s.Equal("test-bucket", storage.bucket)
|
||||||
|
s.Equal("", storage.prefix)
|
||||||
|
s.Equal(int64(0), storage.maxSizeBytes)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *S3StorageTestSuite) TestPrefixNormalization() {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
inputPrefix string
|
||||||
|
expectedPrefix string
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "prefix with trailing slash",
|
||||||
|
inputPrefix: "packages/",
|
||||||
|
expectedPrefix: "packages",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "prefix without trailing slash",
|
||||||
|
inputPrefix: "packages",
|
||||||
|
expectedPrefix: "packages",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "empty prefix",
|
||||||
|
inputPrefix: "",
|
||||||
|
expectedPrefix: "",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "nested prefix with trailing slash",
|
||||||
|
inputPrefix: "cache/packages/",
|
||||||
|
expectedPrefix: "cache/packages",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
s.Run(tt.name, func() {
|
||||||
|
config := Config{
|
||||||
|
Bucket: "test-bucket",
|
||||||
|
Prefix: tt.inputPrefix,
|
||||||
|
AccessKeyID: "test",
|
||||||
|
SecretAccessKey: "test",
|
||||||
|
}
|
||||||
|
|
||||||
|
storage, err := New(config)
|
||||||
|
s.Require().NoError(err)
|
||||||
|
s.Equal(tt.expectedPrefix, storage.prefix)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *S3StorageTestSuite) TestClose() {
|
||||||
|
config := Config{
|
||||||
|
Bucket: "test-bucket",
|
||||||
|
AccessKeyID: "test",
|
||||||
|
SecretAccessKey: "test",
|
||||||
|
}
|
||||||
|
|
||||||
|
storage, err := New(config)
|
||||||
|
s.Require().NoError(err)
|
||||||
|
|
||||||
|
// Close should not error
|
||||||
|
err = storage.Close()
|
||||||
|
s.NoError(err)
|
||||||
|
}
|
||||||
+266
-303
@@ -1,42 +1,43 @@
|
|||||||
package smb
|
package smb
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"bytes"
|
|
||||||
"context"
|
"context"
|
||||||
"crypto/md5" // #nosec G501 -- MD5 used for file checksums, not cryptographic security
|
|
||||||
"crypto/sha256"
|
|
||||||
"encoding/hex"
|
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"net"
|
"net"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"strings"
|
"strings"
|
||||||
"sync"
|
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/hirochachacha/go-smb2"
|
"github.com/hirochachacha/go-smb2"
|
||||||
"github.com/lukaszraczylo/gohoarder/pkg/errors"
|
"github.com/lukaszraczylo/gohoarder/pkg/errors"
|
||||||
"github.com/lukaszraczylo/gohoarder/pkg/metrics"
|
|
||||||
"github.com/lukaszraczylo/gohoarder/pkg/storage"
|
"github.com/lukaszraczylo/gohoarder/pkg/storage"
|
||||||
"github.com/rs/zerolog/log"
|
"github.com/rs/zerolog/log"
|
||||||
)
|
)
|
||||||
|
|
||||||
// SMBStorage implements storage.StorageBackend for SMB/CIFS shares
|
// Config holds SMB storage configuration
|
||||||
type SMBStorage struct {
|
type Config struct {
|
||||||
host string
|
Host string
|
||||||
share string
|
Share string
|
||||||
basePath string
|
Path string
|
||||||
username string
|
Username string
|
||||||
password string
|
Password string
|
||||||
quota int64
|
Domain string
|
||||||
mu sync.RWMutex
|
Port int
|
||||||
used int64
|
MaxSizeBytes int64
|
||||||
connPool chan *smbConnection
|
PoolSize int
|
||||||
poolSize int
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// smbConnection wraps an SMB session and share
|
// SMBStorage implements storage.StorageBackend using SMB/CIFS
|
||||||
|
type SMBStorage struct {
|
||||||
|
connPool chan *smbConnection
|
||||||
|
config Config
|
||||||
|
maxSizeBytes int64
|
||||||
|
poolSize int
|
||||||
|
}
|
||||||
|
|
||||||
|
// smbConnection represents a pooled SMB connection
|
||||||
type smbConnection struct {
|
type smbConnection struct {
|
||||||
conn net.Conn
|
conn net.Conn
|
||||||
session *smb2.Session
|
session *smb2.Session
|
||||||
@@ -44,27 +45,14 @@ type smbConnection struct {
|
|||||||
lastUse time.Time
|
lastUse time.Time
|
||||||
}
|
}
|
||||||
|
|
||||||
// Config holds SMB configuration
|
|
||||||
type Config struct {
|
|
||||||
Host string // SMB server hostname or IP
|
|
||||||
Port int // SMB server port (default: 445)
|
|
||||||
Share string // SMB share name
|
|
||||||
BasePath string // Base path within the share
|
|
||||||
Username string // SMB username
|
|
||||||
Password string // SMB password
|
|
||||||
Domain string // SMB domain (optional)
|
|
||||||
Quota int64 // Quota in bytes (0 = unlimited)
|
|
||||||
PoolSize int // Connection pool size (default: 5)
|
|
||||||
}
|
|
||||||
|
|
||||||
// New creates a new SMB storage backend
|
// New creates a new SMB storage backend
|
||||||
func New(ctx context.Context, cfg Config) (*SMBStorage, error) {
|
func New(cfg Config) (*SMBStorage, error) {
|
||||||
if cfg.Host == "" {
|
if cfg.Host == "" {
|
||||||
return nil, errors.New(errors.ErrCodeInvalidConfig, "SMB host is required")
|
return nil, fmt.Errorf("SMB host is required")
|
||||||
}
|
}
|
||||||
|
|
||||||
if cfg.Share == "" {
|
if cfg.Share == "" {
|
||||||
return nil, errors.New(errors.ErrCodeInvalidConfig, "SMB share is required")
|
return nil, fmt.Errorf("SMB share is required")
|
||||||
}
|
}
|
||||||
|
|
||||||
if cfg.Port == 0 {
|
if cfg.Port == 0 {
|
||||||
@@ -75,64 +63,68 @@ func New(ctx context.Context, cfg Config) (*SMBStorage, error) {
|
|||||||
cfg.PoolSize = 5 // Default pool size
|
cfg.PoolSize = 5 // Default pool size
|
||||||
}
|
}
|
||||||
|
|
||||||
smbStorage := &SMBStorage{
|
// Normalize path
|
||||||
host: fmt.Sprintf("%s:%d", cfg.Host, cfg.Port),
|
cfg.Path = strings.Trim(cfg.Path, "/\\")
|
||||||
share: cfg.Share,
|
|
||||||
basePath: strings.Trim(cfg.BasePath, "/\\"),
|
storage := &SMBStorage{
|
||||||
username: cfg.Username,
|
config: cfg,
|
||||||
password: cfg.Password,
|
maxSizeBytes: cfg.MaxSizeBytes,
|
||||||
quota: cfg.Quota,
|
poolSize: cfg.PoolSize,
|
||||||
connPool: make(chan *smbConnection, cfg.PoolSize),
|
connPool: make(chan *smbConnection, cfg.PoolSize),
|
||||||
poolSize: cfg.PoolSize,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Initialize connection pool
|
// Pre-populate connection pool
|
||||||
for i := 0; i < cfg.PoolSize; i++ {
|
for i := 0; i < cfg.PoolSize; i++ {
|
||||||
conn, err := smbStorage.createConnection(ctx)
|
conn, err := storage.createConnection()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
// Clean up any created connections
|
log.Warn().Err(err).Int("attempt", i).Msg("Failed to create initial SMB connection")
|
||||||
close(smbStorage.connPool)
|
continue
|
||||||
for c := range smbStorage.connPool {
|
|
||||||
c.close()
|
|
||||||
}
|
|
||||||
return nil, errors.Wrap(err, errors.ErrCodeStorageFailure, "failed to create SMB connection pool")
|
|
||||||
}
|
}
|
||||||
smbStorage.connPool <- conn
|
storage.connPool <- conn
|
||||||
}
|
}
|
||||||
|
|
||||||
// Calculate initial usage
|
log.Info().
|
||||||
if err := smbStorage.calculateUsage(ctx); err != nil {
|
Str("host", cfg.Host).
|
||||||
log.Warn().Err(err).Msg("Failed to calculate initial SMB storage usage")
|
Int("port", cfg.Port).
|
||||||
}
|
Str("share", cfg.Share).
|
||||||
|
Str("path", cfg.Path).
|
||||||
|
Int("pool_size", cfg.PoolSize).
|
||||||
|
Msg("SMB storage initialized")
|
||||||
|
|
||||||
return smbStorage, nil
|
return storage, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// createConnection creates a new SMB connection
|
// createConnection creates a new SMB connection
|
||||||
func (s *SMBStorage) createConnection(ctx context.Context) (*smbConnection, error) {
|
func (s *SMBStorage) createConnection() (*smbConnection, error) {
|
||||||
conn, err := net.Dial("tcp", s.host)
|
// Connect to SMB server (use net.JoinHostPort for IPv6 compatibility)
|
||||||
|
addr := net.JoinHostPort(s.config.Host, fmt.Sprintf("%d", s.config.Port))
|
||||||
|
conn, err := net.DialTimeout("tcp", addr, 10*time.Second)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, fmt.Errorf("failed to connect to SMB server: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
dialer := &smb2.Dialer{
|
// Create SMB dialer
|
||||||
|
d := &smb2.Dialer{
|
||||||
Initiator: &smb2.NTLMInitiator{
|
Initiator: &smb2.NTLMInitiator{
|
||||||
User: s.username,
|
User: s.config.Username,
|
||||||
Password: s.password,
|
Password: s.config.Password,
|
||||||
|
Domain: s.config.Domain,
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
session, err := dialer.Dial(conn)
|
// Establish SMB session
|
||||||
|
session, err := d.Dial(conn)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
conn.Close() // #nosec G104 -- Cleanup, error not critical
|
_ = conn.Close()
|
||||||
return nil, err
|
return nil, fmt.Errorf("failed to establish SMB session: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
share, err := session.Mount(s.share)
|
// Mount share
|
||||||
|
share, err := session.Mount(s.config.Share)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
_ = session.Logoff() // #nosec G104 -- SMB cleanup
|
_ = session.Logoff()
|
||||||
conn.Close() // #nosec G104 -- Cleanup, error not critical
|
_ = conn.Close()
|
||||||
return nil, err
|
return nil, fmt.Errorf("failed to mount SMB share: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
return &smbConnection{
|
return &smbConnection{
|
||||||
@@ -143,25 +135,34 @@ func (s *SMBStorage) createConnection(ctx context.Context) (*smbConnection, erro
|
|||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// getConnection gets a connection from the pool
|
// getConnection gets a connection from the pool or creates a new one
|
||||||
func (s *SMBStorage) getConnection(ctx context.Context) (*smbConnection, error) {
|
func (s *SMBStorage) getConnection() (*smbConnection, error) {
|
||||||
select {
|
select {
|
||||||
case conn := <-s.connPool:
|
case conn := <-s.connPool:
|
||||||
|
// Check if connection is still valid (not older than 5 minutes idle)
|
||||||
|
if time.Since(conn.lastUse) > 5*time.Minute {
|
||||||
|
conn.close()
|
||||||
|
return s.createConnection()
|
||||||
|
}
|
||||||
conn.lastUse = time.Now()
|
conn.lastUse = time.Now()
|
||||||
return conn, nil
|
return conn, nil
|
||||||
case <-ctx.Done():
|
default:
|
||||||
return nil, ctx.Err()
|
// Pool is empty, create new connection
|
||||||
case <-time.After(30 * time.Second):
|
return s.createConnection()
|
||||||
return nil, errors.New(errors.ErrCodeStorageFailure, "timeout waiting for SMB connection")
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// returnConnection returns a connection to the pool
|
// returnConnection returns a connection to the pool
|
||||||
func (s *SMBStorage) returnConnection(conn *smbConnection) {
|
func (s *SMBStorage) returnConnection(conn *smbConnection) {
|
||||||
|
if conn == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
select {
|
select {
|
||||||
case s.connPool <- conn:
|
case s.connPool <- conn:
|
||||||
|
// Successfully returned to pool
|
||||||
default:
|
default:
|
||||||
// Pool is full, close the connection
|
// Pool is full, close connection
|
||||||
conn.close()
|
conn.close()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -169,189 +170,161 @@ func (s *SMBStorage) returnConnection(conn *smbConnection) {
|
|||||||
// close closes an SMB connection
|
// close closes an SMB connection
|
||||||
func (c *smbConnection) close() {
|
func (c *smbConnection) close() {
|
||||||
if c.share != nil {
|
if c.share != nil {
|
||||||
_ = c.share.Umount() // #nosec G104 -- SMB cleanup
|
if err := c.share.Umount(); err != nil {
|
||||||
|
log.Warn().Err(err).Msg("Failed to unmount SMB share")
|
||||||
|
}
|
||||||
}
|
}
|
||||||
if c.session != nil {
|
if c.session != nil {
|
||||||
_ = c.session.Logoff() // #nosec G104 -- SMB cleanup
|
if err := c.session.Logoff(); err != nil {
|
||||||
|
log.Warn().Err(err).Msg("Failed to logoff SMB session")
|
||||||
|
}
|
||||||
}
|
}
|
||||||
if c.conn != nil {
|
if c.conn != nil {
|
||||||
c.conn.Close() // #nosec G104 -- Cleanup, error not critical
|
if err := c.conn.Close(); err != nil {
|
||||||
|
log.Warn().Err(err).Msg("Failed to close SMB connection")
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Get retrieves a file from SMB share
|
// Get retrieves data from SMB share
|
||||||
func (s *SMBStorage) Get(ctx context.Context, key string) (io.ReadCloser, error) {
|
func (s *SMBStorage) Get(ctx context.Context, key string) (io.ReadCloser, error) {
|
||||||
conn, err := s.getConnection(ctx)
|
conn, err := s.getConnection()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, errors.Wrap(err, errors.ErrCodeStorageFailure, "failed to get SMB connection")
|
||||||
}
|
}
|
||||||
|
defer s.returnConnection(conn)
|
||||||
|
|
||||||
path := s.keyToPath(key)
|
path := s.keyToPath(key)
|
||||||
|
|
||||||
|
log.Debug().Str("key", path).Msg("Getting file from SMB")
|
||||||
|
|
||||||
// Open file
|
// Open file
|
||||||
file, err := conn.share.Open(path)
|
file, err := conn.share.Open(path)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
s.returnConnection(conn)
|
|
||||||
if os.IsNotExist(err) {
|
if os.IsNotExist(err) {
|
||||||
metrics.RecordStorageOperation("smb", "get", "not_found")
|
return nil, errors.NotFound(fmt.Sprintf("SMB file not found: %s", key))
|
||||||
return nil, errors.NotFound(fmt.Sprintf("file not found: %s", key))
|
|
||||||
}
|
}
|
||||||
metrics.RecordStorageOperation("smb", "get", "error")
|
|
||||||
return nil, errors.Wrap(err, errors.ErrCodeStorageFailure, "failed to open SMB file")
|
return nil, errors.Wrap(err, errors.ErrCodeStorageFailure, "failed to open SMB file")
|
||||||
}
|
}
|
||||||
|
|
||||||
// Read entire file into memory and close SMB connection
|
// Read entire file into memory (SMB files must be read completely before closing connection)
|
||||||
// This is necessary because we need to return the connection to the pool
|
|
||||||
data, err := io.ReadAll(file)
|
data, err := io.ReadAll(file)
|
||||||
file.Close() // #nosec G104 -- Cleanup, error not critical
|
if closeErr := file.Close(); closeErr != nil {
|
||||||
s.returnConnection(conn)
|
log.Warn().Err(closeErr).Str("path", path).Msg("Failed to close SMB file after reading")
|
||||||
|
}
|
||||||
if err != nil {
|
if err != nil {
|
||||||
metrics.RecordStorageOperation("smb", "get", "error")
|
|
||||||
return nil, errors.Wrap(err, errors.ErrCodeStorageFailure, "failed to read SMB file")
|
return nil, errors.Wrap(err, errors.ErrCodeStorageFailure, "failed to read SMB file")
|
||||||
}
|
}
|
||||||
|
|
||||||
metrics.RecordStorageOperation("smb", "get", "success")
|
// Return as ReadCloser
|
||||||
return io.NopCloser(bytes.NewReader(data)), nil
|
return io.NopCloser(strings.NewReader(string(data))), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// Put stores a file on SMB share
|
// Put stores data on SMB share
|
||||||
func (s *SMBStorage) Put(ctx context.Context, key string, data io.Reader, opts *storage.PutOptions) error {
|
func (s *SMBStorage) Put(ctx context.Context, key string, data io.Reader, opts *storage.PutOptions) error {
|
||||||
conn, err := s.getConnection(ctx)
|
conn, err := s.getConnection()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return errors.Wrap(err, errors.ErrCodeStorageFailure, "failed to get SMB connection")
|
||||||
}
|
}
|
||||||
defer s.returnConnection(conn)
|
defer s.returnConnection(conn)
|
||||||
|
|
||||||
path := s.keyToPath(key)
|
path := s.keyToPath(key)
|
||||||
dir := filepath.Dir(path)
|
|
||||||
|
|
||||||
// Create directory structure
|
log.Debug().Str("key", path).Msg("Putting file to SMB")
|
||||||
if err := conn.share.MkdirAll(dir, 0755); err != nil {
|
|
||||||
metrics.RecordStorageOperation("smb", "put", "error")
|
// Ensure directory exists
|
||||||
|
dir := filepath.Dir(path)
|
||||||
|
if err := s.ensureDir(conn, dir); err != nil {
|
||||||
return errors.Wrap(err, errors.ErrCodeStorageFailure, "failed to create SMB directory")
|
return errors.Wrap(err, errors.ErrCodeStorageFailure, "failed to create SMB directory")
|
||||||
}
|
}
|
||||||
|
|
||||||
// Read data into buffer to calculate checksums and size
|
// Read data into buffer to check quota
|
||||||
var buf bytes.Buffer
|
buf := new(strings.Builder)
|
||||||
md5Hash := md5.New() // #nosec G401 -- MD5 used for file integrity check, not cryptographic security
|
size, err := io.Copy(buf, data)
|
||||||
sha256Hash := sha256.New()
|
|
||||||
multiWriter := io.MultiWriter(&buf, md5Hash, sha256Hash)
|
|
||||||
|
|
||||||
written, err := io.Copy(multiWriter, data)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
metrics.RecordStorageOperation("smb", "put", "error")
|
|
||||||
return errors.Wrap(err, errors.ErrCodeStorageFailure, "failed to read data")
|
return errors.Wrap(err, errors.ErrCodeStorageFailure, "failed to read data")
|
||||||
}
|
}
|
||||||
|
|
||||||
// Check quota
|
// Check quota if set
|
||||||
if s.quota > 0 {
|
if s.maxSizeBytes > 0 {
|
||||||
s.mu.RLock()
|
currentUsage, err := s.calculateUsage(conn)
|
||||||
used := s.used
|
if err != nil {
|
||||||
s.mu.RUnlock()
|
log.Warn().Err(err).Msg("Failed to calculate current usage, skipping quota check")
|
||||||
|
} else if currentUsage+size > s.maxSizeBytes {
|
||||||
if used+written > s.quota {
|
return errors.QuotaExceeded(s.maxSizeBytes)
|
||||||
metrics.RecordStorageOperation("smb", "put", "quota_exceeded")
|
|
||||||
return errors.QuotaExceeded(s.quota)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Verify checksums if provided
|
// Create/overwrite file
|
||||||
if opts != nil {
|
file, err := conn.share.Create(path)
|
||||||
md5Sum := hex.EncodeToString(md5Hash.Sum(nil))
|
|
||||||
sha256Sum := hex.EncodeToString(sha256Hash.Sum(nil))
|
|
||||||
|
|
||||||
if opts.ChecksumMD5 != "" && opts.ChecksumMD5 != md5Sum {
|
|
||||||
metrics.RecordStorageOperation("smb", "put", "checksum_error")
|
|
||||||
return errors.New(errors.ErrCodeChecksumMismatch, "MD5 checksum mismatch")
|
|
||||||
}
|
|
||||||
|
|
||||||
if opts.ChecksumSHA256 != "" && opts.ChecksumSHA256 != sha256Sum {
|
|
||||||
metrics.RecordStorageOperation("smb", "put", "checksum_error")
|
|
||||||
return errors.New(errors.ErrCodeChecksumMismatch, "SHA256 checksum mismatch")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Create temp file for atomic write
|
|
||||||
tempPath := path + ".tmp"
|
|
||||||
file, err := conn.share.Create(tempPath)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
metrics.RecordStorageOperation("smb", "put", "error")
|
return errors.Wrap(err, errors.ErrCodeStorageFailure, "failed to create SMB file")
|
||||||
return errors.Wrap(err, errors.ErrCodeStorageFailure, "failed to create SMB temp file")
|
|
||||||
}
|
}
|
||||||
|
defer file.Close()
|
||||||
|
|
||||||
// Write data
|
// Write data
|
||||||
_, err = io.Copy(file, bytes.NewReader(buf.Bytes()))
|
_, err = file.Write([]byte(buf.String()))
|
||||||
file.Close() // #nosec G104 -- Cleanup, error not critical
|
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
_ = conn.share.Remove(tempPath) // #nosec G104 -- SMB cleanup
|
|
||||||
metrics.RecordStorageOperation("smb", "put", "error")
|
|
||||||
return errors.Wrap(err, errors.ErrCodeStorageFailure, "failed to write SMB file")
|
return errors.Wrap(err, errors.ErrCodeStorageFailure, "failed to write SMB file")
|
||||||
}
|
}
|
||||||
|
|
||||||
// Atomic rename
|
|
||||||
if err := conn.share.Rename(tempPath, path); err != nil {
|
|
||||||
_ = conn.share.Remove(tempPath) // #nosec G104 -- SMB cleanup
|
|
||||||
metrics.RecordStorageOperation("smb", "put", "error")
|
|
||||||
return errors.Wrap(err, errors.ErrCodeStorageFailure, "failed to rename SMB temp file")
|
|
||||||
}
|
|
||||||
|
|
||||||
// Update usage
|
|
||||||
s.mu.Lock()
|
|
||||||
s.used += written
|
|
||||||
currentUsed := s.used
|
|
||||||
s.mu.Unlock()
|
|
||||||
|
|
||||||
metrics.RecordStorageOperation("smb", "put", "success")
|
|
||||||
metrics.UpdateCacheSize("smb", currentUsed)
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// Delete removes a file from SMB share
|
// ensureDir ensures a directory exists on SMB share
|
||||||
func (s *SMBStorage) Delete(ctx context.Context, key string) error {
|
func (s *SMBStorage) ensureDir(conn *smbConnection, path string) error {
|
||||||
conn, err := s.getConnection(ctx)
|
if path == "" || path == "." || path == "/" {
|
||||||
if err != nil {
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Try to stat the directory
|
||||||
|
_, err := conn.share.Stat(path)
|
||||||
|
if err == nil {
|
||||||
|
return nil // Directory exists
|
||||||
|
}
|
||||||
|
|
||||||
|
// Create parent directory first
|
||||||
|
parent := filepath.Dir(path)
|
||||||
|
if parent != path && parent != "." && parent != "/" {
|
||||||
|
if err := s.ensureDir(conn, parent); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Create this directory
|
||||||
|
err = conn.share.Mkdir(path, 0755)
|
||||||
|
if err != nil && !os.IsExist(err) {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Delete removes data from SMB share
|
||||||
|
func (s *SMBStorage) Delete(ctx context.Context, key string) error {
|
||||||
|
conn, err := s.getConnection()
|
||||||
|
if err != nil {
|
||||||
|
return errors.Wrap(err, errors.ErrCodeStorageFailure, "failed to get SMB connection")
|
||||||
|
}
|
||||||
defer s.returnConnection(conn)
|
defer s.returnConnection(conn)
|
||||||
|
|
||||||
path := s.keyToPath(key)
|
path := s.keyToPath(key)
|
||||||
|
|
||||||
// Get size before deletion
|
log.Debug().Str("key", path).Msg("Deleting file from SMB")
|
||||||
info, err := conn.share.Stat(path)
|
|
||||||
if err != nil {
|
|
||||||
if os.IsNotExist(err) {
|
|
||||||
metrics.RecordStorageOperation("smb", "delete", "not_found")
|
|
||||||
return errors.NotFound(fmt.Sprintf("file not found: %s", key))
|
|
||||||
}
|
|
||||||
metrics.RecordStorageOperation("smb", "delete", "error")
|
|
||||||
return errors.Wrap(err, errors.ErrCodeStorageFailure, "failed to stat SMB file")
|
|
||||||
}
|
|
||||||
|
|
||||||
size := info.Size()
|
err = conn.share.Remove(path)
|
||||||
|
if err != nil && !os.IsNotExist(err) {
|
||||||
if err := conn.share.Remove(path); err != nil {
|
|
||||||
metrics.RecordStorageOperation("smb", "delete", "error")
|
|
||||||
return errors.Wrap(err, errors.ErrCodeStorageFailure, "failed to delete SMB file")
|
return errors.Wrap(err, errors.ErrCodeStorageFailure, "failed to delete SMB file")
|
||||||
}
|
}
|
||||||
|
|
||||||
// Update usage
|
|
||||||
s.mu.Lock()
|
|
||||||
s.used -= size
|
|
||||||
currentUsed := s.used
|
|
||||||
s.mu.Unlock()
|
|
||||||
|
|
||||||
metrics.RecordStorageOperation("smb", "delete", "success")
|
|
||||||
metrics.UpdateCacheSize("smb", currentUsed)
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// Exists checks if a file exists on SMB share
|
// Exists checks if data exists on SMB share
|
||||||
func (s *SMBStorage) Exists(ctx context.Context, key string) (bool, error) {
|
func (s *SMBStorage) Exists(ctx context.Context, key string) (bool, error) {
|
||||||
conn, err := s.getConnection(ctx)
|
conn, err := s.getConnection()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return false, err
|
return false, errors.Wrap(err, errors.ErrCodeStorageFailure, "failed to get SMB connection")
|
||||||
}
|
}
|
||||||
defer s.returnConnection(conn)
|
defer s.returnConnection(conn)
|
||||||
|
|
||||||
@@ -368,57 +341,90 @@ func (s *SMBStorage) Exists(ctx context.Context, key string) (bool, error) {
|
|||||||
return true, nil
|
return true, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// List lists files with prefix on SMB share
|
// List returns a list of objects with the given prefix
|
||||||
func (s *SMBStorage) List(ctx context.Context, prefix string, opts *storage.ListOptions) ([]storage.StorageObject, error) {
|
func (s *SMBStorage) List(ctx context.Context, prefix string, opts *storage.ListOptions) ([]storage.StorageObject, error) {
|
||||||
conn, err := s.getConnection(ctx)
|
conn, err := s.getConnection()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, errors.Wrap(err, errors.ErrCodeStorageFailure, "failed to get SMB connection")
|
||||||
}
|
}
|
||||||
defer s.returnConnection(conn)
|
defer s.returnConnection(conn)
|
||||||
|
|
||||||
searchPath := s.keyToPath(prefix)
|
basePath := s.keyToPath(prefix)
|
||||||
|
|
||||||
|
log.Debug().Str("prefix", basePath).Msg("Listing files in SMB")
|
||||||
|
|
||||||
var objects []storage.StorageObject
|
var objects []storage.StorageObject
|
||||||
|
|
||||||
err = s.walkPath(conn.share, searchPath, func(path string, info os.FileInfo) error {
|
// Walk the directory tree
|
||||||
|
err = s.walkPath(conn, basePath, func(path string, info os.FileInfo) error {
|
||||||
if info.IsDir() {
|
if info.IsDir() {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Convert path back to key
|
||||||
key := s.pathToKey(path)
|
key := s.pathToKey(path)
|
||||||
|
|
||||||
objects = append(objects, storage.StorageObject{
|
objects = append(objects, storage.StorageObject{
|
||||||
Key: key,
|
Key: key,
|
||||||
Size: info.Size(),
|
Size: info.Size(),
|
||||||
Modified: info.ModTime(),
|
Modified: info.ModTime(),
|
||||||
})
|
})
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
})
|
})
|
||||||
|
|
||||||
if err != nil && !os.IsNotExist(err) {
|
if err != nil {
|
||||||
return nil, errors.Wrap(err, errors.ErrCodeStorageFailure, "failed to list SMB files")
|
return nil, errors.Wrap(err, errors.ErrCodeStorageFailure, "failed to list SMB files")
|
||||||
}
|
}
|
||||||
|
|
||||||
// Apply pagination if requested
|
|
||||||
if opts != nil {
|
|
||||||
start := opts.Offset
|
|
||||||
end := len(objects)
|
|
||||||
if opts.MaxResults > 0 && start+opts.MaxResults < end {
|
|
||||||
end = start + opts.MaxResults
|
|
||||||
}
|
|
||||||
if start < len(objects) {
|
|
||||||
objects = objects[start:end]
|
|
||||||
} else {
|
|
||||||
objects = []storage.StorageObject{}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return objects, nil
|
return objects, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// Stat gets file metadata from SMB share
|
// walkPath walks a directory tree on SMB share
|
||||||
func (s *SMBStorage) Stat(ctx context.Context, key string) (*storage.StorageInfo, error) {
|
func (s *SMBStorage) walkPath(conn *smbConnection, root string, fn func(string, os.FileInfo) error) error {
|
||||||
conn, err := s.getConnection(ctx)
|
// Check if root exists
|
||||||
|
info, err := conn.share.Stat(root)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
if os.IsNotExist(err) {
|
||||||
|
return nil // Empty directory
|
||||||
|
}
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// If root is a file, process it directly
|
||||||
|
if !info.IsDir() {
|
||||||
|
return fn(root, info)
|
||||||
|
}
|
||||||
|
|
||||||
|
// List directory contents
|
||||||
|
entries, err := conn.share.ReadDir(root)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, entry := range entries {
|
||||||
|
fullPath := filepath.Join(root, entry.Name())
|
||||||
|
|
||||||
|
if err := fn(fullPath, entry); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Recurse into subdirectories
|
||||||
|
if entry.IsDir() {
|
||||||
|
if err := s.walkPath(conn, fullPath, fn); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Stat returns metadata about stored data
|
||||||
|
func (s *SMBStorage) Stat(ctx context.Context, key string) (*storage.StorageInfo, error) {
|
||||||
|
conn, err := s.getConnection()
|
||||||
|
if err != nil {
|
||||||
|
return nil, errors.Wrap(err, errors.ErrCodeStorageFailure, "failed to get SMB connection")
|
||||||
}
|
}
|
||||||
defer s.returnConnection(conn)
|
defer s.returnConnection(conn)
|
||||||
|
|
||||||
@@ -427,7 +433,7 @@ func (s *SMBStorage) Stat(ctx context.Context, key string) (*storage.StorageInfo
|
|||||||
info, err := conn.share.Stat(path)
|
info, err := conn.share.Stat(path)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if os.IsNotExist(err) {
|
if os.IsNotExist(err) {
|
||||||
return nil, errors.NotFound(fmt.Sprintf("file not found: %s", key))
|
return nil, errors.NotFound(fmt.Sprintf("SMB file not found: %s", key))
|
||||||
}
|
}
|
||||||
return nil, errors.Wrap(err, errors.ErrCodeStorageFailure, "failed to stat SMB file")
|
return nil, errors.Wrap(err, errors.ErrCodeStorageFailure, "failed to stat SMB file")
|
||||||
}
|
}
|
||||||
@@ -439,35 +445,35 @@ func (s *SMBStorage) Stat(ctx context.Context, key string) (*storage.StorageInfo
|
|||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// GetQuota returns quota information
|
// GetQuota returns current usage and quota information
|
||||||
func (s *SMBStorage) GetQuota(ctx context.Context) (*storage.QuotaInfo, error) {
|
func (s *SMBStorage) GetQuota(ctx context.Context) (*storage.QuotaInfo, error) {
|
||||||
s.mu.RLock()
|
conn, err := s.getConnection()
|
||||||
used := s.used
|
if err != nil {
|
||||||
s.mu.RUnlock()
|
return nil, errors.Wrap(err, errors.ErrCodeStorageFailure, "failed to get SMB connection")
|
||||||
|
}
|
||||||
|
defer s.returnConnection(conn)
|
||||||
|
|
||||||
available := s.quota - used
|
usage, err := s.calculateUsage(conn)
|
||||||
if available < 0 {
|
if err != nil {
|
||||||
available = 0
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
return &storage.QuotaInfo{
|
return &storage.QuotaInfo{
|
||||||
Used: used,
|
Used: usage,
|
||||||
Available: available,
|
Limit: s.maxSizeBytes,
|
||||||
Limit: s.quota,
|
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// Health checks SMB health
|
// Health checks if the SMB backend is healthy
|
||||||
func (s *SMBStorage) Health(ctx context.Context) error {
|
func (s *SMBStorage) Health(ctx context.Context) error {
|
||||||
conn, err := s.getConnection(ctx)
|
conn, err := s.getConnection()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return errors.Wrap(err, errors.ErrCodeStorageFailure, "SMB health check failed - connection error")
|
return errors.Wrap(err, errors.ErrCodeStorageFailure, "SMB health check failed: cannot get connection")
|
||||||
}
|
}
|
||||||
defer s.returnConnection(conn)
|
defer s.returnConnection(conn)
|
||||||
|
|
||||||
// Try to stat the base path
|
// Try to stat the base path
|
||||||
path := s.keyToPath("")
|
_, err = conn.share.Stat(s.config.Path)
|
||||||
_, err = conn.share.Stat(path)
|
|
||||||
if err != nil && !os.IsNotExist(err) {
|
if err != nil && !os.IsNotExist(err) {
|
||||||
return errors.Wrap(err, errors.ErrCodeStorageFailure, "SMB health check failed")
|
return errors.Wrap(err, errors.ErrCodeStorageFailure, "SMB health check failed")
|
||||||
}
|
}
|
||||||
@@ -475,105 +481,62 @@ func (s *SMBStorage) Health(ctx context.Context) error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// Close closes the storage backend
|
// Close closes the SMB storage backend
|
||||||
func (s *SMBStorage) Close() error {
|
func (s *SMBStorage) Close() error {
|
||||||
close(s.connPool)
|
close(s.connPool)
|
||||||
|
|
||||||
|
// Close all connections in pool
|
||||||
for conn := range s.connPool {
|
for conn := range s.connPool {
|
||||||
conn.close()
|
conn.close()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
log.Info().Msg("SMB storage closed")
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// keyToPath converts a storage key to SMB path
|
// keyToPath converts a storage key to SMB path
|
||||||
func (s *SMBStorage) keyToPath(key string) string {
|
func (s *SMBStorage) keyToPath(key string) string {
|
||||||
key = strings.TrimPrefix(key, "/")
|
// Normalize separators to backslash for SMB
|
||||||
key = filepath.Clean(key)
|
key = strings.ReplaceAll(key, "/", "\\")
|
||||||
|
|
||||||
// Remove path traversal attempts
|
if s.config.Path == "" {
|
||||||
for strings.HasPrefix(key, "../") || strings.HasPrefix(key, "..\\") {
|
return key
|
||||||
key = strings.TrimPrefix(key, "../")
|
|
||||||
key = strings.TrimPrefix(key, "..\\")
|
|
||||||
}
|
}
|
||||||
|
|
||||||
key = filepath.Clean(key)
|
// Use backslash for SMB paths
|
||||||
if key == ".." || strings.HasPrefix(key, "../") || strings.HasPrefix(key, "..\\") {
|
return s.config.Path + "\\" + key
|
||||||
key = ""
|
|
||||||
}
|
|
||||||
|
|
||||||
if s.basePath != "" {
|
|
||||||
return filepath.Join(s.basePath, key)
|
|
||||||
}
|
|
||||||
return key
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// pathToKey converts an SMB path back to a storage key
|
// pathToKey converts an SMB path to storage key
|
||||||
func (s *SMBStorage) pathToKey(path string) string {
|
func (s *SMBStorage) pathToKey(path string) string {
|
||||||
if s.basePath != "" {
|
// Remove base path
|
||||||
path = strings.TrimPrefix(path, s.basePath)
|
if s.config.Path != "" {
|
||||||
path = strings.TrimPrefix(path, "/")
|
path = strings.TrimPrefix(path, s.config.Path+"\\")
|
||||||
path = strings.TrimPrefix(path, "\\")
|
|
||||||
}
|
}
|
||||||
return filepath.ToSlash(path)
|
|
||||||
|
// Convert backslashes to forward slashes for consistency
|
||||||
|
return strings.ReplaceAll(path, "\\", "/")
|
||||||
}
|
}
|
||||||
|
|
||||||
// walkPath recursively walks an SMB directory
|
// calculateUsage calculates total storage usage
|
||||||
func (s *SMBStorage) walkPath(share *smb2.Share, path string, fn func(string, os.FileInfo) error) error {
|
func (s *SMBStorage) calculateUsage(conn *smbConnection) (int64, error) {
|
||||||
info, err := share.Stat(path)
|
var totalSize int64
|
||||||
if err != nil {
|
|
||||||
return err
|
basePath := s.config.Path
|
||||||
|
if basePath == "" {
|
||||||
|
basePath = "\\"
|
||||||
}
|
}
|
||||||
|
|
||||||
if !info.IsDir() {
|
err := s.walkPath(conn, basePath, func(path string, info os.FileInfo) error {
|
||||||
return fn(path, info)
|
|
||||||
}
|
|
||||||
|
|
||||||
entries, err := share.ReadDir(path)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, entry := range entries {
|
|
||||||
entryPath := filepath.Join(path, entry.Name())
|
|
||||||
if entry.IsDir() {
|
|
||||||
if err := s.walkPath(share, entryPath, fn); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
if err := fn(entryPath, entry); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// calculateUsage calculates current SMB storage usage
|
|
||||||
func (s *SMBStorage) calculateUsage(ctx context.Context) error {
|
|
||||||
conn, err := s.getConnection(ctx)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
defer s.returnConnection(conn)
|
|
||||||
|
|
||||||
var total int64
|
|
||||||
basePath := s.keyToPath("")
|
|
||||||
|
|
||||||
err = s.walkPath(conn.share, basePath, func(path string, info os.FileInfo) error {
|
|
||||||
if !info.IsDir() {
|
if !info.IsDir() {
|
||||||
total += info.Size()
|
totalSize += info.Size()
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
})
|
})
|
||||||
|
|
||||||
if err != nil && !os.IsNotExist(err) {
|
if err != nil {
|
||||||
return err
|
return 0, fmt.Errorf("failed to calculate usage: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
s.mu.Lock()
|
return totalSize, nil
|
||||||
s.used = total
|
|
||||||
s.mu.Unlock()
|
|
||||||
|
|
||||||
metrics.UpdateCacheSize("smb", total)
|
|
||||||
return nil
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,327 @@
|
|||||||
|
package smb
|
||||||
|
|
||||||
|
import (
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/stretchr/testify/suite"
|
||||||
|
)
|
||||||
|
|
||||||
|
type SMBStorageTestSuite struct {
|
||||||
|
suite.Suite
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSMBStorageTestSuite(t *testing.T) {
|
||||||
|
suite.Run(t, new(SMBStorageTestSuite))
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *SMBStorageTestSuite) TestNewSMBStorage() {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
errorMsg string
|
||||||
|
config Config
|
||||||
|
expectError bool
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "valid config",
|
||||||
|
config: Config{
|
||||||
|
Host: "fileserver.example.com",
|
||||||
|
Port: 445,
|
||||||
|
Share: "gohoarder",
|
||||||
|
Path: "packages",
|
||||||
|
Username: "testuser",
|
||||||
|
Password: "testpass",
|
||||||
|
Domain: "CORP",
|
||||||
|
MaxSizeBytes: 1024 * 1024,
|
||||||
|
PoolSize: 5,
|
||||||
|
},
|
||||||
|
expectError: false,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "missing host",
|
||||||
|
config: Config{
|
||||||
|
Share: "gohoarder",
|
||||||
|
Username: "testuser",
|
||||||
|
Password: "testpass",
|
||||||
|
},
|
||||||
|
expectError: true,
|
||||||
|
errorMsg: "host is required",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "missing share",
|
||||||
|
config: Config{
|
||||||
|
Host: "fileserver.example.com",
|
||||||
|
Username: "testuser",
|
||||||
|
Password: "testpass",
|
||||||
|
},
|
||||||
|
expectError: true,
|
||||||
|
errorMsg: "share is required",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "default port",
|
||||||
|
config: Config{
|
||||||
|
Host: "fileserver.example.com",
|
||||||
|
Share: "gohoarder",
|
||||||
|
Username: "testuser",
|
||||||
|
Password: "testpass",
|
||||||
|
},
|
||||||
|
expectError: false,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "default pool size",
|
||||||
|
config: Config{
|
||||||
|
Host: "fileserver.example.com",
|
||||||
|
Share: "gohoarder",
|
||||||
|
Username: "testuser",
|
||||||
|
Password: "testpass",
|
||||||
|
},
|
||||||
|
expectError: false,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
s.Run(tt.name, func() {
|
||||||
|
storage, err := New(tt.config)
|
||||||
|
|
||||||
|
if tt.expectError {
|
||||||
|
s.Error(err)
|
||||||
|
if tt.errorMsg != "" {
|
||||||
|
s.Contains(err.Error(), tt.errorMsg)
|
||||||
|
}
|
||||||
|
s.Nil(storage)
|
||||||
|
} else {
|
||||||
|
// Note: This will fail in actual execution since we can't connect to a real SMB server
|
||||||
|
// But it tests the validation logic
|
||||||
|
if err != nil {
|
||||||
|
// Connection errors are expected in unit tests
|
||||||
|
s.Contains(err.Error(), "Failed to create initial SMB connection")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *SMBStorageTestSuite) TestKeyToPath() {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
basePath string
|
||||||
|
key string
|
||||||
|
expectedWin string // Expected Windows-style path
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "simple key with base path",
|
||||||
|
basePath: "packages",
|
||||||
|
key: "test/file.txt",
|
||||||
|
expectedWin: "packages\\test\\file.txt",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "simple key without base path",
|
||||||
|
basePath: "",
|
||||||
|
key: "test/file.txt",
|
||||||
|
expectedWin: "test\\file.txt",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "nested key",
|
||||||
|
basePath: "cache",
|
||||||
|
key: "deep/nested/path/file.txt",
|
||||||
|
expectedWin: "cache\\deep\\nested\\path\\file.txt",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "key with backslashes",
|
||||||
|
basePath: "packages",
|
||||||
|
key: "test\\file.txt",
|
||||||
|
expectedWin: "packages\\test\\file.txt",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
s.Run(tt.name, func() {
|
||||||
|
storage := &SMBStorage{
|
||||||
|
config: Config{
|
||||||
|
Path: tt.basePath,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
result := storage.keyToPath(tt.key)
|
||||||
|
s.Equal(tt.expectedWin, result)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *SMBStorageTestSuite) TestPathToKey() {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
basePath string
|
||||||
|
path string
|
||||||
|
expected string
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "windows path with base path",
|
||||||
|
basePath: "packages",
|
||||||
|
path: "packages\\test\\file.txt",
|
||||||
|
expected: "test/file.txt",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "windows path without base path",
|
||||||
|
basePath: "",
|
||||||
|
path: "test\\file.txt",
|
||||||
|
expected: "test/file.txt",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "nested windows path",
|
||||||
|
basePath: "cache",
|
||||||
|
path: "cache\\deep\\nested\\path\\file.txt",
|
||||||
|
expected: "deep/nested/path/file.txt",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
s.Run(tt.name, func() {
|
||||||
|
storage := &SMBStorage{
|
||||||
|
config: Config{
|
||||||
|
Path: tt.basePath,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
result := storage.pathToKey(tt.path)
|
||||||
|
s.Equal(tt.expected, result)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *SMBStorageTestSuite) TestConfigDefaults() {
|
||||||
|
config := Config{
|
||||||
|
Host: "fileserver.example.com",
|
||||||
|
Share: "gohoarder",
|
||||||
|
Username: "testuser",
|
||||||
|
Password: "testpass",
|
||||||
|
}
|
||||||
|
|
||||||
|
// This will fail to connect, but we can verify the config validation
|
||||||
|
_, err := New(config)
|
||||||
|
|
||||||
|
// We expect a connection error, not a validation error
|
||||||
|
if err != nil {
|
||||||
|
s.NotContains(err.Error(), "host is required")
|
||||||
|
s.NotContains(err.Error(), "share is required")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *SMBStorageTestSuite) TestPathNormalization() {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
inputPath string
|
||||||
|
expectedPath string
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "path with trailing slash",
|
||||||
|
inputPath: "packages/",
|
||||||
|
expectedPath: "packages",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "path with trailing backslash",
|
||||||
|
inputPath: "packages\\",
|
||||||
|
expectedPath: "packages",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "path without trailing slash",
|
||||||
|
inputPath: "packages",
|
||||||
|
expectedPath: "packages",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "empty path",
|
||||||
|
inputPath: "",
|
||||||
|
expectedPath: "",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "nested path with trailing slash",
|
||||||
|
inputPath: "cache/packages/",
|
||||||
|
expectedPath: "cache/packages",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
s.Run(tt.name, func() {
|
||||||
|
config := Config{
|
||||||
|
Host: "fileserver.example.com",
|
||||||
|
Share: "gohoarder",
|
||||||
|
Path: tt.inputPath,
|
||||||
|
Username: "testuser",
|
||||||
|
Password: "testpass",
|
||||||
|
}
|
||||||
|
|
||||||
|
// This will fail to connect, but we can check the config
|
||||||
|
storage, _ := New(config)
|
||||||
|
if storage != nil {
|
||||||
|
s.Equal(tt.expectedPath, storage.config.Path)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *SMBStorageTestSuite) TestPoolSizeDefaults() {
|
||||||
|
config := Config{
|
||||||
|
Host: "fileserver.example.com",
|
||||||
|
Share: "gohoarder",
|
||||||
|
Username: "testuser",
|
||||||
|
Password: "testpass",
|
||||||
|
}
|
||||||
|
|
||||||
|
storage, _ := New(config)
|
||||||
|
if storage != nil {
|
||||||
|
s.Equal(5, storage.poolSize) // Default pool size
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *SMBStorageTestSuite) TestPortDefaults() {
|
||||||
|
config := Config{
|
||||||
|
Host: "fileserver.example.com",
|
||||||
|
Share: "gohoarder",
|
||||||
|
Username: "testuser",
|
||||||
|
Password: "testpass",
|
||||||
|
}
|
||||||
|
|
||||||
|
storage, _ := New(config)
|
||||||
|
if storage != nil {
|
||||||
|
s.Equal(445, storage.config.Port) // Default SMB port
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *SMBStorageTestSuite) TestClose() {
|
||||||
|
// Create a storage instance (will fail to connect but that's ok)
|
||||||
|
config := Config{
|
||||||
|
Host: "fileserver.example.com",
|
||||||
|
Share: "gohoarder",
|
||||||
|
Username: "testuser",
|
||||||
|
Password: "testpass",
|
||||||
|
}
|
||||||
|
|
||||||
|
storage, _ := New(config)
|
||||||
|
if storage != nil {
|
||||||
|
// Close should not panic
|
||||||
|
err := storage.Close()
|
||||||
|
s.NoError(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *SMBStorageTestSuite) TestConnectionPoolChannel() {
|
||||||
|
config := Config{
|
||||||
|
Host: "fileserver.example.com",
|
||||||
|
Share: "gohoarder",
|
||||||
|
Username: "testuser",
|
||||||
|
Password: "testpass",
|
||||||
|
PoolSize: 10,
|
||||||
|
}
|
||||||
|
|
||||||
|
storage, _ := New(config)
|
||||||
|
if storage != nil {
|
||||||
|
// Verify pool channel capacity
|
||||||
|
s.NotNil(storage.connPool)
|
||||||
|
s.Equal(10, cap(storage.connPool))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *SMBStorageTestSuite) TestSMBConnectionStruct() {
|
||||||
|
// Verify smbConnection structure exists and has required fields
|
||||||
|
conn := &smbConnection{}
|
||||||
|
s.NotNil(conn)
|
||||||
|
}
|
||||||
@@ -56,8 +56,8 @@ func TestNew(t *testing.T) {
|
|||||||
func TestString(t *testing.T) {
|
func TestString(t *testing.T) {
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
name string
|
name string
|
||||||
uuid UUID
|
|
||||||
expected string
|
expected string
|
||||||
|
uuid UUID
|
||||||
}{
|
}{
|
||||||
{
|
{
|
||||||
name: "zero UUID",
|
name: "zero UUID",
|
||||||
|
|||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user