mirror of
https://github.com/lukaszraczylo/gohoarder.git
synced 2026-07-22 06:20:09 +00:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
 dependabot[bot]andGitHub
|
bf5e609a68 |
@@ -151,17 +151,6 @@ release:
|
||||
# Uses PRE-BUILT binaries from native builds (no Docker compilation - much faster!)
|
||||
# GoReleaser injects the platform-specific binary into each Docker image automatically
|
||||
# This avoids slow QEMU emulation for cross-architecture builds
|
||||
#
|
||||
# Split/Merge Build Support (GoReleaser Pro):
|
||||
# - Run `goreleaser release --split` on multiple CI runners in parallel
|
||||
# - Each runner builds a subset of targets (e.g., runner 1: linux/amd64, runner 2: linux/arm64)
|
||||
# - Run `goreleaser release --merge` to combine all partial builds into final release
|
||||
# - Significantly speeds up builds by parallelizing across multiple machines
|
||||
#
|
||||
# Binary Path Convention:
|
||||
# - Binaries must be in linux/${TARGETARCH}/ subdirectory format
|
||||
# - GoReleaser automatically places binaries there when using --split
|
||||
# - COPY linux/${TARGETARCH}/binary_name in Dockerfiles
|
||||
dockers_v2:
|
||||
# 1. Application Engine - Main GoHoarder server
|
||||
# Uses pre-built binary from 'gohoarder' build (no Docker compilation)
|
||||
|
||||
@@ -1,4 +1,3 @@
|
||||
// Package commands hosts the cobra commands wired into the gohoarder CLI.
|
||||
package commands
|
||||
|
||||
import (
|
||||
@@ -36,11 +35,11 @@ func runServe(cmd *cobra.Command, args []string) error {
|
||||
}
|
||||
|
||||
// Initialize logger
|
||||
if logErr := logger.Init(logger.Config{
|
||||
if err := logger.Init(logger.Config{
|
||||
Level: cfg.Logging.Level,
|
||||
Format: cfg.Logging.Format,
|
||||
}); logErr != nil {
|
||||
return fmt.Errorf("failed to initialize logger: %w", logErr)
|
||||
}); err != nil {
|
||||
return fmt.Errorf("failed to initialize logger: %w", err)
|
||||
}
|
||||
|
||||
log.Info().
|
||||
|
||||
@@ -23,16 +23,16 @@ var VersionCmd = &cobra.Command{
|
||||
if jsonOutput {
|
||||
data, err := json.MarshalIndent(info, "", " ")
|
||||
if err != nil {
|
||||
_, _ = fmt.Fprintf(cmd.OutOrStderr(), "Error: %v\n", err)
|
||||
fmt.Fprintf(cmd.OutOrStderr(), "Error: %v\n", err)
|
||||
return
|
||||
}
|
||||
_, _ = fmt.Fprintln(cmd.OutOrStdout(), string(data))
|
||||
fmt.Fprintln(cmd.OutOrStdout(), string(data))
|
||||
} else {
|
||||
_, _ = fmt.Fprintf(cmd.OutOrStdout(), "GoHoarder %s\n", info.Version)
|
||||
_, _ = fmt.Fprintf(cmd.OutOrStdout(), "Git Commit: %s\n", info.GitCommit)
|
||||
_, _ = fmt.Fprintf(cmd.OutOrStdout(), "Built: %s\n", info.BuildTime)
|
||||
_, _ = fmt.Fprintf(cmd.OutOrStdout(), "Go Version: %s\n", info.GoVersion)
|
||||
_, _ = fmt.Fprintf(cmd.OutOrStdout(), "Platform: %s\n", info.Platform)
|
||||
fmt.Fprintf(cmd.OutOrStdout(), "GoHoarder %s\n", info.Version)
|
||||
fmt.Fprintf(cmd.OutOrStdout(), "Git Commit: %s\n", info.GitCommit)
|
||||
fmt.Fprintf(cmd.OutOrStdout(), "Built: %s\n", info.BuildTime)
|
||||
fmt.Fprintf(cmd.OutOrStdout(), "Go Version: %s\n", info.GoVersion)
|
||||
fmt.Fprintf(cmd.OutOrStdout(), "Platform: %s\n", info.Platform)
|
||||
}
|
||||
},
|
||||
}
|
||||
|
||||
+2
-2
@@ -25,10 +25,10 @@ import (
|
||||
type MigrationConfig struct {
|
||||
Driver string
|
||||
DSN string
|
||||
Timeout time.Duration
|
||||
Action string // migrate, rollback, rollback-to, list
|
||||
TargetID string // For rollback-to
|
||||
LogLevel string
|
||||
Timeout time.Duration
|
||||
}
|
||||
|
||||
func main() {
|
||||
@@ -97,7 +97,7 @@ func RunMigration(cfg MigrationConfig) error {
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to get sql.DB: %w", err)
|
||||
}
|
||||
defer func() { _ = sqlDB.Close() }()
|
||||
defer sqlDB.Close()
|
||||
|
||||
// Wait for database to be ready
|
||||
if err := waitForDB(ctx, sqlDB, 60*time.Second); err != nil {
|
||||
|
||||
@@ -1,129 +0,0 @@
|
||||
# Example GoHoarder values for Traefik IngressRoute
|
||||
#
|
||||
# This configuration uses Traefik IngressRoute instead of standard Kubernetes Ingress
|
||||
# to better handle scoped npm packages with encoded slashes.
|
||||
#
|
||||
# IMPORTANT: You must also configure your Traefik deployment to allow encoded slashes.
|
||||
# See docs/TRAEFIK_ENCODED_SLASHES.md for Traefik configuration steps.
|
||||
|
||||
nameOverride: "gohoarder"
|
||||
fullnameOverride: ""
|
||||
|
||||
global:
|
||||
domain: "i.raczylo.com"
|
||||
imagePullSecrets: []
|
||||
|
||||
replicaCount:
|
||||
server: 1
|
||||
frontend: 1
|
||||
scanner: 1
|
||||
|
||||
image:
|
||||
server:
|
||||
repository: ghcr.io/lukaszraczylo/gohoarder-server
|
||||
pullPolicy: IfNotPresent
|
||||
tag: "latest"
|
||||
frontend:
|
||||
repository: ghcr.io/lukaszraczylo/gohoarder-frontend
|
||||
pullPolicy: IfNotPresent
|
||||
tag: "latest"
|
||||
scanner:
|
||||
repository: ghcr.io/lukaszraczylo/gohoarder-scanner
|
||||
pullPolicy: IfNotPresent
|
||||
tag: "latest"
|
||||
|
||||
# Ingress configuration for Traefik
|
||||
ingress:
|
||||
enabled: true
|
||||
className: "traefik" # This triggers IngressRoute creation instead of Ingress
|
||||
|
||||
# Your GoHoarder hostname
|
||||
host: "hoarder.i.raczylo.com"
|
||||
|
||||
# TLS configuration
|
||||
tls:
|
||||
enabled: true
|
||||
secretName: "cert-i.raczylo.com"
|
||||
|
||||
# Traefik-specific settings
|
||||
# These are used by the IngressRoute template
|
||||
traefik:
|
||||
# Reference middlewares from the traefik namespace
|
||||
# These will be applied to all routes
|
||||
middlewares:
|
||||
- default-chain # Your existing middleware chain from traefik namespace
|
||||
|
||||
# ServersTransport configuration (optional)
|
||||
transport:
|
||||
serverName: ""
|
||||
|
||||
# Storage configuration
|
||||
storage:
|
||||
backend: "filesystem"
|
||||
filesystem:
|
||||
storageClass: "longhorn"
|
||||
size: "100Gi"
|
||||
accessMode: "ReadWriteOnce"
|
||||
|
||||
# Metadata storage
|
||||
metadata:
|
||||
backend: "sqlite"
|
||||
sqlite:
|
||||
persistence:
|
||||
enabled: false # Using emptyDir for metadata
|
||||
walMode: false
|
||||
|
||||
# Cache configuration
|
||||
cache:
|
||||
defaultTTL: "168h" # 7 days
|
||||
maxSizeBytes: 536870912000 # 500GB
|
||||
|
||||
# Package handlers
|
||||
handlers:
|
||||
npm:
|
||||
enabled: true
|
||||
upstreamRegistry: "https://registry.npmjs.org"
|
||||
pypi:
|
||||
enabled: true
|
||||
upstreamUrl: "https://pypi.org"
|
||||
go:
|
||||
enabled: true
|
||||
upstreamProxy: "https://proxy.golang.org"
|
||||
|
||||
# Authentication
|
||||
auth:
|
||||
enabled: true
|
||||
adminApiKey: "" # Will be auto-generated if empty
|
||||
|
||||
# Logging
|
||||
logging:
|
||||
level: "info"
|
||||
format: "json"
|
||||
|
||||
# Resource limits
|
||||
server:
|
||||
resources:
|
||||
limits:
|
||||
cpu: 2000m
|
||||
memory: 2Gi
|
||||
requests:
|
||||
cpu: 500m
|
||||
memory: 512Mi
|
||||
|
||||
frontend:
|
||||
resources:
|
||||
limits:
|
||||
cpu: 500m
|
||||
memory: 512Mi
|
||||
requests:
|
||||
cpu: 100m
|
||||
memory: 128Mi
|
||||
|
||||
scanner:
|
||||
resources:
|
||||
limits:
|
||||
cpu: 2000m
|
||||
memory: 4Gi
|
||||
requests:
|
||||
cpu: 500m
|
||||
memory: 1Gi
|
||||
@@ -12,7 +12,7 @@
|
||||
},
|
||||
"dependencies": {
|
||||
"@vueuse/core": "^14.1.0",
|
||||
"axios": "^1.13.2",
|
||||
"axios": "^1.13.5",
|
||||
"class-variance-authority": "^0.7.1",
|
||||
"clsx": "^2.1.1",
|
||||
"lucide-vue-next": "^0.562.0",
|
||||
|
||||
Generated
+6
-20
@@ -12,8 +12,8 @@ importers:
|
||||
specifier: ^14.1.0
|
||||
version: 14.1.0([email protected]([email protected]))
|
||||
axios:
|
||||
specifier: ^1.13.2
|
||||
version: 1.13.2
|
||||
specifier: ^1.13.5
|
||||
version: 1.13.5
|
||||
class-variance-authority:
|
||||
specifier: ^0.7.1
|
||||
version: 0.7.1
|
||||
@@ -426,67 +426,56 @@ packages:
|
||||
resolution: {integrity: sha512-EHMUcDwhtdRGlXZsGSIuXSYwD5kOT9NVnx9sqzYiwAc91wfYOE1g1djOEDseZJKKqtHAHGwnGPQu3kytmfaXLQ==}
|
||||
cpu: [arm]
|
||||
os: [linux]
|
||||
libc: [glibc]
|
||||
|
||||
'@rollup/[email protected]':
|
||||
resolution: {integrity: sha512-+pBrqEjaakN2ySv5RVrj/qLytYhPKEUwk+e3SFU5jTLHIcAtqh2rLrd/OkbNuHJpsBgxsD8ccJt5ga/SeG0JmA==}
|
||||
cpu: [arm]
|
||||
os: [linux]
|
||||
libc: [musl]
|
||||
|
||||
'@rollup/[email protected]':
|
||||
resolution: {integrity: sha512-NSqc7rE9wuUaRBsBp5ckQ5CVz5aIRKCwsoa6WMF7G01sX3/qHUw/z4pv+D+ahL1EIKy6Enpcnz1RY8pf7bjwng==}
|
||||
cpu: [arm64]
|
||||
os: [linux]
|
||||
libc: [glibc]
|
||||
|
||||
'@rollup/[email protected]':
|
||||
resolution: {integrity: sha512-gr5vDbg3Bakga5kbdpqx81m2n9IX8M6gIMlQQIXiLTNeQW6CucvuInJ91EuCJ/JYvc+rcLLsDFcfAD1K7fMofg==}
|
||||
cpu: [arm64]
|
||||
os: [linux]
|
||||
libc: [musl]
|
||||
|
||||
'@rollup/[email protected]':
|
||||
resolution: {integrity: sha512-gsrtB1NA3ZYj2vq0Rzkylo9ylCtW/PhpLEivlgWe0bpgtX5+9j9EZa0wtZiCjgu6zmSeZWyI/e2YRX1URozpIw==}
|
||||
cpu: [loong64]
|
||||
os: [linux]
|
||||
libc: [glibc]
|
||||
|
||||
'@rollup/[email protected]':
|
||||
resolution: {integrity: sha512-y3qNOfTBStmFNq+t4s7Tmc9hW2ENtPg8FeUD/VShI7rKxNW7O4fFeaYbMsd3tpFlIg1Q8IapFgy7Q9i2BqeBvA==}
|
||||
cpu: [ppc64]
|
||||
os: [linux]
|
||||
libc: [glibc]
|
||||
|
||||
'@rollup/[email protected]':
|
||||
resolution: {integrity: sha512-89sepv7h2lIVPsFma8iwmccN7Yjjtgz0Rj/Ou6fEqg3HDhpCa+Et+YSufy27i6b0Wav69Qv4WBNl3Rs6pwhebQ==}
|
||||
cpu: [riscv64]
|
||||
os: [linux]
|
||||
libc: [glibc]
|
||||
|
||||
'@rollup/[email protected]':
|
||||
resolution: {integrity: sha512-ZcU77ieh0M2Q8Ur7D5X7KvK+UxbXeDHwiOt/CPSBTI1fBmeDMivW0dPkdqkT4rOgDjrDDBUed9x4EgraIKoR2A==}
|
||||
cpu: [riscv64]
|
||||
os: [linux]
|
||||
libc: [musl]
|
||||
|
||||
'@rollup/[email protected]':
|
||||
resolution: {integrity: sha512-2AdWy5RdDF5+4YfG/YesGDDtbyJlC9LHmL6rZw6FurBJ5n4vFGupsOBGfwMRjBYH7qRQowT8D/U4LoSvVwOhSQ==}
|
||||
cpu: [s390x]
|
||||
os: [linux]
|
||||
libc: [glibc]
|
||||
|
||||
'@rollup/[email protected]':
|
||||
resolution: {integrity: sha512-WGt5J8Ij/rvyqpFexxk3ffKqqbLf9AqrTBbWDk7ApGUzaIs6V+s2s84kAxklFwmMF/vBNGrVdYgbblCOFFezMQ==}
|
||||
cpu: [x64]
|
||||
os: [linux]
|
||||
libc: [glibc]
|
||||
|
||||
'@rollup/[email protected]':
|
||||
resolution: {integrity: sha512-JzQmb38ATzHjxlPHuTH6tE7ojnMKM2kYNzt44LO/jJi8BpceEC8QuXYA908n8r3CNuG/B3BV8VR3Hi1rYtmPiw==}
|
||||
cpu: [x64]
|
||||
os: [linux]
|
||||
libc: [musl]
|
||||
|
||||
'@rollup/[email protected]':
|
||||
resolution: {integrity: sha512-huT3fd0iC7jigGh7n3q/+lfPcXxBi+om/Rs3yiFxjvSxbSB6aohDFXbWvlspaqjeOh+hx7DDHS+5Es5qRkWkZg==}
|
||||
@@ -749,8 +738,8 @@ packages:
|
||||
resolution: {integrity: sha512-wvUjBtSGN7+7SjNpq/9M2Tg350UZD3q62IFZLbRAR1bSMlCo1ZaeW+BJ+D090e4hIIZLBcTDWe4Mh4jvUDajzQ==}
|
||||
engines: {node: '>= 0.4'}
|
||||
|
||||
[email protected].2:
|
||||
resolution: {integrity: sha512-VPk9ebNqPcy5lRGuSlKx752IlDatOjT9paPlm8A7yOuW2Fbvp4X3JznJtT4f0GzGLLiWE9W8onz51SqLYwzGaA==}
|
||||
[email protected].5:
|
||||
resolution: {integrity: sha512-cz4ur7Vb0xS4/KUN0tPWe44eqxrIu31me+fbang3ijiNscE129POzipJJA6zniq2C/Z6sJCjMimjS8Lc/GAs8Q==}
|
||||
|
||||
[email protected]:
|
||||
resolution: {integrity: sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==}
|
||||
@@ -1062,6 +1051,7 @@ packages:
|
||||
|
||||
[email protected]:
|
||||
resolution: {integrity: sha512-DfXN8DfhJ7NH3Oe7cFmu3NCu1wKbkReJ8TorzSAFbSKrlNaQSKfIzqYqVY8zlbs2NLBbWpRiU52GX2PbaBVNkg==}
|
||||
deprecated: Old versions of glob are not supported, and contain widely publicized security vulnerabilities, which have been fixed in the current version. Please update. Support for old versions may be purchased (at exorbitant rates) by contacting [email protected]
|
||||
hasBin: true
|
||||
|
||||
[email protected]:
|
||||
@@ -1277,28 +1267,24 @@ packages:
|
||||
engines: {node: '>= 12.0.0'}
|
||||
cpu: [arm64]
|
||||
os: [linux]
|
||||
libc: [glibc]
|
||||
|
||||
[email protected]:
|
||||
resolution: {integrity: sha512-5Vh9dGeblpTxWHpOx8iauV02popZDsCYMPIgiuw97OJ5uaDsL86cnqSFs5LZkG3ghHoX5isLgWzMs+eD1YzrnA==}
|
||||
engines: {node: '>= 12.0.0'}
|
||||
cpu: [arm64]
|
||||
os: [linux]
|
||||
libc: [musl]
|
||||
|
||||
[email protected]:
|
||||
resolution: {integrity: sha512-Cfd46gdmj1vQ+lR6VRTTadNHu6ALuw2pKR9lYq4FnhvgBc4zWY1EtZcAc6EffShbb1MFrIPfLDXD6Xprbnni4w==}
|
||||
engines: {node: '>= 12.0.0'}
|
||||
cpu: [x64]
|
||||
os: [linux]
|
||||
libc: [glibc]
|
||||
|
||||
[email protected]:
|
||||
resolution: {integrity: sha512-XJaLUUFXb6/QG2lGIW6aIk6jKdtjtcffUT0NKvIqhSBY3hh9Ch+1LCeH80dR9q9LBjG3ewbDjnumefsLsP6aiA==}
|
||||
engines: {node: '>= 12.0.0'}
|
||||
cpu: [x64]
|
||||
os: [linux]
|
||||
libc: [musl]
|
||||
|
||||
[email protected]:
|
||||
resolution: {integrity: sha512-FZn+vaj7zLv//D/192WFFVA0RgHawIcHqLX9xuWiQt7P0PtdFEVaxgF9rjM/IRYHQXNnk61/H/gb2Ei+kUQ4xQ==}
|
||||
@@ -2557,7 +2543,7 @@ snapshots:
|
||||
dependencies:
|
||||
possible-typed-array-names: 1.1.0
|
||||
|
||||
[email protected].2:
|
||||
[email protected].5:
|
||||
dependencies:
|
||||
follow-redirects: 1.15.11
|
||||
form-data: 4.0.5
|
||||
|
||||
@@ -3,12 +3,10 @@ import { mount } from '@vue/test-utils'
|
||||
import { createPinia, setActivePinia } from 'pinia'
|
||||
import Dashboard from './Dashboard.vue'
|
||||
import { usePackageStore } from '../stores/packages'
|
||||
import { useRealtimeStore, __resetRealtimeSingleton } from '../stores/realtime'
|
||||
|
||||
describe('Dashboard.vue', () => {
|
||||
beforeEach(() => {
|
||||
setActivePinia(createPinia())
|
||||
__resetRealtimeSingleton()
|
||||
// Mock the fetch functions to prevent actual API calls
|
||||
const store = usePackageStore()
|
||||
vi.spyOn(store, 'fetchStats').mockResolvedValue()
|
||||
@@ -215,33 +213,4 @@ describe('Dashboard.vue', () => {
|
||||
expect(wrapper.text()).toContain('0')
|
||||
expect(wrapper.text()).toContain('0 B')
|
||||
})
|
||||
|
||||
it('renders realtime lastStats over polled stats when present', async () => {
|
||||
const wrapper = mount(Dashboard)
|
||||
const store = usePackageStore()
|
||||
const rt = useRealtimeStore()
|
||||
|
||||
store.loading = false
|
||||
store.stats = {
|
||||
registry: '',
|
||||
total_packages: 10,
|
||||
total_size: 1024,
|
||||
max_cache_size: 10737418240,
|
||||
total_downloads: 5,
|
||||
scanned_packages: 0,
|
||||
vulnerable_packages: 0,
|
||||
blocked_packages: 0,
|
||||
}
|
||||
rt.lastStats = {
|
||||
total_packages: 999,
|
||||
total_size: 2048,
|
||||
total_downloads: 777,
|
||||
scanned_packages: 0,
|
||||
}
|
||||
await wrapper.vm.$nextTick()
|
||||
|
||||
// Realtime values should be visible (override polled values).
|
||||
expect(wrapper.text()).toContain('999')
|
||||
expect(wrapper.text()).toContain('777')
|
||||
})
|
||||
})
|
||||
|
||||
@@ -1,23 +1,6 @@
|
||||
<template>
|
||||
<div>
|
||||
<div class="flex items-center justify-between mb-8">
|
||||
<h2 class="text-3xl font-bold text-gray-900">Dashboard</h2>
|
||||
<div
|
||||
class="flex items-center gap-2 px-3 py-1 rounded-full border text-xs font-medium"
|
||||
:class="
|
||||
rt.connected
|
||||
? 'bg-emerald-50 border-emerald-200 text-emerald-700'
|
||||
: 'bg-gray-100 border-gray-200 text-gray-500'
|
||||
"
|
||||
data-testid="ws-live-indicator"
|
||||
>
|
||||
<span
|
||||
class="w-2 h-2 rounded-full"
|
||||
:class="rt.connected ? 'bg-emerald-500 animate-pulse' : 'bg-gray-400'"
|
||||
></span>
|
||||
{{ rt.connected ? 'Live' : 'Offline' }}
|
||||
</div>
|
||||
</div>
|
||||
<h2 class="text-3xl font-bold text-gray-900 mb-8">Dashboard</h2>
|
||||
|
||||
<!-- Error Alert -->
|
||||
<Alert v-if="error" variant="destructive" class="mb-4">
|
||||
@@ -39,7 +22,7 @@
|
||||
<div class="space-y-2">
|
||||
<p class="text-sm font-medium text-muted-foreground">Total Packages</p>
|
||||
<p class="text-3xl font-bold text-foreground tracking-tight">
|
||||
{{ formatNumber(displayStats?.total_packages || 0) }}
|
||||
{{ formatNumber(stats?.total_packages || 0) }}
|
||||
</p>
|
||||
</div>
|
||||
<div class="w-12 h-12 bg-slate-100 rounded-xl flex items-center justify-center">
|
||||
@@ -55,7 +38,7 @@
|
||||
<div class="space-y-2">
|
||||
<p class="text-sm font-medium text-muted-foreground">Total Size</p>
|
||||
<p class="text-3xl font-bold text-foreground tracking-tight">
|
||||
{{ formatBytes(displayStats?.total_size || 0) }}
|
||||
{{ formatBytes(stats?.total_size || 0) }}
|
||||
</p>
|
||||
</div>
|
||||
<div class="w-12 h-12 bg-sky-50 rounded-xl flex items-center justify-center">
|
||||
@@ -71,7 +54,7 @@
|
||||
<div class="space-y-2">
|
||||
<p class="text-sm font-medium text-muted-foreground">Total Downloads</p>
|
||||
<p class="text-3xl font-bold text-foreground tracking-tight">
|
||||
{{ formatNumber(displayStats?.total_downloads || 0) }}
|
||||
{{ formatNumber(stats?.total_downloads || 0) }}
|
||||
</p>
|
||||
</div>
|
||||
<div class="w-12 h-12 bg-emerald-50 rounded-xl flex items-center justify-center">
|
||||
@@ -87,7 +70,7 @@
|
||||
<div class="space-y-2">
|
||||
<p class="text-sm font-medium text-muted-foreground">Scanned Packages</p>
|
||||
<p class="text-3xl font-bold text-foreground tracking-tight">
|
||||
{{ formatNumber(displayStats?.scanned_packages || 0) }}
|
||||
{{ formatNumber(stats?.scanned_packages || 0) }}
|
||||
</p>
|
||||
</div>
|
||||
<div class="w-12 h-12 bg-violet-50 rounded-xl flex items-center justify-center">
|
||||
@@ -98,31 +81,6 @@
|
||||
</Card>
|
||||
</div>
|
||||
|
||||
<!-- Realtime Feed -->
|
||||
<Card v-if="recentRealtimeEvents.length > 0" class="border-0 shadow-lg mb-10" data-testid="ws-event-feed">
|
||||
<CardContent class="p-6">
|
||||
<h3 class="text-xl font-semibold text-foreground mb-4">
|
||||
<i class="fas fa-bolt mr-2 text-amber-500"></i>Live Activity
|
||||
</h3>
|
||||
<ul class="space-y-2">
|
||||
<li
|
||||
v-for="(evt, idx) in recentRealtimeEvents"
|
||||
:key="`${evt.timestamp}-${idx}`"
|
||||
class="flex items-center gap-3 text-sm"
|
||||
>
|
||||
<component :is="iconForEvent(evt.type)" class="w-4 h-4 text-muted-foreground" />
|
||||
<span class="font-medium">{{ labelForEvent(evt.type) }}</span>
|
||||
<span class="text-muted-foreground truncate">
|
||||
{{ describeEvent(evt) }}
|
||||
</span>
|
||||
<span class="ml-auto text-xs text-muted-foreground">
|
||||
{{ formatTimestamp(evt.timestamp) }}
|
||||
</span>
|
||||
</li>
|
||||
</ul>
|
||||
</CardContent>
|
||||
</Card>
|
||||
|
||||
<!-- Downloads Chart -->
|
||||
<Card class="border-0 shadow-lg mb-10">
|
||||
<CardContent class="p-6">
|
||||
@@ -220,19 +178,10 @@
|
||||
</template>
|
||||
|
||||
<script setup lang="ts">
|
||||
import { computed, onMounted, ref, watch, type Component } from 'vue'
|
||||
import { computed, onMounted, ref, watch } from 'vue'
|
||||
import { storeToRefs } from 'pinia'
|
||||
import axios from 'axios'
|
||||
import {
|
||||
Package as PackageIcon,
|
||||
Download as DownloadIcon,
|
||||
Trash2 as TrashIcon,
|
||||
ShieldCheck as ShieldIcon,
|
||||
Activity as ActivityIcon,
|
||||
} from 'lucide-vue-next'
|
||||
import { usePackageStore } from '../stores/packages'
|
||||
import { useRealtimeStore } from '../stores/realtime'
|
||||
import type { EventType, RealtimeEvent } from '@/lib/ws'
|
||||
import { Alert, AlertDescription } from '@/components/ui/alert'
|
||||
import { Card, CardContent } from '@/components/ui/card'
|
||||
import { Badge } from '@/components/ui/badge'
|
||||
@@ -242,56 +191,6 @@ import { getRegistryBadgeClass } from '@/composables/useBadgeStyles'
|
||||
const store = usePackageStore()
|
||||
const { packages, stats, loading, error } = storeToRefs(store)
|
||||
|
||||
const rt = useRealtimeStore()
|
||||
const { lastStats, events: rtEvents } = storeToRefs(rt)
|
||||
|
||||
// Realtime stats override polled stats when available; otherwise fall back.
|
||||
const displayStats = computed(() => lastStats.value ?? stats.value)
|
||||
|
||||
const recentRealtimeEvents = computed<RealtimeEvent[]>(() => {
|
||||
const list = rtEvents.value
|
||||
return list.slice(Math.max(list.length - 5, 0)).reverse()
|
||||
})
|
||||
|
||||
const EVENT_LABELS: Record<EventType, string> = {
|
||||
package_cached: 'Cached',
|
||||
package_deleted: 'Deleted',
|
||||
package_downloaded: 'Downloaded',
|
||||
scan_complete: 'Scan complete',
|
||||
stats_update: 'Stats update',
|
||||
}
|
||||
|
||||
const EVENT_ICONS: Record<EventType, Component> = {
|
||||
package_cached: PackageIcon,
|
||||
package_deleted: TrashIcon,
|
||||
package_downloaded: DownloadIcon,
|
||||
scan_complete: ShieldIcon,
|
||||
stats_update: ActivityIcon,
|
||||
}
|
||||
|
||||
function iconForEvent(type: EventType): Component {
|
||||
return EVENT_ICONS[type] ?? ActivityIcon
|
||||
}
|
||||
|
||||
function labelForEvent(type: EventType): string {
|
||||
return EVENT_LABELS[type] ?? type
|
||||
}
|
||||
|
||||
function describeEvent(evt: RealtimeEvent): string {
|
||||
const p = evt.payload
|
||||
const name = typeof p.name === 'string' ? p.name : ''
|
||||
const version = typeof p.version === 'string' ? `@${p.version}` : ''
|
||||
const registry = typeof p.registry === 'string' ? `[${p.registry}] ` : ''
|
||||
if (evt.type === 'stats_update') return ''
|
||||
if (name) return `${registry}${name}${version}`
|
||||
return ''
|
||||
}
|
||||
|
||||
function formatTimestamp(ts: number): string {
|
||||
if (!ts) return ''
|
||||
return new Date(ts).toLocaleTimeString()
|
||||
}
|
||||
|
||||
// Chart periods and data
|
||||
const selectedPeriod = ref<string>('1day')
|
||||
const chartPeriods = [
|
||||
@@ -387,10 +286,6 @@ watch(selectedPeriod, () => {
|
||||
})
|
||||
|
||||
onMounted(async () => {
|
||||
// Connect to realtime stream first so we can pick up live events while
|
||||
// initial polling resolves. Singleton survives navigation, so we
|
||||
// intentionally do not disconnect on unmount.
|
||||
rt.connect()
|
||||
await store.fetchStats()
|
||||
await store.fetchPackages()
|
||||
await fetchTimeSeriesData()
|
||||
|
||||
@@ -1,56 +1,22 @@
|
||||
import { describe, it, expect, beforeEach, vi } from 'vitest'
|
||||
import { mount } from '@vue/test-utils'
|
||||
import { createPinia, setActivePinia } from 'pinia'
|
||||
import { createRouter, createMemoryHistory, type Router } from 'vue-router'
|
||||
import { defineComponent, h } from 'vue'
|
||||
import PackageList from './PackageList.vue'
|
||||
import { usePackageStore } from '../stores/packages'
|
||||
|
||||
const Stub = defineComponent({ render: () => h('div') })
|
||||
|
||||
function createTestRouter(): Router {
|
||||
return createRouter({
|
||||
history: createMemoryHistory(),
|
||||
routes: [
|
||||
{ path: '/', name: 'dashboard', component: Stub },
|
||||
{ path: '/packages/:registry?', name: 'packages', component: Stub, props: true },
|
||||
{
|
||||
path: '/package/:registry/:name+/:version',
|
||||
name: 'package-details',
|
||||
component: Stub,
|
||||
props: true,
|
||||
},
|
||||
],
|
||||
})
|
||||
}
|
||||
|
||||
async function mountWithRouter() {
|
||||
const router = createTestRouter()
|
||||
router.push('/packages')
|
||||
await router.isReady()
|
||||
return mount(PackageList, {
|
||||
global: {
|
||||
plugins: [router],
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
describe('PackageList.vue', () => {
|
||||
beforeEach(() => {
|
||||
// Create a fresh pinia instance before each test
|
||||
setActivePinia(createPinia())
|
||||
// Prevent real network calls from store actions on mount
|
||||
const store = usePackageStore()
|
||||
vi.spyOn(store, 'fetchPackages').mockResolvedValue()
|
||||
})
|
||||
|
||||
it('renders package list component', async () => {
|
||||
const wrapper = await mountWithRouter()
|
||||
it('renders package list component', () => {
|
||||
const wrapper = mount(PackageList)
|
||||
expect(wrapper.find('h2').text()).toBe('Packages')
|
||||
})
|
||||
|
||||
it('displays loading state when loading', async () => {
|
||||
const wrapper = await mountWithRouter()
|
||||
const wrapper = mount(PackageList)
|
||||
const store = usePackageStore()
|
||||
|
||||
store.loading = true
|
||||
@@ -60,7 +26,7 @@ describe('PackageList.vue', () => {
|
||||
})
|
||||
|
||||
it('displays error message when error occurs', async () => {
|
||||
const wrapper = await mountWithRouter()
|
||||
const wrapper = mount(PackageList)
|
||||
const store = usePackageStore()
|
||||
|
||||
store.error = 'Failed to fetch packages'
|
||||
@@ -70,7 +36,7 @@ describe('PackageList.vue', () => {
|
||||
})
|
||||
|
||||
it('displays empty state when no packages', async () => {
|
||||
const wrapper = await mountWithRouter()
|
||||
const wrapper = mount(PackageList)
|
||||
const store = usePackageStore()
|
||||
|
||||
store.loading = false
|
||||
@@ -81,7 +47,7 @@ describe('PackageList.vue', () => {
|
||||
})
|
||||
|
||||
it('displays package accordion when packages exist', async () => {
|
||||
const wrapper = await mountWithRouter()
|
||||
const wrapper = mount(PackageList)
|
||||
const store = usePackageStore()
|
||||
|
||||
store.loading = false
|
||||
@@ -103,19 +69,17 @@ describe('PackageList.vue', () => {
|
||||
expect(wrapper.text()).toContain('1 version')
|
||||
})
|
||||
|
||||
it('calls fetchPackages on mount', async () => {
|
||||
it('calls fetchPackages on mount', () => {
|
||||
const store = usePackageStore()
|
||||
// beforeEach already spied with mockResolvedValue; reuse that spy
|
||||
const fetchSpy = store.fetchPackages as ReturnType<typeof vi.spyOn>
|
||||
fetchSpy.mockClear()
|
||||
const fetchSpy = vi.spyOn(store, 'fetchPackages')
|
||||
|
||||
await mountWithRouter()
|
||||
mount(PackageList)
|
||||
|
||||
expect(fetchSpy).toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('groups packages and displays version counts', async () => {
|
||||
const wrapper = await mountWithRouter()
|
||||
const wrapper = mount(PackageList)
|
||||
const store = usePackageStore()
|
||||
|
||||
store.loading = false
|
||||
@@ -148,7 +112,7 @@ describe('PackageList.vue', () => {
|
||||
})
|
||||
|
||||
it('formats bytes correctly', async () => {
|
||||
const wrapper = await mountWithRouter()
|
||||
const wrapper = mount(PackageList)
|
||||
const store = usePackageStore()
|
||||
|
||||
store.loading = false
|
||||
@@ -170,7 +134,7 @@ describe('PackageList.vue', () => {
|
||||
})
|
||||
|
||||
it('applies correct registry badge classes', async () => {
|
||||
const wrapper = await mountWithRouter()
|
||||
const wrapper = mount(PackageList)
|
||||
const store = usePackageStore()
|
||||
|
||||
store.loading = false
|
||||
@@ -215,10 +179,9 @@ describe('PackageList.vue', () => {
|
||||
expect(wrapper.text()).toContain('go')
|
||||
|
||||
// Verify badge component is used with correct classes
|
||||
// (matches getRegistryBadgeClass in src/composables/useBadgeStyles.ts)
|
||||
const html = wrapper.html()
|
||||
expect(html).toContain('bg-red-100') // npm badge
|
||||
expect(html).toContain('bg-blue-100') // pypi badge
|
||||
expect(html).toContain('bg-cyan-100') // go badge
|
||||
expect(html).toContain('bg-blue-100') // npm badge
|
||||
expect(html).toContain('bg-green-100') // pypi badge
|
||||
expect(html).toContain('bg-yellow-100') // go badge
|
||||
})
|
||||
})
|
||||
|
||||
@@ -172,10 +172,7 @@ describe('Stats.vue', () => {
|
||||
}
|
||||
await wrapper.vm.$nextTick()
|
||||
|
||||
// Scope to the registry icon containers (w-12 h-12 rounded-full),
|
||||
// excluding the storage progress bar which also uses rounded-full.
|
||||
const containers = wrapper.findAll('.w-12.h-12.rounded-full')
|
||||
expect(containers).toHaveLength(3)
|
||||
const containers = wrapper.findAll('.rounded-full')
|
||||
expect(containers[0].classes()).toContain('bg-red-100') // npm
|
||||
expect(containers[1].classes()).toContain('bg-blue-100') // pypi
|
||||
expect(containers[2].classes()).toContain('bg-cyan-100') // go
|
||||
|
||||
@@ -1,223 +0,0 @@
|
||||
import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'
|
||||
import { WSClient, __wsInternals, type RealtimeEvent } from './ws'
|
||||
|
||||
/**
|
||||
* Minimal WebSocket stub mimicking the parts of the browser API the WSClient
|
||||
* touches: readyState, onopen, onmessage, onclose, onerror, send, close.
|
||||
*/
|
||||
class StubWebSocket {
|
||||
static OPEN = 1
|
||||
static CLOSED = 3
|
||||
static instances: StubWebSocket[] = []
|
||||
|
||||
url: string
|
||||
readyState = 0
|
||||
sent: string[] = []
|
||||
onopen: ((ev: Event) => void) | null = null
|
||||
onmessage: ((ev: MessageEvent) => void) | null = null
|
||||
onclose: ((ev: CloseEvent) => void) | null = null
|
||||
onerror: ((ev: Event) => void) | null = null
|
||||
|
||||
constructor(url: string) {
|
||||
this.url = url
|
||||
StubWebSocket.instances.push(this)
|
||||
}
|
||||
|
||||
// Test helpers
|
||||
open(): void {
|
||||
this.readyState = StubWebSocket.OPEN
|
||||
this.onopen?.(new Event('open'))
|
||||
}
|
||||
|
||||
receive(data: unknown): void {
|
||||
const text = typeof data === 'string' ? data : JSON.stringify(data)
|
||||
this.onmessage?.({ data: text } as MessageEvent)
|
||||
}
|
||||
|
||||
triggerClose(): void {
|
||||
this.readyState = StubWebSocket.CLOSED
|
||||
this.onclose?.({} as CloseEvent)
|
||||
}
|
||||
|
||||
// Real API
|
||||
send(payload: string): void {
|
||||
this.sent.push(payload)
|
||||
}
|
||||
|
||||
close(): void {
|
||||
this.readyState = StubWebSocket.CLOSED
|
||||
this.onclose?.({} as CloseEvent)
|
||||
}
|
||||
}
|
||||
|
||||
const originalWebSocket = (globalThis as { WebSocket?: typeof WebSocket }).WebSocket
|
||||
|
||||
beforeEach(() => {
|
||||
StubWebSocket.instances = []
|
||||
;(globalThis as unknown as { WebSocket: unknown }).WebSocket = StubWebSocket
|
||||
})
|
||||
|
||||
afterEach(() => {
|
||||
;(globalThis as unknown as { WebSocket: unknown }).WebSocket = originalWebSocket
|
||||
vi.useRealTimers()
|
||||
})
|
||||
|
||||
describe('parseEvent', () => {
|
||||
it('normalizes backend envelope (data + RFC3339 timestamp)', () => {
|
||||
const evt = __wsInternals.parseEvent(
|
||||
JSON.stringify({
|
||||
type: 'package_cached',
|
||||
timestamp: '2026-04-28T12:00:00Z',
|
||||
data: { name: 'lodash', version: '4.17.21', registry: 'npm' },
|
||||
}),
|
||||
)
|
||||
expect(evt).not.toBeNull()
|
||||
expect(evt!.type).toBe('package_cached')
|
||||
expect(evt!.payload).toMatchObject({ name: 'lodash' })
|
||||
expect(evt!.timestamp).toBe(Date.parse('2026-04-28T12:00:00Z'))
|
||||
})
|
||||
|
||||
it('drops malformed JSON', () => {
|
||||
expect(__wsInternals.parseEvent('not-json{{')).toBeNull()
|
||||
})
|
||||
|
||||
it('drops unknown event types (e.g. control frames)', () => {
|
||||
expect(
|
||||
__wsInternals.parseEvent(JSON.stringify({ type: 'pong' })),
|
||||
).toBeNull()
|
||||
})
|
||||
|
||||
it('accepts numeric timestamp', () => {
|
||||
const evt = __wsInternals.parseEvent(
|
||||
JSON.stringify({ type: 'stats_update', timestamp: 1234567890, data: {} }),
|
||||
)
|
||||
expect(evt!.timestamp).toBe(1234567890)
|
||||
})
|
||||
})
|
||||
|
||||
describe('WSClient', () => {
|
||||
it('connects and dispatches typed events to subscribers', () => {
|
||||
const client = new WSClient('ws://test/ws')
|
||||
const received: RealtimeEvent[] = []
|
||||
client.on('package_cached', (e) => received.push(e))
|
||||
client.connect()
|
||||
|
||||
const sock = StubWebSocket.instances[0]
|
||||
expect(sock).toBeDefined()
|
||||
sock.open()
|
||||
sock.receive({
|
||||
type: 'package_cached',
|
||||
timestamp: '2026-04-28T00:00:00Z',
|
||||
data: { name: 'foo' },
|
||||
})
|
||||
|
||||
expect(received).toHaveLength(1)
|
||||
expect(received[0].type).toBe('package_cached')
|
||||
expect(received[0].payload.name).toBe('foo')
|
||||
client.close()
|
||||
})
|
||||
|
||||
it('forwards events to wildcard subscribers', () => {
|
||||
const client = new WSClient('ws://test/ws')
|
||||
const seen: string[] = []
|
||||
client.on('*', (e) => seen.push(e.type))
|
||||
client.connect()
|
||||
const sock = StubWebSocket.instances[0]
|
||||
sock.open()
|
||||
sock.receive({ type: 'scan_complete', data: {} })
|
||||
sock.receive({ type: 'package_deleted', data: {} })
|
||||
expect(seen).toEqual(['scan_complete', 'package_deleted'])
|
||||
client.close()
|
||||
})
|
||||
|
||||
it('unsubscribe stops further dispatches', () => {
|
||||
const client = new WSClient('ws://test/ws')
|
||||
let count = 0
|
||||
const off = client.on('package_cached', () => {
|
||||
count += 1
|
||||
})
|
||||
client.connect()
|
||||
const sock = StubWebSocket.instances[0]
|
||||
sock.open()
|
||||
sock.receive({ type: 'package_cached', data: {} })
|
||||
expect(count).toBe(1)
|
||||
off()
|
||||
sock.receive({ type: 'package_cached', data: {} })
|
||||
expect(count).toBe(1)
|
||||
client.close()
|
||||
})
|
||||
|
||||
it('auto-reconnects with backoff after unexpected close', () => {
|
||||
vi.useFakeTimers()
|
||||
const client = new WSClient('ws://test/ws')
|
||||
client.connect()
|
||||
const first = StubWebSocket.instances[0]
|
||||
first.open()
|
||||
first.triggerClose()
|
||||
|
||||
// First reconnect happens after RECONNECT_BASE_MS (1s).
|
||||
vi.advanceTimersByTime(__wsInternals.RECONNECT_BASE_MS)
|
||||
expect(StubWebSocket.instances.length).toBe(2)
|
||||
|
||||
// Second drop -> next backoff is 2s.
|
||||
StubWebSocket.instances[1].open()
|
||||
StubWebSocket.instances[1].triggerClose()
|
||||
vi.advanceTimersByTime(__wsInternals.RECONNECT_BASE_MS * 2)
|
||||
expect(StubWebSocket.instances.length).toBe(3)
|
||||
|
||||
client.close()
|
||||
})
|
||||
|
||||
it('does not reconnect after explicit close', () => {
|
||||
vi.useFakeTimers()
|
||||
const client = new WSClient('ws://test/ws')
|
||||
client.connect()
|
||||
const sock = StubWebSocket.instances[0]
|
||||
sock.open()
|
||||
client.close()
|
||||
vi.advanceTimersByTime(60_000)
|
||||
expect(StubWebSocket.instances.length).toBe(1)
|
||||
})
|
||||
|
||||
it('emits heartbeat ping every 25s', () => {
|
||||
vi.useFakeTimers()
|
||||
const client = new WSClient('ws://test/ws')
|
||||
client.connect()
|
||||
const sock = StubWebSocket.instances[0]
|
||||
sock.open()
|
||||
expect(sock.sent).toHaveLength(0)
|
||||
vi.advanceTimersByTime(__wsInternals.HEARTBEAT_INTERVAL_MS)
|
||||
expect(sock.sent).toHaveLength(1)
|
||||
expect(JSON.parse(sock.sent[0])).toEqual({ action: 'ping' })
|
||||
vi.advanceTimersByTime(__wsInternals.HEARTBEAT_INTERVAL_MS)
|
||||
expect(sock.sent).toHaveLength(2)
|
||||
client.close()
|
||||
})
|
||||
|
||||
it('queues outbound messages while disconnected and flushes on open', () => {
|
||||
const client = new WSClient('ws://test/ws')
|
||||
client.send({ action: 'subscribe', data: ['package_cached'] })
|
||||
client.connect()
|
||||
const sock = StubWebSocket.instances[0]
|
||||
expect(sock.sent).toHaveLength(0)
|
||||
sock.open()
|
||||
expect(sock.sent).toHaveLength(1)
|
||||
expect(JSON.parse(sock.sent[0])).toMatchObject({ action: 'subscribe' })
|
||||
client.close()
|
||||
})
|
||||
|
||||
it('drops malformed inbound JSON without throwing', () => {
|
||||
const client = new WSClient('ws://test/ws')
|
||||
let count = 0
|
||||
client.on('*', () => {
|
||||
count += 1
|
||||
})
|
||||
client.connect()
|
||||
const sock = StubWebSocket.instances[0]
|
||||
sock.open()
|
||||
sock.receive('not-json{{{')
|
||||
sock.receive({ type: 'unknown_type', data: {} })
|
||||
expect(count).toBe(0)
|
||||
client.close()
|
||||
})
|
||||
})
|
||||
@@ -1,301 +0,0 @@
|
||||
/**
|
||||
* Lightweight WebSocket client with auto-reconnect, heartbeat, and pub/sub.
|
||||
*
|
||||
* Backend (pkg/websocket/server.go) emits envelopes shaped like:
|
||||
* { "type": "package_cached", "timestamp": "2026-04-28T12:34:56Z", "data": {...} }
|
||||
*
|
||||
* We normalize this into RealtimeEvent { type, payload, timestamp(number ms) }.
|
||||
*/
|
||||
|
||||
export type EventType =
|
||||
| 'package_deleted'
|
||||
| 'package_cached'
|
||||
| 'package_downloaded'
|
||||
| 'scan_complete'
|
||||
| 'stats_update'
|
||||
|
||||
export interface RealtimeEvent {
|
||||
type: EventType
|
||||
payload: Record<string, unknown>
|
||||
timestamp: number
|
||||
}
|
||||
|
||||
type WildcardListener = '*'
|
||||
type ListenerKey = EventType | WildcardListener
|
||||
type Listener = (e: RealtimeEvent) => void
|
||||
|
||||
const KNOWN_EVENT_TYPES: ReadonlySet<string> = new Set<EventType>([
|
||||
'package_deleted',
|
||||
'package_cached',
|
||||
'package_downloaded',
|
||||
'scan_complete',
|
||||
'stats_update',
|
||||
])
|
||||
|
||||
const RECONNECT_BASE_MS = 1_000
|
||||
const RECONNECT_MAX_MS = 30_000
|
||||
const HEARTBEAT_INTERVAL_MS = 25_000
|
||||
const RECONNECT_QUEUE_CAP = 50
|
||||
|
||||
interface OutboundMessage {
|
||||
action: string
|
||||
data?: unknown
|
||||
}
|
||||
|
||||
function defaultUrl(): string {
|
||||
if (typeof window === 'undefined' || !window.location) {
|
||||
return 'ws://localhost/ws'
|
||||
}
|
||||
const proto = window.location.protocol === 'https:' ? 'wss:' : 'ws:'
|
||||
return `${proto}//${window.location.host}/ws`
|
||||
}
|
||||
|
||||
function isKnownEventType(value: unknown): value is EventType {
|
||||
return typeof value === 'string' && KNOWN_EVENT_TYPES.has(value)
|
||||
}
|
||||
|
||||
function coerceTimestamp(raw: unknown): number {
|
||||
if (typeof raw === 'number' && Number.isFinite(raw)) {
|
||||
return raw
|
||||
}
|
||||
if (typeof raw === 'string') {
|
||||
const parsed = Date.parse(raw)
|
||||
if (!Number.isNaN(parsed)) return parsed
|
||||
}
|
||||
return Date.now()
|
||||
}
|
||||
|
||||
function parseEvent(text: string): RealtimeEvent | null {
|
||||
let parsed: unknown
|
||||
try {
|
||||
parsed = JSON.parse(text)
|
||||
} catch {
|
||||
return null
|
||||
}
|
||||
if (!parsed || typeof parsed !== 'object') return null
|
||||
const obj = parsed as Record<string, unknown>
|
||||
|
||||
// Server may also emit `{ type: "pong" }` style control frames; ignore those.
|
||||
if (!isKnownEventType(obj.type)) return null
|
||||
|
||||
const payloadSource = obj.payload ?? obj.data
|
||||
const payload =
|
||||
payloadSource && typeof payloadSource === 'object'
|
||||
? (payloadSource as Record<string, unknown>)
|
||||
: {}
|
||||
|
||||
return {
|
||||
type: obj.type,
|
||||
payload,
|
||||
timestamp: coerceTimestamp(obj.timestamp),
|
||||
}
|
||||
}
|
||||
|
||||
export class WSClient {
|
||||
readonly url: string
|
||||
private socket: WebSocket | null = null
|
||||
private listeners: Map<ListenerKey, Set<Listener>> = new Map()
|
||||
private heartbeatTimer: ReturnType<typeof setInterval> | null = null
|
||||
private reconnectTimer: ReturnType<typeof setTimeout> | null = null
|
||||
private reconnectAttempts = 0
|
||||
private explicitlyClosed = false
|
||||
private outboundQueue: OutboundMessage[] = []
|
||||
private _connected = false
|
||||
|
||||
constructor(url?: string) {
|
||||
this.url = url ?? defaultUrl()
|
||||
}
|
||||
|
||||
get connected(): boolean {
|
||||
return this._connected
|
||||
}
|
||||
|
||||
connect(): void {
|
||||
this.explicitlyClosed = false
|
||||
this.openSocket()
|
||||
}
|
||||
|
||||
close(): void {
|
||||
this.explicitlyClosed = true
|
||||
this.clearReconnect()
|
||||
this.stopHeartbeat()
|
||||
this._connected = false
|
||||
if (this.socket) {
|
||||
try {
|
||||
this.socket.close()
|
||||
} catch {
|
||||
/* swallow */
|
||||
}
|
||||
this.socket = null
|
||||
}
|
||||
}
|
||||
|
||||
on(type: ListenerKey, cb: Listener): () => void {
|
||||
let bucket = this.listeners.get(type)
|
||||
if (!bucket) {
|
||||
bucket = new Set()
|
||||
this.listeners.set(type, bucket)
|
||||
}
|
||||
bucket.add(cb)
|
||||
return () => {
|
||||
const current = this.listeners.get(type)
|
||||
if (!current) return
|
||||
current.delete(cb)
|
||||
if (current.size === 0) this.listeners.delete(type)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Send a message. If not connected, queue (capped) and flush on open.
|
||||
*/
|
||||
send(msg: OutboundMessage): void {
|
||||
if (this.socket && this._connected && this.socket.readyState === 1) {
|
||||
try {
|
||||
this.socket.send(JSON.stringify(msg))
|
||||
return
|
||||
} catch {
|
||||
/* fall through to queue */
|
||||
}
|
||||
}
|
||||
if (this.outboundQueue.length >= RECONNECT_QUEUE_CAP) {
|
||||
this.outboundQueue.shift()
|
||||
}
|
||||
this.outboundQueue.push(msg)
|
||||
}
|
||||
|
||||
private openSocket(): void {
|
||||
const Ctor: typeof WebSocket | undefined =
|
||||
typeof WebSocket !== 'undefined'
|
||||
? WebSocket
|
||||
: (globalThis as unknown as { WebSocket?: typeof WebSocket }).WebSocket
|
||||
if (!Ctor) {
|
||||
// No WebSocket available (e.g. SSR); silently bail.
|
||||
return
|
||||
}
|
||||
|
||||
let socket: WebSocket
|
||||
try {
|
||||
socket = new Ctor(this.url)
|
||||
} catch {
|
||||
this.scheduleReconnect()
|
||||
return
|
||||
}
|
||||
this.socket = socket
|
||||
|
||||
socket.onopen = () => {
|
||||
this._connected = true
|
||||
this.reconnectAttempts = 0
|
||||
this.startHeartbeat()
|
||||
this.flushQueue()
|
||||
}
|
||||
|
||||
socket.onmessage = (ev: MessageEvent) => {
|
||||
const data = ev.data
|
||||
if (typeof data !== 'string') return
|
||||
const evt = parseEvent(data)
|
||||
if (!evt) return
|
||||
this.dispatch(evt)
|
||||
}
|
||||
|
||||
socket.onerror = () => {
|
||||
// onclose handles reconnect; nothing to do here.
|
||||
}
|
||||
|
||||
socket.onclose = () => {
|
||||
this._connected = false
|
||||
this.stopHeartbeat()
|
||||
this.socket = null
|
||||
if (!this.explicitlyClosed) {
|
||||
this.scheduleReconnect()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private dispatch(event: RealtimeEvent): void {
|
||||
const exact = this.listeners.get(event.type)
|
||||
if (exact) {
|
||||
for (const cb of exact) {
|
||||
try {
|
||||
cb(event)
|
||||
} catch (err) {
|
||||
console.error('[ws] listener error', err)
|
||||
}
|
||||
}
|
||||
}
|
||||
const wild = this.listeners.get('*')
|
||||
if (wild) {
|
||||
for (const cb of wild) {
|
||||
try {
|
||||
cb(event)
|
||||
} catch (err) {
|
||||
console.error('[ws] listener error', err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private flushQueue(): void {
|
||||
if (!this.socket || !this._connected) return
|
||||
const queued = this.outboundQueue.splice(0)
|
||||
for (const msg of queued) {
|
||||
try {
|
||||
this.socket.send(JSON.stringify(msg))
|
||||
} catch {
|
||||
// requeue and bail
|
||||
this.outboundQueue.unshift(msg)
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private startHeartbeat(): void {
|
||||
this.stopHeartbeat()
|
||||
this.heartbeatTimer = setInterval(() => {
|
||||
if (this.socket && this._connected) {
|
||||
try {
|
||||
this.socket.send(JSON.stringify({ action: 'ping' }))
|
||||
} catch {
|
||||
/* swallow; reconnect path will handle */
|
||||
}
|
||||
}
|
||||
}, HEARTBEAT_INTERVAL_MS)
|
||||
}
|
||||
|
||||
private stopHeartbeat(): void {
|
||||
if (this.heartbeatTimer !== null) {
|
||||
clearInterval(this.heartbeatTimer)
|
||||
this.heartbeatTimer = null
|
||||
}
|
||||
}
|
||||
|
||||
private scheduleReconnect(): void {
|
||||
if (this.explicitlyClosed) return
|
||||
this.clearReconnect()
|
||||
const delay = Math.min(
|
||||
RECONNECT_MAX_MS,
|
||||
RECONNECT_BASE_MS * 2 ** this.reconnectAttempts,
|
||||
)
|
||||
this.reconnectAttempts += 1
|
||||
this.reconnectTimer = setTimeout(() => {
|
||||
this.reconnectTimer = null
|
||||
this.openSocket()
|
||||
}, delay)
|
||||
}
|
||||
|
||||
private clearReconnect(): void {
|
||||
if (this.reconnectTimer !== null) {
|
||||
clearTimeout(this.reconnectTimer)
|
||||
this.reconnectTimer = null
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Test-only helpers — exported under a namespace so production code does not
|
||||
// reach into reconnect math accidentally.
|
||||
export const __wsInternals = {
|
||||
RECONNECT_BASE_MS,
|
||||
RECONNECT_MAX_MS,
|
||||
HEARTBEAT_INTERVAL_MS,
|
||||
parseEvent,
|
||||
defaultUrl,
|
||||
}
|
||||
@@ -1,65 +0,0 @@
|
||||
import { describe, it, expect, beforeEach, afterEach } from 'vitest'
|
||||
import { createPinia, setActivePinia } from 'pinia'
|
||||
import { useRealtimeStore, __resetRealtimeSingleton } from './realtime'
|
||||
import type { RealtimeEvent } from '@/lib/ws'
|
||||
|
||||
function makeEvent(
|
||||
type: RealtimeEvent['type'],
|
||||
payload: Record<string, unknown> = {},
|
||||
timestamp = Date.now(),
|
||||
): RealtimeEvent {
|
||||
return { type, payload, timestamp }
|
||||
}
|
||||
|
||||
describe('useRealtimeStore', () => {
|
||||
beforeEach(() => {
|
||||
setActivePinia(createPinia())
|
||||
})
|
||||
|
||||
afterEach(() => {
|
||||
__resetRealtimeSingleton()
|
||||
})
|
||||
|
||||
it('appends events on ingest', () => {
|
||||
const store = useRealtimeStore()
|
||||
store.ingest(makeEvent('package_cached', { name: 'a' }))
|
||||
store.ingest(makeEvent('package_deleted', { name: 'b' }))
|
||||
expect(store.events).toHaveLength(2)
|
||||
expect(store.events[0].type).toBe('package_cached')
|
||||
})
|
||||
|
||||
it('caps events at 100 with FIFO eviction', () => {
|
||||
const store = useRealtimeStore()
|
||||
for (let i = 0; i < 150; i += 1) {
|
||||
store.ingest(makeEvent('package_cached', { i }))
|
||||
}
|
||||
expect(store.events).toHaveLength(100)
|
||||
// First event should be index 50 (0..49 dropped)
|
||||
expect((store.events[0].payload as { i: number }).i).toBe(50)
|
||||
expect((store.events[99].payload as { i: number }).i).toBe(149)
|
||||
})
|
||||
|
||||
it('updates lastStats only on stats_update events', () => {
|
||||
const store = useRealtimeStore()
|
||||
expect(store.lastStats).toBeNull()
|
||||
store.ingest(makeEvent('package_cached', { name: 'x' }))
|
||||
expect(store.lastStats).toBeNull()
|
||||
store.ingest(
|
||||
makeEvent('stats_update', {
|
||||
total_packages: 42,
|
||||
total_size: 1024,
|
||||
}),
|
||||
)
|
||||
expect(store.lastStats).toMatchObject({ total_packages: 42, total_size: 1024 })
|
||||
})
|
||||
|
||||
it('exposes filtered computed slices', () => {
|
||||
const store = useRealtimeStore()
|
||||
store.ingest(makeEvent('package_cached'))
|
||||
store.ingest(makeEvent('package_downloaded'))
|
||||
store.ingest(makeEvent('scan_complete'))
|
||||
store.ingest(makeEvent('stats_update'))
|
||||
expect(store.recentCacheActivity).toHaveLength(2)
|
||||
expect(store.recentScans).toHaveLength(1)
|
||||
})
|
||||
})
|
||||
@@ -1,118 +0,0 @@
|
||||
import { defineStore } from 'pinia'
|
||||
import { computed, ref } from 'vue'
|
||||
import { WSClient, type EventType, type RealtimeEvent } from '@/lib/ws'
|
||||
|
||||
const EVENTS_CAP = 100
|
||||
|
||||
export interface RealtimeStats {
|
||||
registry?: string
|
||||
total_packages?: number
|
||||
total_size?: number
|
||||
max_cache_size?: number
|
||||
total_downloads?: number
|
||||
scanned_packages?: number
|
||||
vulnerable_packages?: number
|
||||
blocked_packages?: number
|
||||
[key: string]: unknown
|
||||
}
|
||||
|
||||
let singletonClient: WSClient | null = null
|
||||
|
||||
function getClient(url?: string): WSClient {
|
||||
if (!singletonClient) {
|
||||
singletonClient = new WSClient(url)
|
||||
}
|
||||
return singletonClient
|
||||
}
|
||||
|
||||
// Test-only reset hook. Avoids leaking a singleton across tests.
|
||||
export function __resetRealtimeSingleton(): void {
|
||||
if (singletonClient) {
|
||||
singletonClient.close()
|
||||
}
|
||||
singletonClient = null
|
||||
}
|
||||
|
||||
export const useRealtimeStore = defineStore('realtime', () => {
|
||||
const connected = ref(false)
|
||||
const events = ref<RealtimeEvent[]>([])
|
||||
const lastStats = ref<RealtimeStats | null>(null)
|
||||
const unsubscribers = ref<Array<() => void>>([])
|
||||
|
||||
const recentCacheActivity = computed(() =>
|
||||
events.value.filter(
|
||||
(e) => e.type === 'package_cached' || e.type === 'package_downloaded',
|
||||
),
|
||||
)
|
||||
|
||||
const recentScans = computed(() =>
|
||||
events.value.filter((e) => e.type === 'scan_complete'),
|
||||
)
|
||||
|
||||
function appendEvent(event: RealtimeEvent): void {
|
||||
events.value.push(event)
|
||||
if (events.value.length > EVENTS_CAP) {
|
||||
events.value.splice(0, events.value.length - EVENTS_CAP)
|
||||
}
|
||||
if (event.type === 'stats_update') {
|
||||
lastStats.value = event.payload as RealtimeStats
|
||||
}
|
||||
}
|
||||
|
||||
function ingest(event: RealtimeEvent): void {
|
||||
appendEvent(event)
|
||||
}
|
||||
|
||||
function connect(url?: string): void {
|
||||
// Bail in SSR / non-browser test environments where there is no
|
||||
// WebSocket constructor. Keeps Dashboard.spec.ts (which mounts the
|
||||
// component without a WS stub) from leaking timers.
|
||||
const hasWS =
|
||||
typeof globalThis !== 'undefined' &&
|
||||
typeof (globalThis as { WebSocket?: unknown }).WebSocket !== 'undefined'
|
||||
if (!hasWS) return
|
||||
|
||||
const client = getClient(url)
|
||||
if (unsubscribers.value.length === 0) {
|
||||
const types: EventType[] = [
|
||||
'package_cached',
|
||||
'package_deleted',
|
||||
'package_downloaded',
|
||||
'scan_complete',
|
||||
'stats_update',
|
||||
]
|
||||
for (const t of types) {
|
||||
unsubscribers.value.push(client.on(t, ingest))
|
||||
}
|
||||
// Poll connected status — WSClient does not yet emit connection events;
|
||||
// a 1s tick is acceptable here and avoids tight coupling.
|
||||
const tick = setInterval(() => {
|
||||
connected.value = client.connected
|
||||
}, 1_000)
|
||||
unsubscribers.value.push(() => clearInterval(tick))
|
||||
}
|
||||
client.connect()
|
||||
// Reflect the current state immediately so tests/UX don't wait a tick.
|
||||
connected.value = client.connected
|
||||
}
|
||||
|
||||
function disconnect(): void {
|
||||
for (const off of unsubscribers.value) off()
|
||||
unsubscribers.value = []
|
||||
if (singletonClient) {
|
||||
singletonClient.close()
|
||||
}
|
||||
connected.value = false
|
||||
}
|
||||
|
||||
return {
|
||||
connected,
|
||||
events,
|
||||
lastStats,
|
||||
recentCacheActivity,
|
||||
recentScans,
|
||||
connect,
|
||||
disconnect,
|
||||
ingest,
|
||||
}
|
||||
})
|
||||
@@ -3,58 +3,59 @@ module github.com/lukaszraczylo/gohoarder
|
||||
go 1.25.5
|
||||
|
||||
require (
|
||||
github.com/aws/aws-sdk-go-v2 v1.42.1
|
||||
github.com/aws/aws-sdk-go-v2/config v1.32.29
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.19.28
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.105.0
|
||||
github.com/go-gormigrate/gormigrate/v2 v2.1.6
|
||||
github.com/go-sql-driver/mysql v1.10.0
|
||||
github.com/goccy/go-json v0.10.6
|
||||
github.com/gofiber/fiber/v2 v2.52.14
|
||||
github.com/aws/aws-sdk-go-v2 v1.41.1
|
||||
github.com/aws/aws-sdk-go-v2/config v1.32.7
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.19.7
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.96.0
|
||||
github.com/go-gormigrate/gormigrate/v2 v2.1.5
|
||||
github.com/go-sql-driver/mysql v1.9.3
|
||||
github.com/goccy/go-json v0.10.5
|
||||
github.com/gofiber/fiber/v2 v2.52.11
|
||||
github.com/gorilla/websocket v1.5.3
|
||||
github.com/hirochachacha/go-smb2 v1.1.0
|
||||
github.com/lib/pq v1.12.3
|
||||
github.com/lib/pq v1.11.1
|
||||
github.com/prometheus/client_golang v1.23.2
|
||||
github.com/rs/zerolog v1.35.1
|
||||
github.com/rs/zerolog v1.34.0
|
||||
github.com/spf13/cobra v1.10.2
|
||||
github.com/spf13/viper v1.21.0
|
||||
github.com/stretchr/testify v1.11.1
|
||||
github.com/testcontainers/testcontainers-go v0.40.0
|
||||
github.com/testcontainers/testcontainers-go/modules/mysql v0.40.0
|
||||
github.com/testcontainers/testcontainers-go/modules/postgres v0.40.0
|
||||
golang.org/x/crypto v0.54.0
|
||||
golang.org/x/sync v0.22.0
|
||||
golang.org/x/time v0.15.0
|
||||
golang.org/x/crypto v0.48.0
|
||||
golang.org/x/sync v0.19.0
|
||||
golang.org/x/time v0.14.0
|
||||
gorm.io/driver/mysql v1.6.0
|
||||
gorm.io/driver/postgres v1.6.0
|
||||
gorm.io/driver/sqlite v1.6.0
|
||||
gorm.io/gorm v1.31.2
|
||||
gorm.io/gorm v1.31.1
|
||||
)
|
||||
|
||||
require (
|
||||
dario.cat/mergo v1.0.2 // indirect
|
||||
filippo.io/edwards25519 v1.2.0 // indirect
|
||||
filippo.io/edwards25519 v1.1.0 // indirect
|
||||
github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c // indirect
|
||||
github.com/Microsoft/go-winio v0.6.2 // indirect
|
||||
github.com/andybalholm/brotli v1.2.2 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.14 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.30 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.30 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.30 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.31 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.13 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.23 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.30 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.31 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/signin v1.4.0 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.32.0 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.37.0 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.44.0 // indirect
|
||||
github.com/aws/smithy-go v1.27.3 // indirect
|
||||
github.com/andybalholm/brotli v1.2.0 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.17 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.17 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.17 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.17 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.4 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.8 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.17 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.17 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/signin v1.0.5 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.30.9 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.13 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.41.6 // indirect
|
||||
github.com/aws/smithy-go v1.24.0 // indirect
|
||||
github.com/beorn7/perks v1.0.1 // indirect
|
||||
github.com/cenkalti/backoff/v4 v4.3.0 // indirect
|
||||
github.com/cespare/xxhash/v2 v2.3.0 // indirect
|
||||
github.com/clipperhouse/uax29/v2 v2.7.0 // indirect
|
||||
github.com/clipperhouse/uax29/v2 v2.6.0 // indirect
|
||||
github.com/containerd/errdefs v1.0.0 // indirect
|
||||
github.com/containerd/errdefs/pkg v0.3.0 // indirect
|
||||
github.com/containerd/log v0.1.0 // indirect
|
||||
@@ -67,7 +68,7 @@ require (
|
||||
github.com/docker/go-units v0.5.0 // indirect
|
||||
github.com/ebitengine/purego v0.9.1 // indirect
|
||||
github.com/felixge/httpsnoop v1.0.4 // indirect
|
||||
github.com/fsnotify/fsnotify v1.10.1 // indirect
|
||||
github.com/fsnotify/fsnotify v1.9.0 // indirect
|
||||
github.com/geoffgarside/ber v1.2.0 // indirect
|
||||
github.com/go-logr/logr v1.4.3 // indirect
|
||||
github.com/go-logr/stdr v1.2.2 // indirect
|
||||
@@ -78,17 +79,17 @@ require (
|
||||
github.com/inconshreveable/mousetrap v1.1.0 // indirect
|
||||
github.com/jackc/pgpassfile v1.0.0 // indirect
|
||||
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
|
||||
github.com/jackc/pgx/v5 v5.10.0 // indirect
|
||||
github.com/jackc/pgx/v5 v5.8.0 // indirect
|
||||
github.com/jackc/puddle/v2 v2.2.2 // indirect
|
||||
github.com/jinzhu/inflection v1.0.0 // indirect
|
||||
github.com/jinzhu/now v1.1.5 // indirect
|
||||
github.com/klauspost/compress v1.19.0 // indirect
|
||||
github.com/klauspost/compress v1.18.4 // indirect
|
||||
github.com/lufia/plan9stats v0.0.0-20251013123823-9fd1530e3ec3 // indirect
|
||||
github.com/magiconair/properties v1.8.10 // indirect
|
||||
github.com/mattn/go-colorable v0.1.15 // indirect
|
||||
github.com/mattn/go-isatty v0.0.22 // indirect
|
||||
github.com/mattn/go-runewidth v0.0.24 // indirect
|
||||
github.com/mattn/go-sqlite3 v1.14.47 // indirect
|
||||
github.com/mattn/go-colorable v0.1.14 // indirect
|
||||
github.com/mattn/go-isatty v0.0.20 // indirect
|
||||
github.com/mattn/go-runewidth v0.0.19 // indirect
|
||||
github.com/mattn/go-sqlite3 v1.14.33 // indirect
|
||||
github.com/moby/docker-image-spec v1.3.1 // indirect
|
||||
github.com/moby/go-archive v0.2.0 // indirect
|
||||
github.com/moby/patternmatcher v0.6.0 // indirect
|
||||
@@ -100,13 +101,13 @@ require (
|
||||
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
|
||||
github.com/opencontainers/go-digest v1.0.0 // indirect
|
||||
github.com/opencontainers/image-spec v1.1.1 // indirect
|
||||
github.com/pelletier/go-toml/v2 v2.4.3 // indirect
|
||||
github.com/pelletier/go-toml/v2 v2.2.4 // indirect
|
||||
github.com/pkg/errors v0.9.1 // indirect
|
||||
github.com/pmezard/go-difflib v1.0.0 // indirect
|
||||
github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect
|
||||
github.com/prometheus/client_model v0.6.2 // indirect
|
||||
github.com/prometheus/common v0.69.0 // indirect
|
||||
github.com/prometheus/procfs v0.21.1 // indirect
|
||||
github.com/prometheus/common v0.67.5 // indirect
|
||||
github.com/prometheus/procfs v0.19.2 // indirect
|
||||
github.com/sagikazarmark/locafero v0.12.0 // indirect
|
||||
github.com/shirou/gopsutil/v4 v4.25.12 // indirect
|
||||
github.com/sirupsen/logrus v1.9.3 // indirect
|
||||
@@ -118,16 +119,17 @@ require (
|
||||
github.com/tklauser/go-sysconf v0.3.16 // indirect
|
||||
github.com/tklauser/numcpus v0.11.0 // indirect
|
||||
github.com/valyala/bytebufferpool v1.0.0 // indirect
|
||||
github.com/valyala/fasthttp v1.72.0 // indirect
|
||||
github.com/valyala/fasthttp v1.69.0 // indirect
|
||||
github.com/yusufpapurcu/wmi v1.2.4 // indirect
|
||||
go.opentelemetry.io/auto/sdk v1.2.1 // indirect
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.64.0 // indirect
|
||||
go.opentelemetry.io/otel v1.39.0 // indirect
|
||||
go.opentelemetry.io/otel/metric v1.39.0 // indirect
|
||||
go.opentelemetry.io/otel/trace v1.39.0 // indirect
|
||||
go.yaml.in/yaml/v2 v2.4.3 // indirect
|
||||
go.yaml.in/yaml/v3 v3.0.4 // indirect
|
||||
golang.org/x/sys v0.47.0 // indirect
|
||||
golang.org/x/text v0.40.0 // indirect
|
||||
golang.org/x/sys v0.41.0 // indirect
|
||||
golang.org/x/text v0.34.0 // indirect
|
||||
google.golang.org/grpc v1.78.0 // indirect
|
||||
google.golang.org/protobuf v1.36.11 // indirect
|
||||
gopkg.in/yaml.v3 v3.0.1 // indirect
|
||||
|
||||
@@ -1,59 +1,61 @@
|
||||
dario.cat/mergo v1.0.2 h1:85+piFYR1tMbRrLcDwR18y4UKJ3aH1Tbzi24VRW1TK8=
|
||||
dario.cat/mergo v1.0.2/go.mod h1:E/hbnu0NxMFBjpMIE34DRGLWqDy0g5FuKDhCb31ngxA=
|
||||
filippo.io/edwards25519 v1.2.0 h1:crnVqOiS4jqYleHd9vaKZ+HKtHfllngJIiOpNpoJsjo=
|
||||
filippo.io/edwards25519 v1.2.0/go.mod h1:xzAOLCNug/yB62zG1bQ8uziwrIqIuxhctzJT18Q77mc=
|
||||
filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA=
|
||||
filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4=
|
||||
github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6 h1:He8afgbRMd7mFxO99hRNu+6tazq8nFF9lIwo9JFroBk=
|
||||
github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6/go.mod h1:8o94RPi1/7XTJvwPpRSzSUedZrtlirdB3r9Z20bi2f8=
|
||||
github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c h1:udKWzYgxTojEKWjV8V+WSxDXJ4NFATAsZjh8iIbsQIg=
|
||||
github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E=
|
||||
github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY=
|
||||
github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU=
|
||||
github.com/andybalholm/brotli v1.2.2 h1:HzTuoo2ErYQqf5qvcJInB8uvqSVxRttzkFexPWtnceM=
|
||||
github.com/andybalholm/brotli v1.2.2/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY=
|
||||
github.com/aws/aws-sdk-go-v2 v1.42.1 h1:9eOTgu1z/dVtYpNZ3/8/XbbaX0x/BqE3HUzAzs6K0ek=
|
||||
github.com/aws/aws-sdk-go-v2 v1.42.1/go.mod h1:5pKeft2eJj+gElQ38Jqg4ibCqh+/AK33/0X3hip7IjM=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.14 h1:3IZY0XAJquT3aHzbkHfPzy4ACPcEjVG0x87KOwtpqGY=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.14/go.mod h1:zwM6veDkhGgQFqkBy+uT28AAYpLu+uFMlPl+rCg/73E=
|
||||
github.com/aws/aws-sdk-go-v2/config v1.32.29 h1:BcMHHnpiWKogf+gGfpj3K1w+Sktz29XDo/cPSAPO3FU=
|
||||
github.com/aws/aws-sdk-go-v2/config v1.32.29/go.mod h1:+Kbhn8Es4kPUph3F/0W7avykytc+Jh2Ld9/msv9ljV4=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.19.28 h1:zTXJSsNcoO91/mTXsZoYf0AK8dvNPiA58/VtyGXR+wM=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.19.28/go.mod h1:Kd9E0JzDBW/q1xbsHFrev/GnbAf5J0Ng8xoyc7HZ91Q=
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.30 h1:/hi1JADLEW9YYryEz1w4GQu0EtP23pP553Cf9KgsDV4=
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.30/go.mod h1:/3AOgy4K17Dm4ucMZVC/MJkzy5kmfKUcINRHZyo0koQ=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.30 h1:xM/Is9cKMHa8Jj8zkvWhvrFkZsXJV9E+BB4g0HW0duQ=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.30/go.mod h1:WueJeNDZvK1fMYEWJIkcivBfEzUkTpBhzlrUKKY8EuA=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.30 h1:jn46zC9LdsVR/ZpMIJqMqb8hHv31BlLx3ulVqNspUOk=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.30/go.mod h1:1hTMsAgbdS/AtUi4bw8+gUuh1pceo+eXRLfpSuSQj3M=
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.31 h1:3GUprIsfmGcC5SACIyB0e7E0BM1O1b3Erl5CePYIAeQ=
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.31/go.mod h1:7PuV1yl5e2xnUbm+RqvVg5i2iBM8EyijZNoI9wsOoOc=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.13 h1:mbRIur/BiHK6SKPjoBIXSE/hJ6g6JGRLuxQy1jGjlN4=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.13/go.mod h1:ITg9em2KbJx1s0y4aqRX5OYWG6HBZ5TVR//OdpEZ2CQ=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.23 h1:9Fjh6fi/U5JEStVZijmaMpUwE/gvBJj7x2B/PjbO9To=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.23/go.mod h1:iMoT2f1tClxrWAAnKCXjZQ6LOmfLrMG14wmnWpM+F14=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.30 h1:/Z5jmNrKsSD7EmDjzAPsm/3L9IuOkzaynklJZ1qX7S4=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.30/go.mod h1:lEzEZnOosE7zi8Z6royW1cFJTD9fpab4Ul1SBrllewk=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.31 h1:uao4A3QZ5UmB326V6KF+qRpv9Tjz7IlnlnTbbANntlU=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.31/go.mod h1:I/1+z0VwL1GhQyLgkoHDlygpUZ+iTAwOQ/NsftiUL2I=
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.105.0 h1:XptwLL+UHXgafYMIHTy59IRovLbhz3znkxY2uS/pbXU=
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.105.0/go.mod h1:zdmCoFO/dSI7GlrwsPqFJI+WlFnSU4Tc8TJnlXrM1Do=
|
||||
github.com/aws/aws-sdk-go-v2/service/signin v1.4.0 h1:sLzmJGCMv+C8KqiJgEqDLB6vxaJGmobRh4rr//ZpA3w=
|
||||
github.com/aws/aws-sdk-go-v2/service/signin v1.4.0/go.mod h1:mxC0nT/C8wMMS97DemZPzvUZxvIt+2Iq+eS3JdFZGgg=
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.32.0 h1:qjMmry/cBDee1E/2gyvel0uRYCi3mwRZ2hf6N+GAodo=
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.32.0/go.mod h1:u8af9Nqkmqnr96f7v9nHqzZT9XBwbXEkTiqT4ROuJSE=
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.37.0 h1:fpOlDPI55HdszaxapEGk6HsGosOUaM2YPWJpjMgp8UI=
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.37.0/go.mod h1:DMPWJBjYs6+3+f/qhBFEFPPlQ6NlhWjai3dJNvipJ84=
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.44.0 h1:bLZ0PolJ8J+HkJHztcXORUpHXBye2U8298lCEMi6ZCU=
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.44.0/go.mod h1:9gdl4RrflIdpDb2TlXshWgR1F9TeCkvqDx77Vpr4Z/Q=
|
||||
github.com/aws/smithy-go v1.27.3 h1:F3Zb497UhhskkfpJmfkXswyo+t0sh9OTBnIHjogWbVY=
|
||||
github.com/aws/smithy-go v1.27.3/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc=
|
||||
github.com/andybalholm/brotli v1.2.0 h1:ukwgCxwYrmACq68yiUqwIWnGY0cTPox/M94sVwToPjQ=
|
||||
github.com/andybalholm/brotli v1.2.0/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY=
|
||||
github.com/aws/aws-sdk-go-v2 v1.41.1 h1:ABlyEARCDLN034NhxlRUSZr4l71mh+T5KAeGh6cerhU=
|
||||
github.com/aws/aws-sdk-go-v2 v1.41.1/go.mod h1:MayyLB8y+buD9hZqkCW3kX1AKq07Y5pXxtgB+rRFhz0=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4 h1:489krEF9xIGkOaaX3CE/Be2uWjiXrkCH6gUX+bZA/BU=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4/go.mod h1:IOAPF6oT9KCsceNTvvYMNHy0+kMF8akOjeDvPENWxp4=
|
||||
github.com/aws/aws-sdk-go-v2/config v1.32.7 h1:vxUyWGUwmkQ2g19n7JY/9YL8MfAIl7bTesIUykECXmY=
|
||||
github.com/aws/aws-sdk-go-v2/config v1.32.7/go.mod h1:2/Qm5vKUU/r7Y+zUk/Ptt2MDAEKAfUtKc1+3U1Mo3oY=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.19.7 h1:tHK47VqqtJxOymRrNtUXN5SP/zUTvZKeLx4tH6PGQc8=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.19.7/go.mod h1:qOZk8sPDrxhf+4Wf4oT2urYJrYt3RejHSzgAquYeppw=
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.17 h1:I0GyV8wiYrP8XpA70g1HBcQO1JlQxCMTW9npl5UbDHY=
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.17/go.mod h1:tyw7BOl5bBe/oqvoIeECFJjMdzXoa/dfVz3QQ5lgHGA=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.17 h1:xOLELNKGp2vsiteLsvLPwxC+mYmO6OZ8PYgiuPJzF8U=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.17/go.mod h1:5M5CI3D12dNOtH3/mk6minaRwI2/37ifCURZISxA/IQ=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.17 h1:WWLqlh79iO48yLkj1v3ISRNiv+3KdQoZ6JWyfcsyQik=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.17/go.mod h1:EhG22vHRrvF8oXSTYStZhJc1aUgKtnJe+aOiFEV90cM=
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 h1:WKuaxf++XKWlHWu9ECbMlha8WOEGm0OUEZqm4K/Gcfk=
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4/go.mod h1:ZWy7j6v1vWGmPReu0iSGvRiise4YI5SkR3OHKTZ6Wuc=
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.17 h1:JqcdRG//czea7Ppjb+g/n4o8i/R50aTBHkA7vu0lK+k=
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.17/go.mod h1:CO+WeGmIdj/MlPel2KwID9Gt7CNq4M65HUfBW97liM0=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.4 h1:0ryTNEdJbzUCEWkVXEXoqlXV72J5keC1GvILMOuD00E=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.4/go.mod h1:HQ4qwNZh32C3CBeO6iJLQlgtMzqeG17ziAA/3KDJFow=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.8 h1:Z5EiPIzXKewUQK0QTMkutjiaPVeVYXX7KIqhXu/0fXs=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.8/go.mod h1:FsTpJtvC4U1fyDXk7c71XoDv3HlRm8V3NiYLeYLh5YE=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.17 h1:RuNSMoozM8oXlgLG/n6WLaFGoea7/CddrCfIiSA+xdY=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.17/go.mod h1:F2xxQ9TZz5gDWsclCtPQscGpP0VUOc8RqgFM3vDENmU=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.17 h1:bGeHBsGZx0Dvu/eJC0Lh9adJa3M1xREcndxLNZlve2U=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.17/go.mod h1:dcW24lbU0CzHusTE8LLHhRLI42ejmINN8Lcr22bwh/g=
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.96.0 h1:oeu8VPlOre74lBA/PMhxa5vewaMIMmILM+RraSyB8KA=
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.96.0/go.mod h1:5jggDlZ2CLQhwJBiZJb4vfk4f0GxWdEDruWKEJ1xOdo=
|
||||
github.com/aws/aws-sdk-go-v2/service/signin v1.0.5 h1:VrhDvQib/i0lxvr3zqlUwLwJP4fpmpyD9wYG1vfSu+Y=
|
||||
github.com/aws/aws-sdk-go-v2/service/signin v1.0.5/go.mod h1:k029+U8SY30/3/ras4G/Fnv/b88N4mAfliNn08Dem4M=
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.30.9 h1:v6EiMvhEYBoHABfbGB4alOYmCIrcgyPPiBE1wZAEbqk=
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.30.9/go.mod h1:yifAsgBxgJWn3ggx70A3urX2AN49Y5sJTD1UQFlfqBw=
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.13 h1:gd84Omyu9JLriJVCbGApcLzVR3XtmC4ZDPcAI6Ftvds=
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.13/go.mod h1:sTGThjphYE4Ohw8vJiRStAcu3rbjtXRsdNB0TvZ5wwo=
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.41.6 h1:5fFjR/ToSOzB2OQ/XqWpZBmNvmP/pJ1jOWYlFDJTjRQ=
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.41.6/go.mod h1:qgFDZQSD/Kys7nJnVqYlWKnh0SSdMjAi0uSwON4wgYQ=
|
||||
github.com/aws/smithy-go v1.24.0 h1:LpilSUItNPFr1eY85RYgTIg5eIEPtvFbskaFcmmIUnk=
|
||||
github.com/aws/smithy-go v1.24.0/go.mod h1:LEj2LM3rBRQJxPZTB4KuzZkaZYnZPnvgIhb4pu07mx0=
|
||||
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
|
||||
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
|
||||
github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8=
|
||||
github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE=
|
||||
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
|
||||
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
||||
github.com/clipperhouse/uax29/v2 v2.7.0 h1:+gs4oBZ2gPfVrKPthwbMzWZDaAFPGYK72F0NJv2v7Vk=
|
||||
github.com/clipperhouse/uax29/v2 v2.7.0/go.mod h1:EFJ2TJMRUaplDxHKj1qAEhCtQPW2tJSwu5BF98AuoVM=
|
||||
github.com/clipperhouse/uax29/v2 v2.6.0 h1:z0cDbUV+aPASdFb2/ndFnS9ts/WNXgTNNGFoKXuhpos=
|
||||
github.com/clipperhouse/uax29/v2 v2.6.0/go.mod h1:Wn1g7MK6OoeDT0vL+Q0SQLDz/KpfsVRgg6W7ihQeh4g=
|
||||
github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI=
|
||||
github.com/containerd/errdefs v1.0.0/go.mod h1:+YBYIdtsnF4Iw6nWZhJcqGSg/dwvV7tyJ/kCkyJ2k+M=
|
||||
github.com/containerd/errdefs/pkg v0.3.0 h1:9IKJ06FvyNlexW690DXuQNx2KA2cUJXx151Xdx3ZPPE=
|
||||
@@ -62,6 +64,7 @@ github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I=
|
||||
github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo=
|
||||
github.com/containerd/platforms v0.2.1 h1:zvwtM3rz2YHPQsF2CHYM8+KtB5dvhISiXh5ZpSBQv6A=
|
||||
github.com/containerd/platforms v0.2.1/go.mod h1:XHCb+2/hzowdiut9rkudds9bE5yJ7npe7dG/wG+uFPw=
|
||||
github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
|
||||
github.com/cpuguy83/dockercfg v0.3.2 h1:DlJTyZGBDlXqUZ2Dk2Q3xHs/FtnooJJVaad2S9GKorA=
|
||||
github.com/cpuguy83/dockercfg v0.3.2/go.mod h1:sugsbF4//dDlL/i+S+rtpIWp+5h0BHJHfjj5/jFyUJc=
|
||||
github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
|
||||
@@ -84,13 +87,13 @@ github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2
|
||||
github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
|
||||
github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8=
|
||||
github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
|
||||
github.com/fsnotify/fsnotify v1.10.1 h1:b0/UzAf9yR5rhf3RPm9gf3ehBPpf0oZKIjtpKrx59Ho=
|
||||
github.com/fsnotify/fsnotify v1.10.1/go.mod h1:TLheqan6HD6GBK6PrDWyDPBaEV8LspOxvPSjC+bVfgo=
|
||||
github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k=
|
||||
github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0=
|
||||
github.com/geoffgarside/ber v1.1.0/go.mod h1:jVPKeCbj6MvQZhwLYsGwaGI52oUorHoHKNecGT85ZCc=
|
||||
github.com/geoffgarside/ber v1.2.0 h1:/loowoRcs/MWLYmGX9QtIAbA+V/FrnVLsMMPhwiRm64=
|
||||
github.com/geoffgarside/ber v1.2.0/go.mod h1:jVPKeCbj6MvQZhwLYsGwaGI52oUorHoHKNecGT85ZCc=
|
||||
github.com/go-gormigrate/gormigrate/v2 v2.1.6 h1:VtX+l1Stj2v5RGubVQk0LS/8EPGXR+ldcOyCmlmKoyg=
|
||||
github.com/go-gormigrate/gormigrate/v2 v2.1.6/go.mod h1:PZpedQc4tWaxn6kvXicwhinh3L0seLpMc5ReKRX5id4=
|
||||
github.com/go-gormigrate/gormigrate/v2 v2.1.5 h1:1OyorA5LtdQw12cyJDEHuTrEV3GiXiIhS4/QTTa/SM8=
|
||||
github.com/go-gormigrate/gormigrate/v2 v2.1.5/go.mod h1:mj9ekk/7CPF3VjopaFvWKN2v7fN3D9d3eEOAXRhi/+M=
|
||||
github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
|
||||
github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI=
|
||||
github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
|
||||
@@ -99,14 +102,15 @@ github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre
|
||||
github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
|
||||
github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE=
|
||||
github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78=
|
||||
github.com/go-sql-driver/mysql v1.10.0 h1:Q+1LV8DkHJvSYAdR83XzuhDaTykuDx0l6fkXxoWCWfw=
|
||||
github.com/go-sql-driver/mysql v1.10.0/go.mod h1:M+cqaI7+xxXGG9swrdeUIoPG3Y3KCkF0pZej+SK+nWk=
|
||||
github.com/go-sql-driver/mysql v1.9.3 h1:U/N249h2WzJ3Ukj8SowVFjdtZKfu9vlLZxjPXV1aweo=
|
||||
github.com/go-sql-driver/mysql v1.9.3/go.mod h1:qn46aNg1333BRMNU69Lq93t8du/dwxI64Gl8i5p1WMU=
|
||||
github.com/go-viper/mapstructure/v2 v2.5.0 h1:vM5IJoUAy3d7zRSVtIwQgBj7BiWtMPfmPEgAXnvj1Ro=
|
||||
github.com/go-viper/mapstructure/v2 v2.5.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM=
|
||||
github.com/goccy/go-json v0.10.6 h1:p8HrPJzOakx/mn/bQtjgNjdTcN+/S6FcG2CTtQOrHVU=
|
||||
github.com/goccy/go-json v0.10.6/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M=
|
||||
github.com/gofiber/fiber/v2 v2.52.14 h1:Of3L+9qVFaQNwPlcmEdl5IIodHz8BSE0j37R7rWu4pE=
|
||||
github.com/gofiber/fiber/v2 v2.52.14/go.mod h1:YEcBbO/FB+5M1IZNBP9FO3J9281zgPAreiI1oqg8nDw=
|
||||
github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4=
|
||||
github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M=
|
||||
github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
|
||||
github.com/gofiber/fiber/v2 v2.52.11 h1:5f4yzKLcBcF8ha1GQTWB+mpblWz3Vz6nSAbTL31HkWs=
|
||||
github.com/gofiber/fiber/v2 v2.52.11/go.mod h1:YEcBbO/FB+5M1IZNBP9FO3J9281zgPAreiI1oqg8nDw=
|
||||
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
|
||||
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
|
||||
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
||||
@@ -123,36 +127,39 @@ github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsI
|
||||
github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
|
||||
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo=
|
||||
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM=
|
||||
github.com/jackc/pgx/v5 v5.10.0 h1:VhSvgU2jSli8o3AqIEOTJr7rZwAEUVo4E4XhR94Zfr0=
|
||||
github.com/jackc/pgx/v5 v5.10.0/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4=
|
||||
github.com/jackc/pgx/v5 v5.8.0 h1:TYPDoleBBme0xGSAX3/+NujXXtpZn9HBONkQC7IEZSo=
|
||||
github.com/jackc/pgx/v5 v5.8.0/go.mod h1:QVeDInX2m9VyzvNeiCJVjCkNFqzsNb43204HshNSZKw=
|
||||
github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo=
|
||||
github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
|
||||
github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E=
|
||||
github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
|
||||
github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ=
|
||||
github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
|
||||
github.com/klauspost/compress v1.19.0 h1:sXLILfc9jV2QYWkzFOPWStmcUVH2RHEB1JCdY2oVvCQ=
|
||||
github.com/klauspost/compress v1.19.0/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
|
||||
github.com/klauspost/compress v1.18.4 h1:RPhnKRAQ4Fh8zU2FY/6ZFDwTVTxgJ/EMydqSTzE9a2c=
|
||||
github.com/klauspost/compress v1.18.4/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4=
|
||||
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
|
||||
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
|
||||
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
|
||||
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
|
||||
github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
|
||||
github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
|
||||
github.com/lib/pq v1.12.3 h1:tTWxr2YLKwIvK90ZXEw8GP7UFHtcbTtty8zsI+YjrfQ=
|
||||
github.com/lib/pq v1.12.3/go.mod h1:/p+8NSbOcwzAEI7wiMXFlgydTwcgTr3OSKMsD2BitpA=
|
||||
github.com/lib/pq v1.11.1 h1:wuChtj2hfsGmmx3nf1m7xC2XpK6OtelS2shMY+bGMtI=
|
||||
github.com/lib/pq v1.11.1/go.mod h1:/p+8NSbOcwzAEI7wiMXFlgydTwcgTr3OSKMsD2BitpA=
|
||||
github.com/lufia/plan9stats v0.0.0-20251013123823-9fd1530e3ec3 h1:PwQumkgq4/acIiZhtifTV5OUqqiP82UAl0h87xj/l9k=
|
||||
github.com/lufia/plan9stats v0.0.0-20251013123823-9fd1530e3ec3/go.mod h1:autxFIvghDt3jPTLoqZ9OZ7s9qTGNAWmYCjVFWPX/zg=
|
||||
github.com/magiconair/properties v1.8.10 h1:s31yESBquKXCV9a/ScB3ESkOjUYYv+X0rg8SYxI99mE=
|
||||
github.com/magiconair/properties v1.8.10/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0=
|
||||
github.com/mattn/go-colorable v0.1.15 h1:+u9SLTRGnXv73cEsnsmoZBom+dMU88B2M0aDcWy0/jY=
|
||||
github.com/mattn/go-colorable v0.1.15/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8=
|
||||
github.com/mattn/go-isatty v0.0.22 h1:j8l17JJ9i6VGPUFUYoTUKPSgKe/83EYU2zBC7YNKMw4=
|
||||
github.com/mattn/go-isatty v0.0.22/go.mod h1:ZXfXG4SQHsB/w3ZeOYbR0PrPwLy+n6xiMrJlRFqopa4=
|
||||
github.com/mattn/go-runewidth v0.0.24 h1:cpokDiIn0MGnhdHwuWnJBITySJ20QyNGnY2kR/ay2DU=
|
||||
github.com/mattn/go-runewidth v0.0.24/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhgLpndooCuJAs=
|
||||
github.com/mattn/go-sqlite3 v1.14.47 h1:jOBI62gS7nKeZv+as1oGEy0+1qISgXwH/QBlR6KbfIo=
|
||||
github.com/mattn/go-sqlite3 v1.14.47/go.mod h1:6JTjA44L93a0QCyJef5YvlPoKXntQPjzWv5gtm9sB6w=
|
||||
github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
|
||||
github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE=
|
||||
github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8=
|
||||
github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
|
||||
github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
|
||||
github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
|
||||
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
|
||||
github.com/mattn/go-runewidth v0.0.19 h1:v++JhqYnZuu5jSKrk9RbgF5v4CGUjqRfBm05byFGLdw=
|
||||
github.com/mattn/go-runewidth v0.0.19/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhgLpndooCuJAs=
|
||||
github.com/mattn/go-sqlite3 v1.14.33 h1:A5blZ5ulQo2AtayQ9/limgHEkFreKj1Dv226a1K73s0=
|
||||
github.com/mattn/go-sqlite3 v1.14.33/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
|
||||
github.com/mdelapenya/tlscert v0.2.0 h1:7H81W6Z/4weDvZBNOfQte5GpIMo0lGYEeWbkGp5LJHI=
|
||||
github.com/mdelapenya/tlscert v0.2.0/go.mod h1:O4njj3ELLnJjGdkN7M/vIVCpZ+Cf0L6muqOG4tLSl8o=
|
||||
github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0=
|
||||
@@ -179,8 +186,8 @@ github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8
|
||||
github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
|
||||
github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040=
|
||||
github.com/opencontainers/image-spec v1.1.1/go.mod h1:qpqAh3Dmcf36wStyyWU+kCeDgrGnAve2nCC8+7h8Q0M=
|
||||
github.com/pelletier/go-toml/v2 v2.4.3 h1:GTRvJQutkOSftxIFD5xw9aepkYNuPWmVJpffdDPYVpY=
|
||||
github.com/pelletier/go-toml/v2 v2.4.3/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY=
|
||||
github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4=
|
||||
github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY=
|
||||
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
|
||||
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
@@ -191,14 +198,15 @@ github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h
|
||||
github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg=
|
||||
github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk=
|
||||
github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE=
|
||||
github.com/prometheus/common v0.69.0 h1:OA85nJQS/T/MaYh/Q2CcgDKSGWqNIgrBDvDH85CuiNk=
|
||||
github.com/prometheus/common v0.69.0/go.mod h1:ZzL3f6u94qUxh9p+tJTrF+FvBS1XXbbRAZCQkytAL0Y=
|
||||
github.com/prometheus/procfs v0.21.1 h1:GljZCt+zSTS+NZq88cyQ1LjZ+RCHp3uVuabBWA5+OJI=
|
||||
github.com/prometheus/procfs v0.21.1/go.mod h1:aB55Cww9pdSJVHk0hUf0inxWyyjPogFIjmHKYgMKmtY=
|
||||
github.com/prometheus/common v0.67.5 h1:pIgK94WWlQt1WLwAC5j2ynLaBRDiinoAb86HZHTUGI4=
|
||||
github.com/prometheus/common v0.67.5/go.mod h1:SjE/0MzDEEAyrdr5Gqc6G+sXI67maCxzaT3A2+HqjUw=
|
||||
github.com/prometheus/procfs v0.19.2 h1:zUMhqEW66Ex7OXIiDkll3tl9a1ZdilUOd/F6ZXw4Vws=
|
||||
github.com/prometheus/procfs v0.19.2/go.mod h1:M0aotyiemPhBCM0z5w87kL22CxfcH05ZpYlu+b4J7mw=
|
||||
github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ=
|
||||
github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc=
|
||||
github.com/rs/zerolog v1.35.1 h1:m7xQeoiLIiV0BCEY4Hs+j2NG4Gp2o2KPKmhnnLiazKI=
|
||||
github.com/rs/zerolog v1.35.1/go.mod h1:EjML9kdfa/RMA7h/6z6pYmq1ykOuA8/mjWaEvGI+jcw=
|
||||
github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0=
|
||||
github.com/rs/zerolog v1.34.0 h1:k43nTLIwcTVQAncfCw4KZ2VY6ukYoZaBPNOE8txlOeY=
|
||||
github.com/rs/zerolog v1.34.0/go.mod h1:bJsvje4Z08ROH4Nhs5iH600c3IkWhwp44iRc54W6wYQ=
|
||||
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
|
||||
github.com/sagikazarmark/locafero v0.12.0 h1:/NQhBAkUb4+fH1jivKHWusDYFjMOOKU88eegjfxfHb4=
|
||||
github.com/sagikazarmark/locafero v0.12.0/go.mod h1:sZh36u/YSZ918v0Io+U9ogLYQJ9tLLBmM4eneO6WwsI=
|
||||
@@ -238,8 +246,8 @@ github.com/tklauser/numcpus v0.11.0 h1:nSTwhKH5e1dMNsCdVBukSZrURJRoHbSEQjdEbY+9R
|
||||
github.com/tklauser/numcpus v0.11.0/go.mod h1:z+LwcLq54uWZTX0u/bGobaV34u6V7KNlTZejzM6/3MQ=
|
||||
github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
|
||||
github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
|
||||
github.com/valyala/fasthttp v1.72.0 h1:R7kYdoWhn1ye1fVpP+cDHDJwYm3NkwLliwgzJ/Abg7M=
|
||||
github.com/valyala/fasthttp v1.72.0/go.mod h1:zsbLTYqcpIktdQytlVBwIjY9La5d6bs990nBxWg8efk=
|
||||
github.com/valyala/fasthttp v1.69.0 h1:fNLLESD2SooWeh2cidsuFtOcrEi4uB4m1mPrkJMZyVI=
|
||||
github.com/valyala/fasthttp v1.69.0/go.mod h1:4wA4PfAraPlAsJ5jMSqCE2ug5tqUPwKXxVj8oNECGcw=
|
||||
github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU=
|
||||
github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E=
|
||||
github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0=
|
||||
@@ -266,35 +274,38 @@ go.opentelemetry.io/proto/otlp v1.0.0 h1:T0TX0tmXU8a3CbNXzEKGeU5mIVOdf0oykP+u2lI
|
||||
go.opentelemetry.io/proto/otlp v1.0.0/go.mod h1:Sy6pihPLfYHkr3NkUbEhGHFhINUSI/v80hjKIs5JXpM=
|
||||
go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
|
||||
go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
|
||||
go.yaml.in/yaml/v2 v2.4.4 h1:tuyd0P+2Ont/d6e2rl3be67goVK4R6deVxCUX5vyPaQ=
|
||||
go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ=
|
||||
go.yaml.in/yaml/v2 v2.4.3 h1:6gvOSjQoTB3vt1l+CU+tSyi/HOjfOjRLJ4YwYZGwRO0=
|
||||
go.yaml.in/yaml/v2 v2.4.3/go.mod h1:zSxWcmIDjOzPXpjlTTbAsKokqkDNAVtZO0WOMiT90s8=
|
||||
go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc=
|
||||
go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
|
||||
golang.org/x/crypto v0.54.0 h1:YLIA59K4fiNzHzjnZt2tUJQjQtUWfWbeHBqKtk3eScw=
|
||||
golang.org/x/crypto v0.54.0/go.mod h1:KWL8ny2AZdGR2cWmzeHrp2azQPGogOv+HeQaVEXC2dk=
|
||||
golang.org/x/crypto v0.48.0 h1:/VRzVqiRSggnhY7gNRxPauEQ5Drw9haKdM0jqfcCFts=
|
||||
golang.org/x/crypto v0.48.0/go.mod h1:r0kV5h3qnFPlQnBSrULhlsRfryS2pmewsg+XfMgkVos=
|
||||
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.56.0 h1:Rw8j/hFzGvJUZwNBXnAtf5sVDVt+65SK2C7IxCxZt5o=
|
||||
golang.org/x/net v0.56.0/go.mod h1:D3Ku6r+V6JROoZK144D2XfMHFcMq/0zSfLelVTCFKec=
|
||||
golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek=
|
||||
golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
|
||||
golang.org/x/net v0.49.0 h1:eeHFmOGUTtaaPSGNmjBKpbng9MulQsJURQUAfUwY++o=
|
||||
golang.org/x/net v0.49.0/go.mod h1:/ysNB2EvaqvesRkuLAyjI1ycPZlQHM3q01F02UY/MV8=
|
||||
golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4=
|
||||
golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
|
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs=
|
||||
golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
|
||||
golang.org/x/term v0.45.0 h1:NwWyBmoJCbfTHpxrWoZ9C6/VxOf7ic219I8xZZFdrf0=
|
||||
golang.org/x/term v0.45.0/go.mod h1:9aqxs0blBcrm/n0L9QW0aRVD+ktan8ssZromtqJC43w=
|
||||
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.41.0 h1:Ivj+2Cp/ylzLiEU89QhWblYnOE9zerudt9Ftecq2C6k=
|
||||
golang.org/x/sys v0.41.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
|
||||
golang.org/x/term v0.40.0 h1:36e4zGLqU4yhjlmxEaagx2KuYbJq3EwY8K943ZsHcvg=
|
||||
golang.org/x/term v0.40.0/go.mod h1:w2P8uVp06p2iyKKuvXIm7N/y0UCRt3UfJTfZ7oOpglM=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.40.0 h1:Ub2Z6/xjgF1WrYQz2nuITOEegKFtiIy+rieRJ5lHZKs=
|
||||
golang.org/x/text v0.40.0/go.mod h1:hpnzDAfGV753zIKo+wk3u1bVKCGPbrnF7+7LBF/UHVY=
|
||||
golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U=
|
||||
golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno=
|
||||
golang.org/x/text v0.34.0 h1:oL/Qq0Kdaqxa1KbNeMKwQq0reLCCaFtqu2eNuSeNHbk=
|
||||
golang.org/x/text v0.34.0/go.mod h1:homfLqTYRFyVYemLBFl5GgL/DWEiH5wcsQ5gSh1yziA=
|
||||
golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI=
|
||||
golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4=
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20251222181119-0a764e51fe1b h1:uA40e2M6fYRBf0+8uN5mLlqUtV192iiksiICIBkYJ1E=
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20251222181119-0a764e51fe1b/go.mod h1:Xa7le7qx2vmqB/SzWUBa7KdMjpdpAHlh5QCSnjessQk=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20251222181119-0a764e51fe1b h1:Mv8VFug0MP9e5vUxfBcE3vUkV6CImK3cMNMIDFjmzxU=
|
||||
@@ -315,7 +326,7 @@ gorm.io/driver/postgres v1.6.0 h1:2dxzU8xJ+ivvqTRph34QX+WrRaJlmfyPqXmoGVjMBa4=
|
||||
gorm.io/driver/postgres v1.6.0/go.mod h1:vUw0mrGgrTK+uPHEhAdV4sfFELrByKVGnaVRkXDhtWo=
|
||||
gorm.io/driver/sqlite v1.6.0 h1:WHRRrIiulaPiPFmDcod6prc4l2VGVWHz80KspNsxSfQ=
|
||||
gorm.io/driver/sqlite v1.6.0/go.mod h1:AO9V1qIQddBESngQUKWL9yoH93HIeA1X6V633rBwyT8=
|
||||
gorm.io/gorm v1.31.2 h1:3o8FXNo9v9S858gil+3LlZA1LkCOzgb4g5BL64FgaCo=
|
||||
gorm.io/gorm v1.31.2/go.mod h1:XyQVbO2k6YkOis7C2437jSit3SsDK72s7n7rsSHd+Gs=
|
||||
gorm.io/gorm v1.31.1 h1:7CA8FTFz/gRfgqgpeKIBcervUn3xSyPUmr6B2WXJ7kg=
|
||||
gorm.io/gorm v1.31.1/go.mod h1:XyQVbO2k6YkOis7C2437jSit3SsDK72s7n7rsSHd+Gs=
|
||||
gotest.tools/v3 v3.5.2 h1:7koQfIKdy+I8UTetycgUqXWSDwpgv193Ka+qRsmBY8Q=
|
||||
gotest.tools/v3 v3.5.2/go.mod h1:LtdLGcnqToBH83WByAAi/wiwSFCArdFIUV/xxN4pcjA=
|
||||
|
||||
@@ -1,129 +0,0 @@
|
||||
{{- if and .Values.ingress.enabled (eq .Values.ingress.className "traefik") -}}
|
||||
---
|
||||
# Traefik IngressRoute for package registry paths
|
||||
# This handles package downloads including scoped packages with encoded slashes
|
||||
apiVersion: traefik.io/v1alpha1
|
||||
kind: IngressRoute
|
||||
metadata:
|
||||
name: {{ include "gohoarder.fullname" . }}-packages
|
||||
labels:
|
||||
{{- include "gohoarder.labels" . | nindent 4 }}
|
||||
spec:
|
||||
entryPoints:
|
||||
- websecure
|
||||
routes:
|
||||
# Package registry routes with high priority
|
||||
# PathPrefix matching to avoid Go's encoded slash rejection in regex
|
||||
- match: Host(`{{ .Values.ingress.host | default (printf "%s.%s" "gohoarder" .Values.global.domain) }}`) && PathPrefix(`/npm/`)
|
||||
kind: Rule
|
||||
priority: 100
|
||||
services:
|
||||
- name: {{ include "gohoarder.fullname" . }}-frontend
|
||||
port: {{ .Values.frontend.service.port }}
|
||||
middlewares:
|
||||
- name: {{ include "gohoarder.fullname" . }}-strip-encoded-slash
|
||||
namespace: {{ .Release.Namespace }}
|
||||
{{- if .Values.ingress.traefik }}
|
||||
{{- if .Values.ingress.traefik.middlewares }}
|
||||
{{- range .Values.ingress.traefik.middlewares }}
|
||||
- name: {{ . }}
|
||||
namespace: traefik
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
||||
- match: Host(`{{ .Values.ingress.host | default (printf "%s.%s" "gohoarder" .Values.global.domain) }}`) && PathPrefix(`/pypi/`)
|
||||
kind: Rule
|
||||
priority: 100
|
||||
services:
|
||||
- name: {{ include "gohoarder.fullname" . }}-frontend
|
||||
port: {{ .Values.frontend.service.port }}
|
||||
middlewares:
|
||||
- name: {{ include "gohoarder.fullname" . }}-strip-encoded-slash
|
||||
namespace: {{ .Release.Namespace }}
|
||||
{{- if .Values.ingress.traefik }}
|
||||
{{- if .Values.ingress.traefik.middlewares }}
|
||||
{{- range .Values.ingress.traefik.middlewares }}
|
||||
- name: {{ . }}
|
||||
namespace: traefik
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
||||
- match: Host(`{{ .Values.ingress.host | default (printf "%s.%s" "gohoarder" .Values.global.domain) }}`) && PathPrefix(`/go/`)
|
||||
kind: Rule
|
||||
priority: 100
|
||||
services:
|
||||
- name: {{ include "gohoarder.fullname" . }}-frontend
|
||||
port: {{ .Values.frontend.service.port }}
|
||||
middlewares:
|
||||
- name: {{ include "gohoarder.fullname" . }}-strip-encoded-slash
|
||||
namespace: {{ .Release.Namespace }}
|
||||
{{- if .Values.ingress.traefik }}
|
||||
{{- if .Values.ingress.traefik.middlewares }}
|
||||
{{- range .Values.ingress.traefik.middlewares }}
|
||||
- name: {{ . }}
|
||||
namespace: traefik
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
||||
# All other routes (frontend, API, etc.) with lower priority
|
||||
- match: Host(`{{ .Values.ingress.host | default (printf "%s.%s" "gohoarder" .Values.global.domain) }}`)
|
||||
kind: Rule
|
||||
priority: 50
|
||||
services:
|
||||
- name: {{ include "gohoarder.fullname" . }}-frontend
|
||||
port: {{ .Values.frontend.service.port }}
|
||||
{{- if .Values.ingress.traefik }}
|
||||
{{- if .Values.ingress.traefik.middlewares }}
|
||||
middlewares:
|
||||
{{- range .Values.ingress.traefik.middlewares }}
|
||||
- name: {{ . }}
|
||||
namespace: traefik
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
||||
{{- if .Values.ingress.tls.enabled }}
|
||||
tls:
|
||||
secretName: {{ .Values.ingress.tls.secretName }}
|
||||
{{- end }}
|
||||
|
||||
---
|
||||
# Middleware to handle encoded slashes in package paths
|
||||
# Note: This middleware attempts to work around Traefik's encoded slash limitation
|
||||
# but may not fully resolve the issue due to Go stdlib constraints
|
||||
apiVersion: traefik.io/v1alpha1
|
||||
kind: Middleware
|
||||
metadata:
|
||||
name: {{ include "gohoarder.fullname" . }}-strip-encoded-slash
|
||||
labels:
|
||||
{{- include "gohoarder.labels" . | nindent 4 }}
|
||||
spec:
|
||||
# Use headers to pass original path information
|
||||
# The backend can reconstruct the original URL from these headers
|
||||
headers:
|
||||
customRequestHeaders:
|
||||
X-Original-URI: "{{ .Request.URL.Path }}"
|
||||
X-Forwarded-Path: "{{ .Request.URL.Path }}"
|
||||
|
||||
---
|
||||
# ServersTransport configuration for backend communication
|
||||
# This ensures the backend connection doesn't have additional restrictions
|
||||
apiVersion: traefik.io/v1alpha1
|
||||
kind: ServersTransport
|
||||
metadata:
|
||||
name: {{ include "gohoarder.fullname" . }}-transport
|
||||
labels:
|
||||
{{- include "gohoarder.labels" . | nindent 4 }}
|
||||
spec:
|
||||
insecureSkipVerify: false
|
||||
{{- if .Values.ingress.traefik }}
|
||||
{{- if .Values.ingress.traefik.transport }}
|
||||
{{- with .Values.ingress.traefik.transport }}
|
||||
serverName: {{ .serverName | default "" }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
@@ -514,16 +514,6 @@ ingress:
|
||||
enabled: false
|
||||
secretName: "gohoarder-tls"
|
||||
|
||||
# Traefik-specific configuration (when className: "traefik")
|
||||
# Used by IngressRoute template for advanced routing
|
||||
traefik:
|
||||
# Middleware references (from traefik namespace)
|
||||
# Example: ["default-chain", "rate-limit"]
|
||||
middlewares: []
|
||||
# ServersTransport configuration
|
||||
transport:
|
||||
serverName: ""
|
||||
|
||||
# Autoscaling configuration
|
||||
autoscaling:
|
||||
enabled: false
|
||||
|
||||
@@ -1,5 +1,3 @@
|
||||
// Package version exposes build-time version metadata (semver, commit, build
|
||||
// time, Go toolchain version) for the binary, populated via -ldflags.
|
||||
package version
|
||||
|
||||
import "runtime"
|
||||
|
||||
+26
-74
@@ -1,5 +1,3 @@
|
||||
// Package analytics tracks and analyzes package download events with
|
||||
// in-memory aggregation and periodic background flushing.
|
||||
package analytics
|
||||
|
||||
import (
|
||||
@@ -48,22 +46,15 @@ type PopularPackage struct {
|
||||
Trend float64 // Growth rate
|
||||
}
|
||||
|
||||
// Engine tracks and analyzes package downloads.
|
||||
//
|
||||
// Lock ordering: never hold both downloadsMu and statsMu at the same time.
|
||||
// Code paths must acquire only one of the two; if they need both views, they
|
||||
// must take a snapshot under the first lock and release it before taking the
|
||||
// second.
|
||||
// Engine tracks and analyzes package downloads
|
||||
type Engine struct {
|
||||
stats map[string]*PackageStats
|
||||
flushTicker *time.Ticker
|
||||
stopChan chan struct{}
|
||||
doneChan chan struct{}
|
||||
downloads []PackageDownload
|
||||
maxEvents int
|
||||
downloadsMu sync.RWMutex
|
||||
statsMu sync.RWMutex
|
||||
closeOnce sync.Once
|
||||
}
|
||||
|
||||
// Config holds analytics engine configuration
|
||||
@@ -87,7 +78,6 @@ func NewEngine(cfg Config) *Engine {
|
||||
maxEvents: cfg.MaxEvents,
|
||||
flushTicker: time.NewTicker(cfg.FlushInterval),
|
||||
stopChan: make(chan struct{}),
|
||||
doneChan: make(chan struct{}),
|
||||
}
|
||||
|
||||
// Load existing stats from metadata store
|
||||
@@ -104,26 +94,19 @@ func NewEngine(cfg Config) *Engine {
|
||||
return engine
|
||||
}
|
||||
|
||||
// TrackDownload records a package download event.
|
||||
//
|
||||
// Locks are taken sequentially (never nested) to avoid the lock-ordering
|
||||
// inversion between downloadsMu and statsMu that exists elsewhere in the
|
||||
// engine: append to the buffer under downloadsMu, release it, then update
|
||||
// stats under statsMu.
|
||||
// TrackDownload records a package download event
|
||||
func (e *Engine) TrackDownload(download PackageDownload) {
|
||||
// Append to buffer and decide whether buffer is full.
|
||||
e.downloadsMu.Lock()
|
||||
e.downloads = append(e.downloads, download)
|
||||
bufferFull := len(e.downloads) >= e.maxEvents
|
||||
e.downloadsMu.Unlock()
|
||||
defer e.downloadsMu.Unlock()
|
||||
|
||||
// Update in-memory stats outside downloadsMu to keep the two locks
|
||||
// strictly disjoint.
|
||||
// Add to event buffer
|
||||
e.downloads = append(e.downloads, download)
|
||||
|
||||
// Update in-memory stats
|
||||
e.updateStats(download)
|
||||
|
||||
// Flush if buffer is full. The flush goroutine acquires downloadsMu
|
||||
// itself; we must not hold any lock when spawning it.
|
||||
if bufferFull {
|
||||
// Flush if buffer is full
|
||||
if len(e.downloads) >= e.maxEvents {
|
||||
go e.flush()
|
||||
}
|
||||
|
||||
@@ -200,47 +183,28 @@ func (e *Engine) GetTopPackages(limit int) []PopularPackage {
|
||||
return packages
|
||||
}
|
||||
|
||||
// GetTrendingPackages returns packages with growing popularity.
|
||||
//
|
||||
// The implementation takes a snapshot of stats under statsMu, releases it,
|
||||
// and only then queries the downloads buffer. This preserves the invariant
|
||||
// that downloadsMu and statsMu are never held simultaneously.
|
||||
// GetTrendingPackages returns packages with growing popularity
|
||||
func (e *Engine) GetTrendingPackages(limit int) []PopularPackage {
|
||||
// Snapshot stats under statsMu only.
|
||||
type statsSnapshot struct {
|
||||
registry string
|
||||
name string
|
||||
downloads int64
|
||||
}
|
||||
e.statsMu.RLock()
|
||||
snapshot := make([]statsSnapshot, 0, len(e.stats))
|
||||
for _, stats := range e.stats {
|
||||
snapshot = append(snapshot, statsSnapshot{
|
||||
registry: stats.Registry,
|
||||
name: stats.Name,
|
||||
downloads: stats.TotalDownloads,
|
||||
})
|
||||
}
|
||||
e.statsMu.RUnlock()
|
||||
defer e.statsMu.RUnlock()
|
||||
|
||||
sevenDaysAgo := time.Now().Add(-7 * 24 * time.Hour)
|
||||
|
||||
packages := make([]PopularPackage, 0, len(snapshot))
|
||||
for _, s := range snapshot {
|
||||
// Calculate recent downloads (last 7 days). Acquires downloadsMu
|
||||
// only; statsMu is no longer held here.
|
||||
recent := e.getRecentDownloads(s.registry, s.name, sevenDaysAgo)
|
||||
packages := make([]PopularPackage, 0)
|
||||
for _, stats := range e.stats {
|
||||
// Calculate recent downloads (last 7 days)
|
||||
recent := e.getRecentDownloads(stats.Registry, stats.Name, sevenDaysAgo)
|
||||
|
||||
// Calculate trend (simple growth rate)
|
||||
trend := 0.0
|
||||
if s.downloads > 0 {
|
||||
trend = float64(recent) / float64(s.downloads) * 100
|
||||
if stats.TotalDownloads > 0 {
|
||||
trend = float64(recent) / float64(stats.TotalDownloads) * 100
|
||||
}
|
||||
|
||||
packages = append(packages, PopularPackage{
|
||||
Registry: s.registry,
|
||||
Name: s.name,
|
||||
Downloads: s.downloads,
|
||||
Registry: stats.Registry,
|
||||
Name: stats.Name,
|
||||
Downloads: stats.TotalDownloads,
|
||||
RecentDownloads: recent,
|
||||
Trend: trend,
|
||||
})
|
||||
@@ -333,10 +297,8 @@ func (e *Engine) GetTotalStats() map[string]interface{} {
|
||||
}
|
||||
}
|
||||
|
||||
// flushLoop periodically flushes download events to metadata store.
|
||||
// Closes doneChan after the final flush so Close can wait for it.
|
||||
// flushLoop periodically flushes download events to metadata store
|
||||
func (e *Engine) flushLoop() {
|
||||
defer close(e.doneChan)
|
||||
for {
|
||||
select {
|
||||
case <-e.flushTicker.C:
|
||||
@@ -374,22 +336,12 @@ func (e *Engine) loadStats() {
|
||||
log.Debug().Msg("Loading analytics stats from metadata store")
|
||||
}
|
||||
|
||||
// Close stops the analytics engine. Safe to call multiple times.
|
||||
//
|
||||
// Shutdown sequence:
|
||||
// 1. signal stopChan (sync.Once-guarded so we never close twice)
|
||||
// 2. stop the ticker
|
||||
// 3. wait for flushLoop to drain and run its final flush via doneChan
|
||||
//
|
||||
// We do not call flush() directly here — flushLoop owns the final flush, so
|
||||
// there is exactly one flush at shutdown.
|
||||
// Close stops the analytics engine
|
||||
func (e *Engine) Close() {
|
||||
e.closeOnce.Do(func() {
|
||||
close(e.stopChan)
|
||||
e.flushTicker.Stop()
|
||||
<-e.doneChan
|
||||
log.Info().Msg("Analytics engine stopped")
|
||||
})
|
||||
close(e.stopChan)
|
||||
e.flushTicker.Stop()
|
||||
e.flush() // Final flush
|
||||
log.Info().Msg("Analytics engine stopped")
|
||||
}
|
||||
|
||||
// GetRegistryStats returns per-registry statistics
|
||||
|
||||
+63
-360
@@ -1,15 +1,11 @@
|
||||
// Package app wires the GoHoarder HTTP application together: config,
|
||||
// storage, metadata, scanners, and route handlers.
|
||||
package app
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"os/signal"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
@@ -33,7 +29,6 @@ import (
|
||||
"github.com/lukaszraczylo/gohoarder/pkg/scanner"
|
||||
"github.com/lukaszraczylo/gohoarder/pkg/storage"
|
||||
"github.com/lukaszraczylo/gohoarder/pkg/storage/filesystem"
|
||||
nfsstore "github.com/lukaszraczylo/gohoarder/pkg/storage/nfs"
|
||||
"github.com/lukaszraczylo/gohoarder/pkg/storage/s3"
|
||||
"github.com/lukaszraczylo/gohoarder/pkg/storage/smb"
|
||||
"github.com/lukaszraczylo/gohoarder/pkg/vcs"
|
||||
@@ -59,24 +54,23 @@ type App struct {
|
||||
cdnMiddleware *cdn.Middleware
|
||||
}
|
||||
|
||||
// New creates a new application instance.
|
||||
// The local receiver is named "a" to avoid shadowing the package name "app".
|
||||
// New creates a new application instance
|
||||
func New(cfg *config.Config) (*App, error) {
|
||||
a := &App{
|
||||
app := &App{
|
||||
config: cfg,
|
||||
}
|
||||
|
||||
// Initialize components
|
||||
if err := a.initializeComponents(); err != nil {
|
||||
if err := app.initializeComponents(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Setup HTTP server and routes
|
||||
if err := a.setupServer(); err != nil {
|
||||
if err := app.setupServer(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return a, nil
|
||||
return app, nil
|
||||
}
|
||||
|
||||
// initializeComponents initializes all application components
|
||||
@@ -111,18 +105,6 @@ func (a *App) initializeComponents() error {
|
||||
MaxSizeBytes: a.config.Cache.MaxSizeBytes,
|
||||
PoolSize: 5, // Default connection pool size
|
||||
})
|
||||
case "nfs":
|
||||
// SyncWrites defaults to true (durable). Pointer-bool lets operators
|
||||
// opt out via explicit `sync_writes: false` in config.
|
||||
syncWrites := true
|
||||
if a.config.Storage.NFS.SyncWrites != nil {
|
||||
syncWrites = *a.config.Storage.NFS.SyncWrites
|
||||
}
|
||||
a.storage, err = nfsstore.New(nfsstore.Config{
|
||||
Path: a.config.Storage.Path,
|
||||
MaxSize: a.config.Cache.MaxSizeBytes,
|
||||
SyncWrites: syncWrites,
|
||||
}, log.Logger)
|
||||
default:
|
||||
log.Warn().
|
||||
Str("backend", a.config.Storage.Backend).
|
||||
@@ -218,88 +200,36 @@ func (a *App) initializeComponents() error {
|
||||
FlushInterval: 5 * time.Minute,
|
||||
})
|
||||
|
||||
// Initialize cache manager with scanner and analytics. MaxPackageSize is
|
||||
// taken from Security.Scanners.Static (the static analyser already exposes
|
||||
// this knob and it doubles as a cache hard cap).
|
||||
// Initialize cache manager with scanner and analytics
|
||||
log.Info().Msg("Initializing cache manager")
|
||||
cleanupInterval := a.config.Cache.CleanupInterval
|
||||
if cleanupInterval == 0 {
|
||||
cleanupInterval = 5 * time.Minute
|
||||
}
|
||||
a.cache, err = cache.New(a.storage, a.metadata, a.scanManager, a.analyticsEngine, cache.Config{
|
||||
DefaultTTL: a.config.Cache.DefaultTTL,
|
||||
CleanupInterval: cleanupInterval,
|
||||
MaxPackageSize: a.config.Security.Scanners.Static.MaxPackageSize,
|
||||
CleanupInterval: 5 * time.Minute,
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to initialize cache: %w", err)
|
||||
}
|
||||
|
||||
// Initialize network client. Pull values from cfg.Network where set;
|
||||
// fall back to existing literals so unset fields preserve current
|
||||
// behaviour. cfg.Network.* uses different semantic names (per-IP rate,
|
||||
// retry struct, etc.) so we map explicitly rather than blindly assign.
|
||||
// Initialize network client
|
||||
log.Info().Msg("Initializing network client")
|
||||
netTimeout := a.config.Network.ReadTimeout
|
||||
if netTimeout == 0 {
|
||||
netTimeout = 5 * time.Minute
|
||||
}
|
||||
maxRetries := a.config.Network.Retry.MaxAttempts
|
||||
if maxRetries == 0 {
|
||||
maxRetries = 3
|
||||
}
|
||||
retryDelay := a.config.Network.Retry.InitialBackoff
|
||||
if retryDelay == 0 {
|
||||
retryDelay = 1 * time.Second
|
||||
}
|
||||
rateLimit := float64(a.config.Network.RateLimit.PerIP)
|
||||
if rateLimit == 0 {
|
||||
rateLimit = 100
|
||||
}
|
||||
rateBurst := a.config.Network.RateLimit.BurstSize
|
||||
if rateBurst == 0 {
|
||||
rateBurst = 10
|
||||
}
|
||||
cbThreshold := a.config.Network.CircuitBreaker.Threshold
|
||||
if cbThreshold == 0 {
|
||||
cbThreshold = 5
|
||||
}
|
||||
cbTimeout := a.config.Network.CircuitBreaker.Timeout
|
||||
if cbTimeout == 0 {
|
||||
cbTimeout = 30 * time.Second
|
||||
}
|
||||
a.networkClient = network.NewClient(network.Config{
|
||||
Timeout: netTimeout,
|
||||
MaxRetries: maxRetries,
|
||||
RetryDelay: retryDelay,
|
||||
RateLimit: rateLimit,
|
||||
RateBurst: rateBurst,
|
||||
Timeout: 5 * time.Minute,
|
||||
MaxRetries: 3,
|
||||
RetryDelay: 1 * time.Second,
|
||||
RateLimit: 100,
|
||||
RateBurst: 10,
|
||||
CircuitBreaker: network.CircuitBreakerConfig{
|
||||
Enabled: true,
|
||||
FailureThreshold: cbThreshold,
|
||||
FailureThreshold: 5,
|
||||
SuccessThreshold: 2,
|
||||
Timeout: cbTimeout,
|
||||
Timeout: 30 * time.Second,
|
||||
},
|
||||
UserAgent: "GoHoarder/1.0",
|
||||
})
|
||||
|
||||
// Initialize authentication manager. NewWithStore persists keys via the
|
||||
// metadata layer; Load hydrates the in-memory cache from prior state.
|
||||
// Bootstrap creates an admin from env on first boot when no admin exists.
|
||||
// Initialize authentication manager
|
||||
log.Info().Msg("Initializing authentication manager")
|
||||
authCfg := auth.Config{BcryptCost: a.config.Auth.BcryptCost}
|
||||
a.authManager = auth.NewWithStore(authCfg, a.metadata)
|
||||
loadCtx, loadCancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
if err := a.authManager.Load(loadCtx); err != nil {
|
||||
// Empty key set is recoverable: log and continue rather than fail boot.
|
||||
log.Warn().Err(err).Msg("Failed to load API keys from metadata store; starting with empty key set")
|
||||
}
|
||||
loadCancel()
|
||||
bootCtx, bootCancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
if err := auth.BootstrapAdminFromEnv(bootCtx, a.metadata, authCfg); err != nil {
|
||||
log.Warn().Err(err).Msg("Failed to bootstrap admin API key from environment")
|
||||
}
|
||||
bootCancel()
|
||||
a.authManager = auth.New()
|
||||
|
||||
// Initialize rescan worker if enabled
|
||||
if a.config.Security.Enabled && a.config.Security.RescanInterval > 0 {
|
||||
@@ -312,23 +242,17 @@ func (a *App) initializeComponents() error {
|
||||
a.wsServer = websocket.NewServer(websocket.Config{
|
||||
ReadBufferSize: 1024,
|
||||
WriteBufferSize: 1024,
|
||||
CheckOrigin: buildCheckOrigin(a.config.Server.AllowedOrigins),
|
||||
CheckOrigin: func(_ *http.Request) bool {
|
||||
return true // Allow all origins in development
|
||||
},
|
||||
})
|
||||
|
||||
// Wire WebSocket server as the broadcaster for cache + scanner so
|
||||
// lifecycle events (cached/downloaded/scan_*) reach connected clients.
|
||||
// websocket.Server satisfies events.Broadcaster via BroadcastEvent.
|
||||
a.cache.SetBroadcaster(a.wsServer)
|
||||
a.scanManager.SetBroadcaster(a.wsServer)
|
||||
|
||||
// Initialize pre-warming worker. Operator-controlled via cfg.Prewarming;
|
||||
// defaults are baked into config.Default() so unset fields stay sane.
|
||||
// Initialize pre-warming worker
|
||||
log.Info().Msg("Initializing pre-warming worker")
|
||||
a.prewarmWorker = prewarming.NewWorker(prewarming.Config{
|
||||
Enabled: a.config.Prewarming.Enabled,
|
||||
Interval: a.config.Prewarming.Interval,
|
||||
MaxConcurrent: a.config.Prewarming.MaxConcurrent,
|
||||
TopPackages: a.config.Prewarming.TopPackages,
|
||||
Enabled: false, // Disabled by default
|
||||
Interval: 1 * time.Hour,
|
||||
MaxConcurrent: 5,
|
||||
CacheManager: a.cache,
|
||||
Analytics: a.analyticsEngine,
|
||||
NetworkClient: a.networkClient,
|
||||
@@ -388,75 +312,29 @@ func (a *App) setupServer() error {
|
||||
AppName: "GoHoarder v1.0",
|
||||
})
|
||||
|
||||
// Auth middleware factories. Built once, reused on every gated route.
|
||||
// When auth is disabled the variables stay nil and we skip wiring; routes
|
||||
// are mounted bare to preserve backward-compatible public access.
|
||||
authEnabled := a.config.Auth.Enabled
|
||||
var (
|
||||
mwAuth fiber.Handler
|
||||
mwAdmin fiber.Handler
|
||||
mwOptional fiber.Handler
|
||||
)
|
||||
if authEnabled {
|
||||
mwAuth = RequireAuth(a.authManager)
|
||||
mwAdmin = RequireRole(a.authManager, string(auth.RoleAdmin))
|
||||
mwOptional = OptionalAuth(a.authManager)
|
||||
_ = mwOptional // reserved for future endpoints with anonymous fallback
|
||||
}
|
||||
|
||||
// Health endpoints — ALWAYS public (k8s liveness/readiness probes).
|
||||
// Health and metrics endpoints (adapted from net/http)
|
||||
a.app.Get("/health", adaptor.HTTPHandlerFunc(a.healthChecker.HealthHandler()))
|
||||
a.app.Get("/health/ready", adaptor.HTTPHandlerFunc(a.healthChecker.ReadyHandler()))
|
||||
a.app.Get("/metrics", adaptor.HTTPHandler(metrics.Handler()))
|
||||
|
||||
// Metrics endpoint — public by default, optionally auth-gated.
|
||||
if authEnabled && a.config.Metrics.RequireAuth {
|
||||
a.app.Get("/metrics", mwAuth, adaptor.HTTPHandler(metrics.Handler()))
|
||||
} else {
|
||||
a.app.Get("/metrics", adaptor.HTTPHandler(metrics.Handler()))
|
||||
}
|
||||
// WebSocket endpoint (adapted from net/http)
|
||||
a.app.Get("/ws", adaptor.HTTPHandlerFunc(a.wsServer.HandleWebSocket))
|
||||
|
||||
// WebSocket endpoint — gated when auth enabled. Clients must send the
|
||||
// API key via Authorization or X-API-Key on the upgrade request.
|
||||
if authEnabled {
|
||||
a.app.Get("/ws", mwAuth, adaptor.HTTPHandlerFunc(a.wsServer.HandleWebSocket))
|
||||
} else {
|
||||
a.app.Get("/ws", adaptor.HTTPHandlerFunc(a.wsServer.HandleWebSocket))
|
||||
}
|
||||
// API endpoints
|
||||
a.app.Get("/api/config", a.handleConfig)
|
||||
a.app.All("/api/packages/*", a.handlePackages) // Handles packages and vulnerabilities
|
||||
a.app.Get("/api/stats", a.handleStats)
|
||||
a.app.Get("/api/stats/timeseries", a.handleTimeSeriesStats)
|
||||
a.app.Get("/api/info", a.handleInfo)
|
||||
|
||||
// API endpoints. Read endpoints require any valid key; package DELETE
|
||||
// requires admin (split route from the All() so role-check runs only on
|
||||
// destructive verbs). The combined handler still dispatches by method.
|
||||
if authEnabled {
|
||||
a.app.Get("/api/config", mwAuth, a.handleConfig)
|
||||
a.app.Get("/api/packages/*", mwAuth, a.handlePackages)
|
||||
a.app.Delete("/api/packages/*", mwAdmin, a.handlePackages)
|
||||
a.app.All("/api/packages/*", mwAuth, a.handlePackages) // OPTIONS, preflight, vulnerabilities
|
||||
a.app.Get("/api/stats", mwAuth, a.handleStats)
|
||||
a.app.Get("/api/stats/timeseries", mwAuth, a.handleTimeSeriesStats)
|
||||
a.app.Get("/api/info", mwAuth, a.handleInfo)
|
||||
|
||||
a.app.Get("/api/analytics/top", mwAuth, a.handleAnalyticsTopPackages)
|
||||
a.app.Get("/api/analytics/trending", mwAuth, a.handleAnalyticsTrendingPackages)
|
||||
a.app.Get("/api/analytics/trends", mwAuth, a.handleAnalyticsTrends)
|
||||
a.app.Get("/api/analytics/total", mwAuth, a.handleAnalyticsTotalStats)
|
||||
a.app.Get("/api/analytics/registry/:registry", mwAuth, a.handleAnalyticsRegistryStats)
|
||||
a.app.Get("/api/analytics/package/:registry/:name", mwAuth, a.handleAnalyticsPackageStats)
|
||||
a.app.Get("/api/analytics/search", mwAuth, a.handleAnalyticsSearch)
|
||||
} else {
|
||||
a.app.Get("/api/config", a.handleConfig)
|
||||
a.app.All("/api/packages/*", a.handlePackages)
|
||||
a.app.Get("/api/stats", a.handleStats)
|
||||
a.app.Get("/api/stats/timeseries", a.handleTimeSeriesStats)
|
||||
a.app.Get("/api/info", a.handleInfo)
|
||||
|
||||
a.app.Get("/api/analytics/top", a.handleAnalyticsTopPackages)
|
||||
a.app.Get("/api/analytics/trending", a.handleAnalyticsTrendingPackages)
|
||||
a.app.Get("/api/analytics/trends", a.handleAnalyticsTrends)
|
||||
a.app.Get("/api/analytics/total", a.handleAnalyticsTotalStats)
|
||||
a.app.Get("/api/analytics/registry/:registry", a.handleAnalyticsRegistryStats)
|
||||
a.app.Get("/api/analytics/package/:registry/:name", a.handleAnalyticsPackageStats)
|
||||
a.app.Get("/api/analytics/search", a.handleAnalyticsSearch)
|
||||
}
|
||||
// Analytics endpoints
|
||||
a.app.Get("/api/analytics/top", a.handleAnalyticsTopPackages)
|
||||
a.app.Get("/api/analytics/trending", a.handleAnalyticsTrendingPackages)
|
||||
a.app.Get("/api/analytics/trends", a.handleAnalyticsTrends)
|
||||
a.app.Get("/api/analytics/total", a.handleAnalyticsTotalStats)
|
||||
a.app.Get("/api/analytics/registry/:registry", a.handleAnalyticsRegistryStats)
|
||||
a.app.Get("/api/analytics/package/:registry/:name", a.handleAnalyticsPackageStats)
|
||||
a.app.Get("/api/analytics/search", a.handleAnalyticsSearch)
|
||||
|
||||
// Admin endpoints (bypass management)
|
||||
a.app.All("/api/admin/bypasses/:id?", a.requireAdmin, a.handleAdminBypasses)
|
||||
@@ -490,62 +368,28 @@ func (a *App) setupServer() error {
|
||||
}
|
||||
}
|
||||
|
||||
// Per-registry proxies. Mounted only when the corresponding handler is
|
||||
// enabled in config. Each Upstream falls back to the canonical URL if
|
||||
// unset so partially-configured deployments keep working.
|
||||
if a.config.Handlers.Go.Enabled {
|
||||
goUpstream := a.config.Handlers.Go.UpstreamProxy
|
||||
if goUpstream == "" {
|
||||
goUpstream = "https://proxy.golang.org"
|
||||
}
|
||||
sumDB := a.config.Handlers.Go.ChecksumDB
|
||||
if sumDB == "" {
|
||||
sumDB = "https://sum.golang.org"
|
||||
}
|
||||
goProxyHandler := goproxy.New(a.cache, a.networkClient, goproxy.Config{
|
||||
Upstream: goUpstream,
|
||||
SumDBURL: sumDB,
|
||||
CredStore: credStore,
|
||||
})
|
||||
goProxyWithCDN := a.cdnMiddleware.Handler(http.StripPrefix("/go", goProxyHandler))
|
||||
if authEnabled {
|
||||
a.app.All("/go/*", mwAuth, adaptor.HTTPHandler(goProxyWithCDN))
|
||||
} else {
|
||||
a.app.All("/go/*", adaptor.HTTPHandler(goProxyWithCDN))
|
||||
}
|
||||
}
|
||||
// Go proxy with CDN caching
|
||||
goProxyHandler := goproxy.New(a.cache, a.networkClient, goproxy.Config{
|
||||
Upstream: "https://proxy.golang.org",
|
||||
SumDBURL: "https://sum.golang.org",
|
||||
CredStore: credStore,
|
||||
})
|
||||
goProxyWithCDN := a.cdnMiddleware.Handler(http.StripPrefix("/go", goProxyHandler))
|
||||
a.app.All("/go/*", adaptor.HTTPHandler(goProxyWithCDN))
|
||||
|
||||
if a.config.Handlers.NPM.Enabled {
|
||||
npmUpstream := a.config.Handlers.NPM.UpstreamRegistry
|
||||
if npmUpstream == "" {
|
||||
npmUpstream = "https://registry.npmjs.org"
|
||||
}
|
||||
npmProxyHandler := npm.New(a.cache, a.networkClient, npm.Config{
|
||||
Upstream: npmUpstream,
|
||||
})
|
||||
npmProxyWithCDN := a.cdnMiddleware.Handler(http.StripPrefix("/npm", npmProxyHandler))
|
||||
if authEnabled {
|
||||
a.app.All("/npm/*", mwAuth, adaptor.HTTPHandler(npmProxyWithCDN))
|
||||
} else {
|
||||
a.app.All("/npm/*", adaptor.HTTPHandler(npmProxyWithCDN))
|
||||
}
|
||||
}
|
||||
// NPM proxy with CDN caching
|
||||
npmProxyHandler := npm.New(a.cache, a.networkClient, npm.Config{
|
||||
Upstream: "https://registry.npmjs.org",
|
||||
})
|
||||
npmProxyWithCDN := a.cdnMiddleware.Handler(http.StripPrefix("/npm", npmProxyHandler))
|
||||
a.app.All("/npm/*", adaptor.HTTPHandler(npmProxyWithCDN))
|
||||
|
||||
if a.config.Handlers.PyPI.Enabled {
|
||||
pypiUpstream := a.config.Handlers.PyPI.SimpleAPIURL
|
||||
if pypiUpstream == "" {
|
||||
pypiUpstream = "https://pypi.org/simple"
|
||||
}
|
||||
pypiProxyHandler := pypi.New(a.cache, a.networkClient, pypi.Config{
|
||||
Upstream: pypiUpstream,
|
||||
})
|
||||
pypiProxyWithCDN := a.cdnMiddleware.Handler(http.StripPrefix("/pypi", pypiProxyHandler))
|
||||
if authEnabled {
|
||||
a.app.All("/pypi/*", mwAuth, adaptor.HTTPHandler(pypiProxyWithCDN))
|
||||
} else {
|
||||
a.app.All("/pypi/*", adaptor.HTTPHandler(pypiProxyWithCDN))
|
||||
}
|
||||
}
|
||||
// PyPI proxy with CDN caching
|
||||
pypiProxyHandler := pypi.New(a.cache, a.networkClient, pypi.Config{
|
||||
Upstream: "https://pypi.org/simple",
|
||||
})
|
||||
pypiProxyWithCDN := a.cdnMiddleware.Handler(http.StripPrefix("/pypi", pypiProxyHandler))
|
||||
a.app.All("/pypi/*", adaptor.HTTPHandler(pypiProxyWithCDN))
|
||||
|
||||
// Serve frontend static files
|
||||
frontendDir := "frontend/dist"
|
||||
@@ -598,24 +442,10 @@ func (a *App) Run() error {
|
||||
// Start download data aggregation worker (runs every hour)
|
||||
go a.startAggregationWorker(ctx)
|
||||
|
||||
// Start periodic stats broadcaster (every 30s) so connected WS clients
|
||||
// see live counter updates without polling.
|
||||
go a.startStatsBroadcaster(ctx)
|
||||
|
||||
// Start Fiber server in goroutine. Branch on TLS to pick Listen vs ListenTLS.
|
||||
// Start Fiber server in goroutine
|
||||
errChan := make(chan error, 1)
|
||||
go func() {
|
||||
addr := fmt.Sprintf("%s:%d", a.config.Server.Host, a.config.Server.Port)
|
||||
if a.config.Server.TLS.Enabled {
|
||||
log.Info().
|
||||
Str("addr", addr).
|
||||
Str("cert_file", a.config.Server.TLS.CertFile).
|
||||
Msg("Starting Fiber server with TLS")
|
||||
if err := a.app.ListenTLS(addr, a.config.Server.TLS.CertFile, a.config.Server.TLS.KeyFile); err != nil {
|
||||
errChan <- err
|
||||
}
|
||||
return
|
||||
}
|
||||
log.Info().
|
||||
Str("addr", addr).
|
||||
Msg("Starting Fiber server")
|
||||
@@ -650,11 +480,6 @@ func (a *App) Shutdown() error {
|
||||
log.Error().Err(err).Msg("Error shutting down Fiber server")
|
||||
}
|
||||
|
||||
// Drain async auth writes (LastUsedAt updates) before closing metadata.
|
||||
if a.authManager != nil {
|
||||
a.authManager.Close()
|
||||
}
|
||||
|
||||
// Stop pre-warming worker
|
||||
a.prewarmWorker.Stop()
|
||||
|
||||
@@ -680,46 +505,6 @@ func (a *App) Shutdown() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// startStatsBroadcaster periodically pulls aggregate cache stats and pushes
|
||||
// them onto the WebSocket broadcast channel so connected dashboards see
|
||||
// live counters without polling. Lightweight: a single GetStats call every
|
||||
// 30s, drop-on-overflow at the WS layer.
|
||||
func (a *App) startStatsBroadcaster(ctx context.Context) {
|
||||
if a.wsServer == nil {
|
||||
return
|
||||
}
|
||||
const interval = 30 * time.Second
|
||||
ticker := time.NewTicker(interval)
|
||||
defer ticker.Stop()
|
||||
|
||||
emit := func() {
|
||||
statsCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
|
||||
defer cancel()
|
||||
stats, err := a.cache.GetStats(statsCtx, "")
|
||||
if err != nil {
|
||||
log.Debug().Err(err).Msg("stats broadcaster: GetStats failed")
|
||||
return
|
||||
}
|
||||
a.wsServer.BroadcastEvent("stats_update", map[string]interface{}{
|
||||
"stats": stats,
|
||||
"timestamp": time.Now().UTC().Format(time.RFC3339),
|
||||
})
|
||||
}
|
||||
|
||||
// Emit once immediately so newly-connected clients see fresh data.
|
||||
emit()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
log.Info().Msg("Stats broadcaster stopped")
|
||||
return
|
||||
case <-ticker.C:
|
||||
emit()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// startAggregationWorker runs download data aggregation periodically
|
||||
func (a *App) startAggregationWorker(ctx context.Context) {
|
||||
log.Info().Msg("Starting download data aggregation worker (runs every hour)")
|
||||
@@ -761,85 +546,3 @@ func getOrDefaultStr(value, defaultValue string) string {
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
// buildCheckOrigin returns a websocket Origin validator.
|
||||
//
|
||||
// Behaviour:
|
||||
// - allowed entries may be exact origins ("https://app.example.com")
|
||||
// or wildcard host patterns ("https://*.example.com").
|
||||
// - when allowed is empty, only same-origin upgrades are permitted: the
|
||||
// Origin host must match the Host header of the incoming request.
|
||||
// - requests with no Origin header are accepted (non-browser clients).
|
||||
//
|
||||
// We deliberately do NOT default to allow-all to avoid Cross-Site WebSocket
|
||||
// Hijacking (CSWSH).
|
||||
func buildCheckOrigin(allowed []string) func(*http.Request) bool {
|
||||
// Pre-parse allowlist once.
|
||||
type originRule struct {
|
||||
scheme string
|
||||
host string // exact host, or "" if hostSuffix is set
|
||||
hostGlob string // suffix match including leading "."
|
||||
}
|
||||
rules := make([]originRule, 0, len(allowed))
|
||||
for _, raw := range allowed {
|
||||
raw = strings.TrimSpace(raw)
|
||||
if raw == "" {
|
||||
continue
|
||||
}
|
||||
u, err := url.Parse(raw)
|
||||
if err != nil || u.Scheme == "" || u.Host == "" {
|
||||
log.Warn().Str("entry", raw).Msg("Ignoring invalid AllowedOrigins entry")
|
||||
continue
|
||||
}
|
||||
r := originRule{scheme: strings.ToLower(u.Scheme)}
|
||||
host := strings.ToLower(u.Host)
|
||||
if strings.HasPrefix(host, "*.") {
|
||||
// "*.example.com" → match any subdomain plus the apex.
|
||||
r.hostGlob = host[1:] // ".example.com"
|
||||
} else {
|
||||
r.host = host
|
||||
}
|
||||
rules = append(rules, r)
|
||||
}
|
||||
|
||||
return func(r *http.Request) bool {
|
||||
origin := r.Header.Get("Origin")
|
||||
if origin == "" {
|
||||
// Non-browser clients (e.g., CLI, server-to-server) omit Origin.
|
||||
return true
|
||||
}
|
||||
ou, err := url.Parse(origin)
|
||||
if err != nil || ou.Scheme == "" || ou.Host == "" {
|
||||
log.Warn().Str("origin", origin).Msg("Rejecting WebSocket upgrade: malformed Origin header")
|
||||
return false
|
||||
}
|
||||
originScheme := strings.ToLower(ou.Scheme)
|
||||
originHost := strings.ToLower(ou.Host)
|
||||
|
||||
// Empty allowlist → same-origin only.
|
||||
if len(rules) == 0 {
|
||||
return strings.EqualFold(originHost, r.Host)
|
||||
}
|
||||
|
||||
for _, rule := range rules {
|
||||
if rule.scheme != originScheme {
|
||||
continue
|
||||
}
|
||||
if rule.host != "" && rule.host == originHost {
|
||||
return true
|
||||
}
|
||||
if rule.hostGlob != "" {
|
||||
// Match apex (".example.com" → "example.com") and any subdomain.
|
||||
apex := strings.TrimPrefix(rule.hostGlob, ".")
|
||||
if originHost == apex || strings.HasSuffix(originHost, rule.hostGlob) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
log.Warn().
|
||||
Str("origin", origin).
|
||||
Str("host", r.Host).
|
||||
Msg("Rejecting WebSocket upgrade: Origin not in allowlist")
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
@@ -286,9 +286,9 @@ func (s *AnalyticsHandlersTestSuite) TestHandleAnalyticsSearch() {
|
||||
|
||||
if !tt.expectError {
|
||||
var result struct {
|
||||
Query string `json:"query"`
|
||||
Results []analytics.PackageStats `json:"results"`
|
||||
Total int `json:"total"`
|
||||
Query string `json:"query"`
|
||||
}
|
||||
err = json.NewDecoder(resp.Body).Decode(&result)
|
||||
s.NoError(err)
|
||||
|
||||
@@ -44,8 +44,8 @@ func (s *AuthHandlersTestSuite) TestHandleGenerateAPIKey() {
|
||||
tests := []struct {
|
||||
requestBody map[string]string
|
||||
name string
|
||||
expectedRole string
|
||||
expectedStatus int
|
||||
expectedRole string
|
||||
expectKey bool
|
||||
}{
|
||||
{
|
||||
@@ -139,9 +139,9 @@ func (s *AuthHandlersTestSuite) TestHandleGenerateAPIKey() {
|
||||
|
||||
func (s *AuthHandlersTestSuite) TestHandleListAPIKeys() {
|
||||
// Generate some test keys first
|
||||
_, _, _ = s.authManager.GenerateAPIKey("test-key-1", auth.RoleReadOnly, nil)
|
||||
_, _, _ = s.authManager.GenerateAPIKey("test-key-2", auth.RoleReadWrite, nil)
|
||||
_, _, _ = s.authManager.GenerateAPIKey("test-key-3", auth.RoleAdmin, nil)
|
||||
s.authManager.GenerateAPIKey("test-key-1", auth.RoleReadOnly, nil)
|
||||
s.authManager.GenerateAPIKey("test-key-2", auth.RoleReadWrite, nil)
|
||||
s.authManager.GenerateAPIKey("test-key-3", auth.RoleAdmin, nil)
|
||||
|
||||
req := httptest.NewRequest("GET", "/api/admin/keys", nil)
|
||||
resp, err := s.app.Test(req, 5000) // 5 second timeout for CI environments
|
||||
|
||||
@@ -1,193 +0,0 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"strings"
|
||||
|
||||
"github.com/gofiber/fiber/v2"
|
||||
"github.com/lukaszraczylo/gohoarder/pkg/auth"
|
||||
"github.com/lukaszraczylo/gohoarder/pkg/errors"
|
||||
"github.com/rs/zerolog/log"
|
||||
)
|
||||
|
||||
// Locals keys used by the auth middleware. Exported so other handlers in the
|
||||
// app package (and the integrator) can fetch the resolved key/role without
|
||||
// guessing at string literals.
|
||||
const (
|
||||
LocalAuthKey = "auth_key"
|
||||
LocalAuthRole = "auth_role"
|
||||
)
|
||||
|
||||
// extractAPIKey pulls the raw API key from either the Authorization bearer
|
||||
// header or the X-API-Key header. Returns ("", false) when neither is set or
|
||||
// the Authorization header has the wrong shape.
|
||||
func extractAPIKey(c *fiber.Ctx) (string, bool) {
|
||||
if h := strings.TrimSpace(c.Get("Authorization")); h != "" {
|
||||
// Expect: "Bearer <token>". Be tolerant of casing on the scheme.
|
||||
parts := strings.SplitN(h, " ", 2)
|
||||
if len(parts) == 2 && strings.EqualFold(parts[0], "bearer") {
|
||||
token := strings.TrimSpace(parts[1])
|
||||
if token != "" {
|
||||
return token, true
|
||||
}
|
||||
}
|
||||
}
|
||||
if h := strings.TrimSpace(c.Get("X-API-Key")); h != "" {
|
||||
return h, true
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
|
||||
// writeAuthError sends a structured error response matching the project's
|
||||
// errors envelope (errors.Error JSON shape). Uses the HTTPStatusCode map so
|
||||
// callers don't need to remember which status maps to which code.
|
||||
func writeAuthError(c *fiber.Ctx, code, message string) error {
|
||||
status, ok := errors.HTTPStatusCode[code]
|
||||
if !ok {
|
||||
status = fiber.StatusInternalServerError
|
||||
}
|
||||
return c.Status(status).JSON(errors.New(code, message))
|
||||
}
|
||||
|
||||
// requestIDFromCtx returns the request ID set by an upstream middleware or
|
||||
// the X-Request-ID header. Empty string when neither is available — callers
|
||||
// should treat empty as "no correlation id".
|
||||
func requestIDFromCtx(c *fiber.Ctx) string {
|
||||
if v := c.Locals("request_id"); v != nil {
|
||||
if s, ok := v.(string); ok && s != "" {
|
||||
return s
|
||||
}
|
||||
}
|
||||
return c.Get("X-Request-ID")
|
||||
}
|
||||
|
||||
// validateAndAttach is the shared core of all auth middlewares: pull the key,
|
||||
// validate via the manager, and on success store it in Locals. Returns the
|
||||
// resolved key plus a boolean indicating whether a key was present at all
|
||||
// (false when no header). The error is non-nil only when validation failed
|
||||
// (i.e. a key was present but invalid/expired).
|
||||
func validateAndAttach(c *fiber.Ctx, authMgr *auth.Manager) (key *auth.APIKey, present bool, err error) {
|
||||
rawKey, ok := extractAPIKey(c)
|
||||
if !ok {
|
||||
return nil, false, nil
|
||||
}
|
||||
apiKey, err := authMgr.ValidateAPIKey(c.Context(), rawKey)
|
||||
if err != nil {
|
||||
return nil, true, err
|
||||
}
|
||||
c.Locals(LocalAuthKey, apiKey)
|
||||
c.Locals(LocalAuthRole, string(apiKey.Role))
|
||||
return apiKey, true, nil
|
||||
}
|
||||
|
||||
// RequireAuth returns a fiber.Handler that rejects requests without a valid
|
||||
// API key. On success the resolved *auth.APIKey is stored in c.Locals under
|
||||
// LocalAuthKey.
|
||||
func RequireAuth(authMgr *auth.Manager) fiber.Handler {
|
||||
return func(c *fiber.Ctx) error {
|
||||
key, present, err := validateAndAttach(c, authMgr)
|
||||
if !present {
|
||||
log.Debug().
|
||||
Str("path", c.Path()).
|
||||
Str("method", c.Method()).
|
||||
Str("request_id", requestIDFromCtx(c)).
|
||||
Msg("auth: missing API key")
|
||||
return writeAuthError(c, errors.ErrCodeUnauthorized, "missing API key; provide Authorization: Bearer <key> or X-API-Key header")
|
||||
}
|
||||
if err != nil {
|
||||
log.Warn().
|
||||
Err(err).
|
||||
Str("path", c.Path()).
|
||||
Str("method", c.Method()).
|
||||
Str("request_id", requestIDFromCtx(c)).
|
||||
Msg("auth: invalid API key")
|
||||
return writeAuthError(c, errors.ErrCodeInvalidAPIKey, "invalid or expired API key")
|
||||
}
|
||||
log.Debug().
|
||||
Str("key_id", key.ID).
|
||||
Str("role", string(key.Role)).
|
||||
Str("request_id", requestIDFromCtx(c)).
|
||||
Msg("auth: key validated")
|
||||
return c.Next()
|
||||
}
|
||||
}
|
||||
|
||||
// RequireRole returns a fiber.Handler that authenticates the caller and then
|
||||
// requires the resolved key's role to be present in the supplied list (any-of
|
||||
// semantics). An empty roles list behaves like RequireAuth.
|
||||
func RequireRole(authMgr *auth.Manager, roles ...string) fiber.Handler {
|
||||
return func(c *fiber.Ctx) error {
|
||||
key, present, err := validateAndAttach(c, authMgr)
|
||||
if !present {
|
||||
return writeAuthError(c, errors.ErrCodeUnauthorized, "missing API key; provide Authorization: Bearer <key> or X-API-Key header")
|
||||
}
|
||||
if err != nil {
|
||||
return writeAuthError(c, errors.ErrCodeInvalidAPIKey, "invalid or expired API key")
|
||||
}
|
||||
if len(roles) == 0 {
|
||||
return c.Next()
|
||||
}
|
||||
actual := string(key.Role)
|
||||
for _, r := range roles {
|
||||
if r == actual {
|
||||
return c.Next()
|
||||
}
|
||||
}
|
||||
log.Warn().
|
||||
Str("key_id", key.ID).
|
||||
Str("role", actual).
|
||||
Strs("required_roles", roles).
|
||||
Str("request_id", requestIDFromCtx(c)).
|
||||
Msg("auth: role mismatch")
|
||||
return writeAuthError(c, errors.ErrCodeForbidden, "insufficient role for this resource")
|
||||
}
|
||||
}
|
||||
|
||||
// RequirePermission returns a fiber.Handler that authenticates the caller and
|
||||
// then requires the resolved key to hold at least one of the supplied
|
||||
// permissions. An empty perms list behaves like RequireAuth.
|
||||
func RequirePermission(authMgr *auth.Manager, perms ...string) fiber.Handler {
|
||||
return func(c *fiber.Ctx) error {
|
||||
key, present, err := validateAndAttach(c, authMgr)
|
||||
if !present {
|
||||
return writeAuthError(c, errors.ErrCodeUnauthorized, "missing API key; provide Authorization: Bearer <key> or X-API-Key header")
|
||||
}
|
||||
if err != nil {
|
||||
return writeAuthError(c, errors.ErrCodeInvalidAPIKey, "invalid or expired API key")
|
||||
}
|
||||
if len(perms) == 0 {
|
||||
return c.Next()
|
||||
}
|
||||
for _, p := range perms {
|
||||
if key.HasPermission(auth.Permission(p)) {
|
||||
return c.Next()
|
||||
}
|
||||
}
|
||||
log.Warn().
|
||||
Str("key_id", key.ID).
|
||||
Str("role", string(key.Role)).
|
||||
Strs("required_perms", perms).
|
||||
Str("request_id", requestIDFromCtx(c)).
|
||||
Msg("auth: permission denied")
|
||||
return writeAuthError(c, errors.ErrCodeForbidden, "insufficient permissions for this resource")
|
||||
}
|
||||
}
|
||||
|
||||
// OptionalAuth returns a fiber.Handler that attempts to validate any provided
|
||||
// API key but never rejects the request. Handy for endpoints whose behavior
|
||||
// changes when the caller is authenticated (e.g. per-key rate limits) but
|
||||
// which still serve anonymous traffic. Locals are populated on success.
|
||||
func OptionalAuth(authMgr *auth.Manager) fiber.Handler {
|
||||
return func(c *fiber.Ctx) error {
|
||||
_, present, err := validateAndAttach(c, authMgr)
|
||||
if present && err != nil {
|
||||
// Key was provided but invalid — log for observability, continue
|
||||
// anonymously rather than 401-ing.
|
||||
log.Debug().
|
||||
Err(err).
|
||||
Str("path", c.Path()).
|
||||
Str("request_id", requestIDFromCtx(c)).
|
||||
Msg("auth: optional auth ignored invalid key")
|
||||
}
|
||||
return c.Next()
|
||||
}
|
||||
}
|
||||
@@ -1,287 +0,0 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"io"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/gofiber/fiber/v2"
|
||||
"github.com/lukaszraczylo/gohoarder/pkg/auth"
|
||||
"github.com/lukaszraczylo/gohoarder/pkg/errors"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// mwTestSetup builds a fiber app wired with the given middleware on /protected.
|
||||
// The handler echoes the resolved auth_key locals so tests can assert that
|
||||
// downstream handlers see them. Returns the app plus a freshly issued raw key
|
||||
// for the supplied role.
|
||||
func mwTestSetup(t *testing.T, role auth.Role, mw fiber.Handler) (*fiber.App, *auth.Manager, string, *auth.APIKey) {
|
||||
t.Helper()
|
||||
mgr := auth.New()
|
||||
apiKey, raw, err := mgr.GenerateAPIKey("test-"+string(role), role, nil)
|
||||
require.NoError(t, err)
|
||||
|
||||
app := fiber.New(fiber.Config{DisableStartupMessage: true})
|
||||
app.Use("/protected", mw)
|
||||
app.Get("/protected", func(c *fiber.Ctx) error {
|
||||
got, _ := c.Locals(LocalAuthKey).(*auth.APIKey)
|
||||
gotRole, _ := c.Locals(LocalAuthRole).(string)
|
||||
body := fiber.Map{"ok": true, "have_key": got != nil}
|
||||
if got != nil {
|
||||
body["key_id"] = got.ID
|
||||
body["role"] = gotRole
|
||||
}
|
||||
return c.Status(fiber.StatusOK).JSON(body)
|
||||
})
|
||||
return app, mgr, raw, apiKey
|
||||
}
|
||||
|
||||
func decodeErrorBody(t *testing.T, body io.Reader) errors.Error {
|
||||
t.Helper()
|
||||
var e errors.Error
|
||||
require.NoError(t, json.NewDecoder(body).Decode(&e))
|
||||
return e
|
||||
}
|
||||
|
||||
func TestRequireAuth_NoHeader_Returns401(t *testing.T) {
|
||||
app, _, _, _ := mwTestSetup(t, auth.RoleReadOnly, RequireAuth(auth.New()))
|
||||
req := httptest.NewRequest("GET", "/protected", nil)
|
||||
resp, err := app.Test(req, 5000)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, fiber.StatusUnauthorized, resp.StatusCode)
|
||||
body := decodeErrorBody(t, resp.Body)
|
||||
require.Equal(t, errors.ErrCodeUnauthorized, body.Code)
|
||||
}
|
||||
|
||||
func TestRequireAuth_BadKey_Returns401(t *testing.T) {
|
||||
mgr := auth.New()
|
||||
_, _, err := mgr.GenerateAPIKey("real", auth.RoleReadOnly, nil)
|
||||
require.NoError(t, err)
|
||||
|
||||
app, _, _, _ := mwTestSetup(t, auth.RoleReadOnly, RequireAuth(mgr))
|
||||
req := httptest.NewRequest("GET", "/protected", nil)
|
||||
req.Header.Set("Authorization", "Bearer not-a-real-key")
|
||||
resp, err := app.Test(req, 5000)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, fiber.StatusUnauthorized, resp.StatusCode)
|
||||
body := decodeErrorBody(t, resp.Body)
|
||||
require.Equal(t, errors.ErrCodeInvalidAPIKey, body.Code)
|
||||
}
|
||||
|
||||
func TestRequireAuth_BadHeaderShape_Returns401(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
header string
|
||||
value string
|
||||
}{
|
||||
{"non-bearer scheme", "Authorization", "Basic abcdef"},
|
||||
{"bearer no token", "Authorization", "Bearer "},
|
||||
{"empty x-api-key", "X-API-Key", ""},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
tc := tc
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
mgr := auth.New()
|
||||
app := fiber.New(fiber.Config{DisableStartupMessage: true})
|
||||
app.Use("/protected", RequireAuth(mgr))
|
||||
app.Get("/protected", func(c *fiber.Ctx) error { return c.SendStatus(200) })
|
||||
req := httptest.NewRequest("GET", "/protected", nil)
|
||||
if tc.value != "" {
|
||||
req.Header.Set(tc.header, tc.value)
|
||||
}
|
||||
resp, err := app.Test(req, 5000)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, fiber.StatusUnauthorized, resp.StatusCode)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRequireAuth_ValidBearer_Returns200_AndPopulatesLocals(t *testing.T) {
|
||||
mgr := auth.New()
|
||||
apiKey, raw, err := mgr.GenerateAPIKey("good", auth.RoleReadWrite, nil)
|
||||
require.NoError(t, err)
|
||||
|
||||
app := fiber.New(fiber.Config{DisableStartupMessage: true})
|
||||
app.Use("/protected", RequireAuth(mgr))
|
||||
app.Get("/protected", func(c *fiber.Ctx) error {
|
||||
k, _ := c.Locals(LocalAuthKey).(*auth.APIKey)
|
||||
require.NotNil(t, k)
|
||||
require.Equal(t, apiKey.ID, k.ID)
|
||||
require.Equal(t, string(auth.RoleReadWrite), c.Locals(LocalAuthRole))
|
||||
return c.Status(200).JSON(fiber.Map{"key_id": k.ID})
|
||||
})
|
||||
|
||||
req := httptest.NewRequest("GET", "/protected", nil)
|
||||
req.Header.Set("Authorization", "Bearer "+raw)
|
||||
resp, err := app.Test(req, 5000)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, 200, resp.StatusCode)
|
||||
|
||||
var body map[string]string
|
||||
require.NoError(t, json.NewDecoder(resp.Body).Decode(&body))
|
||||
require.Equal(t, apiKey.ID, body["key_id"])
|
||||
}
|
||||
|
||||
func TestRequireAuth_ValidXAPIKey_Returns200(t *testing.T) {
|
||||
mgr := auth.New()
|
||||
_, raw, err := mgr.GenerateAPIKey("good", auth.RoleReadOnly, nil)
|
||||
require.NoError(t, err)
|
||||
|
||||
app := fiber.New(fiber.Config{DisableStartupMessage: true})
|
||||
app.Use("/protected", RequireAuth(mgr))
|
||||
app.Get("/protected", func(c *fiber.Ctx) error { return c.SendStatus(200) })
|
||||
|
||||
req := httptest.NewRequest("GET", "/protected", nil)
|
||||
req.Header.Set("X-API-Key", raw)
|
||||
resp, err := app.Test(req, 5000)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, 200, resp.StatusCode)
|
||||
}
|
||||
|
||||
func TestRequireAuth_ExpiredKey_Returns401(t *testing.T) {
|
||||
mgr := auth.New()
|
||||
d := -1 * time.Hour
|
||||
_, raw, err := mgr.GenerateAPIKey("expired", auth.RoleReadOnly, &d)
|
||||
require.NoError(t, err)
|
||||
|
||||
app := fiber.New(fiber.Config{DisableStartupMessage: true})
|
||||
app.Use("/protected", RequireAuth(mgr))
|
||||
app.Get("/protected", func(c *fiber.Ctx) error { return c.SendStatus(200) })
|
||||
|
||||
req := httptest.NewRequest("GET", "/protected", nil)
|
||||
req.Header.Set("Authorization", "Bearer "+raw)
|
||||
resp, err := app.Test(req, 5000)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, fiber.StatusUnauthorized, resp.StatusCode)
|
||||
}
|
||||
|
||||
func TestRequireRole_Match_Returns200(t *testing.T) {
|
||||
mgr := auth.New()
|
||||
_, raw, err := mgr.GenerateAPIKey("admin", auth.RoleAdmin, nil)
|
||||
require.NoError(t, err)
|
||||
|
||||
app := fiber.New(fiber.Config{DisableStartupMessage: true})
|
||||
app.Use("/protected", RequireRole(mgr, string(auth.RoleAdmin), string(auth.RoleReadWrite)))
|
||||
app.Get("/protected", func(c *fiber.Ctx) error { return c.SendStatus(200) })
|
||||
|
||||
req := httptest.NewRequest("GET", "/protected", nil)
|
||||
req.Header.Set("Authorization", "Bearer "+raw)
|
||||
resp, err := app.Test(req, 5000)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, 200, resp.StatusCode)
|
||||
}
|
||||
|
||||
func TestRequireRole_Mismatch_Returns403(t *testing.T) {
|
||||
mgr := auth.New()
|
||||
_, raw, err := mgr.GenerateAPIKey("ro", auth.RoleReadOnly, nil)
|
||||
require.NoError(t, err)
|
||||
|
||||
app := fiber.New(fiber.Config{DisableStartupMessage: true})
|
||||
app.Use("/protected", RequireRole(mgr, string(auth.RoleAdmin)))
|
||||
app.Get("/protected", func(c *fiber.Ctx) error { return c.SendStatus(200) })
|
||||
|
||||
req := httptest.NewRequest("GET", "/protected", nil)
|
||||
req.Header.Set("Authorization", "Bearer "+raw)
|
||||
resp, err := app.Test(req, 5000)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, fiber.StatusForbidden, resp.StatusCode)
|
||||
body := decodeErrorBody(t, resp.Body)
|
||||
require.Equal(t, errors.ErrCodeForbidden, body.Code)
|
||||
}
|
||||
|
||||
func TestRequireRole_NoHeader_Returns401(t *testing.T) {
|
||||
mgr := auth.New()
|
||||
app := fiber.New(fiber.Config{DisableStartupMessage: true})
|
||||
app.Use("/protected", RequireRole(mgr, string(auth.RoleAdmin)))
|
||||
app.Get("/protected", func(c *fiber.Ctx) error { return c.SendStatus(200) })
|
||||
|
||||
req := httptest.NewRequest("GET", "/protected", nil)
|
||||
resp, err := app.Test(req, 5000)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, fiber.StatusUnauthorized, resp.StatusCode)
|
||||
}
|
||||
|
||||
func TestRequirePermission_Match_Returns200(t *testing.T) {
|
||||
mgr := auth.New()
|
||||
_, raw, err := mgr.GenerateAPIKey("rw", auth.RoleReadWrite, nil)
|
||||
require.NoError(t, err)
|
||||
|
||||
app := fiber.New(fiber.Config{DisableStartupMessage: true})
|
||||
app.Use("/protected", RequirePermission(mgr, string(auth.PermissionWritePackage)))
|
||||
app.Get("/protected", func(c *fiber.Ctx) error { return c.SendStatus(200) })
|
||||
|
||||
req := httptest.NewRequest("GET", "/protected", nil)
|
||||
req.Header.Set("Authorization", "Bearer "+raw)
|
||||
resp, err := app.Test(req, 5000)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, 200, resp.StatusCode)
|
||||
}
|
||||
|
||||
func TestRequirePermission_Mismatch_Returns403(t *testing.T) {
|
||||
mgr := auth.New()
|
||||
_, raw, err := mgr.GenerateAPIKey("ro", auth.RoleReadOnly, nil)
|
||||
require.NoError(t, err)
|
||||
|
||||
app := fiber.New(fiber.Config{DisableStartupMessage: true})
|
||||
app.Use("/protected", RequirePermission(mgr, string(auth.PermissionDeletePackage)))
|
||||
app.Get("/protected", func(c *fiber.Ctx) error { return c.SendStatus(200) })
|
||||
|
||||
req := httptest.NewRequest("GET", "/protected", nil)
|
||||
req.Header.Set("Authorization", "Bearer "+raw)
|
||||
resp, err := app.Test(req, 5000)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, fiber.StatusForbidden, resp.StatusCode)
|
||||
}
|
||||
|
||||
func TestOptionalAuth_NoHeader_Continues(t *testing.T) {
|
||||
mgr := auth.New()
|
||||
app := fiber.New(fiber.Config{DisableStartupMessage: true})
|
||||
app.Use("/protected", OptionalAuth(mgr))
|
||||
app.Get("/protected", func(c *fiber.Ctx) error {
|
||||
require.Nil(t, c.Locals(LocalAuthKey))
|
||||
return c.SendStatus(200)
|
||||
})
|
||||
req := httptest.NewRequest("GET", "/protected", nil)
|
||||
resp, err := app.Test(req, 5000)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, 200, resp.StatusCode)
|
||||
}
|
||||
|
||||
func TestOptionalAuth_BadKey_ContinuesAnonymously(t *testing.T) {
|
||||
mgr := auth.New()
|
||||
app := fiber.New(fiber.Config{DisableStartupMessage: true})
|
||||
app.Use("/protected", OptionalAuth(mgr))
|
||||
app.Get("/protected", func(c *fiber.Ctx) error {
|
||||
// Bad key should NOT populate locals.
|
||||
require.Nil(t, c.Locals(LocalAuthKey))
|
||||
return c.SendStatus(200)
|
||||
})
|
||||
req := httptest.NewRequest("GET", "/protected", nil)
|
||||
req.Header.Set("Authorization", "Bearer not-real")
|
||||
resp, err := app.Test(req, 5000)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, 200, resp.StatusCode)
|
||||
}
|
||||
|
||||
func TestOptionalAuth_GoodKey_PopulatesLocals(t *testing.T) {
|
||||
mgr := auth.New()
|
||||
apiKey, raw, err := mgr.GenerateAPIKey("good", auth.RoleAdmin, nil)
|
||||
require.NoError(t, err)
|
||||
|
||||
app := fiber.New(fiber.Config{DisableStartupMessage: true})
|
||||
app.Use("/protected", OptionalAuth(mgr))
|
||||
app.Get("/protected", func(c *fiber.Ctx) error {
|
||||
k, _ := c.Locals(LocalAuthKey).(*auth.APIKey)
|
||||
require.NotNil(t, k)
|
||||
require.Equal(t, apiKey.ID, k.ID)
|
||||
return c.SendStatus(200)
|
||||
})
|
||||
|
||||
req := httptest.NewRequest("GET", "/protected", nil)
|
||||
req.Header.Set("Authorization", "Bearer "+raw)
|
||||
resp, err := app.Test(req, 5000)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, 200, resp.StatusCode)
|
||||
}
|
||||
+23
-281
@@ -1,48 +1,20 @@
|
||||
// Package auth implements API key issuance, validation, and credential
|
||||
// extraction for upstream-registry pass-through authentication.
|
||||
package auth
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"encoding/base64"
|
||||
stderrors "errors"
|
||||
"fmt"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/lukaszraczylo/gohoarder/pkg/errors"
|
||||
"github.com/lukaszraczylo/gohoarder/pkg/metadata"
|
||||
"github.com/rs/zerolog/log"
|
||||
"golang.org/x/crypto/bcrypt"
|
||||
)
|
||||
|
||||
// Config controls Manager behaviour. Zero value is valid: bcrypt cost defaults
|
||||
// to bcrypt.DefaultCost, store is nil (in-memory only).
|
||||
type Config struct {
|
||||
// BcryptCost is the cost parameter passed to bcrypt.GenerateFromPassword.
|
||||
// 0 means use bcrypt.DefaultCost. Tests typically use bcrypt.MinCost.
|
||||
BcryptCost int
|
||||
}
|
||||
|
||||
// Manager handles authentication and authorization.
|
||||
//
|
||||
// Persistence model:
|
||||
// - All keys live in an in-memory map keyed by APIKey.ID for O(1) revoke
|
||||
// and fast bcrypt iteration during validation.
|
||||
// - When a metadata.MetadataStore is wired in via NewWithStore, mutations
|
||||
// (Generate, Revoke) are mirrored to the store synchronously and
|
||||
// LastUsedAt updates are flushed asynchronously to avoid blocking the
|
||||
// hot validation path on a DB round-trip.
|
||||
// - When the store is nil, Manager is purely in-memory (back-compat for
|
||||
// tests and unconfigured deployments).
|
||||
// Manager handles authentication and authorization
|
||||
type Manager struct {
|
||||
keys map[string]*APIKey
|
||||
store metadata.MetadataStore
|
||||
cfg Config
|
||||
mu sync.RWMutex
|
||||
bgWG sync.WaitGroup
|
||||
closed bool
|
||||
keys map[string]*APIKey
|
||||
mu sync.RWMutex
|
||||
}
|
||||
|
||||
// APIKey represents an API key
|
||||
@@ -51,7 +23,6 @@ type APIKey struct {
|
||||
Name string
|
||||
HashedKey string
|
||||
Role Role
|
||||
Project string
|
||||
CreatedAt time.Time
|
||||
ExpiresAt *time.Time
|
||||
LastUsedAt time.Time
|
||||
@@ -67,15 +38,6 @@ const (
|
||||
RoleAdmin Role = "admin"
|
||||
)
|
||||
|
||||
// Persistent role identifiers stored in metadata.APIKey.Role. Different from
|
||||
// the in-memory Role values for backward compatibility with existing API
|
||||
// surfaces while matching the storage schema described in the spec.
|
||||
const (
|
||||
storedRoleReadOnly = "read_only"
|
||||
storedRoleReadWrite = "read_write"
|
||||
storedRoleAdmin = "admin"
|
||||
)
|
||||
|
||||
// Permission represents a specific permission
|
||||
type Permission string
|
||||
|
||||
@@ -90,63 +52,14 @@ const (
|
||||
PermissionManageBypasses Permission = "bypasses:manage"
|
||||
)
|
||||
|
||||
// New creates a new authentication manager (in-memory only).
|
||||
// Equivalent to NewWithStore(Config{}, nil).
|
||||
// New creates a new authentication manager
|
||||
func New() *Manager {
|
||||
return NewWithStore(Config{}, nil)
|
||||
}
|
||||
|
||||
// NewWithStore creates a Manager backed by an optional metadata store.
|
||||
// Pass store=nil for purely in-memory mode (tests, ephemeral setups).
|
||||
func NewWithStore(cfg Config, store metadata.MetadataStore) *Manager {
|
||||
if cfg.BcryptCost == 0 {
|
||||
cfg.BcryptCost = bcrypt.DefaultCost
|
||||
}
|
||||
return &Manager{
|
||||
keys: make(map[string]*APIKey),
|
||||
store: store,
|
||||
cfg: cfg,
|
||||
keys: make(map[string]*APIKey),
|
||||
}
|
||||
}
|
||||
|
||||
// Load pulls all non-revoked keys from the store into the in-memory map.
|
||||
// Safe to call multiple times: it replaces the in-memory snapshot.
|
||||
// No-op when store is nil or returns ErrNotImplemented.
|
||||
func (m *Manager) Load(ctx context.Context) error {
|
||||
if m.store == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
stored, err := m.store.ListAPIKeys(ctx)
|
||||
if err != nil {
|
||||
if stderrors.Is(err, metadata.ErrNotImplemented) {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
loaded := make(map[string]*APIKey, len(stored))
|
||||
for _, k := range stored {
|
||||
if k == nil || k.Revoked {
|
||||
continue
|
||||
}
|
||||
loaded[k.ID] = storedToMemory(k)
|
||||
}
|
||||
|
||||
m.mu.Lock()
|
||||
m.keys = loaded
|
||||
m.mu.Unlock()
|
||||
|
||||
log.Info().Int("count", len(loaded)).Msg("Loaded API keys from metadata store")
|
||||
return nil
|
||||
}
|
||||
|
||||
// GenerateAPIKey generates a new API key.
|
||||
//
|
||||
// When a store is configured, the key is persisted before returning. If the
|
||||
// store rejects the write the in-memory state is rolled back and the error
|
||||
// is propagated; we never return a key the caller cannot actually use after
|
||||
// a restart.
|
||||
// GenerateAPIKey generates a new API key
|
||||
func (m *Manager) GenerateAPIKey(name string, role Role, expiresIn *time.Duration) (*APIKey, string, error) {
|
||||
// Generate random key
|
||||
keyBytes := make([]byte, 32)
|
||||
@@ -156,8 +69,8 @@ func (m *Manager) GenerateAPIKey(name string, role Role, expiresIn *time.Duratio
|
||||
|
||||
rawKey := base64.URLEncoding.EncodeToString(keyBytes)
|
||||
|
||||
// Hash the key with the configured cost.
|
||||
hashedKey, err := bcrypt.GenerateFromPassword([]byte(rawKey), m.cfg.BcryptCost)
|
||||
// Hash the key
|
||||
hashedKey, err := bcrypt.GenerateFromPassword([]byte(rawKey), bcrypt.DefaultCost)
|
||||
if err != nil {
|
||||
return nil, "", errors.Wrap(err, errors.ErrCodeInternalServer, "failed to hash key")
|
||||
}
|
||||
@@ -182,57 +95,24 @@ func (m *Manager) GenerateAPIKey(name string, role Role, expiresIn *time.Duratio
|
||||
m.keys[apiKey.ID] = apiKey
|
||||
m.mu.Unlock()
|
||||
|
||||
if m.store != nil {
|
||||
stored := memoryToStored(apiKey)
|
||||
if err := m.store.SaveAPIKey(context.Background(), stored); err != nil {
|
||||
if !stderrors.Is(err, metadata.ErrNotImplemented) {
|
||||
// Rollback in-memory insert so caller sees a consistent failure.
|
||||
m.mu.Lock()
|
||||
delete(m.keys, apiKey.ID)
|
||||
m.mu.Unlock()
|
||||
return nil, "", errors.Wrap(err, errors.ErrCodeStorageFailure, "failed to persist api key")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return apiKey, rawKey, nil
|
||||
}
|
||||
|
||||
// ValidateAPIKey validates an API key and returns the associated key object.
|
||||
// Hot path: snapshots candidates, runs bcrypt without holding the lock.
|
||||
// LastUsedAt is updated in-memory under the write lock and flushed to the
|
||||
// store via a fire-and-forget goroutine.
|
||||
// ValidateAPIKey validates an API key and returns the associated key object
|
||||
func (m *Manager) ValidateAPIKey(ctx context.Context, rawKey string) (*APIKey, error) {
|
||||
// Phase 1: snapshot non-expired candidates under RLock. We hold pointers
|
||||
// to APIKey; APIKey.HashedKey is an immutable string so reading it
|
||||
// without a lock is safe even if the key is later removed from the map.
|
||||
m.mu.RLock()
|
||||
candidates := make([]*APIKey, 0, len(m.keys))
|
||||
now := time.Now()
|
||||
defer m.mu.RUnlock()
|
||||
|
||||
for _, apiKey := range m.keys {
|
||||
if apiKey.ExpiresAt != nil && now.After(*apiKey.ExpiresAt) {
|
||||
// Check if key is expired
|
||||
if apiKey.ExpiresAt != nil && time.Now().After(*apiKey.ExpiresAt) {
|
||||
continue
|
||||
}
|
||||
candidates = append(candidates, apiKey)
|
||||
}
|
||||
m.mu.RUnlock()
|
||||
|
||||
// Phase 2: run bcrypt comparisons without holding any lock so concurrent
|
||||
// auth checks can proceed in parallel.
|
||||
for _, apiKey := range candidates {
|
||||
// Compare hashed key
|
||||
if err := bcrypt.CompareHashAndPassword([]byte(apiKey.HashedKey), []byte(rawKey)); err == nil {
|
||||
// Phase 3: briefly take the write lock to update LastUsedAt.
|
||||
// Re-check the key still exists in the map to avoid resurrecting
|
||||
// a revoked key's metadata.
|
||||
usedAt := time.Now()
|
||||
m.mu.Lock()
|
||||
if _, ok := m.keys[apiKey.ID]; ok {
|
||||
apiKey.LastUsedAt = usedAt
|
||||
}
|
||||
m.mu.Unlock()
|
||||
|
||||
// Async store update: never block validation on a DB write.
|
||||
m.persistLastUsedAsync(apiKey.ID, usedAt)
|
||||
// Update last used
|
||||
apiKey.LastUsedAt = time.Now()
|
||||
return apiKey, nil
|
||||
}
|
||||
}
|
||||
@@ -240,63 +120,16 @@ func (m *Manager) ValidateAPIKey(ctx context.Context, rawKey string) (*APIKey, e
|
||||
return nil, errors.New(errors.ErrCodeUnauthorized, "invalid API key")
|
||||
}
|
||||
|
||||
// persistLastUsedAsync flushes a LastUsedAt update to the store off the hot
|
||||
// path. Errors are logged but not propagated — the validation succeeded
|
||||
// in-memory and that is the source of truth for liveness.
|
||||
func (m *Manager) persistLastUsedAsync(id string, t time.Time) {
|
||||
if m.store == nil {
|
||||
return
|
||||
}
|
||||
|
||||
m.mu.Lock()
|
||||
if m.closed {
|
||||
m.mu.Unlock()
|
||||
return
|
||||
}
|
||||
m.bgWG.Add(1)
|
||||
m.mu.Unlock()
|
||||
|
||||
go func() {
|
||||
defer m.bgWG.Done()
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
if err := m.store.UpdateAPIKeyLastUsed(ctx, id, t); err != nil {
|
||||
if !stderrors.Is(err, metadata.ErrNotImplemented) {
|
||||
log.Warn().Err(err).Str("key_id", id).Msg("Failed to persist api key LastUsedAt")
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
// RevokeAPIKey revokes an API key. With a store configured, revocation is
|
||||
// persisted (Revoked=true) so the key remains queryable for audit but never
|
||||
// authenticates again.
|
||||
// RevokeAPIKey revokes an API key
|
||||
func (m *Manager) RevokeAPIKey(keyID string) error {
|
||||
m.mu.Lock()
|
||||
existing, ok := m.keys[keyID]
|
||||
if !ok {
|
||||
m.mu.Unlock()
|
||||
defer m.mu.Unlock()
|
||||
|
||||
if _, exists := m.keys[keyID]; !exists {
|
||||
return errors.NotFound("API key not found")
|
||||
}
|
||||
|
||||
delete(m.keys, keyID)
|
||||
m.mu.Unlock()
|
||||
|
||||
if m.store != nil {
|
||||
stored := memoryToStored(existing)
|
||||
stored.Revoked = true
|
||||
if err := m.store.SaveAPIKey(context.Background(), stored); err != nil {
|
||||
if !stderrors.Is(err, metadata.ErrNotImplemented) {
|
||||
// Rollback: re-add to memory so subsequent validation still
|
||||
// sees it (rather than silently leaving a window where the
|
||||
// key works in-memory only on this replica).
|
||||
m.mu.Lock()
|
||||
m.keys[keyID] = existing
|
||||
m.mu.Unlock()
|
||||
return errors.Wrap(err, errors.ErrCodeStorageFailure, "failed to persist revocation")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -312,19 +145,6 @@ func (m *Manager) ListAPIKeys() []*APIKey {
|
||||
return keys
|
||||
}
|
||||
|
||||
// Close waits for any in-flight background writes (LastUsedAt updates) to
|
||||
// finish. Safe to call multiple times.
|
||||
func (m *Manager) Close() {
|
||||
m.mu.Lock()
|
||||
if m.closed {
|
||||
m.mu.Unlock()
|
||||
return
|
||||
}
|
||||
m.closed = true
|
||||
m.mu.Unlock()
|
||||
m.bgWG.Wait()
|
||||
}
|
||||
|
||||
// HasPermission checks if an API key has a specific permission
|
||||
func (k *APIKey) HasPermission(permission Permission) bool {
|
||||
for _, p := range k.Permissions {
|
||||
@@ -365,87 +185,9 @@ func getPermissionsForRole(role Role) []Permission {
|
||||
}
|
||||
}
|
||||
|
||||
// generateID generates a unique ID. Panics on crypto/rand failure: a
|
||||
// zero-byte ID would be a security catastrophe (collisions, predictability).
|
||||
// generateID generates a unique ID
|
||||
func generateID() string {
|
||||
b := make([]byte, 16)
|
||||
if _, err := rand.Read(b); err != nil {
|
||||
panic(fmt.Sprintf("auth: crypto/rand read failed: %v", err))
|
||||
}
|
||||
_, _ = rand.Read(b) // #nosec G104 -- Rand read always succeeds
|
||||
return base64.URLEncoding.EncodeToString(b)
|
||||
}
|
||||
|
||||
// memoryToStored converts an in-memory APIKey to its metadata-layer form.
|
||||
// Permissions are deliberately not encoded here: they are derived from Role
|
||||
// at load time, keeping the persisted representation compact.
|
||||
func memoryToStored(k *APIKey) *metadata.APIKey {
|
||||
var lastUsed *time.Time
|
||||
if !k.LastUsedAt.IsZero() {
|
||||
t := k.LastUsedAt
|
||||
lastUsed = &t
|
||||
}
|
||||
|
||||
return &metadata.APIKey{
|
||||
ID: k.ID,
|
||||
KeyHash: k.HashedKey,
|
||||
Project: k.Project,
|
||||
Role: roleToStored(k.Role),
|
||||
CreatedAt: k.CreatedAt,
|
||||
ExpiresAt: k.ExpiresAt,
|
||||
LastUsedAt: lastUsed,
|
||||
Revoked: false,
|
||||
}
|
||||
}
|
||||
|
||||
// storedToMemory converts a metadata.APIKey to its runtime form. Permissions
|
||||
// are derived from Role to keep storage minimal.
|
||||
func storedToMemory(k *metadata.APIKey) *APIKey {
|
||||
role := storedToRole(k.Role)
|
||||
out := &APIKey{
|
||||
ID: k.ID,
|
||||
Name: "", // not persisted
|
||||
HashedKey: k.KeyHash,
|
||||
Role: role,
|
||||
Project: k.Project,
|
||||
CreatedAt: k.CreatedAt,
|
||||
ExpiresAt: k.ExpiresAt,
|
||||
Permissions: getPermissionsForRole(role),
|
||||
}
|
||||
if k.LastUsedAt != nil {
|
||||
out.LastUsedAt = *k.LastUsedAt
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// roleToStored maps the in-memory Role enum to its persisted string form.
|
||||
func roleToStored(r Role) string {
|
||||
switch r {
|
||||
case RoleReadOnly:
|
||||
return storedRoleReadOnly
|
||||
case RoleReadWrite:
|
||||
return storedRoleReadWrite
|
||||
case RoleAdmin:
|
||||
return storedRoleAdmin
|
||||
default:
|
||||
return string(r)
|
||||
}
|
||||
}
|
||||
|
||||
// storedToRole inverts roleToStored. Unknown values fall back to RoleReadOnly
|
||||
// (least-privilege) rather than failing — guards against schema drift.
|
||||
func storedToRole(s string) Role {
|
||||
switch s {
|
||||
case storedRoleAdmin:
|
||||
return RoleAdmin
|
||||
case storedRoleReadWrite:
|
||||
return RoleReadWrite
|
||||
case storedRoleReadOnly:
|
||||
return RoleReadOnly
|
||||
}
|
||||
// Tolerate legacy in-memory values that may have been persisted.
|
||||
switch Role(s) {
|
||||
case RoleAdmin, RoleReadWrite, RoleReadOnly:
|
||||
return Role(s)
|
||||
}
|
||||
return RoleReadOnly
|
||||
}
|
||||
|
||||
@@ -1,372 +0,0 @@
|
||||
package auth
|
||||
|
||||
import (
|
||||
"context"
|
||||
stderrors "errors"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/lukaszraczylo/gohoarder/pkg/metadata"
|
||||
"golang.org/x/crypto/bcrypt"
|
||||
)
|
||||
|
||||
// fakeStore is a minimal in-memory metadata.MetadataStore. Only the API key
|
||||
// methods are exercised; everything else returns ErrNotImplemented to make
|
||||
// accidental calls obvious.
|
||||
type fakeStore struct {
|
||||
keys map[string]*metadata.APIKey
|
||||
saveErr error
|
||||
listErr error
|
||||
updateErr error
|
||||
mu sync.Mutex
|
||||
saveCalls int
|
||||
updateCalls int
|
||||
disabled bool // when true, every API key method returns ErrNotImplemented
|
||||
}
|
||||
|
||||
func newFakeStore() *fakeStore {
|
||||
return &fakeStore{keys: make(map[string]*metadata.APIKey)}
|
||||
}
|
||||
|
||||
func (f *fakeStore) SaveAPIKey(ctx context.Context, key *metadata.APIKey) error {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
if f.disabled {
|
||||
return metadata.ErrNotImplemented
|
||||
}
|
||||
f.saveCalls++
|
||||
if f.saveErr != nil {
|
||||
return f.saveErr
|
||||
}
|
||||
cp := *key
|
||||
f.keys[key.ID] = &cp
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *fakeStore) GetAPIKey(ctx context.Context, id string) (*metadata.APIKey, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
if f.disabled {
|
||||
return nil, metadata.ErrNotImplemented
|
||||
}
|
||||
k, ok := f.keys[id]
|
||||
if !ok {
|
||||
return nil, stderrors.New("not found")
|
||||
}
|
||||
cp := *k
|
||||
return &cp, nil
|
||||
}
|
||||
|
||||
func (f *fakeStore) ListAPIKeys(ctx context.Context) ([]*metadata.APIKey, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
if f.disabled {
|
||||
return nil, metadata.ErrNotImplemented
|
||||
}
|
||||
if f.listErr != nil {
|
||||
return nil, f.listErr
|
||||
}
|
||||
out := make([]*metadata.APIKey, 0, len(f.keys))
|
||||
for _, k := range f.keys {
|
||||
cp := *k
|
||||
out = append(out, &cp)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (f *fakeStore) DeleteAPIKey(ctx context.Context, id string) error {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
if f.disabled {
|
||||
return metadata.ErrNotImplemented
|
||||
}
|
||||
delete(f.keys, id)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *fakeStore) UpdateAPIKeyLastUsed(ctx context.Context, id string, t time.Time) error {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
if f.disabled {
|
||||
return metadata.ErrNotImplemented
|
||||
}
|
||||
f.updateCalls++
|
||||
if f.updateErr != nil {
|
||||
return f.updateErr
|
||||
}
|
||||
if k, ok := f.keys[id]; ok {
|
||||
tt := t
|
||||
k.LastUsedAt = &tt
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// All other methods are unreachable from auth.Manager — return ErrNotImplemented
|
||||
// so accidental usage in future tests fails loudly.
|
||||
func (f *fakeStore) SavePackage(context.Context, *metadata.Package) error {
|
||||
return metadata.ErrNotImplemented
|
||||
}
|
||||
func (f *fakeStore) GetPackage(context.Context, string, string, string) (*metadata.Package, error) {
|
||||
return nil, metadata.ErrNotImplemented
|
||||
}
|
||||
func (f *fakeStore) DeletePackage(context.Context, string, string, string) error {
|
||||
return metadata.ErrNotImplemented
|
||||
}
|
||||
func (f *fakeStore) ListPackages(context.Context, *metadata.ListOptions) ([]*metadata.Package, error) {
|
||||
return nil, metadata.ErrNotImplemented
|
||||
}
|
||||
func (f *fakeStore) UpdateDownloadCount(context.Context, string, string, string) error {
|
||||
return metadata.ErrNotImplemented
|
||||
}
|
||||
func (f *fakeStore) GetStats(context.Context, string) (*metadata.Stats, error) {
|
||||
return nil, metadata.ErrNotImplemented
|
||||
}
|
||||
func (f *fakeStore) SaveScanResult(context.Context, *metadata.ScanResult) error {
|
||||
return metadata.ErrNotImplemented
|
||||
}
|
||||
func (f *fakeStore) GetScanResult(context.Context, string, string, string) (*metadata.ScanResult, error) {
|
||||
return nil, metadata.ErrNotImplemented
|
||||
}
|
||||
func (f *fakeStore) SaveCVEBypass(context.Context, *metadata.CVEBypass) error {
|
||||
return metadata.ErrNotImplemented
|
||||
}
|
||||
func (f *fakeStore) GetActiveCVEBypasses(context.Context) ([]*metadata.CVEBypass, error) {
|
||||
return nil, metadata.ErrNotImplemented
|
||||
}
|
||||
func (f *fakeStore) ListCVEBypasses(context.Context, *metadata.BypassListOptions) ([]*metadata.CVEBypass, error) {
|
||||
return nil, metadata.ErrNotImplemented
|
||||
}
|
||||
func (f *fakeStore) DeleteCVEBypass(context.Context, string) error { return metadata.ErrNotImplemented }
|
||||
func (f *fakeStore) CleanupExpiredBypasses(context.Context) (int, error) {
|
||||
return 0, metadata.ErrNotImplemented
|
||||
}
|
||||
func (f *fakeStore) Count(context.Context) (int, error) { return 0, metadata.ErrNotImplemented }
|
||||
func (f *fakeStore) Health(context.Context) error { return metadata.ErrNotImplemented }
|
||||
func (f *fakeStore) GetTimeSeriesStats(context.Context, string, string) (*metadata.TimeSeriesStats, error) {
|
||||
return nil, metadata.ErrNotImplemented
|
||||
}
|
||||
func (f *fakeStore) AggregateDownloadData(context.Context) error { return metadata.ErrNotImplemented }
|
||||
func (f *fakeStore) Close() error { return nil }
|
||||
|
||||
// fastCfg uses bcrypt.MinCost so tests that hash multiple keys stay snappy.
|
||||
func fastCfg() Config { return Config{BcryptCost: bcrypt.MinCost} }
|
||||
|
||||
func TestManager_GenerateAPIKey_PersistsToStore(t *testing.T) {
|
||||
store := newFakeStore()
|
||||
m := NewWithStore(fastCfg(), store)
|
||||
defer m.Close()
|
||||
|
||||
apiKey, raw, err := m.GenerateAPIKey("svc-token", RoleReadWrite, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("GenerateAPIKey: %v", err)
|
||||
}
|
||||
if raw == "" {
|
||||
t.Fatalf("raw key empty")
|
||||
}
|
||||
if apiKey.ID == "" {
|
||||
t.Fatalf("apiKey.ID empty")
|
||||
}
|
||||
|
||||
if store.saveCalls != 1 {
|
||||
t.Errorf("saveCalls = %d, want 1", store.saveCalls)
|
||||
}
|
||||
persisted, err := store.GetAPIKey(context.Background(), apiKey.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("GetAPIKey persisted: %v", err)
|
||||
}
|
||||
if persisted.Role != storedRoleReadWrite {
|
||||
t.Errorf("persisted Role = %q, want %q", persisted.Role, storedRoleReadWrite)
|
||||
}
|
||||
if persisted.Revoked {
|
||||
t.Errorf("persisted Revoked = true, want false")
|
||||
}
|
||||
}
|
||||
|
||||
func TestManager_GenerateAPIKey_StoreFailure_RollsBack(t *testing.T) {
|
||||
store := newFakeStore()
|
||||
store.saveErr = stderrors.New("disk full")
|
||||
m := NewWithStore(fastCfg(), store)
|
||||
defer m.Close()
|
||||
|
||||
_, _, err := m.GenerateAPIKey("svc", RoleReadOnly, nil)
|
||||
if err == nil {
|
||||
t.Fatalf("want error from store, got nil")
|
||||
}
|
||||
if got := len(m.ListAPIKeys()); got != 0 {
|
||||
t.Errorf("in-memory keys = %d, want 0 after rollback", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestManager_RevokeAPIKey_PersistsRevocation(t *testing.T) {
|
||||
store := newFakeStore()
|
||||
m := NewWithStore(fastCfg(), store)
|
||||
defer m.Close()
|
||||
|
||||
key, _, err := m.GenerateAPIKey("svc", RoleAdmin, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("Generate: %v", err)
|
||||
}
|
||||
|
||||
if revokeErr := m.RevokeAPIKey(key.ID); revokeErr != nil {
|
||||
t.Fatalf("Revoke: %v", revokeErr)
|
||||
}
|
||||
persisted, err := store.GetAPIKey(context.Background(), key.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("GetAPIKey after revoke: %v", err)
|
||||
}
|
||||
if !persisted.Revoked {
|
||||
t.Errorf("persisted Revoked = false, want true")
|
||||
}
|
||||
if got := len(m.ListAPIKeys()); got != 0 {
|
||||
t.Errorf("in-memory keys after revoke = %d, want 0", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestManager_RevokeAPIKey_StoreFailure_KeepsInMemory(t *testing.T) {
|
||||
store := newFakeStore()
|
||||
m := NewWithStore(fastCfg(), store)
|
||||
defer m.Close()
|
||||
|
||||
key, _, err := m.GenerateAPIKey("svc", RoleAdmin, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("Generate: %v", err)
|
||||
}
|
||||
|
||||
store.saveErr = stderrors.New("DB exploded")
|
||||
if err := m.RevokeAPIKey(key.ID); err == nil {
|
||||
t.Fatalf("want error from Revoke, got nil")
|
||||
}
|
||||
if got := len(m.ListAPIKeys()); got != 1 {
|
||||
t.Errorf("in-memory keys = %d, want 1 (rollback)", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestManager_Load_HydratesFromStore(t *testing.T) {
|
||||
store := newFakeStore()
|
||||
|
||||
// Pre-populate the store directly.
|
||||
for _, tc := range []struct {
|
||||
id string
|
||||
role string
|
||||
revoked bool
|
||||
}{
|
||||
{"a", storedRoleAdmin, false},
|
||||
{"b", storedRoleReadOnly, false},
|
||||
{"c", storedRoleReadWrite, true}, // revoked: should be skipped
|
||||
} {
|
||||
_ = store.SaveAPIKey(context.Background(), &metadata.APIKey{
|
||||
ID: tc.id, KeyHash: "h", Role: tc.role, Revoked: tc.revoked, CreatedAt: time.Now(),
|
||||
})
|
||||
}
|
||||
|
||||
m := NewWithStore(fastCfg(), store)
|
||||
defer m.Close()
|
||||
|
||||
if err := m.Load(context.Background()); err != nil {
|
||||
t.Fatalf("Load: %v", err)
|
||||
}
|
||||
keys := m.ListAPIKeys()
|
||||
if len(keys) != 2 {
|
||||
t.Fatalf("loaded %d keys, want 2 (revoked excluded)", len(keys))
|
||||
}
|
||||
for _, k := range keys {
|
||||
if k.Role != RoleAdmin && k.Role != RoleReadOnly {
|
||||
t.Errorf("unexpected role %q", k.Role)
|
||||
}
|
||||
if len(k.Permissions) == 0 {
|
||||
t.Errorf("permissions empty for role %q (should be derived)", k.Role)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestManager_Load_NilStore_NoOp(t *testing.T) {
|
||||
m := NewWithStore(fastCfg(), nil)
|
||||
defer m.Close()
|
||||
if err := m.Load(context.Background()); err != nil {
|
||||
t.Fatalf("Load with nil store: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestManager_Load_NotImplemented_NoOp(t *testing.T) {
|
||||
store := newFakeStore()
|
||||
store.disabled = true
|
||||
m := NewWithStore(fastCfg(), store)
|
||||
defer m.Close()
|
||||
if err := m.Load(context.Background()); err != nil {
|
||||
t.Fatalf("Load with disabled store: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestManager_ValidateAPIKey_AsyncLastUsed(t *testing.T) {
|
||||
store := newFakeStore()
|
||||
m := NewWithStore(fastCfg(), store)
|
||||
defer m.Close()
|
||||
|
||||
_, raw, err := m.GenerateAPIKey("svc", RoleReadOnly, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("Generate: %v", err)
|
||||
}
|
||||
|
||||
if _, err := m.ValidateAPIKey(context.Background(), raw); err != nil {
|
||||
t.Fatalf("Validate: %v", err)
|
||||
}
|
||||
|
||||
// Wait for the fire-and-forget goroutine.
|
||||
m.Close()
|
||||
|
||||
if store.updateCalls < 1 {
|
||||
t.Errorf("UpdateAPIKeyLastUsed not called (calls=%d)", store.updateCalls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestManager_ValidateAPIKey_RejectsUnknown(t *testing.T) {
|
||||
m := NewWithStore(fastCfg(), nil)
|
||||
defer m.Close()
|
||||
if _, err := m.ValidateAPIKey(context.Background(), "no-such-key"); err == nil {
|
||||
t.Fatalf("Validate(unknown): want error")
|
||||
}
|
||||
}
|
||||
|
||||
func TestManager_InMemoryMode_StillWorks(t *testing.T) {
|
||||
// Back-compat: the legacy New() constructor must continue working
|
||||
// without a store.
|
||||
m := New()
|
||||
defer m.Close()
|
||||
apiKey, raw, err := m.GenerateAPIKey("legacy", RoleReadOnly, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("Generate: %v", err)
|
||||
}
|
||||
got, err := m.ValidateAPIKey(context.Background(), raw)
|
||||
if err != nil {
|
||||
t.Fatalf("Validate: %v", err)
|
||||
}
|
||||
if got.ID != apiKey.ID {
|
||||
t.Errorf("validated wrong key: got %q want %q", got.ID, apiKey.ID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestManager_RoleRoundTrip(t *testing.T) {
|
||||
cases := []struct {
|
||||
role Role
|
||||
stored string
|
||||
}{
|
||||
{RoleReadOnly, storedRoleReadOnly},
|
||||
{RoleReadWrite, storedRoleReadWrite},
|
||||
{RoleAdmin, storedRoleAdmin},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(string(tc.role), func(t *testing.T) {
|
||||
if got := roleToStored(tc.role); got != tc.stored {
|
||||
t.Errorf("roleToStored(%q) = %q, want %q", tc.role, got, tc.stored)
|
||||
}
|
||||
if got := storedToRole(tc.stored); got != tc.role {
|
||||
t.Errorf("storedToRole(%q) = %q, want %q", tc.stored, got, tc.role)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
if got := storedToRole("garbage"); got != RoleReadOnly {
|
||||
t.Errorf("storedToRole(garbage) = %q, want least-privilege RoleReadOnly", got)
|
||||
}
|
||||
}
|
||||
@@ -1,99 +0,0 @@
|
||||
package auth
|
||||
|
||||
import (
|
||||
"context"
|
||||
stderrors "errors"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"github.com/lukaszraczylo/gohoarder/pkg/errors"
|
||||
"github.com/lukaszraczylo/gohoarder/pkg/metadata"
|
||||
"github.com/rs/zerolog/log"
|
||||
"golang.org/x/crypto/bcrypt"
|
||||
)
|
||||
|
||||
// BootstrapAdminEnvVar is the env var consulted by BootstrapAdminFromEnv.
|
||||
// Exported so callers and operators can reference a single source of truth.
|
||||
const BootstrapAdminEnvVar = "GOHOARDER_BOOTSTRAP_ADMIN_KEY"
|
||||
|
||||
// bootstrapProject is the marker project assigned to the env-bootstrapped
|
||||
// admin key. Callers can list keys filtered by this project to inventory
|
||||
// bootstrap-origin credentials.
|
||||
const bootstrapProject = "bootstrap"
|
||||
|
||||
// BootstrapAdminFromEnv creates an initial admin API key from the
|
||||
// GOHOARDER_BOOTSTRAP_ADMIN_KEY environment variable when no admin key
|
||||
// already exists in the store.
|
||||
//
|
||||
// Behaviour matrix:
|
||||
// - env unset/empty -> no-op, nil
|
||||
// - store nil -> no-op, nil (caller must provide a store)
|
||||
// - store has any non-revoked
|
||||
// admin key -> log and return nil (idempotent)
|
||||
// - else -> bcrypt-hash env value, persist as admin
|
||||
//
|
||||
// The env var holds the RAW key the operator wants to use; we hash it with
|
||||
// the configured bcrypt cost (default DefaultCost) before storing. The
|
||||
// operator is expected to clear the env var after first successful boot.
|
||||
func BootstrapAdminFromEnv(ctx context.Context, store metadata.MetadataStore, cfg Config) error {
|
||||
rawKey := os.Getenv(BootstrapAdminEnvVar)
|
||||
if rawKey == "" {
|
||||
return nil
|
||||
}
|
||||
if store == nil {
|
||||
log.Warn().Msg("Bootstrap admin key requested but no metadata store configured; skipping")
|
||||
return nil
|
||||
}
|
||||
|
||||
cost := cfg.BcryptCost
|
||||
if cost == 0 {
|
||||
cost = bcrypt.DefaultCost
|
||||
}
|
||||
|
||||
// Idempotency: if any non-revoked admin already exists, skip.
|
||||
existing, err := store.ListAPIKeys(ctx)
|
||||
if err != nil {
|
||||
if stderrors.Is(err, metadata.ErrNotImplemented) {
|
||||
log.Warn().Msg("Bootstrap admin: store does not implement API key persistence; skipping")
|
||||
return nil
|
||||
}
|
||||
return errors.Wrap(err, errors.ErrCodeStorageFailure, "bootstrap admin: failed to list existing keys")
|
||||
}
|
||||
|
||||
for _, k := range existing {
|
||||
if k == nil || k.Revoked {
|
||||
continue
|
||||
}
|
||||
if k.Role == storedRoleAdmin {
|
||||
log.Info().Msg("Bootstrap admin skipped — admin key already exists")
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
hashed, err := bcrypt.GenerateFromPassword([]byte(rawKey), cost)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, errors.ErrCodeInternalServer, "bootstrap admin: failed to hash key")
|
||||
}
|
||||
|
||||
now := time.Now()
|
||||
key := &metadata.APIKey{
|
||||
ID: generateID(),
|
||||
KeyHash: string(hashed),
|
||||
Project: bootstrapProject,
|
||||
Role: storedRoleAdmin,
|
||||
CreatedAt: now,
|
||||
Revoked: false,
|
||||
}
|
||||
|
||||
if err := store.SaveAPIKey(ctx, key); err != nil {
|
||||
return errors.Wrap(err, errors.ErrCodeStorageFailure, "bootstrap admin: failed to save key")
|
||||
}
|
||||
|
||||
// SECURITY: never log the raw key or its hash.
|
||||
log.Info().
|
||||
Str("key_id", key.ID).
|
||||
Str("project", bootstrapProject).
|
||||
Msg("Bootstrap admin created — delete " + BootstrapAdminEnvVar + " after first run")
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -1,142 +0,0 @@
|
||||
package auth
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/lukaszraczylo/gohoarder/pkg/metadata"
|
||||
"golang.org/x/crypto/bcrypt"
|
||||
)
|
||||
|
||||
func TestBootstrapAdminFromEnv_EnvEmpty_NoOp(t *testing.T) {
|
||||
t.Setenv(BootstrapAdminEnvVar, "")
|
||||
store := newFakeStore()
|
||||
|
||||
if err := BootstrapAdminFromEnv(context.Background(), store, fastCfg()); err != nil {
|
||||
t.Fatalf("BootstrapAdminFromEnv: %v", err)
|
||||
}
|
||||
if got := len(store.keys); got != 0 {
|
||||
t.Errorf("store has %d keys, want 0 when env empty", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBootstrapAdminFromEnv_NilStore_NoOp(t *testing.T) {
|
||||
t.Setenv(BootstrapAdminEnvVar, "secret")
|
||||
if err := BootstrapAdminFromEnv(context.Background(), nil, fastCfg()); err != nil {
|
||||
t.Fatalf("BootstrapAdminFromEnv(nil store): %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBootstrapAdminFromEnv_FreshStore_CreatesAdmin(t *testing.T) {
|
||||
const raw = "super-secret-bootstrap-key"
|
||||
t.Setenv(BootstrapAdminEnvVar, raw)
|
||||
store := newFakeStore()
|
||||
|
||||
if err := BootstrapAdminFromEnv(context.Background(), store, fastCfg()); err != nil {
|
||||
t.Fatalf("BootstrapAdminFromEnv: %v", err)
|
||||
}
|
||||
|
||||
if got := len(store.keys); got != 1 {
|
||||
t.Fatalf("store has %d keys, want 1", got)
|
||||
}
|
||||
|
||||
var created *metadata.APIKey
|
||||
for _, k := range store.keys {
|
||||
created = k
|
||||
break
|
||||
}
|
||||
if created.Role != storedRoleAdmin {
|
||||
t.Errorf("Role = %q, want %q", created.Role, storedRoleAdmin)
|
||||
}
|
||||
if created.Project != bootstrapProject {
|
||||
t.Errorf("Project = %q, want %q", created.Project, bootstrapProject)
|
||||
}
|
||||
if created.Revoked {
|
||||
t.Errorf("Revoked = true, want false")
|
||||
}
|
||||
if created.ID == "" {
|
||||
t.Errorf("ID empty")
|
||||
}
|
||||
// Hash must verify against the raw env value.
|
||||
if err := bcrypt.CompareHashAndPassword([]byte(created.KeyHash), []byte(raw)); err != nil {
|
||||
t.Errorf("bcrypt mismatch: %v", err)
|
||||
}
|
||||
// Hash must NOT equal the raw key (would indicate plaintext storage).
|
||||
if created.KeyHash == raw {
|
||||
t.Errorf("KeyHash equals raw key — plaintext storage bug")
|
||||
}
|
||||
}
|
||||
|
||||
func TestBootstrapAdminFromEnv_AdminAlreadyExists_Idempotent(t *testing.T) {
|
||||
t.Setenv(BootstrapAdminEnvVar, "another-secret")
|
||||
store := newFakeStore()
|
||||
// Pre-existing admin.
|
||||
_ = store.SaveAPIKey(context.Background(), &metadata.APIKey{
|
||||
ID: "existing-admin",
|
||||
KeyHash: "$2a$04$dummy",
|
||||
Role: storedRoleAdmin,
|
||||
Project: "default",
|
||||
CreatedAt: time.Now(),
|
||||
})
|
||||
|
||||
if err := BootstrapAdminFromEnv(context.Background(), store, fastCfg()); err != nil {
|
||||
t.Fatalf("BootstrapAdminFromEnv: %v", err)
|
||||
}
|
||||
if got := len(store.keys); got != 1 {
|
||||
t.Errorf("store has %d keys, want 1 (no new admin created)", got)
|
||||
}
|
||||
if _, ok := store.keys["existing-admin"]; !ok {
|
||||
t.Errorf("existing admin was removed/overwritten")
|
||||
}
|
||||
}
|
||||
|
||||
func TestBootstrapAdminFromEnv_RevokedAdmin_StillCreatesNewOne(t *testing.T) {
|
||||
t.Setenv(BootstrapAdminEnvVar, "fresh-secret")
|
||||
store := newFakeStore()
|
||||
// Revoked admin should NOT count as "admin already exists".
|
||||
_ = store.SaveAPIKey(context.Background(), &metadata.APIKey{
|
||||
ID: "old-admin",
|
||||
KeyHash: "x",
|
||||
Role: storedRoleAdmin,
|
||||
Project: "bootstrap",
|
||||
Revoked: true,
|
||||
CreatedAt: time.Now(),
|
||||
})
|
||||
|
||||
if err := BootstrapAdminFromEnv(context.Background(), store, fastCfg()); err != nil {
|
||||
t.Fatalf("BootstrapAdminFromEnv: %v", err)
|
||||
}
|
||||
if got := len(store.keys); got != 2 {
|
||||
t.Errorf("store has %d keys, want 2 (revoked + fresh)", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBootstrapAdminFromEnv_NonAdminKeysIgnored(t *testing.T) {
|
||||
t.Setenv(BootstrapAdminEnvVar, "secret-x")
|
||||
store := newFakeStore()
|
||||
_ = store.SaveAPIKey(context.Background(), &metadata.APIKey{
|
||||
ID: "reader",
|
||||
KeyHash: "x",
|
||||
Role: storedRoleReadOnly,
|
||||
Project: "default",
|
||||
CreatedAt: time.Now(),
|
||||
})
|
||||
|
||||
if err := BootstrapAdminFromEnv(context.Background(), store, fastCfg()); err != nil {
|
||||
t.Fatalf("BootstrapAdminFromEnv: %v", err)
|
||||
}
|
||||
if got := len(store.keys); got != 2 {
|
||||
t.Errorf("store has %d keys, want 2 (read-only existing + new admin)", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBootstrapAdminFromEnv_StoreNotImplemented_NoOp(t *testing.T) {
|
||||
t.Setenv(BootstrapAdminEnvVar, "secret-y")
|
||||
store := newFakeStore()
|
||||
store.disabled = true
|
||||
|
||||
if err := BootstrapAdminFromEnv(context.Background(), store, fastCfg()); err != nil {
|
||||
t.Fatalf("BootstrapAdminFromEnv: %v", err)
|
||||
}
|
||||
}
|
||||
+4
-6
@@ -14,18 +14,16 @@ func NewCredentialHasher() *CredentialHasher {
|
||||
return &CredentialHasher{}
|
||||
}
|
||||
|
||||
// Hash generates a hash of credentials for use in cache keys.
|
||||
// Returns "public" if no credentials provided.
|
||||
// Returns the full 64-char SHA256 hex digest otherwise: truncating to 8 bytes
|
||||
// gives a ~2^32 birthday bound which is unsafe for a security-sensitive
|
||||
// cache key (cross-credential cache poisoning).
|
||||
// Hash generates a short hash of credentials for use in cache keys
|
||||
// Returns "public" if no credentials provided
|
||||
func (h *CredentialHasher) Hash(credentials string) string {
|
||||
if credentials == "" {
|
||||
return "public"
|
||||
}
|
||||
|
||||
// Use SHA256 and take first 16 characters (8 bytes)
|
||||
hash := sha256.Sum256([]byte(credentials))
|
||||
return hex.EncodeToString(hash[:])
|
||||
return hex.EncodeToString(hash[:8])
|
||||
}
|
||||
|
||||
// GenerateCacheKey generates a cache key that includes credential hash
|
||||
|
||||
@@ -14,20 +14,16 @@ type ValidationResult struct {
|
||||
|
||||
// ValidationCache caches credential validation results to reduce upstream checks
|
||||
type ValidationCache struct {
|
||||
cache map[string]*ValidationResult
|
||||
stopCh chan struct{}
|
||||
stopOnce sync.Once
|
||||
mu sync.RWMutex
|
||||
ttl time.Duration
|
||||
cache map[string]*ValidationResult
|
||||
mu sync.RWMutex
|
||||
ttl time.Duration
|
||||
}
|
||||
|
||||
// NewValidationCache creates a new validation cache. Callers must call Stop()
|
||||
// during shutdown to terminate the background cleanup goroutine.
|
||||
// NewValidationCache creates a new validation cache
|
||||
func NewValidationCache(ttl time.Duration) *ValidationCache {
|
||||
vc := &ValidationCache{
|
||||
cache: make(map[string]*ValidationResult),
|
||||
stopCh: make(chan struct{}),
|
||||
ttl: ttl,
|
||||
cache: make(map[string]*ValidationResult),
|
||||
ttl: ttl,
|
||||
}
|
||||
|
||||
// Start cleanup goroutine
|
||||
@@ -36,42 +32,25 @@ func NewValidationCache(ttl time.Duration) *ValidationCache {
|
||||
return vc
|
||||
}
|
||||
|
||||
// Stop terminates the background cleanup goroutine. Safe to call multiple
|
||||
// times; subsequent calls are no-ops.
|
||||
func (vc *ValidationCache) Stop() {
|
||||
vc.stopOnce.Do(func() {
|
||||
close(vc.stopCh)
|
||||
})
|
||||
}
|
||||
|
||||
// Get retrieves a validation result from cache
|
||||
// Returns (allowed bool, cached bool, reason string)
|
||||
func (vc *ValidationCache) Get(credHash, packageURL string) (bool, bool, string) {
|
||||
key := credHash + ":" + packageURL
|
||||
|
||||
// Fast path: read-locked lookup.
|
||||
vc.mu.RLock()
|
||||
defer vc.mu.RUnlock()
|
||||
|
||||
key := credHash + ":" + packageURL
|
||||
result, exists := vc.cache[key]
|
||||
|
||||
if !exists {
|
||||
vc.mu.RUnlock()
|
||||
return false, false, ""
|
||||
}
|
||||
expired := time.Now().After(result.ExpiresAt)
|
||||
if !expired {
|
||||
allowed, reason := result.Allowed, result.Reason
|
||||
vc.mu.RUnlock()
|
||||
return allowed, true, reason
|
||||
}
|
||||
vc.mu.RUnlock()
|
||||
|
||||
// Expired: drop it under the write lock so callers don't all stampede
|
||||
// the upstream while waiting for the periodic cleanup goroutine.
|
||||
vc.mu.Lock()
|
||||
if cur, ok := vc.cache[key]; ok && time.Now().After(cur.ExpiresAt) {
|
||||
delete(vc.cache, key)
|
||||
// Check if expired
|
||||
if time.Now().After(result.ExpiresAt) {
|
||||
return false, false, ""
|
||||
}
|
||||
vc.mu.Unlock()
|
||||
return false, false, ""
|
||||
|
||||
return result.Allowed, true, result.Reason
|
||||
}
|
||||
|
||||
// Set stores a validation result in cache
|
||||
@@ -112,25 +91,19 @@ func (vc *ValidationCache) Size() int {
|
||||
return len(vc.cache)
|
||||
}
|
||||
|
||||
// cleanupExpired removes expired entries periodically. Exits when Stop is
|
||||
// called.
|
||||
// cleanupExpired removes expired entries periodically
|
||||
func (vc *ValidationCache) cleanupExpired() {
|
||||
ticker := time.NewTicker(1 * time.Minute)
|
||||
defer ticker.Stop()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-vc.stopCh:
|
||||
return
|
||||
case <-ticker.C:
|
||||
vc.mu.Lock()
|
||||
now := time.Now()
|
||||
for key, result := range vc.cache {
|
||||
if now.After(result.ExpiresAt) {
|
||||
delete(vc.cache, key)
|
||||
}
|
||||
for range ticker.C {
|
||||
vc.mu.Lock()
|
||||
now := time.Now()
|
||||
for key, result := range vc.cache {
|
||||
if now.After(result.ExpiresAt) {
|
||||
delete(vc.cache, key)
|
||||
}
|
||||
vc.mu.Unlock()
|
||||
}
|
||||
vc.mu.Unlock()
|
||||
}
|
||||
}
|
||||
|
||||
+27
-27
@@ -50,11 +50,11 @@ func (v *NPMValidator) ValidateAccess(ctx context.Context, packageURL string, cr
|
||||
|
||||
resp, err := v.client.Do(req)
|
||||
if err != nil {
|
||||
// Network error - fail closed. Caller decides any fallback policy.
|
||||
log.Warn().Err(err).Str("url", packageURL).Msg("Validation request failed, denying access")
|
||||
return false, fmt.Errorf("validation failed: %w", err)
|
||||
// Network error - allow cache fallback with warning
|
||||
log.Warn().Err(err).Str("url", packageURL).Msg("Validation request failed, allowing cache fallback")
|
||||
return true, fmt.Errorf("validation failed: %w (allowing cache fallback)", err)
|
||||
}
|
||||
defer func() { _ = resp.Body.Close() }() // #nosec G104 -- Cleanup, error not critical
|
||||
defer resp.Body.Close() // #nosec G104 -- Cleanup, error not critical
|
||||
|
||||
// Check status code
|
||||
switch resp.StatusCode {
|
||||
@@ -65,9 +65,9 @@ func (v *NPMValidator) ValidateAccess(ctx context.Context, packageURL string, cr
|
||||
// Access denied
|
||||
return false, fmt.Errorf("access denied: HTTP %d", resp.StatusCode)
|
||||
default:
|
||||
// Unexpected status - fail closed.
|
||||
log.Warn().Int("status", resp.StatusCode).Str("url", packageURL).Msg("Unexpected validation status, denying access")
|
||||
return false, fmt.Errorf("unexpected status %d", resp.StatusCode)
|
||||
// Unexpected status - allow cache fallback with warning
|
||||
log.Warn().Int("status", resp.StatusCode).Str("url", packageURL).Msg("Unexpected validation status, allowing cache fallback")
|
||||
return true, fmt.Errorf("unexpected status %d (allowing cache fallback)", resp.StatusCode)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -101,11 +101,11 @@ func (v *PyPIValidator) ValidateAccess(ctx context.Context, packageURL string, c
|
||||
|
||||
resp, err := v.client.Do(req)
|
||||
if err != nil {
|
||||
// Network error - fail closed. Caller decides any fallback policy.
|
||||
log.Warn().Err(err).Str("url", packageURL).Msg("Validation request failed, denying access")
|
||||
return false, fmt.Errorf("validation failed: %w", err)
|
||||
// Network error - allow cache fallback with warning
|
||||
log.Warn().Err(err).Str("url", packageURL).Msg("Validation request failed, allowing cache fallback")
|
||||
return true, fmt.Errorf("validation failed: %w (allowing cache fallback)", err)
|
||||
}
|
||||
defer func() { _ = resp.Body.Close() }() // #nosec G104 -- Cleanup, error not critical
|
||||
defer resp.Body.Close() // #nosec G104 -- Cleanup, error not critical
|
||||
|
||||
// Check status code
|
||||
switch resp.StatusCode {
|
||||
@@ -116,9 +116,9 @@ func (v *PyPIValidator) ValidateAccess(ctx context.Context, packageURL string, c
|
||||
// Access denied
|
||||
return false, fmt.Errorf("access denied: HTTP %d", resp.StatusCode)
|
||||
default:
|
||||
// Unexpected status - fail closed.
|
||||
log.Warn().Int("status", resp.StatusCode).Str("url", packageURL).Msg("Unexpected validation status, denying access")
|
||||
return false, fmt.Errorf("unexpected status %d", resp.StatusCode)
|
||||
// Unexpected status - allow cache fallback with warning
|
||||
log.Warn().Int("status", resp.StatusCode).Str("url", packageURL).Msg("Unexpected validation status, allowing cache fallback")
|
||||
return true, fmt.Errorf("unexpected status %d (allowing cache fallback)", resp.StatusCode)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -171,13 +171,13 @@ func (v *GoValidator) validateGitHub(ctx context.Context, modulePath, credential
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
defer func() { _ = os.RemoveAll(tempDir) }()
|
||||
defer os.RemoveAll(tempDir)
|
||||
|
||||
// Create .netrc file with credentials
|
||||
netrcPath := filepath.Join(tempDir, ".netrc")
|
||||
netrcContent := fmt.Sprintf("machine github.com\nlogin oauth2\npassword %s\n", token)
|
||||
if writeErr := os.WriteFile(netrcPath, []byte(netrcContent), 0600); writeErr != nil {
|
||||
return false, writeErr
|
||||
if err := os.WriteFile(netrcPath, []byte(netrcContent), 0600); err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
// Run git ls-remote (lightweight, just checks access)
|
||||
@@ -199,9 +199,9 @@ func (v *GoValidator) validateGitHub(ctx context.Context, modulePath, credential
|
||||
return false, fmt.Errorf("access denied: %s", strings.TrimSpace(errMsg))
|
||||
}
|
||||
|
||||
// Other error (network, etc.) - fail closed.
|
||||
log.Warn().Err(err).Str("module", modulePath).Msg("Git validation failed, denying access")
|
||||
return false, fmt.Errorf("validation error: %w", err)
|
||||
// Other error (network, etc.) - allow cache fallback
|
||||
log.Warn().Err(err).Str("module", modulePath).Msg("Git validation failed, allowing cache fallback")
|
||||
return true, fmt.Errorf("validation error (allowing cache): %w", err)
|
||||
}
|
||||
|
||||
// Success - repository accessible
|
||||
@@ -227,13 +227,13 @@ func (v *GoValidator) validateGitLab(ctx context.Context, modulePath, credential
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
defer func() { _ = os.RemoveAll(tempDir) }()
|
||||
defer os.RemoveAll(tempDir)
|
||||
|
||||
// Create .netrc file with credentials
|
||||
netrcPath := filepath.Join(tempDir, ".netrc")
|
||||
netrcContent := fmt.Sprintf("machine gitlab.com\nlogin oauth2\npassword %s\n", token)
|
||||
if writeErr := os.WriteFile(netrcPath, []byte(netrcContent), 0600); writeErr != nil {
|
||||
return false, writeErr
|
||||
if err := os.WriteFile(netrcPath, []byte(netrcContent), 0600); err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
// Run git ls-remote
|
||||
@@ -252,8 +252,8 @@ func (v *GoValidator) validateGitLab(ctx context.Context, modulePath, credential
|
||||
return false, fmt.Errorf("access denied: %s", strings.TrimSpace(errMsg))
|
||||
}
|
||||
|
||||
log.Warn().Err(err).Str("module", modulePath).Msg("Git validation failed, denying access")
|
||||
return false, fmt.Errorf("validation error: %w", err)
|
||||
log.Warn().Err(err).Str("module", modulePath).Msg("Git validation failed, allowing cache fallback")
|
||||
return true, fmt.Errorf("validation error (allowing cache): %w", err)
|
||||
}
|
||||
|
||||
return true, nil
|
||||
@@ -276,8 +276,8 @@ func (v *GoValidator) validateGit(ctx context.Context, modulePath, credentials s
|
||||
return false, fmt.Errorf("access denied: %s", strings.TrimSpace(errMsg))
|
||||
}
|
||||
|
||||
log.Warn().Err(err).Str("module", modulePath).Msg("Git validation failed, denying access")
|
||||
return false, fmt.Errorf("validation error: %w", err)
|
||||
log.Warn().Err(err).Str("module", modulePath).Msg("Git validation failed, allowing cache fallback")
|
||||
return true, fmt.Errorf("validation error (allowing cache): %w", err)
|
||||
}
|
||||
|
||||
return true, nil
|
||||
|
||||
Vendored
+64
-184
@@ -1,5 +1,3 @@
|
||||
// Package cache implements the unified cache manager that coordinates
|
||||
// metadata, storage, scanning, and singleflight for upstream packages.
|
||||
package cache
|
||||
|
||||
import (
|
||||
@@ -9,18 +7,16 @@ import (
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"strings"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/lukaszraczylo/gohoarder/pkg/analytics"
|
||||
"github.com/lukaszraczylo/gohoarder/pkg/errors"
|
||||
"github.com/lukaszraczylo/gohoarder/pkg/events"
|
||||
"github.com/lukaszraczylo/gohoarder/pkg/metadata"
|
||||
"github.com/lukaszraczylo/gohoarder/pkg/metrics"
|
||||
"github.com/lukaszraczylo/gohoarder/pkg/storage"
|
||||
"github.com/lukaszraczylo/gohoarder/pkg/uuid"
|
||||
"github.com/lukaszraczylo/gohoarder/pkg/websocket"
|
||||
"github.com/rs/zerolog/log"
|
||||
"golang.org/x/sync/singleflight"
|
||||
)
|
||||
@@ -39,43 +35,14 @@ type AnalyticsInterface interface {
|
||||
|
||||
// Manager coordinates caching operations between storage and metadata
|
||||
type Manager struct {
|
||||
storage storage.StorageBackend
|
||||
metadata metadata.MetadataStore
|
||||
scanner ScannerInterface
|
||||
analytics AnalyticsInterface
|
||||
broadcaster events.Broadcaster
|
||||
stopCh chan struct{}
|
||||
sf singleflight.Group
|
||||
config Config
|
||||
cleanupWG sync.WaitGroup
|
||||
mu sync.RWMutex
|
||||
bcMu sync.RWMutex
|
||||
closeOnce sync.Once
|
||||
evicting bool
|
||||
}
|
||||
|
||||
// SetBroadcaster wires an events.Broadcaster onto the manager so cache
|
||||
// lifecycle events (cached / downloaded) are published. Pass nil to
|
||||
// disable broadcasting. Safe to call after construction; concurrent
|
||||
// access is guarded.
|
||||
func (m *Manager) SetBroadcaster(b events.Broadcaster) {
|
||||
m.bcMu.Lock()
|
||||
m.broadcaster = b
|
||||
m.bcMu.Unlock()
|
||||
}
|
||||
|
||||
// emit publishes an event via the configured broadcaster, if any.
|
||||
// Fire-and-forget: never blocks the caller. The websocket server's
|
||||
// BroadcastEvent is itself non-blocking (channel-drop on overflow),
|
||||
// so we call directly rather than spawning a goroutine.
|
||||
func (m *Manager) emit(eventType string, payload map[string]interface{}) {
|
||||
m.bcMu.RLock()
|
||||
b := m.broadcaster
|
||||
m.bcMu.RUnlock()
|
||||
if b == nil {
|
||||
return
|
||||
}
|
||||
b.BroadcastEvent(eventType, payload)
|
||||
storage storage.StorageBackend
|
||||
metadata metadata.MetadataStore
|
||||
scanner ScannerInterface
|
||||
analytics AnalyticsInterface
|
||||
sf singleflight.Group
|
||||
config Config
|
||||
mu sync.RWMutex
|
||||
evicting bool
|
||||
}
|
||||
|
||||
// Config holds cache manager configuration
|
||||
@@ -84,7 +51,6 @@ type Config struct {
|
||||
CleanupInterval time.Duration // How often to run cleanup
|
||||
EvictionThreshold float64 // Trigger eviction when usage > threshold (0.0-1.0)
|
||||
MaxConcurrent int // Max concurrent upstream fetches
|
||||
MaxPackageSize int64 // Maximum package size in bytes (0 = default 2GB)
|
||||
}
|
||||
|
||||
// CacheEntry represents a cached package
|
||||
@@ -132,21 +98,15 @@ func New(storage storage.StorageBackend, metadata metadata.MetadataStore, scanne
|
||||
config.MaxConcurrent = 100
|
||||
}
|
||||
|
||||
if config.MaxPackageSize == 0 {
|
||||
config.MaxPackageSize = 2 * 1024 * 1024 * 1024 // 2GB default
|
||||
}
|
||||
|
||||
manager := &Manager{
|
||||
storage: storage,
|
||||
metadata: metadata,
|
||||
scanner: scanner,
|
||||
analytics: analytics,
|
||||
config: config,
|
||||
stopCh: make(chan struct{}),
|
||||
}
|
||||
|
||||
// Start background cleanup worker
|
||||
manager.cleanupWG.Add(1)
|
||||
go manager.cleanupWorker()
|
||||
|
||||
return manager, nil
|
||||
@@ -158,9 +118,6 @@ func (m *Manager) Get(ctx context.Context, registry, name, version string, fetch
|
||||
key := fmt.Sprintf("%s/%s/%s", registry, name, version)
|
||||
|
||||
result, err, _ := m.sf.Do(key, func() (interface{}, error) {
|
||||
// getOrFetch returns a CacheEntry with Data == nil. Each caller
|
||||
// re-opens its own storage reader below to avoid sharing a
|
||||
// single io.ReadCloser across concurrent waiters.
|
||||
return m.getOrFetch(ctx, registry, name, version, fetchFunc)
|
||||
})
|
||||
|
||||
@@ -168,25 +125,7 @@ func (m *Manager) Get(ctx context.Context, registry, name, version string, fetch
|
||||
return nil, err
|
||||
}
|
||||
|
||||
entry := result.(*CacheEntry)
|
||||
if entry == nil || entry.Package == nil {
|
||||
return nil, errors.New(errors.ErrCodeStorageFailure, "cache entry missing package metadata")
|
||||
}
|
||||
|
||||
// Open a fresh ReadCloser per caller from storage.
|
||||
data, err := m.storage.Get(ctx, entry.Package.StorageKey)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, errors.ErrCodeStorageFailure, "failed to retrieve cached package")
|
||||
}
|
||||
|
||||
// Return a copy so concurrent waiters don't share the Data field.
|
||||
return &CacheEntry{
|
||||
Data: data,
|
||||
Package: entry.Package,
|
||||
UpstreamURL: entry.UpstreamURL,
|
||||
CacheControl: entry.CacheControl,
|
||||
FromCache: entry.FromCache,
|
||||
}, nil
|
||||
return result.(*CacheEntry), nil
|
||||
}
|
||||
|
||||
// getOrFetch implements the actual get-or-fetch logic
|
||||
@@ -201,20 +140,16 @@ func (m *Manager) getOrFetch(ctx context.Context, registry, name, version string
|
||||
// Delete expired package
|
||||
_ = m.deletePackage(ctx, pkg) // #nosec G104 -- Async cleanup
|
||||
} else {
|
||||
// Probe storage by opening then immediately closing the reader.
|
||||
// Singleflight callers can't share a live ReadCloser; each caller in
|
||||
// Get() opens its own reader after the singleflight returns.
|
||||
data, getErr := m.storage.Get(ctx, pkg.StorageKey)
|
||||
if getErr == nil {
|
||||
_ = data.Close() // #nosec G104 -- probe only; Get() reopens per caller
|
||||
|
||||
// Try to get from storage
|
||||
data, err := m.storage.Get(ctx, pkg.StorageKey)
|
||||
if err == nil {
|
||||
// Cache hit!
|
||||
metrics.RecordCacheHit(registry)
|
||||
|
||||
// Update download count (log errors for debugging)
|
||||
if updErr := m.metadata.UpdateDownloadCount(ctx, registry, name, version); updErr != nil {
|
||||
if err := m.metadata.UpdateDownloadCount(ctx, registry, name, version); err != nil {
|
||||
log.Warn().
|
||||
Err(updErr).
|
||||
Err(err).
|
||||
Str("registry", registry).
|
||||
Str("package", name).
|
||||
Str("version", version).
|
||||
@@ -249,30 +184,20 @@ func (m *Manager) getOrFetch(ctx context.Context, registry, name, version string
|
||||
|
||||
// Check for vulnerabilities if scanner is enabled
|
||||
if m.scanner != nil {
|
||||
blocked, reason, vulnErr := m.scanner.CheckVulnerabilities(ctx, registry, name, version)
|
||||
if vulnErr != nil {
|
||||
log.Warn().Err(vulnErr).Str("package", name).Msg("Failed to check vulnerabilities")
|
||||
blocked, reason, err := m.scanner.CheckVulnerabilities(ctx, registry, name, version)
|
||||
if err != nil {
|
||||
log.Warn().Err(err).Str("package", name).Msg("Failed to check vulnerabilities")
|
||||
}
|
||||
if blocked {
|
||||
metrics.RecordCacheHit(registry) // Record as blocked
|
||||
_ = data.Close() // #nosec G104 // Close the data reader
|
||||
return nil, errors.New(errors.ErrCodeSecurityViolation, reason)
|
||||
}
|
||||
}
|
||||
|
||||
// Broadcast cache-hit serve event. Fire-and-forget; the
|
||||
// underlying transport is non-blocking. Only emitted on
|
||||
// the cache-hit path — the miss-then-fetch path is
|
||||
// covered by EventPackageCached emitted from store().
|
||||
m.emit(string(websocket.EventPackageDownloaded), map[string]interface{}{
|
||||
"registry": registry,
|
||||
"name": name,
|
||||
"version": version,
|
||||
"cache_hit": true,
|
||||
})
|
||||
|
||||
// Data is intentionally nil; Get() opens a fresh reader per caller.
|
||||
return &CacheEntry{
|
||||
Package: pkg,
|
||||
Data: nil,
|
||||
Data: data,
|
||||
FromCache: true,
|
||||
}, nil
|
||||
}
|
||||
@@ -298,7 +223,7 @@ func (m *Manager) getOrFetch(ctx context.Context, registry, name, version string
|
||||
metrics.RecordUpstreamRequest(registry, "error")
|
||||
return nil, errors.Wrap(err, errors.ErrCodeUpstreamFailure, "failed to fetch from upstream")
|
||||
}
|
||||
defer func() { _ = data.Close() }() // #nosec G104 -- Cleanup, error not critical
|
||||
defer data.Close() // #nosec G104 -- Cleanup, error not critical
|
||||
|
||||
metrics.RecordUpstreamRequest(registry, "success")
|
||||
|
||||
@@ -309,13 +234,10 @@ func (m *Manager) getOrFetch(ctx context.Context, registry, name, version string
|
||||
}
|
||||
|
||||
// Skip security scan wait for metadata entries (index pages, lists, etc.)
|
||||
// Also skip Go module metadata files (.mod, .info)
|
||||
isMetadataEntry := version == "list" || version == "page" || version == "latest" || version == "metadata" ||
|
||||
strings.HasSuffix(name, ".mod") || strings.HasSuffix(name, ".info")
|
||||
isMetadataEntry := version == "list" || version == "page" || version == "latest" || version == "metadata"
|
||||
|
||||
// Wait briefly for initial scan to complete if scanner is enabled
|
||||
// This prevents serving vulnerable packages on first request.
|
||||
// SECURITY: timeouts MUST fail closed — never serve unscanned content.
|
||||
// This prevents serving vulnerable packages on first request
|
||||
if m.scanner != nil && !isMetadataEntry {
|
||||
// Wait up to 30 seconds for scan to complete
|
||||
scanCtx, cancel := context.WithTimeout(ctx, 30*time.Second)
|
||||
@@ -324,18 +246,16 @@ func (m *Manager) getOrFetch(ctx context.Context, registry, name, version string
|
||||
ticker := time.NewTicker(100 * time.Millisecond)
|
||||
defer ticker.Stop()
|
||||
|
||||
scanWait:
|
||||
for {
|
||||
select {
|
||||
case <-scanCtx.Done():
|
||||
// Fail closed: do NOT serve unscanned packages on timeout/cancel.
|
||||
// Package remains cached; subsequent requests can retry once
|
||||
// the scan completes.
|
||||
// Timeout or context cancelled - proceed anyway
|
||||
// Package is cached, will be blocked on next request if vulnerable
|
||||
log.Warn().
|
||||
Str("package", name).
|
||||
Str("version", version).
|
||||
Msg("Scan timeout - refusing to serve unscanned package (fail-closed)")
|
||||
return nil, errors.New(errors.ErrCodeServiceUnavailable, "package scan in progress, retry shortly")
|
||||
Msg("Scan timeout - allowing first download, will block on subsequent requests if vulnerable")
|
||||
goto servePkg
|
||||
|
||||
case <-ticker.C:
|
||||
// First check if scan has completed by checking the SecurityScanned flag
|
||||
@@ -388,11 +308,18 @@ func (m *Manager) getOrFetch(ctx context.Context, registry, name, version string
|
||||
Str("package", name).
|
||||
Str("version", version).
|
||||
Msg("Scan completed, package is clean")
|
||||
break scanWait
|
||||
goto servePkg
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
servePkg:
|
||||
// Re-open from storage for consistency
|
||||
storedData, err := m.storage.Get(ctx, storedPkg.StorageKey)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, errors.ErrCodeStorageFailure, "failed to retrieve just-stored package")
|
||||
}
|
||||
|
||||
// Track download count for first-time download (cache miss)
|
||||
// This ensures download count increments regardless of cache hit/miss
|
||||
if err := m.metadata.UpdateDownloadCount(ctx, registry, name, version); err != nil {
|
||||
@@ -409,10 +336,9 @@ func (m *Manager) getOrFetch(ctx context.Context, registry, name, version string
|
||||
m.trackDownload(registry, name, version, storedPkg.Size)
|
||||
}
|
||||
|
||||
// Data is intentionally nil; Get() opens a fresh reader per caller.
|
||||
return &CacheEntry{
|
||||
Package: storedPkg,
|
||||
Data: nil,
|
||||
Data: storedData,
|
||||
FromCache: false,
|
||||
UpstreamURL: upstreamURL,
|
||||
}, nil
|
||||
@@ -429,21 +355,11 @@ func (m *Manager) store(ctx context.Context, registry, name, version string, dat
|
||||
var buf []byte
|
||||
var err error
|
||||
|
||||
// Cap upstream read to MaxPackageSize to prevent OOM on hostile/oversized
|
||||
// upstream responses. Read one extra byte so we can detect overflow.
|
||||
maxSize := m.config.MaxPackageSize
|
||||
if maxSize <= 0 {
|
||||
maxSize = 2 * 1024 * 1024 * 1024 // 2GB safety floor
|
||||
}
|
||||
limited := io.LimitReader(data, maxSize+1)
|
||||
buf, err = io.ReadAll(limited)
|
||||
// Read all data
|
||||
buf, err = io.ReadAll(data)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, errors.ErrCodeUpstreamFailure, "failed to read upstream data")
|
||||
}
|
||||
if int64(len(buf)) > maxSize {
|
||||
return nil, errors.New(errors.ErrCodePayloadTooLarge,
|
||||
fmt.Sprintf("upstream package exceeds max size (%d bytes)", maxSize))
|
||||
}
|
||||
|
||||
// Calculate checksums
|
||||
h := sha256.New()
|
||||
@@ -457,7 +373,7 @@ func (m *Manager) store(ctx context.Context, registry, name, version string, dat
|
||||
if err == nil && quota.Limit > 0 {
|
||||
if quota.Used+size > quota.Limit {
|
||||
// Trigger eviction
|
||||
if evictErr := m.evict(ctx, size); evictErr != nil {
|
||||
if err := m.evict(ctx, size); err != nil {
|
||||
return nil, errors.QuotaExceeded(quota.Limit)
|
||||
}
|
||||
}
|
||||
@@ -493,40 +409,15 @@ func (m *Manager) store(ctx context.Context, registry, name, version string, dat
|
||||
Metadata: make(map[string]string),
|
||||
}
|
||||
|
||||
// Persist metadata for ALL entries (including metadata pages and Go .mod/.info).
|
||||
// Skipping persistence for metadata entries caused unconditional upstream re-fetch
|
||||
// on every metadata request. SavePackage upserts safely; the metadata-entry flag
|
||||
// below is still used to skip security scanning (these are not scannable packages).
|
||||
//
|
||||
// TRADEOFF: metadata pages share the cache's DefaultTTL and are not refreshed
|
||||
// based on upstream Cache-Control. Plumbing per-response TTL from registry handlers
|
||||
// is out of scope here and tracked separately.
|
||||
isMetadataEntry := version == "list" || version == "page" || version == "latest" || version == "metadata" ||
|
||||
strings.HasSuffix(name, ".mod") || strings.HasSuffix(name, ".info")
|
||||
if err := m.metadata.SavePackage(ctx, pkg); err != nil {
|
||||
// Clean up storage if metadata save fails
|
||||
_ = m.storage.Delete(ctx, storageKey) // #nosec G104 -- Cleanup, error logged
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Broadcast cache-store event. The scan_status reflects the
|
||||
// initial state ("pending" if scanning is enabled and applicable;
|
||||
// "skipped" for metadata entries; "disabled" when scanner is nil).
|
||||
scanStatus := "disabled"
|
||||
if m.scanner != nil {
|
||||
if isMetadataEntry {
|
||||
scanStatus = "skipped"
|
||||
} else {
|
||||
scanStatus = "pending"
|
||||
// Save metadata (skip metadata entries like index pages, lists, etc.)
|
||||
isMetadataEntry := version == "list" || version == "page" || version == "latest" || version == "metadata"
|
||||
if !isMetadataEntry {
|
||||
if err := m.metadata.SavePackage(ctx, pkg); err != nil {
|
||||
// Clean up storage if metadata save fails
|
||||
_ = m.storage.Delete(ctx, storageKey) // #nosec G104 -- Cleanup, error logged
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
m.emit(string(websocket.EventPackageCached), map[string]interface{}{
|
||||
"registry": registry,
|
||||
"name": name,
|
||||
"version": version,
|
||||
"size": size,
|
||||
"scan_status": scanStatus,
|
||||
})
|
||||
|
||||
// Scan package if scanner is enabled (run in background to not block cache operations)
|
||||
// Skip scanning metadata entries (index pages, lists, etc.)
|
||||
@@ -550,24 +441,29 @@ func (m *Manager) store(ctx context.Context, registry, name, version string, dat
|
||||
cleanupFunc = func() {} // No cleanup needed for direct path
|
||||
log.Debug().Str("package", name).Str("path", filePath).Msg("Scanning package from storage path")
|
||||
} else {
|
||||
// Fallback: Create temp file for remote storage (S3, SMB, etc.).
|
||||
// Use os.CreateTemp so the OS picks a safe, unique filename — this
|
||||
// prevents path traversal via storageKey containing "..".
|
||||
tempFile, err := os.CreateTemp(os.TempDir(), "gohoarder-scan-*")
|
||||
// Fallback: Create temp file for remote storage (S3, SMB, etc.)
|
||||
tempFilePath := filepath.Join(os.TempDir(), storageKey)
|
||||
|
||||
// Create parent directories if they don't exist
|
||||
if err := os.MkdirAll(filepath.Dir(tempFilePath), 0750); err != nil {
|
||||
log.Error().Err(err).Str("package", name).Msg("Failed to create temp directory for scanning")
|
||||
return
|
||||
}
|
||||
|
||||
tempFile, err := os.Create(tempFilePath) // #nosec G304 -- Temp file path is constructed from validated package name
|
||||
if err != nil {
|
||||
log.Error().Err(err).Str("package", name).Msg("Failed to create temp file for scanning")
|
||||
return
|
||||
}
|
||||
tempFilePath := tempFile.Name()
|
||||
|
||||
// Write package data to temp file
|
||||
if _, err := tempFile.Write(buf); err != nil {
|
||||
_ = tempFile.Close() // #nosec G104 -- Cleanup, error not critical
|
||||
tempFile.Close() // #nosec G104 -- Cleanup, error not critical
|
||||
_ = os.Remove(tempFilePath) // #nosec G104 -- Cleanup, error not critical
|
||||
log.Error().Err(err).Str("package", name).Msg("Failed to write temp file for scanning")
|
||||
return
|
||||
}
|
||||
_ = tempFile.Close() // #nosec G104 -- Cleanup, error not critical
|
||||
tempFile.Close() // #nosec G104 -- Cleanup, error not critical
|
||||
|
||||
filePath = tempFilePath
|
||||
cleanupFunc = func() { _ = os.Remove(tempFilePath) } // #nosec G104 -- Cleanup
|
||||
@@ -662,22 +558,14 @@ func (m *Manager) evict(ctx context.Context, needed int64) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// cleanupWorker runs periodic cleanup of expired packages.
|
||||
// Exits when stopCh is closed (via Close()).
|
||||
// cleanupWorker runs periodic cleanup of expired packages
|
||||
func (m *Manager) cleanupWorker() {
|
||||
defer m.cleanupWG.Done()
|
||||
|
||||
ticker := time.NewTicker(m.config.CleanupInterval)
|
||||
defer ticker.Stop()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-m.stopCh:
|
||||
return
|
||||
case <-ticker.C:
|
||||
ctx := context.Background()
|
||||
m.cleanup(ctx)
|
||||
}
|
||||
for range ticker.C {
|
||||
ctx := context.Background()
|
||||
m.cleanup(ctx)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -750,18 +638,10 @@ func (m *Manager) trackDownload(registry, name, version string, size int64) {
|
||||
m.analytics.TrackDownload(download)
|
||||
}
|
||||
|
||||
// Close closes the cache manager.
|
||||
// Stops the cleanup worker, then closes storage and metadata backends.
|
||||
// Safe to call multiple times.
|
||||
// Close closes the cache manager
|
||||
func (m *Manager) Close() error {
|
||||
var err error
|
||||
|
||||
// Stop cleanup worker (idempotent via sync.Once).
|
||||
m.closeOnce.Do(func() {
|
||||
close(m.stopCh)
|
||||
m.cleanupWG.Wait()
|
||||
})
|
||||
|
||||
if closeErr := m.storage.Close(); closeErr != nil {
|
||||
err = closeErr
|
||||
}
|
||||
|
||||
Vendored
-233
@@ -6,52 +6,16 @@ import (
|
||||
"errors"
|
||||
"io"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/lukaszraczylo/gohoarder/pkg/metadata"
|
||||
"github.com/lukaszraczylo/gohoarder/pkg/storage"
|
||||
"github.com/lukaszraczylo/gohoarder/pkg/websocket"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/mock"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// fakeBroadcaster records BroadcastEvent calls for assertions.
|
||||
type fakeBroadcaster struct {
|
||||
events []fakeBroadcastEvent
|
||||
mu sync.Mutex
|
||||
}
|
||||
|
||||
type fakeBroadcastEvent struct {
|
||||
Payload any
|
||||
Type string
|
||||
}
|
||||
|
||||
func (f *fakeBroadcaster) BroadcastEvent(eventType string, payload any) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
f.events = append(f.events, fakeBroadcastEvent{Type: eventType, Payload: payload})
|
||||
}
|
||||
|
||||
func (f *fakeBroadcaster) snapshot() []fakeBroadcastEvent {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
out := make([]fakeBroadcastEvent, len(f.events))
|
||||
copy(out, f.events)
|
||||
return out
|
||||
}
|
||||
|
||||
func (f *fakeBroadcaster) typesOnly() []string {
|
||||
snap := f.snapshot()
|
||||
out := make([]string, len(snap))
|
||||
for i, e := range snap {
|
||||
out[i] = e.Type
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// MockStorageBackend is a mock for storage.StorageBackend
|
||||
type MockStorageBackend struct {
|
||||
mock.Mock
|
||||
@@ -230,29 +194,6 @@ func (m *MockMetadataStore) AggregateDownloadData(ctx context.Context) error {
|
||||
return args.Error(0)
|
||||
}
|
||||
|
||||
// API key methods are not used by the cache package; stub them out so the
|
||||
// mock continues to satisfy metadata.MetadataStore after the interface gained
|
||||
// API key persistence methods.
|
||||
func (m *MockMetadataStore) SaveAPIKey(ctx context.Context, key *metadata.APIKey) error {
|
||||
return metadata.ErrNotImplemented
|
||||
}
|
||||
|
||||
func (m *MockMetadataStore) GetAPIKey(ctx context.Context, id string) (*metadata.APIKey, error) {
|
||||
return nil, metadata.ErrNotImplemented
|
||||
}
|
||||
|
||||
func (m *MockMetadataStore) ListAPIKeys(ctx context.Context) ([]*metadata.APIKey, error) {
|
||||
return nil, metadata.ErrNotImplemented
|
||||
}
|
||||
|
||||
func (m *MockMetadataStore) DeleteAPIKey(ctx context.Context, id string) error {
|
||||
return metadata.ErrNotImplemented
|
||||
}
|
||||
|
||||
func (m *MockMetadataStore) UpdateAPIKeyLastUsed(ctx context.Context, id string, t time.Time) error {
|
||||
return metadata.ErrNotImplemented
|
||||
}
|
||||
|
||||
// TestNew tests cache manager creation
|
||||
func TestNew(t *testing.T) {
|
||||
tests := []struct {
|
||||
@@ -1040,177 +981,3 @@ func TestConcurrentGet(t *testing.T) {
|
||||
// Verify at least one call was made (singleflight may deduplicate others)
|
||||
mockMetadata.AssertCalled(t, "GetPackage", mock.Anything, "npm", "concurrent", "1.0.0")
|
||||
}
|
||||
|
||||
// TestBroadcaster_CacheHit verifies EventPackageDownloaded fires on a
|
||||
// cache-hit serve and no other events are emitted.
|
||||
func TestBroadcaster_CacheHit(t *testing.T) {
|
||||
mockStorage := &MockStorageBackend{}
|
||||
mockMetadata := &MockMetadataStore{}
|
||||
|
||||
now := time.Now()
|
||||
expiresAt := now.Add(24 * time.Hour)
|
||||
pkg := &metadata.Package{
|
||||
ID: "test-id",
|
||||
Registry: "npm",
|
||||
Name: "react",
|
||||
Version: "18.2.0",
|
||||
StorageKey: "npm/react/18.2.0",
|
||||
CachedAt: now,
|
||||
LastAccessed: now,
|
||||
ExpiresAt: &expiresAt,
|
||||
}
|
||||
mockMetadata.On("GetPackage", mock.Anything, "npm", "react", "18.2.0").Return(pkg, nil)
|
||||
mockStorage.On("Get", mock.Anything, "npm/react/18.2.0").Return(io.NopCloser(strings.NewReader("cached data")), nil)
|
||||
mockMetadata.On("UpdateDownloadCount", mock.Anything, "npm", "react", "18.2.0").Return(nil)
|
||||
|
||||
manager, err := New(mockStorage, mockMetadata, nil, nil, Config{})
|
||||
require.NoError(t, err)
|
||||
|
||||
bc := &fakeBroadcaster{}
|
||||
manager.SetBroadcaster(bc)
|
||||
|
||||
entry, err := manager.Get(context.Background(), "npm", "react", "18.2.0", nil)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, entry)
|
||||
|
||||
events := bc.snapshot()
|
||||
require.Len(t, events, 1, "expected exactly one event on cache hit")
|
||||
assert.Equal(t, string(websocket.EventPackageDownloaded), events[0].Type)
|
||||
|
||||
payload, ok := events[0].Payload.(map[string]interface{})
|
||||
require.True(t, ok, "payload should be map[string]interface{}")
|
||||
assert.Equal(t, "npm", payload["registry"])
|
||||
assert.Equal(t, "react", payload["name"])
|
||||
assert.Equal(t, "18.2.0", payload["version"])
|
||||
assert.Equal(t, true, payload["cache_hit"])
|
||||
}
|
||||
|
||||
// TestBroadcaster_CacheMissStore verifies EventPackageCached fires on a
|
||||
// cache-miss-then-store path and EventPackageDownloaded does NOT fire
|
||||
// on that path (avoid double counting).
|
||||
func TestBroadcaster_CacheMissStore(t *testing.T) {
|
||||
mockStorage := &MockStorageBackend{}
|
||||
mockMetadata := &MockMetadataStore{}
|
||||
|
||||
mockMetadata.On("GetPackage", mock.Anything, "npm", "lodash", "4.17.21").Return(nil, errors.New("not found"))
|
||||
mockStorage.On("GetQuota", mock.Anything).Return(&storage.QuotaInfo{Used: 100, Available: 900, Limit: 1000}, nil)
|
||||
mockStorage.On("Put", mock.Anything, "npm/lodash/4.17.21", mock.Anything, mock.Anything).Return(nil)
|
||||
mockMetadata.On("SavePackage", mock.Anything, mock.Anything).Return(nil)
|
||||
mockStorage.On("Get", mock.Anything, "npm/lodash/4.17.21").Return(io.NopCloser(strings.NewReader("upstream data")), nil)
|
||||
mockMetadata.On("UpdateDownloadCount", mock.Anything, "npm", "lodash", "4.17.21").Return(nil)
|
||||
|
||||
manager, err := New(mockStorage, mockMetadata, nil, nil, Config{})
|
||||
require.NoError(t, err)
|
||||
|
||||
bc := &fakeBroadcaster{}
|
||||
manager.SetBroadcaster(bc)
|
||||
|
||||
fetch := func(ctx context.Context) (io.ReadCloser, string, error) {
|
||||
return io.NopCloser(strings.NewReader("upstream data")), "https://registry.npmjs.org/lodash", nil
|
||||
}
|
||||
entry, err := manager.Get(context.Background(), "npm", "lodash", "4.17.21", fetch)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, entry)
|
||||
|
||||
types := bc.typesOnly()
|
||||
require.Contains(t, types, string(websocket.EventPackageCached))
|
||||
require.NotContains(t, types, string(websocket.EventPackageDownloaded),
|
||||
"miss-then-fetch path should not emit EventPackageDownloaded")
|
||||
|
||||
// Inspect EventPackageCached payload.
|
||||
var cachedEv *fakeBroadcastEvent
|
||||
for i := range bc.events {
|
||||
if bc.events[i].Type == string(websocket.EventPackageCached) {
|
||||
cachedEv = &bc.events[i]
|
||||
break
|
||||
}
|
||||
}
|
||||
require.NotNil(t, cachedEv)
|
||||
payload, ok := cachedEv.Payload.(map[string]interface{})
|
||||
require.True(t, ok)
|
||||
assert.Equal(t, "npm", payload["registry"])
|
||||
assert.Equal(t, "lodash", payload["name"])
|
||||
assert.Equal(t, "4.17.21", payload["version"])
|
||||
assert.Equal(t, int64(len("upstream data")), payload["size"])
|
||||
// Scanner is nil in this test → scan_status should be "disabled".
|
||||
assert.Equal(t, "disabled", payload["scan_status"])
|
||||
}
|
||||
|
||||
// TestBroadcaster_NoEmitOnError ensures error paths (storage Put fail,
|
||||
// metadata Save fail) do NOT emit EventPackageCached.
|
||||
func TestBroadcaster_NoEmitOnError(t *testing.T) {
|
||||
t.Run("storage put fails", func(t *testing.T) {
|
||||
mockStorage := &MockStorageBackend{}
|
||||
mockMetadata := &MockMetadataStore{}
|
||||
|
||||
mockMetadata.On("GetPackage", mock.Anything, "npm", "fail", "1.0.0").Return(nil, errors.New("not found"))
|
||||
mockStorage.On("GetQuota", mock.Anything).Return(&storage.QuotaInfo{Used: 100, Available: 900, Limit: 1000}, nil)
|
||||
mockStorage.On("Put", mock.Anything, "npm/fail/1.0.0", mock.Anything, mock.Anything).Return(errors.New("storage error"))
|
||||
|
||||
manager, err := New(mockStorage, mockMetadata, nil, nil, Config{})
|
||||
require.NoError(t, err)
|
||||
|
||||
bc := &fakeBroadcaster{}
|
||||
manager.SetBroadcaster(bc)
|
||||
|
||||
fetch := func(ctx context.Context) (io.ReadCloser, string, error) {
|
||||
return io.NopCloser(strings.NewReader("data")), "https://registry.npmjs.org/fail", nil
|
||||
}
|
||||
_, err = manager.Get(context.Background(), "npm", "fail", "1.0.0", fetch)
|
||||
require.Error(t, err)
|
||||
|
||||
assert.Empty(t, bc.snapshot(), "no events should be emitted when storage Put fails")
|
||||
})
|
||||
|
||||
t.Run("metadata save fails", func(t *testing.T) {
|
||||
mockStorage := &MockStorageBackend{}
|
||||
mockMetadata := &MockMetadataStore{}
|
||||
|
||||
mockMetadata.On("GetPackage", mock.Anything, "npm", "meta-fail", "1.0.0").Return(nil, errors.New("not found"))
|
||||
mockStorage.On("GetQuota", mock.Anything).Return(&storage.QuotaInfo{Used: 100, Available: 900, Limit: 1000}, nil)
|
||||
mockStorage.On("Put", mock.Anything, "npm/meta-fail/1.0.0", mock.Anything, mock.Anything).Return(nil)
|
||||
mockMetadata.On("SavePackage", mock.Anything, mock.Anything).Return(errors.New("metadata error"))
|
||||
mockStorage.On("Delete", mock.Anything, "npm/meta-fail/1.0.0").Return(nil)
|
||||
|
||||
manager, err := New(mockStorage, mockMetadata, nil, nil, Config{})
|
||||
require.NoError(t, err)
|
||||
|
||||
bc := &fakeBroadcaster{}
|
||||
manager.SetBroadcaster(bc)
|
||||
|
||||
fetch := func(ctx context.Context) (io.ReadCloser, string, error) {
|
||||
return io.NopCloser(strings.NewReader("data")), "https://registry.npmjs.org/meta-fail", nil
|
||||
}
|
||||
_, err = manager.Get(context.Background(), "npm", "meta-fail", "1.0.0", fetch)
|
||||
require.Error(t, err)
|
||||
|
||||
assert.Empty(t, bc.snapshot(), "no events should be emitted when metadata SavePackage fails")
|
||||
})
|
||||
|
||||
t.Run("nil broadcaster is safe", func(t *testing.T) {
|
||||
mockStorage := &MockStorageBackend{}
|
||||
mockMetadata := &MockMetadataStore{}
|
||||
|
||||
now := time.Now()
|
||||
expiresAt := now.Add(24 * time.Hour)
|
||||
pkg := &metadata.Package{
|
||||
ID: "id",
|
||||
Registry: "npm",
|
||||
Name: "x",
|
||||
Version: "1",
|
||||
StorageKey: "npm/x/1",
|
||||
CachedAt: now,
|
||||
LastAccessed: now,
|
||||
ExpiresAt: &expiresAt,
|
||||
}
|
||||
mockMetadata.On("GetPackage", mock.Anything, "npm", "x", "1").Return(pkg, nil)
|
||||
mockStorage.On("Get", mock.Anything, "npm/x/1").Return(io.NopCloser(strings.NewReader("d")), nil)
|
||||
mockMetadata.On("UpdateDownloadCount", mock.Anything, "npm", "x", "1").Return(nil)
|
||||
|
||||
manager, err := New(mockStorage, mockMetadata, nil, nil, Config{})
|
||||
require.NoError(t, err)
|
||||
// No SetBroadcaster — manager.broadcaster is nil.
|
||||
_, err = manager.Get(context.Background(), "npm", "x", "1", nil)
|
||||
require.NoError(t, err)
|
||||
})
|
||||
}
|
||||
|
||||
@@ -1,5 +1,3 @@
|
||||
// Package cdn provides CDN-friendly HTTP middleware (ETag, Cache-Control)
|
||||
// for proxying cached package responses.
|
||||
package cdn
|
||||
|
||||
import (
|
||||
|
||||
+6
-6
@@ -32,7 +32,7 @@ func TestCDNMiddlewareTestSuite(t *testing.T) {
|
||||
func (s *CDNMiddlewareTestSuite) TestCacheControlHeader() {
|
||||
handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.WriteHeader(http.StatusOK)
|
||||
_, _ = w.Write([]byte("test response"))
|
||||
w.Write([]byte("test response"))
|
||||
})
|
||||
|
||||
wrappedHandler := s.middleware.Handler(handler)
|
||||
@@ -51,7 +51,7 @@ func (s *CDNMiddlewareTestSuite) TestCacheControlHeader() {
|
||||
func (s *CDNMiddlewareTestSuite) TestETagGeneration() {
|
||||
handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.WriteHeader(http.StatusOK)
|
||||
_, _ = w.Write([]byte("test response content"))
|
||||
w.Write([]byte("test response content"))
|
||||
})
|
||||
|
||||
wrappedHandler := s.middleware.Handler(handler)
|
||||
@@ -71,7 +71,7 @@ func (s *CDNMiddlewareTestSuite) TestETagConsistencyAcrossRequests() {
|
||||
responseBody := []byte("test response content")
|
||||
handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.WriteHeader(http.StatusOK)
|
||||
_, _ = w.Write(responseBody)
|
||||
w.Write(responseBody)
|
||||
})
|
||||
|
||||
wrappedHandler := s.middleware.Handler(handler)
|
||||
@@ -95,7 +95,7 @@ func (s *CDNMiddlewareTestSuite) TestETagConsistencyAcrossRequests() {
|
||||
func (s *CDNMiddlewareTestSuite) TestVaryHeader() {
|
||||
handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.WriteHeader(http.StatusOK)
|
||||
_, _ = w.Write([]byte("test"))
|
||||
w.Write([]byte("test"))
|
||||
})
|
||||
|
||||
wrappedHandler := s.middleware.Handler(handler)
|
||||
@@ -246,7 +246,7 @@ func (s *CDNMiddlewareTestSuite) TestETagConsistency() {
|
||||
func (s *CDNMiddlewareTestSuite) TestNoCacheFor4xxErrors() {
|
||||
handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
_, _ = w.Write([]byte("not found"))
|
||||
w.Write([]byte("not found"))
|
||||
})
|
||||
|
||||
wrappedHandler := s.middleware.Handler(handler)
|
||||
@@ -264,7 +264,7 @@ func (s *CDNMiddlewareTestSuite) TestNoCacheFor4xxErrors() {
|
||||
func (s *CDNMiddlewareTestSuite) TestNoCacheFor5xxErrors() {
|
||||
handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
_, _ = w.Write([]byte("error"))
|
||||
w.Write([]byte("error"))
|
||||
})
|
||||
|
||||
wrappedHandler := s.middleware.Handler(handler)
|
||||
|
||||
+25
-69
@@ -1,5 +1,3 @@
|
||||
// Package config defines the typed configuration schema and loader for
|
||||
// GoHoarder, sourced from YAML files and environment variables.
|
||||
package config
|
||||
|
||||
import (
|
||||
@@ -9,32 +7,25 @@ import (
|
||||
|
||||
// Config is the main configuration struct
|
||||
type Config struct {
|
||||
Metadata MetadataConfig `mapstructure:"metadata" json:"metadata"`
|
||||
Logging LoggingConfig `mapstructure:"logging" json:"logging"`
|
||||
Storage StorageConfig `mapstructure:"storage" json:"storage"`
|
||||
Handlers HandlersConfig `mapstructure:"handlers" json:"handlers"`
|
||||
Cache CacheConfig `mapstructure:"cache" json:"cache"`
|
||||
Server ServerConfig `mapstructure:"server" json:"server"`
|
||||
Security SecurityConfig `mapstructure:"security" json:"security"`
|
||||
Network NetworkConfig `mapstructure:"network" json:"network"`
|
||||
Prewarming PrewarmingConfig `mapstructure:"prewarming" json:"prewarming"`
|
||||
Auth AuthConfig `mapstructure:"auth" json:"auth"`
|
||||
Metrics MetricsConfig `mapstructure:"metrics" json:"metrics"`
|
||||
Storage StorageConfig `mapstructure:"storage" json:"storage"`
|
||||
Cache CacheConfig `mapstructure:"cache" json:"cache"`
|
||||
Security SecurityConfig `mapstructure:"security" json:"security"`
|
||||
Metadata MetadataConfig `mapstructure:"metadata" json:"metadata"`
|
||||
Handlers HandlersConfig `mapstructure:"handlers" json:"handlers"`
|
||||
Server ServerConfig `mapstructure:"server" json:"server"`
|
||||
Logging LoggingConfig `mapstructure:"logging" json:"logging"`
|
||||
Network NetworkConfig `mapstructure:"network" json:"network"`
|
||||
Auth AuthConfig `mapstructure:"auth" json:"auth"`
|
||||
}
|
||||
|
||||
// ServerConfig contains HTTP server configuration
|
||||
type ServerConfig struct {
|
||||
Host string `mapstructure:"host" json:"host"`
|
||||
TLS TLSConfig `mapstructure:"tls" json:"tls"`
|
||||
// AllowedOrigins is the WebSocket Origin allowlist. Each entry is either
|
||||
// an exact origin (e.g. "https://app.example.com") or a wildcard host
|
||||
// pattern (e.g. "https://*.example.com"). When empty, only same-origin
|
||||
// WebSocket upgrades are allowed.
|
||||
AllowedOrigins []string `mapstructure:"allowed_origins" json:"allowed_origins"`
|
||||
Port int `mapstructure:"port" json:"port"`
|
||||
ReadTimeout time.Duration `mapstructure:"read_timeout" json:"read_timeout"`
|
||||
WriteTimeout time.Duration `mapstructure:"write_timeout" json:"write_timeout"`
|
||||
IdleTimeout time.Duration `mapstructure:"idle_timeout" json:"idle_timeout"`
|
||||
TLS TLSConfig `mapstructure:"tls" json:"tls"`
|
||||
Host string `mapstructure:"host" json:"host"`
|
||||
Port int `mapstructure:"port" json:"port"`
|
||||
ReadTimeout time.Duration `mapstructure:"read_timeout" json:"read_timeout"`
|
||||
WriteTimeout time.Duration `mapstructure:"write_timeout" json:"write_timeout"`
|
||||
IdleTimeout time.Duration `mapstructure:"idle_timeout" json:"idle_timeout"`
|
||||
}
|
||||
|
||||
// TLSConfig contains TLS/HTTPS configuration
|
||||
@@ -48,21 +39,12 @@ type TLSConfig struct {
|
||||
type StorageConfig struct {
|
||||
Options map[string]interface{} `mapstructure:"options" json:"options"`
|
||||
SMB SMBConfig `mapstructure:"smb" json:"smb"`
|
||||
NFS NFSConfig `mapstructure:"nfs" json:"nfs"`
|
||||
Backend string `mapstructure:"backend" json:"backend"`
|
||||
Path string `mapstructure:"path" json:"path"`
|
||||
Filesystem FilesystemConfig `mapstructure:"filesystem" json:"filesystem"`
|
||||
S3 S3Config `mapstructure:"s3" json:"s3"`
|
||||
}
|
||||
|
||||
// NFSConfig contains NFS-specific storage configuration. The path is taken
|
||||
// from StorageConfig.Path (mount point). This struct only carries flags
|
||||
// specific to NFS semantics. SyncWrites pointer-bool so absent config
|
||||
// defaults to true (durable by default).
|
||||
type NFSConfig struct {
|
||||
SyncWrites *bool `mapstructure:"sync_writes" json:"sync_writes"`
|
||||
}
|
||||
|
||||
// FilesystemConfig contains local filesystem storage configuration
|
||||
type FilesystemConfig struct {
|
||||
BasePath string `mapstructure:"base_path" json:"base_path"`
|
||||
@@ -93,17 +75,15 @@ type SMBConfig struct {
|
||||
type MetadataConfig struct {
|
||||
Backend string `mapstructure:"backend" json:"backend"`
|
||||
Connection string `mapstructure:"connection" json:"connection"`
|
||||
LogLevel string `mapstructure:"log_level" json:"log_level"` // "silent", "error", "warn", "info"
|
||||
SQLite SQLiteConfig `mapstructure:"sqlite" json:"sqlite"`
|
||||
PostgreSQL PostgreSQLConfig `mapstructure:"postgresql" json:"postgresql"`
|
||||
|
||||
SQLite SQLiteConfig `mapstructure:"sqlite" json:"sqlite"`
|
||||
|
||||
MySQL MySQLConfig `mapstructure:"mysql" json:"mysql"`
|
||||
MySQL MySQLConfig `mapstructure:"mysql" json:"mysql"`
|
||||
|
||||
// GORM-specific settings
|
||||
MaxOpenConns int `mapstructure:"max_open_conns" json:"max_open_conns"`
|
||||
MaxIdleConns int `mapstructure:"max_idle_conns" json:"max_idle_conns"`
|
||||
ConnMaxLifetime int `mapstructure:"conn_max_lifetime" json:"conn_max_lifetime"` // seconds
|
||||
MaxOpenConns int `mapstructure:"max_open_conns" json:"max_open_conns"`
|
||||
MaxIdleConns int `mapstructure:"max_idle_conns" json:"max_idle_conns"`
|
||||
ConnMaxLifetime int `mapstructure:"conn_max_lifetime" json:"conn_max_lifetime"` // seconds
|
||||
LogLevel string `mapstructure:"log_level" json:"log_level"` // "silent", "error", "warn", "info"
|
||||
}
|
||||
|
||||
// SQLiteConfig contains SQLite-specific configuration
|
||||
@@ -115,21 +95,21 @@ type SQLiteConfig struct {
|
||||
// PostgreSQLConfig contains PostgreSQL-specific configuration
|
||||
type PostgreSQLConfig struct {
|
||||
Host string `mapstructure:"host" json:"host"`
|
||||
Port int `mapstructure:"port" json:"port"`
|
||||
Database string `mapstructure:"database" json:"database"`
|
||||
User string `mapstructure:"user" json:"user"`
|
||||
Password string `mapstructure:"password" json:"-"`
|
||||
SSLMode string `mapstructure:"ssl_mode" json:"ssl_mode"`
|
||||
Port int `mapstructure:"port" json:"port"`
|
||||
}
|
||||
|
||||
// MySQLConfig contains MySQL/MariaDB-specific configuration
|
||||
type MySQLConfig struct {
|
||||
Host string `mapstructure:"host" json:"host"`
|
||||
Port int `mapstructure:"port" json:"port"`
|
||||
Database string `mapstructure:"database" json:"database"`
|
||||
User string `mapstructure:"user" json:"user"`
|
||||
Password string `mapstructure:"password" json:"-"` // Don't serialize
|
||||
Charset string `mapstructure:"charset" json:"charset"`
|
||||
Port int `mapstructure:"port" json:"port"`
|
||||
ParseTime bool `mapstructure:"parse_time" json:"parse_time"`
|
||||
}
|
||||
|
||||
@@ -144,10 +124,10 @@ type CacheConfig struct {
|
||||
|
||||
// SecurityConfig contains security scanning configuration
|
||||
type SecurityConfig struct {
|
||||
Scanners ScannersConfig `mapstructure:"scanners" json:"scanners"`
|
||||
BlockOnSeverity string `mapstructure:"block_on_severity" json:"block_on_severity"`
|
||||
AllowedPackages []string `mapstructure:"allowed_packages" json:"allowed_packages"`
|
||||
IgnoredCVEs []string `mapstructure:"ignored_cves" json:"ignored_cves"`
|
||||
Scanners ScannersConfig `mapstructure:"scanners" json:"scanners"`
|
||||
BlockThresholds VulnerabilityThresholds `mapstructure:"block_thresholds" json:"block_thresholds"`
|
||||
RescanInterval time.Duration `mapstructure:"rescan_interval" json:"rescan_interval"`
|
||||
Enabled bool `mapstructure:"enabled" json:"enabled"`
|
||||
@@ -167,8 +147,8 @@ type VulnerabilityThresholds struct {
|
||||
type ScannersConfig struct {
|
||||
Trivy TrivyConfig `mapstructure:"trivy" json:"trivy"`
|
||||
GHSA GHSAConfig `mapstructure:"ghsa" json:"ghsa"`
|
||||
OSV OSVConfig `mapstructure:"osv" json:"osv"`
|
||||
Static StaticConfig `mapstructure:"static" json:"static"`
|
||||
OSV OSVConfig `mapstructure:"osv" json:"osv"`
|
||||
Grype GrypeConfig `mapstructure:"grype" json:"grype"`
|
||||
Govulncheck GovulncheckConfig `mapstructure:"govulncheck" json:"govulncheck"`
|
||||
NpmAudit NpmAuditConfig `mapstructure:"npm_audit" json:"npm_audit"`
|
||||
@@ -276,21 +256,6 @@ type LoggingConfig struct {
|
||||
Format string `mapstructure:"format" json:"format"` // json, pretty
|
||||
}
|
||||
|
||||
// PrewarmingConfig controls the popular-package pre-warming worker.
|
||||
type PrewarmingConfig struct {
|
||||
Interval time.Duration `mapstructure:"interval" json:"interval"`
|
||||
MaxConcurrent int `mapstructure:"max_concurrent" json:"max_concurrent"`
|
||||
TopPackages int `mapstructure:"top_packages" json:"top_packages"`
|
||||
Enabled bool `mapstructure:"enabled" json:"enabled"`
|
||||
}
|
||||
|
||||
// MetricsConfig controls /metrics endpoint behaviour.
|
||||
type MetricsConfig struct {
|
||||
// RequireAuth, when true and Auth.Enabled is also true, gates /metrics
|
||||
// behind RequireAuth middleware. Default false (Prometheus-friendly).
|
||||
RequireAuth bool `mapstructure:"require_auth" json:"require_auth"`
|
||||
}
|
||||
|
||||
// HandlersConfig contains package manager handler configurations
|
||||
type HandlersConfig struct {
|
||||
Go GoHandlerConfig `mapstructure:"go" json:"go"`
|
||||
@@ -434,15 +399,6 @@ func Default() *Config {
|
||||
Level: "info",
|
||||
Format: "json",
|
||||
},
|
||||
Prewarming: PrewarmingConfig{
|
||||
Enabled: false,
|
||||
Interval: 1 * time.Hour,
|
||||
MaxConcurrent: 5,
|
||||
TopPackages: 100,
|
||||
},
|
||||
Metrics: MetricsConfig{
|
||||
RequireAuth: false,
|
||||
},
|
||||
Handlers: HandlersConfig{
|
||||
Go: GoHandlerConfig{
|
||||
Enabled: true,
|
||||
|
||||
@@ -287,13 +287,13 @@ auth:
|
||||
s.Run(tt.name, func() {
|
||||
// Write config file
|
||||
configPath := filepath.Join(s.tempDir, "config.yaml")
|
||||
err := os.WriteFile(configPath, []byte(tt.configYAML), 0600)
|
||||
err := os.WriteFile(configPath, []byte(tt.configYAML), 0644)
|
||||
s.Require().NoError(err)
|
||||
|
||||
// Set environment variables
|
||||
for k, v := range tt.envVars {
|
||||
_ = os.Setenv(k, v) // #nosec G104 -- test setup, error not actionable
|
||||
defer os.Unsetenv(k) //nolint:errcheck // test cleanup
|
||||
os.Setenv(k, v)
|
||||
defer os.Unsetenv(k)
|
||||
}
|
||||
|
||||
// Load config
|
||||
|
||||
@@ -1,5 +1,3 @@
|
||||
// Package errors provides typed application errors with structured codes
|
||||
// and wrapping helpers used across the GoHoarder service.
|
||||
package errors
|
||||
|
||||
import (
|
||||
|
||||
@@ -1,24 +0,0 @@
|
||||
// Package events defines the cross-package event broadcaster interface.
|
||||
//
|
||||
// Sub-systems such as cache and scanner emit lifecycle events but must
|
||||
// not import pkg/websocket directly (would create circular import once
|
||||
// websocket grows callers in those packages or in app wiring).
|
||||
//
|
||||
// The Broadcaster contract is intentionally minimal: a single method
|
||||
// that accepts an event type string and an arbitrary payload. The
|
||||
// concrete implementation (websocket.Server) decides how to marshal
|
||||
// and dispatch the payload.
|
||||
package events
|
||||
|
||||
// Broadcaster is the minimal contract sub-systems use to publish events.
|
||||
//
|
||||
// Implementations MUST be safe for concurrent use and MUST NOT block
|
||||
// the caller (callers expect fire-and-forget semantics). If the
|
||||
// underlying transport is full or unavailable, implementations should
|
||||
// drop the event and log, never block or panic.
|
||||
type Broadcaster interface {
|
||||
// BroadcastEvent publishes an event of the given type with the
|
||||
// supplied payload. Payload is typically map[string]any but
|
||||
// implementations may accept arbitrary serialisable values.
|
||||
BroadcastEvent(eventType string, payload any)
|
||||
}
|
||||
@@ -1,5 +1,3 @@
|
||||
// Package health exposes liveness and readiness checks consumed by HTTP
|
||||
// probes and orchestrators.
|
||||
package health
|
||||
|
||||
import (
|
||||
@@ -134,10 +132,9 @@ func (c *Checker) HealthHandler() http.HandlerFunc {
|
||||
}
|
||||
|
||||
statusCode := http.StatusOK
|
||||
switch healthData.Status {
|
||||
case StatusUnhealthy:
|
||||
if healthData.Status == StatusUnhealthy {
|
||||
statusCode = http.StatusServiceUnavailable
|
||||
case StatusDegraded:
|
||||
} else if healthData.Status == StatusDegraded {
|
||||
statusCode = http.StatusOK // 200 but degraded
|
||||
}
|
||||
|
||||
|
||||
@@ -1,5 +1,3 @@
|
||||
// Package logger configures the zerolog-based application logger used by
|
||||
// the GoHoarder server and CLI commands.
|
||||
package logger
|
||||
|
||||
import (
|
||||
|
||||
@@ -1,5 +1,3 @@
|
||||
// Package file implements a JSON-on-disk metadata store used for local
|
||||
// development and tests.
|
||||
package file
|
||||
|
||||
import (
|
||||
@@ -546,29 +544,3 @@ func (s *Store) Close() error {
|
||||
// Nothing to close for file-based store
|
||||
return nil
|
||||
}
|
||||
|
||||
// SaveAPIKey is a stub: file-backed metadata does not persist API keys.
|
||||
// The auth.Manager treats ErrNotImplemented as "in-memory only" mode.
|
||||
func (s *Store) SaveAPIKey(ctx context.Context, key *metadata.APIKey) error {
|
||||
return metadata.ErrNotImplemented
|
||||
}
|
||||
|
||||
// GetAPIKey is a stub. See SaveAPIKey.
|
||||
func (s *Store) GetAPIKey(ctx context.Context, id string) (*metadata.APIKey, error) {
|
||||
return nil, metadata.ErrNotImplemented
|
||||
}
|
||||
|
||||
// ListAPIKeys is a stub. See SaveAPIKey.
|
||||
func (s *Store) ListAPIKeys(ctx context.Context) ([]*metadata.APIKey, error) {
|
||||
return nil, metadata.ErrNotImplemented
|
||||
}
|
||||
|
||||
// DeleteAPIKey is a stub. See SaveAPIKey.
|
||||
func (s *Store) DeleteAPIKey(ctx context.Context, id string) error {
|
||||
return metadata.ErrNotImplemented
|
||||
}
|
||||
|
||||
// UpdateAPIKeyLastUsed is a stub. See SaveAPIKey.
|
||||
func (s *Store) UpdateAPIKeyLastUsed(ctx context.Context, id string, t time.Time) error {
|
||||
return metadata.ErrNotImplemented
|
||||
}
|
||||
|
||||
@@ -66,7 +66,7 @@ func (w *AggregationWorker) AggregateHourly() error {
|
||||
log.Debug().Msg("Starting hourly aggregation")
|
||||
|
||||
// Get dialect name
|
||||
dialectName := w.db.Name()
|
||||
dialectName := w.db.Dialector.Name()
|
||||
|
||||
// Calculate cutoff time (aggregate events older than 5 minutes to avoid partial data)
|
||||
cutoff := time.Now().Add(-5 * time.Minute).Truncate(time.Hour)
|
||||
@@ -147,22 +147,16 @@ func (w *AggregationWorker) AggregateHourly() error {
|
||||
return err
|
||||
}
|
||||
|
||||
// Delete aggregated events in the SAME transaction using the SAME cutoff
|
||||
// that drove the aggregation. This guarantees an event is counted exactly
|
||||
// once: prior to this commit it lives in download_events; after this commit
|
||||
// it's reflected only in download_stats_hourly. Any later run sees only
|
||||
// fresh events (downloaded_at >= cutoff at write time) and cannot
|
||||
// double-count rows that were already aggregated.
|
||||
// Trade-off: raw events older than `cutoff` are not retained for debugging.
|
||||
deleteResult := tx.Exec("DELETE FROM download_events WHERE downloaded_at < ?", cutoff)
|
||||
// Delete aggregated events (older than 24 hours to keep recent data for debugging)
|
||||
deleteOlder := time.Now().Add(-24 * time.Hour)
|
||||
deleteResult := tx.Exec("DELETE FROM download_events WHERE downloaded_at < ?", deleteOlder)
|
||||
if deleteResult.Error != nil {
|
||||
return deleteResult.Error
|
||||
}
|
||||
|
||||
// Also update package-level stats (NULL package_id = registry totals)
|
||||
var registryAggSQL string
|
||||
switch dialectName {
|
||||
case "postgres":
|
||||
if dialectName == "postgres" {
|
||||
registryAggSQL = `
|
||||
INSERT INTO download_stats_hourly (registry_id, package_id, time_bucket, download_count, unique_ips, auth_downloads, created_at, updated_at)
|
||||
SELECT
|
||||
@@ -184,7 +178,7 @@ func (w *AggregationWorker) AggregateHourly() error {
|
||||
auth_downloads = EXCLUDED.auth_downloads,
|
||||
updated_at = NOW()
|
||||
`
|
||||
case "mysql":
|
||||
} else if dialectName == "mysql" {
|
||||
registryAggSQL = `
|
||||
INSERT INTO download_stats_hourly (registry_id, package_id, time_bucket, download_count, unique_ips, auth_downloads, created_at, updated_at)
|
||||
SELECT
|
||||
@@ -205,7 +199,7 @@ func (w *AggregationWorker) AggregateHourly() error {
|
||||
auth_downloads = VALUES(auth_downloads),
|
||||
updated_at = NOW()
|
||||
`
|
||||
default:
|
||||
} else {
|
||||
// SQLite
|
||||
registryAggSQL = `
|
||||
INSERT OR REPLACE INTO download_stats_hourly (registry_id, package_id, time_bucket, download_count, unique_ips, auth_downloads, created_at, updated_at)
|
||||
@@ -243,7 +237,7 @@ func (w *AggregationWorker) AggregateDaily() error {
|
||||
startTime := time.Now()
|
||||
log.Debug().Msg("Starting daily aggregation")
|
||||
|
||||
dialectName := w.db.Name()
|
||||
dialectName := w.db.Dialector.Name()
|
||||
|
||||
// Aggregate yesterday's data
|
||||
yesterday := time.Now().AddDate(0, 0, -1).Truncate(24 * time.Hour)
|
||||
|
||||
@@ -1,115 +0,0 @@
|
||||
package gormstore
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"github.com/lukaszraczylo/gohoarder/pkg/errors"
|
||||
"github.com/lukaszraczylo/gohoarder/pkg/metadata"
|
||||
"gorm.io/gorm"
|
||||
"gorm.io/gorm/clause"
|
||||
)
|
||||
|
||||
// SaveAPIKey persists an API key. Insert-or-update by primary key.
|
||||
func (s *GORMStoreV2) SaveAPIKey(ctx context.Context, key *metadata.APIKey) error {
|
||||
if key == nil || key.ID == "" {
|
||||
return errors.New(errors.ErrCodeBadRequest, "api key id required")
|
||||
}
|
||||
|
||||
model := &APIKeyModel{
|
||||
ID: key.ID,
|
||||
KeyHash: key.KeyHash,
|
||||
Project: key.Project,
|
||||
Role: key.Role,
|
||||
Permissions: key.Permissions,
|
||||
CreatedAt: key.CreatedAt,
|
||||
ExpiresAt: key.ExpiresAt,
|
||||
LastUsedAt: key.LastUsedAt,
|
||||
Revoked: key.Revoked,
|
||||
}
|
||||
if model.CreatedAt.IsZero() {
|
||||
model.CreatedAt = time.Now()
|
||||
}
|
||||
|
||||
// Upsert on primary key. Updates all columns on conflict — keeps the call
|
||||
// site simple (Generate then Revoke both flow through here).
|
||||
err := s.db.WithContext(ctx).
|
||||
Clauses(clause.OnConflict{
|
||||
Columns: []clause.Column{{Name: "id"}},
|
||||
UpdateAll: true,
|
||||
}).
|
||||
Create(model).Error
|
||||
if err != nil {
|
||||
return errors.Wrap(err, errors.ErrCodeStorageFailure, "failed to save api key")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetAPIKey retrieves an API key by ID. Returns a not-found error if missing.
|
||||
func (s *GORMStoreV2) GetAPIKey(ctx context.Context, id string) (*metadata.APIKey, error) {
|
||||
var model APIKeyModel
|
||||
err := s.db.WithContext(ctx).Where("id = ?", id).First(&model).Error
|
||||
if err != nil {
|
||||
if err == gorm.ErrRecordNotFound {
|
||||
return nil, errors.New(errors.ErrCodeNotFound, "api key not found")
|
||||
}
|
||||
return nil, errors.Wrap(err, errors.ErrCodeStorageFailure, "failed to get api key")
|
||||
}
|
||||
return apiKeyModelToMetadata(&model), nil
|
||||
}
|
||||
|
||||
// ListAPIKeys returns all API keys (revoked included). Caller filters as needed.
|
||||
func (s *GORMStoreV2) ListAPIKeys(ctx context.Context) ([]*metadata.APIKey, error) {
|
||||
var models []APIKeyModel
|
||||
if err := s.db.WithContext(ctx).Order("created_at DESC").Find(&models).Error; err != nil {
|
||||
return nil, errors.Wrap(err, errors.ErrCodeStorageFailure, "failed to list api keys")
|
||||
}
|
||||
out := make([]*metadata.APIKey, len(models))
|
||||
for i := range models {
|
||||
out[i] = apiKeyModelToMetadata(&models[i])
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// DeleteAPIKey hard-deletes an API key by ID. Use SaveAPIKey with Revoked=true
|
||||
// for audit-friendly revocation; this method is for explicit purges.
|
||||
func (s *GORMStoreV2) DeleteAPIKey(ctx context.Context, id string) error {
|
||||
result := s.db.WithContext(ctx).Where("id = ?", id).Delete(&APIKeyModel{})
|
||||
if result.Error != nil {
|
||||
return errors.Wrap(result.Error, errors.ErrCodeStorageFailure, "failed to delete api key")
|
||||
}
|
||||
if result.RowsAffected == 0 {
|
||||
return errors.New(errors.ErrCodeNotFound, "api key not found")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// UpdateAPIKeyLastUsed updates only last_used_at. Cheap (single column UPDATE).
|
||||
// Auth manager calls this asynchronously per-request; do not perform extra
|
||||
// work here without measuring impact.
|
||||
func (s *GORMStoreV2) UpdateAPIKeyLastUsed(ctx context.Context, id string, t time.Time) error {
|
||||
result := s.db.WithContext(ctx).
|
||||
Model(&APIKeyModel{}).
|
||||
Where("id = ?", id).
|
||||
Update("last_used_at", t)
|
||||
if result.Error != nil {
|
||||
return errors.Wrap(result.Error, errors.ErrCodeStorageFailure, "failed to update api key last used")
|
||||
}
|
||||
// Missing rows are not fatal: caller validated the key against an
|
||||
// in-memory snapshot that may have lagged a concurrent revoke.
|
||||
return nil
|
||||
}
|
||||
|
||||
func apiKeyModelToMetadata(m *APIKeyModel) *metadata.APIKey {
|
||||
return &metadata.APIKey{
|
||||
ID: m.ID,
|
||||
KeyHash: m.KeyHash,
|
||||
Project: m.Project,
|
||||
Role: m.Role,
|
||||
Permissions: m.Permissions,
|
||||
CreatedAt: m.CreatedAt,
|
||||
ExpiresAt: m.ExpiresAt,
|
||||
LastUsedAt: m.LastUsedAt,
|
||||
Revoked: m.Revoked,
|
||||
}
|
||||
}
|
||||
@@ -1,231 +0,0 @@
|
||||
package gormstore
|
||||
|
||||
import (
|
||||
"context"
|
||||
stderrors "errors"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/lukaszraczylo/gohoarder/pkg/metadata"
|
||||
)
|
||||
|
||||
// newAPIKeyTestStore spins up an isolated SQLite store. Each test gets a
|
||||
// fresh DSN to avoid the shared-cache cross-test bleeding that the suite
|
||||
// uses elsewhere (we want a clean api_keys table).
|
||||
func newAPIKeyTestStore(t *testing.T) *GORMStoreV2 {
|
||||
t.Helper()
|
||||
// Use :memory: in the DSN so NewV2 skips the background aggregation
|
||||
// worker (it would otherwise contend on the shared cache and produce
|
||||
// "database table is locked" errors mid-test). MaxOpenConns=1 pins the
|
||||
// connection so the in-memory database survives across statements
|
||||
// (sqlite drops the DB when the only connection closes).
|
||||
cfg := Config{
|
||||
Driver: "sqlite",
|
||||
DSN: "file::memory:?_busy_timeout=5000",
|
||||
MaxOpenConns: 1,
|
||||
MaxIdleConns: 1,
|
||||
ConnMaxLifetime: time.Hour,
|
||||
LogLevel: "silent",
|
||||
}
|
||||
store, err := NewV2(cfg)
|
||||
if err != nil {
|
||||
t.Fatalf("NewV2: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
// Truncate first so cross-test isolation holds with shared cache.
|
||||
store.db.Exec("DELETE FROM api_keys")
|
||||
_ = store.Close()
|
||||
})
|
||||
return store
|
||||
}
|
||||
|
||||
func TestAPIKey_RoundTrip(t *testing.T) {
|
||||
store := newAPIKeyTestStore(t)
|
||||
ctx := context.Background()
|
||||
|
||||
expires := time.Now().Add(24 * time.Hour).UTC().Truncate(time.Second)
|
||||
key := &metadata.APIKey{
|
||||
ID: "test-id-1",
|
||||
KeyHash: "$2a$04$abcdefghijklmnopqrstuv",
|
||||
Project: "default",
|
||||
Role: "admin",
|
||||
CreatedAt: time.Now().UTC().Truncate(time.Second),
|
||||
ExpiresAt: &expires,
|
||||
}
|
||||
|
||||
if err := store.SaveAPIKey(ctx, key); err != nil {
|
||||
t.Fatalf("SaveAPIKey: %v", err)
|
||||
}
|
||||
|
||||
got, err := store.GetAPIKey(ctx, key.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("GetAPIKey: %v", err)
|
||||
}
|
||||
if got.ID != key.ID {
|
||||
t.Errorf("ID = %q, want %q", got.ID, key.ID)
|
||||
}
|
||||
if got.KeyHash != key.KeyHash {
|
||||
t.Errorf("KeyHash mismatch")
|
||||
}
|
||||
if got.Project != "default" {
|
||||
t.Errorf("Project = %q, want default", got.Project)
|
||||
}
|
||||
if got.Role != "admin" {
|
||||
t.Errorf("Role = %q, want admin", got.Role)
|
||||
}
|
||||
if got.Revoked {
|
||||
t.Errorf("Revoked = true, want false")
|
||||
}
|
||||
if got.ExpiresAt == nil || !got.ExpiresAt.Equal(expires) {
|
||||
t.Errorf("ExpiresAt = %v, want %v", got.ExpiresAt, expires)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAPIKey_Upsert(t *testing.T) {
|
||||
store := newAPIKeyTestStore(t)
|
||||
ctx := context.Background()
|
||||
|
||||
key := &metadata.APIKey{
|
||||
ID: "upsert-id",
|
||||
KeyHash: "hash1",
|
||||
Project: "p1",
|
||||
Role: "read_only",
|
||||
CreatedAt: time.Now().UTC().Truncate(time.Second),
|
||||
}
|
||||
if err := store.SaveAPIKey(ctx, key); err != nil {
|
||||
t.Fatalf("first SaveAPIKey: %v", err)
|
||||
}
|
||||
|
||||
// Update via Save (revoke).
|
||||
key.Revoked = true
|
||||
key.KeyHash = "hash2"
|
||||
if err := store.SaveAPIKey(ctx, key); err != nil {
|
||||
t.Fatalf("second SaveAPIKey: %v", err)
|
||||
}
|
||||
|
||||
got, err := store.GetAPIKey(ctx, key.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("GetAPIKey: %v", err)
|
||||
}
|
||||
if !got.Revoked {
|
||||
t.Errorf("Revoked = false after revoke")
|
||||
}
|
||||
if got.KeyHash != "hash2" {
|
||||
t.Errorf("KeyHash = %q, want hash2", got.KeyHash)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAPIKey_List_OrdersByCreatedDesc(t *testing.T) {
|
||||
store := newAPIKeyTestStore(t)
|
||||
ctx := context.Background()
|
||||
|
||||
now := time.Now().UTC().Truncate(time.Second)
|
||||
keys := []*metadata.APIKey{
|
||||
{ID: "k-old", KeyHash: "h", Project: "p", Role: "read_only", CreatedAt: now.Add(-2 * time.Hour)},
|
||||
{ID: "k-mid", KeyHash: "h", Project: "p", Role: "read_only", CreatedAt: now.Add(-1 * time.Hour)},
|
||||
{ID: "k-new", KeyHash: "h", Project: "p", Role: "read_only", CreatedAt: now},
|
||||
}
|
||||
for _, k := range keys {
|
||||
if err := store.SaveAPIKey(ctx, k); err != nil {
|
||||
t.Fatalf("save %s: %v", k.ID, err)
|
||||
}
|
||||
}
|
||||
|
||||
listed, err := store.ListAPIKeys(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("ListAPIKeys: %v", err)
|
||||
}
|
||||
if len(listed) != 3 {
|
||||
t.Fatalf("len = %d, want 3", len(listed))
|
||||
}
|
||||
wantOrder := []string{"k-new", "k-mid", "k-old"}
|
||||
for i, want := range wantOrder {
|
||||
if listed[i].ID != want {
|
||||
t.Errorf("listed[%d].ID = %q, want %q", i, listed[i].ID, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAPIKey_UpdateLastUsed(t *testing.T) {
|
||||
store := newAPIKeyTestStore(t)
|
||||
ctx := context.Background()
|
||||
|
||||
key := &metadata.APIKey{
|
||||
ID: "lu-id",
|
||||
KeyHash: "h",
|
||||
Project: "p",
|
||||
Role: "admin",
|
||||
CreatedAt: time.Now().UTC().Truncate(time.Second),
|
||||
}
|
||||
if err := store.SaveAPIKey(ctx, key); err != nil {
|
||||
t.Fatalf("SaveAPIKey: %v", err)
|
||||
}
|
||||
|
||||
used := time.Now().UTC().Truncate(time.Second).Add(time.Minute)
|
||||
if err := store.UpdateAPIKeyLastUsed(ctx, key.ID, used); err != nil {
|
||||
t.Fatalf("UpdateAPIKeyLastUsed: %v", err)
|
||||
}
|
||||
|
||||
got, err := store.GetAPIKey(ctx, key.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("GetAPIKey: %v", err)
|
||||
}
|
||||
if got.LastUsedAt == nil || !got.LastUsedAt.Equal(used) {
|
||||
t.Errorf("LastUsedAt = %v, want %v", got.LastUsedAt, used)
|
||||
}
|
||||
|
||||
// Updating a missing key must not error: validation can race revoke.
|
||||
if err := store.UpdateAPIKeyLastUsed(ctx, "no-such-id", used); err != nil {
|
||||
t.Errorf("UpdateAPIKeyLastUsed(missing) returned err: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAPIKey_Delete(t *testing.T) {
|
||||
store := newAPIKeyTestStore(t)
|
||||
ctx := context.Background()
|
||||
|
||||
key := &metadata.APIKey{
|
||||
ID: "del-id",
|
||||
KeyHash: "h",
|
||||
Project: "p",
|
||||
Role: "read_only",
|
||||
CreatedAt: time.Now().UTC().Truncate(time.Second),
|
||||
}
|
||||
if err := store.SaveAPIKey(ctx, key); err != nil {
|
||||
t.Fatalf("SaveAPIKey: %v", err)
|
||||
}
|
||||
if err := store.DeleteAPIKey(ctx, key.ID); err != nil {
|
||||
t.Fatalf("DeleteAPIKey: %v", err)
|
||||
}
|
||||
if _, err := store.GetAPIKey(ctx, key.ID); err == nil {
|
||||
t.Errorf("GetAPIKey after delete: want error, got nil")
|
||||
}
|
||||
// Second delete returns not-found.
|
||||
if err := store.DeleteAPIKey(ctx, key.ID); err == nil {
|
||||
t.Errorf("second DeleteAPIKey: want error, got nil")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAPIKey_SaveAPIKey_RejectsEmptyID(t *testing.T) {
|
||||
store := newAPIKeyTestStore(t)
|
||||
ctx := context.Background()
|
||||
|
||||
err := store.SaveAPIKey(ctx, &metadata.APIKey{KeyHash: "h"})
|
||||
if err == nil {
|
||||
t.Fatalf("want error for empty ID")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAPIKey_GetMissing_ReturnsNotFound(t *testing.T) {
|
||||
store := newAPIKeyTestStore(t)
|
||||
ctx := context.Background()
|
||||
|
||||
_, err := store.GetAPIKey(ctx, "missing")
|
||||
if err == nil {
|
||||
t.Fatalf("want error")
|
||||
}
|
||||
// Sanity: error chain doesn't accidentally surface ErrNotImplemented.
|
||||
if stderrors.Is(err, metadata.ErrNotImplemented) {
|
||||
t.Errorf("got ErrNotImplemented for missing key, expected not-found")
|
||||
}
|
||||
}
|
||||
@@ -1,5 +1,3 @@
|
||||
// Package gormstore implements the GORM-backed metadata store used for
|
||||
// production persistence (Postgres, MySQL, SQLite).
|
||||
package gormstore
|
||||
|
||||
import (
|
||||
@@ -15,13 +13,13 @@ type Config struct {
|
||||
Driver string // "sqlite", "postgres", "mysql"
|
||||
DSN string // Data Source Name
|
||||
|
||||
// GORM settings
|
||||
LogLevel string // "silent", "error", "warn", "info"
|
||||
|
||||
// Connection pool
|
||||
MaxOpenConns int
|
||||
MaxIdleConns int
|
||||
ConnMaxLifetime time.Duration
|
||||
|
||||
// GORM settings
|
||||
LogLevel string // "silent", "error", "warn", "info"
|
||||
}
|
||||
|
||||
// Validate validates the configuration
|
||||
|
||||
@@ -5,7 +5,6 @@ import (
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/lukaszraczylo/gohoarder/pkg/errors"
|
||||
@@ -24,7 +23,6 @@ type GORMStoreV2 struct {
|
||||
registryCache map[string]int32 // Cache registry name -> ID mapping
|
||||
aggregationWorker *AggregationWorker
|
||||
partitionManager *PartitionManager
|
||||
registryCacheMu sync.RWMutex // Protects registryCache for concurrent access
|
||||
}
|
||||
|
||||
// NewV2 creates a new GORM-based metadata store with V2 schema
|
||||
@@ -104,7 +102,7 @@ func NewV2(cfg Config) (*GORMStoreV2, error) {
|
||||
}
|
||||
|
||||
// Seed default registries if empty
|
||||
if store.registryCacheLen() == 0 {
|
||||
if len(store.registryCache) == 0 {
|
||||
if err := store.seedDefaultRegistries(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -135,21 +133,12 @@ func (s *GORMStoreV2) loadRegistryCache() error {
|
||||
return errors.Wrap(err, errors.ErrCodeStorageFailure, "failed to load registries")
|
||||
}
|
||||
|
||||
s.registryCacheMu.Lock()
|
||||
for _, r := range registries {
|
||||
s.registryCache[r.Name] = r.ID
|
||||
}
|
||||
s.registryCacheMu.Unlock()
|
||||
return nil
|
||||
}
|
||||
|
||||
// registryCacheLen returns current cache size (used after loading default seed data)
|
||||
func (s *GORMStoreV2) registryCacheLen() int {
|
||||
s.registryCacheMu.RLock()
|
||||
defer s.registryCacheMu.RUnlock()
|
||||
return len(s.registryCache)
|
||||
}
|
||||
|
||||
// seedDefaultRegistries creates default registry entries
|
||||
func (s *GORMStoreV2) seedDefaultRegistries() error {
|
||||
defaultRegistries := []RegistryModel{
|
||||
@@ -162,9 +151,7 @@ func (s *GORMStoreV2) seedDefaultRegistries() error {
|
||||
if err := s.db.Create(®).Error; err != nil {
|
||||
return errors.Wrap(err, errors.ErrCodeStorageFailure, "failed to seed registry: "+reg.Name)
|
||||
}
|
||||
s.registryCacheMu.Lock()
|
||||
s.registryCache[reg.Name] = reg.ID
|
||||
s.registryCacheMu.Unlock()
|
||||
}
|
||||
|
||||
log.Info().Msg("Seeded default registries: npm, pypi, go")
|
||||
@@ -173,13 +160,9 @@ func (s *GORMStoreV2) seedDefaultRegistries() error {
|
||||
|
||||
// getRegistryID returns the registry ID from cache or database
|
||||
func (s *GORMStoreV2) getRegistryID(name string) (int32, error) {
|
||||
// Fast path: read lock for cache hit
|
||||
s.registryCacheMu.RLock()
|
||||
if id, ok := s.registryCache[name]; ok {
|
||||
s.registryCacheMu.RUnlock()
|
||||
return id, nil
|
||||
}
|
||||
s.registryCacheMu.RUnlock()
|
||||
|
||||
// Not in cache, try to load from database
|
||||
var reg RegistryModel
|
||||
@@ -190,15 +173,7 @@ func (s *GORMStoreV2) getRegistryID(name string) (int32, error) {
|
||||
return 0, errors.Wrap(err, errors.ErrCodeStorageFailure, "failed to query registry")
|
||||
}
|
||||
|
||||
// Promote to write lock to populate cache
|
||||
s.registryCacheMu.Lock()
|
||||
// Re-check: another goroutine may have populated between RUnlock and Lock
|
||||
if existing, ok := s.registryCache[name]; ok {
|
||||
s.registryCacheMu.Unlock()
|
||||
return existing, nil
|
||||
}
|
||||
s.registryCache[name] = reg.ID
|
||||
s.registryCacheMu.Unlock()
|
||||
return reg.ID, nil
|
||||
}
|
||||
|
||||
@@ -237,29 +212,10 @@ func (s *GORMStoreV2) SavePackage(ctx context.Context, pkg *metadata.Package) er
|
||||
AuthProvider: pkg.AuthProvider,
|
||||
}
|
||||
|
||||
// Upsert package: first try to update, if no rows affected then create.
|
||||
// IMPORTANT: use map[string]interface{} so that zero values
|
||||
// (e.g. RequiresAuth=false, Size=0) are written explicitly. With Updates(struct)
|
||||
// GORM silently skips zero fields, which leaves stale security-relevant flags.
|
||||
updateFields := map[string]interface{}{
|
||||
"registry_id": registryID,
|
||||
"name": pkg.Name,
|
||||
"version": pkg.Version,
|
||||
"storage_key": pkg.StorageKey,
|
||||
"size": pkg.Size,
|
||||
"checksum_md5": pkg.ChecksumMD5,
|
||||
"checksum_sha256": pkg.ChecksumSHA256,
|
||||
"upstream_url": pkg.UpstreamURL,
|
||||
"cached_at": pkg.CachedAt,
|
||||
"last_accessed": pkg.LastAccessed,
|
||||
"expires_at": pkg.ExpiresAt,
|
||||
"requires_auth": pkg.RequiresAuth,
|
||||
"auth_provider": pkg.AuthProvider,
|
||||
}
|
||||
|
||||
// Upsert package: first try to update, if no rows affected then create
|
||||
result := tx.Model(&PackageModel{}).
|
||||
Where("registry_id = ? AND name = ? AND version = ?", registryID, pkg.Name, pkg.Version).
|
||||
Updates(updateFields)
|
||||
Updates(model)
|
||||
|
||||
if result.Error != nil {
|
||||
return errors.Wrap(result.Error, errors.ErrCodeStorageFailure, "failed to update package")
|
||||
@@ -270,14 +226,6 @@ func (s *GORMStoreV2) SavePackage(ctx context.Context, pkg *metadata.Package) er
|
||||
if err := tx.Create(model).Error; err != nil {
|
||||
return errors.Wrap(err, errors.ErrCodeStorageFailure, "failed to create package")
|
||||
}
|
||||
} else {
|
||||
// On update, fetch the row's primary key for downstream metadata save
|
||||
if err := tx.Model(&PackageModel{}).
|
||||
Select("id").
|
||||
Where("registry_id = ? AND name = ? AND version = ?", registryID, pkg.Name, pkg.Version).
|
||||
First(model).Error; err != nil {
|
||||
return errors.Wrap(err, errors.ErrCodeStorageFailure, "failed to reload package id after update")
|
||||
}
|
||||
}
|
||||
|
||||
// Save metadata if present
|
||||
@@ -467,15 +415,16 @@ func (s *GORMStoreV2) ListPackages(ctx context.Context, opts *metadata.ListOptio
|
||||
|
||||
// Convert to metadata.Package
|
||||
packages := make([]*metadata.Package, len(models))
|
||||
// Snapshot id->name mapping under read lock once to avoid repeated locking
|
||||
s.registryCacheMu.RLock()
|
||||
idToName := make(map[int32]string, len(s.registryCache))
|
||||
for name, id := range s.registryCache {
|
||||
idToName[id] = name
|
||||
}
|
||||
s.registryCacheMu.RUnlock()
|
||||
for i, model := range models {
|
||||
packages[i] = s.modelToPackage(&model, idToName[model.RegistryID])
|
||||
// Get registry name from cache
|
||||
var regName string
|
||||
for name, id := range s.registryCache {
|
||||
if id == model.RegistryID {
|
||||
regName = name
|
||||
break
|
||||
}
|
||||
}
|
||||
packages[i] = s.modelToPackage(&model, regName)
|
||||
}
|
||||
|
||||
return packages, nil
|
||||
@@ -540,9 +489,6 @@ func (s *GORMStoreV2) GetStats(ctx context.Context, registry string) (*metadata.
|
||||
// Filter out metadata entries (npm metadata pages, pypi pages, etc.)
|
||||
query = query.Where("version NOT IN (?)", []string{"list", "latest", "metadata", "page"})
|
||||
|
||||
// Filter out Go module metadata files (.mod, .info) - only count actual packages (.zip)
|
||||
query = query.Where("name NOT LIKE ?", "%.mod").Where("name NOT LIKE ?", "%.info")
|
||||
|
||||
// Filter by registry if specified
|
||||
if registry != "" {
|
||||
registryID, err := s.getRegistryID(registry)
|
||||
|
||||
@@ -73,7 +73,7 @@ func (s *GORMStoreV2TestSuite) TearDownTest() {
|
||||
s.store.db.Exec(fmt.Sprintf("DELETE FROM %s", table))
|
||||
}
|
||||
|
||||
_ = s.store.Close()
|
||||
s.store.Close()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -352,7 +352,7 @@ func (s *GORMStoreV2TestSuite) Test_V2_Count() {
|
||||
CachedAt: time.Now(),
|
||||
LastAccessed: time.Now(),
|
||||
}
|
||||
err = s.store.SavePackage(s.ctx, pkg)
|
||||
err := s.store.SavePackage(s.ctx, pkg)
|
||||
s.NoError(err)
|
||||
}
|
||||
|
||||
@@ -376,9 +376,9 @@ func (s *GORMStoreV2TestSuite) Test_V2_GetStats() {
|
||||
}
|
||||
|
||||
// Update download counts
|
||||
_ = s.store.UpdateDownloadCount(s.ctx, "npm", "pkg1", "1.0.0")
|
||||
_ = s.store.UpdateDownloadCount(s.ctx, "npm", "pkg1", "1.0.0")
|
||||
_ = s.store.UpdateDownloadCount(s.ctx, "npm", "pkg2", "1.0.0")
|
||||
s.store.UpdateDownloadCount(s.ctx, "npm", "pkg1", "1.0.0")
|
||||
s.store.UpdateDownloadCount(s.ctx, "npm", "pkg1", "1.0.0")
|
||||
s.store.UpdateDownloadCount(s.ctx, "npm", "pkg2", "1.0.0")
|
||||
|
||||
// Get stats for all registries
|
||||
statsAll, err := s.store.GetStats(s.ctx, "")
|
||||
@@ -486,7 +486,7 @@ func (s *GORMStoreV2TestSuite) Test_V2_ConcurrentUpdates() {
|
||||
// SQLite: Sequential updates only (write lock prevents concurrent writes)
|
||||
updateCount := 5
|
||||
for i := 0; i < updateCount; i++ {
|
||||
err = s.store.UpdateDownloadCount(s.ctx, "npm", "concurrent-test", "1.0.0")
|
||||
err := s.store.UpdateDownloadCount(s.ctx, "npm", "concurrent-test", "1.0.0")
|
||||
s.NoError(err)
|
||||
}
|
||||
|
||||
|
||||
@@ -1,8 +1,6 @@
|
||||
package gormstore
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"github.com/go-gormigrate/gormigrate/v2"
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
@@ -32,51 +30,13 @@ func GetMigrations() []*gormigrate.Migration {
|
||||
// return tx.Exec("ALTER TABLE packages DROP COLUMN new_field").Error
|
||||
// },
|
||||
// },
|
||||
{
|
||||
// Behavior change: aggregation worker now deletes raw download_events
|
||||
// in the same transaction as the hourly aggregation, using the same
|
||||
// cutoff. This prevents double-counting in multi-replica deployments
|
||||
// where the previous logic kept events for 24h and re-aggregated
|
||||
// them every run. No schema change required; we purge any stale
|
||||
// events older than the current aggregation cutoff so the new
|
||||
// invariant (events ⇔ unaggregated) holds from now on.
|
||||
ID: "202604280001",
|
||||
Migrate: func(tx *gorm.DB) error {
|
||||
// Best-effort cleanup. Safe to run repeatedly.
|
||||
cutoff := time.Now().Add(-5 * time.Minute)
|
||||
if err := tx.Exec(
|
||||
"DELETE FROM download_events WHERE downloaded_at < ?",
|
||||
cutoff,
|
||||
).Error; err != nil {
|
||||
// Table may not exist yet on a fresh install; tolerate.
|
||||
return nil
|
||||
}
|
||||
return nil
|
||||
},
|
||||
Rollback: func(tx *gorm.DB) error {
|
||||
// Behavior change only — nothing to roll back schema-wise.
|
||||
return nil
|
||||
},
|
||||
},
|
||||
{
|
||||
// Add api_keys table for persistent authentication keys. Idempotent:
|
||||
// AutoMigrate is a no-op if the table already exists with the
|
||||
// expected columns.
|
||||
ID: "202604280002",
|
||||
Migrate: func(tx *gorm.DB) error {
|
||||
return tx.AutoMigrate(&APIKeyModel{})
|
||||
},
|
||||
Rollback: func(tx *gorm.DB) error {
|
||||
return tx.Migrator().DropTable("api_keys")
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// migrateToV2 creates the complete V2 schema
|
||||
func migrateToV2(tx *gorm.DB) error {
|
||||
// Get dialect name for database-specific features
|
||||
dialectName := tx.Name()
|
||||
dialectName := tx.Dialector.Name()
|
||||
|
||||
// Step 1: Create all tables using GORM AutoMigrate
|
||||
// This handles cross-database compatibility automatically
|
||||
@@ -99,12 +59,11 @@ func migrateToV2(tx *gorm.DB) error {
|
||||
}
|
||||
|
||||
// Step 3: Create database-specific optimizations
|
||||
switch dialectName {
|
||||
case "postgres":
|
||||
if dialectName == "postgres" {
|
||||
if err := createPostgreSQLOptimizations(tx); err != nil {
|
||||
return err
|
||||
}
|
||||
case "mysql":
|
||||
} else if dialectName == "mysql" {
|
||||
if err := createMySQLOptimizations(tx); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -233,7 +192,6 @@ func createMySQLOptimizations(tx *gorm.DB) error {
|
||||
func rollbackFromV2(tx *gorm.DB) error {
|
||||
// Drop in reverse order to respect foreign keys
|
||||
tables := []string{
|
||||
"api_keys",
|
||||
"audit_log",
|
||||
"download_stats_daily",
|
||||
"download_stats_hourly",
|
||||
@@ -248,13 +206,13 @@ func rollbackFromV2(tx *gorm.DB) error {
|
||||
}
|
||||
|
||||
// Drop PostgreSQL-specific objects
|
||||
if tx.Name() == "postgres" {
|
||||
if tx.Dialector.Name() == "postgres" {
|
||||
tx.Exec("DROP VIEW IF EXISTS v_vulnerable_packages")
|
||||
tx.Exec("DROP FUNCTION IF EXISTS create_next_month_partitions()")
|
||||
}
|
||||
|
||||
// Drop MySQL-specific objects
|
||||
if tx.Name() == "mysql" {
|
||||
if tx.Dialector.Name() == "mysql" {
|
||||
tx.Exec("DROP VIEW IF EXISTS v_vulnerable_packages")
|
||||
}
|
||||
|
||||
|
||||
@@ -18,13 +18,13 @@ type BaseModel struct {
|
||||
// RegistryModel represents package registries (normalized)
|
||||
// This eliminates repetition of "npm", "pypi", "go" across millions of rows
|
||||
type RegistryModel struct {
|
||||
BaseModel
|
||||
ID int32 `gorm:"primaryKey;autoIncrement"`
|
||||
Name string `gorm:"uniqueIndex:idx_registry_name;not null;size:50"` // npm, pypi, go
|
||||
DisplayName string `gorm:"not null;size:100"` // NPM Registry, PyPI, Go Modules
|
||||
UpstreamURL string `gorm:"not null;size:512"` // https://registry.npmjs.org
|
||||
ID int32 `gorm:"primaryKey;autoIncrement"`
|
||||
Enabled bool `gorm:"not null;default:true;index:idx_registry_enabled"`
|
||||
ScanByDefault bool `gorm:"not null;default:true"`
|
||||
BaseModel
|
||||
}
|
||||
|
||||
func (RegistryModel) TableName() string {
|
||||
@@ -33,50 +33,45 @@ func (RegistryModel) TableName() string {
|
||||
|
||||
// PackageModel represents the core package data (optimized)
|
||||
type PackageModel struct {
|
||||
|
||||
// Cache management
|
||||
CachedAt time.Time `gorm:"not null;index:idx_package_cached_at"`
|
||||
LastAccessed time.Time `gorm:"not null;index:idx_package_last_accessed"` // For LRU eviction
|
||||
ExpiresAt *time.Time `gorm:"index:idx_package_expires_at"` // For cache invalidation
|
||||
LastScannedAt *time.Time `gorm:"index:idx_package_last_scanned"`
|
||||
Metadata *PackageMetadataModel `gorm:"foreignKey:PackageID;constraint:OnUpdate:CASCADE,OnDelete:CASCADE"`
|
||||
|
||||
// Relationships
|
||||
Registry RegistryModel `gorm:"foreignKey:RegistryID;constraint:OnUpdate:CASCADE,OnDelete:RESTRICT"`
|
||||
|
||||
BaseModel
|
||||
|
||||
Name string `gorm:"not null;size:255;index:idx_package_name;index:idx_package_registry_name_version,priority:2"`
|
||||
Version string `gorm:"not null;size:100;index:idx_package_registry_name_version,priority:3"`
|
||||
ID int64 `gorm:"primaryKey;autoIncrement"`
|
||||
RegistryID int32 `gorm:"not null;index:idx_package_registry_name_version,priority:1"` // Foreign key to registries
|
||||
Name string `gorm:"not null;size:255;index:idx_package_name;index:idx_package_registry_name_version,priority:2"`
|
||||
Version string `gorm:"not null;size:100;index:idx_package_registry_name_version,priority:3"`
|
||||
|
||||
// Storage information
|
||||
StorageKey string `gorm:"not null;uniqueIndex:idx_package_storage_key;size:512"`
|
||||
Size int64 `gorm:"not null;index:idx_package_size"` // For storage quota queries
|
||||
ChecksumMD5 string `gorm:"size:32;index:idx_package_md5"`
|
||||
ChecksumSHA256 string `gorm:"size:64;index:idx_package_sha256"`
|
||||
UpstreamURL string `gorm:"size:1024"`
|
||||
|
||||
HighestSeverity string `gorm:"size:20;index:idx_package_severity"` // critical, high, medium, low, none
|
||||
AuthProvider string `gorm:"size:50;index:idx_package_auth_provider"` // github, gitlab, custom
|
||||
|
||||
ScanResults []ScanResultModel `gorm:"foreignKey:PackageID;constraint:OnUpdate:CASCADE,OnDelete:CASCADE"`
|
||||
Vulnerabilities []PackageVulnerabilityModel `gorm:"foreignKey:PackageID;constraint:OnUpdate:CASCADE,OnDelete:CASCADE"`
|
||||
ID int64 `gorm:"primaryKey;autoIncrement"`
|
||||
Size int64 `gorm:"not null;index:idx_package_size"` // For storage quota queries
|
||||
AccessCount int64 `gorm:"not null;default:0;index:idx_package_access_count"` // Total access count (denormalized for performance)
|
||||
|
||||
VulnerabilityCount int `gorm:"not null;default:0;index:idx_package_vuln_count"` // Denormalized for fast filtering
|
||||
CriticalCount int `gorm:"not null;default:0"` // Count of critical vulnerabilities
|
||||
HighCount int `gorm:"not null;default:0"` // Count of high vulnerabilities
|
||||
ModerateCount int `gorm:"not null;default:0"` // Count of moderate vulnerabilities
|
||||
LowCount int `gorm:"not null;default:0"` // Count of low vulnerabilities
|
||||
|
||||
RegistryID int32 `gorm:"not null;index:idx_package_registry_name_version,priority:1"` // Foreign key to registries
|
||||
// Cache management
|
||||
CachedAt time.Time `gorm:"not null;index:idx_package_cached_at"`
|
||||
LastAccessed time.Time `gorm:"not null;index:idx_package_last_accessed"` // For LRU eviction
|
||||
ExpiresAt *time.Time `gorm:"index:idx_package_expires_at"` // For cache invalidation
|
||||
AccessCount int64 `gorm:"not null;default:0;index:idx_package_access_count"` // Total access count (denormalized for performance)
|
||||
|
||||
// Security
|
||||
SecurityScanned bool `gorm:"not null;default:false;index:idx_package_security_scanned"`
|
||||
SecurityScanned bool `gorm:"not null;default:false;index:idx_package_security_scanned"`
|
||||
LastScannedAt *time.Time `gorm:"index:idx_package_last_scanned"`
|
||||
VulnerabilityCount int `gorm:"not null;default:0;index:idx_package_vuln_count"` // Denormalized for fast filtering
|
||||
HighestSeverity string `gorm:"size:20;index:idx_package_severity"` // critical, high, medium, low, none
|
||||
CriticalCount int `gorm:"not null;default:0"` // Count of critical vulnerabilities
|
||||
HighCount int `gorm:"not null;default:0"` // Count of high vulnerabilities
|
||||
ModerateCount int `gorm:"not null;default:0"` // Count of moderate vulnerabilities
|
||||
LowCount int `gorm:"not null;default:0"` // Count of low vulnerabilities
|
||||
|
||||
// Authentication
|
||||
RequiresAuth bool `gorm:"not null;default:false;index:idx_package_requires_auth"`
|
||||
RequiresAuth bool `gorm:"not null;default:false;index:idx_package_requires_auth"`
|
||||
AuthProvider string `gorm:"size:50;index:idx_package_auth_provider"` // github, gitlab, custom
|
||||
|
||||
BaseModel
|
||||
|
||||
// Relationships
|
||||
Registry RegistryModel `gorm:"foreignKey:RegistryID;constraint:OnUpdate:CASCADE,OnDelete:RESTRICT"`
|
||||
Metadata *PackageMetadataModel `gorm:"foreignKey:PackageID;constraint:OnUpdate:CASCADE,OnDelete:CASCADE"`
|
||||
ScanResults []ScanResultModel `gorm:"foreignKey:PackageID;constraint:OnUpdate:CASCADE,OnDelete:CASCADE"`
|
||||
Vulnerabilities []PackageVulnerabilityModel `gorm:"foreignKey:PackageID;constraint:OnUpdate:CASCADE,OnDelete:CASCADE"`
|
||||
}
|
||||
|
||||
func (PackageModel) TableName() string {
|
||||
@@ -94,15 +89,15 @@ func (p *PackageModel) BeforeCreate(tx *gorm.DB) error {
|
||||
// PackageMetadataModel stores structured package metadata (1:1 with packages)
|
||||
// Separated from main table to reduce row size and improve query performance
|
||||
type PackageMetadataModel struct {
|
||||
RawMetadata JSONBField `gorm:"type:jsonb"` // Full metadata as JSONB (PostgreSQL) or JSON
|
||||
BaseModel
|
||||
PackageID int64 `gorm:"primaryKey;not null"` // 1:1 relationship
|
||||
Author string `gorm:"size:255;index:idx_metadata_author"`
|
||||
License string `gorm:"size:100;index:idx_metadata_license"`
|
||||
Homepage string `gorm:"size:512"`
|
||||
Repository string `gorm:"size:512"`
|
||||
Description string `gorm:"type:text"`
|
||||
Keywords PostgresArray `gorm:"type:text"` // JSONB array for PostgreSQL, JSON for MySQL/SQLite
|
||||
PackageID int64 `gorm:"primaryKey;not null"` // 1:1 relationship
|
||||
Keywords PostgresArray `gorm:"type:text"` // JSONB array for PostgreSQL, JSON for MySQL/SQLite
|
||||
RawMetadata JSONBField `gorm:"type:jsonb"` // Full metadata as JSONB (PostgreSQL) or JSON
|
||||
BaseModel
|
||||
}
|
||||
|
||||
func (PackageMetadataModel) TableName() string {
|
||||
@@ -111,22 +106,21 @@ func (PackageMetadataModel) TableName() string {
|
||||
|
||||
// ScanResultModel represents security scan results (optimized)
|
||||
type ScanResultModel struct {
|
||||
ScannedAt time.Time `gorm:"not null;index:idx_scan_scanned_at"`
|
||||
Details JSONBField `gorm:"type:jsonb"` // Scanner-specific details
|
||||
ID int64 `gorm:"primaryKey;autoIncrement"`
|
||||
PackageID int64 `gorm:"not null;index:idx_scan_package_scanner,priority:1"` // Foreign key
|
||||
Scanner string `gorm:"not null;size:50;index:idx_scan_scanner;index:idx_scan_package_scanner,priority:2"`
|
||||
ScannedAt time.Time `gorm:"not null;index:idx_scan_scanned_at"`
|
||||
Status string `gorm:"not null;size:20;index:idx_scan_status"` // success, failed, pending
|
||||
VulnCount int `gorm:"not null;default:0;index:idx_scan_vuln_count"`
|
||||
CriticalCount int `gorm:"not null;default:0"`
|
||||
HighCount int `gorm:"not null;default:0"`
|
||||
MediumCount int `gorm:"not null;default:0"`
|
||||
LowCount int `gorm:"not null;default:0"`
|
||||
ScanDuration int `gorm:"not null;default:0"` // milliseconds
|
||||
Details JSONBField `gorm:"type:jsonb"` // Scanner-specific details
|
||||
BaseModel
|
||||
|
||||
Scanner string `gorm:"not null;size:50;index:idx_scan_scanner;index:idx_scan_package_scanner,priority:2"`
|
||||
Status string `gorm:"not null;size:20;index:idx_scan_status"` // success, failed, pending
|
||||
|
||||
Package PackageModel `gorm:"foreignKey:PackageID;constraint:OnUpdate:CASCADE,OnDelete:CASCADE"`
|
||||
ID int64 `gorm:"primaryKey;autoIncrement"`
|
||||
PackageID int64 `gorm:"not null;index:idx_scan_package_scanner,priority:1"` // Foreign key
|
||||
VulnCount int `gorm:"not null;default:0;index:idx_scan_vuln_count"`
|
||||
CriticalCount int `gorm:"not null;default:0"`
|
||||
HighCount int `gorm:"not null;default:0"`
|
||||
MediumCount int `gorm:"not null;default:0"`
|
||||
LowCount int `gorm:"not null;default:0"`
|
||||
ScanDuration int `gorm:"not null;default:0"` // milliseconds
|
||||
Package PackageModel `gorm:"foreignKey:PackageID;constraint:OnUpdate:CASCADE,OnDelete:CASCADE"`
|
||||
}
|
||||
|
||||
func (ScanResultModel) TableName() string {
|
||||
@@ -135,16 +129,16 @@ func (ScanResultModel) TableName() string {
|
||||
|
||||
// VulnerabilityModel represents unique vulnerabilities (normalized)
|
||||
type VulnerabilityModel struct {
|
||||
PublishedAt time.Time `gorm:"not null;index:idx_vuln_published"`
|
||||
BaseModel
|
||||
ID int64 `gorm:"primaryKey;autoIncrement"`
|
||||
CVEID string `gorm:"uniqueIndex:idx_vuln_cve_id;not null;size:50"` // CVE-2021-12345
|
||||
Title string `gorm:"not null;size:512"`
|
||||
Description string `gorm:"type:text"`
|
||||
Severity string `gorm:"not null;size:20;index:idx_vuln_severity"` // critical, high, medium, low
|
||||
FixedVersion string `gorm:"size:100"` // First version where it's fixed
|
||||
References PostgresArray `gorm:"type:text"` // URLs to advisories
|
||||
ID int64 `gorm:"primaryKey;autoIncrement"`
|
||||
CVSS float32 `gorm:"index:idx_vuln_cvss"` // CVSS score for sorting
|
||||
CVSS float32 `gorm:"index:idx_vuln_cvss"` // CVSS score for sorting
|
||||
PublishedAt time.Time `gorm:"not null;index:idx_vuln_published"`
|
||||
FixedVersion string `gorm:"size:100"` // First version where it's fixed
|
||||
References PostgresArray `gorm:"type:text"` // URLs to advisories
|
||||
BaseModel
|
||||
}
|
||||
|
||||
func (VulnerabilityModel) TableName() string {
|
||||
@@ -153,18 +147,17 @@ func (VulnerabilityModel) TableName() string {
|
||||
|
||||
// PackageVulnerabilityModel is a many-to-many relationship between packages and vulnerabilities
|
||||
type PackageVulnerabilityModel struct {
|
||||
DetectedAt time.Time `gorm:"not null;index:idx_pkg_vuln_detected"`
|
||||
BypassID *int64 `gorm:"index:idx_pkg_vuln_bypass_id"` // Reference to bypass if applicable
|
||||
Vulnerability VulnerabilityModel `gorm:"foreignKey:VulnerabilityID;constraint:OnUpdate:CASCADE,OnDelete:CASCADE"`
|
||||
ID int64 `gorm:"primaryKey;autoIncrement"`
|
||||
PackageID int64 `gorm:"not null;index:idx_pkg_vuln_package,priority:1;index:idx_pkg_vuln_composite,priority:1"`
|
||||
VulnerabilityID int64 `gorm:"not null;index:idx_pkg_vuln_vuln,priority:1;index:idx_pkg_vuln_composite,priority:2"`
|
||||
Scanner string `gorm:"not null;size:50;index:idx_pkg_vuln_scanner"`
|
||||
DetectedAt time.Time `gorm:"not null;index:idx_pkg_vuln_detected"`
|
||||
Bypassed bool `gorm:"not null;default:false;index:idx_pkg_vuln_bypassed"`
|
||||
BypassID *int64 `gorm:"index:idx_pkg_vuln_bypass_id"` // Reference to bypass if applicable
|
||||
BaseModel
|
||||
|
||||
Scanner string `gorm:"not null;size:50;index:idx_pkg_vuln_scanner"`
|
||||
|
||||
Package PackageModel `gorm:"foreignKey:PackageID;constraint:OnUpdate:CASCADE,OnDelete:CASCADE"`
|
||||
ID int64 `gorm:"primaryKey;autoIncrement"`
|
||||
PackageID int64 `gorm:"not null;index:idx_pkg_vuln_package,priority:1;index:idx_pkg_vuln_composite,priority:1"`
|
||||
VulnerabilityID int64 `gorm:"not null;index:idx_pkg_vuln_vuln,priority:1;index:idx_pkg_vuln_composite,priority:2"`
|
||||
Bypassed bool `gorm:"not null;default:false;index:idx_pkg_vuln_bypassed"`
|
||||
Package PackageModel `gorm:"foreignKey:PackageID;constraint:OnUpdate:CASCADE,OnDelete:CASCADE"`
|
||||
Vulnerability VulnerabilityModel `gorm:"foreignKey:VulnerabilityID;constraint:OnUpdate:CASCADE,OnDelete:CASCADE"`
|
||||
}
|
||||
|
||||
func (PackageVulnerabilityModel) TableName() string {
|
||||
@@ -173,22 +166,22 @@ func (PackageVulnerabilityModel) TableName() string {
|
||||
|
||||
// CVEBypassModel represents CVE bypass rules (improved)
|
||||
type CVEBypassModel struct {
|
||||
ExpiresAt time.Time `gorm:"not null;index:idx_bypass_expires_at"`
|
||||
LastUsedAt *time.Time `gorm:"index:idx_bypass_last_used"`
|
||||
ID int64 `gorm:"primaryKey;autoIncrement"`
|
||||
Type string `gorm:"not null;size:20;index:idx_bypass_type"` // cve, package, registry
|
||||
Target string `gorm:"not null;size:512;index:idx_bypass_target"` // CVE-ID, package name, etc.
|
||||
Reason string `gorm:"not null;type:text"`
|
||||
CreatedBy string `gorm:"not null;size:255;index:idx_bypass_created_by"`
|
||||
ExpiresAt time.Time `gorm:"not null;index:idx_bypass_expires_at"`
|
||||
NotifyOnExpiry bool `gorm:"not null;default:false"`
|
||||
Active bool `gorm:"not null;default:true;index:idx_bypass_active"`
|
||||
UsageCount int64 `gorm:"not null;default:0"` // How many times this bypass has been used
|
||||
LastUsedAt *time.Time `gorm:"index:idx_bypass_last_used"`
|
||||
|
||||
// Scope limiting (optional)
|
||||
RegistryID *int32 `gorm:"index:idx_bypass_registry"` // NULL = all registries
|
||||
PackageID *int64 `gorm:"index:idx_bypass_package"` // NULL = all packages
|
||||
|
||||
BaseModel
|
||||
Type string `gorm:"not null;size:20;index:idx_bypass_type"` // cve, package, registry
|
||||
Target string `gorm:"not null;size:512;index:idx_bypass_target"` // CVE-ID, package name, etc.
|
||||
Reason string `gorm:"not null;type:text"`
|
||||
CreatedBy string `gorm:"not null;size:255;index:idx_bypass_created_by"`
|
||||
ID int64 `gorm:"primaryKey;autoIncrement"`
|
||||
UsageCount int64 `gorm:"not null;default:0"` // How many times this bypass has been used
|
||||
NotifyOnExpiry bool `gorm:"not null;default:false"`
|
||||
Active bool `gorm:"not null;default:true;index:idx_bypass_active"`
|
||||
}
|
||||
|
||||
func (CVEBypassModel) TableName() string {
|
||||
@@ -198,17 +191,17 @@ func (CVEBypassModel) TableName() string {
|
||||
// DownloadEventModel represents raw download events (partitioned by month)
|
||||
// This table should use PostgreSQL partitioning or time-series DB features
|
||||
type DownloadEventModel struct {
|
||||
DownloadedAt time.Time `gorm:"not null;index:idx_download_time;index:idx_download_package,priority:2"` // Partition key
|
||||
UserAgent string `gorm:"size:512"` // For analytics
|
||||
IPAddress string `gorm:"size:45;index:idx_download_ip"` // IPv6 support
|
||||
Username string `gorm:"size:255;index:idx_download_user"`
|
||||
ID int64 `gorm:"primaryKey;autoIncrement"`
|
||||
PackageID int64 `gorm:"not null;index:idx_download_package,priority:1"`
|
||||
RegistryID int32 `gorm:"not null;index:idx_download_registry"`
|
||||
DownloadedAt time.Time `gorm:"not null;index:idx_download_time;index:idx_download_package,priority:2"` // Partition key
|
||||
UserAgent string `gorm:"size:512"` // For analytics
|
||||
IPAddress string `gorm:"size:45;index:idx_download_ip"` // IPv6 support
|
||||
Authenticated bool `gorm:"not null;default:false"`
|
||||
Username string `gorm:"size:255;index:idx_download_user"`
|
||||
|
||||
// No BaseModel - this is append-only, no updates/deletes on individual rows
|
||||
// Partitioned tables handle cleanup via DROP PARTITION
|
||||
ID int64 `gorm:"primaryKey;autoIncrement"`
|
||||
PackageID int64 `gorm:"not null;index:idx_download_package,priority:1"`
|
||||
RegistryID int32 `gorm:"not null;index:idx_download_registry"`
|
||||
Authenticated bool `gorm:"not null;default:false"`
|
||||
}
|
||||
|
||||
func (DownloadEventModel) TableName() string {
|
||||
@@ -217,16 +210,15 @@ func (DownloadEventModel) TableName() string {
|
||||
|
||||
// DownloadStatsHourlyModel represents pre-aggregated hourly statistics (partitioned)
|
||||
type DownloadStatsHourlyModel struct {
|
||||
TimeBucket time.Time `gorm:"not null;index:idx_stats_hourly_composite,priority:2"` // Truncated to hour
|
||||
PackageID *int64 `gorm:"index:idx_stats_hourly_package"` // NULL = all packages in registry
|
||||
ID int64 `gorm:"primaryKey;autoIncrement"`
|
||||
RegistryID int32 `gorm:"not null;index:idx_stats_hourly_composite,priority:1"`
|
||||
PackageID *int64 `gorm:"index:idx_stats_hourly_package"` // NULL = all packages in registry
|
||||
TimeBucket time.Time `gorm:"not null;index:idx_stats_hourly_composite,priority:2"` // Truncated to hour
|
||||
DownloadCount int64 `gorm:"not null;default:0"`
|
||||
UniqueIPs int64 `gorm:"not null;default:0"` // Unique downloaders
|
||||
AuthDownloads int64 `gorm:"not null;default:0"` // Authenticated downloads
|
||||
|
||||
BaseModel
|
||||
ID int64 `gorm:"primaryKey;autoIncrement"`
|
||||
DownloadCount int64 `gorm:"not null;default:0"`
|
||||
UniqueIPs int64 `gorm:"not null;default:0"` // Unique downloaders
|
||||
AuthDownloads int64 `gorm:"not null;default:0"` // Authenticated downloads
|
||||
|
||||
RegistryID int32 `gorm:"not null;index:idx_stats_hourly_composite,priority:1"`
|
||||
}
|
||||
|
||||
func (DownloadStatsHourlyModel) TableName() string {
|
||||
@@ -235,16 +227,16 @@ func (DownloadStatsHourlyModel) TableName() string {
|
||||
|
||||
// DownloadStatsDailyModel represents pre-aggregated daily statistics
|
||||
type DownloadStatsDailyModel struct {
|
||||
TimeBucket time.Time `gorm:"not null;index:idx_stats_daily_composite,priority:2"` // Truncated to day
|
||||
ID int64 `gorm:"primaryKey;autoIncrement"`
|
||||
RegistryID int32 `gorm:"not null;index:idx_stats_daily_composite,priority:1"`
|
||||
PackageID *int64 `gorm:"index:idx_stats_daily_package"` // NULL = all packages in registry
|
||||
TopUserAgents JSONBField `gorm:"type:jsonb"` // Top 10 user agents
|
||||
TimeBucket time.Time `gorm:"not null;index:idx_stats_daily_composite,priority:2"` // Truncated to day
|
||||
DownloadCount int64 `gorm:"not null;default:0"`
|
||||
UniqueIPs int64 `gorm:"not null;default:0"`
|
||||
AuthDownloads int64 `gorm:"not null;default:0"`
|
||||
TopUserAgents JSONBField `gorm:"type:jsonb"` // Top 10 user agents
|
||||
|
||||
BaseModel
|
||||
ID int64 `gorm:"primaryKey;autoIncrement"`
|
||||
DownloadCount int64 `gorm:"not null;default:0"`
|
||||
UniqueIPs int64 `gorm:"not null;default:0"`
|
||||
AuthDownloads int64 `gorm:"not null;default:0"`
|
||||
RegistryID int32 `gorm:"not null;index:idx_stats_daily_composite,priority:1"`
|
||||
}
|
||||
|
||||
func (DownloadStatsDailyModel) TableName() string {
|
||||
@@ -253,43 +245,23 @@ func (DownloadStatsDailyModel) TableName() string {
|
||||
|
||||
// AuditLogModel tracks all important changes (optional, for compliance)
|
||||
type AuditLogModel struct {
|
||||
Timestamp time.Time `gorm:"not null;index:idx_audit_timestamp"`
|
||||
Changes JSONBField `gorm:"type:jsonb"` // Before/after values
|
||||
ID int64 `gorm:"primaryKey;autoIncrement"`
|
||||
EntityType string `gorm:"not null;size:50;index:idx_audit_entity_type"` // package, bypass, registry
|
||||
Action string `gorm:"not null;size:20;index:idx_audit_action"` // create, update, delete
|
||||
EntityID int64 `gorm:"not null;index:idx_audit_entity_id"`
|
||||
Action string `gorm:"not null;size:20;index:idx_audit_action"` // create, update, delete
|
||||
Username string `gorm:"not null;size:255;index:idx_audit_username"`
|
||||
Timestamp time.Time `gorm:"not null;index:idx_audit_timestamp"`
|
||||
Changes JSONBField `gorm:"type:jsonb"` // Before/after values
|
||||
IPAddress string `gorm:"size:45"`
|
||||
UserAgent string `gorm:"size:512"`
|
||||
|
||||
// No BaseModel - append-only audit log
|
||||
ID int64 `gorm:"primaryKey;autoIncrement"`
|
||||
EntityID int64 `gorm:"not null;index:idx_audit_entity_id"`
|
||||
}
|
||||
|
||||
func (AuditLogModel) TableName() string {
|
||||
return "audit_log"
|
||||
}
|
||||
|
||||
// APIKeyModel represents a persisted authentication API key.
|
||||
// Index on (id, project, revoked) is provided via composite index
|
||||
// idx_api_key_lookup so the auth manager can quickly enumerate
|
||||
// non-revoked keys per project at startup.
|
||||
type APIKeyModel struct {
|
||||
CreatedAt time.Time `gorm:"not null"`
|
||||
ExpiresAt *time.Time `gorm:"index:idx_api_key_expires"`
|
||||
LastUsedAt *time.Time `gorm:"index:idx_api_key_last_used"`
|
||||
ID string `gorm:"primaryKey;size:64;index:idx_api_key_lookup,priority:1"`
|
||||
KeyHash string `gorm:"not null;size:255"` // bcrypt hash
|
||||
Project string `gorm:"not null;size:255;index:idx_api_key_project;index:idx_api_key_lookup,priority:2"`
|
||||
Role string `gorm:"not null;size:32;index:idx_api_key_role"` // read_only, read_write, admin
|
||||
Permissions string `gorm:"type:text"` // JSON-encoded list (optional)
|
||||
Revoked bool `gorm:"not null;default:false;index:idx_api_key_revoked;index:idx_api_key_lookup,priority:3"`
|
||||
}
|
||||
|
||||
func (APIKeyModel) TableName() string {
|
||||
return "api_keys"
|
||||
}
|
||||
|
||||
// JSONBField is a custom type for JSONB (PostgreSQL) / JSON (MySQL/SQLite)
|
||||
type JSONBField map[string]interface{}
|
||||
|
||||
@@ -352,6 +324,5 @@ func GetAllModels() []interface{} {
|
||||
&DownloadStatsHourlyModel{},
|
||||
&DownloadStatsDailyModel{},
|
||||
&AuditLogModel{},
|
||||
&APIKeyModel{},
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2,20 +2,12 @@ package gormstore
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"regexp"
|
||||
"time"
|
||||
|
||||
"github.com/rs/zerolog/log"
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
// Whitelist regexes for partition table names — defense in depth before
|
||||
// we interpolate them into raw DDL.
|
||||
var (
|
||||
downloadEventPartitionRe = regexp.MustCompile(`^download_events_\d{4}_\d{2}$`)
|
||||
auditLogPartitionRe = regexp.MustCompile(`^audit_log_\d{4}_\d{2}$`)
|
||||
)
|
||||
|
||||
// PartitionManager handles automatic partition creation and cleanup for PostgreSQL
|
||||
type PartitionManager struct {
|
||||
db *gorm.DB
|
||||
@@ -29,7 +21,7 @@ func NewPartitionManager(db *gorm.DB) *PartitionManager {
|
||||
// EnsurePartitions ensures required partitions exist for current and future months
|
||||
func (pm *PartitionManager) EnsurePartitions() error {
|
||||
// Check if we're using PostgreSQL
|
||||
if pm.db.Name() != "postgres" {
|
||||
if pm.db.Dialector.Name() != "postgres" {
|
||||
log.Debug().Msg("Partitioning only supported on PostgreSQL, skipping")
|
||||
return nil
|
||||
}
|
||||
@@ -294,7 +286,7 @@ func (pm *PartitionManager) createPartitionFunction() error {
|
||||
|
||||
// CleanupOldPartitions drops partitions older than the retention period
|
||||
func (pm *PartitionManager) CleanupOldPartitions(retentionMonths int) error {
|
||||
if pm.db.Name() != "postgres" {
|
||||
if pm.db.Dialector.Name() != "postgres" {
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -308,45 +300,39 @@ func (pm *PartitionManager) CleanupOldPartitions(retentionMonths int) error {
|
||||
|
||||
// Find and drop old download_events partitions
|
||||
var downloadPartitions []string
|
||||
if err := pm.db.Raw(`
|
||||
err := pm.db.Raw(`
|
||||
SELECT tablename FROM pg_tables
|
||||
WHERE tablename LIKE 'download_events_%'
|
||||
AND tablename < 'download_events_' || ?
|
||||
`, cutoffPartition).Scan(&downloadPartitions).Error; err != nil {
|
||||
`, cutoffPartition).Scan(&downloadPartitions).Error
|
||||
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, partition := range downloadPartitions {
|
||||
// Defense in depth: even though tablename came from pg_tables, verify it
|
||||
// matches the expected partition naming scheme before interpolating into DDL.
|
||||
if !downloadEventPartitionRe.MatchString(partition) {
|
||||
log.Warn().Str("partition", partition).Msg("Skipping partition with unexpected name")
|
||||
continue
|
||||
}
|
||||
log.Info().Str("partition", partition).Msg("Dropping old partition")
|
||||
if dropErr := pm.db.Exec(fmt.Sprintf("DROP TABLE IF EXISTS %s", partition)).Error; dropErr != nil {
|
||||
log.Error().Err(dropErr).Str("partition", partition).Msg("Failed to drop partition")
|
||||
if err := pm.db.Exec(fmt.Sprintf("DROP TABLE IF EXISTS %s", partition)).Error; err != nil {
|
||||
log.Error().Err(err).Str("partition", partition).Msg("Failed to drop partition")
|
||||
}
|
||||
}
|
||||
|
||||
// Find and drop old audit_log partitions
|
||||
var auditPartitions []string
|
||||
if err := pm.db.Raw(`
|
||||
err = pm.db.Raw(`
|
||||
SELECT tablename FROM pg_tables
|
||||
WHERE tablename LIKE 'audit_log_%'
|
||||
AND tablename < 'audit_log_' || ?
|
||||
`, cutoffPartition).Scan(&auditPartitions).Error; err != nil {
|
||||
`, cutoffPartition).Scan(&auditPartitions).Error
|
||||
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, partition := range auditPartitions {
|
||||
if !auditLogPartitionRe.MatchString(partition) {
|
||||
log.Warn().Str("partition", partition).Msg("Skipping audit partition with unexpected name")
|
||||
continue
|
||||
}
|
||||
log.Info().Str("partition", partition).Msg("Dropping old audit partition")
|
||||
if dropErr := pm.db.Exec(fmt.Sprintf("DROP TABLE IF EXISTS %s", partition)).Error; dropErr != nil {
|
||||
log.Error().Err(dropErr).Str("partition", partition).Msg("Failed to drop audit partition")
|
||||
if err := pm.db.Exec(fmt.Sprintf("DROP TABLE IF EXISTS %s", partition)).Error; err != nil {
|
||||
log.Error().Err(err).Str("partition", partition).Msg("Failed to drop audit partition")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -355,7 +341,7 @@ func (pm *PartitionManager) CleanupOldPartitions(retentionMonths int) error {
|
||||
|
||||
// GetPartitionInfo returns information about current partitions
|
||||
func (pm *PartitionManager) GetPartitionInfo() (map[string]interface{}, error) {
|
||||
if pm.db.Name() != "postgres" {
|
||||
if pm.db.Dialector.Name() != "postgres" {
|
||||
return map[string]interface{}{"status": "not_applicable"}, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -1,19 +1,11 @@
|
||||
// Package metadata defines the Store interface and shared model types for
|
||||
// package metadata persistence backends.
|
||||
package metadata
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// ErrNotImplemented signals that an optional MetadataStore method is not
|
||||
// supported by the current backend (e.g. a backend may not persist API keys).
|
||||
// Callers should treat it as a non-fatal capability check.
|
||||
var ErrNotImplemented = errors.New("metadata: operation not implemented by this backend")
|
||||
|
||||
// Store is an alias for MetadataStore for convenience
|
||||
type Store = MetadataStore
|
||||
|
||||
@@ -70,48 +62,10 @@ type MetadataStore interface {
|
||||
// AggregateDownloadData aggregates raw download events and cleans up old data
|
||||
AggregateDownloadData(ctx context.Context) error
|
||||
|
||||
// SaveAPIKey persists (insert or update) an API key record. Backends that
|
||||
// do not implement persistent API keys MUST return ErrNotImplemented.
|
||||
SaveAPIKey(ctx context.Context, key *APIKey) error
|
||||
|
||||
// GetAPIKey retrieves a single API key by ID. Returns ErrNotImplemented if
|
||||
// the backend does not persist API keys. Returns a not-found error wrapped
|
||||
// from the backend if the key does not exist.
|
||||
GetAPIKey(ctx context.Context, id string) (*APIKey, error)
|
||||
|
||||
// ListAPIKeys returns all API keys (including revoked) so callers can
|
||||
// decide whether to filter. Returns ErrNotImplemented for backends that
|
||||
// do not persist API keys.
|
||||
ListAPIKeys(ctx context.Context) ([]*APIKey, error)
|
||||
|
||||
// DeleteAPIKey removes an API key by ID. Returns ErrNotImplemented for
|
||||
// backends that do not persist API keys.
|
||||
DeleteAPIKey(ctx context.Context, id string) error
|
||||
|
||||
// UpdateAPIKeyLastUsed updates the last-used timestamp without rewriting
|
||||
// the rest of the row. Implementations should make this cheap; auth.Manager
|
||||
// may invoke it asynchronously on every successful validation.
|
||||
UpdateAPIKeyLastUsed(ctx context.Context, id string, t time.Time) error
|
||||
|
||||
// Close closes the metadata store
|
||||
Close() error
|
||||
}
|
||||
|
||||
// APIKey is the storage-layer representation of an authentication API key.
|
||||
// It is intentionally separate from auth.APIKey to avoid an import cycle:
|
||||
// the auth package depends on metadata, never the other way around.
|
||||
type APIKey struct {
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
ExpiresAt *time.Time `json:"expires_at,omitempty"`
|
||||
LastUsedAt *time.Time `json:"last_used_at,omitempty"`
|
||||
ID string `json:"id"`
|
||||
KeyHash string `json:"key_hash"`
|
||||
Project string `json:"project"`
|
||||
Role string `json:"role"`
|
||||
Permissions string `json:"permissions,omitempty"` // JSON-encoded list (optional)
|
||||
Revoked bool `json:"revoked"`
|
||||
}
|
||||
|
||||
// Package represents package metadata
|
||||
type Package struct {
|
||||
CachedAt time.Time `json:"cached_at"`
|
||||
|
||||
@@ -1,5 +1,3 @@
|
||||
// Package metrics exposes Prometheus instrumentation for cache, storage,
|
||||
// scanner, and HTTP request flows.
|
||||
package metrics
|
||||
|
||||
import (
|
||||
|
||||
@@ -1,5 +1,3 @@
|
||||
// Package network provides resilient HTTP client primitives with retry,
|
||||
// timeout, and circuit-breaker support for upstream registry calls.
|
||||
package network
|
||||
|
||||
import (
|
||||
@@ -237,7 +235,7 @@ func (c *Client) do(ctx context.Context, req *http.Request) (*http.Response, err
|
||||
|
||||
// Check if response is retryable
|
||||
if c.isRetryable(resp.StatusCode) {
|
||||
_ = resp.Body.Close() // #nosec G104 -- Cleanup, error not critical
|
||||
resp.Body.Close() // #nosec G104 -- Cleanup, error not critical
|
||||
lastErr = fmt.Errorf("received retryable status code: %d", resp.StatusCode)
|
||||
if c.circuitBreaker != nil {
|
||||
c.circuitBreaker.RecordFailure()
|
||||
|
||||
@@ -1,5 +1,3 @@
|
||||
// Package prewarming runs background workers that pre-fetch hot packages
|
||||
// to reduce first-request latency.
|
||||
package prewarming
|
||||
|
||||
import (
|
||||
@@ -204,7 +202,7 @@ func (w *Worker) prewarmPackage(ctx context.Context, pkg PackageInfo, workerID i
|
||||
Msg("Failed to fetch package for pre-warming")
|
||||
return
|
||||
}
|
||||
defer func() { _ = body.Close() }() // #nosec G104 -- Cleanup, error not critical
|
||||
defer body.Close() // #nosec G104 -- Cleanup, error not critical
|
||||
|
||||
if statusCode != 200 {
|
||||
log.Warn().
|
||||
|
||||
@@ -1,5 +1,3 @@
|
||||
// Package goproxy implements the HTTP handler that proxies Go module
|
||||
// requests through the GoHoarder cache.
|
||||
package goproxy
|
||||
|
||||
import (
|
||||
@@ -127,7 +125,7 @@ func (h *Handler) handleList(ctx context.Context, w http.ResponseWriter, r *http
|
||||
return nil, "", err
|
||||
}
|
||||
if statusCode != http.StatusOK {
|
||||
_ = body.Close() // #nosec G104 -- Cleanup, error not critical
|
||||
body.Close() // #nosec G104 -- Cleanup, error not critical
|
||||
return nil, "", fmt.Errorf("upstream returned status %d", statusCode)
|
||||
}
|
||||
return body, url, nil
|
||||
@@ -138,7 +136,7 @@ func (h *Handler) handleList(ctx context.Context, w http.ResponseWriter, r *http
|
||||
http.Error(w, "Failed to fetch version list", http.StatusBadGateway)
|
||||
return
|
||||
}
|
||||
defer func() { _ = entry.Data.Close() }() // #nosec G104 -- Cleanup, error not critical
|
||||
defer entry.Data.Close() // #nosec G104 -- Cleanup, error not critical
|
||||
|
||||
w.Header().Set("Content-Type", "text/plain; charset=UTF-8")
|
||||
_, _ = io.Copy(w, entry.Data) // #nosec G104 -- HTTP response write
|
||||
@@ -167,7 +165,7 @@ func (h *Handler) handleInfo(ctx context.Context, w http.ResponseWriter, r *http
|
||||
return nil, "", err
|
||||
}
|
||||
if statusCode != http.StatusOK {
|
||||
_ = body.Close() // #nosec G104 -- Cleanup, error not critical
|
||||
body.Close() // #nosec G104 -- Cleanup, error not critical
|
||||
return nil, "", fmt.Errorf("upstream returned status %d", statusCode)
|
||||
}
|
||||
return body, url, nil
|
||||
@@ -178,7 +176,7 @@ func (h *Handler) handleInfo(ctx context.Context, w http.ResponseWriter, r *http
|
||||
http.Error(w, "Failed to fetch version info", http.StatusBadGateway)
|
||||
return
|
||||
}
|
||||
defer func() { _ = entry.Data.Close() }() // #nosec G104 -- Cleanup, error not critical
|
||||
defer entry.Data.Close() // #nosec G104 -- Cleanup, error not critical
|
||||
|
||||
w.Header().Set("Content-Type", "application/json; charset=UTF-8")
|
||||
_, _ = io.Copy(w, entry.Data) // #nosec G104 -- HTTP response write
|
||||
@@ -207,7 +205,7 @@ func (h *Handler) handleMod(ctx context.Context, w http.ResponseWriter, r *http.
|
||||
return nil, "", err
|
||||
}
|
||||
if statusCode != http.StatusOK {
|
||||
_ = body.Close() // #nosec G104 -- Cleanup, error not critical
|
||||
body.Close() // #nosec G104 -- Cleanup, error not critical
|
||||
return nil, "", fmt.Errorf("upstream returned status %d", statusCode)
|
||||
}
|
||||
return body, url, nil
|
||||
@@ -218,7 +216,7 @@ func (h *Handler) handleMod(ctx context.Context, w http.ResponseWriter, r *http.
|
||||
http.Error(w, "Failed to fetch go.mod", http.StatusBadGateway)
|
||||
return
|
||||
}
|
||||
defer func() { _ = entry.Data.Close() }() // #nosec G104 -- Cleanup, error not critical
|
||||
defer entry.Data.Close() // #nosec G104 -- Cleanup, error not critical
|
||||
|
||||
w.Header().Set("Content-Type", "text/plain; charset=UTF-8")
|
||||
_, _ = io.Copy(w, entry.Data) // #nosec G104 -- HTTP response write
|
||||
@@ -261,7 +259,7 @@ func (h *Handler) handleZip(ctx context.Context, w http.ResponseWriter, r *http.
|
||||
// If upstream failed with 404 or 403, try git fallback (private modules)
|
||||
if statusCode == http.StatusNotFound || statusCode == http.StatusForbidden {
|
||||
if body != nil {
|
||||
_ = body.Close() // #nosec G104 -- Cleanup, error not critical
|
||||
body.Close() // #nosec G104 -- Cleanup, error not critical
|
||||
}
|
||||
|
||||
log.Debug().
|
||||
@@ -275,7 +273,7 @@ func (h *Handler) handleZip(ctx context.Context, w http.ResponseWriter, r *http.
|
||||
|
||||
// Other errors
|
||||
if body != nil {
|
||||
_ = body.Close() // #nosec G104 -- Cleanup, error not critical
|
||||
body.Close() // #nosec G104 -- Cleanup, error not critical
|
||||
}
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
@@ -296,7 +294,7 @@ func (h *Handler) handleZip(ctx context.Context, w http.ResponseWriter, r *http.
|
||||
http.Error(w, "Failed to fetch module zip", http.StatusBadGateway)
|
||||
return
|
||||
}
|
||||
defer func() { _ = entry.Data.Close() }() // #nosec G104 -- Cleanup, error not critical
|
||||
defer entry.Data.Close() // #nosec G104 -- Cleanup, error not critical
|
||||
|
||||
// CRITICAL SECURITY CHECK: If module requires auth, validate credentials
|
||||
if entry.Package != nil && entry.Package.RequiresAuth {
|
||||
@@ -374,7 +372,7 @@ func (h *Handler) handleLatest(ctx context.Context, w http.ResponseWriter, r *ht
|
||||
return nil, "", err
|
||||
}
|
||||
if statusCode != http.StatusOK {
|
||||
_ = body.Close() // #nosec G104 -- Cleanup, error not critical
|
||||
body.Close() // #nosec G104 -- Cleanup, error not critical
|
||||
return nil, "", fmt.Errorf("upstream returned status %d", statusCode)
|
||||
}
|
||||
return body, url, nil
|
||||
@@ -385,7 +383,7 @@ func (h *Handler) handleLatest(ctx context.Context, w http.ResponseWriter, r *ht
|
||||
http.Error(w, "Failed to fetch latest version", http.StatusBadGateway)
|
||||
return
|
||||
}
|
||||
defer func() { _ = entry.Data.Close() }() // #nosec G104 -- Cleanup, error not critical
|
||||
defer entry.Data.Close() // #nosec G104 -- Cleanup, error not critical
|
||||
|
||||
w.Header().Set("Content-Type", "application/json; charset=UTF-8")
|
||||
_, _ = io.Copy(w, entry.Data) // #nosec G104 -- HTTP response write
|
||||
@@ -407,7 +405,7 @@ func (h *Handler) handleSumDB(ctx context.Context, w http.ResponseWriter, r *htt
|
||||
http.Error(w, "Failed to fetch from sumdb", http.StatusBadGateway)
|
||||
return
|
||||
}
|
||||
defer func() { _ = body.Close() }() // #nosec G104 -- Cleanup, error not critical
|
||||
defer body.Close() // #nosec G104 -- Cleanup, error not critical
|
||||
|
||||
if statusCode != http.StatusOK {
|
||||
log.Error().Int("status", statusCode).Str("url", url).Msg("Sumdb returned non-OK status")
|
||||
@@ -464,8 +462,8 @@ func (h *Handler) fetchModuleFromGit(ctx context.Context, modulePath, version, c
|
||||
defer h.gitFetcher.Cleanup(srcPath)
|
||||
|
||||
// 2. Validate module
|
||||
if validateErr := h.moduleBuilder.ValidateModule(ctx, srcPath, modulePath); validateErr != nil {
|
||||
return nil, "", fmt.Errorf("module validation failed: %w", validateErr)
|
||||
if err := h.moduleBuilder.ValidateModule(ctx, srcPath, modulePath); err != nil {
|
||||
return nil, "", fmt.Errorf("module validation failed: %w", err)
|
||||
}
|
||||
|
||||
// 3. Build module zip
|
||||
|
||||
+23
-51
@@ -1,5 +1,3 @@
|
||||
// Package npm implements the HTTP handler that proxies npm registry
|
||||
// requests through the GoHoarder cache.
|
||||
package npm
|
||||
|
||||
import (
|
||||
@@ -81,12 +79,12 @@ func (h *Handler) handleMetadata(ctx context.Context, w http.ResponseWriter, r *
|
||||
packageName := extractPackageName(path)
|
||||
|
||||
entry, err := h.cache.Get(ctx, "npm", packageName, "metadata", func(ctx context.Context) (io.ReadCloser, string, error) {
|
||||
body, statusCode, fetchErr := h.client.Get(ctx, url, nil)
|
||||
if fetchErr != nil {
|
||||
return nil, "", fetchErr
|
||||
body, statusCode, err := h.client.Get(ctx, url, nil)
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
if statusCode != http.StatusOK {
|
||||
_ = body.Close()
|
||||
body.Close() // #nosec G104 -- Cleanup, error not critical
|
||||
return nil, "", fmt.Errorf("upstream returned status %d", statusCode)
|
||||
}
|
||||
return body, url, nil
|
||||
@@ -97,24 +95,20 @@ func (h *Handler) handleMetadata(ctx context.Context, w http.ResponseWriter, r *
|
||||
http.Error(w, "Failed to fetch package metadata", http.StatusBadGateway)
|
||||
return
|
||||
}
|
||||
defer func() {
|
||||
if cerr := entry.Data.Close(); cerr != nil {
|
||||
log.Warn().Err(cerr).Msg("Failed to close NPM metadata body")
|
||||
}
|
||||
}()
|
||||
defer entry.Data.Close() // #nosec G104 -- Cleanup, error not critical
|
||||
|
||||
// Read metadata into memory for URL rewriting
|
||||
var buf bytes.Buffer
|
||||
if _, copyErr := io.Copy(&buf, entry.Data); copyErr != nil {
|
||||
log.Error().Err(copyErr).Msg("Failed to read metadata")
|
||||
if _, err := io.Copy(&buf, entry.Data); err != nil {
|
||||
log.Error().Err(err).Msg("Failed to read metadata")
|
||||
http.Error(w, "Failed to read metadata", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
// Parse JSON metadata
|
||||
var metadata map[string]interface{}
|
||||
if jsonErr := json.Unmarshal(buf.Bytes(), &metadata); jsonErr != nil {
|
||||
log.Error().Err(jsonErr).Msg("Failed to parse metadata JSON")
|
||||
if err := json.Unmarshal(buf.Bytes(), &metadata); err != nil {
|
||||
log.Error().Err(err).Msg("Failed to parse metadata JSON")
|
||||
http.Error(w, "Failed to parse metadata", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
@@ -144,10 +138,8 @@ func (h *Handler) handleTarball(ctx context.Context, w http.ResponseWriter, r *h
|
||||
credHash := h.credHasher.Hash(credentials)
|
||||
|
||||
// Construct proper upstream URL with /-/ format
|
||||
// Format: https://registry.npmjs.org/<package>/-/<filename>
|
||||
// For unscoped: filename is "<package>-<version>.tgz"
|
||||
// For scoped @scope/pkg: filename is "<pkg>-<version>.tgz" (no scope, no leading @)
|
||||
tarballFilename := tarballPrefix(packageName) + version + ".tgz"
|
||||
// Format: https://registry.npmjs.org/package/-/package-version.tgz
|
||||
tarballFilename := strings.ReplaceAll(packageName, "/", "-") + "-" + version + ".tgz"
|
||||
url := fmt.Sprintf("%s/%s/-/%s", h.upstream, packageName, tarballFilename)
|
||||
|
||||
log.Debug().
|
||||
@@ -167,12 +159,12 @@ func (h *Handler) handleTarball(ctx context.Context, w http.ResponseWriter, r *h
|
||||
headers["Authorization"] = credentials
|
||||
}
|
||||
|
||||
body, statusCode, fetchErr := h.client.Get(ctx, url, headers)
|
||||
if fetchErr != nil {
|
||||
return nil, "", fetchErr
|
||||
body, statusCode, err := h.client.Get(ctx, url, headers)
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
if statusCode != http.StatusOK {
|
||||
_ = body.Close()
|
||||
body.Close() // #nosec G104 -- Cleanup, error not critical
|
||||
return nil, "", fmt.Errorf("upstream returned status %d", statusCode)
|
||||
}
|
||||
return body, url, nil
|
||||
@@ -191,11 +183,7 @@ func (h *Handler) handleTarball(ctx context.Context, w http.ResponseWriter, r *h
|
||||
http.Error(w, "Failed to fetch package tarball", http.StatusBadGateway)
|
||||
return
|
||||
}
|
||||
defer func() {
|
||||
if cerr := entry.Data.Close(); cerr != nil {
|
||||
log.Warn().Err(cerr).Msg("Failed to close NPM tarball body")
|
||||
}
|
||||
}()
|
||||
defer entry.Data.Close() // #nosec G104 -- Cleanup, error not critical
|
||||
|
||||
// CRITICAL SECURITY CHECK: If package requires auth, validate credentials
|
||||
if entry.Package != nil && entry.Package.RequiresAuth {
|
||||
@@ -263,11 +251,7 @@ func (h *Handler) handleSpecial(ctx context.Context, w http.ResponseWriter, r *h
|
||||
http.Error(w, "Failed to fetch from upstream", http.StatusBadGateway)
|
||||
return
|
||||
}
|
||||
defer func() {
|
||||
if cerr := body.Close(); cerr != nil {
|
||||
log.Warn().Err(cerr).Msg("Failed to close NPM special endpoint body")
|
||||
}
|
||||
}()
|
||||
defer body.Close() // #nosec G104 -- Cleanup, error not critical
|
||||
|
||||
w.WriteHeader(statusCode)
|
||||
_, _ = io.Copy(w, body) // #nosec G104 -- HTTP response write
|
||||
@@ -306,19 +290,6 @@ func extractPackageName(path string) string {
|
||||
return path
|
||||
}
|
||||
|
||||
// tarballPrefix returns the filename prefix used by npm tarballs for a given
|
||||
// package. For scoped packages (@scope/pkg) only the bare name is used as a
|
||||
// prefix — the @scope segment is not part of the filename.
|
||||
func tarballPrefix(packageName string) string {
|
||||
bare := packageName
|
||||
if strings.HasPrefix(bare, "@") {
|
||||
if idx := strings.Index(bare, "/"); idx >= 0 {
|
||||
bare = bare[idx+1:]
|
||||
}
|
||||
}
|
||||
return bare + "-"
|
||||
}
|
||||
|
||||
// extractTarballInfo extracts package name and version from tarball path
|
||||
func extractTarballInfo(path string) (string, string) {
|
||||
// Format: /@scope/package/-/package-version.tgz
|
||||
@@ -333,9 +304,9 @@ func extractTarballInfo(path string) (string, string) {
|
||||
tarballName = strings.TrimSuffix(tarballName, ".tgz")
|
||||
tarballName = strings.TrimSuffix(tarballName, ".tar.gz")
|
||||
|
||||
// Remove package name prefix to get version. For scoped packages the
|
||||
// tarball filename uses only the bare package name (no @scope/ prefix).
|
||||
version := strings.TrimPrefix(tarballName, tarballPrefix(packageName))
|
||||
// Remove package name prefix to get version
|
||||
prefix := strings.ReplaceAll(packageName, "/", "-") + "-"
|
||||
version := strings.TrimPrefix(tarballName, prefix)
|
||||
|
||||
return packageName, version
|
||||
}
|
||||
@@ -363,8 +334,9 @@ func extractTarballInfo(path string) (string, string) {
|
||||
tarballName = strings.TrimSuffix(tarballName, ".tgz")
|
||||
tarballName = strings.TrimSuffix(tarballName, ".tar.gz")
|
||||
|
||||
// Remove package name prefix to get version (scope-aware).
|
||||
version := strings.TrimPrefix(tarballName, tarballPrefix(packageName))
|
||||
// Remove package name prefix to get version
|
||||
prefix := strings.ReplaceAll(packageName, "/", "-") + "-"
|
||||
version := strings.TrimPrefix(tarballName, prefix)
|
||||
|
||||
return packageName, version
|
||||
}
|
||||
|
||||
@@ -1,81 +0,0 @@
|
||||
package npm
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// TestScopedTarballFilename verifies that scoped packages produce the correct
|
||||
// upstream tarball filename. NPM registry expects:
|
||||
//
|
||||
// @scope/pkg v1.0.0 -> https://registry.npmjs.org/@scope/pkg/-/pkg-1.0.0.tgz
|
||||
//
|
||||
// The filename portion must NOT include the leading "@scope-" prefix.
|
||||
func TestScopedTarballFilename(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
packageName string
|
||||
version string
|
||||
wantURL string
|
||||
}{
|
||||
{
|
||||
name: "unscoped package",
|
||||
packageName: "express",
|
||||
version: "4.18.2",
|
||||
wantURL: "https://registry.npmjs.org/express/-/express-4.18.2.tgz",
|
||||
},
|
||||
{
|
||||
name: "scoped package",
|
||||
packageName: "@types/node",
|
||||
version: "20.0.0",
|
||||
wantURL: "https://registry.npmjs.org/@types/node/-/node-20.0.0.tgz",
|
||||
},
|
||||
{
|
||||
name: "scoped package with hyphen in name",
|
||||
packageName: "@babel/preset-env",
|
||||
version: "7.22.0",
|
||||
wantURL: "https://registry.npmjs.org/@babel/preset-env/-/preset-env-7.22.0.tgz",
|
||||
},
|
||||
}
|
||||
|
||||
upstream := "https://registry.npmjs.org"
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
bareName := tt.packageName
|
||||
if strings.HasPrefix(bareName, "@") {
|
||||
if idx := strings.Index(bareName, "/"); idx >= 0 {
|
||||
bareName = bareName[idx+1:]
|
||||
}
|
||||
}
|
||||
tarballFilename := bareName + "-" + tt.version + ".tgz"
|
||||
gotURL := fmt.Sprintf("%s/%s/-/%s", upstream, tt.packageName, tarballFilename)
|
||||
if gotURL != tt.wantURL {
|
||||
t.Errorf("got %q, want %q", gotURL, tt.wantURL)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractTarballInfo_Scoped(t *testing.T) {
|
||||
// Sanity check that extractTarballInfo correctly parses scoped paths so
|
||||
// the construction pipeline above is consistent end-to-end.
|
||||
tests := []struct {
|
||||
path string
|
||||
wantPackage string
|
||||
wantVersion string
|
||||
}{
|
||||
{"/@types/node/-/node-20.0.0.tgz", "@types/node", "20.0.0"},
|
||||
{"/@babel/preset-env/-/preset-env-7.22.0.tgz", "@babel/preset-env", "7.22.0"},
|
||||
{"/express/-/express-4.18.2.tgz", "express", "4.18.2"},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.path, func(t *testing.T) {
|
||||
gotPkg, gotVer := extractTarballInfo(tt.path)
|
||||
if gotPkg != tt.wantPackage || gotVer != tt.wantVersion {
|
||||
t.Errorf("extractTarballInfo(%q) = (%q,%q), want (%q,%q)",
|
||||
tt.path, gotPkg, gotVer, tt.wantPackage, tt.wantVersion)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
+9
-80
@@ -1,5 +1,3 @@
|
||||
// Package pypi implements the HTTP handler that proxies PyPI registry
|
||||
// requests through the GoHoarder cache.
|
||||
package pypi
|
||||
|
||||
import (
|
||||
@@ -8,7 +6,6 @@ import (
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
@@ -20,36 +17,6 @@ import (
|
||||
"github.com/rs/zerolog/log"
|
||||
)
|
||||
|
||||
// defaultAllowedPyPIHosts is the hardcoded SSRF allowlist for hosts that
|
||||
// original_url query params may target. Subdomains of pythonhosted.org are
|
||||
// also accepted (handled in isAllowedPyPIHost).
|
||||
var defaultAllowedPyPIHosts = []string{
|
||||
"pypi.org",
|
||||
"files.pythonhosted.org",
|
||||
"pythonhosted.org",
|
||||
}
|
||||
|
||||
// isAllowedPyPIHost reports whether host is on the allowlist or a subdomain
|
||||
// of pythonhosted.org. Comparison is case-insensitive on host only.
|
||||
func isAllowedPyPIHost(host string, allowed []string) bool {
|
||||
host = strings.ToLower(host)
|
||||
// Strip optional port
|
||||
if i := strings.IndexByte(host, ':'); i >= 0 {
|
||||
host = host[:i]
|
||||
}
|
||||
for _, a := range allowed {
|
||||
a = strings.ToLower(a)
|
||||
if host == a {
|
||||
return true
|
||||
}
|
||||
}
|
||||
// Allow any subdomain of pythonhosted.org (e.g. files.pythonhosted.org)
|
||||
if strings.HasSuffix(host, ".pythonhosted.org") {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// Handler implements the PyPI Simple API (PEP 503)
|
||||
type Handler struct {
|
||||
cache *cache.Manager
|
||||
@@ -59,15 +26,11 @@ type Handler struct {
|
||||
credValidator *auth.PyPIValidator
|
||||
validationCache *auth.ValidationCache
|
||||
upstream string
|
||||
allowedHosts []string
|
||||
}
|
||||
|
||||
// Config holds PyPI proxy configuration
|
||||
type Config struct {
|
||||
Upstream string // Upstream PyPI index (e.g., pypi.org/simple)
|
||||
// AllowedHosts is an SSRF allowlist for hosts that original_url query
|
||||
// params may target. If empty, defaultAllowedPyPIHosts is used.
|
||||
AllowedHosts []string
|
||||
}
|
||||
|
||||
// New creates a new PyPI proxy handler
|
||||
@@ -76,16 +39,10 @@ func New(cacheManager *cache.Manager, client *network.Client, config Config) *Ha
|
||||
config.Upstream = "https://pypi.org/simple"
|
||||
}
|
||||
|
||||
allowed := config.AllowedHosts
|
||||
if len(allowed) == 0 {
|
||||
allowed = defaultAllowedPyPIHosts
|
||||
}
|
||||
|
||||
return &Handler{
|
||||
cache: cacheManager,
|
||||
client: client,
|
||||
upstream: config.Upstream,
|
||||
allowedHosts: allowed,
|
||||
credExtractor: auth.NewCredentialExtractor(),
|
||||
credHasher: auth.NewCredentialHasher(),
|
||||
credValidator: auth.NewPyPIValidator(),
|
||||
@@ -130,7 +87,7 @@ func (h *Handler) handleIndex(ctx context.Context, w http.ResponseWriter, r *htt
|
||||
return nil, "", err
|
||||
}
|
||||
if statusCode != http.StatusOK {
|
||||
_ = body.Close()
|
||||
body.Close() // #nosec G104 -- Cleanup, error not critical
|
||||
return nil, "", fmt.Errorf("upstream returned status %d", statusCode)
|
||||
}
|
||||
return body, url, nil
|
||||
@@ -141,11 +98,7 @@ func (h *Handler) handleIndex(ctx context.Context, w http.ResponseWriter, r *htt
|
||||
http.Error(w, "Failed to fetch PyPI index", http.StatusBadGateway)
|
||||
return
|
||||
}
|
||||
defer func() {
|
||||
if cerr := entry.Data.Close(); cerr != nil {
|
||||
log.Warn().Err(cerr).Msg("Failed to close PyPI index body")
|
||||
}
|
||||
}()
|
||||
defer entry.Data.Close() // #nosec G104 -- Cleanup, error not critical
|
||||
|
||||
w.Header().Set("Content-Type", "text/html; charset=UTF-8")
|
||||
_, _ = io.Copy(w, entry.Data) // #nosec G104 -- HTTP response write
|
||||
@@ -162,7 +115,7 @@ func (h *Handler) handlePackagePage(ctx context.Context, w http.ResponseWriter,
|
||||
return nil, "", err
|
||||
}
|
||||
if statusCode != http.StatusOK {
|
||||
_ = body.Close()
|
||||
body.Close() // #nosec G104 -- Cleanup, error not critical
|
||||
return nil, "", fmt.Errorf("upstream returned status %d", statusCode)
|
||||
}
|
||||
return body, url, nil
|
||||
@@ -173,11 +126,7 @@ func (h *Handler) handlePackagePage(ctx context.Context, w http.ResponseWriter,
|
||||
http.Error(w, "Failed to fetch package page", http.StatusBadGateway)
|
||||
return
|
||||
}
|
||||
defer func() {
|
||||
if cerr := entry.Data.Close(); cerr != nil {
|
||||
log.Warn().Err(cerr).Msg("Failed to close PyPI package page body")
|
||||
}
|
||||
}()
|
||||
defer entry.Data.Close() // #nosec G104 -- Cleanup, error not critical
|
||||
|
||||
// Read page into memory for URL rewriting
|
||||
var buf bytes.Buffer
|
||||
@@ -226,23 +175,6 @@ func (h *Handler) handlePackageFile(ctx context.Context, w http.ResponseWriter,
|
||||
if !strings.HasPrefix(originalURL, "http://") && !strings.HasPrefix(originalURL, "https://") {
|
||||
originalURL = "https://pypi.org" + originalURL
|
||||
}
|
||||
|
||||
// SSRF protection: validate parsed host against allowlist before
|
||||
// fetching. Rejects 169.254.169.254, internal services, etc.
|
||||
parsed, parseErr := url.Parse(originalURL)
|
||||
if parseErr != nil || parsed.Host == "" || (parsed.Scheme != "http" && parsed.Scheme != "https") {
|
||||
log.Warn().Str("original_url", originalURL).Msg("Rejected invalid original_url")
|
||||
http.Error(w, "Invalid original_url", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
if !isAllowedPyPIHost(parsed.Host, h.allowedHosts) {
|
||||
log.Warn().
|
||||
Str("original_url", originalURL).
|
||||
Str("host", parsed.Host).
|
||||
Msg("Rejected original_url host not on allowlist")
|
||||
http.Error(w, "original_url host not allowed", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
log.Debug().
|
||||
@@ -267,7 +199,7 @@ func (h *Handler) handlePackageFile(ctx context.Context, w http.ResponseWriter,
|
||||
return nil, "", err
|
||||
}
|
||||
if statusCode != http.StatusOK {
|
||||
_ = body.Close()
|
||||
body.Close() // #nosec G104 -- Cleanup, error not critical
|
||||
return nil, "", fmt.Errorf("upstream returned status %d", statusCode)
|
||||
}
|
||||
return body, originalURL, nil
|
||||
@@ -286,11 +218,7 @@ func (h *Handler) handlePackageFile(ctx context.Context, w http.ResponseWriter,
|
||||
http.Error(w, "Failed to fetch package file", http.StatusBadGateway)
|
||||
return
|
||||
}
|
||||
defer func() {
|
||||
if cerr := entry.Data.Close(); cerr != nil {
|
||||
log.Warn().Err(cerr).Msg("Failed to close PyPI package file body")
|
||||
}
|
||||
}()
|
||||
defer entry.Data.Close() // #nosec G104 -- Cleanup, error not critical
|
||||
|
||||
// CRITICAL SECURITY CHECK: If package requires auth, validate credentials
|
||||
if entry.Package != nil && entry.Package.RequiresAuth {
|
||||
@@ -468,8 +396,9 @@ func rewritePackagePageURLs(html, packageName, proxyBaseURL string) string {
|
||||
// This preserves the original CDN URL so we can fetch from the correct location
|
||||
baseURL := strings.TrimSuffix(proxyBaseURL, "/simple")
|
||||
|
||||
// URL encode the original URL — covers &, =, ?, #, +, /, etc.
|
||||
encodedURL := url.QueryEscape(originalURL)
|
||||
// URL encode the original URL
|
||||
encodedURL := strings.ReplaceAll(originalURL, "&", "%26")
|
||||
encodedURL = strings.ReplaceAll(encodedURL, "=", "%3D")
|
||||
|
||||
newURL := fmt.Sprintf(`href="%s/%s/%s?original_url=%s"`, baseURL, packageName, filenameMatch, encodedURL)
|
||||
return newURL
|
||||
|
||||
@@ -1,139 +0,0 @@
|
||||
package pypi
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"testing"
|
||||
|
||||
"github.com/lukaszraczylo/gohoarder/pkg/auth"
|
||||
)
|
||||
|
||||
func TestIsAllowedPyPIHost(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
host string
|
||||
allowed []string
|
||||
want bool
|
||||
}{
|
||||
{"pypi.org allowed", "pypi.org", defaultAllowedPyPIHosts, true},
|
||||
{"files.pythonhosted.org allowed", "files.pythonhosted.org", defaultAllowedPyPIHosts, true},
|
||||
{"subdomain of pythonhosted.org allowed", "cdn.pythonhosted.org", defaultAllowedPyPIHosts, true},
|
||||
{"case insensitive", "PyPI.ORG", defaultAllowedPyPIHosts, true},
|
||||
{"with port", "pypi.org:443", defaultAllowedPyPIHosts, true},
|
||||
{"AWS metadata blocked", "169.254.169.254", defaultAllowedPyPIHosts, false},
|
||||
{"GCP metadata blocked", "metadata.google.internal", defaultAllowedPyPIHosts, false},
|
||||
{"localhost blocked", "localhost", defaultAllowedPyPIHosts, false},
|
||||
{"loopback blocked", "127.0.0.1", defaultAllowedPyPIHosts, false},
|
||||
{"private RFC1918 blocked", "10.0.0.1", defaultAllowedPyPIHosts, false},
|
||||
{"attacker domain blocked", "evil.example.com", defaultAllowedPyPIHosts, false},
|
||||
{"empty host blocked", "", defaultAllowedPyPIHosts, false},
|
||||
{"trailing-substring attack blocked", "evilpythonhosted.org", defaultAllowedPyPIHosts, false},
|
||||
{"prefix attack blocked", "pypi.org.evil.com", defaultAllowedPyPIHosts, false},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := isAllowedPyPIHost(tt.host, tt.allowed)
|
||||
if got != tt.want {
|
||||
t.Errorf("isAllowedPyPIHost(%q) = %v, want %v", tt.host, got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// newHandlerForSSRFTest builds a Handler with only the fields needed to reach
|
||||
// the SSRF guard. cache is nil intentionally — the guard rejects before any
|
||||
// cache call.
|
||||
func newHandlerForSSRFTest() *Handler {
|
||||
return &Handler{
|
||||
credExtractor: auth.NewCredentialExtractor(),
|
||||
credHasher: auth.NewCredentialHasher(),
|
||||
upstream: "https://pypi.org/simple",
|
||||
allowedHosts: defaultAllowedPyPIHosts,
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandlePackageFile_SSRFRejected(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
originalURL string
|
||||
}{
|
||||
{"AWS metadata IP", "http://169.254.169.254/"},
|
||||
{"GCP metadata host", "http://metadata.google.internal/computeMetadata/v1/"},
|
||||
{"localhost", "http://localhost:8080/secret"},
|
||||
{"private network", "http://10.0.0.5/"},
|
||||
{"file scheme", "file:///etc/passwd"},
|
||||
{"gopher scheme", "gopher://internal/"},
|
||||
{"unrelated public host", "https://evil.example.com/payload.whl"},
|
||||
}
|
||||
|
||||
h := newHandlerForSSRFTest()
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
path := "/example/example-1.0.0.whl"
|
||||
q := url.Values{}
|
||||
q.Set("original_url", tt.originalURL)
|
||||
req := httptest.NewRequest(http.MethodGet, path+"?"+q.Encode(), nil)
|
||||
w := httptest.NewRecorder()
|
||||
|
||||
h.handlePackageFile(req.Context(), w, req, path)
|
||||
|
||||
if w.Code != http.StatusBadRequest {
|
||||
t.Errorf("expected 400 for SSRF target %q, got %d (body=%q)", tt.originalURL, w.Code, w.Body.String())
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandlePackageFile_AllowedHostNotRejected(t *testing.T) {
|
||||
// Sanity check: an allowlisted host should NOT be rejected at the SSRF
|
||||
// guard. We don't have a real cache so the call will fail later with a
|
||||
// non-400 status — that's fine, we only assert the SSRF guard didn't
|
||||
// fire.
|
||||
h := newHandlerForSSRFTest()
|
||||
|
||||
path := "/example/example-1.0.0.whl"
|
||||
q := url.Values{}
|
||||
q.Set("original_url", "https://files.pythonhosted.org/packages/abc/example-1.0.0.whl")
|
||||
req := httptest.NewRequest(http.MethodGet, path+"?"+q.Encode(), nil)
|
||||
w := httptest.NewRecorder()
|
||||
|
||||
defer func() {
|
||||
// nil cache will panic when reached — recover and treat as "guard
|
||||
// did not block", which is the property we care about.
|
||||
_ = recover()
|
||||
}()
|
||||
|
||||
h.handlePackageFile(req.Context(), w, req, path)
|
||||
|
||||
if w.Code == http.StatusBadRequest {
|
||||
t.Errorf("allowlisted host wrongly rejected with 400: %s", w.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestRewritePackagePageURLs_QueryEscape(t *testing.T) {
|
||||
// Original URL contains characters that strings.ReplaceAll(&,=) would miss:
|
||||
// '?', '#', '+'. Verify url.QueryEscape handles them.
|
||||
html := `<a href="https://files.pythonhosted.org/packages/x/y+z/foo-1.0.whl?token=abc#frag">link</a>`
|
||||
out := rewritePackagePageURLs(html, "foo", "http://proxy/pypi")
|
||||
|
||||
// '+' must be encoded (otherwise PyPI would interpret as space)
|
||||
if !contains(out, "original_url=https%3A%2F%2Ffiles.pythonhosted.org%2Fpackages%2Fx%2Fy%2Bz%2Ffoo-1.0.whl%3Ftoken%3Dabc%23frag") {
|
||||
t.Errorf("expected fully URL-encoded original_url, got: %s", out)
|
||||
}
|
||||
}
|
||||
|
||||
func contains(s, sub string) bool {
|
||||
return len(s) >= len(sub) && (indexOf(s, sub) >= 0)
|
||||
}
|
||||
|
||||
func indexOf(s, sub string) int {
|
||||
for i := 0; i+len(sub) <= len(s); i++ {
|
||||
if s[i:i+len(sub)] == sub {
|
||||
return i
|
||||
}
|
||||
}
|
||||
return -1
|
||||
}
|
||||
+11
-198
@@ -1,5 +1,3 @@
|
||||
// Package ghsa implements a vulnerability scanner backed by the GitHub
|
||||
// Security Advisory Database.
|
||||
package ghsa
|
||||
|
||||
import (
|
||||
@@ -8,8 +6,6 @@ import (
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
@@ -109,7 +105,7 @@ func (s *Scanner) Health(ctx context.Context) error {
|
||||
if err != nil {
|
||||
return fmt.Errorf("github advisory database not accessible: %w", err)
|
||||
}
|
||||
defer func() { _ = resp.Body.Close() }() // #nosec G104 -- Cleanup, error not critical
|
||||
defer resp.Body.Close() // #nosec G104 -- Cleanup, error not critical
|
||||
|
||||
// Accept any 2xx or 403 (rate limit) as healthy
|
||||
// Rate limits are expected without a GitHub token and shouldn't fail health checks
|
||||
@@ -140,13 +136,9 @@ func (s *Scanner) mapRegistryToEcosystem(registry string) string {
|
||||
|
||||
// queryAdvisories queries GitHub Advisory Database for a package
|
||||
func (s *Scanner) queryAdvisories(ctx context.Context, ecosystem, packageName string) ([]GHSAAdvisory, error) {
|
||||
endpoint := fmt.Sprintf(
|
||||
"https://api.github.com/advisories?ecosystem=%s&affects=%s&per_page=100",
|
||||
url.QueryEscape(ecosystem),
|
||||
url.QueryEscape(packageName),
|
||||
)
|
||||
url := fmt.Sprintf("https://api.github.com/advisories?ecosystem=%s&affects=%s&per_page=100", ecosystem, packageName)
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", endpoint, nil)
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", url, nil)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create request: %w", err)
|
||||
}
|
||||
@@ -160,7 +152,7 @@ func (s *Scanner) queryAdvisories(ctx context.Context, ecosystem, packageName st
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to query advisories: %w", err)
|
||||
}
|
||||
defer func() { _ = resp.Body.Close() }() // #nosec G104 -- Cleanup, error not critical
|
||||
defer resp.Body.Close() // #nosec G104 -- Cleanup, error not critical
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
@@ -175,194 +167,15 @@ func (s *Scanner) queryAdvisories(ctx context.Context, ecosystem, packageName st
|
||||
return advisories, nil
|
||||
}
|
||||
|
||||
// filterAffectedAdvisories filters advisories that affect the given version.
|
||||
// Each advisory may have multiple GHSAVulnerability entries; if any of them
|
||||
// applies to our installed version (per its vulnerable_version_range), the
|
||||
// advisory is considered affecting.
|
||||
//
|
||||
// Fail-closed: if the version or any range cannot be parsed, the advisory is
|
||||
// included. We err on the side of reporting a possible vulnerability rather
|
||||
// than silently dropping it.
|
||||
// filterAffectedAdvisories filters advisories that affect the given version
|
||||
func (s *Scanner) filterAffectedAdvisories(advisories []GHSAAdvisory, version string) []GHSAAdvisory {
|
||||
if version == "" {
|
||||
// Without a target version, conservatively include everything.
|
||||
return append([]GHSAAdvisory(nil), advisories...)
|
||||
}
|
||||
// Check if this version is affected
|
||||
// GitHub API already filters by package, but we need to check version ranges
|
||||
// For now, we'll include all advisories that match the package
|
||||
// A more sophisticated implementation would parse version ranges
|
||||
affected := append([]GHSAAdvisory(nil), advisories...)
|
||||
|
||||
out := make([]GHSAAdvisory, 0, len(advisories))
|
||||
for _, adv := range advisories {
|
||||
if advisoryAffectsVersion(adv, version) {
|
||||
out = append(out, adv)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// advisoryAffectsVersion reports whether the given installed version falls
|
||||
// within any of the advisory's vulnerable version ranges.
|
||||
func advisoryAffectsVersion(adv GHSAAdvisory, version string) bool {
|
||||
// If the advisory carries no per-vuln range info, conservatively include.
|
||||
if len(adv.Vulnerabilities) == 0 {
|
||||
return true
|
||||
}
|
||||
|
||||
for _, v := range adv.Vulnerabilities {
|
||||
rangeExpr := strings.TrimSpace(v.VulnerableVersions)
|
||||
if rangeExpr == "" {
|
||||
// No range — assume affected.
|
||||
return true
|
||||
}
|
||||
matched, ok := versionInRange(version, rangeExpr)
|
||||
if !ok {
|
||||
// Parse error → fail-closed: include.
|
||||
log.Debug().
|
||||
Str("ghsa_id", adv.GHSAID).
|
||||
Str("range", rangeExpr).
|
||||
Str("version", version).
|
||||
Msg("Could not parse GHSA vulnerable_version_range, including advisory")
|
||||
return true
|
||||
}
|
||||
if matched {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// versionInRange reports whether version satisfies expr. Returns (matched, ok)
|
||||
// where ok=false signals a parse error. Supported forms (comma separated AND):
|
||||
//
|
||||
// "= X"
|
||||
// "< X"
|
||||
// "<= X"
|
||||
// "> X"
|
||||
// ">= X"
|
||||
// ">= X, < Y"
|
||||
//
|
||||
// All clauses must be satisfied for the range to match.
|
||||
func versionInRange(version, expr string) (bool, bool) {
|
||||
clauses := strings.Split(expr, ",")
|
||||
for _, c := range clauses {
|
||||
c = strings.TrimSpace(c)
|
||||
if c == "" {
|
||||
continue
|
||||
}
|
||||
op, bound, ok := splitOpAndVersion(c)
|
||||
if !ok {
|
||||
return false, false
|
||||
}
|
||||
cmp, ok := compareVersions(version, bound)
|
||||
if !ok {
|
||||
return false, false
|
||||
}
|
||||
switch op {
|
||||
case "=", "==":
|
||||
if cmp != 0 {
|
||||
return false, true
|
||||
}
|
||||
case "<":
|
||||
if cmp >= 0 {
|
||||
return false, true
|
||||
}
|
||||
case "<=":
|
||||
if cmp > 0 {
|
||||
return false, true
|
||||
}
|
||||
case ">":
|
||||
if cmp <= 0 {
|
||||
return false, true
|
||||
}
|
||||
case ">=":
|
||||
if cmp < 0 {
|
||||
return false, true
|
||||
}
|
||||
default:
|
||||
return false, false
|
||||
}
|
||||
}
|
||||
return true, true
|
||||
}
|
||||
|
||||
// splitOpAndVersion parses "<op> <version>" pairs (e.g. ">= 1.2.3").
|
||||
func splitOpAndVersion(clause string) (op, ver string, ok bool) {
|
||||
clause = strings.TrimSpace(clause)
|
||||
// Longer operators first to avoid prefix shadowing.
|
||||
for _, candidate := range []string{">=", "<=", "==", "=", ">", "<"} {
|
||||
if strings.HasPrefix(clause, candidate) {
|
||||
rest := strings.TrimSpace(strings.TrimPrefix(clause, candidate))
|
||||
if rest == "" {
|
||||
return "", "", false
|
||||
}
|
||||
return candidate, rest, true
|
||||
}
|
||||
}
|
||||
return "", "", false
|
||||
}
|
||||
|
||||
// compareVersions compares two dot-separated version strings.
|
||||
// Returns (cmp, ok). Numeric segments are compared numerically.
|
||||
// A pre-release suffix (anything after '-' or '+') is treated as lower-priority
|
||||
// than the same version without one, matching common semver intuition for
|
||||
// the cases we expect from the GitHub Advisory Database.
|
||||
func compareVersions(a, b string) (int, bool) {
|
||||
aBase, aPre := splitPreRelease(a)
|
||||
bBase, bPre := splitPreRelease(b)
|
||||
|
||||
aParts := strings.Split(aBase, ".")
|
||||
bParts := strings.Split(bBase, ".")
|
||||
|
||||
n := len(aParts)
|
||||
if len(bParts) > n {
|
||||
n = len(bParts)
|
||||
}
|
||||
for i := 0; i < n; i++ {
|
||||
var av, bv int
|
||||
var err error
|
||||
if i < len(aParts) {
|
||||
av, err = strconv.Atoi(aParts[i])
|
||||
if err != nil {
|
||||
return 0, false
|
||||
}
|
||||
}
|
||||
if i < len(bParts) {
|
||||
bv, err = strconv.Atoi(bParts[i])
|
||||
if err != nil {
|
||||
return 0, false
|
||||
}
|
||||
}
|
||||
if av != bv {
|
||||
if av < bv {
|
||||
return -1, true
|
||||
}
|
||||
return 1, true
|
||||
}
|
||||
}
|
||||
|
||||
// Bases equal; compare pre-release. No pre-release > has pre-release.
|
||||
switch {
|
||||
case aPre == "" && bPre == "":
|
||||
return 0, true
|
||||
case aPre == "" && bPre != "":
|
||||
return 1, true
|
||||
case aPre != "" && bPre == "":
|
||||
return -1, true
|
||||
default:
|
||||
return strings.Compare(aPre, bPre), true
|
||||
}
|
||||
}
|
||||
|
||||
// splitPreRelease separates "1.2.3-rc1" into ("1.2.3", "rc1"). Build metadata
|
||||
// after '+' is stripped (per semver).
|
||||
func splitPreRelease(v string) (base, pre string) {
|
||||
v = strings.TrimSpace(v)
|
||||
v = strings.TrimPrefix(v, "v")
|
||||
if i := strings.Index(v, "+"); i >= 0 {
|
||||
v = v[:i]
|
||||
}
|
||||
if i := strings.Index(v, "-"); i >= 0 {
|
||||
return v[:i], v[i+1:]
|
||||
}
|
||||
return v, ""
|
||||
return affected
|
||||
}
|
||||
|
||||
// emptyResult returns an empty scan result
|
||||
|
||||
@@ -1,160 +0,0 @@
|
||||
package ghsa
|
||||
|
||||
import (
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestVersionInRange(t *testing.T) {
|
||||
// matched: whether the version satisfies the range expression.
|
||||
// ok: whether the parser/comparator could evaluate the inputs.
|
||||
cases := []struct {
|
||||
name string
|
||||
version string
|
||||
expr string
|
||||
matched bool
|
||||
ok bool
|
||||
}{
|
||||
{"single LT match", "1.2.3", "< 2.0.0", true, true},
|
||||
{"single LT no match", "2.5.0", "< 2.0.0", false, true},
|
||||
{"single GTE match", "2.5.0", ">= 2.0.0", true, true},
|
||||
{"single GTE no match", "1.0.0", ">= 2.0.0", false, true},
|
||||
{"range hit", "1.5.0", ">= 1.0.0, < 2.0.0", true, true},
|
||||
{"range below", "0.9.0", ">= 1.0.0, < 2.0.0", false, true},
|
||||
{"range above", "2.0.0", ">= 1.0.0, < 2.0.0", false, true},
|
||||
{"range upper bound exclusive", "2.0.0", ">= 1.0.0, < 2.0.0", false, true},
|
||||
{"range lower bound inclusive", "1.0.0", ">= 1.0.0, < 2.0.0", true, true},
|
||||
{"equality match", "1.2.3", "= 1.2.3", true, true},
|
||||
{"equality miss", "1.2.4", "= 1.2.3", false, true},
|
||||
{"with v prefix on bound", "1.2.3", ">= v1.0.0", true, true},
|
||||
{"shorter version coerces", "1.0", ">= 1.0.0", true, true},
|
||||
{"pre-release lower than release", "1.0.0-rc1", ">= 1.0.0", false, true},
|
||||
{"pre-release greater than older", "1.0.0-rc1", ">= 0.9.0", true, true},
|
||||
{"malformed operator", "1.0.0", "~ 1.0.0", false, false},
|
||||
{"malformed version", "abc", ">= 1.0.0", false, false},
|
||||
{"empty bound after op", "1.0.0", ">=", false, false},
|
||||
}
|
||||
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
matched, ok := versionInRange(tc.version, tc.expr)
|
||||
if matched != tc.matched || ok != tc.ok {
|
||||
t.Fatalf("versionInRange(%q, %q) = (%v, %v), want (%v, %v)",
|
||||
tc.version, tc.expr, matched, ok, tc.matched, tc.ok)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestAdvisoryAffectsVersion(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
version string
|
||||
adv GHSAAdvisory
|
||||
want bool
|
||||
}{
|
||||
{
|
||||
name: "advisory with no vulnerabilities is conservatively included",
|
||||
adv: GHSAAdvisory{GHSAID: "GHSA-xxxx", Vulnerabilities: nil},
|
||||
version: "1.0.0",
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "matching range marks advisory as affecting",
|
||||
adv: GHSAAdvisory{
|
||||
GHSAID: "GHSA-aaaa",
|
||||
Vulnerabilities: []GHSAVulnerability{
|
||||
{VulnerableVersions: ">= 1.0.0, < 2.0.0"},
|
||||
},
|
||||
},
|
||||
version: "1.5.0",
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "non-matching range excludes advisory",
|
||||
adv: GHSAAdvisory{
|
||||
GHSAID: "GHSA-bbbb",
|
||||
Vulnerabilities: []GHSAVulnerability{
|
||||
{VulnerableVersions: ">= 2.0.0"},
|
||||
},
|
||||
},
|
||||
version: "1.0.0",
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
name: "any matching range across multiple vulns is affecting",
|
||||
adv: GHSAAdvisory{
|
||||
GHSAID: "GHSA-cccc",
|
||||
Vulnerabilities: []GHSAVulnerability{
|
||||
{VulnerableVersions: "< 0.5.0"},
|
||||
{VulnerableVersions: ">= 1.0.0, < 1.2.0"},
|
||||
},
|
||||
},
|
||||
version: "1.1.0",
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "empty range falls back to affecting (fail-closed)",
|
||||
adv: GHSAAdvisory{
|
||||
GHSAID: "GHSA-dddd",
|
||||
Vulnerabilities: []GHSAVulnerability{{VulnerableVersions: ""}},
|
||||
},
|
||||
version: "1.0.0",
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "unparseable range falls back to affecting (fail-closed)",
|
||||
adv: GHSAAdvisory{
|
||||
GHSAID: "GHSA-eeee",
|
||||
Vulnerabilities: []GHSAVulnerability{{VulnerableVersions: "~> 1.0"}},
|
||||
},
|
||||
version: "1.0.0",
|
||||
want: true,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
got := advisoryAffectsVersion(tc.adv, tc.version)
|
||||
if got != tc.want {
|
||||
t.Fatalf("advisoryAffectsVersion(%q) = %v, want %v",
|
||||
tc.version, got, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestFilterAffectedAdvisoriesEmptyVersion(t *testing.T) {
|
||||
// Without a target version we can't compare ranges, so all advisories
|
||||
// are conservatively included.
|
||||
s := &Scanner{}
|
||||
in := []GHSAAdvisory{
|
||||
{GHSAID: "A", Vulnerabilities: []GHSAVulnerability{{VulnerableVersions: ">= 2.0.0"}}},
|
||||
{GHSAID: "B"},
|
||||
}
|
||||
out := s.filterAffectedAdvisories(in, "")
|
||||
if len(out) != len(in) {
|
||||
t.Fatalf("expected all advisories with empty version, got %d/%d", len(out), len(in))
|
||||
}
|
||||
}
|
||||
|
||||
func TestFilterAffectedAdvisoriesFiltersByRange(t *testing.T) {
|
||||
s := &Scanner{}
|
||||
in := []GHSAAdvisory{
|
||||
{ // matches
|
||||
GHSAID: "MATCH",
|
||||
Vulnerabilities: []GHSAVulnerability{
|
||||
{VulnerableVersions: ">= 1.0.0, < 2.0.0"},
|
||||
},
|
||||
},
|
||||
{ // does not match
|
||||
GHSAID: "MISS",
|
||||
Vulnerabilities: []GHSAVulnerability{
|
||||
{VulnerableVersions: ">= 3.0.0"},
|
||||
},
|
||||
},
|
||||
}
|
||||
out := s.filterAffectedAdvisories(in, "1.5.0")
|
||||
if len(out) != 1 || out[0].GHSAID != "MATCH" {
|
||||
t.Fatalf("expected only MATCH, got %+v", out)
|
||||
}
|
||||
}
|
||||
@@ -1,5 +1,3 @@
|
||||
// Package govulncheck wraps the `govulncheck` CLI to scan Go modules for
|
||||
// known vulnerabilities.
|
||||
package govulncheck
|
||||
|
||||
import (
|
||||
@@ -8,7 +6,6 @@ import (
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
@@ -69,30 +66,15 @@ func (s *Scanner) Scan(ctx context.Context, registry, packageName, version strin
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create temp dir: %w", err)
|
||||
}
|
||||
defer func() { _ = os.RemoveAll(tmpDir) }()
|
||||
defer os.RemoveAll(tmpDir)
|
||||
|
||||
// Extract the .zip file
|
||||
if extractErr := s.extractZip(filePath, tmpDir); extractErr != nil {
|
||||
return nil, fmt.Errorf("failed to extract zip: %w", extractErr)
|
||||
if err := s.extractZip(filePath, tmpDir); err != nil {
|
||||
return nil, fmt.Errorf("failed to extract zip: %w", err)
|
||||
}
|
||||
|
||||
// Locate the Go module root (directory containing go.mod). Go modules
|
||||
// in the proxy zip layout are nested under <module>@<version>/.
|
||||
moduleDir, err := findGoModDir(tmpDir)
|
||||
if err != nil {
|
||||
log.Warn().
|
||||
Err(err).
|
||||
Str("package", packageName).
|
||||
Str("version", version).
|
||||
Msg("Could not locate go.mod in extracted module, skipping govulncheck")
|
||||
return s.skippedResult(registry, packageName, version, "no go.mod in extracted module"), nil
|
||||
}
|
||||
|
||||
// Run govulncheck in source mode against the module's package set.
|
||||
// -mode=binary requires a compiled binary which we do not have; the
|
||||
// default (source) mode wants a Go source tree with a go.mod.
|
||||
cmd := exec.CommandContext(ctx, "govulncheck", "-json", "./...") // #nosec G204 -- fixed args, cwd is controlled temp dir
|
||||
cmd.Dir = moduleDir
|
||||
// Run govulncheck
|
||||
cmd := exec.CommandContext(ctx, "govulncheck", "-json", "-mode=binary", tmpDir) // #nosec G204 -- govulncheck command with temp directory
|
||||
output, _ := cmd.CombinedOutput()
|
||||
|
||||
// govulncheck returns non-zero when vulnerabilities are found
|
||||
@@ -146,59 +128,6 @@ func (s *Scanner) extractZip(zipPath, destDir string) error {
|
||||
return cmd.Run()
|
||||
}
|
||||
|
||||
// findGoModDir walks the directory tree under root looking for a directory
|
||||
// that contains a go.mod file. The Go module proxy ships zips with layout
|
||||
// "<module>@<version>/...", so the module root is typically one or two
|
||||
// levels below the extraction directory. Returns an error if none is found.
|
||||
func findGoModDir(root string) (string, error) {
|
||||
// Quick check: does the root itself contain go.mod?
|
||||
if _, err := os.Stat(filepath.Join(root, "go.mod")); err == nil {
|
||||
return root, nil
|
||||
}
|
||||
|
||||
var found string
|
||||
err := filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return nil // keep searching
|
||||
}
|
||||
if d.IsDir() {
|
||||
return nil
|
||||
}
|
||||
if d.Name() == "go.mod" {
|
||||
found = filepath.Dir(path)
|
||||
return filepath.SkipAll
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if found == "" {
|
||||
return "", fmt.Errorf("go.mod not found under %s", root)
|
||||
}
|
||||
return found, nil
|
||||
}
|
||||
|
||||
// skippedResult returns a clean ScanResult marked as skipped with an
|
||||
// explanation. Using clean (not error) here because the package is simply
|
||||
// not a Go module we can analyse — not a scanner failure.
|
||||
func (s *Scanner) skippedResult(registry, packageName, version, reason string) *metadata.ScanResult {
|
||||
return &metadata.ScanResult{
|
||||
ID: uuid.New().String(),
|
||||
Registry: registry,
|
||||
PackageName: packageName,
|
||||
PackageVersion: version,
|
||||
Scanner: ScannerName,
|
||||
ScannedAt: time.Now(),
|
||||
Status: metadata.ScanStatusClean,
|
||||
VulnerabilityCount: 0,
|
||||
Vulnerabilities: []metadata.Vulnerability{},
|
||||
Details: map[string]interface{}{
|
||||
"skipped": reason,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// convertResult converts govulncheck findings to our ScanResult format
|
||||
func (s *Scanner) convertResult(vulns []GovulncheckVuln, registry, packageName, version string) *metadata.ScanResult {
|
||||
vulnerabilities := make([]metadata.Vulnerability, 0)
|
||||
|
||||
@@ -1,5 +1,3 @@
|
||||
// Package grype wraps the Anchore `grype` CLI to scan packages for
|
||||
// known vulnerabilities.
|
||||
package grype
|
||||
|
||||
import (
|
||||
|
||||
@@ -1,5 +1,3 @@
|
||||
// Package npmaudit wraps the `npm audit` CLI to surface vulnerability
|
||||
// findings for npm packages.
|
||||
package npmaudit
|
||||
|
||||
import (
|
||||
@@ -68,7 +66,7 @@ func (s *Scanner) Scan(ctx context.Context, registry, packageName, version strin
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create temp dir: %w", err)
|
||||
}
|
||||
defer func() { _ = os.RemoveAll(tmpDir) }()
|
||||
defer os.RemoveAll(tmpDir)
|
||||
|
||||
// Extract the .tgz file
|
||||
if err := s.extractTgz(filePath, tmpDir); err != nil {
|
||||
@@ -82,36 +80,8 @@ func (s *Scanner) Scan(ctx context.Context, registry, packageName, version strin
|
||||
packageDir = tmpDir
|
||||
}
|
||||
|
||||
// npm tarballs ship only package.json — there is no lockfile. We must
|
||||
// generate one before `npm audit` can resolve the dependency tree.
|
||||
// NOTE: this performs network egress (npm registry lookups for
|
||||
// transitive deps). Acceptable here because the scanner runs server-
|
||||
// side and the operator already trusts upstream resolution to cache
|
||||
// the package; we use --ignore-scripts to avoid running install hooks.
|
||||
log.Info().
|
||||
Str("scanner", ScannerName).
|
||||
Str("package", packageName).
|
||||
Msg("Generating package-lock.json for npm audit (network egress)")
|
||||
|
||||
installCmd := exec.CommandContext(ctx, "npm", "install",
|
||||
"--package-lock-only",
|
||||
"--omit=dev",
|
||||
"--ignore-scripts",
|
||||
"--no-audit",
|
||||
)
|
||||
installCmd.Dir = packageDir
|
||||
if installOut, err := installCmd.CombinedOutput(); err != nil {
|
||||
log.Warn().
|
||||
Err(err).
|
||||
Str("package", packageName).
|
||||
Str("output", string(installOut)).
|
||||
Msg("npm install --package-lock-only failed; returning scan-error")
|
||||
return s.scanErrorResult(registry, packageName, version,
|
||||
fmt.Sprintf("npm install --package-lock-only failed: %v", err)), nil
|
||||
}
|
||||
|
||||
// Run npm audit against the freshly generated lockfile.
|
||||
cmd := exec.CommandContext(ctx, "npm", "audit", "--json")
|
||||
// Run npm audit
|
||||
cmd := exec.CommandContext(ctx, "npm", "audit", "--json", "--package-lock-only")
|
||||
cmd.Dir = packageDir
|
||||
output, _ := cmd.CombinedOutput() // npm audit returns non-zero when vulns found
|
||||
|
||||
@@ -120,9 +90,8 @@ func (s *Scanner) Scan(ctx context.Context, registry, packageName, version strin
|
||||
if len(output) > 0 {
|
||||
if err := json.Unmarshal(output, &auditResult); err != nil {
|
||||
log.Warn().Err(err).Msg("Failed to parse npm audit output")
|
||||
// Parse failure means we couldn't determine vulnerability state — fail closed.
|
||||
return s.scanErrorResult(registry, packageName, version,
|
||||
fmt.Sprintf("failed to parse npm audit output: %v", err)), nil
|
||||
// Return clean result on parse error
|
||||
return s.emptyResult(registry, packageName, version), nil
|
||||
}
|
||||
}
|
||||
|
||||
@@ -154,11 +123,7 @@ func (s *Scanner) extractTgz(tgzPath, destDir string) error {
|
||||
}
|
||||
|
||||
// emptyResult returns an empty scan result
|
||||
|
||||
// scanErrorResult returns a result marked as scan-error so the manager merge
|
||||
// and CheckVulnerabilities can fail closed. Use this when the scan could not
|
||||
// complete and we therefore have no signal about vulnerabilities.
|
||||
func (s *Scanner) scanErrorResult(registry, packageName, version, reason string) *metadata.ScanResult {
|
||||
func (s *Scanner) emptyResult(registry, packageName, version string) *metadata.ScanResult {
|
||||
return &metadata.ScanResult{
|
||||
ID: uuid.New().String(),
|
||||
Registry: registry,
|
||||
@@ -166,12 +131,10 @@ func (s *Scanner) scanErrorResult(registry, packageName, version, reason string)
|
||||
PackageVersion: version,
|
||||
Scanner: ScannerName,
|
||||
ScannedAt: time.Now(),
|
||||
Status: metadata.ScanStatusError,
|
||||
Status: metadata.ScanStatusClean,
|
||||
VulnerabilityCount: 0,
|
||||
Vulnerabilities: []metadata.Vulnerability{},
|
||||
Details: map[string]interface{}{
|
||||
"error": reason,
|
||||
},
|
||||
Details: map[string]interface{}{},
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1,4 +1,3 @@
|
||||
// Package osv implements a vulnerability scanner backed by the OSV.dev API.
|
||||
package osv
|
||||
|
||||
import (
|
||||
@@ -159,7 +158,7 @@ func (s *Scanner) Scan(ctx context.Context, registry, packageName, version strin
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("OSV API request failed: %w", err)
|
||||
}
|
||||
defer func() { _ = resp.Body.Close() }() // #nosec G104 -- Cleanup, error not critical
|
||||
defer resp.Body.Close() // #nosec G104 -- Cleanup, error not critical
|
||||
|
||||
// Read response
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
@@ -373,7 +372,7 @@ func (s *Scanner) Health(ctx context.Context) error {
|
||||
if err != nil {
|
||||
return fmt.Errorf("OSV API not reachable: %w", err)
|
||||
}
|
||||
defer func() { _ = resp.Body.Close() }() // #nosec G104 -- Cleanup, error not critical
|
||||
defer resp.Body.Close() // #nosec G104 -- Cleanup, error not critical
|
||||
|
||||
log.Debug().Int("status", resp.StatusCode).Msg("OSV health check passed")
|
||||
return nil
|
||||
|
||||
@@ -1,5 +1,3 @@
|
||||
// Package pipaudit wraps the `pip-audit` CLI to scan Python wheels and
|
||||
// source distributions for known vulnerabilities.
|
||||
package pipaudit
|
||||
|
||||
import (
|
||||
@@ -9,7 +7,6 @@ import (
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/lukaszraczylo/gohoarder/pkg/config"
|
||||
@@ -69,7 +66,7 @@ func (s *Scanner) Scan(ctx context.Context, registry, packageName, version strin
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create temp dir: %w", err)
|
||||
}
|
||||
defer func() { _ = os.RemoveAll(tmpDir) }()
|
||||
defer os.RemoveAll(tmpDir)
|
||||
|
||||
// Copy the wheel/tar.gz file to temp directory
|
||||
tmpFile := filepath.Join(tmpDir, filepath.Base(filePath))
|
||||
@@ -77,30 +74,16 @@ func (s *Scanner) Scan(ctx context.Context, registry, packageName, version strin
|
||||
return nil, fmt.Errorf("failed to copy file: %w", err)
|
||||
}
|
||||
|
||||
// Build the appropriate pip-audit invocation based on artifact type.
|
||||
// `-r` expects requirements.txt — passing a wheel/tarball there is wrong.
|
||||
// Wheels can be scanned directly via positional arg. Source distributions
|
||||
// (tarballs) need to be extracted; if they contain a pyproject.toml we
|
||||
// can scan that, otherwise we fail closed.
|
||||
cmd, prepErr := s.buildAuditCmd(ctx, tmpDir, tmpFile)
|
||||
if prepErr != nil {
|
||||
log.Warn().
|
||||
Err(prepErr).
|
||||
Str("package", packageName).
|
||||
Str("version", version).
|
||||
Msg("pip-audit could not prepare input artifact, returning scan-error")
|
||||
return s.scanErrorResult(registry, packageName, version, prepErr.Error()), nil
|
||||
}
|
||||
output, _ := cmd.CombinedOutput() // pip-audit returns non-zero when vulns found
|
||||
// Run pip-audit on the package file
|
||||
cmd := exec.CommandContext(ctx, "pip-audit", "-r", tmpFile, "--format", "json") // #nosec G204 -- pip-audit command with temp file
|
||||
output, _ := cmd.CombinedOutput() // pip-audit returns non-zero when vulns found
|
||||
|
||||
// Parse pip-audit output
|
||||
var auditResult PipAuditResult
|
||||
if len(output) > 0 {
|
||||
if err := json.Unmarshal(output, &auditResult); err != nil {
|
||||
log.Warn().Err(err).Msg("Failed to parse pip-audit output")
|
||||
// Parse failure → no signal → fail closed.
|
||||
return s.scanErrorResult(registry, packageName, version,
|
||||
fmt.Sprintf("failed to parse pip-audit output: %v", err)), nil
|
||||
return s.emptyResult(registry, packageName, version), nil
|
||||
}
|
||||
}
|
||||
|
||||
@@ -134,84 +117,8 @@ func (s *Scanner) copyFile(src, dst string) error {
|
||||
return os.WriteFile(dst, input, 0600)
|
||||
}
|
||||
|
||||
// buildAuditCmd constructs the right pip-audit command for the input artifact.
|
||||
//
|
||||
// - .whl -> pip-audit <wheel> --format json
|
||||
// - .tar.gz / .tgz / .zip (sdist) -> extract; if pyproject.toml exists
|
||||
// run `pip-audit --pyproject <pyproject> --format json`; otherwise error.
|
||||
//
|
||||
// extractDir is used as a workspace for sdist extraction.
|
||||
func (s *Scanner) buildAuditCmd(ctx context.Context, extractDir, artifact string) (*exec.Cmd, error) {
|
||||
lower := strings.ToLower(artifact)
|
||||
switch {
|
||||
case strings.HasSuffix(lower, ".whl"):
|
||||
// pip-audit can scan a wheel directly via positional argument.
|
||||
return exec.CommandContext(ctx, "pip-audit", artifact, "--format", "json"), nil // #nosec G204 -- artifact path is in controlled tmp dir
|
||||
|
||||
case strings.HasSuffix(lower, ".tar.gz"),
|
||||
strings.HasSuffix(lower, ".tgz"),
|
||||
strings.HasSuffix(lower, ".zip"):
|
||||
// Source distributions must be unpacked first.
|
||||
sdistDir := filepath.Join(extractDir, "sdist")
|
||||
if err := os.MkdirAll(sdistDir, 0o750); err != nil {
|
||||
return nil, fmt.Errorf("create sdist dir: %w", err)
|
||||
}
|
||||
if err := s.extractSdist(artifact, sdistDir); err != nil {
|
||||
return nil, fmt.Errorf("extract sdist: %w", err)
|
||||
}
|
||||
pyproject, err := findPyProject(sdistDir)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("no pyproject.toml in sdist: %w", err)
|
||||
}
|
||||
return exec.CommandContext(ctx, "pip-audit", "--pyproject", pyproject, "--format", "json"), nil // #nosec G204 -- pyproject path under controlled tmp dir
|
||||
|
||||
default:
|
||||
return nil, fmt.Errorf("unsupported pip artifact extension: %s", filepath.Base(artifact))
|
||||
}
|
||||
}
|
||||
|
||||
// extractSdist unpacks a Python source distribution into destDir.
|
||||
func (s *Scanner) extractSdist(archive, destDir string) error {
|
||||
lower := strings.ToLower(archive)
|
||||
switch {
|
||||
case strings.HasSuffix(lower, ".tar.gz"), strings.HasSuffix(lower, ".tgz"):
|
||||
return exec.Command("tar", "-xzf", archive, "-C", destDir).Run()
|
||||
case strings.HasSuffix(lower, ".zip"):
|
||||
return exec.Command("unzip", "-q", archive, "-d", destDir).Run()
|
||||
default:
|
||||
return fmt.Errorf("unknown archive type: %s", archive)
|
||||
}
|
||||
}
|
||||
|
||||
// findPyProject returns the path to a pyproject.toml within root, walking
|
||||
// one level deep (sdists typically extract to <pkg>-<ver>/).
|
||||
func findPyProject(root string) (string, error) {
|
||||
var found string
|
||||
err := filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
if d.IsDir() {
|
||||
return nil
|
||||
}
|
||||
if d.Name() == "pyproject.toml" {
|
||||
found = path
|
||||
return filepath.SkipAll
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if found == "" {
|
||||
return "", fmt.Errorf("pyproject.toml not found under %s", root)
|
||||
}
|
||||
return found, nil
|
||||
}
|
||||
|
||||
// scanErrorResult returns a result marked scan-error so manager merge and
|
||||
// CheckVulnerabilities can fail closed when this scanner could not run.
|
||||
func (s *Scanner) scanErrorResult(registry, packageName, version, reason string) *metadata.ScanResult {
|
||||
// emptyResult returns an empty scan result
|
||||
func (s *Scanner) emptyResult(registry, packageName, version string) *metadata.ScanResult {
|
||||
return &metadata.ScanResult{
|
||||
ID: uuid.New().String(),
|
||||
Registry: registry,
|
||||
@@ -219,17 +126,13 @@ func (s *Scanner) scanErrorResult(registry, packageName, version, reason string)
|
||||
PackageVersion: version,
|
||||
Scanner: ScannerName,
|
||||
ScannedAt: time.Now(),
|
||||
Status: metadata.ScanStatusError,
|
||||
Status: metadata.ScanStatusClean,
|
||||
VulnerabilityCount: 0,
|
||||
Vulnerabilities: []metadata.Vulnerability{},
|
||||
Details: map[string]interface{}{
|
||||
"error": reason,
|
||||
},
|
||||
Details: map[string]interface{}{},
|
||||
}
|
||||
}
|
||||
|
||||
// emptyResult returns an empty scan result
|
||||
|
||||
// convertResult converts pip-audit output to our ScanResult format
|
||||
func (s *Scanner) convertResult(auditResult *PipAuditResult, registry, packageName, version string) *metadata.ScanResult {
|
||||
vulnerabilities := make([]metadata.Vulnerability, 0)
|
||||
|
||||
@@ -2,7 +2,6 @@ package scanner
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/lukaszraczylo/gohoarder/pkg/metadata"
|
||||
@@ -16,7 +15,6 @@ type RescanWorker struct {
|
||||
storage storage.StorageBackend
|
||||
manager *Manager
|
||||
stopCh chan struct{}
|
||||
stopOnce sync.Once
|
||||
interval time.Duration
|
||||
}
|
||||
|
||||
@@ -66,11 +64,9 @@ func (w *RescanWorker) Start(ctx context.Context) {
|
||||
}
|
||||
}
|
||||
|
||||
// Stop stops the rescan worker. Safe to call multiple times.
|
||||
// Stop stops the rescan worker
|
||||
func (w *RescanWorker) Stop() {
|
||||
w.stopOnce.Do(func() {
|
||||
close(w.stopCh)
|
||||
})
|
||||
close(w.stopCh)
|
||||
}
|
||||
|
||||
// rescanPackages re-scans packages that need updating
|
||||
|
||||
+10
-117
@@ -1,18 +1,11 @@
|
||||
// Package scanner orchestrates pluggable vulnerability scanners and
|
||||
// records their results against cached packages.
|
||||
package scanner
|
||||
|
||||
import (
|
||||
"context"
|
||||
stderrors "errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/lukaszraczylo/gohoarder/pkg/config"
|
||||
hoardererrors "github.com/lukaszraczylo/gohoarder/pkg/errors"
|
||||
"github.com/lukaszraczylo/gohoarder/pkg/events"
|
||||
"github.com/lukaszraczylo/gohoarder/pkg/metadata"
|
||||
"github.com/lukaszraczylo/gohoarder/pkg/scanner/ghsa"
|
||||
"github.com/lukaszraczylo/gohoarder/pkg/scanner/govulncheck"
|
||||
@@ -21,8 +14,6 @@ import (
|
||||
"github.com/lukaszraczylo/gohoarder/pkg/scanner/osv"
|
||||
"github.com/lukaszraczylo/gohoarder/pkg/scanner/pipaudit"
|
||||
"github.com/lukaszraczylo/gohoarder/pkg/scanner/trivy"
|
||||
"github.com/lukaszraczylo/gohoarder/pkg/uuid"
|
||||
"github.com/lukaszraczylo/gohoarder/pkg/websocket"
|
||||
"github.com/rs/zerolog/log"
|
||||
)
|
||||
|
||||
@@ -46,34 +37,11 @@ type DatabaseUpdater interface {
|
||||
// Manager manages multiple security scanners
|
||||
type Manager struct {
|
||||
metadataStore metadata.MetadataStore
|
||||
broadcaster events.Broadcaster
|
||||
scanners []Scanner
|
||||
config config.SecurityConfig
|
||||
bcMu sync.RWMutex
|
||||
scanners []Scanner
|
||||
enabled bool
|
||||
}
|
||||
|
||||
// SetBroadcaster wires an events.Broadcaster onto the scanner so scan
|
||||
// lifecycle events are published. Pass nil to disable broadcasting.
|
||||
// Safe for concurrent use.
|
||||
func (m *Manager) SetBroadcaster(b events.Broadcaster) {
|
||||
m.bcMu.Lock()
|
||||
m.broadcaster = b
|
||||
m.bcMu.Unlock()
|
||||
}
|
||||
|
||||
// emit publishes an event via the configured broadcaster, if any.
|
||||
// Non-blocking: the underlying transport handles overflow by dropping.
|
||||
func (m *Manager) emit(eventType string, payload map[string]interface{}) {
|
||||
m.bcMu.RLock()
|
||||
b := m.broadcaster
|
||||
m.bcMu.RUnlock()
|
||||
if b == nil {
|
||||
return
|
||||
}
|
||||
b.BroadcastEvent(eventType, payload)
|
||||
}
|
||||
|
||||
// New creates a new scanner manager with configured scanners
|
||||
func New(cfg config.SecurityConfig, metadataStore metadata.MetadataStore) (*Manager, error) {
|
||||
manager := &Manager{
|
||||
@@ -210,42 +178,11 @@ func (m *Manager) ScanPackage(ctx context.Context, registry, packageName, versio
|
||||
Msg("Scan completed")
|
||||
}
|
||||
|
||||
// If no scanners succeeded, persist a synthetic error result so callers
|
||||
// fail closed (no scan == blocked) rather than silently leaving
|
||||
// SecurityScanned=false which would allow the package to be served.
|
||||
// If no scanners succeeded, return
|
||||
if len(scanResults) == 0 {
|
||||
log.Warn().
|
||||
Str("package", packageName).
|
||||
Msg("All scanners failed, saving scan-error result for fail-closed enforcement")
|
||||
|
||||
errResult := &metadata.ScanResult{
|
||||
ID: uuid.New().String(),
|
||||
Registry: registry,
|
||||
PackageName: packageName,
|
||||
PackageVersion: version,
|
||||
Scanner: "all",
|
||||
ScannedAt: time.Now(),
|
||||
Status: metadata.ScanStatusError,
|
||||
VulnerabilityCount: 0,
|
||||
Vulnerabilities: []metadata.Vulnerability{},
|
||||
Details: map[string]interface{}{
|
||||
"error": "all configured scanners failed for this package",
|
||||
},
|
||||
}
|
||||
if err := m.metadataStore.SaveScanResult(ctx, errResult); err != nil {
|
||||
log.Error().
|
||||
Err(err).
|
||||
Str("package", packageName).
|
||||
Msg("Failed to save scan-error result")
|
||||
return err
|
||||
}
|
||||
m.emit(string(websocket.EventScanComplete), map[string]interface{}{
|
||||
"registry": registry,
|
||||
"name": packageName,
|
||||
"version": version,
|
||||
"status": string(errResult.Status),
|
||||
"vulnerability_count": errResult.VulnerabilityCount,
|
||||
})
|
||||
Msg("All scanners failed, no results to save")
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -269,14 +206,6 @@ func (m *Manager) ScanPackage(ctx context.Context, registry, packageName, versio
|
||||
Strs("scanners", scannerNames).
|
||||
Msg("Consolidated scan results saved")
|
||||
|
||||
m.emit(string(websocket.EventScanComplete), map[string]interface{}{
|
||||
"registry": registry,
|
||||
"name": packageName,
|
||||
"version": version,
|
||||
"status": string(mergedResult.Status),
|
||||
"vulnerability_count": mergedResult.VulnerabilityCount,
|
||||
})
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -356,21 +285,11 @@ func (m *Manager) mergeResults(results []*metadata.ScanResult, scannerNames []st
|
||||
}
|
||||
}
|
||||
|
||||
// Update status to worst case.
|
||||
// Order: vulnerable > error > pending > clean. Vulnerable wins because
|
||||
// the cache layer must surface the actual vulns. Otherwise, if any
|
||||
// scanner errored, propagate so CheckVulnerabilities can fail closed.
|
||||
switch result.Status {
|
||||
case metadata.ScanStatusVulnerable:
|
||||
// Update status to worst case
|
||||
if result.Status == metadata.ScanStatusVulnerable {
|
||||
merged.Status = metadata.ScanStatusVulnerable
|
||||
case metadata.ScanStatusError:
|
||||
if merged.Status != metadata.ScanStatusVulnerable {
|
||||
merged.Status = metadata.ScanStatusError
|
||||
}
|
||||
case metadata.ScanStatusPending:
|
||||
if merged.Status != metadata.ScanStatusVulnerable && merged.Status != metadata.ScanStatusError {
|
||||
merged.Status = metadata.ScanStatusPending
|
||||
}
|
||||
} else if result.Status == metadata.ScanStatusPending && merged.Status != metadata.ScanStatusVulnerable {
|
||||
merged.Status = metadata.ScanStatusPending
|
||||
}
|
||||
}
|
||||
|
||||
@@ -440,37 +359,11 @@ func (m *Manager) CheckVulnerabilities(ctx context.Context, registry, packageNam
|
||||
}
|
||||
}
|
||||
|
||||
// Get latest scan result.
|
||||
// SECURITY: Fail closed when no scan exists or backend errors.
|
||||
// Previously this returned (false, "", nil) which allowed unscanned
|
||||
// packages through — a fail-open bypass. Cache layer is expected to
|
||||
// wait for the initial scan via SecurityScanned flag; once that flag
|
||||
// is set, GetScanResult MUST return a record. A missing record at this
|
||||
// point indicates either a cleared/lost scan or a transient error;
|
||||
// either way we block.
|
||||
// Get latest scan result
|
||||
result, err := m.metadataStore.GetScanResult(ctx, registry, packageName, version)
|
||||
if err != nil {
|
||||
var hErr *hoardererrors.Error
|
||||
if stderrors.As(err, &hErr) && hErr.Code == hoardererrors.ErrCodeNotFound {
|
||||
return true, "no scan available - fail closed", nil
|
||||
}
|
||||
// Real backend error (DB transient, etc.) — also block.
|
||||
log.Warn().
|
||||
Err(err).
|
||||
Str("package", packageName).
|
||||
Msg("Failed to retrieve scan result, failing closed")
|
||||
return true, "scan lookup failed - fail closed", nil
|
||||
}
|
||||
if result == nil {
|
||||
// File-backed metadata store returns (nil, nil) on not-found.
|
||||
return true, "no scan available - fail closed", nil
|
||||
}
|
||||
|
||||
// If the scan itself errored (all scanners failed for this package),
|
||||
// block. A scan-error record means we don't actually know whether the
|
||||
// package is safe.
|
||||
if result.Status == metadata.ScanStatusError {
|
||||
return true, "scan failed - fail closed", nil
|
||||
// No scan result found - allow download (will be scanned after)
|
||||
return false, "", nil
|
||||
}
|
||||
|
||||
// Build set of bypassed CVEs for fast lookup
|
||||
|
||||
@@ -1,256 +0,0 @@
|
||||
package scanner
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/lukaszraczylo/gohoarder/pkg/config"
|
||||
hoardererrors "github.com/lukaszraczylo/gohoarder/pkg/errors"
|
||||
"github.com/lukaszraczylo/gohoarder/pkg/metadata"
|
||||
"github.com/lukaszraczylo/gohoarder/pkg/websocket"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// fakeBroadcaster records BroadcastEvent calls for assertions.
|
||||
type fakeBroadcaster struct {
|
||||
events []fakeBroadcastEvent
|
||||
mu sync.Mutex
|
||||
}
|
||||
|
||||
type fakeBroadcastEvent struct {
|
||||
Payload any
|
||||
Type string
|
||||
}
|
||||
|
||||
func (f *fakeBroadcaster) BroadcastEvent(eventType string, payload any) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
f.events = append(f.events, fakeBroadcastEvent{Type: eventType, Payload: payload})
|
||||
}
|
||||
|
||||
func (f *fakeBroadcaster) snapshot() []fakeBroadcastEvent {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
out := make([]fakeBroadcastEvent, len(f.events))
|
||||
copy(out, f.events)
|
||||
return out
|
||||
}
|
||||
|
||||
// stubScanner is a minimal Scanner implementation for tests.
|
||||
type stubScanner struct {
|
||||
result *metadata.ScanResult
|
||||
err error
|
||||
name string
|
||||
}
|
||||
|
||||
func (s *stubScanner) Name() string { return s.name }
|
||||
|
||||
func (s *stubScanner) Scan(_ context.Context, registry, packageName, version, _ string) (*metadata.ScanResult, error) {
|
||||
if s.err != nil {
|
||||
return nil, s.err
|
||||
}
|
||||
r := *s.result
|
||||
r.Registry = registry
|
||||
r.PackageName = packageName
|
||||
r.PackageVersion = version
|
||||
return &r, nil
|
||||
}
|
||||
|
||||
func (s *stubScanner) Health(context.Context) error { return nil }
|
||||
|
||||
// stubMetadataStore is a minimal MetadataStore — only SaveScanResult is exercised.
|
||||
type stubMetadataStore struct {
|
||||
saveErr error
|
||||
savedResults []*metadata.ScanResult
|
||||
mu sync.Mutex
|
||||
}
|
||||
|
||||
func (m *stubMetadataStore) SaveScanResult(_ context.Context, r *metadata.ScanResult) error {
|
||||
if m.saveErr != nil {
|
||||
return m.saveErr
|
||||
}
|
||||
m.mu.Lock()
|
||||
m.savedResults = append(m.savedResults, r)
|
||||
m.mu.Unlock()
|
||||
return nil
|
||||
}
|
||||
|
||||
// All other MetadataStore methods are unused by ScanPackage — stub to satisfy interface.
|
||||
func (m *stubMetadataStore) SavePackage(context.Context, *metadata.Package) error { return nil }
|
||||
func (m *stubMetadataStore) GetPackage(context.Context, string, string, string) (*metadata.Package, error) {
|
||||
return nil, hoardererrors.NotFound("not implemented")
|
||||
}
|
||||
func (m *stubMetadataStore) DeletePackage(context.Context, string, string, string) error { return nil }
|
||||
func (m *stubMetadataStore) ListPackages(context.Context, *metadata.ListOptions) ([]*metadata.Package, error) {
|
||||
return nil, nil
|
||||
}
|
||||
func (m *stubMetadataStore) UpdateDownloadCount(context.Context, string, string, string) error {
|
||||
return nil
|
||||
}
|
||||
func (m *stubMetadataStore) GetStats(context.Context, string) (*metadata.Stats, error) {
|
||||
return nil, nil
|
||||
}
|
||||
func (m *stubMetadataStore) GetScanResult(context.Context, string, string, string) (*metadata.ScanResult, error) {
|
||||
return nil, nil
|
||||
}
|
||||
func (m *stubMetadataStore) Count(context.Context) (int, error) { return 0, nil }
|
||||
func (m *stubMetadataStore) Health(context.Context) error { return nil }
|
||||
func (m *stubMetadataStore) Close() error { return nil }
|
||||
func (m *stubMetadataStore) SaveCVEBypass(context.Context, *metadata.CVEBypass) error {
|
||||
return nil
|
||||
}
|
||||
func (m *stubMetadataStore) GetActiveCVEBypasses(context.Context) ([]*metadata.CVEBypass, error) {
|
||||
return nil, nil
|
||||
}
|
||||
func (m *stubMetadataStore) ListCVEBypasses(context.Context, *metadata.BypassListOptions) ([]*metadata.CVEBypass, error) {
|
||||
return nil, nil
|
||||
}
|
||||
func (m *stubMetadataStore) DeleteCVEBypass(context.Context, string) error { return nil }
|
||||
func (m *stubMetadataStore) CleanupExpiredBypasses(context.Context) (int, error) {
|
||||
return 0, nil
|
||||
}
|
||||
func (m *stubMetadataStore) GetTimeSeriesStats(context.Context, string, string) (*metadata.TimeSeriesStats, error) {
|
||||
return nil, nil
|
||||
}
|
||||
func (m *stubMetadataStore) AggregateDownloadData(context.Context) error { return nil }
|
||||
func (m *stubMetadataStore) SaveAPIKey(context.Context, *metadata.APIKey) error {
|
||||
return metadata.ErrNotImplemented
|
||||
}
|
||||
func (m *stubMetadataStore) GetAPIKey(context.Context, string) (*metadata.APIKey, error) {
|
||||
return nil, metadata.ErrNotImplemented
|
||||
}
|
||||
func (m *stubMetadataStore) ListAPIKeys(context.Context) ([]*metadata.APIKey, error) {
|
||||
return nil, metadata.ErrNotImplemented
|
||||
}
|
||||
func (m *stubMetadataStore) DeleteAPIKey(context.Context, string) error {
|
||||
return metadata.ErrNotImplemented
|
||||
}
|
||||
func (m *stubMetadataStore) UpdateAPIKeyLastUsed(context.Context, string, time.Time) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func newTestManager(t *testing.T, store metadata.MetadataStore) *Manager {
|
||||
t.Helper()
|
||||
// Enabled=true but all built-in scanners disabled — we'll register
|
||||
// our own stub via RegisterScanner.
|
||||
cfg := config.SecurityConfig{Enabled: true}
|
||||
mgr, err := New(cfg, store)
|
||||
require.NoError(t, err)
|
||||
return mgr
|
||||
}
|
||||
|
||||
// TestBroadcaster_ScanCompleteSuccess verifies EventScanComplete fires
|
||||
// after a successful scan with the expected payload shape.
|
||||
func TestBroadcaster_ScanCompleteSuccess(t *testing.T) {
|
||||
store := &stubMetadataStore{}
|
||||
mgr := newTestManager(t, store)
|
||||
|
||||
result := &metadata.ScanResult{
|
||||
ID: "r1",
|
||||
Scanner: "stub",
|
||||
ScannedAt: time.Now(),
|
||||
Status: metadata.ScanStatusClean,
|
||||
VulnerabilityCount: 0,
|
||||
Vulnerabilities: []metadata.Vulnerability{},
|
||||
}
|
||||
mgr.RegisterScanner(&stubScanner{name: "stub", result: result})
|
||||
|
||||
bc := &fakeBroadcaster{}
|
||||
mgr.SetBroadcaster(bc)
|
||||
|
||||
err := mgr.ScanPackage(context.Background(), "npm", "react", "18.2.0", "/tmp/dummy")
|
||||
require.NoError(t, err)
|
||||
|
||||
events := bc.snapshot()
|
||||
require.Len(t, events, 1)
|
||||
assert.Equal(t, string(websocket.EventScanComplete), events[0].Type)
|
||||
|
||||
payload, ok := events[0].Payload.(map[string]interface{})
|
||||
require.True(t, ok)
|
||||
assert.Equal(t, "npm", payload["registry"])
|
||||
assert.Equal(t, "react", payload["name"])
|
||||
assert.Equal(t, "18.2.0", payload["version"])
|
||||
assert.Equal(t, string(metadata.ScanStatusClean), payload["status"])
|
||||
assert.Equal(t, 0, payload["vulnerability_count"])
|
||||
}
|
||||
|
||||
// TestBroadcaster_ScanCompleteAllScannersFailed verifies that the
|
||||
// synthetic-error path still fires EventScanComplete with status=error.
|
||||
func TestBroadcaster_ScanCompleteAllScannersFailed(t *testing.T) {
|
||||
store := &stubMetadataStore{}
|
||||
mgr := newTestManager(t, store)
|
||||
|
||||
mgr.RegisterScanner(&stubScanner{name: "broken", err: errors.New("scanner exploded")})
|
||||
|
||||
bc := &fakeBroadcaster{}
|
||||
mgr.SetBroadcaster(bc)
|
||||
|
||||
err := mgr.ScanPackage(context.Background(), "pypi", "requests", "2.31.0", "/tmp/dummy")
|
||||
require.NoError(t, err)
|
||||
|
||||
events := bc.snapshot()
|
||||
require.Len(t, events, 1)
|
||||
assert.Equal(t, string(websocket.EventScanComplete), events[0].Type)
|
||||
|
||||
payload := events[0].Payload.(map[string]interface{})
|
||||
assert.Equal(t, "pypi", payload["registry"])
|
||||
assert.Equal(t, "requests", payload["name"])
|
||||
assert.Equal(t, "2.31.0", payload["version"])
|
||||
assert.Equal(t, string(metadata.ScanStatusError), payload["status"])
|
||||
}
|
||||
|
||||
// TestBroadcaster_NoEmitOnSaveError verifies no event is emitted when
|
||||
// the metadata store fails to persist the scan result.
|
||||
func TestBroadcaster_NoEmitOnSaveError(t *testing.T) {
|
||||
store := &stubMetadataStore{saveErr: errors.New("db down")}
|
||||
mgr := newTestManager(t, store)
|
||||
|
||||
result := &metadata.ScanResult{
|
||||
ID: "r2",
|
||||
Scanner: "stub",
|
||||
ScannedAt: time.Now(),
|
||||
Status: metadata.ScanStatusClean,
|
||||
Vulnerabilities: []metadata.Vulnerability{},
|
||||
}
|
||||
mgr.RegisterScanner(&stubScanner{name: "stub", result: result})
|
||||
|
||||
bc := &fakeBroadcaster{}
|
||||
mgr.SetBroadcaster(bc)
|
||||
|
||||
err := mgr.ScanPackage(context.Background(), "npm", "x", "1", "/tmp/dummy")
|
||||
require.Error(t, err)
|
||||
assert.Empty(t, bc.snapshot(), "no event should fire when SaveScanResult fails")
|
||||
}
|
||||
|
||||
// TestBroadcaster_DisabledNoEmit verifies disabled scanner manager
|
||||
// silently no-ops and emits nothing.
|
||||
func TestBroadcaster_DisabledNoEmit(t *testing.T) {
|
||||
store := &stubMetadataStore{}
|
||||
cfg := config.SecurityConfig{Enabled: false}
|
||||
mgr, err := New(cfg, store)
|
||||
require.NoError(t, err)
|
||||
|
||||
bc := &fakeBroadcaster{}
|
||||
mgr.SetBroadcaster(bc)
|
||||
|
||||
err = mgr.ScanPackage(context.Background(), "npm", "x", "1", "/tmp/dummy")
|
||||
require.NoError(t, err)
|
||||
assert.Empty(t, bc.snapshot())
|
||||
}
|
||||
|
||||
// TestBroadcaster_NilBroadcasterSafe ensures nil broadcaster is safe.
|
||||
func TestBroadcaster_NilBroadcasterSafe(t *testing.T) {
|
||||
store := &stubMetadataStore{}
|
||||
mgr := newTestManager(t, store)
|
||||
mgr.RegisterScanner(&stubScanner{name: "stub", result: &metadata.ScanResult{
|
||||
ID: "r", Scanner: "stub", ScannedAt: time.Now(), Status: metadata.ScanStatusClean,
|
||||
}})
|
||||
|
||||
// No SetBroadcaster.
|
||||
err := mgr.ScanPackage(context.Background(), "npm", "x", "1", "/tmp/dummy")
|
||||
require.NoError(t, err)
|
||||
}
|
||||
@@ -1,5 +1,3 @@
|
||||
// Package trivy wraps the Aqua Security `trivy` CLI to scan packages for
|
||||
// known vulnerabilities and license issues.
|
||||
package trivy
|
||||
|
||||
import (
|
||||
|
||||
@@ -1,5 +1,3 @@
|
||||
// Package filesystem implements the local-disk Storage backend used for
|
||||
// development and single-node deployments.
|
||||
package filesystem
|
||||
|
||||
import (
|
||||
@@ -57,11 +55,7 @@ func (fs *FilesystemStorage) Get(ctx context.Context, key string) (io.ReadCloser
|
||||
default:
|
||||
}
|
||||
|
||||
path, err := fs.keyToPath(key)
|
||||
if err != nil {
|
||||
metrics.RecordStorageOperation("filesystem", "get", "error")
|
||||
return nil, err
|
||||
}
|
||||
path := fs.keyToPath(key)
|
||||
|
||||
file, err := os.Open(path) // #nosec G304 -- Path is sanitized storage key
|
||||
if err != nil {
|
||||
@@ -86,17 +80,13 @@ func (fs *FilesystemStorage) Put(ctx context.Context, key string, data io.Reader
|
||||
default:
|
||||
}
|
||||
|
||||
path, err := fs.keyToPath(key)
|
||||
if err != nil {
|
||||
metrics.RecordStorageOperation("filesystem", "put", "error")
|
||||
return err
|
||||
}
|
||||
path := fs.keyToPath(key)
|
||||
dir := filepath.Dir(path)
|
||||
|
||||
// Create directory
|
||||
if mkErr := os.MkdirAll(dir, 0750); mkErr != nil {
|
||||
if err := os.MkdirAll(dir, 0750); err != nil {
|
||||
metrics.RecordStorageOperation("filesystem", "put", "error")
|
||||
return errors.Wrap(mkErr, errors.ErrCodeStorageFailure, "failed to create directory")
|
||||
return errors.Wrap(err, errors.ErrCodeStorageFailure, "failed to create directory")
|
||||
}
|
||||
|
||||
// Create temp file for atomic write
|
||||
@@ -115,7 +105,7 @@ func (fs *FilesystemStorage) Put(ctx context.Context, key string, data io.Reader
|
||||
|
||||
written, err := io.Copy(multiWriter, data)
|
||||
if err != nil {
|
||||
_ = tempFile.Close() // #nosec G104 -- Cleanup, error not critical
|
||||
tempFile.Close() // #nosec G104 -- Cleanup, error not critical
|
||||
_ = os.Remove(tempPath) // #nosec G104 -- Cleanup, error not critical
|
||||
metrics.RecordStorageOperation("filesystem", "put", "error")
|
||||
return errors.Wrap(err, errors.ErrCodeStorageFailure, "failed to write data")
|
||||
@@ -127,6 +117,17 @@ func (fs *FilesystemStorage) Put(ctx context.Context, key string, data io.Reader
|
||||
return errors.Wrap(err, errors.ErrCodeStorageFailure, "failed to close temp file")
|
||||
}
|
||||
|
||||
// Check quota
|
||||
fs.mu.Lock()
|
||||
if fs.quota > 0 && fs.used+written > fs.quota {
|
||||
fs.mu.Unlock()
|
||||
_ = os.Remove(tempPath) // #nosec G104 -- Cleanup, error not critical
|
||||
metrics.RecordStorageOperation("filesystem", "put", "quota_exceeded")
|
||||
return errors.QuotaExceeded(fs.quota)
|
||||
}
|
||||
fs.used += written
|
||||
fs.mu.Unlock()
|
||||
|
||||
// Verify checksums if provided
|
||||
if opts != nil {
|
||||
md5Sum := hex.EncodeToString(md5Hash.Sum(nil))
|
||||
@@ -145,25 +146,21 @@ func (fs *FilesystemStorage) Put(ctx context.Context, key string, data io.Reader
|
||||
}
|
||||
}
|
||||
|
||||
// Atomic rename and quota update under lock so that fs.used reflects
|
||||
// only successfully renamed files. Quota check happens before increment
|
||||
// to avoid transient inflation seen by concurrent Puts.
|
||||
fs.mu.Lock()
|
||||
if fs.quota > 0 && fs.used+written > fs.quota {
|
||||
fs.mu.Unlock()
|
||||
_ = os.Remove(tempPath) // #nosec G104 -- Cleanup, error not critical
|
||||
metrics.RecordStorageOperation("filesystem", "put", "quota_exceeded")
|
||||
return errors.QuotaExceeded(fs.quota)
|
||||
}
|
||||
// Atomic rename
|
||||
if err := os.Rename(tempPath, path); err != nil {
|
||||
fs.mu.Unlock()
|
||||
_ = os.Remove(tempPath) // #nosec G104 -- Cleanup, error not critical
|
||||
fs.mu.Lock()
|
||||
fs.used -= written
|
||||
currentUsed := fs.used
|
||||
fs.mu.Unlock()
|
||||
metrics.RecordStorageOperation("filesystem", "put", "error")
|
||||
metrics.UpdateCacheSize("filesystem", currentUsed)
|
||||
return errors.Wrap(err, errors.ErrCodeStorageFailure, "failed to rename temp file")
|
||||
}
|
||||
fs.used += written
|
||||
|
||||
fs.mu.RLock()
|
||||
currentUsed := fs.used
|
||||
fs.mu.Unlock()
|
||||
fs.mu.RUnlock()
|
||||
|
||||
metrics.RecordStorageOperation("filesystem", "put", "success")
|
||||
metrics.UpdateCacheSize("filesystem", currentUsed)
|
||||
@@ -178,11 +175,7 @@ func (fs *FilesystemStorage) Delete(ctx context.Context, key string) error {
|
||||
default:
|
||||
}
|
||||
|
||||
path, err := fs.keyToPath(key)
|
||||
if err != nil {
|
||||
metrics.RecordStorageOperation("filesystem", "delete", "error")
|
||||
return err
|
||||
}
|
||||
path := fs.keyToPath(key)
|
||||
|
||||
// Get size before deletion
|
||||
info, err := os.Stat(path)
|
||||
@@ -220,11 +213,8 @@ func (fs *FilesystemStorage) Exists(ctx context.Context, key string) (bool, erro
|
||||
default:
|
||||
}
|
||||
|
||||
path, err := fs.keyToPath(key)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
_, err = os.Stat(path)
|
||||
path := fs.keyToPath(key)
|
||||
_, err := os.Stat(path)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return false, nil
|
||||
@@ -242,13 +232,10 @@ func (fs *FilesystemStorage) List(ctx context.Context, prefix string, opts *stor
|
||||
default:
|
||||
}
|
||||
|
||||
searchPath, err := fs.keyToPath(prefix)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
searchPath := fs.keyToPath(prefix)
|
||||
var objects []storage.StorageObject
|
||||
|
||||
err = filepath.Walk(searchPath, func(path string, info os.FileInfo, err error) error {
|
||||
err := filepath.Walk(searchPath, func(path string, info os.FileInfo, err error) error {
|
||||
if err != nil {
|
||||
return nil // Skip errors
|
||||
}
|
||||
@@ -297,10 +284,7 @@ func (fs *FilesystemStorage) Stat(ctx context.Context, key string) (*storage.Sto
|
||||
default:
|
||||
}
|
||||
|
||||
path, err := fs.keyToPath(key)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
path := fs.keyToPath(key)
|
||||
info, err := os.Stat(path)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
@@ -347,7 +331,7 @@ func (fs *FilesystemStorage) Health(ctx context.Context) error {
|
||||
if err != nil {
|
||||
return errors.Wrap(err, errors.ErrCodeStorageFailure, "cannot write to storage")
|
||||
}
|
||||
_ = f.Close() // #nosec G104 -- Cleanup, error not critical
|
||||
f.Close() // #nosec G104 -- Cleanup, error not critical
|
||||
_ = os.Remove(tempPath) // #nosec G104 -- Cleanup, error not critical
|
||||
|
||||
return nil
|
||||
@@ -368,10 +352,7 @@ func (fs *FilesystemStorage) GetLocalPath(ctx context.Context, key string) (stri
|
||||
default:
|
||||
}
|
||||
|
||||
path, err := fs.keyToPath(key)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
path := fs.keyToPath(key)
|
||||
|
||||
// Verify file exists
|
||||
if _, err := os.Stat(path); err != nil {
|
||||
@@ -384,46 +365,27 @@ func (fs *FilesystemStorage) GetLocalPath(ctx context.Context, key string) (stri
|
||||
return path, nil
|
||||
}
|
||||
|
||||
// keyToPath converts a storage key to filesystem path.
|
||||
// It sanitizes the key to prevent path traversal and verifies that the
|
||||
// resulting absolute path stays within the configured base directory as a
|
||||
// defense-in-depth check on top of filepath.Clean/Join semantics.
|
||||
func (fs *FilesystemStorage) keyToPath(key string) (string, error) {
|
||||
// keyToPath converts a storage key to filesystem path
|
||||
func (fs *FilesystemStorage) keyToPath(key string) string {
|
||||
// Sanitize key to prevent path traversal
|
||||
cleaned := filepath.Clean(key)
|
||||
key = filepath.Clean(key)
|
||||
|
||||
// Remove any leading slashes or dots
|
||||
cleaned = strings.TrimPrefix(cleaned, "/")
|
||||
key = strings.TrimPrefix(key, "/")
|
||||
|
||||
// Keep removing ../ until there are no more
|
||||
for strings.HasPrefix(cleaned, "../") || strings.HasPrefix(cleaned, "..\\") {
|
||||
cleaned = strings.TrimPrefix(cleaned, "../")
|
||||
cleaned = strings.TrimPrefix(cleaned, "..\\")
|
||||
for strings.HasPrefix(key, "../") || strings.HasPrefix(key, "..\\") {
|
||||
key = strings.TrimPrefix(key, "../")
|
||||
key = strings.TrimPrefix(key, "..\\")
|
||||
}
|
||||
|
||||
// Final clean and ensure it's within base path
|
||||
cleaned = filepath.Clean(cleaned)
|
||||
if cleaned == ".." || strings.HasPrefix(cleaned, "../") || strings.HasPrefix(cleaned, "..\\") {
|
||||
cleaned = ""
|
||||
key = filepath.Clean(key)
|
||||
if key == ".." || strings.HasPrefix(key, "../") || strings.HasPrefix(key, "..\\") {
|
||||
key = ""
|
||||
}
|
||||
|
||||
target := filepath.Join(fs.basePath, cleaned)
|
||||
|
||||
// Defense-in-depth: verify the resolved absolute path is contained
|
||||
// within the base directory.
|
||||
targetAbs, err := filepath.Abs(target)
|
||||
if err != nil {
|
||||
return "", errors.Wrap(err, errors.ErrCodeStorageFailure, "failed to resolve path")
|
||||
}
|
||||
baseAbs, err := filepath.Abs(fs.basePath)
|
||||
if err != nil {
|
||||
return "", errors.Wrap(err, errors.ErrCodeStorageFailure, "failed to resolve base path")
|
||||
}
|
||||
if targetAbs != baseAbs && !strings.HasPrefix(targetAbs, baseAbs+string(os.PathSeparator)) {
|
||||
return "", errors.New(errors.ErrCodeStorageFailure, fmt.Sprintf("path traversal rejected: %s", key))
|
||||
}
|
||||
|
||||
return target, nil
|
||||
return filepath.Join(fs.basePath, key)
|
||||
}
|
||||
|
||||
// calculateUsage calculates current storage usage
|
||||
|
||||
@@ -531,8 +531,8 @@ func (s *FilesystemStorageTestSuite) TestConcurrentReadsAndWrites() {
|
||||
key := fmt.Sprintf("shared/file-%d.txt", j%10)
|
||||
reader, err := s.fs.Get(ctx, key)
|
||||
if err == nil {
|
||||
_, _ = io.ReadAll(reader)
|
||||
_ = reader.Close() // #nosec G104 -- Cleanup, error not critical
|
||||
io.ReadAll(reader)
|
||||
reader.Close() // #nosec G104 -- Cleanup, error not critical
|
||||
}
|
||||
}
|
||||
}(i)
|
||||
@@ -546,7 +546,7 @@ func (s *FilesystemStorageTestSuite) TestConcurrentReadsAndWrites() {
|
||||
for j := 0; j < numOps; j++ {
|
||||
key := fmt.Sprintf("shared/writer-%d-%d.txt", id, j)
|
||||
data := fmt.Sprintf("writer-%d-%d", id, j)
|
||||
_ = s.fs.Put(ctx, key, strings.NewReader(data), nil)
|
||||
s.fs.Put(ctx, key, strings.NewReader(data), nil)
|
||||
}
|
||||
}(i)
|
||||
}
|
||||
@@ -608,15 +608,15 @@ func (s *FilesystemStorageTestSuite) TestAtomicWrite() {
|
||||
case <-stopReading:
|
||||
return
|
||||
default:
|
||||
reader, getErr := s.fs.Get(ctx, key)
|
||||
if getErr != nil {
|
||||
readErrors <- getErr
|
||||
reader, err := s.fs.Get(ctx, key)
|
||||
if err != nil {
|
||||
readErrors <- err
|
||||
continue
|
||||
}
|
||||
data, readErr := io.ReadAll(reader)
|
||||
data, err := io.ReadAll(reader)
|
||||
reader.Close() // #nosec G104 -- Cleanup, error not critical
|
||||
if readErr != nil {
|
||||
readErrors <- readErr
|
||||
if err != nil {
|
||||
readErrors <- err
|
||||
continue
|
||||
}
|
||||
// Data should be either "initial" or "updated", never partial
|
||||
@@ -663,8 +663,7 @@ func (s *FilesystemStorageTestSuite) TestPathSanitization() {
|
||||
s.NoError(err) // Should succeed but sanitize path
|
||||
|
||||
// Verify file is inside base directory
|
||||
sanitized, sanitizeErr := s.fs.keyToPath(path)
|
||||
s.NoError(sanitizeErr)
|
||||
sanitized := s.fs.keyToPath(path)
|
||||
s.True(strings.HasPrefix(sanitized, s.tempDir),
|
||||
"Sanitized path %s should be inside %s", sanitized, s.tempDir)
|
||||
})
|
||||
@@ -729,7 +728,7 @@ func BenchmarkFilesystemPut(b *testing.B) {
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
key := fmt.Sprintf("bench/file-%d.txt", i)
|
||||
_ = fs.Put(ctx, key, strings.NewReader(data), nil)
|
||||
fs.Put(ctx, key, strings.NewReader(data), nil)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -745,14 +744,14 @@ func BenchmarkFilesystemGet(b *testing.B) {
|
||||
data := strings.Repeat("x", 1024)
|
||||
|
||||
// Setup: Create test file
|
||||
_ = fs.Put(ctx, "bench/test.txt", strings.NewReader(data), nil)
|
||||
fs.Put(ctx, "bench/test.txt", strings.NewReader(data), nil)
|
||||
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
reader, _ := fs.Get(ctx, "bench/test.txt")
|
||||
if reader != nil {
|
||||
_, _ = io.ReadAll(reader)
|
||||
_ = reader.Close() // #nosec G104 -- Cleanup, error not critical
|
||||
io.ReadAll(reader)
|
||||
reader.Close() // #nosec G104 -- Cleanup, error not critical
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,5 +1,3 @@
|
||||
// Package storage defines the pluggable Storage backend interface used to
|
||||
// persist cached package payloads (filesystem, S3, NFS, etc.).
|
||||
package storage
|
||||
|
||||
import (
|
||||
|
||||
@@ -1,280 +0,0 @@
|
||||
// Package nfs implements an NFS-backed storage backend.
|
||||
//
|
||||
// NFS is, from Go's perspective, an ordinary mounted filesystem. The user is
|
||||
// expected to mount the export at cfg.Path before starting the application;
|
||||
// this package does NOT perform mount(8) calls. It wraps the filesystem
|
||||
// backend and adds NFS-specific safety:
|
||||
//
|
||||
// - Best-effort mount-type detection (Linux: /proc/mounts). On non-Linux
|
||||
// platforms detection is skipped silently. A non-NFS mount is logged at
|
||||
// Warn level but is NOT a fatal error so tests/CI can run on local
|
||||
// filesystems.
|
||||
//
|
||||
// - Optional per-write fsync (SyncWrites, default true) to flush NFS client
|
||||
// caches and improve durability across NFS-cached metadata. Stale handles
|
||||
// and "silent" write losses are common NFS pitfalls.
|
||||
//
|
||||
// - A richer Health probe that round-trips a marker file (write, fsync,
|
||||
// read, delete) to surface stale handles or read-after-write
|
||||
// inconsistencies the bare filesystem health check would miss.
|
||||
package nfs
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strings"
|
||||
|
||||
"github.com/lukaszraczylo/gohoarder/pkg/errors"
|
||||
"github.com/lukaszraczylo/gohoarder/pkg/storage"
|
||||
"github.com/lukaszraczylo/gohoarder/pkg/storage/filesystem"
|
||||
"github.com/rs/zerolog"
|
||||
)
|
||||
|
||||
// Config holds NFS storage configuration. The struct is intentionally
|
||||
// self-contained so callers can map their own config (e.g.
|
||||
// pkg/config.StorageConfig) without import cycles.
|
||||
type Config struct {
|
||||
// Path is the local mount point of the NFS export. Required.
|
||||
Path string
|
||||
// MaxSize is the optional quota in bytes (0 = unlimited). Forwarded to
|
||||
// the underlying filesystem backend.
|
||||
MaxSize int64
|
||||
// SyncWrites, when true (default), forces fsync after every successful
|
||||
// Put so data is flushed through the NFS client cache to the server.
|
||||
SyncWrites bool
|
||||
}
|
||||
|
||||
// Storage implements storage.StorageBackend on top of an NFS-mounted path.
|
||||
type Storage struct {
|
||||
fs *filesystem.FilesystemStorage
|
||||
logger zerolog.Logger
|
||||
path string
|
||||
syncWrites bool
|
||||
}
|
||||
|
||||
// New constructs an NFS storage backend rooted at cfg.Path.
|
||||
//
|
||||
// cfg.Path must already exist and be a directory; the caller is responsible
|
||||
// for the actual NFS mount. Mount-type detection is best-effort.
|
||||
func New(cfg Config, logger zerolog.Logger) (*Storage, error) {
|
||||
if cfg.Path == "" {
|
||||
return nil, errors.New(errors.ErrCodeStorageFailure, "nfs: path is required")
|
||||
}
|
||||
|
||||
info, err := os.Stat(cfg.Path)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, errors.ErrCodeStorageFailure, "nfs: path does not exist or is inaccessible")
|
||||
}
|
||||
if !info.IsDir() {
|
||||
return nil, errors.New(errors.ErrCodeStorageFailure, fmt.Sprintf("nfs: path is not a directory: %s", cfg.Path))
|
||||
}
|
||||
|
||||
// Best-effort mount-type detection. Non-fatal: warn only.
|
||||
if mountType, ok := detectMountType(cfg.Path); ok {
|
||||
if !isNFSMountType(mountType) {
|
||||
logger.Warn().
|
||||
Str("path", cfg.Path).
|
||||
Str("mount_type", mountType).
|
||||
Msg("nfs: configured path is not on an NFS mount; proceeding anyway")
|
||||
} else {
|
||||
logger.Info().
|
||||
Str("path", cfg.Path).
|
||||
Str("mount_type", mountType).
|
||||
Msg("nfs: detected NFS mount")
|
||||
}
|
||||
} else {
|
||||
// Detection unavailable (non-Linux or /proc/mounts unreadable).
|
||||
logger.Debug().
|
||||
Str("path", cfg.Path).
|
||||
Str("os", runtime.GOOS).
|
||||
Msg("nfs: mount-type detection skipped")
|
||||
}
|
||||
|
||||
fs, err := filesystem.New(cfg.Path, cfg.MaxSize)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &Storage{
|
||||
fs: fs,
|
||||
logger: logger,
|
||||
path: cfg.Path,
|
||||
syncWrites: cfg.SyncWrites,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Get delegates to the underlying filesystem backend.
|
||||
func (s *Storage) Get(ctx context.Context, key string) (io.ReadCloser, error) {
|
||||
return s.fs.Get(ctx, key)
|
||||
}
|
||||
|
||||
// Put delegates to the filesystem backend and, when SyncWrites is enabled,
|
||||
// fsyncs the resulting file to flush the NFS client cache.
|
||||
func (s *Storage) Put(ctx context.Context, key string, data io.Reader, opts *storage.PutOptions) error {
|
||||
if err := s.fs.Put(ctx, key, data, opts); err != nil {
|
||||
return err
|
||||
}
|
||||
if !s.syncWrites {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Resolve the on-disk path via the LocalPathProvider contract the
|
||||
// filesystem backend implements. Failure to fsync is logged but not
|
||||
// returned: the write itself succeeded; durability is best-effort.
|
||||
path, err := s.fs.GetLocalPath(ctx, key)
|
||||
if err != nil {
|
||||
s.logger.Warn().Err(err).Str("key", key).Msg("nfs: post-put path lookup failed; skipping fsync")
|
||||
return nil
|
||||
}
|
||||
f, err := os.OpenFile(path, os.O_RDWR, 0) // #nosec G304 -- path resolved by sanitizing backend
|
||||
if err != nil {
|
||||
s.logger.Warn().Err(err).Str("key", key).Msg("nfs: post-put open failed; skipping fsync")
|
||||
return nil
|
||||
}
|
||||
if syncErr := f.Sync(); syncErr != nil {
|
||||
s.logger.Warn().Err(syncErr).Str("key", key).Msg("nfs: post-put fsync failed")
|
||||
}
|
||||
_ = f.Close() // #nosec G104 -- close after sync, error not actionable
|
||||
return nil
|
||||
}
|
||||
|
||||
// Delete delegates to the underlying filesystem backend.
|
||||
func (s *Storage) Delete(ctx context.Context, key string) error {
|
||||
return s.fs.Delete(ctx, key)
|
||||
}
|
||||
|
||||
// Exists delegates to the underlying filesystem backend.
|
||||
func (s *Storage) Exists(ctx context.Context, key string) (bool, error) {
|
||||
return s.fs.Exists(ctx, key)
|
||||
}
|
||||
|
||||
// List delegates to the underlying filesystem backend.
|
||||
func (s *Storage) List(ctx context.Context, prefix string, opts *storage.ListOptions) ([]storage.StorageObject, error) {
|
||||
return s.fs.List(ctx, prefix, opts)
|
||||
}
|
||||
|
||||
// Stat delegates to the underlying filesystem backend.
|
||||
func (s *Storage) Stat(ctx context.Context, key string) (*storage.StorageInfo, error) {
|
||||
return s.fs.Stat(ctx, key)
|
||||
}
|
||||
|
||||
// GetQuota delegates to the underlying filesystem backend.
|
||||
func (s *Storage) GetQuota(ctx context.Context) (*storage.QuotaInfo, error) {
|
||||
return s.fs.GetQuota(ctx)
|
||||
}
|
||||
|
||||
// Health checks both the underlying filesystem and runs an NFS-specific
|
||||
// round-trip probe (write, fsync, read, delete) to surface stale handles or
|
||||
// cache-coherency issues that a bare stat would miss.
|
||||
func (s *Storage) Health(ctx context.Context) error {
|
||||
if err := s.fs.Health(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
probePath := filepath.Join(s.path, ".nfs_health_probe")
|
||||
payload := []byte("nfs-health-probe")
|
||||
|
||||
f, err := os.Create(probePath) // #nosec G304 -- path under configured base, fixed name
|
||||
if err != nil {
|
||||
return errors.Wrap(err, errors.ErrCodeStorageFailure, "nfs: cannot create health probe file")
|
||||
}
|
||||
if _, writeErr := f.Write(payload); writeErr != nil {
|
||||
_ = f.Close() // #nosec G104 -- cleanup
|
||||
_ = os.Remove(probePath) // #nosec G104 -- cleanup
|
||||
return errors.Wrap(writeErr, errors.ErrCodeStorageFailure, "nfs: cannot write health probe")
|
||||
}
|
||||
if syncErr := f.Sync(); syncErr != nil {
|
||||
_ = f.Close() // #nosec G104 -- cleanup
|
||||
_ = os.Remove(probePath) // #nosec G104 -- cleanup
|
||||
return errors.Wrap(syncErr, errors.ErrCodeStorageFailure, "nfs: fsync of health probe failed")
|
||||
}
|
||||
if closeErr := f.Close(); closeErr != nil {
|
||||
_ = os.Remove(probePath) // #nosec G104 -- cleanup
|
||||
return errors.Wrap(closeErr, errors.ErrCodeStorageFailure, "nfs: close of health probe failed")
|
||||
}
|
||||
|
||||
got, err := os.ReadFile(probePath) // #nosec G304 -- fixed probe path
|
||||
if err != nil {
|
||||
_ = os.Remove(probePath) // #nosec G104 -- cleanup
|
||||
return errors.Wrap(err, errors.ErrCodeStorageFailure, "nfs: read-back of health probe failed (possible stale handle)")
|
||||
}
|
||||
if string(got) != string(payload) {
|
||||
_ = os.Remove(probePath) // #nosec G104 -- cleanup
|
||||
return errors.New(errors.ErrCodeStorageFailure, "nfs: health probe payload mismatch (cache coherency issue?)")
|
||||
}
|
||||
if err := os.Remove(probePath); err != nil {
|
||||
return errors.Wrap(err, errors.ErrCodeStorageFailure, "nfs: cannot remove health probe file")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Close delegates to the underlying filesystem backend.
|
||||
func (s *Storage) Close() error {
|
||||
return s.fs.Close()
|
||||
}
|
||||
|
||||
// GetLocalPath exposes direct on-disk paths for scanning (NFS exports look
|
||||
// like local files to callers). Implements storage.LocalPathProvider.
|
||||
func (s *Storage) GetLocalPath(ctx context.Context, key string) (string, error) {
|
||||
return s.fs.GetLocalPath(ctx, key)
|
||||
}
|
||||
|
||||
// detectMountType returns the filesystem type backing path. Linux-only: on
|
||||
// other platforms the second return value is false. Implementation walks
|
||||
// /proc/mounts and selects the longest matching mount point, which is the
|
||||
// canonical way to find which mount owns a path.
|
||||
func detectMountType(path string) (string, bool) {
|
||||
if runtime.GOOS != "linux" {
|
||||
return "", false
|
||||
}
|
||||
|
||||
abs, err := filepath.Abs(path)
|
||||
if err != nil {
|
||||
return "", false
|
||||
}
|
||||
|
||||
data, err := os.ReadFile("/proc/mounts")
|
||||
if err != nil {
|
||||
return "", false
|
||||
}
|
||||
|
||||
var (
|
||||
bestMount string
|
||||
bestType string
|
||||
)
|
||||
for _, line := range strings.Split(string(data), "\n") {
|
||||
fields := strings.Fields(line)
|
||||
if len(fields) < 3 {
|
||||
continue
|
||||
}
|
||||
mountPoint := fields[1]
|
||||
fsType := fields[2]
|
||||
if abs == mountPoint || strings.HasPrefix(abs, strings.TrimRight(mountPoint, "/")+"/") {
|
||||
if len(mountPoint) > len(bestMount) {
|
||||
bestMount = mountPoint
|
||||
bestType = fsType
|
||||
}
|
||||
}
|
||||
}
|
||||
if bestMount == "" {
|
||||
return "", false
|
||||
}
|
||||
return bestType, true
|
||||
}
|
||||
|
||||
// isNFSMountType returns true for NFS family mount types.
|
||||
func isNFSMountType(t string) bool {
|
||||
switch t {
|
||||
case "nfs", "nfs4":
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
@@ -1,239 +0,0 @@
|
||||
package nfs
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/lukaszraczylo/gohoarder/pkg/storage"
|
||||
"github.com/rs/zerolog"
|
||||
)
|
||||
|
||||
// newTestStorage builds an NFS Storage rooted at t.TempDir(). It also returns
|
||||
// a buffer capturing the logger output so detection-related tests can assert
|
||||
// on log lines without requiring a real NFS mount.
|
||||
func newTestStorage(t *testing.T, syncWrites bool) (*Storage, *bytes.Buffer) {
|
||||
t.Helper()
|
||||
dir := t.TempDir()
|
||||
logBuf := &bytes.Buffer{}
|
||||
logger := zerolog.New(logBuf)
|
||||
s, err := New(Config{
|
||||
Path: dir,
|
||||
MaxSize: 1 << 20, // 1 MiB
|
||||
SyncWrites: syncWrites,
|
||||
}, logger)
|
||||
if err != nil {
|
||||
t.Fatalf("New: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { _ = s.Close() })
|
||||
return s, logBuf
|
||||
}
|
||||
|
||||
func TestNew_RejectsMissingPath(t *testing.T) {
|
||||
logger := zerolog.New(io.Discard)
|
||||
if _, err := New(Config{Path: ""}, logger); err == nil {
|
||||
t.Fatal("expected error for empty path")
|
||||
}
|
||||
}
|
||||
|
||||
func TestNew_RejectsNonDirectory(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
file := filepath.Join(dir, "not-a-dir")
|
||||
if err := os.WriteFile(file, []byte("x"), 0o600); err != nil {
|
||||
t.Fatalf("setup: %v", err)
|
||||
}
|
||||
logger := zerolog.New(io.Discard)
|
||||
if _, err := New(Config{Path: file}, logger); err == nil {
|
||||
t.Fatal("expected error when path is a file")
|
||||
}
|
||||
}
|
||||
|
||||
func TestNew_RejectsNonexistentPath(t *testing.T) {
|
||||
logger := zerolog.New(io.Discard)
|
||||
if _, err := New(Config{Path: "/nonexistent/path/does/not/exist"}, logger); err == nil {
|
||||
t.Fatal("expected error for missing path")
|
||||
}
|
||||
}
|
||||
|
||||
// TestNew_LogsWarnOnNonNFSMount: on Linux the temp dir lives on a non-NFS fs,
|
||||
// so detection should fire and log a warn. On other OSes detection is skipped
|
||||
// and we just assert New succeeds.
|
||||
func TestNew_LogsWarnOnNonNFSMount(t *testing.T) {
|
||||
s, logBuf := newTestStorage(t, true)
|
||||
if s == nil {
|
||||
t.Fatal("expected storage")
|
||||
}
|
||||
|
||||
if runtime.GOOS != "linux" {
|
||||
t.Skipf("mount detection only runs on linux; got %s", runtime.GOOS)
|
||||
}
|
||||
|
||||
out := logBuf.String()
|
||||
// Either a warn ("not on an NFS mount") or, if /proc/mounts is unreadable
|
||||
// inside the sandbox, a debug "detection skipped". Both are acceptable;
|
||||
// what we never want is a hard error.
|
||||
if !strings.Contains(out, "not on an NFS mount") &&
|
||||
!strings.Contains(out, "detection skipped") &&
|
||||
!strings.Contains(out, "detected NFS mount") {
|
||||
t.Fatalf("expected mount-detection log line, got: %q", out)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRoundTrip_PutGetStatDelete(t *testing.T) {
|
||||
s, _ := newTestStorage(t, true)
|
||||
ctx := context.Background()
|
||||
|
||||
const key = "pkgs/example/1.0.0/data.bin"
|
||||
payload := []byte("hello-nfs-roundtrip")
|
||||
|
||||
if err := s.Put(ctx, key, bytes.NewReader(payload), nil); err != nil {
|
||||
t.Fatalf("Put: %v", err)
|
||||
}
|
||||
|
||||
exists, err := s.Exists(ctx, key)
|
||||
if err != nil || !exists {
|
||||
t.Fatalf("Exists: got (%v, %v), want (true, nil)", exists, err)
|
||||
}
|
||||
|
||||
rc, err := s.Get(ctx, key)
|
||||
if err != nil {
|
||||
t.Fatalf("Get: %v", err)
|
||||
}
|
||||
got, err := io.ReadAll(rc)
|
||||
_ = rc.Close()
|
||||
if err != nil {
|
||||
t.Fatalf("ReadAll: %v", err)
|
||||
}
|
||||
if !bytes.Equal(got, payload) {
|
||||
t.Fatalf("payload mismatch: got %q want %q", got, payload)
|
||||
}
|
||||
|
||||
info, err := s.Stat(ctx, key)
|
||||
if err != nil {
|
||||
t.Fatalf("Stat: %v", err)
|
||||
}
|
||||
if info.Size != int64(len(payload)) {
|
||||
t.Fatalf("Stat size: got %d want %d", info.Size, len(payload))
|
||||
}
|
||||
|
||||
if delErr := s.Delete(ctx, key); delErr != nil {
|
||||
t.Fatalf("Delete: %v", delErr)
|
||||
}
|
||||
exists, err = s.Exists(ctx, key)
|
||||
if err != nil {
|
||||
t.Fatalf("Exists after delete: %v", err)
|
||||
}
|
||||
if exists {
|
||||
t.Fatal("expected key to be gone after delete")
|
||||
}
|
||||
}
|
||||
|
||||
func TestPut_NoSyncPath(t *testing.T) {
|
||||
// Same flow as round-trip, but with SyncWrites=false to exercise the
|
||||
// non-fsync branch.
|
||||
s, _ := newTestStorage(t, false)
|
||||
ctx := context.Background()
|
||||
if err := s.Put(ctx, "no-sync.txt", strings.NewReader("data"), nil); err != nil {
|
||||
t.Fatalf("Put: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestList(t *testing.T) {
|
||||
s, _ := newTestStorage(t, true)
|
||||
ctx := context.Background()
|
||||
keys := []string{"a/one.txt", "a/two.txt", "b/three.txt"}
|
||||
for _, k := range keys {
|
||||
if err := s.Put(ctx, k, strings.NewReader(k), nil); err != nil {
|
||||
t.Fatalf("Put %s: %v", k, err)
|
||||
}
|
||||
}
|
||||
|
||||
objs, err := s.List(ctx, "a", &storage.ListOptions{})
|
||||
if err != nil {
|
||||
t.Fatalf("List: %v", err)
|
||||
}
|
||||
if len(objs) != 2 {
|
||||
t.Fatalf("List(a): got %d objs want 2 (%v)", len(objs), objsKeys(objs))
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetQuota(t *testing.T) {
|
||||
s, _ := newTestStorage(t, true)
|
||||
ctx := context.Background()
|
||||
|
||||
q, err := s.GetQuota(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("GetQuota: %v", err)
|
||||
}
|
||||
if q.Limit != 1<<20 {
|
||||
t.Fatalf("Limit: got %d want %d", q.Limit, 1<<20)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHealth_OK(t *testing.T) {
|
||||
s, _ := newTestStorage(t, true)
|
||||
if err := s.Health(context.Background()); err != nil {
|
||||
t.Fatalf("Health: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHealth_LeavesNoProbeFile(t *testing.T) {
|
||||
s, _ := newTestStorage(t, true)
|
||||
if err := s.Health(context.Background()); err != nil {
|
||||
t.Fatalf("Health: %v", err)
|
||||
}
|
||||
probe := filepath.Join(s.path, ".nfs_health_probe")
|
||||
if _, err := os.Stat(probe); !os.IsNotExist(err) {
|
||||
t.Fatalf("expected probe file removed; stat err=%v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHealth_FailsWhenPathRemoved(t *testing.T) {
|
||||
s, _ := newTestStorage(t, true)
|
||||
// Remove the entire base dir under the backend's feet to simulate a
|
||||
// missing/stale mount. Health must surface that as an error.
|
||||
if err := os.RemoveAll(s.path); err != nil {
|
||||
t.Fatalf("setup: %v", err)
|
||||
}
|
||||
if err := s.Health(context.Background()); err == nil {
|
||||
t.Fatal("expected Health to fail after path removed")
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetLocalPath(t *testing.T) {
|
||||
s, _ := newTestStorage(t, true)
|
||||
ctx := context.Background()
|
||||
const key = "local/path/test.txt"
|
||||
if err := s.Put(ctx, key, strings.NewReader("data"), nil); err != nil {
|
||||
t.Fatalf("Put: %v", err)
|
||||
}
|
||||
p, err := s.GetLocalPath(ctx, key)
|
||||
if err != nil {
|
||||
t.Fatalf("GetLocalPath: %v", err)
|
||||
}
|
||||
if !strings.HasPrefix(p, s.path) {
|
||||
t.Fatalf("expected path under base; got %s (base %s)", p, s.path)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStorageBackend_InterfaceConformance(t *testing.T) {
|
||||
// Compile-time check the wrapper satisfies the public interface.
|
||||
var _ storage.StorageBackend = (*Storage)(nil)
|
||||
var _ storage.LocalPathProvider = (*Storage)(nil)
|
||||
}
|
||||
|
||||
// objsKeys is a small helper used in failure messages.
|
||||
func objsKeys(objs []storage.StorageObject) string {
|
||||
keys := make([]string, 0, len(objs))
|
||||
for _, o := range objs {
|
||||
keys = append(keys, o.Key)
|
||||
}
|
||||
b, _ := json.Marshal(keys)
|
||||
return string(b)
|
||||
}
|
||||
@@ -1,5 +1,3 @@
|
||||
// Package s3 implements the S3-compatible Storage backend (AWS S3,
|
||||
// MinIO, etc.).
|
||||
package s3
|
||||
|
||||
import (
|
||||
@@ -141,9 +139,9 @@ func (s *S3Storage) Put(ctx context.Context, key string, data io.Reader, opts *s
|
||||
|
||||
// Check quota if set
|
||||
if s.maxSizeBytes > 0 {
|
||||
currentUsage, usageErr := s.calculateUsage(ctx)
|
||||
if usageErr != nil {
|
||||
log.Warn().Err(usageErr).Msg("Failed to calculate current usage, skipping quota check")
|
||||
currentUsage, err := s.calculateUsage(ctx)
|
||||
if err != nil {
|
||||
log.Warn().Err(err).Msg("Failed to calculate current usage, skipping quota check")
|
||||
} else if currentUsage+size > s.maxSizeBytes {
|
||||
return errors.QuotaExceeded(s.maxSizeBytes)
|
||||
}
|
||||
|
||||
@@ -17,9 +17,9 @@ func TestS3StorageTestSuite(t *testing.T) {
|
||||
func (s *S3StorageTestSuite) TestNewS3Storage() {
|
||||
tests := []struct {
|
||||
name string
|
||||
errorMsg string
|
||||
config Config
|
||||
expectError bool
|
||||
errorMsg string
|
||||
}{
|
||||
{
|
||||
name: "valid config with credentials",
|
||||
@@ -175,8 +175,8 @@ func (s *S3StorageTestSuite) TestStripPrefix() {
|
||||
|
||||
func (s *S3StorageTestSuite) TestIsNotFoundError() {
|
||||
tests := []struct {
|
||||
err error
|
||||
name string
|
||||
err error
|
||||
expected bool
|
||||
}{
|
||||
{
|
||||
|
||||
+25
-59
@@ -1,4 +1,3 @@
|
||||
// Package smb implements the SMB/CIFS-backed Storage backend.
|
||||
package smb
|
||||
|
||||
import (
|
||||
@@ -189,17 +188,14 @@ func (c *smbConnection) close() {
|
||||
|
||||
// Get retrieves data from SMB share
|
||||
func (s *SMBStorage) Get(ctx context.Context, key string) (io.ReadCloser, error) {
|
||||
path, err := s.keyToPath(key)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
conn, err := s.getConnection()
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, errors.ErrCodeStorageFailure, "failed to get SMB connection")
|
||||
}
|
||||
defer s.returnConnection(conn)
|
||||
|
||||
path := s.keyToPath(key)
|
||||
|
||||
log.Debug().Str("key", path).Msg("Getting file from SMB")
|
||||
|
||||
// Open file
|
||||
@@ -226,23 +222,20 @@ func (s *SMBStorage) Get(ctx context.Context, key string) (io.ReadCloser, error)
|
||||
|
||||
// Put stores data on SMB share
|
||||
func (s *SMBStorage) Put(ctx context.Context, key string, data io.Reader, opts *storage.PutOptions) error {
|
||||
path, err := s.keyToPath(key)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
conn, err := s.getConnection()
|
||||
if err != nil {
|
||||
return errors.Wrap(err, errors.ErrCodeStorageFailure, "failed to get SMB connection")
|
||||
}
|
||||
defer s.returnConnection(conn)
|
||||
|
||||
path := s.keyToPath(key)
|
||||
|
||||
log.Debug().Str("key", path).Msg("Putting file to SMB")
|
||||
|
||||
// Ensure directory exists
|
||||
dir := filepath.Dir(path)
|
||||
if dirErr := s.ensureDir(conn, dir); dirErr != nil {
|
||||
return errors.Wrap(dirErr, errors.ErrCodeStorageFailure, "failed to create SMB directory")
|
||||
if err := s.ensureDir(conn, dir); err != nil {
|
||||
return errors.Wrap(err, errors.ErrCodeStorageFailure, "failed to create SMB directory")
|
||||
}
|
||||
|
||||
// Read data into buffer to check quota
|
||||
@@ -254,9 +247,9 @@ func (s *SMBStorage) Put(ctx context.Context, key string, data io.Reader, opts *
|
||||
|
||||
// Check quota if set
|
||||
if s.maxSizeBytes > 0 {
|
||||
currentUsage, usageErr := s.calculateUsage(conn)
|
||||
if usageErr != nil {
|
||||
log.Warn().Err(usageErr).Msg("Failed to calculate current usage, skipping quota check")
|
||||
currentUsage, err := s.calculateUsage(conn)
|
||||
if err != nil {
|
||||
log.Warn().Err(err).Msg("Failed to calculate current usage, skipping quota check")
|
||||
} else if currentUsage+size > s.maxSizeBytes {
|
||||
return errors.QuotaExceeded(s.maxSizeBytes)
|
||||
}
|
||||
@@ -267,11 +260,7 @@ func (s *SMBStorage) Put(ctx context.Context, key string, data io.Reader, opts *
|
||||
if err != nil {
|
||||
return errors.Wrap(err, errors.ErrCodeStorageFailure, "failed to create SMB file")
|
||||
}
|
||||
defer func() {
|
||||
if closeErr := file.Close(); closeErr != nil {
|
||||
log.Warn().Err(closeErr).Str("path", path).Msg("Failed to close SMB file after writing")
|
||||
}
|
||||
}()
|
||||
defer file.Close()
|
||||
|
||||
// Write data
|
||||
_, err = file.Write([]byte(buf.String()))
|
||||
@@ -297,8 +286,8 @@ func (s *SMBStorage) ensureDir(conn *smbConnection, path string) error {
|
||||
// Create parent directory first
|
||||
parent := filepath.Dir(path)
|
||||
if parent != path && parent != "." && parent != "/" {
|
||||
if parentErr := s.ensureDir(conn, parent); parentErr != nil {
|
||||
return parentErr
|
||||
if err := s.ensureDir(conn, parent); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
@@ -313,17 +302,14 @@ func (s *SMBStorage) ensureDir(conn *smbConnection, path string) error {
|
||||
|
||||
// Delete removes data from SMB share
|
||||
func (s *SMBStorage) Delete(ctx context.Context, key string) error {
|
||||
path, err := s.keyToPath(key)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
conn, err := s.getConnection()
|
||||
if err != nil {
|
||||
return errors.Wrap(err, errors.ErrCodeStorageFailure, "failed to get SMB connection")
|
||||
}
|
||||
defer s.returnConnection(conn)
|
||||
|
||||
path := s.keyToPath(key)
|
||||
|
||||
log.Debug().Str("key", path).Msg("Deleting file from SMB")
|
||||
|
||||
err = conn.share.Remove(path)
|
||||
@@ -336,17 +322,14 @@ func (s *SMBStorage) Delete(ctx context.Context, key string) error {
|
||||
|
||||
// Exists checks if data exists on SMB share
|
||||
func (s *SMBStorage) Exists(ctx context.Context, key string) (bool, error) {
|
||||
path, err := s.keyToPath(key)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
conn, err := s.getConnection()
|
||||
if err != nil {
|
||||
return false, errors.Wrap(err, errors.ErrCodeStorageFailure, "failed to get SMB connection")
|
||||
}
|
||||
defer s.returnConnection(conn)
|
||||
|
||||
path := s.keyToPath(key)
|
||||
|
||||
_, err = conn.share.Stat(path)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
@@ -360,17 +343,14 @@ func (s *SMBStorage) Exists(ctx context.Context, key string) (bool, error) {
|
||||
|
||||
// List returns a list of objects with the given prefix
|
||||
func (s *SMBStorage) List(ctx context.Context, prefix string, opts *storage.ListOptions) ([]storage.StorageObject, error) {
|
||||
basePath, err := s.keyToPath(prefix)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
conn, err := s.getConnection()
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, errors.ErrCodeStorageFailure, "failed to get SMB connection")
|
||||
}
|
||||
defer s.returnConnection(conn)
|
||||
|
||||
basePath := s.keyToPath(prefix)
|
||||
|
||||
log.Debug().Str("prefix", basePath).Msg("Listing files in SMB")
|
||||
|
||||
var objects []storage.StorageObject
|
||||
@@ -442,17 +422,14 @@ func (s *SMBStorage) walkPath(conn *smbConnection, root string, fn func(string,
|
||||
|
||||
// Stat returns metadata about stored data
|
||||
func (s *SMBStorage) Stat(ctx context.Context, key string) (*storage.StorageInfo, error) {
|
||||
path, err := s.keyToPath(key)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
conn, err := s.getConnection()
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, errors.ErrCodeStorageFailure, "failed to get SMB connection")
|
||||
}
|
||||
defer s.returnConnection(conn)
|
||||
|
||||
path := s.keyToPath(key)
|
||||
|
||||
info, err := conn.share.Stat(path)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
@@ -517,28 +494,17 @@ func (s *SMBStorage) Close() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// keyToPath converts a storage key to SMB path. It rejects keys that
|
||||
// contain traversal segments ("..") or empty segments to prevent escaping
|
||||
// the configured base path.
|
||||
func (s *SMBStorage) keyToPath(key string) (string, error) {
|
||||
// Reject traversal attempts. Split by both forward and backslash so
|
||||
// callers can use either separator on input.
|
||||
normalized := strings.ReplaceAll(key, "\\", "/")
|
||||
for _, seg := range strings.Split(normalized, "/") {
|
||||
if seg == ".." || seg == "." {
|
||||
return "", errors.New(errors.ErrCodeStorageFailure, fmt.Sprintf("invalid key segment %q in %q", seg, key))
|
||||
}
|
||||
}
|
||||
|
||||
// keyToPath converts a storage key to SMB path
|
||||
func (s *SMBStorage) keyToPath(key string) string {
|
||||
// Normalize separators to backslash for SMB
|
||||
winKey := strings.ReplaceAll(key, "/", "\\")
|
||||
key = strings.ReplaceAll(key, "/", "\\")
|
||||
|
||||
if s.config.Path == "" {
|
||||
return winKey, nil
|
||||
return key
|
||||
}
|
||||
|
||||
// Use backslash for SMB paths
|
||||
return s.config.Path + "\\" + winKey, nil
|
||||
return s.config.Path + "\\" + key
|
||||
}
|
||||
|
||||
// pathToKey converts an SMB path to storage key
|
||||
|
||||
@@ -141,8 +141,7 @@ func (s *SMBStorageTestSuite) TestKeyToPath() {
|
||||
},
|
||||
}
|
||||
|
||||
result, err := storage.keyToPath(tt.key)
|
||||
s.NoError(err)
|
||||
result := storage.keyToPath(tt.key)
|
||||
s.Equal(tt.expectedWin, result)
|
||||
})
|
||||
}
|
||||
|
||||
@@ -1,5 +1,3 @@
|
||||
// Package uuid generates RFC 4122 v4 UUIDs used as identifiers across
|
||||
// the GoHoarder service.
|
||||
package uuid
|
||||
|
||||
import (
|
||||
|
||||
@@ -1,5 +1,3 @@
|
||||
// Package vcs provides Git/VCS helpers for resolving Go modules and
|
||||
// extracting repository credentials.
|
||||
package vcs
|
||||
|
||||
import (
|
||||
|
||||
+6
-46
@@ -6,33 +6,12 @@ import (
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/rs/zerolog/log"
|
||||
)
|
||||
|
||||
// versionPattern restricts version strings to characters that cannot be
|
||||
// confused with git command-line options. We additionally reject any leading
|
||||
// '-' to defend against argv option-injection (e.g. "--upload-pack=...").
|
||||
var versionPattern = regexp.MustCompile(`^v?[0-9A-Za-z.\-+]+$`)
|
||||
|
||||
// validateVersion ensures a user-supplied version is safe to pass as an argv
|
||||
// argument to git. Empty input and option-like prefixes are rejected.
|
||||
func validateVersion(version string) error {
|
||||
if version == "" {
|
||||
return fmt.Errorf("version is required")
|
||||
}
|
||||
if strings.HasPrefix(version, "-") {
|
||||
return fmt.Errorf("invalid version %q: must not start with '-'", version)
|
||||
}
|
||||
if !versionPattern.MatchString(version) {
|
||||
return fmt.Errorf("invalid version %q: must match %s", version, versionPattern.String())
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// GitFetcher handles git repository operations
|
||||
type GitFetcher struct {
|
||||
credStore *CredentialStore
|
||||
@@ -60,12 +39,6 @@ func NewGitFetcher(workDir string, credStore *CredentialStore) *GitFetcher {
|
||||
// FetchModule clones a git repository and checks out a specific version
|
||||
// Returns the path to the checked-out source directory
|
||||
func (g *GitFetcher) FetchModule(ctx context.Context, modulePath, version, credentials string) (string, error) {
|
||||
// Validate version before it reaches any git argv to prevent
|
||||
// option-injection (e.g. "--upload-pack=..." passed as a "version").
|
||||
if err := validateVersion(version); err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
// Create context with timeout
|
||||
ctx, cancel := context.WithTimeout(ctx, g.timeout)
|
||||
defer cancel()
|
||||
@@ -142,10 +115,8 @@ func (g *GitFetcher) modulePathToRepoURL(modulePath string) (string, error) {
|
||||
owner := parts[1]
|
||||
repo := parts[2]
|
||||
|
||||
// NOTE: Go module major-version suffixes appear as separate path
|
||||
// segments (e.g. "github.com/owner/repo/v2"), never as a "v" prefix on
|
||||
// the repo name. Stripping a leading "v" from the repo segment would
|
||||
// corrupt legitimate names like "vault", "vitess", "vim-go".
|
||||
// Remove version suffix if present (e.g., /v2, /v3)
|
||||
repo = strings.TrimPrefix(repo, "v")
|
||||
|
||||
repoURL := fmt.Sprintf("https://%s/%s/%s.git", host, owner, repo)
|
||||
return repoURL, nil
|
||||
@@ -252,12 +223,7 @@ func (g *GitFetcher) extractHost(repoURL string) string {
|
||||
|
||||
// shallowClone performs a shallow clone of a specific version
|
||||
func (g *GitFetcher) shallowClone(ctx context.Context, repoURL, version, cloneDir string, credentialHelper map[string]string) error {
|
||||
if err := validateVersion(version); err != nil {
|
||||
return err
|
||||
}
|
||||
// "--" separates options from positional args; combined with version
|
||||
// validation it prevents option-injection.
|
||||
cmd := exec.CommandContext(ctx, "git", "clone", "--depth", "1", "--branch", version, "--", repoURL, cloneDir)
|
||||
cmd := exec.CommandContext(ctx, "git", "clone", "--depth", "1", "--branch", version, repoURL, cloneDir)
|
||||
cmd.Env = append(os.Environ(), g.envMapToSlice(credentialHelper)...)
|
||||
|
||||
output, err := cmd.CombinedOutput()
|
||||
@@ -270,7 +236,7 @@ func (g *GitFetcher) shallowClone(ctx context.Context, repoURL, version, cloneDi
|
||||
|
||||
// fullClone performs a full clone of the repository
|
||||
func (g *GitFetcher) fullClone(ctx context.Context, repoURL, cloneDir string, credentialHelper map[string]string) error {
|
||||
cmd := exec.CommandContext(ctx, "git", "clone", "--", repoURL, cloneDir)
|
||||
cmd := exec.CommandContext(ctx, "git", "clone", repoURL, cloneDir)
|
||||
cmd.Env = append(os.Environ(), g.envMapToSlice(credentialHelper)...)
|
||||
|
||||
output, err := cmd.CombinedOutput()
|
||||
@@ -281,15 +247,9 @@ func (g *GitFetcher) fullClone(ctx context.Context, repoURL, cloneDir string, cr
|
||||
return nil
|
||||
}
|
||||
|
||||
// checkout checks out a specific version (tag, branch, or commit) in
|
||||
// detached-HEAD mode. Detached mode avoids ambiguity between local branch
|
||||
// refs and remote tags/commits and prevents creating unintended local
|
||||
// branches when the version happens to share a name with one.
|
||||
// checkout checks out a specific version (tag, branch, or commit)
|
||||
func (g *GitFetcher) checkout(ctx context.Context, repoDir, version string) error {
|
||||
if err := validateVersion(version); err != nil {
|
||||
return err
|
||||
}
|
||||
cmd := exec.CommandContext(ctx, "git", "checkout", "--detach", "--", version)
|
||||
cmd := exec.CommandContext(ctx, "git", "checkout", version)
|
||||
cmd.Dir = repoDir
|
||||
cmd.Env = append(os.Environ(), "GIT_TERMINAL_PROMPT=0")
|
||||
|
||||
|
||||
+2
-2
@@ -57,7 +57,7 @@ func (b *ModuleBuilder) BuildModuleZip(ctx context.Context, srcPath, modulePath,
|
||||
prefix := fmt.Sprintf("%s@%s/", modulePath, version)
|
||||
for _, relPath := range files {
|
||||
if err := b.addFileToZip(zipWriter, srcPath, relPath, prefix); err != nil {
|
||||
_ = zipWriter.Close() // #nosec G104 -- best-effort cleanup on error path
|
||||
zipWriter.Close() // #nosec G104 -- Cleanup, error not critical
|
||||
return nil, fmt.Errorf("failed to add file %s: %w", relPath, err)
|
||||
}
|
||||
}
|
||||
@@ -152,7 +152,7 @@ func (b *ModuleBuilder) addFileToZip(zipWriter *zip.Writer, srcPath, relPath, pr
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() { _ = file.Close() }() // #nosec G104 -- read-only file, close error not actionable
|
||||
defer file.Close() // #nosec G104 -- Cleanup, error not critical
|
||||
|
||||
if _, err := io.Copy(writer, file); err != nil {
|
||||
return err
|
||||
|
||||
+13
-59
@@ -1,5 +1,3 @@
|
||||
// Package websocket implements the realtime event broadcasting server used
|
||||
// by the dashboard frontend.
|
||||
package websocket
|
||||
|
||||
import (
|
||||
@@ -39,16 +37,6 @@ type Client struct {
|
||||
server *Server
|
||||
subscriptions map[EventType]bool
|
||||
mu sync.RWMutex
|
||||
closeOnce sync.Once
|
||||
}
|
||||
|
||||
// closeSend safely closes the client's send channel exactly once.
|
||||
// Safe to call concurrently from multiple goroutines (e.g. run loop
|
||||
// unregister and shutdown closeAllClients).
|
||||
func (c *Client) closeSend() {
|
||||
c.closeOnce.Do(func() {
|
||||
close(c.send)
|
||||
})
|
||||
}
|
||||
|
||||
// Server manages WebSocket connections and event broadcasting
|
||||
@@ -80,7 +68,7 @@ func NewServer(cfg Config) *Server {
|
||||
clients: make(map[*Client]bool),
|
||||
broadcast: make(chan Event, 256),
|
||||
register: make(chan *Client),
|
||||
unregister: make(chan *Client, 256),
|
||||
unregister: make(chan *Client),
|
||||
upgrader: websocket.Upgrader{
|
||||
ReadBufferSize: cfg.ReadBufferSize,
|
||||
WriteBufferSize: cfg.WriteBufferSize,
|
||||
@@ -121,7 +109,7 @@ func (s *Server) run(ctx context.Context) {
|
||||
s.mu.Lock()
|
||||
if _, ok := s.clients[client]; ok {
|
||||
delete(s.clients, client)
|
||||
client.closeSend()
|
||||
close(client.send)
|
||||
}
|
||||
s.mu.Unlock()
|
||||
log.Debug().
|
||||
@@ -159,15 +147,10 @@ func (s *Server) broadcastEvent(event Event) {
|
||||
select {
|
||||
case client.send <- message:
|
||||
default:
|
||||
// Client send buffer full - schedule unregister.
|
||||
// Non-blocking send; unregister chan is buffered, so we
|
||||
// avoid spawning goroutines that pile up under a stuck
|
||||
// run() consumer. If the buffer is full the client will
|
||||
// be cleaned up on its next failed write/ping.
|
||||
select {
|
||||
case s.unregister <- client:
|
||||
default:
|
||||
}
|
||||
// Client send buffer full - close connection
|
||||
go func(c *Client) {
|
||||
s.unregister <- c
|
||||
}(client)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -190,11 +173,9 @@ func (s *Server) pingClients() {
|
||||
time.Now().Add(10*time.Second),
|
||||
); err != nil {
|
||||
log.Debug().Err(err).Msg("Failed to ping client")
|
||||
// Non-blocking send; see broadcastEvent for rationale.
|
||||
select {
|
||||
case s.unregister <- client:
|
||||
default:
|
||||
}
|
||||
go func(c *Client) {
|
||||
s.unregister <- c
|
||||
}(client)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -205,8 +186,8 @@ func (s *Server) closeAllClients() {
|
||||
defer s.mu.Unlock()
|
||||
|
||||
for client := range s.clients {
|
||||
_ = client.conn.Close() // #nosec G104 -- Cleanup, error not critical
|
||||
client.closeSend()
|
||||
client.conn.Close() // #nosec G104 -- Cleanup, error not critical
|
||||
close(client.send)
|
||||
}
|
||||
s.clients = make(map[*Client]bool)
|
||||
}
|
||||
@@ -226,33 +207,6 @@ func (s *Server) Broadcast(eventType EventType, data map[string]interface{}) {
|
||||
}
|
||||
}
|
||||
|
||||
// BroadcastEvent satisfies the events.Broadcaster contract used by
|
||||
// cache/scanner sub-systems. It accepts a string event type and an
|
||||
// arbitrary payload (typically map[string]any) and forwards to the
|
||||
// typed Broadcast path.
|
||||
//
|
||||
// Non-blocking: enqueues onto the broadcast channel and drops the
|
||||
// event if the channel is full. Safe for concurrent use.
|
||||
func (s *Server) BroadcastEvent(eventType string, payload any) {
|
||||
var data map[string]interface{}
|
||||
switch p := payload.(type) {
|
||||
case map[string]interface{}:
|
||||
data = p
|
||||
case nil:
|
||||
data = map[string]interface{}{}
|
||||
default:
|
||||
// Wrap unknown payloads so the wire format stays consistent.
|
||||
// Sub-systems are expected to pass map[string]any; this branch
|
||||
// is defensive only.
|
||||
log.Warn().
|
||||
Str("event_type", eventType).
|
||||
Msg("BroadcastEvent received non-map payload; wrapping under 'payload' key")
|
||||
data = map[string]interface{}{"payload": payload}
|
||||
}
|
||||
|
||||
s.Broadcast(EventType(eventType), data)
|
||||
}
|
||||
|
||||
// HandleWebSocket upgrades HTTP connection to WebSocket
|
||||
func (s *Server) HandleWebSocket(w http.ResponseWriter, r *http.Request) {
|
||||
conn, err := s.upgrader.Upgrade(w, r, nil)
|
||||
@@ -283,7 +237,7 @@ func (s *Server) HandleWebSocket(w http.ResponseWriter, r *http.Request) {
|
||||
func (c *Client) readPump() {
|
||||
defer func() {
|
||||
c.server.unregister <- c
|
||||
_ = c.conn.Close() // #nosec G104 -- Cleanup, error not critical
|
||||
c.conn.Close() // #nosec G104 -- Cleanup, error not critical
|
||||
}()
|
||||
|
||||
_ = c.conn.SetReadDeadline(time.Now().Add(60 * time.Second)) // #nosec G104 -- Websocket deadline
|
||||
@@ -311,7 +265,7 @@ func (c *Client) writePump() {
|
||||
ticker := time.NewTicker(54 * time.Second)
|
||||
defer func() {
|
||||
ticker.Stop()
|
||||
_ = c.conn.Close() // #nosec G104 -- Cleanup, error not critical
|
||||
c.conn.Close() // #nosec G104 -- Cleanup, error not critical
|
||||
}()
|
||||
|
||||
for {
|
||||
|
||||
@@ -1,184 +0,0 @@
|
||||
package websocket
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"runtime"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
gws "github.com/gorilla/websocket"
|
||||
)
|
||||
|
||||
// TestCloseSendIdempotent ensures double-close on client.send is safe.
|
||||
func TestCloseSendIdempotent(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
calls int
|
||||
}{
|
||||
{name: "single", calls: 1},
|
||||
{name: "double", calls: 2},
|
||||
{name: "many", calls: 16},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
c := &Client{send: make(chan []byte, 1)}
|
||||
var wg sync.WaitGroup
|
||||
wg.Add(tc.calls)
|
||||
for i := 0; i < tc.calls; i++ {
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
t.Errorf("closeSend panicked: %v", r)
|
||||
}
|
||||
}()
|
||||
c.closeSend()
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestUnregisterChannelBuffered verifies unregister channel is buffered so
|
||||
// non-blocking sends from broadcastEvent/pingClients don't block or leak
|
||||
// goroutines under a stuck consumer.
|
||||
func TestUnregisterChannelBuffered(t *testing.T) {
|
||||
s := NewServer(Config{})
|
||||
if cap(s.unregister) < 256 {
|
||||
t.Fatalf("unregister channel cap=%d, want >=256", cap(s.unregister))
|
||||
}
|
||||
}
|
||||
|
||||
// TestConcurrentConnectionsCloseNoPanic spawns N concurrent websocket clients,
|
||||
// closes them all, and asserts no panic. Also performs a coarse goroutine-leak
|
||||
// check (pre/post counts within a small delta).
|
||||
func TestConcurrentConnectionsCloseNoPanic(t *testing.T) {
|
||||
startGoros := runtime.NumGoroutine()
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
|
||||
srv := NewServer(Config{})
|
||||
srv.Start(ctx)
|
||||
|
||||
httpSrv := httptest.NewServer(http.HandlerFunc(srv.HandleWebSocket))
|
||||
defer httpSrv.Close()
|
||||
|
||||
wsURL, err := url.Parse(httpSrv.URL)
|
||||
if err != nil {
|
||||
t.Fatalf("parse url: %v", err)
|
||||
}
|
||||
wsURL.Scheme = "ws"
|
||||
|
||||
const N = 32
|
||||
conns := make([]*gws.Conn, 0, N)
|
||||
for i := 0; i < N; i++ {
|
||||
c, _, err := gws.DefaultDialer.Dial(wsURL.String(), nil)
|
||||
if err != nil {
|
||||
t.Fatalf("dial %d: %v", i, err)
|
||||
}
|
||||
conns = append(conns, c)
|
||||
}
|
||||
|
||||
// Allow registrations to settle.
|
||||
deadline := time.Now().Add(2 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
srv.mu.RLock()
|
||||
n := len(srv.clients)
|
||||
srv.mu.RUnlock()
|
||||
if n == N {
|
||||
break
|
||||
}
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
|
||||
// Concurrent close from many goroutines simulates burst disconnect.
|
||||
var wg sync.WaitGroup
|
||||
for _, c := range conns {
|
||||
wg.Add(1)
|
||||
go func(c *gws.Conn) {
|
||||
defer wg.Done()
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
t.Errorf("close panicked: %v", r)
|
||||
}
|
||||
}()
|
||||
_ = c.Close()
|
||||
}(c)
|
||||
}
|
||||
wg.Wait()
|
||||
|
||||
// Trigger shutdown — closeAllClients runs concurrently with any pending
|
||||
// in-loop unregisters. This is the double-close race scenario.
|
||||
cancel()
|
||||
|
||||
// Give run loop and pumps time to exit.
|
||||
time.Sleep(500 * time.Millisecond)
|
||||
runtime.GC()
|
||||
|
||||
endGoros := runtime.NumGoroutine()
|
||||
// Allow some slack: test runtime, pollers, etc. We just want to catch
|
||||
// catastrophic per-connection leaks (would scale with N=32).
|
||||
if endGoros > startGoros+8 {
|
||||
t.Errorf("possible goroutine leak: start=%d end=%d", startGoros, endGoros)
|
||||
}
|
||||
}
|
||||
|
||||
// TestBroadcastWithFullClientBufferDoesNotLeakGoroutines verifies the
|
||||
// non-blocking unregister send path: a client with a full send buffer should
|
||||
// be unregistered without spawning per-event goroutines.
|
||||
func TestBroadcastWithFullClientBufferDoesNotLeakGoroutines(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
|
||||
s := NewServer(Config{})
|
||||
// Don't Start — drive run-loop interactions manually so we can test the
|
||||
// non-blocking unregister send without races on the real loop.
|
||||
go s.run(ctx)
|
||||
|
||||
// Inject a fake client with a full send buffer.
|
||||
c := &Client{
|
||||
send: make(chan []byte, 1),
|
||||
subscriptions: make(map[EventType]bool),
|
||||
}
|
||||
c.send <- []byte("filler") // buffer now full
|
||||
s.register <- c
|
||||
|
||||
// Wait for registration.
|
||||
deadline := time.Now().Add(time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
s.mu.RLock()
|
||||
_, ok := s.clients[c]
|
||||
s.mu.RUnlock()
|
||||
if ok {
|
||||
break
|
||||
}
|
||||
time.Sleep(5 * time.Millisecond)
|
||||
}
|
||||
|
||||
// Broadcast many events. Buffer is full, so each will hit the default
|
||||
// branch and try to non-blocking-send to s.unregister.
|
||||
for i := 0; i < 1000; i++ {
|
||||
s.Broadcast(EventStatsUpdate, map[string]interface{}{"n": i})
|
||||
}
|
||||
|
||||
// Wait for client to be unregistered.
|
||||
deadline = time.Now().Add(2 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
s.mu.RLock()
|
||||
_, ok := s.clients[c]
|
||||
n := len(s.clients)
|
||||
s.mu.RUnlock()
|
||||
if !ok && n == 0 {
|
||||
return
|
||||
}
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
t.Fatal("client was not unregistered after broadcast spam with full buffer")
|
||||
}
|
||||
Reference in New Issue
Block a user