mirror of
https://github.com/lukaszraczylo/gohoarder.git
synced 2026-07-13 04:57:11 +00:00
e39d6a0f0d
Multi-agent audit + fix pass covering bugs, security, dead config wiring,
and missing features. All quality gates green: build/vet/test -race/govulncheck/
golangci-lint. Frontend tests 47/47.
SECURITY & CORRECTNESS
- Scanner pipeline fail-closed: cache.go scan timeout now 503 (was goto servePkg);
scanner.go all-scanners-fail saves ScanStatusError; CheckVulnerabilities blocks
on missing/error result instead of allowing.
- Scanner argv corrections: govulncheck source-mode + go.mod discovery;
npm-audit generates lockfile via npm install --package-lock-only --ignore-scripts;
pip-audit per-extension dispatch (wheel direct / sdist extract+pyproject).
- GHSA version-range filtering implemented (was always-include); url.QueryEscape
on package name.
- pypi SSRF closed via host allowlist on original_url; url.QueryEscape on
rewriteURL output.
- Path traversal: cache temp file uses os.CreateTemp; smb keyToPath sanitized;
filesystem keyToPath returns error + filepath.Abs prefix-verify.
- Goroutine leaks plugged: cache.cleanupWorker stop channel; auth.ValidationCache
Stop() with sync.Once; ws.unregister buffered with non-blocking send.
- WebSocket double-close panic fixed via sync.Once Client.closeSend().
- Race conditions: gormstore.registryCache sync.RWMutex (was concurrent map
panic); auth.LastUsedAt no longer mutated under RLock; analytics strict
lock-ordering invariant (statsMu before downloadsMu).
- Auth validator fails CLOSED on transport/unknown-status errors (was returning
true,err 'allow cache fallback').
- Credential cache hash full SHA256 (was 8-byte truncate; eliminates 2^32
collision risk).
- filesystem.fs.used incremented after successful rename (no quota inflation
race).
- gormstore aggregation events deleted in same tx as insert (no double-counting).
- gormstore.SavePackage uses Updates(map) so zero-value security flags
(RequiresAuth=false) can be cleared.
- partition_manager validates partition name regex before DROP TABLE.
- vcs/git: version regex validation rejects --option-injection; checkout uses
--detach -- separator; removed TrimPrefix(repo,'v') that corrupted vault/
vitess/vim-go.
- WebSocket CheckOrigin allowlist (was return true; CSWSH closed); same-origin
default + ServerConfig.AllowedOrigins.
- fiber CVE GO-2026-4543 patched (v2.52.10 -> v2.52.12).
FUNCTIONAL FEATURES
- API key DB persistence: APIKeyModel + migration 202604280002, full CRUD,
async LastUsedAt updates with WaitGroup-tracked goroutines drained on Close.
- Admin bootstrap via GOHOARDER_BOOTSTRAP_ADMIN_KEY env var; idempotent
(skipped when non-revoked admin already exists).
- Auth middleware factory in pkg/app: RequireAuth, RequireRole, RequirePermission,
OptionalAuth. Mounted on proxy + read APIs when Auth.Enabled=true.
DELETE /api/packages/* requires admin role. /health public for k8s probes.
- NFS storage backend: pkg/storage/nfs wraps filesystem with /proc/mounts
detection (Linux), post-write fsync, read-after-write health probe.
- WebSocket real-time pipeline: pkg/events Broadcaster interface; cache + scanner
managers emit EventPackageCached / EventPackageDownloaded / EventScanComplete;
app.go runs 30s EventStatsUpdate ticker. Frontend has full WS client (reconnect
with exponential backoff 1s->30s, 25s heartbeat, subscriber pattern), Pinia
realtime store, Dashboard live indicator + activity feed + reactive stats
override.
- TLS termination: fiber.ListenTLS when Server.TLS.Enabled.
- Pre-warming: Prewarming.Enabled flag wired (was hardcoded false); Interval,
MaxConcurrent, TopPackages plumbed.
- NetworkConfig wired (timeouts, retry, rate-limit, circuit-breaker).
- AuthConfig.BcryptCost wired.
- Security.Scanners.Static.MaxPackageSize plumbed to cache.io.LimitReader.
- Handlers.{Go,NPM,PyPI}.Enabled gates route mounting.
- Graceful shutdown: authManager.Close drains async writes.
LINT/HYGIENE
- 80 golangci-lint issues resolved: errcheck (Close/Write/RemoveAll wrapped),
gofmt, gosec G104/G306, govet shadow renames + fieldalignment reorders,
staticcheck ST1000 pkg comments + QF1003 tagged switches + QF1008 embedded
field selectors + QF1001 De Morgan's law.
BEHAVIOR CHANGES
- Scanner now fail-closed: deployments without scanner binaries
(trivy/govulncheck/npm-audit/pip-audit/grype) BLOCK packages instead of
serving unscanned. Add binaries or plan a future allow-on-scan-error flag.
- WS origin defaults to same-origin; cross-origin dashboards must set
server.allowed_origins.
- Validator no longer fails open; private packages won't serve from cache when
validation can't be performed.
- download_events retention dropped from 24h to ~5min (deleted in agg tx).
Migration 202604280001 purges pre-upgrade events.
- DELETE /api/packages/* requires admin role when auth enabled.
NEW PACKAGES
- pkg/events - Broadcaster interface
- pkg/storage/nfs - NFS-aware filesystem wrapper
DEFERRED (out of scope this pass)
- Static scanner implementation (config defines AllowedLicenses/MaxPackageSize/
BlockSuspicious but pkg/scanner/static/ does not exist).
- AuthConfig.AuditLog wiring (no audit log infra yet).
- CacheConfig.TTLOverrides passthrough (would touch cache.Config beyond
integrator scope).
363 lines
9.0 KiB
Go
363 lines
9.0 KiB
Go
// Package network provides resilient HTTP client primitives with retry,
|
|
// timeout, and circuit-breaker support for upstream registry calls.
|
|
package network
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"io"
|
|
"net/http"
|
|
"sync"
|
|
"time"
|
|
|
|
"github.com/lukaszraczylo/gohoarder/pkg/errors"
|
|
"github.com/lukaszraczylo/gohoarder/pkg/metrics"
|
|
"github.com/rs/zerolog/log"
|
|
"golang.org/x/time/rate"
|
|
)
|
|
|
|
// Client is an HTTP client with resilience features
|
|
type Client struct {
|
|
client *http.Client
|
|
rateLimiter *rate.Limiter
|
|
circuitBreaker *CircuitBreaker
|
|
retryConfig RetryConfig
|
|
}
|
|
|
|
// Config holds client configuration
|
|
type Config struct {
|
|
UserAgent string
|
|
CircuitBreaker CircuitBreakerConfig
|
|
Timeout time.Duration
|
|
MaxRetries int
|
|
RetryDelay time.Duration
|
|
RateLimit float64
|
|
RateBurst int
|
|
MaxConnsPerHost int
|
|
}
|
|
|
|
// RetryConfig holds retry configuration
|
|
type RetryConfig struct {
|
|
FixedDelays []time.Duration
|
|
MaxAttempts int
|
|
InitialDelay time.Duration
|
|
MaxDelay time.Duration
|
|
Multiplier float64
|
|
}
|
|
|
|
// CircuitBreakerConfig holds circuit breaker configuration
|
|
type CircuitBreakerConfig struct {
|
|
Enabled bool
|
|
FailureThreshold int // Failures before opening
|
|
SuccessThreshold int // Successes before closing
|
|
Timeout time.Duration // How long to stay open
|
|
HalfOpenMaxCalls int // Max calls in half-open state
|
|
}
|
|
|
|
// CircuitBreakerState represents circuit breaker state
|
|
type CircuitBreakerState int
|
|
|
|
const (
|
|
StateClosed CircuitBreakerState = iota
|
|
StateOpen
|
|
StateHalfOpen
|
|
)
|
|
|
|
// CircuitBreaker implements the circuit breaker pattern
|
|
type CircuitBreaker struct {
|
|
lastFailureTime time.Time
|
|
config CircuitBreakerConfig
|
|
state CircuitBreakerState
|
|
failures int
|
|
successes int
|
|
halfOpenCalls int
|
|
mu sync.RWMutex
|
|
}
|
|
|
|
// NewClient creates a new HTTP client with resilience features
|
|
func NewClient(config Config) *Client {
|
|
if config.Timeout == 0 {
|
|
config.Timeout = 30 * time.Second
|
|
}
|
|
|
|
if config.MaxRetries == 0 {
|
|
config.MaxRetries = 3
|
|
}
|
|
|
|
if config.RetryDelay == 0 {
|
|
config.RetryDelay = 1 * time.Second
|
|
}
|
|
|
|
if config.UserAgent == "" {
|
|
config.UserAgent = "GoHoarder/1.0"
|
|
}
|
|
|
|
if config.MaxConnsPerHost == 0 {
|
|
config.MaxConnsPerHost = 100
|
|
}
|
|
|
|
transport := &http.Transport{
|
|
MaxIdleConns: 100,
|
|
MaxIdleConnsPerHost: config.MaxConnsPerHost,
|
|
MaxConnsPerHost: config.MaxConnsPerHost,
|
|
IdleConnTimeout: 90 * time.Second,
|
|
DisableCompression: false,
|
|
}
|
|
|
|
httpClient := &http.Client{
|
|
Timeout: config.Timeout,
|
|
Transport: transport,
|
|
}
|
|
|
|
var rateLimiter *rate.Limiter
|
|
if config.RateLimit > 0 {
|
|
if config.RateBurst == 0 {
|
|
config.RateBurst = int(config.RateLimit)
|
|
}
|
|
rateLimiter = rate.NewLimiter(rate.Limit(config.RateLimit), config.RateBurst)
|
|
}
|
|
|
|
var cb *CircuitBreaker
|
|
if config.CircuitBreaker.Enabled {
|
|
if config.CircuitBreaker.FailureThreshold == 0 {
|
|
config.CircuitBreaker.FailureThreshold = 5
|
|
}
|
|
if config.CircuitBreaker.SuccessThreshold == 0 {
|
|
config.CircuitBreaker.SuccessThreshold = 2
|
|
}
|
|
if config.CircuitBreaker.Timeout == 0 {
|
|
config.CircuitBreaker.Timeout = 60 * time.Second
|
|
}
|
|
if config.CircuitBreaker.HalfOpenMaxCalls == 0 {
|
|
config.CircuitBreaker.HalfOpenMaxCalls = 3
|
|
}
|
|
|
|
cb = &CircuitBreaker{
|
|
config: config.CircuitBreaker,
|
|
state: StateClosed,
|
|
}
|
|
}
|
|
|
|
return &Client{
|
|
client: httpClient,
|
|
rateLimiter: rateLimiter,
|
|
circuitBreaker: cb,
|
|
retryConfig: RetryConfig{
|
|
MaxAttempts: config.MaxRetries,
|
|
InitialDelay: config.RetryDelay,
|
|
MaxDelay: 30 * time.Second,
|
|
Multiplier: 2.0,
|
|
// Fixed delays: 1s, 5s, 10s for retry attempts 1, 2, 3
|
|
FixedDelays: []time.Duration{1 * time.Second, 5 * time.Second, 10 * time.Second},
|
|
},
|
|
}
|
|
}
|
|
|
|
// Get performs a GET request with resilience features
|
|
func (c *Client) Get(ctx context.Context, url string, headers map[string]string) (io.ReadCloser, int, error) {
|
|
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
|
|
if err != nil {
|
|
return nil, 0, errors.Wrap(err, errors.ErrCodeUpstreamError, "failed to create request")
|
|
}
|
|
|
|
for key, value := range headers {
|
|
req.Header.Set(key, value)
|
|
}
|
|
|
|
resp, err := c.do(ctx, req)
|
|
if err != nil {
|
|
return nil, 0, err
|
|
}
|
|
|
|
return resp.Body, resp.StatusCode, nil
|
|
}
|
|
|
|
// do executes an HTTP request with retries and circuit breaker
|
|
func (c *Client) do(ctx context.Context, req *http.Request) (*http.Response, error) {
|
|
// Check circuit breaker
|
|
if c.circuitBreaker != nil {
|
|
if !c.circuitBreaker.AllowRequest() {
|
|
metrics.UpdateCircuitBreakerState("upstream", int(StateOpen))
|
|
return nil, errors.New(errors.ErrCodeCircuitOpen, "circuit breaker is open")
|
|
}
|
|
}
|
|
|
|
// Apply rate limiting
|
|
if c.rateLimiter != nil {
|
|
if err := c.rateLimiter.Wait(ctx); err != nil {
|
|
return nil, errors.Wrap(err, errors.ErrCodeRateLimited, "rate limit exceeded")
|
|
}
|
|
}
|
|
|
|
// Execute with retries
|
|
var lastErr error
|
|
delay := c.retryConfig.InitialDelay
|
|
|
|
for attempt := 0; attempt < c.retryConfig.MaxAttempts; attempt++ {
|
|
if attempt > 0 {
|
|
// Calculate delay: use fixed delays if configured, otherwise exponential backoff
|
|
if len(c.retryConfig.FixedDelays) > 0 {
|
|
// Use fixed delay schedule
|
|
delayIndex := attempt - 1
|
|
if delayIndex < len(c.retryConfig.FixedDelays) {
|
|
delay = c.retryConfig.FixedDelays[delayIndex]
|
|
} else {
|
|
// Use last delay if we run out of configured delays
|
|
delay = c.retryConfig.FixedDelays[len(c.retryConfig.FixedDelays)-1]
|
|
}
|
|
} else {
|
|
// Exponential backoff
|
|
delay = time.Duration(float64(delay) * c.retryConfig.Multiplier)
|
|
if delay > c.retryConfig.MaxDelay {
|
|
delay = c.retryConfig.MaxDelay
|
|
}
|
|
}
|
|
|
|
select {
|
|
case <-ctx.Done():
|
|
return nil, ctx.Err()
|
|
case <-time.After(delay):
|
|
}
|
|
|
|
log.Debug().
|
|
Str("url", req.URL.String()).
|
|
Int("attempt", attempt+1).
|
|
Dur("delay", delay).
|
|
Msg("Retrying request")
|
|
}
|
|
|
|
resp, err := c.client.Do(req)
|
|
if err != nil {
|
|
lastErr = err
|
|
if c.circuitBreaker != nil {
|
|
c.circuitBreaker.RecordFailure()
|
|
}
|
|
continue
|
|
}
|
|
|
|
// Check if response is retryable
|
|
if c.isRetryable(resp.StatusCode) {
|
|
_ = resp.Body.Close() // #nosec G104 -- Cleanup, error not critical
|
|
lastErr = fmt.Errorf("received retryable status code: %d", resp.StatusCode)
|
|
if c.circuitBreaker != nil {
|
|
c.circuitBreaker.RecordFailure()
|
|
}
|
|
continue
|
|
}
|
|
|
|
// Success
|
|
if c.circuitBreaker != nil {
|
|
c.circuitBreaker.RecordSuccess()
|
|
metrics.UpdateCircuitBreakerState("upstream", int(StateClosed))
|
|
}
|
|
|
|
return resp, nil
|
|
}
|
|
|
|
// All retries exhausted
|
|
if c.circuitBreaker != nil {
|
|
c.circuitBreaker.RecordFailure()
|
|
}
|
|
|
|
if lastErr != nil {
|
|
return nil, errors.Wrap(lastErr, errors.ErrCodeUpstreamFailure, "all retry attempts failed")
|
|
}
|
|
|
|
return nil, errors.New(errors.ErrCodeUpstreamFailure, "request failed without error")
|
|
}
|
|
|
|
// isRetryable checks if a status code should trigger a retry
|
|
func (c *Client) isRetryable(statusCode int) bool {
|
|
// Retry on server errors and some client errors
|
|
return statusCode >= 500 || statusCode == 408 || statusCode == 429
|
|
}
|
|
|
|
// AllowRequest checks if a request is allowed by the circuit breaker
|
|
func (cb *CircuitBreaker) AllowRequest() bool {
|
|
cb.mu.Lock()
|
|
defer cb.mu.Unlock()
|
|
|
|
switch cb.state {
|
|
case StateClosed:
|
|
return true
|
|
|
|
case StateOpen:
|
|
// Check if timeout has elapsed
|
|
if time.Since(cb.lastFailureTime) > cb.config.Timeout {
|
|
cb.state = StateHalfOpen
|
|
cb.halfOpenCalls = 0
|
|
cb.successes = 0
|
|
log.Info().Msg("Circuit breaker transitioning to half-open")
|
|
metrics.UpdateCircuitBreakerState("upstream", int(StateHalfOpen))
|
|
return true
|
|
}
|
|
return false
|
|
|
|
case StateHalfOpen:
|
|
// Allow limited requests in half-open state
|
|
if cb.halfOpenCalls < cb.config.HalfOpenMaxCalls {
|
|
cb.halfOpenCalls++
|
|
return true
|
|
}
|
|
return false
|
|
|
|
default:
|
|
return false
|
|
}
|
|
}
|
|
|
|
// RecordSuccess records a successful request
|
|
func (cb *CircuitBreaker) RecordSuccess() {
|
|
cb.mu.Lock()
|
|
defer cb.mu.Unlock()
|
|
|
|
switch cb.state {
|
|
case StateClosed:
|
|
cb.failures = 0
|
|
|
|
case StateHalfOpen:
|
|
cb.successes++
|
|
if cb.successes >= cb.config.SuccessThreshold {
|
|
cb.state = StateClosed
|
|
cb.failures = 0
|
|
cb.successes = 0
|
|
cb.halfOpenCalls = 0
|
|
log.Info().Msg("Circuit breaker closed")
|
|
metrics.UpdateCircuitBreakerState("upstream", int(StateClosed))
|
|
}
|
|
}
|
|
}
|
|
|
|
// RecordFailure records a failed request
|
|
func (cb *CircuitBreaker) RecordFailure() {
|
|
cb.mu.Lock()
|
|
defer cb.mu.Unlock()
|
|
|
|
cb.lastFailureTime = time.Now()
|
|
|
|
switch cb.state {
|
|
case StateClosed:
|
|
cb.failures++
|
|
if cb.failures >= cb.config.FailureThreshold {
|
|
cb.state = StateOpen
|
|
log.Warn().Int("failures", cb.failures).Msg("Circuit breaker opened")
|
|
metrics.UpdateCircuitBreakerState("upstream", int(StateOpen))
|
|
}
|
|
|
|
case StateHalfOpen:
|
|
// Single failure in half-open returns to open
|
|
cb.state = StateOpen
|
|
cb.halfOpenCalls = 0
|
|
cb.successes = 0
|
|
log.Warn().Msg("Circuit breaker re-opened from half-open")
|
|
metrics.UpdateCircuitBreakerState("upstream", int(StateOpen))
|
|
}
|
|
}
|
|
|
|
// GetState returns the current circuit breaker state
|
|
func (cb *CircuitBreaker) GetState() CircuitBreakerState {
|
|
cb.mu.RLock()
|
|
defer cb.mu.RUnlock()
|
|
return cb.state
|
|
}
|