mirror of
https://github.com/lukaszraczylo/gohoarder.git
synced 2026-10-18 01:17:46 +00:00
fix: remediate severity-ordered review findings across cache, scanners, and proxies
- H2: gormstore soft-delete + storage_key UNIQUE collision; re-save after eviction succeeds
- H1: cache getOrFetch cache-hit branch no longer runs CheckVulnerabilities on metadata entries
- M2: cache store() streams upstream body to temp file + SHA256 instead of double-buffering
- H3/L3: FetchResult/FetchFunc plumbing (+RequiresAuth/AuthProvider); proxy closures adapted;
goproxy strips git+ prefix from UpstreamURL
- H4: scanner normalizes bypass package names; CheckVulnerabilities builds normalized packageKey
- M4: requireAdmin routes through extractAPIKey -> ValidateAPIKey -> admin role check
- M5: network client keeps only final RecordFailure (one count per attempt cycle)
- M3: normalizeOriginHost + buildCheckOrigin rewrite; origin_test.go added
- M1: app lifecycle cancel func; Run -> Shutdown on listen error (sole cache owner); JoinHostPort
- M6: VulnerabilityThresholds defaults all -1 so BlockOnSeverity governs
- L4: metadata.NormalizeSeverity at 4 severity-count sites (MEDIUM->MODERATE consistency)
- L1: handleDeleteBypass uses structured *errors.Error + ErrCodeNotFound
- L2: pypi extractVersionFromFilename (wheel/2nd segment, non-wheel/last digit segment);
cacheVersion uses full unique basename; regression tests added
- L5: document per-instance ValidationCache key scoping; rename packageURL param to resource
All: go build ./... && go vet ./... && go test ./... pass; golangci-lint clean.
This commit is contained in:
+93
-31
@@ -5,10 +5,12 @@ package app
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"os/signal"
|
||||
"strconv"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
@@ -44,6 +46,7 @@ import (
|
||||
// App represents the main application
|
||||
type App struct {
|
||||
config *config.Config
|
||||
cancel context.CancelFunc
|
||||
app *fiber.App
|
||||
healthChecker *health.Checker
|
||||
cache *cache.Manager
|
||||
@@ -580,20 +583,13 @@ func (a *App) setupServer() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Run starts the application
|
||||
func (a *App) Run() error {
|
||||
ctx := context.Background()
|
||||
|
||||
// Start WebSocket server
|
||||
a.wsServer.Start(ctx)
|
||||
|
||||
// Start pre-warming worker
|
||||
a.prewarmWorker.Start(ctx)
|
||||
|
||||
// Start rescan worker if enabled
|
||||
if a.rescanWorker != nil {
|
||||
go a.rescanWorker.Start(ctx)
|
||||
}
|
||||
// Cancellable worker context: aggregation + stats broadcaster goroutines
|
||||
// watch ctx.Done() and exit, so Shutdown can stop them before closing
|
||||
// the storage/metadata backends they touch.
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
a.cancel = cancel
|
||||
defer cancel()
|
||||
|
||||
// Start download data aggregation worker (runs every hour)
|
||||
go a.startAggregationWorker(ctx)
|
||||
@@ -605,7 +601,7 @@ func (a *App) Run() error {
|
||||
// Start Fiber server in goroutine. Branch on TLS to pick Listen vs ListenTLS.
|
||||
errChan := make(chan error, 1)
|
||||
go func() {
|
||||
addr := fmt.Sprintf("%s:%d", a.config.Server.Host, a.config.Server.Port)
|
||||
addr := net.JoinHostPort(a.config.Server.Host, strconv.Itoa(a.config.Server.Port))
|
||||
if a.config.Server.TLS.Enabled {
|
||||
log.Info().
|
||||
Str("addr", addr).
|
||||
@@ -630,6 +626,13 @@ func (a *App) Run() error {
|
||||
|
||||
select {
|
||||
case err := <-errChan:
|
||||
// Server failed to start or stopped unexpectedly. Cancel the worker
|
||||
// contexts and run the same teardown Shutdown performs so background
|
||||
// goroutines don't keep running against closed storage/metadata.
|
||||
shutdownErr := a.Shutdown()
|
||||
if shutdownErr != nil {
|
||||
log.Error().Err(shutdownErr).Msg("Error during shutdown after server error")
|
||||
}
|
||||
return fmt.Errorf("server error: %w", err)
|
||||
case sig := <-sigChan:
|
||||
log.Info().
|
||||
@@ -640,7 +643,6 @@ func (a *App) Run() error {
|
||||
// Graceful shutdown
|
||||
return a.Shutdown()
|
||||
}
|
||||
|
||||
// Shutdown gracefully shuts down the application
|
||||
func (a *App) Shutdown() error {
|
||||
log.Info().Msg("Starting graceful shutdown")
|
||||
@@ -650,6 +652,12 @@ func (a *App) Shutdown() error {
|
||||
log.Error().Err(err).Msg("Error shutting down Fiber server")
|
||||
}
|
||||
|
||||
// Cancel worker contexts first so the aggregation and stats broadcaster
|
||||
// goroutines stop before the storage/metadata backends they touch close.
|
||||
if a.cancel != nil {
|
||||
a.cancel()
|
||||
}
|
||||
|
||||
// Drain async auth writes (LastUsedAt updates) before closing metadata.
|
||||
if a.authManager != nil {
|
||||
a.authManager.Close()
|
||||
@@ -666,14 +674,15 @@ func (a *App) Shutdown() error {
|
||||
// Close analytics engine
|
||||
a.analyticsEngine.Close() // #nosec G104 -- Cleanup, error not critical
|
||||
|
||||
// Close storage
|
||||
if err := a.storage.Close(); err != nil {
|
||||
log.Error().Err(err).Msg("Error closing storage")
|
||||
}
|
||||
|
||||
// Close metadata store
|
||||
if err := a.metadata.Close(); err != nil {
|
||||
log.Error().Err(err).Msg("Error closing metadata store")
|
||||
// Close cache manager. This stops the cache cleanup worker and closes the
|
||||
// storage + metadata backends (a.cache is the sole owner of both, sharing
|
||||
// the same pointers with the App). Calling Close on storage/metadata here
|
||||
// instead of via a.cache.Close() would double-close and leave the cleanup
|
||||
// worker running against closed backends.
|
||||
if a.cache != nil {
|
||||
if err := a.cache.Close(); err != nil {
|
||||
log.Error().Err(err).Msg("Error closing cache")
|
||||
}
|
||||
}
|
||||
|
||||
log.Info().Msg("Shutdown complete")
|
||||
@@ -762,6 +771,41 @@ func getOrDefaultStr(value, defaultValue string) string {
|
||||
return value
|
||||
}
|
||||
|
||||
// normalizeOriginHost lowercases a host and strips the scheme's default port
|
||||
// (443 for https, 80 for http) so "https://app.example.com:443" and
|
||||
// "https://app.example.com" are treated as the same origin. Non-default ports
|
||||
// are preserved. Returns ("", false) for an empty/invalid host.
|
||||
func normalizeOriginHost(scheme, host string) (string, bool) {
|
||||
host = strings.ToLower(strings.TrimSpace(host))
|
||||
if host == "" {
|
||||
return "", false
|
||||
}
|
||||
// Split off a port, if present. url.Parse stores the full host; use
|
||||
// net.SplitHostPort so IPv6 literals like [::1]:8080 are handled.
|
||||
h := host
|
||||
var port string
|
||||
if hh, p, err := net.SplitHostPort(host); err == nil {
|
||||
h = hh
|
||||
port = p
|
||||
}
|
||||
defaultPort := ""
|
||||
switch scheme {
|
||||
case "https":
|
||||
defaultPort = "443"
|
||||
case "http":
|
||||
defaultPort = "80"
|
||||
}
|
||||
if port != "" && port == defaultPort {
|
||||
return h, true
|
||||
}
|
||||
// Rebuild the host with the port (when present) for exact comparisons.
|
||||
if port != "" {
|
||||
// Reuse the original port form; net.JoinHostPort preserves IPv6 brackets.
|
||||
return net.JoinHostPort(h, port), true
|
||||
}
|
||||
return h, true
|
||||
}
|
||||
|
||||
// buildCheckOrigin returns a websocket Origin validator.
|
||||
//
|
||||
// Behaviour:
|
||||
@@ -771,13 +815,18 @@ func getOrDefaultStr(value, defaultValue string) string {
|
||||
// Origin host must match the Host header of the incoming request.
|
||||
// - requests with no Origin header are accepted (non-browser clients).
|
||||
//
|
||||
// Scheme-default ports (443/80) are normalized out of both the allowlist and
|
||||
// the incoming Origin, so "https://app.example.com" and
|
||||
// "https://app.example.com:443" are treated identically. Wildcard matching is
|
||||
// performed on the hostname only.
|
||||
//
|
||||
// We deliberately do NOT default to allow-all to avoid Cross-Site WebSocket
|
||||
// Hijacking (CSWSH).
|
||||
func buildCheckOrigin(allowed []string) func(*http.Request) bool {
|
||||
// Pre-parse allowlist once.
|
||||
type originRule struct {
|
||||
scheme string
|
||||
host string // exact host, or "" if hostSuffix is set
|
||||
host string // exact normalized host:port (default port stripped), or "" if hostGlob is set
|
||||
hostGlob string // suffix match including leading "."
|
||||
}
|
||||
rules := make([]originRule, 0, len(allowed))
|
||||
@@ -791,13 +840,18 @@ func buildCheckOrigin(allowed []string) func(*http.Request) bool {
|
||||
log.Warn().Str("entry", raw).Msg("Ignoring invalid AllowedOrigins entry")
|
||||
continue
|
||||
}
|
||||
r := originRule{scheme: strings.ToLower(u.Scheme)}
|
||||
host := strings.ToLower(u.Host)
|
||||
if strings.HasPrefix(host, "*.") {
|
||||
scheme := strings.ToLower(u.Scheme)
|
||||
normHost, ok := normalizeOriginHost(scheme, u.Host)
|
||||
if !ok {
|
||||
log.Warn().Str("entry", raw).Msg("Ignoring invalid AllowedOrigins entry")
|
||||
continue
|
||||
}
|
||||
r := originRule{scheme: scheme}
|
||||
if strings.HasPrefix(normHost, "*.") {
|
||||
// "*.example.com" → match any subdomain plus the apex.
|
||||
r.hostGlob = host[1:] // ".example.com"
|
||||
r.hostGlob = normHost[1:] // ".example.com"
|
||||
} else {
|
||||
r.host = host
|
||||
r.host = normHost
|
||||
}
|
||||
rules = append(rules, r)
|
||||
}
|
||||
@@ -814,11 +868,19 @@ func buildCheckOrigin(allowed []string) func(*http.Request) bool {
|
||||
return false
|
||||
}
|
||||
originScheme := strings.ToLower(ou.Scheme)
|
||||
originHost := strings.ToLower(ou.Host)
|
||||
originHost, ok := normalizeOriginHost(originScheme, ou.Host)
|
||||
if !ok {
|
||||
log.Warn().Str("origin", origin).Msg("Rejecting WebSocket upgrade: malformed Origin header")
|
||||
return false
|
||||
}
|
||||
|
||||
// Empty allowlist → same-origin only.
|
||||
if len(rules) == 0 {
|
||||
return strings.EqualFold(originHost, r.Host)
|
||||
reqHost, ok := normalizeOriginHost(originScheme, r.Host)
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
return originHost == reqHost
|
||||
}
|
||||
|
||||
for _, rule := range rules {
|
||||
|
||||
+1
-1
@@ -139,7 +139,7 @@ func (a *App) handleListPackages(c *fiber.Ctx) error {
|
||||
// Count vulnerabilities by severity
|
||||
severityCounts := make(map[string]int)
|
||||
for _, vuln := range scanResult.Vulnerabilities {
|
||||
severityCounts[strings.ToUpper(vuln.Severity)]++
|
||||
severityCounts[metadata.NormalizeSeverity(vuln.Severity)]++
|
||||
}
|
||||
|
||||
// Check if package should be blocked based on thresholds
|
||||
|
||||
@@ -1,36 +1,28 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/gofiber/fiber/v2"
|
||||
"github.com/lukaszraczylo/gohoarder/pkg/auth"
|
||||
"github.com/lukaszraczylo/gohoarder/pkg/errors"
|
||||
"github.com/lukaszraczylo/gohoarder/pkg/metadata"
|
||||
"github.com/lukaszraczylo/gohoarder/pkg/uuid"
|
||||
"github.com/rs/zerolog/log"
|
||||
)
|
||||
|
||||
// requireAdmin middleware checks for admin authentication
|
||||
// requireAdmin middleware checks for admin authentication. It accepts the API
|
||||
// key via either the Authorization Bearer header or the X-API-Key header
|
||||
// (shared extractAPIKey), so admin clients are not limited to the bearer form.
|
||||
func (a *App) requireAdmin(c *fiber.Ctx) error {
|
||||
// Get API key from Authorization header
|
||||
authHeader := c.Get("Authorization")
|
||||
if authHeader == "" {
|
||||
// Get API key from Authorization or X-API-Key header
|
||||
apiKey, ok := extractAPIKey(c)
|
||||
if !ok {
|
||||
return c.Status(fiber.StatusUnauthorized).JSON(fiber.Map{
|
||||
"error": "missing authorization header",
|
||||
"error": "missing API key; provide Authorization: Bearer <key> or X-API-Key header",
|
||||
})
|
||||
}
|
||||
|
||||
// Extract bearer token
|
||||
parts := strings.SplitN(authHeader, " ", 2)
|
||||
if len(parts) != 2 || strings.ToLower(parts[0]) != "bearer" {
|
||||
return c.Status(fiber.StatusUnauthorized).JSON(fiber.Map{
|
||||
"error": "invalid authorization header format, expected: Bearer <token>",
|
||||
})
|
||||
}
|
||||
|
||||
apiKey := parts[1]
|
||||
|
||||
// Validate API key
|
||||
key, err := a.authManager.ValidateAPIKey(c.Context(), apiKey)
|
||||
if err != nil {
|
||||
@@ -304,7 +296,7 @@ func (a *App) handleDeleteBypass(c *fiber.Ctx) error {
|
||||
|
||||
// Delete bypass
|
||||
if err := a.metadata.DeleteCVEBypass(ctx, bypassID); err != nil {
|
||||
if strings.Contains(err.Error(), "not found") {
|
||||
if appErr, ok := err.(*errors.Error); ok && appErr.Code == errors.ErrCodeNotFound {
|
||||
return c.Status(fiber.StatusNotFound).JSON(fiber.Map{"error": "bypass not found"})
|
||||
}
|
||||
log.Error().Err(err).Msg("Failed to delete bypass")
|
||||
|
||||
@@ -77,11 +77,19 @@ func (a *App) handleVulnerabilities(c *fiber.Ctx) error {
|
||||
bypasses = []*metadata.CVEBypass{}
|
||||
}
|
||||
|
||||
// Build bypass map for fast lookup
|
||||
// Build bypass map for fast lookup.
|
||||
// Normalize Go cache-key names (modulePath/@v/version.zip) to the
|
||||
// natural module path so admin bypass targets like
|
||||
// "go/github.com/foo/[email protected]" match the package.
|
||||
bypassName := name
|
||||
if registry == "go" {
|
||||
if idx := strings.Index(bypassName, "/@v/"); idx >= 0 {
|
||||
bypassName = bypassName[:idx]
|
||||
}
|
||||
}
|
||||
packageKey := registry + "/" + bypassName + "@" + version
|
||||
packageKeyNoVersion := registry + "/" + bypassName
|
||||
bypassedCVEs := make(map[string]*metadata.CVEBypass)
|
||||
packageKey := registry + "/" + name + "@" + version
|
||||
packageKeyNoVersion := registry + "/" + name
|
||||
|
||||
for _, bypass := range bypasses {
|
||||
if bypass.Type == metadata.BypassTypeCVE && bypass.Active {
|
||||
// Check if bypass applies to this package
|
||||
@@ -110,7 +118,7 @@ func (a *App) handleVulnerabilities(c *fiber.Ctx) error {
|
||||
}
|
||||
} else {
|
||||
// Count non-bypassed vulnerabilities by severity
|
||||
severityCounts[strings.ToUpper(vuln.Severity)]++
|
||||
severityCounts[metadata.NormalizeSeverity(vuln.Severity)]++
|
||||
}
|
||||
|
||||
enrichedVuln := map[string]interface{}{
|
||||
|
||||
@@ -0,0 +1,73 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func reqWithOrigin(origin string) *http.Request {
|
||||
r, _ := http.NewRequest(http.MethodGet, "/ws", nil)
|
||||
if origin != "" {
|
||||
r.Header.Set("Origin", origin)
|
||||
}
|
||||
r.Host = "app.example.com"
|
||||
return r
|
||||
}
|
||||
|
||||
// TestBuildCheckOrigin_DefaultPortNormalization verifies that scheme-default
|
||||
// ports (443/80) are ignored, so "https://app.example.com" and
|
||||
// "https://app.example.com:443" are treated as the same origin.
|
||||
func TestBuildCheckOrigin_DefaultPortNormalization(t *testing.T) {
|
||||
check := buildCheckOrigin([]string{"https://app.example.com"})
|
||||
|
||||
assert.True(t, check(reqWithOrigin("https://app.example.com")))
|
||||
assert.True(t, check(reqWithOrigin("https://app.example.com:443")))
|
||||
assert.False(t, check(reqWithOrigin("https://app.example.com:8443")))
|
||||
|
||||
// Default http port strips too.
|
||||
checkHTTP := buildCheckOrigin([]string{"http://app.example.com"})
|
||||
assert.True(t, checkHTTP(reqWithOrigin("http://app.example.com:80")))
|
||||
assert.True(t, checkHTTP(reqWithOrigin("http://app.example.com")))
|
||||
}
|
||||
|
||||
// TestBuildCheckOrigin_NonDefaultPortPreserved verifies a non-default port is
|
||||
// not stripped and must match exactly.
|
||||
func TestBuildCheckOrigin_NonDefaultPortPreserved(t *testing.T) {
|
||||
check := buildCheckOrigin([]string{"https://app.example.com:8443"})
|
||||
|
||||
assert.True(t, check(reqWithOrigin("https://app.example.com:8443")))
|
||||
assert.False(t, check(reqWithOrigin("https://app.example.com")))
|
||||
}
|
||||
|
||||
// TestBuildCheckOrigin_Wildcard verifies wildcard host patterns match the apex
|
||||
// and subdomains, and are unaffected by ports on the incoming origin.
|
||||
func TestBuildCheckOrigin_Wildcard(t *testing.T) {
|
||||
check := buildCheckOrigin([]string{"https://*.example.com"})
|
||||
|
||||
assert.True(t, check(reqWithOrigin("https://example.com")))
|
||||
assert.True(t, check(reqWithOrigin("https://cdn.example.com")))
|
||||
assert.True(t, check(reqWithOrigin("https://cdn.example.com:443")))
|
||||
assert.False(t, check(reqWithOrigin("https://example.org")))
|
||||
}
|
||||
|
||||
// TestBuildCheckOrigin_SchemeMismatch verifies the scheme must match.
|
||||
func TestBuildCheckOrigin_SchemeMismatch(t *testing.T) {
|
||||
check := buildCheckOrigin([]string{"https://app.example.com"})
|
||||
|
||||
assert.False(t, check(reqWithOrigin("http://app.example.com")))
|
||||
}
|
||||
|
||||
// TestBuildCheckOrigin_SameOriginFallback verifies that with an empty allowlist
|
||||
// only a same-origin upgrade (matching Host) is permitted.
|
||||
func TestBuildCheckOrigin_SameOriginFallback(t *testing.T) {
|
||||
check := buildCheckOrigin(nil)
|
||||
|
||||
r := reqWithOrigin("https://app.example.com")
|
||||
assert.True(t, check(r))
|
||||
|
||||
// Host defaults to app.example.com; a different Origin host is rejected.
|
||||
r2 := reqWithOrigin("https://other.example.com")
|
||||
assert.False(t, check(r2))
|
||||
}
|
||||
Reference in New Issue
Block a user