lukaszraczylo/gohoarder
1
0
Fork 0
mirror of https://github.com/lukaszraczylo/gohoarder.git synced 2026-06-13 02:36:48 +00:00
Code Issues Packages Projects Releases Wiki Activity

fixes

Browse Source
This commit is contained in:
Lukasz Raczylo
2026-01-02 11:49:08 +00:00
parent 3b8e171fdb
commit 1cbf6c5d9e
27 changed files with 779 additions and 384 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/app/app.go
+1 -1
View File
@@ -148,7 +148,7 @@ func (a *App) initializeComponents() error {
// Initialize rescan worker if enabled
if a.config.Security.Enabled && a.config.Security.RescanInterval > 0 {
log.Info().Dur("interval", a.config.Security.RescanInterval).Msg("Initializing package rescan worker")
a.rescanWorker = scanner.NewRescanWorker(a.scanManager, a.metadata, a.config.Security.RescanInterval)
a.rescanWorker = scanner.NewRescanWorker(a.scanManager, a.metadata, a.storage, a.config.Security.RescanInterval)
}
// Initialize analytics engine
pkg/app/handlers.go
+5 -5
View File
@@ -58,8 +58,8 @@ func (a *App) handleListPackages(w http.ResponseWriter, r *http.Request) {
// Filter, clean, and deduplicate packages
seen := make(map[string]*metadata.Package)
for _, pkg := range allPackages {
// Skip metadata entries
if pkg.Version == "list" || pkg.Version == "latest" {
// Skip metadata entries (npm metadata pages, pypi pages, etc.)
if pkg.Version == "list" || pkg.Version == "latest" || pkg.Version == "metadata" || pkg.Version == "page" {
continue
}
@@ -121,7 +121,7 @@ func (a *App) handleListPackages(w http.ResponseWriter, r *http.Request) {
"counts": map[string]int{
"critical": severityCounts["CRITICAL"],
"high": severityCounts["HIGH"],
"medium": severityCounts["MEDIUM"],
"moderate": severityCounts["MODERATE"],
"low": severityCounts["LOW"],
},
"total": scanResult.VulnerabilityCount,
@@ -330,8 +330,8 @@ func (a *App) handleStats(w http.ResponseWriter, r *http.Request) {
registryStats := make(map[string]map[string]interface{})
for _, pkg := range packages {
// Skip metadata entries
if pkg.Version == "list" || pkg.Version == "latest" {
// Skip metadata entries (npm metadata pages, pypi pages, etc.)
if pkg.Version == "list" || pkg.Version == "latest" || pkg.Version == "metadata" || pkg.Version == "page" {
continue
}
totalSize += pkg.Size
pkg/app/handlers_vulnerabilities.go
+2 -2
View File
@@ -150,10 +150,10 @@ func (a *App) handleVulnerabilities(w http.ResponseWriter, r *http.Request) {
"severity_counts": map[string]int{
"critical": severityCounts["CRITICAL"],
"high": severityCounts["HIGH"],
"medium": severityCounts["MEDIUM"],
"moderate": severityCounts["MODERATE"],
"low": severityCounts["LOW"],
},
"bypassed_count": len(scanResult.Vulnerabilities) - (severityCounts["CRITICAL"] + severityCounts["HIGH"] + severityCounts["MEDIUM"] + severityCounts["LOW"]),
"bypassed_count": len(scanResult.Vulnerabilities) - (severityCounts["CRITICAL"] + severityCounts["HIGH"] + severityCounts["MODERATE"] + severityCounts["LOW"]),
}
errors.WriteJSONSimple(w, http.StatusOK, response)
pkg/metadata/interface.go
+21 -1
View File
@@ -2,6 +2,7 @@ package metadata
import (
"context"
"strings"
"time"
)
@@ -95,7 +96,7 @@ type ScanResult struct {
// Vulnerability represents a security vulnerability
type Vulnerability struct {
ID string `json:"id"` // CVE-xxx, GHSA-xxx, etc.
Severity string `json:"severity"` // critical, high, medium, low
Severity string `json:"severity"` // critical, high, moderate, low
Title string `json:"title"`
Description string `json:"description"`
References []string `json:"references"`
@@ -103,6 +104,25 @@ type Vulnerability struct {
DetectedBy []string `json:"detected_by,omitempty"` // List of scanners that detected this vulnerability
}
// NormalizeSeverity normalizes severity names to standard values
// Ensures consistent naming: CRITICAL, HIGH, MODERATE, LOW
func NormalizeSeverity(severity string) string {
normalized := strings.ToUpper(strings.TrimSpace(severity))
// Map MEDIUM to MODERATE for consistency
if normalized == "MEDIUM" {
return "MODERATE"
}
// Ensure we only return valid severity levels
switch normalized {
case "CRITICAL", "HIGH", "MODERATE", "LOW":
return normalized
default:
return "LOW" // Default unknown severities to LOW
}
}
// ScanStatus represents scan result status
type ScanStatus string
pkg/metadata/sqlite/sqlite.go
+19 -1
View File
@@ -449,7 +449,25 @@ func (s *SQLiteStore) SaveScanResult(ctx context.Context, result *metadata.ScanR
// Update package security_scanned flag
updateQuery := `UPDATE packages SET security_scanned = 1 WHERE registry = ? AND name = ? AND version = ?`
s.db.ExecContext(ctx, updateQuery, result.Registry, result.PackageName, result.PackageVersion)
updateResult, err := s.db.ExecContext(ctx, updateQuery, result.Registry, result.PackageName, result.PackageVersion)
if err != nil {
log.Warn().
Err(err).
Str("registry", result.Registry).
Str("package", result.PackageName).
Str("version", result.PackageVersion).
Msg("Failed to update security_scanned flag")
// Don't return error - scan result is already saved
} else {
rowsAffected, _ := updateResult.RowsAffected()
if rowsAffected == 0 {
log.Warn().
Str("registry", result.Registry).
Str("package", result.PackageName).
Str("version", result.PackageVersion).
Msg("Package not found when updating security_scanned flag - possibly name mismatch")
}
}
return nil
}
pkg/scanner/osv/osv.go
+20 -10
View File
@@ -253,32 +253,42 @@ func (s *Scanner) convertOSVResult(osvResp *OSVResponse, registry, packageName,
// determineSeverity extracts severity from OSV vulnerability
func (s *Scanner) determineSeverity(vuln *OSVVulnerability) string {
var rawSeverity string
// Try to get severity from CVSS
for _, sev := range vuln.Severity {
if sev.Type == "CVSS_V3" || sev.Type == "CVSS_V2" {
// Parse CVSS score to severity
score := sev.Score
if strings.Contains(strings.ToUpper(score), "CRITICAL") {
return "CRITICAL"
rawSeverity = "CRITICAL"
} else if strings.Contains(strings.ToUpper(score), "HIGH") {
return "HIGH"
} else if strings.Contains(strings.ToUpper(score), "MEDIUM") {
return "MEDIUM"
rawSeverity = "HIGH"
} else if strings.Contains(strings.ToUpper(score), "MEDIUM") || strings.Contains(strings.ToUpper(score), "MODERATE") {
rawSeverity = "MODERATE"
} else if strings.Contains(strings.ToUpper(score), "LOW") {
return "LOW"
rawSeverity = "LOW"
}
if rawSeverity != "" {
break
}
}
}
// Check database_specific for severity
if vuln.DatabaseSpecific != nil {
// Check database_specific for severity if not found in CVSS
if rawSeverity == "" && vuln.DatabaseSpecific != nil {
if sev, ok := vuln.DatabaseSpecific["severity"].(string); ok {
return strings.ToUpper(sev)
rawSeverity = sev
}
}
// Default to MEDIUM if unknown
return "MEDIUM"
// Default to MODERATE if unknown
if rawSeverity == "" {
rawSeverity = "MODERATE"
}
// Normalize to standard severity values
return metadata.NormalizeSeverity(rawSeverity)
}
// findFixedVersion extracts the fixed version from OSV affected ranges
pkg/scanner/rescanner.go
+85 -10
View File
@@ -5,6 +5,7 @@ import (
"time"
"github.com/lukaszraczylo/gohoarder/pkg/metadata"
"github.com/lukaszraczylo/gohoarder/pkg/storage"
"github.com/rs/zerolog/log"
)
@@ -12,15 +13,17 @@ import (
type RescanWorker struct {
manager *Manager
metadataStore metadata.MetadataStore
storage storage.StorageBackend
interval time.Duration
stopCh chan struct{}
}
// NewRescanWorker creates a new rescan worker
func NewRescanWorker(manager *Manager, metadataStore metadata.MetadataStore, interval time.Duration) *RescanWorker {
func NewRescanWorker(manager *Manager, metadataStore metadata.MetadataStore, storageBackend storage.StorageBackend, interval time.Duration) *RescanWorker {
return &RescanWorker{
manager: manager,
metadataStore: metadataStore,
storage: storageBackend,
interval: interval,
stopCh: make(chan struct{}),
}
@@ -40,8 +43,12 @@ func (w *RescanWorker) Start(ctx context.Context) {
ticker := time.NewTicker(w.interval)
defer ticker.Stop()
// Run initial scan immediately
// Run initial scan immediately on startup
log.Info().Msg("Running initial package scan on startup")
w.rescanPackages(ctx)
log.Info().
Dur("next_scan", w.interval).
Msg("Initial scan complete, next scan scheduled")
for {
select {
@@ -64,7 +71,7 @@ func (w *RescanWorker) Stop() {
// rescanPackages re-scans packages that need updating
func (w *RescanWorker) rescanPackages(ctx context.Context) {
log.Info().Msg("Starting package rescan cycle")
log.Info().Msg("Starting package rescan cycle - checking all packages for scan status")
// Get all packages
packages, err := w.metadataStore.ListPackages(ctx, &metadata.ListOptions{})
@@ -78,6 +85,12 @@ func (w *RescanWorker) rescanPackages(ctx context.Context) {
failed := 0
for _, pkg := range packages {
// Skip metadata entries (npm metadata pages, pypi pages, etc.)
if pkg.Version == "list" || pkg.Version == "latest" || pkg.Version == "metadata" || pkg.Version == "page" {
skipped++
continue
}
// Check if package needs rescanning
needsRescan, err := w.needsRescan(ctx, pkg)
if err != nil {
@@ -95,19 +108,57 @@ func (w *RescanWorker) rescanPackages(ctx context.Context) {
continue
}
// Rescan the package
// Note: We need the file path - we'll need to reconstruct it or get it from storage
// For now, we'll just log and skip actual rescanning
log.Info().
Str("registry", pkg.Registry).
Str("package", pkg.Name).
Str("version", pkg.Version).
Msg("Package needs rescanning")
// TODO: Implement actual rescanning by:
// 1. Retrieving package file from storage
// 2. Scanning it
// This would require access to storage backend
// Get file path from storage using the storage key from the package metadata
if pkg.StorageKey == "" {
log.Warn().
Str("registry", pkg.Registry).
Str("package", pkg.Name).
Str("version", pkg.Version).
Msg("Package has no storage key, skipping rescan")
failed++
continue
}
filePath, err := w.getPackageFilePath(ctx, pkg.StorageKey)
if err != nil {
log.Warn().
Err(err).
Str("registry", pkg.Registry).
Str("package", pkg.Name).
Str("version", pkg.Version).
Str("storage_key", pkg.StorageKey).
Msg("Failed to get package file path, skipping rescan")
failed++
continue
}
if filePath == "" {
log.Debug().
Str("registry", pkg.Registry).
Str("package", pkg.Name).
Str("version", pkg.Version).
Msg("No local file path available, skipping rescan")
skipped++
continue
}
// Perform the actual scan
if err := w.manager.ScanPackage(ctx, pkg.Registry, pkg.Name, pkg.Version, filePath); err != nil {
log.Error().
Err(err).
Str("registry", pkg.Registry).
Str("package", pkg.Name).
Str("version", pkg.Version).
Msg("Failed to rescan package")
failed++
continue
}
scanned++
}
@@ -126,6 +177,19 @@ func (w *RescanWorker) needsRescan(ctx context.Context, pkg *metadata.Package) (
scanResult, err := w.metadataStore.GetScanResult(ctx, pkg.Registry, pkg.Name, pkg.Version)
if err != nil {
// No scan result - needs scanning
log.Debug().
Str("package", pkg.Name).
Str("version", pkg.Version).
Msg("Package has no scan result, needs scanning")
return true, nil
}
// If package is not marked as scanned but has scan result, it's a stale state - rescan
if !pkg.SecurityScanned {
log.Info().
Str("package", pkg.Name).
Str("version", pkg.Version).
Msg("Package has scan result but security_scanned flag is false, needs update")
return true, nil
}
@@ -137,3 +201,14 @@ func (w *RescanWorker) needsRescan(ctx context.Context, pkg *metadata.Package) (
return false, nil
}
// getPackageFilePath retrieves the local file path for a package from storage
func (w *RescanWorker) getPackageFilePath(ctx context.Context, storageKey string) (string, error) {
// Check if storage backend supports local paths
if localProvider, ok := w.storage.(storage.LocalPathProvider); ok {
return localProvider.GetLocalPath(ctx, storageKey)
}
// If storage doesn't support local paths (S3, SMB), we can't rescan
return "", nil
}
pkg/scanner/scanner.go
+11 -8
View File
@@ -260,7 +260,8 @@ func (m *Manager) compareSeverity(s1, s2 string) int {
severityOrder := map[string]int{
"CRITICAL": 4,
"HIGH": 3,
"MEDIUM": 2,
"MODERATE": 2,
"MEDIUM": 2, // Support both for backwards compatibility
"LOW": 1,
"UNKNOWN": 0,
}
@@ -353,10 +354,11 @@ func (m *Manager) CheckVulnerabilities(ctx context.Context, registry, packageNam
severityCounts["HIGH"], thresholds.High), nil
}
// Check medium
if thresholds.Medium >= 0 && severityCounts["MEDIUM"] > thresholds.Medium {
return true, fmt.Sprintf("Package has %d MEDIUM vulnerabilities (threshold: %d)",
severityCounts["MEDIUM"], thresholds.Medium), nil
// Check moderate (medium)
moderateCount := severityCounts["MODERATE"] + severityCounts["MEDIUM"] // Support both for backwards compatibility
if thresholds.Medium >= 0 && moderateCount > thresholds.Medium {
return true, fmt.Sprintf("Package has %d MODERATE vulnerabilities (threshold: %d)",
moderateCount, thresholds.Medium), nil
}
// Check low
@@ -379,9 +381,10 @@ func (m *Manager) CheckVulnerabilities(ctx context.Context, registry, packageNam
if severityCounts["CRITICAL"] > 0 || severityCounts["HIGH"] > 0 {
return true, fmt.Sprintf("Package has HIGH or CRITICAL vulnerabilities"), nil
}
case "MEDIUM":
if severityCounts["CRITICAL"] > 0 || severityCounts["HIGH"] > 0 || severityCounts["MEDIUM"] > 0 {
return true, fmt.Sprintf("Package has MEDIUM, HIGH, or CRITICAL vulnerabilities"), nil
case "MODERATE", "MEDIUM":
moderateCount := severityCounts["MODERATE"] + severityCounts["MEDIUM"]
if severityCounts["CRITICAL"] > 0 || severityCounts["HIGH"] > 0 || moderateCount > 0 {
return true, fmt.Sprintf("Package has MODERATE, HIGH, or CRITICAL vulnerabilities"), nil
}
case "LOW":
if len(result.Vulnerabilities) > 0 {
pkg/scanner/trivy/trivy.go
+5 -2
View File
@@ -187,13 +187,16 @@ func (s *Scanner) convertTrivyResult(trivyResult *TrivyResult, registry, package
// Aggregate all vulnerabilities from all results
for _, result := range trivyResult.Results {
for _, vuln := range result.Vulnerabilities {
// Normalize severity to standard values (CRITICAL, HIGH, MODERATE, LOW)
normalizedSeverity := metadata.NormalizeSeverity(vuln.Severity)
// Count by severity
severityCounts[strings.ToUpper(vuln.Severity)]++
severityCounts[normalizedSeverity]++
// Add to vulnerabilities list
vulnerabilities = append(vulnerabilities, metadata.Vulnerability{
ID: vuln.VulnerabilityID,
Severity: strings.ToUpper(vuln.Severity),
Severity: normalizedSeverity,
Title: vuln.Title,
Description: vuln.Description,
References: vuln.References,