This commit is contained in:
2026-01-02 04:02:02 +00:00
commit 3b8e171fdb
117 changed files with 21570 additions and 0 deletions
+528
View File
@@ -0,0 +1,528 @@
package file
import (
"context"
"encoding/json"
"fmt"
"os"
"path/filepath"
"sync"
"time"
"github.com/lukaszraczylo/gohoarder/pkg/metadata"
"github.com/rs/zerolog/log"
)
// Store implements a file-based metadata store
type Store struct {
basePath string
mu sync.RWMutex
}
// Config holds file store configuration
type Config struct {
Path string
}
// New creates a new file-based metadata store
func New(cfg Config) (*Store, error) {
if cfg.Path == "" {
cfg.Path = "./metadata"
}
// Create directory if it doesn't exist
if err := os.MkdirAll(cfg.Path, 0755); err != nil {
return nil, fmt.Errorf("failed to create metadata directory: %w", err)
}
log.Info().
Str("path", cfg.Path).
Msg("File-based metadata store initialized")
return &Store{
basePath: cfg.Path,
}, nil
}
// SavePackage saves package metadata
func (s *Store) SavePackage(ctx context.Context, pkg *metadata.Package) error {
s.mu.Lock()
defer s.mu.Unlock()
// Create registry directory
regDir := filepath.Join(s.basePath, pkg.Registry)
if err := os.MkdirAll(regDir, 0755); err != nil {
return err
}
// Save to file
filename := filepath.Join(regDir, fmt.Sprintf("%s-%s.json", pkg.Name, pkg.Version))
data, err := json.MarshalIndent(pkg, "", " ")
if err != nil {
return err
}
return os.WriteFile(filename, data, 0644)
}
// GetPackage retrieves package metadata
func (s *Store) GetPackage(ctx context.Context, registry, name, version string) (*metadata.Package, error) {
s.mu.RLock()
defer s.mu.RUnlock()
filename := filepath.Join(s.basePath, registry, fmt.Sprintf("%s-%s.json", name, version))
data, err := os.ReadFile(filename)
if err != nil {
if os.IsNotExist(err) {
return nil, nil
}
return nil, err
}
var pkg metadata.Package
if err := json.Unmarshal(data, &pkg); err != nil {
return nil, err
}
return &pkg, nil
}
// ListPackages lists all packages
func (s *Store) ListPackages(ctx context.Context, opts *metadata.ListOptions) ([]*metadata.Package, error) {
s.mu.RLock()
defer s.mu.RUnlock()
var packages []*metadata.Package
// Walk through all files
err := filepath.Walk(s.basePath, func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}
if info.IsDir() || filepath.Ext(path) != ".json" {
return nil
}
data, err := os.ReadFile(path)
if err != nil {
return nil // Skip files we can't read
}
var pkg metadata.Package
if err := json.Unmarshal(data, &pkg); err != nil {
return nil // Skip invalid JSON
}
packages = append(packages, &pkg)
return nil
})
if err != nil {
return nil, err
}
// Apply pagination if options provided
if opts != nil {
if opts.Offset >= len(packages) {
return []*metadata.Package{}, nil
}
end := opts.Offset + opts.Limit
if end > len(packages) {
end = len(packages)
}
return packages[opts.Offset:end], nil
}
return packages, nil
}
// DeletePackage deletes package metadata
func (s *Store) DeletePackage(ctx context.Context, registry, name, version string) error {
s.mu.Lock()
defer s.mu.Unlock()
filename := filepath.Join(s.basePath, registry, fmt.Sprintf("%s-%s.json", name, version))
if err := os.Remove(filename); err != nil && !os.IsNotExist(err) {
return err
}
return nil
}
// SaveScanResult saves scan result
func (s *Store) SaveScanResult(ctx context.Context, result *metadata.ScanResult) error {
s.mu.Lock()
defer s.mu.Unlock()
// Create scans directory
scanDir := filepath.Join(s.basePath, "scans", result.Registry, result.PackageName)
if err := os.MkdirAll(scanDir, 0755); err != nil {
return err
}
// Save to file with timestamp
timestamp := time.Now().Unix()
filename := filepath.Join(scanDir, fmt.Sprintf("%s-%d.json", result.PackageVersion, timestamp))
data, err := json.MarshalIndent(result, "", " ")
if err != nil {
return err
}
return os.WriteFile(filename, data, 0644)
}
// UpdateDownloadCount increments download counter
func (s *Store) UpdateDownloadCount(ctx context.Context, registry, name, version string) error {
s.mu.Lock()
defer s.mu.Unlock()
// Load package
pkg, err := s.GetPackage(ctx, registry, name, version)
if err != nil || pkg == nil {
return err
}
// Increment counter
pkg.DownloadCount++
pkg.LastAccessed = time.Now()
// Save back
return s.SavePackage(ctx, pkg)
}
// GetStats returns statistics for a registry
func (s *Store) GetStats(ctx context.Context, registry string) (*metadata.Stats, error) {
s.mu.RLock()
defer s.mu.RUnlock()
stats := &metadata.Stats{
Registry: registry,
LastUpdated: time.Now(),
}
// Walk through files and calculate stats
err := filepath.Walk(s.basePath, func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}
if info.IsDir() || filepath.Ext(path) != ".json" {
return nil
}
data, err := os.ReadFile(path)
if err != nil {
return nil
}
var pkg metadata.Package
if err := json.Unmarshal(data, &pkg); err != nil {
return nil
}
// Filter by registry if specified
if registry != "" && pkg.Registry != registry {
return nil
}
stats.TotalPackages++
stats.TotalSize += pkg.Size
stats.TotalDownloads += pkg.DownloadCount
if pkg.SecurityScanned {
stats.ScannedPackages++
}
return nil
})
if err != nil {
return nil, err
}
return stats, nil
}
// GetScanResult retrieves latest scan result
func (s *Store) GetScanResult(ctx context.Context, registry, name, version string) (*metadata.ScanResult, error) {
s.mu.RLock()
defer s.mu.RUnlock()
scanDir := filepath.Join(s.basePath, "scans", registry, name)
pattern := filepath.Join(scanDir, fmt.Sprintf("%s-*.json", version))
matches, err := filepath.Glob(pattern)
if err != nil {
return nil, err
}
if len(matches) == 0 {
return nil, nil
}
// Get the latest file
latestFile := matches[len(matches)-1]
data, err := os.ReadFile(latestFile)
if err != nil {
return nil, err
}
var result metadata.ScanResult
if err := json.Unmarshal(data, &result); err != nil {
return nil, err
}
return &result, nil
}
// Count returns total number of packages
func (s *Store) Count(ctx context.Context) (int, error) {
s.mu.RLock()
defer s.mu.RUnlock()
count := 0
err := filepath.Walk(s.basePath, func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}
if !info.IsDir() && filepath.Ext(path) == ".json" && filepath.Dir(path) != filepath.Join(s.basePath, "scans") {
count++
}
return nil
})
if err != nil {
return 0, err
}
return count, nil
}
// Health checks if the store is healthy
func (s *Store) Health(ctx context.Context) error {
// Check if directory is accessible
_, err := os.Stat(s.basePath)
return err
}
// SaveCVEBypass saves a CVE bypass (admin only)
func (s *Store) SaveCVEBypass(ctx context.Context, bypass *metadata.CVEBypass) error {
s.mu.Lock()
defer s.mu.Unlock()
// Create bypasses directory
bypassesDir := filepath.Join(s.basePath, "bypasses")
if err := os.MkdirAll(bypassesDir, 0755); err != nil {
return err
}
// Save to file
filename := filepath.Join(bypassesDir, fmt.Sprintf("%s.json", bypass.ID))
data, err := json.MarshalIndent(bypass, "", " ")
if err != nil {
return err
}
return os.WriteFile(filename, data, 0644)
}
// GetActiveCVEBypasses retrieves all active (non-expired) CVE bypasses
func (s *Store) GetActiveCVEBypasses(ctx context.Context) ([]*metadata.CVEBypass, error) {
s.mu.RLock()
defer s.mu.RUnlock()
bypassesDir := filepath.Join(s.basePath, "bypasses")
var bypasses []*metadata.CVEBypass
now := time.Now()
// Read all bypass files
err := filepath.Walk(bypassesDir, func(path string, info os.FileInfo, err error) error {
if err != nil {
if os.IsNotExist(err) {
return nil // bypasses directory doesn't exist yet
}
return err
}
if info.IsDir() || filepath.Ext(path) != ".json" {
return nil
}
data, err := os.ReadFile(path)
if err != nil {
return err
}
var bypass metadata.CVEBypass
if err := json.Unmarshal(data, &bypass); err != nil {
log.Warn().Err(err).Str("file", path).Msg("Failed to unmarshal bypass")
return nil
}
// Only include active and non-expired bypasses
if bypass.Active && bypass.ExpiresAt.After(now) {
bypasses = append(bypasses, &bypass)
}
return nil
})
if err != nil {
return nil, err
}
return bypasses, nil
}
// ListCVEBypasses lists all CVE bypasses (including expired)
func (s *Store) ListCVEBypasses(ctx context.Context, opts *metadata.BypassListOptions) ([]*metadata.CVEBypass, error) {
s.mu.RLock()
defer s.mu.RUnlock()
bypassesDir := filepath.Join(s.basePath, "bypasses")
var bypasses []*metadata.CVEBypass
now := time.Now()
// Read all bypass files
err := filepath.Walk(bypassesDir, func(path string, info os.FileInfo, err error) error {
if err != nil {
if os.IsNotExist(err) {
return nil // bypasses directory doesn't exist yet
}
return err
}
if info.IsDir() || filepath.Ext(path) != ".json" {
return nil
}
data, err := os.ReadFile(path)
if err != nil {
return err
}
var bypass metadata.CVEBypass
if err := json.Unmarshal(data, &bypass); err != nil {
log.Warn().Err(err).Str("file", path).Msg("Failed to unmarshal bypass")
return nil
}
// Apply filters if options provided
if opts != nil {
if opts.Type != "" && bypass.Type != opts.Type {
return nil
}
if !opts.IncludeExpired && bypass.ExpiresAt.Before(now) {
return nil
}
if opts.ActiveOnly && !bypass.Active {
return nil
}
}
bypasses = append(bypasses, &bypass)
return nil
})
if err != nil {
return nil, err
}
// Apply limit and offset if specified
if opts != nil {
if opts.Offset > 0 && opts.Offset < len(bypasses) {
bypasses = bypasses[opts.Offset:]
} else if opts.Offset >= len(bypasses) {
return []*metadata.CVEBypass{}, nil
}
if opts.Limit > 0 && opts.Limit < len(bypasses) {
bypasses = bypasses[:opts.Limit]
}
}
return bypasses, nil
}
// DeleteCVEBypass deletes a CVE bypass by ID
func (s *Store) DeleteCVEBypass(ctx context.Context, id string) error {
s.mu.Lock()
defer s.mu.Unlock()
filename := filepath.Join(s.basePath, "bypasses", fmt.Sprintf("%s.json", id))
err := os.Remove(filename)
if err != nil {
if os.IsNotExist(err) {
return fmt.Errorf("CVE bypass not found: %s", id)
}
return err
}
return nil
}
// CleanupExpiredBypasses removes expired bypasses
func (s *Store) CleanupExpiredBypasses(ctx context.Context) (int, error) {
s.mu.Lock()
defer s.mu.Unlock()
bypassesDir := filepath.Join(s.basePath, "bypasses")
count := 0
now := time.Now()
// Read all bypass files
err := filepath.Walk(bypassesDir, func(path string, info os.FileInfo, err error) error {
if err != nil {
if os.IsNotExist(err) {
return nil // bypasses directory doesn't exist yet
}
return err
}
if info.IsDir() || filepath.Ext(path) != ".json" {
return nil
}
data, err := os.ReadFile(path)
if err != nil {
return err
}
var bypass metadata.CVEBypass
if err := json.Unmarshal(data, &bypass); err != nil {
log.Warn().Err(err).Str("file", path).Msg("Failed to unmarshal bypass")
return nil
}
// Delete if expired
if bypass.ExpiresAt.Before(now) {
if err := os.Remove(path); err != nil {
log.Warn().Err(err).Str("file", path).Msg("Failed to delete expired bypass")
} else {
count++
}
}
return nil
})
if err != nil {
return 0, err
}
return count, nil
}
// Close closes the store
func (s *Store) Close() error {
// Nothing to close for file-based store
return nil
}
+170
View File
@@ -0,0 +1,170 @@
package metadata
import (
"context"
"time"
)
// Store is an alias for MetadataStore for convenience
type Store = MetadataStore
// MetadataStore defines the interface for package metadata storage
type MetadataStore interface {
// SavePackage saves package metadata
SavePackage(ctx context.Context, pkg *Package) error
// GetPackage retrieves package metadata
GetPackage(ctx context.Context, registry, name, version string) (*Package, error)
// DeletePackage deletes package metadata
DeletePackage(ctx context.Context, registry, name, version string) error
// ListPackages lists packages with optional filtering
ListPackages(ctx context.Context, opts *ListOptions) ([]*Package, error)
// UpdateDownloadCount increments download counter
UpdateDownloadCount(ctx context.Context, registry, name, version string) error
// GetStats returns statistics
GetStats(ctx context.Context, registry string) (*Stats, error)
// SaveScanResult saves security scan result
SaveScanResult(ctx context.Context, result *ScanResult) error
// GetScanResult retrieves security scan result
GetScanResult(ctx context.Context, registry, name, version string) (*ScanResult, error)
// SaveCVEBypass saves a CVE bypass (admin only)
SaveCVEBypass(ctx context.Context, bypass *CVEBypass) error
// GetActiveCVEBypasses retrieves all active (non-expired) CVE bypasses
GetActiveCVEBypasses(ctx context.Context) ([]*CVEBypass, error)
// ListCVEBypasses lists all CVE bypasses (including expired)
ListCVEBypasses(ctx context.Context, opts *BypassListOptions) ([]*CVEBypass, error)
// DeleteCVEBypass deletes a CVE bypass by ID
DeleteCVEBypass(ctx context.Context, id string) error
// CleanupExpiredBypasses removes expired bypasses
CleanupExpiredBypasses(ctx context.Context) (int, error)
// Count returns total number of packages
Count(ctx context.Context) (int, error)
// Health checks metadata store health
Health(ctx context.Context) error
// Close closes the metadata store
Close() error
}
// Package represents package metadata
type Package struct {
ID string `json:"id"`
Registry string `json:"registry"` // npm, pypi, go
Name string `json:"name"` // Package name
Version string `json:"version"` // Package version
StorageKey string `json:"storage_key"` // Key in storage backend
Size int64 `json:"size"` // Package size in bytes
ChecksumMD5 string `json:"checksum_md5"` // MD5 checksum
ChecksumSHA256 string `json:"checksum_sha256"` // SHA256 checksum
UpstreamURL string `json:"upstream_url"` // Original upstream URL
CachedAt time.Time `json:"cached_at"` // When cached
LastAccessed time.Time `json:"last_accessed"` // Last access time
ExpiresAt *time.Time `json:"expires_at"` // Expiration time (nil = never)
DownloadCount int64 `json:"download_count"` // Download counter
Metadata map[string]string `json:"metadata"` // Additional metadata
SecurityScanned bool `json:"security_scanned"` // Has been scanned
}
// ScanResult represents a security scan result
type ScanResult struct {
ID string `json:"id"`
Registry string `json:"registry"`
PackageName string `json:"package_name"`
PackageVersion string `json:"package_version"`
Scanner string `json:"scanner"` // trivy, osv, etc.
ScannedAt time.Time `json:"scanned_at"`
Status ScanStatus `json:"status"` // clean, vulnerable, error
VulnerabilityCount int `json:"vulnerability_count"`
Vulnerabilities []Vulnerability `json:"vulnerabilities"`
Details map[string]interface{} `json:"details"` // Scanner-specific details
}
// Vulnerability represents a security vulnerability
type Vulnerability struct {
ID string `json:"id"` // CVE-xxx, GHSA-xxx, etc.
Severity string `json:"severity"` // critical, high, medium, low
Title string `json:"title"`
Description string `json:"description"`
References []string `json:"references"`
FixedIn string `json:"fixed_in"` // Version where fixed
DetectedBy []string `json:"detected_by,omitempty"` // List of scanners that detected this vulnerability
}
// ScanStatus represents scan result status
type ScanStatus string
const (
ScanStatusClean ScanStatus = "clean"
ScanStatusVulnerable ScanStatus = "vulnerable"
ScanStatusError ScanStatus = "error"
ScanStatusPending ScanStatus = "pending"
)
// Stats represents metadata statistics
type Stats struct {
Registry string `json:"registry"`
TotalPackages int64 `json:"total_packages"`
TotalSize int64 `json:"total_size"`
TotalDownloads int64 `json:"total_downloads"`
ScannedPackages int64 `json:"scanned_packages"`
VulnerablePackages int64 `json:"vulnerable_packages"`
LastUpdated time.Time `json:"last_updated"`
}
// CVEBypass represents a temporary bypass for a CVE or package
type CVEBypass struct {
ID string `json:"id"` // Unique bypass ID
Type BypassType `json:"type"` // cve, package
Target string `json:"target"` // CVE ID (e.g., "CVE-2021-23337") or package (e.g., "npm/[email protected]")
Reason string `json:"reason"` // Why this bypass was created
CreatedBy string `json:"created_by"` // Admin user who created it
CreatedAt time.Time `json:"created_at"` // When created
ExpiresAt time.Time `json:"expires_at"` // When it expires
AppliesTo string `json:"applies_to,omitempty"` // Optional: limit to specific package (for CVE bypasses)
NotifyOnExpiry bool `json:"notify_on_expiry"` // Send notification when expired
Active bool `json:"active"` // Can be deactivated without deletion
}
// BypassType represents the type of bypass
type BypassType string
const (
BypassTypeCVE BypassType = "cve" // Bypass specific CVE
BypassTypePackage BypassType = "package" // Bypass entire package
)
// BypassListOptions contains options for listing CVE bypasses
type BypassListOptions struct {
Type BypassType // Filter by type
IncludeExpired bool // Include expired bypasses
ActiveOnly bool // Only active bypasses
Limit int // Max results
Offset int // Pagination offset
}
// ListOptions contains options for listing packages
type ListOptions struct {
Registry string // Filter by registry
NamePrefix string // Filter by name prefix
MinSize int64 // Minimum package size
MaxSize int64 // Maximum package size
ScannedOnly bool // Only scanned packages
SinceDate time.Time // Packages cached since date
Limit int // Max results
Offset int // Pagination offset
SortBy string // Sort field (name, size, cached_at, download_count)
SortDesc bool // Sort descending
}
+707
View File
@@ -0,0 +1,707 @@
package sqlite
import (
"context"
"database/sql"
"fmt"
"sync"
"time"
goccy_json "github.com/goccy/go-json"
_ "modernc.org/sqlite"
"github.com/lukaszraczylo/gohoarder/pkg/errors"
"github.com/lukaszraczylo/gohoarder/pkg/metadata"
"github.com/rs/zerolog/log"
)
// SQLiteStore implements metadata.MetadataStore using SQLite
type SQLiteStore struct {
db *sql.DB
mu sync.RWMutex
}
// Config holds SQLite configuration
type Config struct {
Path string // Database file path
MaxOpenConns int // Maximum open connections
MaxIdleConns int // Maximum idle connections
}
const schema = `
CREATE TABLE IF NOT EXISTS packages (
id TEXT PRIMARY KEY,
registry TEXT NOT NULL,
name TEXT NOT NULL,
version TEXT NOT NULL,
storage_key TEXT NOT NULL,
size INTEGER NOT NULL,
checksum_md5 TEXT,
checksum_sha256 TEXT,
upstream_url TEXT,
cached_at DATETIME NOT NULL,
last_accessed DATETIME NOT NULL,
expires_at DATETIME,
download_count INTEGER DEFAULT 0,
metadata TEXT,
security_scanned BOOLEAN DEFAULT 0,
UNIQUE(registry, name, version)
);
CREATE INDEX IF NOT EXISTS idx_packages_registry ON packages(registry);
CREATE INDEX IF NOT EXISTS idx_packages_name ON packages(name);
CREATE INDEX IF NOT EXISTS idx_packages_cached_at ON packages(cached_at);
CREATE INDEX IF NOT EXISTS idx_packages_last_accessed ON packages(last_accessed);
CREATE INDEX IF NOT EXISTS idx_packages_expires_at ON packages(expires_at);
CREATE TABLE IF NOT EXISTS scan_results (
id TEXT PRIMARY KEY,
registry TEXT NOT NULL,
package_name TEXT NOT NULL,
package_version TEXT NOT NULL,
scanner TEXT NOT NULL,
scanned_at DATETIME NOT NULL,
status TEXT NOT NULL,
vulnerability_count INTEGER DEFAULT 0,
vulnerabilities TEXT,
details TEXT,
UNIQUE(registry, package_name, package_version, scanner)
);
CREATE INDEX IF NOT EXISTS idx_scan_results_registry ON scan_results(registry);
CREATE INDEX IF NOT EXISTS idx_scan_results_package ON scan_results(package_name);
CREATE INDEX IF NOT EXISTS idx_scan_results_status ON scan_results(status);
CREATE TABLE IF NOT EXISTS cve_bypasses (
id TEXT PRIMARY KEY,
type TEXT NOT NULL,
target TEXT NOT NULL,
reason TEXT NOT NULL,
created_by TEXT NOT NULL,
created_at DATETIME NOT NULL,
expires_at DATETIME NOT NULL,
applies_to TEXT,
notify_on_expiry BOOLEAN DEFAULT 0,
active BOOLEAN DEFAULT 1
);
CREATE INDEX IF NOT EXISTS idx_cve_bypasses_type ON cve_bypasses(type);
CREATE INDEX IF NOT EXISTS idx_cve_bypasses_target ON cve_bypasses(target);
CREATE INDEX IF NOT EXISTS idx_cve_bypasses_expires_at ON cve_bypasses(expires_at);
CREATE INDEX IF NOT EXISTS idx_cve_bypasses_active ON cve_bypasses(active);
`
// New creates a new SQLite metadata store
func New(cfg Config) (*SQLiteStore, error) {
if cfg.Path == "" {
return nil, errors.New(errors.ErrCodeInvalidConfig, "SQLite database path is required")
}
if cfg.MaxOpenConns == 0 {
cfg.MaxOpenConns = 10
}
if cfg.MaxIdleConns == 0 {
cfg.MaxIdleConns = 5
}
// Open database with WAL mode for better concurrency
dsn := fmt.Sprintf("%s?_journal_mode=WAL&_busy_timeout=5000&_synchronous=NORMAL&_cache_size=2000", cfg.Path)
db, err := sql.Open("sqlite", dsn)
if err != nil {
return nil, errors.Wrap(err, errors.ErrCodeStorageFailure, "failed to open SQLite database")
}
db.SetMaxOpenConns(cfg.MaxOpenConns)
db.SetMaxIdleConns(cfg.MaxIdleConns)
db.SetConnMaxLifetime(time.Hour)
// Create schema
if _, err := db.Exec(schema); err != nil {
db.Close()
return nil, errors.Wrap(err, errors.ErrCodeStorageFailure, "failed to create SQLite schema")
}
return &SQLiteStore{
db: db,
}, nil
}
// SavePackage saves package metadata
func (s *SQLiteStore) SavePackage(ctx context.Context, pkg *metadata.Package) error {
s.mu.Lock()
defer s.mu.Unlock()
// Serialize metadata
metadataJSON, err := goccy_json.Marshal(pkg.Metadata)
if err != nil {
return errors.Wrap(err, errors.ErrCodeInternalServer, "failed to serialize package metadata")
}
var expiresAt interface{}
if pkg.ExpiresAt != nil {
expiresAt = pkg.ExpiresAt
}
query := `
INSERT INTO packages (
id, registry, name, version, storage_key, size,
checksum_md5, checksum_sha256, upstream_url,
cached_at, last_accessed, expires_at, download_count,
metadata, security_scanned
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(registry, name, version) DO UPDATE SET
storage_key = excluded.storage_key,
size = excluded.size,
checksum_md5 = excluded.checksum_md5,
checksum_sha256 = excluded.checksum_sha256,
upstream_url = excluded.upstream_url,
last_accessed = excluded.last_accessed,
expires_at = excluded.expires_at,
metadata = excluded.metadata,
security_scanned = excluded.security_scanned
`
_, err = s.db.ExecContext(ctx, query,
pkg.ID, pkg.Registry, pkg.Name, pkg.Version, pkg.StorageKey, pkg.Size,
pkg.ChecksumMD5, pkg.ChecksumSHA256, pkg.UpstreamURL,
pkg.CachedAt, pkg.LastAccessed, expiresAt, pkg.DownloadCount,
string(metadataJSON), pkg.SecurityScanned,
)
if err != nil {
return errors.Wrap(err, errors.ErrCodeStorageFailure, "failed to save package metadata")
}
return nil
}
// GetPackage retrieves package metadata
func (s *SQLiteStore) GetPackage(ctx context.Context, registry, name, version string) (*metadata.Package, error) {
s.mu.RLock()
defer s.mu.RUnlock()
query := `
SELECT id, registry, name, version, storage_key, size,
checksum_md5, checksum_sha256, upstream_url,
cached_at, last_accessed, expires_at, download_count,
metadata, security_scanned
FROM packages
WHERE registry = ? AND name = ? AND version = ?
`
var pkg metadata.Package
var metadataJSON string
var expiresAt sql.NullTime
err := s.db.QueryRowContext(ctx, query, registry, name, version).Scan(
&pkg.ID, &pkg.Registry, &pkg.Name, &pkg.Version, &pkg.StorageKey, &pkg.Size,
&pkg.ChecksumMD5, &pkg.ChecksumSHA256, &pkg.UpstreamURL,
&pkg.CachedAt, &pkg.LastAccessed, &expiresAt, &pkg.DownloadCount,
&metadataJSON, &pkg.SecurityScanned,
)
if err == sql.ErrNoRows {
return nil, errors.NotFound(fmt.Sprintf("package not found: %s/%s@%s", registry, name, version))
}
if err != nil {
return nil, errors.Wrap(err, errors.ErrCodeStorageFailure, "failed to get package metadata")
}
if expiresAt.Valid {
pkg.ExpiresAt = &expiresAt.Time
}
// Deserialize metadata
if metadataJSON != "" {
if err := goccy_json.Unmarshal([]byte(metadataJSON), &pkg.Metadata); err != nil {
log.Warn().Err(err).Msg("Failed to deserialize package metadata")
}
}
return &pkg, nil
}
// DeletePackage deletes package metadata
func (s *SQLiteStore) DeletePackage(ctx context.Context, registry, name, version string) error {
s.mu.Lock()
defer s.mu.Unlock()
query := "DELETE FROM packages WHERE registry = ? AND name = ? AND version = ?"
result, err := s.db.ExecContext(ctx, query, registry, name, version)
if err != nil {
return errors.Wrap(err, errors.ErrCodeStorageFailure, "failed to delete package metadata")
}
rows, _ := result.RowsAffected()
if rows == 0 {
return errors.NotFound(fmt.Sprintf("package not found: %s/%s@%s", registry, name, version))
}
return nil
}
// ListPackages lists packages with optional filtering
func (s *SQLiteStore) ListPackages(ctx context.Context, opts *metadata.ListOptions) ([]*metadata.Package, error) {
s.mu.RLock()
defer s.mu.RUnlock()
query := "SELECT id, registry, name, version, storage_key, size, checksum_md5, checksum_sha256, upstream_url, cached_at, last_accessed, expires_at, download_count, metadata, security_scanned FROM packages WHERE 1=1"
args := []interface{}{}
if opts != nil {
if opts.Registry != "" {
query += " AND registry = ?"
args = append(args, opts.Registry)
}
if opts.NamePrefix != "" {
query += " AND name LIKE ?"
args = append(args, opts.NamePrefix+"%")
}
if opts.MinSize > 0 {
query += " AND size >= ?"
args = append(args, opts.MinSize)
}
if opts.MaxSize > 0 {
query += " AND size <= ?"
args = append(args, opts.MaxSize)
}
if opts.ScannedOnly {
query += " AND security_scanned = 1"
}
if !opts.SinceDate.IsZero() {
query += " AND cached_at >= ?"
args = append(args, opts.SinceDate)
}
// Sorting
sortBy := "cached_at"
if opts.SortBy != "" {
sortBy = opts.SortBy
}
sortOrder := "ASC"
if opts.SortDesc {
sortOrder = "DESC"
}
query += fmt.Sprintf(" ORDER BY %s %s", sortBy, sortOrder)
// Pagination
if opts.Limit > 0 {
query += " LIMIT ?"
args = append(args, opts.Limit)
}
if opts.Offset > 0 {
query += " OFFSET ?"
args = append(args, opts.Offset)
}
}
rows, err := s.db.QueryContext(ctx, query, args...)
if err != nil {
return nil, errors.Wrap(err, errors.ErrCodeStorageFailure, "failed to list packages")
}
defer rows.Close()
var packages []*metadata.Package
for rows.Next() {
var pkg metadata.Package
var metadataJSON string
var expiresAt sql.NullTime
err := rows.Scan(
&pkg.ID, &pkg.Registry, &pkg.Name, &pkg.Version, &pkg.StorageKey, &pkg.Size,
&pkg.ChecksumMD5, &pkg.ChecksumSHA256, &pkg.UpstreamURL,
&pkg.CachedAt, &pkg.LastAccessed, &expiresAt, &pkg.DownloadCount,
&metadataJSON, &pkg.SecurityScanned,
)
if err != nil {
return nil, errors.Wrap(err, errors.ErrCodeStorageFailure, "failed to scan package row")
}
if expiresAt.Valid {
pkg.ExpiresAt = &expiresAt.Time
}
if metadataJSON != "" {
goccy_json.Unmarshal([]byte(metadataJSON), &pkg.Metadata)
}
packages = append(packages, &pkg)
}
return packages, nil
}
// UpdateDownloadCount increments download counter
func (s *SQLiteStore) UpdateDownloadCount(ctx context.Context, registry, name, version string) error {
s.mu.Lock()
defer s.mu.Unlock()
query := `
UPDATE packages
SET download_count = download_count + 1,
last_accessed = ?
WHERE registry = ? AND name = ? AND version = ?
`
_, err := s.db.ExecContext(ctx, query, time.Now(), registry, name, version)
if err != nil {
return errors.Wrap(err, errors.ErrCodeStorageFailure, "failed to update download count")
}
return nil
}
// GetStats returns statistics
func (s *SQLiteStore) GetStats(ctx context.Context, registry string) (*metadata.Stats, error) {
s.mu.RLock()
defer s.mu.RUnlock()
query := `
SELECT
COUNT(*) as total_packages,
COALESCE(SUM(size), 0) as total_size,
COALESCE(SUM(download_count), 0) as total_downloads,
COALESCE(SUM(CASE WHEN security_scanned = 1 THEN 1 ELSE 0 END), 0) as scanned_packages
FROM packages
`
args := []interface{}{}
if registry != "" {
query += " WHERE registry = ?"
args = append(args, registry)
}
var stats metadata.Stats
stats.Registry = registry
stats.LastUpdated = time.Now()
err := s.db.QueryRowContext(ctx, query, args...).Scan(
&stats.TotalPackages,
&stats.TotalSize,
&stats.TotalDownloads,
&stats.ScannedPackages,
)
if err != nil {
return nil, errors.Wrap(err, errors.ErrCodeStorageFailure, "failed to get stats")
}
// Count vulnerable packages
vulnQuery := `SELECT COUNT(*) FROM scan_results WHERE status = 'vulnerable'`
vulnArgs := []interface{}{}
if registry != "" {
vulnQuery += " AND registry = ?"
vulnArgs = append(vulnArgs, registry)
}
s.db.QueryRowContext(ctx, vulnQuery, vulnArgs...).Scan(&stats.VulnerablePackages)
return &stats, nil
}
// SaveScanResult saves security scan result
func (s *SQLiteStore) SaveScanResult(ctx context.Context, result *metadata.ScanResult) error {
s.mu.Lock()
defer s.mu.Unlock()
// Serialize vulnerabilities and details
vulnJSON, err := goccy_json.Marshal(result.Vulnerabilities)
if err != nil {
return errors.Wrap(err, errors.ErrCodeInternalServer, "failed to serialize vulnerabilities")
}
detailsJSON, err := goccy_json.Marshal(result.Details)
if err != nil {
return errors.Wrap(err, errors.ErrCodeInternalServer, "failed to serialize scan details")
}
query := `
INSERT INTO scan_results (
id, registry, package_name, package_version, scanner,
scanned_at, status, vulnerability_count, vulnerabilities, details
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(registry, package_name, package_version, scanner) DO UPDATE SET
scanned_at = excluded.scanned_at,
status = excluded.status,
vulnerability_count = excluded.vulnerability_count,
vulnerabilities = excluded.vulnerabilities,
details = excluded.details
`
_, err = s.db.ExecContext(ctx, query,
result.ID, result.Registry, result.PackageName, result.PackageVersion, result.Scanner,
result.ScannedAt, result.Status, result.VulnerabilityCount,
string(vulnJSON), string(detailsJSON),
)
if err != nil {
return errors.Wrap(err, errors.ErrCodeStorageFailure, "failed to save scan result")
}
// Update package security_scanned flag
updateQuery := `UPDATE packages SET security_scanned = 1 WHERE registry = ? AND name = ? AND version = ?`
s.db.ExecContext(ctx, updateQuery, result.Registry, result.PackageName, result.PackageVersion)
return nil
}
// GetScanResult retrieves security scan result
func (s *SQLiteStore) GetScanResult(ctx context.Context, registry, name, version string) (*metadata.ScanResult, error) {
s.mu.RLock()
defer s.mu.RUnlock()
query := `
SELECT id, registry, package_name, package_version, scanner,
scanned_at, status, vulnerability_count, vulnerabilities, details
FROM scan_results
WHERE registry = ? AND package_name = ? AND package_version = ?
ORDER BY scanned_at DESC
LIMIT 1
`
var result metadata.ScanResult
var vulnJSON, detailsJSON string
err := s.db.QueryRowContext(ctx, query, registry, name, version).Scan(
&result.ID, &result.Registry, &result.PackageName, &result.PackageVersion, &result.Scanner,
&result.ScannedAt, &result.Status, &result.VulnerabilityCount,
&vulnJSON, &detailsJSON,
)
if err == sql.ErrNoRows {
return nil, errors.NotFound(fmt.Sprintf("scan result not found: %s/%s@%s", registry, name, version))
}
if err != nil {
return nil, errors.Wrap(err, errors.ErrCodeStorageFailure, "failed to get scan result")
}
// Deserialize
if vulnJSON != "" {
goccy_json.Unmarshal([]byte(vulnJSON), &result.Vulnerabilities)
}
if detailsJSON != "" {
goccy_json.Unmarshal([]byte(detailsJSON), &result.Details)
}
return &result, nil
}
// Count returns total number of packages
func (s *SQLiteStore) Count(ctx context.Context) (int, error) {
s.mu.RLock()
defer s.mu.RUnlock()
var count int
query := "SELECT COUNT(*) FROM packages"
err := s.db.QueryRowContext(ctx, query).Scan(&count)
if err != nil {
return 0, errors.Wrap(err, errors.ErrCodeStorageFailure, "failed to count packages")
}
return count, nil
}
// Health checks metadata store health
func (s *SQLiteStore) Health(ctx context.Context) error {
return s.db.PingContext(ctx)
}
// SaveCVEBypass saves a CVE bypass (admin only)
func (s *SQLiteStore) SaveCVEBypass(ctx context.Context, bypass *metadata.CVEBypass) error {
s.mu.Lock()
defer s.mu.Unlock()
query := `
INSERT INTO cve_bypasses (
id, type, target, reason, created_by, created_at,
expires_at, applies_to, notify_on_expiry, active
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(id) DO UPDATE SET
type = excluded.type,
target = excluded.target,
reason = excluded.reason,
expires_at = excluded.expires_at,
applies_to = excluded.applies_to,
notify_on_expiry = excluded.notify_on_expiry,
active = excluded.active
`
_, err := s.db.ExecContext(ctx, query,
bypass.ID, bypass.Type, bypass.Target, bypass.Reason, bypass.CreatedBy,
bypass.CreatedAt, bypass.ExpiresAt, bypass.AppliesTo,
bypass.NotifyOnExpiry, bypass.Active,
)
if err != nil {
return errors.Wrap(err, errors.ErrCodeStorageFailure, "failed to save CVE bypass")
}
return nil
}
// GetActiveCVEBypasses retrieves all active (non-expired) CVE bypasses
func (s *SQLiteStore) GetActiveCVEBypasses(ctx context.Context) ([]*metadata.CVEBypass, error) {
s.mu.RLock()
defer s.mu.RUnlock()
query := `
SELECT id, type, target, reason, created_by, created_at,
expires_at, applies_to, notify_on_expiry, active
FROM cve_bypasses
WHERE active = 1 AND expires_at > ?
ORDER BY created_at DESC
`
rows, err := s.db.QueryContext(ctx, query, time.Now())
if err != nil {
return nil, errors.Wrap(err, errors.ErrCodeStorageFailure, "failed to get active CVE bypasses")
}
defer rows.Close()
var bypasses []*metadata.CVEBypass
for rows.Next() {
var bypass metadata.CVEBypass
var appliesTo sql.NullString
err := rows.Scan(
&bypass.ID, &bypass.Type, &bypass.Target, &bypass.Reason, &bypass.CreatedBy,
&bypass.CreatedAt, &bypass.ExpiresAt, &appliesTo,
&bypass.NotifyOnExpiry, &bypass.Active,
)
if err != nil {
return nil, errors.Wrap(err, errors.ErrCodeStorageFailure, "failed to scan CVE bypass row")
}
if appliesTo.Valid {
bypass.AppliesTo = appliesTo.String
}
bypasses = append(bypasses, &bypass)
}
return bypasses, nil
}
// ListCVEBypasses lists all CVE bypasses (including expired)
func (s *SQLiteStore) ListCVEBypasses(ctx context.Context, opts *metadata.BypassListOptions) ([]*metadata.CVEBypass, error) {
s.mu.RLock()
defer s.mu.RUnlock()
query := `
SELECT id, type, target, reason, created_by, created_at,
expires_at, applies_to, notify_on_expiry, active
FROM cve_bypasses
WHERE 1=1
`
args := []interface{}{}
if opts != nil {
if opts.Type != "" {
query += " AND type = ?"
args = append(args, opts.Type)
}
if !opts.IncludeExpired {
query += " AND expires_at > ?"
args = append(args, time.Now())
}
if opts.ActiveOnly {
query += " AND active = 1"
}
query += " ORDER BY created_at DESC"
if opts.Limit > 0 {
query += " LIMIT ?"
args = append(args, opts.Limit)
}
if opts.Offset > 0 {
query += " OFFSET ?"
args = append(args, opts.Offset)
}
}
rows, err := s.db.QueryContext(ctx, query, args...)
if err != nil {
return nil, errors.Wrap(err, errors.ErrCodeStorageFailure, "failed to list CVE bypasses")
}
defer rows.Close()
var bypasses []*metadata.CVEBypass
for rows.Next() {
var bypass metadata.CVEBypass
var appliesTo sql.NullString
err := rows.Scan(
&bypass.ID, &bypass.Type, &bypass.Target, &bypass.Reason, &bypass.CreatedBy,
&bypass.CreatedAt, &bypass.ExpiresAt, &appliesTo,
&bypass.NotifyOnExpiry, &bypass.Active,
)
if err != nil {
return nil, errors.Wrap(err, errors.ErrCodeStorageFailure, "failed to scan CVE bypass row")
}
if appliesTo.Valid {
bypass.AppliesTo = appliesTo.String
}
bypasses = append(bypasses, &bypass)
}
return bypasses, nil
}
// DeleteCVEBypass deletes a CVE bypass by ID
func (s *SQLiteStore) DeleteCVEBypass(ctx context.Context, id string) error {
s.mu.Lock()
defer s.mu.Unlock()
query := "DELETE FROM cve_bypasses WHERE id = ?"
result, err := s.db.ExecContext(ctx, query, id)
if err != nil {
return errors.Wrap(err, errors.ErrCodeStorageFailure, "failed to delete CVE bypass")
}
rows, _ := result.RowsAffected()
if rows == 0 {
return errors.NotFound(fmt.Sprintf("CVE bypass not found: %s", id))
}
return nil
}
// CleanupExpiredBypasses removes expired bypasses
func (s *SQLiteStore) CleanupExpiredBypasses(ctx context.Context) (int, error) {
s.mu.Lock()
defer s.mu.Unlock()
query := "DELETE FROM cve_bypasses WHERE expires_at <= ?"
result, err := s.db.ExecContext(ctx, query, time.Now())
if err != nil {
return 0, errors.Wrap(err, errors.ErrCodeStorageFailure, "failed to cleanup expired CVE bypasses")
}
rows, _ := result.RowsAffected()
return int(rows), nil
}
// Close closes the metadata store
func (s *SQLiteStore) Close() error {
return s.db.Close()
}