Files
gohoarder/pkg/metadata/sqlite/sqlite.go
T
2026-01-02 04:02:02 +00:00

708 lines
19 KiB
Go

package sqlite
import (
"context"
"database/sql"
"fmt"
"sync"
"time"
goccy_json "github.com/goccy/go-json"
_ "modernc.org/sqlite"
"github.com/lukaszraczylo/gohoarder/pkg/errors"
"github.com/lukaszraczylo/gohoarder/pkg/metadata"
"github.com/rs/zerolog/log"
)
// SQLiteStore implements metadata.MetadataStore using SQLite
type SQLiteStore struct {
db *sql.DB
mu sync.RWMutex
}
// Config holds SQLite configuration
type Config struct {
Path string // Database file path
MaxOpenConns int // Maximum open connections
MaxIdleConns int // Maximum idle connections
}
const schema = `
CREATE TABLE IF NOT EXISTS packages (
id TEXT PRIMARY KEY,
registry TEXT NOT NULL,
name TEXT NOT NULL,
version TEXT NOT NULL,
storage_key TEXT NOT NULL,
size INTEGER NOT NULL,
checksum_md5 TEXT,
checksum_sha256 TEXT,
upstream_url TEXT,
cached_at DATETIME NOT NULL,
last_accessed DATETIME NOT NULL,
expires_at DATETIME,
download_count INTEGER DEFAULT 0,
metadata TEXT,
security_scanned BOOLEAN DEFAULT 0,
UNIQUE(registry, name, version)
);
CREATE INDEX IF NOT EXISTS idx_packages_registry ON packages(registry);
CREATE INDEX IF NOT EXISTS idx_packages_name ON packages(name);
CREATE INDEX IF NOT EXISTS idx_packages_cached_at ON packages(cached_at);
CREATE INDEX IF NOT EXISTS idx_packages_last_accessed ON packages(last_accessed);
CREATE INDEX IF NOT EXISTS idx_packages_expires_at ON packages(expires_at);
CREATE TABLE IF NOT EXISTS scan_results (
id TEXT PRIMARY KEY,
registry TEXT NOT NULL,
package_name TEXT NOT NULL,
package_version TEXT NOT NULL,
scanner TEXT NOT NULL,
scanned_at DATETIME NOT NULL,
status TEXT NOT NULL,
vulnerability_count INTEGER DEFAULT 0,
vulnerabilities TEXT,
details TEXT,
UNIQUE(registry, package_name, package_version, scanner)
);
CREATE INDEX IF NOT EXISTS idx_scan_results_registry ON scan_results(registry);
CREATE INDEX IF NOT EXISTS idx_scan_results_package ON scan_results(package_name);
CREATE INDEX IF NOT EXISTS idx_scan_results_status ON scan_results(status);
CREATE TABLE IF NOT EXISTS cve_bypasses (
id TEXT PRIMARY KEY,
type TEXT NOT NULL,
target TEXT NOT NULL,
reason TEXT NOT NULL,
created_by TEXT NOT NULL,
created_at DATETIME NOT NULL,
expires_at DATETIME NOT NULL,
applies_to TEXT,
notify_on_expiry BOOLEAN DEFAULT 0,
active BOOLEAN DEFAULT 1
);
CREATE INDEX IF NOT EXISTS idx_cve_bypasses_type ON cve_bypasses(type);
CREATE INDEX IF NOT EXISTS idx_cve_bypasses_target ON cve_bypasses(target);
CREATE INDEX IF NOT EXISTS idx_cve_bypasses_expires_at ON cve_bypasses(expires_at);
CREATE INDEX IF NOT EXISTS idx_cve_bypasses_active ON cve_bypasses(active);
`
// New creates a new SQLite metadata store
func New(cfg Config) (*SQLiteStore, error) {
if cfg.Path == "" {
return nil, errors.New(errors.ErrCodeInvalidConfig, "SQLite database path is required")
}
if cfg.MaxOpenConns == 0 {
cfg.MaxOpenConns = 10
}
if cfg.MaxIdleConns == 0 {
cfg.MaxIdleConns = 5
}
// Open database with WAL mode for better concurrency
dsn := fmt.Sprintf("%s?_journal_mode=WAL&_busy_timeout=5000&_synchronous=NORMAL&_cache_size=2000", cfg.Path)
db, err := sql.Open("sqlite", dsn)
if err != nil {
return nil, errors.Wrap(err, errors.ErrCodeStorageFailure, "failed to open SQLite database")
}
db.SetMaxOpenConns(cfg.MaxOpenConns)
db.SetMaxIdleConns(cfg.MaxIdleConns)
db.SetConnMaxLifetime(time.Hour)
// Create schema
if _, err := db.Exec(schema); err != nil {
db.Close()
return nil, errors.Wrap(err, errors.ErrCodeStorageFailure, "failed to create SQLite schema")
}
return &SQLiteStore{
db: db,
}, nil
}
// SavePackage saves package metadata
func (s *SQLiteStore) SavePackage(ctx context.Context, pkg *metadata.Package) error {
s.mu.Lock()
defer s.mu.Unlock()
// Serialize metadata
metadataJSON, err := goccy_json.Marshal(pkg.Metadata)
if err != nil {
return errors.Wrap(err, errors.ErrCodeInternalServer, "failed to serialize package metadata")
}
var expiresAt interface{}
if pkg.ExpiresAt != nil {
expiresAt = pkg.ExpiresAt
}
query := `
INSERT INTO packages (
id, registry, name, version, storage_key, size,
checksum_md5, checksum_sha256, upstream_url,
cached_at, last_accessed, expires_at, download_count,
metadata, security_scanned
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(registry, name, version) DO UPDATE SET
storage_key = excluded.storage_key,
size = excluded.size,
checksum_md5 = excluded.checksum_md5,
checksum_sha256 = excluded.checksum_sha256,
upstream_url = excluded.upstream_url,
last_accessed = excluded.last_accessed,
expires_at = excluded.expires_at,
metadata = excluded.metadata,
security_scanned = excluded.security_scanned
`
_, err = s.db.ExecContext(ctx, query,
pkg.ID, pkg.Registry, pkg.Name, pkg.Version, pkg.StorageKey, pkg.Size,
pkg.ChecksumMD5, pkg.ChecksumSHA256, pkg.UpstreamURL,
pkg.CachedAt, pkg.LastAccessed, expiresAt, pkg.DownloadCount,
string(metadataJSON), pkg.SecurityScanned,
)
if err != nil {
return errors.Wrap(err, errors.ErrCodeStorageFailure, "failed to save package metadata")
}
return nil
}
// GetPackage retrieves package metadata
func (s *SQLiteStore) GetPackage(ctx context.Context, registry, name, version string) (*metadata.Package, error) {
s.mu.RLock()
defer s.mu.RUnlock()
query := `
SELECT id, registry, name, version, storage_key, size,
checksum_md5, checksum_sha256, upstream_url,
cached_at, last_accessed, expires_at, download_count,
metadata, security_scanned
FROM packages
WHERE registry = ? AND name = ? AND version = ?
`
var pkg metadata.Package
var metadataJSON string
var expiresAt sql.NullTime
err := s.db.QueryRowContext(ctx, query, registry, name, version).Scan(
&pkg.ID, &pkg.Registry, &pkg.Name, &pkg.Version, &pkg.StorageKey, &pkg.Size,
&pkg.ChecksumMD5, &pkg.ChecksumSHA256, &pkg.UpstreamURL,
&pkg.CachedAt, &pkg.LastAccessed, &expiresAt, &pkg.DownloadCount,
&metadataJSON, &pkg.SecurityScanned,
)
if err == sql.ErrNoRows {
return nil, errors.NotFound(fmt.Sprintf("package not found: %s/%s@%s", registry, name, version))
}
if err != nil {
return nil, errors.Wrap(err, errors.ErrCodeStorageFailure, "failed to get package metadata")
}
if expiresAt.Valid {
pkg.ExpiresAt = &expiresAt.Time
}
// Deserialize metadata
if metadataJSON != "" {
if err := goccy_json.Unmarshal([]byte(metadataJSON), &pkg.Metadata); err != nil {
log.Warn().Err(err).Msg("Failed to deserialize package metadata")
}
}
return &pkg, nil
}
// DeletePackage deletes package metadata
func (s *SQLiteStore) DeletePackage(ctx context.Context, registry, name, version string) error {
s.mu.Lock()
defer s.mu.Unlock()
query := "DELETE FROM packages WHERE registry = ? AND name = ? AND version = ?"
result, err := s.db.ExecContext(ctx, query, registry, name, version)
if err != nil {
return errors.Wrap(err, errors.ErrCodeStorageFailure, "failed to delete package metadata")
}
rows, _ := result.RowsAffected()
if rows == 0 {
return errors.NotFound(fmt.Sprintf("package not found: %s/%s@%s", registry, name, version))
}
return nil
}
// ListPackages lists packages with optional filtering
func (s *SQLiteStore) ListPackages(ctx context.Context, opts *metadata.ListOptions) ([]*metadata.Package, error) {
s.mu.RLock()
defer s.mu.RUnlock()
query := "SELECT id, registry, name, version, storage_key, size, checksum_md5, checksum_sha256, upstream_url, cached_at, last_accessed, expires_at, download_count, metadata, security_scanned FROM packages WHERE 1=1"
args := []interface{}{}
if opts != nil {
if opts.Registry != "" {
query += " AND registry = ?"
args = append(args, opts.Registry)
}
if opts.NamePrefix != "" {
query += " AND name LIKE ?"
args = append(args, opts.NamePrefix+"%")
}
if opts.MinSize > 0 {
query += " AND size >= ?"
args = append(args, opts.MinSize)
}
if opts.MaxSize > 0 {
query += " AND size <= ?"
args = append(args, opts.MaxSize)
}
if opts.ScannedOnly {
query += " AND security_scanned = 1"
}
if !opts.SinceDate.IsZero() {
query += " AND cached_at >= ?"
args = append(args, opts.SinceDate)
}
// Sorting
sortBy := "cached_at"
if opts.SortBy != "" {
sortBy = opts.SortBy
}
sortOrder := "ASC"
if opts.SortDesc {
sortOrder = "DESC"
}
query += fmt.Sprintf(" ORDER BY %s %s", sortBy, sortOrder)
// Pagination
if opts.Limit > 0 {
query += " LIMIT ?"
args = append(args, opts.Limit)
}
if opts.Offset > 0 {
query += " OFFSET ?"
args = append(args, opts.Offset)
}
}
rows, err := s.db.QueryContext(ctx, query, args...)
if err != nil {
return nil, errors.Wrap(err, errors.ErrCodeStorageFailure, "failed to list packages")
}
defer rows.Close()
var packages []*metadata.Package
for rows.Next() {
var pkg metadata.Package
var metadataJSON string
var expiresAt sql.NullTime
err := rows.Scan(
&pkg.ID, &pkg.Registry, &pkg.Name, &pkg.Version, &pkg.StorageKey, &pkg.Size,
&pkg.ChecksumMD5, &pkg.ChecksumSHA256, &pkg.UpstreamURL,
&pkg.CachedAt, &pkg.LastAccessed, &expiresAt, &pkg.DownloadCount,
&metadataJSON, &pkg.SecurityScanned,
)
if err != nil {
return nil, errors.Wrap(err, errors.ErrCodeStorageFailure, "failed to scan package row")
}
if expiresAt.Valid {
pkg.ExpiresAt = &expiresAt.Time
}
if metadataJSON != "" {
goccy_json.Unmarshal([]byte(metadataJSON), &pkg.Metadata)
}
packages = append(packages, &pkg)
}
return packages, nil
}
// UpdateDownloadCount increments download counter
func (s *SQLiteStore) UpdateDownloadCount(ctx context.Context, registry, name, version string) error {
s.mu.Lock()
defer s.mu.Unlock()
query := `
UPDATE packages
SET download_count = download_count + 1,
last_accessed = ?
WHERE registry = ? AND name = ? AND version = ?
`
_, err := s.db.ExecContext(ctx, query, time.Now(), registry, name, version)
if err != nil {
return errors.Wrap(err, errors.ErrCodeStorageFailure, "failed to update download count")
}
return nil
}
// GetStats returns statistics
func (s *SQLiteStore) GetStats(ctx context.Context, registry string) (*metadata.Stats, error) {
s.mu.RLock()
defer s.mu.RUnlock()
query := `
SELECT
COUNT(*) as total_packages,
COALESCE(SUM(size), 0) as total_size,
COALESCE(SUM(download_count), 0) as total_downloads,
COALESCE(SUM(CASE WHEN security_scanned = 1 THEN 1 ELSE 0 END), 0) as scanned_packages
FROM packages
`
args := []interface{}{}
if registry != "" {
query += " WHERE registry = ?"
args = append(args, registry)
}
var stats metadata.Stats
stats.Registry = registry
stats.LastUpdated = time.Now()
err := s.db.QueryRowContext(ctx, query, args...).Scan(
&stats.TotalPackages,
&stats.TotalSize,
&stats.TotalDownloads,
&stats.ScannedPackages,
)
if err != nil {
return nil, errors.Wrap(err, errors.ErrCodeStorageFailure, "failed to get stats")
}
// Count vulnerable packages
vulnQuery := `SELECT COUNT(*) FROM scan_results WHERE status = 'vulnerable'`
vulnArgs := []interface{}{}
if registry != "" {
vulnQuery += " AND registry = ?"
vulnArgs = append(vulnArgs, registry)
}
s.db.QueryRowContext(ctx, vulnQuery, vulnArgs...).Scan(&stats.VulnerablePackages)
return &stats, nil
}
// SaveScanResult saves security scan result
func (s *SQLiteStore) SaveScanResult(ctx context.Context, result *metadata.ScanResult) error {
s.mu.Lock()
defer s.mu.Unlock()
// Serialize vulnerabilities and details
vulnJSON, err := goccy_json.Marshal(result.Vulnerabilities)
if err != nil {
return errors.Wrap(err, errors.ErrCodeInternalServer, "failed to serialize vulnerabilities")
}
detailsJSON, err := goccy_json.Marshal(result.Details)
if err != nil {
return errors.Wrap(err, errors.ErrCodeInternalServer, "failed to serialize scan details")
}
query := `
INSERT INTO scan_results (
id, registry, package_name, package_version, scanner,
scanned_at, status, vulnerability_count, vulnerabilities, details
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(registry, package_name, package_version, scanner) DO UPDATE SET
scanned_at = excluded.scanned_at,
status = excluded.status,
vulnerability_count = excluded.vulnerability_count,
vulnerabilities = excluded.vulnerabilities,
details = excluded.details
`
_, err = s.db.ExecContext(ctx, query,
result.ID, result.Registry, result.PackageName, result.PackageVersion, result.Scanner,
result.ScannedAt, result.Status, result.VulnerabilityCount,
string(vulnJSON), string(detailsJSON),
)
if err != nil {
return errors.Wrap(err, errors.ErrCodeStorageFailure, "failed to save scan result")
}
// Update package security_scanned flag
updateQuery := `UPDATE packages SET security_scanned = 1 WHERE registry = ? AND name = ? AND version = ?`
s.db.ExecContext(ctx, updateQuery, result.Registry, result.PackageName, result.PackageVersion)
return nil
}
// GetScanResult retrieves security scan result
func (s *SQLiteStore) GetScanResult(ctx context.Context, registry, name, version string) (*metadata.ScanResult, error) {
s.mu.RLock()
defer s.mu.RUnlock()
query := `
SELECT id, registry, package_name, package_version, scanner,
scanned_at, status, vulnerability_count, vulnerabilities, details
FROM scan_results
WHERE registry = ? AND package_name = ? AND package_version = ?
ORDER BY scanned_at DESC
LIMIT 1
`
var result metadata.ScanResult
var vulnJSON, detailsJSON string
err := s.db.QueryRowContext(ctx, query, registry, name, version).Scan(
&result.ID, &result.Registry, &result.PackageName, &result.PackageVersion, &result.Scanner,
&result.ScannedAt, &result.Status, &result.VulnerabilityCount,
&vulnJSON, &detailsJSON,
)
if err == sql.ErrNoRows {
return nil, errors.NotFound(fmt.Sprintf("scan result not found: %s/%s@%s", registry, name, version))
}
if err != nil {
return nil, errors.Wrap(err, errors.ErrCodeStorageFailure, "failed to get scan result")
}
// Deserialize
if vulnJSON != "" {
goccy_json.Unmarshal([]byte(vulnJSON), &result.Vulnerabilities)
}
if detailsJSON != "" {
goccy_json.Unmarshal([]byte(detailsJSON), &result.Details)
}
return &result, nil
}
// Count returns total number of packages
func (s *SQLiteStore) Count(ctx context.Context) (int, error) {
s.mu.RLock()
defer s.mu.RUnlock()
var count int
query := "SELECT COUNT(*) FROM packages"
err := s.db.QueryRowContext(ctx, query).Scan(&count)
if err != nil {
return 0, errors.Wrap(err, errors.ErrCodeStorageFailure, "failed to count packages")
}
return count, nil
}
// Health checks metadata store health
func (s *SQLiteStore) Health(ctx context.Context) error {
return s.db.PingContext(ctx)
}
// SaveCVEBypass saves a CVE bypass (admin only)
func (s *SQLiteStore) SaveCVEBypass(ctx context.Context, bypass *metadata.CVEBypass) error {
s.mu.Lock()
defer s.mu.Unlock()
query := `
INSERT INTO cve_bypasses (
id, type, target, reason, created_by, created_at,
expires_at, applies_to, notify_on_expiry, active
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(id) DO UPDATE SET
type = excluded.type,
target = excluded.target,
reason = excluded.reason,
expires_at = excluded.expires_at,
applies_to = excluded.applies_to,
notify_on_expiry = excluded.notify_on_expiry,
active = excluded.active
`
_, err := s.db.ExecContext(ctx, query,
bypass.ID, bypass.Type, bypass.Target, bypass.Reason, bypass.CreatedBy,
bypass.CreatedAt, bypass.ExpiresAt, bypass.AppliesTo,
bypass.NotifyOnExpiry, bypass.Active,
)
if err != nil {
return errors.Wrap(err, errors.ErrCodeStorageFailure, "failed to save CVE bypass")
}
return nil
}
// GetActiveCVEBypasses retrieves all active (non-expired) CVE bypasses
func (s *SQLiteStore) GetActiveCVEBypasses(ctx context.Context) ([]*metadata.CVEBypass, error) {
s.mu.RLock()
defer s.mu.RUnlock()
query := `
SELECT id, type, target, reason, created_by, created_at,
expires_at, applies_to, notify_on_expiry, active
FROM cve_bypasses
WHERE active = 1 AND expires_at > ?
ORDER BY created_at DESC
`
rows, err := s.db.QueryContext(ctx, query, time.Now())
if err != nil {
return nil, errors.Wrap(err, errors.ErrCodeStorageFailure, "failed to get active CVE bypasses")
}
defer rows.Close()
var bypasses []*metadata.CVEBypass
for rows.Next() {
var bypass metadata.CVEBypass
var appliesTo sql.NullString
err := rows.Scan(
&bypass.ID, &bypass.Type, &bypass.Target, &bypass.Reason, &bypass.CreatedBy,
&bypass.CreatedAt, &bypass.ExpiresAt, &appliesTo,
&bypass.NotifyOnExpiry, &bypass.Active,
)
if err != nil {
return nil, errors.Wrap(err, errors.ErrCodeStorageFailure, "failed to scan CVE bypass row")
}
if appliesTo.Valid {
bypass.AppliesTo = appliesTo.String
}
bypasses = append(bypasses, &bypass)
}
return bypasses, nil
}
// ListCVEBypasses lists all CVE bypasses (including expired)
func (s *SQLiteStore) ListCVEBypasses(ctx context.Context, opts *metadata.BypassListOptions) ([]*metadata.CVEBypass, error) {
s.mu.RLock()
defer s.mu.RUnlock()
query := `
SELECT id, type, target, reason, created_by, created_at,
expires_at, applies_to, notify_on_expiry, active
FROM cve_bypasses
WHERE 1=1
`
args := []interface{}{}
if opts != nil {
if opts.Type != "" {
query += " AND type = ?"
args = append(args, opts.Type)
}
if !opts.IncludeExpired {
query += " AND expires_at > ?"
args = append(args, time.Now())
}
if opts.ActiveOnly {
query += " AND active = 1"
}
query += " ORDER BY created_at DESC"
if opts.Limit > 0 {
query += " LIMIT ?"
args = append(args, opts.Limit)
}
if opts.Offset > 0 {
query += " OFFSET ?"
args = append(args, opts.Offset)
}
}
rows, err := s.db.QueryContext(ctx, query, args...)
if err != nil {
return nil, errors.Wrap(err, errors.ErrCodeStorageFailure, "failed to list CVE bypasses")
}
defer rows.Close()
var bypasses []*metadata.CVEBypass
for rows.Next() {
var bypass metadata.CVEBypass
var appliesTo sql.NullString
err := rows.Scan(
&bypass.ID, &bypass.Type, &bypass.Target, &bypass.Reason, &bypass.CreatedBy,
&bypass.CreatedAt, &bypass.ExpiresAt, &appliesTo,
&bypass.NotifyOnExpiry, &bypass.Active,
)
if err != nil {
return nil, errors.Wrap(err, errors.ErrCodeStorageFailure, "failed to scan CVE bypass row")
}
if appliesTo.Valid {
bypass.AppliesTo = appliesTo.String
}
bypasses = append(bypasses, &bypass)
}
return bypasses, nil
}
// DeleteCVEBypass deletes a CVE bypass by ID
func (s *SQLiteStore) DeleteCVEBypass(ctx context.Context, id string) error {
s.mu.Lock()
defer s.mu.Unlock()
query := "DELETE FROM cve_bypasses WHERE id = ?"
result, err := s.db.ExecContext(ctx, query, id)
if err != nil {
return errors.Wrap(err, errors.ErrCodeStorageFailure, "failed to delete CVE bypass")
}
rows, _ := result.RowsAffected()
if rows == 0 {
return errors.NotFound(fmt.Sprintf("CVE bypass not found: %s", id))
}
return nil
}
// CleanupExpiredBypasses removes expired bypasses
func (s *SQLiteStore) CleanupExpiredBypasses(ctx context.Context) (int, error) {
s.mu.Lock()
defer s.mu.Unlock()
query := "DELETE FROM cve_bypasses WHERE expires_at <= ?"
result, err := s.db.ExecContext(ctx, query, time.Now())
if err != nil {
return 0, errors.Wrap(err, errors.ErrCodeStorageFailure, "failed to cleanup expired CVE bypasses")
}
rows, _ := result.RowsAffected()
return int(rows), nil
}
// Close closes the metadata store
func (s *SQLiteStore) Close() error {
return s.db.Close()
}