mirror of
https://github.com/lukaszraczylo/gohoarder.git
synced 2026-06-15 00:51:22 +00:00
refactor: reorganize struct fields, add new handlers and storage backends
- [x] Reorder struct fields across codebase for consistency - [x] Add analytics event handlers and tests - [x] Add authentication API key management handlers and tests - [x] Add pre-warming control handlers and tests - [x] Implement S3 storage backend with tests - [x] Implement SMB/CIFS storage backend with tests - [x] Add CDN middleware tests - [x] Integrate analytics tracking into cache manager - [x] Add S3 and SMB storage initialization in app setup - [x] Add CDN caching to proxy handlers - [x] Remove distributed locking (Redis lock manager) - [x] Remove proxy common package and utilities - [x] Remove standalone HTTP server package - [x] Remove logger middleware - [x] Simplify error handling utilities - [x] Update config with S3 and SMB options - [x] Update cache manager signature to include analytics
This commit is contained in:
Vendored
+47
-14
@@ -11,6 +11,7 @@ import (
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/lukaszraczylo/gohoarder/pkg/analytics"
|
||||
"github.com/lukaszraczylo/gohoarder/pkg/errors"
|
||||
"github.com/lukaszraczylo/gohoarder/pkg/metadata"
|
||||
"github.com/lukaszraczylo/gohoarder/pkg/metrics"
|
||||
@@ -27,15 +28,21 @@ type ScannerInterface interface {
|
||||
CheckVulnerabilities(ctx context.Context, registry, packageName, version string) (blocked bool, reason string, err error)
|
||||
}
|
||||
|
||||
// AnalyticsInterface defines the interface for analytics tracking
|
||||
type AnalyticsInterface interface {
|
||||
TrackDownload(download analytics.PackageDownload)
|
||||
}
|
||||
|
||||
// Manager coordinates caching operations between storage and metadata
|
||||
type Manager struct {
|
||||
storage storage.StorageBackend
|
||||
metadata metadata.MetadataStore
|
||||
scanner ScannerInterface
|
||||
config Config
|
||||
sf singleflight.Group
|
||||
mu sync.RWMutex
|
||||
evicting bool
|
||||
storage storage.StorageBackend
|
||||
metadata metadata.MetadataStore
|
||||
scanner ScannerInterface
|
||||
analytics AnalyticsInterface
|
||||
sf singleflight.Group
|
||||
config Config
|
||||
mu sync.RWMutex
|
||||
evicting bool
|
||||
}
|
||||
|
||||
// Config holds cache manager configuration
|
||||
@@ -48,15 +55,15 @@ type Config struct {
|
||||
|
||||
// CacheEntry represents a cached package
|
||||
type CacheEntry struct {
|
||||
Package *metadata.Package
|
||||
Data io.ReadCloser
|
||||
FromCache bool
|
||||
Package *metadata.Package
|
||||
UpstreamURL string
|
||||
CacheControl string
|
||||
FromCache bool
|
||||
}
|
||||
|
||||
// New creates a new cache manager
|
||||
func New(storage storage.StorageBackend, metadata metadata.MetadataStore, scanner ScannerInterface, config Config) (*Manager, error) {
|
||||
func New(storage storage.StorageBackend, metadata metadata.MetadataStore, scanner ScannerInterface, analytics AnalyticsInterface, config Config) (*Manager, error) {
|
||||
if storage == nil {
|
||||
return nil, errors.New(errors.ErrCodeInvalidConfig, "storage backend is required")
|
||||
}
|
||||
@@ -70,6 +77,11 @@ func New(storage storage.StorageBackend, metadata metadata.MetadataStore, scanne
|
||||
log.Info().Msg("Cache manager initialized with security scanning enabled")
|
||||
}
|
||||
|
||||
// Analytics is optional - can be nil if analytics tracking is disabled
|
||||
if analytics != nil {
|
||||
log.Info().Msg("Cache manager initialized with analytics tracking enabled")
|
||||
}
|
||||
|
||||
if config.DefaultTTL == 0 {
|
||||
config.DefaultTTL = 7 * 24 * time.Hour // 7 days default
|
||||
}
|
||||
@@ -87,10 +99,11 @@ func New(storage storage.StorageBackend, metadata metadata.MetadataStore, scanne
|
||||
}
|
||||
|
||||
manager := &Manager{
|
||||
storage: storage,
|
||||
metadata: metadata,
|
||||
scanner: scanner,
|
||||
config: config,
|
||||
storage: storage,
|
||||
metadata: metadata,
|
||||
scanner: scanner,
|
||||
analytics: analytics,
|
||||
config: config,
|
||||
}
|
||||
|
||||
// Start background cleanup worker
|
||||
@@ -134,6 +147,11 @@ func (m *Manager) getOrFetch(ctx context.Context, registry, name, version string
|
||||
metrics.RecordCacheHit(registry)
|
||||
_ = m.metadata.UpdateDownloadCount(ctx, registry, name, version) // #nosec G104 -- Async update, error logged
|
||||
|
||||
// Track download in analytics if enabled
|
||||
if m.analytics != nil {
|
||||
m.trackDownload(registry, name, version, pkg.Size)
|
||||
}
|
||||
|
||||
// Check for vulnerabilities if scanner is enabled
|
||||
if m.scanner != nil {
|
||||
blocked, reason, err := m.scanner.CheckVulnerabilities(ctx, registry, name, version)
|
||||
@@ -552,6 +570,21 @@ func (m *Manager) Health(ctx context.Context) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// trackDownload tracks a package download event in analytics
|
||||
func (m *Manager) trackDownload(registry, name, version string, size int64) {
|
||||
download := analytics.PackageDownload{
|
||||
Registry: registry,
|
||||
Name: name,
|
||||
Version: version,
|
||||
Timestamp: time.Now(),
|
||||
BytesSize: size,
|
||||
ClientIP: "", // TODO: Extract from context if available
|
||||
UserAgent: "", // TODO: Extract from context if available
|
||||
}
|
||||
|
||||
m.analytics.TrackDownload(download)
|
||||
}
|
||||
|
||||
// Close closes the cache manager
|
||||
func (m *Manager) Close() error {
|
||||
var err error
|
||||
|
||||
Vendored
+22
-22
@@ -197,12 +197,12 @@ func (m *MockMetadataStore) AggregateDownloadData(ctx context.Context) error {
|
||||
// TestNew tests cache manager creation
|
||||
func TestNew(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
storage storage.StorageBackend
|
||||
metadata metadata.MetadataStore
|
||||
name string
|
||||
errContains string
|
||||
config Config
|
||||
wantErr bool
|
||||
errContains string
|
||||
}{
|
||||
// GOOD: Valid configuration
|
||||
{
|
||||
@@ -262,7 +262,7 @@ func TestNew(t *testing.T) {
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
manager, err := New(tt.storage, tt.metadata, nil, tt.config)
|
||||
manager, err := New(tt.storage, tt.metadata, nil, nil, tt.config)
|
||||
|
||||
if tt.wantErr {
|
||||
require.Error(t, err)
|
||||
@@ -295,15 +295,15 @@ func TestNew(t *testing.T) {
|
||||
// TestGet tests cache retrieval with various scenarios
|
||||
func TestGet(t *testing.T) {
|
||||
tests := []struct {
|
||||
setupMock func(*MockStorageBackend, *MockMetadataStore)
|
||||
fetchFunc func(context.Context) (io.ReadCloser, string, error)
|
||||
name string
|
||||
registry string
|
||||
packageName string
|
||||
version string
|
||||
setupMock func(*MockStorageBackend, *MockMetadataStore)
|
||||
fetchFunc func(context.Context) (io.ReadCloser, string, error)
|
||||
errContains string
|
||||
wantFromCache bool
|
||||
wantErr bool
|
||||
errContains string
|
||||
}{
|
||||
// GOOD: Cache hit
|
||||
{
|
||||
@@ -489,7 +489,7 @@ func TestGet(t *testing.T) {
|
||||
tt.setupMock(mockStorage, mockMetadata)
|
||||
}
|
||||
|
||||
manager, err := New(mockStorage, mockMetadata, nil, Config{
|
||||
manager, err := New(mockStorage, mockMetadata, nil, nil, Config{
|
||||
DefaultTTL: 24 * time.Hour,
|
||||
CleanupInterval: 1 * time.Hour,
|
||||
})
|
||||
@@ -523,13 +523,13 @@ func TestGet(t *testing.T) {
|
||||
// TestDelete tests package deletion
|
||||
func TestDelete(t *testing.T) {
|
||||
tests := []struct {
|
||||
setupMock func(*MockStorageBackend, *MockMetadataStore)
|
||||
name string
|
||||
registry string
|
||||
packageName string
|
||||
version string
|
||||
setupMock func(*MockStorageBackend, *MockMetadataStore)
|
||||
wantErr bool
|
||||
errContains string
|
||||
wantErr bool
|
||||
}{
|
||||
// GOOD: Successful deletion
|
||||
{
|
||||
@@ -615,7 +615,7 @@ func TestDelete(t *testing.T) {
|
||||
tt.setupMock(mockStorage, mockMetadata)
|
||||
}
|
||||
|
||||
manager, err := New(mockStorage, mockMetadata, nil, Config{})
|
||||
manager, err := New(mockStorage, mockMetadata, nil, nil, Config{})
|
||||
require.NoError(t, err)
|
||||
|
||||
ctx := context.Background()
|
||||
@@ -639,10 +639,10 @@ func TestDelete(t *testing.T) {
|
||||
// TestHealth tests health check functionality
|
||||
func TestHealth(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
setupMock func(*MockStorageBackend, *MockMetadataStore)
|
||||
wantErr bool
|
||||
name string
|
||||
errContains string
|
||||
wantErr bool
|
||||
}{
|
||||
// GOOD: Both healthy
|
||||
{
|
||||
@@ -692,7 +692,7 @@ func TestHealth(t *testing.T) {
|
||||
tt.setupMock(mockStorage, mockMetadata)
|
||||
}
|
||||
|
||||
manager, err := New(mockStorage, mockMetadata, nil, Config{})
|
||||
manager, err := New(mockStorage, mockMetadata, nil, nil, Config{})
|
||||
require.NoError(t, err)
|
||||
|
||||
ctx := context.Background()
|
||||
@@ -727,7 +727,7 @@ func TestGetStats(t *testing.T) {
|
||||
|
||||
mockMetadata.On("GetStats", mock.Anything, "npm").Return(expectedStats, nil)
|
||||
|
||||
manager, err := New(mockStorage, mockMetadata, nil, Config{})
|
||||
manager, err := New(mockStorage, mockMetadata, nil, nil, Config{})
|
||||
require.NoError(t, err)
|
||||
|
||||
ctx := context.Background()
|
||||
@@ -741,8 +741,8 @@ func TestGetStats(t *testing.T) {
|
||||
// TestClose tests manager cleanup
|
||||
func TestClose(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
setupMock func(*MockStorageBackend, *MockMetadataStore)
|
||||
name string
|
||||
wantErr bool
|
||||
}{
|
||||
// GOOD: Clean close
|
||||
@@ -792,7 +792,7 @@ func TestClose(t *testing.T) {
|
||||
tt.setupMock(mockStorage, mockMetadata)
|
||||
}
|
||||
|
||||
manager, err := New(mockStorage, mockMetadata, nil, Config{})
|
||||
manager, err := New(mockStorage, mockMetadata, nil, nil, Config{})
|
||||
require.NoError(t, err)
|
||||
|
||||
err = manager.Close() // #nosec G104 -- Cleanup, error not critical
|
||||
@@ -812,11 +812,11 @@ func TestClose(t *testing.T) {
|
||||
// TestEvict tests LRU eviction
|
||||
func TestEvict(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
needed int64
|
||||
setupMock func(*MockStorageBackend, *MockMetadataStore)
|
||||
wantErr bool
|
||||
name string
|
||||
errContains string
|
||||
needed int64
|
||||
wantErr bool
|
||||
}{
|
||||
// GOOD: Successful eviction
|
||||
{
|
||||
@@ -881,7 +881,7 @@ func TestEvict(t *testing.T) {
|
||||
tt.setupMock(mockStorage, mockMetadata)
|
||||
}
|
||||
|
||||
manager, err := New(mockStorage, mockMetadata, nil, Config{})
|
||||
manager, err := New(mockStorage, mockMetadata, nil, nil, Config{})
|
||||
require.NoError(t, err)
|
||||
|
||||
ctx := context.Background()
|
||||
@@ -907,7 +907,7 @@ func TestGenerateStorageKey(t *testing.T) {
|
||||
mockStorage := &MockStorageBackend{}
|
||||
mockMetadata := &MockMetadataStore{}
|
||||
|
||||
manager, err := New(mockStorage, mockMetadata, nil, Config{})
|
||||
manager, err := New(mockStorage, mockMetadata, nil, nil, Config{})
|
||||
require.NoError(t, err)
|
||||
|
||||
tests := []struct {
|
||||
@@ -954,7 +954,7 @@ func TestConcurrentGet(t *testing.T) {
|
||||
io.NopCloser(bytes.NewReader([]byte("test data"))), nil).Maybe()
|
||||
mockMetadata.On("UpdateDownloadCount", mock.Anything, "npm", "concurrent", "1.0.0").Return(nil).Maybe()
|
||||
|
||||
manager, err := New(mockStorage, mockMetadata, nil, Config{})
|
||||
manager, err := New(mockStorage, mockMetadata, nil, nil, Config{})
|
||||
require.NoError(t, err)
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
Reference in New Issue
Block a user