mirror of
https://github.com/lukaszraczylo/gohoarder.git
synced 2026-07-22 06:20:09 +00:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
1baf0993de | ||
|
|
434a15076e | ||
|
|
4e7350363d | ||
|
|
38edd735b6 |
@@ -19,4 +19,4 @@ sources:
|
||||
maintainers:
|
||||
- name: Lukasz Raczylo
|
||||
email: [email protected]
|
||||
icon: https://raw.githubusercontent.com/lukaszraczylo/gohoarder/main/docs/logo.png
|
||||
# icon: https://raw.githubusercontent.com/lukaszraczylo/gohoarder/main/docs/logo.png # TODO: Add logo
|
||||
|
||||
Vendored
+47
-1
@@ -145,7 +145,37 @@ func (m *Manager) getOrFetch(ctx context.Context, registry, name, version string
|
||||
if err == nil {
|
||||
// Cache hit!
|
||||
metrics.RecordCacheHit(registry)
|
||||
_ = m.metadata.UpdateDownloadCount(ctx, registry, name, version) // #nosec G104 -- Async update, error logged
|
||||
|
||||
// Update download count (log errors for debugging)
|
||||
if err := m.metadata.UpdateDownloadCount(ctx, registry, name, version); err != nil {
|
||||
log.Warn().
|
||||
Err(err).
|
||||
Str("registry", registry).
|
||||
Str("package", name).
|
||||
Str("version", version).
|
||||
Msg("Failed to update download count - package may not exist in database")
|
||||
|
||||
// Try to save package to database if it doesn't exist
|
||||
// This handles the case where storage has files but database was migrated/reset
|
||||
if saveErr := m.metadata.SavePackage(ctx, pkg); saveErr != nil {
|
||||
log.Error().
|
||||
Err(saveErr).
|
||||
Str("registry", registry).
|
||||
Str("package", name).
|
||||
Str("version", version).
|
||||
Msg("Failed to save package to database")
|
||||
} else {
|
||||
// Retry download count update after saving package
|
||||
if retryErr := m.metadata.UpdateDownloadCount(ctx, registry, name, version); retryErr != nil {
|
||||
log.Error().
|
||||
Err(retryErr).
|
||||
Str("registry", registry).
|
||||
Str("package", name).
|
||||
Str("version", version).
|
||||
Msg("Failed to update download count even after saving package")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Track download in analytics if enabled
|
||||
if m.analytics != nil {
|
||||
@@ -290,6 +320,22 @@ servePkg:
|
||||
return nil, errors.Wrap(err, errors.ErrCodeStorageFailure, "failed to retrieve just-stored package")
|
||||
}
|
||||
|
||||
// Track download count for first-time download (cache miss)
|
||||
// This ensures download count increments regardless of cache hit/miss
|
||||
if err := m.metadata.UpdateDownloadCount(ctx, registry, name, version); err != nil {
|
||||
log.Warn().
|
||||
Err(err).
|
||||
Str("registry", registry).
|
||||
Str("package", name).
|
||||
Str("version", version).
|
||||
Msg("Failed to update download count for newly cached package")
|
||||
}
|
||||
|
||||
// Track download in analytics if enabled
|
||||
if m.analytics != nil {
|
||||
m.trackDownload(registry, name, version, storedPkg.Size)
|
||||
}
|
||||
|
||||
return &CacheEntry{
|
||||
Package: storedPkg,
|
||||
Data: storedData,
|
||||
|
||||
Vendored
+3
@@ -343,6 +343,7 @@ func TestGet(t *testing.T) {
|
||||
s.On("Put", mock.Anything, "npm/lodash/4.17.21", mock.Anything, mock.Anything).Return(nil)
|
||||
m.On("SavePackage", mock.Anything, mock.Anything).Return(nil)
|
||||
s.On("Get", mock.Anything, "npm/lodash/4.17.21").Return(io.NopCloser(strings.NewReader("upstream data")), nil)
|
||||
m.On("UpdateDownloadCount", mock.Anything, "npm", "lodash", "4.17.21").Return(nil)
|
||||
},
|
||||
fetchFunc: func(ctx context.Context) (io.ReadCloser, string, error) {
|
||||
return io.NopCloser(strings.NewReader("upstream data")), "https://registry.npmjs.org/lodash", nil
|
||||
@@ -374,6 +375,7 @@ func TestGet(t *testing.T) {
|
||||
s.On("Put", mock.Anything, "npm/expired-pkg/1.0.0", mock.Anything, mock.Anything).Return(nil)
|
||||
m.On("SavePackage", mock.Anything, mock.Anything).Return(nil)
|
||||
s.On("Get", mock.Anything, "npm/expired-pkg/1.0.0").Return(io.NopCloser(strings.NewReader("refreshed data")), nil)
|
||||
m.On("UpdateDownloadCount", mock.Anything, "npm", "expired-pkg", "1.0.0").Return(nil)
|
||||
},
|
||||
fetchFunc: func(ctx context.Context) (io.ReadCloser, string, error) {
|
||||
return io.NopCloser(strings.NewReader("refreshed data")), "https://registry.npmjs.org/expired-pkg", nil
|
||||
@@ -435,6 +437,7 @@ func TestGet(t *testing.T) {
|
||||
m.On("SavePackage", mock.Anything, mock.Anything).Return(nil)
|
||||
// Second Get succeeds (after re-storing)
|
||||
s.On("Get", mock.Anything, "npm/inconsistent/1.0.0").Return(io.NopCloser(strings.NewReader("recovered data")), nil).Once()
|
||||
m.On("UpdateDownloadCount", mock.Anything, "npm", "inconsistent", "1.0.0").Return(nil)
|
||||
},
|
||||
fetchFunc: func(ctx context.Context) (io.ReadCloser, string, error) {
|
||||
return io.NopCloser(strings.NewReader("recovered data")), "https://registry.npmjs.org/inconsistent", nil
|
||||
|
||||
@@ -583,29 +583,38 @@ func (s *GORMStoreV2) GetTimeSeriesStats(ctx context.Context, period string, reg
|
||||
// Determine which table to query based on period
|
||||
var tableName string
|
||||
var since time.Time
|
||||
var bucketDuration time.Duration
|
||||
|
||||
switch period {
|
||||
case "1h":
|
||||
tableName = "download_stats_hourly"
|
||||
since = time.Now().Add(-1 * time.Hour)
|
||||
since = time.Now().Add(-1 * time.Hour).Truncate(time.Hour)
|
||||
bucketDuration = time.Hour
|
||||
case "1day":
|
||||
tableName = "download_stats_hourly"
|
||||
since = time.Now().Add(-24 * time.Hour)
|
||||
since = time.Now().Add(-24 * time.Hour).Truncate(time.Hour)
|
||||
bucketDuration = time.Hour
|
||||
case "7day":
|
||||
tableName = "download_stats_daily"
|
||||
since = time.Now().Add(-7 * 24 * time.Hour)
|
||||
since = time.Now().Add(-7 * 24 * time.Hour).Truncate(24 * time.Hour)
|
||||
bucketDuration = 24 * time.Hour
|
||||
case "30day":
|
||||
tableName = "download_stats_daily"
|
||||
since = time.Now().Add(-30 * 24 * time.Hour)
|
||||
since = time.Now().Add(-30 * 24 * time.Hour).Truncate(24 * time.Hour)
|
||||
bucketDuration = 24 * time.Hour
|
||||
default:
|
||||
tableName = "download_stats_hourly"
|
||||
since = time.Now().Add(-24 * time.Hour)
|
||||
since = time.Now().Add(-24 * time.Hour).Truncate(time.Hour)
|
||||
bucketDuration = time.Hour
|
||||
}
|
||||
|
||||
now := time.Now()
|
||||
|
||||
// Try to get registry-level aggregate stats first
|
||||
query := s.db.WithContext(ctx).
|
||||
Table(tableName).
|
||||
Select("time_bucket as timestamp, download_count as value").
|
||||
Where("time_bucket >= ?", since)
|
||||
Where("time_bucket >= ? AND time_bucket <= ?", since, now)
|
||||
|
||||
// Filter by registry if specified
|
||||
if registry != "" {
|
||||
@@ -630,10 +639,45 @@ func (s *GORMStoreV2) GetTimeSeriesStats(ctx context.Context, period string, reg
|
||||
return nil, errors.Wrap(err, errors.ErrCodeStorageFailure, "failed to get time series stats")
|
||||
}
|
||||
|
||||
// If no aggregate data found, try summing package-level stats
|
||||
if len(results) == 0 {
|
||||
sumQuery := s.db.WithContext(ctx).
|
||||
Table(tableName).
|
||||
Select("time_bucket as timestamp, SUM(download_count) as value").
|
||||
Where("time_bucket >= ? AND time_bucket <= ?", since, now)
|
||||
|
||||
if registry != "" {
|
||||
registryID, err := s.getRegistryID(registry)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
sumQuery = sumQuery.Where("registry_id = ? AND package_id IS NOT NULL", registryID)
|
||||
} else {
|
||||
sumQuery = sumQuery.Where("package_id IS NOT NULL")
|
||||
}
|
||||
|
||||
sumQuery = sumQuery.Group("time_bucket").Order("time_bucket ASC")
|
||||
|
||||
if err := sumQuery.Scan(&results).Error; err != nil {
|
||||
return nil, errors.Wrap(err, errors.ErrCodeStorageFailure, "failed to get time series stats from package data")
|
||||
}
|
||||
}
|
||||
|
||||
// Create a map of existing data points
|
||||
dataMap := make(map[time.Time]int64)
|
||||
for _, r := range results {
|
||||
dataMap[r.Timestamp] = r.Value
|
||||
}
|
||||
|
||||
// Generate continuous time series with 0 values for missing periods
|
||||
for t := since; t.Before(now) || t.Equal(now); t = t.Add(bucketDuration) {
|
||||
value := int64(0)
|
||||
if v, exists := dataMap[t]; exists {
|
||||
value = v
|
||||
}
|
||||
stats.DataPoints = append(stats.DataPoints, &metadata.TimeSeriesDataPoint{
|
||||
Timestamp: r.Timestamp,
|
||||
Value: r.Value,
|
||||
Timestamp: t,
|
||||
Value: value,
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user