Compare commits

...
4 Commits
Author SHA1 Message Date
lukaszraczylo 1baf0993de fix: remove missing logo reference from Helm chart
The referenced logo file (docs/logo.png) doesn't exist, causing
Artifact Hub to fail with 404 errors when indexing the chart.

Commented out the icon line until a logo is created.

Resolves: "error getting logo image https://raw.githubusercontent.com/
lukaszraczylo/gohoarder/main/docs/logo.png: unexpected status code
received: 404"
2026-01-04 14:25:57 +00:00
lukaszraczylo 434a15076e fix: generate continuous time series data with zero-filled gaps for charts
Issues fixed:
1. Charts not rendering correctly due to sparse data (missing time buckets)
2. Period "1h" returning empty data when aggregate stats not yet available

Changes:
- Generate continuous time series with 0 values for all time buckets
- Truncate start time to hour/day boundaries for consistent bucketing
- Fallback to package-level stats aggregation when registry totals unavailable
- Add proper time range filtering (since <= time_bucket <= now)

Behavior now:
- All time periods (1h, 1day, 7day, 30day) return complete data sets
- Missing hours/days are filled with value: 0
- Chart libraries can render continuous lines/bars correctly
- No more empty data for "1h" period

Example output (1 hour period):
Before: [{"timestamp":"14:00","value":5}, {"timestamp":"15:00","value":3}]
After:  [{"timestamp":"13:00","value":0}, {"timestamp":"14:00","value":5},
         {"timestamp":"15:00","value":3}, {"timestamp":"16:00","value":0}]

Resolves: "Chart doesn't generate correctly - it should automatically
fill 0 for the empty periods to render correctly"
2026-01-04 14:23:08 +00:00
lukaszraczylo 4e7350363d feat: track download counts for both cache hits and cache misses
Previously, download counts only incremented on cache hits (when package
was served from cache). First-time downloads (cache misses) were not counted.

Changes:
- Add UpdateDownloadCount() call when serving newly cached packages
- This ensures every download through the proxy increments the counter
- Analytics tracking also added for cache misses

Behavior now:
- First download (cache miss): counter = 1
- Second download (cache hit): counter = 2
- Third download (cache hit): counter = 3
- etc.

Updated all relevant tests to expect the additional UpdateDownloadCount call.

Resolves user requirement: "I want the counters to increase whenever
package is downloaded via proxy - regardless of it being new download
or cached download"
2026-01-04 13:50:42 +00:00
lukaszraczylo 38edd735b6 fix: improve download statistics tracking resilience
Problem:
- Download counts were not incrementing when packages existed in storage
  but not in the database (e.g., after database migration/reset)
- UpdateDownloadCount() was failing silently when package didn't exist in DB
- Error was completely ignored despite comment claiming "error logged"

Changes:
1. Log errors when UpdateDownloadCount() fails instead of silently ignoring
2. Auto-save package to database if UpdateDownloadCount() fails
3. Retry download count update after saving package
4. Provide detailed error logging for troubleshooting

This fixes the issue where:
- Database is migrated but storage still has cached packages
- Cache hits occur but download events aren't recorded
- Statistics show zero downloads despite actual usage

Resolves user report: "I cleaned go mod which redownloaded the modules
through the proxy but counters did not increased"
2026-01-04 13:44:36 +00:00
4 changed files with 103 additions and 10 deletions
+1 -1
View File
@@ -19,4 +19,4 @@ sources:
maintainers:
- name: Lukasz Raczylo
email: [email protected]
icon: https://raw.githubusercontent.com/lukaszraczylo/gohoarder/main/docs/logo.png
# icon: https://raw.githubusercontent.com/lukaszraczylo/gohoarder/main/docs/logo.png # TODO: Add logo
+47 -1
View File
@@ -145,7 +145,37 @@ func (m *Manager) getOrFetch(ctx context.Context, registry, name, version string
if err == nil {
// Cache hit!
metrics.RecordCacheHit(registry)
_ = m.metadata.UpdateDownloadCount(ctx, registry, name, version) // #nosec G104 -- Async update, error logged
// Update download count (log errors for debugging)
if err := m.metadata.UpdateDownloadCount(ctx, registry, name, version); err != nil {
log.Warn().
Err(err).
Str("registry", registry).
Str("package", name).
Str("version", version).
Msg("Failed to update download count - package may not exist in database")
// Try to save package to database if it doesn't exist
// This handles the case where storage has files but database was migrated/reset
if saveErr := m.metadata.SavePackage(ctx, pkg); saveErr != nil {
log.Error().
Err(saveErr).
Str("registry", registry).
Str("package", name).
Str("version", version).
Msg("Failed to save package to database")
} else {
// Retry download count update after saving package
if retryErr := m.metadata.UpdateDownloadCount(ctx, registry, name, version); retryErr != nil {
log.Error().
Err(retryErr).
Str("registry", registry).
Str("package", name).
Str("version", version).
Msg("Failed to update download count even after saving package")
}
}
}
// Track download in analytics if enabled
if m.analytics != nil {
@@ -290,6 +320,22 @@ servePkg:
return nil, errors.Wrap(err, errors.ErrCodeStorageFailure, "failed to retrieve just-stored package")
}
// Track download count for first-time download (cache miss)
// This ensures download count increments regardless of cache hit/miss
if err := m.metadata.UpdateDownloadCount(ctx, registry, name, version); err != nil {
log.Warn().
Err(err).
Str("registry", registry).
Str("package", name).
Str("version", version).
Msg("Failed to update download count for newly cached package")
}
// Track download in analytics if enabled
if m.analytics != nil {
m.trackDownload(registry, name, version, storedPkg.Size)
}
return &CacheEntry{
Package: storedPkg,
Data: storedData,
+3
View File
@@ -343,6 +343,7 @@ func TestGet(t *testing.T) {
s.On("Put", mock.Anything, "npm/lodash/4.17.21", mock.Anything, mock.Anything).Return(nil)
m.On("SavePackage", mock.Anything, mock.Anything).Return(nil)
s.On("Get", mock.Anything, "npm/lodash/4.17.21").Return(io.NopCloser(strings.NewReader("upstream data")), nil)
m.On("UpdateDownloadCount", mock.Anything, "npm", "lodash", "4.17.21").Return(nil)
},
fetchFunc: func(ctx context.Context) (io.ReadCloser, string, error) {
return io.NopCloser(strings.NewReader("upstream data")), "https://registry.npmjs.org/lodash", nil
@@ -374,6 +375,7 @@ func TestGet(t *testing.T) {
s.On("Put", mock.Anything, "npm/expired-pkg/1.0.0", mock.Anything, mock.Anything).Return(nil)
m.On("SavePackage", mock.Anything, mock.Anything).Return(nil)
s.On("Get", mock.Anything, "npm/expired-pkg/1.0.0").Return(io.NopCloser(strings.NewReader("refreshed data")), nil)
m.On("UpdateDownloadCount", mock.Anything, "npm", "expired-pkg", "1.0.0").Return(nil)
},
fetchFunc: func(ctx context.Context) (io.ReadCloser, string, error) {
return io.NopCloser(strings.NewReader("refreshed data")), "https://registry.npmjs.org/expired-pkg", nil
@@ -435,6 +437,7 @@ func TestGet(t *testing.T) {
m.On("SavePackage", mock.Anything, mock.Anything).Return(nil)
// Second Get succeeds (after re-storing)
s.On("Get", mock.Anything, "npm/inconsistent/1.0.0").Return(io.NopCloser(strings.NewReader("recovered data")), nil).Once()
m.On("UpdateDownloadCount", mock.Anything, "npm", "inconsistent", "1.0.0").Return(nil)
},
fetchFunc: func(ctx context.Context) (io.ReadCloser, string, error) {
return io.NopCloser(strings.NewReader("recovered data")), "https://registry.npmjs.org/inconsistent", nil
+52 -8
View File
@@ -583,29 +583,38 @@ func (s *GORMStoreV2) GetTimeSeriesStats(ctx context.Context, period string, reg
// Determine which table to query based on period
var tableName string
var since time.Time
var bucketDuration time.Duration
switch period {
case "1h":
tableName = "download_stats_hourly"
since = time.Now().Add(-1 * time.Hour)
since = time.Now().Add(-1 * time.Hour).Truncate(time.Hour)
bucketDuration = time.Hour
case "1day":
tableName = "download_stats_hourly"
since = time.Now().Add(-24 * time.Hour)
since = time.Now().Add(-24 * time.Hour).Truncate(time.Hour)
bucketDuration = time.Hour
case "7day":
tableName = "download_stats_daily"
since = time.Now().Add(-7 * 24 * time.Hour)
since = time.Now().Add(-7 * 24 * time.Hour).Truncate(24 * time.Hour)
bucketDuration = 24 * time.Hour
case "30day":
tableName = "download_stats_daily"
since = time.Now().Add(-30 * 24 * time.Hour)
since = time.Now().Add(-30 * 24 * time.Hour).Truncate(24 * time.Hour)
bucketDuration = 24 * time.Hour
default:
tableName = "download_stats_hourly"
since = time.Now().Add(-24 * time.Hour)
since = time.Now().Add(-24 * time.Hour).Truncate(time.Hour)
bucketDuration = time.Hour
}
now := time.Now()
// Try to get registry-level aggregate stats first
query := s.db.WithContext(ctx).
Table(tableName).
Select("time_bucket as timestamp, download_count as value").
Where("time_bucket >= ?", since)
Where("time_bucket >= ? AND time_bucket <= ?", since, now)
// Filter by registry if specified
if registry != "" {
@@ -630,10 +639,45 @@ func (s *GORMStoreV2) GetTimeSeriesStats(ctx context.Context, period string, reg
return nil, errors.Wrap(err, errors.ErrCodeStorageFailure, "failed to get time series stats")
}
// If no aggregate data found, try summing package-level stats
if len(results) == 0 {
sumQuery := s.db.WithContext(ctx).
Table(tableName).
Select("time_bucket as timestamp, SUM(download_count) as value").
Where("time_bucket >= ? AND time_bucket <= ?", since, now)
if registry != "" {
registryID, err := s.getRegistryID(registry)
if err != nil {
return nil, err
}
sumQuery = sumQuery.Where("registry_id = ? AND package_id IS NOT NULL", registryID)
} else {
sumQuery = sumQuery.Where("package_id IS NOT NULL")
}
sumQuery = sumQuery.Group("time_bucket").Order("time_bucket ASC")
if err := sumQuery.Scan(&results).Error; err != nil {
return nil, errors.Wrap(err, errors.ErrCodeStorageFailure, "failed to get time series stats from package data")
}
}
// Create a map of existing data points
dataMap := make(map[time.Time]int64)
for _, r := range results {
dataMap[r.Timestamp] = r.Value
}
// Generate continuous time series with 0 values for missing periods
for t := since; t.Before(now) || t.Equal(now); t = t.Add(bucketDuration) {
value := int64(0)
if v, exists := dataMap[t]; exists {
value = v
}
stats.DataPoints = append(stats.DataPoints, &metadata.TimeSeriesDataPoint{
Timestamp: r.Timestamp,
Value: r.Value,
Timestamp: t,
Value: value,
})
}