mirror of
https://github.com/lukaszraczylo/gohoarder.git
synced 2026-10-15 00:58:34 +00:00
fix: remediate severity-ordered review findings across cache, scanners, and proxies
- H2: gormstore soft-delete + storage_key UNIQUE collision; re-save after eviction succeeds
- H1: cache getOrFetch cache-hit branch no longer runs CheckVulnerabilities on metadata entries
- M2: cache store() streams upstream body to temp file + SHA256 instead of double-buffering
- H3/L3: FetchResult/FetchFunc plumbing (+RequiresAuth/AuthProvider); proxy closures adapted;
goproxy strips git+ prefix from UpstreamURL
- H4: scanner normalizes bypass package names; CheckVulnerabilities builds normalized packageKey
- M4: requireAdmin routes through extractAPIKey -> ValidateAPIKey -> admin role check
- M5: network client keeps only final RecordFailure (one count per attempt cycle)
- M3: normalizeOriginHost + buildCheckOrigin rewrite; origin_test.go added
- M1: app lifecycle cancel func; Run -> Shutdown on listen error (sole cache owner); JoinHostPort
- M6: VulnerabilityThresholds defaults all -1 so BlockOnSeverity governs
- L4: metadata.NormalizeSeverity at 4 severity-count sites (MEDIUM->MODERATE consistency)
- L1: handleDeleteBypass uses structured *errors.Error + ErrCodeNotFound
- L2: pypi extractVersionFromFilename (wheel/2nd segment, non-wheel/last digit segment);
cacheVersion uses full unique basename; regression tests added
- L5: document per-instance ValidationCache key scoping; rename packageURL param to resource
All: go build ./... && go vet ./... && go test ./... pass; golangci-lint clean.
This commit is contained in:
@@ -115,7 +115,7 @@ func (h *Handler) handleList(ctx context.Context, w http.ResponseWriter, r *http
|
||||
// Extract credentials from request
|
||||
credentials := h.credExtractor.Extract(r)
|
||||
|
||||
entry, err := h.cache.Get(ctx, "go", modulePath, "list", func(ctx context.Context) (io.ReadCloser, string, error) {
|
||||
entry, err := h.cache.Get(ctx, "go", modulePath, "list", func(ctx context.Context) (*cache.FetchResult, error) {
|
||||
// Prepare headers for upstream request
|
||||
headers := make(map[string]string)
|
||||
if credentials != "" {
|
||||
@@ -124,13 +124,16 @@ func (h *Handler) handleList(ctx context.Context, w http.ResponseWriter, r *http
|
||||
|
||||
body, statusCode, err := h.client.Get(ctx, url, headers)
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
return nil, err
|
||||
}
|
||||
if statusCode == http.StatusUnauthorized || statusCode == http.StatusForbidden {
|
||||
return &cache.FetchResult{Data: body, UpstreamURL: url, RequiresAuth: true, AuthProvider: h.credValidator.ProviderFor(modulePath)}, nil
|
||||
}
|
||||
if statusCode != http.StatusOK {
|
||||
_ = body.Close() // #nosec G104 -- Cleanup, error not critical
|
||||
return nil, "", fmt.Errorf("upstream returned status %d", statusCode)
|
||||
return nil, fmt.Errorf("upstream returned status %d", statusCode)
|
||||
}
|
||||
return body, url, nil
|
||||
return &cache.FetchResult{Data: body, UpstreamURL: url}, nil
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
@@ -155,7 +158,7 @@ func (h *Handler) handleInfo(ctx context.Context, w http.ResponseWriter, r *http
|
||||
// Extract credentials from request
|
||||
credentials := h.credExtractor.Extract(r)
|
||||
|
||||
entry, err := h.cache.Get(ctx, "go", cacheKey, version, func(ctx context.Context) (io.ReadCloser, string, error) {
|
||||
entry, err := h.cache.Get(ctx, "go", cacheKey, version, func(ctx context.Context) (*cache.FetchResult, error) {
|
||||
// Prepare headers for upstream request
|
||||
headers := make(map[string]string)
|
||||
if credentials != "" {
|
||||
@@ -164,13 +167,16 @@ func (h *Handler) handleInfo(ctx context.Context, w http.ResponseWriter, r *http
|
||||
|
||||
body, statusCode, err := h.client.Get(ctx, url, headers)
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
return nil, err
|
||||
}
|
||||
if statusCode == http.StatusUnauthorized || statusCode == http.StatusForbidden {
|
||||
return &cache.FetchResult{Data: body, UpstreamURL: url, RequiresAuth: true, AuthProvider: h.credValidator.ProviderFor(modulePath)}, nil
|
||||
}
|
||||
if statusCode != http.StatusOK {
|
||||
_ = body.Close() // #nosec G104 -- Cleanup, error not critical
|
||||
return nil, "", fmt.Errorf("upstream returned status %d", statusCode)
|
||||
return nil, fmt.Errorf("upstream returned status %d", statusCode)
|
||||
}
|
||||
return body, url, nil
|
||||
return &cache.FetchResult{Data: body, UpstreamURL: url}, nil
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
@@ -188,14 +194,14 @@ func (h *Handler) handleInfo(ctx context.Context, w http.ResponseWriter, r *http
|
||||
func (h *Handler) handleMod(ctx context.Context, w http.ResponseWriter, r *http.Request, path string) {
|
||||
url := h.upstream + path
|
||||
modulePath := h.extractModulePath(path)
|
||||
|
||||
// Extract credentials from request
|
||||
credentials := h.credExtractor.Extract(r)
|
||||
version := h.extractVersion(path, ".mod")
|
||||
// Use .mod suffix to distinguish from .info and .zip in cache
|
||||
cacheKey := modulePath + "/@v/" + version + ".mod"
|
||||
|
||||
// Extract credentials from request
|
||||
credentials := h.credExtractor.Extract(r)
|
||||
|
||||
entry, err := h.cache.Get(ctx, "go", cacheKey, version, func(ctx context.Context) (io.ReadCloser, string, error) {
|
||||
entry, err := h.cache.Get(ctx, "go", cacheKey, version, func(ctx context.Context) (*cache.FetchResult, error) {
|
||||
// Prepare headers for upstream request
|
||||
headers := make(map[string]string)
|
||||
if credentials != "" {
|
||||
@@ -204,13 +210,16 @@ func (h *Handler) handleMod(ctx context.Context, w http.ResponseWriter, r *http.
|
||||
|
||||
body, statusCode, err := h.client.Get(ctx, url, headers)
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
return nil, err
|
||||
}
|
||||
if statusCode == http.StatusUnauthorized || statusCode == http.StatusForbidden {
|
||||
return &cache.FetchResult{Data: body, UpstreamURL: url, RequiresAuth: true, AuthProvider: h.credValidator.ProviderFor(modulePath)}, nil
|
||||
}
|
||||
if statusCode != http.StatusOK {
|
||||
_ = body.Close() // #nosec G104 -- Cleanup, error not critical
|
||||
return nil, "", fmt.Errorf("upstream returned status %d", statusCode)
|
||||
return nil, fmt.Errorf("upstream returned status %d", statusCode)
|
||||
}
|
||||
return body, url, nil
|
||||
return &cache.FetchResult{Data: body, UpstreamURL: url}, nil
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
@@ -245,7 +254,7 @@ func (h *Handler) handleZip(ctx context.Context, w http.ResponseWriter, r *http.
|
||||
Bool("has_credentials", credentials != "").
|
||||
Msg("Handling Go module zip request")
|
||||
|
||||
entry, err := h.cache.Get(ctx, "go", cacheKey, version, func(ctx context.Context) (io.ReadCloser, string, error) {
|
||||
entry, err := h.cache.Get(ctx, "go", cacheKey, version, func(ctx context.Context) (*cache.FetchResult, error) {
|
||||
// Prepare headers for upstream request
|
||||
headers := make(map[string]string)
|
||||
if credentials != "" {
|
||||
@@ -255,7 +264,7 @@ func (h *Handler) handleZip(ctx context.Context, w http.ResponseWriter, r *http.
|
||||
// Try upstream proxy first (fast path for public modules)
|
||||
body, statusCode, err := h.client.Get(ctx, url, headers)
|
||||
if err == nil && statusCode == http.StatusOK {
|
||||
return body, url, nil
|
||||
return &cache.FetchResult{Data: body, UpstreamURL: url}, nil
|
||||
}
|
||||
|
||||
// If upstream failed with 404 or 403, try git fallback (private modules)
|
||||
@@ -278,9 +287,9 @@ func (h *Handler) handleZip(ctx context.Context, w http.ResponseWriter, r *http.
|
||||
_ = body.Close() // #nosec G104 -- Cleanup, error not critical
|
||||
}
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
return nil, err
|
||||
}
|
||||
return nil, "", fmt.Errorf("upstream returned status %d", statusCode)
|
||||
return nil, fmt.Errorf("upstream returned status %d", statusCode)
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
@@ -299,6 +308,10 @@ func (h *Handler) handleZip(ctx context.Context, w http.ResponseWriter, r *http.
|
||||
defer func() { _ = entry.Data.Close() }() // #nosec G104 -- Cleanup, error not critical
|
||||
|
||||
// CRITICAL SECURITY CHECK: If module requires auth, validate credentials
|
||||
// The auth validation cache is owned by this handler instance only, so its
|
||||
// key may use the module path (modulePath) rather than a URL. The module path
|
||||
// uniquely identifies the private module within this registry's cache; there
|
||||
// is no cross-registry collision because npm/pypi use their own instances.
|
||||
if entry.Package != nil && entry.Package.RequiresAuth {
|
||||
// Check validation cache first
|
||||
allowed, cached, reason := h.validationCache.Get(credHash, modulePath)
|
||||
@@ -352,6 +365,7 @@ func (h *Handler) handleZip(ctx context.Context, w http.ResponseWriter, r *http.
|
||||
|
||||
w.Header().Set("Content-Type", "application/zip")
|
||||
_, _ = io.Copy(w, entry.Data) // #nosec G104 -- HTTP response write
|
||||
|
||||
}
|
||||
|
||||
// handleLatest handles /@latest requests
|
||||
@@ -362,7 +376,7 @@ func (h *Handler) handleLatest(ctx context.Context, w http.ResponseWriter, r *ht
|
||||
// Extract credentials from request
|
||||
credentials := h.credExtractor.Extract(r)
|
||||
|
||||
entry, err := h.cache.Get(ctx, "go", modulePath, "latest", func(ctx context.Context) (io.ReadCloser, string, error) {
|
||||
entry, err := h.cache.Get(ctx, "go", modulePath, "latest", func(ctx context.Context) (*cache.FetchResult, error) {
|
||||
// Prepare headers for upstream request
|
||||
headers := make(map[string]string)
|
||||
if credentials != "" {
|
||||
@@ -371,13 +385,16 @@ func (h *Handler) handleLatest(ctx context.Context, w http.ResponseWriter, r *ht
|
||||
|
||||
body, statusCode, err := h.client.Get(ctx, url, headers)
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
return nil, err
|
||||
}
|
||||
if statusCode == http.StatusUnauthorized || statusCode == http.StatusForbidden {
|
||||
return &cache.FetchResult{Data: body, UpstreamURL: url, RequiresAuth: true, AuthProvider: h.credValidator.ProviderFor(modulePath)}, nil
|
||||
}
|
||||
if statusCode != http.StatusOK {
|
||||
_ = body.Close() // #nosec G104 -- Cleanup, error not critical
|
||||
return nil, "", fmt.Errorf("upstream returned status %d", statusCode)
|
||||
return nil, fmt.Errorf("upstream returned status %d", statusCode)
|
||||
}
|
||||
return body, url, nil
|
||||
return &cache.FetchResult{Data: body, UpstreamURL: url}, nil
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
@@ -450,7 +467,7 @@ func (h *Handler) extractModulePath(path string) string {
|
||||
}
|
||||
|
||||
// fetchModuleFromGit fetches a Go module directly from git repository
|
||||
func (h *Handler) fetchModuleFromGit(ctx context.Context, modulePath, version, credentials string) (io.ReadCloser, string, error) {
|
||||
func (h *Handler) fetchModuleFromGit(ctx context.Context, modulePath, version, credentials string) (*cache.FetchResult, error) {
|
||||
log.Info().
|
||||
Str("module", modulePath).
|
||||
Str("version", version).
|
||||
@@ -459,23 +476,25 @@ func (h *Handler) fetchModuleFromGit(ctx context.Context, modulePath, version, c
|
||||
// 1. Fetch module source from git
|
||||
srcPath, err := h.gitFetcher.FetchModule(ctx, modulePath, version, credentials)
|
||||
if err != nil {
|
||||
return nil, "", fmt.Errorf("git fetch failed: %w", err)
|
||||
return nil, fmt.Errorf("git fetch failed: %w", err)
|
||||
}
|
||||
defer h.gitFetcher.Cleanup(srcPath)
|
||||
|
||||
// 2. Validate module
|
||||
if validateErr := h.moduleBuilder.ValidateModule(ctx, srcPath, modulePath); validateErr != nil {
|
||||
return nil, "", fmt.Errorf("module validation failed: %w", validateErr)
|
||||
return nil, fmt.Errorf("module validation failed: %w", validateErr)
|
||||
}
|
||||
|
||||
// 3. Build module zip
|
||||
zipReader, err := h.moduleBuilder.BuildModuleZip(ctx, srcPath, modulePath, version)
|
||||
if err != nil {
|
||||
return nil, "", fmt.Errorf("module zip build failed: %w", err)
|
||||
return nil, fmt.Errorf("module zip build failed: %w", err)
|
||||
}
|
||||
|
||||
// Create source URL for logging
|
||||
sourceURL := fmt.Sprintf("git+https://%s@%s", modulePath, version)
|
||||
// Create source URL for logging. Private modules fetched from git require
|
||||
// auth, so mark the result accordingly. FIX-L3: strip the git+ scheme so the
|
||||
// stored UpstreamURL is a fetchable HTTPS URL.
|
||||
sourceURL := fmt.Sprintf("https://%s@%s", modulePath, version)
|
||||
|
||||
log.Info().
|
||||
Str("module", modulePath).
|
||||
@@ -483,5 +502,10 @@ func (h *Handler) fetchModuleFromGit(ctx context.Context, modulePath, version, c
|
||||
Str("source", sourceURL).
|
||||
Msg("Successfully built module from git")
|
||||
|
||||
return zipReader, sourceURL, nil
|
||||
return &cache.FetchResult{
|
||||
Data: zipReader,
|
||||
UpstreamURL: sourceURL,
|
||||
RequiresAuth: true,
|
||||
AuthProvider: h.credValidator.ProviderFor(modulePath),
|
||||
}, nil
|
||||
}
|
||||
|
||||
+18
-10
@@ -80,16 +80,19 @@ func (h *Handler) handleMetadata(ctx context.Context, w http.ResponseWriter, r *
|
||||
url := h.upstream + path
|
||||
packageName := extractPackageName(path)
|
||||
|
||||
entry, err := h.cache.Get(ctx, "npm", packageName, "metadata", func(ctx context.Context) (io.ReadCloser, string, error) {
|
||||
entry, err := h.cache.Get(ctx, "npm", packageName, "metadata", func(ctx context.Context) (*cache.FetchResult, error) {
|
||||
body, statusCode, fetchErr := h.client.Get(ctx, url, nil)
|
||||
if fetchErr != nil {
|
||||
return nil, "", fetchErr
|
||||
return nil, fetchErr
|
||||
}
|
||||
if statusCode == http.StatusUnauthorized || statusCode == http.StatusForbidden {
|
||||
return &cache.FetchResult{Data: body, UpstreamURL: url, RequiresAuth: true, AuthProvider: h.credValidator.Provider()}, nil
|
||||
}
|
||||
if statusCode != http.StatusOK {
|
||||
_ = body.Close()
|
||||
return nil, "", fmt.Errorf("upstream returned status %d", statusCode)
|
||||
return nil, fmt.Errorf("upstream returned status %d", statusCode)
|
||||
}
|
||||
return body, url, nil
|
||||
return &cache.FetchResult{Data: body, UpstreamURL: url}, nil
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
@@ -158,9 +161,7 @@ func (h *Handler) handleTarball(ctx context.Context, w http.ResponseWriter, r *h
|
||||
Str("cred_hash", credHash).
|
||||
Bool("has_credentials", credentials != "").
|
||||
Msg("Handling tarball request")
|
||||
|
||||
// Try to get from cache first (with credential-aware key)
|
||||
entry, err := h.cache.Get(ctx, "npm", packageName, version, func(ctx context.Context) (io.ReadCloser, string, error) {
|
||||
entry, err := h.cache.Get(ctx, "npm", packageName, version, func(ctx context.Context) (*cache.FetchResult, error) {
|
||||
// Prepare headers for upstream request
|
||||
headers := make(map[string]string)
|
||||
if credentials != "" {
|
||||
@@ -169,13 +170,16 @@ func (h *Handler) handleTarball(ctx context.Context, w http.ResponseWriter, r *h
|
||||
|
||||
body, statusCode, fetchErr := h.client.Get(ctx, url, headers)
|
||||
if fetchErr != nil {
|
||||
return nil, "", fetchErr
|
||||
return nil, fetchErr
|
||||
}
|
||||
if statusCode == http.StatusUnauthorized || statusCode == http.StatusForbidden {
|
||||
return &cache.FetchResult{Data: body, UpstreamURL: url, RequiresAuth: true, AuthProvider: h.credValidator.Provider()}, nil
|
||||
}
|
||||
if statusCode != http.StatusOK {
|
||||
_ = body.Close()
|
||||
return nil, "", fmt.Errorf("upstream returned status %d", statusCode)
|
||||
return nil, fmt.Errorf("upstream returned status %d", statusCode)
|
||||
}
|
||||
return body, url, nil
|
||||
return &cache.FetchResult{Data: body, UpstreamURL: url}, nil
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
@@ -198,6 +202,10 @@ func (h *Handler) handleTarball(ctx context.Context, w http.ResponseWriter, r *h
|
||||
}()
|
||||
|
||||
// CRITICAL SECURITY CHECK: If package requires auth, validate credentials
|
||||
// The validation cache is owned by this handler instance only, so the key
|
||||
// uses the upstream tarball URL. Uniquely identifies the artifact within
|
||||
// this registry; no cross-registry collision (goproxy/pypi keep their own
|
||||
// independent instances).
|
||||
if entry.Package != nil && entry.Package.RequiresAuth {
|
||||
// Check validation cache first
|
||||
allowed, cached, reason := h.validationCache.Get(credHash, url)
|
||||
|
||||
+77
-41
@@ -124,16 +124,19 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
func (h *Handler) handleIndex(ctx context.Context, w http.ResponseWriter, r *http.Request) {
|
||||
url := h.upstream + "/"
|
||||
|
||||
entry, err := h.cache.Get(ctx, "pypi", "index", "latest", func(ctx context.Context) (io.ReadCloser, string, error) {
|
||||
entry, err := h.cache.Get(ctx, "pypi", "index", "latest", func(ctx context.Context) (*cache.FetchResult, error) {
|
||||
body, statusCode, err := h.client.Get(ctx, url, nil)
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
return nil, err
|
||||
}
|
||||
if statusCode == http.StatusUnauthorized || statusCode == http.StatusForbidden {
|
||||
return &cache.FetchResult{Data: body, UpstreamURL: url, RequiresAuth: true, AuthProvider: h.credValidator.Provider()}, nil
|
||||
}
|
||||
if statusCode != http.StatusOK {
|
||||
_ = body.Close()
|
||||
return nil, "", fmt.Errorf("upstream returned status %d", statusCode)
|
||||
return nil, fmt.Errorf("upstream returned status %d", statusCode)
|
||||
}
|
||||
return body, url, nil
|
||||
return &cache.FetchResult{Data: body, UpstreamURL: url}, nil
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
@@ -155,17 +158,19 @@ func (h *Handler) handleIndex(ctx context.Context, w http.ResponseWriter, r *htt
|
||||
func (h *Handler) handlePackagePage(ctx context.Context, w http.ResponseWriter, r *http.Request, path string) {
|
||||
url := h.upstream + path
|
||||
packageName := extractPackageName(path)
|
||||
|
||||
entry, err := h.cache.Get(ctx, "pypi", packageName, "page", func(ctx context.Context) (io.ReadCloser, string, error) {
|
||||
entry, err := h.cache.Get(ctx, "pypi", packageName, "page", func(ctx context.Context) (*cache.FetchResult, error) {
|
||||
body, statusCode, err := h.client.Get(ctx, url, nil)
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
return nil, err
|
||||
}
|
||||
if statusCode == http.StatusUnauthorized || statusCode == http.StatusForbidden {
|
||||
return &cache.FetchResult{Data: body, UpstreamURL: url, RequiresAuth: true, AuthProvider: h.credValidator.Provider()}, nil
|
||||
}
|
||||
if statusCode != http.StatusOK {
|
||||
_ = body.Close()
|
||||
return nil, "", fmt.Errorf("upstream returned status %d", statusCode)
|
||||
return nil, fmt.Errorf("upstream returned status %d", statusCode)
|
||||
}
|
||||
return body, url, nil
|
||||
return &cache.FetchResult{Data: body, UpstreamURL: url}, nil
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
@@ -199,16 +204,17 @@ func (h *Handler) handlePackagePage(ctx context.Context, w http.ResponseWriter,
|
||||
func (h *Handler) handlePackageFile(ctx context.Context, w http.ResponseWriter, r *http.Request, path string) {
|
||||
packageName, version := extractPackageFileInfo(path)
|
||||
|
||||
// Make version unique by appending file type to avoid cache collisions
|
||||
// between .whl and .metadata files with same version
|
||||
cacheVersion := version
|
||||
if strings.HasSuffix(path, ".metadata") {
|
||||
cacheVersion = version + ".metadata"
|
||||
} else if strings.HasSuffix(path, ".whl") {
|
||||
cacheVersion = version + ".whl"
|
||||
} else if strings.HasSuffix(path, ".tar.gz") {
|
||||
cacheVersion = version + ".tar.gz"
|
||||
// cacheVersion is the cache key's version portion. It must be unique per
|
||||
// distinct artifact. PyPI publishes many files per release (multiple
|
||||
// wheels with different build tags, sdists, zips, .egg files, and
|
||||
// .metadata files for PEP 658), so the bare version is too coarse:
|
||||
// distinct files of the same version would collide on the same cache
|
||||
// key. The full unique basename guarantees one cache entry per artifact.
|
||||
filename := path
|
||||
if i := strings.LastIndex(filename, "/"); i >= 0 {
|
||||
filename = filename[i+1:]
|
||||
}
|
||||
cacheVersion := filename
|
||||
|
||||
// Extract credentials from request
|
||||
credentials := h.credExtractor.Extract(r)
|
||||
@@ -255,7 +261,7 @@ func (h *Handler) handlePackageFile(ctx context.Context, w http.ResponseWriter,
|
||||
Bool("has_credentials", credentials != "").
|
||||
Msg("Handling PyPI package file request")
|
||||
|
||||
entry, err := h.cache.Get(ctx, "pypi", packageName, cacheVersion, func(ctx context.Context) (io.ReadCloser, string, error) {
|
||||
entry, err := h.cache.Get(ctx, "pypi", packageName, cacheVersion, func(ctx context.Context) (*cache.FetchResult, error) {
|
||||
// Prepare headers for upstream request
|
||||
headers := make(map[string]string)
|
||||
if credentials != "" {
|
||||
@@ -264,13 +270,16 @@ func (h *Handler) handlePackageFile(ctx context.Context, w http.ResponseWriter,
|
||||
|
||||
body, statusCode, err := h.client.Get(ctx, originalURL, headers)
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
return nil, err
|
||||
}
|
||||
if statusCode == http.StatusUnauthorized || statusCode == http.StatusForbidden {
|
||||
return &cache.FetchResult{Data: body, UpstreamURL: originalURL, RequiresAuth: true, AuthProvider: h.credValidator.Provider()}, nil
|
||||
}
|
||||
if statusCode != http.StatusOK {
|
||||
_ = body.Close()
|
||||
return nil, "", fmt.Errorf("upstream returned status %d", statusCode)
|
||||
return nil, fmt.Errorf("upstream returned status %d", statusCode)
|
||||
}
|
||||
return body, originalURL, nil
|
||||
return &cache.FetchResult{Data: body, UpstreamURL: originalURL}, nil
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
@@ -293,6 +302,10 @@ func (h *Handler) handlePackageFile(ctx context.Context, w http.ResponseWriter,
|
||||
}()
|
||||
|
||||
// CRITICAL SECURITY CHECK: If package requires auth, validate credentials
|
||||
// The validation cache is owned by this handler instance only, so the key
|
||||
// uses the upstream artifact URL (originalURL). Uniquely identifies the
|
||||
// artifact within this registry; no cross-registry collision (goproxy/npm
|
||||
// keep their own independent instances).
|
||||
if entry.Package != nil && entry.Package.RequiresAuth {
|
||||
// Check validation cache first
|
||||
allowed, cached, reason := h.validationCache.Get(credHash, originalURL)
|
||||
@@ -395,37 +408,60 @@ func extractPackageName(path string) string {
|
||||
func extractPackageFileInfo(path string) (string, string) {
|
||||
// Format: /package-name/package-name-version.whl
|
||||
// or: /package-name/package-name-version.tar.gz
|
||||
|
||||
packageName := extractPackageName(path)
|
||||
|
||||
// Extract filename
|
||||
parts := strings.Split(path, "/")
|
||||
if len(parts) < 2 {
|
||||
return packageName, ""
|
||||
}
|
||||
|
||||
filename := parts[len(parts)-1]
|
||||
base := parts[len(parts)-1]
|
||||
isWheel := strings.HasSuffix(base, ".whl")
|
||||
base = strings.TrimSuffix(base, ".whl")
|
||||
base = strings.TrimSuffix(base, ".tar.gz")
|
||||
base = strings.TrimSuffix(base, ".zip")
|
||||
base = strings.TrimSuffix(base, ".egg")
|
||||
base = strings.TrimSuffix(base, ".metadata")
|
||||
|
||||
// Remove extension
|
||||
filename = strings.TrimSuffix(filename, ".whl")
|
||||
filename = strings.TrimSuffix(filename, ".tar.gz")
|
||||
filename = strings.TrimSuffix(filename, ".zip")
|
||||
filename = strings.TrimSuffix(filename, ".egg")
|
||||
return packageName, extractVersionFromFilename(base, isWheel)
|
||||
}
|
||||
|
||||
// Extract version
|
||||
// Filename format: package-name-version or package_name-version
|
||||
// Version typically starts after last dash before build tags
|
||||
versionParts := strings.Split(filename, "-")
|
||||
if len(versionParts) >= 2 {
|
||||
// Simple heuristic: version is the part that starts with a digit
|
||||
for i := 1; i < len(versionParts); i++ {
|
||||
if len(versionParts[i]) > 0 && versionParts[i][0] >= '0' && versionParts[i][0] <= '9' {
|
||||
return packageName, versionParts[i]
|
||||
}
|
||||
// extractVersionFromFilename extracts the version from a PyPI distribution
|
||||
// filename with the extension already stripped.
|
||||
//
|
||||
// Wheels (PEP 427) are formatted as
|
||||
//
|
||||
// {distribution}-{version}(-{build})?-{python}-{abi}-{platform}
|
||||
//
|
||||
// and the distribution is normalized (lowercase, runs of '-', '.', '_'
|
||||
// collapsed to '_'), so it never contains '-' and the version is always the
|
||||
// second dash-separated segment. For sdists/zips/eggs the filename is
|
||||
// {name}-{version}; PyPI normalizes name separators to '_', so the version
|
||||
// is always the last dash-separated segment (PEP 440 versions never contain
|
||||
// '-').
|
||||
//
|
||||
// The historical first-digit-segment heuristic is avoided because it mangles
|
||||
// digit-embedded distribution names (e.g. "2captcha-1.0.0" → "2captcha") and
|
||||
// intermediate digit segments (e.g. "foo-2019bar-1.0" → "2019bar"). The last
|
||||
// digit-starting segment is correct for those cases.
|
||||
func extractVersionFromFilename(base string, isWheel bool) string {
|
||||
if isWheel {
|
||||
// The distribution never contains '-', so the version immediately
|
||||
// follows the first '-'.
|
||||
if parts := strings.SplitN(base, "-", 3); len(parts) >= 2 && parts[1] != "" && parts[1][0] >= '0' && parts[1][0] <= '9' {
|
||||
return parts[1]
|
||||
}
|
||||
}
|
||||
|
||||
return packageName, filename
|
||||
// Non-wheel fallback (and defensive wheel edge case): the last
|
||||
// dash-separated segment that starts with a digit.
|
||||
parts := strings.Split(base, "-")
|
||||
for i := len(parts) - 1; i >= 1; i-- {
|
||||
if len(parts[i]) > 0 && parts[i][0] >= '0' && parts[i][0] <= '9' {
|
||||
return parts[i]
|
||||
}
|
||||
}
|
||||
return base
|
||||
}
|
||||
|
||||
// getProxyBaseURL constructs the proxy base URL from the request
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/lukaszraczylo/gohoarder/pkg/auth"
|
||||
@@ -137,3 +138,72 @@ func indexOf(s, sub string) int {
|
||||
}
|
||||
return -1
|
||||
}
|
||||
|
||||
func TestExtractPackageFileInfo(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
path string
|
||||
wantName string
|
||||
wantVer string
|
||||
}{
|
||||
{"numpy wheel", "/numpy/numpy-1.26.0rc1-cp312-cp312-manylinux.whl", "numpy", "1.26.0rc1"},
|
||||
{"underscore wheel name", "/typing_extensions/typing_extensions-4.8.0-py3-none-any.whl", "typing_extensions", "4.8.0"},
|
||||
{"hyphenated sdist", "/my-pkg/my-pkg-2.0.1.tar.gz", "my-pkg", "2.0.1"},
|
||||
{"hyphen name underscore file", "/typed-ast/typed_ast-1.5.4.tar.gz", "typed-ast", "1.5.4"},
|
||||
{"digit-embedded name", "/2captcha/2captcha-1.0.0.tar.gz", "2captcha", "1.0.0"},
|
||||
{"digit intermediate segment", "/foo/foo-2019bar-1.0.zip", "foo", "1.0"},
|
||||
{"egg file", "/foo/foo-1.0.egg", "foo", "1.0"},
|
||||
{"metadata file", "/foo/foo-1.0.metadata", "foo", "1.0"},
|
||||
{"no version", "/foo", "foo", "foo"},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
gotName, gotVer := extractPackageFileInfo(tt.path)
|
||||
if gotName != tt.wantName {
|
||||
t.Errorf("extractPackageFileInfo(%q) name = %q, want %q", tt.path, gotName, tt.wantName)
|
||||
}
|
||||
if gotVer != tt.wantVer {
|
||||
t.Errorf("extractPackageFileInfo(%q) version = %q, want %q", tt.path, gotVer, tt.wantVer)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestCacheVersionUniquePerArtifact asserts that distinct PyPI artifacts of
|
||||
// the same logical version produce distinct cache keys, and that the bare
|
||||
// version returned for logging stays stable.
|
||||
func TestCacheVersionUniquePerArtifact(t *testing.T) {
|
||||
// Three wheels of the same version differing only in build/tag, plus a
|
||||
// .zip and a .metadata file for the same version. The cache key uses the
|
||||
// full unique basename, so keys must all differ; the extracted version
|
||||
// must remain the bare stable version for logging/bypass logic.
|
||||
paths := []string{
|
||||
"/foo/foo-1.0.0-cp312-cp312-manylinux.whl",
|
||||
"/foo/foo-1.0.0-cp311-cp311-manylinux.whl",
|
||||
"/foo/foo-1.0.0-py3-none-any.whl",
|
||||
"/foo/foo-1.0.0.zip",
|
||||
"/foo/foo-1.0.0.metadata",
|
||||
}
|
||||
|
||||
seenKeys := make(map[string]bool)
|
||||
for _, path := range paths {
|
||||
_, ver := extractPackageFileInfo(path)
|
||||
if ver != "1.0.0" {
|
||||
t.Errorf("extractPackageFileInfo(%q) version = %q, want stable %q", path, ver, "1.0.0")
|
||||
}
|
||||
|
||||
filename := path
|
||||
if i := strings.LastIndex(filename, "/"); i >= 0 {
|
||||
filename = filename[i+1:]
|
||||
}
|
||||
if seenKeys[filename] {
|
||||
t.Errorf("cache key %q duplicated across artifacts", filename)
|
||||
}
|
||||
seenKeys[filename] = true
|
||||
}
|
||||
|
||||
if len(seenKeys) != len(paths) {
|
||||
t.Errorf("expected %d distinct cache keys, got %d", len(paths), len(seenKeys))
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user