lukaszraczylo/gohoarder
1
0
Fork 0
mirror of https://github.com/lukaszraczylo/gohoarder.git synced 2026-10-17 01:17:31 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix: remediate severity-ordered review findings across cache, scanners, and proxies

Browse Source 
- H2: gormstore soft-delete + storage_key UNIQUE collision; re-save after eviction succeeds
- H1: cache getOrFetch cache-hit branch no longer runs CheckVulnerabilities on metadata entries
- M2: cache store() streams upstream body to temp file + SHA256 instead of double-buffering
- H3/L3: FetchResult/FetchFunc plumbing (+RequiresAuth/AuthProvider); proxy closures adapted;
        goproxy strips git+ prefix from UpstreamURL
- H4: scanner normalizes bypass package names; CheckVulnerabilities builds normalized packageKey
- M4: requireAdmin routes through extractAPIKey -> ValidateAPIKey -> admin role check
- M5: network client keeps only final RecordFailure (one count per attempt cycle)
- M3: normalizeOriginHost + buildCheckOrigin rewrite; origin_test.go added
- M1: app lifecycle cancel func; Run -> Shutdown on listen error (sole cache owner); JoinHostPort
- M6: VulnerabilityThresholds defaults all -1 so BlockOnSeverity governs
- L4: metadata.NormalizeSeverity at 4 severity-count sites (MEDIUM->MODERATE consistency)
- L1: handleDeleteBypass uses structured *errors.Error + ErrCodeNotFound
- L2: pypi extractVersionFromFilename (wheel/2nd segment, non-wheel/last digit segment);
      cacheVersion uses full unique basename; regression tests added
- L5: document per-instance ValidationCache key scoping; rename packageURL param to resource

All: go build ./... && go vet ./... && go test ./... pass; golangci-lint clean.
This commit is contained in:
lukaszraczylo
2026-09-04 14:24:13 +01:00
parent edeb2eca0c
commit 171386f7a7
20 changed files with 790 additions and 239 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/proxy/pypi/pypi.go
+77 -41
View File
@@ -124,16 +124,19 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
func (h *Handler) handleIndex(ctx context.Context, w http.ResponseWriter, r *http.Request) {
url := h.upstream + "/"
entry, err := h.cache.Get(ctx, "pypi", "index", "latest", func(ctx context.Context) (io.ReadCloser, string, error) {
entry, err := h.cache.Get(ctx, "pypi", "index", "latest", func(ctx context.Context) (*cache.FetchResult, error) {
body, statusCode, err := h.client.Get(ctx, url, nil)
if err != nil {
return nil, "", err
return nil, err
}
if statusCode == http.StatusUnauthorized || statusCode == http.StatusForbidden {
return &cache.FetchResult{Data: body, UpstreamURL: url, RequiresAuth: true, AuthProvider: h.credValidator.Provider()}, nil
}
if statusCode != http.StatusOK {
_ = body.Close()
return nil, "", fmt.Errorf("upstream returned status %d", statusCode)
return nil, fmt.Errorf("upstream returned status %d", statusCode)
}
return body, url, nil
return &cache.FetchResult{Data: body, UpstreamURL: url}, nil
})
if err != nil {
@@ -155,17 +158,19 @@ func (h *Handler) handleIndex(ctx context.Context, w http.ResponseWriter, r *htt
func (h *Handler) handlePackagePage(ctx context.Context, w http.ResponseWriter, r *http.Request, path string) {
url := h.upstream + path
packageName := extractPackageName(path)
entry, err := h.cache.Get(ctx, "pypi", packageName, "page", func(ctx context.Context) (io.ReadCloser, string, error) {
entry, err := h.cache.Get(ctx, "pypi", packageName, "page", func(ctx context.Context) (*cache.FetchResult, error) {
body, statusCode, err := h.client.Get(ctx, url, nil)
if err != nil {
return nil, "", err
return nil, err
}
if statusCode == http.StatusUnauthorized || statusCode == http.StatusForbidden {
return &cache.FetchResult{Data: body, UpstreamURL: url, RequiresAuth: true, AuthProvider: h.credValidator.Provider()}, nil
}
if statusCode != http.StatusOK {
_ = body.Close()
return nil, "", fmt.Errorf("upstream returned status %d", statusCode)
return nil, fmt.Errorf("upstream returned status %d", statusCode)
}
return body, url, nil
return &cache.FetchResult{Data: body, UpstreamURL: url}, nil
})
if err != nil {
@@ -199,16 +204,17 @@ func (h *Handler) handlePackagePage(ctx context.Context, w http.ResponseWriter,
func (h *Handler) handlePackageFile(ctx context.Context, w http.ResponseWriter, r *http.Request, path string) {
packageName, version := extractPackageFileInfo(path)
// Make version unique by appending file type to avoid cache collisions
// between .whl and .metadata files with same version
cacheVersion := version
if strings.HasSuffix(path, ".metadata") {
cacheVersion = version + ".metadata"
} else if strings.HasSuffix(path, ".whl") {
cacheVersion = version + ".whl"
} else if strings.HasSuffix(path, ".tar.gz") {
cacheVersion = version + ".tar.gz"
// cacheVersion is the cache key's version portion. It must be unique per
// distinct artifact. PyPI publishes many files per release (multiple
// wheels with different build tags, sdists, zips, .egg files, and
// .metadata files for PEP 658), so the bare version is too coarse:
// distinct files of the same version would collide on the same cache
// key. The full unique basename guarantees one cache entry per artifact.
filename := path
if i := strings.LastIndex(filename, "/"); i >= 0 {
filename = filename[i+1:]
}
cacheVersion := filename
// Extract credentials from request
credentials := h.credExtractor.Extract(r)
@@ -255,7 +261,7 @@ func (h *Handler) handlePackageFile(ctx context.Context, w http.ResponseWriter,
Bool("has_credentials", credentials != "").
Msg("Handling PyPI package file request")
entry, err := h.cache.Get(ctx, "pypi", packageName, cacheVersion, func(ctx context.Context) (io.ReadCloser, string, error) {
entry, err := h.cache.Get(ctx, "pypi", packageName, cacheVersion, func(ctx context.Context) (*cache.FetchResult, error) {
// Prepare headers for upstream request
headers := make(map[string]string)
if credentials != "" {
@@ -264,13 +270,16 @@ func (h *Handler) handlePackageFile(ctx context.Context, w http.ResponseWriter,
body, statusCode, err := h.client.Get(ctx, originalURL, headers)
if err != nil {
return nil, "", err
return nil, err
}
if statusCode == http.StatusUnauthorized || statusCode == http.StatusForbidden {
return &cache.FetchResult{Data: body, UpstreamURL: originalURL, RequiresAuth: true, AuthProvider: h.credValidator.Provider()}, nil
}
if statusCode != http.StatusOK {
_ = body.Close()
return nil, "", fmt.Errorf("upstream returned status %d", statusCode)
return nil, fmt.Errorf("upstream returned status %d", statusCode)
}
return body, originalURL, nil
return &cache.FetchResult{Data: body, UpstreamURL: originalURL}, nil
})
if err != nil {
@@ -293,6 +302,10 @@ func (h *Handler) handlePackageFile(ctx context.Context, w http.ResponseWriter,
}()
// CRITICAL SECURITY CHECK: If package requires auth, validate credentials
// The validation cache is owned by this handler instance only, so the key
// uses the upstream artifact URL (originalURL). Uniquely identifies the
// artifact within this registry; no cross-registry collision (goproxy/npm
// keep their own independent instances).
if entry.Package != nil && entry.Package.RequiresAuth {
// Check validation cache first
allowed, cached, reason := h.validationCache.Get(credHash, originalURL)
@@ -395,37 +408,60 @@ func extractPackageName(path string) string {
func extractPackageFileInfo(path string) (string, string) {
// Format: /package-name/package-name-version.whl
// or: /package-name/package-name-version.tar.gz
packageName := extractPackageName(path)
// Extract filename
parts := strings.Split(path, "/")
if len(parts) < 2 {
return packageName, ""
}
filename := parts[len(parts)-1]
base := parts[len(parts)-1]
isWheel := strings.HasSuffix(base, ".whl")
base = strings.TrimSuffix(base, ".whl")
base = strings.TrimSuffix(base, ".tar.gz")
base = strings.TrimSuffix(base, ".zip")
base = strings.TrimSuffix(base, ".egg")
base = strings.TrimSuffix(base, ".metadata")
// Remove extension
filename = strings.TrimSuffix(filename, ".whl")
filename = strings.TrimSuffix(filename, ".tar.gz")
filename = strings.TrimSuffix(filename, ".zip")
filename = strings.TrimSuffix(filename, ".egg")
return packageName, extractVersionFromFilename(base, isWheel)
}
// Extract version
// Filename format: package-name-version or package_name-version
// Version typically starts after last dash before build tags
versionParts := strings.Split(filename, "-")
if len(versionParts) >= 2 {
// Simple heuristic: version is the part that starts with a digit
for i := 1; i < len(versionParts); i++ {
if len(versionParts[i]) > 0 && versionParts[i][0] >= '0' && versionParts[i][0] <= '9' {
return packageName, versionParts[i]
}
// extractVersionFromFilename extracts the version from a PyPI distribution
// filename with the extension already stripped.
//
// Wheels (PEP 427) are formatted as
//
// {distribution}-{version}(-{build})?-{python}-{abi}-{platform}
//
// and the distribution is normalized (lowercase, runs of '-', '.', '_'
// collapsed to '_'), so it never contains '-' and the version is always the
// second dash-separated segment. For sdists/zips/eggs the filename is
// {name}-{version}; PyPI normalizes name separators to '_', so the version
// is always the last dash-separated segment (PEP 440 versions never contain
// '-').
//
// The historical first-digit-segment heuristic is avoided because it mangles
// digit-embedded distribution names (e.g. "2captcha-1.0.0" → "2captcha") and
// intermediate digit segments (e.g. "foo-2019bar-1.0" → "2019bar"). The last
// digit-starting segment is correct for those cases.
func extractVersionFromFilename(base string, isWheel bool) string {
if isWheel {
// The distribution never contains '-', so the version immediately
// follows the first '-'.
if parts := strings.SplitN(base, "-", 3); len(parts) >= 2 && parts[1] != "" && parts[1][0] >= '0' && parts[1][0] <= '9' {
return parts[1]
}
}
return packageName, filename
// Non-wheel fallback (and defensive wheel edge case): the last
// dash-separated segment that starts with a digit.
parts := strings.Split(base, "-")
for i := len(parts) - 1; i >= 1; i-- {
if len(parts[i]) > 0 && parts[i][0] >= '0' && parts[i][0] <= '9' {
return parts[i]
}
}
return base
}
// getProxyBaseURL constructs the proxy base URL from the request
pkg/proxy/pypi/pypi_test.go
+70
View File
@@ -4,6 +4,7 @@ import (
"net/http"
"net/http/httptest"
"net/url"
"strings"
"testing"
"github.com/lukaszraczylo/gohoarder/pkg/auth"
@@ -137,3 +138,72 @@ func indexOf(s, sub string) int {
}
return -1
}
func TestExtractPackageFileInfo(t *testing.T) {
tests := []struct {
name string
path string
wantName string
wantVer string
}{
{"numpy wheel", "/numpy/numpy-1.26.0rc1-cp312-cp312-manylinux.whl", "numpy", "1.26.0rc1"},
{"underscore wheel name", "/typing_extensions/typing_extensions-4.8.0-py3-none-any.whl", "typing_extensions", "4.8.0"},
{"hyphenated sdist", "/my-pkg/my-pkg-2.0.1.tar.gz", "my-pkg", "2.0.1"},
{"hyphen name underscore file", "/typed-ast/typed_ast-1.5.4.tar.gz", "typed-ast", "1.5.4"},
{"digit-embedded name", "/2captcha/2captcha-1.0.0.tar.gz", "2captcha", "1.0.0"},
{"digit intermediate segment", "/foo/foo-2019bar-1.0.zip", "foo", "1.0"},
{"egg file", "/foo/foo-1.0.egg", "foo", "1.0"},
{"metadata file", "/foo/foo-1.0.metadata", "foo", "1.0"},
{"no version", "/foo", "foo", "foo"},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
gotName, gotVer := extractPackageFileInfo(tt.path)
if gotName != tt.wantName {
t.Errorf("extractPackageFileInfo(%q) name = %q, want %q", tt.path, gotName, tt.wantName)
}
if gotVer != tt.wantVer {
t.Errorf("extractPackageFileInfo(%q) version = %q, want %q", tt.path, gotVer, tt.wantVer)
}
})
}
}
// TestCacheVersionUniquePerArtifact asserts that distinct PyPI artifacts of
// the same logical version produce distinct cache keys, and that the bare
// version returned for logging stays stable.
func TestCacheVersionUniquePerArtifact(t *testing.T) {
// Three wheels of the same version differing only in build/tag, plus a
// .zip and a .metadata file for the same version. The cache key uses the
// full unique basename, so keys must all differ; the extracted version
// must remain the bare stable version for logging/bypass logic.
paths := []string{
"/foo/foo-1.0.0-cp312-cp312-manylinux.whl",
"/foo/foo-1.0.0-cp311-cp311-manylinux.whl",
"/foo/foo-1.0.0-py3-none-any.whl",
"/foo/foo-1.0.0.zip",
"/foo/foo-1.0.0.metadata",
}
seenKeys := make(map[string]bool)
for _, path := range paths {
_, ver := extractPackageFileInfo(path)
if ver != "1.0.0" {
t.Errorf("extractPackageFileInfo(%q) version = %q, want stable %q", path, ver, "1.0.0")
}
filename := path
if i := strings.LastIndex(filename, "/"); i >= 0 {
filename = filename[i+1:]
}
if seenKeys[filename] {
t.Errorf("cache key %q duplicated across artifacts", filename)
}
seenKeys[filename] = true
}
if len(seenKeys) != len(paths) {
t.Errorf("expected %d distinct cache keys, got %d", len(paths), len(seenKeys))
}
}