mirror of
https://github.com/lukaszraczylo/gohoarder.git
synced 2026-06-15 00:51:22 +00:00
fixes
This commit is contained in:
@@ -0,0 +1,415 @@
|
||||
package filesystem
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/md5"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"github.com/lukaszraczylo/gohoarder/pkg/errors"
|
||||
"github.com/lukaszraczylo/gohoarder/pkg/metrics"
|
||||
"github.com/lukaszraczylo/gohoarder/pkg/storage"
|
||||
"github.com/rs/zerolog/log"
|
||||
)
|
||||
|
||||
// FilesystemStorage implements storage.StorageBackend for local filesystem
|
||||
type FilesystemStorage struct {
|
||||
basePath string
|
||||
quota int64
|
||||
mu sync.RWMutex
|
||||
used int64
|
||||
}
|
||||
|
||||
// New creates a new filesystem storage backend
|
||||
func New(basePath string, quota int64) (*FilesystemStorage, error) {
|
||||
// Create base directory if it doesn't exist
|
||||
if err := os.MkdirAll(basePath, 0755); err != nil {
|
||||
return nil, errors.Wrap(err, errors.ErrCodeStorageFailure, "failed to create base directory")
|
||||
}
|
||||
|
||||
fs := &FilesystemStorage{
|
||||
basePath: basePath,
|
||||
quota: quota,
|
||||
}
|
||||
|
||||
// Calculate initial usage
|
||||
if err := fs.calculateUsage(); err != nil {
|
||||
log.Warn().Err(err).Msg("Failed to calculate initial storage usage")
|
||||
}
|
||||
|
||||
return fs, nil
|
||||
}
|
||||
|
||||
// Get retrieves a file
|
||||
func (fs *FilesystemStorage) Get(ctx context.Context, key string) (io.ReadCloser, error) {
|
||||
// Check context
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil, ctx.Err()
|
||||
default:
|
||||
}
|
||||
|
||||
path := fs.keyToPath(key)
|
||||
|
||||
file, err := os.Open(path)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
metrics.RecordStorageOperation("filesystem", "get", "not_found")
|
||||
return nil, errors.NotFound(fmt.Sprintf("file not found: %s", key))
|
||||
}
|
||||
metrics.RecordStorageOperation("filesystem", "get", "error")
|
||||
return nil, errors.Wrap(err, errors.ErrCodeStorageFailure, "failed to open file")
|
||||
}
|
||||
|
||||
metrics.RecordStorageOperation("filesystem", "get", "success")
|
||||
return file, nil
|
||||
}
|
||||
|
||||
// Put stores a file atomically
|
||||
func (fs *FilesystemStorage) Put(ctx context.Context, key string, data io.Reader, opts *storage.PutOptions) error {
|
||||
// Check context
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
default:
|
||||
}
|
||||
|
||||
path := fs.keyToPath(key)
|
||||
dir := filepath.Dir(path)
|
||||
|
||||
// Create directory
|
||||
if err := os.MkdirAll(dir, 0755); err != nil {
|
||||
metrics.RecordStorageOperation("filesystem", "put", "error")
|
||||
return errors.Wrap(err, errors.ErrCodeStorageFailure, "failed to create directory")
|
||||
}
|
||||
|
||||
// Create temp file for atomic write
|
||||
tempPath := path + ".tmp"
|
||||
tempFile, err := os.Create(tempPath)
|
||||
if err != nil {
|
||||
metrics.RecordStorageOperation("filesystem", "put", "error")
|
||||
return errors.Wrap(err, errors.ErrCodeStorageFailure, "failed to create temp file")
|
||||
}
|
||||
|
||||
// Calculate checksums while writing
|
||||
// NOTE: MD5 is used for integrity verification (checksums), not cryptographic security
|
||||
md5Hash := md5.New()
|
||||
sha256Hash := sha256.New()
|
||||
multiWriter := io.MultiWriter(tempFile, md5Hash, sha256Hash)
|
||||
|
||||
written, err := io.Copy(multiWriter, data)
|
||||
if err != nil {
|
||||
tempFile.Close()
|
||||
os.Remove(tempPath)
|
||||
metrics.RecordStorageOperation("filesystem", "put", "error")
|
||||
return errors.Wrap(err, errors.ErrCodeStorageFailure, "failed to write data")
|
||||
}
|
||||
|
||||
if err := tempFile.Close(); err != nil {
|
||||
os.Remove(tempPath)
|
||||
metrics.RecordStorageOperation("filesystem", "put", "error")
|
||||
return errors.Wrap(err, errors.ErrCodeStorageFailure, "failed to close temp file")
|
||||
}
|
||||
|
||||
// Check quota
|
||||
fs.mu.Lock()
|
||||
if fs.quota > 0 && fs.used+written > fs.quota {
|
||||
fs.mu.Unlock()
|
||||
os.Remove(tempPath)
|
||||
metrics.RecordStorageOperation("filesystem", "put", "quota_exceeded")
|
||||
return errors.QuotaExceeded(fs.quota)
|
||||
}
|
||||
fs.used += written
|
||||
fs.mu.Unlock()
|
||||
|
||||
// Verify checksums if provided
|
||||
if opts != nil {
|
||||
md5Sum := hex.EncodeToString(md5Hash.Sum(nil))
|
||||
sha256Sum := hex.EncodeToString(sha256Hash.Sum(nil))
|
||||
|
||||
if opts.ChecksumMD5 != "" && opts.ChecksumMD5 != md5Sum {
|
||||
os.Remove(tempPath)
|
||||
metrics.RecordStorageOperation("filesystem", "put", "checksum_error")
|
||||
return errors.New(errors.ErrCodeChecksumMismatch, "MD5 checksum mismatch")
|
||||
}
|
||||
|
||||
if opts.ChecksumSHA256 != "" && opts.ChecksumSHA256 != sha256Sum {
|
||||
os.Remove(tempPath)
|
||||
metrics.RecordStorageOperation("filesystem", "put", "checksum_error")
|
||||
return errors.New(errors.ErrCodeChecksumMismatch, "SHA256 checksum mismatch")
|
||||
}
|
||||
}
|
||||
|
||||
// Atomic rename
|
||||
if err := os.Rename(tempPath, path); err != nil {
|
||||
os.Remove(tempPath)
|
||||
fs.mu.Lock()
|
||||
fs.used -= written
|
||||
currentUsed := fs.used
|
||||
fs.mu.Unlock()
|
||||
metrics.RecordStorageOperation("filesystem", "put", "error")
|
||||
metrics.UpdateCacheSize("filesystem", currentUsed)
|
||||
return errors.Wrap(err, errors.ErrCodeStorageFailure, "failed to rename temp file")
|
||||
}
|
||||
|
||||
fs.mu.RLock()
|
||||
currentUsed := fs.used
|
||||
fs.mu.RUnlock()
|
||||
|
||||
metrics.RecordStorageOperation("filesystem", "put", "success")
|
||||
metrics.UpdateCacheSize("filesystem", currentUsed)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Delete removes a file
|
||||
func (fs *FilesystemStorage) Delete(ctx context.Context, key string) error {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
default:
|
||||
}
|
||||
|
||||
path := fs.keyToPath(key)
|
||||
|
||||
// Get size before deletion
|
||||
info, err := os.Stat(path)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
metrics.RecordStorageOperation("filesystem", "delete", "not_found")
|
||||
return errors.NotFound(fmt.Sprintf("file not found: %s", key))
|
||||
}
|
||||
metrics.RecordStorageOperation("filesystem", "delete", "error")
|
||||
return errors.Wrap(err, errors.ErrCodeStorageFailure, "failed to stat file")
|
||||
}
|
||||
|
||||
size := info.Size()
|
||||
|
||||
if err := os.Remove(path); err != nil {
|
||||
metrics.RecordStorageOperation("filesystem", "delete", "error")
|
||||
return errors.Wrap(err, errors.ErrCodeStorageFailure, "failed to delete file")
|
||||
}
|
||||
|
||||
fs.mu.Lock()
|
||||
fs.used -= size
|
||||
currentUsed := fs.used
|
||||
fs.mu.Unlock()
|
||||
|
||||
metrics.RecordStorageOperation("filesystem", "delete", "success")
|
||||
metrics.UpdateCacheSize("filesystem", currentUsed)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Exists checks if a file exists
|
||||
func (fs *FilesystemStorage) Exists(ctx context.Context, key string) (bool, error) {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return false, ctx.Err()
|
||||
default:
|
||||
}
|
||||
|
||||
path := fs.keyToPath(key)
|
||||
_, err := os.Stat(path)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return false, nil
|
||||
}
|
||||
return false, errors.Wrap(err, errors.ErrCodeStorageFailure, "failed to check existence")
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// List lists files with prefix
|
||||
func (fs *FilesystemStorage) List(ctx context.Context, prefix string, opts *storage.ListOptions) ([]storage.StorageObject, error) {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil, ctx.Err()
|
||||
default:
|
||||
}
|
||||
|
||||
searchPath := fs.keyToPath(prefix)
|
||||
var objects []storage.StorageObject
|
||||
|
||||
err := filepath.Walk(searchPath, func(path string, info os.FileInfo, err error) error {
|
||||
if err != nil {
|
||||
return nil // Skip errors
|
||||
}
|
||||
|
||||
if info.IsDir() {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Convert path back to key
|
||||
relPath, _ := filepath.Rel(fs.basePath, path)
|
||||
key := filepath.ToSlash(relPath)
|
||||
|
||||
objects = append(objects, storage.StorageObject{
|
||||
Key: key,
|
||||
Size: info.Size(),
|
||||
Modified: info.ModTime(),
|
||||
})
|
||||
|
||||
return nil
|
||||
})
|
||||
|
||||
if err != nil && !os.IsNotExist(err) {
|
||||
return nil, errors.Wrap(err, errors.ErrCodeStorageFailure, "failed to list files")
|
||||
}
|
||||
|
||||
// Apply pagination if requested
|
||||
if opts != nil {
|
||||
start := opts.Offset
|
||||
end := len(objects)
|
||||
if opts.MaxResults > 0 && start+opts.MaxResults < end {
|
||||
end = start + opts.MaxResults
|
||||
}
|
||||
if start < len(objects) {
|
||||
objects = objects[start:end]
|
||||
}
|
||||
}
|
||||
|
||||
return objects, nil
|
||||
}
|
||||
|
||||
// Stat gets file metadata
|
||||
func (fs *FilesystemStorage) Stat(ctx context.Context, key string) (*storage.StorageInfo, error) {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil, ctx.Err()
|
||||
default:
|
||||
}
|
||||
|
||||
path := fs.keyToPath(key)
|
||||
info, err := os.Stat(path)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return nil, errors.NotFound(fmt.Sprintf("file not found: %s", key))
|
||||
}
|
||||
return nil, errors.Wrap(err, errors.ErrCodeStorageFailure, "failed to stat file")
|
||||
}
|
||||
|
||||
return &storage.StorageInfo{
|
||||
Key: key,
|
||||
Size: info.Size(),
|
||||
Modified: info.ModTime(),
|
||||
}, nil
|
||||
}
|
||||
|
||||
// GetQuota returns quota information
|
||||
func (fs *FilesystemStorage) GetQuota(ctx context.Context) (*storage.QuotaInfo, error) {
|
||||
fs.mu.RLock()
|
||||
used := fs.used
|
||||
fs.mu.RUnlock()
|
||||
|
||||
available := fs.quota - used
|
||||
if available < 0 {
|
||||
available = 0
|
||||
}
|
||||
|
||||
return &storage.QuotaInfo{
|
||||
Used: used,
|
||||
Available: available,
|
||||
Limit: fs.quota,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Health checks filesystem health
|
||||
func (fs *FilesystemStorage) Health(ctx context.Context) error {
|
||||
// Check if base path is accessible
|
||||
if _, err := os.Stat(fs.basePath); err != nil {
|
||||
return errors.Wrap(err, errors.ErrCodeStorageFailure, "base path not accessible")
|
||||
}
|
||||
|
||||
// Try to create a temp file (sanitize path to prevent traversal)
|
||||
tempPath := filepath.Clean(filepath.Join(fs.basePath, ".health_check"))
|
||||
f, err := os.Create(tempPath)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, errors.ErrCodeStorageFailure, "cannot write to storage")
|
||||
}
|
||||
f.Close()
|
||||
os.Remove(tempPath)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Close closes the storage backend
|
||||
func (fs *FilesystemStorage) Close() error {
|
||||
// Nothing to close for filesystem
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetLocalPath returns the local filesystem path for a storage key
|
||||
// This implements storage.LocalPathProvider interface
|
||||
func (fs *FilesystemStorage) GetLocalPath(ctx context.Context, key string) (string, error) {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return "", ctx.Err()
|
||||
default:
|
||||
}
|
||||
|
||||
path := fs.keyToPath(key)
|
||||
|
||||
// Verify file exists
|
||||
if _, err := os.Stat(path); err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return "", errors.NotFound(fmt.Sprintf("file not found: %s", key))
|
||||
}
|
||||
return "", errors.Wrap(err, errors.ErrCodeStorageFailure, "failed to stat file")
|
||||
}
|
||||
|
||||
return path, nil
|
||||
}
|
||||
|
||||
// keyToPath converts a storage key to filesystem path
|
||||
func (fs *FilesystemStorage) keyToPath(key string) string {
|
||||
// Sanitize key to prevent path traversal
|
||||
key = filepath.Clean(key)
|
||||
|
||||
// Remove any leading slashes or dots
|
||||
key = strings.TrimPrefix(key, "/")
|
||||
|
||||
// Keep removing ../ until there are no more
|
||||
for strings.HasPrefix(key, "../") || strings.HasPrefix(key, "..\\") {
|
||||
key = strings.TrimPrefix(key, "../")
|
||||
key = strings.TrimPrefix(key, "..\\")
|
||||
}
|
||||
|
||||
// Final clean and ensure it's within base path
|
||||
key = filepath.Clean(key)
|
||||
if key == ".." || strings.HasPrefix(key, "../") || strings.HasPrefix(key, "..\\") {
|
||||
key = ""
|
||||
}
|
||||
|
||||
return filepath.Join(fs.basePath, key)
|
||||
}
|
||||
|
||||
// calculateUsage calculates current storage usage
|
||||
func (fs *FilesystemStorage) calculateUsage() error {
|
||||
var total int64
|
||||
|
||||
err := filepath.Walk(fs.basePath, func(path string, info os.FileInfo, err error) error {
|
||||
if err != nil {
|
||||
return nil // Skip errors
|
||||
}
|
||||
if !info.IsDir() {
|
||||
total += info.Size()
|
||||
}
|
||||
return nil
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
fs.mu.Lock()
|
||||
fs.used = total
|
||||
fs.mu.Unlock()
|
||||
|
||||
metrics.UpdateCacheSize("filesystem", total)
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,757 @@
|
||||
package filesystem
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/lukaszraczylo/gohoarder/pkg/storage"
|
||||
"github.com/stretchr/testify/suite"
|
||||
)
|
||||
|
||||
type FilesystemStorageTestSuite struct {
|
||||
suite.Suite
|
||||
tempDir string
|
||||
fs *FilesystemStorage
|
||||
}
|
||||
|
||||
func (s *FilesystemStorageTestSuite) SetupTest() {
|
||||
var err error
|
||||
s.tempDir, err = os.MkdirTemp("", "gohoarder-test-*")
|
||||
s.Require().NoError(err)
|
||||
|
||||
s.fs, err = New(s.tempDir, 1024*1024) // 1MB quota
|
||||
s.Require().NoError(err)
|
||||
}
|
||||
|
||||
func (s *FilesystemStorageTestSuite) TearDownTest() {
|
||||
if s.fs != nil {
|
||||
s.fs.Close()
|
||||
}
|
||||
if s.tempDir != "" {
|
||||
os.RemoveAll(s.tempDir)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFilesystemStorageTestSuite(t *testing.T) {
|
||||
suite.Run(t, new(FilesystemStorageTestSuite))
|
||||
}
|
||||
|
||||
// Test Put operation
|
||||
func (s *FilesystemStorageTestSuite) TestPut() {
|
||||
tests := []struct {
|
||||
name string
|
||||
key string
|
||||
data string
|
||||
opts *storage.PutOptions
|
||||
expectError bool
|
||||
errorCheck func(error) bool
|
||||
}{
|
||||
{
|
||||
name: "successful put",
|
||||
key: "test/file.txt",
|
||||
data: "hello world",
|
||||
opts: nil,
|
||||
expectError: false,
|
||||
},
|
||||
{
|
||||
name: "put with valid MD5 checksum",
|
||||
key: "test/checksummed.txt",
|
||||
data: "test data",
|
||||
opts: &storage.PutOptions{ChecksumMD5: "eb733a00c0c9d336e65691a37ab54293"},
|
||||
expectError: false,
|
||||
},
|
||||
{
|
||||
name: "put with invalid MD5 checksum",
|
||||
key: "test/bad-checksum.txt",
|
||||
data: "test data",
|
||||
opts: &storage.PutOptions{ChecksumMD5: "invalid"},
|
||||
expectError: true,
|
||||
},
|
||||
{
|
||||
name: "put with nested path",
|
||||
key: "deep/nested/path/file.txt",
|
||||
data: "nested content",
|
||||
opts: nil,
|
||||
expectError: false,
|
||||
},
|
||||
{
|
||||
name: "put with path traversal attempt",
|
||||
key: "../../../etc/passwd",
|
||||
data: "malicious",
|
||||
opts: nil,
|
||||
expectError: false, // Should be sanitized, not error
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
s.Run(tt.name, func() {
|
||||
ctx := context.Background()
|
||||
reader := strings.NewReader(tt.data)
|
||||
|
||||
err := s.fs.Put(ctx, tt.key, reader, tt.opts)
|
||||
|
||||
if tt.expectError {
|
||||
s.Error(err)
|
||||
} else {
|
||||
s.NoError(err)
|
||||
// Verify file exists
|
||||
exists, err := s.fs.Exists(ctx, tt.key)
|
||||
s.NoError(err)
|
||||
s.True(exists)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Test Get operation
|
||||
func (s *FilesystemStorageTestSuite) TestGet() {
|
||||
ctx := context.Background()
|
||||
|
||||
// Setup: Put a test file
|
||||
testData := "test content for retrieval"
|
||||
err := s.fs.Put(ctx, "test/get.txt", strings.NewReader(testData), nil)
|
||||
s.Require().NoError(err)
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
key string
|
||||
expectError bool
|
||||
expectData string
|
||||
}{
|
||||
{
|
||||
name: "get existing file",
|
||||
key: "test/get.txt",
|
||||
expectError: false,
|
||||
expectData: testData,
|
||||
},
|
||||
{
|
||||
name: "get non-existent file",
|
||||
key: "does/not/exist.txt",
|
||||
expectError: true,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
s.Run(tt.name, func() {
|
||||
reader, err := s.fs.Get(ctx, tt.key)
|
||||
|
||||
if tt.expectError {
|
||||
s.Error(err)
|
||||
s.Nil(reader)
|
||||
} else {
|
||||
s.NoError(err)
|
||||
s.NotNil(reader)
|
||||
defer reader.Close()
|
||||
|
||||
data, err := io.ReadAll(reader)
|
||||
s.NoError(err)
|
||||
s.Equal(tt.expectData, string(data))
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Test Delete operation
|
||||
func (s *FilesystemStorageTestSuite) TestDelete() {
|
||||
ctx := context.Background()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
setupKey string
|
||||
deleteKey string
|
||||
expectError bool
|
||||
}{
|
||||
{
|
||||
name: "delete existing file",
|
||||
setupKey: "test/delete-me.txt",
|
||||
deleteKey: "test/delete-me.txt",
|
||||
expectError: false,
|
||||
},
|
||||
{
|
||||
name: "delete non-existent file",
|
||||
setupKey: "",
|
||||
deleteKey: "does/not/exist.txt",
|
||||
expectError: true,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
s.Run(tt.name, func() {
|
||||
// Setup
|
||||
if tt.setupKey != "" {
|
||||
err := s.fs.Put(ctx, tt.setupKey, strings.NewReader("to be deleted"), nil)
|
||||
s.Require().NoError(err)
|
||||
}
|
||||
|
||||
// Test delete
|
||||
err := s.fs.Delete(ctx, tt.deleteKey)
|
||||
|
||||
if tt.expectError {
|
||||
s.Error(err)
|
||||
} else {
|
||||
s.NoError(err)
|
||||
// Verify file no longer exists
|
||||
exists, err := s.fs.Exists(ctx, tt.deleteKey)
|
||||
s.NoError(err)
|
||||
s.False(exists)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Test Exists operation
|
||||
func (s *FilesystemStorageTestSuite) TestExists() {
|
||||
ctx := context.Background()
|
||||
|
||||
// Setup: Put a test file
|
||||
err := s.fs.Put(ctx, "test/exists.txt", strings.NewReader("content"), nil)
|
||||
s.Require().NoError(err)
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
key string
|
||||
exists bool
|
||||
}{
|
||||
{
|
||||
name: "existing file",
|
||||
key: "test/exists.txt",
|
||||
exists: true,
|
||||
},
|
||||
{
|
||||
name: "non-existent file",
|
||||
key: "test/does-not-exist.txt",
|
||||
exists: false,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
s.Run(tt.name, func() {
|
||||
exists, err := s.fs.Exists(ctx, tt.key)
|
||||
s.NoError(err)
|
||||
s.Equal(tt.exists, exists)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Test List operation
|
||||
func (s *FilesystemStorageTestSuite) TestList() {
|
||||
ctx := context.Background()
|
||||
|
||||
// Setup: Create multiple files
|
||||
files := []string{
|
||||
"packages/npm/react/17.0.1/package.json",
|
||||
"packages/npm/react/17.0.2/package.json",
|
||||
"packages/npm/vue/3.0.0/package.json",
|
||||
"packages/pypi/django/3.2.0/wheel.whl",
|
||||
}
|
||||
|
||||
for _, file := range files {
|
||||
err := s.fs.Put(ctx, file, strings.NewReader("content"), nil)
|
||||
s.Require().NoError(err)
|
||||
}
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
prefix string
|
||||
opts *storage.ListOptions
|
||||
expectedCount int
|
||||
expectedKeys []string
|
||||
}{
|
||||
{
|
||||
name: "list all npm packages",
|
||||
prefix: "packages/npm",
|
||||
opts: nil,
|
||||
expectedCount: 3,
|
||||
},
|
||||
{
|
||||
name: "list react packages",
|
||||
prefix: "packages/npm/react",
|
||||
opts: nil,
|
||||
expectedCount: 2,
|
||||
},
|
||||
{
|
||||
name: "list with pagination",
|
||||
prefix: "packages/npm",
|
||||
opts: &storage.ListOptions{MaxResults: 2, Offset: 0},
|
||||
expectedCount: 2,
|
||||
},
|
||||
{
|
||||
name: "list with offset",
|
||||
prefix: "packages/npm",
|
||||
opts: &storage.ListOptions{MaxResults: 2, Offset: 1},
|
||||
expectedCount: 2,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
s.Run(tt.name, func() {
|
||||
objects, err := s.fs.List(ctx, tt.prefix, tt.opts)
|
||||
s.NoError(err)
|
||||
s.Equal(tt.expectedCount, len(objects))
|
||||
|
||||
// Verify objects have required fields
|
||||
for _, obj := range objects {
|
||||
s.NotEmpty(obj.Key)
|
||||
s.Greater(obj.Size, int64(0))
|
||||
s.False(obj.Modified.IsZero())
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Test Stat operation
|
||||
func (s *FilesystemStorageTestSuite) TestStat() {
|
||||
ctx := context.Background()
|
||||
|
||||
// Setup: Put a test file
|
||||
testData := "stat test content"
|
||||
testKey := "test/stat.txt"
|
||||
err := s.fs.Put(ctx, testKey, strings.NewReader(testData), nil)
|
||||
s.Require().NoError(err)
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
key string
|
||||
expectError bool
|
||||
}{
|
||||
{
|
||||
name: "stat existing file",
|
||||
key: testKey,
|
||||
expectError: false,
|
||||
},
|
||||
{
|
||||
name: "stat non-existent file",
|
||||
key: "does/not/exist.txt",
|
||||
expectError: true,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
s.Run(tt.name, func() {
|
||||
info, err := s.fs.Stat(ctx, tt.key)
|
||||
|
||||
if tt.expectError {
|
||||
s.Error(err)
|
||||
s.Nil(info)
|
||||
} else {
|
||||
s.NoError(err)
|
||||
s.NotNil(info)
|
||||
s.Equal(tt.key, info.Key)
|
||||
s.Equal(int64(len(testData)), info.Size)
|
||||
s.False(info.Modified.IsZero())
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Test Quota enforcement
|
||||
func (s *FilesystemStorageTestSuite) TestQuotaEnforcement() {
|
||||
ctx := context.Background()
|
||||
|
||||
// Create a new filesystem with small quota (100 bytes)
|
||||
smallQuotaDir, err := os.MkdirTemp("", "gohoarder-quota-*")
|
||||
s.Require().NoError(err)
|
||||
defer os.RemoveAll(smallQuotaDir)
|
||||
|
||||
smallFs, err := New(smallQuotaDir, 100)
|
||||
s.Require().NoError(err)
|
||||
defer smallFs.Close()
|
||||
|
||||
// First write should succeed
|
||||
err = smallFs.Put(ctx, "file1.txt", strings.NewReader("small content"), nil)
|
||||
s.NoError(err)
|
||||
|
||||
// Large write should fail due to quota
|
||||
largeData := strings.Repeat("x", 200)
|
||||
err = smallFs.Put(ctx, "large.txt", strings.NewReader(largeData), nil)
|
||||
s.Error(err)
|
||||
|
||||
// Verify quota info
|
||||
quotaInfo, err := smallFs.GetQuota(ctx)
|
||||
s.NoError(err)
|
||||
s.Equal(int64(100), quotaInfo.Limit)
|
||||
s.Greater(quotaInfo.Used, int64(0))
|
||||
s.LessOrEqual(quotaInfo.Used, quotaInfo.Limit)
|
||||
}
|
||||
|
||||
// Test GetQuota operation
|
||||
func (s *FilesystemStorageTestSuite) TestGetQuota() {
|
||||
ctx := context.Background()
|
||||
|
||||
// Put some files
|
||||
err := s.fs.Put(ctx, "file1.txt", strings.NewReader("content1"), nil)
|
||||
s.Require().NoError(err)
|
||||
err = s.fs.Put(ctx, "file2.txt", strings.NewReader("content2"), nil)
|
||||
s.Require().NoError(err)
|
||||
|
||||
quotaInfo, err := s.fs.GetQuota(ctx)
|
||||
s.NoError(err)
|
||||
s.NotNil(quotaInfo)
|
||||
s.Equal(int64(1024*1024), quotaInfo.Limit)
|
||||
s.Greater(quotaInfo.Used, int64(0))
|
||||
s.Greater(quotaInfo.Available, int64(0))
|
||||
s.Equal(quotaInfo.Limit, quotaInfo.Used+quotaInfo.Available)
|
||||
}
|
||||
|
||||
// Test Health check
|
||||
func (s *FilesystemStorageTestSuite) TestHealth() {
|
||||
ctx := context.Background()
|
||||
|
||||
// Healthy filesystem
|
||||
err := s.fs.Health(ctx)
|
||||
s.NoError(err)
|
||||
|
||||
// Unhealthy filesystem (removed directory)
|
||||
badDir := filepath.Join(s.tempDir, "nonexistent")
|
||||
badFs := &FilesystemStorage{basePath: badDir}
|
||||
err = badFs.Health(ctx)
|
||||
s.Error(err)
|
||||
}
|
||||
|
||||
// Test Context cancellation
|
||||
func (s *FilesystemStorageTestSuite) TestContextCancellation() {
|
||||
// Create cancelled context
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
fn func() error
|
||||
}{
|
||||
{
|
||||
name: "Get with cancelled context",
|
||||
fn: func() error {
|
||||
_, err := s.fs.Get(ctx, "test.txt")
|
||||
return err
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "Put with cancelled context",
|
||||
fn: func() error {
|
||||
return s.fs.Put(ctx, "test.txt", strings.NewReader("data"), nil)
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "Delete with cancelled context",
|
||||
fn: func() error {
|
||||
return s.fs.Delete(ctx, "test.txt")
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "Exists with cancelled context",
|
||||
fn: func() error {
|
||||
_, err := s.fs.Exists(ctx, "test.txt")
|
||||
return err
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "List with cancelled context",
|
||||
fn: func() error {
|
||||
_, err := s.fs.List(ctx, "test", nil)
|
||||
return err
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "Stat with cancelled context",
|
||||
fn: func() error {
|
||||
_, err := s.fs.Stat(ctx, "test.txt")
|
||||
return err
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
s.Run(tt.name, func() {
|
||||
err := tt.fn()
|
||||
s.Error(err)
|
||||
s.Equal(context.Canceled, err)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Test concurrent access (race condition testing)
|
||||
func (s *FilesystemStorageTestSuite) TestConcurrentAccess() {
|
||||
ctx := context.Background()
|
||||
numGoroutines := 10
|
||||
numOperations := 100
|
||||
|
||||
var wg sync.WaitGroup
|
||||
|
||||
// Concurrent writes
|
||||
for i := 0; i < numGoroutines; i++ {
|
||||
wg.Add(1)
|
||||
go func(id int) {
|
||||
defer wg.Done()
|
||||
for j := 0; j < numOperations; j++ {
|
||||
key := fmt.Sprintf("concurrent/%d/%d.txt", id, j)
|
||||
data := fmt.Sprintf("data-%d-%d", id, j)
|
||||
err := s.fs.Put(ctx, key, strings.NewReader(data), nil)
|
||||
s.NoError(err)
|
||||
}
|
||||
}(i)
|
||||
}
|
||||
|
||||
wg.Wait()
|
||||
|
||||
// Verify all files exist
|
||||
objects, err := s.fs.List(ctx, "concurrent", nil)
|
||||
s.NoError(err)
|
||||
s.Equal(numGoroutines*numOperations, len(objects))
|
||||
}
|
||||
|
||||
// Test concurrent reads and writes
|
||||
func (s *FilesystemStorageTestSuite) TestConcurrentReadsAndWrites() {
|
||||
ctx := context.Background()
|
||||
|
||||
// Setup: Create some initial files
|
||||
for i := 0; i < 10; i++ {
|
||||
key := fmt.Sprintf("shared/file-%d.txt", i)
|
||||
err := s.fs.Put(ctx, key, strings.NewReader(fmt.Sprintf("initial-%d", i)), nil)
|
||||
s.Require().NoError(err)
|
||||
}
|
||||
|
||||
var wg sync.WaitGroup
|
||||
numReaders := 5
|
||||
numWriters := 5
|
||||
numOps := 50
|
||||
|
||||
// Concurrent readers
|
||||
for i := 0; i < numReaders; i++ {
|
||||
wg.Add(1)
|
||||
go func(id int) {
|
||||
defer wg.Done()
|
||||
for j := 0; j < numOps; j++ {
|
||||
key := fmt.Sprintf("shared/file-%d.txt", j%10)
|
||||
reader, err := s.fs.Get(ctx, key)
|
||||
if err == nil {
|
||||
io.ReadAll(reader)
|
||||
reader.Close()
|
||||
}
|
||||
}
|
||||
}(i)
|
||||
}
|
||||
|
||||
// Concurrent writers
|
||||
for i := 0; i < numWriters; i++ {
|
||||
wg.Add(1)
|
||||
go func(id int) {
|
||||
defer wg.Done()
|
||||
for j := 0; j < numOps; j++ {
|
||||
key := fmt.Sprintf("shared/writer-%d-%d.txt", id, j)
|
||||
data := fmt.Sprintf("writer-%d-%d", id, j)
|
||||
s.fs.Put(ctx, key, strings.NewReader(data), nil)
|
||||
}
|
||||
}(i)
|
||||
}
|
||||
|
||||
wg.Wait()
|
||||
|
||||
// Verify quota tracking is consistent
|
||||
quotaInfo, err := s.fs.GetQuota(ctx)
|
||||
s.NoError(err)
|
||||
s.Greater(quotaInfo.Used, int64(0))
|
||||
}
|
||||
|
||||
// Test Delete updates quota correctly
|
||||
func (s *FilesystemStorageTestSuite) TestDeleteUpdatesQuota() {
|
||||
ctx := context.Background()
|
||||
|
||||
// Put a file
|
||||
testData := "test data for quota tracking"
|
||||
err := s.fs.Put(ctx, "quota/test.txt", strings.NewReader(testData), nil)
|
||||
s.Require().NoError(err)
|
||||
|
||||
// Get quota before delete
|
||||
quotaBefore, err := s.fs.GetQuota(ctx)
|
||||
s.Require().NoError(err)
|
||||
|
||||
// Delete the file
|
||||
err = s.fs.Delete(ctx, "quota/test.txt")
|
||||
s.NoError(err)
|
||||
|
||||
// Get quota after delete
|
||||
quotaAfter, err := s.fs.GetQuota(ctx)
|
||||
s.NoError(err)
|
||||
|
||||
// Quota should have decreased
|
||||
s.Less(quotaAfter.Used, quotaBefore.Used)
|
||||
}
|
||||
|
||||
// Test atomic write behavior
|
||||
func (s *FilesystemStorageTestSuite) TestAtomicWrite() {
|
||||
ctx := context.Background()
|
||||
key := "atomic/test.txt"
|
||||
|
||||
// Initial write
|
||||
err := s.fs.Put(ctx, key, strings.NewReader("initial"), nil)
|
||||
s.Require().NoError(err)
|
||||
|
||||
// Concurrent readers should never see partial writes
|
||||
var wg sync.WaitGroup
|
||||
stopReading := make(chan struct{})
|
||||
readErrors := make(chan error, 100)
|
||||
|
||||
// Start readers
|
||||
for i := 0; i < 5; i++ {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
for {
|
||||
select {
|
||||
case <-stopReading:
|
||||
return
|
||||
default:
|
||||
reader, err := s.fs.Get(ctx, key)
|
||||
if err != nil {
|
||||
readErrors <- err
|
||||
continue
|
||||
}
|
||||
data, err := io.ReadAll(reader)
|
||||
reader.Close()
|
||||
if err != nil {
|
||||
readErrors <- err
|
||||
continue
|
||||
}
|
||||
// Data should be either "initial" or "updated", never partial
|
||||
content := string(data)
|
||||
if content != "initial" && content != "updated" {
|
||||
readErrors <- fmt.Errorf("read partial data: %s", content)
|
||||
}
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
// Perform update
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
err = s.fs.Put(ctx, key, strings.NewReader("updated"), nil)
|
||||
s.NoError(err)
|
||||
|
||||
// Stop readers
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
close(stopReading)
|
||||
wg.Wait()
|
||||
close(readErrors)
|
||||
|
||||
// Check for read errors
|
||||
for err := range readErrors {
|
||||
s.NoError(err)
|
||||
}
|
||||
}
|
||||
|
||||
// Test path sanitization
|
||||
func (s *FilesystemStorageTestSuite) TestPathSanitization() {
|
||||
ctx := context.Background()
|
||||
|
||||
maliciousPaths := []string{
|
||||
"../../../etc/passwd",
|
||||
"/../secret.txt",
|
||||
"./../../outside.txt",
|
||||
"//etc/passwd",
|
||||
}
|
||||
|
||||
for _, path := range maliciousPaths {
|
||||
s.Run(fmt.Sprintf("sanitize_%s", path), func() {
|
||||
err := s.fs.Put(ctx, path, strings.NewReader("malicious"), nil)
|
||||
s.NoError(err) // Should succeed but sanitize path
|
||||
|
||||
// Verify file is inside base directory
|
||||
sanitized := s.fs.keyToPath(path)
|
||||
s.True(strings.HasPrefix(sanitized, s.tempDir),
|
||||
"Sanitized path %s should be inside %s", sanitized, s.tempDir)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Test checksum validation
|
||||
func (s *FilesystemStorageTestSuite) TestChecksumValidation() {
|
||||
ctx := context.Background()
|
||||
|
||||
testData := "checksum test data"
|
||||
// Correct checksums calculated for "checksum test data"
|
||||
correctMD5 := "7dd7323e8ce3e087972f93d3711ef62b"
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
opts *storage.PutOptions
|
||||
expectError bool
|
||||
}{
|
||||
{
|
||||
name: "valid MD5",
|
||||
opts: &storage.PutOptions{ChecksumMD5: correctMD5},
|
||||
expectError: false,
|
||||
},
|
||||
{
|
||||
name: "invalid MD5",
|
||||
opts: &storage.PutOptions{ChecksumMD5: "invalid"},
|
||||
expectError: true,
|
||||
},
|
||||
{
|
||||
name: "empty checksum (no validation)",
|
||||
opts: &storage.PutOptions{ChecksumMD5: ""},
|
||||
expectError: false,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
s.Run(tt.name, func() {
|
||||
key := fmt.Sprintf("checksum/%s.txt", tt.name)
|
||||
err := s.fs.Put(ctx, key, strings.NewReader(testData), tt.opts)
|
||||
|
||||
if tt.expectError {
|
||||
s.Error(err)
|
||||
} else {
|
||||
s.NoError(err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Benchmark Put operation
|
||||
func BenchmarkFilesystemPut(b *testing.B) {
|
||||
tempDir, _ := os.MkdirTemp("", "gohoarder-bench-*")
|
||||
defer os.RemoveAll(tempDir)
|
||||
|
||||
fs, _ := New(tempDir, 1024*1024*1024) // 1GB quota
|
||||
defer fs.Close()
|
||||
|
||||
ctx := context.Background()
|
||||
data := strings.Repeat("x", 1024) // 1KB
|
||||
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
key := fmt.Sprintf("bench/file-%d.txt", i)
|
||||
fs.Put(ctx, key, strings.NewReader(data), nil)
|
||||
}
|
||||
}
|
||||
|
||||
// Benchmark Get operation
|
||||
func BenchmarkFilesystemGet(b *testing.B) {
|
||||
tempDir, _ := os.MkdirTemp("", "gohoarder-bench-*")
|
||||
defer os.RemoveAll(tempDir)
|
||||
|
||||
fs, _ := New(tempDir, 1024*1024*1024)
|
||||
defer fs.Close()
|
||||
|
||||
ctx := context.Background()
|
||||
data := strings.Repeat("x", 1024)
|
||||
|
||||
// Setup: Create test file
|
||||
fs.Put(ctx, "bench/test.txt", strings.NewReader(data), nil)
|
||||
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
reader, _ := fs.Get(ctx, "bench/test.txt")
|
||||
if reader != nil {
|
||||
io.ReadAll(reader)
|
||||
reader.Close()
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user