mirror of
https://github.com/lukaszraczylo/gohoarder.git
synced 2026-07-06 04:04:57 +00:00
refactor: reorganize struct fields, add new handlers and storage backends
- [x] Reorder struct fields across codebase for consistency - [x] Add analytics event handlers and tests - [x] Add authentication API key management handlers and tests - [x] Add pre-warming control handlers and tests - [x] Implement S3 storage backend with tests - [x] Implement SMB/CIFS storage backend with tests - [x] Add CDN middleware tests - [x] Integrate analytics tracking into cache manager - [x] Add S3 and SMB storage initialization in app setup - [x] Add CDN caching to proxy handlers - [x] Remove distributed locking (Redis lock manager) - [x] Remove proxy common package and utilities - [x] Remove standalone HTTP server package - [x] Remove logger middleware - [x] Simplify error handling utilities - [x] Update config with S3 and SMB options - [x] Update cache manager signature to include analytics
This commit is contained in:
@@ -17,8 +17,8 @@ import (
|
||||
|
||||
type FilesystemStorageTestSuite struct {
|
||||
suite.Suite
|
||||
tempDir string
|
||||
fs *FilesystemStorage
|
||||
tempDir string
|
||||
}
|
||||
|
||||
func (s *FilesystemStorageTestSuite) SetupTest() {
|
||||
@@ -46,12 +46,12 @@ func TestFilesystemStorageTestSuite(t *testing.T) {
|
||||
// Test Put operation
|
||||
func (s *FilesystemStorageTestSuite) TestPut() {
|
||||
tests := []struct {
|
||||
opts *storage.PutOptions
|
||||
errorCheck func(error) bool
|
||||
name string
|
||||
key string
|
||||
data string
|
||||
opts *storage.PutOptions
|
||||
expectError bool
|
||||
errorCheck func(error) bool
|
||||
}{
|
||||
{
|
||||
name: "successful put",
|
||||
@@ -122,8 +122,8 @@ func (s *FilesystemStorageTestSuite) TestGet() {
|
||||
tests := []struct {
|
||||
name string
|
||||
key string
|
||||
expectError bool
|
||||
expectData string
|
||||
expectError bool
|
||||
}{
|
||||
{
|
||||
name: "get existing file",
|
||||
@@ -258,11 +258,11 @@ func (s *FilesystemStorageTestSuite) TestList() {
|
||||
}
|
||||
|
||||
tests := []struct {
|
||||
opts *storage.ListOptions
|
||||
name string
|
||||
prefix string
|
||||
opts *storage.ListOptions
|
||||
expectedCount int
|
||||
expectedKeys []string
|
||||
expectedCount int
|
||||
}{
|
||||
{
|
||||
name: "list all npm packages",
|
||||
@@ -422,8 +422,8 @@ func (s *FilesystemStorageTestSuite) TestContextCancellation() {
|
||||
cancel()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
fn func() error
|
||||
name string
|
||||
}{
|
||||
{
|
||||
name: "Get with cancelled context",
|
||||
@@ -679,8 +679,8 @@ func (s *FilesystemStorageTestSuite) TestChecksumValidation() {
|
||||
correctMD5 := "7dd7323e8ce3e087972f93d3711ef62b"
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
opts *storage.PutOptions
|
||||
name string
|
||||
expectError bool
|
||||
}{
|
||||
{
|
||||
|
||||
@@ -52,21 +52,21 @@ type ListOptions struct {
|
||||
|
||||
// StorageObject represents a stored object
|
||||
type StorageObject struct {
|
||||
Key string
|
||||
Size int64
|
||||
Modified time.Time
|
||||
Key string
|
||||
ETag string
|
||||
Size int64
|
||||
}
|
||||
|
||||
// StorageInfo contains detailed object information
|
||||
type StorageInfo struct {
|
||||
Key string
|
||||
Size int64
|
||||
Modified time.Time
|
||||
ETag string
|
||||
ContentType string
|
||||
Metadata map[string]string
|
||||
Checksums *Checksums
|
||||
Key string
|
||||
ETag string
|
||||
ContentType string
|
||||
Size int64
|
||||
}
|
||||
|
||||
// Checksums contains file checksums
|
||||
|
||||
+189
-244
@@ -3,14 +3,10 @@ package s3
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/md5" // #nosec G501 -- MD5 used for S3 Content-MD5 header, not cryptographic security
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
stderrors "errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
"github.com/aws/aws-sdk-go-v2/config"
|
||||
@@ -18,261 +14,210 @@ import (
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3/types"
|
||||
"github.com/lukaszraczylo/gohoarder/pkg/errors"
|
||||
"github.com/lukaszraczylo/gohoarder/pkg/metrics"
|
||||
"github.com/lukaszraczylo/gohoarder/pkg/storage"
|
||||
"github.com/rs/zerolog/log"
|
||||
)
|
||||
|
||||
// S3Storage implements storage.StorageBackend for AWS S3
|
||||
type S3Storage struct {
|
||||
client *s3.Client
|
||||
bucket string
|
||||
prefix string
|
||||
quota int64
|
||||
mu sync.RWMutex
|
||||
used int64
|
||||
}
|
||||
|
||||
// Config holds S3 configuration
|
||||
// Config holds S3 storage configuration
|
||||
type Config struct {
|
||||
Bucket string
|
||||
Region string
|
||||
Endpoint string // For S3-compatible services (MinIO, etc.)
|
||||
Bucket string
|
||||
Prefix string
|
||||
AccessKeyID string
|
||||
SecretAccessKey string
|
||||
Prefix string // Optional prefix for all keys
|
||||
Quota int64 // Quota in bytes (0 = unlimited)
|
||||
ForcePathStyle bool // For S3-compatible services
|
||||
Endpoint string // Optional: for S3-compatible services like MinIO
|
||||
ForcePathStyle bool // Optional: for S3-compatible services
|
||||
MaxSizeBytes int64
|
||||
}
|
||||
|
||||
// S3Storage implements storage.StorageBackend using AWS S3
|
||||
type S3Storage struct {
|
||||
client *s3.Client
|
||||
bucket string
|
||||
prefix string
|
||||
maxSizeBytes int64
|
||||
}
|
||||
|
||||
// New creates a new S3 storage backend
|
||||
func New(ctx context.Context, cfg Config) (*S3Storage, error) {
|
||||
func New(cfg Config) (*S3Storage, error) {
|
||||
if cfg.Bucket == "" {
|
||||
return nil, errors.New(errors.ErrCodeInvalidConfig, "S3 bucket is required")
|
||||
return nil, fmt.Errorf("S3 bucket is required")
|
||||
}
|
||||
|
||||
if cfg.Region == "" {
|
||||
return nil, errors.New(errors.ErrCodeInvalidConfig, "S3 region is required")
|
||||
cfg.Region = "us-east-1" // Default region
|
||||
}
|
||||
|
||||
// Build AWS config
|
||||
var awsCfg aws.Config
|
||||
var awsConfig aws.Config
|
||||
var err error
|
||||
|
||||
// Build config options
|
||||
configOpts := []func(*config.LoadOptions) error{
|
||||
config.WithRegion(cfg.Region),
|
||||
}
|
||||
|
||||
// Add credentials if provided
|
||||
if cfg.AccessKeyID != "" && cfg.SecretAccessKey != "" {
|
||||
// Use static credentials
|
||||
awsCfg, err = config.LoadDefaultConfig(ctx,
|
||||
config.WithRegion(cfg.Region),
|
||||
config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(
|
||||
configOpts = append(configOpts, config.WithCredentialsProvider(
|
||||
credentials.NewStaticCredentialsProvider(
|
||||
cfg.AccessKeyID,
|
||||
cfg.SecretAccessKey,
|
||||
"",
|
||||
)),
|
||||
)
|
||||
} else {
|
||||
// Use default credential chain
|
||||
awsCfg, err = config.LoadDefaultConfig(ctx,
|
||||
config.WithRegion(cfg.Region),
|
||||
)
|
||||
),
|
||||
))
|
||||
}
|
||||
|
||||
awsConfig, err = config.LoadDefaultConfig(context.Background(), configOpts...)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, errors.ErrCodeStorageFailure, "failed to load AWS config")
|
||||
return nil, fmt.Errorf("failed to load AWS config: %w", err)
|
||||
}
|
||||
|
||||
// Create S3 client
|
||||
var s3Options []func(*s3.Options)
|
||||
|
||||
if cfg.Endpoint != "" {
|
||||
s3Options = append(s3Options, func(o *s3.Options) {
|
||||
// Create S3 client with service-specific options
|
||||
client := s3.NewFromConfig(awsConfig, func(o *s3.Options) {
|
||||
// Use custom endpoint if provided (for MinIO, S3-compatible services, etc.)
|
||||
if cfg.Endpoint != "" {
|
||||
o.BaseEndpoint = aws.String(cfg.Endpoint)
|
||||
o.UsePathStyle = cfg.ForcePathStyle
|
||||
})
|
||||
}
|
||||
if cfg.ForcePathStyle {
|
||||
o.UsePathStyle = true
|
||||
}
|
||||
})
|
||||
|
||||
storage := &S3Storage{
|
||||
client: client,
|
||||
bucket: cfg.Bucket,
|
||||
prefix: strings.TrimSuffix(cfg.Prefix, "/"),
|
||||
maxSizeBytes: cfg.MaxSizeBytes,
|
||||
}
|
||||
|
||||
client := s3.NewFromConfig(awsCfg, s3Options...)
|
||||
log.Info().
|
||||
Str("bucket", cfg.Bucket).
|
||||
Str("region", cfg.Region).
|
||||
Str("prefix", cfg.Prefix).
|
||||
Msg("S3 storage initialized")
|
||||
|
||||
s3Storage := &S3Storage{
|
||||
client: client,
|
||||
bucket: cfg.Bucket,
|
||||
prefix: strings.TrimSuffix(cfg.Prefix, "/"),
|
||||
quota: cfg.Quota,
|
||||
}
|
||||
|
||||
// Calculate initial usage
|
||||
if err := s3Storage.calculateUsage(ctx); err != nil {
|
||||
log.Warn().Err(err).Msg("Failed to calculate initial S3 storage usage")
|
||||
}
|
||||
|
||||
return s3Storage, nil
|
||||
return storage, nil
|
||||
}
|
||||
|
||||
// Get retrieves a file from S3
|
||||
// Get retrieves data from S3
|
||||
func (s *S3Storage) Get(ctx context.Context, key string) (io.ReadCloser, error) {
|
||||
s3Key := s.buildKey(key)
|
||||
fullKey := s.buildKey(key)
|
||||
|
||||
input := &s3.GetObjectInput{
|
||||
log.Debug().Str("key", fullKey).Msg("Getting object from S3")
|
||||
|
||||
result, err := s.client.GetObject(ctx, &s3.GetObjectInput{
|
||||
Bucket: aws.String(s.bucket),
|
||||
Key: aws.String(s3Key),
|
||||
}
|
||||
Key: aws.String(fullKey),
|
||||
})
|
||||
|
||||
result, err := s.client.GetObject(ctx, input)
|
||||
if err != nil {
|
||||
if isNotFoundError(err) {
|
||||
metrics.RecordStorageOperation("s3", "get", "not_found")
|
||||
return nil, errors.NotFound(fmt.Sprintf("file not found: %s", key))
|
||||
return nil, errors.NotFound(fmt.Sprintf("S3 object not found: %s", key))
|
||||
}
|
||||
metrics.RecordStorageOperation("s3", "get", "error")
|
||||
return nil, errors.Wrap(err, errors.ErrCodeStorageFailure, "failed to get object from S3")
|
||||
}
|
||||
|
||||
metrics.RecordStorageOperation("s3", "get", "success")
|
||||
return result.Body, nil
|
||||
}
|
||||
|
||||
// Put stores a file in S3
|
||||
// Put stores data in S3
|
||||
func (s *S3Storage) Put(ctx context.Context, key string, data io.Reader, opts *storage.PutOptions) error {
|
||||
s3Key := s.buildKey(key)
|
||||
fullKey := s.buildKey(key)
|
||||
|
||||
// Read data into buffer to calculate checksums and size
|
||||
var buf bytes.Buffer
|
||||
md5Hash := md5.New() // #nosec G401 -- MD5 used for S3 integrity check, not cryptographic security
|
||||
sha256Hash := sha256.New()
|
||||
multiWriter := io.MultiWriter(&buf, md5Hash, sha256Hash)
|
||||
|
||||
written, err := io.Copy(multiWriter, data)
|
||||
// Read data into buffer to get size
|
||||
buf := new(bytes.Buffer)
|
||||
size, err := io.Copy(buf, data)
|
||||
if err != nil {
|
||||
metrics.RecordStorageOperation("s3", "put", "error")
|
||||
return errors.Wrap(err, errors.ErrCodeStorageFailure, "failed to read data")
|
||||
return fmt.Errorf("failed to read data: %w", err)
|
||||
}
|
||||
|
||||
// Check quota before upload
|
||||
if s.quota > 0 {
|
||||
s.mu.RLock()
|
||||
used := s.used
|
||||
s.mu.RUnlock()
|
||||
log.Debug().
|
||||
Str("key", fullKey).
|
||||
Int64("size", size).
|
||||
Msg("Putting object to S3")
|
||||
|
||||
if used+written > s.quota {
|
||||
metrics.RecordStorageOperation("s3", "put", "quota_exceeded")
|
||||
return errors.QuotaExceeded(s.quota)
|
||||
// Check quota if set
|
||||
if s.maxSizeBytes > 0 {
|
||||
currentUsage, err := s.calculateUsage(ctx)
|
||||
if err != nil {
|
||||
log.Warn().Err(err).Msg("Failed to calculate current usage, skipping quota check")
|
||||
} else if currentUsage+size > s.maxSizeBytes {
|
||||
return errors.QuotaExceeded(s.maxSizeBytes)
|
||||
}
|
||||
}
|
||||
|
||||
// Verify checksums if provided
|
||||
if opts != nil {
|
||||
md5Sum := hex.EncodeToString(md5Hash.Sum(nil))
|
||||
sha256Sum := hex.EncodeToString(sha256Hash.Sum(nil))
|
||||
|
||||
if opts.ChecksumMD5 != "" && opts.ChecksumMD5 != md5Sum {
|
||||
metrics.RecordStorageOperation("s3", "put", "checksum_error")
|
||||
return errors.New(errors.ErrCodeChecksumMismatch, "MD5 checksum mismatch")
|
||||
}
|
||||
|
||||
if opts.ChecksumSHA256 != "" && opts.ChecksumSHA256 != sha256Sum {
|
||||
metrics.RecordStorageOperation("s3", "put", "checksum_error")
|
||||
return errors.New(errors.ErrCodeChecksumMismatch, "SHA256 checksum mismatch")
|
||||
}
|
||||
}
|
||||
|
||||
// Prepare metadata
|
||||
metadata := make(map[string]string)
|
||||
// Convert metadata to S3 metadata format
|
||||
s3Metadata := make(map[string]string)
|
||||
if opts != nil && opts.Metadata != nil {
|
||||
metadata = opts.Metadata
|
||||
}
|
||||
|
||||
// Build put input
|
||||
input := &s3.PutObjectInput{
|
||||
Bucket: aws.String(s.bucket),
|
||||
Key: aws.String(s3Key),
|
||||
Body: bytes.NewReader(buf.Bytes()),
|
||||
Metadata: metadata,
|
||||
}
|
||||
|
||||
if opts != nil && opts.ContentType != "" {
|
||||
input.ContentType = aws.String(opts.ContentType)
|
||||
for k, v := range opts.Metadata {
|
||||
s3Metadata[k] = v
|
||||
}
|
||||
}
|
||||
|
||||
// Upload to S3
|
||||
_, err = s.client.PutObject(ctx, input)
|
||||
_, err = s.client.PutObject(ctx, &s3.PutObjectInput{
|
||||
Bucket: aws.String(s.bucket),
|
||||
Key: aws.String(fullKey),
|
||||
Body: bytes.NewReader(buf.Bytes()),
|
||||
Metadata: s3Metadata,
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
metrics.RecordStorageOperation("s3", "put", "error")
|
||||
return errors.Wrap(err, errors.ErrCodeStorageFailure, "failed to upload to S3")
|
||||
return errors.Wrap(err, errors.ErrCodeStorageFailure, "failed to put object to S3")
|
||||
}
|
||||
|
||||
// Update usage
|
||||
s.mu.Lock()
|
||||
s.used += written
|
||||
currentUsed := s.used
|
||||
s.mu.Unlock()
|
||||
|
||||
metrics.RecordStorageOperation("s3", "put", "success")
|
||||
metrics.UpdateCacheSize("s3", currentUsed)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Delete removes a file from S3
|
||||
// Delete removes data from S3
|
||||
func (s *S3Storage) Delete(ctx context.Context, key string) error {
|
||||
s3Key := s.buildKey(key)
|
||||
fullKey := s.buildKey(key)
|
||||
|
||||
// Get size before deletion for quota tracking
|
||||
statInfo, err := s.Stat(ctx, key)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
log.Debug().Str("key", fullKey).Msg("Deleting object from S3")
|
||||
|
||||
input := &s3.DeleteObjectInput{
|
||||
_, err := s.client.DeleteObject(ctx, &s3.DeleteObjectInput{
|
||||
Bucket: aws.String(s.bucket),
|
||||
Key: aws.String(s3Key),
|
||||
}
|
||||
Key: aws.String(fullKey),
|
||||
})
|
||||
|
||||
_, err = s.client.DeleteObject(ctx, input)
|
||||
if err != nil {
|
||||
metrics.RecordStorageOperation("s3", "delete", "error")
|
||||
return errors.Wrap(err, errors.ErrCodeStorageFailure, "failed to delete from S3")
|
||||
return errors.Wrap(err, errors.ErrCodeStorageFailure, "failed to delete object from S3")
|
||||
}
|
||||
|
||||
// Update usage
|
||||
s.mu.Lock()
|
||||
s.used -= statInfo.Size
|
||||
currentUsed := s.used
|
||||
s.mu.Unlock()
|
||||
|
||||
metrics.RecordStorageOperation("s3", "delete", "success")
|
||||
metrics.UpdateCacheSize("s3", currentUsed)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Exists checks if a file exists in S3
|
||||
// Exists checks if data exists in S3
|
||||
func (s *S3Storage) Exists(ctx context.Context, key string) (bool, error) {
|
||||
s3Key := s.buildKey(key)
|
||||
fullKey := s.buildKey(key)
|
||||
|
||||
input := &s3.HeadObjectInput{
|
||||
_, err := s.client.HeadObject(ctx, &s3.HeadObjectInput{
|
||||
Bucket: aws.String(s.bucket),
|
||||
Key: aws.String(s3Key),
|
||||
}
|
||||
Key: aws.String(fullKey),
|
||||
})
|
||||
|
||||
_, err := s.client.HeadObject(ctx, input)
|
||||
if err != nil {
|
||||
if isNotFoundError(err) {
|
||||
return false, nil
|
||||
}
|
||||
return false, errors.Wrap(err, errors.ErrCodeStorageFailure, "failed to check existence in S3")
|
||||
return false, errors.Wrap(err, errors.ErrCodeStorageFailure, "failed to check object existence in S3")
|
||||
}
|
||||
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// List lists files with prefix in S3
|
||||
// List returns a list of objects with the given prefix
|
||||
func (s *S3Storage) List(ctx context.Context, prefix string, opts *storage.ListOptions) ([]storage.StorageObject, error) {
|
||||
s3Prefix := s.buildKey(prefix)
|
||||
fullPrefix := s.buildKey(prefix)
|
||||
|
||||
input := &s3.ListObjectsV2Input{
|
||||
Bucket: aws.String(s.bucket),
|
||||
Prefix: aws.String(s3Prefix),
|
||||
}
|
||||
log.Debug().Str("prefix", fullPrefix).Msg("Listing objects in S3")
|
||||
|
||||
var objects []storage.StorageObject
|
||||
paginator := s3.NewListObjectsV2Paginator(s.client, input)
|
||||
paginator := s3.NewListObjectsV2Paginator(s.client, &s3.ListObjectsV2Input{
|
||||
Bucket: aws.String(s.bucket),
|
||||
Prefix: aws.String(fullPrefix),
|
||||
})
|
||||
|
||||
for paginator.HasMorePages() {
|
||||
page, err := paginator.NextPage(ctx)
|
||||
@@ -281,56 +226,58 @@ func (s *S3Storage) List(ctx context.Context, prefix string, opts *storage.ListO
|
||||
}
|
||||
|
||||
for _, obj := range page.Contents {
|
||||
key := s.stripPrefix(*obj.Key)
|
||||
objects = append(objects, storage.StorageObject{
|
||||
Key: key,
|
||||
Size: *obj.Size,
|
||||
Modified: *obj.LastModified,
|
||||
ETag: strings.Trim(*obj.ETag, "\""),
|
||||
})
|
||||
}
|
||||
}
|
||||
if obj.Key != nil {
|
||||
// Strip prefix from key
|
||||
key := s.stripPrefix(*obj.Key)
|
||||
|
||||
// Apply pagination if requested
|
||||
if opts != nil {
|
||||
start := opts.Offset
|
||||
end := len(objects)
|
||||
if opts.MaxResults > 0 && start+opts.MaxResults < end {
|
||||
end = start + opts.MaxResults
|
||||
}
|
||||
if start < len(objects) {
|
||||
objects = objects[start:end]
|
||||
} else {
|
||||
objects = []storage.StorageObject{}
|
||||
object := storage.StorageObject{
|
||||
Key: key,
|
||||
Size: aws.ToInt64(obj.Size),
|
||||
}
|
||||
|
||||
if obj.LastModified != nil {
|
||||
object.Modified = *obj.LastModified
|
||||
}
|
||||
|
||||
if obj.ETag != nil {
|
||||
object.ETag = *obj.ETag
|
||||
}
|
||||
|
||||
objects = append(objects, object)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return objects, nil
|
||||
}
|
||||
|
||||
// Stat gets file metadata from S3
|
||||
// Stat returns metadata about stored data
|
||||
func (s *S3Storage) Stat(ctx context.Context, key string) (*storage.StorageInfo, error) {
|
||||
s3Key := s.buildKey(key)
|
||||
fullKey := s.buildKey(key)
|
||||
|
||||
input := &s3.HeadObjectInput{
|
||||
result, err := s.client.HeadObject(ctx, &s3.HeadObjectInput{
|
||||
Bucket: aws.String(s.bucket),
|
||||
Key: aws.String(s3Key),
|
||||
}
|
||||
Key: aws.String(fullKey),
|
||||
})
|
||||
|
||||
result, err := s.client.HeadObject(ctx, input)
|
||||
if err != nil {
|
||||
if isNotFoundError(err) {
|
||||
return nil, errors.NotFound(fmt.Sprintf("file not found: %s", key))
|
||||
return nil, errors.NotFound(fmt.Sprintf("S3 object not found: %s", key))
|
||||
}
|
||||
return nil, errors.Wrap(err, errors.ErrCodeStorageFailure, "failed to stat object in S3")
|
||||
}
|
||||
|
||||
info := &storage.StorageInfo{
|
||||
Key: key,
|
||||
Size: *result.ContentLength,
|
||||
Modified: *result.LastModified,
|
||||
ETag: strings.Trim(*result.ETag, "\""),
|
||||
Metadata: result.Metadata,
|
||||
Key: key,
|
||||
Size: aws.ToInt64(result.ContentLength),
|
||||
}
|
||||
|
||||
if result.LastModified != nil {
|
||||
info.Modified = *result.LastModified
|
||||
}
|
||||
|
||||
if result.ETag != nil {
|
||||
info.ETag = *result.ETag
|
||||
}
|
||||
|
||||
if result.ContentType != nil {
|
||||
@@ -340,33 +287,27 @@ func (s *S3Storage) Stat(ctx context.Context, key string) (*storage.StorageInfo,
|
||||
return info, nil
|
||||
}
|
||||
|
||||
// GetQuota returns quota information
|
||||
// GetQuota returns current usage and quota information
|
||||
func (s *S3Storage) GetQuota(ctx context.Context) (*storage.QuotaInfo, error) {
|
||||
s.mu.RLock()
|
||||
used := s.used
|
||||
s.mu.RUnlock()
|
||||
|
||||
available := s.quota - used
|
||||
if available < 0 {
|
||||
available = 0
|
||||
usage, err := s.calculateUsage(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &storage.QuotaInfo{
|
||||
Used: used,
|
||||
Available: available,
|
||||
Limit: s.quota,
|
||||
Used: usage,
|
||||
Limit: s.maxSizeBytes,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Health checks S3 health
|
||||
// Health checks if the S3 backend is healthy
|
||||
func (s *S3Storage) Health(ctx context.Context) error {
|
||||
// Try to list bucket to verify connectivity
|
||||
input := &s3.ListObjectsV2Input{
|
||||
// Try to list objects (lightweight operation)
|
||||
_, err := s.client.ListObjectsV2(ctx, &s3.ListObjectsV2Input{
|
||||
Bucket: aws.String(s.bucket),
|
||||
MaxKeys: aws.Int32(1),
|
||||
}
|
||||
})
|
||||
|
||||
_, err := s.client.ListObjectsV2(ctx, input)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, errors.ErrCodeStorageFailure, "S3 health check failed")
|
||||
}
|
||||
@@ -374,60 +315,51 @@ func (s *S3Storage) Health(ctx context.Context) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Close closes the storage backend
|
||||
// Close closes the S3 storage backend
|
||||
func (s *S3Storage) Close() error {
|
||||
// No cleanup needed for S3 client
|
||||
log.Info().Msg("S3 storage closed")
|
||||
return nil
|
||||
}
|
||||
|
||||
// buildKey builds the full S3 key with prefix
|
||||
// buildKey constructs the full S3 key with prefix
|
||||
func (s *S3Storage) buildKey(key string) string {
|
||||
key = strings.TrimPrefix(key, "/")
|
||||
if s.prefix != "" {
|
||||
return s.prefix + "/" + key
|
||||
if s.prefix == "" {
|
||||
return key
|
||||
}
|
||||
return key
|
||||
return s.prefix + "/" + key
|
||||
}
|
||||
|
||||
// stripPrefix removes the configured prefix from an S3 key
|
||||
func (s *S3Storage) stripPrefix(s3Key string) string {
|
||||
if s.prefix != "" {
|
||||
return strings.TrimPrefix(s3Key, s.prefix+"/")
|
||||
// stripPrefix removes the prefix from an S3 key
|
||||
func (s *S3Storage) stripPrefix(key string) string {
|
||||
if s.prefix == "" {
|
||||
return key
|
||||
}
|
||||
return s3Key
|
||||
return strings.TrimPrefix(key, s.prefix+"/")
|
||||
}
|
||||
|
||||
// calculateUsage calculates current S3 storage usage
|
||||
func (s *S3Storage) calculateUsage(ctx context.Context) error {
|
||||
var total int64
|
||||
// calculateUsage calculates total storage usage
|
||||
func (s *S3Storage) calculateUsage(ctx context.Context) (int64, error) {
|
||||
var totalSize int64
|
||||
|
||||
input := &s3.ListObjectsV2Input{
|
||||
paginator := s3.NewListObjectsV2Paginator(s.client, &s3.ListObjectsV2Input{
|
||||
Bucket: aws.String(s.bucket),
|
||||
}
|
||||
|
||||
if s.prefix != "" {
|
||||
input.Prefix = aws.String(s.prefix + "/")
|
||||
}
|
||||
|
||||
paginator := s3.NewListObjectsV2Paginator(s.client, input)
|
||||
Prefix: aws.String(s.prefix),
|
||||
})
|
||||
|
||||
for paginator.HasMorePages() {
|
||||
page, err := paginator.NextPage(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
return 0, fmt.Errorf("failed to calculate usage: %w", err)
|
||||
}
|
||||
|
||||
for _, obj := range page.Contents {
|
||||
total += *obj.Size
|
||||
if obj.Size != nil {
|
||||
totalSize += aws.ToInt64(obj.Size)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
s.mu.Lock()
|
||||
s.used = total
|
||||
s.mu.Unlock()
|
||||
|
||||
metrics.UpdateCacheSize("s3", total)
|
||||
return nil
|
||||
return totalSize, nil
|
||||
}
|
||||
|
||||
// isNotFoundError checks if an error is a "not found" error
|
||||
@@ -436,8 +368,21 @@ func isNotFoundError(err error) bool {
|
||||
return false
|
||||
}
|
||||
|
||||
// Check for specific S3 error types
|
||||
var notFound *types.NotFound
|
||||
var noSuchKey *types.NoSuchKey
|
||||
|
||||
return stderrors.As(err, ¬Found) || stderrors.As(err, &noSuchKey)
|
||||
// Use errors.As to check for wrapped errors
|
||||
if ok := stderrors.As(err, ¬Found); ok {
|
||||
return true
|
||||
}
|
||||
if ok := stderrors.As(err, &noSuchKey); ok {
|
||||
return true
|
||||
}
|
||||
|
||||
// Check error message as fallback
|
||||
errMsg := err.Error()
|
||||
return strings.Contains(errMsg, "NoSuchKey") ||
|
||||
strings.Contains(errMsg, "NotFound") ||
|
||||
strings.Contains(errMsg, "404")
|
||||
}
|
||||
|
||||
@@ -0,0 +1,271 @@
|
||||
package s3
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/suite"
|
||||
)
|
||||
|
||||
type S3StorageTestSuite struct {
|
||||
suite.Suite
|
||||
}
|
||||
|
||||
func TestS3StorageTestSuite(t *testing.T) {
|
||||
suite.Run(t, new(S3StorageTestSuite))
|
||||
}
|
||||
|
||||
func (s *S3StorageTestSuite) TestNewS3Storage() {
|
||||
tests := []struct {
|
||||
name string
|
||||
config Config
|
||||
expectError bool
|
||||
errorMsg string
|
||||
}{
|
||||
{
|
||||
name: "valid config with credentials",
|
||||
config: Config{
|
||||
Region: "us-east-1",
|
||||
Bucket: "test-bucket",
|
||||
Prefix: "packages/",
|
||||
AccessKeyID: "AKIAIOSFODNN7EXAMPLE",
|
||||
SecretAccessKey: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY",
|
||||
MaxSizeBytes: 1024 * 1024,
|
||||
},
|
||||
expectError: false,
|
||||
},
|
||||
{
|
||||
name: "valid config with custom endpoint",
|
||||
config: Config{
|
||||
Region: "us-east-1",
|
||||
Bucket: "test-bucket",
|
||||
Endpoint: "https://minio.example.com",
|
||||
AccessKeyID: "minioadmin",
|
||||
SecretAccessKey: "minioadmin",
|
||||
ForcePathStyle: true,
|
||||
},
|
||||
expectError: false,
|
||||
},
|
||||
{
|
||||
name: "valid config with default region",
|
||||
config: Config{
|
||||
Bucket: "test-bucket",
|
||||
AccessKeyID: "test",
|
||||
SecretAccessKey: "test",
|
||||
},
|
||||
expectError: false,
|
||||
},
|
||||
{
|
||||
name: "missing bucket",
|
||||
config: Config{
|
||||
Region: "us-east-1",
|
||||
AccessKeyID: "test",
|
||||
SecretAccessKey: "test",
|
||||
},
|
||||
expectError: true,
|
||||
errorMsg: "bucket is required",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
s.Run(tt.name, func() {
|
||||
storage, err := New(tt.config)
|
||||
|
||||
if tt.expectError {
|
||||
s.Error(err)
|
||||
if tt.errorMsg != "" {
|
||||
s.Contains(err.Error(), tt.errorMsg)
|
||||
}
|
||||
s.Nil(storage)
|
||||
} else {
|
||||
s.NoError(err)
|
||||
s.NotNil(storage)
|
||||
s.Equal(tt.config.Bucket, storage.bucket)
|
||||
s.Equal(tt.config.MaxSizeBytes, storage.maxSizeBytes)
|
||||
|
||||
// Test prefix normalization
|
||||
if tt.config.Prefix != "" {
|
||||
s.NotContains(storage.prefix, "/", "prefix should not end with /")
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func (s *S3StorageTestSuite) TestBuildKey() {
|
||||
tests := []struct {
|
||||
name string
|
||||
prefix string
|
||||
key string
|
||||
expected string
|
||||
}{
|
||||
{
|
||||
name: "with prefix",
|
||||
prefix: "packages",
|
||||
key: "test/file.txt",
|
||||
expected: "packages/test/file.txt",
|
||||
},
|
||||
{
|
||||
name: "without prefix",
|
||||
prefix: "",
|
||||
key: "test/file.txt",
|
||||
expected: "test/file.txt",
|
||||
},
|
||||
{
|
||||
name: "with trailing slash in prefix",
|
||||
prefix: "packages/",
|
||||
key: "test/file.txt",
|
||||
expected: "packages/test/file.txt",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
s.Run(tt.name, func() {
|
||||
storage := &S3Storage{
|
||||
prefix: tt.prefix,
|
||||
}
|
||||
// Normalize prefix like in New()
|
||||
if storage.prefix != "" && storage.prefix[len(storage.prefix)-1] == '/' {
|
||||
storage.prefix = storage.prefix[:len(storage.prefix)-1]
|
||||
}
|
||||
|
||||
result := storage.buildKey(tt.key)
|
||||
s.Equal(tt.expected, result)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func (s *S3StorageTestSuite) TestStripPrefix() {
|
||||
tests := []struct {
|
||||
name string
|
||||
prefix string
|
||||
key string
|
||||
expected string
|
||||
}{
|
||||
{
|
||||
name: "with prefix",
|
||||
prefix: "packages",
|
||||
key: "packages/test/file.txt",
|
||||
expected: "test/file.txt",
|
||||
},
|
||||
{
|
||||
name: "without prefix",
|
||||
prefix: "",
|
||||
key: "test/file.txt",
|
||||
expected: "test/file.txt",
|
||||
},
|
||||
{
|
||||
name: "key without prefix but prefix set",
|
||||
prefix: "packages",
|
||||
key: "test/file.txt",
|
||||
expected: "test/file.txt",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
s.Run(tt.name, func() {
|
||||
storage := &S3Storage{
|
||||
prefix: tt.prefix,
|
||||
}
|
||||
|
||||
result := storage.stripPrefix(tt.key)
|
||||
s.Equal(tt.expected, result)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func (s *S3StorageTestSuite) TestIsNotFoundError() {
|
||||
tests := []struct {
|
||||
name string
|
||||
err error
|
||||
expected bool
|
||||
}{
|
||||
{
|
||||
name: "nil error",
|
||||
err: nil,
|
||||
expected: false,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
s.Run(tt.name, func() {
|
||||
result := isNotFoundError(tt.err)
|
||||
s.Equal(tt.expected, result)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func (s *S3StorageTestSuite) TestConfigDefaults() {
|
||||
config := Config{
|
||||
Bucket: "test-bucket",
|
||||
AccessKeyID: "test",
|
||||
SecretAccessKey: "test",
|
||||
}
|
||||
|
||||
storage, err := New(config)
|
||||
s.Require().NoError(err)
|
||||
s.NotNil(storage)
|
||||
|
||||
// Verify defaults
|
||||
s.Equal("test-bucket", storage.bucket)
|
||||
s.Equal("", storage.prefix)
|
||||
s.Equal(int64(0), storage.maxSizeBytes)
|
||||
}
|
||||
|
||||
func (s *S3StorageTestSuite) TestPrefixNormalization() {
|
||||
tests := []struct {
|
||||
name string
|
||||
inputPrefix string
|
||||
expectedPrefix string
|
||||
}{
|
||||
{
|
||||
name: "prefix with trailing slash",
|
||||
inputPrefix: "packages/",
|
||||
expectedPrefix: "packages",
|
||||
},
|
||||
{
|
||||
name: "prefix without trailing slash",
|
||||
inputPrefix: "packages",
|
||||
expectedPrefix: "packages",
|
||||
},
|
||||
{
|
||||
name: "empty prefix",
|
||||
inputPrefix: "",
|
||||
expectedPrefix: "",
|
||||
},
|
||||
{
|
||||
name: "nested prefix with trailing slash",
|
||||
inputPrefix: "cache/packages/",
|
||||
expectedPrefix: "cache/packages",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
s.Run(tt.name, func() {
|
||||
config := Config{
|
||||
Bucket: "test-bucket",
|
||||
Prefix: tt.inputPrefix,
|
||||
AccessKeyID: "test",
|
||||
SecretAccessKey: "test",
|
||||
}
|
||||
|
||||
storage, err := New(config)
|
||||
s.Require().NoError(err)
|
||||
s.Equal(tt.expectedPrefix, storage.prefix)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func (s *S3StorageTestSuite) TestClose() {
|
||||
config := Config{
|
||||
Bucket: "test-bucket",
|
||||
AccessKeyID: "test",
|
||||
SecretAccessKey: "test",
|
||||
}
|
||||
|
||||
storage, err := New(config)
|
||||
s.Require().NoError(err)
|
||||
|
||||
// Close should not error
|
||||
err = storage.Close()
|
||||
s.NoError(err)
|
||||
}
|
||||
+266
-303
@@ -1,42 +1,43 @@
|
||||
package smb
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/md5" // #nosec G501 -- MD5 used for file checksums, not cryptographic security
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/hirochachacha/go-smb2"
|
||||
"github.com/lukaszraczylo/gohoarder/pkg/errors"
|
||||
"github.com/lukaszraczylo/gohoarder/pkg/metrics"
|
||||
"github.com/lukaszraczylo/gohoarder/pkg/storage"
|
||||
"github.com/rs/zerolog/log"
|
||||
)
|
||||
|
||||
// SMBStorage implements storage.StorageBackend for SMB/CIFS shares
|
||||
type SMBStorage struct {
|
||||
host string
|
||||
share string
|
||||
basePath string
|
||||
username string
|
||||
password string
|
||||
quota int64
|
||||
mu sync.RWMutex
|
||||
used int64
|
||||
connPool chan *smbConnection
|
||||
poolSize int
|
||||
// Config holds SMB storage configuration
|
||||
type Config struct {
|
||||
Host string
|
||||
Share string
|
||||
Path string
|
||||
Username string
|
||||
Password string
|
||||
Domain string
|
||||
Port int
|
||||
MaxSizeBytes int64
|
||||
PoolSize int
|
||||
}
|
||||
|
||||
// smbConnection wraps an SMB session and share
|
||||
// SMBStorage implements storage.StorageBackend using SMB/CIFS
|
||||
type SMBStorage struct {
|
||||
connPool chan *smbConnection
|
||||
config Config
|
||||
maxSizeBytes int64
|
||||
poolSize int
|
||||
}
|
||||
|
||||
// smbConnection represents a pooled SMB connection
|
||||
type smbConnection struct {
|
||||
conn net.Conn
|
||||
session *smb2.Session
|
||||
@@ -44,27 +45,14 @@ type smbConnection struct {
|
||||
lastUse time.Time
|
||||
}
|
||||
|
||||
// Config holds SMB configuration
|
||||
type Config struct {
|
||||
Host string // SMB server hostname or IP
|
||||
Port int // SMB server port (default: 445)
|
||||
Share string // SMB share name
|
||||
BasePath string // Base path within the share
|
||||
Username string // SMB username
|
||||
Password string // SMB password
|
||||
Domain string // SMB domain (optional)
|
||||
Quota int64 // Quota in bytes (0 = unlimited)
|
||||
PoolSize int // Connection pool size (default: 5)
|
||||
}
|
||||
|
||||
// New creates a new SMB storage backend
|
||||
func New(ctx context.Context, cfg Config) (*SMBStorage, error) {
|
||||
func New(cfg Config) (*SMBStorage, error) {
|
||||
if cfg.Host == "" {
|
||||
return nil, errors.New(errors.ErrCodeInvalidConfig, "SMB host is required")
|
||||
return nil, fmt.Errorf("SMB host is required")
|
||||
}
|
||||
|
||||
if cfg.Share == "" {
|
||||
return nil, errors.New(errors.ErrCodeInvalidConfig, "SMB share is required")
|
||||
return nil, fmt.Errorf("SMB share is required")
|
||||
}
|
||||
|
||||
if cfg.Port == 0 {
|
||||
@@ -75,64 +63,68 @@ func New(ctx context.Context, cfg Config) (*SMBStorage, error) {
|
||||
cfg.PoolSize = 5 // Default pool size
|
||||
}
|
||||
|
||||
smbStorage := &SMBStorage{
|
||||
host: fmt.Sprintf("%s:%d", cfg.Host, cfg.Port),
|
||||
share: cfg.Share,
|
||||
basePath: strings.Trim(cfg.BasePath, "/\\"),
|
||||
username: cfg.Username,
|
||||
password: cfg.Password,
|
||||
quota: cfg.Quota,
|
||||
connPool: make(chan *smbConnection, cfg.PoolSize),
|
||||
poolSize: cfg.PoolSize,
|
||||
// Normalize path
|
||||
cfg.Path = strings.Trim(cfg.Path, "/\\")
|
||||
|
||||
storage := &SMBStorage{
|
||||
config: cfg,
|
||||
maxSizeBytes: cfg.MaxSizeBytes,
|
||||
poolSize: cfg.PoolSize,
|
||||
connPool: make(chan *smbConnection, cfg.PoolSize),
|
||||
}
|
||||
|
||||
// Initialize connection pool
|
||||
// Pre-populate connection pool
|
||||
for i := 0; i < cfg.PoolSize; i++ {
|
||||
conn, err := smbStorage.createConnection(ctx)
|
||||
conn, err := storage.createConnection()
|
||||
if err != nil {
|
||||
// Clean up any created connections
|
||||
close(smbStorage.connPool)
|
||||
for c := range smbStorage.connPool {
|
||||
c.close()
|
||||
}
|
||||
return nil, errors.Wrap(err, errors.ErrCodeStorageFailure, "failed to create SMB connection pool")
|
||||
log.Warn().Err(err).Int("attempt", i).Msg("Failed to create initial SMB connection")
|
||||
continue
|
||||
}
|
||||
smbStorage.connPool <- conn
|
||||
storage.connPool <- conn
|
||||
}
|
||||
|
||||
// Calculate initial usage
|
||||
if err := smbStorage.calculateUsage(ctx); err != nil {
|
||||
log.Warn().Err(err).Msg("Failed to calculate initial SMB storage usage")
|
||||
}
|
||||
log.Info().
|
||||
Str("host", cfg.Host).
|
||||
Int("port", cfg.Port).
|
||||
Str("share", cfg.Share).
|
||||
Str("path", cfg.Path).
|
||||
Int("pool_size", cfg.PoolSize).
|
||||
Msg("SMB storage initialized")
|
||||
|
||||
return smbStorage, nil
|
||||
return storage, nil
|
||||
}
|
||||
|
||||
// createConnection creates a new SMB connection
|
||||
func (s *SMBStorage) createConnection(ctx context.Context) (*smbConnection, error) {
|
||||
conn, err := net.Dial("tcp", s.host)
|
||||
func (s *SMBStorage) createConnection() (*smbConnection, error) {
|
||||
// Connect to SMB server (use net.JoinHostPort for IPv6 compatibility)
|
||||
addr := net.JoinHostPort(s.config.Host, fmt.Sprintf("%d", s.config.Port))
|
||||
conn, err := net.DialTimeout("tcp", addr, 10*time.Second)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return nil, fmt.Errorf("failed to connect to SMB server: %w", err)
|
||||
}
|
||||
|
||||
dialer := &smb2.Dialer{
|
||||
// Create SMB dialer
|
||||
d := &smb2.Dialer{
|
||||
Initiator: &smb2.NTLMInitiator{
|
||||
User: s.username,
|
||||
Password: s.password,
|
||||
User: s.config.Username,
|
||||
Password: s.config.Password,
|
||||
Domain: s.config.Domain,
|
||||
},
|
||||
}
|
||||
|
||||
session, err := dialer.Dial(conn)
|
||||
// Establish SMB session
|
||||
session, err := d.Dial(conn)
|
||||
if err != nil {
|
||||
conn.Close() // #nosec G104 -- Cleanup, error not critical
|
||||
return nil, err
|
||||
_ = conn.Close()
|
||||
return nil, fmt.Errorf("failed to establish SMB session: %w", err)
|
||||
}
|
||||
|
||||
share, err := session.Mount(s.share)
|
||||
// Mount share
|
||||
share, err := session.Mount(s.config.Share)
|
||||
if err != nil {
|
||||
_ = session.Logoff() // #nosec G104 -- SMB cleanup
|
||||
conn.Close() // #nosec G104 -- Cleanup, error not critical
|
||||
return nil, err
|
||||
_ = session.Logoff()
|
||||
_ = conn.Close()
|
||||
return nil, fmt.Errorf("failed to mount SMB share: %w", err)
|
||||
}
|
||||
|
||||
return &smbConnection{
|
||||
@@ -143,25 +135,34 @@ func (s *SMBStorage) createConnection(ctx context.Context) (*smbConnection, erro
|
||||
}, nil
|
||||
}
|
||||
|
||||
// getConnection gets a connection from the pool
|
||||
func (s *SMBStorage) getConnection(ctx context.Context) (*smbConnection, error) {
|
||||
// getConnection gets a connection from the pool or creates a new one
|
||||
func (s *SMBStorage) getConnection() (*smbConnection, error) {
|
||||
select {
|
||||
case conn := <-s.connPool:
|
||||
// Check if connection is still valid (not older than 5 minutes idle)
|
||||
if time.Since(conn.lastUse) > 5*time.Minute {
|
||||
conn.close()
|
||||
return s.createConnection()
|
||||
}
|
||||
conn.lastUse = time.Now()
|
||||
return conn, nil
|
||||
case <-ctx.Done():
|
||||
return nil, ctx.Err()
|
||||
case <-time.After(30 * time.Second):
|
||||
return nil, errors.New(errors.ErrCodeStorageFailure, "timeout waiting for SMB connection")
|
||||
default:
|
||||
// Pool is empty, create new connection
|
||||
return s.createConnection()
|
||||
}
|
||||
}
|
||||
|
||||
// returnConnection returns a connection to the pool
|
||||
func (s *SMBStorage) returnConnection(conn *smbConnection) {
|
||||
if conn == nil {
|
||||
return
|
||||
}
|
||||
|
||||
select {
|
||||
case s.connPool <- conn:
|
||||
// Successfully returned to pool
|
||||
default:
|
||||
// Pool is full, close the connection
|
||||
// Pool is full, close connection
|
||||
conn.close()
|
||||
}
|
||||
}
|
||||
@@ -169,189 +170,161 @@ func (s *SMBStorage) returnConnection(conn *smbConnection) {
|
||||
// close closes an SMB connection
|
||||
func (c *smbConnection) close() {
|
||||
if c.share != nil {
|
||||
_ = c.share.Umount() // #nosec G104 -- SMB cleanup
|
||||
if err := c.share.Umount(); err != nil {
|
||||
log.Warn().Err(err).Msg("Failed to unmount SMB share")
|
||||
}
|
||||
}
|
||||
if c.session != nil {
|
||||
_ = c.session.Logoff() // #nosec G104 -- SMB cleanup
|
||||
if err := c.session.Logoff(); err != nil {
|
||||
log.Warn().Err(err).Msg("Failed to logoff SMB session")
|
||||
}
|
||||
}
|
||||
if c.conn != nil {
|
||||
c.conn.Close() // #nosec G104 -- Cleanup, error not critical
|
||||
if err := c.conn.Close(); err != nil {
|
||||
log.Warn().Err(err).Msg("Failed to close SMB connection")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Get retrieves a file from SMB share
|
||||
// Get retrieves data from SMB share
|
||||
func (s *SMBStorage) Get(ctx context.Context, key string) (io.ReadCloser, error) {
|
||||
conn, err := s.getConnection(ctx)
|
||||
conn, err := s.getConnection()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return nil, errors.Wrap(err, errors.ErrCodeStorageFailure, "failed to get SMB connection")
|
||||
}
|
||||
defer s.returnConnection(conn)
|
||||
|
||||
path := s.keyToPath(key)
|
||||
|
||||
log.Debug().Str("key", path).Msg("Getting file from SMB")
|
||||
|
||||
// Open file
|
||||
file, err := conn.share.Open(path)
|
||||
if err != nil {
|
||||
s.returnConnection(conn)
|
||||
if os.IsNotExist(err) {
|
||||
metrics.RecordStorageOperation("smb", "get", "not_found")
|
||||
return nil, errors.NotFound(fmt.Sprintf("file not found: %s", key))
|
||||
return nil, errors.NotFound(fmt.Sprintf("SMB file not found: %s", key))
|
||||
}
|
||||
metrics.RecordStorageOperation("smb", "get", "error")
|
||||
return nil, errors.Wrap(err, errors.ErrCodeStorageFailure, "failed to open SMB file")
|
||||
}
|
||||
|
||||
// Read entire file into memory and close SMB connection
|
||||
// This is necessary because we need to return the connection to the pool
|
||||
// Read entire file into memory (SMB files must be read completely before closing connection)
|
||||
data, err := io.ReadAll(file)
|
||||
file.Close() // #nosec G104 -- Cleanup, error not critical
|
||||
s.returnConnection(conn)
|
||||
|
||||
if closeErr := file.Close(); closeErr != nil {
|
||||
log.Warn().Err(closeErr).Str("path", path).Msg("Failed to close SMB file after reading")
|
||||
}
|
||||
if err != nil {
|
||||
metrics.RecordStorageOperation("smb", "get", "error")
|
||||
return nil, errors.Wrap(err, errors.ErrCodeStorageFailure, "failed to read SMB file")
|
||||
}
|
||||
|
||||
metrics.RecordStorageOperation("smb", "get", "success")
|
||||
return io.NopCloser(bytes.NewReader(data)), nil
|
||||
// Return as ReadCloser
|
||||
return io.NopCloser(strings.NewReader(string(data))), nil
|
||||
}
|
||||
|
||||
// Put stores a file on SMB share
|
||||
// Put stores data on SMB share
|
||||
func (s *SMBStorage) Put(ctx context.Context, key string, data io.Reader, opts *storage.PutOptions) error {
|
||||
conn, err := s.getConnection(ctx)
|
||||
conn, err := s.getConnection()
|
||||
if err != nil {
|
||||
return err
|
||||
return errors.Wrap(err, errors.ErrCodeStorageFailure, "failed to get SMB connection")
|
||||
}
|
||||
defer s.returnConnection(conn)
|
||||
|
||||
path := s.keyToPath(key)
|
||||
dir := filepath.Dir(path)
|
||||
|
||||
// Create directory structure
|
||||
if err := conn.share.MkdirAll(dir, 0755); err != nil {
|
||||
metrics.RecordStorageOperation("smb", "put", "error")
|
||||
log.Debug().Str("key", path).Msg("Putting file to SMB")
|
||||
|
||||
// Ensure directory exists
|
||||
dir := filepath.Dir(path)
|
||||
if err := s.ensureDir(conn, dir); err != nil {
|
||||
return errors.Wrap(err, errors.ErrCodeStorageFailure, "failed to create SMB directory")
|
||||
}
|
||||
|
||||
// Read data into buffer to calculate checksums and size
|
||||
var buf bytes.Buffer
|
||||
md5Hash := md5.New() // #nosec G401 -- MD5 used for file integrity check, not cryptographic security
|
||||
sha256Hash := sha256.New()
|
||||
multiWriter := io.MultiWriter(&buf, md5Hash, sha256Hash)
|
||||
|
||||
written, err := io.Copy(multiWriter, data)
|
||||
// Read data into buffer to check quota
|
||||
buf := new(strings.Builder)
|
||||
size, err := io.Copy(buf, data)
|
||||
if err != nil {
|
||||
metrics.RecordStorageOperation("smb", "put", "error")
|
||||
return errors.Wrap(err, errors.ErrCodeStorageFailure, "failed to read data")
|
||||
}
|
||||
|
||||
// Check quota
|
||||
if s.quota > 0 {
|
||||
s.mu.RLock()
|
||||
used := s.used
|
||||
s.mu.RUnlock()
|
||||
|
||||
if used+written > s.quota {
|
||||
metrics.RecordStorageOperation("smb", "put", "quota_exceeded")
|
||||
return errors.QuotaExceeded(s.quota)
|
||||
// Check quota if set
|
||||
if s.maxSizeBytes > 0 {
|
||||
currentUsage, err := s.calculateUsage(conn)
|
||||
if err != nil {
|
||||
log.Warn().Err(err).Msg("Failed to calculate current usage, skipping quota check")
|
||||
} else if currentUsage+size > s.maxSizeBytes {
|
||||
return errors.QuotaExceeded(s.maxSizeBytes)
|
||||
}
|
||||
}
|
||||
|
||||
// Verify checksums if provided
|
||||
if opts != nil {
|
||||
md5Sum := hex.EncodeToString(md5Hash.Sum(nil))
|
||||
sha256Sum := hex.EncodeToString(sha256Hash.Sum(nil))
|
||||
|
||||
if opts.ChecksumMD5 != "" && opts.ChecksumMD5 != md5Sum {
|
||||
metrics.RecordStorageOperation("smb", "put", "checksum_error")
|
||||
return errors.New(errors.ErrCodeChecksumMismatch, "MD5 checksum mismatch")
|
||||
}
|
||||
|
||||
if opts.ChecksumSHA256 != "" && opts.ChecksumSHA256 != sha256Sum {
|
||||
metrics.RecordStorageOperation("smb", "put", "checksum_error")
|
||||
return errors.New(errors.ErrCodeChecksumMismatch, "SHA256 checksum mismatch")
|
||||
}
|
||||
}
|
||||
|
||||
// Create temp file for atomic write
|
||||
tempPath := path + ".tmp"
|
||||
file, err := conn.share.Create(tempPath)
|
||||
// Create/overwrite file
|
||||
file, err := conn.share.Create(path)
|
||||
if err != nil {
|
||||
metrics.RecordStorageOperation("smb", "put", "error")
|
||||
return errors.Wrap(err, errors.ErrCodeStorageFailure, "failed to create SMB temp file")
|
||||
return errors.Wrap(err, errors.ErrCodeStorageFailure, "failed to create SMB file")
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
// Write data
|
||||
_, err = io.Copy(file, bytes.NewReader(buf.Bytes()))
|
||||
file.Close() // #nosec G104 -- Cleanup, error not critical
|
||||
|
||||
_, err = file.Write([]byte(buf.String()))
|
||||
if err != nil {
|
||||
_ = conn.share.Remove(tempPath) // #nosec G104 -- SMB cleanup
|
||||
metrics.RecordStorageOperation("smb", "put", "error")
|
||||
return errors.Wrap(err, errors.ErrCodeStorageFailure, "failed to write SMB file")
|
||||
}
|
||||
|
||||
// Atomic rename
|
||||
if err := conn.share.Rename(tempPath, path); err != nil {
|
||||
_ = conn.share.Remove(tempPath) // #nosec G104 -- SMB cleanup
|
||||
metrics.RecordStorageOperation("smb", "put", "error")
|
||||
return errors.Wrap(err, errors.ErrCodeStorageFailure, "failed to rename SMB temp file")
|
||||
}
|
||||
|
||||
// Update usage
|
||||
s.mu.Lock()
|
||||
s.used += written
|
||||
currentUsed := s.used
|
||||
s.mu.Unlock()
|
||||
|
||||
metrics.RecordStorageOperation("smb", "put", "success")
|
||||
metrics.UpdateCacheSize("smb", currentUsed)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Delete removes a file from SMB share
|
||||
func (s *SMBStorage) Delete(ctx context.Context, key string) error {
|
||||
conn, err := s.getConnection(ctx)
|
||||
if err != nil {
|
||||
// ensureDir ensures a directory exists on SMB share
|
||||
func (s *SMBStorage) ensureDir(conn *smbConnection, path string) error {
|
||||
if path == "" || path == "." || path == "/" {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Try to stat the directory
|
||||
_, err := conn.share.Stat(path)
|
||||
if err == nil {
|
||||
return nil // Directory exists
|
||||
}
|
||||
|
||||
// Create parent directory first
|
||||
parent := filepath.Dir(path)
|
||||
if parent != path && parent != "." && parent != "/" {
|
||||
if err := s.ensureDir(conn, parent); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// Create this directory
|
||||
err = conn.share.Mkdir(path, 0755)
|
||||
if err != nil && !os.IsExist(err) {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Delete removes data from SMB share
|
||||
func (s *SMBStorage) Delete(ctx context.Context, key string) error {
|
||||
conn, err := s.getConnection()
|
||||
if err != nil {
|
||||
return errors.Wrap(err, errors.ErrCodeStorageFailure, "failed to get SMB connection")
|
||||
}
|
||||
defer s.returnConnection(conn)
|
||||
|
||||
path := s.keyToPath(key)
|
||||
|
||||
// Get size before deletion
|
||||
info, err := conn.share.Stat(path)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
metrics.RecordStorageOperation("smb", "delete", "not_found")
|
||||
return errors.NotFound(fmt.Sprintf("file not found: %s", key))
|
||||
}
|
||||
metrics.RecordStorageOperation("smb", "delete", "error")
|
||||
return errors.Wrap(err, errors.ErrCodeStorageFailure, "failed to stat SMB file")
|
||||
}
|
||||
log.Debug().Str("key", path).Msg("Deleting file from SMB")
|
||||
|
||||
size := info.Size()
|
||||
|
||||
if err := conn.share.Remove(path); err != nil {
|
||||
metrics.RecordStorageOperation("smb", "delete", "error")
|
||||
err = conn.share.Remove(path)
|
||||
if err != nil && !os.IsNotExist(err) {
|
||||
return errors.Wrap(err, errors.ErrCodeStorageFailure, "failed to delete SMB file")
|
||||
}
|
||||
|
||||
// Update usage
|
||||
s.mu.Lock()
|
||||
s.used -= size
|
||||
currentUsed := s.used
|
||||
s.mu.Unlock()
|
||||
|
||||
metrics.RecordStorageOperation("smb", "delete", "success")
|
||||
metrics.UpdateCacheSize("smb", currentUsed)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Exists checks if a file exists on SMB share
|
||||
// Exists checks if data exists on SMB share
|
||||
func (s *SMBStorage) Exists(ctx context.Context, key string) (bool, error) {
|
||||
conn, err := s.getConnection(ctx)
|
||||
conn, err := s.getConnection()
|
||||
if err != nil {
|
||||
return false, err
|
||||
return false, errors.Wrap(err, errors.ErrCodeStorageFailure, "failed to get SMB connection")
|
||||
}
|
||||
defer s.returnConnection(conn)
|
||||
|
||||
@@ -368,57 +341,90 @@ func (s *SMBStorage) Exists(ctx context.Context, key string) (bool, error) {
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// List lists files with prefix on SMB share
|
||||
// List returns a list of objects with the given prefix
|
||||
func (s *SMBStorage) List(ctx context.Context, prefix string, opts *storage.ListOptions) ([]storage.StorageObject, error) {
|
||||
conn, err := s.getConnection(ctx)
|
||||
conn, err := s.getConnection()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return nil, errors.Wrap(err, errors.ErrCodeStorageFailure, "failed to get SMB connection")
|
||||
}
|
||||
defer s.returnConnection(conn)
|
||||
|
||||
searchPath := s.keyToPath(prefix)
|
||||
basePath := s.keyToPath(prefix)
|
||||
|
||||
log.Debug().Str("prefix", basePath).Msg("Listing files in SMB")
|
||||
|
||||
var objects []storage.StorageObject
|
||||
|
||||
err = s.walkPath(conn.share, searchPath, func(path string, info os.FileInfo) error {
|
||||
// Walk the directory tree
|
||||
err = s.walkPath(conn, basePath, func(path string, info os.FileInfo) error {
|
||||
if info.IsDir() {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Convert path back to key
|
||||
key := s.pathToKey(path)
|
||||
|
||||
objects = append(objects, storage.StorageObject{
|
||||
Key: key,
|
||||
Size: info.Size(),
|
||||
Modified: info.ModTime(),
|
||||
})
|
||||
|
||||
return nil
|
||||
})
|
||||
|
||||
if err != nil && !os.IsNotExist(err) {
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, errors.ErrCodeStorageFailure, "failed to list SMB files")
|
||||
}
|
||||
|
||||
// Apply pagination if requested
|
||||
if opts != nil {
|
||||
start := opts.Offset
|
||||
end := len(objects)
|
||||
if opts.MaxResults > 0 && start+opts.MaxResults < end {
|
||||
end = start + opts.MaxResults
|
||||
}
|
||||
if start < len(objects) {
|
||||
objects = objects[start:end]
|
||||
} else {
|
||||
objects = []storage.StorageObject{}
|
||||
}
|
||||
}
|
||||
|
||||
return objects, nil
|
||||
}
|
||||
|
||||
// Stat gets file metadata from SMB share
|
||||
func (s *SMBStorage) Stat(ctx context.Context, key string) (*storage.StorageInfo, error) {
|
||||
conn, err := s.getConnection(ctx)
|
||||
// walkPath walks a directory tree on SMB share
|
||||
func (s *SMBStorage) walkPath(conn *smbConnection, root string, fn func(string, os.FileInfo) error) error {
|
||||
// Check if root exists
|
||||
info, err := conn.share.Stat(root)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
if os.IsNotExist(err) {
|
||||
return nil // Empty directory
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// If root is a file, process it directly
|
||||
if !info.IsDir() {
|
||||
return fn(root, info)
|
||||
}
|
||||
|
||||
// List directory contents
|
||||
entries, err := conn.share.ReadDir(root)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, entry := range entries {
|
||||
fullPath := filepath.Join(root, entry.Name())
|
||||
|
||||
if err := fn(fullPath, entry); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Recurse into subdirectories
|
||||
if entry.IsDir() {
|
||||
if err := s.walkPath(conn, fullPath, fn); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Stat returns metadata about stored data
|
||||
func (s *SMBStorage) Stat(ctx context.Context, key string) (*storage.StorageInfo, error) {
|
||||
conn, err := s.getConnection()
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, errors.ErrCodeStorageFailure, "failed to get SMB connection")
|
||||
}
|
||||
defer s.returnConnection(conn)
|
||||
|
||||
@@ -427,7 +433,7 @@ func (s *SMBStorage) Stat(ctx context.Context, key string) (*storage.StorageInfo
|
||||
info, err := conn.share.Stat(path)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return nil, errors.NotFound(fmt.Sprintf("file not found: %s", key))
|
||||
return nil, errors.NotFound(fmt.Sprintf("SMB file not found: %s", key))
|
||||
}
|
||||
return nil, errors.Wrap(err, errors.ErrCodeStorageFailure, "failed to stat SMB file")
|
||||
}
|
||||
@@ -439,35 +445,35 @@ func (s *SMBStorage) Stat(ctx context.Context, key string) (*storage.StorageInfo
|
||||
}, nil
|
||||
}
|
||||
|
||||
// GetQuota returns quota information
|
||||
// GetQuota returns current usage and quota information
|
||||
func (s *SMBStorage) GetQuota(ctx context.Context) (*storage.QuotaInfo, error) {
|
||||
s.mu.RLock()
|
||||
used := s.used
|
||||
s.mu.RUnlock()
|
||||
conn, err := s.getConnection()
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, errors.ErrCodeStorageFailure, "failed to get SMB connection")
|
||||
}
|
||||
defer s.returnConnection(conn)
|
||||
|
||||
available := s.quota - used
|
||||
if available < 0 {
|
||||
available = 0
|
||||
usage, err := s.calculateUsage(conn)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &storage.QuotaInfo{
|
||||
Used: used,
|
||||
Available: available,
|
||||
Limit: s.quota,
|
||||
Used: usage,
|
||||
Limit: s.maxSizeBytes,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Health checks SMB health
|
||||
// Health checks if the SMB backend is healthy
|
||||
func (s *SMBStorage) Health(ctx context.Context) error {
|
||||
conn, err := s.getConnection(ctx)
|
||||
conn, err := s.getConnection()
|
||||
if err != nil {
|
||||
return errors.Wrap(err, errors.ErrCodeStorageFailure, "SMB health check failed - connection error")
|
||||
return errors.Wrap(err, errors.ErrCodeStorageFailure, "SMB health check failed: cannot get connection")
|
||||
}
|
||||
defer s.returnConnection(conn)
|
||||
|
||||
// Try to stat the base path
|
||||
path := s.keyToPath("")
|
||||
_, err = conn.share.Stat(path)
|
||||
_, err = conn.share.Stat(s.config.Path)
|
||||
if err != nil && !os.IsNotExist(err) {
|
||||
return errors.Wrap(err, errors.ErrCodeStorageFailure, "SMB health check failed")
|
||||
}
|
||||
@@ -475,105 +481,62 @@ func (s *SMBStorage) Health(ctx context.Context) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Close closes the storage backend
|
||||
// Close closes the SMB storage backend
|
||||
func (s *SMBStorage) Close() error {
|
||||
close(s.connPool)
|
||||
|
||||
// Close all connections in pool
|
||||
for conn := range s.connPool {
|
||||
conn.close()
|
||||
}
|
||||
|
||||
log.Info().Msg("SMB storage closed")
|
||||
return nil
|
||||
}
|
||||
|
||||
// keyToPath converts a storage key to SMB path
|
||||
func (s *SMBStorage) keyToPath(key string) string {
|
||||
key = strings.TrimPrefix(key, "/")
|
||||
key = filepath.Clean(key)
|
||||
// Normalize separators to backslash for SMB
|
||||
key = strings.ReplaceAll(key, "/", "\\")
|
||||
|
||||
// Remove path traversal attempts
|
||||
for strings.HasPrefix(key, "../") || strings.HasPrefix(key, "..\\") {
|
||||
key = strings.TrimPrefix(key, "../")
|
||||
key = strings.TrimPrefix(key, "..\\")
|
||||
if s.config.Path == "" {
|
||||
return key
|
||||
}
|
||||
|
||||
key = filepath.Clean(key)
|
||||
if key == ".." || strings.HasPrefix(key, "../") || strings.HasPrefix(key, "..\\") {
|
||||
key = ""
|
||||
}
|
||||
|
||||
if s.basePath != "" {
|
||||
return filepath.Join(s.basePath, key)
|
||||
}
|
||||
return key
|
||||
// Use backslash for SMB paths
|
||||
return s.config.Path + "\\" + key
|
||||
}
|
||||
|
||||
// pathToKey converts an SMB path back to a storage key
|
||||
// pathToKey converts an SMB path to storage key
|
||||
func (s *SMBStorage) pathToKey(path string) string {
|
||||
if s.basePath != "" {
|
||||
path = strings.TrimPrefix(path, s.basePath)
|
||||
path = strings.TrimPrefix(path, "/")
|
||||
path = strings.TrimPrefix(path, "\\")
|
||||
// Remove base path
|
||||
if s.config.Path != "" {
|
||||
path = strings.TrimPrefix(path, s.config.Path+"\\")
|
||||
}
|
||||
return filepath.ToSlash(path)
|
||||
|
||||
// Convert backslashes to forward slashes for consistency
|
||||
return strings.ReplaceAll(path, "\\", "/")
|
||||
}
|
||||
|
||||
// walkPath recursively walks an SMB directory
|
||||
func (s *SMBStorage) walkPath(share *smb2.Share, path string, fn func(string, os.FileInfo) error) error {
|
||||
info, err := share.Stat(path)
|
||||
if err != nil {
|
||||
return err
|
||||
// calculateUsage calculates total storage usage
|
||||
func (s *SMBStorage) calculateUsage(conn *smbConnection) (int64, error) {
|
||||
var totalSize int64
|
||||
|
||||
basePath := s.config.Path
|
||||
if basePath == "" {
|
||||
basePath = "\\"
|
||||
}
|
||||
|
||||
if !info.IsDir() {
|
||||
return fn(path, info)
|
||||
}
|
||||
|
||||
entries, err := share.ReadDir(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, entry := range entries {
|
||||
entryPath := filepath.Join(path, entry.Name())
|
||||
if entry.IsDir() {
|
||||
if err := s.walkPath(share, entryPath, fn); err != nil {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
if err := fn(entryPath, entry); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// calculateUsage calculates current SMB storage usage
|
||||
func (s *SMBStorage) calculateUsage(ctx context.Context) error {
|
||||
conn, err := s.getConnection(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer s.returnConnection(conn)
|
||||
|
||||
var total int64
|
||||
basePath := s.keyToPath("")
|
||||
|
||||
err = s.walkPath(conn.share, basePath, func(path string, info os.FileInfo) error {
|
||||
err := s.walkPath(conn, basePath, func(path string, info os.FileInfo) error {
|
||||
if !info.IsDir() {
|
||||
total += info.Size()
|
||||
totalSize += info.Size()
|
||||
}
|
||||
return nil
|
||||
})
|
||||
|
||||
if err != nil && !os.IsNotExist(err) {
|
||||
return err
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("failed to calculate usage: %w", err)
|
||||
}
|
||||
|
||||
s.mu.Lock()
|
||||
s.used = total
|
||||
s.mu.Unlock()
|
||||
|
||||
metrics.UpdateCacheSize("smb", total)
|
||||
return nil
|
||||
return totalSize, nil
|
||||
}
|
||||
|
||||
@@ -0,0 +1,327 @@
|
||||
package smb
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/suite"
|
||||
)
|
||||
|
||||
type SMBStorageTestSuite struct {
|
||||
suite.Suite
|
||||
}
|
||||
|
||||
func TestSMBStorageTestSuite(t *testing.T) {
|
||||
suite.Run(t, new(SMBStorageTestSuite))
|
||||
}
|
||||
|
||||
func (s *SMBStorageTestSuite) TestNewSMBStorage() {
|
||||
tests := []struct {
|
||||
name string
|
||||
errorMsg string
|
||||
config Config
|
||||
expectError bool
|
||||
}{
|
||||
{
|
||||
name: "valid config",
|
||||
config: Config{
|
||||
Host: "fileserver.example.com",
|
||||
Port: 445,
|
||||
Share: "gohoarder",
|
||||
Path: "packages",
|
||||
Username: "testuser",
|
||||
Password: "testpass",
|
||||
Domain: "CORP",
|
||||
MaxSizeBytes: 1024 * 1024,
|
||||
PoolSize: 5,
|
||||
},
|
||||
expectError: false,
|
||||
},
|
||||
{
|
||||
name: "missing host",
|
||||
config: Config{
|
||||
Share: "gohoarder",
|
||||
Username: "testuser",
|
||||
Password: "testpass",
|
||||
},
|
||||
expectError: true,
|
||||
errorMsg: "host is required",
|
||||
},
|
||||
{
|
||||
name: "missing share",
|
||||
config: Config{
|
||||
Host: "fileserver.example.com",
|
||||
Username: "testuser",
|
||||
Password: "testpass",
|
||||
},
|
||||
expectError: true,
|
||||
errorMsg: "share is required",
|
||||
},
|
||||
{
|
||||
name: "default port",
|
||||
config: Config{
|
||||
Host: "fileserver.example.com",
|
||||
Share: "gohoarder",
|
||||
Username: "testuser",
|
||||
Password: "testpass",
|
||||
},
|
||||
expectError: false,
|
||||
},
|
||||
{
|
||||
name: "default pool size",
|
||||
config: Config{
|
||||
Host: "fileserver.example.com",
|
||||
Share: "gohoarder",
|
||||
Username: "testuser",
|
||||
Password: "testpass",
|
||||
},
|
||||
expectError: false,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
s.Run(tt.name, func() {
|
||||
storage, err := New(tt.config)
|
||||
|
||||
if tt.expectError {
|
||||
s.Error(err)
|
||||
if tt.errorMsg != "" {
|
||||
s.Contains(err.Error(), tt.errorMsg)
|
||||
}
|
||||
s.Nil(storage)
|
||||
} else {
|
||||
// Note: This will fail in actual execution since we can't connect to a real SMB server
|
||||
// But it tests the validation logic
|
||||
if err != nil {
|
||||
// Connection errors are expected in unit tests
|
||||
s.Contains(err.Error(), "Failed to create initial SMB connection")
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func (s *SMBStorageTestSuite) TestKeyToPath() {
|
||||
tests := []struct {
|
||||
name string
|
||||
basePath string
|
||||
key string
|
||||
expectedWin string // Expected Windows-style path
|
||||
}{
|
||||
{
|
||||
name: "simple key with base path",
|
||||
basePath: "packages",
|
||||
key: "test/file.txt",
|
||||
expectedWin: "packages\\test\\file.txt",
|
||||
},
|
||||
{
|
||||
name: "simple key without base path",
|
||||
basePath: "",
|
||||
key: "test/file.txt",
|
||||
expectedWin: "test\\file.txt",
|
||||
},
|
||||
{
|
||||
name: "nested key",
|
||||
basePath: "cache",
|
||||
key: "deep/nested/path/file.txt",
|
||||
expectedWin: "cache\\deep\\nested\\path\\file.txt",
|
||||
},
|
||||
{
|
||||
name: "key with backslashes",
|
||||
basePath: "packages",
|
||||
key: "test\\file.txt",
|
||||
expectedWin: "packages\\test\\file.txt",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
s.Run(tt.name, func() {
|
||||
storage := &SMBStorage{
|
||||
config: Config{
|
||||
Path: tt.basePath,
|
||||
},
|
||||
}
|
||||
|
||||
result := storage.keyToPath(tt.key)
|
||||
s.Equal(tt.expectedWin, result)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func (s *SMBStorageTestSuite) TestPathToKey() {
|
||||
tests := []struct {
|
||||
name string
|
||||
basePath string
|
||||
path string
|
||||
expected string
|
||||
}{
|
||||
{
|
||||
name: "windows path with base path",
|
||||
basePath: "packages",
|
||||
path: "packages\\test\\file.txt",
|
||||
expected: "test/file.txt",
|
||||
},
|
||||
{
|
||||
name: "windows path without base path",
|
||||
basePath: "",
|
||||
path: "test\\file.txt",
|
||||
expected: "test/file.txt",
|
||||
},
|
||||
{
|
||||
name: "nested windows path",
|
||||
basePath: "cache",
|
||||
path: "cache\\deep\\nested\\path\\file.txt",
|
||||
expected: "deep/nested/path/file.txt",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
s.Run(tt.name, func() {
|
||||
storage := &SMBStorage{
|
||||
config: Config{
|
||||
Path: tt.basePath,
|
||||
},
|
||||
}
|
||||
|
||||
result := storage.pathToKey(tt.path)
|
||||
s.Equal(tt.expected, result)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func (s *SMBStorageTestSuite) TestConfigDefaults() {
|
||||
config := Config{
|
||||
Host: "fileserver.example.com",
|
||||
Share: "gohoarder",
|
||||
Username: "testuser",
|
||||
Password: "testpass",
|
||||
}
|
||||
|
||||
// This will fail to connect, but we can verify the config validation
|
||||
_, err := New(config)
|
||||
|
||||
// We expect a connection error, not a validation error
|
||||
if err != nil {
|
||||
s.NotContains(err.Error(), "host is required")
|
||||
s.NotContains(err.Error(), "share is required")
|
||||
}
|
||||
}
|
||||
|
||||
func (s *SMBStorageTestSuite) TestPathNormalization() {
|
||||
tests := []struct {
|
||||
name string
|
||||
inputPath string
|
||||
expectedPath string
|
||||
}{
|
||||
{
|
||||
name: "path with trailing slash",
|
||||
inputPath: "packages/",
|
||||
expectedPath: "packages",
|
||||
},
|
||||
{
|
||||
name: "path with trailing backslash",
|
||||
inputPath: "packages\\",
|
||||
expectedPath: "packages",
|
||||
},
|
||||
{
|
||||
name: "path without trailing slash",
|
||||
inputPath: "packages",
|
||||
expectedPath: "packages",
|
||||
},
|
||||
{
|
||||
name: "empty path",
|
||||
inputPath: "",
|
||||
expectedPath: "",
|
||||
},
|
||||
{
|
||||
name: "nested path with trailing slash",
|
||||
inputPath: "cache/packages/",
|
||||
expectedPath: "cache/packages",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
s.Run(tt.name, func() {
|
||||
config := Config{
|
||||
Host: "fileserver.example.com",
|
||||
Share: "gohoarder",
|
||||
Path: tt.inputPath,
|
||||
Username: "testuser",
|
||||
Password: "testpass",
|
||||
}
|
||||
|
||||
// This will fail to connect, but we can check the config
|
||||
storage, _ := New(config)
|
||||
if storage != nil {
|
||||
s.Equal(tt.expectedPath, storage.config.Path)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func (s *SMBStorageTestSuite) TestPoolSizeDefaults() {
|
||||
config := Config{
|
||||
Host: "fileserver.example.com",
|
||||
Share: "gohoarder",
|
||||
Username: "testuser",
|
||||
Password: "testpass",
|
||||
}
|
||||
|
||||
storage, _ := New(config)
|
||||
if storage != nil {
|
||||
s.Equal(5, storage.poolSize) // Default pool size
|
||||
}
|
||||
}
|
||||
|
||||
func (s *SMBStorageTestSuite) TestPortDefaults() {
|
||||
config := Config{
|
||||
Host: "fileserver.example.com",
|
||||
Share: "gohoarder",
|
||||
Username: "testuser",
|
||||
Password: "testpass",
|
||||
}
|
||||
|
||||
storage, _ := New(config)
|
||||
if storage != nil {
|
||||
s.Equal(445, storage.config.Port) // Default SMB port
|
||||
}
|
||||
}
|
||||
|
||||
func (s *SMBStorageTestSuite) TestClose() {
|
||||
// Create a storage instance (will fail to connect but that's ok)
|
||||
config := Config{
|
||||
Host: "fileserver.example.com",
|
||||
Share: "gohoarder",
|
||||
Username: "testuser",
|
||||
Password: "testpass",
|
||||
}
|
||||
|
||||
storage, _ := New(config)
|
||||
if storage != nil {
|
||||
// Close should not panic
|
||||
err := storage.Close()
|
||||
s.NoError(err)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *SMBStorageTestSuite) TestConnectionPoolChannel() {
|
||||
config := Config{
|
||||
Host: "fileserver.example.com",
|
||||
Share: "gohoarder",
|
||||
Username: "testuser",
|
||||
Password: "testpass",
|
||||
PoolSize: 10,
|
||||
}
|
||||
|
||||
storage, _ := New(config)
|
||||
if storage != nil {
|
||||
// Verify pool channel capacity
|
||||
s.NotNil(storage.connPool)
|
||||
s.Equal(10, cap(storage.connPool))
|
||||
}
|
||||
}
|
||||
|
||||
func (s *SMBStorageTestSuite) TestSMBConnectionStruct() {
|
||||
// Verify smbConnection structure exists and has required fields
|
||||
conn := &smbConnection{}
|
||||
s.NotNil(conn)
|
||||
}
|
||||
Reference in New Issue
Block a user