fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! General improvements

chart/Chart.yaml

@@ -10,9 +10,9 @@ description: |
 
 type: application
 
-version: 0.2.35
+version: 0.2.36
 
-appVersion: "0.2.35"
+appVersion: "0.2.36"
 
 home: https://github.com/lukaszraczylo/kubernetes-images-sync-operator
 

chart/values.yaml

@@ -12,7 +12,7 @@ sa:
         - ALL
     image:
       repository: ghcr.io/lukaszraczylo/kubernetes-images-sync-operator
-      tag: 0.2.35
+      tag: 0.2.36
     resources:
       limits:
         cpu: 500m

docker-image-worker/export.py

@@ -2,41 +2,79 @@
 import os
 import sys
 import argparse
-from botocore.exceptions import ClientError
+import logging
+from botocore.exceptions import ClientError, BotoCoreError
 from tenacity import retry, stop_after_attempt, wait_fixed
 
+# Configure logging
+logging.basicConfig(
+    level=logging.INFO,
+    format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
+)
+logger = logging.getLogger(__name__)
+
 sys.path.append(os.path.dirname(os.path.abspath(__file__)))
 from s3_utils import get_s3_client, parse_s3_path, add_common_arguments, validate_args
 
+def log_error_details(e):
+    """Log detailed error information from AWS exceptions"""
+    if hasattr(e, 'response'):
+        error_code = e.response.get('Error', {}).get('Code', 'Unknown')
+        error_message = e.response.get('Error', {}).get('Message', str(e))
+        request_id = e.response.get('ResponseMetadata', {}).get('RequestId', 'Unknown')
+        logger.error(f"AWS Error Details:")
+        logger.error(f"- Error Code: {error_code}")
+        logger.error(f"- Error Message: {error_message}")
+        logger.error(f"- Request ID: {request_id}")
+        logger.error(f"- Full Response: {e.response}")
+    else:
+        logger.error(f"Non-AWS Error: {str(e)}")
+
 @retry(stop=stop_after_attempt(5), wait=wait_fixed(5))
 def transfer_file(source, destination, use_role=False, role_name=None, aws_access_key_id=None, aws_secret_access_key=None, endpoint_url=None, region=None):
     """
     Transfer a file from a local source to either a local destination or an S3 bucket
     """
     if not os.path.isfile(source):
-        print(f"Error: Source file '{source}' does not exist or is not a file.")
+        logger.error(f"Error: Source file '{source}' does not exist or is not a file.")
         return False
 
     if destination.startswith('s3://'):
         # Uploading to S3
-        s3_client = get_s3_client(use_role, role_name, aws_access_key_id, aws_secret_access_key, endpoint_url, region)
-        bucket, s3_key = parse_s3_path(destination)
         try:
-            s3_client.upload_file(source, bucket, s3_key)
+            logger.info(f"Attempting to upload {source} to {destination}")
-            print(f"File {source} uploaded successfully to {destination}")
+            s3_client = get_s3_client(use_role, role_name, aws_access_key_id, aws_secret_access_key, endpoint_url, region)
-        except ClientError as e:
+            bucket, s3_key = parse_s3_path(destination)
-            print(f"Error uploading file: {str(e)}")
+            
+            try:
+                s3_client.upload_file(source, bucket, s3_key)
+                logger.info(f"File {source} uploaded successfully to {destination}")
+            except ClientError as e:
+                log_error_details(e)
+                if "AccessDenied" in str(e):
+                    logger.error("Access denied. Please check:")
+                    logger.error("1. IAM role/user permissions")
+                    logger.error("2. S3 bucket permissions")
+                    logger.error("3. Web identity token configuration")
+                return False
+            except BotoCoreError as e:
+                logger.error(f"Boto3 error during upload: {str(e)}")
+                return False
+            
+        except Exception as e:
+            logger.error(f"Unexpected error during S3 client creation or upload: {str(e)}")
             return False
     else:
         # Copying to local destination
         try:
             import shutil
+            logger.info(f"Attempting to copy {source} to local destination {destination}")
             # Create destination directory if it doesn't exist
             os.makedirs(os.path.dirname(destination), exist_ok=True)
             shutil.copy2(source, destination)
-            print(f"File {source} copied successfully to {destination}")
+            logger.info(f"File {source} copied successfully to {destination}")
         except IOError as e:
-            print(f"Error copying file: {str(e)}")
+            logger.error(f"Error copying file: {str(e)}")
             return False
     return True
 
@@ -61,7 +99,7 @@ if __name__ == "__main__":
     )
 
     if success:
-        print("Transfer completed successfully.")
+        logger.info("Transfer completed successfully.")
     else:
-        print("Transfer failed.")
+        logger.error("Transfer failed.")
-        exit(1)
+        exit(1)

docker-image-worker/s3_utils.py

@@ -1,37 +1,77 @@
 import boto3
 from botocore.exceptions import ClientError
 
+import os
+import logging
+
 def get_s3_client(use_role=False, role_name=None, aws_access_key_id=None, aws_secret_access_key=None, endpoint_url=None, region=None):
     """
     Create and return an S3 client based on the provided authentication method, endpoint, and region.
     """
+    logging.basicConfig(level=logging.INFO)
+    logger = logging.getLogger(__name__)
+
     client_kwargs = {}
     
+    # Log authentication method being attempted
+    logger.info("Attempting S3 client creation with:")
+    logger.info(f"- Region: {region if region else 'default'}")
+    logger.info(f"- Endpoint URL: {endpoint_url if endpoint_url else 'default'}")
+    
     if endpoint_url:
         client_kwargs['endpoint_url'] = endpoint_url
     elif region:
         client_kwargs['region_name'] = region
 
+    # Check for AWS Web Identity token
+    token_file = os.environ.get('AWS_WEB_IDENTITY_TOKEN_FILE')
+    role_arn = os.environ.get('AWS_ROLE_ARN')
+    if token_file or role_arn:
+        logger.info("AWS Web Identity configuration detected:")
+        logger.info(f"- Token file path: {token_file}")
+        logger.info(f"- Role ARN: {role_arn}")
+        logger.info(f"- Session name: {os.environ.get('AWS_ROLE_SESSION_NAME', 'default')}")
+
     if aws_access_key_id and aws_secret_access_key:
+        logger.info("Using explicit AWS credentials")
         # Use explicit credentials if provided
         client_kwargs['aws_access_key_id'] = aws_access_key_id
         client_kwargs['aws_secret_access_key'] = aws_secret_access_key
         return boto3.client('s3', **client_kwargs)
     elif use_role and role_name:
         # Assume specific role if requested
-        sts_client = boto3.client('sts')
+        logger.info(f"Attempting to assume role: {role_name}")
-        assumed_role_object = sts_client.assume_role(
+        try:
+            sts_client = boto3.client('sts')
+            # Get current identity for logging
+            identity = sts_client.get_caller_identity()
+            logger.info(f"Current identity: {identity['Arn']}")
+            
+            assumed_role_object = sts_client.assume_role(
             RoleArn=f"arn:aws:iam::{boto3.client('sts').get_caller_identity()['Account']}:role/{role_name}",
             RoleSessionName="AssumeRoleSession"
         )
-        credentials = assumed_role_object['Credentials']
+            credentials = assumed_role_object['Credentials']
-        client_kwargs['aws_access_key_id'] = credentials['AccessKeyId']
+            client_kwargs['aws_access_key_id'] = credentials['AccessKeyId']
-        client_kwargs['aws_secret_access_key'] = credentials['SecretAccessKey']
+            client_kwargs['aws_secret_access_key'] = credentials['SecretAccessKey']
-        client_kwargs['aws_session_token'] = credentials['SessionToken']
+            client_kwargs['aws_session_token'] = credentials['SessionToken']
-        return boto3.client('s3', **client_kwargs)
+            return boto3.client('s3', **client_kwargs)
+        except Exception as e:
+            logger.error(f"Failed to assume role {role_name}: {str(e)}")
+            raise
     else:
         # Use default credentials (environment, instance profile, or pod service account)
-        return boto3.client('s3', **client_kwargs)
+        logger.info("Using default credential provider chain")
+        try:
+            client = boto3.client('s3', **client_kwargs)
+            # Try to get caller identity to verify credentials
+            sts = boto3.client('sts')
+            identity = sts.get_caller_identity()
+            logger.info(f"Successfully authenticated as: {identity['Arn']}")
+            return client
+        except Exception as e:
+            logger.error(f"Failed to create S3 client: {str(e)}")
+            raise
 
 def parse_s3_path(s3_path):
     """