diff --git a/chart/Chart.yaml b/chart/Chart.yaml index 8b9c67a..bc02e96 100644 --- a/chart/Chart.yaml +++ b/chart/Chart.yaml @@ -10,9 +10,9 @@ description: | type: application -version: 0.2.35 +version: 0.2.36 -appVersion: "0.2.35" +appVersion: "0.2.36" home: https://github.com/lukaszraczylo/kubernetes-images-sync-operator diff --git a/chart/values.yaml b/chart/values.yaml index a51d9d8..b2540f4 100644 --- a/chart/values.yaml +++ b/chart/values.yaml @@ -12,7 +12,7 @@ sa: - ALL image: repository: ghcr.io/lukaszraczylo/kubernetes-images-sync-operator - tag: 0.2.35 + tag: 0.2.36 resources: limits: cpu: 500m diff --git a/docker-image-worker/export.py b/docker-image-worker/export.py index a7dd63b..6cdd6c9 100755 --- a/docker-image-worker/export.py +++ b/docker-image-worker/export.py @@ -2,41 +2,79 @@ import os import sys import argparse -from botocore.exceptions import ClientError +import logging +from botocore.exceptions import ClientError, BotoCoreError from tenacity import retry, stop_after_attempt, wait_fixed +# Configure logging +logging.basicConfig( + level=logging.INFO, + format='%(asctime)s - %(name)s - %(levelname)s - %(message)s' +) +logger = logging.getLogger(__name__) + sys.path.append(os.path.dirname(os.path.abspath(__file__))) from s3_utils import get_s3_client, parse_s3_path, add_common_arguments, validate_args +def log_error_details(e): + """Log detailed error information from AWS exceptions""" + if hasattr(e, 'response'): + error_code = e.response.get('Error', {}).get('Code', 'Unknown') + error_message = e.response.get('Error', {}).get('Message', str(e)) + request_id = e.response.get('ResponseMetadata', {}).get('RequestId', 'Unknown') + logger.error(f"AWS Error Details:") + logger.error(f"- Error Code: {error_code}") + logger.error(f"- Error Message: {error_message}") + logger.error(f"- Request ID: {request_id}") + logger.error(f"- Full Response: {e.response}") + else: + logger.error(f"Non-AWS Error: {str(e)}") + @retry(stop=stop_after_attempt(5), wait=wait_fixed(5)) def transfer_file(source, destination, use_role=False, role_name=None, aws_access_key_id=None, aws_secret_access_key=None, endpoint_url=None, region=None): """ Transfer a file from a local source to either a local destination or an S3 bucket """ if not os.path.isfile(source): - print(f"Error: Source file '{source}' does not exist or is not a file.") + logger.error(f"Error: Source file '{source}' does not exist or is not a file.") return False if destination.startswith('s3://'): # Uploading to S3 - s3_client = get_s3_client(use_role, role_name, aws_access_key_id, aws_secret_access_key, endpoint_url, region) - bucket, s3_key = parse_s3_path(destination) try: - s3_client.upload_file(source, bucket, s3_key) - print(f"File {source} uploaded successfully to {destination}") - except ClientError as e: - print(f"Error uploading file: {str(e)}") + logger.info(f"Attempting to upload {source} to {destination}") + s3_client = get_s3_client(use_role, role_name, aws_access_key_id, aws_secret_access_key, endpoint_url, region) + bucket, s3_key = parse_s3_path(destination) + + try: + s3_client.upload_file(source, bucket, s3_key) + logger.info(f"File {source} uploaded successfully to {destination}") + except ClientError as e: + log_error_details(e) + if "AccessDenied" in str(e): + logger.error("Access denied. Please check:") + logger.error("1. IAM role/user permissions") + logger.error("2. S3 bucket permissions") + logger.error("3. Web identity token configuration") + return False + except BotoCoreError as e: + logger.error(f"Boto3 error during upload: {str(e)}") + return False + + except Exception as e: + logger.error(f"Unexpected error during S3 client creation or upload: {str(e)}") return False else: # Copying to local destination try: import shutil + logger.info(f"Attempting to copy {source} to local destination {destination}") # Create destination directory if it doesn't exist os.makedirs(os.path.dirname(destination), exist_ok=True) shutil.copy2(source, destination) - print(f"File {source} copied successfully to {destination}") + logger.info(f"File {source} copied successfully to {destination}") except IOError as e: - print(f"Error copying file: {str(e)}") + logger.error(f"Error copying file: {str(e)}") return False return True @@ -61,7 +99,7 @@ if __name__ == "__main__": ) if success: - print("Transfer completed successfully.") + logger.info("Transfer completed successfully.") else: - print("Transfer failed.") - exit(1) \ No newline at end of file + logger.error("Transfer failed.") + exit(1) diff --git a/docker-image-worker/s3_utils.py b/docker-image-worker/s3_utils.py index 656547e..7de2f64 100644 --- a/docker-image-worker/s3_utils.py +++ b/docker-image-worker/s3_utils.py @@ -1,37 +1,77 @@ import boto3 from botocore.exceptions import ClientError +import os +import logging + def get_s3_client(use_role=False, role_name=None, aws_access_key_id=None, aws_secret_access_key=None, endpoint_url=None, region=None): """ Create and return an S3 client based on the provided authentication method, endpoint, and region. """ + logging.basicConfig(level=logging.INFO) + logger = logging.getLogger(__name__) + client_kwargs = {} + # Log authentication method being attempted + logger.info("Attempting S3 client creation with:") + logger.info(f"- Region: {region if region else 'default'}") + logger.info(f"- Endpoint URL: {endpoint_url if endpoint_url else 'default'}") + if endpoint_url: client_kwargs['endpoint_url'] = endpoint_url elif region: client_kwargs['region_name'] = region + # Check for AWS Web Identity token + token_file = os.environ.get('AWS_WEB_IDENTITY_TOKEN_FILE') + role_arn = os.environ.get('AWS_ROLE_ARN') + if token_file or role_arn: + logger.info("AWS Web Identity configuration detected:") + logger.info(f"- Token file path: {token_file}") + logger.info(f"- Role ARN: {role_arn}") + logger.info(f"- Session name: {os.environ.get('AWS_ROLE_SESSION_NAME', 'default')}") + if aws_access_key_id and aws_secret_access_key: + logger.info("Using explicit AWS credentials") # Use explicit credentials if provided client_kwargs['aws_access_key_id'] = aws_access_key_id client_kwargs['aws_secret_access_key'] = aws_secret_access_key return boto3.client('s3', **client_kwargs) elif use_role and role_name: # Assume specific role if requested - sts_client = boto3.client('sts') - assumed_role_object = sts_client.assume_role( + logger.info(f"Attempting to assume role: {role_name}") + try: + sts_client = boto3.client('sts') + # Get current identity for logging + identity = sts_client.get_caller_identity() + logger.info(f"Current identity: {identity['Arn']}") + + assumed_role_object = sts_client.assume_role( RoleArn=f"arn:aws:iam::{boto3.client('sts').get_caller_identity()['Account']}:role/{role_name}", RoleSessionName="AssumeRoleSession" ) - credentials = assumed_role_object['Credentials'] - client_kwargs['aws_access_key_id'] = credentials['AccessKeyId'] - client_kwargs['aws_secret_access_key'] = credentials['SecretAccessKey'] - client_kwargs['aws_session_token'] = credentials['SessionToken'] - return boto3.client('s3', **client_kwargs) + credentials = assumed_role_object['Credentials'] + client_kwargs['aws_access_key_id'] = credentials['AccessKeyId'] + client_kwargs['aws_secret_access_key'] = credentials['SecretAccessKey'] + client_kwargs['aws_session_token'] = credentials['SessionToken'] + return boto3.client('s3', **client_kwargs) + except Exception as e: + logger.error(f"Failed to assume role {role_name}: {str(e)}") + raise else: # Use default credentials (environment, instance profile, or pod service account) - return boto3.client('s3', **client_kwargs) + logger.info("Using default credential provider chain") + try: + client = boto3.client('s3', **client_kwargs) + # Try to get caller identity to verify credentials + sts = boto3.client('sts') + identity = sts.get_caller_identity() + logger.info(f"Successfully authenticated as: {identity['Arn']}") + return client + except Exception as e: + logger.error(f"Failed to create S3 client: {str(e)}") + raise def parse_s3_path(s3_path): """