lukaszraczylo/kubernetes-images-sync-operator
1
0
Fork 0
mirror of https://github.com/lukaszraczylo/kubernetes-images-sync-operator.git synced 2026-06-05 22:53:39 +00:00
Code Issues Packages Projects Releases Wiki Activity

fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! General improvements

Browse Source
This commit is contained in:
Lukasz Raczylo
2025-01-10 13:57:40 +00:00
parent cabf085e6c
commit 202cfc3869
4 changed files with 102 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files
chart/Chart.yaml
+2 -2
View File
@@ -10,9 +10,9 @@ description: |
type: application
version: 0.2.35
version: 0.2.36
appVersion: "0.2.35"
appVersion: "0.2.36"
home: https://github.com/lukaszraczylo/kubernetes-images-sync-operator
chart/values.yaml
+1 -1
View File
@@ -12,7 +12,7 @@ sa:
- ALL
image:
repository: ghcr.io/lukaszraczylo/kubernetes-images-sync-operator
tag: 0.2.35
tag: 0.2.36
resources:
limits:
cpu: 500m
docker-image-worker/export.py
+51 -13
View File
@@ -2,41 +2,79 @@
import os
import sys
import argparse
from botocore.exceptions import ClientError
import logging
from botocore.exceptions import ClientError, BotoCoreError
from tenacity import retry, stop_after_attempt, wait_fixed
# Configure logging
logging.basicConfig(
level=logging.INFO,
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
)
logger = logging.getLogger(__name__)
sys.path.append(os.path.dirname(os.path.abspath(__file__)))
from s3_utils import get_s3_client, parse_s3_path, add_common_arguments, validate_args
def log_error_details(e):
"""Log detailed error information from AWS exceptions"""
if hasattr(e, 'response'):
error_code = e.response.get('Error', {}).get('Code', 'Unknown')
error_message = e.response.get('Error', {}).get('Message', str(e))
request_id = e.response.get('ResponseMetadata', {}).get('RequestId', 'Unknown')
logger.error(f"AWS Error Details:")
logger.error(f"- Error Code: {error_code}")
logger.error(f"- Error Message: {error_message}")
logger.error(f"- Request ID: {request_id}")
logger.error(f"- Full Response: {e.response}")
else:
logger.error(f"Non-AWS Error: {str(e)}")
@retry(stop=stop_after_attempt(5), wait=wait_fixed(5))
def transfer_file(source, destination, use_role=False, role_name=None, aws_access_key_id=None, aws_secret_access_key=None, endpoint_url=None, region=None):
"""
Transfer a file from a local source to either a local destination or an S3 bucket
"""
if not os.path.isfile(source):
print(f"Error: Source file '{source}' does not exist or is not a file.")
logger.error(f"Error: Source file '{source}' does not exist or is not a file.")
return False
if destination.startswith('s3://'):
# Uploading to S3
s3_client = get_s3_client(use_role, role_name, aws_access_key_id, aws_secret_access_key, endpoint_url, region)
bucket, s3_key = parse_s3_path(destination)
try:
s3_client.upload_file(source, bucket, s3_key)
print(f"File {source} uploaded successfully to {destination}")
except ClientError as e:
print(f"Error uploading file: {str(e)}")
logger.info(f"Attempting to upload {source} to {destination}")
s3_client = get_s3_client(use_role, role_name, aws_access_key_id, aws_secret_access_key, endpoint_url, region)
bucket, s3_key = parse_s3_path(destination)
try:
s3_client.upload_file(source, bucket, s3_key)
logger.info(f"File {source} uploaded successfully to {destination}")
except ClientError as e:
log_error_details(e)
if "AccessDenied" in str(e):
logger.error("Access denied. Please check:")
logger.error("1. IAM role/user permissions")
logger.error("2. S3 bucket permissions")
logger.error("3. Web identity token configuration")
return False
except BotoCoreError as e:
logger.error(f"Boto3 error during upload: {str(e)}")
return False
except Exception as e:
logger.error(f"Unexpected error during S3 client creation or upload: {str(e)}")
return False
else:
# Copying to local destination
try:
import shutil
logger.info(f"Attempting to copy {source} to local destination {destination}")
# Create destination directory if it doesn't exist
os.makedirs(os.path.dirname(destination), exist_ok=True)
shutil.copy2(source, destination)
print(f"File {source} copied successfully to {destination}")
logger.info(f"File {source} copied successfully to {destination}")
except IOError as e:
print(f"Error copying file: {str(e)}")
logger.error(f"Error copying file: {str(e)}")
return False
return True
@@ -61,7 +99,7 @@ if __name__ == "__main__":
)
if success:
print("Transfer completed successfully.")
logger.info("Transfer completed successfully.")
else:
print("Transfer failed.")
exit(1)
logger.error("Transfer failed.")
exit(1)
docker-image-worker/s3_utils.py
+48 -8
View File
@@ -1,37 +1,77 @@
import boto3
from botocore.exceptions import ClientError
import os
import logging
def get_s3_client(use_role=False, role_name=None, aws_access_key_id=None, aws_secret_access_key=None, endpoint_url=None, region=None):
"""
Create and return an S3 client based on the provided authentication method, endpoint, and region.
"""
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)
client_kwargs = {}
# Log authentication method being attempted
logger.info("Attempting S3 client creation with:")
logger.info(f"- Region: {region if region else 'default'}")
logger.info(f"- Endpoint URL: {endpoint_url if endpoint_url else 'default'}")
if endpoint_url:
client_kwargs['endpoint_url'] = endpoint_url
elif region:
client_kwargs['region_name'] = region
# Check for AWS Web Identity token
token_file = os.environ.get('AWS_WEB_IDENTITY_TOKEN_FILE')
role_arn = os.environ.get('AWS_ROLE_ARN')
if token_file or role_arn:
logger.info("AWS Web Identity configuration detected:")
logger.info(f"- Token file path: {token_file}")
logger.info(f"- Role ARN: {role_arn}")
logger.info(f"- Session name: {os.environ.get('AWS_ROLE_SESSION_NAME', 'default')}")
if aws_access_key_id and aws_secret_access_key:
logger.info("Using explicit AWS credentials")
# Use explicit credentials if provided
client_kwargs['aws_access_key_id'] = aws_access_key_id
client_kwargs['aws_secret_access_key'] = aws_secret_access_key
return boto3.client('s3', **client_kwargs)
elif use_role and role_name:
# Assume specific role if requested
sts_client = boto3.client('sts')
assumed_role_object = sts_client.assume_role(
logger.info(f"Attempting to assume role: {role_name}")
try:
sts_client = boto3.client('sts')
# Get current identity for logging
identity = sts_client.get_caller_identity()
logger.info(f"Current identity: {identity['Arn']}")
assumed_role_object = sts_client.assume_role(
RoleArn=f"arn:aws:iam::{boto3.client('sts').get_caller_identity()['Account']}:role/{role_name}",
RoleSessionName="AssumeRoleSession"
)
credentials = assumed_role_object['Credentials']
client_kwargs['aws_access_key_id'] = credentials['AccessKeyId']
client_kwargs['aws_secret_access_key'] = credentials['SecretAccessKey']
client_kwargs['aws_session_token'] = credentials['SessionToken']
return boto3.client('s3', **client_kwargs)
credentials = assumed_role_object['Credentials']
client_kwargs['aws_access_key_id'] = credentials['AccessKeyId']
client_kwargs['aws_secret_access_key'] = credentials['SecretAccessKey']
client_kwargs['aws_session_token'] = credentials['SessionToken']
return boto3.client('s3', **client_kwargs)
except Exception as e:
logger.error(f"Failed to assume role {role_name}: {str(e)}")
raise
else:
# Use default credentials (environment, instance profile, or pod service account)
return boto3.client('s3', **client_kwargs)
logger.info("Using default credential provider chain")
try:
client = boto3.client('s3', **client_kwargs)
# Try to get caller identity to verify credentials
sts = boto3.client('sts')
identity = sts.get_caller_identity()
logger.info(f"Successfully authenticated as: {identity['Arn']}")
return client
except Exception as e:
logger.error(f"Failed to create S3 client: {str(e)}")
raise
def parse_s3_path(s3_path):
"""