mirror of
https://github.com/lukaszraczylo/kubernetes-images-sync-operator.git
synced 2026-06-11 23:38:45 +00:00
More fixes, moving from python to golang worker.
This commit is contained in:
@@ -1,40 +0,0 @@
|
||||
FROM ubuntu:22.04
|
||||
|
||||
ENV DEBIAN_FRONTEND=noninteractive
|
||||
ARG TARGETPLATFORM
|
||||
ARG TARGETARCH
|
||||
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
curl \
|
||||
gnupg2 \
|
||||
python3-pip \
|
||||
sudo \
|
||||
jq \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
RUN echo "deb [arch=${TARGETARCH}] https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/unstable/xUbuntu_22.04/ /" | tee /etc/apt/sources.list.d/devel:kubic:libcontainers:unstable.list \
|
||||
&& curl -fsSL "https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/unstable/xUbuntu_22.04/Release.key" | apt-key add -
|
||||
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
uidmap \
|
||||
fuse-overlayfs \
|
||||
podman \
|
||||
netavark \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
RUN adduser --disabled-password --gecos "" --uid 1001 runner \
|
||||
&& groupadd docker --gid 123 \
|
||||
&& usermod -aG sudo,docker runner \
|
||||
&& echo "%sudo ALL=(ALL:ALL) NOPASSWD:ALL" > /etc/sudoers \
|
||||
&& echo "Defaults env_keep += \"DEBIAN_FRONTEND\"" >> /etc/sudoers
|
||||
|
||||
WORKDIR /home/runner
|
||||
|
||||
COPY storage.conf containers.conf registries.conf /home/runner/.config/containers/
|
||||
COPY requirements.txt export.py cleanup.py s3_utils.py podman-preauth.sh ./
|
||||
USER runner
|
||||
RUN sudo chown -R runner:runner /home/runner/.config \
|
||||
&& python3 -m pip install --no-cache-dir --only-binary=:all: -r requirements.txt \
|
||||
&& sudo chmod +x podman-preauth.sh
|
||||
ENTRYPOINT ["/home/runner/podman-preauth.sh"]
|
||||
CMD ["bash", "-c"]
|
||||
@@ -1,67 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
import os
|
||||
import sys
|
||||
import argparse
|
||||
from botocore.exceptions import ClientError
|
||||
from tenacity import retry, stop_after_attempt, wait_fixed
|
||||
|
||||
sys.path.append(os.path.dirname(os.path.abspath(__file__)))
|
||||
|
||||
from s3_utils import get_s3_client, parse_s3_path, add_common_arguments, validate_args
|
||||
|
||||
@retry(stop=stop_after_attempt(5), wait=wait_fixed(5))
|
||||
def remove_directory(destination, use_role=False, role_name=None, aws_access_key_id=None, aws_secret_access_key=None, endpoint_url=None, region=None):
|
||||
"""
|
||||
Remove a directory recursively, either local or in an S3 bucket
|
||||
"""
|
||||
if destination.startswith('s3://'):
|
||||
# Removing from S3
|
||||
s3_client = get_s3_client(use_role, role_name, aws_access_key_id, aws_secret_access_key, endpoint_url, region)
|
||||
bucket, prefix = parse_s3_path(destination)
|
||||
try:
|
||||
paginator = s3_client.get_paginator('list_objects_v2')
|
||||
for page in paginator.paginate(Bucket=bucket, Prefix=prefix):
|
||||
if 'Contents' in page:
|
||||
objects_to_delete = [{'Key': obj['Key']} for obj in page['Contents']]
|
||||
s3_client.delete_objects(Bucket=bucket, Delete={'Objects': objects_to_delete})
|
||||
print(f"Directory {destination} removed successfully from S3")
|
||||
except ClientError as e:
|
||||
print(f"Error removing directory from S3: {str(e)}")
|
||||
return False
|
||||
else:
|
||||
# Removing local directory
|
||||
try:
|
||||
import shutil
|
||||
if os.path.exists(destination):
|
||||
shutil.rmtree(destination)
|
||||
print(f"Directory {destination} removed successfully")
|
||||
else:
|
||||
print(f"Directory {destination} does not exist")
|
||||
except IOError as e:
|
||||
print(f"Error removing directory: {str(e)}")
|
||||
return False
|
||||
return True
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser(description="Remove a directory recursively, either local or in an S3 bucket.")
|
||||
parser.add_argument("destination", help="The directory path (local) or S3 path (e.g., 's3://bucket/prefix') to remove")
|
||||
add_common_arguments(parser)
|
||||
|
||||
args = parser.parse_args()
|
||||
validate_args(args, parser)
|
||||
|
||||
success = remove_directory(
|
||||
args.destination,
|
||||
args.use_role,
|
||||
args.role_name,
|
||||
args.aws_access_key_id,
|
||||
args.aws_secret_access_key,
|
||||
args.endpoint_url,
|
||||
args.region
|
||||
)
|
||||
|
||||
if success:
|
||||
print("Cleanup completed successfully.")
|
||||
else:
|
||||
print("Cleanup failed.")
|
||||
exit(1)
|
||||
@@ -1,106 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
import os
|
||||
import sys
|
||||
import argparse
|
||||
import logging
|
||||
from botocore.exceptions import ClientError, BotoCoreError
|
||||
from tenacity import retry, stop_after_attempt, wait_fixed
|
||||
|
||||
# Configure logging
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
|
||||
)
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
sys.path.append(os.path.dirname(os.path.abspath(__file__)))
|
||||
from s3_utils import get_s3_client, parse_s3_path, add_common_arguments, validate_args
|
||||
|
||||
def log_error_details(e):
|
||||
"""Log detailed error information from AWS exceptions"""
|
||||
if hasattr(e, 'response'):
|
||||
error_code = e.response.get('Error', {}).get('Code', 'Unknown')
|
||||
error_message = e.response.get('Error', {}).get('Message', str(e))
|
||||
request_id = e.response.get('ResponseMetadata', {}).get('RequestId', 'Unknown')
|
||||
logger.error(f"AWS Error Details:")
|
||||
logger.error(f"- Error Code: {error_code}")
|
||||
logger.error(f"- Error Message: {error_message}")
|
||||
logger.error(f"- Request ID: {request_id}")
|
||||
logger.error(f"- Full Response: {e.response}")
|
||||
else:
|
||||
logger.error(f"Non-AWS Error: {str(e)}")
|
||||
|
||||
@retry(stop=stop_after_attempt(5), wait=wait_fixed(5))
|
||||
def transfer_file(source, destination, use_role=False, role_name=None, use_current_role=False, aws_access_key_id=None, aws_secret_access_key=None, endpoint_url=None, region=None):
|
||||
"""
|
||||
Transfer a file from a local source to either a local destination or an S3 bucket
|
||||
"""
|
||||
if not os.path.isfile(source):
|
||||
logger.error(f"Error: Source file '{source}' does not exist or is not a file.")
|
||||
return False
|
||||
|
||||
if destination.startswith('s3://'):
|
||||
# Uploading to S3
|
||||
try:
|
||||
logger.info(f"Attempting to upload {source} to {destination}")
|
||||
s3_client = get_s3_client(use_role, role_name, use_current_role, aws_access_key_id, aws_secret_access_key, endpoint_url, region)
|
||||
bucket, s3_key = parse_s3_path(destination)
|
||||
|
||||
try:
|
||||
s3_client.upload_file(source, bucket, s3_key)
|
||||
logger.info(f"File {source} uploaded successfully to {destination}")
|
||||
except ClientError as e:
|
||||
log_error_details(e)
|
||||
if "AccessDenied" in str(e):
|
||||
logger.error("Access denied. Please check:")
|
||||
logger.error("1. IAM role/user permissions")
|
||||
logger.error("2. S3 bucket permissions")
|
||||
logger.error("3. Web identity token configuration")
|
||||
return False
|
||||
except BotoCoreError as e:
|
||||
logger.error(f"Boto3 error during upload: {str(e)}")
|
||||
return False
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Unexpected error during S3 client creation or upload: {str(e)}")
|
||||
return False
|
||||
else:
|
||||
# Copying to local destination
|
||||
try:
|
||||
import shutil
|
||||
logger.info(f"Attempting to copy {source} to local destination {destination}")
|
||||
# Create destination directory if it doesn't exist
|
||||
os.makedirs(os.path.dirname(destination), exist_ok=True)
|
||||
shutil.copy2(source, destination)
|
||||
logger.info(f"File {source} copied successfully to {destination}")
|
||||
except IOError as e:
|
||||
logger.error(f"Error copying file: {str(e)}")
|
||||
return False
|
||||
return True
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser(description="Transfer a file from a local source to either a local destination or an S3 bucket.")
|
||||
parser.add_argument("source", help="The local source file path")
|
||||
parser.add_argument("destination", help="The destination file path (local) or S3 path (e.g., 's3://bucket/key')")
|
||||
add_common_arguments(parser)
|
||||
|
||||
args = parser.parse_args()
|
||||
validate_args(args, parser)
|
||||
|
||||
success = transfer_file(
|
||||
args.source,
|
||||
args.destination,
|
||||
args.use_role,
|
||||
args.role_name,
|
||||
args.use_current_role,
|
||||
args.aws_access_key_id,
|
||||
args.aws_secret_access_key,
|
||||
args.endpoint_url,
|
||||
args.region
|
||||
)
|
||||
|
||||
if success:
|
||||
logger.info("Transfer completed successfully.")
|
||||
else:
|
||||
logger.error("Transfer failed.")
|
||||
exit(1)
|
||||
@@ -1,4 +0,0 @@
|
||||
boto3
|
||||
botocore
|
||||
jmespath
|
||||
tenacity
|
||||
@@ -1,228 +0,0 @@
|
||||
import boto3
|
||||
from botocore.exceptions import ClientError
|
||||
|
||||
import os
|
||||
import logging
|
||||
|
||||
def get_s3_client(use_role=False, role_name=None, use_current_role=False, aws_access_key_id=None, aws_secret_access_key=None, endpoint_url=None, region=None):
|
||||
"""
|
||||
Create and return an S3 client based on the provided authentication method, endpoint, and region.
|
||||
"""
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
client_kwargs = {}
|
||||
|
||||
# Log authentication method being attempted
|
||||
logger.info("Attempting S3 client creation with:")
|
||||
logger.info(f"- Region: {region if region else 'default'}")
|
||||
logger.info(f"- Endpoint URL: {endpoint_url if endpoint_url else 'default'}")
|
||||
|
||||
if endpoint_url:
|
||||
client_kwargs['endpoint_url'] = endpoint_url
|
||||
if region:
|
||||
client_kwargs['region_name'] = region
|
||||
|
||||
# Check for AWS Web Identity token
|
||||
token_file = os.environ.get('AWS_WEB_IDENTITY_TOKEN_FILE')
|
||||
role_arn = os.environ.get('AWS_ROLE_ARN')
|
||||
if token_file or role_arn:
|
||||
logger.info("AWS Web Identity configuration detected:")
|
||||
logger.info(f"- Token file path: {token_file}")
|
||||
logger.info(f"- Role ARN: {role_arn}")
|
||||
logger.info(f"- Session name: {os.environ.get('AWS_ROLE_SESSION_NAME', 'default')}")
|
||||
|
||||
if aws_access_key_id and aws_secret_access_key:
|
||||
logger.info("Using explicit AWS credentials")
|
||||
# Use explicit credentials if provided
|
||||
client_kwargs['aws_access_key_id'] = aws_access_key_id
|
||||
client_kwargs['aws_secret_access_key'] = aws_secret_access_key
|
||||
return boto3.client('s3', **client_kwargs)
|
||||
elif use_role and role_name:
|
||||
# Assume specific role if requested
|
||||
logger.info(f"Attempting to assume role: {role_name}")
|
||||
try:
|
||||
sts_client = boto3.client('sts')
|
||||
# Get current identity for logging
|
||||
identity = sts_client.get_caller_identity()
|
||||
logger.info(f"Current identity: {identity['Arn']}")
|
||||
|
||||
assumed_role_object = sts_client.assume_role(
|
||||
RoleArn=f"arn:aws:iam::{boto3.client('sts').get_caller_identity()['Account']}:role/{role_name}",
|
||||
RoleSessionName="AssumeRoleSession"
|
||||
)
|
||||
credentials = assumed_role_object['Credentials']
|
||||
client_kwargs['aws_access_key_id'] = credentials['AccessKeyId']
|
||||
client_kwargs['aws_secret_access_key'] = credentials['SecretAccessKey']
|
||||
client_kwargs['aws_session_token'] = credentials['SessionToken']
|
||||
return boto3.client('s3', **client_kwargs)
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to assume role {role_name}: {str(e)}")
|
||||
raise
|
||||
elif use_current_role:
|
||||
# Use the current role (e.g., from Kubernetes service account)
|
||||
logger.info("Using current role from environment")
|
||||
try:
|
||||
# Log environment for debugging
|
||||
for key, value in sorted(os.environ.items()):
|
||||
if any(k in key.lower() for k in ['aws', 'role', 'auth', 'token', 'credential']):
|
||||
logger.info(f"Environment: {key}={value}")
|
||||
|
||||
# Get the AWS region from environment or parameter
|
||||
aws_region = os.environ.get('AWS_REGION') or os.environ.get('AWS_DEFAULT_REGION')
|
||||
if not aws_region and not region:
|
||||
raise ValueError("AWS region must be specified either through region parameter or AWS_REGION environment variable")
|
||||
|
||||
# Use region from parameter only if not set in environment
|
||||
if not aws_region:
|
||||
aws_region = region
|
||||
# Set it in environment for other AWS clients
|
||||
os.environ['AWS_REGION'] = region
|
||||
|
||||
logger.info(f"Using AWS region: {aws_region}")
|
||||
|
||||
# Create an STS client in the correct region
|
||||
sts_kwargs = {'endpoint_url': f'https://sts.{aws_region}.amazonaws.com'}
|
||||
if not os.environ.get('AWS_REGION') and not os.environ.get('AWS_DEFAULT_REGION'):
|
||||
sts_kwargs['region_name'] = aws_region
|
||||
sts = boto3.client('sts', **sts_kwargs)
|
||||
|
||||
# Read the web identity token
|
||||
token_file = os.environ.get('AWS_WEB_IDENTITY_TOKEN_FILE')
|
||||
role_arn = os.environ.get('AWS_ROLE_ARN')
|
||||
|
||||
if not token_file or not role_arn:
|
||||
raise ValueError("AWS_WEB_IDENTITY_TOKEN_FILE and AWS_ROLE_ARN must be set")
|
||||
|
||||
with open(token_file, 'r') as f:
|
||||
token = f.read().strip()
|
||||
|
||||
logger.info("Successfully read web identity token")
|
||||
logger.info(f"Using role ARN: {role_arn}")
|
||||
|
||||
# Assume role with web identity using regional endpoint
|
||||
try:
|
||||
response = sts.assume_role_with_web_identity(
|
||||
RoleArn=role_arn,
|
||||
RoleSessionName=os.environ.get('AWS_ROLE_SESSION_NAME', 'WebIdentitySession'),
|
||||
WebIdentityToken=token
|
||||
)
|
||||
|
||||
# Get the temporary credentials
|
||||
credentials = response['Credentials']
|
||||
|
||||
# Create the S3 client with the temporary credentials
|
||||
s3_kwargs = {
|
||||
'aws_access_key_id': credentials['AccessKeyId'],
|
||||
'aws_secret_access_key': credentials['SecretAccessKey'],
|
||||
'aws_session_token': credentials['SessionToken']
|
||||
}
|
||||
# Only set region_name if not already in environment
|
||||
if not os.environ.get('AWS_REGION') and not os.environ.get('AWS_DEFAULT_REGION'):
|
||||
s3_kwargs['region_name'] = aws_region
|
||||
# Add any additional kwargs
|
||||
s3_kwargs.update(client_kwargs)
|
||||
client = boto3.client('s3', **s3_kwargs)
|
||||
|
||||
logger.info(f"Successfully assumed role with web identity: {response['AssumedRoleUser']['Arn']}")
|
||||
|
||||
# Test the credentials
|
||||
try:
|
||||
# Try to get caller identity first
|
||||
sts_test = boto3.client(
|
||||
'sts',
|
||||
region_name=aws_region,
|
||||
aws_access_key_id=credentials['AccessKeyId'],
|
||||
aws_secret_access_key=credentials['SecretAccessKey'],
|
||||
aws_session_token=credentials['SessionToken']
|
||||
)
|
||||
identity = sts_test.get_caller_identity()
|
||||
logger.info(f"Successfully verified credentials as: {identity['Arn']}")
|
||||
|
||||
# Then try S3 access
|
||||
bucket_name = os.environ.get('BUCKET_NAME', 'default-bucket')
|
||||
try:
|
||||
client.head_bucket(Bucket=bucket_name)
|
||||
logger.info(f"Successfully verified S3 access to bucket: {bucket_name}")
|
||||
except ClientError as e:
|
||||
error_code = e.response['Error']['Code']
|
||||
if error_code == '404':
|
||||
logger.warning(f"Bucket {bucket_name} does not exist, but credentials work")
|
||||
else:
|
||||
logger.warning(f"S3 access check failed: {error_code} - {e.response['Error']['Message']}")
|
||||
except Exception as e:
|
||||
logger.warning(f"Could not verify credentials: {str(e)}")
|
||||
|
||||
return client
|
||||
|
||||
except ClientError as e:
|
||||
error_code = e.response['Error']['Code']
|
||||
error_message = e.response['Error']['Message']
|
||||
logger.error("Failed to assume role with web identity:")
|
||||
logger.error(f"Error Code: {error_code}")
|
||||
logger.error(f"Error Message: {error_message}")
|
||||
logger.error("Trust policy might need to be updated to allow sts:AssumeRoleWithWebIdentity")
|
||||
logger.error("Current role ARN: " + role_arn)
|
||||
logger.error("Token file path: " + token_file)
|
||||
raise
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to use current role: {str(e)}")
|
||||
logger.error("Current environment:")
|
||||
for key, value in sorted(os.environ.items()):
|
||||
if any(k in key.lower() for k in ['aws', 'role', 'auth', 'token', 'credential']):
|
||||
logger.error(f" {key}: {value}")
|
||||
raise
|
||||
else:
|
||||
# Use default credentials (environment, instance profile, or pod service account)
|
||||
logger.info("Using default credential provider chain")
|
||||
try:
|
||||
client = boto3.client('s3', **client_kwargs)
|
||||
# Try to get caller identity to verify credentials
|
||||
sts = boto3.client('sts')
|
||||
identity = sts.get_caller_identity()
|
||||
logger.info(f"Successfully authenticated as: {identity['Arn']}")
|
||||
return client
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to create S3 client: {str(e)}")
|
||||
raise
|
||||
|
||||
def parse_s3_path(s3_path):
|
||||
"""
|
||||
Parse an S3 path into bucket and key
|
||||
"""
|
||||
parts = s3_path.replace('s3://', '').split('/', 1)
|
||||
bucket = parts[0]
|
||||
key = parts[1] if len(parts) > 1 else ''
|
||||
return bucket, key
|
||||
|
||||
def add_common_arguments(parser):
|
||||
"""
|
||||
Add common command-line arguments to an ArgumentParser object
|
||||
"""
|
||||
auth_group = parser.add_mutually_exclusive_group()
|
||||
auth_group.add_argument("--use_role", action="store_true", help="Use IAM role for authentication")
|
||||
auth_group.add_argument("--use_current_role", action="store_true", help="Use current AWS role (e.g. from Kubernetes service account)")
|
||||
parser.add_argument("--role_name", help="The name of the IAM role to assume (only when --use_role is set)")
|
||||
parser.add_argument("--aws_access_key_id", help="AWS access key ID")
|
||||
parser.add_argument("--aws_secret_access_key", help="AWS secret access key")
|
||||
parser.add_argument("--endpoint_url", help="S3-compatible endpoint URL")
|
||||
parser.add_argument("--region", help="AWS region (ignored if endpoint_url is specified)")
|
||||
|
||||
def validate_args(args, parser):
|
||||
"""
|
||||
Validate command-line arguments
|
||||
"""
|
||||
if args.destination.startswith('s3://'):
|
||||
# Check for conflicting auth methods
|
||||
if args.use_role and not args.role_name:
|
||||
parser.error("--role_name is required when using --use_role")
|
||||
|
||||
if args.role_name and not args.use_role:
|
||||
parser.error("--role_name can only be used with --use_role")
|
||||
|
||||
if args.use_current_role and (args.aws_access_key_id or args.aws_secret_access_key):
|
||||
parser.error("When using current role (--use_current_role), access key and secret should not be specified")
|
||||
|
||||
# If using explicit credentials, require both key and secret
|
||||
if (args.aws_access_key_id or args.aws_secret_access_key) and not (args.aws_access_key_id and args.aws_secret_access_key):
|
||||
parser.error("Both --aws_access_key_id and --aws_secret_access_key must be provided when using access key authentication")
|
||||
Reference in New Issue
Block a user