From 930bcd2ee44d3e8b8fa33b87095fd9b13e23b106 Mon Sep 17 00:00:00 2001 From: Lukasz Raczylo Date: Wed, 11 Sep 2024 12:00:18 +0100 Subject: [PATCH] Dummy commit. --- Makefile | 3 +- README.md | 6 +++- chart/Chart.yaml | 4 +-- chart/values.yaml | 2 +- docker-image-worker/cleanup.py | 63 ++-------------------------------- docker-image-worker/export.py | 63 ++-------------------------------- internal/shared/jobs.go | 2 +- 7 files changed, 17 insertions(+), 126 deletions(-) diff --git a/Makefile b/Makefile index 73eafb2..7943d97 100644 --- a/Makefile +++ b/Makefile @@ -229,7 +229,8 @@ release-chart: cd ../helm-charts/; git add -A charts/packages; git fix; git push; cd ../helm-charts/charts/${CHART_NAME}; cr upload --config ../../chart-releaser.yaml --skip-existing; cd ../helm-charts/charts/${CHART_NAME}; rm -fr .cr-index; mkdir .cr-index; cr index --config ../../chart-releaser.yaml; cp .cr-index/index.yaml ../../index.yaml; - ../helm-charts; git fix; git push + echo "index.yaml updated" + cd ../helm-charts; git fix; git push # go-install-tool will 'go install' any package with custom target and name of binary, if it doesn't exist # $1 - target path with name of binary diff --git a/README.md b/README.md index e787f63..429a177 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # kubernetes-images-sync-operator -Kubernetes operator backing up images into the local / S3 compatible storage +Kubernetes operator backing up images into the local / S3 compatible storage. ## Description @@ -63,6 +63,10 @@ spec: maxConcurrentJobs: 1 ``` +## Worth knowing + +* If you provide roleARN, you also need to set the useRole to true. + #### Random fluff Copyright 2024. diff --git a/chart/Chart.yaml b/chart/Chart.yaml index 7e25d46..c76df32 100644 --- a/chart/Chart.yaml +++ b/chart/Chart.yaml @@ -10,9 +10,9 @@ description: | type: application -version: 0.1.11 +version: 0.1.14 -appVersion: "0.1.11" +appVersion: "0.1.14" home: https://github.com/lukaszraczylo/kubernetes-images-sync-operator diff --git a/chart/values.yaml b/chart/values.yaml index 211f702..d99e057 100644 --- a/chart/values.yaml +++ b/chart/values.yaml @@ -11,7 +11,7 @@ cmRaczyloCom: - ALL image: repository: ghcr.io/lukaszraczylo/kubernetes-images-sync-operator - tag: 0.1.11 + tag: 0.1.14 resources: limits: cpu: 500m diff --git a/docker-image-worker/cleanup.py b/docker-image-worker/cleanup.py index 6fa44fc..973663d 100755 --- a/docker-image-worker/cleanup.py +++ b/docker-image-worker/cleanup.py @@ -1,39 +1,8 @@ #!/usr/bin/env python3 import os -import boto3 import argparse from botocore.exceptions import ClientError - -def get_s3_client(use_role=False, role_name=None, aws_access_key_id=None, aws_secret_access_key=None, endpoint_url=None, region=None): - """ - Create and return an S3 client based on the provided authentication method, endpoint, and region. - """ - client_kwargs = {} - - if endpoint_url: - client_kwargs['endpoint_url'] = endpoint_url - elif region: - client_kwargs['region_name'] = region - - if use_role: - if role_name: - # Assume the specified role - sts_client = boto3.client('sts') - assumed_role_object = sts_client.assume_role( - RoleArn=f"arn:aws:iam::{boto3.client('sts').get_caller_identity()['Account']}:role/{role_name}", - RoleSessionName="AssumeRoleSession" - ) - credentials = assumed_role_object['Credentials'] - client_kwargs['aws_access_key_id'] = credentials['AccessKeyId'] - client_kwargs['aws_secret_access_key'] = credentials['SecretAccessKey'] - client_kwargs['aws_session_token'] = credentials['SessionToken'] - return boto3.client('s3', **client_kwargs) - elif aws_access_key_id and aws_secret_access_key: - client_kwargs['aws_access_key_id'] = aws_access_key_id - client_kwargs['aws_secret_access_key'] = aws_secret_access_key - return boto3.client('s3', **client_kwargs) - else: - raise ValueError("Either use_role must be True, or both aws_access_key_id and aws_secret_access_key must be provided") +from s3_utils import get_s3_client, parse_s3_path, add_common_arguments, validate_args def remove_directory(destination, use_role=False, role_name=None, aws_access_key_id=None, aws_secret_access_key=None, endpoint_url=None, region=None): """ @@ -67,39 +36,13 @@ def remove_directory(destination, use_role=False, role_name=None, aws_access_key return False return True -def parse_s3_path(s3_path): - """ - Parse an S3 path into bucket and key - """ - parts = s3_path.replace('s3://', '').split('/', 1) - bucket = parts[0] - key = parts[1] if len(parts) > 1 else '' - return bucket, key - if __name__ == "__main__": parser = argparse.ArgumentParser(description="Remove a directory recursively, either local or in an S3 bucket.") parser.add_argument("destination", help="The directory path (local) or S3 path (e.g., 's3://bucket/prefix') to remove") - parser.add_argument("--use_role", action="store_true", help="Use IAM role for authentication") - parser.add_argument("--role_name", help="The name of the IAM role to assume") - parser.add_argument("--aws_access_key_id", help="AWS access key ID") - parser.add_argument("--aws_secret_access_key", help="AWS secret access key") - parser.add_argument("--endpoint_url", help="S3-compatible endpoint URL") - parser.add_argument("--region", help="AWS region (ignored if endpoint_url is specified)") + add_common_arguments(parser) args = parser.parse_args() - - if args.destination.startswith('s3://'): - if args.use_role and (args.aws_access_key_id or args.aws_secret_access_key or args.endpoint_url): - parser.error("When using IAM role (--use_role), access key, secret, and endpoint URL should not be specified.") - - if (args.aws_access_key_id or args.aws_secret_access_key) and not (args.aws_access_key_id and args.aws_secret_access_key): - parser.error("Both --aws_access_key_id and --aws_secret_access_key must be provided when using access key authentication.") - - if not args.use_role and not (args.aws_access_key_id and args.aws_secret_access_key): - parser.error("Either --use_role or both --aws_access_key_id and --aws_secret_access_key must be provided for S3 operations.") - - if args.use_role and args.role_name and (args.aws_access_key_id or args.aws_secret_access_key): - parser.error("When using a specific role (--role_name), access key and secret should not be specified.") + validate_args(args, parser) success = remove_directory( args.destination, diff --git a/docker-image-worker/export.py b/docker-image-worker/export.py index c9d67c9..11dec94 100755 --- a/docker-image-worker/export.py +++ b/docker-image-worker/export.py @@ -1,39 +1,8 @@ #!/usr/bin/env python3 import os -import boto3 import argparse from botocore.exceptions import ClientError - -def get_s3_client(use_role=False, role_name=None, aws_access_key_id=None, aws_secret_access_key=None, endpoint_url=None, region=None): - """ - Create and return an S3 client based on the provided authentication method, endpoint, and region. - """ - client_kwargs = {} - - if endpoint_url: - client_kwargs['endpoint_url'] = endpoint_url - elif region: - client_kwargs['region_name'] = region - - if use_role: - if role_name: - # Assume the specified role - sts_client = boto3.client('sts') - assumed_role_object = sts_client.assume_role( - RoleArn=f"arn:aws:iam::{boto3.client('sts').get_caller_identity()['Account']}:role/{role_name}", - RoleSessionName="AssumeRoleSession" - ) - credentials = assumed_role_object['Credentials'] - client_kwargs['aws_access_key_id'] = credentials['AccessKeyId'] - client_kwargs['aws_secret_access_key'] = credentials['SecretAccessKey'] - client_kwargs['aws_session_token'] = credentials['SessionToken'] - return boto3.client('s3', **client_kwargs) - elif aws_access_key_id and aws_secret_access_key: - client_kwargs['aws_access_key_id'] = aws_access_key_id - client_kwargs['aws_secret_access_key'] = aws_secret_access_key - return boto3.client('s3', **client_kwargs) - else: - raise ValueError("Either use_role must be True, or both aws_access_key_id and aws_secret_access_key must be provided") +from s3_utils import get_s3_client, parse_s3_path, add_common_arguments, validate_args def transfer_file(source, destination, use_role=False, role_name=None, aws_access_key_id=None, aws_secret_access_key=None, endpoint_url=None, region=None): """ @@ -66,40 +35,14 @@ def transfer_file(source, destination, use_role=False, role_name=None, aws_acces return False return True -def parse_s3_path(s3_path): - """ - Parse an S3 path into bucket and key - """ - parts = s3_path.replace('s3://', '').split('/', 1) - bucket = parts[0] - key = parts[1] if len(parts) > 1 else '' - return bucket, key - if __name__ == "__main__": parser = argparse.ArgumentParser(description="Transfer a file from a local source to either a local destination or an S3 bucket.") parser.add_argument("source", help="The local source file path") parser.add_argument("destination", help="The destination file path (local) or S3 path (e.g., 's3://bucket/key')") - parser.add_argument("--use_role", action="store_true", help="Use IAM role for authentication") - parser.add_argument("--role_name", help="The name of the IAM role to assume") - parser.add_argument("--aws_access_key_id", help="AWS access key ID") - parser.add_argument("--aws_secret_access_key", help="AWS secret access key") - parser.add_argument("--endpoint_url", help="S3-compatible endpoint URL") - parser.add_argument("--region", help="AWS region (ignored if endpoint_url is specified)") + add_common_arguments(parser) args = parser.parse_args() - - if args.destination.startswith('s3://'): - if args.use_role and (args.aws_access_key_id or args.aws_secret_access_key or args.endpoint_url): - parser.error("When using IAM role (--use_role), access key, secret, and endpoint URL should not be specified.") - - if (args.aws_access_key_id or args.aws_secret_access_key) and not (args.aws_access_key_id and args.aws_secret_access_key): - parser.error("Both --aws_access_key_id and --aws_secret_access_key must be provided when using access key authentication.") - - if not args.use_role and not (args.aws_access_key_id and args.aws_secret_access_key): - parser.error("Either --use_role or both --aws_access_key_id and --aws_secret_access_key must be provided for S3 transfers.") - - if args.use_role and args.role_name and (args.aws_access_key_id or args.aws_secret_access_key): - parser.error("When using a specific role (--role_name), access key and secret should not be specified.") + validate_args(args, parser) success = transfer_file( args.source, diff --git a/internal/shared/jobs.go b/internal/shared/jobs.go index 4509223..5c97a40 100644 --- a/internal/shared/jobs.go +++ b/internal/shared/jobs.go @@ -64,7 +64,7 @@ func CreateJob[T any](params JobParams, setupFunc func(T) []string) *batchv1.Job func SetupS3Params(s3Config raczylocomv1.ClusterImageStorageS3) []string { params := []string{} if s3Config.UseRole { - params = append(params, "--use-role") + params = append(params, "--use_role") } else { params = append(params, fmt.Sprintf("--aws_access_key_id='%s'", s3Config.AccessKey)) params = append(params, fmt.Sprintf("--aws_secret_access_key='%s'", s3Config.SecretKey))