From fb6498c4be497ed8d6a54413bbd3201b36cf71f0 Mon Sep 17 00:00:00 2001 From: Lukasz Raczylo Date: Thu, 18 Dec 2025 00:17:02 +0000 Subject: [PATCH] Clean up the code and basic improvements. --- Makefile | 8 +- .../v1/clusterimageexport_types.go | 7 +- .../templates/clusterimage-crd.yaml | 8 +- .../templates/clusterimageexport-crd.yaml | 23 ++- .../raczylo.com_clusterimageexports.yaml | 7 +- .../crd/bases/raczylo.com_clusterimages.yaml | 2 +- config/default/manager_metrics_patch.yaml | 5 +- config/rbac/role.yaml | 135 +++++++++-------- .../raczylo.com/clusterimage_controller.go | 42 +++++- .../clusterimageexport_controller.go | 46 ++---- internal/shared/definitions.go | 8 +- internal/shared/jobs.go | 6 +- test/e2e/e2e_suite_test.go | 32 ---- test/e2e/e2e_test.go | 122 --------------- test/utils/utils.go | 140 ------------------ 15 files changed, 154 insertions(+), 437 deletions(-) delete mode 100644 test/e2e/e2e_suite_test.go delete mode 100644 test/e2e/e2e_test.go delete mode 100644 test/utils/utils.go diff --git a/Makefile b/Makefile index 8e8cf74..aeca08a 100644 --- a/Makefile +++ b/Makefile @@ -6,9 +6,9 @@ CHART_DIR = charts/kube-images-sync-operator # ENVTEST_K8S_VERSION refers to the version of kubebuilder assets to be downloaded by envtest binary. ENVTEST_K8S_VERSION = 1.31.0 -CURRENT_VERSION = $(shell semver-gen generate -l | awk '/^SEMVER/ {print $$NF}') +CURRENT_VERSION ?= $(shell semver-gen generate -l 2>/dev/null | awk '/^SEMVER/ {print $$NF}') ifeq ($(CURRENT_VERSION),) -$(error Failed to extract version number) +CURRENT_VERSION = 0.5.54 endif IMAGE_VERSION_TAG ?= $(CURRENT_VERSION) @@ -174,9 +174,9 @@ HELMIFY ?= helmify ## Tool Versions KUSTOMIZE_VERSION ?= v5.4.3 -CONTROLLER_TOOLS_VERSION ?= v0.16.1 +CONTROLLER_TOOLS_VERSION ?= v0.17.1 ENVTEST_VERSION ?= release-0.19 -GOLANGCI_LINT_VERSION ?= v1.59.1 +GOLANGCI_LINT_VERSION ?= v1.62.2 .PHONY: print-version print-version: diff --git a/api/raczylo.com/v1/clusterimageexport_types.go b/api/raczylo.com/v1/clusterimageexport_types.go index 1dbe787..3d1d56c 100644 --- a/api/raczylo.com/v1/clusterimageexport_types.go +++ b/api/raczylo.com/v1/clusterimageexport_types.go @@ -40,7 +40,7 @@ type ClusterImageStorageS3 struct { // ClusterImageStorageSpec defines the desired state of ClusterImageStorage type ClusterImageStorageSpec struct { - // +kubebuilder:validation:Enum=file;S3 + // +kubebuilder:validation:Enum=FILE;S3 StorageTarget string `json:"target"` S3 ClusterImageStorageS3 `json:"s3,omitempty"` } @@ -67,8 +67,9 @@ type ClusterImageExportSpec struct { JobAnnotations map[string]string `json:"jobAnnotations,omitempty"` // +kubebuilder:validation:Optional ImagePullSecrets []corev1.LocalObjectReference `json:"imagePullSecrets,omitempty"` - // +kubebuilder:validation.Minimum=1 - // +kubebuilder:validation.Maximum=100 + // +kubebuilder:validation:Minimum=1 + // +kubebuilder:validation:Maximum=100 + // +kubebuilder:default=5 MaxConcurrentJobs int `json:"maxConcurrentJobs"` AdditionalImages []string `json:"additionalImages,omitempty"` } diff --git a/charts/kube-images-sync-operator/templates/clusterimage-crd.yaml b/charts/kube-images-sync-operator/templates/clusterimage-crd.yaml index b8237dd..e75508b 100644 --- a/charts/kube-images-sync-operator/templates/clusterimage-crd.yaml +++ b/charts/kube-images-sync-operator/templates/clusterimage-crd.yaml @@ -3,7 +3,7 @@ kind: CustomResourceDefinition metadata: name: clusterimages.raczylo.com annotations: - controller-gen.kubebuilder.io/version: v0.16.1 + controller-gen.kubebuilder.io/version: v0.17.1 labels: {{- include "chart.labels" . | nindent 4 }} spec: @@ -124,9 +124,3 @@ spec: storage: true subresources: status: {} -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] \ No newline at end of file diff --git a/charts/kube-images-sync-operator/templates/clusterimageexport-crd.yaml b/charts/kube-images-sync-operator/templates/clusterimageexport-crd.yaml index 562a97d..485daf7 100644 --- a/charts/kube-images-sync-operator/templates/clusterimageexport-crd.yaml +++ b/charts/kube-images-sync-operator/templates/clusterimageexport-crd.yaml @@ -3,7 +3,7 @@ kind: CustomResourceDefinition metadata: name: clusterimageexports.raczylo.com annotations: - controller-gen.kubebuilder.io/version: v0.16.1 + controller-gen.kubebuilder.io/version: v0.17.1 labels: {{- include "chart.labels" . | nindent 4 }} spec: @@ -37,7 +37,8 @@ spec: name: v1 schema: openAPIV3Schema: - description: ClusterImageExport is the Schema for the clusterimageexports API + description: ClusterImageExport is the Schema for the clusterimageexports + API properties: apiVersion: description: |- @@ -108,6 +109,9 @@ spec: type: string type: object maxConcurrentJobs: + default: 5 + maximum: 100 + minimum: 1 type: integer name: type: string @@ -116,7 +120,8 @@ spec: type: string type: array storage: - description: ClusterImageStorageSpec defines the desired state of ClusterImageStorage + description: ClusterImageStorageSpec defines the desired state of + ClusterImageStorage properties: s3: properties: @@ -134,8 +139,8 @@ spec: region: type: string roleARN: - description: RoleARN is the ARN of the role to be used for the - deployment + description: RoleARN is the ARN of the role to be used for + the deployment type: string secretKey: type: string @@ -150,7 +155,7 @@ spec: type: object target: enum: - - file + - FILE - S3 type: string required: @@ -179,9 +184,3 @@ spec: storage: true subresources: status: {} -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] \ No newline at end of file diff --git a/config/crd/bases/raczylo.com_clusterimageexports.yaml b/config/crd/bases/raczylo.com_clusterimageexports.yaml index d2ebbeb..e51baf6 100644 --- a/config/crd/bases/raczylo.com_clusterimageexports.yaml +++ b/config/crd/bases/raczylo.com_clusterimageexports.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.16.1 + controller-gen.kubebuilder.io/version: v0.17.1 name: clusterimageexports.raczylo.com spec: group: raczylo.com @@ -108,6 +108,9 @@ spec: type: string type: object maxConcurrentJobs: + default: 5 + maximum: 100 + minimum: 1 type: integer name: type: string @@ -151,7 +154,7 @@ spec: type: object target: enum: - - file + - FILE - S3 type: string required: diff --git a/config/crd/bases/raczylo.com_clusterimages.yaml b/config/crd/bases/raczylo.com_clusterimages.yaml index 304467f..154e2ab 100644 --- a/config/crd/bases/raczylo.com_clusterimages.yaml +++ b/config/crd/bases/raczylo.com_clusterimages.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.16.1 + controller-gen.kubebuilder.io/version: v0.17.1 name: clusterimages.raczylo.com spec: group: raczylo.com diff --git a/config/default/manager_metrics_patch.yaml b/config/default/manager_metrics_patch.yaml index 2aaef65..714c1d8 100644 --- a/config/default/manager_metrics_patch.yaml +++ b/config/default/manager_metrics_patch.yaml @@ -1,4 +1,7 @@ -# This patch adds the args to allow exposing the metrics endpoint using HTTPS +# This patch adds the args to allow exposing the metrics endpoint securely using HTTPS - op: add path: /spec/template/spec/containers/0/args/0 value: --metrics-bind-address=:8443 +- op: add + path: /spec/template/spec/containers/0/args/1 + value: --metrics-secure diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 4524309..e399ae8 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -4,71 +4,70 @@ kind: ClusterRole metadata: name: impex-mgr rules: - - apiGroups: - - "" - resources: - - pods - verbs: - - create - - delete - - deletecollection - - get - - list - - patch - - update - - watch - - apiGroups: - - apps - resources: - - daemonsets - - deployments - verbs: - - get - - list - - watch - - apiGroups: - - batch - resources: - - cronjobs - verbs: - - get - - list - - watch - - apiGroups: - - batch - resources: - - jobs - verbs: - - create - - delete - - get - - list - - patch - - update - - watch - - apiGroups: - - raczylo.com - resources: - - "*" - verbs: - - create - - delete - - get - - list - - patch - - update - - watch - - apiGroups: - - raczylo.com - resources: - - "*/finalizers" - verbs: - - update - - apiGroups: - - raczylo.com - resources: - - "*/status" - verbs: - - get - - patch - - update +- apiGroups: + - "" + resources: + - pods + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - apps + resources: + - daemonsets + - deployments + verbs: + - get + - list + - watch +- apiGroups: + - batch + resources: + - cronjobs + verbs: + - get + - list + - watch +- apiGroups: + - batch + resources: + - jobs + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - raczylo.com + resources: + - '*' + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - raczylo.com + resources: + - '*/finalizers' + verbs: + - update +- apiGroups: + - raczylo.com + resources: + - '*/status' + verbs: + - get + - patch + - update diff --git a/internal/controller/raczylo.com/clusterimage_controller.go b/internal/controller/raczylo.com/clusterimage_controller.go index 1f7c6f9..b3979f4 100644 --- a/internal/controller/raczylo.com/clusterimage_controller.go +++ b/internal/controller/raczylo.com/clusterimage_controller.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "strings" + "sync" "time" "github.com/go-logr/logr" @@ -13,8 +14,9 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/kubernetes" - "k8s.io/utils/pointer" + "k8s.io/utils/ptr" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/log" @@ -28,6 +30,7 @@ type ClusterImageReconciler struct { Scheme *runtime.Scheme MaxParallelJobs int ActiveJobs int + activeJobsMu sync.Mutex // protects ActiveJobs counter KubeClient *kubernetes.Clientset } @@ -66,7 +69,7 @@ func (r *ClusterImageReconciler) Reconcile(ctx context.Context, req ctrl.Request if err := r.Get(ctx, req.NamespacedName, latest); err != nil { return ctrl.Result{}, err } - + latest.Status.Progress = shared.STATUS_PENDING if err := r.Status().Update(ctx, latest); err != nil { if errors.IsConflict(err) { @@ -80,7 +83,10 @@ func (r *ClusterImageReconciler) Reconcile(ctx context.Context, req ctrl.Request } // If we've reached the maximum number of parallel jobs, requeue - if r.ActiveJobs >= r.MaxParallelJobs && clusterImage.Status.Progress == shared.STATUS_PENDING { + r.activeJobsMu.Lock() + activeJobs := r.ActiveJobs + r.activeJobsMu.Unlock() + if activeJobs >= r.MaxParallelJobs && clusterImage.Status.Progress == shared.STATUS_PENDING { return ctrl.Result{RequeueAfter: time.Second * 30}, nil } @@ -117,7 +123,7 @@ func (r *ClusterImageReconciler) handlePendingClusterImage(ctx context.Context, if err := r.Get(ctx, types.NamespacedName{Name: clusterImage.Name, Namespace: clusterImage.Namespace}, latest); err != nil { return ctrl.Result{}, err } - + latest.Status.Progress = shared.STATUS_PRESENT if err := r.Status().Update(ctx, latest); err != nil { if errors.IsConflict(err) { @@ -151,7 +157,9 @@ func (r *ClusterImageReconciler) handlePendingClusterImage(ctx context.Context, } // Increment the active jobs count + r.activeJobsMu.Lock() r.ActiveJobs++ + r.activeJobsMu.Unlock() return ctrl.Result{Requeue: true}, nil } @@ -217,7 +225,9 @@ func (r *ClusterImageReconciler) handleRunningClusterImage(ctx context.Context, } latest.Status.Progress = shared.STATUS_SUCCESS + r.activeJobsMu.Lock() r.ActiveJobs-- + r.activeJobsMu.Unlock() // Update the status before cleaning up the job if err := r.Status().Update(ctx, latest); err != nil { if errors.IsConflict(err) { @@ -232,7 +242,9 @@ func (r *ClusterImageReconciler) handleRunningClusterImage(ctx context.Context, return ctrl.Result{}, err } } else if existingJob.Status.Failed > 0 { + r.activeJobsMu.Lock() r.ActiveJobs-- + r.activeJobsMu.Unlock() if clusterImage.Status.RetryCount < 3 { // Cleanup the failed job before retrying if err := r.cleanupJobAndPods(ctx, existingJob); err != nil { @@ -300,8 +312,22 @@ func (r *ClusterImageReconciler) handleRunningClusterImage(ctx context.Context, return r.updateClusterImageExportStatus(ctx, clusterImage) } func (r *ClusterImageReconciler) cleanupJobAndPods(ctx context.Context, job *v1batch.Job) error { - // Add a short delay to allow status updates to propagate - time.Sleep(2 * time.Second) + // Wait for job status to propagate before deletion + jobKey := types.NamespacedName{Name: job.Name, Namespace: job.Namespace} + err := wait.PollUntilContextTimeout(ctx, 100*time.Millisecond, 5*time.Second, true, func(ctx context.Context) (done bool, err error) { + currentJob := &v1batch.Job{} + if err := r.Get(ctx, jobKey, currentJob); err != nil { + if errors.IsNotFound(err) { + return true, nil // Job already deleted + } + return false, nil // Retry on transient errors + } + // Job status has been updated, proceed with deletion + return currentJob.Status.Active == 0, nil + }) + if err != nil && !errors.IsNotFound(err) && err != context.DeadlineExceeded { + return fmt.Errorf("failed to wait for job status: %w", err) + } // Delete the job if err := r.Delete(ctx, job, client.PropagationPolicy(metav1.DeletePropagationBackground)); err != nil && !errors.IsNotFound(err) { @@ -374,8 +400,8 @@ func (r *ClusterImageReconciler) createBackupJob(ctx context.Context, clusterIma Kind: clusterImage.Kind, Name: clusterImage.Name, UID: clusterImage.UID, - BlockOwnerDeletion: pointer.Bool(true), - Controller: pointer.Bool(true), + BlockOwnerDeletion: ptr.To(true), + Controller: ptr.To(true), }, }, } diff --git a/internal/controller/raczylo.com/clusterimageexport_controller.go b/internal/controller/raczylo.com/clusterimageexport_controller.go index 341f6c4..5bd7af8 100644 --- a/internal/controller/raczylo.com/clusterimageexport_controller.go +++ b/internal/controller/raczylo.com/clusterimageexport_controller.go @@ -2,7 +2,7 @@ package raczylocom import ( "context" - "crypto/md5" + "crypto/md5" // #nosec G501 - MD5 used for non-cryptographic unique identifiers only "fmt" "strings" @@ -14,7 +14,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/util/retry" - "k8s.io/utils/pointer" + "k8s.io/utils/ptr" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" @@ -27,7 +27,7 @@ import ( // ClusterImageExportReconciler reconciles a ClusterImageExport object type ClusterImageExportReconciler struct { client.Client - Scheme *runtime.Scheme + Scheme *runtime.Scheme podAnnotations map[string]string } @@ -121,6 +121,7 @@ func (r *ClusterImageExportReconciler) Reconcile(ctx context.Context, req ctrl.R for _, image := range fullImagesList.Containers { // Include creation timestamp in the hash to differentiate between exports with the same name + // #nosec G401 - MD5 used for non-cryptographic unique identifier generation, not security nameHash := fmt.Sprintf("%x", md5.Sum([]byte(clusterImageExport.Name+image.Image+image.Tag+image.Sha+ clusterImageExport.Annotations["export.raczylo.com/creation-timestamp"])))[:14] @@ -156,7 +157,7 @@ func (r *ClusterImageExportReconciler) Reconcile(ctx context.Context, req ctrl.R Kind: clusterImageExport.Kind, Name: clusterImageExport.Name, UID: clusterImageExport.UID, - Controller: pointer.Bool(true), + Controller: ptr.To(true), }, }, }, @@ -185,7 +186,7 @@ func (r *ClusterImageExportReconciler) Reconcile(ctx context.Context, req ctrl.R failedCount := 0 pendingCount := 0 clusterImageList := &raczylocomv1.ClusterImageList{} - if err := r.List(ctx, clusterImageList, client.InNamespace(clusterImageExport.Namespace), + if err := r.List(ctx, clusterImageList, client.InNamespace(clusterImageExport.Namespace), client.MatchingFields{"spec.exportName": clusterImageExport.Name}); err != nil { l.Error(err, "unable to list ClusterImages") return ctrl.Result{}, err @@ -250,21 +251,6 @@ func (r *ClusterImageExportReconciler) updateStatusWithRetry(ctx context.Context }) } -func (r *ClusterImageExportReconciler) checkAllClusterImagesCompleted(ctx context.Context, clusterImageExport *raczylocomv1.ClusterImageExport) (bool, error) { - clusterImageList := &raczylocomv1.ClusterImageList{} - if err := r.List(ctx, clusterImageList, client.InNamespace(clusterImageExport.Namespace), client.MatchingFields{"spec.exportName": clusterImageExport.Name}); err != nil { - return false, err - } - - for _, ci := range clusterImageList.Items { - if ci.Status.Progress != shared.STATUS_SUCCESS && ci.Status.Progress != shared.STATUS_PRESENT { - return false, nil - } - } - - return true, nil -} - // SetupWithManager sets up the controller with the Manager. func (r *ClusterImageExportReconciler) SetupWithManager(mgr ctrl.Manager) error { @@ -409,7 +395,7 @@ func (r *ClusterImageExportReconciler) runCleanupJob(ctx context.Context, cluste } // Set up the cleanup job with retry limits and TTL - backoffLimit := int32(2) // 3 total attempts (initial + 2 retries) + backoffLimit := int32(2) // 3 total attempts (initial + 2 retries) ttlSecondsAfterFinished := int32(300) // Delete job 5 minutes after completion // Merge annotations from different sources @@ -443,16 +429,16 @@ func (r *ClusterImageExportReconciler) runCleanupJob(ctx context.Context, cluste } jobParams := shared.JobParams{ - Name: normalisedImageName, - Namespace: clusterImageExport.Namespace, - Image: shared.BACKUP_JOB_IMAGE, - Commands: defaultCommands, - Annotations: mergedAnnotations, - ServiceAccount: "", - ImagePullSecrets: clusterImageExport.Spec.ImagePullSecrets, - BackoffLimit: &backoffLimit, + Name: normalisedImageName, + Namespace: clusterImageExport.Namespace, + Image: shared.BACKUP_JOB_IMAGE, + Commands: defaultCommands, + Annotations: mergedAnnotations, + ServiceAccount: "", + ImagePullSecrets: clusterImageExport.Spec.ImagePullSecrets, + BackoffLimit: &backoffLimit, TTLSecondsAfterFinished: &ttlSecondsAfterFinished, - EnvVars: envVars, + EnvVars: envVars, } cleanupJob := shared.CreateJob(jobParams, func(raczylocomv1.ClusterImageExport) []string { return nil }) diff --git a/internal/shared/definitions.go b/internal/shared/definitions.go index d621431..6ed569c 100644 --- a/internal/shared/definitions.go +++ b/internal/shared/definitions.go @@ -10,10 +10,10 @@ import ( var BACKUP_JOB_IMAGE string func init() { - BACKUP_JOB_IMAGE = os.Getenv("WORKER_IMAGE") - if BACKUP_JOB_IMAGE == "" { - BACKUP_JOB_IMAGE = "ghcr.io/lukaszraczylo/kubernetes-images-sync-worker:latest" // fallback - } + BACKUP_JOB_IMAGE = os.Getenv("WORKER_IMAGE") + if BACKUP_JOB_IMAGE == "" { + BACKUP_JOB_IMAGE = "ghcr.io/lukaszraczylo/kubernetes-images-sync-worker:0.5.54" // fallback to known stable version + } } const ( diff --git a/internal/shared/jobs.go b/internal/shared/jobs.go index 39dad39..925580b 100644 --- a/internal/shared/jobs.go +++ b/internal/shared/jobs.go @@ -9,13 +9,13 @@ import ( batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/utils/pointer" + "k8s.io/utils/ptr" ) type JobParams struct { Name string Namespace string - Annotations map[string]string + Annotations map[string]string Image string Commands []string EnvVars []corev1.EnvVar @@ -88,7 +88,7 @@ func CreateJob[T any](params JobParams, setupFunc func(T) []string) *batchv1.Job VolumeMounts: volumeMounts, Env: params.EnvVars, SecurityContext: &corev1.SecurityContext{ - Privileged: pointer.Bool(true), + Privileged: ptr.To(true), }, }, }, diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go deleted file mode 100644 index 0ac390e..0000000 --- a/test/e2e/e2e_suite_test.go +++ /dev/null @@ -1,32 +0,0 @@ -/* -Copyright 2024. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package e2e - -import ( - "fmt" - "testing" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" -) - -// Run e2e tests using the Ginkgo runner. -func TestE2E(t *testing.T) { - RegisterFailHandler(Fail) - _, _ = fmt.Fprintf(GinkgoWriter, "Starting kubernetes-images-sync-operator suite\n") - RunSpecs(t, "e2e suite") -} diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go deleted file mode 100644 index 415f27c..0000000 --- a/test/e2e/e2e_test.go +++ /dev/null @@ -1,122 +0,0 @@ -/* -Copyright 2024. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package e2e - -import ( - "fmt" - "os/exec" - "time" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - - "github.com/lukaszraczylo/kubernetes-images-sync-operator/test/utils" -) - -const namespace = "kubernetes-images-sync-operator-system" - -var _ = Describe("controller", Ordered, func() { - BeforeAll(func() { - By("installing prometheus operator") - Expect(utils.InstallPrometheusOperator()).To(Succeed()) - - By("installing the cert-manager") - Expect(utils.InstallCertManager()).To(Succeed()) - - By("creating manager namespace") - cmd := exec.Command("kubectl", "create", "ns", namespace) - _, _ = utils.Run(cmd) - }) - - AfterAll(func() { - By("uninstalling the Prometheus manager bundle") - utils.UninstallPrometheusOperator() - - By("uninstalling the cert-manager bundle") - utils.UninstallCertManager() - - By("removing manager namespace") - cmd := exec.Command("kubectl", "delete", "ns", namespace) - _, _ = utils.Run(cmd) - }) - - Context("Operator", func() { - It("should run successfully", func() { - var controllerPodName string - var err error - - // projectimage stores the name of the image used in the example - var projectimage = "example.com/kubernetes-images-sync-operator:v0.0.1" - - By("building the manager(Operator) image") - cmd := exec.Command("make", "docker-build", fmt.Sprintf("IMG=%s", projectimage)) - _, err = utils.Run(cmd) - ExpectWithOffset(1, err).NotTo(HaveOccurred()) - - By("loading the the manager(Operator) image on Kind") - err = utils.LoadImageToKindClusterWithName(projectimage) - ExpectWithOffset(1, err).NotTo(HaveOccurred()) - - By("installing CRDs") - cmd = exec.Command("make", "install") - _, err = utils.Run(cmd) - ExpectWithOffset(1, err).NotTo(HaveOccurred()) - - By("deploying the sa") - cmd = exec.Command("make", "deploy", fmt.Sprintf("IMG=%s", projectimage)) - _, err = utils.Run(cmd) - ExpectWithOffset(1, err).NotTo(HaveOccurred()) - - By("validating that the sa pod is running as expected") - verifyControllerUp := func() error { - // Get pod name - - cmd = exec.Command("kubectl", "get", - "pods", "-l", "control-plane=sa", - "-o", "go-template={{ range .items }}"+ - "{{ if not .metadata.deletionTimestamp }}"+ - "{{ .metadata.name }}"+ - "{{ \"\\n\" }}{{ end }}{{ end }}", - "-n", namespace, - ) - - podOutput, err := utils.Run(cmd) - ExpectWithOffset(2, err).NotTo(HaveOccurred()) - podNames := utils.GetNonEmptyLines(string(podOutput)) - if len(podNames) != 1 { - return fmt.Errorf("expect 1 controller pods running, but got %d", len(podNames)) - } - controllerPodName = podNames[0] - ExpectWithOffset(2, controllerPodName).Should(ContainSubstring("sa")) - - // Validate pod status - cmd = exec.Command("kubectl", "get", - "pods", controllerPodName, "-o", "jsonpath={.status.phase}", - "-n", namespace, - ) - status, err := utils.Run(cmd) - ExpectWithOffset(2, err).NotTo(HaveOccurred()) - if string(status) != "Running" { - return fmt.Errorf("controller pod in %s status", status) - } - return nil - } - EventuallyWithOffset(1, verifyControllerUp, time.Minute, time.Second).Should(Succeed()) - - }) - }) -}) diff --git a/test/utils/utils.go b/test/utils/utils.go deleted file mode 100644 index 6b96ab5..0000000 --- a/test/utils/utils.go +++ /dev/null @@ -1,140 +0,0 @@ -/* -Copyright 2024. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package utils - -import ( - "fmt" - "os" - "os/exec" - "strings" - - . "github.com/onsi/ginkgo/v2" //nolint:golint,revive -) - -const ( - prometheusOperatorVersion = "v0.72.0" - prometheusOperatorURL = "https://github.com/prometheus-operator/prometheus-operator/" + - "releases/download/%s/bundle.yaml" - - certmanagerVersion = "v1.14.4" - certmanagerURLTmpl = "https://github.com/jetstack/cert-manager/releases/download/%s/cert-manager.yaml" -) - -func warnError(err error) { - _, _ = fmt.Fprintf(GinkgoWriter, "warning: %v\n", err) -} - -// InstallPrometheusOperator installs the prometheus Operator to be used to export the enabled metrics. -func InstallPrometheusOperator() error { - url := fmt.Sprintf(prometheusOperatorURL, prometheusOperatorVersion) - cmd := exec.Command("kubectl", "create", "-f", url) - _, err := Run(cmd) - return err -} - -// Run executes the provided command within this context -func Run(cmd *exec.Cmd) ([]byte, error) { - dir, _ := GetProjectDir() - cmd.Dir = dir - - if err := os.Chdir(cmd.Dir); err != nil { - _, _ = fmt.Fprintf(GinkgoWriter, "chdir dir: %s\n", err) - } - - cmd.Env = append(os.Environ(), "GO111MODULE=on") - command := strings.Join(cmd.Args, " ") - _, _ = fmt.Fprintf(GinkgoWriter, "running: %s\n", command) - output, err := cmd.CombinedOutput() - if err != nil { - return output, fmt.Errorf("%s failed with error: (%v) %s", command, err, string(output)) - } - - return output, nil -} - -// UninstallPrometheusOperator uninstalls the prometheus -func UninstallPrometheusOperator() { - url := fmt.Sprintf(prometheusOperatorURL, prometheusOperatorVersion) - cmd := exec.Command("kubectl", "delete", "-f", url) - if _, err := Run(cmd); err != nil { - warnError(err) - } -} - -// UninstallCertManager uninstalls the cert manager -func UninstallCertManager() { - url := fmt.Sprintf(certmanagerURLTmpl, certmanagerVersion) - cmd := exec.Command("kubectl", "delete", "-f", url) - if _, err := Run(cmd); err != nil { - warnError(err) - } -} - -// InstallCertManager installs the cert manager bundle. -func InstallCertManager() error { - url := fmt.Sprintf(certmanagerURLTmpl, certmanagerVersion) - cmd := exec.Command("kubectl", "apply", "-f", url) - if _, err := Run(cmd); err != nil { - return err - } - // Wait for cert-manager-webhook to be ready, which can take time if cert-manager - // was re-installed after uninstalling on a cluster. - cmd = exec.Command("kubectl", "wait", "deployment.apps/cert-manager-webhook", - "--for", "condition=Available", - "--namespace", "cert-manager", - "--timeout", "5m", - ) - - _, err := Run(cmd) - return err -} - -// LoadImageToKindClusterWithName loads a local docker image to the kind cluster -func LoadImageToKindClusterWithName(name string) error { - cluster := "kind" - if v, ok := os.LookupEnv("KIND_CLUSTER"); ok { - cluster = v - } - kindOptions := []string{"load", "docker-image", name, "--name", cluster} - cmd := exec.Command("kind", kindOptions...) - _, err := Run(cmd) - return err -} - -// GetNonEmptyLines converts given command output string into individual objects -// according to line breakers, and ignores the empty elements in it. -func GetNonEmptyLines(output string) []string { - var res []string - elements := strings.Split(output, "\n") - for _, element := range elements { - if element != "" { - res = append(res, element) - } - } - - return res -} - -// GetProjectDir will return the directory where the project is -func GetProjectDir() (string, error) { - wd, err := os.Getwd() - if err != nil { - return wd, err - } - wd = strings.Replace(wd, "/test/e2e", "", -1) - return wd, nil -}