lukaszraczylo/kubernetes-images-sync-operator
1
0
Fork 0
mirror of https://github.com/lukaszraczylo/kubernetes-images-sync-operator.git synced 2026-06-05 22:53:39 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add ability to provide annotations to the jobs.

Browse Source
This commit is contained in:
Lukasz Raczylo
2024-09-11 13:09:54 +01:00
parent bbb41681df
commit fffbae11d8
33 changed files with 143 additions and 85 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.gitignore
+3 -1
View File
@@ -26,4 +26,6 @@ go.work
*.swo
*~
**/.DS_Store
config/samples
config/samples
**/__pycache__
Makefile
+1 -1
View File
@@ -54,7 +54,7 @@ help: ## Display this help.
.PHONY: manifests
manifests: controller-gen ## Generate WebhookConfiguration, ClusterRole and CustomResourceDefinition objects.
$(CONTROLLER_GEN) rbac:roleName=mr-raczylo-com crd webhook paths="./..." output:crd:artifacts:config=config/crd/bases
$(CONTROLLER_GEN) rbac:roleName=impex-mgr crd webhook paths="./..." output:crd:artifacts:config=config/crd/bases
.PHONY: generate
generate: controller-gen ## Generate code containing DeepCopy, DeepCopyInto, and DeepCopyObject method implementations.
README.md
+2
View File
@@ -30,6 +30,8 @@ kind: ClusterImageExport
metadata:
name: backup-20240901
spec:
jobAnnotations:
my-fancy-export: 11-09-2024
# Excludes will remove all images with listed wording from the backup list
# excludes:
# - nginx
api/raczylo.com/v1/clusterimage_types.go
+9 -8
View File
@@ -28,14 +28,15 @@ import (
// +kubebuilder:printcolumn:name="Path",type="string",JSONPath=".spec.exportPath"
// +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp"
type ClusterImageSpec struct {
Image string `json:"image,omitempty"`
Tag string `json:"tag,omitempty"`
Sha string `json:"sha,omitempty"`
FullName string `json:"fullName,omitempty"` // Because I'm lazy and it's easier to pull that way
Storage string `json:"storage,omitempty"`
ExportName string `json:"exportName"`
ExportPath string `json:"exportPath,omitempty"`
ImageNamespace string `json:"imageNamespace,omitempty"`
Image string `json:"image,omitempty"`
Tag string `json:"tag,omitempty"`
Sha string `json:"sha,omitempty"`
FullName string `json:"fullName,omitempty"` // Because I'm lazy and it's easier to pull that way
Storage string `json:"storage,omitempty"`
ExportName string `json:"exportName"`
ExportPath string `json:"exportPath,omitempty"`
ImageNamespace string `json:"imageNamespace,omitempty"`
JobAnnotations map[string]string `json:"jobAnnotations,omitempty"`
}
// ClusterImageStatus defines the observed state of ClusterImage
api/raczylo.com/v1/clusterimageexport_types.go
+3 -2
View File
@@ -60,8 +60,9 @@ type ClusterImageExportSpec struct {
// Base path for the export - both file and S3
// +kubebuilder:validation:MinLength=1
// +kubebuilder:validation:MaxLength=255
BasePath string `json:"basePath"`
Storage ClusterImageStorageSpec `json:"storage"`
BasePath string `json:"basePath"`
Storage ClusterImageStorageSpec `json:"storage"`
JobAnnotations map[string]string `json:"jobAnnotations,omitempty"`
// +kubebuilder:validation.Minimum=1
// +kubebuilder:validation.Maximum=100
MaxConcurrentJobs int `json:"maxConcurrentJobs"`
api/raczylo.com/v1/zz_generated.deepcopy.go
+15 -1
View File
@@ -29,7 +29,7 @@ func (in *ClusterImage) DeepCopyInto(out *ClusterImage) {
*out = *in
out.TypeMeta = in.TypeMeta
in.ObjectMeta.DeepCopyInto(&out.ObjectMeta)
out.Spec = in.Spec
in.Spec.DeepCopyInto(&out.Spec)
out.Status = in.Status
}
@@ -135,6 +135,13 @@ func (in *ClusterImageExportSpec) DeepCopyInto(out *ClusterImageExportSpec) {
copy(*out, *in)
}
out.Storage = in.Storage
if in.JobAnnotations != nil {
in, out := &in.JobAnnotations, &out.JobAnnotations
*out = make(map[string]string, len(*in))
for key, val := range *in {
(*out)[key] = val
}
}
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ClusterImageExportSpec.
@@ -197,6 +204,13 @@ func (in *ClusterImageList) DeepCopyObject() runtime.Object {
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *ClusterImageSpec) DeepCopyInto(out *ClusterImageSpec) {
*out = *in
if in.JobAnnotations != nil {
in, out := &in.JobAnnotations, &out.JobAnnotations
*out = make(map[string]string, len(*in))
for key, val := range *in {
(*out)[key] = val
}
}
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ClusterImageSpec.
chart/Chart.yaml
+2 -2
View File
@@ -10,9 +10,9 @@ description: |
type: application
version: 0.1.18
version: 0.1.20
appVersion: "0.1.18"
appVersion: "0.1.20"
home: https://github.com/lukaszraczylo/kubernetes-images-sync-operator
chart/templates/clusterimage-crd.yaml
+4
View File
@@ -78,6 +78,10 @@ spec:
type: string
imageNamespace:
type: string
jobAnnotations:
additionalProperties:
type: string
type: object
sha:
type: string
storage:
chart/templates/clusterimageexport-crd.yaml
+4
View File
@@ -75,6 +75,10 @@ spec:
items:
type: string
type: array
jobAnnotations:
additionalProperties:
type: string
type: object
maxConcurrentJobs:
type: integer
name:
chart/templates/cm-raczylo-com-metrics-service.yaml
-14
View File
@@ -1,14 +0,0 @@
apiVersion: v1
kind: Service
metadata:
name: {{ include "chart.fullname" . }}-cm-raczylo-com-metrics-service
labels:
control-plane: cm-raczylo-com
{{- include "chart.labels" . | nindent 4 }}
spec:
type: {{ .Values.cmRaczyloComMetricsService.type }}
selector:
control-plane: cm-raczylo-com
{{- include "chart.selectorLabels" . | nindent 4 }}
ports:
{{- .Values.cmRaczyloComMetricsService.ports | toYaml | nindent 2 }}
chart/templates/deployment.yaml
+15 -11
View File
@@ -1,32 +1,36 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: {{ include "chart.fullname" . }}-cm-raczylo-com
name: {{ include "chart.fullname" . }}-sa
labels:
control-plane: cm-raczylo-com
control-plane: sa
{{- include "chart.labels" . | nindent 4 }}
spec:
replicas: {{ .Values.cmRaczyloCom.replicas }}
replicas: {{ .Values.sa.replicas }}
selector:
matchLabels:
control-plane: cm-raczylo-com
control-plane: sa
{{- include "chart.selectorLabels" . | nindent 6 }}
template:
metadata:
labels:
control-plane: cm-raczylo-com
control-plane: sa
{{- include "chart.selectorLabels" . | nindent 8 }}
annotations:
kubectl.kubernetes.io/default-container: manager
spec:
containers:
- args: {{- toYaml .Values.cmRaczyloCom.manager.args | nindent 8 }}
- args: {{- toYaml .Values.sa.manager.args | nindent 8 }}
command:
- /manager
env:
- name: POD_SERVICE_ACCOUNT
valueFrom:
fieldRef:
fieldPath: spec.serviceAccountName
- name: KUBERNETES_CLUSTER_DOMAIN
value: {{ quote .Values.kubernetesClusterDomain }}
image: {{ .Values.cmRaczyloCom.manager.image.repository }}:{{ .Values.cmRaczyloCom.manager.image.tag
image: {{ .Values.sa.manager.image.repository }}:{{ .Values.sa.manager.image.tag
| default .Chart.AppVersion }}
livenessProbe:
httpGet:
@@ -41,10 +45,10 @@ spec:
port: 8081
initialDelaySeconds: 5
periodSeconds: 10
resources: {{- toYaml .Values.cmRaczyloCom.manager.resources | nindent 10 }}
securityContext: {{- toYaml .Values.cmRaczyloCom.manager.containerSecurityContext
| nindent 10 }}
resources: {{- toYaml .Values.sa.manager.resources | nindent 10 }}
securityContext: {{- toYaml .Values.sa.manager.containerSecurityContext | nindent
10 }}
securityContext:
runAsNonRoot: true
serviceAccountName: {{ include "chart.fullname" . }}-cm-raczylo-com
serviceAccountName: {{ include "chart.fullname" . }}-sa
terminationGracePeriodSeconds: 10
chart/templates/mr-raczylo-com-rbac.yaml → chart/templates/impex-mgr-rbac.yaml
+1 -1
View File
@@ -1,7 +1,7 @@
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: {{ include "chart.fullname" . }}-mr-raczylo-com
name: {{ include "chart.fullname" . }}-impex-mgr
labels:
{{- include "chart.labels" . | nindent 4 }}
rules:
chart/templates/mr-raczylo-combinding-rbac.yaml → chart/templates/impex-mgrbinding-rbac.yaml
+3 -3
View File
@@ -1,14 +1,14 @@
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: {{ include "chart.fullname" . }}-mr-raczylo-combinding
name: {{ include "chart.fullname" . }}-impex-mgrbinding
labels:
{{- include "chart.labels" . | nindent 4 }}
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: '{{ include "chart.fullname" . }}-mr-raczylo-com'
name: '{{ include "chart.fullname" . }}-impex-mgr'
subjects:
- kind: ServiceAccount
name: '{{ include "chart.fullname" . }}-cm-raczylo-com'
name: '{{ include "chart.fullname" . }}-sa'
namespace: '{{ .Release.Namespace }}'
chart/templates/metrics-auth-raczylobinding-rbac.yaml
+1 -1
View File
@@ -10,5 +10,5 @@ roleRef:
name: '{{ include "chart.fullname" . }}-metrics-auth-raczylo'
subjects:
- kind: ServiceAccount
name: '{{ include "chart.fullname" . }}-cm-raczylo-com'
name: '{{ include "chart.fullname" . }}-sa'
namespace: '{{ .Release.Namespace }}'
chart/templates/raczylo-com-leaderbinding-rbac.yaml
+1 -1
View File
@@ -10,5 +10,5 @@ roleRef:
name: '{{ include "chart.fullname" . }}-raczylo-com-leader'
subjects:
- kind: ServiceAccount
name: '{{ include "chart.fullname" . }}-cm-raczylo-com'
name: '{{ include "chart.fullname" . }}-sa'
namespace: '{{ .Release.Namespace }}'
chart/templates/sa-metrics-service.yaml
+14
View File
@@ -0,0 +1,14 @@
apiVersion: v1
kind: Service
metadata:
name: {{ include "chart.fullname" . }}-sa-metrics-service
labels:
control-plane: sa
{{- include "chart.labels" . | nindent 4 }}
spec:
type: {{ .Values.saMetricsService.type }}
selector:
control-plane: sa
{{- include "chart.selectorLabels" . | nindent 4 }}
ports:
{{- .Values.saMetricsService.ports | toYaml | nindent 2 }}
chart/templates/serviceaccount.yaml
+2 -2
View File
@@ -1,8 +1,8 @@
apiVersion: v1
kind: ServiceAccount
metadata:
name: {{ include "chart.fullname" . }}-cm-raczylo-com
name: {{ include "chart.fullname" . }}-sa
labels:
{{- include "chart.labels" . | nindent 4 }}
annotations:
{{- toYaml .Values.cmRaczyloCom.serviceAccount.annotations | nindent 4 }}
{{- toYaml .Values.sa.serviceAccount.annotations | nindent 4 }}
chart/values.yaml
+4 -4
View File
@@ -1,4 +1,5 @@
cmRaczyloCom:
kubernetesClusterDomain: cluster.local
sa:
manager:
args:
- --metrics-bind-address=:8443
@@ -11,7 +12,7 @@ cmRaczyloCom:
- ALL
image:
repository: ghcr.io/lukaszraczylo/kubernetes-images-sync-operator
tag: 0.1.18
tag: 0.1.20
resources:
limits:
cpu: 500m
@@ -22,11 +23,10 @@ cmRaczyloCom:
replicas: 1
serviceAccount:
annotations: {}
cmRaczyloComMetricsService:
saMetricsService:
ports:
- name: https
port: 8443
protocol: TCP
targetPort: 8443
type: ClusterIP
kubernetesClusterDomain: cluster.local
config/crd/bases/raczylo.com_clusterimageexports.yaml
+4
View File
@@ -75,6 +75,10 @@ spec:
items:
type: string
type: array
jobAnnotations:
additionalProperties:
type: string
type: object
maxConcurrentJobs:
type: integer
name:
config/crd/bases/raczylo.com_clusterimages.yaml
+4
View File
@@ -77,6 +77,10 @@ spec:
type: string
imageNamespace:
type: string
jobAnnotations:
additionalProperties:
type: string
type: object
sha:
type: string
storage:
config/default/metrics_service.yaml
+3 -3
View File
@@ -2,10 +2,10 @@ apiVersion: v1
kind: Service
metadata:
labels:
control-plane: cm-raczylo-com
control-plane: sa
app.kubernetes.io/name: kubernetes-images-sync-operator
app.kubernetes.io/managed-by: kustomize
name: cm-raczylo-com-metrics-service
name: sa-metrics-service
namespace: system
spec:
ports:
@@ -14,4 +14,4 @@ spec:
protocol: TCP
targetPort: 8443
selector:
control-plane: cm-raczylo-com
control-plane: sa
config/manager/manager.yaml
+11 -6
View File
@@ -2,7 +2,7 @@ apiVersion: v1
kind: Namespace
metadata:
labels:
control-plane: cm-raczylo-com
control-plane: sa
app.kubernetes.io/name: kubernetes-images-sync-operator
app.kubernetes.io/managed-by: kustomize
name: system
@@ -10,23 +10,23 @@ metadata:
apiVersion: apps/v1
kind: Deployment
metadata:
name: cm-raczylo-com
name: sa
namespace: system
labels:
control-plane: cm-raczylo-com
control-plane: sa
app.kubernetes.io/name: kubernetes-images-sync-operator
app.kubernetes.io/managed-by: kustomize
spec:
selector:
matchLabels:
control-plane: cm-raczylo-com
control-plane: sa
replicas: 1
template:
metadata:
annotations:
kubectl.kubernetes.io/default-container: manager
labels:
control-plane: cm-raczylo-com
control-plane: sa
spec:
# TODO(user): Uncomment the following code to configure the nodeAffinity expression
# according to the platforms which are supported by your solution.
@@ -63,6 +63,11 @@ spec:
args:
- --leader-elect
- --health-probe-bind-address=:8081
env:
- name: POD_SERVICE_ACCOUNT
valueFrom:
fieldRef:
fieldPath: spec.serviceAccountName
image: controller:latest
name: manager
securityContext:
@@ -91,5 +96,5 @@ spec:
requests:
cpu: 10m
memory: 64Mi
serviceAccountName: cm-raczylo-com
serviceAccountName: sa
terminationGracePeriodSeconds: 10
config/network-policy/allow-metrics-traffic.yaml
+1 -1
View File
@@ -12,7 +12,7 @@ metadata:
spec:
podSelector:
matchLabels:
control-plane: cm-raczylo-com
control-plane: sa
policyTypes:
- Ingress
ingress:
config/prometheus/monitor.yaml
+3 -3
View File
@@ -3,10 +3,10 @@ apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
labels:
control-plane: cm-raczylo-com
control-plane: sa
app.kubernetes.io/name: kubernetes-images-sync-operator
app.kubernetes.io/managed-by: kustomize
name: cm-raczylo-com-metrics-monitor
name: sa-metrics-monitor
namespace: system
spec:
endpoints:
@@ -27,4 +27,4 @@ spec:
insecureSkipVerify: true
selector:
matchLabels:
control-plane: cm-raczylo-com
control-plane: sa
config/rbac/leader_election_role_binding.yaml
+1 -1
View File
@@ -11,5 +11,5 @@ roleRef:
name: raczylo-com-leader
subjects:
- kind: ServiceAccount
name: cm-raczylo-com
name: sa
namespace: system
config/rbac/metrics_auth_role_binding.yaml
+1 -1
View File
@@ -8,5 +8,5 @@ roleRef:
name: metrics-auth-raczylo
subjects:
- kind: ServiceAccount
name: cm-raczylo-com
name: sa
namespace: system
config/rbac/role.yaml
+1 -1
View File
@@ -2,7 +2,7 @@
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: mr-raczylo-com
name: impex-mgr
rules:
- apiGroups:
- ""
config/rbac/role_binding.yaml
+3 -3
View File
@@ -4,12 +4,12 @@ metadata:
labels:
app.kubernetes.io/name: kubernetes-images-sync-operator
app.kubernetes.io/managed-by: kustomize
name: mr-raczylo-combinding
name: impex-mgrbinding
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: mr-raczylo-com
name: impex-mgr
subjects:
- kind: ServiceAccount
name: cm-raczylo-com
name: sa
namespace: system
config/rbac/service_account.yaml
+1 -1
View File
@@ -4,5 +4,5 @@ metadata:
labels:
app.kubernetes.io/name: kubernetes-images-sync-operator
app.kubernetes.io/managed-by: kustomize
name: cm-raczylo-com
name: sa
namespace: system
internal/controller/raczylo.com/clusterimage_controller.go
+8 -4
View File
@@ -3,6 +3,7 @@ package raczylocom
import (
"context"
"fmt"
"os"
"strings"
"time"
@@ -34,6 +35,7 @@ type ClusterImageReconciler struct {
// +kubebuilder:rbac:groups=raczylo.com,resources=*/finalizers,verbs=update
// # additional RBAC rules - create and manage jobs
// +kubebuilder:rbac:groups=batch,resources=jobs,verbs=get;list;watch;create;update;patch;delete
// +kubebuilder:rbac:groups="",resources=pods,verbs=get;list;watch;create;update;patch;delete
func (r *ClusterImageReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
l := log.FromContext(ctx)
@@ -227,10 +229,12 @@ func (r *ClusterImageReconciler) createBackupJob(ctx context.Context, clusterIma
defaultCommands = append(defaultCommands, "rm -f /tmp/"+normalisedImageName+".tar")
jobParams := shared.JobParams{
Name: fmt.Sprintf("img-export-%s", clusterImage.Name),
Namespace: clusterImage.Namespace,
Image: shared.BACKUP_JOB_IMAGE,
Commands: defaultCommands,
Name: fmt.Sprintf("img-export-%s", clusterImage.Name),
Namespace: clusterImage.Namespace,
Image: shared.BACKUP_JOB_IMAGE,
Annotations: clusterImage.Spec.JobAnnotations,
Commands: defaultCommands,
ServiceAccount: os.Getenv("POD_SERVICE_ACCOUNT"),
OwnerReferences: []metav1.OwnerReference{
{
APIVersion: clusterImage.APIVersion,
internal/controller/raczylo.com/clusterimageexport_controller.go
+8 -4
View File
@@ -4,6 +4,7 @@ import (
"context"
"crypto/md5"
"fmt"
"os"
"strings"
"time"
@@ -138,6 +139,7 @@ func (r *ClusterImageExportReconciler) Reconcile(ctx context.Context, req ctrl.R
Storage: clusterImageExport.Spec.Storage.StorageTarget,
ExportName: clusterImageExport.Name,
ExportPath: clusterImageExport.Spec.BasePath,
JobAnnotations: clusterImageExport.Spec.JobAnnotations,
},
}
@@ -267,10 +269,12 @@ func (r *ClusterImageExportReconciler) runCleanupJob(ctx context.Context, cluste
}
jobParams := shared.JobParams{
Name: normalisedImageName,
Namespace: clusterImageExport.Namespace,
Image: shared.BACKUP_JOB_IMAGE,
Commands: defaultCommands,
Name: normalisedImageName,
Namespace: clusterImageExport.Namespace,
Image: shared.BACKUP_JOB_IMAGE,
Commands: defaultCommands,
Annotations: clusterImageExport.Spec.JobAnnotations,
ServiceAccount: os.Getenv("POD_SERVICE_ACCOUNT"),
}
cleanupJob := shared.CreateJob(jobParams, func(raczylocomv1.ClusterImageExport) []string { return nil })
internal/shared/jobs.go
+6 -1
View File
@@ -14,10 +14,12 @@ import (
type JobParams struct {
Name string
Namespace string
Annotations map[string]string
Image string
Commands []string
EnvVars []corev1.EnvVar
OwnerReferences []metav1.OwnerReference
ServiceAccount string
}
func CreateJob[T any](params JobParams, setupFunc func(T) []string) *batchv1.Job {
@@ -29,6 +31,7 @@ func CreateJob[T any](params JobParams, setupFunc func(T) []string) *batchv1.Job
Labels: map[string]string{
"app": "image-export",
},
Annotations: params.Annotations,
},
Spec: batchv1.JobSpec{
Template: corev1.PodTemplateSpec{
@@ -36,9 +39,11 @@ func CreateJob[T any](params JobParams, setupFunc func(T) []string) *batchv1.Job
Labels: map[string]string{
"app": "image-export",
},
Annotations: params.Annotations,
},
Spec: corev1.PodSpec{
RestartPolicy: corev1.RestartPolicyOnFailure,
RestartPolicy: corev1.RestartPolicyOnFailure,
ServiceAccountName: params.ServiceAccount,
Containers: []corev1.Container{
{
Name: "export",
test/e2e/e2e_test.go
+4 -4
View File
@@ -76,17 +76,17 @@ var _ = Describe("controller", Ordered, func() {
_, err = utils.Run(cmd)
ExpectWithOffset(1, err).NotTo(HaveOccurred())
By("deploying the cm-raczylo-com")
By("deploying the sa")
cmd = exec.Command("make", "deploy", fmt.Sprintf("IMG=%s", projectimage))
_, err = utils.Run(cmd)
ExpectWithOffset(1, err).NotTo(HaveOccurred())
By("validating that the cm-raczylo-com pod is running as expected")
By("validating that the sa pod is running as expected")
verifyControllerUp := func() error {
// Get pod name
cmd = exec.Command("kubectl", "get",
"pods", "-l", "control-plane=cm-raczylo-com",
"pods", "-l", "control-plane=sa",
"-o", "go-template={{ range .items }}"+
"{{ if not .metadata.deletionTimestamp }}"+
"{{ .metadata.name }}"+
@@ -101,7 +101,7 @@ var _ = Describe("controller", Ordered, func() {
return fmt.Errorf("expect 1 controller pods running, but got %d", len(podNames))
}
controllerPodName = podNames[0]
ExpectWithOffset(2, controllerPodName).Should(ContainSubstring("cm-raczylo-com"))
ExpectWithOffset(2, controllerPodName).Should(ContainSubstring("sa"))
// Validate pod status
cmd = exec.Command("kubectl", "get",