Initial commit.

This commit is contained in:
2023-02-19 20:46:21 +00:00
commit 984e1a0f7b
45 changed files with 11131 additions and 0 deletions
+71
View File
@@ -0,0 +1,71 @@
package controllers
import (
corev1 "k8s.io/api/core/v1"
)
func (cp *connPackage) checkGroupsStatus() {
groupsTotal := len(cp.mj.Spec.Groups)
totalJobs := 0
completedJobs := 0
didAnyJobAbort := false
changePresent := false
groupsCompleted := 0
// Check if all groups have completed and set ManagedJob status to "succeeded" if so.
if cp.mj.Spec.Status != ExecutionStatusSucceeded && groupsTotal > 0 {
for _, group := range cp.mj.Spec.Groups {
groupJobsTotal := len(group.Jobs)
groupJobsCompleted := 0
for _, job := range group.Jobs {
if job.Status == ExecutionStatusSucceeded {
groupJobsCompleted++
completedJobs++
}
if job.Status == ExecutionStatusFailed || job.Status == ExecutionStatusAborted {
didAnyJobAbort = true
}
totalJobs++
}
if groupJobsTotal == groupJobsCompleted {
// All the jobs in the group are completed.
if group.Status != ExecutionStatusSucceeded {
group.Status = ExecutionStatusSucceeded
changePresent = true
}
groupsCompleted++
}
}
}
if groupsTotal == groupsCompleted && cp.mj.Spec.Status != ExecutionStatusSucceeded && cp.mj.Status != ExecutionStatusSucceeded {
cp.mj.Spec.Status = ExecutionStatusSucceeded
changePresent = true
cp.r.Recorder.Eventf(cp.mj, corev1.EventTypeNormal, "Completed", "All jobs completed")
}
// Update status if any job aborted.
if didAnyJobAbort && cp.mj.Spec.Status != ExecutionStatusFailed {
cp.mj.Spec.Status = ExecutionStatusFailed
changePresent = true
cp.r.Recorder.Eventf(cp.mj, corev1.EventTypeNormal, "Aborted", "One of the jobs aborted")
}
// Update status to "running" if not already set.
// if cp.mj.Spec.Status != ExecutionStatusRunning && cp.mj.Spec.Status != ExecutionStatusSucceeded {
// cp.mj.Spec.Status = ExecutionStatusRunning
// cp.mj.Status = ExecutionStatusRunning
// changePresent = true
// }
// Check if the ManagedJob status has changed.
statusChanged := cp.mj.Spec.Status != cp.mj.Status
// Update status and send event if it has changed.
if statusChanged || changePresent {
cp.updateCRDStatusDirectly()
// cp.r.Client.Status().Update(cp.ctx, cp.mj)
}
}
+261
View File
@@ -0,0 +1,261 @@
package controllers
import (
"github.com/lukaszraczylo/pandati"
kbatch "k8s.io/api/batch/v1"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/labels"
jobsmanagerv1beta1 "raczylo.com/jobs-manager-operator/api/v1beta1"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/log"
)
/* Compile parameters from top to the job level */
type compiledParams struct {
FromEnv []corev1.EnvFromSource
Env []corev1.EnvVar
Volumes []corev1.Volume
VolumeMounts []corev1.VolumeMount
ServiceAccount string
RestartPolicy string
ImagePullSecrets []corev1.LocalObjectReference
ImagePullPolicy string
Labels map[string]string
}
func (cp *connPackage) compileParameters(params ...jobsmanagerv1beta1.ManagedJobParameters) jobsmanagerv1beta1.ManagedJobParameters {
cparams := jobsmanagerv1beta1.ManagedJobParameters{}
for _, params := range params {
if !pandati.IsZero(params) {
if params.FromEnv != nil {
cparams.FromEnv = append(cparams.FromEnv, params.FromEnv...)
}
if params.Env != nil {
cparams.Env = append(cparams.Env, params.Env...)
}
// if params.Volumes != nil {
// cparams.Volumes = append(cparams.Volumes, params.Volumes...)
// }
if params.VolumeMounts != nil {
cparams.VolumeMounts = append(cparams.VolumeMounts, params.VolumeMounts...)
}
if params.ServiceAccount != "" {
cparams.ServiceAccount = params.ServiceAccount
}
if params.RestartPolicy != "" {
cparams.RestartPolicy = params.RestartPolicy
}
if params.ImagePullSecrets != nil {
cparams.ImagePullSecrets = append(cparams.ImagePullSecrets, params.ImagePullSecrets...)
}
if params.Labels != nil {
for k, v := range params.Labels {
cparams.Labels[k] = v
}
}
}
}
return cparams
}
type previousJobsAndGroups struct {
GroupID string
JobID string
}
func (cp *connPackage) buildJobsDependencyTree() error {
changePending := false
for i, group := range cp.mj.Spec.Groups {
var group_previous string
if group.Parallel {
group_previous = group.Name
} else {
if i > 0 {
group_previous = cp.mj.Spec.Groups[i-1].Name
} else {
group_previous = group.Name
}
}
for j, job := range group.Jobs {
if job.Dependencies.Group == "" || job.Dependencies.Job == "" {
changePending = true
}
job.Dependencies.Group = group_previous
if job.Parallel {
job.Dependencies.Job = job.Name
} else {
if j > 0 {
job.Dependencies.Job = group.Jobs[j-1].Name
} else {
job.Dependencies.Job = job.Name
}
}
}
}
if changePending {
cp.updateCRDStatusDirectly()
}
return nil
}
func (cp *connPackage) checkJobStatus() {
var childJobs kbatch.JobList
labelSelector := labels.SelectorFromSet(labels.Set{
"jobmanager.raczylo.com/workflow-name": cp.mj.Name,
})
listOptions := &client.ListOptions{LabelSelector: labelSelector, Namespace: cp.mj.Namespace}
err := cp.r.Client.List(cp.ctx, &childJobs, listOptions)
if err != nil {
// log.Log.Error(err, "unable to list child jobs")
return
}
changePresent := false
for _, childJob := range childJobs.Items {
for _, group := range cp.mj.Spec.Groups {
for _, job := range group.Jobs {
generatedJobName := jobNameGenerator(cp.mj.Name, group.Name, job.Name)
if childJob.Name == generatedJobName {
if childJob.Status.Succeeded > 0 && job.Status != ExecutionStatusSucceeded {
job.Status = ExecutionStatusSucceeded
changePresent = true
cp.r.Recorder.Eventf(cp.mj, corev1.EventTypeNormal, "Completed", "Job %s completed", childJob.Name)
} else if childJob.Status.Failed > 0 && job.Status != ExecutionStatusFailed {
job.Status = ExecutionStatusFailed
changePresent = true
cp.r.Recorder.Eventf(cp.mj, corev1.EventTypeNormal, "Failed", "Job %s failed", childJob.Name)
} else if childJob.Status.Active > 0 && job.Status != ExecutionStatusRunning {
changePresent = true
job.Status = ExecutionStatusRunning
}
}
}
}
}
if changePresent {
cp.updateCRDStatusDirectly()
}
}
func (cp *connPackage) runPendingJobs() {
copyMJ := cp.mj.DeepCopy()
changePresent := false
for _, group := range cp.mj.Spec.Groups {
for _, job := range group.Jobs {
if job.Status == ExecutionStatusPending {
if job.Dependencies.Group == group.Name && job.Dependencies.Job == job.Name {
job.CompiledParams = cp.compileParameters(job.Params, group.Params, cp.mj.Spec.Params)
err := cp.executeJob(job, group)
if err != nil {
// log.Log.Info("Unable to execute job", "group", group.Name, "job", job.Name, "error", err)
continue
}
job.Status = ExecutionStatusRunning
changePresent = true
continue
} else {
for _, group2 := range copyMJ.Spec.Groups {
for _, job2 := range group2.Jobs {
if job2.Name == job.Dependencies.Job && group2.Name == job.Dependencies.Group {
switch job2.Status {
case ExecutionStatusSucceeded:
job.CompiledParams = cp.compileParameters(job.Params, group.Params, cp.mj.Spec.Params)
err := cp.executeJob(job, group)
if err != nil {
log.Log.Info("Unable to execute job", "group", group.Name, "job", job.Name, "error", err)
continue
}
job.Status = ExecutionStatusRunning
changePresent = true
case ExecutionStatusFailed:
job.Status = ExecutionStatusAborted
changePresent = true
}
}
}
}
}
}
}
}
if changePresent {
cp.updateCRDStatusDirectly()
}
}
func (cp *connPackage) executeJob(j *jobsmanagerv1beta1.ManagedJobDefinition, g *jobsmanagerv1beta1.ManagedJobGroup) (err error) {
generatedJobName := jobNameGenerator(cp.mj.Name, g.Name, j.Name)
convertRetries := func(retries int) *int32 {
if retries == 0 {
return nil
}
retries32 := int32(retries)
return &retries32
}
// compile labels
labels := map[string]string{
"jobmanager.raczylo.com/workflow-name": cp.mj.Name,
"jobmanager.raczylo.com/group-name": g.Name,
"jobmanager.raczylo.com/job-name": generatedJobName,
"jobmanager.raczylo.com/job-id": j.Name,
}
// merge labels with j.Parameters.Labels
for k, v := range j.CompiledParams.Labels {
labels[k] = v
}
job_handler := kbatch.Job{
ObjectMeta: metav1.ObjectMeta{
Name: generatedJobName,
Namespace: cp.mj.Namespace,
},
Spec: kbatch.JobSpec{
Template: corev1.PodTemplateSpec{
ObjectMeta: metav1.ObjectMeta{
Name: generatedJobName,
Namespace: cp.mj.Namespace,
Labels: labels,
},
Spec: corev1.PodSpec{
Volumes: j.CompiledParams.Volumes,
ImagePullSecrets: j.CompiledParams.ImagePullSecrets,
ServiceAccountName: j.CompiledParams.ServiceAccount,
Containers: []corev1.Container{
{
Name: generatedJobName,
Image: j.Image,
Args: j.Args,
ImagePullPolicy: corev1.PullPolicy(j.CompiledParams.ImagePullPolicy),
EnvFrom: j.CompiledParams.FromEnv,
Env: j.CompiledParams.Env,
VolumeMounts: j.CompiledParams.VolumeMounts,
},
},
RestartPolicy: corev1.RestartPolicy(j.CompiledParams.RestartPolicy),
},
},
BackoffLimit: convertRetries(cp.mj.Spec.Retries),
},
}
getMetaRefForWorkflowData, err := cp.getOwnerReference()
if err != nil {
return err
}
job_handler.SetOwnerReferences([]metav1.OwnerReference{getMetaRefForWorkflowData})
err = cp.r.Client.Create(cp.ctx, &job_handler)
if err != nil || pandati.IsZero(job_handler) {
log.Log.Error(err, "Unable to create job", "job", job_handler.Name)
return err
}
cp.r.Recorder.Eventf(cp.mj, corev1.EventTypeNormal, "Created", "Created job %s", job_handler.Name)
return nil
}
+17
View File
@@ -0,0 +1,17 @@
package controllers
// +kubebuilder:validation:Enum=Allow;Forbid;Replace
type ExecutionStatus string
const (
ExecutionStatusPending string = "pending"
ExecutionStatusRunning string = "running"
ExecutionStatusSucceeded string = "succeeded"
ExecutionStatusFailed string = "failed"
ExecutionStatusAborted string = "aborted"
ExecutionStatusUnknown string = "unknown"
)
var (
jobOwnerKey = ".metadata.controller"
)
+127
View File
@@ -0,0 +1,127 @@
package controllers
import (
"context"
"encoding/json"
"fmt"
"strings"
"sync"
"github.com/kr/pretty"
"github.com/lukaszraczylo/pandati"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
"raczylo.com/jobs-manager-operator/api/v1beta1"
jobsmanagerv1beta1 "raczylo.com/jobs-manager-operator/api/v1beta1"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/log"
)
func jobNameGenerator(name ...string) string {
// join name parts with "-" and convert to lowercase
return strings.ToLower(strings.Join(name, "-"))
}
type jobStatusUpdate struct {
Job *jobsmanagerv1beta1.ManagedJob
PatchedResource string
Status string
}
type connPackage struct {
r *ManagedJobReconciler
ctx context.Context
req ctrl.Request
mtx sync.Mutex
mj *jobsmanagerv1beta1.ManagedJob
}
func (cp *connPackage) getOwnerReference() (metav1.OwnerReference, error) {
mj := &jobsmanagerv1beta1.ManagedJob{}
err := cp.r.Client.Get(cp.ctx, cp.req.NamespacedName, mj)
if err != nil {
return metav1.OwnerReference{}, err
}
t := true
return metav1.OwnerReference{
APIVersion: v1beta1.GroupVersion.String(),
Kind: "ManagedJob",
Name: mj.Name,
UID: mj.UID,
Controller: &t,
}, nil
}
type patchStringValue struct {
Op string `json:"op"`
Path string `json:"path"`
Value string `json:"value"`
}
func (cp *connPackage) getLatestMainJobAndPatch(proposedChanges *jobsmanagerv1beta1.ManagedJob) (bool, error) {
mj := jobsmanagerv1beta1.ManagedJob{}
err := cp.r.Client.Get(cp.ctx, cp.req.NamespacedName, &mj)
if err != nil {
return false, err
}
diff, diffIdentical, _ := pandati.CompareStructsReplaced(mj, proposedChanges)
if !diffIdentical {
if err != nil {
log.Log.Error(err, "Unable to marshal ManagedJob")
}
var payloadBytesArr []patchStringValue
for _, d := range diff {
if strings.HasPrefix(d.Key, "/spec") {
operation := "replace"
if d.OldValue == "" || d.OldValue == nil {
operation = "add"
}
payload := patchStringValue{
Op: operation,
Path: d.Key,
Value: d.Value.(string),
}
payloadBytesArr = append(payloadBytesArr, payload)
}
}
patchPayload, err := json.Marshal(payloadBytesArr)
if err != nil {
log.Log.Error(err, "Unable to marshal ManagedJob")
}
fmt.Printf("PatchPayload: %# v", pretty.Formatter(fmt.Sprintf("%s", patchPayload)))
kubepath := client.RawPatch(types.JSONPatchType, patchPayload)
err = cp.r.Client.Patch(cp.ctx, &mj, kubepath)
// if err != nil {
// log.Log.Error(err, "Unable to patch ManagedJob")
// }
return true, err
}
return false, nil
}
func (cp *connPackage) updateCRDStatusDirectly() error {
cp.mtx.Lock()
// defer cp.mtx.Unlock()
// val, err := cp.getLatestMainJobAndPatch(cp.mj)
// if err != nil {
// log.Log.Error(err, "Unable to get latest ManagedJob")
// return err
// }
// if !val && err == nil {
err := cp.r.Update(cp.ctx, cp.mj)
if err != nil {
// log.Log.Info("Error", err.Error(), "more", "Unable to update ManagedJob status directly")
}
// get updated ManagedJob
err = cp.r.Client.Get(cp.ctx, cp.req.NamespacedName, cp.mj)
if err != nil {
log.Log.Error(err, "Unable to get updated ManagedJob")
}
// }
cp.mtx.Unlock()
return err
}
+74
View File
@@ -0,0 +1,74 @@
/*
Copyright 2023.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package controllers
import (
"context"
kbatch "k8s.io/api/batch/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/client-go/tools/record"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/log"
jobsmanagerv1beta1 "raczylo.com/jobs-manager-operator/api/v1beta1"
)
// ManagedJobReconciler reconciles a ManagedJob object
type ManagedJobReconciler struct {
client.Client
Scheme *runtime.Scheme
Recorder record.EventRecorder
}
//+kubebuilder:rbac:groups=jobsmanager.raczylo.com,resources=managedjobs,verbs=get;list;watch;create;update;patch;delete
//+kubebuilder:rbac:groups=jobsmanager.raczylo.com,resources=managedjobs/status,verbs=get;update;patch
//+kubebuilder:rbac:groups=jobsmanager.raczylo.com,resources=managedjobs/finalizers,verbs=update
func (r *ManagedJobReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
_ = log.FromContext(ctx)
cp := &connPackage{
r: r,
ctx: ctx,
req: req,
}
var managedJob jobsmanagerv1beta1.ManagedJob
if err := r.Get(ctx, req.NamespacedName, &managedJob); err != nil {
return ctrl.Result{}, client.IgnoreNotFound(err)
}
cp.mj = &managedJob
cp.buildJobsDependencyTree()
cp.checkJobStatus()
cp.runPendingJobs()
cp.checkGroupsStatus()
// fmt.Printf("Reconcile: %# v", pretty.Formatter(r.Updater))
return ctrl.Result{}, nil
}
// SetupWithManager sets up the controller with the Manager.
func (r *ManagedJobReconciler) SetupWithManager(mgr ctrl.Manager) error {
return ctrl.NewControllerManagedBy(mgr).
For(&jobsmanagerv1beta1.ManagedJob{}).
Owns(&kbatch.Job{}).
Complete(r)
}
+80
View File
@@ -0,0 +1,80 @@
/*
Copyright 2023.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package controllers
import (
"path/filepath"
"testing"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
"k8s.io/client-go/kubernetes/scheme"
"k8s.io/client-go/rest"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/envtest"
logf "sigs.k8s.io/controller-runtime/pkg/log"
"sigs.k8s.io/controller-runtime/pkg/log/zap"
jobsmanagerv1beta1 "raczylo.com/jobs-manager-operator/api/v1beta1"
//+kubebuilder:scaffold:imports
)
// These tests use Ginkgo (BDD-style Go testing framework). Refer to
// http://onsi.github.io/ginkgo/ to learn more about Ginkgo.
var cfg *rest.Config
var k8sClient client.Client
var testEnv *envtest.Environment
func TestAPIs(t *testing.T) {
RegisterFailHandler(Fail)
RunSpecs(t, "Controller Suite")
}
var _ = BeforeSuite(func() {
logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true)))
By("bootstrapping test environment")
testEnv = &envtest.Environment{
CRDDirectoryPaths: []string{filepath.Join("..", "config", "crd", "bases")},
ErrorIfCRDPathMissing: true,
}
var err error
// cfg is defined in this file globally.
cfg, err = testEnv.Start()
Expect(err).NotTo(HaveOccurred())
Expect(cfg).NotTo(BeNil())
err = jobsmanagerv1beta1.AddToScheme(scheme.Scheme)
Expect(err).NotTo(HaveOccurred())
//+kubebuilder:scaffold:scheme
k8sClient, err = client.New(cfg, client.Options{Scheme: scheme.Scheme})
Expect(err).NotTo(HaveOccurred())
Expect(k8sClient).NotTo(BeNil())
})
var _ = AfterSuite(func() {
By("tearing down the test environment")
err := testEnv.Stop()
Expect(err).NotTo(HaveOccurred())
})