mirror of
https://github.com/lukaszraczylo/jobs-manager-operator.git
synced 2026-07-10 06:43:01 +00:00
Multiple fixes (#29)
* Multiple fixes - add goreleaser to the build / release process - add kubectl plugin for job graphs visualization - add installation scripts - update dependencies * Update the release & CRD content. * Next set of improvements. Code Quality - Label constants: Added LabelWorkflowName, LabelGroupName, LabelJobName, LabelJobID in controllers/definitions.go - Removed commented debug code: Cleaned up dead code from multiple files - Removed unused dependencyTree field: Cleaned connPackage struct - Fixed snake_case variables: Changed to camelCase (runGroup, groupDep, runJob, jobDep, k8sJob) Kubernetes Best Practices - Finalizers: Implemented handleDeletion() and deleteChildJobs() for proper cleanup - Status enum validation: Added +kubebuilder:validation:Enum=pending;running;succeeded;failed;aborted - ImagePullPolicy default: Created getImagePullPolicy() helper that defaults to IfNotPresent - Resource limits support: Added Resources *corev1.ResourceRequirements to ManagedJobParameters Observability - Prometheus metrics: Created controllers/metrics.go with counters (jobs created/succeeded/failed), histogram (reconciliation duration), and gauge (active jobs) - Structured logging: Added logger field to connPackage, used context-based logging throughout Configuration - Leader election ID: Made configurable via --leader-election-id flag - Development mode: Made configurable via --dev-mode flag and LOG_LEVEL env var Performance - Dependency lookup optimization: Changed from O(n*m) to O(1) using lookup maps (jobDepMap, groupDepMap) - Reconciliation backoff: Added RequeueAfter: 30*time.Second when workflow is running Documentation & Testing - Godoc documentation: Added comprehensive comments to API types and controller - Unit tests: Added helpers_test.go with tests for all helper functions - Integration tests: Added managedjob_controller_test.go with Ginkgo/Gomega tests * Add the helm chart release. * Add reasonable test coverage.
This commit is contained in:
@@ -0,0 +1,732 @@
|
||||
package controllers
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"github.com/stretchr/testify/suite"
|
||||
kbatch "k8s.io/api/batch/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/types"
|
||||
ctrl "sigs.k8s.io/controller-runtime"
|
||||
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
|
||||
"sigs.k8s.io/controller-runtime/pkg/reconcile"
|
||||
|
||||
jobsmanagerv1beta1 "raczylo.com/jobs-manager-operator/api/v1beta1"
|
||||
)
|
||||
|
||||
// ReconcilerTestSuite contains all reconciler tests
|
||||
type ReconcilerTestSuite struct {
|
||||
suite.Suite
|
||||
ctx context.Context
|
||||
cancel context.CancelFunc
|
||||
client *MockClient
|
||||
reconciler *ManagedJobReconciler
|
||||
}
|
||||
|
||||
func (s *ReconcilerTestSuite) SetupTest() {
|
||||
s.ctx, s.cancel = context.WithTimeout(context.Background(), 30*time.Second)
|
||||
s.client = NewMockClient()
|
||||
s.reconciler = &ManagedJobReconciler{
|
||||
Client: s.client,
|
||||
Scheme: s.client.Scheme(),
|
||||
Recorder: NewFakeRecorder(),
|
||||
}
|
||||
}
|
||||
|
||||
func (s *ReconcilerTestSuite) TearDownTest() {
|
||||
s.cancel()
|
||||
}
|
||||
|
||||
func TestReconcilerSuite(t *testing.T) {
|
||||
suite.Run(t, new(ReconcilerTestSuite))
|
||||
}
|
||||
|
||||
// ==================== GOOD SCENARIOS ====================
|
||||
|
||||
func (s *ReconcilerTestSuite) TestReconcile_Good_NewManagedJob_AddsFinalizer() {
|
||||
// Arrange
|
||||
mj := NewTestManagedJob("test-workflow", "default", []*jobsmanagerv1beta1.ManagedJobGroup{
|
||||
NewTestGroup("group1", []*jobsmanagerv1beta1.ManagedJobDefinition{
|
||||
NewTestJobDef("job1", "busybox:latest"),
|
||||
}),
|
||||
})
|
||||
s.client.AddManagedJob(mj)
|
||||
|
||||
req := reconcile.Request{
|
||||
NamespacedName: types.NamespacedName{Name: "test-workflow", Namespace: "default"},
|
||||
}
|
||||
|
||||
// Act
|
||||
result, err := s.reconciler.Reconcile(s.ctx, req)
|
||||
|
||||
// Assert
|
||||
s.NoError(err)
|
||||
s.True(result.RequeueAfter > 0, "should requeue after adding finalizer")
|
||||
|
||||
updated := s.client.GetManagedJobByKey("default/test-workflow")
|
||||
s.True(controllerutil.ContainsFinalizer(updated, FinalizerName), "finalizer should be added")
|
||||
}
|
||||
|
||||
func (s *ReconcilerTestSuite) TestReconcile_Good_SingleJobWorkflow_CreatesJob() {
|
||||
// Arrange
|
||||
mj := NewTestManagedJob("test-workflow", "default", []*jobsmanagerv1beta1.ManagedJobGroup{
|
||||
NewTestGroup("group1", []*jobsmanagerv1beta1.ManagedJobDefinition{
|
||||
NewTestJobDef("job1", "busybox:latest"),
|
||||
}),
|
||||
})
|
||||
controllerutil.AddFinalizer(mj, FinalizerName)
|
||||
s.client.AddManagedJob(mj)
|
||||
|
||||
req := reconcile.Request{
|
||||
NamespacedName: types.NamespacedName{Name: "test-workflow", Namespace: "default"},
|
||||
}
|
||||
|
||||
// Act - first reconcile generates dependency tree
|
||||
_, err := s.reconciler.Reconcile(s.ctx, req)
|
||||
s.NoError(err)
|
||||
|
||||
// Act - second reconcile runs jobs
|
||||
_, err = s.reconciler.Reconcile(s.ctx, req)
|
||||
s.NoError(err)
|
||||
|
||||
// Assert
|
||||
s.GreaterOrEqual(s.client.CreateCalls, 1, "should have created at least one job")
|
||||
}
|
||||
|
||||
func (s *ReconcilerTestSuite) TestReconcile_Good_CompletedJob_UpdatesStatus() {
|
||||
// Arrange
|
||||
mj := NewTestManagedJob("test-workflow", "default", []*jobsmanagerv1beta1.ManagedJobGroup{
|
||||
NewTestGroup("group1", []*jobsmanagerv1beta1.ManagedJobDefinition{
|
||||
NewTestJobDef("job1", "busybox:latest"),
|
||||
}),
|
||||
})
|
||||
controllerutil.AddFinalizer(mj, FinalizerName)
|
||||
mj.Spec.Groups[0].Jobs[0].Status = ExecutionStatusRunning
|
||||
mj.Spec.Groups[0].Status = ExecutionStatusRunning
|
||||
s.client.AddManagedJob(mj)
|
||||
|
||||
// Add a completed K8s job
|
||||
k8sJob := NewTestK8sJob("test-workflow-group1-job1", "default", "test-workflow", "group1", kbatch.JobStatus{
|
||||
Succeeded: 1,
|
||||
})
|
||||
s.client.AddJob(k8sJob)
|
||||
|
||||
req := reconcile.Request{
|
||||
NamespacedName: types.NamespacedName{Name: "test-workflow", Namespace: "default"},
|
||||
}
|
||||
|
||||
// Act
|
||||
_, err := s.reconciler.Reconcile(s.ctx, req)
|
||||
|
||||
// Assert
|
||||
s.NoError(err)
|
||||
}
|
||||
|
||||
func (s *ReconcilerTestSuite) TestReconcile_Good_RunningWorkflow_RequeuesAfterDelay() {
|
||||
// Arrange
|
||||
mj := NewTestManagedJob("test-workflow", "default", []*jobsmanagerv1beta1.ManagedJobGroup{
|
||||
NewTestGroup("group1", []*jobsmanagerv1beta1.ManagedJobDefinition{
|
||||
NewTestJobDef("job1", "busybox:latest"),
|
||||
}),
|
||||
})
|
||||
controllerutil.AddFinalizer(mj, FinalizerName)
|
||||
mj.Status = ExecutionStatusRunning
|
||||
mj.Spec.Groups[0].Status = ExecutionStatusRunning
|
||||
mj.Spec.Groups[0].Jobs[0].Status = ExecutionStatusRunning
|
||||
s.client.AddManagedJob(mj)
|
||||
|
||||
// Add running K8s job
|
||||
k8sJob := NewTestK8sJob("test-workflow-group1-job1", "default", "test-workflow", "group1", kbatch.JobStatus{
|
||||
Active: 1,
|
||||
})
|
||||
s.client.AddJob(k8sJob)
|
||||
|
||||
req := reconcile.Request{
|
||||
NamespacedName: types.NamespacedName{Name: "test-workflow", Namespace: "default"},
|
||||
}
|
||||
|
||||
// Act
|
||||
result, err := s.reconciler.Reconcile(s.ctx, req)
|
||||
|
||||
// Assert
|
||||
s.NoError(err)
|
||||
s.Equal(RequeueDelay, result.RequeueAfter, "should requeue after delay for running workflow")
|
||||
}
|
||||
|
||||
// ==================== NOT GOOD SCENARIOS ====================
|
||||
|
||||
func (s *ReconcilerTestSuite) TestReconcile_NotGood_ManagedJobNotFound_ReturnsNoError() {
|
||||
// Arrange - no ManagedJob added to client
|
||||
req := reconcile.Request{
|
||||
NamespacedName: types.NamespacedName{Name: "nonexistent", Namespace: "default"},
|
||||
}
|
||||
|
||||
// Act
|
||||
result, err := s.reconciler.Reconcile(s.ctx, req)
|
||||
|
||||
// Assert
|
||||
s.NoError(err, "should not return error for not found")
|
||||
s.Zero(result.RequeueAfter, "should not requeue for not found")
|
||||
}
|
||||
|
||||
func (s *ReconcilerTestSuite) TestReconcile_NotGood_FailedJob_UpdatesStatusToFailed() {
|
||||
// Arrange
|
||||
mj := NewTestManagedJob("test-workflow", "default", []*jobsmanagerv1beta1.ManagedJobGroup{
|
||||
NewTestGroup("group1", []*jobsmanagerv1beta1.ManagedJobDefinition{
|
||||
NewTestJobDef("job1", "busybox:latest"),
|
||||
}),
|
||||
})
|
||||
controllerutil.AddFinalizer(mj, FinalizerName)
|
||||
mj.Spec.Groups[0].Jobs[0].Status = ExecutionStatusRunning
|
||||
mj.Spec.Groups[0].Status = ExecutionStatusRunning
|
||||
s.client.AddManagedJob(mj)
|
||||
|
||||
// Add a failed K8s job
|
||||
k8sJob := NewTestK8sJob("test-workflow-group1-job1", "default", "test-workflow", "group1", kbatch.JobStatus{
|
||||
Failed: 1,
|
||||
})
|
||||
s.client.AddJob(k8sJob)
|
||||
|
||||
req := reconcile.Request{
|
||||
NamespacedName: types.NamespacedName{Name: "test-workflow", Namespace: "default"},
|
||||
}
|
||||
|
||||
// Act
|
||||
_, err := s.reconciler.Reconcile(s.ctx, req)
|
||||
|
||||
// Assert
|
||||
s.NoError(err)
|
||||
}
|
||||
|
||||
func (s *ReconcilerTestSuite) TestReconcile_NotGood_DeletionInProgress_CleansUpJobs() {
|
||||
// Arrange
|
||||
now := metav1.Now()
|
||||
mj := NewTestManagedJob("test-workflow", "default", []*jobsmanagerv1beta1.ManagedJobGroup{
|
||||
NewTestGroup("group1", []*jobsmanagerv1beta1.ManagedJobDefinition{
|
||||
NewTestJobDef("job1", "busybox:latest"),
|
||||
}),
|
||||
})
|
||||
controllerutil.AddFinalizer(mj, FinalizerName)
|
||||
mj.DeletionTimestamp = &now
|
||||
s.client.AddManagedJob(mj)
|
||||
|
||||
// Add child job to be deleted
|
||||
k8sJob := NewTestK8sJob("test-workflow-group1-job1", "default", "test-workflow", "group1", kbatch.JobStatus{})
|
||||
s.client.AddJob(k8sJob)
|
||||
|
||||
req := reconcile.Request{
|
||||
NamespacedName: types.NamespacedName{Name: "test-workflow", Namespace: "default"},
|
||||
}
|
||||
|
||||
// Act
|
||||
_, err := s.reconciler.Reconcile(s.ctx, req)
|
||||
|
||||
// Assert
|
||||
s.NoError(err)
|
||||
s.GreaterOrEqual(s.client.DeleteCalls, 1, "should have deleted child jobs")
|
||||
}
|
||||
|
||||
// ==================== REALLY BAD SCENARIOS ====================
|
||||
|
||||
func (s *ReconcilerTestSuite) TestReconcile_ReallyBad_GetError_ReturnsError() {
|
||||
// Arrange
|
||||
s.client.GetError = ErrServerError
|
||||
|
||||
req := reconcile.Request{
|
||||
NamespacedName: types.NamespacedName{Name: "test-workflow", Namespace: "default"},
|
||||
}
|
||||
|
||||
// Act
|
||||
_, err := s.reconciler.Reconcile(s.ctx, req)
|
||||
|
||||
// Assert
|
||||
s.Error(err)
|
||||
}
|
||||
|
||||
func (s *ReconcilerTestSuite) TestReconcile_ReallyBad_UpdateConflict_ReturnsError() {
|
||||
// Arrange
|
||||
mj := NewTestManagedJob("test-workflow", "default", []*jobsmanagerv1beta1.ManagedJobGroup{
|
||||
NewTestGroup("group1", []*jobsmanagerv1beta1.ManagedJobDefinition{
|
||||
NewTestJobDef("job1", "busybox:latest"),
|
||||
}),
|
||||
})
|
||||
s.client.AddManagedJob(mj)
|
||||
s.client.UpdateError = ErrConflict
|
||||
|
||||
req := reconcile.Request{
|
||||
NamespacedName: types.NamespacedName{Name: "test-workflow", Namespace: "default"},
|
||||
}
|
||||
|
||||
// Act
|
||||
_, err := s.reconciler.Reconcile(s.ctx, req)
|
||||
|
||||
// Assert
|
||||
s.Error(err, "should return error on update conflict")
|
||||
}
|
||||
|
||||
func (s *ReconcilerTestSuite) TestReconcile_ReallyBad_ContextTimeout_ReturnsError() {
|
||||
// Arrange
|
||||
mj := NewTestManagedJob("test-workflow", "default", []*jobsmanagerv1beta1.ManagedJobGroup{
|
||||
NewTestGroup("group1", []*jobsmanagerv1beta1.ManagedJobDefinition{
|
||||
NewTestJobDef("job1", "busybox:latest"),
|
||||
}),
|
||||
})
|
||||
s.client.AddManagedJob(mj)
|
||||
|
||||
// Cancel context immediately
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
|
||||
req := reconcile.Request{
|
||||
NamespacedName: types.NamespacedName{Name: "test-workflow", Namespace: "default"},
|
||||
}
|
||||
|
||||
// Act
|
||||
_, err := s.reconciler.Reconcile(ctx, req)
|
||||
|
||||
// Assert
|
||||
s.Error(err, "should return error on context timeout")
|
||||
}
|
||||
|
||||
func (s *ReconcilerTestSuite) TestReconcile_ReallyBad_ListJobsError_Continues() {
|
||||
// Arrange
|
||||
mj := NewTestManagedJob("test-workflow", "default", []*jobsmanagerv1beta1.ManagedJobGroup{
|
||||
NewTestGroup("group1", []*jobsmanagerv1beta1.ManagedJobDefinition{
|
||||
NewTestJobDef("job1", "busybox:latest"),
|
||||
}),
|
||||
})
|
||||
controllerutil.AddFinalizer(mj, FinalizerName)
|
||||
s.client.AddManagedJob(mj)
|
||||
s.client.ListError = ErrServerError
|
||||
|
||||
req := reconcile.Request{
|
||||
NamespacedName: types.NamespacedName{Name: "test-workflow", Namespace: "default"},
|
||||
}
|
||||
|
||||
// Act - should not panic, should handle gracefully
|
||||
_, err := s.reconciler.Reconcile(s.ctx, req)
|
||||
|
||||
// Assert - list error is logged but doesn't fail reconciliation
|
||||
s.NoError(err)
|
||||
}
|
||||
|
||||
// ==================== MATRIX TEST: WORKFLOW SCENARIOS ====================
|
||||
|
||||
func TestReconcile_WorkflowScenarios(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
scenario TestScenario
|
||||
setupMJ func() *jobsmanagerv1beta1.ManagedJob
|
||||
setupJobs func() []*kbatch.Job
|
||||
clientSetup func(*MockClient)
|
||||
expectError bool
|
||||
expectRequeue bool
|
||||
expectRequeueAfter time.Duration
|
||||
validateResult func(*testing.T, *MockClient, ctrl.Result)
|
||||
}{
|
||||
// GOOD SCENARIOS
|
||||
{
|
||||
name: "good_simple_workflow_starts",
|
||||
scenario: ScenarioGood,
|
||||
setupMJ: func() *jobsmanagerv1beta1.ManagedJob {
|
||||
mj := NewTestManagedJob("workflow", "default", []*jobsmanagerv1beta1.ManagedJobGroup{
|
||||
NewTestGroup("init", []*jobsmanagerv1beta1.ManagedJobDefinition{
|
||||
NewTestJobDef("setup", "busybox:latest"),
|
||||
}),
|
||||
})
|
||||
controllerutil.AddFinalizer(mj, FinalizerName)
|
||||
return mj
|
||||
},
|
||||
setupJobs: func() []*kbatch.Job { return nil },
|
||||
expectError: false,
|
||||
expectRequeue: false,
|
||||
},
|
||||
{
|
||||
name: "good_multi_group_workflow",
|
||||
scenario: ScenarioGood,
|
||||
setupMJ: func() *jobsmanagerv1beta1.ManagedJob {
|
||||
mj := NewTestManagedJob("workflow", "default", []*jobsmanagerv1beta1.ManagedJobGroup{
|
||||
NewTestGroup("init", []*jobsmanagerv1beta1.ManagedJobDefinition{
|
||||
NewTestJobDef("setup", "busybox:latest"),
|
||||
}),
|
||||
NewTestGroup("main", []*jobsmanagerv1beta1.ManagedJobDefinition{
|
||||
NewTestJobDef("process", "busybox:latest"),
|
||||
}, NewTestDependency("init", ExecutionStatusPending)),
|
||||
})
|
||||
controllerutil.AddFinalizer(mj, FinalizerName)
|
||||
return mj
|
||||
},
|
||||
setupJobs: func() []*kbatch.Job { return nil },
|
||||
expectError: false,
|
||||
expectRequeue: false,
|
||||
},
|
||||
{
|
||||
name: "good_all_jobs_completed",
|
||||
scenario: ScenarioGood,
|
||||
setupMJ: func() *jobsmanagerv1beta1.ManagedJob {
|
||||
mj := NewTestManagedJob("workflow", "default", []*jobsmanagerv1beta1.ManagedJobGroup{
|
||||
NewTestGroup("init", []*jobsmanagerv1beta1.ManagedJobDefinition{
|
||||
NewTestJobDef("setup", "busybox:latest"),
|
||||
}),
|
||||
})
|
||||
controllerutil.AddFinalizer(mj, FinalizerName)
|
||||
mj.Spec.Groups[0].Jobs[0].Status = ExecutionStatusSucceeded
|
||||
mj.Spec.Groups[0].Status = ExecutionStatusSucceeded
|
||||
return mj
|
||||
},
|
||||
setupJobs: func() []*kbatch.Job {
|
||||
return []*kbatch.Job{
|
||||
NewTestK8sJob("workflow-init-setup", "default", "workflow", "init", kbatch.JobStatus{Succeeded: 1}),
|
||||
}
|
||||
},
|
||||
expectError: false,
|
||||
expectRequeue: false,
|
||||
},
|
||||
|
||||
// NOT GOOD SCENARIOS
|
||||
{
|
||||
name: "notgood_job_failed_workflow_continues",
|
||||
scenario: ScenarioNotGood,
|
||||
setupMJ: func() *jobsmanagerv1beta1.ManagedJob {
|
||||
mj := NewTestManagedJob("workflow", "default", []*jobsmanagerv1beta1.ManagedJobGroup{
|
||||
NewTestGroup("init", []*jobsmanagerv1beta1.ManagedJobDefinition{
|
||||
NewTestJobDef("setup", "busybox:latest"),
|
||||
}),
|
||||
})
|
||||
controllerutil.AddFinalizer(mj, FinalizerName)
|
||||
mj.Spec.Groups[0].Jobs[0].Status = ExecutionStatusRunning
|
||||
return mj
|
||||
},
|
||||
setupJobs: func() []*kbatch.Job {
|
||||
return []*kbatch.Job{
|
||||
NewTestK8sJob("workflow-init-setup", "default", "workflow", "init", kbatch.JobStatus{Failed: 1}),
|
||||
}
|
||||
},
|
||||
expectError: false,
|
||||
expectRequeue: false,
|
||||
},
|
||||
{
|
||||
name: "notgood_dependent_job_aborted",
|
||||
scenario: ScenarioNotGood,
|
||||
setupMJ: func() *jobsmanagerv1beta1.ManagedJob {
|
||||
mj := NewTestManagedJob("workflow", "default", []*jobsmanagerv1beta1.ManagedJobGroup{
|
||||
NewTestGroup("init", []*jobsmanagerv1beta1.ManagedJobDefinition{
|
||||
NewTestJobDef("setup", "busybox:latest"),
|
||||
NewTestJobDef("verify", "busybox:latest", NewTestDependency("setup", ExecutionStatusFailed)),
|
||||
}),
|
||||
})
|
||||
controllerutil.AddFinalizer(mj, FinalizerName)
|
||||
return mj
|
||||
},
|
||||
setupJobs: func() []*kbatch.Job {
|
||||
return []*kbatch.Job{
|
||||
NewTestK8sJob("workflow-init-setup", "default", "workflow", "init", kbatch.JobStatus{Failed: 1}),
|
||||
}
|
||||
},
|
||||
expectError: false,
|
||||
expectRequeue: false,
|
||||
},
|
||||
|
||||
// REALLY BAD SCENARIOS
|
||||
{
|
||||
name: "reallybad_api_server_unavailable",
|
||||
scenario: ScenarioReallyBad,
|
||||
setupMJ: func() *jobsmanagerv1beta1.ManagedJob {
|
||||
return NewTestManagedJob("workflow", "default", nil)
|
||||
},
|
||||
setupJobs: func() []*kbatch.Job { return nil },
|
||||
clientSetup: func(c *MockClient) {
|
||||
c.GetError = ErrNetworkFailure
|
||||
},
|
||||
expectError: true,
|
||||
expectRequeue: false,
|
||||
},
|
||||
{
|
||||
name: "reallybad_create_job_fails",
|
||||
scenario: ScenarioReallyBad,
|
||||
setupMJ: func() *jobsmanagerv1beta1.ManagedJob {
|
||||
mj := NewTestManagedJob("workflow", "default", []*jobsmanagerv1beta1.ManagedJobGroup{
|
||||
NewTestGroup("init", []*jobsmanagerv1beta1.ManagedJobDefinition{
|
||||
NewTestJobDef("setup", "busybox:latest"),
|
||||
}),
|
||||
})
|
||||
controllerutil.AddFinalizer(mj, FinalizerName)
|
||||
return mj
|
||||
},
|
||||
setupJobs: func() []*kbatch.Job { return nil },
|
||||
clientSetup: func(c *MockClient) {
|
||||
c.CreateError = ErrForbidden
|
||||
},
|
||||
expectError: false, // Job creation failure doesn't stop reconciliation
|
||||
expectRequeue: false,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
client := NewMockClient()
|
||||
reconciler := &ManagedJobReconciler{
|
||||
Client: client,
|
||||
Scheme: client.Scheme(),
|
||||
Recorder: NewFakeRecorder(),
|
||||
}
|
||||
|
||||
// Setup client behavior
|
||||
if tt.clientSetup != nil {
|
||||
tt.clientSetup(client)
|
||||
}
|
||||
|
||||
// Add ManagedJob
|
||||
mj := tt.setupMJ()
|
||||
if mj != nil {
|
||||
client.AddManagedJob(mj)
|
||||
}
|
||||
|
||||
// Add Jobs
|
||||
for _, job := range tt.setupJobs() {
|
||||
client.AddJob(job)
|
||||
}
|
||||
|
||||
req := reconcile.Request{
|
||||
NamespacedName: types.NamespacedName{Name: "workflow", Namespace: "default"},
|
||||
}
|
||||
|
||||
// Act
|
||||
result, err := reconciler.Reconcile(ctx, req)
|
||||
|
||||
// Assert
|
||||
if tt.expectError {
|
||||
assert.Error(t, err, "expected error for scenario: %s", tt.scenario)
|
||||
} else {
|
||||
assert.NoError(t, err, "expected no error for scenario: %s", tt.scenario)
|
||||
}
|
||||
|
||||
if tt.expectRequeue {
|
||||
assert.True(t, result.RequeueAfter > 0, "expected requeue")
|
||||
}
|
||||
|
||||
if tt.expectRequeueAfter > 0 {
|
||||
assert.Equal(t, tt.expectRequeueAfter, result.RequeueAfter)
|
||||
}
|
||||
|
||||
if tt.validateResult != nil {
|
||||
tt.validateResult(t, client, result)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// ==================== EDGE CASES ====================
|
||||
|
||||
func TestReconcile_EdgeCases(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
description string
|
||||
setup func(*MockClient) *jobsmanagerv1beta1.ManagedJob
|
||||
validate func(*testing.T, *MockClient, ctrl.Result, error)
|
||||
}{
|
||||
{
|
||||
name: "empty_groups",
|
||||
description: "ManagedJob with no groups should complete immediately",
|
||||
setup: func(c *MockClient) *jobsmanagerv1beta1.ManagedJob {
|
||||
mj := NewTestManagedJob("empty", "default", []*jobsmanagerv1beta1.ManagedJobGroup{})
|
||||
controllerutil.AddFinalizer(mj, FinalizerName)
|
||||
return mj
|
||||
},
|
||||
validate: func(t *testing.T, c *MockClient, r ctrl.Result, err error) {
|
||||
assert.NoError(t, err)
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "group_with_no_jobs",
|
||||
description: "Group with no jobs should be marked as succeeded",
|
||||
setup: func(c *MockClient) *jobsmanagerv1beta1.ManagedJob {
|
||||
mj := NewTestManagedJob("no-jobs", "default", []*jobsmanagerv1beta1.ManagedJobGroup{
|
||||
NewTestGroup("empty-group", []*jobsmanagerv1beta1.ManagedJobDefinition{}),
|
||||
})
|
||||
controllerutil.AddFinalizer(mj, FinalizerName)
|
||||
return mj
|
||||
},
|
||||
validate: func(t *testing.T, c *MockClient, r ctrl.Result, err error) {
|
||||
assert.NoError(t, err)
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "circular_dependency_protection",
|
||||
description: "Jobs with circular dependencies should not cause infinite loop",
|
||||
setup: func(c *MockClient) *jobsmanagerv1beta1.ManagedJob {
|
||||
mj := NewTestManagedJob("circular", "default", []*jobsmanagerv1beta1.ManagedJobGroup{
|
||||
NewTestGroup("group1", []*jobsmanagerv1beta1.ManagedJobDefinition{
|
||||
NewTestJobDef("job-a", "busybox", NewTestDependency("job-b", ExecutionStatusPending)),
|
||||
NewTestJobDef("job-b", "busybox", NewTestDependency("job-a", ExecutionStatusPending)),
|
||||
}),
|
||||
})
|
||||
controllerutil.AddFinalizer(mj, FinalizerName)
|
||||
return mj
|
||||
},
|
||||
validate: func(t *testing.T, c *MockClient, r ctrl.Result, err error) {
|
||||
assert.NoError(t, err, "should handle circular deps gracefully")
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "rapid_status_changes",
|
||||
description: "Multiple rapid reconciliations should be idempotent",
|
||||
setup: func(c *MockClient) *jobsmanagerv1beta1.ManagedJob {
|
||||
mj := NewTestManagedJob("rapid", "default", []*jobsmanagerv1beta1.ManagedJobGroup{
|
||||
NewTestGroup("group1", []*jobsmanagerv1beta1.ManagedJobDefinition{
|
||||
NewTestJobDef("job1", "busybox"),
|
||||
}),
|
||||
})
|
||||
controllerutil.AddFinalizer(mj, FinalizerName)
|
||||
return mj
|
||||
},
|
||||
validate: func(t *testing.T, c *MockClient, r ctrl.Result, err error) {
|
||||
assert.NoError(t, err)
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "very_long_job_name",
|
||||
description: "Long names should be handled (K8s has 63 char limit)",
|
||||
setup: func(c *MockClient) *jobsmanagerv1beta1.ManagedJob {
|
||||
mj := NewTestManagedJob("workflow", "default", []*jobsmanagerv1beta1.ManagedJobGroup{
|
||||
NewTestGroup("very-long-group-name-that-exceeds-normal", []*jobsmanagerv1beta1.ManagedJobDefinition{
|
||||
NewTestJobDef("extremely-long-job-name-here", "busybox"),
|
||||
}),
|
||||
})
|
||||
controllerutil.AddFinalizer(mj, FinalizerName)
|
||||
return mj
|
||||
},
|
||||
validate: func(t *testing.T, c *MockClient, r ctrl.Result, err error) {
|
||||
assert.NoError(t, err)
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "special_characters_in_name",
|
||||
description: "Names with special characters",
|
||||
setup: func(c *MockClient) *jobsmanagerv1beta1.ManagedJob {
|
||||
mj := NewTestManagedJob("test-workflow", "default", []*jobsmanagerv1beta1.ManagedJobGroup{
|
||||
NewTestGroup("group-1", []*jobsmanagerv1beta1.ManagedJobDefinition{
|
||||
NewTestJobDef("job-1", "busybox"),
|
||||
}),
|
||||
})
|
||||
controllerutil.AddFinalizer(mj, FinalizerName)
|
||||
return mj
|
||||
},
|
||||
validate: func(t *testing.T, c *MockClient, r ctrl.Result, err error) {
|
||||
assert.NoError(t, err)
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
client := NewMockClient()
|
||||
reconciler := &ManagedJobReconciler{
|
||||
Client: client,
|
||||
Scheme: client.Scheme(),
|
||||
Recorder: NewFakeRecorder(),
|
||||
}
|
||||
|
||||
mj := tt.setup(client)
|
||||
client.AddManagedJob(mj)
|
||||
|
||||
req := reconcile.Request{
|
||||
NamespacedName: types.NamespacedName{Name: mj.Name, Namespace: mj.Namespace},
|
||||
}
|
||||
|
||||
result, err := reconciler.Reconcile(context.Background(), req)
|
||||
tt.validate(t, client, result, err)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// ==================== KUBERNETES VOLATILITY TESTS ====================
|
||||
|
||||
func TestReconcile_KubernetesVolatility(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
description string
|
||||
clientSetup func(*MockClient)
|
||||
expectError bool
|
||||
}{
|
||||
{
|
||||
name: "intermittent_api_failure",
|
||||
description: "API fails on first call but succeeds on retry",
|
||||
clientSetup: func(c *MockClient) {
|
||||
c.FailOnNthCall["get"] = 1
|
||||
},
|
||||
expectError: true,
|
||||
},
|
||||
{
|
||||
name: "list_timeout",
|
||||
description: "List operation times out",
|
||||
clientSetup: func(c *MockClient) {
|
||||
c.ListError = ErrTimeout
|
||||
},
|
||||
expectError: false, // List error is handled gracefully
|
||||
},
|
||||
{
|
||||
name: "create_conflict",
|
||||
description: "Job already exists when creating",
|
||||
clientSetup: func(c *MockClient) {
|
||||
c.CreateError = errors.New("already exists")
|
||||
},
|
||||
expectError: false, // Already exists is handled
|
||||
},
|
||||
{
|
||||
name: "update_resource_version_conflict",
|
||||
description: "Optimistic locking conflict on update",
|
||||
clientSetup: func(c *MockClient) {
|
||||
c.SimulateConflictOnUpdate = true
|
||||
},
|
||||
expectError: false, // First update should succeed
|
||||
},
|
||||
{
|
||||
name: "forbidden_permission",
|
||||
description: "Insufficient RBAC permissions",
|
||||
clientSetup: func(c *MockClient) {
|
||||
c.CreateError = ErrForbidden
|
||||
},
|
||||
expectError: false, // Logged but doesn't fail reconciliation
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
client := NewMockClient()
|
||||
tt.clientSetup(client)
|
||||
|
||||
reconciler := &ManagedJobReconciler{
|
||||
Client: client,
|
||||
Scheme: client.Scheme(),
|
||||
Recorder: NewFakeRecorder(),
|
||||
}
|
||||
|
||||
mj := NewTestManagedJob("workflow", "default", []*jobsmanagerv1beta1.ManagedJobGroup{
|
||||
NewTestGroup("group1", []*jobsmanagerv1beta1.ManagedJobDefinition{
|
||||
NewTestJobDef("job1", "busybox"),
|
||||
}),
|
||||
})
|
||||
controllerutil.AddFinalizer(mj, FinalizerName)
|
||||
client.AddManagedJob(mj)
|
||||
|
||||
req := reconcile.Request{
|
||||
NamespacedName: types.NamespacedName{Name: "workflow", Namespace: "default"},
|
||||
}
|
||||
|
||||
_, err := reconciler.Reconcile(context.Background(), req)
|
||||
|
||||
if tt.expectError {
|
||||
require.Error(t, err, tt.description)
|
||||
} else {
|
||||
require.NoError(t, err, tt.description)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user