lukaszraczylo/kubemirror
1
0
Fork 0
mirror of https://github.com/lukaszraczylo/kubemirror.git synced 2026-06-10 23:09:14 +00:00
Code Issues Packages Projects Releases Wiki Activity

improvements jan2025 (#6)

Browse Source 
* feat(controller): add lazy watcher, improve resource usage and add pattern validation

- [x] Add cache sync health check for readiness probe verification
- [x] Create namespace lister with API reader support for fresh label queries
- [x] Add pattern validation with warning logs for invalid glob patterns
- [x] Implement lazy watcher initialization mode to scan for active resources
- [x] Add requeue delay to namespace reconciler for cache settlement
- [x] Replace custom containsString with slices.Contains from stdlib
- [x] Add structured logging context to reconcilers (kind, group, version)
- [x] Improve error variable naming for clarity in nested conditions
- [x] Add nil-safe label access in namespace reconciler setup
- [x] Add APIReader to namespace and source reconcilers for fresh data
- [x] Improve type assertions with proper error handling in mirror operations
- [x] Reorder struct fields for consistency and readability
- [x] Add comprehensive pattern validation tests and validation API

* feat(controller): add lazy watcher, improve resource usage and add pattern validation

- [x] Add circuit breaker for reconciliation failure tracking and prevention
- [x] Implement granular registration state tracking (not-registered, source-only, fully-registered)
- [x] Add lazy controller initialization for active resource types only
- [x] Consolidate namespace listing into single API call for efficiency
- [x] Add mirror creation verification to catch webhook rejections
- [x] Implement high-cardinality resource detection and warnings
- [x] Add source deletion check in mirror reconciler to prevent races
- [x] Preserve transformation annotations on errors in mirror reconciliation
- [x] Expand constants documentation with labels vs annotations design rationale
- [x] Add comprehensive test coverage for circuit breaker and registration states
- [x] Add mutation-safety tests for hash computation

* fixup! feat(controller): add lazy watcher, improve resource usage and add pattern validation
This commit is contained in:
Lukasz Raczylo
2026-01-14 13:07:11 +00:00
committed by GitHub
parent 4f8e2783cf
commit 096dca47d1
22 changed files with 1937 additions and 266 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/controller/dynamic_manager.go
+225 -49
View File
@@ -18,6 +18,32 @@ import (
"github.com/lukaszraczylo/kubemirror/pkg/filter"
)
// RegistrationState tracks the granular state of controller registration
type RegistrationState int
const (
// StateNotRegistered means no controllers are registered for this GVK
StateNotRegistered RegistrationState = iota
// StateSourceOnly means only the source controller is registered (partial failure)
StateSourceOnly
// StateFullyRegistered means both source and mirror controllers are registered
StateFullyRegistered
)
// String returns a human-readable representation of the registration state
func (rs RegistrationState) String() string {
switch rs {
case StateNotRegistered:
return "not-registered"
case StateSourceOnly:
return "source-only"
case StateFullyRegistered:
return "fully-registered"
default:
return "unknown"
}
}
// DynamicControllerManager manages lazy initialization of controllers
// for resource types that actually have resources marked for mirroring.
//
@@ -30,22 +56,20 @@ import (
// 3. Dynamically registers controllers only for resource types in use
// 4. Optionally unregisters controllers for resource types no longer in use
type DynamicControllerManager struct {
client client.Client
mgr ctrl.Manager
config *config.Config
filter *filter.NamespaceFilter
namespaceLister NamespaceLister
scanInterval time.Duration
// Tracking state
mu sync.RWMutex
registeredControllers map[string]bool // GVK string -> registered
activeResourceTypes map[string]schema.GroupVersionKind
availableResourceTypes []config.ResourceType
// Reconciler factories
client client.Client
apiReader client.Reader // Direct API reader (bypasses cache)
mgr ctrl.Manager
namespaceLister NamespaceLister
config *config.Config
filter *filter.NamespaceFilter
registrationState map[string]RegistrationState // Granular registration state tracking
activeResourceTypes map[string]schema.GroupVersionKind
sourceReconcilerFactory SourceReconcilerFactory
mirrorReconcilerFactory MirrorReconcilerFactory
availableResourceTypes []config.ResourceType
scanInterval time.Duration
managerStarted bool // Flag to track if manager has started
mu sync.RWMutex
}
// SourceReconcilerFactory creates source reconcilers for a given GVK
@@ -57,14 +81,15 @@ type MirrorReconcilerFactory func(gvk schema.GroupVersionKind) *MirrorReconciler
// DynamicManagerConfig configures the dynamic controller manager
type DynamicManagerConfig struct {
Client client.Client
APIReader client.Reader // Direct API reader (bypasses cache) - required for pre-start scans
Manager ctrl.Manager
NamespaceLister NamespaceLister
Config *config.Config
Filter *filter.NamespaceFilter
NamespaceLister NamespaceLister
AvailableResources []config.ResourceType
ScanInterval time.Duration // How often to scan for new resources (default: 5m)
SourceReconcilerFactory SourceReconcilerFactory
MirrorReconcilerFactory MirrorReconcilerFactory
AvailableResources []config.ResourceType
ScanInterval time.Duration
}
// NewDynamicControllerManager creates a new dynamic controller manager
@@ -75,38 +100,56 @@ func NewDynamicControllerManager(cfg DynamicManagerConfig) *DynamicControllerMan
return &DynamicControllerManager{
client: cfg.Client,
apiReader: cfg.APIReader,
mgr: cfg.Manager,
config: cfg.Config,
filter: cfg.Filter,
namespaceLister: cfg.NamespaceLister,
scanInterval: cfg.ScanInterval,
registeredControllers: make(map[string]bool),
registrationState: make(map[string]RegistrationState),
activeResourceTypes: make(map[string]schema.GroupVersionKind),
managerStarted: false,
availableResourceTypes: cfg.AvailableResources,
sourceReconcilerFactory: cfg.SourceReconcilerFactory,
mirrorReconcilerFactory: cfg.MirrorReconcilerFactory,
}
}
// Start begins the dynamic controller management loop
// Start begins the dynamic controller management loop.
// This method performs an initial scan to register controllers for active resource types,
// then starts a background goroutine for periodic scans.
// IMPORTANT: This should be called BEFORE mgr.Start() to ensure controllers are registered
// before the manager starts. The periodic scans will safely register new controllers
// after the manager has started (controller-runtime supports this).
func (d *DynamicControllerManager) Start(ctx context.Context) error {
logger := log.FromContext(ctx).WithName("dynamic-controller-manager")
// Initial scan and registration
// Initial scan and registration (before main manager starts)
logger.Info("performing initial scan for active resource types")
if err := d.scanAndRegister(ctx); err != nil {
return fmt.Errorf("initial scan failed: %w", err)
}
// Start periodic scanning
// Start periodic scanning (will run after main manager starts)
go d.run(ctx)
logger.Info("dynamic controller manager started",
"scanInterval", d.scanInterval,
"initialControllersRegistered", d.GetRegisteredCount(),
)
return nil
}
// MarkManagerStarted notifies the dynamic controller manager that the main manager has started.
// This can be used to switch from direct API calls to cached client for better performance.
// Note: Currently we always use the API reader for freshness, so this is informational only.
func (d *DynamicControllerManager) MarkManagerStarted() {
d.mu.Lock()
defer d.mu.Unlock()
d.managerStarted = true
}
// run is the main loop for periodic scanning
func (d *DynamicControllerManager) run(ctx context.Context) {
logger := log.FromContext(ctx).WithName("dynamic-controller-manager")
@@ -140,50 +183,111 @@ func (d *DynamicControllerManager) scanAndRegister(ctx context.Context) error {
defer d.mu.Unlock()
// Track changes
var newlyRegistered, alreadyRegistered int
var newlyRegistered, alreadyRegistered, partialRetried int
// Register controllers for active resource types
for gvkStr, gvk := range activeTypes {
if d.registeredControllers[gvkStr] {
state := d.registrationState[gvkStr]
switch state {
case StateFullyRegistered:
// Already fully registered, nothing to do
alreadyRegistered++
continue
}
// Register new controller
if err := d.registerController(ctx, gvk); err != nil {
logger.Error(err, "failed to register controller",
"gvk", gvkStr,
case StateSourceOnly:
// Partial registration - retry mirror controller only
partialRetried++
if err := d.registerMirrorControllerOnly(ctx, gvk); err != nil {
logger.Error(err, "failed to complete partial registration (mirror controller)",
"gvk", gvkStr,
"currentState", state.String(),
)
continue
}
d.registrationState[gvkStr] = StateFullyRegistered
logger.Info("completed partial registration",
"group", gvk.Group,
"version", gvk.Version,
"kind", gvk.Kind,
)
case StateNotRegistered:
// New registration - register both controllers
newState, err := d.registerController(ctx, gvk)
if err != nil {
logger.Error(err, "failed to register controller",
"gvk", gvkStr,
"achievedState", newState.String(),
)
// Save partial state if source was registered
if newState == StateSourceOnly {
d.registrationState[gvkStr] = newState
d.activeResourceTypes[gvkStr] = gvk
logger.Info("partial registration - source controller only",
"group", gvk.Group,
"version", gvk.Version,
"kind", gvk.Kind,
)
}
continue
}
d.registrationState[gvkStr] = StateFullyRegistered
d.activeResourceTypes[gvkStr] = gvk
newlyRegistered++
logger.Info("registered controller for active resource type",
"group", gvk.Group,
"version", gvk.Version,
"kind", gvk.Kind,
)
continue
}
}
d.registeredControllers[gvkStr] = true
d.activeResourceTypes[gvkStr] = gvk
newlyRegistered++
logger.Info("registered controller for active resource type",
"group", gvk.Group,
"version", gvk.Version,
"kind", gvk.Kind,
)
// Count fully registered controllers
fullyRegistered := 0
for _, state := range d.registrationState {
if state == StateFullyRegistered {
fullyRegistered++
}
}
logger.Info("scan completed",
"activeResourceTypes", len(activeTypes),
"alreadyRegistered", alreadyRegistered,
"newlyRegistered", newlyRegistered,
"totalRegistered", len(d.registeredControllers),
"partialRetried", partialRetried,
"fullyRegistered", fullyRegistered,
)
return nil
}
// getReader returns the appropriate reader based on whether the manager has started.
// Before manager starts, we must use the API reader (direct API calls).
// After manager starts, we can use the cached client for better performance.
func (d *DynamicControllerManager) getReader() client.Reader {
d.mu.RLock()
defer d.mu.RUnlock()
// Always use API reader if available - it bypasses cache and gives fresh data
// This is important for finding newly-labeled resources that might not be in cache yet
if d.apiReader != nil {
return d.apiReader
}
return d.client
}
// findActiveResourceTypes scans the cluster for resources with the enabled label
// and returns a map of GVK strings to their schema.GroupVersionKind
func (d *DynamicControllerManager) findActiveResourceTypes(ctx context.Context) (map[string]schema.GroupVersionKind, error) {
logger := log.FromContext(ctx).WithName("dynamic-controller-manager")
activeTypes := make(map[string]schema.GroupVersionKind)
reader := d.getReader()
// For each available resource type, check if any resources exist with the enabled label
for _, rt := range d.availableResourceTypes {
gvk := rt.GroupVersionKind()
@@ -204,7 +308,7 @@ func (d *DynamicControllerManager) findActiveResourceTypes(ctx context.Context)
},
}
if err := d.client.List(ctx, list, opts...); err != nil {
if err := reader.List(ctx, list, opts...); err != nil {
// Ignore errors for resource types that don't exist or we can't access
logger.V(2).Info("failed to list resources (ignoring)",
"gvk", gvkStr,
@@ -226,8 +330,10 @@ func (d *DynamicControllerManager) findActiveResourceTypes(ctx context.Context)
return activeTypes, nil
}
// registerController registers source and mirror controllers for a GVK
func (d *DynamicControllerManager) registerController(ctx context.Context, gvk schema.GroupVersionKind) error {
// registerController registers source and mirror controllers for a GVK.
// Returns the achieved registration state and any error.
// If source registration succeeds but mirror fails, returns StateSourceOnly to allow retry.
func (d *DynamicControllerManager) registerController(ctx context.Context, gvk schema.GroupVersionKind) (RegistrationState, error) {
logger := log.FromContext(ctx).WithName("dynamic-controller-manager")
// Create source reconciler using factory
@@ -235,9 +341,37 @@ func (d *DynamicControllerManager) registerController(ctx context.Context, gvk s
// Register source controller
if err := sourceReconciler.SetupWithManagerForResourceType(d.mgr, gvk); err != nil {
return fmt.Errorf("failed to register source controller: %w", err)
return StateNotRegistered, fmt.Errorf("failed to register source controller: %w", err)
}
// Source registered successfully, now try mirror
logger.V(1).Info("source controller registered",
"group", gvk.Group,
"version", gvk.Version,
"kind", gvk.Kind,
)
// Create mirror reconciler using factory
mirrorReconciler := d.mirrorReconcilerFactory(gvk)
// Register mirror controller
if err := mirrorReconciler.SetupWithManager(d.mgr, gvk); err != nil {
// Source is registered but mirror failed - return partial state
return StateSourceOnly, fmt.Errorf("source registered but mirror failed: %w", err)
}
logger.Info("registered both controllers",
"group", gvk.Group,
"version", gvk.Version,
"kind", gvk.Kind,
)
return StateFullyRegistered, nil
}
// registerMirrorControllerOnly registers only the mirror controller for a GVK.
// Used to complete partial registrations where source was registered but mirror failed.
func (d *DynamicControllerManager) registerMirrorControllerOnly(ctx context.Context, gvk schema.GroupVersionKind) error {
// Create mirror reconciler using factory
mirrorReconciler := d.mirrorReconcilerFactory(gvk)
@@ -246,11 +380,53 @@ func (d *DynamicControllerManager) registerController(ctx context.Context, gvk s
return fmt.Errorf("failed to register mirror controller: %w", err)
}
logger.Info("registered controllers",
"group", gvk.Group,
"version", gvk.Version,
"kind", gvk.Kind,
)
return nil
}
// GetRegisteredCount returns the number of fully registered controllers
func (d *DynamicControllerManager) GetRegisteredCount() int {
d.mu.RLock()
defer d.mu.RUnlock()
count := 0
for _, state := range d.registrationState {
if state == StateFullyRegistered {
count++
}
}
return count
}
// GetRegistrationState returns the registration state for a specific GVK
func (d *DynamicControllerManager) GetRegistrationState(gvkStr string) RegistrationState {
d.mu.RLock()
defer d.mu.RUnlock()
return d.registrationState[gvkStr]
}
// GetRegistrationStats returns counts of controllers in each state
func (d *DynamicControllerManager) GetRegistrationStats() (fullyRegistered, sourceOnly, notRegistered int) {
d.mu.RLock()
defer d.mu.RUnlock()
for _, state := range d.registrationState {
switch state {
case StateFullyRegistered:
fullyRegistered++
case StateSourceOnly:
sourceOnly++
default:
notRegistered++
}
}
return
}
// GetActiveResourceTypes returns a copy of the active resource types map
func (d *DynamicControllerManager) GetActiveResourceTypes() map[string]schema.GroupVersionKind {
d.mu.RLock()
defer d.mu.RUnlock()
result := make(map[string]schema.GroupVersionKind, len(d.activeResourceTypes))
for k, v := range d.activeResourceTypes {
result[k] = v
}
return result
}
pkg/controller/dynamic_manager_test.go
+111 -16
View File
@@ -21,11 +21,17 @@ import (
// These are intentionally not exported methods on DynamicControllerManager
// to avoid exposing them in production code
// getRegisteredCount returns the number of currently registered controllers (test helper)
// getRegisteredCount returns the number of fully registered controllers (test helper)
func getRegisteredCount(d *DynamicControllerManager) int {
d.mu.RLock()
defer d.mu.RUnlock()
return len(d.registeredControllers)
count := 0
for _, state := range d.registrationState {
if state == StateFullyRegistered {
count++
}
}
return count
}
// getActiveResourceTypes returns the currently active resource types (test helper)
@@ -49,8 +55,8 @@ func TestDynamicControllerManager_FindActiveResourceTypes(t *testing.T) {
name string
availableResources []config.ResourceType
existingResources []*unstructured.Unstructured
expectedActiveCount int
expectedActiveTypes []string
expectedActiveCount int
}{
{
name: "no resources marked for mirroring",
@@ -242,9 +248,9 @@ func TestDynamicControllerManager_FindActiveResourceTypes(t *testing.T) {
func TestDynamicControllerManager_GetRegisteredCount(t *testing.T) {
mgr := &DynamicControllerManager{
registeredControllers: map[string]bool{
"Secret.v1.": true,
"ConfigMap.v1.": true,
registrationState: map[string]RegistrationState{
"Secret.v1.": StateFullyRegistered,
"ConfigMap.v1.": StateFullyRegistered,
},
}
@@ -252,6 +258,19 @@ func TestDynamicControllerManager_GetRegisteredCount(t *testing.T) {
assert.Equal(t, 2, count)
}
func TestDynamicControllerManager_GetRegisteredCount_PartialStates(t *testing.T) {
mgr := &DynamicControllerManager{
registrationState: map[string]RegistrationState{
"Secret.v1.": StateFullyRegistered,
"ConfigMap.v1.": StateSourceOnly, // Partial - shouldn't count
"Deployment.v1.": StateNotRegistered, // Not registered - shouldn't count
},
}
count := getRegisteredCount(mgr)
assert.Equal(t, 1, count, "only fully registered controllers should be counted")
}
func TestDynamicControllerManager_GetActiveResourceTypes(t *testing.T) {
secretGVK := schema.GroupVersionKind{Group: "", Version: "v1", Kind: "Secret"}
configMapGVK := schema.GroupVersionKind{Group: "", Version: "v1", Kind: "ConfigMap"}
@@ -319,22 +338,22 @@ func TestDynamicControllerManager_ScanInterval(t *testing.T) {
func TestDynamicControllerManager_RegistrationTracking(t *testing.T) {
// Test that registration tracking works correctly
mgr := &DynamicControllerManager{
registeredControllers: make(map[string]bool),
activeResourceTypes: make(map[string]schema.GroupVersionKind),
registrationState: make(map[string]RegistrationState),
activeResourceTypes: make(map[string]schema.GroupVersionKind),
}
gvk := schema.GroupVersionKind{Group: "", Version: "v1", Kind: "Secret"}
gvkStr := "Secret.v1."
// Initially not registered
assert.False(t, mgr.registeredControllers[gvkStr])
assert.Equal(t, StateNotRegistered, mgr.registrationState[gvkStr])
assert.Equal(t, 0, getRegisteredCount(mgr))
// Mark as registered
mgr.registeredControllers[gvkStr] = true
// Mark as fully registered
mgr.registrationState[gvkStr] = StateFullyRegistered
mgr.activeResourceTypes[gvkStr] = gvk
assert.True(t, mgr.registeredControllers[gvkStr])
assert.Equal(t, StateFullyRegistered, mgr.registrationState[gvkStr])
assert.Equal(t, 1, getRegisteredCount(mgr))
activeTypes := getActiveResourceTypes(mgr)
@@ -342,11 +361,87 @@ func TestDynamicControllerManager_RegistrationTracking(t *testing.T) {
assert.Equal(t, gvk, activeTypes[0])
}
func TestDynamicControllerManager_PartialRegistration(t *testing.T) {
// Test that partial registration (source only) is tracked correctly
mgr := &DynamicControllerManager{
registrationState: make(map[string]RegistrationState),
activeResourceTypes: make(map[string]schema.GroupVersionKind),
}
gvk := schema.GroupVersionKind{Group: "", Version: "v1", Kind: "Secret"}
gvkStr := "Secret.v1."
// Mark as partially registered (source only)
mgr.registrationState[gvkStr] = StateSourceOnly
mgr.activeResourceTypes[gvkStr] = gvk
// Should not count as registered
assert.Equal(t, StateSourceOnly, mgr.registrationState[gvkStr])
assert.Equal(t, 0, getRegisteredCount(mgr), "partial registration should not count as fully registered")
// But should be in active resource types
activeTypes := getActiveResourceTypes(mgr)
assert.Equal(t, 1, len(activeTypes))
// Complete the registration
mgr.registrationState[gvkStr] = StateFullyRegistered
assert.Equal(t, 1, getRegisteredCount(mgr), "should now count as fully registered")
}
func TestDynamicControllerManager_GetRegistrationStats(t *testing.T) {
mgr := &DynamicControllerManager{
registrationState: map[string]RegistrationState{
"Secret.v1.": StateFullyRegistered,
"ConfigMap.v1.": StateFullyRegistered,
"Deployment.v1.": StateSourceOnly,
"Service.v1.": StateSourceOnly,
"Ingress.v1.": StateNotRegistered,
},
}
fullyReg, sourceOnly, notReg := mgr.GetRegistrationStats()
assert.Equal(t, 2, fullyReg, "should have 2 fully registered")
assert.Equal(t, 2, sourceOnly, "should have 2 source-only")
assert.Equal(t, 1, notReg, "should have 1 not registered")
}
func TestDynamicControllerManager_GetRegistrationState(t *testing.T) {
mgr := &DynamicControllerManager{
registrationState: map[string]RegistrationState{
"Secret.v1.": StateFullyRegistered,
"ConfigMap.v1.": StateSourceOnly,
},
}
assert.Equal(t, StateFullyRegistered, mgr.GetRegistrationState("Secret.v1."))
assert.Equal(t, StateSourceOnly, mgr.GetRegistrationState("ConfigMap.v1."))
assert.Equal(t, StateNotRegistered, mgr.GetRegistrationState("Unknown.v1."), "unknown GVK should be not registered")
}
func TestRegistrationState_String(t *testing.T) {
tests := []struct {
expected string
state RegistrationState
}{
{"not-registered", StateNotRegistered},
{"source-only", StateSourceOnly},
{"fully-registered", StateFullyRegistered},
{"unknown", RegistrationState(99)},
}
for _, tt := range tests {
t.Run(tt.expected, func(t *testing.T) {
assert.Equal(t, tt.expected, tt.state.String())
})
}
}
// TestDynamicControllerManager_ConcurrentAccess tests thread-safety
func TestDynamicControllerManager_ConcurrentAccess(t *testing.T) {
mgr := &DynamicControllerManager{
registeredControllers: make(map[string]bool),
activeResourceTypes: make(map[string]schema.GroupVersionKind),
registrationState: make(map[string]RegistrationState),
activeResourceTypes: make(map[string]schema.GroupVersionKind),
}
// Simulate concurrent reads and writes
@@ -356,7 +451,7 @@ func TestDynamicControllerManager_ConcurrentAccess(t *testing.T) {
go func() {
for i := 0; i < 100; i++ {
mgr.mu.Lock()
mgr.registeredControllers["test"] = true
mgr.registrationState["test"] = StateFullyRegistered
mgr.mu.Unlock()
time.Sleep(1 * time.Millisecond)
}
@@ -381,7 +476,7 @@ func TestDynamicControllerManager_ConcurrentAccess(t *testing.T) {
}
// Should not panic and should have final state
assert.True(t, mgr.registeredControllers["test"])
assert.Equal(t, StateFullyRegistered, mgr.registrationState["test"])
}
func TestDynamicControllerManager_UnstructuredResourceHandling(t *testing.T) {
pkg/controller/mirror.go
+53 -14
View File
@@ -160,8 +160,17 @@ func createUnstructuredMirror(source runtime.Object, targetNamespace, sourceHash
}
// buildMirrorAnnotations builds the ownership annotations for a mirror resource.
// Returns empty map if source doesn't implement metav1.Object.
func buildMirrorAnnotations(source runtime.Object, sourceHash string) map[string]string {
sourceObj, _ := source.(metav1.Object)
sourceObj, ok := source.(metav1.Object)
if !ok {
// This should never happen for valid Kubernetes resources.
// Return minimal annotations with just the hash.
return map[string]string{
constants.AnnotationSourceContentHash: sourceHash,
constants.AnnotationLastSyncTime: time.Now().UTC().Format(time.RFC3339),
}
}
annotations := map[string]string{
constants.AnnotationSourceNamespace: sourceObj.GetNamespace(),
@@ -196,24 +205,34 @@ func UpdateMirror(mirror, source runtime.Object) error {
// Update based on type
switch m := mirror.(type) {
case *corev1.Secret:
src := source.(*corev1.Secret)
src, ok := source.(*corev1.Secret)
if !ok {
return fmt.Errorf("mirror is Secret but source is %T", source)
}
m.Data = src.Data
m.Type = src.Type
updateMirrorAnnotations(m, source, sourceHash)
case *corev1.ConfigMap:
src := source.(*corev1.ConfigMap)
src, ok := source.(*corev1.ConfigMap)
if !ok {
return fmt.Errorf("mirror is ConfigMap but source is %T", source)
}
m.Data = src.Data
m.BinaryData = src.BinaryData
updateMirrorAnnotations(m, source, sourceHash)
default:
// Unstructured
if err := updateUnstructuredMirror(mirror, source, sourceHash); err != nil {
err = updateUnstructuredMirror(mirror, source, sourceHash)
if err != nil {
return err
}
}
// Apply transformations after updating data (only if transformation rules exist)
mirrorObj, _ := mirror.(metav1.Object)
mirrorObj, ok := mirror.(metav1.Object)
if !ok {
return fmt.Errorf("mirror does not implement metav1.Object, got %T", mirror)
}
targetNamespace := mirrorObj.GetNamespace()
transformed, err := applyTransformations(source, mirror, targetNamespace)
if err != nil {
@@ -280,8 +299,6 @@ func convertToByteMap(data map[string]interface{}) map[string][]byte {
// updateMirrorAnnotations updates the ownership annotations on a mirror.
func updateMirrorAnnotations(mirror metav1.Object, source runtime.Object, sourceHash string) {
sourceObj, _ := source.(metav1.Object)
annotations := mirror.GetAnnotations()
if annotations == nil {
annotations = make(map[string]string)
@@ -290,12 +307,16 @@ func updateMirrorAnnotations(mirror metav1.Object, source runtime.Object, source
annotations[constants.AnnotationSourceContentHash] = sourceHash
annotations[constants.AnnotationLastSyncTime] = time.Now().UTC().Format(time.RFC3339)
if sourceObj.GetGeneration() > 0 {
annotations[constants.AnnotationSourceGeneration] = fmt.Sprintf("%d", sourceObj.GetGeneration())
}
// Safely extract source metadata if available
sourceObj, ok := source.(metav1.Object)
if ok {
if sourceObj.GetGeneration() > 0 {
annotations[constants.AnnotationSourceGeneration] = fmt.Sprintf("%d", sourceObj.GetGeneration())
}
if sourceObj.GetResourceVersion() != "" {
annotations[constants.AnnotationSourceResourceVersion] = sourceObj.GetResourceVersion()
if sourceObj.GetResourceVersion() != "" {
annotations[constants.AnnotationSourceResourceVersion] = sourceObj.GetResourceVersion()
}
}
mirror.SetAnnotations(annotations)
@@ -304,8 +325,14 @@ func updateMirrorAnnotations(mirror metav1.Object, source runtime.Object, source
// updateUnstructuredMirror updates an unstructured mirror.
// Uses generic field introspection to handle any resource type (Secrets, ConfigMaps, CRDs).
func updateUnstructuredMirror(mirror, source runtime.Object, sourceHash string) error {
m := mirror.(*unstructured.Unstructured)
s := source.(*unstructured.Unstructured)
m, ok := mirror.(*unstructured.Unstructured)
if !ok {
return fmt.Errorf("mirror is not *unstructured.Unstructured, got %T", mirror)
}
s, ok := source.(*unstructured.Unstructured)
if !ok {
return fmt.Errorf("source is not *unstructured.Unstructured, got %T", source)
}
// Fields to skip (Kubernetes-managed fields, not user content)
// These are managed by Kubernetes API server or controllers
@@ -416,6 +443,16 @@ func applyTransformations(source, mirror runtime.Object, targetNamespace string)
return mirror, nil
}
// Save original annotations to restore on failure
originalAnnotations := mirrorObj.GetAnnotations()
var savedAnnotations map[string]string
if originalAnnotations != nil {
savedAnnotations = make(map[string]string, len(originalAnnotations))
for k, v := range originalAnnotations {
savedAnnotations[k] = v
}
}
mirrorAnnotations := mirrorObj.GetAnnotations()
if mirrorAnnotations == nil {
mirrorAnnotations = make(map[string]string)
@@ -437,6 +474,8 @@ func applyTransformations(source, mirror runtime.Object, targetNamespace string)
// Apply transformations (transformer reads rules from mirror's annotations now)
transformed, err := t.Transform(mirror, ctx)
if err != nil {
// Restore original annotations on failure to avoid leaving mirror in inconsistent state
mirrorObj.SetAnnotations(savedAnnotations)
return nil, err
}
pkg/controller/mirror_reconciler.go
+21 -4
View File
@@ -24,7 +24,13 @@ type MirrorReconciler struct {
// Reconcile checks if a mirrored resource's source still exists, and deletes the mirror if orphaned.
func (r *MirrorReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
logger := log.FromContext(ctx)
logger := log.FromContext(ctx).WithValues(
"mirrorNamespace", req.Namespace,
"mirrorName", req.Name,
"kind", r.GVK.Kind,
"group", r.GVK.Group,
"version", r.GVK.Version,
)
// Fetch the mirror resource
mirror := &unstructured.Unstructured{}
@@ -73,9 +79,10 @@ func (r *MirrorReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctr
"sourceName", sourceName,
"sourceUID", sourceUID)
if err := r.Delete(ctx, mirror); err != nil {
logger.Error(err, "failed to delete orphaned mirror")
return ctrl.Result{}, err
deleteErr := r.Delete(ctx, mirror)
if deleteErr != nil {
logger.Error(deleteErr, "failed to delete orphaned mirror")
return ctrl.Result{}, deleteErr
}
logger.Info("orphaned mirror deleted successfully",
@@ -91,6 +98,16 @@ func (r *MirrorReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctr
return ctrl.Result{}, err
}
// Check if source is being deleted - if so, let the SourceReconciler handle cleanup
// This prevents race conditions where both reconcilers try to delete mirrors
if !source.GetDeletionTimestamp().IsZero() {
logger.V(1).Info("source is being deleted, skipping mirror check (SourceReconciler will handle cleanup)",
"mirror", req.NamespacedName,
"sourceNamespace", sourceNs,
"sourceName", sourceName)
return ctrl.Result{}, nil
}
// Source exists - verify UID matches
actualUID := string(source.GetUID())
if actualUID != sourceUID {
pkg/controller/namespace_lister.go
+81 -4
View File
@@ -13,6 +13,10 @@ import (
// KubernetesNamespaceLister implements NamespaceLister using the Kubernetes API.
type KubernetesNamespaceLister struct {
client client.Client
// apiReader provides direct API access bypassing cache (optional).
// When set, it's used for label-based queries where cache staleness
// can cause missed namespaces after label changes.
apiReader client.Reader
}
// NewKubernetesNamespaceLister creates a new KubernetesNamespaceLister.
@@ -22,6 +26,25 @@ func NewKubernetesNamespaceLister(client client.Client) *KubernetesNamespaceList
}
}
// NewKubernetesNamespaceListerWithAPIReader creates a KubernetesNamespaceLister
// that uses direct API reads for label-based queries. This is more expensive
// but ensures fresh data for critical queries like allow-mirrors label lookups.
func NewKubernetesNamespaceListerWithAPIReader(c client.Client, apiReader client.Reader) *KubernetesNamespaceLister {
return &KubernetesNamespaceLister{
client: c,
apiReader: apiReader,
}
}
// getReader returns the appropriate reader to use.
// Returns apiReader if available (for fresh reads), otherwise falls back to cached client.
func (k *KubernetesNamespaceLister) getReader() client.Reader {
if k.apiReader != nil {
return k.apiReader
}
return k.client
}
// ListNamespaces returns all namespace names in the cluster.
func (k *KubernetesNamespaceLister) ListNamespaces(ctx context.Context) ([]string, error) {
namespaceList := &corev1.NamespaceList{}
@@ -38,11 +61,15 @@ func (k *KubernetesNamespaceLister) ListNamespaces(ctx context.Context) ([]strin
}
// ListAllowMirrorsNamespaces returns namespaces that have the allow-mirrors label.
// Uses direct API reads if apiReader is configured to avoid cache staleness issues.
func (k *KubernetesNamespaceLister) ListAllowMirrorsNamespaces(ctx context.Context) ([]string, error) {
namespaceList := &corev1.NamespaceList{}
// List namespaces with the allow-mirrors label
if err := k.client.List(ctx, namespaceList, client.MatchingLabels{
// Use direct API reader for label queries to ensure fresh data.
// This is critical because cache staleness can cause namespaces with
// newly added allow-mirrors labels to be missed.
reader := k.getReader()
if err := reader.List(ctx, namespaceList, client.MatchingLabels{
constants.LabelAllowMirrors: "true",
}); err != nil {
return nil, err
@@ -58,11 +85,13 @@ func (k *KubernetesNamespaceLister) ListAllowMirrorsNamespaces(ctx context.Conte
// ListOptOutNamespaces returns namespaces that have explicitly opted out of mirrors.
// These are namespaces with allow-mirrors="false".
// Uses direct API reads if apiReader is configured to avoid cache staleness issues.
func (k *KubernetesNamespaceLister) ListOptOutNamespaces(ctx context.Context) ([]string, error) {
namespaceList := &corev1.NamespaceList{}
// List namespaces with allow-mirrors label set to false
if err := k.client.List(ctx, namespaceList, client.MatchingLabels{
// Use direct API reader for label queries to ensure fresh data.
reader := k.getReader()
if err := reader.List(ctx, namespaceList, client.MatchingLabels{
constants.LabelAllowMirrors: "false",
}); err != nil {
return nil, err
@@ -75,3 +104,51 @@ func (k *KubernetesNamespaceLister) ListOptOutNamespaces(ctx context.Context) ([
return names, nil
}
// NamespaceInfo contains categorized namespace information from a single API call.
// This is more efficient than making 3 separate API calls.
type NamespaceInfo struct {
// All contains all namespace names in the cluster
All []string
// AllowMirrors contains namespaces with allow-mirrors="true" label
AllowMirrors []string
// OptOut contains namespaces with allow-mirrors="false" label
OptOut []string
}
// ListNamespacesWithLabels returns all namespaces categorized by their allow-mirrors label
// in a single API call. This is more efficient than calling ListNamespaces,
// ListAllowMirrorsNamespaces, and ListOptOutNamespaces separately.
// Uses direct API reads if apiReader is configured to ensure fresh data.
func (k *KubernetesNamespaceLister) ListNamespacesWithLabels(ctx context.Context) (*NamespaceInfo, error) {
namespaceList := &corev1.NamespaceList{}
// Use direct API reader if available for fresh data
reader := k.getReader()
if err := reader.List(ctx, namespaceList); err != nil {
return nil, err
}
info := &NamespaceInfo{
All: make([]string, 0, len(namespaceList.Items)),
AllowMirrors: make([]string, 0),
OptOut: make([]string, 0),
}
for _, ns := range namespaceList.Items {
info.All = append(info.All, ns.Name)
// Check allow-mirrors label value
if ns.Labels != nil {
labelValue := ns.Labels[constants.LabelAllowMirrors]
switch labelValue {
case "true":
info.AllowMirrors = append(info.AllowMirrors, ns.Name)
case "false":
info.OptOut = append(info.OptOut, ns.Name)
}
}
}
return info, nil
}
pkg/controller/namespace_reconciler.go
+69 -32
View File
@@ -4,6 +4,8 @@ package controller
import (
"context"
"fmt"
"slices"
"time"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/errors"
@@ -20,21 +22,31 @@ import (
"github.com/lukaszraczylo/kubemirror/pkg/filter"
)
const (
// cacheSettleDelay is the time to wait after namespace label changes
// to allow informer caches to sync. This addresses the race condition
// where namespace watch events fire before the cache is updated.
cacheSettleDelay = 3 * time.Second
)
// NamespaceReconciler watches for namespace CREATE and UPDATE events
// and triggers reconciliation of source resources that match the new namespace.
type NamespaceReconciler struct {
client.Client
NamespaceLister NamespaceLister
APIReader client.Reader
Scheme *runtime.Scheme
Config *config.Config
Filter *filter.NamespaceFilter
NamespaceLister NamespaceLister
// ResourceTypes contains all discovered resource types to reconcile
ResourceTypes []config.ResourceType
ResourceTypes []config.ResourceType
}
// Reconcile processes namespace events and creates mirrors for matching sources.
func (r *NamespaceReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
logger := log.FromContext(ctx).WithValues("namespace", req.Name)
logger := log.FromContext(ctx).WithValues(
"namespace", req.Name,
"reconciler", "namespace",
)
// Fetch the namespace
namespace := &corev1.Namespace{}
@@ -76,7 +88,11 @@ func (r *NamespaceReconciler) Reconcile(ctx context.Context, req ctrl.Request) (
return ctrl.Result{}, fmt.Errorf("failed to reconcile %d source resources", totalErrors)
}
return ctrl.Result{}, nil
// Requeue with delay to catch any updates missed due to cache staleness.
// This is particularly important for namespace label changes where the
// informer cache may not yet reflect the new label state. The delay allows
// the cache to settle and ensures all relevant source resources are reconciled.
return ctrl.Result{RequeueAfter: cacheSettleDelay}, nil
}
// reconcileResourceType finds and reconciles all sources of a specific resource type
@@ -125,13 +141,7 @@ func (r *NamespaceReconciler) reconcileResourceType(ctx context.Context, rt conf
}
// Check if the new namespace matches this source's targets
var isTarget bool
for _, target := range targetNamespaces {
if target == namespaceName {
isTarget = true
break
}
}
isTarget := slices.Contains(targetNamespaces, namespaceName)
if isTarget {
// Create or update mirror in the namespace
@@ -222,30 +232,47 @@ func (r *NamespaceReconciler) resolveTargetNamespaces(ctx context.Context, sourc
return nil, nil
}
// Get all namespaces
allNamespaces, err := r.NamespaceLister.ListNamespaces(ctx)
// Validate patterns and log warnings for invalid ones
validationResults, allValid := filter.ValidatePatterns(patterns)
if !allValid {
logger := log.FromContext(ctx)
invalidPatterns := filter.InvalidPatterns(validationResults)
for _, invalid := range invalidPatterns {
logger.Info("invalid glob pattern in target-namespaces annotation, pattern will be skipped",
"pattern", invalid.Pattern,
"error", invalid.Error.Error(),
"source", source.GetName(),
"namespace", source.GetNamespace(),
)
}
// Filter to only valid patterns
var validPatterns []string
for _, result := range validationResults {
if result.Valid {
validPatterns = append(validPatterns, result.Pattern)
}
}
patterns = validPatterns
// If no valid patterns remain, return empty
if len(patterns) == 0 {
return nil, nil
}
}
// Get all namespace info in a single API call (more efficient than 3 separate calls)
nsInfo, err := r.NamespaceLister.ListNamespacesWithLabels(ctx)
if err != nil {
return nil, fmt.Errorf("failed to list namespaces: %w", err)
}
// Get namespaces with allow-mirrors label
allowMirrorsNamespaces, err := r.NamespaceLister.ListAllowMirrorsNamespaces(ctx)
if err != nil {
return nil, fmt.Errorf("failed to list allow-mirrors namespaces: %w", err)
}
// Get namespaces that have explicitly opted out (allow-mirrors="false")
optOutNamespaces, err := r.NamespaceLister.ListOptOutNamespaces(ctx)
if err != nil {
return nil, fmt.Errorf("failed to list opt-out namespaces: %w", err)
}
// Resolve target namespaces
// Resolve target namespaces using the pre-categorized namespace info
targetNamespaces := filter.ResolveTargetNamespaces(
patterns,
allNamespaces,
allowMirrorsNamespaces,
optOutNamespaces,
nsInfo.All,
nsInfo.AllowMirrors,
nsInfo.OptOut,
source.GetNamespace(),
r.Filter,
)
@@ -292,8 +319,18 @@ func (r *NamespaceReconciler) SetupWithManager(mgr ctrl.Manager) error {
}
// Check if allow-mirrors label changed
oldLabel := oldNs.Labels[constants.LabelAllowMirrors]
newLabel := newNs.Labels[constants.LabelAllowMirrors]
// Use GetLabels() to safely handle nil labels map
oldLabels := oldNs.GetLabels()
newLabels := newNs.GetLabels()
// Get label values with nil-safe access
var oldLabel, newLabel string
if oldLabels != nil {
oldLabel = oldLabels[constants.LabelAllowMirrors]
}
if newLabels != nil {
newLabel = newLabels[constants.LabelAllowMirrors]
}
return oldLabel != newLabel
},
pkg/controller/namespace_reconciler_test.go
+23 -1
View File
@@ -282,9 +282,9 @@ func makeUnstructuredMirror(name, namespace, sourceNs, sourceName string) *unstr
// Mock namespace lister for testing
type mockNamespaceLister struct {
namespaces []string
allowMirrors map[string]bool
optOut map[string]bool
namespaces []string
}
func (m *mockNamespaceLister) ListNamespaces(ctx context.Context) ([]string, error) {
@@ -310,3 +310,25 @@ func (m *mockNamespaceLister) ListOptOutNamespaces(ctx context.Context) ([]strin
}
return result, nil
}
func (m *mockNamespaceLister) ListNamespacesWithLabels(ctx context.Context) (*NamespaceInfo, error) {
info := &NamespaceInfo{
All: m.namespaces,
AllowMirrors: make([]string, 0),
OptOut: make([]string, 0),
}
for ns, allowed := range m.allowMirrors {
if allowed {
info.AllowMirrors = append(info.AllowMirrors, ns)
}
}
for ns, optedOut := range m.optOut {
if optedOut {
info.OptOut = append(info.OptOut, ns)
}
}
return info, nil
}
pkg/controller/source_reconciler.go
+130 -59
View File
@@ -4,6 +4,8 @@ package controller
import (
"context"
"fmt"
"slices"
"time"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/errors"
@@ -21,6 +23,7 @@ import (
"sigs.k8s.io/controller-runtime/pkg/predicate"
"sigs.k8s.io/controller-runtime/pkg/reconcile"
"github.com/lukaszraczylo/kubemirror/pkg/circuitbreaker"
"github.com/lukaszraczylo/kubemirror/pkg/config"
"github.com/lukaszraczylo/kubemirror/pkg/constants"
"github.com/lukaszraczylo/kubemirror/pkg/filter"
@@ -30,12 +33,13 @@ import (
// SourceReconciler reconciles source resources that need mirroring.
type SourceReconciler struct {
client.Client
NamespaceLister NamespaceLister
APIReader client.Reader
Scheme *runtime.Scheme
Config *config.Config
Filter *filter.NamespaceFilter
NamespaceLister NamespaceLister
GVK schema.GroupVersionKind // The resource type this reconciler handles
APIReader client.Reader // Direct API reader (bypasses cache)
CircuitBreaker *circuitbreaker.CircuitBreaker
GVK schema.GroupVersionKind
}
// NamespaceLister provides a list of all namespaces in the cluster.
@@ -44,6 +48,8 @@ type NamespaceLister interface {
ListNamespaces(ctx context.Context) ([]string, error)
ListAllowMirrorsNamespaces(ctx context.Context) ([]string, error)
ListOptOutNamespaces(ctx context.Context) ([]string, error)
// ListNamespacesWithLabels returns all namespace info in a single API call (preferred)
ListNamespacesWithLabels(ctx context.Context) (*NamespaceInfo, error)
}
// +kubebuilder:rbac:groups=core,resources=secrets,verbs=get;list;watch;create;update;patch;delete
@@ -115,7 +121,13 @@ func (r *SourceReconciler) getSourceWithFreshness(ctx context.Context, key clien
// Reconcile processes a single source resource.
func (r *SourceReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
logger := log.FromContext(ctx).WithValues("namespace", req.Namespace, "name", req.Name)
logger := log.FromContext(ctx).WithValues(
"namespace", req.Namespace,
"name", req.Name,
"kind", r.GVK.Kind,
"group", r.GVK.Group,
"version", r.GVK.Version,
)
// Fetch the source resource with optional freshness verification
source, err := r.getSourceWithFreshness(ctx, req.NamespacedName, r.GVK)
@@ -136,24 +148,40 @@ func (r *SourceReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctr
return ctrl.Result{}, nil
}
// Check circuit breaker - skip if circuit is open (too many failures)
if r.CircuitBreaker != nil {
if !r.CircuitBreaker.AllowRequest(req.Namespace, req.Name, r.GVK.Kind) {
cbState := r.CircuitBreaker.GetState(req.Namespace, req.Name, r.GVK.Kind)
failCount := r.CircuitBreaker.GetFailureCount(req.Namespace, req.Name, r.GVK.Kind)
logger.Info("circuit breaker open, skipping reconciliation",
"state", cbState.String(),
"consecutiveFailures", failCount,
"lastError", r.CircuitBreaker.GetLastError(req.Namespace, req.Name, r.GVK.Kind))
// Requeue after circuit breaker reset timeout to try again
return ctrl.Result{RequeueAfter: 5 * time.Minute}, nil
}
}
// Check if resource is enabled for mirroring
// Check if resource is being deleted
if !sourceObj.GetDeletionTimestamp().IsZero() {
// Resource is being deleted - clean up mirrors and remove finalizer
if containsString(sourceObj.GetFinalizers(), constants.FinalizerName) {
if slices.Contains(sourceObj.GetFinalizers(), constants.FinalizerName) {
logger.Info("source being deleted, cleaning up all mirrors")
if err := r.deleteAllMirrors(ctx, sourceObj); err != nil {
logger.Error(err, "failed to delete all mirrors during source deletion")
return ctrl.Result{}, err
deleteErr := r.deleteAllMirrors(ctx, sourceObj)
if deleteErr != nil {
logger.Error(deleteErr, "failed to delete all mirrors during source deletion")
return ctrl.Result{}, deleteErr
}
// Remove finalizer to allow resource deletion
logger.Info("removing finalizer from source resource")
finalizers := removeString(sourceObj.GetFinalizers(), constants.FinalizerName)
sourceObj.SetFinalizers(finalizers)
if err := r.Update(ctx, source); err != nil {
logger.Error(err, "failed to remove finalizer")
return ctrl.Result{}, err
updateErr := r.Update(ctx, source)
if updateErr != nil {
logger.Error(updateErr, "failed to remove finalizer")
return ctrl.Result{}, updateErr
}
logger.Info("finalizer removed, resource can now be deleted")
}
@@ -162,7 +190,7 @@ func (r *SourceReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctr
if !isEnabledForMirroring(sourceObj) {
// Resource is disabled - remove finalizer if present and delete all mirrors
if containsString(sourceObj.GetFinalizers(), constants.FinalizerName) {
if slices.Contains(sourceObj.GetFinalizers(), constants.FinalizerName) {
return r.handleDisabled(ctx, sourceObj)
}
// No finalizer, just skip
@@ -170,13 +198,14 @@ func (r *SourceReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctr
}
// Add finalizer if not present
if !containsString(sourceObj.GetFinalizers(), constants.FinalizerName) {
if !slices.Contains(sourceObj.GetFinalizers(), constants.FinalizerName) {
logger.Info("adding finalizer to source resource")
finalizers := append(sourceObj.GetFinalizers(), constants.FinalizerName)
sourceObj.SetFinalizers(finalizers)
if err := r.Update(ctx, source); err != nil {
logger.Error(err, "failed to add finalizer")
return ctrl.Result{}, err
addFinalizerErr := r.Update(ctx, source)
if addFinalizerErr != nil {
logger.Error(addFinalizerErr, "failed to add finalizer")
return ctrl.Result{}, addFinalizerErr
}
logger.Info("finalizer added")
// Requeue to continue with reconciliation after finalizer is added
@@ -187,6 +216,9 @@ func (r *SourceReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctr
targetNamespaces, err := r.resolveTargetNamespaces(ctx, sourceObj)
if err != nil {
logger.Error(err, "failed to resolve target namespaces")
if r.CircuitBreaker != nil {
r.CircuitBreaker.RecordFailure(req.Namespace, req.Name, r.GVK.Kind, err)
}
return ctrl.Result{}, err
}
@@ -200,8 +232,9 @@ func (r *SourceReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctr
// Reconcile each target namespace
var reconciledCount, errorCount int
for _, targetNs := range targetNamespaces {
if err := r.reconcileMirror(ctx, source, sourceObj, targetNs); err != nil {
logger.Error(err, "failed to reconcile mirror", "targetNamespace", targetNs)
reconcileErr := r.reconcileMirror(ctx, source, sourceObj, targetNs)
if reconcileErr != nil {
logger.Error(reconcileErr, "failed to reconcile mirror", "targetNamespace", targetNs)
errorCount++
} else {
reconciledCount++
@@ -220,6 +253,9 @@ func (r *SourceReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctr
// Update status annotation with last sync info
if err := r.updateLastSyncStatus(ctx, source, sourceObj, reconciledCount, errorCount); err != nil {
logger.Error(err, "failed to update sync status")
if r.CircuitBreaker != nil {
r.CircuitBreaker.RecordFailure(req.Namespace, req.Name, r.GVK.Kind, err)
}
return ctrl.Result{}, err
}
@@ -230,7 +266,22 @@ func (r *SourceReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctr
// Return error if there were errors (controller-runtime will automatically requeue with exponential backoff)
if errorCount > 0 {
return ctrl.Result{}, fmt.Errorf("failed to reconcile %d/%d mirrors", errorCount, len(targetNamespaces))
err := fmt.Errorf("failed to reconcile %d/%d mirrors", errorCount, len(targetNamespaces))
// Record failure with circuit breaker
if r.CircuitBreaker != nil {
state, justOpened := r.CircuitBreaker.RecordFailure(req.Namespace, req.Name, r.GVK.Kind, err)
if justOpened {
logger.Info("circuit breaker opened due to repeated failures",
"state", state.String(),
"consecutiveFailures", r.CircuitBreaker.GetFailureCount(req.Namespace, req.Name, r.GVK.Kind))
}
}
return ctrl.Result{}, err
}
// Record success with circuit breaker
if r.CircuitBreaker != nil {
r.CircuitBreaker.RecordSuccess(req.Namespace, req.Name, r.GVK.Kind)
}
return ctrl.Result{}, nil
@@ -247,7 +298,7 @@ func (r *SourceReconciler) handleDisabled(ctx context.Context, sourceObj metav1.
}
// Remove finalizer if present
if containsString(sourceObj.GetFinalizers(), constants.FinalizerName) {
if slices.Contains(sourceObj.GetFinalizers(), constants.FinalizerName) {
logger.Info("removing finalizer from disabled resource")
finalizers := removeString(sourceObj.GetFinalizers(), constants.FinalizerName)
sourceObj.SetFinalizers(finalizers)
@@ -301,9 +352,9 @@ func (r *SourceReconciler) reconcileMirror(ctx context.Context, source runtime.O
}
// Check if update is needed
needsSync, err := hash.NeedsSync(source, existing, existing.GetAnnotations())
if err != nil {
return fmt.Errorf("failed to check if sync needed: %w", err)
needsSync, syncCheckErr := hash.NeedsSync(source, existing, existing.GetAnnotations())
if syncCheckErr != nil {
return fmt.Errorf("failed to check if sync needed: %w", syncCheckErr)
}
if !needsSync {
@@ -312,12 +363,14 @@ func (r *SourceReconciler) reconcileMirror(ctx context.Context, source runtime.O
}
// Update mirror
if err := UpdateMirror(existing, source); err != nil {
return fmt.Errorf("failed to update mirror: %w", err)
updateErr := UpdateMirror(existing, source)
if updateErr != nil {
return fmt.Errorf("failed to update mirror: %w", updateErr)
}
if err := r.Update(ctx, existing); err != nil {
return fmt.Errorf("failed to update mirror in cluster: %w", err)
clusterUpdateErr := r.Update(ctx, existing)
if clusterUpdateErr != nil {
return fmt.Errorf("failed to update mirror in cluster: %w", clusterUpdateErr)
}
logger.V(1).Info("mirror updated")
@@ -330,11 +383,21 @@ func (r *SourceReconciler) reconcileMirror(ctx context.Context, source runtime.O
return fmt.Errorf("failed to create mirror: %w", err)
}
if err := r.Create(ctx, mirror.(client.Object)); err != nil {
mirrorObj := mirror.(client.Object)
if err := r.Create(ctx, mirrorObj); err != nil {
return fmt.Errorf("failed to create mirror in cluster: %w", err)
}
logger.V(1).Info("mirror created")
// Verify mirror was actually created (catches webhook rejections, quota issues)
verifyMirror := &unstructured.Unstructured{}
verifyMirror.SetGroupVersionKind(sourceUnstructured.GroupVersionKind())
verifyKey := client.ObjectKey{Namespace: targetNs, Name: sourceObj.GetName()}
if verifyErr := r.Get(ctx, verifyKey, verifyMirror); verifyErr != nil {
logger.Error(verifyErr, "mirror creation verification failed - mirror may have been rejected")
return fmt.Errorf("mirror creation verification failed: %w", verifyErr)
}
logger.V(1).Info("mirror created and verified")
return nil
}
@@ -384,11 +447,12 @@ func (r *SourceReconciler) deleteAllMirrors(ctx context.Context, sourceObj metav
func (r *SourceReconciler) cleanupOrphanedMirrors(ctx context.Context, sourceObj metav1.Object, targetNamespaces []string) (int, error) {
logger := log.FromContext(ctx)
// List all namespaces
allNamespaces, err := r.NamespaceLister.ListNamespaces(ctx)
// List all namespaces using unified method for consistency
nsInfo, err := r.NamespaceLister.ListNamespacesWithLabels(ctx)
if err != nil {
return 0, fmt.Errorf("failed to list namespaces: %w", err)
}
allNamespaces := nsInfo.All
// Get GVK from source object
sourceUnstructured, ok := sourceObj.(*unstructured.Unstructured)
@@ -472,30 +536,47 @@ func (r *SourceReconciler) resolveTargetNamespaces(ctx context.Context, sourceOb
return nil, nil
}
// Get all namespaces
allNamespaces, err := r.NamespaceLister.ListNamespaces(ctx)
// Validate patterns and log warnings for invalid ones
validationResults, allValid := filter.ValidatePatterns(patterns)
if !allValid {
logger := log.FromContext(ctx)
invalidPatterns := filter.InvalidPatterns(validationResults)
for _, invalid := range invalidPatterns {
logger.Info("invalid glob pattern in target-namespaces annotation, pattern will be skipped",
"pattern", invalid.Pattern,
"error", invalid.Error.Error(),
"source", sourceObj.GetName(),
"namespace", sourceObj.GetNamespace(),
)
}
// Filter to only valid patterns
var validPatterns []string
for _, result := range validationResults {
if result.Valid {
validPatterns = append(validPatterns, result.Pattern)
}
}
patterns = validPatterns
// If no valid patterns remain, return empty
if len(patterns) == 0 {
return nil, nil
}
}
// Get all namespace info in a single API call (more efficient than 3 separate calls)
nsInfo, err := r.NamespaceLister.ListNamespacesWithLabels(ctx)
if err != nil {
return nil, fmt.Errorf("failed to list namespaces: %w", err)
}
// Get namespaces with allow-mirrors label
allowMirrorsNamespaces, err := r.NamespaceLister.ListAllowMirrorsNamespaces(ctx)
if err != nil {
return nil, fmt.Errorf("failed to list allow-mirrors namespaces: %w", err)
}
// Get namespaces that have explicitly opted out (allow-mirrors="false")
optOutNamespaces, err := r.NamespaceLister.ListOptOutNamespaces(ctx)
if err != nil {
return nil, fmt.Errorf("failed to list opt-out namespaces: %w", err)
}
// Resolve target namespaces
// Resolve target namespaces using the pre-categorized namespace info
targetNamespaces := filter.ResolveTargetNamespaces(
patterns,
allNamespaces,
allowMirrorsNamespaces,
optOutNamespaces,
nsInfo.All,
nsInfo.AllowMirrors,
nsInfo.OptOut,
sourceObj.GetNamespace(),
r.Filter,
)
@@ -611,16 +692,6 @@ func (r *SourceReconciler) mapMirrorToSource(ctx context.Context, obj client.Obj
}
}
// containsString checks if a slice contains a string.
func containsString(slice []string, s string) bool {
for _, item := range slice {
if item == s {
return true
}
}
return false
}
// removeString removes a string from a slice.
func removeString(slice []string, s string) []string {
result := make([]string, 0, len(slice))
pkg/controller/source_reconciler_test.go
+87 -14
View File
@@ -134,6 +134,14 @@ func (m *MockNamespaceLister) ListOptOutNamespaces(ctx context.Context) ([]strin
return args.Get(0).([]string), args.Error(1)
}
func (m *MockNamespaceLister) ListNamespacesWithLabels(ctx context.Context) (*NamespaceInfo, error) {
args := m.Called(ctx)
if args.Get(0) == nil {
return nil, args.Error(1)
}
return args.Get(0).(*NamespaceInfo), args.Error(1)
}
func TestIsEnabledForMirroring(t *testing.T) {
tests := []struct {
obj metav1.Object
@@ -280,9 +288,12 @@ func TestSourceReconciler_resolveTargetNamespaces(t *testing.T) {
mockLister := new(MockNamespaceLister)
if tt.expectListCalls {
mockLister.On("ListNamespaces", mock.Anything).Return(tt.allNamespaces, nil)
mockLister.On("ListAllowMirrorsNamespaces", mock.Anything).Return(tt.allowMirrorsNamespaces, nil)
mockLister.On("ListOptOutNamespaces", mock.Anything).Return([]string{}, nil)
nsInfo := &NamespaceInfo{
All: tt.allNamespaces,
AllowMirrors: tt.allowMirrorsNamespaces,
OptOut: []string{},
}
mockLister.On("ListNamespacesWithLabels", mock.Anything).Return(nsInfo, nil)
}
r := &SourceReconciler{
@@ -442,12 +453,15 @@ func BenchmarkIsEnabledForMirroring(b *testing.B) {
func BenchmarkResolveTargetNamespaces(b *testing.B) {
mockLister := new(MockNamespaceLister)
allNamespaces := make([]string, 100)
for i := 0; i < 100; i++ {
for i := range 100 {
allNamespaces[i] = fmt.Sprintf("namespace-%d", i)
}
mockLister.On("ListNamespaces", mock.Anything).Return(allNamespaces, nil)
mockLister.On("ListAllowMirrorsNamespaces", mock.Anything).Return(allNamespaces[:50], nil)
mockLister.On("ListOptOutNamespaces", mock.Anything).Return([]string{}, nil)
nsInfo := &NamespaceInfo{
All: allNamespaces,
AllowMirrors: allNamespaces[:50],
OptOut: []string{},
}
mockLister.On("ListNamespacesWithLabels", mock.Anything).Return(nsInfo, nil)
r := &SourceReconciler{
Config: &config.Config{},
@@ -517,7 +531,12 @@ func TestSourceReconciler_cleanupOrphanedMirrors(t *testing.T) {
// Setup: all namespaces in cluster
allNamespaces := []string{"default", "app-1", "app-2", "app-3", "prod-1"}
mockLister.On("ListNamespaces", mock.Anything).Return(allNamespaces, nil)
nsInfo := &NamespaceInfo{
All: allNamespaces,
AllowMirrors: []string{},
OptOut: []string{},
}
mockLister.On("ListNamespacesWithLabels", mock.Anything).Return(nsInfo, nil)
// Current target list (after annotation change): only app-1 and app-2
targetNamespaces := []string{"app-1", "app-2"}
@@ -624,14 +643,35 @@ func TestSourceReconciler_Reconcile_AnnotationChange_AllToAllLabeled(t *testing.
mockLister := new(MockNamespaceLister)
mockFilter := filter.NewNamespaceFilter(nil, nil)
mockLister.On("ListNamespaces", mock.Anything).Return(allNamespaces, nil)
mockLister.On("ListAllowMirrorsNamespaces", mock.Anything).Return(allowMirrorsNamespaces, nil)
mockLister.On("ListOptOutNamespaces", mock.Anything).Return([]string{}, nil)
nsInfo := &NamespaceInfo{
All: allNamespaces,
AllowMirrors: allowMirrorsNamespaces,
OptOut: []string{},
}
mockLister.On("ListNamespacesWithLabels", mock.Anything).Return(nsInfo, nil)
// Mock Get for source
mockClient.On("Get", mock.Anything, types.NamespacedName{Namespace: "default", Name: "test-secret"}, mock.Anything).
Return(nil, source)
// Helper to create a mock mirror for verification
createMirror := func(ns string) *unstructured.Unstructured {
return &unstructured.Unstructured{
Object: map[string]any{
"apiVersion": "v1",
"kind": "Secret",
"metadata": map[string]any{
"name": "test-secret",
"namespace": ns,
"labels": map[string]any{
constants.LabelManagedBy: constants.ControllerName,
constants.LabelMirror: "true",
},
},
},
}
}
// Mock reconcileMirror calls for app-1 and app-2 (current targets)
notFoundErr := errors.NewNotFound(schema.GroupResource{Group: "", Resource: "secrets"}, "test-secret")
mockClient.On("Get", mock.Anything, types.NamespacedName{Namespace: "app-1", Name: "test-secret"}, mock.Anything).
@@ -639,12 +679,18 @@ func TestSourceReconciler_Reconcile_AnnotationChange_AllToAllLabeled(t *testing.
mockClient.On("Create", mock.Anything, mock.MatchedBy(func(obj client.Object) bool {
return obj.GetNamespace() == "app-1"
}), mock.Anything).Return(nil).Once()
// Verification Get after Create
mockClient.On("Get", mock.Anything, types.NamespacedName{Namespace: "app-1", Name: "test-secret"}, mock.Anything).
Return(nil, createMirror("app-1")).Once()
mockClient.On("Get", mock.Anything, types.NamespacedName{Namespace: "app-2", Name: "test-secret"}, mock.Anything).
Return(notFoundErr, nil).Once()
mockClient.On("Create", mock.Anything, mock.MatchedBy(func(obj client.Object) bool {
return obj.GetNamespace() == "app-2"
}), mock.Anything).Return(nil).Once()
// Verification Get after Create
mockClient.On("Get", mock.Anything, types.NamespacedName{Namespace: "app-2", Name: "test-secret"}, mock.Anything).
Return(nil, createMirror("app-2")).Once()
// Mock cleanup: check orphaned namespaces app-3, prod-1, prod-2
mockClient.On("Get", mock.Anything, types.NamespacedName{Namespace: "app-3", Name: "test-secret"}, mock.Anything).
@@ -751,9 +797,12 @@ func TestSourceReconciler_Reconcile_AnnotationChange_PatternChange(t *testing.T)
mockLister := new(MockNamespaceLister)
mockFilter := filter.NewNamespaceFilter(nil, nil)
mockLister.On("ListNamespaces", mock.Anything).Return(allNamespaces, nil)
mockLister.On("ListAllowMirrorsNamespaces", mock.Anything).Return([]string{}, nil)
mockLister.On("ListOptOutNamespaces", mock.Anything).Return([]string{}, nil)
nsInfo := &NamespaceInfo{
All: allNamespaces,
AllowMirrors: []string{},
OptOut: []string{},
}
mockLister.On("ListNamespacesWithLabels", mock.Anything).Return(nsInfo, nil)
// Mock Get for source
mockClient.On("Get", mock.Anything, types.NamespacedName{Namespace: "default", Name: "app-config"}, mock.Anything).
@@ -761,18 +810,42 @@ func TestSourceReconciler_Reconcile_AnnotationChange_PatternChange(t *testing.T)
notFoundErr := errors.NewNotFound(schema.GroupResource{Group: "", Resource: "configmaps"}, "app-config")
// Helper to create a mock mirror for verification
createMirror := func(ns string) *unstructured.Unstructured {
return &unstructured.Unstructured{
Object: map[string]any{
"apiVersion": "v1",
"kind": "ConfigMap",
"metadata": map[string]any{
"name": "app-config",
"namespace": ns,
"labels": map[string]any{
constants.LabelManagedBy: constants.ControllerName,
constants.LabelMirror: "true",
},
},
},
}
}
// Mock reconcileMirror for prod-1 and prod-2 (new targets)
mockClient.On("Get", mock.Anything, types.NamespacedName{Namespace: "prod-1", Name: "app-config"}, mock.Anything).
Return(notFoundErr, nil).Once()
mockClient.On("Create", mock.Anything, mock.MatchedBy(func(obj client.Object) bool {
return obj.GetNamespace() == "prod-1"
}), mock.Anything).Return(nil).Once()
// Verification Get after Create
mockClient.On("Get", mock.Anything, types.NamespacedName{Namespace: "prod-1", Name: "app-config"}, mock.Anything).
Return(nil, createMirror("prod-1")).Once()
mockClient.On("Get", mock.Anything, types.NamespacedName{Namespace: "prod-2", Name: "app-config"}, mock.Anything).
Return(notFoundErr, nil).Once()
mockClient.On("Create", mock.Anything, mock.MatchedBy(func(obj client.Object) bool {
return obj.GetNamespace() == "prod-2"
}), mock.Anything).Return(nil).Once()
// Verification Get after Create
mockClient.On("Get", mock.Anything, types.NamespacedName{Namespace: "prod-2", Name: "app-config"}, mock.Anything).
Return(nil, createMirror("prod-2")).Once()
// Mock cleanup: delete orphaned mirrors in app-1, app-2, app-3
for _, ns := range []string{"app-1", "app-2", "app-3"} {