mirror of
https://github.com/lukaszraczylo/kubemirror.git
synced 2026-10-13 04:37:33 +00:00
perf(controller): repair mirror drift from watch events instead of resync
Mirror update events were filtered out, so an in-place edit of a mirrored copy was invisible to the controller. The periodic resync did not catch it either: NeedsSync only compares the source content hash with the recorded annotation, and tampered mirror data leaves both unchanged. Changes: - The mirror watch now passes real update events (resourceVersion changed, object is a managed mirror) and maps them to the source. Informer resync replays keep an unchanged resourceVersion and stay filtered. - reconcileMirror detects in-place drift by rebuilding the expected mirror in memory and comparing content hashes. Metadata differences never count as drift, so the controller's own writes converge in one no-op pass. - The resync-period default moves from 10m to 1h. Watches now cover source changes, mirror deletion, mirror drift, and namespace changes, so the resync is only a safety net for missed watch events. The chart default moves with it. - Add predicate and tamper-repair regression tests; drift repair is verified to make no direct apiserver reads.
This commit is contained in:
@@ -378,8 +378,19 @@ func (r *SourceReconciler) reconcileMirror(ctx context.Context, source runtime.O
|
||||
}
|
||||
|
||||
if !needsSync {
|
||||
logger.V(2).Info("mirror is up to date")
|
||||
return nil
|
||||
// The source is unchanged since the last sync, but the mirrored
|
||||
// copy itself may have been edited in place (its bookkeeping
|
||||
// annotations would still match the source). Rebuild the expected
|
||||
// mirror in memory and compare content hashes to catch that.
|
||||
drifted, driftErr := mirrorContentDrifted(source, existing, targetNs)
|
||||
if driftErr != nil {
|
||||
return fmt.Errorf("failed to check mirror drift: %w", driftErr)
|
||||
}
|
||||
if !drifted {
|
||||
logger.V(2).Info("mirror is up to date")
|
||||
return nil
|
||||
}
|
||||
logger.Info("mirror content drifted from expected state, repairing")
|
||||
}
|
||||
|
||||
// Update mirror
|
||||
@@ -421,6 +432,30 @@ func (r *SourceReconciler) reconcileMirror(ctx context.Context, source runtime.O
|
||||
return nil
|
||||
}
|
||||
|
||||
// mirrorContentDrifted reports whether the live mirror's content diverged from
|
||||
// what mirroring the source into targetNs should produce. It rebuilds the
|
||||
// expected mirror in memory (including transformations) and compares content
|
||||
// hashes, so metadata-only differences never count as drift. This is what
|
||||
// makes drift repair event-driven instead of depending on the periodic resync.
|
||||
func mirrorContentDrifted(source runtime.Object, existing *unstructured.Unstructured, targetNs string) (bool, error) {
|
||||
expected, err := CreateMirror(source, targetNs)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("failed to build expected mirror: %w", err)
|
||||
}
|
||||
|
||||
expectedHash, err := hash.ComputeContentHash(expected)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("failed to hash expected mirror: %w", err)
|
||||
}
|
||||
|
||||
existingHash, err := hash.ComputeContentHash(existing)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("failed to hash existing mirror: %w", err)
|
||||
}
|
||||
|
||||
return expectedHash != existingHash, nil
|
||||
}
|
||||
|
||||
// deleteAllMirrors deletes all mirrors that this source owns across the cluster.
|
||||
// It verifies ownership (managed-by label + source-reference annotation) before
|
||||
// deleting anything to avoid destroying unrelated resources that happen to share
|
||||
@@ -680,26 +715,43 @@ func (r *SourceReconciler) SetupWithManagerForResourceType(
|
||||
mirrorObj := &unstructured.Unstructured{}
|
||||
mirrorObj.SetGroupVersionKind(gvk)
|
||||
|
||||
// Create predicates to only watch mirror deletions
|
||||
mirrorDeletePredicate := predicate.Funcs{
|
||||
CreateFunc: func(e event.CreateEvent) bool { return false },
|
||||
UpdateFunc: func(e event.UpdateEvent) bool { return false },
|
||||
DeleteFunc: func(e event.DeleteEvent) bool { return IsMirrorResource(e.Object) },
|
||||
GenericFunc: func(e event.GenericEvent) bool { return false },
|
||||
}
|
||||
|
||||
return ctrl.NewControllerManagedBy(mgr).
|
||||
For(obj).
|
||||
Named(controllerName).
|
||||
// Watch mirror resources - when deleted, enqueue source for reconciliation
|
||||
// Watch mirror resources - deletes trigger recreation, real writes
|
||||
// trigger drift repair. This keeps mirrors converged without relying
|
||||
// on the periodic resync.
|
||||
Watches(
|
||||
mirrorObj,
|
||||
handler.EnqueueRequestsFromMapFunc(r.mapMirrorToSource),
|
||||
builder.WithPredicates(mirrorDeletePredicate),
|
||||
builder.WithPredicates(mirrorEventPredicate()),
|
||||
).
|
||||
Complete(r)
|
||||
}
|
||||
|
||||
// mirrorEventPredicate gates the mirror watch. Deletes of managed mirrors
|
||||
// enqueue the source so the mirror is recreated. Updates enqueue the source
|
||||
// so in-place edits to a mirror are repaired event-driven; informer resync
|
||||
// replays carry an unchanged resourceVersion and are filtered out, so only
|
||||
// real writes pass. The controller's own mirror writes pass too and converge
|
||||
// in one extra reconcile that no-ops on the content comparison.
|
||||
func mirrorEventPredicate() predicate.Funcs {
|
||||
return predicate.Funcs{
|
||||
CreateFunc: func(e event.CreateEvent) bool { return false },
|
||||
UpdateFunc: func(e event.UpdateEvent) bool {
|
||||
if e.ObjectOld == nil || e.ObjectNew == nil {
|
||||
return false
|
||||
}
|
||||
if e.ObjectOld.GetResourceVersion() == e.ObjectNew.GetResourceVersion() {
|
||||
return false
|
||||
}
|
||||
return IsMirrorResource(e.ObjectNew)
|
||||
},
|
||||
DeleteFunc: func(e event.DeleteEvent) bool { return IsMirrorResource(e.Object) },
|
||||
GenericFunc: func(e event.GenericEvent) bool { return false },
|
||||
}
|
||||
}
|
||||
|
||||
// mapMirrorToSource maps a mirror resource to its source for reconciliation.
|
||||
func (r *SourceReconciler) mapMirrorToSource(ctx context.Context, obj client.Object) []reconcile.Request {
|
||||
// Only process if this is a mirror
|
||||
|
||||
Reference in New Issue
Block a user