perf(controller): stop per-target apiserver reads in mirror reconcile

The mirror reconcile path issued one direct apiserver GET per target
namespace on every pass. With N sources and M target namespaces each
resync produced N*M GET requests. On a 65-namespace cluster this
generated 63 GET/s and overloaded one control-plane node.

Changes:
- reconcileMirror now trusts the informer cache. Content drift is
  detected via the source hash annotation. A stale resourceVersion
  makes Update return a conflict, which requeues the source.
- The verify-source-freshness flag now defaults to false. The Helm
  chart passes the value explicitly, so the chart default of false
  matches the binary behaviour.
- Reconcile performs one namespace LIST per pass. Target resolution
  and orphan cleanup share the result.
- Add load regression tests that count direct API reads in the
  reconcile hot path.
This commit is contained in:
2026-08-29 13:44:54 +01:00
parent 8c0bb0c5b0
commit eaa1246fb7
6 changed files with 131 additions and 41 deletions
+94 -3
View File
@@ -381,7 +381,7 @@ func TestSourceReconciler_resolveTargetNamespaces(t *testing.T) {
},
}
got, err := r.resolveTargetNamespaces(context.Background(), sourceObj)
got, _, err := r.resolveTargetNamespaces(context.Background(), sourceObj)
if tt.wantError {
require.Error(t, err)
@@ -555,7 +555,7 @@ func BenchmarkResolveTargetNamespaces(b *testing.B) {
b.ResetTimer()
b.ReportAllocs()
for i := 0; i < b.N; i++ {
_, _ = r.resolveTargetNamespaces(ctx, sourceObj)
_, _, _ = r.resolveTargetNamespaces(ctx, sourceObj)
}
}
@@ -640,7 +640,7 @@ func TestSourceReconciler_cleanupOrphanedMirrors(t *testing.T) {
}
ctx := context.Background()
deletedCount, err := r.cleanupOrphanedMirrors(ctx, sourceObj, targetNamespaces)
deletedCount, err := r.cleanupOrphanedMirrors(ctx, sourceObj, targetNamespaces, nil)
require.NoError(t, err)
assert.Equal(t, 1, deletedCount, "should have deleted 1 orphaned mirror")
@@ -1273,3 +1273,94 @@ func TestSourceReconciler_updateLastSyncStatus_writesWhenChanged(t *testing.T) {
assert.Equal(t, "reconciled:3,errors:0", source.GetAnnotations()[constants.AnnotationSyncStatus])
mockClient.AssertCalled(t, "Update", mock.Anything, mock.Anything, mock.Anything)
}
// countingReader is a client.Reader that counts direct API reads. It stands in
// for the manager's APIReader so tests can assert the reconcile hot path never
// bypasses the informer cache.
type countingReader struct {
gets int
lists int
}
func (c *countingReader) Get(ctx context.Context, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error {
c.gets++
return errors.NewNotFound(schema.GroupResource{Resource: "secrets"}, key.Name)
}
func (c *countingReader) List(ctx context.Context, list client.ObjectList, opts ...client.ListOption) error {
c.lists++
return nil
}
// newMirrorTestSource builds an unstructured Secret source enabled for mirroring.
func newMirrorTestSource(t *testing.T) *unstructured.Unstructured {
t.Helper()
source := &unstructured.Unstructured{}
source.SetGroupVersionKind(schema.GroupVersionKind{Version: "v1", Kind: "Secret"})
source.SetNamespace("default")
source.SetName("test-secret")
source.SetLabels(map[string]string{constants.LabelEnabled: "true"})
source.SetAnnotations(map[string]string{
constants.AnnotationSync: "true",
constants.AnnotationTargetNamespaces: "all",
})
require.NoError(t, unstructured.SetNestedField(source.Object, "Opaque", "type"))
return source
}
// TestSourceReconciler_reconcileMirror_NoDirectAPIReads is a load regression
// test: with an up-to-date mirror in the cache, reconcileMirror must complete
// without a single direct API read. A per-target APIReader.Get here multiplies
// into (sources x target namespaces) apiserver GETs on every resync and has
// melted a control-plane node before.
func TestSourceReconciler_reconcileMirror_NoDirectAPIReads(t *testing.T) {
source := newMirrorTestSource(t)
mirrorObj, err := CreateMirror(source, "target-ns")
require.NoError(t, err)
mirror, ok := mirrorObj.(*unstructured.Unstructured)
require.True(t, ok, "unstructured source must produce an unstructured mirror")
mirror.SetResourceVersion("100")
mockClient := new(MockClient)
mockClient.On("Get", mock.Anything, types.NamespacedName{Namespace: "target-ns", Name: "test-secret"}, mock.Anything).
Return(nil, mirror)
reader := &countingReader{}
r := &SourceReconciler{
Client: mockClient,
APIReader: reader,
Config: &config.Config{VerifySourceFreshness: true},
GVK: source.GroupVersionKind(),
}
require.NoError(t, r.reconcileMirror(context.Background(), source, source, "target-ns"))
assert.Zero(t, reader.gets, "steady-state mirror reconcile must not GET from the apiserver directly")
assert.Zero(t, reader.lists, "steady-state mirror reconcile must not LIST from the apiserver directly")
mockClient.AssertNotCalled(t, "Update", mock.Anything, mock.Anything, mock.Anything)
}
// TestSourceReconciler_getSourceWithFreshness_DefaultNoDirectRead locks the
// default behaviour: freshness verification is opt-in, so the default config
// must serve the source straight from the informer cache.
func TestSourceReconciler_getSourceWithFreshness_DefaultNoDirectRead(t *testing.T) {
source := newMirrorTestSource(t)
mockClient := new(MockClient)
mockClient.On("Get", mock.Anything, types.NamespacedName{Namespace: "default", Name: "test-secret"}, mock.Anything).
Return(nil, source)
reader := &countingReader{}
r := &SourceReconciler{
Client: mockClient,
APIReader: reader,
Config: &config.Config{},
GVK: source.GroupVersionKind(),
}
got, err := r.getSourceWithFreshness(context.Background(), client.ObjectKey{Namespace: "default", Name: "test-secret"}, r.GVK)
require.NoError(t, err)
assert.Equal(t, "test-secret", got.GetName())
assert.Zero(t, reader.gets, "default config must not verify freshness against the apiserver")
}