improvements jan2025 (#6)

* feat(controller): add lazy watcher, improve resource usage and add pattern validation

- [x] Add cache sync health check for readiness probe verification
- [x] Create namespace lister with API reader support for fresh label queries
- [x] Add pattern validation with warning logs for invalid glob patterns
- [x] Implement lazy watcher initialization mode to scan for active resources
- [x] Add requeue delay to namespace reconciler for cache settlement
- [x] Replace custom containsString with slices.Contains from stdlib
- [x] Add structured logging context to reconcilers (kind, group, version)
- [x] Improve error variable naming for clarity in nested conditions
- [x] Add nil-safe label access in namespace reconciler setup
- [x] Add APIReader to namespace and source reconcilers for fresh data
- [x] Improve type assertions with proper error handling in mirror operations
- [x] Reorder struct fields for consistency and readability
- [x] Add comprehensive pattern validation tests and validation API

* feat(controller): add lazy watcher, improve resource usage and add pattern validation

- [x] Add circuit breaker for reconciliation failure tracking and prevention
- [x] Implement granular registration state tracking (not-registered, source-only, fully-registered)
- [x] Add lazy controller initialization for active resource types only
- [x] Consolidate namespace listing into single API call for efficiency
- [x] Add mirror creation verification to catch webhook rejections
- [x] Implement high-cardinality resource detection and warnings
- [x] Add source deletion check in mirror reconciler to prevent races
- [x] Preserve transformation annotations on errors in mirror reconciliation
- [x] Expand constants documentation with labels vs annotations design rationale
- [x] Add comprehensive test coverage for circuit breaker and registration states
- [x] Add mutation-safety tests for hash computation

* fixup! feat(controller): add lazy watcher, improve resource usage and add pattern validation

pkg/discovery/discovery.go

@@ -10,10 +10,13 @@ import (
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/client-go/discovery"
 	"k8s.io/client-go/rest"
+	ctrl "sigs.k8s.io/controller-runtime"
 
 	"github.com/lukaszraczylo/kubemirror/pkg/config"
 )
 
+var discoveryLog = ctrl.Log.WithName("discovery")
+
 // ResourceDiscovery discovers all mirrorable resource types in a cluster.
 type ResourceDiscovery struct {
 	discoveryClient discovery.DiscoveryInterface
@@ -34,6 +37,8 @@ func NewResourceDiscovery(cfg *rest.Config) (*ResourceDiscovery, error) {
 // DiscoverMirrorableResources discovers all resource types that can be mirrored.
 // It filters out resources that shouldn't be mirrored based on a deny list.
 func (d *ResourceDiscovery) DiscoverMirrorableResources(ctx context.Context) ([]config.ResourceType, error) {
+	logger := discoveryLog.WithName("discover")
+
 	// Get all API resources in the cluster
 	_, apiResourceLists, err := d.discoveryClient.ServerGroupsAndResources()
 	if err != nil {
@@ -42,10 +47,12 @@ func (d *ResourceDiscovery) DiscoverMirrorableResources(ctx context.Context) ([]
 		if !discovery.IsGroupDiscoveryFailedError(err) {
 			return nil, fmt.Errorf("failed to discover API resources: %w", err)
 		}
+		logger.V(1).Info("some API groups had discovery errors, continuing with available resources")
 	}
 
 	var resources []config.ResourceType
 	seen := make(map[string]bool) // Deduplicate
+	var deniedCount int
 
 	for _, apiResourceList := range apiResourceLists {
 		gv, err := schema.ParseGroupVersion(apiResourceList.GroupVersion)
@@ -71,9 +78,23 @@ func (d *ResourceDiscovery) DiscoverMirrorableResources(ctx context.Context) ([]
 
 			// Skip denied resource types
 			if isDeniedResourceType(apiResource.Kind) {
+				deniedCount++
+				logger.V(2).Info("skipping denied resource type",
+					"kind", apiResource.Kind,
+					"group", gv.Group,
+					"version", gv.Version)
 				continue
 			}
 
+			// Warn about potentially high-cardinality resource types that aren't in deny list
+			if isHighCardinalityResource(apiResource.Kind) {
+				logger.Info("WARNING: discovered potentially high-cardinality resource type",
+					"kind", apiResource.Kind,
+					"group", gv.Group,
+					"version", gv.Version,
+					"recommendation", "Consider adding to deny list if high volume is observed")
+			}
+
 			rt := config.ResourceType{
 				Group:   gv.Group,
 				Version: gv.Version,
@@ -91,6 +112,10 @@ func (d *ResourceDiscovery) DiscoverMirrorableResources(ctx context.Context) ([]
 		}
 	}
 
+	logger.Info("resource discovery complete",
+		"discovered", len(resources),
+		"denied", deniedCount)
+
 	return resources, nil
 }
 
@@ -316,3 +341,34 @@ var deniedKinds = map[string]bool{
 func isDeniedResourceType(kind string) bool {
 	return deniedKinds[kind]
 }
+
+// highCardinalityKinds are resource types that might generate high volumes of objects.
+// These aren't denied by default but warrant monitoring when discovered.
+var highCardinalityKinds = map[string]bool{
+	// Resources that might have many instances per namespace
+	"ServiceAccount":          true, // Often auto-created per deployment
+	"Role":                    true, // Can be many per namespace
+	"RoleBinding":             true, // Can be many per namespace
+	"NetworkPolicy":           true, // Can be many per namespace
+	"LimitRange":              true, // Usually few but triggers on all namespace changes
+	"ResourceQuota":           true, // Usually few but triggers on all namespace changes
+	"HorizontalPodAutoscaler": true, // One per deployment/statefulset
+
+	// CRD resources that might have high cardinality
+	"ServiceEntry":       true, // Istio - can have many
+	"VirtualService":     true, // Istio - can have many
+	"DestinationRule":    true, // Istio - can have many
+	"EnvoyFilter":        true, // Istio - can have many
+	"Sidecar":            true, // Istio - can have many
+	"PeerAuthentication": true, // Istio - can have many
+
+	// Prometheus-style monitoring resources
+	"ServiceMonitor": true, // Often one per service
+	"PodMonitor":     true, // Often one per pod type
+	"PrometheusRule": true, // Can have many rules
+}
+
+// isHighCardinalityResource checks if a resource type might generate high volumes.
+func isHighCardinalityResource(kind string) bool {
+	return highCardinalityKinds[kind]
+}