fix: stop reconciling on standby management clusters

exp/internal/controllers/machinepool_controller.go

@@ -46,6 +46,7 @@ import (
 	"sigs.k8s.io/cluster-api/controllers/external"
 	expv1 "sigs.k8s.io/cluster-api/exp/api/v1beta1"
 	"sigs.k8s.io/cluster-api/internal/util/ssa"
+	"sigs.k8s.io/cluster-api/pkg/standby"
 	"sigs.k8s.io/cluster-api/util"
 	"sigs.k8s.io/cluster-api/util/conditions"
 	"sigs.k8s.io/cluster-api/util/finalizers"
@@ -132,7 +133,13 @@ func (r *MachinePoolReconciler) SetupWithManager(ctx context.Context, mgr ctrl.M
 			),
 		).
 		WatchesRawSource(r.ClusterCache.GetClusterSource("machinepool", clusterToMachinePools)).
-		Build(r)
+		Build(standby.WrapClusterNamedReconcilerWithConfigMapDetector(
+			mgr.GetAPIReader(),
+			"machinepool",
+			func() client.Object { return &expv1.MachinePool{} },
+			func(obj client.Object) string { return obj.(*expv1.MachinePool).Spec.ClusterName },
+			r,
+		))
 	if err != nil {
 		return errors.Wrap(err, "failed setting up with a controller manager")
 	}

internal/controllers/cluster/cluster_controller.go

@@ -49,6 +49,7 @@ import (
 	expv1 "sigs.k8s.io/cluster-api/exp/api/v1beta1"
 	"sigs.k8s.io/cluster-api/feature"
 	"sigs.k8s.io/cluster-api/internal/hooks"
+	"sigs.k8s.io/cluster-api/pkg/standby"
 	"sigs.k8s.io/cluster-api/util"
 	"sigs.k8s.io/cluster-api/util/collections"
 	"sigs.k8s.io/cluster-api/util/conditions"
@@ -122,7 +123,7 @@ func (r *Reconciler) SetupWithManager(ctx context.Context, mgr ctrl.Manager, opt
 	c, err := b.
 		WithOptions(options).
 		WithEventFilter(predicates.ResourceHasFilterLabel(mgr.GetScheme(), predicateLog, r.WatchFilterValue)).
-		Build(r)
+		Build(standby.WrapClusterReconcilerWithConfigMapDetector(mgr.GetAPIReader(), "cluster", r))
 
 	if err != nil {
 		return errors.Wrap(err, "failed setting up with a controller manager")

internal/controllers/clusterresourceset/clusterresourceset_controller.go

@@ -44,6 +44,7 @@ import (
 	clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
 	"sigs.k8s.io/cluster-api/controllers/clustercache"
 	resourcepredicates "sigs.k8s.io/cluster-api/internal/controllers/clusterresourceset/predicates"
+	"sigs.k8s.io/cluster-api/pkg/standby"
 	"sigs.k8s.io/cluster-api/util"
 	"sigs.k8s.io/cluster-api/util/conditions"
 	v1beta2conditions "sigs.k8s.io/cluster-api/util/conditions/v1beta2"
@@ -66,6 +67,9 @@ type Reconciler struct {
 	Client       client.Client
 	ClusterCache clustercache.ClusterCache
 
+	// StandbyDetector detects if the management cluster is in standby mode.
+	StandbyDetector standby.Detector
+
 	// WatchFilterValue is the label value used to filter events prior to reconciliation.
 	WatchFilterValue string
 }
@@ -74,6 +78,9 @@ func (r *Reconciler) SetupWithManager(ctx context.Context, mgr ctrl.Manager, opt
 	if r.Client == nil || r.ClusterCache == nil {
 		return errors.New("Client and ClusterCache must not be nil")
 	}
+	if r.StandbyDetector == nil {
+		r.StandbyDetector = standby.NewConfigMapDetector(mgr.GetAPIReader())
+	}
 
 	predicateLog := ctrl.LoggerFrom(ctx).WithValues("controller", "clusterresourceset")
 	err := ctrl.NewControllerManagedBy(mgr).
@@ -181,10 +188,14 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (_ ctrl.Re
 		})
 		return ctrl.Result{}, err
 	}
+	clusters, skippedNonGlobalClusters, err := r.filterClustersForStandby(ctx, clusters)
+	if err != nil {
+		return ctrl.Result{}, err
+	}
 
 	// Handle deletion reconciliation loop.
 	if !clusterResourceSet.ObjectMeta.DeletionTimestamp.IsZero() {
-		return ctrl.Result{}, r.reconcileDelete(ctx, clusters, clusterResourceSet)
+		return r.reconcileDelete(ctx, clusters, clusterResourceSet, skippedNonGlobalClusters)
 	}
 
 	errs := []error{}
@@ -219,7 +230,7 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (_ ctrl.Re
 }
 
 // reconcileDelete removes the deleted ClusterResourceSet from all the ClusterResourceSetBindings it is added to.
-func (r *Reconciler) reconcileDelete(ctx context.Context, clusters []*clusterv1.Cluster, crs *addonsv1.ClusterResourceSet) error {
+func (r *Reconciler) reconcileDelete(ctx context.Context, clusters []*clusterv1.Cluster, crs *addonsv1.ClusterResourceSet, skippedNonGlobalClusters bool) (ctrl.Result, error) {
 	for _, cluster := range clusters {
 		log := ctrl.LoggerFrom(ctx, "Cluster", klog.KObj(cluster))
 
@@ -230,16 +241,20 @@ func (r *Reconciler) reconcileDelete(ctx context.Context, clusters []*clusterv1.
 		}
 		if err := r.Client.Get(ctx, clusterResourceSetBindingKey, clusterResourceSetBinding); err != nil {
 			if !apierrors.IsNotFound(err) {
-				return errors.Wrapf(err, "failed to get ClusterResourceSetBinding during ClusterResourceSet deletion")
+				return ctrl.Result{}, errors.Wrapf(err, "failed to get ClusterResourceSetBinding during ClusterResourceSet deletion")
+			}
+			if skippedNonGlobalClusters {
+				logSkipRemoveFinalizerForStandby(ctx, crs)
+				return ctrl.Result{RequeueAfter: standby.RequeueAfter}, nil
 			}
 			controllerutil.RemoveFinalizer(crs, addonsv1.ClusterResourceSetFinalizer)
-			return nil
+			return ctrl.Result{}, nil
 		}
 
 		// Initialize the patch helper.
 		patchHelper, err := patch.NewHelper(clusterResourceSetBinding, r.Client)
 		if err != nil {
-			return err
+			return ctrl.Result{}, err
 		}
 
 		clusterResourceSetBinding.RemoveBinding(crs)
@@ -256,12 +271,20 @@ func (r *Reconciler) reconcileDelete(ctx context.Context, clusters []*clusterv1.
 				log.Error(err, "Failed to delete empty ClusterResourceSetBinding")
 			}
 		} else if err := patchHelper.Patch(ctx, clusterResourceSetBinding); err != nil {
-			return err
+			return ctrl.Result{}, err
 		}
 	}
 
+	if skippedNonGlobalClusters {
+		logSkipRemoveFinalizerForStandby(ctx, crs)
+		return ctrl.Result{RequeueAfter: standby.RequeueAfter}, nil
+	}
 	controllerutil.RemoveFinalizer(crs, addonsv1.ClusterResourceSetFinalizer)
-	return nil
+	return ctrl.Result{}, nil
+}
+
+func logSkipRemoveFinalizerForStandby(ctx context.Context, crs *addonsv1.ClusterResourceSet) {
+	ctrl.LoggerFrom(ctx).Info("Refusing to remove ClusterResourceSet finalizer because management cluster is DR standby and non-global clusters were skipped", "ClusterResourceSet", klog.KObj(crs), "finalizer", addonsv1.ClusterResourceSetFinalizer, "globalCluster", standby.GlobalClusterName, "requeueAfter", standby.RequeueAfter)
 }
 
 // getClustersByClusterResourceSetSelector fetches Clusters matched by the ClusterResourceSet's label selector that are in the same namespace as the ClusterResourceSet object.
@@ -294,6 +317,30 @@ func (r *Reconciler) getClustersByClusterResourceSetSelector(ctx context.Context
 	return clusters, nil
 }
 
+func (r *Reconciler) filterClustersForStandby(ctx context.Context, clusters []*clusterv1.Cluster) ([]*clusterv1.Cluster, bool, error) {
+	if r.StandbyDetector == nil {
+		return clusters, false, nil
+	}
+	isStandby, err := r.StandbyDetector.IsStandby(ctx)
+	if err != nil {
+		return nil, false, err
+	}
+	if !isStandby {
+		return clusters, false, nil
+	}
+
+	filtered := []*clusterv1.Cluster{}
+	skippedNonGlobalClusters := false
+	for _, cluster := range clusters {
+		if cluster.Name == standby.GlobalClusterName {
+			filtered = append(filtered, cluster)
+			continue
+		}
+		skippedNonGlobalClusters = true
+	}
+	return filtered, skippedNonGlobalClusters, nil
+}
+
 // ApplyClusterResourceSet applies resources in a ClusterResourceSet to a Cluster. Once applied, a record will be added to the
 // cluster's ClusterResourceSetBinding.
 // In ApplyOnce strategy, resources are applied only once to a particular cluster. ClusterResourceSetBinding is used to check if a resource is applied before.

internal/controllers/clusterresourceset/clusterresourceset_controller_test.go

@@ -17,6 +17,7 @@ limitations under the License.
 package clusterresourceset
 
 import (
+	"context"
 	"crypto/sha1" //nolint: gosec
 	"fmt"
 	"testing"
@@ -34,6 +35,7 @@ import (
 	addonsv1 "sigs.k8s.io/cluster-api/api/addons/v1beta1"
 	clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
 	"sigs.k8s.io/cluster-api/internal/test/envtest"
+	"sigs.k8s.io/cluster-api/pkg/standby"
 	"sigs.k8s.io/cluster-api/util"
 	"sigs.k8s.io/cluster-api/util/conditions"
 	v1beta2conditions "sigs.k8s.io/cluster-api/util/conditions/v1beta2"
@@ -43,6 +45,91 @@ const (
 	timeout = time.Second * 15
 )
 
+type crsStandbyDetector struct {
+	standby bool
+	err     error
+}
+
+func (d crsStandbyDetector) IsStandby(context.Context) (bool, error) {
+	return d.standby, d.err
+}
+
+func TestFilterClustersForStandby(t *testing.T) {
+	testCases := []struct {
+		name            string
+		detector        standby.Detector
+		clusters        []*clusterv1.Cluster
+		expectedNames   []string
+		expectedSkipped bool
+		expectError     bool
+	}{
+		{
+			name:          "active returns all clusters",
+			detector:      crsStandbyDetector{},
+			clusters:      []*clusterv1.Cluster{{ObjectMeta: metav1.ObjectMeta{Name: standby.GlobalClusterName}}, {ObjectMeta: metav1.ObjectMeta{Name: "business"}}},
+			expectedNames: []string{standby.GlobalClusterName, "business"},
+		},
+		{
+			name:            "standby returns global cluster only",
+			detector:        crsStandbyDetector{standby: true},
+			clusters:        []*clusterv1.Cluster{{ObjectMeta: metav1.ObjectMeta{Name: standby.GlobalClusterName}}, {ObjectMeta: metav1.ObjectMeta{Name: "business"}}},
+			expectedNames:   []string{standby.GlobalClusterName},
+			expectedSkipped: true,
+		},
+		{
+			name:            "standby returns empty without global cluster",
+			detector:        crsStandbyDetector{standby: true},
+			clusters:        []*clusterv1.Cluster{{ObjectMeta: metav1.ObjectMeta{Name: "business"}}},
+			expectedNames:   []string{},
+			expectedSkipped: true,
+		},
+		{
+			name:        "detector error fails closed",
+			detector:    crsStandbyDetector{err: errors.New("detector error")},
+			clusters:    []*clusterv1.Cluster{{ObjectMeta: metav1.ObjectMeta{Name: standby.GlobalClusterName}}},
+			expectError: true,
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			g := NewWithT(t)
+			r := &Reconciler{StandbyDetector: tc.detector}
+
+			clusters, skipped, err := r.filterClustersForStandby(context.Background(), tc.clusters)
+			if tc.expectError {
+				g.Expect(err).To(HaveOccurred())
+				return
+			}
+			g.Expect(err).NotTo(HaveOccurred())
+
+			clusterNames := []string{}
+			for _, cluster := range clusters {
+				clusterNames = append(clusterNames, cluster.Name)
+			}
+			g.Expect(clusterNames).To(Equal(tc.expectedNames))
+			g.Expect(skipped).To(Equal(tc.expectedSkipped))
+		})
+	}
+}
+
+func TestReconcileDeleteSkipsFinalizerRemovalForStandbySkippedClusters(t *testing.T) {
+	g := NewWithT(t)
+	clusterResourceSet := &addonsv1.ClusterResourceSet{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:       "crs",
+			Namespace:  metav1.NamespaceDefault,
+			Finalizers: []string{addonsv1.ClusterResourceSetFinalizer},
+		},
+	}
+	r := &Reconciler{}
+
+	res, err := r.reconcileDelete(context.Background(), nil, clusterResourceSet, true)
+	g.Expect(err).NotTo(HaveOccurred())
+	g.Expect(res.RequeueAfter).To(Equal(standby.RequeueAfter))
+	g.Expect(clusterResourceSet.Finalizers).To(ContainElement(addonsv1.ClusterResourceSetFinalizer))
+}
+
 func TestClusterResourceSetReconciler(t *testing.T) {
 	var (
 		clusterResourceSetName      string

internal/controllers/clusterresourcesetbinding/clusterresourcesetbinding_controller.go

@@ -34,6 +34,7 @@ import (
 	clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
 	"sigs.k8s.io/cluster-api/feature"
 	"sigs.k8s.io/cluster-api/internal/hooks"
+	"sigs.k8s.io/cluster-api/pkg/standby"
 	"sigs.k8s.io/cluster-api/util"
 	"sigs.k8s.io/cluster-api/util/patch"
 	"sigs.k8s.io/cluster-api/util/predicates"
@@ -64,7 +65,19 @@ func (r *Reconciler) SetupWithManager(ctx context.Context, mgr ctrl.Manager, opt
 		).
 		WithOptions(options).
 		WithEventFilter(predicates.ResourceNotPausedAndHasFilterLabel(mgr.GetScheme(), predicateLog, r.WatchFilterValue)).
-		Complete(r)
+		Complete(standby.WrapClusterNamedReconcilerWithConfigMapDetector(
+			mgr.GetAPIReader(),
+			"clusterresourcesetbinding",
+			func() client.Object { return &addonsv1.ClusterResourceSetBinding{} },
+			func(obj client.Object) string {
+				binding := obj.(*addonsv1.ClusterResourceSetBinding)
+				if binding.Spec.ClusterName != "" {
+					return binding.Spec.ClusterName
+				}
+				return binding.Name
+			},
+			r,
+		))
 	if err != nil {
 		return errors.Wrap(err, "failed setting up with a controller manager")
 	}

internal/controllers/machine/machine_controller.go

@@ -55,6 +55,7 @@ import (
 	"sigs.k8s.io/cluster-api/feature"
 	"sigs.k8s.io/cluster-api/internal/contract"
 	"sigs.k8s.io/cluster-api/internal/controllers/machine/drain"
+	"sigs.k8s.io/cluster-api/pkg/standby"
 	"sigs.k8s.io/cluster-api/util"
 	"sigs.k8s.io/cluster-api/util/annotations"
 	"sigs.k8s.io/cluster-api/util/cache"
@@ -172,7 +173,13 @@ func (r *Reconciler) SetupWithManager(ctx context.Context, mgr ctrl.Manager, opt
 			handler.EnqueueRequestsFromMapFunc(mdToMachines),
 			builder.WithPredicates(predicates.ResourceIsChanged(mgr.GetScheme(), *r.predicateLog)),
 		).
-		Build(r)
+		Build(standby.WrapClusterNamedReconcilerWithConfigMapDetector(
+			mgr.GetAPIReader(),
+			"machine",
+			func() client.Object { return &clusterv1.Machine{} },
+			func(obj client.Object) string { return obj.(*clusterv1.Machine).Spec.ClusterName },
+			r,
+		))
 	if err != nil {
 		return errors.Wrap(err, "failed setting up with a controller manager")
 	}
@@ -726,7 +733,7 @@ func (r *Reconciler) isDeleteNodeAllowed(ctx context.Context, cluster *clusterv1
 		providerID = *machine.Spec.ProviderID
 	} else if infraMachine != nil {
 		// Fallback to retrieve from infraMachine.
-		if providerIDFromInfraMachine, err := contract.InfrastructureMachine().ProviderID().Get(infraMachine); err == nil {
+		if providerIDFromInfraMachine, err := contract.InfrastructureMachine().ProviderID().Get(infraMachine); err == nil && providerIDFromInfraMachine != nil {
 			providerID = *providerIDFromInfraMachine
 		}
 	}

internal/controllers/machinedeployment/machinedeployment_controller.go

@@ -39,6 +39,7 @@ import (
 	clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
 	"sigs.k8s.io/cluster-api/controllers/external"
 	"sigs.k8s.io/cluster-api/internal/util/ssa"
+	"sigs.k8s.io/cluster-api/pkg/standby"
 	"sigs.k8s.io/cluster-api/util"
 	"sigs.k8s.io/cluster-api/util/conditions"
 	utilconversion "sigs.k8s.io/cluster-api/util/conversion"
@@ -108,7 +109,13 @@ func (r *Reconciler) SetupWithManager(ctx context.Context, mgr ctrl.Manager, opt
 				predicates.ClusterPausedTransitions(mgr.GetScheme(), predicateLog),
 			)),
 			// TODO: should this wait for Cluster.Status.InfrastructureReady similar to Infra Machine resources?
-		).Complete(r)
+		).Complete(standby.WrapClusterNamedReconcilerWithConfigMapDetector(
+		mgr.GetAPIReader(),
+		"machinedeployment",
+		func() client.Object { return &clusterv1.MachineDeployment{} },
+		func(obj client.Object) string { return obj.(*clusterv1.MachineDeployment).Spec.ClusterName },
+		r,
+	))
 	if err != nil {
 		return errors.Wrap(err, "failed setting up with a controller manager")
 	}

internal/controllers/machinehealthcheck/machinehealthcheck_controller.go

@@ -49,6 +49,7 @@ import (
 	"sigs.k8s.io/cluster-api/controllers/clustercache"
 	"sigs.k8s.io/cluster-api/controllers/external"
 	"sigs.k8s.io/cluster-api/internal/controllers/machine"
+	"sigs.k8s.io/cluster-api/pkg/standby"
 	"sigs.k8s.io/cluster-api/util"
 	"sigs.k8s.io/cluster-api/util/annotations"
 	"sigs.k8s.io/cluster-api/util/conditions"
@@ -118,7 +119,13 @@ func (r *Reconciler) SetupWithManager(ctx context.Context, mgr ctrl.Manager, opt
 			),
 		).
 		WatchesRawSource(r.ClusterCache.GetClusterSource("machinehealthcheck", r.clusterToMachineHealthCheck)).
-		Build(r)
+		Build(standby.WrapClusterNamedReconcilerWithConfigMapDetector(
+			mgr.GetAPIReader(),
+			"machinehealthcheck",
+			func() client.Object { return &clusterv1.MachineHealthCheck{} },
+			func(obj client.Object) string { return obj.(*clusterv1.MachineHealthCheck).Spec.ClusterName },
+			r,
+		))
 	if err != nil {
 		return errors.Wrap(err, "failed setting up with a controller manager")
 	}