Skip to content

Commit

Permalink
Handle finalizers early in Reconciles
Browse files Browse the repository at this point in the history
Signed-off-by: Stefan Büringer [email protected]
  • Loading branch information
sbueringer committed Oct 14, 2024
1 parent 82ee71d commit 1f9ec2e
Show file tree
Hide file tree
Showing 16 changed files with 266 additions and 122 deletions.
22 changes: 6 additions & 16 deletions controlplane/kubeadm/internal/controllers/controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,7 @@ import (
"sigs.k8s.io/cluster-api/util/annotations"
"sigs.k8s.io/cluster-api/util/collections"
"sigs.k8s.io/cluster-api/util/conditions"
"sigs.k8s.io/cluster-api/util/finalizers"
"sigs.k8s.io/cluster-api/util/patch"
"sigs.k8s.io/cluster-api/util/predicates"
"sigs.k8s.io/cluster-api/util/secret"
Expand Down Expand Up @@ -149,6 +150,11 @@ func (r *KubeadmControlPlaneReconciler) Reconcile(ctx context.Context, req ctrl.
return ctrl.Result{}, err
}

// Add finalizer first if not set to avoid the race condition between init and delete.
if finalizerAdded, err := finalizers.EnsureFinalizer(ctx, r.Client, kcp, controlplanev1.KubeadmControlPlaneFinalizer); err != nil || finalizerAdded {
return ctrl.Result{}, err
}

// Fetch the Cluster.
cluster, err := util.GetOwnerCluster(ctx, r.Client, kcp.ObjectMeta)
if err != nil {
Expand Down Expand Up @@ -176,22 +182,6 @@ func (r *KubeadmControlPlaneReconciler) Reconcile(ctx context.Context, req ctrl.
return ctrl.Result{Requeue: true}, nil
}

// Add finalizer first if not set to avoid the race condition between init and delete.
// Note: Finalizers in general can only be added when the deletionTimestamp is not set.
if kcp.ObjectMeta.DeletionTimestamp.IsZero() && !controllerutil.ContainsFinalizer(kcp, controlplanev1.KubeadmControlPlaneFinalizer) {
controllerutil.AddFinalizer(kcp, controlplanev1.KubeadmControlPlaneFinalizer)

// patch and return right away instead of reusing the main defer,
// because the main defer may take too much time to get cluster status
// Patch ObservedGeneration only if the reconciliation completed successfully
patchOpts := []patch.Option{patch.WithStatusObservedGeneration{}}
if err := patchHelper.Patch(ctx, kcp, patchOpts...); err != nil {
return ctrl.Result{}, errors.Wrapf(err, "failed to add finalizer")
}

return ctrl.Result{}, nil
}

// Initialize the control plane scope; this includes also checking for orphan machines and
// adopt them if necessary.
controlPlane, adoptableMachineFound, err := r.initControlPlaneScope(ctx, cluster, kcp)
Expand Down
3 changes: 3 additions & 0 deletions controlplane/kubeadm/internal/controllers/controller_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -317,6 +317,9 @@ func TestReconcileNoCluster(t *testing.T) {
Name: "foo",
},
},
Finalizers: []string{
controlplanev1.KubeadmControlPlaneFinalizer,
},
},
Spec: controlplanev1.KubeadmControlPlaneSpec{
Version: "v1.16.6",
Expand Down
13 changes: 6 additions & 7 deletions exp/addons/internal/controllers/clusterresourceset_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@ import (
resourcepredicates "sigs.k8s.io/cluster-api/exp/addons/internal/controllers/predicates"
"sigs.k8s.io/cluster-api/util"
"sigs.k8s.io/cluster-api/util/conditions"
"sigs.k8s.io/cluster-api/util/finalizers"
"sigs.k8s.io/cluster-api/util/patch"
"sigs.k8s.io/cluster-api/util/predicates"
)
Expand Down Expand Up @@ -122,6 +123,11 @@ func (r *ClusterResourceSetReconciler) Reconcile(ctx context.Context, req ctrl.R
return ctrl.Result{}, err
}

// Add finalizer first if not set to avoid the race condition between init and delete.
if finalizerAdded, err := finalizers.EnsureFinalizer(ctx, r.Client, clusterResourceSet, addonsv1.ClusterResourceSetFinalizer); err != nil || finalizerAdded {
return ctrl.Result{}, err
}

// Initialize the patch helper.
patchHelper, err := patch.NewHelper(clusterResourceSet, r.Client)
if err != nil {
Expand Down Expand Up @@ -152,13 +158,6 @@ func (r *ClusterResourceSetReconciler) Reconcile(ctx context.Context, req ctrl.R
return ctrl.Result{}, r.reconcileDelete(ctx, clusters, clusterResourceSet)
}

// Add finalizer first if not set to avoid the race condition between init and delete.
// Note: Finalizers in general can only be added when the deletionTimestamp is not set.
if !controllerutil.ContainsFinalizer(clusterResourceSet, addonsv1.ClusterResourceSetFinalizer) {
controllerutil.AddFinalizer(clusterResourceSet, addonsv1.ClusterResourceSetFinalizer)
return ctrl.Result{}, nil
}

errs := []error{}
errClusterLockedOccurred := false
for _, cluster := range clusters {
Expand Down
15 changes: 7 additions & 8 deletions exp/internal/controllers/machinepool_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@ import (
"sigs.k8s.io/cluster-api/util"
"sigs.k8s.io/cluster-api/util/annotations"
"sigs.k8s.io/cluster-api/util/conditions"
"sigs.k8s.io/cluster-api/util/finalizers"
"sigs.k8s.io/cluster-api/util/patch"
"sigs.k8s.io/cluster-api/util/predicates"
)
Expand Down Expand Up @@ -151,9 +152,14 @@ func (r *MachinePoolReconciler) Reconcile(ctx context.Context, req ctrl.Request)
return ctrl.Result{}, err
}

log = log.WithValues("Cluster", klog.KRef(mp.ObjectMeta.Namespace, mp.Spec.ClusterName))
log = log.WithValues("Cluster", klog.KRef(mp.Namespace, mp.Spec.ClusterName))
ctx = ctrl.LoggerInto(ctx, log)

// Add finalizer first if not set to avoid the race condition between init and delete.
if finalizerAdded, err := finalizers.EnsureFinalizer(ctx, r.Client, mp, expv1.MachinePoolFinalizer); err != nil || finalizerAdded {
return ctrl.Result{}, err
}

cluster, err := util.GetClusterByName(ctx, r.Client, mp.ObjectMeta.Namespace, mp.Spec.ClusterName)
if err != nil {
log.Error(err, "Failed to get Cluster for MachinePool.", "MachinePool", klog.KObj(mp), "Cluster", klog.KRef(mp.ObjectMeta.Namespace, mp.Spec.ClusterName))
Expand Down Expand Up @@ -223,13 +229,6 @@ func (r *MachinePoolReconciler) Reconcile(ctx context.Context, req ctrl.Request)
return ctrl.Result{}, err
}

// Add finalizer first if not set to avoid the race condition between init and delete.
// Note: Finalizers in general can only be added when the deletionTimestamp is not set.
if !controllerutil.ContainsFinalizer(mp, expv1.MachinePoolFinalizer) {
controllerutil.AddFinalizer(mp, expv1.MachinePoolFinalizer)
return ctrl.Result{}, nil
}

// Handle normal reconciliation loop.
scope := &scope{
cluster: cluster,
Expand Down
13 changes: 6 additions & 7 deletions internal/controllers/cluster/cluster_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,7 @@ import (
"sigs.k8s.io/cluster-api/util/annotations"
"sigs.k8s.io/cluster-api/util/collections"
"sigs.k8s.io/cluster-api/util/conditions"
"sigs.k8s.io/cluster-api/util/finalizers"
"sigs.k8s.io/cluster-api/util/patch"
"sigs.k8s.io/cluster-api/util/predicates"
)
Expand Down Expand Up @@ -120,6 +121,11 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (_ ctrl.Re
return ctrl.Result{}, err
}

// Add finalizer first if not set to avoid the race condition between init and delete.
if finalizerAdded, err := finalizers.EnsureFinalizer(ctx, r.Client, cluster, clusterv1.ClusterFinalizer); err != nil || finalizerAdded {
return ctrl.Result{}, err
}

// Return early if the object or Cluster is paused.
if annotations.IsPaused(cluster, cluster) {
log.Info("Reconciliation is paused for this object")
Expand Down Expand Up @@ -152,13 +158,6 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (_ ctrl.Re
return r.reconcileDelete(ctx, cluster)
}

// Add finalizer first if not set to avoid the race condition between init and delete.
// Note: Finalizers in general can only be added when the deletionTimestamp is not set.
if !controllerutil.ContainsFinalizer(cluster, clusterv1.ClusterFinalizer) {
controllerutil.AddFinalizer(cluster, clusterv1.ClusterFinalizer)
return ctrl.Result{}, nil
}

// Handle normal reconciliation loop.
return r.reconcile(ctx, cluster)
}
Expand Down
19 changes: 9 additions & 10 deletions internal/controllers/machine/machine_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,7 @@ import (
"sigs.k8s.io/cluster-api/util/annotations"
"sigs.k8s.io/cluster-api/util/collections"
"sigs.k8s.io/cluster-api/util/conditions"
"sigs.k8s.io/cluster-api/util/finalizers"
clog "sigs.k8s.io/cluster-api/util/log"
"sigs.k8s.io/cluster-api/util/patch"
"sigs.k8s.io/cluster-api/util/predicates"
Expand Down Expand Up @@ -173,16 +174,21 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (_ ctrl.Re
return ctrl.Result{}, err
}

log := ctrl.LoggerFrom(ctx).WithValues("Cluster", klog.KRef(m.Namespace, m.Spec.ClusterName))
ctx = ctrl.LoggerInto(ctx, log)

// Add finalizer first if not set to avoid the race condition between init and delete.
if finalizerAdded, err := finalizers.EnsureFinalizer(ctx, r.Client, m, clusterv1.MachineFinalizer); err != nil || finalizerAdded {
return ctrl.Result{}, err
}

// AddOwners adds the owners of Machine as k/v pairs to the logger.
// Specifically, it will add KubeadmControlPlane, MachineSet and MachineDeployment.
ctx, log, err := clog.AddOwners(ctx, r.Client, m)
if err != nil {
return ctrl.Result{}, err
}

log = log.WithValues("Cluster", klog.KRef(m.ObjectMeta.Namespace, m.Spec.ClusterName))
ctx = ctrl.LoggerInto(ctx, log)

cluster, err := util.GetClusterByName(ctx, r.Client, m.ObjectMeta.Namespace, m.Spec.ClusterName)
if err != nil {
return ctrl.Result{}, errors.Wrapf(err, "failed to get cluster %q for machine %q in namespace %q",
Expand Down Expand Up @@ -220,13 +226,6 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (_ ctrl.Re
}
}()

// Add finalizer first if not set to avoid the race condition between init and delete.
// Note: Finalizers in general can only be added when the deletionTimestamp is not set.
if !controllerutil.ContainsFinalizer(m, clusterv1.MachineFinalizer) && m.ObjectMeta.DeletionTimestamp.IsZero() {
controllerutil.AddFinalizer(m, clusterv1.MachineFinalizer)
return ctrl.Result{}, nil
}

alwaysReconcile := []machineReconcileFunc{
r.reconcileMachineOwnerAndLabels,
r.reconcileBootstrap,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@ import (
"sigs.k8s.io/cluster-api/util/annotations"
"sigs.k8s.io/cluster-api/util/conditions"
utilconversion "sigs.k8s.io/cluster-api/util/conversion"
"sigs.k8s.io/cluster-api/util/finalizers"
clog "sigs.k8s.io/cluster-api/util/log"
"sigs.k8s.io/cluster-api/util/patch"
"sigs.k8s.io/cluster-api/util/predicates"
Expand Down Expand Up @@ -131,6 +132,11 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (_ ctrl.Re
log = log.WithValues("Cluster", klog.KRef(deployment.Namespace, deployment.Spec.ClusterName))
ctx = ctrl.LoggerInto(ctx, log)

// Add finalizer first if not set to avoid the race condition between init and delete.
if finalizerAdded, err := finalizers.EnsureFinalizer(ctx, r.Client, deployment, clusterv1.MachineDeploymentFinalizer); err != nil || finalizerAdded {
return ctrl.Result{}, err
}

cluster, err := util.GetClusterByName(ctx, r.Client, deployment.Namespace, deployment.Spec.ClusterName)
if err != nil {
return ctrl.Result{}, err
Expand Down Expand Up @@ -165,13 +171,6 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (_ ctrl.Re
return ctrl.Result{}, r.reconcileDelete(ctx, deployment)
}

// Add finalizer first if not set to avoid the race condition between init and delete.
// Note: Finalizers in general can only be added when the deletionTimestamp is not set.
if !controllerutil.ContainsFinalizer(deployment, clusterv1.MachineDeploymentFinalizer) {
controllerutil.AddFinalizer(deployment, clusterv1.MachineDeploymentFinalizer)
return ctrl.Result{}, nil
}

err = r.reconcile(ctx, cluster, deployment)
if err != nil {
r.recorder.Eventf(deployment, corev1.EventTypeWarning, "ReconcileError", "%v", err)
Expand Down
19 changes: 9 additions & 10 deletions internal/controllers/machineset/machineset_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,7 @@ import (
"sigs.k8s.io/cluster-api/util/collections"
"sigs.k8s.io/cluster-api/util/conditions"
utilconversion "sigs.k8s.io/cluster-api/util/conversion"
"sigs.k8s.io/cluster-api/util/finalizers"
"sigs.k8s.io/cluster-api/util/labels/format"
clog "sigs.k8s.io/cluster-api/util/log"
"sigs.k8s.io/cluster-api/util/patch"
Expand Down Expand Up @@ -149,16 +150,21 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (_ ctrl.Re
return ctrl.Result{}, err
}

log := ctrl.LoggerFrom(ctx).WithValues("Cluster", klog.KRef(machineSet.Namespace, machineSet.Spec.ClusterName))
ctx = ctrl.LoggerInto(ctx, log)

// Add finalizer first if not set to avoid the race condition between init and delete.
if finalizerAdded, err := finalizers.EnsureFinalizer(ctx, r.Client, machineSet, clusterv1.MachineSetFinalizer); err != nil || finalizerAdded {
return ctrl.Result{}, err
}

// AddOwners adds the owners of MachineSet as k/v pairs to the logger.
// Specifically, it will add MachineDeployment.
ctx, log, err := clog.AddOwners(ctx, r.Client, machineSet)
if err != nil {
return ctrl.Result{}, err
}

log = log.WithValues("Cluster", klog.KRef(machineSet.ObjectMeta.Namespace, machineSet.Spec.ClusterName))
ctx = ctrl.LoggerInto(ctx, log)

cluster, err := util.GetClusterByName(ctx, r.Client, machineSet.ObjectMeta.Namespace, machineSet.Spec.ClusterName)
if err != nil {
return ctrl.Result{}, err
Expand Down Expand Up @@ -188,13 +194,6 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (_ ctrl.Re
return ctrl.Result{}, r.reconcileDelete(ctx, machineSet)
}

// Add finalizer first if not set to avoid the race condition between init and delete.
// Note: Finalizers in general can only be added when the deletionTimestamp is not set.
if !controllerutil.ContainsFinalizer(machineSet, clusterv1.MachineSetFinalizer) {
controllerutil.AddFinalizer(machineSet, clusterv1.MachineSetFinalizer)
return ctrl.Result{}, nil
}

result, err := r.reconcile(ctx, cluster, machineSet)
if err != nil {
// Requeue if the reconcile failed because the ClusterCacheTracker was locked for
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ import (
"sigs.k8s.io/cluster-api/internal/controllers/topology/machineset"
"sigs.k8s.io/cluster-api/util"
"sigs.k8s.io/cluster-api/util/annotations"
"sigs.k8s.io/cluster-api/util/finalizers"
"sigs.k8s.io/cluster-api/util/labels"
"sigs.k8s.io/cluster-api/util/patch"
"sigs.k8s.io/cluster-api/util/predicates"
Expand Down Expand Up @@ -124,6 +125,17 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (_ ctrl.Re
log = log.WithValues("Cluster", klog.KRef(md.Namespace, md.Spec.ClusterName))
ctx = ctrl.LoggerInto(ctx, log)

// Return early if the MachineDeployment is not topology owned.
if !labels.IsTopologyOwned(md) {
log.Info(fmt.Sprintf("Reconciliation is skipped because the MachineDeployment does not have the %q label", clusterv1.ClusterTopologyOwnedLabel))
return ctrl.Result{}, nil
}

// Add finalizer first if not set to avoid the race condition between init and delete.
if finalizerAdded, err := finalizers.EnsureFinalizer(ctx, r.Client, md, clusterv1.MachineDeploymentTopologyFinalizer); err != nil || finalizerAdded {
return ctrl.Result{}, err
}

cluster, err := util.GetClusterByName(ctx, r.Client, md.Namespace, md.Spec.ClusterName)
if err != nil {
return ctrl.Result{}, err
Expand All @@ -135,12 +147,6 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (_ ctrl.Re
return ctrl.Result{}, nil
}

// Return early if the MachineDeployment is not topology owned.
if !labels.IsTopologyOwned(md) {
log.Info(fmt.Sprintf("Reconciliation is skipped because the MachineDeployment does not have the %q label", clusterv1.ClusterTopologyOwnedLabel))
return ctrl.Result{}, nil
}

// Create a patch helper to add or remove the finalizer from the MachineDeployment.
patchHelper, err := patch.NewHelper(md, r.Client)
if err != nil {
Expand All @@ -157,13 +163,6 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (_ ctrl.Re
return ctrl.Result{}, r.reconcileDelete(ctx, md)
}

// Add finalizer first if not set to avoid the race condition between init and delete.
// Note: Finalizers in general can only be added when the deletionTimestamp is not set.
if !controllerutil.ContainsFinalizer(md, clusterv1.MachineDeploymentTopologyFinalizer) {
controllerutil.AddFinalizer(md, clusterv1.MachineDeploymentTopologyFinalizer)
return ctrl.Result{}, nil
}

return ctrl.Result{}, nil
}

Expand Down
Loading

0 comments on commit 1f9ec2e

Please sign in to comment.