From 34202b3213873a655c44e538a3b517a55cce6199 Mon Sep 17 00:00:00 2001 From: fabriziopandini Date: Sun, 13 Oct 2024 13:26:08 +0200 Subject: [PATCH] Refactor ControlPlane contract --- .../v1beta1/kubeadm_control_plane_types.go | 7 +- ...cluster.x-k8s.io_kubeadmcontrolplanes.yaml | 7 +- docs/book/src/SUMMARY.md | 1 - .../src/developer/core/controllers/cluster.md | 8 +- .../core/controllers/control-plane.md | 47 - .../providers/contracts/bootstrap-config.md | 15 +- .../providers/contracts/control-plane.md | 1011 +++++++++++++---- .../providers/contracts/infra-cluster.md | 31 +- .../providers/contracts/infra-machine.md | 15 +- 9 files changed, 842 insertions(+), 300 deletions(-) delete mode 100644 docs/book/src/developer/core/controllers/control-plane.md diff --git a/controlplane/kubeadm/api/v1beta1/kubeadm_control_plane_types.go b/controlplane/kubeadm/api/v1beta1/kubeadm_control_plane_types.go index c920e0481974..46f587036a1f 100644 --- a/controlplane/kubeadm/api/v1beta1/kubeadm_control_plane_types.go +++ b/controlplane/kubeadm/api/v1beta1/kubeadm_control_plane_types.go @@ -271,8 +271,11 @@ type KubeadmControlPlaneStatus struct { // +optional UnavailableReplicas int32 `json:"unavailableReplicas"` - // Initialized denotes whether or not the control plane has the - // uploaded kubeadm-config configmap. + // Initialized denotes that the KubeadmControlPlane API Server is initialized and thus + // it can accept requests. + // NOTE: this field is part of the Cluster API contract and it is used to orchestrate provisioning. + // The value of this field is never updated after provisioning is completed. Please use conditions + // to check the operational state of the control plane. // +optional Initialized bool `json:"initialized"` diff --git a/controlplane/kubeadm/config/crd/bases/controlplane.cluster.x-k8s.io_kubeadmcontrolplanes.yaml b/controlplane/kubeadm/config/crd/bases/controlplane.cluster.x-k8s.io_kubeadmcontrolplanes.yaml index a9180b04fc8a..f47d7da87394 100644 --- a/controlplane/kubeadm/config/crd/bases/controlplane.cluster.x-k8s.io_kubeadmcontrolplanes.yaml +++ b/controlplane/kubeadm/config/crd/bases/controlplane.cluster.x-k8s.io_kubeadmcontrolplanes.yaml @@ -4465,8 +4465,11 @@ spec: type: string initialized: description: |- - Initialized denotes whether or not the control plane has the - uploaded kubeadm-config configmap. + Initialized denotes that the KubeadmControlPlane API Server is initialized and thus + it can accept requests. + NOTE: this field is part of the Cluster API contract and it is used to orchestrate provisioning. + The value of this field is never updated after provisioning is completed. Please use conditions + to check the operational state of the control plane. type: boolean lastRemediation: description: LastRemediation stores info about last remediation performed. diff --git a/docs/book/src/SUMMARY.md b/docs/book/src/SUMMARY.md index 6eab4c8285e1..8bf8fa477298 100644 --- a/docs/book/src/SUMMARY.md +++ b/docs/book/src/SUMMARY.md @@ -78,7 +78,6 @@ - [Machine](./developer/core/controllers/machine.md) - [MachinePool](./developer/core/controllers/machine-pool.md) - [MachineHealthCheck](./developer/core/controllers/machine-health-check.md) - - [Control Plane](./developer/core/controllers/control-plane.md) - [Logging](developer/core/logging.md) - [Testing](developer/core/testing.md) - [Developing E2E tests](developer/core/e2e.md) diff --git a/docs/book/src/developer/core/controllers/cluster.md b/docs/book/src/developer/core/controllers/cluster.md index ac648fd2f10e..046cd39a6f9c 100644 --- a/docs/book/src/developer/core/controllers/cluster.md +++ b/docs/book/src/developer/core/controllers/cluster.md @@ -18,8 +18,12 @@ Among those rules: Similarly, in order to support different solutions for control plane management, The Cluster resource references an ControlPlane object, e.g. KubeadmControlPlane, EKSControlPlane etc. -The [ControlPlane resource contract](../../providers/contracts/control-plane.md) defines a set of rules a provider is expected to comply with in order to allow -the expected interactions with the Cluster controller. +Among those rules: +- ControlPlane SHOULD report a [controlplane endpoint](../../providers/contracts/control-plane.md#controlplane-endpoint) for the Cluster +- ControlPlane MUST report when Cluster's infrastructure is [fully provisioned](../../providers/contracts/control-plane.md#controlplane-initialization-completed) +- ControlPlane MUST manage a [KubeConfig secret](../../providers/contracts/control-plane.md#cluster-kubeconfig-management) +- ControlPlane SHOULD report [conditions](../../providers/contracts/control-plane.md#controlplane-conditions) +- ControlPlane SHOULD report [terminal failures](../../providers/contracts/control-plane.md#controlplane-terminal-failures) Considering all the info above, the Cluster controller's main responsibilities are: diff --git a/docs/book/src/developer/core/controllers/control-plane.md b/docs/book/src/developer/core/controllers/control-plane.md deleted file mode 100644 index 69bd4b6a74c8..000000000000 --- a/docs/book/src/developer/core/controllers/control-plane.md +++ /dev/null @@ -1,47 +0,0 @@ -# Control Plane Controller - -![](../../../images/control-plane-controller.png) - -The Control Plane controller's main responsibilities are: - -* Managing a set of machines that represent a Kubernetes control plane. -* Provide information about the state of the control plane to downstream - consumers. -* Create/manage a secret with the kubeconfig file for accessing the workload cluster. - -A reference implementation is managed within the core Cluster API project as the -Kubeadm control plane controller (`KubeadmControlPlane`). In this document, -we refer to an example `ImplementationControlPlane` where not otherwise specified. - -## Example usage - -```yaml -apiVersion: controlplane.cluster.x-k8s.io/v1beta1 -kind: KubeadmControlPlane -metadata: - name: kcp-1 - namespace: default -spec: - machineTemplate: - infrastructureRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: DockerMachineTemplate - name: docker-machine-template-1 - namespace: default - replicas: 3 - version: v1.21.2 -``` - -[scale]: https://kubernetes.io/docs/tasks/access-kubernetes-api/custom-resources/custom-resource-definitions/#scale-subresource - -## Kubeconfig management - -Control Plane providers are expected to create and maintain a Kubeconfig -secret for operators to gain initial access to the cluster. -The given secret must be labelled with the key-pair `cluster.x-k8s.io/cluster-name=${CLUSTER_NAME}` -to make it stored and retrievable in the cache used by CAPI managers. If a provider uses -client certificates for authentication in these Kubeconfigs, the client -certificate should be kept with a reasonably short expiration period and -periodically regenerated to keep a valid set of credentials available. As an -example, the Kubeadm Control Plane provider uses a year of validity and -refreshes the certificate after 6 months. diff --git a/docs/book/src/developer/providers/contracts/bootstrap-config.md b/docs/book/src/developer/providers/contracts/bootstrap-config.md index 339f5ea7775b..f37bb6482861 100644 --- a/docs/book/src/developer/providers/contracts/bootstrap-config.md +++ b/docs/book/src/developer/providers/contracts/bootstrap-config.md @@ -221,7 +221,10 @@ Each BootstrapConfig MUST report when the bootstrap data secret is fully provisi ```go type FooConfigStatus struct { - // Ready denotes that the foo bootstrap data secret is fully provisioned. + // ready denotes that the foo bootstrap data secret is fully provisioned. + // NOTE: this field is part of the Cluster API contract and it is used to orchestrate provisioning. + // The value of this field is never updated after provisioning is completed. Please use conditions + // to check the operational state of the bootstrap config. // +optional Ready bool `json:"ready"` @@ -297,7 +300,7 @@ setting `status.failureReason` and `status.failureMessage` in the BootstrapConfi ```go type FooConfigStatus struct { - // FailureReason will be set in the event that there is a terminal problem reconciling the FooConfig + // failureReason will be set in the event that there is a terminal problem reconciling the FooConfig // and will contain a succinct value suitable for machine interpretation. // // This field should not be set for transitive errors that can be fixed automatically or with manual intervention, @@ -305,7 +308,7 @@ type FooConfigStatus struct { // +optional FailureReason *capierrors.ClusterStatusError `json:"failureReason,omitempty"` - // FailureMessage will be set in the event that there is a terminal problem reconciling the FooConfig + // failureMessage will be set in the event that there is a terminal problem reconciling the FooConfig // and will contain a more verbose string suitable for logging and human consumption. // // This field should not be set for transitive errors that can be fixed automatically or with manual intervention, @@ -432,6 +435,12 @@ Please, read carefully the page linked above to fully understand implications an The clusterctl command is designed to work with all the providers compliant with the rules defined in the [clusterctl provider contract]. +### BootstrapConfig: pausing + +Providers SHOULD implement the pause behaviour for every object with a reconciliation loop. This is done by checking if `spec.paused` is set on the Cluster object and by checking for the `cluster.x-k8s.io/paused` annotation on the BootstrapConfig object. + +If implementing the pause behavior, providers SHOULD surface the paused status of an object using the Paused condition: `Status.Conditions[Paused]`. + ## Typical BootstrapConfig reconciliation workflow A bootstrap provider must respond to changes to its BootstrapConfig resources. This process is diff --git a/docs/book/src/developer/providers/contracts/control-plane.md b/docs/book/src/developer/providers/contracts/control-plane.md index 036d87a8d923..7fbb335624c0 100644 --- a/docs/book/src/developer/providers/contracts/control-plane.md +++ b/docs/book/src/developer/providers/contracts/control-plane.md @@ -1,52 +1,311 @@ -# Control Plane Provider Specification +# Contract rules for ControlPlane -## Overview +Control plane providers MUST implement an ControlPlane resource. -The general expectation of a control plane controller is to instantiate a -Kubernetes control plane consisting of the following services: +The goal of an ControlPlane resource is to instantiate a Kubernetes control plane; a Kubernetes control plane +at least contains the following components: +- Kubernetes API Server +- Kubernetes Controller Manager +- Kubernetes Scheduler +- etcd (if not externally managed) -#### Required Control Plane Services +Optional control plane components are +- Cloud controller manager +- Cluster DNS (e.g. CoreDNS) +- Service proxy (e.g. kube-proxy) -* etcd -* Kubernetes API Server -* Kubernetes Controller Manager -* Kubernetes Scheduler +Instead, CNI should be left to user to apply once control plane is instantiated. -#### Optional Control Plane Services +The ControlPlane resource will be referenced by one of the Cluster API core resources, Cluster. -* Cloud controller manager -* Cluster DNS (e.g. CoreDNS) -* Service proxy (e.g. kube-proxy) +The [Cluster's controller](../../core/controllers/cluster.md) will be responsible to coordinate operations of the ControlPlane, +and the interaction between the Cluster's controller and the ControlPlane resource is based on the contract +rules defined in this page. -#### Prohibited Services +Once contract rules are satisfied by a ControlPlane implementation, other implementation details +could be addressed according to the specific needs (Cluster API is not prescriptive). -* CNI - should be left to user to apply once control plane is instantiated. +Nevertheless, it is always recommended to take a look at Cluster API controllers, +in-tree providers, other providers and use them as a reference implementation (unless custom solutions are required +in order to address very specific needs). -### Relationship to other Cluster API types +In order to facilitate the initial design for each ControlPlane resource, a few [implementation best practices] +are explicitly called out in dedicated pages. -The Cluster controller will set an OwnerReference on the Control Plane. The Control Plane controller should normally take no action during reconciliation until it sees the ownerReference. +On top of that special consideration MUST be done to ensure +security around private key material required to create and run the Kubernetes control plane. -A Control Plane controller implementation must either supply a controlPlaneEndpoint (via its own `spec.controlPlaneEndpoint` field), -or rely on `spec.controlPlaneEndpoint` in its parent Cluster object. + -* `scale` subresource with the following signature: +## Rules (contract version v1beta1) + +| Rule | Mandatory | Note | +|----------------------------------------------------------------------|-----------|------------------------------------------------------------------------------------------------------------------------------------| +| [All resources: scope] | Yes | | +| [All resources: `TypeMeta` and `ObjectMeta`field] | Yes | | +| [All resources: `APIVersion` field value] | Yes | | +| [ControlPlane, ControlPlaneList resource definition] | Yes | | +| [ControlPlane: endpoint] | No | Mandatory if control plane endpoint is not provided by other means. | +| [ControlPlane: replicas] | No | Mandatory if control plane as a notion of number of instances; also mandatory for cluster class support. | +| [ControlPlane: version] | No | Mandatory if control plane allows direct management of the Kubernetes version in use; Mandatory for cluster class support. | +| [ControlPlane: machines] | No | Mandatory if control plane instances are represented with a set of Cluster API Machines; also mandatory for cluster class support. | +| [ControlPlane: initialization completed] | Yes | | +| [ControlPlane: conditions] | No | | +| [ControlPlane: terminal failures] | No | | +| [ControlPlaneTemplate, ControlPlaneTemplateList resource definition] | No | Mandatory for ClusterClasses support | +| [Cluster kubeconfig management] | Yes | | +| [Cluster certificate management] | No | | +| [Machine placement] | No | | +| [Metadata propagation] | No | | +| [MinReadySeconds and UpToDate propagation] | No | | +| [Support for running multiple instances] | No | Mandatory for clusterctl CLI support | +| [Clusterctl support] | No | Mandatory for clusterctl CLI support | +| [ControlPlane: pausing] | No | | + +### All resources: scope + +All resources MUST be namespace-scoped. + +### All resources: `TypeMeta` and `ObjectMeta` field + +All resources MUST have the standard Kubernetes `TypeMeta` and `ObjectMeta` fields. + +### All resources: `APIVersion` field value + +In Kubernetes `APIVersion` is a combination of API group and version. +Special consideration MUST applies to both API group and version for all the resources Cluster API interacts with. + +#### All resources: API group + +The domain for Cluster API resources is `cluster.x-k8s.io`, and control plane providers under the Kubernetes SIGS org +generally use `controlplane.cluster.x-k8s.io` as API group. + +If your provider uses a different API group, you MUST grant full read/write RBAC permissions for resources in your API group +to the Cluster API core controllers. The canonical way to do so is via a `ClusterRole` resource with the [aggregation label] +`cluster.x-k8s.io/aggregate-to-manager: "true"`. + +The following is an example ClusterRole for a `FooControlPlane` resource in the `controlplane.foo.com` API group: + +```yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: capi-foo-controlplane + labels: + cluster.x-k8s.io/aggregate-to-manager: "true" +rules: +- apiGroups: + - controlplane.foo.com + resources: + - foocontrolplanes + - foocontrolplanetemplates + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +``` + +Note: The write permissions allow the Cluster controller to set owner references and labels on the ControlPlane resources; +when using ClusterClass and managed topologies, also ControlPlaneTemplates are managed directly by Cluster API. + +#### All resources: version + +The resource Version defines the stability of the API and its backward compatibility guarantees. +Examples include `v1alpha1`, `v1beta1`, `v1`, etc. and are governed by the [Kubernetes API Deprecation Policy]. + +Your provider SHOULD abide by the same policies. + +Note: The version of your provider does not need to be in sync with the version of core Cluster API resources. +Instead, prefer choosing a version that matches the stability of the provider API and its backward compatibility guarantees. + +Additionally: + +Providers MUST set `cluster.x-k8s.io/` label on the InfraCluster Custom Resource Definitions. + +The label is a map from a Cluster API contract version to your Custom Resource Definition versions. +The value is an underscore-delimited (_) list of versions. Each value MUST point to an available version in your CRD Spec. + +The label allows Cluster API controllers to perform automatic conversions for object references, the controllers will pick +the last available version in the list if multiple versions are found. + +To apply the label to CRDs it’s possible to use commonLabels in your `kustomize.yaml` file, usually in `config/crd`: + +```yaml +commonLabels: + cluster.x-k8s.io/v1alpha2: v1alpha1 + cluster.x-k8s.io/v1alpha3: v1alpha2 + cluster.x-k8s.io/v1beta1: v1beta1 +``` + +An example of this is in the [Kubeadm Bootstrap provider](https://github.com/kubernetes-sigs/cluster-api/blob/release-1.1/controlplane/kubeadm/config/crd/kustomization.yaml). + +### ControlPlane, ControlPlaneList resource definition + +You MUST define a ControlPlane resource. +The ControlPlane resource name must have the format produced by `sigs.k8s.io/cluster-api/util/contract.CalculateCRDName(Group, Kind)`. + +Note: Cluster API is using such a naming convention to avoid an expensive CRD lookup operation when looking for labels from +the CRD definition of the ControlPlane resource. + +It is a generally applied convention to use names in the format `${env}ControlPlane`, where ${env} is a, possibly short, name +for the control plane implementation in question. For example `KubeadmControlPlane` is an implementation of a control plane +using kubeadm as a bootstrapper tool. + +```go +// +kubebuilder:object:root=true +// +kubebuilder:resource:path=foocontrolplanes,shortName=foocp,scope=Namespaced,categories=cluster-api +// +kubebuilder:storageversion +// +kubebuilder:subresource:status + +// FooControlPlane is the Schema for foocontrolplanes. +type FooControlPlane struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + Spec FooControlPlaneSpec `json:"spec,omitempty"` + Status FooControlPlaneStatus `json:"status,omitempty"` +} + +type FooControlPlaneSpec struct { + // See other rules for more details about mandatory/optional fields in ControlPlane spec. + // Other fields SHOULD be added based on the needs of your provider. +} + +type FooControlPlaneStatus struct { + // See other rules for more details about mandatory/optional fields in ControlPlane status. + // Other fields SHOULD be added based on the needs of your provider. +} +``` + +For each ControlPlane resource, you MUST also add the corresponding list resource. +The list resource MUST be named as `List`. + +```go +// +kubebuilder:object:root=true + +// FooControlPlaneList contains a list of foocontrolplanes. +type FooControlPlaneList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []FooControlPlane `json:"items"` +} +``` + +### ControlPlane: endpoint + +Each Cluster needs a control plane endpoint to sit in front of control plane machines. +Control plane endpoint can be provided in three ways in Cluster API: by the users, by the control plane provider or +by the infrastructure provider. + +In case you are developing a control plane provider which is responsible to provide a control plane endpoint for +each Cluster, the host and port of the generated control plane endpoint MUST surface on `spec.controlPlaneEndpoint` +in the ControlPlane resource. + +```go +type FooControlPlane struct { + // controlPlaneEndpoint represents the endpoint used to communicate with the control plane. + // +optional + ControlPlaneEndpoint APIEndpoint `json:"controlPlaneEndpoint"` + + // See other rules for more details about mandatory/optional fields in ControlPlane spec. + // Other fields SHOULD be added based on the needs of your provider. +} + +// APIEndpoint represents a reachable Kubernetes API endpoint. +type APIEndpoint struct { + // host is the hostname on which the API server is serving. + Host string `json:"host"` + + // port is the port on which the API server is serving. + Port int32 `json:"port"` +} +``` + +Once `spec.controlPlaneEndpoint` is set on the ControlPlane resource and the [ControlPlane: initialization completed], +the Cluster controller will surface this info in Cluster's `spec.controlPlaneEndpoint`. + +If instead you are developing a control plane provider which is NOT responsible to provide a control plane endpoint, +the implementer should exit reconciliation until it sees Cluster's `spec.controlPlaneEndpoint` populated. + +### ControlPlane: replicas + +In case you are developing a control plane provider which allows control of the number of replicas of the +Kubernetes control plane instances in your control plane, following fields MUST be implemented in +the ControlPlane `spec`. + +```go +type FooControlPlaneSpec struct { + // replicas represent the number of desired replicas. + // This is a pointer to distinguish between explicit zero and not specified. + // +optional + Replicas *int32 `json:"replicas,omitempty"` + + // See other rules for more details about mandatory/optional fields in ControlPlane spec. + // Other fields SHOULD be added based on the needs of your provider. +} +``` + +Following fields MUST be implemented in the ControlPlane `status`. + +```go +type FooControlPlaneStatus struct { + // selector is the label selector in string format to avoid introspection + // by clients, and is used to provide the CRD-based integration for the + // scale subresource and additional integrations for things like kubectl + // describe. The string will be in the same format as the query-param syntax. + // More info about label selectors: http://kubernetes.io/docs/user-guide/labels#label-selectors + // +optional + Selector string `json:"selector,omitempty"` + + // replicas is the total number of non-terminated machines targeted by this control plane + // (their labels match the selector). + // +optional + Replicas int32 `json:"replicas"` + + // updatedReplicas is the total number of non-terminated machines targeted by this control plane + // that have the desired template spec. + // +optional + UpdatedReplicas int32 `json:"updatedReplicas"` + + // readyReplicas is the total number of fully running and ready control plane machines. + // +optional + ReadyReplicas int32 `json:"readyReplicas"` + + // unavailableReplicas is the total number of unavailable machines targeted by this control plane. + // This is the total number of machines that are still required for the deployment to have 100% available capacity. + // They may either be machines that are running but not yet ready or machines + // that still have not been created. + // +optional + UnavailableReplicas int32 `json:"unavailableReplicas"` + + // See other rules for more details about mandatory/optional fields in ControlPlane status. + // Other fields SHOULD be added based on the needs of your provider. +} +``` + +As you might have already noticed from the `status.selector` field, the ControlPlane custom resource definition MUST +support the `scale` subresource with the following signature: ``` yaml scale: @@ -59,196 +318,502 @@ status: {} More information about the [scale subresource can be found in the Kubernetes documentation][scale]. -#### Required `spec` fields for implementations using version - -* `version` - is a string representing the Kubernetes version to be used - by the control plane machines. The value must be a valid semantic version; - also if the value provided by the user does not start with the v prefix, it - must be added. - -#### Required `spec` fields for implementations using Machines - -* `machineTemplate` - is a struct containing details of the control plane - machine template. - -* `machineTemplate.metadata` - is a struct containing info about metadata for control plane - machines. - -* `machineTemplate.metadata.labels` - is a map of string keys and values that can be used - to organize and categorize control plane machines. - -* `machineTemplate.metadata.annotations` - is a map of string keys and values containing - arbitrary metadata to be applied to control plane machines. - -* `machineTemplate.infrastructureRef` - is a corev1.ObjectReference to a custom resource - offered by an infrastructure provider. The namespace in the ObjectReference must - be in the same namespace of the control plane object. - -* `machineTemplate.nodeDrainTimeout` - is a *metav1.Duration defining the total amount of time - that the controller will spend on draining a control plane node. - The default value is 0, meaning that the node can be drained without any time limitations. - -* `machineTemplate.nodeVolumeDetachTimeout` - is a *metav1.Duration defining how long the controller - will spend on waiting for all volumes to be detached. - The default value is 0, meaning that the volume can be detached without any time limitations. - -* `machineTemplate.nodeDeletionTimeout` - is a *metav1.Duration defining how long the controller - will attempt to delete the Node that is hosted by a Machine after the Machine is marked for - deletion. A duration of 0 will retry deletion indefinitely. It defaults to 10 seconds on the - Machine. - -#### Optional `spec` fields for implementations providing endpoints - -The `ImplementationControlPlane` object may provide a `spec.controlPlaneEndpoint` field to inform the Cluster -controller where the endpoint is located. - -Implementers might opt to choose the `APIEndpoint` struct exposed by Cluster API types, or the following: - - - - - - - - - - - - - - - - - -
Field Type Description
hostString - The hostname on which the API server is serving. -
portInteger - The port on which the API server is serving. -
- -#### Required `status` fields - -The `ImplementationControlPlane` object **must** have a `status` object. - -The `status` object **must** have the following fields defined: - - - - - - - - - - - - - - - - - - - - -
Field Type Description Implementation in Kubeadm Control Plane Controller
initializedBoolean - a boolean field that is true when the target cluster has - completed initialization such that at least once, the - target's control plane has been contactable. - - Transitions to initialized when the controller detects that kubeadm has uploaded - a kubeadm-config configmap, which occurs at the end of kubeadm provisioning. -
readyBoolean - Ready denotes that the target API Server is ready to receive requests. -
- -#### Required `status` fields for implementations using replicas - -Where the `ImplementationControlPlane` has a concept of replicas, e.g. most -high availability control planes, then the `status` object **must** have the -following fields defined: - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Field Type Description Implementation in Kubeadm Control Plane Controller
readyReplicasIntegerTotal number of fully running and ready control plane instances.Is equal to the number of fully running and ready control plane machines
replicasIntegerTotal number of non-terminated control plane instances, - i.e. the state machine for this instance - of the control plane is able to transition to ready.Is equal to the number of non-terminated control plane machines
selectorString`selector` is the label selector in string format to avoid - introspection by clients, and is used to provide the CRD-based integration - for the scale subresource and additional integrations for things like - kubectl describe. The string will be in the same format as the query-param - syntax. More info about label selectors: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#label-selectors -
unavailableReplicasInteger - Total number of unavailable control plane instances targeted by this control plane, equal - to the desired number of control plane instances - ready instances. - - Total number of unavailable machines targeted by this control - plane. This is the total number of machines that are still required - for the deployment to have 100% available capacity. They may either - be machines that are running but not yet ready or machines that still - have not been created. -
- updatedReplicas - integer - Total number of non-terminated machines targeted by this - control plane that have the desired template spec. - - Total number of non-terminated machines targeted by this - control plane that have the desired template spec. -
- -#### Required `status` fields for implementations using version - -* `version` - is a string representing the minimum Kubernetes version for the - control plane machines in the cluster. - NOTE: The minimum Kubernetes version, and more specifically the API server - version, will be used to determine when a control plane is fully upgraded - (`spec.version == status.version`) and for enforcing [Kubernetes version - skew policies](https://kubernetes.io/releases/version-skew-policy/) in managed topologies. - -#### Optional `status` fields - -The `status` object **may** define several fields: - -* `failureReason` - is a string that explains why an error has occurred, if possible. -* `failureMessage` - is a string that holds the message contained by the error. -* `externalManagedControlPlane` - is a bool that should be set to true if the Node objects do not - exist in the cluster. For example, managed control plane providers for AKS, EKS, GKE, etc, should - set this to `true`. Leaving the field undefined is equivalent to setting the value to `false`. - -Note: once any of `failureReason` or `failureMessage` surface on the cluster who is referencing the control plane object, -they cannot be restored anymore (it is considered a terminal error; the only way to recover is to delete and recreate the cluster). + + +### ControlPlane: version + +In case you are developing a control plane provider which allows control of the version of the +Kubernetes control plane instances in your control plane, following fields MUST be implemented in +the ControlPlane `spec`. + +```go +type FooControlPlaneSpec struct { + // version defines the desired Kubernetes version for the control plane. + // The value must be a valid semantic version; also if the value provided by the user does not start with the v prefix, it + // must be added. + Version string `json:"version"` + + // See other rules for more details about mandatory/optional fields in ControlPlane spec. + // Other fields SHOULD be added based on the needs of your provider. +} +``` + +Following fields MUST be implemented in the ControlPlane `status`. + +```go +type FooControlPlaneStatus struct { + // version represents the minimum Kubernetes version for the control plane machines + // in the cluster. + // +optional + Version *string `json:"version,omitempty"` + + // See other rules for more details about mandatory/optional fields in ControlPlane status. + // Other fields SHOULD be added based on the needs of your provider. +} +``` + +NOTE: The minimum Kubernetes version, and more specifically the API server version, will be used to determine +when a control plane is fully upgraded (spec.version == status.version) and for enforcing Kubernetes version skew +policies when a Cluster derived from a ClusterClass is managed by the Topology controller. + +### ControlPlane: machines + +In case you are developing a control plane provider which uses a Cluster API Machine object to represent each +control plane instance, following fields MUST be implemented in +the ControlPlane `spec`. + +```go +type FooControlPlaneSpec struct { + // machineTemplate contains information about how machines + // should be shaped when creating or updating a control plane. + MachineTemplate FooControlPlaneMachineTemplate `json:"machineTemplate"` + + // See other rules for more details about mandatory/optional fields in ControlPlane spec. + // Other fields SHOULD be added based on the needs of your provider. +} + +type FooControlPlaneMachineTemplate struct { + // Standard object's metadata. + // More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata + // +optional + ObjectMeta clusterv1.ObjectMeta `json:"metadata,omitempty"` + + // infrastructureRef is a required reference to a custom infra machine template resource + // offered by an infrastructure provider. + InfrastructureRef corev1.ObjectReference `json:"infrastructureRef"` + + // nodeDrainTimeout is the total amount of time that the controller will spend on draining a controlplane node + // The default value is 0, meaning that the node can be drained without any time limitations. + // +optional + NodeDrainTimeout *metav1.Duration `json:"nodeDrainTimeout,omitempty"` + + // nodeVolumeDetachTimeout is the total amount of time that the controller will spend on waiting for all volumes + // to be detached. The default value is 0, meaning that the volumes can be detached without any time limitations. + // +optional + NodeVolumeDetachTimeout *metav1.Duration `json:"nodeVolumeDetachTimeout,omitempty"` + + // nodeDeletionTimeout defines how long the machine controller will attempt to delete the Node that the Machine + // hosts after the Machine is marked for deletion. A duration of 0 will retry deletion indefinitely. + // If no value is provided, the default value for this property of the Machine resource will be used. + // +optional + NodeDeletionTimeout *metav1.Duration `json:"nodeDeletionTimeout,omitempty"` + + // Other fields SHOULD be added based on the needs of your provider. +} +``` + +Please note that some of the above fields (`metadata`, `nodeDrainTimeout`, `nodeVolumeDetachTimeout`, `nodeDeletionTimeout`) +must be propagated to machines without triggering rollouts. +See [In place propagation of changes affecting Kubernetes objects only] as well as [Metadata propagation] for more details. + +Additionally, in case you are developing a control plane provider where control plane instances uses a Cluster API Machine +object to represent each control plane instance, but those instances do not show up as a Kubernetes node (for example, +managed control plane providers for AKS, EKS, GKE etc), you SHOULD also implement the following `status` field. + +```go +type FooControlPlaneStatus struct { + // externalManagedControlPlane is a bool that should be set to true if the Node objects do not exist in the cluster. + // +optional + ExternalManagedControlPlane bool `json:"externalManagedControlPlane,omitempty"` + + // See other rules for more details about mandatory/optional fields in ControlPlane status. + // Other fields SHOULD be added based on the needs of your provider. +} +``` + +Please note that by representing each control plane instance as Cluster API machine, each control plane instance +can benefit from several Cluster API behaviours, for example: +- Machine provisioning workflow (in coordination with an InfraMachine and a BootstrapConfig of your choice) +- Machine health checking +- Machine drain and wait for volume detach during deletion + +### ControlPlane: initialization completed + +Each ControlPlane MUST report when the Kubernetes control plane is initialized; usually a control plane is considered +initialized when it can accept requests, no matter if this happens before the control plane is fully provisioned or not. + +For example, in a highly available Kubernetes control plane with three instances of each component, usually the control plane +can be considered initialized after the first instance is up and running. + +A ControlPlane reports when it is initialized by setting `status.initialized` and `status.ready`. + +```go +type FooControlPlaneStatus struct { + // initialized denotes that the foo control plane API Server is initialized and thus + // it can accept requests. + // NOTE: this field is part of the Cluster API contract and it is used to orchestrate provisioning. + // The value of this field is never updated after provisioning is completed. Please use conditions + // to check the operational state of the control plane. + Initialized bool `json:"initialized"` + + // ready denotes that the foo control plane is ready to serve requests. + // NOTE: this field is part of the Cluster API contract and it is used to orchestrate provisioning. + // The value of this field is never updated after provisioning is completed. Please use conditions + // to check the operational state of the control plane. + // +optional + Ready bool `json:"ready"` + + // See other rules for more details about mandatory/optional fields in InfraCluster status. + // Other fields SHOULD be added based on the needs of your provider. +} +``` + +Once `status.initialized` and `status.ready` are set, the Cluster "core" controller will bubbles up those info in +Cluster's `status.controlPlaneReady` field and in the `ControlPlaneInitialized` condition. + +If defined, also InfraCluster's `spec.controlPlaneEndpoint` will be surfaced on Cluster's corresponding fields at the same time. + + + +### ControlPlane: conditions + +According to [Kubernetes API Conventions], Conditions provide a standard mechanism for higher-level +status reporting from a controller. + +Providers implementers SHOULD implement `status.conditions` for their ControlPlane resource. +In case conditions are implemented, Cluster API condition type MUST be used. + +If a condition with type `Ready` exist, such condition will be mirrored in Cluster's `ControlPlaneReady` condition. + +Please note that the `Ready` condition is expected to surface the status of the ControlPlane during its own entire lifecycle, +including initial provisioning, the final deletion process, and the period in between these two moments. + +See [Cluster API condition proposal] for more context. + + + +### ControlPlane: terminal failures + +Each ControlPlane SHOULD report when Cluster's enter in a state that cannot be recovered (terminal failure) by +setting `status.failureReason` and `status.failureMessage` in the ControlPlane resource. + +```go +type FoControlPlaneStatus struct { + // FailureReason will be set in the event that there is a terminal problem reconciling the FooControlPlane + // and will contain a succinct value suitable for machine interpretation. + // + // This field should not be set for transitive errors that can be fixed automatically or with manual intervention, + // but instead indicate that something is fundamentally wrong with the FooCluster and that it cannot be recovered. + // +optional + FailureReason *capierrors.ClusterStatusError `json:"failureReason,omitempty"` + + // FailureMessage will be set in the event that there is a terminal problem reconciling the FooControlPlane + // and will contain a more verbose string suitable for logging and human consumption. + // + // This field should not be set for transitive errors that can be fixed automatically or with manual intervention, + // but instead indicate that something is fundamentally wrong with the FooCluster and that it cannot be recovered. + // +optional + FailureMessage *string `json:"failureMessage,omitempty"` + + // See other rules for more details about mandatory/optional fields in ControlPlane status. + // Other fields SHOULD be added based on the needs of your provider. +} +``` + +Once `status.failureReason` and `status.failureMessage` are set on the ControlPlane resource, the Cluster "core" controller +will surface those info in the corresponding fields in Cluster's `status`. + +Please note that once failureReason/failureMessage is set in Cluster's `status`, the only way to recover is to delete and +recreate the Cluster (it is a terminal failure). + + + +### ControlPlaneTemplate, ControlPlaneTemplateList resource definition + +For a given ControlPlane resource, you should also add a corresponding ControlPlaneTemplate resources in order to use it in ClusterClasses. +The template resource MUST be named as `Template`. + +```go +// +kubebuilder:object:root=true +// +kubebuilder:resource:path=foocontrolplanetemplates,scope=Namespaced,categories=cluster-api +// +kubebuilder:storageversion + +// FooControlPlaneTemplate is the Schema for the fooclustertemplates API. +type FooControlPlaneTemplate struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec FooControlPlaneTemplateSpec `json:"spec,omitempty"` +} + +type FooControlPlaneTemplateSpec struct { + Template FooControlPlaneTemplateResource `json:"template"` +} + +type FooControlPlaneTemplateResource struct { + // Standard object's metadata. + // More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata + // +optional + ObjectMeta clusterv1.ObjectMeta `json:"metadata,omitempty"` + Spec FooControlPlaneSpec `json:"spec"` +} +``` + +NOTE: in this example ControlPlaneTemplate's `spec.template.spec` embeds `FooControlPlaneSpec` from ControlPlane. This might not always be +the best choice depending of if/how ControlPlane's spec fields applies to many clusters vs only one. + +For each ControlPlaneTemplate resource, you MUST also add the corresponding list resource. +The list resource MUST be named as `List`. + +```go +// +kubebuilder:object:root=true + +// FooControlPlaneTemplateList contains a list of FooControlPlaneTemplates. +type FooControlPlaneTemplateList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []FooControlPlaneTemplate `json:"items"` +} +``` + +### Cluster kubeconfig management + +Control Plane providers are expected to create and maintain a Kubeconfig secret for Cluster API to gain access to the +workload cluster. + +Such secret might be used also by operators to gain initial access to the cluster, but this secret MUST not be shared +with other users or applications build on top of Cluster API. Instead, follow instruction in [Certificate Management](https://cluster-api.sigs.k8s.io/tasks/certs/) +to create custom certificates for additional users or other applications. + +The kubeconfig secret MUST: +- Be crated in the same namespace where the Cluster exists +- Be named `-kubeconfig` +- Have type `cluster.x-k8s.io/secret` +- Be labelled with the key-pair `cluster.x-k8s.io/cluster-name=${CLUSTER_NAME}`. + Note: this label is required for the secret to retrievable in the cache used by CAPI managers. + +Important! If a control plane provider uses client certificates for authentication in these Kubeconfigs, the client certificate +MUST be kept with a reasonably short expiration period and periodically regenerated to keep a valid set of credentials available. +As an example, the Kubeadm Control Plane provider uses a year of validity and refreshes the certificate after 6 months. + +### Cluster certificate management + +Control Plane providers are expected to create and maintain all the certificates required to create and run a Kubernetes cluster. + +Cluster certificates MUST be stored as a secrets: +- In the same namespace where the Cluster exists +- Following a naming convention `-`; common certificate names are `ca`, `etcd`, `proxy`, `sa` +- Have type `cluster.x-k8s.io/secret` +- Be labelled with the key-pair `cluster.x-k8s.io/cluster-name=${CLUSTER_NAME}`. + Note: this label is required for the secret to retrievable in the cache used by CAPI managers. + +See [Certificate Management] for more context. + +### Machine placement + +Control Plane providers are expected to place machines in failure domains defined in Cluster's `status.failureDomains` field. + +More specifically, Control Plane should be spread across failure domains specifically flagged to host control plane machines. + +### Metadata propagation + +Cluster API defines rules to propagate metadata (labels and annotations) across the hierarchies of objects, down +to Machines and nodes. + +In order to ensure a nice and consistent user experience across the entire Cluster, also ControlPlane providers +are expected to implement similar propagation rules for control plane machines. + +See. [Metadata propagation rules] for more +details about how metadata should be propagated across the hierarchy of Cluster API objects (use KubeadmControlPlane as a reference). + +Also, please note that `metadata` MUST be propagated to control plane instances machines without triggering rollouts. +See [In place propagation of changes affecting Kubernetes objects only] for more details. + +See. [Label Sync Between Machines and underlying Kubernetes Nodes] for more details about how metadata are +propagated to Kubernetes Nodes. + +### MinReadySeconds and UpToDate propagation + + + +### Support for running multiple instances + +Cluster API does not support running multiples instances of the same provider, which someone can +assume an alternative solution to implement multi tenancy; same applies to the clusterctl CLI. + +See [Support running multiple instances of the same provider] for more context. + +However, if you want to make it possible for users to run multiples instances of your provider, your controller's SHOULD: + +- support the `--namespace` flag. +- support the `--watch-filter` flag. + +Please, read carefully the page linked above to fully understand implications and risks related to this option. + +### Clusterctl support + +The clusterctl command is designed to work with all the providers compliant with the rules defined in the [clusterctl provider contract]. + +### ControlPlane: pausing + +Providers SHOULD implement the pause behaviour for every object with a reconciliation loop. This is done by checking if `spec.paused` is set on the Cluster object and by checking for the `cluster.x-k8s.io/paused` annotation on the ControlPlane object. + +If implementing the pause behavior, providers SHOULD surface the paused status of an object using the Paused condition: `Status.Conditions[Paused]`. + +## Typical ControlPlane reconciliation workflow + +A control plane provider must respond to changes to its ControlPlane resources. This process is +typically called reconciliation. The provider must watch for new, updated, and deleted resources and respond +accordingly. + +As a reference you can look at the following workflow to understand how the typical reconciliation workflow +is implemented in ControlPlane controllers: + +![](../../../images/control-plane-controller.png) + +[All resources: Scope]: #all-resources-scope +[All resources: `TypeMeta` and `ObjectMeta`field]: #all-resources-typemeta-and-objectmeta-field +[All resources: `APIVersion` field value]: #all-resources-apiversion-field-value +[aggregation label]: https://kubernetes.io/docs/reference/access-authn-authz/rbac/#aggregated-clusterroles +[Kubernetes API Deprecation Policy]: https://kubernetes.io/docs/reference/using-api/deprecation-policy/ +[ControlPlane, ControlPlaneList resource definition]: #controlplane-controlplanelist-resource-definition +[ControlPlane: endpoint]: #controlplane-endpoint +[ControlPlane: replicas]: #controlplane-replicas +[scale]: https://kubernetes.io/docs/tasks/extend-kubernetes/custom-resources/custom-resource-definitions/#subresources +[ControlPlane: machines]: #controlplane-machines +[In place propagation of changes affecting Kubernetes objects only]: https://github.com/kubernetes-sigs/cluster-api/blob/main/docs/proposals/20221003-In-place-propagation-of-Kubernetes-objects-only-changes.md +[ControlPlane: version]: #controlplane-version +[ControlPlane: initialization completed]: #controlplane-initialization-completed +[ControlPlane: conditions]: #controlplane-conditions +[Kubernetes API Conventions]: https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md#typical-status-properties +[Cluster API condition proposal]: https://github.com/kubernetes-sigs/cluster-api/blob/main/docs/proposals/20200506-conditions.md +[Improving status in CAPI resources]: https://github.com/kubernetes-sigs/cluster-api/blob/main/docs/proposals/20240916-improve-status-in-CAPI-resources.md +[ControlPlane: terminal failures]: #controlplane-terminal-failures +[ControlPlaneTemplate, ControlPlaneTemplateList resource definition]: #controlplanetemplate-controlplanetemplatelist-resource-definition +[Cluster kubeconfig management]: #cluster-kubeconfig-management +[Certificate Management]: https://cluster-api.sigs.k8s.io/tasks/certs/ +[Cluster certificate management]: #cluster-certificate-management +[Machine placement]: #machine-placement +[Metadata propagation]: #metadata-propagation +[Metadata propagation rules]: https://main.cluster-api.sigs.k8s.io/reference/api/metadata-propagation +[Label Sync Between Machines and underlying Kubernetes Nodes]: https://github.com/kubernetes-sigs/cluster-api/blob/main/docs/proposals/20220927-label-sync-between-machine-and-nodes.md +[MinReadySeconds and UpToDate propagation]: #minreadyseconds-and-uptodate-propagation +[Support for running multiple instances]: #support-for-running-multiple-instances +[Support running multiple instances of the same provider]: ../../core/support-multiple-instances.md +[Clusterctl support]: #clusterctl-support +[ControlPlane: pausing]: #controlplane-pausing +[implementation best practices]: ../best-practices.md diff --git a/docs/book/src/developer/providers/contracts/infra-cluster.md b/docs/book/src/developer/providers/contracts/infra-cluster.md index 6a8698ffa0d9..6dd0b9c8b7d8 100644 --- a/docs/book/src/developer/providers/contracts/infra-cluster.md +++ b/docs/book/src/developer/providers/contracts/infra-cluster.md @@ -218,7 +218,7 @@ in the InfraCluster resource. ```go type FooClusterSpec struct { - // ControlPlaneEndpoint represents the endpoint used to communicate with the control plane. + // controlPlaneEndpoint represents the endpoint used to communicate with the control plane. // +optional ControlPlaneEndpoint APIEndpoint `json:"controlPlaneEndpoint"` @@ -228,10 +228,10 @@ type FooClusterSpec struct { // APIEndpoint represents a reachable Kubernetes API endpoint. type APIEndpoint struct { - // The hostname on which the API server is serving. + // host is the hostname on which the API server is serving. Host string `json:"host"` - // The port on which the API server is serving. + // port is the port on which the API server is serving. Port int32 `json:"port"` } ``` @@ -249,7 +249,7 @@ placed in, the list of available failure domains MUST surface on `status.failure ```go type FooClusterStatus struct { - // FailureDomains is a list of failure domain objects synced from the infrastructure provider. + // failureDomains is a list of failure domain objects synced from the infrastructure provider. FailureDomains clusterv1.FailureDomains `json:"failureDomains,omitempty"` // See other rules for more details about mandatory/optional fields in InfraCluster status. @@ -272,7 +272,10 @@ Each InfraCluster MUST report when Cluster's infrastructure is fully provisioned ```go type FooClusterStatus struct { - // Ready denotes that the foo cluster infrastructure is fully provisioned. + // ready denotes that the foo cluster infrastructure is fully provisioned. + // NOTE: this field is part of the Cluster API contract and it is used to orchestrate provisioning. + // The value of this field is never updated after provisioning is completed. Please use conditions + // to check the operational state of the infa cluster. // +optional Ready bool `json:"ready"` @@ -281,7 +284,7 @@ type FooClusterStatus struct { } ``` -Once `status.ready` the Cluster "core" controller will bubbles up this info in Cluster's `status.infrastructureReady`; +Once `status.ready` is set, the Cluster "core" controller will bubbles up this info in Cluster's `status.infrastructureReady`; If defined, also InfraCluster's `spec.controlPlaneEndpoint` and `status.failureDomains` will be surfaced on Cluster's corresponding fields at the same time. @@ -336,12 +339,6 @@ the implication of this choice which are described both in the document above an -### InfraCluster: pausing - -Providers SHOULD implement the pause behaviour for every object with a reconciliation loop. This is done by checking if `spec.paused` is set on the Cluster object and by checking for the `cluster.x-k8s.io/paused` annotation on the InfraCluster object. - -If implementing the pause behavior, providers SHOULD surface the paused status of an object using the Paused condition: `Status.Conditions[Paused]`. - ### InfraCluster: terminal failures Each InfraCluster SHOULD report when Cluster's enter in a state that cannot be recovered (terminal failure) by @@ -349,7 +346,7 @@ setting `status.failureReason` and `status.failureMessage` in the InfraCluster r ```go type FooClusterStatus struct { - // FailureReason will be set in the event that there is a terminal problem reconciling the FooCluster + // failureReason will be set in the event that there is a terminal problem reconciling the FooCluster // and will contain a succinct value suitable for machine interpretation. // // This field should not be set for transitive errors that can be fixed automatically or with manual intervention, @@ -357,7 +354,7 @@ type FooClusterStatus struct { // +optional FailureReason *capierrors.ClusterStatusError `json:"failureReason,omitempty"` - // FailureMessage will be set in the event that there is a terminal problem reconciling the FooCluster + // failureMessage will be set in the event that there is a terminal problem reconciling the FooCluster // and will contain a more verbose string suitable for logging and human consumption. // // This field should not be set for transitive errors that can be fixed automatically or with manual intervention, @@ -475,6 +472,12 @@ Please, read carefully the page linked above to fully understand implications an The clusterctl command is designed to work with all the providers compliant with the rules defined in the [clusterctl provider contract]. +### InfraCluster: pausing + +Providers SHOULD implement the pause behaviour for every object with a reconciliation loop. This is done by checking if `spec.paused` is set on the Cluster object and by checking for the `cluster.x-k8s.io/paused` annotation on the InfraCluster object. + +If implementing the pause behavior, providers SHOULD surface the paused status of an object using the Paused condition: `Status.Conditions[Paused]`. + ## Typical InfraCluster reconciliation workflow A cluster infrastructure provider must respond to changes to its InfraCluster resources. This process is diff --git a/docs/book/src/developer/providers/contracts/infra-machine.md b/docs/book/src/developer/providers/contracts/infra-machine.md index 88f6492e740a..17d80f9b18b4 100644 --- a/docs/book/src/developer/providers/contracts/infra-machine.md +++ b/docs/book/src/developer/providers/contracts/infra-machine.md @@ -204,7 +204,7 @@ Node's Provider id MUST surface on `spec.providerID` in the InfraMachine resour ```go type FooMachineSpec struct { - // ProviderID must match the provider ID as seen on the node object corresponding to this machine. + // providerID must match the provider ID as seen on the node object corresponding to this machine. // For Kubernetes Nodes running on the Foo provider, this value is set by the corresponding CPI component // and it has the format docker:////. // +optional @@ -244,7 +244,7 @@ new corresponding field (also in status). ```go type FooMachineStatus struct { - // FailureDomain is the unique identifier of the failure domain where this Machine has been placed in. + // failureDomain is the unique identifier of the failure domain where this Machine has been placed in. // For this Foo infrastructure provider, the name is equivalent to the name of one of the available regions. FailureDomain *string `json:"failureDomain,omitempty"` @@ -264,7 +264,7 @@ In case you want to surface machine's addresses, you MUST surface them in `statu ```go type FooMachineStatus struct { - // Addresses contains the associated addresses for the machine. + // addresses contains the associated addresses for the machine. // +optional Addresses []clusterv1.MachineAddress `json:"addresses,omitempty"` @@ -285,7 +285,10 @@ Each InfraMachine MUST report when Machine's infrastructure is fully provisioned ```go type FooMachineStatus struct { - // Ready denotes that the foo machine infrastructure is fully provisioned. + // ready denotes that the foo machine infrastructure is fully provisioned. + // NOTE: this field is part of the Cluster API contract and it is used to orchestrate provisioning. + // The value of this field is never updated after provisioning is completed. Please use conditions + // to check the operational state of the infra machine. // +optional Ready bool `json:"ready"` @@ -356,7 +359,7 @@ setting `status.failureReason` and `status.failureMessage` in the InfraMachine r ```go type FooMachineStatus struct { - // FailureReason will be set in the event that there is a terminal problem reconciling the FooMachine + // failureReason will be set in the event that there is a terminal problem reconciling the FooMachine // and will contain a succinct value suitable for machine interpretation. // // This field should not be set for transitive errors that can be fixed automatically or with manual intervention, @@ -364,7 +367,7 @@ type FooMachineStatus struct { // +optional FailureReason *capierrors.ClusterStatusError `json:"failureReason,omitempty"` - // FailureMessage will be set in the event that there is a terminal problem reconciling the FooMachine + // failureMessage will be set in the event that there is a terminal problem reconciling the FooMachine // and will contain a more verbose string suitable for logging and human consumption. // // This field should not be set for transitive errors that can be fixed automatically or with manual intervention,