[WIP] Add support for dedicated chaincode nodes

See #228

Signed-off-by: James Taylor <[email protected]>

cmd/run/main.go

@@ -9,7 +9,8 @@ import (
 	"github.com/hyperledger-labs/fabric-builder-k8s/internal/builder"
 	"github.com/hyperledger-labs/fabric-builder-k8s/internal/log"
 	"github.com/hyperledger-labs/fabric-builder-k8s/internal/util"
-	"k8s.io/apimachinery/pkg/api/validation"
+	apivalidation "k8s.io/apimachinery/pkg/api/validation"
+	"k8s.io/apimachinery/pkg/util/validation"
 )
 
 const (
@@ -56,6 +57,15 @@ func main() {
 		}
 	}
 
+	kubeNodeRole := util.GetOptionalEnv(util.ChaincodeNodeRoleVariable, "")
+	logger.Debugf("%s=%s", util.ChaincodeNodeRoleVariable, kubeNodeRole)
+
+	// TODO: are valid taint values the same?!
+	if msgs := validation.IsValidLabelValue(kubeNodeRole); len(msgs) > 0 {
+		logger.Printf("The FABRIC_K8S_BUILDER_NODE_ROLE environment variable must be a valid Kubernetes label value: %s", msgs[0])
+		os.Exit(1)
+	}
+
 	kubeServiceAccount := util.GetOptionalEnv(util.ChaincodeServiceAccountVariable, util.DefaultServiceAccountName)
 	logger.Debugf("%s=%s", util.ChaincodeServiceAccountVariable, kubeServiceAccount)
 
@@ -67,7 +77,7 @@ func main() {
 		os.Exit(1)
 	}
 
-	if msgs := validation.NameIsDNS1035Label(kubeNamePrefix, true); len(msgs) > 0 {
+	if msgs := apivalidation.NameIsDNS1035Label(kubeNamePrefix, true); len(msgs) > 0 {
 		logger.Printf("The FABRIC_K8S_BUILDER_OBJECT_NAME_PREFIX environment variable must be a valid DNS-1035 label: %s", msgs[0])
 		os.Exit(1)
 	}
@@ -78,6 +88,7 @@ func main() {
 		PeerID:               peerID,
 		KubeconfigPath:       kubeconfigPath,
 		KubeNamespace:        kubeNamespace,
+		KubeNodeRole:         kubeNodeRole,
 		KubeServiceAccount:   kubeServiceAccount,
 		KubeNamePrefix:       kubeNamePrefix,
 	}

cmd/run/main_test.go

@@ -45,6 +45,28 @@ var _ = Describe("Main", func() {
 		),
 	)
 
+	DescribeTable("Running the run command produces the correct error for invalid FABRIC_K8S_BUILDER_NODE_ROLE environment variable values",
+		func(kubeNodeRoleValue, expectedErrorMessage string) {
+			args := []string{"BUILD_OUTPUT_DIR", "RUN_METADATA_DIR"}
+			command := exec.Command(runCmdPath, args...)
+			command.Env = append(os.Environ(),
+				"CORE_PEER_ID=core-peer-id-abcdefghijklmnopqrstuvwxyz-0123456789",
+				"FABRIC_K8S_BUILDER_NODE_ROLE="+kubeNodeRoleValue,
+			)
+			session, err := gexec.Start(command, GinkgoWriter, GinkgoWriter)
+			Expect(err).NotTo(HaveOccurred())
+
+			Eventually(session).Should(gexec.Exit(1))
+			Eventually(
+				session.Err,
+			).Should(gbytes.Say(expectedErrorMessage))
+		},
+		Entry("When the FABRIC_K8S_BUILDER_NODE_ROLE is too long", "long-node-role-is-looooooooooooooooooooooooooooooooooooooooooong", `run \[\d+\]: The FABRIC_K8S_BUILDER_NODE_ROLE environment variable must be a valid Kubernetes label value: must be no more than 63 characters`),
+		Entry("When the FABRIC_K8S_BUILDER_NODE_ROLE contains invalid characters", "invalid*value", `run \[\d+\]: The FABRIC_K8S_BUILDER_NODE_ROLE environment variable must be a valid Kubernetes label value: a valid label must be an empty string or consist of alphanumeric characters, '-', '_' or '\.', and must start and end with an alphanumeric character`),
+		Entry("When the FABRIC_K8S_BUILDER_NODE_ROLE does not start with an alphanumeric character", ".role", `run \[\d+\]: The FABRIC_K8S_BUILDER_NODE_ROLE environment variable must be a valid Kubernetes label value: a valid label must be an empty string or consist of alphanumeric characters, '-', '_' or '\.', and must start and end with an alphanumeric character`),
+		Entry("When the FABRIC_K8S_BUILDER_NODE_ROLE does not end with an alphanumeric character", "role-", `run \[\d+\]: The FABRIC_K8S_BUILDER_NODE_ROLE environment variable must be a valid Kubernetes label value: a valid label must be an empty string or consist of alphanumeric characters, '-', '_' or '\.', and must start and end with an alphanumeric character`),
+	)
+
 	DescribeTable("Running the run command produces the correct error for invalid FABRIC_K8S_BUILDER_OBJECT_NAME_PREFIX environment variable values",
 		func(kubeNamePrefixValue, expectedErrorMessage string) {
 			args := []string{"BUILD_OUTPUT_DIR", "RUN_METADATA_DIR"}

docs/configuring/dedicated-nodes.md

@@ -0,0 +1,15 @@
+# Dedicated nodes
+
+TBC
+
+The `FABRIC_K8S_BUILDER_NODE_ROLE` environment variable can be used to...
+
+For example, if `FABRIC_K8S_BUILDER_NODE_ROLE` is set to `chaincode`, ... using the following command.
+
+```shell
+kubectl label nodes node1 fabric-builder-k8s-role=chaincode
+kubectl taint nodes node1 fabric-builder-k8s-role=chaincode:NoSchedule
+```
+
+More complex requirements should be handled with Dynamic Admission Control using a Mutating Webhook.
+For example, it looks like the namespace-node-affinity webhook could be used to assign node affinity and tolerations to all pods in the FABRIC_K8S_BUILDER_NAMESPACE namespace.

docs/configuring/overview.md

@@ -14,6 +14,7 @@ External builders are configured in the `core.yaml` file, for example:
         - CORE_PEER_ID
         - FABRIC_K8S_BUILDER_DEBUG
         - FABRIC_K8S_BUILDER_NAMESPACE
+        - FABRIC_K8S_BUILDER_NODE_ROLE
         - FABRIC_K8S_BUILDER_OBJECT_NAME_PREFIX
         - FABRIC_K8S_BUILDER_SERVICE_ACCOUNT
         - KUBERNETES_SERVICE_HOST
@@ -30,6 +31,7 @@ The k8s builder is configured using the following environment variables.
 | ------------------------------------- | -------------------------------- | ---------------------------------------------------- |
 | CORE_PEER_ID                          |                                  | The Fabric peer ID (required)                        |
 | FABRIC_K8S_BUILDER_NAMESPACE          | The peer namespace or `default`  | The Kubernetes namespace to run chaincode with       |
+| FABRIC_K8S_BUILDER_NODE_ROLE          |                                  | TBC                                                  |
 | FABRIC_K8S_BUILDER_OBJECT_NAME_PREFIX | `hlfcc`                          | Eye-catcher prefix for Kubernetes object names       |
 | FABRIC_K8S_BUILDER_SERVICE_ACCOUNT    | `default`                        | The Kubernetes service account to run chaincode with |
 | FABRIC_K8S_BUILDER_DEBUG              | `false`                          | Set to `true` to enable k8s builder debug messages   |

internal/builder/run.go

@@ -16,6 +16,7 @@ type Run struct {
 	PeerID               string
 	KubeconfigPath       string
 	KubeNamespace        string
+	KubeNodeRole         string
 	KubeServiceAccount   string
 	KubeNamePrefix       string
 }

internal/util/env.go

@@ -10,6 +10,7 @@ import (
 const (
 	builderVariablePrefix           = "FABRIC_K8S_BUILDER_"
 	ChaincodeNamespaceVariable      = builderVariablePrefix + "NAMESPACE"
+	ChaincodeNodeRoleVariable       = builderVariablePrefix + "NODE_ROLE"
 	ObjectNamePrefixVariable        = builderVariablePrefix + "OBJECT_NAME_PREFIX"
 	ChaincodeServiceAccountVariable = builderVariablePrefix + "SERVICE_ACCOUNT"
 	DebugVariable                   = builderVariablePrefix + "DEBUG"

internal/util/k8s.go

@@ -232,6 +232,32 @@ func getChaincodeJobSpec(
 					Annotations: annotations,
 				},
 				Spec: apiv1.PodSpec{
+					// TODO make affinity and toleration conditional on/use FABRIC_K8S_BUILDER_NODE_ROLE value
+					Affinity: &apiv1.Affinity{
+						NodeAffinity: &apiv1.NodeAffinity{
+							RequiredDuringSchedulingIgnoredDuringExecution: &apiv1.NodeSelector{
+								NodeSelectorTerms: []apiv1.NodeSelectorTerm{
+									{
+										MatchExpressions: []apiv1.NodeSelectorRequirement{
+											{
+												Key:      "fabric-builder-k8s-role",
+												Operator: apiv1.NodeSelectorOpIn,
+												Values:   []string{"chaincode"},
+											},
+										},
+									},
+								},
+							},
+						},
+					},
+					Tolerations: []apiv1.Toleration{
+						{
+							Key:      "fabric-builder-k8s-role",
+							Operator: apiv1.TolerationOpEqual,
+							Value:    "chaincode",
+							Effect:   apiv1.TaintEffectNoSchedule,
+						},
+					},
 					ServiceAccountName: serviceAccount,
 					Containers: []apiv1.Container{
 						{

mkdocs.yml

@@ -106,6 +106,7 @@ nav:
     - Kubernetes permissions: configuring/kubernetes-permissions.md
     - Kubernetes namespace: configuring/kubernetes-namespace.md
     - Kubernetes service account: configuring/kubernetes-service-account.md
+    - Dedicated nodes: configuring/dedicated-nodes.md
   - Tutorials:
     - Developing and debuging chaincode: tutorials/develop-chaincode.md
     - Creating a chaincode package: tutorials/package-chaincode.md