diff --git a/.github/workflows/kind-e2e.yaml b/.github/workflows/kind-e2e.yaml index 5b54629..5bfe215 100644 --- a/.github/workflows/kind-e2e.yaml +++ b/.github/workflows/kind-e2e.yaml @@ -22,7 +22,7 @@ jobs: - v1.27.x - v1.28.x os: - - ubuntu-20.04 # Ubuntu 20.04 uses cgroup v1 + # - ubuntu-20.04 # Ubuntu 20.04 uses cgroup v1 # TODO: temporarily disabled because of openssl commands incompatibility (1.1.1 version vs 3.x) - ubuntu-22.04 # Ubuntu 22.04 uses cgroup v2 runs-on: ${{ matrix.os }} @@ -51,12 +51,14 @@ jobs: - name: Install k8s-pod-cpu-booster run: | - ko apply -f config/ + make --directory config/ mutating-webhook-certs + kustomize build config/ | ko apply -f - - name: Wait for Ready run: | - echo "Waiting for Pods to become ready" - kubectl wait pod --for=condition=Ready -n pod-cpu-booster -l name=pod-cpu-booster + echo "Waiting for k8s-pod-cpu-booster items to become ready" + kubectl wait pod --for=condition=Ready -n pod-cpu-booster -l app=pod-cpu-boost-reset + kubectl wait pod --for=condition=Ready -n pod-cpu-booster -l app=mutating-webhook sleep 5 # because readiness probe is not accurate (Ready != informer is started), but sleeping is enough for now - name: Run e2e Tests diff --git a/README.md b/README.md index 266052b..5ab8e86 100644 --- a/README.md +++ b/README.md @@ -15,11 +15,14 @@ Between startup and `Ready` status, the container benefits from a CPU boost (x10 ## How does it work? -It is deployed as a controller on every node (with a `DaemonSet`). It listens for every pod update; if a pod has `norbjd.github.io/k8s-pod-cpu-booster-enabled: "true"` label: it boosts the CPU at pod startup, and reset the CPU limit when the pod is ready. +It is deployed in two parts: -The CPU boost can be configured with `norbjd.github.io/k8s-pod-cpu-booster-multiplier` annotation: +- a mutating webhook boosting the CPU of pods with `norbjd.github.io/k8s-pod-cpu-booster-enabled: "true"` label, before they are submitted to k8s API +- a controller listening for every update of pods with `norbjd.github.io/k8s-pod-cpu-booster-enabled: "true"` label; when a pod is ready, it will reset its CPU limit -- if specified, it will increase the CPU limit by `x`, where `x` is the value of the annotation (must be an unsigned integer) +The CPU boost can be configured with `norbjd.github.io/k8s-pod-cpu-booster-multiplier` label: + +- if specified, it will increase the CPU limit by `x`, where `x` is the value of the label (must be an unsigned integer) - if unspecified or invalid, it will increase the CPU limit by the default value (`10`) ## Install @@ -27,7 +30,8 @@ The CPU boost can be configured with `norbjd.github.io/k8s-pod-cpu-booster-multi Use `ko`. Example on a `kind` cluster: ```sh -KO_DOCKER_REPO=kind.local ko apply -f config/ +make --directory config/ mutating-webhook-certs # generates self-signed certificates for the webhook +kustomize build config/ | KO_DOCKER_REPO=kind.local ko apply -f - ``` ## Test/Demo @@ -48,25 +52,26 @@ kind load docker-image python:3.11-alpine Install `k8s-pod-cpu-booster`: ```sh -KO_DOCKER_REPO=kind.local ko apply -f config/ +make --directory config/ mutating-webhook-certs # generates self-signed certificates for the webhook +kustomize build config/ | KO_DOCKER_REPO=kind.local ko apply -f - ``` -Start two similar pods with low CPU limits and running `python -m http.server`, with a readiness probe configured to check when the http server is started. The only differences are the name (obviously), and the annotation `norbjd.github.io/k8s-pod-cpu-booster-enabled`: +Start two similar pods with low CPU limits and running `python -m http.server`, with a readiness probe configured to check when the http server is started. The only differences are the name (obviously), and the label `norbjd.github.io/k8s-pod-cpu-booster-enabled`: ```diff --- examples/pod-no-boost.yaml +++ examples/pod-with-default-boost.yaml @@ -4 +4,3 @@ - name: pod-no-boost -+ annotations: ++ labels: + norbjd.github.io/k8s-pod-cpu-booster-enabled: "true" + name: pod-with-default-boost ``` > [!NOTE] -> The CPU boost multiplier can also be configured (see [`pod-with-small-boost.yaml`](https://github.com/norbjd/k8s-pod-cpu-booster/blob/main/examples/pod-with-small-boost.yaml)) by using the `norbjd.github.io/k8s-pod-cpu-booster-multiplier` annotation. +> The CPU boost multiplier can also be configured (see [`pod-with-small-boost.yaml`](https://github.com/norbjd/k8s-pod-cpu-booster/blob/main/examples/pod-with-small-boost.yaml)) by using the `norbjd.github.io/k8s-pod-cpu-booster-multiplier` label. -As a result, the pod `pod-with-default-boost` (with the annotation) will benefit from a CPU boost, but `pod-no-boost` won't: +As a result, the pod `pod-with-default-boost` (with the label) will benefit from a CPU boost, but `pod-no-boost` won't: ```sh kubectl apply -f examples/pod-no-boost.yaml -f examples/pod-with-default-boost.yaml @@ -101,7 +106,8 @@ Cleanup: ```sh kubectl delete -f examples/pod-no-boost.yaml -f examples/pod-with-default-boost.yaml -KO_DOCKER_REPO=kind.local ko delete -f config/ +kustomize build config/ | KO_DOCKER_REPO=kind.local ko delete -f - +make --directory config/ remove-certs kind delete cluster ``` diff --git a/cmd/main.go b/cmd/informer/main.go similarity index 100% rename from cmd/main.go rename to cmd/informer/main.go diff --git a/cmd/webhook/main.go b/cmd/webhook/main.go new file mode 100644 index 0000000..64818ef --- /dev/null +++ b/cmd/webhook/main.go @@ -0,0 +1,27 @@ +package main + +import ( + "flag" + + "github.com/norbjd/k8s-pod-cpu-booster/pkg/webhook" + "k8s.io/klog/v2" +) + +func main() { + klog.InitFlags(nil) + + var port uint + var pathToCertFile string + var pathToKeyFile string + + flag.UintVar(&port, "port", 8443, "listening port") + flag.StringVar(&pathToCertFile, "cert", "", "path to cert file") + flag.StringVar(&pathToKeyFile, "key", "", "path to key file") + + flag.Parse() + + err := webhook.Run(port, pathToCertFile, pathToKeyFile) + if err != nil { + klog.Fatal(err) + } +} diff --git a/config/.gitignore b/config/.gitignore new file mode 100644 index 0000000..be870b4 --- /dev/null +++ b/config/.gitignore @@ -0,0 +1,2 @@ +*.crt +*.key diff --git a/config/100-namespace.yaml b/config/100-namespace.yaml deleted file mode 100644 index 42afbd7..0000000 --- a/config/100-namespace.yaml +++ /dev/null @@ -1,4 +0,0 @@ -apiVersion: v1 -kind: Namespace -metadata: - name: pod-cpu-booster diff --git a/config/200-rbac.yaml b/config/200-rbac.yaml deleted file mode 100644 index a909bec..0000000 --- a/config/200-rbac.yaml +++ /dev/null @@ -1,33 +0,0 @@ -apiVersion: v1 -kind: ServiceAccount -metadata: - namespace: pod-cpu-booster - name: pod-cpu-booster ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: pod-cpu-booster -rules: -- apiGroups: - - "" - resources: - - pods - verbs: - - list - - watch - - get - - update ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: pod-cpu-booster -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: pod-cpu-booster -subjects: -- kind: ServiceAccount - name: pod-cpu-booster - namespace: pod-cpu-booster diff --git a/config/300-pod-cpu-booster.yaml b/config/300-pod-cpu-booster.yaml deleted file mode 100644 index 14120ab..0000000 --- a/config/300-pod-cpu-booster.yaml +++ /dev/null @@ -1,31 +0,0 @@ -apiVersion: apps/v1 -kind: DaemonSet -metadata: - name: pod-cpu-booster - namespace: pod-cpu-booster -spec: - selector: - matchLabels: - name: pod-cpu-booster - template: - metadata: - labels: - name: pod-cpu-booster - spec: - containers: - - name: pod-cpu-booster - image: ko://github.com/norbjd/k8s-pod-cpu-booster/cmd - env: - - name: NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - resources: - limits: - cpu: 100m - memory: 100Mi - requests: - cpu: 100m - memory: 100Mi - serviceAccountName: pod-cpu-booster - terminationGracePeriodSeconds: 0 # TODO: change for production environments diff --git a/config/GNUmakefile b/config/GNUmakefile new file mode 100644 index 0000000..15abdc3 --- /dev/null +++ b/config/GNUmakefile @@ -0,0 +1,13 @@ +# namespace is used in the certificate's CN and SAN +NAMESPACE := $(shell grep 'namespace: ' kustomization.yaml | cut -d ' ' -f 2) + +mutating-webhook-certs: + openssl genrsa -out ca.key 4096 + openssl req -new -x509 -key ca.key -out ca.crt -days 3650 -nodes -subj "/CN=my-self-signed-ca" + openssl req -x509 -CA ca.crt -CAkey ca.key -keyout tls.key -out tls.crt -sha256 -days 3650 -nodes -subj "/CN=mutating-webhook.${NAMESPACE}.svc" -addext "subjectAltName = DNS:mutating-webhook.${NAMESPACE}.svc" + +remove-certs: + rm -f *.key *.crt + +build: + kustomize build . diff --git a/config/kustomization.yaml b/config/kustomization.yaml new file mode 100644 index 0000000..b8fbbd4 --- /dev/null +++ b/config/kustomization.yaml @@ -0,0 +1,26 @@ +--- +namespace: pod-cpu-booster +resources: + - mutating-webhook.yaml + - namespace.yaml + - pod-cpu-boost-reset.yaml +secretGenerator: + - name: mutating-webhook-certs + options: + disableNameSuffixHash: true + type: kubernetes.io/tls + files: + - ca.crt + - tls.crt + - tls.key +replacements: + - source: + kind: Secret + name: mutating-webhook-certs + fieldPath: "data.[ca.crt]" + targets: + - select: + kind: MutatingWebhookConfiguration + name: k8s-pod-cpu-booster + fieldPaths: + - webhooks.[name=k8s-pod-cpu-booster.norbjd.github.io].clientConfig.caBundle diff --git a/config/mutating-webhook.yaml b/config/mutating-webhook.yaml new file mode 100644 index 0000000..19fc97c --- /dev/null +++ b/config/mutating-webhook.yaml @@ -0,0 +1,75 @@ +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: MutatingWebhookConfiguration +metadata: + name: k8s-pod-cpu-booster +webhooks: + - name: k8s-pod-cpu-booster.norbjd.github.io + clientConfig: + caBundle: "" # will be overridden by kustomization replacement + service: # namespace field is overridden by the namespace defined in kustomization.yaml + name: mutating-webhook + path: /mutate + objectSelector: + matchExpressions: + # we don't want that creation of mutating-webhook pods triggers the webhook (otherwise pods won't start) + - key: app + operator: NotIn + values: + - mutating-webhook + - key: norbjd.github.io/k8s-pod-cpu-booster-enabled + operator: In + values: + - "true" + rules: + - apiGroups: [""] + apiVersions: ["v1"] + resources: ["pods"] + operations: ["CREATE"] + scope: Namespaced + sideEffects: None + admissionReviewVersions: ["v1"] +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: mutating-webhook +spec: + replicas: 3 + selector: + matchLabels: + app: mutating-webhook + template: + metadata: + labels: + app: mutating-webhook + spec: + containers: + - name: mutating-webhook + image: ko://github.com/norbjd/k8s-pod-cpu-booster/cmd/webhook + args: + - -v=9 + - -port=8443 + - -cert=/etc/certs/tls.crt + - -key=/etc/certs/tls.key + ports: + - containerPort: 8443 + volumeMounts: + - name: certs + mountPath: /etc/certs + readOnly: true + volumes: + - name: certs + secret: + secretName: mutating-webhook-certs +--- +apiVersion: v1 +kind: Service +metadata: + name: mutating-webhook +spec: + selector: + app: mutating-webhook + ports: + - port: 443 + targetPort: 8443 diff --git a/config/namespace.yaml b/config/namespace.yaml new file mode 100644 index 0000000..c1b7060 --- /dev/null +++ b/config/namespace.yaml @@ -0,0 +1,5 @@ +--- +apiVersion: v1 +kind: Namespace +metadata: + name: doesnt-matter # will be overridden by kustomize diff --git a/config/pod-cpu-boost-reset.yaml b/config/pod-cpu-boost-reset.yaml new file mode 100644 index 0000000..ee7fdb4 --- /dev/null +++ b/config/pod-cpu-boost-reset.yaml @@ -0,0 +1,59 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: pod-cpu-boost-reset +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: pod-cpu-boost-reset +rules: + - apiGroups: + - "" + resources: + - pods + verbs: + - list + - watch + - get + - update +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: pod-cpu-boost-reset +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: pod-cpu-boost-reset +subjects: + - kind: ServiceAccount + name: pod-cpu-boost-reset +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: pod-cpu-boost-reset +spec: + replicas: 1 # for now, we don't support multiple replicas + selector: + matchLabels: + app: pod-cpu-boost-reset + template: + metadata: + labels: + app: pod-cpu-boost-reset + spec: + containers: + - name: pod-cpu-boost-reset + image: ko://github.com/norbjd/k8s-pod-cpu-booster/cmd/informer + resources: + limits: + cpu: 100m + memory: 100Mi + requests: + cpu: 100m + memory: 100Mi + serviceAccountName: pod-cpu-boost-reset + terminationGracePeriodSeconds: 0 # TODO: change for production environments diff --git a/examples/deployment-with-default-boost.yaml b/examples/deployment-with-default-boost.yaml index 1aad20f..c727698 100644 --- a/examples/deployment-with-default-boost.yaml +++ b/examples/deployment-with-default-boost.yaml @@ -9,9 +9,8 @@ spec: app: deployment-with-default-boost template: metadata: - annotations: - norbjd.github.io/k8s-pod-cpu-booster-enabled: "true" labels: + norbjd.github.io/k8s-pod-cpu-booster-enabled: "true" app: deployment-with-default-boost spec: containers: diff --git a/examples/knative-service.yaml b/examples/knative-service.yaml index 5666fa0..f9d348a 100644 --- a/examples/knative-service.yaml +++ b/examples/knative-service.yaml @@ -10,6 +10,7 @@ spec: queue.sidecar.serving.knative.dev/cpu-resource-limit: "300m" queue.sidecar.serving.knative.dev/memory-resource-request: "10M" queue.sidecar.serving.knative.dev/memory-resource-limit: "10M" + labels: norbjd.github.io/k8s-pod-cpu-booster-enabled: "true" norbjd.github.io/k8s-pod-cpu-booster-container: "user-container" spec: diff --git a/examples/pod-with-default-boost.yaml b/examples/pod-with-default-boost.yaml index d7be3a8..5774b34 100644 --- a/examples/pod-with-default-boost.yaml +++ b/examples/pod-with-default-boost.yaml @@ -1,7 +1,7 @@ apiVersion: v1 kind: Pod metadata: - annotations: + labels: norbjd.github.io/k8s-pod-cpu-booster-enabled: "true" name: pod-with-default-boost spec: diff --git a/examples/pod-with-multiple-containers-and-default-boost.yaml b/examples/pod-with-multiple-containers-and-default-boost.yaml index 6bb06cc..3fd8f0c 100644 --- a/examples/pod-with-multiple-containers-and-default-boost.yaml +++ b/examples/pod-with-multiple-containers-and-default-boost.yaml @@ -1,7 +1,7 @@ apiVersion: v1 kind: Pod metadata: - annotations: + labels: norbjd.github.io/k8s-pod-cpu-booster-enabled: "true" norbjd.github.io/k8s-pod-cpu-booster-container: "python" name: pod-with-multiple-containers-and-default-boost diff --git a/examples/pod-with-small-boost.yaml b/examples/pod-with-small-boost.yaml index 0a9b057..34e6e26 100644 --- a/examples/pod-with-small-boost.yaml +++ b/examples/pod-with-small-boost.yaml @@ -1,7 +1,7 @@ apiVersion: v1 kind: Pod metadata: - annotations: + labels: norbjd.github.io/k8s-pod-cpu-booster-enabled: "true" norbjd.github.io/k8s-pod-cpu-booster-multiplier: "3" name: pod-with-small-boost diff --git a/pkg/informer/informer.go b/pkg/informer/informer.go index 4200b8d..5b085ec 100644 --- a/pkg/informer/informer.go +++ b/pkg/informer/informer.go @@ -4,10 +4,9 @@ import ( "context" "errors" "fmt" - "os" - "strconv" "github.com/google/go-cmp/cmp" + "github.com/norbjd/k8s-pod-cpu-booster/pkg/shared" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -21,13 +20,7 @@ import ( ) const ( - cpuBoostStartupAnnotation = "norbjd.github.io/k8s-pod-cpu-booster-enabled" - - cpuBoostMultiplierAnnotation = "norbjd.github.io/k8s-pod-cpu-booster-multiplier" - cpuBoostDefaultMultiplier = uint64(10) - - cpuBoostContainerNameAnnotation = "norbjd.github.io/k8s-pod-cpu-booster-container" - + cpuBoostStartupLabel = "norbjd.github.io/k8s-pod-cpu-booster-enabled" cpuBoostProgressLabelName = "norbjd.github.io/k8s-pod-cpu-booster-progress" cpuBoostInProgressLabelValue = "boosting" cpuBoostDoneLabelValue = "has-been-boosted" @@ -60,91 +53,43 @@ func Run(clientset *kubernetes.Clientset) { <-stopper } -// only check pods running on the current node, assumes NODE_NAME contains the name of the node -// only necessary if we want to deploy pod-cpu-booster as a DaemonSet, otherwise a simple Deployment with 1 replica (to avoid conflicts) would be enough +// only check pods with the CPU boost label set func podCPUBoosterTweakFunc() internalinterfaces.TweakListOptionsFunc { return func(opts *metav1.ListOptions) { - opts.FieldSelector = "spec.nodeName=" + os.Getenv("NODE_NAME") + opts.LabelSelector = metav1.FormatLabelSelector(&metav1.LabelSelector{ + MatchLabels: map[string]string{ + cpuBoostStartupLabel: "true", + }, + }) } } -func getBoostMultiplierFromAnnotations(pod *corev1.Pod) uint64 { - if boostMultiplierAnnotationValue, ok := pod.Annotations[cpuBoostMultiplierAnnotation]; ok { - boostMultiplierAnnotationValueInt, err := strconv.ParseUint(boostMultiplierAnnotationValue, 10, 64) - if err != nil { - klog.Errorf("boost multiplier is not a valid value, will take the default %d instead: %s", - cpuBoostDefaultMultiplier, err.Error()) - return cpuBoostDefaultMultiplier - } - - return boostMultiplierAnnotationValueInt - } - - return cpuBoostDefaultMultiplier -} - func onUpdate(clientset *kubernetes.Clientset, oldObj interface{}, newObj interface{}) { oldPod := oldObj.(*corev1.Pod) newPod := newObj.(*corev1.Pod) klog.Infof("pod %s/%s updated", newPod.Namespace, newPod.Name) klog.V(9).Info(cmp.Diff(oldPod, newPod)) - if podHasBoostAnnotation(newPod) { - if len(newPod.Status.ContainerStatuses) == 0 { - klog.Infof("pod %s/%s has no container statuses, skipping...", newPod.Namespace, newPod.Name) - return - } - - containerNameToBoost := newPod.Annotations[cpuBoostContainerNameAnnotation] - - containerIndex := -1 - - if containerNameToBoost == "" { - if len(newPod.Spec.Containers) > 1 { - klog.Warningf("pod %s/%s contains %d containers but annotation %s is unset, skipping...", - newPod.Namespace, newPod.Name, len(newPod.Spec.Containers), cpuBoostContainerNameAnnotation) - return - } else { - containerIndex = 0 - } - } else { - for i, container := range newPod.Spec.Containers { - if container.Name == containerNameToBoost { - containerIndex = i - break - } - } - - if containerIndex == -1 { - klog.Warningf("pod %s/%s contains no containers named %s (found in annotation %s), skipping...", - newPod.Namespace, newPod.Name, containerNameToBoost, cpuBoostContainerNameAnnotation) - return - } - } + if len(newPod.Status.ContainerStatuses) == 0 { + klog.Infof("pod %s/%s has no container statuses, skipping...", newPod.Namespace, newPod.Name) + return + } - boostMultiplier := getBoostMultiplierFromAnnotations(newPod) + boostInfo, err := shared.RetrieveBoostInfo(newPod) + if err != nil { + klog.ErrorS(err, "cannot retrieve boost info") + return + } - if podJustStartedAndNotReadyYet(newPod) { - klog.Infof("will boost %s/%s (container %s) CPU limit", newPod.Namespace, newPod.Name, containerNameToBoost) - err := boostCPU(clientset, newPod, containerIndex, boostMultiplier) - if err != nil { - klog.Errorf("error while boosting CPU: %s", err.Error()) - } - } else if podIsNowReadyAfterBoosting(newPod) { - klog.Infof("will reset %s/%s (container %s) CPU limit to default", newPod.Namespace, newPod.Name, containerNameToBoost) - err := resetCPUBoost(clientset, newPod, containerIndex, boostMultiplier) - if err != nil { - klog.Errorf("error while resetting CPU boost: %s", err.Error()) - } + if podIsNowReadyAfterBoosting(newPod) { + klog.Infof("will reset %s/%s (container %s) CPU limit to default", newPod.Namespace, newPod.Name, boostInfo.ContainerName) + err := resetCPUBoost(clientset, newPod, boostInfo.ContainerIndex, boostInfo.Multiplier) + if err != nil { + klog.Errorf("error while resetting CPU boost: %s", err.Error()) } } } -func podHasBoostAnnotation(pod *corev1.Pod) bool { - boost, ok := pod.Annotations[cpuBoostStartupAnnotation] - return ok && boost == "true" -} - func podIsNowReadyAfterBoosting(newPod *corev1.Pod) bool { for _, condition := range newPod.Status.Conditions { if condition.Type == "Ready" && condition.Status == "True" { @@ -161,27 +106,6 @@ func podIsNowReadyAfterBoosting(newPod *corev1.Pod) bool { return false } -func podJustStartedAndNotReadyYet(pod *corev1.Pod) bool { - // we have to wait until it's running before changing the CPU otherwise the behavior is undefined (caught this by experimenting) - return pod.Status.Phase == corev1.PodRunning && pod.Labels[cpuBoostProgressLabelName] == "" -} - -func boostCPU(clientset *kubernetes.Clientset, pod *corev1.Pod, containerIndex int, boostMultiplier uint64) error { - container := pod.Spec.Containers[containerIndex] - currentCPULimit := container.Resources.Limits.Cpu() - cpuLimitAfterBoost := resource.NewScaledQuantity(currentCPULimit.ScaledValue(resource.Nano)*int64(boostMultiplier), resource.Nano) - - klog.Infof("Current CPU limit for %s/%s (container %s) is %s, will set new CPU limit to %s", - pod.Namespace, pod.Name, container.Name, currentCPULimit, cpuLimitAfterBoost) - - err := writeCPULimit(clientset, pod, containerIndex, cpuLimitAfterBoost, boost) - if err != nil { - return err - } - - return nil -} - func resetCPUBoost(clientset *kubernetes.Clientset, pod *corev1.Pod, containerIndex int, boostMultiplier uint64) error { container := pod.Spec.Containers[containerIndex] currentCPULimit := container.Resources.Limits.Cpu() @@ -190,7 +114,7 @@ func resetCPUBoost(clientset *kubernetes.Clientset, pod *corev1.Pod, containerIn klog.Infof("Current CPU limit for %s/%s (container %s) is %s, will reset CPU limit to %s", pod.Namespace, pod.Name, container.Name, currentCPULimit, cpuLimitAfterReset) - err := writeCPULimit(clientset, pod, containerIndex, cpuLimitAfterReset, reset) + err := writeCPULimit(clientset, pod, containerIndex, cpuLimitAfterReset) if err != nil { return err } @@ -198,14 +122,7 @@ func resetCPUBoost(clientset *kubernetes.Clientset, pod *corev1.Pod, containerIn return nil } -type action int32 - -const ( - boost action = iota - reset -) - -func writeCPULimit(clientset *kubernetes.Clientset, pod *corev1.Pod, containerIndex int, cpuLimit *resource.Quantity, action action) error { +func writeCPULimit(clientset *kubernetes.Clientset, pod *corev1.Pod, containerIndex int, cpuLimit *resource.Quantity) error { ctx := context.Background() podsClient := clientset.CoreV1().Pods(pod.Namespace) @@ -215,25 +132,7 @@ func writeCPULimit(clientset *kubernetes.Clientset, pod *corev1.Pod, containerIn return fmt.Errorf("failed to get latest version of pod %s/%s: %v", pod.Namespace, pod.Name, getErr) } - if action == boost && result.Labels[cpuBoostProgressLabelName] == cpuBoostInProgressLabelValue { - klog.Info("Already in boosting process, skipping...") - return nil - } - - switch action { - case boost: - if result.Labels == nil { - result.Labels = make(map[string]string) - } - result.Labels[cpuBoostProgressLabelName] = cpuBoostInProgressLabelValue - case reset: - if result.Labels == nil { - result.Labels = make(map[string]string) - } - result.Labels[cpuBoostProgressLabelName] = cpuBoostDoneLabelValue - default: - return fmt.Errorf("unknown action: %d (expected %d or %d)", action, boost, reset) - } + result.Labels[cpuBoostProgressLabelName] = cpuBoostDoneLabelValue container := result.Spec.Containers[containerIndex] diff --git a/pkg/shared/shared.go b/pkg/shared/shared.go new file mode 100644 index 0000000..147eb93 --- /dev/null +++ b/pkg/shared/shared.go @@ -0,0 +1,79 @@ +package shared + +import ( + "fmt" + "strconv" + + corev1 "k8s.io/api/core/v1" + "k8s.io/klog/v2" +) + +const ( + cpuBoostMultiplierLabel = "norbjd.github.io/k8s-pod-cpu-booster-multiplier" + cpuBoostDefaultMultiplier = uint64(10) + cpuBoostContainerNameLabel = "norbjd.github.io/k8s-pod-cpu-booster-container" +) + +type BoostInfo struct { + ContainerIndex int + ContainerName string + Multiplier uint64 +} + +func RetrieveBoostInfo(pod *corev1.Pod) (BoostInfo, error) { + containerIndex, containerName, err := getContainerToBoost(pod) + if err != nil { + return BoostInfo{}, err + } + + boostMultiplier := getBoostMultiplierFromLabels(pod) + + return BoostInfo{ + ContainerIndex: containerIndex, + ContainerName: containerName, + Multiplier: boostMultiplier, + }, nil +} + +func getBoostMultiplierFromLabels(pod *corev1.Pod) uint64 { + if boostMultiplierLabelValue, ok := pod.Labels[cpuBoostMultiplierLabel]; ok { + boostMultiplierLabelValueInt, err := strconv.ParseUint(boostMultiplierLabelValue, 10, 64) + if err != nil { + klog.Warningf("boost multiplier is not a valid value, will take the default %d instead: %s", + cpuBoostDefaultMultiplier, err.Error()) + return cpuBoostDefaultMultiplier + } + + return boostMultiplierLabelValueInt + } + + return cpuBoostDefaultMultiplier +} + +func getContainerToBoost(pod *corev1.Pod) (index int, name string, err error) { + containerNameToBoost := pod.Labels[cpuBoostContainerNameLabel] + containerIndex := -1 + + if containerNameToBoost == "" { + if len(pod.Spec.Containers) > 1 { + return 0, "", fmt.Errorf("pod %s/%s contains %d containers but label %s is unset", + pod.Namespace, pod.Name, len(pod.Spec.Containers), cpuBoostContainerNameLabel) + } else { + containerIndex = 0 + } + } else { + for i, container := range pod.Spec.Containers { + if container.Name == containerNameToBoost { + containerIndex = i + break + } + } + + if containerIndex == -1 { + return 0, "", fmt.Errorf("pod %s/%s contains no containers named %s (found in label %s)", + pod.Namespace, pod.Name, containerNameToBoost, cpuBoostContainerNameLabel) + } + } + + return containerIndex, containerNameToBoost, nil +} diff --git a/pkg/informer/informer_test.go b/pkg/shared/shared_test.go similarity index 59% rename from pkg/informer/informer_test.go rename to pkg/shared/shared_test.go index adac60e..e299765 100644 --- a/pkg/informer/informer_test.go +++ b/pkg/shared/shared_test.go @@ -1,4 +1,4 @@ -package informer +package shared import ( "fmt" @@ -10,24 +10,24 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) -func Test_getBoostMultiplierFromAnnotations(t *testing.T) { - t.Run("should take the default value if no annotation is provided", func(t *testing.T) { - boostMultiplier := getBoostMultiplierFromAnnotations(&corev1.Pod{ +func Test_getBoostMultiplierFromLabels(t *testing.T) { + t.Run("should take the default value if no label is provided", func(t *testing.T) { + boostMultiplier := getBoostMultiplierFromLabels(&corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ - Annotations: nil, + Labels: nil, }, }) assert.Equal(t, cpuBoostDefaultMultiplier, boostMultiplier) }) - t.Run("should take the value if annotation is valid", func(t *testing.T) { + t.Run("should take the value if label is valid", func(t *testing.T) { notDefaultValue := uint64(5) notDefaultValueString := fmt.Sprintf("%d", notDefaultValue) require.NotEqual(t, cpuBoostDefaultMultiplier, notDefaultValueString, "must not use the default value in that test!") - boostMultiplier := getBoostMultiplierFromAnnotations(&corev1.Pod{ + boostMultiplier := getBoostMultiplierFromLabels(&corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ - Annotations: map[string]string{ + Labels: map[string]string{ "norbjd.github.io/k8s-pod-cpu-booster-multiplier": notDefaultValueString, }, }, @@ -35,10 +35,10 @@ func Test_getBoostMultiplierFromAnnotations(t *testing.T) { assert.Equal(t, notDefaultValue, boostMultiplier) }) - t.Run("should fail if annotation value is invalid", func(t *testing.T) { - boostMultiplier := getBoostMultiplierFromAnnotations(&corev1.Pod{ + t.Run("should fail if label value is invalid", func(t *testing.T) { + boostMultiplier := getBoostMultiplierFromLabels(&corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ - Annotations: map[string]string{ + Labels: map[string]string{ "norbjd.github.io/k8s-pod-cpu-booster-multiplier": "not-a-valid-value", }, }, diff --git a/pkg/webhook/webhook.go b/pkg/webhook/webhook.go new file mode 100644 index 0000000..c4007f9 --- /dev/null +++ b/pkg/webhook/webhook.go @@ -0,0 +1,156 @@ +package webhook + +import ( + "crypto/tls" + "encoding/json" + "fmt" + "io" + "net/http" + + "github.com/norbjd/k8s-pod-cpu-booster/pkg/shared" + admissionv1 "k8s.io/api/admission/v1" + v1 "k8s.io/api/admission/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/serializer" + "k8s.io/klog/v2" +) + +var deserializer = serializer.NewCodecFactory(runtime.NewScheme()).UniversalDeserializer() + +func admissionReviewFromRequest(r *http.Request, deserializer runtime.Decoder) (*admissionv1.AdmissionReview, error) { + if r.Header.Get("Content-Type") != "application/json" { + return nil, fmt.Errorf("expected application/json content-type") + } + + var body []byte + if r.Body != nil { + requestData, err := io.ReadAll(r.Body) + if err != nil { + return nil, err + } + body = requestData + } + + admissionReviewRequest := &admissionv1.AdmissionReview{} + if _, _, err := deserializer.Decode(body, nil, admissionReviewRequest); err != nil { + return nil, err + } + + return admissionReviewRequest, nil +} + +func mutatePod(w http.ResponseWriter, r *http.Request) { + klog.V(9).Infof("received message on mutate") + + admissionReviewRequest, err := admissionReviewFromRequest(r, deserializer) + if err != nil { + msg := "error getting admission review from request" + klog.ErrorS(err, msg) + w.WriteHeader(400) + w.Write([]byte(msg)) + return + } + + // decode the pod from the AdmissionReview + // here, we are **sure** the object is a pod, because this is configured in the MutatingWebhookConfiguration's rules + rawRequest := admissionReviewRequest.Request.Object.Raw + pod := corev1.Pod{} + if _, _, err := deserializer.Decode(rawRequest, nil, &pod); err != nil { + msg := "error decoding raw pod" + klog.ErrorS(err, msg) + w.WriteHeader(500) + w.Write([]byte(msg)) + return + } + + boostInfo, err := shared.RetrieveBoostInfo(&pod) + if err != nil { + klog.ErrorS(err, "cannot get boost info") + w.WriteHeader(400) + w.Write([]byte(err.Error())) + return + } + + currentCPURequest := pod.Spec.Containers[boostInfo.ContainerIndex].Resources.Requests.Cpu() + currentCPULimit := pod.Spec.Containers[boostInfo.ContainerIndex].Resources.Limits.Cpu() + + newCPURequest := resource.NewScaledQuantity(currentCPURequest.ScaledValue(resource.Nano)*int64(boostInfo.Multiplier), resource.Nano) + newCPULimit := resource.NewScaledQuantity(currentCPULimit.ScaledValue(resource.Nano)*int64(boostInfo.Multiplier), resource.Nano) + + admissionResponse := &admissionv1.AdmissionResponse{} + patchType := v1.PatchTypeJSONPatch + patch := fmt.Sprintf(` + [ + { + "op": "add", + "path": "/metadata/labels/norbjd.github.io~1k8s-pod-cpu-booster-progress", + "value": "boosting" + }, + { + "op": "replace", + "path": "/spec/containers/%d/resources/requests/cpu", + "value": "%s" + }, + { + "op": "replace", + "path": "/spec/containers/%d/resources/limits/cpu", + "value": "%s" + } + ] + `, boostInfo.ContainerIndex, newCPURequest.String(), boostInfo.ContainerIndex, newCPULimit.String()) + + podName := pod.Name + if podName == "" { + podName = pod.GenerateName + "" + } + + klog.Infof("Current CPU request/limit for %s/%s (container 0) is %s/%s, will set new CPU limit to %s/%s (boost by %d)", + pod.Namespace, podName, currentCPURequest, currentCPULimit, newCPURequest, newCPULimit, boostInfo.Multiplier) + + admissionResponse.Allowed = true + admissionResponse.PatchType = &patchType + admissionResponse.Patch = []byte(patch) + + // Construct the response, which is just another AdmissionReview. + var admissionReviewResponse admissionv1.AdmissionReview + admissionReviewResponse.Response = admissionResponse + admissionReviewResponse.SetGroupVersionKind(admissionReviewRequest.GroupVersionKind()) + admissionReviewResponse.Response.UID = admissionReviewRequest.Request.UID + + resp, err := json.Marshal(admissionReviewResponse) + if err != nil { + msg := "error marshalling response json" + klog.ErrorS(err, msg) + w.WriteHeader(500) + w.Write([]byte(msg)) + return + } + + w.Header().Set("Content-Type", "application/json") + w.Write(resp) +} + +func Run(port uint, certFile, keyFile string) error { + cert, errLoadCert := tls.LoadX509KeyPair(certFile, keyFile) + if errLoadCert != nil { + return errLoadCert + } + + klog.Info("Starting webhook server") + http.HandleFunc("/mutate", mutatePod) + server := http.Server{ + Addr: fmt.Sprintf(":%d", port), + TLSConfig: &tls.Config{ + Certificates: []tls.Certificate{cert}, + }, + ErrorLog: klog.NewStandardLogger("INFO"), + } + + if err := server.ListenAndServeTLS("", ""); err != nil { + return err + } + + return nil +} diff --git a/test/e2e-kind.sh b/test/e2e-kind.sh index dbed6db..a8de0be 100755 --- a/test/e2e-kind.sh +++ b/test/e2e-kind.sh @@ -128,9 +128,14 @@ else # cgroup v2 fi fi -echo "Pod-cpu-booster logs" +echo "mutating-webhook logs" echo "====================" -kubectl logs --tail=-1 -n pod-cpu-booster -l name=pod-cpu-booster +kubectl logs --tail=-1 -n pod-cpu-booster -l app=mutating-webhook --prefix +echo "====================" + +echo "pod-cpu-boost-reset logs" +echo "====================" +kubectl logs --tail=-1 -n pod-cpu-booster -l app=pod-cpu-boost-reset --prefix echo "====================" kubectl delete \ diff --git a/test/e2e/deployment-with-default-boost.yaml b/test/e2e/deployment-with-default-boost.yaml index 1aad20f..c727698 100644 --- a/test/e2e/deployment-with-default-boost.yaml +++ b/test/e2e/deployment-with-default-boost.yaml @@ -9,9 +9,8 @@ spec: app: deployment-with-default-boost template: metadata: - annotations: - norbjd.github.io/k8s-pod-cpu-booster-enabled: "true" labels: + norbjd.github.io/k8s-pod-cpu-booster-enabled: "true" app: deployment-with-default-boost spec: containers: diff --git a/test/e2e/pod-with-default-boost.yaml b/test/e2e/pod-with-default-boost.yaml index d7be3a8..5774b34 100644 --- a/test/e2e/pod-with-default-boost.yaml +++ b/test/e2e/pod-with-default-boost.yaml @@ -1,7 +1,7 @@ apiVersion: v1 kind: Pod metadata: - annotations: + labels: norbjd.github.io/k8s-pod-cpu-booster-enabled: "true" name: pod-with-default-boost spec: