[kube-state-metrics] Allow to enable kube-rbac-proxy for kube-state-m…

…etric (#2888) [kube-state-metrics] Allow to enable kube-rbac-proxy for kube-state-metric (#2888) Signed-off-by: Dorian Jolivald <[email protected]> Co-authored-by: David Calvert <[email protected]>
prometheus-community · Jan 10, 2023 · 206479a · 206479a
1 parent 9754539
commit 206479a
Show file tree

Hide file tree

Showing 7 changed files with 195 additions and 16 deletions.
diff --git a/charts/kube-state-metrics/Chart.yaml b/charts/kube-state-metrics/Chart.yaml
@@ -7,7 +7,7 @@ keywords:
 - prometheus
 - kubernetes
 type: application
-version: 4.27.0
+version: 4.28.0
 appVersion: 2.7.0
 home: https://github.com/kubernetes/kube-state-metrics/
 sources:

diff --git a/charts/kube-state-metrics/README.md b/charts/kube-state-metrics/README.md
@@ -2,14 +2,15 @@
 
 Installs the [kube-state-metrics agent](https://github.com/kubernetes/kube-state-metrics).
 
-## Get Repo Info
-
+## Get Repository Info
+<!-- textlint-disable -->
 ```console
 helm repo add prometheus-community https://prometheus-community.github.io/helm-charts
 helm repo update
 ```
 
 _See [helm repo](https://helm.sh/docs/helm/helm_repo/) for command documentation._
+<!-- textlint-enable -->
 
 ## Install Chart
 
@@ -43,20 +44,19 @@ _See [helm upgrade](https://helm.sh/docs/helm/helm_upgrade/) for command documen
 
 You can upgrade in-place:
 
-1. [get repo info](#get-repo-info)
-1. [upgrade](#upgrading-chart) your existing release name using the new chart repo
-
+1. [get repository info](#get-repository-info)
+1. [upgrade](#upgrading-chart) your existing release name using the new chart repository
 
 ## Upgrading to v3.0.0
 
 v3.0.0 includes kube-state-metrics v2.0, see the [changelog](https://github.com/kubernetes/kube-state-metrics/blob/release-2.0/CHANGELOG.md) for major changes on the application-side.
 
 The upgraded chart now the following changes:
+
 * Dropped support for helm v2 (helm v3 or later is required)
 * collectors key was renamed to resources
 * namespace key was renamed to namespaces
 
-
 ## Configuration
 
 See [Customizing the Chart Before Installing](https://helm.sh/docs/intro/using_helm/#customizing-the-chart-before-installing). To see all configurable options with detailed comments:
@@ -65,4 +65,21 @@ See [Customizing the Chart Before Installing](https://helm.sh/docs/intro/using_h
 helm show values prometheus-community/kube-state-metrics
 ```
 
-You may also run `helm show values` on this chart's [dependencies](#dependencies) for additional options.
+### kube-rbac-proxy
+
+You can enable `kube-state-metrics` endpoint protection using `kube-rbac-proxy`. By setting `kubeRBACProxy.enabled: true`, this chart will deploy one RBAC proxy container per endpoint (metrics & telemetry).
+To authorize access, authenticate your requests (via a `ServiceAccount` for example) with a `ClusterRole` attached such as:
+
+```yaml
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: kube-state-metrics-read
+rules:
+  - apiGroups: [ "" ]
+    resources: ["services/kube-state-metrics"]
+    verbs:
+      - get
+```
+
+See [kube-rbac-proxy examples](https://github.com/brancz/kube-rbac-proxy/tree/master/examples/resource-attributes) for more details.
diff --git a/charts/kube-state-metrics/templates/NOTES.txt b/charts/kube-state-metrics/templates/NOTES.txt
@@ -8,3 +8,16 @@ In your case, {{ template "kube-state-metrics.fullname" . }}.{{ template "kube-s
 They are served either as plaintext or protobuf depending on the Accept header.
 They are designed to be consumed either by Prometheus itself or by a scraper that is compatible with scraping a Prometheus client endpoint.
 
+{{- if .Values.kubeRBACProxy.enabled}}
+
+kube-rbac-proxy endpoint protections is enabled:
+- Metrics endpoints are now HTTPS
+- Ensure that the client authenticates the requests (e.g. via service account) with the following role permissions:
+```
+rules:
+  - apiGroups: [ "" ]
+    resources: ["services/{{ template "kube-state-metrics.fullname" . }}"]
+    verbs:
+      - get
+```
+{{- end }}
diff --git a/charts/kube-state-metrics/templates/deployment.yaml b/charts/kube-state-metrics/templates/deployment.yaml
@@ -40,6 +40,8 @@ spec:
       priorityClassName: {{ .Values.priorityClassName }}
     {{- end }}
       containers:
+      {{- $httpPort := ternary 9090 (.Values.service.port | default 8080) .Values.kubeRBACProxy.enabled}}
+      {{- $telemetryPort := ternary 9091 (.Values.selfMonitor.telemetryPort | default 8081) .Values.kubeRBACProxy.enabled}}
       - name: {{ template "kube-state-metrics.name" . }}
         {{- if .Values.autosharding.enabled }}
         env:
@@ -56,9 +58,7 @@ spec:
         {{-  if .Values.extraArgs  }}
         {{- .Values.extraArgs | toYaml | nindent 8 }}
         {{-  end  }}
-        {{- if .Values.service.port }}
-        - --port={{ .Values.service.port | default 8080}}
-        {{-  end  }}
+        - --port={{ $httpPort }}
         {{-  if .Values.collectors  }}
         - --resources={{ .Values.collectors | join "," }}
         {{-  end  }}
@@ -96,11 +96,16 @@ spec:
         {{- if .Values.kubeconfig.enabled }}
         - --kubeconfig=/opt/k8s/.kube/config
         {{- end }}
+        {{- if .Values.kubeRBACProxy.enabled }}
+        - --telemetry-host=127.0.0.1
+        - --telemetry-port={{ $telemetryPort }}
+        {{- else }}
         {{- if .Values.selfMonitor.telemetryHost }}
         - --telemetry-host={{ .Values.selfMonitor.telemetryHost }}
         {{- end }}
         {{- if .Values.selfMonitor.telemetryPort }}
-        - --telemetry-port={{ .Values.selfMonitor.telemetryPort | default 8081 }}
+        - --telemetry-port={{ $telemetryPort }}
+        {{- end }}
         {{- end }}
         {{- if or (.Values.kubeconfig.enabled) (.Values.volumeMounts) }}
         volumeMounts:
@@ -119,23 +124,25 @@ spec:
         {{- else }}
         image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
         {{- end }}
+        {{- if eq .Values.kubeRBACProxy.enabled false }}
         ports:
         - containerPort: {{ .Values.service.port | default 8080}}
           name: "http"
         {{- if .Values.selfMonitor.enabled }}
-        - containerPort: {{ .Values.selfMonitor.telemetryPort | default 8081 }}
+        - containerPort: {{ $telemetryPort }}
           name: "metrics"
         {{- end }}
+        {{- end }}
         livenessProbe:
           httpGet:
             path: /healthz
-            port: {{ .Values.service.port | default 8080}}
+            port: {{ $httpPort }}
           initialDelaySeconds: 5
           timeoutSeconds: 5
         readinessProbe:
           httpGet:
             path: /
-            port: {{ .Values.service.port | default 8080}}
+            port: {{ $httpPort }}
           initialDelaySeconds: 5
           timeoutSeconds: 5
         {{- if .Values.resources }}
@@ -146,6 +153,86 @@ spec:
         securityContext:
 {{ toYaml .Values.containerSecurityContext | indent 10 }}
 {{- end }}
+        {{-  if .Values.kubeRBACProxy.enabled  }}
+      - name: kube-rbac-proxy-http
+        args:
+        {{-  if .Values.kubeRBACProxy.extraArgs  }}
+        {{- .Values.kubeRBACProxy.extraArgs | toYaml | nindent 8 }}
+        {{-  end  }}
+        - --secure-listen-address=:{{ .Values.service.port | default 8080}}
+        - --upstream=http://127.0.0.1:{{ $httpPort }}/
+        - --proxy-endpoints-port=8888
+        - --config-file=/etc/kube-rbac-proxy-config/config-file.yaml
+        volumeMounts:
+          - name: kube-rbac-proxy-config
+            mountPath: /etc/kube-rbac-proxy-config
+        imagePullPolicy: {{ .Values.kubeRBACProxy.image.pullPolicy }}
+        {{- if .Values.kubeRBACProxy.image.sha }}
+        image: "{{ .Values.kubeRBACProxy.image.repository }}:{{ .Values.kubeRBACProxy.image.tag }}@sha256:{{ .Values.kubeRBACProxy.image.sha }}"
+        {{- else }}
+        image: "{{ .Values.kubeRBACProxy.image.repository }}:{{ .Values.kubeRBACProxy.image.tag }}"
+        {{- end }}
+        ports:
+          - containerPort: {{ .Values.service.port | default 8080}}
+            name: "http"
+          - containerPort: 8888
+            name: "http-healthz"
+        readinessProbe:
+          httpGet:
+            scheme: HTTPS
+            port: 8888
+            path: healthz
+          initialDelaySeconds: 5
+          timeoutSeconds: 5
+        {{- if .Values.kubeRBACProxy.resources }}
+        resources:
+{{ toYaml .Values.kubeRBACProxy.resources | indent 10 }}
+{{- end }}
+{{- if .Values.kubeRBACProxy.containerSecurityContext }}
+        securityContext:
+{{ toYaml .Values.kubeRBACProxy.containerSecurityContext | indent 10 }}
+{{- end }}
+      {{-  if .Values.selfMonitor.enabled  }}
+      - name: kube-rbac-proxy-telemetry
+        args:
+        {{-  if .Values.kubeRBACProxy.extraArgs  }}
+        {{- .Values.kubeRBACProxy.extraArgs | toYaml | nindent 8 }}
+        {{-  end  }}
+        - --secure-listen-address=:{{ .Values.selfMonitor.telemetryPort | default 8081 }}
+        - --upstream=http://127.0.0.1:{{ $telemetryPort }}/
+        - --proxy-endpoints-port=8889
+        - --config-file=/etc/kube-rbac-proxy-config/config-file.yaml
+        volumeMounts:
+          - name: kube-rbac-proxy-config
+            mountPath: /etc/kube-rbac-proxy-config
+        imagePullPolicy: {{ .Values.kubeRBACProxy.image.pullPolicy }}
+        {{- if .Values.kubeRBACProxy.image.sha }}
+        image: "{{ .Values.kubeRBACProxy.image.repository }}:{{ .Values.kubeRBACProxy.image.tag }}@sha256:{{ .Values.kubeRBACProxy.image.sha }}"
+        {{- else }}
+        image: "{{ .Values.kubeRBACProxy.image.repository }}:{{ .Values.kubeRBACProxy.image.tag }}"
+        {{- end }}
+        ports:
+          - containerPort: {{ .Values.selfMonitor.telemetryPort | default 8081 }}
+            name: "metrics"
+          - containerPort: 8889
+            name: "metrics-healthz"
+        readinessProbe:
+          httpGet:
+            scheme: HTTPS
+            port: 8889
+            path: healthz
+          initialDelaySeconds: 5
+          timeoutSeconds: 5
+        {{- if .Values.kubeRBACProxy.resources }}
+        resources:
+{{ toYaml .Values.kubeRBACProxy.resources | indent 10 }}
+{{- end }}
+{{- if .Values.kubeRBACProxy.containerSecurityContext }}
+        securityContext:
+{{ toYaml .Values.kubeRBACProxy.containerSecurityContext | indent 10 }}
+{{- end }}
+      {{- end }}
+      {{- end }}
 {{- if or .Values.imagePullSecrets .Values.global.imagePullSecrets }}
       imagePullSecrets:
         {{- include "kube-state-metrics.imagePullSecrets" (dict "Values" .Values "imagePullSecrets" .Values.imagePullSecrets) | indent 8 }}
@@ -166,13 +253,18 @@ spec:
       topologySpreadConstraints:
 {{ toYaml .Values.topologySpreadConstraints | indent 8 }}
       {{- end }}
-      {{- if or (.Values.kubeconfig.enabled) (.Values.volumes) }}
+      {{- if or (.Values.kubeconfig.enabled) (.Values.volumes) (.Values.kubeRBACProxy.enabled) }}
       volumes:
       {{- if .Values.kubeconfig.enabled}}
         - name: kubeconfig
           secret:
             secretName: {{ template "kube-state-metrics.fullname" . }}-kubeconfig
       {{- end }}
+      {{- if .Values.kubeRBACProxy.enabled}}
+        - name: kube-rbac-proxy-config
+          configMap:
+            name: {{ template "kube-state-metrics.fullname" . }}-rbac-config
+      {{- end }}
       {{- if .Values.volumes }}
 {{ toYaml .Values.volumes | indent 8 }}
       {{- end }}

diff --git a/charts/kube-state-metrics/templates/rbac-configmap.yaml b/charts/kube-state-metrics/templates/rbac-configmap.yaml
@@ -0,0 +1,15 @@
+{{- if .Values.kubeRBACProxy.enabled}}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ template "kube-state-metrics.fullname" . }}-rbac-config
+data:
+  config-file.yaml: |+
+    authorization:
+      resourceAttributes:
+        namespace: {{ template "kube-state-metrics.namespace" . }}
+        apiVersion: v1
+        resource: services
+        subresource: {{ template "kube-state-metrics.fullname" . }}
+        name: {{ template "kube-state-metrics.fullname" . }}
+{{- end }}
diff --git a/charts/kube-state-metrics/templates/role.yaml b/charts/kube-state-metrics/templates/role.yaml
@@ -189,6 +189,16 @@ rules:
     - verticalpodautoscalers
   verbs: ["list", "watch"]
 {{ end -}}
+{{-  if $.Values.kubeRBACProxy.enabled  }}
+- apiGroups: ["authentication.k8s.io"]
+  resources:
+    - tokenreviews
+  verbs: ["create"]
+- apiGroups: ["authorization.k8s.io"]
+  resources:
+    - subjectaccessreviews
+  verbs: ["create"]
+{{- end }}
 {{ if $.Values.rbac.extraRules }}
 {{ toYaml $.Values.rbac.extraRules }}
 {{ end }}

diff --git a/charts/kube-state-metrics/values.yaml b/charts/kube-state-metrics/values.yaml
@@ -77,6 +77,38 @@ rbac:
   #   verbs: ["list", "watch"]
   extraRules: []
 
+# Configure kube-rbac-proxy. When enabled, creates one kube-rbac-proxy container per exposed HTTP endpoint (metrics and telemetry if enabled).
+# The requests are served through the same service but requests are then HTTPS.
+kubeRBACProxy:
+  enabled: false
+  image:
+    repository: quay.io/brancz/kube-rbac-proxy
+    tag: v0.14.0
+    sha: ""
+    pullPolicy: IfNotPresent
+
+  # List of additional cli arguments to configure kube-rbac-prxy
+  # for example: --tls-cipher-suites, --log-file, etc.
+  # all the possible args can be found here: https://github.com/brancz/kube-rbac-proxy#usage
+  extraArgs: []
+
+  ## Specify security settings for a Container
+  ## Allows overrides and additional options compared to (Pod) securityContext
+  ## Ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-container
+  containerSecurityContext: {}
+
+  resources: {}
+    # We usually recommend not to specify default resources and to leave this as a conscious
+    # choice for the user. This also increases chances charts run on environments with little
+    # resources, such as Minikube. If you do want to specify resources, uncomment the following
+    # lines, adjust them as necessary, and remove the curly braces after 'resources:'.
+    # limits:
+    #  cpu: 100m
+    #  memory: 64Mi
+    # requests:
+    #  cpu: 10m
+    #  memory: 32Mi
+
 serviceAccount:
   # Specifies whether a ServiceAccount should be created, require rbac true
   create: true