e2e: use grafana alloy instead of fluent-bit (envoyproxy#4525)

* use grafana alloy instead of fluent-bit Signed-off-by: zirain <[email protected]> * make alloy disabled by default Signed-off-by: zirain <[email protected]> * enable alloy in e2e Signed-off-by: zirain <[email protected]> --------- Signed-off-by: zirain <[email protected]>
guydc · Nov 5, 2024 · 3191d49 · 3191d49
1 parent 2d68e6a
commit 3191d49
Show file tree

Hide file tree

Showing 10 changed files with 405 additions and 149 deletions.
diff --git a/charts/gateway-addons-helm/Chart.lock b/charts/gateway-addons-helm/Chart.lock
@@ -8,6 +8,9 @@ dependencies:
 - name: fluent-bit
   repository: https://fluent.github.io/helm-charts
   version: 0.30.4
+- name: alloy
+  repository: https://grafana.github.io/helm-charts
+  version: 0.9.2
 - name: loki
   repository: https://grafana.github.io/helm-charts
   version: 4.8.0
@@ -17,5 +20,5 @@ dependencies:
 - name: opentelemetry-collector
   repository: https://open-telemetry.github.io/opentelemetry-helm-charts
   version: 0.108.0
-digest: sha256:ea6663bb1358123b96b69d2c5b0b8c20650a43dc39b24c482f0560201fd2cc3a
-generated: "2024-10-19T12:59:47.251089661+02:00"
+digest: sha256:bc634c59972bfd4a01e0f4310a4949095752e659a9b5cb1d9c0fbe9a86f37011
+generated: "2024-10-25T10:55:26.755739+08:00"
diff --git a/charts/gateway-addons-helm/Chart.yaml b/charts/gateway-addons-helm/Chart.yaml
@@ -37,6 +37,10 @@ dependencies:
     repository: https://fluent.github.io/helm-charts
     version: 0.30.4
     condition: fluent-bit.enabled
+  - name: alloy
+    repository: https://grafana.github.io/helm-charts
+    version: 0.9.2
+    condition: alloy.enabled
   - name: loki
     version: 4.8.0
     repository: https://grafana.github.io/helm-charts

diff --git a/charts/gateway-addons-helm/README.md b/charts/gateway-addons-helm/README.md
@@ -22,6 +22,7 @@ An Add-ons Helm chart for Envoy Gateway
 | Repository | Name | Version |
 |------------|------|---------|
 | https://fluent.github.io/helm-charts | fluent-bit | 0.30.4 |
+| https://grafana.github.io/helm-charts | alloy | 0.9.2 |
 | https://grafana.github.io/helm-charts | grafana | 8.0.0 |
 | https://grafana.github.io/helm-charts | loki | 4.8.0 |
 | https://grafana.github.io/helm-charts | tempo | 1.3.1 |
@@ -55,6 +56,9 @@ To uninstall the chart:
 
 | Key | Type | Default | Description |
 |-----|------|---------|-------------|
+| alloy.alloy.configMap.content | string | `"// Write your Alloy config here:\nlogging {\n  level = \"info\"\n  format = \"logfmt\"\n}\nloki.write \"alloy\" {\n  endpoint {\n    url = \"http://loki.monitoring.svc:3100/loki/api/v1/push\"\n  }\n}\n// discovery.kubernetes allows you to find scrape targets from Kubernetes resources.\n// It watches cluster state and ensures targets are continually synced with what is currently running in your cluster.\ndiscovery.kubernetes \"pod\" {\n  role = \"pod\"\n}\n\n// discovery.relabel rewrites the label set of the input targets by applying one or more relabeling rules.\n// If no rules are defined, then the input targets are exported as-is.\ndiscovery.relabel \"pod_logs\" {\n  targets = discovery.kubernetes.pod.targets\n\n  // Label creation - \"namespace\" field from \"__meta_kubernetes_namespace\"\n  rule {\n    source_labels = [\"__meta_kubernetes_namespace\"]\n    action = \"replace\"\n    target_label = \"namespace\"\n  }\n\n  // Label creation - \"pod\" field from \"__meta_kubernetes_pod_name\"\n  rule {\n    source_labels = [\"__meta_kubernetes_pod_name\"]\n    action = \"replace\"\n    target_label = \"pod\"\n  }\n\n  // Label creation - \"container\" field from \"__meta_kubernetes_pod_container_name\"\n  rule {\n    source_labels = [\"__meta_kubernetes_pod_container_name\"]\n    action = \"replace\"\n    target_label = \"container\"\n  }\n\n  // Label creation -  \"app\" field from \"__meta_kubernetes_pod_label_app_kubernetes_io_name\"\n  rule {\n    source_labels = [\"__meta_kubernetes_pod_label_app_kubernetes_io_name\"]\n    action = \"replace\"\n    target_label = \"app\"\n  }\n\n  // Label creation -  \"job\" field from \"__meta_kubernetes_namespace\" and \"__meta_kubernetes_pod_container_name\"\n  // Concatenate values __meta_kubernetes_namespace/__meta_kubernetes_pod_container_name\n  rule {\n    source_labels = [\"__meta_kubernetes_namespace\", \"__meta_kubernetes_pod_container_name\"]\n    action = \"replace\"\n    target_label = \"job\"\n    separator = \"/\"\n    replacement = \"$1\"\n  }\n\n  // Label creation - \"container\" field from \"__meta_kubernetes_pod_uid\" and \"__meta_kubernetes_pod_container_name\"\n  // Concatenate values __meta_kubernetes_pod_uid/__meta_kubernetes_pod_container_name.log\n  rule {\n    source_labels = [\"__meta_kubernetes_pod_uid\", \"__meta_kubernetes_pod_container_name\"]\n    action = \"replace\"\n    target_label = \"__path__\"\n    separator = \"/\"\n    replacement = \"/var/log/pods/*$1/*.log\"\n  }\n\n  // Label creation -  \"container_runtime\" field from \"__meta_kubernetes_pod_container_id\"\n  rule {\n    source_labels = [\"__meta_kubernetes_pod_container_id\"]\n    action = \"replace\"\n    target_label = \"container_runtime\"\n    regex = \"^(\\\\S+):\\\\/\\\\/.+$\"\n    replacement = \"$1\"\n  }\n}\n\n// loki.source.kubernetes tails logs from Kubernetes containers using the Kubernetes API.\nloki.source.kubernetes \"pod_logs\" {\n  targets    = discovery.relabel.pod_logs.output\n  forward_to = [loki.process.pod_logs.receiver]\n}\n// loki.process receives log entries from other Loki components, applies one or more processing stages,\n// and forwards the results to the list of receivers in the component’s arguments.\nloki.process \"pod_logs\" {\n  stage.static_labels {\n      values = {\n        cluster = \"envoy-gateway\",\n      }\n  }\n\n  forward_to = [loki.write.alloy.receiver]\n}"` |  |
+| alloy.enabled | bool | `false` |  |
+| alloy.fullnameOverride | string | `"alloy"` |  |
 | fluent-bit.config.filters | string | `"[FILTER]\n    Name kubernetes\n    Match kube.*\n    Merge_Log On\n    Keep_Log Off\n    K8S-Logging.Parser On\n    K8S-Logging.Exclude On\n\n[FILTER]\n    Name grep\n    Match kube.*\n    Regex $kubernetes['container_name'] ^envoy$\n\n[FILTER]\n    Name parser\n    Match kube.*\n    Key_Name log\n    Parser envoy\n    Reserve_Data True\n"` |  |
 | fluent-bit.config.inputs | string | `"[INPUT]\n    Name tail\n    Path /var/log/containers/*.log\n    multiline.parser docker, cri\n    Tag kube.*\n    Mem_Buf_Limit 5MB\n    Skip_Long_Lines On\n"` |  |
 | fluent-bit.config.outputs | string | `"[OUTPUT]\n    Name                   loki\n    Match                  kube.*\n    Host                   loki.monitoring.svc.cluster.local\n    Port                   3100\n    Labels                 job=fluentbit, app=$kubernetes['labels']['app'], k8s_namespace_name=$kubernetes['namespace_name'], k8s_pod_name=$kubernetes['pod_name'], k8s_container_name=$kubernetes['container_name']\n"` |  |

diff --git a/charts/gateway-addons-helm/values.yaml b/charts/gateway-addons-helm/values.yaml
@@ -60,6 +60,7 @@ prometheus:
 
 
 # Values for Fluent-bit dependency
+# TODO: remove fluent-bit dependency
 fluent-bit:
   enabled: true
   image:
@@ -167,6 +168,109 @@ loki:
   gateway:
     enabled: false
 
+# Values for Alloy dependency
+alloy:
+  enabled: false
+  fullnameOverride: alloy
+  alloy:
+    configMap:
+      content: |-
+        // Write your Alloy config here:
+        logging {
+          level = "info"
+          format = "logfmt"
+        }
+        loki.write "alloy" {
+          endpoint {
+            url = "http://loki.monitoring.svc:3100/loki/api/v1/push"
+          }
+        }
+        // discovery.kubernetes allows you to find scrape targets from Kubernetes resources.
+        // It watches cluster state and ensures targets are continually synced with what is currently running in your cluster.
+        discovery.kubernetes "pod" {
+          role = "pod"
+        }
+  
+        // discovery.relabel rewrites the label set of the input targets by applying one or more relabeling rules.
+        // If no rules are defined, then the input targets are exported as-is.
+        discovery.relabel "pod_logs" {
+          targets = discovery.kubernetes.pod.targets
+
+          // Label creation - "namespace" field from "__meta_kubernetes_namespace"
+          rule {
+            source_labels = ["__meta_kubernetes_namespace"]
+            action = "replace"
+            target_label = "namespace"
+          }
+
+          // Label creation - "pod" field from "__meta_kubernetes_pod_name"
+          rule {
+            source_labels = ["__meta_kubernetes_pod_name"]
+            action = "replace"
+            target_label = "pod"
+          }
+
+          // Label creation - "container" field from "__meta_kubernetes_pod_container_name"
+          rule {
+            source_labels = ["__meta_kubernetes_pod_container_name"]
+            action = "replace"
+            target_label = "container"
+          }
+
+          // Label creation -  "app" field from "__meta_kubernetes_pod_label_app_kubernetes_io_name"
+          rule {
+            source_labels = ["__meta_kubernetes_pod_label_app_kubernetes_io_name"]
+            action = "replace"
+            target_label = "app"
+          }
+
+          // Label creation -  "job" field from "__meta_kubernetes_namespace" and "__meta_kubernetes_pod_container_name"
+          // Concatenate values __meta_kubernetes_namespace/__meta_kubernetes_pod_container_name
+          rule {
+            source_labels = ["__meta_kubernetes_namespace", "__meta_kubernetes_pod_container_name"]
+            action = "replace"
+            target_label = "job"
+            separator = "/"
+            replacement = "$1"
+          }
+
+          // Label creation - "container" field from "__meta_kubernetes_pod_uid" and "__meta_kubernetes_pod_container_name"
+          // Concatenate values __meta_kubernetes_pod_uid/__meta_kubernetes_pod_container_name.log
+          rule {
+            source_labels = ["__meta_kubernetes_pod_uid", "__meta_kubernetes_pod_container_name"]
+            action = "replace"
+            target_label = "__path__"
+            separator = "/"
+            replacement = "/var/log/pods/*$1/*.log"
+          }
+
+          // Label creation -  "container_runtime" field from "__meta_kubernetes_pod_container_id"
+          rule {
+            source_labels = ["__meta_kubernetes_pod_container_id"]
+            action = "replace"
+            target_label = "container_runtime"
+            regex = "^(\\S+):\\/\\/.+$"
+            replacement = "$1"
+          }
+        }
+
+        // loki.source.kubernetes tails logs from Kubernetes containers using the Kubernetes API.
+        loki.source.kubernetes "pod_logs" {
+          targets    = discovery.relabel.pod_logs.output
+          forward_to = [loki.process.pod_logs.receiver]
+        }
+        // loki.process receives log entries from other Loki components, applies one or more processing stages,
+        // and forwards the results to the list of receivers in the component’s arguments.
+        loki.process "pod_logs" {
+          stage.static_labels {
+              values = {
+                cluster = "envoy-gateway",
+              }
+          }
+
+          forward_to = [loki.write.alloy.receiver]
+        }
+
 
 # Values for Tempo dependency
 tempo:

diff --git a/site/content/en/latest/install/gateway-addons-helm-api.md b/site/content/en/latest/install/gateway-addons-helm-api.md
@@ -24,6 +24,7 @@ An Add-ons Helm chart for Envoy Gateway
 | Repository | Name | Version |
 |------------|------|---------|
 | https://fluent.github.io/helm-charts | fluent-bit | 0.30.4 |
+| https://grafana.github.io/helm-charts | alloy | 0.9.2 |
 | https://grafana.github.io/helm-charts | grafana | 8.0.0 |
 | https://grafana.github.io/helm-charts | loki | 4.8.0 |
 | https://grafana.github.io/helm-charts | tempo | 1.3.1 |
@@ -34,6 +35,9 @@ An Add-ons Helm chart for Envoy Gateway
 
 | Key | Type | Default | Description |
 |-----|------|---------|-------------|
+| alloy.alloy.configMap.content | string | `"// Write your Alloy config here:\nlogging {\n  level = \"info\"\n  format = \"logfmt\"\n}\nloki.write \"alloy\" {\n  endpoint {\n    url = \"http://loki.monitoring.svc:3100/loki/api/v1/push\"\n  }\n}\n// discovery.kubernetes allows you to find scrape targets from Kubernetes resources.\n// It watches cluster state and ensures targets are continually synced with what is currently running in your cluster.\ndiscovery.kubernetes \"pod\" {\n  role = \"pod\"\n}\n\n// discovery.relabel rewrites the label set of the input targets by applying one or more relabeling rules.\n// If no rules are defined, then the input targets are exported as-is.\ndiscovery.relabel \"pod_logs\" {\n  targets = discovery.kubernetes.pod.targets\n\n  // Label creation - \"namespace\" field from \"__meta_kubernetes_namespace\"\n  rule {\n    source_labels = [\"__meta_kubernetes_namespace\"]\n    action = \"replace\"\n    target_label = \"namespace\"\n  }\n\n  // Label creation - \"pod\" field from \"__meta_kubernetes_pod_name\"\n  rule {\n    source_labels = [\"__meta_kubernetes_pod_name\"]\n    action = \"replace\"\n    target_label = \"pod\"\n  }\n\n  // Label creation - \"container\" field from \"__meta_kubernetes_pod_container_name\"\n  rule {\n    source_labels = [\"__meta_kubernetes_pod_container_name\"]\n    action = \"replace\"\n    target_label = \"container\"\n  }\n\n  // Label creation -  \"app\" field from \"__meta_kubernetes_pod_label_app_kubernetes_io_name\"\n  rule {\n    source_labels = [\"__meta_kubernetes_pod_label_app_kubernetes_io_name\"]\n    action = \"replace\"\n    target_label = \"app\"\n  }\n\n  // Label creation -  \"job\" field from \"__meta_kubernetes_namespace\" and \"__meta_kubernetes_pod_container_name\"\n  // Concatenate values __meta_kubernetes_namespace/__meta_kubernetes_pod_container_name\n  rule {\n    source_labels = [\"__meta_kubernetes_namespace\", \"__meta_kubernetes_pod_container_name\"]\n    action = \"replace\"\n    target_label = \"job\"\n    separator = \"/\"\n    replacement = \"$1\"\n  }\n\n  // Label creation - \"container\" field from \"__meta_kubernetes_pod_uid\" and \"__meta_kubernetes_pod_container_name\"\n  // Concatenate values __meta_kubernetes_pod_uid/__meta_kubernetes_pod_container_name.log\n  rule {\n    source_labels = [\"__meta_kubernetes_pod_uid\", \"__meta_kubernetes_pod_container_name\"]\n    action = \"replace\"\n    target_label = \"__path__\"\n    separator = \"/\"\n    replacement = \"/var/log/pods/*$1/*.log\"\n  }\n\n  // Label creation -  \"container_runtime\" field from \"__meta_kubernetes_pod_container_id\"\n  rule {\n    source_labels = [\"__meta_kubernetes_pod_container_id\"]\n    action = \"replace\"\n    target_label = \"container_runtime\"\n    regex = \"^(\\\\S+):\\\\/\\\\/.+$\"\n    replacement = \"$1\"\n  }\n}\n\n// loki.source.kubernetes tails logs from Kubernetes containers using the Kubernetes API.\nloki.source.kubernetes \"pod_logs\" {\n  targets    = discovery.relabel.pod_logs.output\n  forward_to = [loki.process.pod_logs.receiver]\n}\n// loki.process receives log entries from other Loki components, applies one or more processing stages,\n// and forwards the results to the list of receivers in the component’s arguments.\nloki.process \"pod_logs\" {\n  stage.static_labels {\n      values = {\n        cluster = \"envoy-gateway\",\n      }\n  }\n\n  forward_to = [loki.write.alloy.receiver]\n}"` |  |
+| alloy.enabled | bool | `false` |  |
+| alloy.fullnameOverride | string | `"alloy"` |  |
 | fluent-bit.config.filters | string | `"[FILTER]\n    Name kubernetes\n    Match kube.*\n    Merge_Log On\n    Keep_Log Off\n    K8S-Logging.Parser On\n    K8S-Logging.Exclude On\n\n[FILTER]\n    Name grep\n    Match kube.*\n    Regex $kubernetes['container_name'] ^envoy$\n\n[FILTER]\n    Name parser\n    Match kube.*\n    Key_Name log\n    Parser envoy\n    Reserve_Data True\n"` |  |
 | fluent-bit.config.inputs | string | `"[INPUT]\n    Name tail\n    Path /var/log/containers/*.log\n    multiline.parser docker, cri\n    Tag kube.*\n    Mem_Buf_Limit 5MB\n    Skip_Long_Lines On\n"` |  |
 | fluent-bit.config.outputs | string | `"[OUTPUT]\n    Name                   loki\n    Match                  kube.*\n    Host                   loki.monitoring.svc.cluster.local\n    Port                   3100\n    Labels                 job=fluentbit, app=$kubernetes['labels']['app'], k8s_namespace_name=$kubernetes['namespace_name'], k8s_pod_name=$kubernetes['pod_name'], k8s_container_name=$kubernetes['container_name']\n"` |  |