environment variable expansion for log config

.github/workflows/build_and_test.yaml

@@ -246,4 +246,8 @@ jobs:
           kubectl exec sidecar -- sh -c "! test -e /tmp/relative/relative.txt" && kubectl exec sidecar -- sh -c "test -e /tmp/relative/change-relative.txt" &&
           kubectl exec sidecar -- sh -c "! test -e /tmp/orig-dir/change-dir.txt" && kubectl exec sidecar -- sh -c "test -e /tmp/new-dir/change-dir.txt" &&
           kubectl exec sidecar -- sh -c "! test -e /tmp/similar-configmap.txt" && kubectl exec sidecar -- sh -c "test -e /tmp/change-similar-configmap.txt" &&
-          kubectl exec sidecar -- sh -c "! test -e /tmp/similar-secret.txt" && kubectl exec sidecar -- sh -c "test -e /tmp/change-similar-secret.txt"
+          kubectl exec sidecar -- sh -c "! test -e /tmp/similar-secret.txt" && kubectl exec sidecar -- sh -c "test -e /tmp/change-similar-secret.txt"
+      - name: Verify environment variable expansion feature in log config
+        run: |
+          cat /tmp/logs/sidecar-pythonscript-logfile.log | jq 'has("msg") and has("level")' -e >/dev/null &&
+          test $(cat /tmp/logs/sidecar-pythonscript-logfile.log | jq '. | select(.level == "DEBUG")' | wc -l) -gt 0

README.md

@@ -99,3 +99,35 @@ If the filename ends with `.url` suffix, the content will be processed as a URL
 | `LOG_FORMAT`               | Set a log format. (JSON or LOGFMT)                                                                                                                                                                                                                                                                                                  | false    | `JSON`                                    | string  |
 | `LOG_TZ`                   | Set the log timezone. (LOCAL or UTC)                                                                                                                                                                                                                                                                                                | false    | `LOCAL`                                   | string  |
 | `LOG_CONFIG`               | Log configuration file path. If not configured, uses the default log config for backward compatibility support. When not configured `LOG_LEVEL, LOG_FORMAT and LOG_TZ` would be used. Refer to [Python logging](https://docs.python.org/3/library/logging.config.html) for log configuration. For sample configuration file  refer to file examples/example_logconfig.yaml | false    | -                                         | string  |
+
+
+## Environment variable expansion in LOG_CONFIG
+Kiwigrid k8s-sidecar supports expansion of environment variables in the log config.
+This can be done by wrapping the name of environment variable in the regex placeholder `$(<env_var_name>)` in the log config.
+At the startup, the k8s-sidecar container will look for the regex wrapper and replace all the matched occurrences with the content of the environment variables.
+
+For instance the below snippet from the log config,
+
+```commandline
+version: 1
+disable_existing_loggers: false
+
+root:
+    level: $(LV_DBG)
+    handlers: [console]
+...
+```
+
+would be read as below, replacing the content of `$(LV_DBG)` with the value of environment variable `LV_DBG` (assuming it set to `DEBUG` here).
+
+```commandline
+version: 1
+disable_existing_loggers: false
+
+root:
+    level: DEBUG
+    handlers: [console]
+...
+```
+
+> **Note:** The k8s-sidecar will terminate if it finds a match for the environment variable expansion placeholder and the corresponding environment variable is not set.

src/logger.py

@@ -1,6 +1,7 @@
 import logging
 import os
 import sys
+import re
 import yaml
 from datetime import datetime
 from typing import Optional
@@ -107,11 +108,27 @@ def add_fields(self, log_record, record, message_dict):
     }
 }
 
+def expand_env(data):
+    placeholder_pattern = r'\$\((.*?)\)'
+
+    def replace_placeholder(s):
+        env = s.group(1)
+        env_value = os.getenv(env)
+        if env_value is None:
+            print(f'unable to expand environment variable {env} in LOG_CONFIG. reason: env variable not set')
+            sys.exit(1)
+        else:
+            return env_value
+
+    processed_data = re.sub(placeholder_pattern, replace_placeholder, data)
+    return yaml.load(processed_data, Loader=yaml.FullLoader)
+
 def get_log_config():
     if log_conf_file != "" :
         try:
             with open(log_conf_file, 'r') as stream:
-                config = yaml.load(stream, Loader=yaml.FullLoader)
+                data = stream.read()
+                config = expand_env(data)
             return config
         except FileNotFoundError:
             msg = "Config file: "+ log_conf_file + " Not Found"

test/resources/sidecar.yaml

@@ -166,22 +166,22 @@ data:
     disable_existing_loggers: false
 
     root:
-        level: DEBUG
+        level: $(ENV_LEVEL_DEBUG)
         handlers: [console]
 
     handlers:
         console:
             class: logging.StreamHandler
-            level: DEBUG
+            level: $(ENV_LEVEL_DEBUG)
             formatter: JSON
 
     formatters:
         JSON:
             (): logger.JsonFormatter
             format: '%(levelname)s %(message)s'
             rename_fields: {
-                    "message": "msg", 
-                    "levelname": "level"
+                    "message": "$(ENV_RENAME_FIELD_MSG)", 
+                    "levelname": "$(ENV_RENAME_FIELD_LEVEL)"
                     }
 ---
 apiVersion: v1
@@ -248,6 +248,12 @@ spec:
           value: "DEBUG"
         - name: LOG_CONFIG
           value: "/etc/k8s-sidecar/log_conf.yaml"
+        - name: ENV_LEVEL_DEBUG
+          value: "DEBUG"
+        - name: ENV_RENAME_FIELD_MSG
+          value: "msg"
+        - name: ENV_RENAME_FIELD_LEVEL
+          value: "level"
   volumes:
     - name: shared-volume
       emptyDir: { }