diff --git a/.github/workflows/build_and_test.yaml b/.github/workflows/build_and_test.yaml index 993a8ffc..c7912fb2 100644 --- a/.github/workflows/build_and_test.yaml +++ b/.github/workflows/build_and_test.yaml @@ -246,4 +246,8 @@ jobs: kubectl exec sidecar -- sh -c "! test -e /tmp/relative/relative.txt" && kubectl exec sidecar -- sh -c "test -e /tmp/relative/change-relative.txt" && kubectl exec sidecar -- sh -c "! test -e /tmp/orig-dir/change-dir.txt" && kubectl exec sidecar -- sh -c "test -e /tmp/new-dir/change-dir.txt" && kubectl exec sidecar -- sh -c "! test -e /tmp/similar-configmap.txt" && kubectl exec sidecar -- sh -c "test -e /tmp/change-similar-configmap.txt" && - kubectl exec sidecar -- sh -c "! test -e /tmp/similar-secret.txt" && kubectl exec sidecar -- sh -c "test -e /tmp/change-similar-secret.txt" \ No newline at end of file + kubectl exec sidecar -- sh -c "! test -e /tmp/similar-secret.txt" && kubectl exec sidecar -- sh -c "test -e /tmp/change-similar-secret.txt" + - name: Verify environment variable expansion feature in log config + run: | + cat /tmp/logs/sidecar-pythonscript-logfile.log | jq 'has("msg") and has("level")' -e >/dev/null && + test $(cat /tmp/logs/sidecar-pythonscript-logfile.log | jq '. | select(.level == "DEBUG")' | wc -l) -gt 0 \ No newline at end of file diff --git a/README.md b/README.md index e97b659b..4f7fc086 100644 --- a/README.md +++ b/README.md @@ -99,3 +99,35 @@ If the filename ends with `.url` suffix, the content will be processed as a URL | `LOG_FORMAT` | Set a log format. (JSON or LOGFMT) | false | `JSON` | string | | `LOG_TZ` | Set the log timezone. (LOCAL or UTC) | false | `LOCAL` | string | | `LOG_CONFIG` | Log configuration file path. If not configured, uses the default log config for backward compatibility support. When not configured `LOG_LEVEL, LOG_FORMAT and LOG_TZ` would be used. Refer to [Python logging](https://docs.python.org/3/library/logging.config.html) for log configuration. For sample configuration file refer to file examples/example_logconfig.yaml | false | - | string | + + +## Environment variable expansion in LOG_CONFIG +Kiwigrid k8s-sidecar supports expansion of environment variables in the log config. +This can be done by wrapping the name of environment variable in the regex placeholder `$()` in the log config. +At the startup, the k8s-sidecar container will look for the regex wrapper and replace all the matched occurrences with the content of the environment variables. + +For instance the below snippet from the log config, + +```commandline +version: 1 +disable_existing_loggers: false + +root: + level: $(LV_DBG) + handlers: [console] +... +``` + +would be read as below, replacing the content of `$(LV_DBG)` with the value of environment variable `LV_DBG` (assuming it set to `DEBUG` here). + +```commandline +version: 1 +disable_existing_loggers: false + +root: + level: DEBUG + handlers: [console] +... +``` + +> **Note:** The k8s-sidecar will terminate if it finds a match for the environment variable expansion placeholder and the corresponding environment variable is not set. diff --git a/src/logger.py b/src/logger.py index e6324abc..27ef75e2 100644 --- a/src/logger.py +++ b/src/logger.py @@ -1,6 +1,7 @@ import logging import os import sys +import re import yaml from datetime import datetime from typing import Optional @@ -107,11 +108,27 @@ def add_fields(self, log_record, record, message_dict): } } +def expand_env(data): + placeholder_pattern = r'\$\((.*?)\)' + + def replace_placeholder(s): + env = s.group(1) + env_value = os.getenv(env) + if env_value is None: + print(f'unable to expand environment variable {env} in LOG_CONFIG. reason: env variable not set') + sys.exit(1) + else: + return env_value + + processed_data = re.sub(placeholder_pattern, replace_placeholder, data) + return yaml.load(processed_data, Loader=yaml.FullLoader) + def get_log_config(): if log_conf_file != "" : try: with open(log_conf_file, 'r') as stream: - config = yaml.load(stream, Loader=yaml.FullLoader) + data = stream.read() + config = expand_env(data) return config except FileNotFoundError: msg = "Config file: "+ log_conf_file + " Not Found" diff --git a/test/resources/sidecar.yaml b/test/resources/sidecar.yaml index 40ca9c44..d4bcc021 100644 --- a/test/resources/sidecar.yaml +++ b/test/resources/sidecar.yaml @@ -166,13 +166,13 @@ data: disable_existing_loggers: false root: - level: DEBUG + level: $(ENV_LEVEL_DEBUG) handlers: [console] handlers: console: class: logging.StreamHandler - level: DEBUG + level: $(ENV_LEVEL_DEBUG) formatter: JSON formatters: @@ -180,8 +180,8 @@ data: (): logger.JsonFormatter format: '%(levelname)s %(message)s' rename_fields: { - "message": "msg", - "levelname": "level" + "message": "$(ENV_RENAME_FIELD_MSG)", + "levelname": "$(ENV_RENAME_FIELD_LEVEL)" } --- apiVersion: v1 @@ -248,6 +248,12 @@ spec: value: "DEBUG" - name: LOG_CONFIG value: "/etc/k8s-sidecar/log_conf.yaml" + - name: ENV_LEVEL_DEBUG + value: "DEBUG" + - name: ENV_RENAME_FIELD_MSG + value: "msg" + - name: ENV_RENAME_FIELD_LEVEL + value: "level" volumes: - name: shared-volume emptyDir: { }