SOLR-15960 Unified use of system properties and environment variables

solr/bin/solr

@@ -112,6 +112,13 @@ elif [ -r "$SOLR_INCLUDE" ]; then
   . "$SOLR_INCLUDE"
 fi
 
+# Export all SOLR variables so they are visible to Solr sub-process
+for var in $(compgen -e); do
+  if [[ "$var" =~ ^SOLR_ ]]; then
+    export "${var?}"
+  fi
+done
+
 # if pid dir is unset, default to $solr_tip/bin
 : "${SOLR_PID_DIR:=$SOLR_TIP/bin}"
 
@@ -1345,12 +1352,10 @@ if [ $# -gt 0 ]; then
         ;;
         -v)
             SOLR_LOG_LEVEL=DEBUG
-            PASS_TO_RUN_EXAMPLE+=("-Dsolr.log.level=$SOLR_LOG_LEVEL")
             shift
         ;;
         -q)
             SOLR_LOG_LEVEL=WARN
-            PASS_TO_RUN_EXAMPLE+=("-Dsolr.log.level=$SOLR_LOG_LEVEL")
             shift
         ;;
         -all)
@@ -1384,15 +1389,6 @@ if [ $# -gt 0 ]; then
   done
 fi
 
-if [[ -n ${SOLR_LOG_LEVEL:-} ]] ; then
-  SOLR_LOG_LEVEL_OPT="-Dsolr.log.level=$SOLR_LOG_LEVEL"
-fi
-
-# Solr modules option
-if [[ -n "${SOLR_MODULES:-}" ]] ; then
-  SCRIPT_SOLR_OPTS+=("-Dsolr.modules=$SOLR_MODULES")
-fi
-
 # Default placement plugin
 if [[ -n "${SOLR_PLACEMENTPLUGIN_DEFAULT:-}" ]] ; then
   SCRIPT_SOLR_OPTS+=("-Dsolr.placementplugin.default=$SOLR_PLACEMENTPLUGIN_DEFAULT")
@@ -1883,7 +1879,7 @@ function start_solr() {
   fi
 
   SOLR_START_OPTS=('-server' "${JAVA_MEM_OPTS[@]}" "${GC_TUNE_ARR[@]}" "${GC_LOG_OPTS[@]}" "${IP_ACL_OPTS[@]}" \
-    "${REMOTE_JMX_OPTS[@]}" "${CLOUD_MODE_OPTS[@]}" ${SOLR_LOG_LEVEL_OPT:-} -Dsolr.log.dir="$SOLR_LOGS_DIR" \
+    "${REMOTE_JMX_OPTS[@]}" "${CLOUD_MODE_OPTS[@]}" -Dsolr.log.dir="$SOLR_LOGS_DIR" \
     "-Djetty.port=$SOLR_PORT" "-DSTOP.PORT=$stop_port" "-DSTOP.KEY=$STOP_KEY" \
     # '-OmitStackTraceInFastThrow' ensures stack traces in errors,
     # users who don't care about useful error msgs can override in SOLR_OPTS with +OmitStackTraceInFastThrow

solr/bin/solr.cmd

@@ -559,13 +559,11 @@ goto parse_args
 
 :set_debug
 set SOLR_LOG_LEVEL=DEBUG
-set "PASS_TO_RUN_EXAMPLE=!PASS_TO_RUN_EXAMPLE! -Dsolr.log.level=%SOLR_LOG_LEVEL%"
 SHIFT
 goto parse_args
 
 :set_warn
 set SOLR_LOG_LEVEL=WARN
-set "PASS_TO_RUN_EXAMPLE=!PASS_TO_RUN_EXAMPLE! -Dsolr.log.level=%SOLR_LOG_LEVEL%"
 SHIFT
 goto parse_args
 
@@ -807,11 +805,6 @@ IF NOT "%SOLR_HOST%"=="" (
 
 set SCRIPT_SOLR_OPTS=
 
-REM Solr modules option
-IF DEFINED SOLR_MODULES (
-  set "SCRIPT_SOLR_OPTS=%SCRIPT_SOLR_OPTS% -Dsolr.modules=%SOLR_MODULES%"
-)
-
 REM Default placement plugin
 IF DEFINED SOLR_PLACEMENTPLUGIN_DEFAULT (
   set "SCRIPT_SOLR_OPTS=%SCRIPT_SOLR_OPTS% -Dsolr.placementplugin.default=%SOLR_PLACEMENTPLUGIN_DEFAULT%"
@@ -1220,7 +1213,6 @@ IF "%SOLR_SSL_ENABLED%"=="true" (
   set "SSL_PORT_PROP=-Dsolr.jetty.https.port=%SOLR_PORT%"
   set "START_OPTS=%START_OPTS% %SOLR_SSL_OPTS% !SSL_PORT_PROP!"
 )
-IF NOT "%SOLR_LOG_LEVEL%"=="" set "START_OPTS=%START_OPTS% -Dsolr.log.level=%SOLR_LOG_LEVEL%"
 
 set SOLR_LOGS_DIR_QUOTED="%SOLR_LOGS_DIR%"
 set SOLR_DATA_HOME_QUOTED="%SOLR_DATA_HOME%"

solr/core/src/java/org/apache/solr/cli/SolrCLI.java

@@ -63,6 +63,7 @@
 import org.apache.solr.common.params.CommonParams;
 import org.apache.solr.common.util.ContentStreamBase;
 import org.apache.solr.common.util.NamedList;
+import org.apache.solr.util.EnvUtils;
 import org.apache.solr.util.StartupLoggingUtils;
 import org.apache.solr.util.configuration.SSLConfigurationsFactory;
 import org.slf4j.Logger;
@@ -187,18 +188,9 @@ public static CommandLine parseCmdLine(String toolName, String[] args, List<Opti
   }
 
   public static String getDefaultSolrUrl() {
-    String scheme = System.getenv("SOLR_URL_SCHEME");
-    if (scheme == null) {
-      scheme = "http";
-    }
-    String host = System.getenv("SOLR_TOOL_HOST");
-    if (host == null) {
-      host = "localhost";
-    }
-    String port = System.getenv("SOLR_PORT");
-    if (port == null) {
-      port = "8983";
-    }
+    String scheme = EnvUtils.getEnv("SOLR_URL_SCHEME", "http");
+    String host = EnvUtils.getEnv("SOLR_TOOL_HOST", "localhost");
+    String port = EnvUtils.getEnv("SOLR_PORT", "8983");
     return String.format(Locale.ROOT, "%s://%s:%s", scheme.toLowerCase(Locale.ROOT), host, port);
   }
 

solr/core/src/java/org/apache/solr/util/EnvUtils.java

@@ -0,0 +1,235 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.util;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.nio.charset.StandardCharsets;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Properties;
+import java.util.SortedMap;
+import java.util.TreeMap;
+import java.util.stream.Collectors;
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.util.StrUtils;
+import org.apache.solr.common.util.Utils;
+
+/**
+ * This class is a unified provider of environment variables and system properties. It exposes a
+ * mutable copy of environment variables with. It also converts 'SOLR_FOO' variables to system
+ * properties 'solr.foo' and provide various convenience accessors for them.
+ */
+public class EnvUtils {
+  private static final SortedMap<String, String> ENV = new TreeMap<>(System.getenv());
+  private static final Map<String, String> CUSTOM_MAPPINGS = new HashMap<>();
+
+  static {
+    try {
+      Properties props = new Properties();
+      try (InputStream stream =
+          EnvUtils.class.getClassLoader().getResourceAsStream("EnvToSyspropMappings.properties")) {
+        props.load(new InputStreamReader(Objects.requireNonNull(stream), StandardCharsets.UTF_8));
+        for (String key : props.stringPropertyNames()) {
+          CUSTOM_MAPPINGS.put(key, props.getProperty(key));
+        }
+        init(false);
+      }
+    } catch (IOException e) {
+      throw new SolrException(
+          SolrException.ErrorCode.INVALID_STATE, "Failed loading env.var->properties mapping", e);
+    }
+  }
+
+  /**
+   * Get all environment variables with SOLR_ prefix.
+   *
+   * @return sorted map of environment variables
+   */
+  public static SortedMap<String, String> getEnvs() {
+    return ENV;
+  }
+
+  /** Get a single environment variable as string */
+  public static String getEnv(String key) {
+    return ENV.get(key);
+  }
+
+  /** Get a single environment variable as string, or default */
+  public static String getEnv(String key, String defaultValue) {
+    return ENV.getOrDefault(key, defaultValue);
+  }
+
+  /** Get an environment variable as long */
+  public static long getEnvAsLong(String key) {
+    return Long.parseLong(ENV.get(key));
+  }
+
+  /** Get an environment variable as long, or default value */
+  public static long getEnvAsLong(String key, long defaultValue) {
+    String value = ENV.get(key);
+    if (value == null) {
+      return defaultValue;
+    }
+    return Long.parseLong(value);
+  }
+
+  /** Get an env var as boolean */
+  public static boolean getEnvAsBool(String key) {
+    return Boolean.parseBoolean(ENV.get(key));
+  }
+
+  /** Get an env var as boolean, or default value */
+  public static boolean getEnvAsBool(String key, boolean defaultValue) {
+    String value = ENV.get(key);
+    if (value == null) {
+      return defaultValue;
+    }
+    return Boolean.parseBoolean(value);
+  }
+
+  /** Get comma separated strings from env as List */
+  public static List<String> getEnvAsList(String key) {
+    return getEnv(key) != null ? stringValueToList(getEnv(key)) : null;
+  }
+
+  /** Get comma separated strings from env as List */
+  public static List<String> getEnvAsList(String key, List<String> defaultValue) {
+    return ENV.get(key) != null ? getEnvAsList(key) : defaultValue;
+  }
+
+  /** Set an environment variable */
+  public static void setEnv(String key, String value) {
+    ENV.put(key, value);
+  }
+
+  /** Set all environment variables */
+  public static void setEnvs(Map<String, String> env) {
+    ENV.clear();
+    ENV.putAll(env);
+  }
+
+  /** Get all Solr system properties as a sorted map */
+  public static SortedMap<String, String> getProps() {
+    SortedMap<String, String> props = new TreeMap<>();
+    for (String key : System.getProperties().stringPropertyNames()) {
+      props.put(key, System.getProperty(key));
+    }
+    return props;
+  }
+
+  /** Get a property as string */
+  public static String getProp(String key) {
+    return getProp(key, null);
+  }
+
+  /** Get a property as string */
+  public static String getProp(String key, String defaultValue) {
+    return System.getProperties().getProperty(key, defaultValue);
+  }
+
+  /** Get property as integer */
+  public static long getPropAsLong(String key) {
+    return Long.parseLong(getProp(key));
+  }
+
+  /** Get property as long, or default value */
+  public static long getPropAsLong(String key, long defaultValue) {
+    String value = getProp(key);
+    if (value == null) {
+      return defaultValue;
+    }
+    return Long.parseLong(value);
+  }
+
+  /** Get property as boolean */
+  public static boolean getPropAsBool(String key) {
+    return Boolean.parseBoolean(getProp(key));
+  }
+
+  /** Get property as boolean, or default value */
+  public static boolean getPropAsBool(String key, boolean defaultValue) {
+    String value = getProp(key);
+    if (value == null) {
+      return defaultValue;
+    }
+    return Boolean.parseBoolean(value);
+  }
+
+  /**
+   * Get comma separated strings from sysprop as List
+   *
+   * @return list of strings, or null if not found
+   */
+  public static List<String> getPropAsList(String key) {
+    return getProp(key) != null ? stringValueToList(getProp(key)) : null;
+  }
+
+  /**
+   * Get comma separated strings from sysprop as List, or default value
+   *
+   * @return list of strings, or provided default if not found
+   */
+  public static List<String> getPropAsList(String key, List<String> defaultValue) {
+    return getProp(key) != null ? getPropAsList(key) : defaultValue;
+  }
+
+  /** Set a system property. Shim to {@link System#setProperty(String, String)} */
+  public static void setProp(String key, String value) {
+    System.setProperty(key, value);
+  }
+
+  /**
+   * Re-reads environment variables and updates the internal map.
+   *
+   * @param overwrite if true, overwrite existing system properties with environment variables
+   */
+  public static void init(boolean overwrite) {
+    // Convert eligible environment variables with SOLR_ prefix to system properties
+    for (String key :
+        ENV.keySet().stream().filter(k -> k.startsWith("SOLR_")).toArray(String[]::new)) {
+      String sysPropKey = envNameToSyspropName(key);
+      // Existing system properties take precedence
+      if (!sysPropKey.isBlank() && (overwrite || getProp(sysPropKey) == null)) {
+        setProp(sysPropKey, ENV.get(key));
+      }
+    }
+  }
+
+  protected static String envNameToSyspropName(String envName) {
+    return CUSTOM_MAPPINGS.containsKey(envName)
+        ? CUSTOM_MAPPINGS.get(envName)
+        : envName.toLowerCase(Locale.ROOT).replace("_", ".");
+  }
+
+  @SuppressWarnings("unchecked")
+  private static List<String> stringValueToList(String string) {
+    if (string.startsWith("[") && string.endsWith("]")) {
+      // Convert a JSON string to a List<String> using Noggit parser
+      return (List<String>) Utils.fromJSONString(string);
+    } else {
+      return StrUtils.splitSmart(string, ",", true).stream()
+          .map(String::trim)
+          .collect(Collectors.toList());
+    }
+  }
+}

solr/core/src/java/org/apache/solr/util/ModuleUtils.java

@@ -22,7 +22,6 @@
 import java.nio.file.Path;
 import java.util.Collection;
 import java.util.Collections;
-import java.util.HashSet;
 import java.util.Set;
 import java.util.regex.Pattern;
 import java.util.stream.Collectors;
@@ -56,17 +55,9 @@ public static Path getModuleLibPath(Path solrInstallDirPath, String moduleName)
    * @return set of raw volume names from sysprop and/or env.var
    */
   static Set<String> resolveFromSyspropOrEnv() {
-    // Fall back to sysprop and env.var if nothing configured through solr.xml
-    Set<String> mods = new HashSet<>();
-    String modulesFromProps = System.getProperty("solr.modules");
-    if (StrUtils.isNotNullOrEmpty(modulesFromProps)) {
-      mods.addAll(StrUtils.splitSmart(modulesFromProps, ',', true));
-    }
-    String modulesFromEnv = System.getenv("SOLR_MODULES");
-    if (StrUtils.isNotNullOrEmpty(modulesFromEnv)) {
-      mods.addAll(StrUtils.splitSmart(modulesFromEnv, ',', true));
-    }
-    return mods.stream().map(String::trim).collect(Collectors.toSet());
+    return EnvUtils.getPropAsList("solr.modules", Collections.emptyList()).stream()
+        .map(String::trim)
+        .collect(Collectors.toSet());
   }
 
   /** Returns true if a module name is valid and exists in the system */