SOLR-16743: Auto reload keystore/truststore on change (#2100)

Add Jetty module to scan and automatically reload key store and trust store changes. Also, add scanner to the SolrJ library to do the same on the client side if configured

solr/CHANGES.txt

@@ -85,6 +85,9 @@ New Features
 
 * SOLR-17079: Allow to declare replica placement plugins in solr.xml  (Vincent Primault)
 
+* SOLR-16743: When using TLS, Solr can now auto-reload the keystore and truststore without the need to restart the process.
+  This is enabled by default when running with TLS and can be disabled or configured in solr.in.sh (Houston Putman, Tomás Fernández Löbbe)
+
 Improvements
 ---------------------
 * SOLR-17053: Distributed search with shards.tolerant: if all shards fail, fail the request (Aparna Suresh via David Smiley)

solr/bin/solr

@@ -208,9 +208,17 @@ if [ -z "${SOLR_SSL_ENABLED:-}" ]; then
 fi
 if [ "$SOLR_SSL_ENABLED" == "true" ]; then
   SOLR_JETTY_CONFIG+=("--module=https" "--lib=$DEFAULT_SERVER_DIR/solr-webapp/webapp/WEB-INF/lib/*")
+  if [ "${SOLR_SSL_RELOAD_ENABLED:-true}" == "true" ]; then
+    SOLR_JETTY_CONFIG+=("--module=ssl-reload")
+    SOLR_SSL_OPTS+=" -Dsolr.keyStoreReload.enabled=true"
+  fi
   SOLR_URL_SCHEME=https
   if [ -n "$SOLR_SSL_KEY_STORE" ]; then
     SOLR_SSL_OPTS+=" -Dsolr.jetty.keystore=$SOLR_SSL_KEY_STORE"
+    if [ "${SOLR_SSL_RELOAD_ENABLED:-true}" == "true" ] && [ "${SOLR_SECURITY_MANAGER_ENABLED:-true}" == "true"  ]; then
+      # In this case we need to allow reads from the parent directory of the keystore
+      SOLR_SSL_OPTS+=" -Dsolr.jetty.keystoreParentPath=$SOLR_SSL_KEY_STORE/.."
+    fi
   fi
   if [ -n "$SOLR_SSL_KEY_STORE_PASSWORD" ]; then
     export SOLR_SSL_KEY_STORE_PASSWORD=$SOLR_SSL_KEY_STORE_PASSWORD
@@ -249,6 +257,10 @@ if [ "$SOLR_SSL_ENABLED" == "true" ]; then
     if [ -n "$SOLR_SSL_CLIENT_KEY_STORE_TYPE" ]; then
       SOLR_SSL_OPTS+=" -Djavax.net.ssl.keyStoreType=$SOLR_SSL_CLIENT_KEY_STORE_TYPE"
     fi
+    if [ "${SOLR_SSL_RELOAD_ENABLED:-true}" == "true" ] && [ "${SOLR_SECURITY_MANAGER_ENABLED:-true}" == "true"  ]; then
+      # In this case we need to allow reads from the parent directory of the keystore
+      SOLR_SSL_OPTS+=" -Djavax.net.ssl.keyStoreParentPath=$SOLR_SSL_CLIENT_KEY_STORE/.."
+    fi
   else
     if [ -n "$SOLR_SSL_KEY_STORE" ]; then
       SOLR_SSL_OPTS+=" -Djavax.net.ssl.keyStore=$SOLR_SSL_KEY_STORE"

solr/bin/solr.cmd

@@ -84,11 +84,29 @@ IF NOT DEFINED SOLR_SSL_ENABLED (
   )
 )
 
+IF NOT DEFINED SOLR_SSL_RELOAD_ENABLED (
+  set "SOLR_SSL_RELOAD_ENABLED=true"
+)
+
+REM Enable java security manager by default (limiting filesystem access and other things)
+IF NOT DEFINED SOLR_SECURITY_MANAGER_ENABLED (
+  set SOLR_SECURITY_MANAGER_ENABLED=true
+)
+
 IF "%SOLR_SSL_ENABLED%"=="true" (
   set "SOLR_JETTY_CONFIG=--module=https --lib="%DEFAULT_SERVER_DIR%\solr-webapp\webapp\WEB-INF\lib\*""
   set SOLR_URL_SCHEME=https
+  IF "%SOLR_SSL_RELOAD_ENABLED%"=="true" (
+    set "SOLR_JETTY_CONFIG=!SOLR_JETTY_CONFIG! --module=ssl-reload"
+    set "SOLR_SSL_OPTS=!SOLR_SSL_OPTS! -Dsolr.keyStoreReload.enabled=true"
+  )
   IF DEFINED SOLR_SSL_KEY_STORE (
     set "SOLR_SSL_OPTS=!SOLR_SSL_OPTS! -Dsolr.jetty.keystore=%SOLR_SSL_KEY_STORE%"
+    IF "%SOLR_SSL_RELOAD_ENABLED%"=="true" (
+      IF "%SOLR_SECURITY_MANAGER_ENABLED%"=="true" (
+        set "SOLR_SSL_OPTS=!SOLR_SSL_OPTS! -Dsolr.jetty.keystoreParentPath=%SOLR_SSL_KEY_STORE%/.."
+      )
+    )
   )
 
   IF DEFINED SOLR_SSL_KEY_STORE_TYPE (
@@ -122,6 +140,11 @@ IF "%SOLR_SSL_ENABLED%"=="true" (
     IF DEFINED SOLR_SSL_CLIENT_KEY_STORE_TYPE (
       set "SOLR_SSL_OPTS=!SOLR_SSL_OPTS! -Djavax.net.ssl.keyStoreType=%SOLR_SSL_CLIENT_KEY_STORE_TYPE%"
     )
+    IF "%SOLR_SSL_RELOAD_ENABLED%"=="true" (
+      IF "%SOLR_SECURITY_MANAGER_ENABLED%"=="true" (
+        set "SOLR_SSL_OPTS=!SOLR_SSL_OPTS! -Djavax.net.ssl.keyStoreParentPath=%SOLR_SSL_CLIENT_KEY_STORE_TYPE%/.."
+      )
+    )
   ) ELSE (
     IF DEFINED SOLR_SSL_KEY_STORE (
       set "SOLR_SSL_OPTS=!SOLR_SSL_OPTS! -Djavax.net.ssl.keyStore=%SOLR_SSL_KEY_STORE%"
@@ -1077,11 +1100,6 @@ IF "%ENABLE_REMOTE_JMX_OPTS%"=="true" (
   set REMOTE_JMX_OPTS=
 )
 
-REM Enable java security manager by default (limiting filesystem access and other things)
-IF NOT DEFINED SOLR_SECURITY_MANAGER_ENABLED (
-  set SOLR_SECURITY_MANAGER_ENABLED=true
-)
-
 IF "%SOLR_SECURITY_MANAGER_ENABLED%"=="true" (
   set SECURITY_MANAGER_OPTS=-Djava.security.manager ^
 -Djava.security.policy="%SOLR_SERVER_DIR%\etc\security.policy" ^

solr/bin/solr.in.sh

@@ -179,6 +179,7 @@
 # Override Key/Trust Store types if necessary
 #SOLR_SSL_KEY_STORE_TYPE=PKCS12
 #SOLR_SSL_TRUST_STORE_TYPE=PKCS12
+#SOLR_SSL_RELOAD_ENABLED=true
 
 # Uncomment if you want to override previously defined SSL values for HTTP client
 # otherwise keep them commented and the above values will automatically be set for HTTP clients

solr/core/src/java/org/apache/solr/cli/CreateTool.java

@@ -202,6 +202,7 @@ protected void createCollection(CommandLine cli) throws Exception {
         new Http2SolrClient.Builder()
             .withIdleTimeout(30, TimeUnit.SECONDS)
             .withConnectionTimeout(15, TimeUnit.SECONDS)
+            .withKeyStoreReloadInterval(-1, TimeUnit.SECONDS)
             .withOptionalBasicAuthCredentials(
                 cli.getOptionValue(SolrCLI.OPTION_CREDENTIALS.getLongOpt()));
     String zkHost = SolrCLI.getZkHost(cli);

solr/core/src/java/org/apache/solr/cli/DeleteTool.java

@@ -107,6 +107,7 @@ protected void deleteCollection(CommandLine cli) throws Exception {
         new Http2SolrClient.Builder()
             .withIdleTimeout(30, TimeUnit.SECONDS)
             .withConnectionTimeout(15, TimeUnit.SECONDS)
+            .withKeyStoreReloadInterval(-1, TimeUnit.SECONDS)
             .withOptionalBasicAuthCredentials(cli.getOptionValue(("credentials")));
 
     String zkHost = SolrCLI.getZkHost(cli);

solr/core/src/java/org/apache/solr/cli/PostLogsTool.java

@@ -30,6 +30,7 @@
 import java.util.Map;
 import java.util.TreeMap;
 import java.util.UUID;
+import java.util.concurrent.TimeUnit;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 import java.util.stream.Collectors;
@@ -90,7 +91,9 @@ public void runImpl(CommandLine cli) throws Exception {
   public void runCommand(String baseUrl, String root, String credentials) throws IOException {
 
     Http2SolrClient.Builder builder =
-        new Http2SolrClient.Builder(baseUrl).withOptionalBasicAuthCredentials(credentials);
+        new Http2SolrClient.Builder(baseUrl)
+            .withKeyStoreReloadInterval(-1, TimeUnit.SECONDS)
+            .withOptionalBasicAuthCredentials(credentials);
     try (SolrClient client = builder.build()) {
       int rec = 0;
       UpdateRequest request = new UpdateRequest();

solr/core/src/java/org/apache/solr/cli/SolrCLI.java

@@ -416,6 +416,7 @@ public static SolrClient getSolrClient(String solrUrl, String credentials, boole
     Http2SolrClient.Builder builder =
         new Http2SolrClient.Builder(solrUrl)
             .withMaxConnectionsPerHost(32)
+            .withKeyStoreReloadInterval(-1, TimeUnit.SECONDS)
             .withOptionalBasicAuthCredentials(credentials);
 
     return builder.build();

solr/packaging/test/test_ssl.bats

@@ -29,6 +29,7 @@ teardown() {
   solr stop -all >/dev/null 2>&1
 }
 
+
 @test "start solr with ssl" {
   # Create a keystore
   export ssl_dir="${BATS_TEST_TMPDIR}/ssl"
@@ -151,8 +152,8 @@ teardown() {
   export SOLR_HOST=localhost
 
   solr start -c
-  solr auth enable -type basicAuth -credentials name:password
   solr assert --started https://localhost:${SOLR_PORT}/solr --timeout 5000
+  solr auth enable -type basicAuth -credentials name:password
 
   run curl -u name:password --basic --cacert "$ssl_dir/solr-ssl.pem" "https://localhost:${SOLR_PORT}/solr/admin/collections?action=CREATE&collection.configName=_default&name=test&numShards=2&replicationFactor=1&router.name=compositeId&wt=json"
   assert_output --partial '"status":0'
@@ -209,13 +210,13 @@ teardown() {
 
   run solr start -c
 
+  solr assert --started https://localhost:${SOLR_PORT}/solr --timeout 5000
+
   export SOLR_SSL_KEY_STORE=
   export SOLR_SSL_KEY_STORE_PASSWORD=
   export SOLR_SSL_TRUST_STORE=
   export SOLR_SSL_TRUST_STORE_PASSWORD=
 
-  solr assert --started https://localhost:${SOLR_PORT}/solr --timeout 5000
-
   run solr create -c test -s 2
   assert_output --partial "Created collection 'test'"
 
@@ -494,3 +495,117 @@ teardown() {
   run solr api -verbose -get "https://localhost:${SOLR_PORT}/solr/test/select?q=*:*&rows=0"
   assert_output --regexp '(unable to find valid certification path to requested target|Server refused connection)'
 }
+
+@test "test keystore reload" {
+  # Create a keystore
+  export ssl_dir="${BATS_TEST_TMPDIR}/ssl"
+  mkdir -p "$ssl_dir"
+  (
+    cd "$ssl_dir"
+    rm -f cert1.keystore.p12 cert1.pem cert2.keystore.p12 cert2.pem
+    # cert and keystore 1
+    keytool -genkeypair -alias cert1 -keyalg RSA -keysize 2048 -keypass secret -storepass secret -validity 9999 -keystore cert1.keystore.p12 -storetype PKCS12 -ext SAN=DNS:localhost,IP:127.0.0.1 -dname "CN=localhost, OU=Organizational Unit, O=Organization, L=Location, ST=State, C=Country"
+    openssl pkcs12 -in cert1.keystore.p12 -out cert1.pem -passin pass:secret -passout pass:secret
+
+    # cert and keystore 2
+    keytool -genkeypair -alias cert2 -keyalg RSA -keysize 2048 -keypass secret -storepass secret -validity 9999 -keystore cert2.keystore.p12 -storetype PKCS12 -ext SAN=DNS:localhost,IP:127.0.0.1 -dname "CN=localhost, OU=Organizational Unit, O=Organization, L=Location, ST=State, C=Country"
+    openssl pkcs12 -in cert2.keystore.p12 -out cert2.pem -passin pass:secret -passout pass:secret
+
+    cp cert1.keystore.p12 server1.keystore.p12
+    cp cert1.keystore.p12 server2.keystore.p12
+  )
+
+  # Set ENV_VARs so that Solr uses this keystore
+  export SOLR_SSL_ENABLED=true
+  export SOLR_SSL_KEY_STORE_PASSWORD=secret
+  export SOLR_SSL_TRUST_STORE_PASSWORD=secret
+  export SOLR_SSL_NEED_CLIENT_AUTH=true
+  export SOLR_SSL_WANT_CLIENT_AUTH=false
+  export SOLR_HOST=localhost
+
+  # server1 will run on $SOLR_PORT and will use server1.keystore
+  export SOLR_SSL_KEY_STORE=$ssl_dir/server1.keystore.p12
+  export SOLR_SSL_TRUST_STORE=$ssl_dir/server1.keystore.p12
+  solr start -c -a "-Dsolr.jetty.sslContext.reload.scanInterval=1 -DsocketTimeout=5000"
+  solr assert --started https://localhost:${SOLR_PORT}/solr --timeout 5000
+
+  # server2 will run on $SOLR2_PORT and will use server2.keystore. Initially, this is the same as server1.keystore
+  export SOLR_SSL_KEY_STORE=$ssl_dir/server2.keystore.p12
+  export SOLR_SSL_TRUST_STORE=$ssl_dir/server2.keystore.p12
+  solr start -c -z localhost:${ZK_PORT} -p ${SOLR2_PORT} -a "-Dsolr.jetty.sslContext.reload.scanInterval=1 -DsocketTimeout=5000"
+  solr assert --started https://localhost:${SOLR2_PORT}/solr --timeout 5000
+
+  # "test" collection is two shards, meaning there must be communication between shards for queries (handled by http shard handler factory)
+  run solr create -c test -s 2
+  assert_output --partial "Created collection 'test'"
+
+  # "test-single-shard" is one shard and one replica, this means that one of the nodes will have to forward requests to the other
+  run solr create -c test-single-shard -s 1
+  assert_output --partial "Created collection 'test-single-shard'"
+
+  run solr api -get "https://localhost:${SOLR_PORT}/solr/test/select?q=*:*"
+  assert_output --partial '"numFound":0'
+  run solr api -get "https://localhost:${SOLR2_PORT}/solr/test/select?q=*:*"
+  assert_output --partial '"numFound":0'
+
+  run solr api -get "https://localhost:${SOLR_PORT}/solr/test-single-shard/select?q=*:*"
+  assert_output --partial '"numFound":0'
+  run solr api -get "https://localhost:${SOLR2_PORT}/solr/test-single-shard/select?q=*:*"
+  assert_output --partial '"numFound":0'
+
+  run ! curl "https://localhost:${SOLR_PORT}/solr/test/select?q=*:*"
+  run ! curl "https://localhost:${SOLR2_PORT}/solr/test/select?q=*:*"
+
+  run ! curl "https://localhost:${SOLR_PORT}/solr/test-single-shard/select?q=*:*"
+  run ! curl "https://localhost:${SOLR2_PORT}/solr/test-single-shard/select?q=*:*"
+
+  export SOLR_SSL_KEY_STORE=$ssl_dir/cert2.keystore.p12
+  export SOLR_SSL_KEY_STORE_PASSWORD=secret
+  export SOLR_SSL_TRUST_STORE=$ssl_dir/cert2.keystore.p12
+  export SOLR_SSL_TRUST_STORE_PASSWORD=secret
+
+  run ! solr api -get "https://localhost:${SOLR_PORT}/solr/test/select?q=*:*"
+
+  (
+    cd "$ssl_dir"
+    # Replace server1 keystore with client's
+    cp cert2.keystore.p12 server1.keystore.p12
+  )
+  # Give some time for the server reload
+  sleep 6
+
+  run solr healthcheck -solrUrl https://localhost:${SOLR_PORT}
+
+  # Server 2 still uses the cert1, so this request should fail
+  run ! solr api -get "https://localhost:${SOLR2_PORT}/solr/test/select?q=query2"
+
+  run ! solr healthcheck -solrUrl https://localhost:${SOLR2_PORT}
+
+  (
+    cd "$ssl_dir"
+    # Replace server2 keystore with client's
+    cp cert2.keystore.p12 server2.keystore.p12
+  )
+  # Give some time for the server reload
+  sleep 6
+
+  run solr healthcheck -solrUrl https://localhost:${SOLR_PORT}
+  run solr healthcheck -solrUrl https://localhost:${SOLR2_PORT}
+
+  run solr api -get "https://localhost:${SOLR_PORT}/solr/test/select?q=query3"
+  assert_output --partial '"numFound":0'
+
+  run solr api -get "https://localhost:${SOLR2_PORT}/solr/test/select?q=query3"
+  assert_output --partial '"numFound":0'
+
+  run solr api -get "https://localhost:${SOLR_PORT}/solr/test-single-shard/select?q=query4"
+  assert_output --partial '"numFound":0'
+
+  run solr api -get "https://localhost:${SOLR2_PORT}/solr/test-single-shard/select?q=query4"
+  assert_output --partial '"numFound":0'
+
+  run solr post -url https://localhost:${SOLR_PORT}/solr/test/update -commit ${SOLR_TIP}/example/exampledocs/books.csv
+
+  run solr api -get "https://localhost:${SOLR_PORT}/solr/test/select?q=*:*"
+  assert_output --partial '"numFound":10'
+}

solr/server/etc/jetty-ssl-context-reload.xml

@@ -0,0 +1,13 @@
+<?xml version="1.0"?>
+<!DOCTYPE Configure PUBLIC "-//Jetty//Configure//EN" "http://www.eclipse.org/jetty/configure_10_0.dtd">
+
+<Configure id="Server" class="org.eclipse.jetty.server.Server">
+    <Call name="addBean">
+        <Arg>
+            <New id="keyStoreScanner" class="org.eclipse.jetty.util.ssl.KeyStoreScanner">
+                <Arg><Ref refid="sslContextFactory"/></Arg>
+                <Set name="scanInterval"><Property name="solr.jetty.sslContext.reload.scanInterval" default="30"/></Set>
+            </New>
+        </Arg>
+    </Call>
+</Configure>

solr/server/etc/security.policy

@@ -193,6 +193,9 @@ grant {
 
   permission java.io.FilePermission "${javax.net.ssl.trustStore}", "read,readlink";
 
+  permission java.io.FilePermission "${solr.jetty.keystoreParentPath}", "read,readlink";
+  permission java.io.FilePermission "${javax.net.ssl.keyStoreParentPath}", "read,readlink";
+
   permission java.io.FilePermission "${solr.install.dir}", "read,write,delete,readlink";
   permission java.io.FilePermission "${solr.install.dir}${/}-", "read,write,delete,readlink";
   permission java.io.FilePermission "${solr.install.symDir}", "read,write,delete,readlink";

solr/server/modules/ssl-reload.mod

@@ -0,0 +1,12 @@
+[description]
+Enables the KeyStore to be reloaded when the KeyStore file changes.
+
+[tags]
+connector
+ssl
+
+[depend]
+ssl
+
+[xml]
+etc/jetty-ssl-context-reload.xml

solr/solr-ref-guide/modules/deployment-guide/pages/enabling-ssl.adoc

@@ -323,6 +323,23 @@ C:\> bin\solr.cmd -cloud -s cloud\node1 -z server1:2181,server2:2181,server3:218
 ====
 --
 
+== Automatically reloading KeyStore/TrustStore
+=== Solr Server
+Solr can automatically reload KeyStore/TrustStore when certificates are updated without restarting. This is enabled by default
+when using SSL, but can be disabled by setting the environment variable `SOLR_SSL_RELOAD_ENABLED` to `false`. By
+default, Solr will check for updates in the KeyStore every 30 seconds, but this interval can be updated by passing the
+system property `solr.jetty.sslContext.reload.scanInterval` with the new interval in seconds on startup.
+Note that the truststore file is not actively monitored, so if you need to apply changes to the truststore, you need
+to update it and after that touch the keystore to trigger a reload.
+
+=== SolrJ client
+Http2SolrClient builder has a method `withKeyStoreReloadInterval(long interval, TimeUnit unit)` to initialize a scanner
+that will watch and update the keystore and truststore for changes. If you are using CloudHttp2SolrClient, you can use
+the `withInternalClientBuilder(Http2SolrClient.Builder internalClientBuilder)` to configure the internal http client
+with a keystore reload interval. The minimum reload interval is 1 second. If not set (or set to 0 or a negative value),
+the keystore/truststore won't be updated in the client.
+
+
 == Example Client Actions
 
 [IMPORTANT]