Merge branch 'main' into SOLR-15960-env-sysprop

.github/PULL_REQUEST_TEMPLATE.md

@@ -35,7 +35,7 @@ Please describe the tests you've developed or run to confirm this patch implemen
 
 Please review the following and check all that apply:
 
-- [ ] I have reviewed the guidelines for [How to Contribute](https://wiki.apache.org/solr/HowToContribute) and my code conforms to the standards described there to the best of my ability.
+- [ ] I have reviewed the guidelines for [How to Contribute](https://github.com/apache/solr/blob/main/CONTRIBUTING.md) and my code conforms to the standards described there to the best of my ability.
 - [ ] I have created a Jira issue and added the issue ID to my pull request title.
 - [ ] I have given Solr maintainers [access](https://help.github.com/en/articles/allowing-changes-to-a-pull-request-branch-created-from-a-fork) to contribute to my PR branch. (optional but recommended)
 - [ ] I have developed this patch against the `main` branch.

dev-tools/scripts/smokeTestRelease.py

@@ -220,6 +220,7 @@ def checkSigs(urlString, version, tmpDir, isSigned, keysFile):
   ents = getDirEntries(urlString)
   artifact = None
   changesURL = None
+  openApiURL = None
   mavenURL = None
   dockerURL = None
   artifactURL = None
@@ -243,6 +244,10 @@ def checkSigs(urlString, version, tmpDir, isSigned, keysFile):
       if text not in ('changes/', 'changes-%s/' % version):
         raise RuntimeError('solr: found %s vs expected changes-%s/' % (text, version))
       changesURL = subURL
+    elif text.startswith('openApi'):
+      if text not in ('openApi/', 'openApi-%s/' % version):
+        raise RuntimeError('solr: found %s vs expected openApi-%s/' % (text, version))
+      openApiURL = subURL
     elif artifact is None:
       artifact = text
       artifactURL = subURL
@@ -296,6 +301,12 @@ def checkSigs(urlString, version, tmpDir, isSigned, keysFile):
     raise RuntimeError('solr is missing changes-%s' % version)
   testChanges(version, changesURL)
 
+  if openApiURL is None:
+    stopGpgAgent(gpgHomeDir, logfile)
+    raise RuntimeError('solr is missing OpenAPI specification' % version)
+  testOpenApi(version, openApiURL)
+
+
   for artifact, urlString in artifacts: # pylint: disable=redefined-argument-from-local
     print('  download %s...' % artifact)
     scriptutil.download(artifact, urlString, tmpDir, force_clean=FORCE_CLEAN)
@@ -349,6 +360,17 @@ def testChanges(version, changesURLString):
   s = load(changesURL)
   checkChangesContent(s, version, changesURL, True)
 
+def testOpenApi(version, openApiDirUrl):
+  print('  check OpenAPI specification...')
+  specFound = False
+  expectedSpecFileName = 'solr-openapi-' + version + '.json'
+  for text, subURL in getDirEntries(openApiDirUrl):
+    if text == expectedSpecFileName:
+      specFound = True
+
+  if not specFound:
+    raise RuntimeError('Did not see %s in %s' % expectedSpecFileName, openApiDirUrl)
+
 
 def testChangesText(dir, version):
   "Checks all CHANGES.txt under this dir."

gradle/testing/randomization/policies/solr-tests.policy

@@ -162,6 +162,11 @@ grant {
   permission javax.security.auth.AuthPermission "createLoginContext.Server";
   permission javax.security.auth.AuthPermission "createLoginContext.Client";
 
+  // Needed by BouncyCastle in jwt-auth tests
+  permission java.security.SecurityPermission "putProviderProperty.BC";
+  permission java.security.SecurityPermission "removeProviderProperty.BC";
+  permission java.security.SecurityPermission "getProperty.org.bouncycastle.x509.allow_non-der_tbscert";
+
   // may only be necessary with Java 7?
   permission javax.security.auth.PrivateCredentialPermission "javax.security.auth.kerberos.KeyTab * \"*\"", "read";
   permission javax.security.auth.PrivateCredentialPermission "sun.security.jgss.krb5.Krb5Util$KeysFromKeyTab * \"*\"", "read";

solr/CHANGES.txt

@@ -124,6 +124,10 @@ Improvements
 
 * SOLR-17063: Do not retain log param references in LogWatcher (Michael Gibney)
 
+* SOLR-17066: SolrClient builders now allow users to specify a "default" collection or core
+  using the `withDefaultCollection` method.  This is preferable to including the collection
+  in the base URL accepted by certain client implementations. (Jason Gerlowski)
+
 Optimizations
 ---------------------
 * SOLR-17084: LBSolrClient (used by CloudSolrClient) now returns the count of core tracked as not live AKA zombies
@@ -152,6 +156,11 @@ Bug Fixes
 * SOLR-17057: JSON Query regression: If "query" is specified with a String (not JSON structure),
   "defType" should parse it.  Since 9.4 defType was ignored.  (David Smiley)
 
+* SOLR-17060: CoreContainer#create may deadlock with concurrent requests for metrics (Alex Deparvu, David Smiley)
+
+* SOLR-17098: ZK Credentials and ACLs are no longer sent to all ZK Servers when using Streaming Expressions.
+  They will only be used when sent to the default ZK Host. (Houston Putman, Jan Høydahl, David Smiley, Gus Heck, Qing Xu)
+
 Dependency Upgrades
 ---------------------
 * SOLR-17012: Update Apache Hadoop to 3.3.6 and Apache Curator to 5.5.0 (Kevin Risden)
@@ -169,6 +178,11 @@ Other Changes
 * SOLR-17091: dev tools script cloud.sh became broken after changes in 9.3 added a new -slim.tgz file it was not expecting
   cloud.sh has been updated to ignore the -slim.tgz version of the tarball.
 
+* SOLR-16949: Restrict certain file types from being uploaded to or downloaded from Config Sets (janhoy, Houston Putman)
+
+* SOLR-16880: Solr now produces an OpenAPI Specification artifact on releases ("solr-openapi-x.y.z.json") that covers
+  Solr's v2 APIs. (Jason Gerlowski, Houston Putman)
+
 ==================  9.4.0 ==================
 New Features
 ---------------------

solr/api/build.gradle

@@ -28,7 +28,7 @@ ext {
     jsClientDir = "${buildDir}/generated/js"
     pythonClientDir = "${buildDir}/generated/python"
     openApiSpecDir = "${buildDir}/generated/openapi"
-    openApiSpecFile = "${project.openApiSpecDir}/openapi.json"
+    openApiSpecFile = "${project.openApiSpecDir}/solr-openapi-${version}.json"
 }
 
 configurations {
@@ -50,6 +50,7 @@ resolve {
   classpath = sourceSets.main.runtimeClasspath
   resourcePackages = ["org.apache.solr.client.api.util", "org.apache.solr.client.api.endpoint"]
   outputDir = file(project.openApiSpecDir)
+  outputFileName = "solr-openapi-${version}"
   prettyPrint = true
 }
 
@@ -91,7 +92,7 @@ tasks.withType(org.openapitools.generator.gradle.plugin.tasks.GenerateTask) {
 
 artifacts {
     // Ensure the OAS is available to other modules who want to generate code (i.e. solrj)
-    openapiSpec resolve.outputDir, {
+    openapiSpec file(openApiSpecFile), {
         builtBy resolve
     }
 

solr/core/build.gradle

@@ -159,6 +159,8 @@ dependencies {
 
   compileOnly 'com.github.stephenc.jcip:jcip-annotations'
 
+  implementation 'com.j256.simplemagic:simplemagic'
+
   // -- Test Dependencies
 
   testRuntimeOnly 'org.slf4j:jcl-over-slf4j'

solr/core/src/java/org/apache/solr/cli/ConfigSetUploadTool.java

@@ -27,6 +27,7 @@
 import org.apache.solr.common.cloud.SolrZkClient;
 import org.apache.solr.common.cloud.ZkMaintenanceUtils;
 import org.apache.solr.core.ConfigSetService;
+import org.apache.solr.util.FileTypeMagicUtil;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -101,6 +102,7 @@ public void runImpl(CommandLine cli) throws Exception {
               + cli.getOptionValue("confname")
               + " to ZooKeeper at "
               + zkHost);
+      FileTypeMagicUtil.assertConfigSetFolderLegal(confPath);
       ZkMaintenanceUtils.uploadToZK(
           zkClient,
           confPath,

solr/core/src/java/org/apache/solr/cloud/ZkConfigSetService.java

@@ -22,6 +22,7 @@
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
+import java.util.Locale;
 import java.util.Map;
 import java.util.Objects;
 import org.apache.solr.client.solrj.cloud.SolrCloudManager;
@@ -39,6 +40,7 @@
 import org.apache.solr.core.CoreDescriptor;
 import org.apache.solr.core.SolrConfig;
 import org.apache.solr.core.SolrResourceLoader;
+import org.apache.solr.util.FileTypeMagicUtil;
 import org.apache.zookeeper.CreateMode;
 import org.apache.zookeeper.KeeperException;
 import org.apache.zookeeper.data.Stat;
@@ -199,6 +201,15 @@ public void uploadFileToConfig(
     try {
       if (ZkMaintenanceUtils.isFileForbiddenInConfigSets(fileName)) {
         log.warn("Not including uploading file to config, as it is a forbidden type: {}", fileName);
+      } else if (FileTypeMagicUtil.isFileForbiddenInConfigset(data)) {
+        String mimeType = FileTypeMagicUtil.INSTANCE.guessMimeType(data);
+        throw new SolrException(
+            SolrException.ErrorCode.BAD_REQUEST,
+            String.format(
+                Locale.ROOT,
+                "Not uploading file %s to config, as it matched the MAGIC signature of a forbidden mime type %s",
+                fileName,
+                mimeType));
       } else {
         // if overwriteOnExists is true then zkClient#makePath failOnExists is set to false
         zkClient.makePath(filePath, data, CreateMode.PERSISTENT, null, !overwriteOnExists, true);
@@ -340,7 +351,15 @@ private void copyData(String fromZkFilePath, String toZkFilePath)
     } else {
       log.debug("Copying zk node {} to {}", fromZkFilePath, toZkFilePath);
       byte[] data = zkClient.getData(fromZkFilePath, null, null, true);
-      zkClient.makePath(toZkFilePath, data, true);
+      if (!FileTypeMagicUtil.isFileForbiddenInConfigset(data)) {
+        zkClient.makePath(toZkFilePath, data, true);
+      } else {
+        String mimeType = FileTypeMagicUtil.INSTANCE.guessMimeType(data);
+        log.warn(
+            "Skipping copy of file {} in ZK, as it matched the MAGIC signature of a forbidden mime type {}",
+            fromZkFilePath,
+            mimeType);
+      }
     }
   }
 

solr/core/src/java/org/apache/solr/core/CoreContainer.java

@@ -829,6 +829,7 @@ private void loadInternal() {
 
     zkSys.initZooKeeper(this, cfg.getCloudConfig());
     if (isZooKeeperAware()) {
+      solrClientCache.setDefaultZKHost(getZkController().getZkServerAddress());
       // initialize ZkClient metrics
       zkSys.getZkMetricsProducer().initializeMetrics(solrMetricsContext, "zkClient");
       pkiAuthenticationSecurityBuilder =
@@ -1298,7 +1299,11 @@ public void shutdown() {
         }
       }
 
-      objectCache.clear();
+      try {
+        objectCache.close();
+      } catch (IOException e) {
+        log.warn("Exception while closing ObjectCache.", e);
+      }
 
       // It's still possible that one of the pending dynamic load operation is waiting, so wake it
       // up if so. Since all the pending operations queues have been drained, there should be

solr/core/src/java/org/apache/solr/core/FileSystemConfigSetService.java

@@ -37,6 +37,7 @@
 import org.apache.solr.common.SolrException;
 import org.apache.solr.common.cloud.ZkMaintenanceUtils;
 import org.apache.solr.common.util.Utils;
+import org.apache.solr.util.FileTypeMagicUtil;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -150,9 +151,17 @@ public void uploadFileToConfig(
     if (ZkMaintenanceUtils.isFileForbiddenInConfigSets(fileName)) {
       log.warn("Not including uploading file to config, as it is a forbidden type: {}", fileName);
     } else {
-      Path filePath = getConfigDir(configName).resolve(normalizePathToOsSeparator(fileName));
-      if (!Files.exists(filePath) || overwriteOnExists) {
-        Files.write(filePath, data);
+      if (!FileTypeMagicUtil.isFileForbiddenInConfigset(data)) {
+        Path filePath = getConfigDir(configName).resolve(normalizePathToOsSeparator(fileName));
+        if (!Files.exists(filePath) || overwriteOnExists) {
+          Files.write(filePath, data);
+        }
+      } else {
+        String mimeType = FileTypeMagicUtil.INSTANCE.guessMimeType(data);
+        log.warn(
+            "Not including uploading file {}, as it matched the MAGIC signature of a forbidden mime type {}",
+            fileName,
+            mimeType);
       }
     }
   }
@@ -205,8 +214,16 @@ public FileVisitResult visitFile(Path file, BasicFileAttributes attrs)
                     "Not including uploading file to config, as it is a forbidden type: {}",
                     file.getFileName());
               } else {
-                Files.copy(
-                    file, target.resolve(source.relativize(file).toString()), REPLACE_EXISTING);
+                if (!FileTypeMagicUtil.isFileForbiddenInConfigset(file)) {
+                  Files.copy(
+                      file, target.resolve(source.relativize(file).toString()), REPLACE_EXISTING);
+                } else {
+                  String mimeType = FileTypeMagicUtil.INSTANCE.guessMimeType(file);
+                  log.warn(
+                      "Not copying file {}, as it matched the MAGIC signature of a forbidden mime type {}",
+                      file.getFileName(),
+                      mimeType);
+                }
               }
               return FileVisitResult.CONTINUE;
             }

solr/core/src/java/org/apache/solr/core/SolrCore.java

@@ -259,6 +259,7 @@ public class SolrCore implements SolrInfoBean, Closeable {
   private Counter newSearcherCounter;
   private Counter newSearcherMaxReachedCounter;
   private Counter newSearcherOtherErrorsCounter;
+  private volatile boolean newSearcherReady = false;
 
   private final String metricTag = SolrMetricProducer.getUniqueMetricTag(this, null);
   private final SolrMetricsContext solrMetricsContext;
@@ -1338,14 +1339,14 @@ public void initializeMetrics(SolrMetricsContext parentContext, String scope) {
         "sizeInBytes",
         Category.INDEX.toString());
     parentContext.gauge(
-        () -> isClosed() ? parentContext.nullNumber() : getSegmentCount(),
+        () -> isClosed() ? parentContext.nullString() : NumberUtils.readableSize(getIndexSize()),
         true,
-        "segments",
+        "size",
         Category.INDEX.toString());
     parentContext.gauge(
-        () -> isClosed() ? parentContext.nullString() : NumberUtils.readableSize(getIndexSize()),
+        () -> isReady() ? getSegmentCount() : parentContext.nullNumber(),
         true,
-        "size",
+        "segments",
         Category.INDEX.toString());
 
     final CloudDescriptor cd = getCoreDescriptor().getCloudDescriptor();
@@ -1899,6 +1900,11 @@ public boolean isClosed() {
     return refCount.get() <= 0;
   }
 
+  /** Returns true if the core is ready for use. It is not initializing or closing/closed. */
+  public boolean isReady() {
+    return !isClosed() && newSearcherReady;
+  }
+
   private Collection<CloseHook> closeHooks = null;
 
   /** Add a close callback hook */
@@ -2107,6 +2113,7 @@ public RefCounted<SolrIndexSearcher> getSearcher() {
    * because it might be closed soon after this method returns; it really depends.
    */
   public <R> R withSearcher(IOFunction<SolrIndexSearcher, R> lambda) throws IOException {
+    assert isReady();
     final RefCounted<SolrIndexSearcher> refCounted = getSearcher();
     try {
       return lambda.apply(refCounted.get());
@@ -2704,7 +2711,6 @@ public RefCounted<SolrIndexSearcher> getSearcher(
 
       if (!success) {
         newSearcherOtherErrorsCounter.inc();
-        ;
         synchronized (searcherLock) {
           onDeckSearchers--;
 
@@ -2734,6 +2740,7 @@ public RefCounted<SolrIndexSearcher> getSearcher(
       // we want to do this after we decrement onDeckSearchers so another thread
       // doesn't increment first and throw a false warning.
       openSearcherLock.unlock();
+      newSearcherReady = true;
     }
   }
 

solr/core/src/java/org/apache/solr/core/backup/BackupManager.java

@@ -40,6 +40,7 @@
 import org.apache.solr.common.util.Utils;
 import org.apache.solr.core.ConfigSetService;
 import org.apache.solr.core.backup.repository.BackupRepository;
+import org.apache.solr.util.FileTypeMagicUtil;
 import org.apache.zookeeper.CreateMode;
 import org.apache.zookeeper.KeeperException;
 import org.slf4j.Logger;
@@ -349,8 +350,16 @@ private void downloadConfigToRepo(ConfigSetService configSetService, String conf
           if (data == null) {
             data = new byte[0];
           }
-          try (OutputStream os = repository.createOutput(uri)) {
-            os.write(data);
+          if (!FileTypeMagicUtil.isFileForbiddenInConfigset(data)) {
+            try (OutputStream os = repository.createOutput(uri)) {
+              os.write(data);
+            }
+          } else {
+            String mimeType = FileTypeMagicUtil.INSTANCE.guessMimeType(data);
+            log.warn(
+                "Not including zookeeper file {} in backup, as it matched the MAGIC signature of a forbidden mime type {}",
+                filePath,
+                mimeType);
           }
         }
       } else {
@@ -379,7 +388,15 @@ private void uploadConfigToSolrCloud(
                 // probably ok since the config file should be small.
                 byte[] arr = new byte[(int) is.length()];
                 is.readBytes(arr, 0, (int) is.length());
-                configSetService.uploadFileToConfig(configName, filePath, arr, false);
+                if (!FileTypeMagicUtil.isFileForbiddenInConfigset(arr)) {
+                  configSetService.uploadFileToConfig(configName, filePath, arr, false);
+                } else {
+                  String mimeType = FileTypeMagicUtil.INSTANCE.guessMimeType(arr);
+                  log.warn(
+                      "Not including zookeeper file {} in restore, as it matched the MAGIC signature of a forbidden mime type {}",
+                      filePath,
+                      mimeType);
+                }
               }
             }
             break;