Merge remote-tracking branch 'upstream/main' into SOLR-14496

build.gradle

@@ -23,7 +23,7 @@ plugins {
   id "com.palantir.consistent-versions" version "2.11.0"
   id "org.owasp.dependencycheck" version "8.0.1"
   id 'ca.cutterslade.analyze' version "1.9.0"
-  id 'de.thetaphi.forbiddenapis' version '3.5' apply false
+  id 'de.thetaphi.forbiddenapis' version '3.6' apply false
   id "de.undercouch.download" version "5.2.0" apply false
   id "net.ltgt.errorprone" version "3.0.1" apply false
   id 'com.diffplug.spotless' version "6.5.2" apply false

gradle/validation/error-prone.gradle

@@ -85,6 +85,7 @@ allprojects { prj ->
             // '-Xep:AutoValueConstructorOrderChecker:OFF', // we don't use autovalue
             '-Xep:BadAnnotationImplementation:ERROR',
             '-Xep:BadShiftAmount:ERROR',
+            '-Xep:BanClassLoader:ERROR',
             '-Xep:BanJNDI:ERROR', // todo - implement with forbidden APIs instead?
             '-Xep:BoxedPrimitiveEquality:ERROR',
             // '-Xep:BundleDeserializationCast:OFF', // we don't use android
@@ -104,6 +105,7 @@ allprojects { prj ->
             '-Xep:DangerousLiteralNull:ERROR',
             '-Xep:DeadException:ERROR',
             '-Xep:DeadThread:ERROR',
+            '-Xep:DereferenceWithNullBranch:ERROR',
             '-Xep:DiscardedPostfixExpression:ERROR',
             // '-Xep:DoNotCall:OFF', // we don't use this annotation
             '-Xep:DoNotMock:ERROR',
@@ -262,6 +264,7 @@ allprojects { prj ->
             '-Xep:AssertThrowsMultipleStatements:WARN',
             '-Xep:AssertionFailureIgnored:WARN',
             '-Xep:AssistedInjectAndInjectOnSameConstructor:WARN',
+            '-Xep:AttemptedNegativeZero:WARN',
             // '-Xep:AutoValueFinalMethods:OFF', // we don't use autovalue
             // '-Xep:AutoValueImmutableFields:OFF', // we don't use autovalue
             // '-Xep:AutoValueSubclassLeaked:OFF', // we don't use autovalue
@@ -284,6 +287,7 @@ allprojects { prj ->
             '-Xep:ClassCanBeStatic:WARN',
             '-Xep:ClassNewInstance:WARN',
             // '-Xep:CloseableProvides:OFF', // we don't use this annotation
+            '-Xep:ClosingStandardOutputStreams:WARN',
             '-Xep:CollectionUndefinedEquality:WARN',
             '-Xep:CollectorShouldNotUseState:WARN',
             '-Xep:ComparableAndComparator:WARN',
@@ -299,6 +303,7 @@ allprojects { prj ->
             // '-Xep:DoNotCallSuggester:OFF', // we don't use this annotation
             // '-Xep:DoNotClaimAnnotations:OFF', // we don't use this annotation
             // '-Xep:DoNotMockAutoValue:OFF', // we don't use autovalue
+            // '-Xep:DoNotUseRuleChain:OFF', // todo could be fixed but not easy
             // '-Xep:DoubleCheckedLocking:OFF', // todo check if useful or comment why not
             '-Xep:EmptyBlockTag:WARN',
             // '-Xep:EmptyCatch:OFF', // todo check if useful or comment why not - might be handled by ECJ?
@@ -325,6 +330,7 @@ allprojects { prj ->
             // '-Xep:FutureReturnValueIgnored:OFF', // todo there are problems that should be fixed
             '-Xep:GetClassOnEnum:WARN',
             '-Xep:HidingField:WARN',
+            '-Xep:ICCProfileGetInstance:WARN',
             '-Xep:IdentityHashMapUsage:WARN',
             // '-Xep:ImmutableAnnotationChecker:OFF', // we don't use this annotation
             '-Xep:ImmutableEnumChecker:WARN',
@@ -343,6 +349,7 @@ allprojects { prj ->
             // '-Xep:InlineMeInliner:OFF', // we don't use this annotation
             // '-Xep:InlineMeSuggester:OFF', // We don't use this annotation
             // '-Xep:InputStreamSlowMultibyteRead:OFF', // todo check if useful or comment why not
+            '-Xep:InlineTrivialConstant:WARN',
             '-Xep:InstanceOfAndCastMatchWrongType:WARN',
             '-Xep:IntLongMath:WARN',
             // '-Xep:InvalidBlockTag:OFF', // this is needed for tags like lucene.internal
@@ -379,6 +386,7 @@ allprojects { prj ->
             '-Xep:LiteEnumValueOf:WARN',
             '-Xep:LiteProtoToString:WARN',
             '-Xep:LockNotBeforeTry:WARN',
+            '-Xep:LockOnNonEnclosingClassLiteral:WARN',
             '-Xep:LogicalAssignment:WARN',
             '-Xep:LongDoubleConversion:WARN',
             '-Xep:LongFloatConversion:WARN',
@@ -390,25 +398,32 @@ allprojects { prj ->
             '-Xep:MissingFail:WARN',
             '-Xep:MissingImplementsComparable:WARN',
             '-Xep:MissingOverride:WARN',
+            // '-Xep:MissingRefasterAnnotation:OFF', // don't use Refaster
             // '-Xep:MissingSummary:OFF', // style preference that we don't want to enforce
             // '-Xep:MixedMutabilityReturnType:OFF', // todo check if useful or comment why not
             '-Xep:MockNotUsedInProduction:WARN',
             '-Xep:ModifiedButNotUsed:WARN',
             '-Xep:ModifyCollectionInEnhancedForLoop:WARN',
             '-Xep:ModifySourceCollectionInStream:WARN',
+            '-Xep:MultimapKeys:WARN',
             '-Xep:MultipleParallelOrSequentialCalls:WARN',
             '-Xep:MultipleUnaryOperatorsInMethodCall:WARN',
+            // '-Xep:MutableGuiceModule:OFF', // we don't use guice
             '-Xep:MutablePublicArray:WARN',
+            '-Xep:NamedLikeContextualKeyword:WARN',
             '-Xep:NarrowCalculation:WARN',
             '-Xep:NarrowingCompoundAssignment:WARN',
             // '-Xep:NegativeCharLiteral:OFF', // todo check if useful or comment why not
             '-Xep:NestedInstanceOfConditions:WARN',
             '-Xep:NewFileSystem:WARN',
+            //'-Xep:NonApiType:OFF', // todo could be fixed but lots of changes
             // '-Xep:NonAtomicVolatileUpdate:OFF', // todo check if useful or comment why not
             '-Xep:NonCanonicalType:WARN',
             '-Xep:NonOverridingEquals:WARN',
+            '-Xep:NotJavadoc:WARN',
             '-Xep:NullOptional:WARN',
             // '-Xep:NullableConstructor:OFF', // we don't use this annotation
+            '-Xep:NullableOptional:WARN',
             // '-Xep:NullablePrimitive:OFF', // we don't use this annotation
             // '-Xep:NullablePrimitiveArray:OFF', // we don't use this annotation
             // '-Xep:NullableVoid:OFF', // we don't use this annotation
@@ -420,9 +435,10 @@ allprojects { prj ->
             '-Xep:OptionalNotPresent:WARN',
             '-Xep:OrphanedFormatString:WARN',
             // '-Xep:OutlineNone:OFF', // we don't use gwt
+            '-Xep:OverridingMethodInconsistentArgumentNamesChecker:WARN',
             '-Xep:OverrideThrowableToString:WARN',
             '-Xep:Overrides:WARN',
-            '-Xep:OverridesGuiceInjectableMethod:WARN',
+            // '-Xep:OverridesGuiceInjectableMethod:OFF', // we don't use guice
             '-Xep:ParameterName:WARN',
             '-Xep:PreconditionsCheckNotNullRepeated:WARN',
             '-Xep:PrimitiveAtomicReference:WARN',
@@ -434,6 +450,7 @@ allprojects { prj ->
             '-Xep:ReachabilityFenceUsage:WARN',
             '-Xep:ReferenceEquality:WARN',
             '-Xep:RethrowReflectiveOperationExceptionAsLinkageError:WARN',
+            '-Xep:ReturnAtTheEndOfVoidFunction:WARN',
             '-Xep:ReturnFromVoid:WARN',
             '-Xep:RobolectricShadowDirectlyOn:WARN',
             '-Xep:RxReturnValueIgnored:WARN',
@@ -446,6 +463,7 @@ allprojects { prj ->
             '-Xep:StaticMockMember:WARN',
             // '-Xep:StreamResourceLeak:OFF', // todo check if useful or comment why not
             '-Xep:StreamToIterable:WARN',
+            // '-Xep:StringCaseLocaleUsage:OFF', // we have forbiddenapis for that
             // '-Xep:StringSplitter:OFF', // todo check if useful or comment why not - might be able to use forbidden-apis for this?
             // '-Xep:SwigMemoryLeak:OFF', // we don't use swig
             // '-Xep:SynchronizeOnNonFinalField:OFF', // todo check if useful or comment why not
@@ -457,6 +475,7 @@ allprojects { prj ->
             '-Xep:ToStringReturnsNull:WARN',
             // '-Xep:TruthAssertExpected:OFF', // we don't use truth
             // '-Xep:TruthConstantAsserts:OFF', // we don't use truth
+            // '-Xep:TruthContainsExactlyElementsInUsage:OFF', // we don't use truth
             // '-Xep:TruthGetOrDefault:OFF', // we don't use truth
             // '-Xep:TruthIncompatibleType:OFF', // we don't use truth
             '-Xep:TypeEquals:WARN',
@@ -468,11 +487,14 @@ allprojects { prj ->
             '-Xep:UnescapedEntity:WARN',
             // '-Xep:UnicodeEscape:OFF', // can't enable since Lucene/Solr tests use unicode a bunch
             // '-Xep:UnnecessaryAssignment:OFF', // we don't use these annotations
+            '-Xep:UnnecessaryAsync:WARN',
             '-Xep:UnnecessaryLambda:WARN',
             '-Xep:UnnecessaryLongToIntConversion:WARN',
             '-Xep:UnnecessaryMethodInvocationMatcher:WARN',
             '-Xep:UnnecessaryMethodReference:WARN',
             // '-Xep:UnnecessaryParentheses:OFF', // style preference that we don't want to enforce
+            '-Xep:UnnecessaryStringBuilder:WARN',
+            // '-Xep:UnnecessaryTestMethodPrefix:OFF', // style preference that we don't want to enforce
             // '-Xep:UnqualifiedYield:OFF', // javac takes care
             '-Xep:UnrecognisedJavadocTag:WARN',
             '-Xep:UnsafeFinalization:WARN',

solr/CHANGES.txt

@@ -72,6 +72,10 @@ New Features
 
 * SOLR-16954: Make Circuit Breakers available for Update Requests (janhoy, Christine Poerschke, Pierre Salagnac)
 
+* SOLR-15056: A new Circuit breaker for percentage of CPU utilization is added. The former "CPU" circuit breaker
+  is now more correctly named LoadAverageCircuitBreaker as it trips on system load average which is not a percentage.
+  Users of legacy CircuitBreakerManager are not affected by this change. (Walter Underwood, janhoy, Christine Poerschke, Atri Sharma)
+
 * SOLR-15771: bin/auth creates reasonable roles and permissions for security: 'search', 'index', 'admin', and 'superadmin' and assigns user superadmin role. (Eric Pugh, janhoy)
 
 * SOLR-15367: Convert "rid" functionality into a default Tracer (Alex Deparvu, David Smiley)
@@ -109,6 +113,8 @@ Improvements
 
 * SOLR-15474: Make Circuit breakers individually pluggable (Atri Sharma, Christine Poerschke, janhoy)
 
+* SOLR-16982: Trip Circuit Breakers only for external requests (janhoy, Christine Poerschke)
+
 * SOLR-16927: Allow SolrClientCache clients to use Jetty HTTP2 clients (Alex Deparvu, David Smiley)
 
 * SOLR-16896, SOLR-16897: Add support of OAuth 2.0/OIDC 'code with PKCE' flow (Lamine Idjeraoui, janhoy, Kevin Risden, Anshum Gupta)
@@ -123,6 +129,8 @@ Improvements
 
 * SOLR-16970: SOLR_OPTS is now able to override options set by the Solr control scripts, "bin/solr" and "bin/solr.cmd". (Houston Putman)
 
+* SOLR-16968: The MemoryCircuitBreaker now uses average heap usage over the last 30 seconds (janhoy, Christine Poerschke)
+
 * SOLR-14886: Suppress stack traces in query response (Isabelle Giguere via Alex Deparvu)
 
 * SOLR-16461: `/solr/coreName/replication?command=backup` now has a v2 equivalent, available at
@@ -134,13 +142,21 @@ Improvements
 
 * SOLR-15440: The Learning To Rank FieldValueFeature now uses DocValues when docValues=true and stored=true are combined. (Christine Poerschke, Tom Gilke)
 
+* SOLR-16959: Make the internal CoresLocator implementation configurable in solr.xml (Vincent Primault via David Smiley)
+
+* SOLR-16967: Some ConfigSet operations formerly required that solrconfig.xml exist but should not have because
+  the name of the file is configurable when creating cores / collections.  (David Smiley)
+
 Optimizations
 ---------------------
 
 * SOLR-16845: BinaryResponseWriter should not attempt cast to Utf8CharSequence (Alex Deparvu via Houston Putman)
 
 * SOLR-16265: reduce memory usage of ContentWriter based requests in Http2SolrClient (Alex Deparvu, Kevin Risden, David Smiley)
 
+* SOLR-16989: Optimize and consolidate reuse of DocValues iterators for value retrieval (Michael Gibney)
+
+* SOLR-17004: ZkStateReader waitForState should check clusterState before using watchers (Kevin Risden)
 
 Bug Fixes
 ---------------------
@@ -192,11 +208,36 @@ Bug Fixes
 
 * SOLR-16971: RealTimeGet with Composite router throws NPE (Alex Deparvu)
 
+* SOLR-16931: ReRankScaler explain breaks with debug=true and in distributed mode (Joel Bernstein)
+
+* SOLR-16983: Fixed a bug that could cause some usages of SolrStream to fail to close InputStreams from the server.
+  Also fixed the usage of ObjectReleaseTracker in SolrTestCaseJ4 to catch these kinds of bugs (hossman)
+
+* SOLR-16925: Fix indentation for JacksonJsonWriter (Houston Putman)
+
+* SOLR-16701: Fix race condition on PRS enabled collection deletion (Patson Luk)
+
+* SOLR-16991: Concurrent requests failing JWT authentication in Admin UI intermittently (Lamine Idjeraoui, janhoy)
+
+* SOLR-16997: OTEL configurator NPE when SOLR_HOST not set (janhoy)
+
+* PR#1963: Fix the admin UI green core-size graph on nodes screen (janhoy)
+
+* SOLR-16980: Connect to SOLR standalone with basic authentication (Alex Deparvu)
+
+* SOLR-16992: Non-reproducible StreamingTest failures -- suggests CloudSolrStream concurency race condition (Alex Deparvu, hossman)
+
+* SOLR-16644: Fixing the entropy warning threshold using scaling based on poolsize (Raghavan Muthuregunathan)
+
 Dependency Upgrades
 ---------------------
 
 * PR#1846: Update io.opentelemetry to 1.29.0 (Alex Deparvu)
 
+* SOLR-16985: Upgrade Lucene to 9.8.0 (Alex Deparvu, Christine Poerschke)
+
+* PR#1971: Update forbiddenapis to 3.6 to support Java 21/22 and commons-io up to 2.14.0 (Uwe Schindler)
+
 Other Changes
 ---------------------
 
@@ -214,6 +255,10 @@ Other Changes
 
 * SOLR-16979: BATS integration tests now start solr instances on a randomly selected port (janhoy)
 
+* SOLR-16978: Be case insensitive when parsing booleans from text (Tomás Fernández Löbbe)
+
+* SOLR-16960: Tests should sometimes run with a Tracer (not no-op) (Alex Deparvu)
+
 ==================  9.3.0 ==================
 
 Upgrade Notes
@@ -376,6 +421,9 @@ Improvements
 
 * SOLR-16878: Use Log4J JUL manager when starting Java. This is necessary for Lucene logs to be included with Solr logs. (Houston Putman, Uwe Schindler)
 
+* SOLR-16397: Reload core v2 endpoints have been updated to be more REST-ful.
+  RELOAD is now available at `POST /api/cores/coreName/reload` (Sanjay Dutt via Jason Gerlowski)
+
 Optimizations
 ---------------------
 

solr/bin/solr

@@ -1915,12 +1915,26 @@ function start_solr() {
         -jar start.jar "${SOLR_JETTY_CONFIG[@]}" $SOLR_JETTY_ADDL_CONFIG \
 	1>"$SOLR_LOGS_DIR/solr-$SOLR_PORT-console.log" 2>&1 & echo $! > "$SOLR_PID_DIR/solr-$SOLR_PORT.pid"
 
-    # check if /proc/sys/kernel/random/entropy_avail exists then check output of cat /proc/sys/kernel/random/entropy_avail to see if less than 300
-    if [[ -f /proc/sys/kernel/random/entropy_avail ]] && (( $(cat /proc/sys/kernel/random/entropy_avail) < 300)); then
-	echo "Warning: Available entropy is low. As a result, use of the UUIDField, SSL, or any other features that require"
-	echo "RNG might not work properly. To check for the amount of available entropy, use 'cat /proc/sys/kernel/random/entropy_avail'."
-	echo ""
+    # Get the current entropy available
+    entropy_avail=$(cat /proc/sys/kernel/random/entropy_avail)
+
+    # Get the pool size
+    pool_size=$(cat /proc/sys/kernel/random/poolsize)
+
+    # Check if entropy is available and pool size is non-zero
+    if [[ $entropy_avail -gt 0 && $pool_size -ne 0 ]]; then
+        # Compute the ratio of entropy available to pool size
+        ratio=$(awk -v ea="$entropy_avail" -v ps="$pool_size" 'BEGIN {print (ea/ps)*100}')
+
+        # Check if the ratio is less than 25%
+        if (( $(echo "$ratio < 25" | bc -l) )); then
+          echo "Warning: Available entropy is low. As a result, use of the UUIDField, SSL, or any other features that require"
+          echo "RNG might not work properly. To check for the amount of available entropy, use 'cat /proc/sys/kernel/random/entropy_avail'."
+        fi
+    else
+        echo "Error: Either no entropy is available or the pool size is zero."
     fi
+
     # no lsof on cygwin though
     if lsof -v 2>&1 | grep -q revision; then
       echo -n "Waiting up to $SOLR_START_WAIT seconds to see Solr running on port $SOLR_PORT"

solr/core/src/java/org/apache/solr/cli/SimplePostTool.java

@@ -426,7 +426,6 @@ private void doWebMode() {
     } catch (MalformedURLException e) {
       fatal("Wrong URL trying to append /extract to " + solrUrl);
     }
-    return;
   }
 
   private void doStdinMode() {

solr/core/src/java/org/apache/solr/cloud/ZkConfigSetService.java

@@ -139,10 +139,7 @@ public String configSetName(CoreDescriptor cd) {
   @Override
   public boolean checkConfigExists(String configName) throws IOException {
     try {
-      Boolean existsSolrConfigXml =
-          zkClient.exists(CONFIGS_ZKNODE + "/" + configName + "/solrconfig.xml", true);
-      if (existsSolrConfigXml == null) return false;
-      return existsSolrConfigXml;
+      return zkClient.exists(CONFIGS_ZKNODE + "/" + configName, true);
     } catch (KeeperException | InterruptedException e) {
       throw new IOException(
           "Error checking whether config exists", SolrZkClient.checkInterrupted(e));

solr/core/src/java/org/apache/solr/core/CoreContainer.java

@@ -26,7 +26,6 @@
 import static org.apache.solr.common.params.CommonParams.METRICS_PATH;
 import static org.apache.solr.common.params.CommonParams.ZK_PATH;
 import static org.apache.solr.common.params.CommonParams.ZK_STATUS_PATH;
-import static org.apache.solr.core.CorePropertiesLocator.PROPERTIES_FILENAME;
 import static org.apache.solr.security.AuthenticationPlugin.AUTHENTICATION_PLUGIN_PROP;
 
 import com.github.benmanes.caffeine.cache.Interner;
@@ -377,21 +376,24 @@ public CoreContainer(Path solrHome, Properties properties) {
    * @see #load()
    */
   public CoreContainer(NodeConfig config) {
-    this(config, new CorePropertiesLocator(config.getCoreRootDirectory()));
+    this(config, CoresLocator.instantiate(config));
   }
 
   public CoreContainer(NodeConfig config, boolean asyncSolrCoreLoad) {
-    this(config, new CorePropertiesLocator(config.getCoreRootDirectory()), asyncSolrCoreLoad);
+    this(config, CoresLocator.instantiate(config), asyncSolrCoreLoad);
   }
 
   /**
-   * Create a new CoreContainer using the given configuration and locator. The container's cores are
-   * not loaded.
+   * Create a new CoreContainer using the given configuration and locator.
+   *
+   * <p>The container's cores are not loaded. This constructor should be used only in tests, as it
+   * overrides {@link CoresLocator}'s instantiation process.
    *
    * @param config a ConfigSolr representation of this container's configuration
    * @param locator a CoresLocator
    * @see #load()
    */
+  @VisibleForTesting
   public CoreContainer(NodeConfig config, CoresLocator locator) {
     this(config, locator, false);
   }
@@ -1945,9 +1947,7 @@ private CoreDescriptor reloadCoreDescriptor(CoreDescriptor oldDesc) {
       return null;
     }
 
-    CorePropertiesLocator cpl = new CorePropertiesLocator(null);
-    CoreDescriptor ret =
-        cpl.buildCoreDescriptor(oldDesc.getInstanceDir().resolve(PROPERTIES_FILENAME), this);
+    CoreDescriptor ret = getCoresLocator().reload(oldDesc, this);
 
     // Ok, this little jewel is all because we still create core descriptors on the fly from lists
     // of properties in tests particularly. Theoretically, there should be _no_ way to create a