SOLR-16974: Global Circuit Breakers (#1919)

Co-authored-by: Christine Poerschke <[email protected]>

solr/bin/solr

@@ -1406,6 +1406,28 @@ if [ "${SOLR_ENABLE_STREAM_BODY:-false}" == "true" ]; then
   SCRIPT_SOLR_OPTS+=("-Dsolr.enableStreamBody=true")
 fi
 
+# Parse global circuit breaker env vars and convert to dot separated, lowercase properties
+if [ -n "${SOLR_CIRCUITBREAKER_UPDATE_CPU:-}" ]; then
+  SOLR_OPTS+=("-Dsolr.circuitbreaker.update.cpu=$SOLR_CIRCUITBREAKER_UPDATE_CPU")
+fi
+if [ -n "${SOLR_CIRCUITBREAKER_UPDATE_MEM:-}" ]; then
+  SOLR_OPTS+=("-Dsolr.circuitbreaker.update.mem=$SOLR_CIRCUITBREAKER_UPDATE_MEM")
+fi
+if [ -n "${SOLR_CIRCUITBREAKER_UPDATE_LOADAVG:-}" ]; then
+  SOLR_OPTS+=("-Dsolr.circuitbreaker.update.loadavg=$SOLR_CIRCUITBREAKER_UPDATE_LOADAVG")
+fi
+if [ -n "${SOLR_CIRCUITBREAKER_QUERY_CPU:-}" ]; then
+  SOLR_OPTS+=("-Dsolr.circuitbreaker.query.cpu=$SOLR_CIRCUITBREAKER_QUERY_CPU")
+fi
+if [ -n "${SOLR_CIRCUITBREAKER_QUERY_MEM:-}" ]; then
+  SOLR_OPTS+=("-Dsolr.circuitbreaker.query.mem=$SOLR_CIRCUITBREAKER_QUERY_MEM")
+fi
+if [ -n "${SOLR_CIRCUITBREAKER_QUERY_LOADAVG:-}" ]; then
+  SOLR_OPTS+=("-Dsolr.circuitbreaker.query.loadavg=$SOLR_CIRCUITBREAKER_QUERY_LOADAVG")
+fi
+
+echo "SOLR_OPTS is now: ${SOLR_OPTS[*]}"
+
 : ${SOLR_SERVER_DIR:=$DEFAULT_SERVER_DIR}
 
 if [ ! -e "$SOLR_SERVER_DIR" ]; then

solr/core/src/java/org/apache/solr/core/SolrCore.java

@@ -246,7 +246,7 @@ public class SolrCore implements SolrInfoBean, Closeable {
   private final ConfigSet configSet;
   // singleton listener for all packages used in schema
 
-  private final CircuitBreakerRegistry circuitBreakerRegistry = new CircuitBreakerRegistry();
+  private final CircuitBreakerRegistry circuitBreakerRegistry;
 
   private final List<Runnable> confListeners = new CopyOnWriteArrayList<>();
 
@@ -1072,6 +1072,7 @@ private SolrCore(
     final CountDownLatch latch = new CountDownLatch(1);
     try {
       this.coreContainer = coreContainer;
+      this.circuitBreakerRegistry = new CircuitBreakerRegistry(coreContainer);
       this.configSet = configSet;
       this.coreDescriptor = Objects.requireNonNull(coreDescriptor, "coreDescriptor cannot be null");
       this.name = Objects.requireNonNull(coreDescriptor.getName());
@@ -3173,6 +3174,9 @@ public <T> T initPlugins(
             type.getSimpleName() + "." + info.name, (SolrMetricProducer) o);
       }
       if (o instanceof CircuitBreaker) {
+        if (o instanceof SolrCoreAware) {
+          ((SolrCoreAware) o).inform(this);
+        }
         circuitBreakerRegistry.register((CircuitBreaker) o);
       }
       if (info.isDefault()) {

solr/core/src/java/org/apache/solr/core/SolrResourceLoader.java

@@ -72,6 +72,7 @@
 import org.apache.solr.schema.SimilarityFactory;
 import org.apache.solr.search.QParserPlugin;
 import org.apache.solr.update.processor.UpdateRequestProcessorFactory;
+import org.apache.solr.util.circuitbreaker.CircuitBreaker;
 import org.apache.solr.util.plugin.SolrCoreAware;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -814,6 +815,7 @@ public Path getInstancePath() {
         new Class<?>[] {
           // DO NOT ADD THINGS TO THIS LIST -- ESPECIALLY THINGS THAT CAN BE CREATED DYNAMICALLY
           // VIA RUNTIME APIS -- UNTIL CAREFULLY CONSIDERING THE ISSUES MENTIONED IN SOLR-8311
+          CircuitBreaker.class,
           CodecFactory.class,
           DirectoryFactory.class,
           ManagedIndexSchemaFactory.class,

solr/core/src/java/org/apache/solr/util/circuitbreaker/CPUCircuitBreaker.java

@@ -20,9 +20,10 @@
 import com.codahale.metrics.Gauge;
 import com.codahale.metrics.Metric;
 import java.lang.invoke.MethodHandles;
-import org.apache.solr.common.util.NamedList;
+import org.apache.solr.core.CoreContainer;
 import org.apache.solr.core.SolrCore;
 import org.apache.solr.metrics.SolrMetricManager;
+import org.apache.solr.util.plugin.SolrCoreAware;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -33,35 +34,30 @@
  * We depend on OperatingSystemMXBean which does not allow a configurable interval of collection of
  * data.
  */
-public class CPUCircuitBreaker extends CircuitBreaker {
+public class CPUCircuitBreaker extends CircuitBreaker implements SolrCoreAware {
   private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
 
-  private boolean enabled = true;
+  private boolean enabled = false;
   private double cpuUsageThreshold;
-  private final SolrCore core;
+  private CoreContainer cc;
 
   private static final ThreadLocal<Double> seenCPUUsage = ThreadLocal.withInitial(() -> 0.0);
 
   private static final ThreadLocal<Double> allowedCPUUsage = ThreadLocal.withInitial(() -> 0.0);
 
-  public CPUCircuitBreaker(SolrCore core) {
+  public CPUCircuitBreaker() {
     super();
-    this.core = core;
   }
 
-  @Override
-  public void init(NamedList<?> args) {
-    super.init(args);
-    double localSeenCPUUsage = calculateLiveCPUUsage();
+  @Deprecated(since = "9.5")
+  public CPUCircuitBreaker(SolrCore core) {
+    this(core.getCoreContainer());
+  }
 
-    if (localSeenCPUUsage < 0) {
-      String msg =
-          "Initialization failure for CPU circuit breaker. Unable to get 'systemCpuLoad', not supported by the JVM?";
-      if (log.isErrorEnabled()) {
-        log.error(msg);
-      }
-      enabled = false;
-    }
+  public CPUCircuitBreaker(CoreContainer coreContainer) {
+    super();
+    this.cc = coreContainer;
+    enableIfSupported();
   }
 
   @Override
@@ -91,7 +87,7 @@ public String getErrorMessage() {
         + allowedCPUUsage.get();
   }
 
-  public void setThreshold(double thresholdValueInPercentage) {
+  public CPUCircuitBreaker setThreshold(double thresholdValueInPercentage) {
     if (thresholdValueInPercentage > 100) {
       throw new IllegalArgumentException("Invalid Invalid threshold value.");
     }
@@ -100,6 +96,7 @@ public void setThreshold(double thresholdValueInPercentage) {
       throw new IllegalStateException("Threshold cannot be less than or equal to zero");
     }
     cpuUsageThreshold = thresholdValueInPercentage;
+    return this;
   }
 
   public double getCpuUsageThreshold() {
@@ -114,12 +111,7 @@ public double getCpuUsageThreshold() {
   protected double calculateLiveCPUUsage() {
     // TODO: Use Codahale Meter to calculate the value
     Metric metric =
-        this.core
-            .getCoreContainer()
-            .getMetricManager()
-            .registry("solr.jvm")
-            .getMetrics()
-            .get("os.systemCpuLoad");
+        this.cc.getMetricManager().registry("solr.jvm").getMetrics().get("os.systemCpuLoad");
 
     if (metric == null) {
       return -1.0;
@@ -137,4 +129,23 @@ protected double calculateLiveCPUUsage() {
 
     return -1.0; // Unable to unpack metric
   }
+
+  @Override
+  public void inform(SolrCore core) {
+    this.cc = core.getCoreContainer();
+    enableIfSupported();
+  }
+
+  private void enableIfSupported() {
+    if (calculateLiveCPUUsage() < 0) {
+      String msg =
+          "Initialization failure for CPU circuit breaker. Unable to get 'systemCpuLoad', not supported by the JVM?";
+      if (log.isErrorEnabled()) {
+        log.error(msg);
+      }
+      enabled = false;
+    } else {
+      enabled = true;
+    }
+  }
 }