Merge remote-tracking branch 'upstream/main' into trigger-on-published

Signed-off-by: Simon Dudley <[email protected]>
siladu · Apr 24, 2024 · dc9aa24 · dc9aa24
2 parents e91160f + 12723ac
commit dc9aa24
Show file tree

Hide file tree

Showing 90 changed files with 1,407 additions and 2,834 deletions.
diff --git a/.github/workflows/acceptance-tests.yml b/.github/workflows/acceptance-tests.yml
@@ -11,7 +11,7 @@ concurrency:
   cancel-in-progress: true
 
 env:
-  GRADLE_OPTS: "-Xmx6g -Dorg.gradle.daemon=false"
+  GRADLE_OPTS: "-Xmx6g"
   total-runners: 16
 
 jobs:
@@ -49,6 +49,10 @@ jobs:
         uses: gradle/actions/setup-gradle@9e899d11ad247ec76be7a60bc1cf9d3abbb9e7f1
         with:
           cache-disabled: true
+      - name: Compile acceptance tests
+        run: ./gradlew :acceptance-tests:tests:testClasses
+      - name: List acceptance tests
+        run: ./gradlew :acceptance-tests:tests:listAcceptanceTestNotPrivacy -Dorg.gradle.caching=true | fgrep org.hyperledger.besu.tests.acceptance > tmp/currentTests.list
       - name: Split tests
         run: .github/workflows/splitTestsByTime.sh tmp/junit-xml-reports-downloaded ${{env.total-runners}} ${{ matrix.runner_index }} > testList.txt
       - name: Upload Timing
@@ -57,6 +61,12 @@ jobs:
         with:
           name: acceptance-tests-timing
           path: 'tmp/timing.tsv'
+      - name: Upload Lists
+        uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3
+        if: matrix.runner_index == 0
+        with:
+          name: acceptance-tests-lists
+          path: 'tmp/*.list'
       - name: format gradle args
         # insert --tests option between each.
         run:  cat testList.txt | sed -e 's/^\| / --tests /g' | tee gradleArgs.txt

diff --git a/.github/workflows/splitTestsByTime.sh b/.github/workflows/splitTestsByTime.sh
@@ -26,27 +26,44 @@ for line in "${sorted[@]}"; do
 	test_time=${line_parts[0]//./} # convert to millis
 	test_time=${test_time##0} # remove leading zeros
 	test_name=${line_parts[1]}
-
-	# Find index of min sum
-	idx_min_sum=0
-	min_sum=${sums[0]}
-	for ((i=0; i<SPLIT_COUNT; i++))
-	do		
-		if [[ ${sums[$i]} -lt $min_sum ]]
-		then
-			idx_min_sum=$i		
-			min_sum=${sums[$i]}
-		fi
-	done
-
-	# Add the test to the min sum list
-	min_sum_tests=${tests[$idx_min_sum]}
-	tests[$idx_min_sum]="${min_sum_tests}${test_name},"
-
-	# Update the sums
-	((sums[idx_min_sum]+=test_time))
 
+  # Does the test still exists?
+  if grep -F -q --line-regexp "$test_name" tmp/currentTests.list
+  then
+    # Find index of min sum
+    idx_min_sum=0
+    min_sum=${sums[0]}
+    for ((i=0; i<SPLIT_COUNT; i++))
+    do
+      if [[ ${sums[$i]} -lt $min_sum ]]
+      then
+        idx_min_sum=$i
+        min_sum=${sums[$i]}
+      fi
+    done
+
+    # Add the test to the min sum list
+    min_sum_tests=${tests[$idx_min_sum]}
+    tests[$idx_min_sum]="${min_sum_tests}${test_name},"
+
+    # Update the sums
+    ((sums[idx_min_sum]+=test_time))
+
+    echo "$test_name" >> tmp/processedTests.list
+  fi
 done
 
+# Any new test?
+grep -F --line-regexp -v -f tmp/processedTests.list tmp/currentTests.list > tmp/newTests.list
+idx_new_test=0
+while read -r new_test_name
+do
+  idx_group=$(( idx_new_test % SPLIT_COUNT ))
+  group=${tests[$idx_group]}
+  tests[$idx_group]="${group}${new_test_name},"
+  idx_new_test=$(( idx_new_test + 1 ))
+done < tmp/newTests.list
+
+
 # return the requests index, without quotes to drop the last trailing space
 echo ${tests[$SPLIT_INDEX]//,/ }
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -9,6 +9,7 @@
 
 ### Upcoming Breaking Changes
 - Receipt compaction will be enabled by default in a future version of Besu. After this change it will not be possible to downgrade to the previous Besu version.
+- PKI-backed QBFT will be removed in a future version of Besu. Other forms of QBFT will remain unchanged. 
 
 ### Deprecations
 
@@ -41,6 +42,8 @@
 - Expose bad block events via the BesuEvents plugin API  [#6848](https://github.com/hyperledger/besu/pull/6848)
 - Add RPC errors metric [#6919](https://github.com/hyperledger/besu/pull/6919/)
 - Add `rlp decode` subcommand to decode IBFT/QBFT extraData to validator list [#6895](https://github.com/hyperledger/besu/pull/6895)
+- Allow users to specify which plugins are registered [#6700](https://github.com/hyperledger/besu/pull/6700)
+- Layered txpool tuning for blob transactions [#6940](https://github.com/hyperledger/besu/pull/6940)
 
 ### Bug fixes
 - Fix txpool dump/restore race condition [#6665](https://github.com/hyperledger/besu/pull/6665)
@@ -51,6 +54,7 @@
 - Fix to avoid broadcasting full blob txs, instead of only the tx announcement, to a subset of nodes [#6835](https://github.com/hyperledger/besu/pull/6835)
 - Snap client fixes discovered during snap server testing [#6847](https://github.com/hyperledger/besu/pull/6847)
 - Correctly initialize the txpool as disabled on creation [#6890](https://github.com/hyperledger/besu/pull/6890)
+- Fix worldstate download halt when using snap sync during initial sync [#6981](https://github.com/hyperledger/besu/pull/6981)
 
 ### Download Links
 

diff --git a/...ance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/BesuNode.java b/...ance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/BesuNode.java
@@ -35,7 +35,6 @@
 import org.hyperledger.besu.ethereum.permissioning.PermissioningConfiguration;
 import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration;
 import org.hyperledger.besu.metrics.prometheus.MetricsConfiguration;
-import org.hyperledger.besu.pki.config.PkiKeyStoreConfiguration;
 import org.hyperledger.besu.tests.acceptance.dsl.condition.Condition;
 import org.hyperledger.besu.tests.acceptance.dsl.node.configuration.NodeConfiguration;
 import org.hyperledger.besu.tests.acceptance.dsl.node.configuration.genesis.GenesisConfigurationProvider;
@@ -132,7 +131,6 @@ public class BesuNode implements NodeConfiguration, RunnableNode, AutoCloseable
   private final List<String> staticNodes;
   private boolean isDnsEnabled = false;
   private Optional<Integer> exitCode = Optional.empty();
-  private Optional<PkiKeyStoreConfiguration> pkiKeyStoreConfiguration = Optional.empty();
   private final boolean isStrictTxReplayProtectionEnabled;
   private final Map<String, String> environment;
 
@@ -169,7 +167,6 @@ public BesuNode(
       final Optional<PrivacyParameters> privacyParameters,
       final List<String> runCommand,
       final Optional<KeyPair> keyPair,
-      final Optional<PkiKeyStoreConfiguration> pkiKeyStoreConfiguration,
       final boolean isStrictTxReplayProtectionEnabled,
       final Map<String, String> environment)
       throws IOException {
@@ -230,7 +227,6 @@ public BesuNode(
     this.staticNodes = staticNodes;
     this.isDnsEnabled = isDnsEnabled;
     privacyParameters.ifPresent(this::setPrivacyParameters);
-    this.pkiKeyStoreConfiguration = pkiKeyStoreConfiguration;
     this.environment = environment;
     LOG.info("Created BesuNode {}", this);
   }
@@ -763,10 +759,6 @@ public List<String> getRunCommand() {
     return runCommand;
   }
 
-  public Optional<PkiKeyStoreConfiguration> getPkiKeyStoreConfiguration() {
-    return pkiKeyStoreConfiguration;
-  }
-
   public boolean isStrictTxReplayProtectionEnabled() {
     return isStrictTxReplayProtectionEnabled;
   }

diff --git a/...l/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/ProcessBesuNodeRunner.java b/...l/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/ProcessBesuNodeRunner.java
@@ -377,33 +377,6 @@ public void startNode(final BesuNode node) {
                       permissioningConfiguration.getNodeSmartContractInterfaceVersion()));
             });
 
-    node.getPkiKeyStoreConfiguration()
-        .ifPresent(
-            pkiConfig -> {
-              params.add("--Xpki-block-creation-enabled");
-
-              params.add("--Xpki-block-creation-keystore-certificate-alias");
-              params.add(pkiConfig.getCertificateAlias());
-
-              params.add("--Xpki-block-creation-keystore-type");
-              params.add(pkiConfig.getKeyStoreType());
-
-              params.add("--Xpki-block-creation-keystore-file");
-              params.add(pkiConfig.getKeyStorePath().toAbsolutePath().toString());
-
-              params.add("--Xpki-block-creation-keystore-password-file");
-              params.add(pkiConfig.getKeyStorePasswordPath().toAbsolutePath().toString());
-
-              params.add("--Xpki-block-creation-truststore-type");
-              params.add(pkiConfig.getTrustStoreType());
-
-              params.add("--Xpki-block-creation-truststore-file");
-              params.add(pkiConfig.getTrustStorePath().toAbsolutePath().toString());
-
-              params.add("--Xpki-block-creation-truststore-password-file");
-              params.add(pkiConfig.getTrustStorePasswordPath().toAbsolutePath().toString());
-            });
-
     params.addAll(node.getExtraCLIOptions());
 
     params.add("--key-value-storage");

diff --git a/...sl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/ThreadBesuNodeRunner.java b/...sl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/ThreadBesuNodeRunner.java
@@ -21,7 +21,6 @@
 import org.hyperledger.besu.cli.config.EthNetworkConfig;
 import org.hyperledger.besu.cli.config.NetworkName;
 import org.hyperledger.besu.config.GenesisConfigFile;
-import org.hyperledger.besu.consensus.qbft.pki.PkiBlockCreationConfigurationProvider;
 import org.hyperledger.besu.controller.BesuController;
 import org.hyperledger.besu.controller.BesuControllerBuilder;
 import org.hyperledger.besu.crypto.KeyPairUtil;
@@ -31,6 +30,7 @@
 import org.hyperledger.besu.ethereum.api.ApiConfiguration;
 import org.hyperledger.besu.ethereum.api.graphql.GraphQLConfiguration;
 import org.hyperledger.besu.ethereum.core.ImmutableMiningParameters;
+import org.hyperledger.besu.ethereum.core.plugins.PluginConfiguration;
 import org.hyperledger.besu.ethereum.eth.EthProtocolConfiguration;
 import org.hyperledger.besu.ethereum.eth.sync.SynchronizerConfiguration;
 import org.hyperledger.besu.ethereum.eth.transactions.ImmutableTransactionPoolConfiguration;
@@ -141,7 +141,8 @@ private BesuPluginContextImpl buildPluginContext(
     besuPluginContext.addService(PermissioningService.class, permissioningService);
     besuPluginContext.addService(PrivacyPluginService.class, new PrivacyPluginServiceImpl());
 
-    besuPluginContext.registerPlugins(pluginsPath);
+    besuPluginContext.registerPlugins(new PluginConfiguration(pluginsPath));
+
     commandLine.parseArgs(node.getConfiguration().getExtraCLIOptions().toArray(new String[0]));
 
     // register built-in plugins
@@ -251,9 +252,6 @@ public void startNode(final BesuNode node) {
         .isRevertReasonEnabled(node.isRevertReasonEnabled())
         .storageProvider(storageProvider)
         .gasLimitCalculator(GasLimitCalculator.constant())
-        .pkiBlockCreationConfiguration(
-            node.getPkiKeyStoreConfiguration()
-                .map(pkiConfig -> new PkiBlockCreationConfigurationProvider().load(pkiConfig)))
         .evmConfiguration(EvmConfiguration.DEFAULT)
         .maxPeers(maxPeers)
         .maxRemotelyInitiatedPeers(15)

diff --git a/...in/java/org/hyperledger/besu/tests/acceptance/dsl/node/configuration/BesuNodeFactory.java b/...in/java/org/hyperledger/besu/tests/acceptance/dsl/node/configuration/BesuNodeFactory.java
@@ -39,7 +39,6 @@
 import org.hyperledger.besu.tests.acceptance.dsl.node.RunnableNode;
 import org.hyperledger.besu.tests.acceptance.dsl.node.configuration.genesis.GenesisConfigurationFactory;
 import org.hyperledger.besu.tests.acceptance.dsl.node.configuration.genesis.GenesisConfigurationFactory.CliqueOptions;
-import org.hyperledger.besu.tests.acceptance.dsl.node.configuration.pki.PkiKeystoreConfigurationFactory;
 
 import java.io.File;
 import java.io.IOException;
@@ -57,8 +56,6 @@
 public class BesuNodeFactory {
 
   private final NodeConfigurationFactory node = new NodeConfigurationFactory();
-  private final PkiKeystoreConfigurationFactory pkiKeystoreConfigurationFactory =
-      new PkiKeystoreConfigurationFactory();
 
   public BesuNode create(final BesuNodeConfiguration config) throws IOException {
     return new BesuNode(
@@ -94,7 +91,6 @@ public BesuNode create(final BesuNodeConfiguration config) throws IOException {
         config.getPrivacyParameters(),
         config.getRunCommand(),
         config.getKeyPair(),
-        config.getPkiKeyStoreConfiguration(),
         config.isStrictTxReplayProtectionEnabled(),
         config.getEnvironment());
   }
@@ -517,31 +513,6 @@ public BesuNode createQbftNode(final String name) throws IOException {
             .build());
   }
 
-  public BesuNode createPkiQbftJKSNode(final String name) throws IOException {
-    return createPkiQbftNode(KeyStoreWrapper.KEYSTORE_TYPE_JKS, name);
-  }
-
-  public BesuNode createPkiQbftPKCS11Node(final String name) throws IOException {
-    return createPkiQbftNode(KeyStoreWrapper.KEYSTORE_TYPE_PKCS11, name);
-  }
-
-  public BesuNode createPkiQbftPKCS12Node(final String name) throws IOException {
-    return createPkiQbftNode(KeyStoreWrapper.KEYSTORE_TYPE_PKCS12, name);
-  }
-
-  public BesuNode createPkiQbftNode(final String type, final String name) throws IOException {
-    return create(
-        new BesuNodeConfigurationBuilder()
-            .name(name)
-            .miningEnabled()
-            .jsonRpcConfiguration(node.createJsonRpcWithQbftEnabledConfig(false))
-            .webSocketConfiguration(node.createWebSocketEnabledConfig())
-            .devMode(false)
-            .genesisConfigProvider(GenesisConfigurationFactory::createQbftGenesisConfig)
-            .pkiBlockCreationEnabled(pkiKeystoreConfigurationFactory.createPkiConfig(type, name))
-            .build());
-  }
-
   public BesuNode createCustomGenesisNode(
       final String name, final String genesisPath, final boolean canBeBootnode) throws IOException {
     return createCustomGenesisNode(name, genesisPath, canBeBootnode, false);
@@ -699,41 +670,6 @@ public BesuNode createQbftNodeWithContractBasedValidators(
             .build());
   }
 
-  public BesuNode createPkiQbftJKSNodeWithValidators(final String name, final String... validators)
-      throws IOException {
-    return createPkiQbftNodeWithValidators(KeyStoreWrapper.KEYSTORE_TYPE_JKS, name, validators);
-  }
-
-  public BesuNode createPkiQbftPKCS11NodeWithValidators(
-      final String name, final String... validators) throws IOException {
-    return createPkiQbftNodeWithValidators(KeyStoreWrapper.KEYSTORE_TYPE_PKCS11, name, validators);
-  }
-
-  public BesuNode createPkiQbftPKCS12NodeWithValidators(
-      final String name, final String... validators) throws IOException {
-    return createPkiQbftNodeWithValidators(KeyStoreWrapper.KEYSTORE_TYPE_PKCS12, name, validators);
-  }
-
-  public BesuNode createPkiQbftNodeWithValidators(
-      final String type, final String name, final String... validators) throws IOException {
-
-    return create(
-        new BesuNodeConfigurationBuilder()
-            .name(name)
-            .miningEnabled()
-            .jsonRpcConfiguration(node.createJsonRpcWithQbftEnabledConfig(false))
-            .webSocketConfiguration(node.createWebSocketEnabledConfig())
-            .devMode(false)
-            .pkiBlockCreationEnabled(pkiKeystoreConfigurationFactory.createPkiConfig(type, name))
-            .genesisConfigProvider(
-                nodes ->
-                    node.createGenesisConfigForValidators(
-                        asList(validators),
-                        nodes,
-                        GenesisConfigurationFactory::createQbftGenesisConfig))
-            .build());
-  }
-
   public BesuNode createNodeWithStaticNodes(final String name, final List<Node> staticNodes)
       throws IOException {