Merge pull request #103 from AntelopeIO/run_system_tests_in_parallel

Split system tests so that they can run in parallel
AntelopeIO · Jun 13, 2024 · efb7d81 · efb7d81
2 parents 229dd4f + 877f89f
commit efb7d81
Show file tree

Hide file tree

Showing 3 changed files with 44 additions and 21 deletions.
diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt
@@ -41,15 +41,18 @@ foreach(TEST_SUITE ${UNIT_TESTS}) # create an independent target for each test s
     COMMAND
       bash -c
       "grep -E 'BOOST_AUTO_TEST_SUITE\\s*[(]' ${TEST_SUITE} | grep -vE '//.*BOOST_AUTO_TEST_SUITE\\s*[(]' | cut -d ')' -f 1 | cut -d '(' -f 2"
-    OUTPUT_VARIABLE SUITE_NAME
-    OUTPUT_STRIP_TRAILING_WHITESPACE) # get the test suite name from the *.cpp file
-  if(NOT "" STREQUAL "${SUITE_NAME}") # ignore empty lines
-    execute_process(
-      COMMAND bash -c "echo ${SUITE_NAME} | sed -e 's/s$//' | sed -e 's/_test$//'"
-      OUTPUT_VARIABLE TRIMMED_SUITE_NAME
-      OUTPUT_STRIP_TRAILING_WHITESPACE) # trim "_test" or "_tests" from the end of ${SUITE_NAME}
-    # to run unit_test with all log from blockchain displayed, put "--verbose" after "--", i.e. "unit_test -- --verbose"
-    add_test(NAME ${TRIMMED_SUITE_NAME}_unit_test COMMAND unit_test --run_test=${SUITE_NAME} --report_level=detailed
-                                                          --color_output)
-  endif()
+      OUTPUT_VARIABLE SUITE_NAMES
+    OUTPUT_STRIP_TRAILING_WHITESPACE) # get all the test suite names from each *.cpp file
+  string(REPLACE "\n" ";" SUITE_NAMES "${SUITE_NAMES}")
+  foreach(SUITE_NAME IN LISTS SUITE_NAMES)
+    if(NOT "" STREQUAL "${SUITE_NAME}") # ignore empty lines
+      execute_process(
+        COMMAND bash -c "echo ${SUITE_NAME} | sed -e 's/s$//' | sed -e 's/_test$//'"
+        OUTPUT_VARIABLE TRIMMED_SUITE_NAME
+        OUTPUT_STRIP_TRAILING_WHITESPACE) # trim "_test" or "_tests" from the end of ${SUITE_NAME}
+      # to run unit_test with all log from blockchain displayed, put "--verbose" after "--", i.e. "unit_test -- --verbose"
+      add_test(NAME ${TRIMMED_SUITE_NAME}_unit_test COMMAND unit_test --run_test=${SUITE_NAME} --report_level=detailed
+                                                            --color_output)
+    endif()
+  endforeach(SUITE_NAME)
 endforeach(TEST_SUITE)
diff --git a/tests/eosio.system_tester.hpp b/tests/eosio.system_tester.hpp
@@ -21,13 +21,13 @@ class eosio_system_tester : public validating_tester {
 public:
 
    void basic_setup() {
-      produce_blocks( 2 );
+      produce_block();
 
       create_accounts({ "eosio.token"_n, "eosio.ram"_n, "eosio.ramfee"_n, "eosio.stake"_n,
                "eosio.bpay"_n, "eosio.vpay"_n, "eosio.saving"_n, "eosio.names"_n, "eosio.rex"_n });
 
 
-      produce_blocks( 100 );
+      produce_block();
       set_code( "eosio.token"_n, contracts::token_wasm());
       set_abi( "eosio.token"_n, contracts::token_abi().data() );
       {
@@ -65,7 +65,7 @@ class eosio_system_tester : public validating_tester {
    }
 
    void remaining_setup() {
-      produce_blocks();
+      produce_block();
 
       // Assumes previous setup steps were done with core token symbol set to CORE_SYM
       create_account_with_resources( "alice1111111"_n, config::system_account_name, core_sym::from_string("1.0000"), false );
@@ -1306,7 +1306,8 @@ class eosio_system_tester : public validating_tester {
             BOOST_REQUIRE_EQUAL( success(), regproducer(p) );
          }
       }
-      produce_blocks( 250);
+      produce_block();
+      produce_block(fc::seconds(1000));
 
       auto trace_auth = validating_tester::push_action(config::system_account_name, updateauth::get_name(), config::system_account_name, mvo()
                                             ("account", name(config::system_account_name).to_string())
@@ -1332,7 +1333,7 @@ class eosio_system_tester : public validating_tester {
                              )
          );
       }
-      produce_blocks( 250 );
+      produce_blocks( 2 * 21 ); // This is minimum number of blocks required by ram_gift in system_tests
 
       auto producer_schedule = control->active_producers();
       BOOST_REQUIRE_EQUAL( 21, producer_schedule.producers.size() );

diff --git a/tests/eosio.system_tests.cpp b/tests/eosio.system_tests.cpp
@@ -24,11 +24,15 @@ FC_REFLECT( connector, (balance)(weight) );
 
 using namespace eosio_system;
 
-BOOST_AUTO_TEST_SUITE(eosio_system_tests)
-
 bool within_error(int64_t a, int64_t b, int64_t err) { return std::abs(a - b) <= err; };
 bool within_one(int64_t a, int64_t b) { return within_error(a, b, 1); }
 
+// Split the tests into multiple suites so that they can run in parallel in CICD to
+// reduce overall CICD time..
+// Each suite is grouped by functionality and takes approximately the same amount of time.
+
+BOOST_AUTO_TEST_SUITE(eosio_system_stake_tests)
+
 BOOST_FIXTURE_TEST_CASE( buysell, eosio_system_tester ) try {
 
    BOOST_REQUIRE_EQUAL( core_sym::from_string("0.0000"), get_balance( "alice1111111" ) );
@@ -743,6 +747,9 @@ BOOST_FIXTURE_TEST_CASE( stake_to_another_user_not_from_refund, eosio_system_tes
 
 } FC_LOG_AND_RETHROW()
 
+BOOST_AUTO_TEST_SUITE_END()
+BOOST_AUTO_TEST_SUITE(eosio_system_producer_tests)
+
 // Tests for voting
 BOOST_FIXTURE_TEST_CASE( producer_register_unregister, eosio_system_tester ) try {
    issue_and_transfer( "alice1111111", core_sym::from_string("1000.0000"),  config::system_account_name );
@@ -1640,6 +1647,9 @@ BOOST_FIXTURE_TEST_CASE(producer_pay, eosio_system_tester, * boost::unit_test::t
    }
 } FC_LOG_AND_RETHROW()
 
+BOOST_AUTO_TEST_SUITE_END()
+BOOST_AUTO_TEST_SUITE(eosio_system_inflation_tests)
+
 BOOST_FIXTURE_TEST_CASE(change_inflation, eosio_system_tester) try {
 
    {
@@ -1773,6 +1783,9 @@ BOOST_AUTO_TEST_CASE(extreme_inflation) try {
    BOOST_REQUIRE_EQUAL(t.wasm_assert_msg("quantity exceeds available supply"), t.push_action("defproducera"_n, "claimrewards"_n, mvo()("owner", "defproducera")));
 } FC_LOG_AND_RETHROW()
 
+BOOST_AUTO_TEST_SUITE_END()
+BOOST_AUTO_TEST_SUITE(eosio_system_multiple_producer_pay_tests)
+
 BOOST_FIXTURE_TEST_CASE(multiple_producer_pay, eosio_system_tester, * boost::unit_test::tolerance(1e-10)) try {
 
    const int64_t secs_per_year  = 52 * 7 * 24 * 3600;
@@ -2152,6 +2165,9 @@ BOOST_FIXTURE_TEST_CASE(multiple_producer_pay, eosio_system_tester, * boost::uni
 } FC_LOG_AND_RETHROW()
 
 
+BOOST_AUTO_TEST_SUITE_END()
+BOOST_AUTO_TEST_SUITE(eosio_system_votepay_tests)
+
 BOOST_FIXTURE_TEST_CASE(multiple_producer_votepay_share, eosio_system_tester, * boost::unit_test::tolerance(1e-10)) try {
 
    const asset net = core_sym::from_string("80.0000");
@@ -2850,9 +2866,6 @@ BOOST_FIXTURE_TEST_CASE(producers_upgrade_system_contract, eosio_system_tester)
    BOOST_REQUIRE( bool(trace) );
    BOOST_REQUIRE_EQUAL( 1, trace->action_traces.size() );
    BOOST_REQUIRE_EQUAL( transaction_receipt::executed, trace->receipt->status );
-
-   produce_blocks( 250 );
-
 } FC_LOG_AND_RETHROW()
 
 BOOST_FIXTURE_TEST_CASE(producer_onblock_check, eosio_system_tester) try {
@@ -3246,6 +3259,9 @@ BOOST_FIXTURE_TEST_CASE( elect_producers /*_and_parameters*/, eosio_system_teste
 } FC_LOG_AND_RETHROW()
 
 
+BOOST_AUTO_TEST_SUITE_END()
+BOOST_AUTO_TEST_SUITE(eosio_system_name_tests)
+
 BOOST_FIXTURE_TEST_CASE( buyname, eosio_system_tester ) try {
    create_accounts_with_resources( { "dan"_n, "sam"_n } );
    transfer( config::system_account_name, "dan", core_sym::from_string( "10000.0000" ) );
@@ -3889,6 +3905,9 @@ BOOST_FIXTURE_TEST_CASE( ram_gift, eosio_system_tester ) try {
 
 } FC_LOG_AND_RETHROW()
 
+BOOST_AUTO_TEST_SUITE_END()
+BOOST_AUTO_TEST_SUITE(eosio_system_rex_tests)
+
 BOOST_FIXTURE_TEST_CASE( rex_rounding_issue, eosio_system_tester ) try {
    const std::vector<name> whales { "whale1"_n, "whale2"_n, "whale3"_n, "whale4"_n , "whale5"_n  };
    const name bob{ "bob"_n }, alice{ "alice"_n };