GH-570 Add pause_at_block test

tests/CMakeLists.txt

@@ -82,6 +82,7 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/production_pause_vote_timeout.py ${CM
 configure_file(${CMAKE_CURRENT_SOURCE_DIR}/production_pause_vote_timeout_test_shape.json ${CMAKE_CURRENT_BINARY_DIR}/production_pause_vote_timeout_test_shape.json COPYONLY)
 configure_file(${CMAKE_CURRENT_SOURCE_DIR}/production_restart.py ${CMAKE_CURRENT_BINARY_DIR}/production_restart.py COPYONLY)
 configure_file(${CMAKE_CURRENT_SOURCE_DIR}/production_restart_test_shape.json ${CMAKE_CURRENT_BINARY_DIR}/production_restart_test_shape.json COPYONLY)
+configure_file(${CMAKE_CURRENT_SOURCE_DIR}/pause_at_block_test.py ${CMAKE_CURRENT_BINARY_DIR}/pause_at_block_test.py COPYONLY)
 configure_file(${CMAKE_CURRENT_SOURCE_DIR}/trx_finality_status_test.py ${CMAKE_CURRENT_BINARY_DIR}/trx_finality_status_test.py COPYONLY)
 configure_file(${CMAKE_CURRENT_SOURCE_DIR}/trx_finality_status_forked_test.py ${CMAKE_CURRENT_BINARY_DIR}/trx_finality_status_forked_test.py COPYONLY)
 configure_file(${CMAKE_CURRENT_SOURCE_DIR}/plugin_http_api_test.py ${CMAKE_CURRENT_BINARY_DIR}/plugin_http_api_test.py COPYONLY)
@@ -225,6 +226,8 @@ add_test(NAME subjective_billing_test COMMAND tests/subjective_billing_test.py -
 set_property(TEST subjective_billing_test PROPERTY LABELS nonparallelizable_tests)
 add_test(NAME get_account_test COMMAND tests/get_account_test.py -v -p 2 -n 3 ${UNSHARE} WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
 set_property(TEST get_account_test PROPERTY LABELS nonparallelizable_tests)
+add_test(NAME pause_at_block_test COMMAND tests/pause_at_block_test.py -v ${UNSHARE} WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
+set_property(TEST pause_at_block_test PROPERTY LABELS nonparallelizable_tests)
 
 add_test(NAME distributed-transactions-test COMMAND tests/distributed-transactions-test.py -d 2 -p 4 -n 6 -v ${UNSHARE} WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
 set_property(TEST distributed-transactions-test PROPERTY LABELS nonparallelizable_tests)

tests/pause_at_block_test.py

@@ -0,0 +1,101 @@
+#!/usr/bin/env python3
+
+import json
+import signal
+
+from TestHarness import Account, Cluster, Node, ReturnType, TestHelper, Utils, WalletMgr
+from TestHarness.TestHelper import AppArgs
+
+###############################################################
+# pause_at_block_test
+#
+# Verify /v1/producer/pause_at_block pauses node in all read modes
+#
+###############################################################
+
+# Parse command line arguments
+args = TestHelper.parse_args({"-v","--dump-error-details","--leave-running","--keep-logs","--unshared"})
+Utils.Debug = args.v
+dumpErrorDetails=args.dump_error_details
+dontKill=args.leave_running
+keepLogs=args.keep_logs
+
+walletMgr=WalletMgr(True)
+cluster=Cluster(unshared=args.unshared, keepRunning=args.leave_running, keepLogs=args.keep_logs)
+cluster.setWalletMgr(walletMgr)
+
+testSuccessful = False
+try:
+    TestHelper.printSystemInfo("BEGIN")
+
+    specificNodeosArgs = {
+        0 : "--enable-stale-production",
+
+        2 : "--read-mode head",
+        3 : "--read-mode speculative",
+        4 : "--read-mode irreversible"
+    }
+    assert cluster.launch(
+        pnodes=2,
+        prodCount=2,
+        totalProducers=2,
+        totalNodes=5,
+        loadSystemContract=False,
+        activateIF=True)
+
+    prodNode = cluster.getNode(0)
+    prodNode2 = cluster.getNode(1)
+    headNode = cluster.getNode(2)
+    specNode = cluster.getNode(3)
+    irrvNode = cluster.getNode(4)
+
+    prodNode.waitForProducer("defproducerb");
+    prodNode.waitForProducer("defproducera");
+
+    blockNum = prodNode.getHeadBlockNum()
+
+    blockNum += 5
+
+    Utils.Print(f"Pausing at block {blockNum}")
+    prodNode2.processUrllibRequest("producer", "pause_at_block", {"block_num":blockNum}),
+    headNode.processUrllibRequest("producer", "pause_at_block", {"block_num":blockNum}),
+    specNode.processUrllibRequest("producer", "pause_at_block", {"block_num":blockNum}),
+    irrvNode.processUrllibRequest("producer", "pause_at_block", {"block_num":blockNum}),
+
+    assert prodNode.waitForLibToAdvance(), "LIB did not advance with paused nodes"
+    # blockNum -1 since waitForBlock uses `> blockNum` instead of `>=`
+    assert prodNode2.waitForBlock(blockNum-1), f"Block {blockNum} did not arrive after pausing"
+
+    Utils.Print(f"Verify paused at block {blockNum}")
+    assert prodNode2.getHeadBlockNum() == blockNum, "Prod Node_01 did not pause at block"
+    assert headNode.getHeadBlockNum() == blockNum, "Head Node_02 did not pause at block"
+    assert specNode.getHeadBlockNum() == blockNum, "Speculative Node_03 did not pause at block"
+    assert irrvNode.getHeadBlockNum() == blockNum, "Irreversible Node_04 did not pause at block"
+
+    Utils.Print(f"Verify prod node still producing blocks")
+    assert prodNode.waitForLibToAdvance(), "LIB did not advance with paused nodes"
+
+    Utils.Print(f"Verify still paused at block {blockNum}")
+    assert prodNode2.getHeadBlockNum() == blockNum, "Prod Node_01 did not pause at block"
+    assert headNode.getHeadBlockNum() == blockNum, "Head Node_02 did not pause at block"
+    assert specNode.getHeadBlockNum() == blockNum, "Speculative Node_03 did not pause at block"
+    assert irrvNode.getHeadBlockNum() == blockNum, "Irreversible Node_04 did not pause at block"
+
+    Utils.Print(f"Resume paused nodes")
+    prodNode2.processUrllibRequest("producer", "resume", {})
+    headNode.processUrllibRequest("producer", "resume", {})
+    specNode.processUrllibRequest("producer", "resume",{})
+    irrvNode.processUrllibRequest("producer", "resume", {})
+
+    Utils.Print(f"Verify nodes resumed")
+    assert prodNode2.waitForLibToAdvance(), "Prod Node_01 did not resume"
+    assert headNode.waitForLibToAdvance(), "Head Node_02 did not resume"
+    assert specNode.waitForLibToAdvance(), "Speculative Node_03 did not resume"
+    assert irrvNode.waitForLibToAdvance(), "Irreversible Node_04 did not resume"
+
+    testSuccessful = True
+finally:
+    TestHelper.shutdown(cluster, walletMgr, testSuccessful, dumpErrorDetails)
+
+exitCode = 0 if testSuccessful else 1
+exit(exitCode)