GH-2125 Make test more robust and allow for multiple fork switches

tests/trx_finality_status_forked_test.py

@@ -82,7 +82,7 @@
     prodNodes=[ prodNode0, prodNode1, prodNode2, prodNode3 ]
 
     prodA=prodNode0 # defproducera
-    prodD=prodNode3 # defproducerc
+    prodD=prodNode3 # defproducerd
 
     # ***   Identify a block where production is stable   ***
 
@@ -159,13 +159,12 @@ def getState(status):
     if not nonProdNode.relaunch():
         errorExit(f"Failure - (non-production) node {nonProdNode.nodeNum} should have restarted")
 
-    while prodD.getInfo()['last_irreversible_block_num'] < transBlockNum:
-        Print("Wait for LIB to move, which indicates prodD may have forked out the branch")
-        assert prodD.waitForLibToAdvance(60), \
-            "ERROR: Network did not reach consensus after bridge node was restarted."
+    Print("Repeatedly check status looking for forked out state until after LIB moves and defproducerd")
+    while True:
         retStatus = prodD.getTransactionStatus(transId)
         state = getState(retStatus)
-        if state == forkedOutState:
+        info = prodD.getInfo()
+        if state == forkedOutState or ( info['head_block_producer'] == 'defproducerd' and info['last_irreversible_block_num'] > transBlockNum ):
             break
 
     assert state == forkedOutState, \
@@ -177,7 +176,7 @@ def getState(status):
         Print(f"node info: {json.dumps(info, indent=1)}")
 
     assert prodD.waitForProducer("defproducerd"), \
-        f"Waiting for prodC to produce, but it never happened" + \
+        f"Waiting for prodD to produce, but it never happened" + \
         f"\n\nprod A info: {json.dumps(prodA.getInfo(), indent=1)}\n\nprod D info: {json.dumps(prodD.getInfo(), indent=1)}"
 
     retStatus = prodD.getTransactionStatus(transId)
@@ -206,6 +205,15 @@ def getState(status):
     retStatus = prodD.getTransactionStatus(transId)
     state = getState(retStatus)
 
+    # it is possible for another fork switch to cause the trx to be forked out again
+    if state == forkedOutState:
+        while True:
+            info = prodD.getInfo()
+            retStatus = prodD.getTransactionStatus(transId)
+            state = getState(retStatus)
+            if state == irreversibleState or ( info['head_block_producer'] == 'defproducerd' and info['last_irreversible_block_num'] > retStatus["block_number"] ):
+                break
+
     assert state == irreversibleState, \
         f"ERROR: getTransactionStatus didn't return \"{irreversibleState}\" state.\n\nstatus: {json.dumps(retStatus, indent=1)}" + \
         f"\n\nprod A info: {json.dumps(prodA.getInfo(), indent=1)}\n\nprod D info: {json.dumps(prodD.getInfo(), indent=1)}"