Add label to ctest tests. Create long running tests build kite configuration....

Add label to ctest tests. Create long running tests build kite configuration. Test script fix to properly handle wait method return value. Compute block finalizational timeout based upon producers cout.

.buildkite/debug.yml

@@ -96,7 +96,7 @@ steps:
         echo "--- :m: Starting MongoDB" && \
         $(which mongod) --fork --logpath "$(pwd)"/mongod.log && \
         echo "+++ :microscope: Running tests" && \
-        ln -s "$(pwd)" /data/job && cd /data/job/build && ctest --output-on-failure
+        ln -s "$(pwd)" /data/job && cd /data/job/build && ctest -LE long_runing_tests --output-on-failure
     retry:
       automatic:
         limit: 1
@@ -116,7 +116,7 @@ steps:
         echo "--- :m: Starting MongoDB" && \
         $(which mongod) --fork --logpath "$(pwd)"/mongod.log && \
         echo "+++ :microscope: Running tests" && \
-        cd /data/job/build && ctest --output-on-failure
+        cd /data/job/build && ctest -LE long_runing_tests --output-on-failure
     retry:
       automatic:
         limit: 1
@@ -140,7 +140,7 @@ steps:
         echo "--- :m: Starting MongoDB" && \
         $(which mongod) --fork --logpath "$(pwd)"/mongod.log && \
         echo "+++ :microscope: Running tests" && \
-        cd /data/job/build && ctest --output-on-failure
+        cd /data/job/build && ctest -LE long_runing_tests --output-on-failure
     retry:
       automatic:
         limit: 1
@@ -164,7 +164,7 @@ steps:
         echo "--- :m: Starting MongoDB" && \
         $(which mongod) --fork --logpath "$(pwd)"/mongod.log && \
         echo "+++ :microscope: Running tests" && \
-        cd /data/job/build && ctest --output-on-failure
+        cd /data/job/build && ctest -LE long_runing_tests --output-on-failure
     retry:
       automatic:
         limit: 1
@@ -188,7 +188,7 @@ steps:
         echo "--- :m: Starting MongoDB" && \
         $(which mongod) --fork --logpath "$(pwd)"/mongod.log && \
         echo "+++ :microscope: Running tests" && \
-        cd /data/job/build && ctest --output-on-failure
+        cd /data/job/build && ctest -LE long_runing_tests --output-on-failure
     retry:
       automatic:
         limit: 1
@@ -212,7 +212,7 @@ steps:
         echo "--- :m: Starting MongoDB" && \
         $(which mongod) --fork --logpath "$(pwd)"/mongod.log && \
         echo "+++ :microscope: Running tests" && \
-        cd /data/job/build && ctest --output-on-failure
+        cd /data/job/build && ctest -LE long_runing_tests --output-on-failure
     retry:
       automatic:
         limit: 1

.buildkite/long_running_tests.yml

+steps:
+  - command: |
+      echo "--- Creating symbolic link to job directory :file_folder:" && \
+      sleep 5 && ln -s "$(pwd)" /data/job && cd /data/job && \
+      echo "+++ Building :hammer:" && \
+      echo 1 | ./eosio_build.sh && \
+      echo "--- Compressing build directory :compression:" && \
+      tar -pczf build.tar.gz build/
+    label: ":darwin: Build"
+    agents:
+      - "role=macos-builder"
+    artifact_paths: "build.tar.gz"
+    timeout: 60
+
+  - command: |
+        echo "+++ :hammer: Building" && \
+        echo 1 | ./eosio_build.sh && \
+        echo "--- :compression: Compressing build directory" && \
+        tar -pczf build.tar.gz build/
+    label: ":ubuntu: Build"
+    agents:
+      - "role=linux-builder"
+    artifact_paths: "build.tar.gz"
+    plugins:
+      docker#v1.1.1:
+        image: "eosio/ci:ubuntu"
+        workdir: /data/job
+    timeout: 60
+    
+  - command: |
+        echo "+++ :hammer: Building" && \
+        echo 1 | ./eosio_build.sh && \
+        echo "--- :compression: Compressing build directory" && \
+        tar -pczf build.tar.gz build/
+    label: ":ubuntu: 18.04 Build"
+    agents:
+      - "role=linux-builder"
+    artifact_paths: "build.tar.gz"
+    plugins:
+      docker#v1.1.1:
+        image: "eosio/ci:ubuntu18"
+        workdir: /data/job
+    timeout: 60
+
+  - command: |
+        echo "+++ :hammer: Building" && \
+        echo 1 | ./eosio_build.sh && \
+        echo "--- :compression: Compressing build directory" && \
+        tar -pczf build.tar.gz build/
+    label: ":fedora: Build"
+    agents:
+      - "role=linux-builder"
+    artifact_paths: "build.tar.gz"
+    plugins:
+      docker#v1.1.1:
+        image: "eosio/ci:fedora"
+        workdir: /data/job
+    timeout: 60
+ 
+  - command: |
+        echo "+++ :hammer: Building" && \
+        echo 1 | ./eosio_build.sh && \
+        echo "--- :compression: Compressing build directory" && \
+        tar -pczf build.tar.gz build/
+    label: ":centos: Build"
+    agents:
+      - "role=linux-builder"
+    artifact_paths: "build.tar.gz"
+    plugins:
+      docker#v1.1.1:
+        image: "eosio/ci:centos"
+        workdir: /data/job
+    timeout: 60
+
+  - command: |
+        echo "+++ :hammer: Building" && \
+        echo 1 | ./eosio_build.sh && \
+        echo "--- :compression: Compressing build directory" && \
+        tar -pczf build.tar.gz build/
+    label: ":aws: Build"
+    agents:
+      - "role=linux-builder"
+    artifact_paths: "build.tar.gz"
+    plugins:
+      docker#v1.1.1:
+        image: "eosio/ci:amazonlinux"
+        workdir: /data/job
+    timeout: 60
+
+  - wait
+
+  - command: |
+        echo "--- :arrow_down: Downloading build directory" && \
+        buildkite-agent artifact download "build.tar.gz" . --step ":darwin: Build" &&
+        tar -zxf build.tar.gz && \
+        echo "--- :m: Starting MongoDB" && \
+        $(which mongod) --fork --logpath "$(pwd)"/mongod.log && \
+        echo "+++ :microscope: Running tests" && \
+        ln -s "$(pwd)" /data/job && cd /data/job/build && ctest -L long_runing_tests --output-on-failure
+    retry:
+      automatic:
+        limit: 1
+    label: ":darwin: Tests"
+    agents:
+      - "role=macos-tester"
+    artifact_paths:
+      - "mongod.log"
+      - "build/genesis.json"
+      - "build/config.ini"
+    timeout: 60
+    
+  - command: |
+        echo "--- :arrow_down: Downloading build directory" && \
+        buildkite-agent artifact download "build.tar.gz" . --step ":ubuntu: Build" &&
+        tar -zxf build.tar.gz && \
+        echo "--- :m: Starting MongoDB" && \
+        $(which mongod) --fork --logpath "$(pwd)"/mongod.log && \
+        echo "+++ :microscope: Running tests" && \
+        cd /data/job/build && ctest -L long_runing_tests --output-on-failure
+    retry:
+      automatic:
+        limit: 1
+    label: ":ubuntu: Tests"
+    agents:
+      - "role=linux-tester"
+    artifact_paths:
+      - "mongod.log"
+      - "build/genesis.json"
+      - "build/config.ini"
+    plugins:
+      docker#v1.1.1:
+        image: "eosio/ci:ubuntu"
+        workdir: /data/job
+    timeout: 60
+  
+  - command: |
+        echo "--- :arrow_down: Downloading build directory" && \
+        buildkite-agent artifact download "build.tar.gz" . --step ":ubuntu: 18.04 Build" &&
+        tar -zxf build.tar.gz && \
+        echo "--- :m: Starting MongoDB" && \
+        $(which mongod) --fork --logpath "$(pwd)"/mongod.log && \
+        echo "+++ :microscope: Running tests" && \
+        cd /data/job/build && ctest -L long_runing_tests --output-on-failure
+    retry:
+      automatic:
+        limit: 1
+    label: ":ubuntu: 18.04 Tests"
+    agents:
+      - "role=linux-tester"
+    artifact_paths:
+      - "mongod.log"
+      - "build/genesis.json"
+      - "build/config.ini"
+    plugins:
+      docker#v1.1.1:
+        image: "eosio/ci:ubuntu18"
+        workdir: /data/job
+    timeout: 60
+
+  - command: |
+        echo "--- :arrow_down: Downloading build directory" && \
+        buildkite-agent artifact download "build.tar.gz" . --step ":fedora: Build" &&
+        tar -zxf build.tar.gz && \
+        echo "--- :m: Starting MongoDB" && \
+        $(which mongod) --fork --logpath "$(pwd)"/mongod.log && \
+        echo "+++ :microscope: Running tests" && \
+        cd /data/job/build && ctest -L long_runing_tests --output-on-failure
+    retry:
+      automatic:
+        limit: 1
+    label: ":fedora: Tests"
+    agents:
+      - "role=linux-tester"
+    artifact_paths:
+      - "mongod.log"
+      - "build/genesis.json"
+      - "build/config.ini"
+    plugins:
+      docker#v1.1.1:
+        image: "eosio/ci:fedora"
+        workdir: /data/job
+    timeout: 60
+
+  - command: |
+        echo "--- :arrow_down: Downloading build directory" && \
+        buildkite-agent artifact download "build.tar.gz" . --step ":centos: Build" &&
+        tar -zxf build.tar.gz && \
+        echo "--- :m: Starting MongoDB" && \
+        $(which mongod) --fork --logpath "$(pwd)"/mongod.log && \
+        echo "+++ :microscope: Running tests" && \
+        cd /data/job/build && ctest -L long_runing_tests --output-on-failure
+    retry:
+      automatic:
+        limit: 1
+    label: ":centos: Tests"
+    agents:
+      - "role=linux-tester"
+    artifact_paths:
+      - "mongod.log"
+      - "build/genesis.json"
+      - "build/config.ini"
+    plugins:
+      docker#v1.1.1:
+        image: "eosio/ci:centos"
+        workdir: /data/job
+    timeout: 60
+
+  - command: |
+        echo "--- :arrow_down: Downloading build directory" && \
+        buildkite-agent artifact download "build.tar.gz" . --step ":aws: Build" &&
+        tar -zxf build.tar.gz && \
+        echo "--- :m: Starting MongoDB" && \
+        $(which mongod) --fork --logpath "$(pwd)"/mongod.log && \
+        echo "+++ :microscope: Running tests" && \
+        cd /data/job/build && ctest -L long_runing_tests --output-on-failure
+    retry:
+      automatic:
+        limit: 1
+    label: ":aws: Tests"
+    agents:
+      - "role=linux-tester"
+    artifact_paths:
+      - "mongod.log"
+      - "build/genesis.json"
+      - "build/config.ini"
+    plugins:
+      docker#v1.1.1:
+        image: "eosio/ci:amazonlinux"
+        workdir: /data/job
+    timeout: 60

.buildkite/pipeline.yml

@@ -96,7 +96,7 @@ steps:
         echo "--- :m: Starting MongoDB" && \
         $(which mongod) --fork --logpath "$(pwd)"/mongod.log && \
         echo "+++ :microscope: Running tests" && \
-        ln -s "$(pwd)" /data/job && cd /data/job/build && ctest --output-on-failure
+        ln -s "$(pwd)" /data/job && cd /data/job/build && ctest -LE long_runing_tests --output-on-failure
     retry:
       automatic:
         limit: 1
@@ -116,7 +116,7 @@ steps:
         echo "--- :m: Starting MongoDB" && \
         $(which mongod) --fork --logpath "$(pwd)"/mongod.log && \
         echo "+++ :microscope: Running tests" && \
-        cd /data/job/build && ctest --output-on-failure
+        cd /data/job/build && ctest -LE long_runing_tests --output-on-failure
     retry:
       automatic:
         limit: 1
@@ -140,7 +140,7 @@ steps:
         echo "--- :m: Starting MongoDB" && \
         $(which mongod) --fork --logpath "$(pwd)"/mongod.log && \
         echo "+++ :microscope: Running tests" && \
-        cd /data/job/build && ctest --output-on-failure
+        cd /data/job/build && ctest -LE long_runing_tests --output-on-failure
     retry:
       automatic:
         limit: 1
@@ -164,7 +164,7 @@ steps:
         echo "--- :m: Starting MongoDB" && \
         $(which mongod) --fork --logpath "$(pwd)"/mongod.log && \
         echo "+++ :microscope: Running tests" && \
-        cd /data/job/build && ctest --output-on-failure
+        cd /data/job/build && ctest -LE long_runing_tests --output-on-failure
     retry:
       automatic:
         limit: 1
@@ -188,7 +188,7 @@ steps:
         echo "--- :m: Starting MongoDB" && \
         $(which mongod) --fork --logpath "$(pwd)"/mongod.log && \
         echo "+++ :microscope: Running tests" && \
-        cd /data/job/build && ctest --output-on-failure
+        cd /data/job/build && ctest -LE long_runing_tests --output-on-failure
     retry:
       automatic:
         limit: 1
@@ -212,7 +212,7 @@ steps:
         echo "--- :m: Starting MongoDB" && \
         $(which mongod) --fork --logpath "$(pwd)"/mongod.log && \
         echo "+++ :microscope: Running tests" && \
-        cd /data/job/build && ctest --output-on-failure
+        cd /data/job/build && ctest -LE long_runing_tests --output-on-failure
     retry:
       automatic:
         limit: 1

tests/CMakeLists.txt

@@ -50,7 +50,6 @@ add_test(NAME p2p_dawn515_test COMMAND tests/p2p_tests/dawn_515/test.sh WORKING_
 #   add_test(NAME nodeos_run_test-mongodb COMMAND tests/nodeos_run_test.py --mongodb -v --dump-error-detail WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
 #endif()
 
-# TODO: Tests removed until working again on master.
 add_test(NAME distributed-transactions-test COMMAND tests/distributed-transactions-test.py -d 2 -p 1 -n 4 -v --clean-run --dump-error-detail WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
 add_test(NAME restart-scenarios-test-resync COMMAND tests/restart-scenarios-test.py -c resync -p4 -v --clean-run --dump-error-details WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
 add_test(NAME restart-scenarios-test-hard_replay COMMAND tests/restart-scenarios-test.py -c hardReplay -p4 -v --clean-run --dump-error-details WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
@@ -58,6 +57,12 @@ add_test(NAME restart-scenarios-test-none COMMAND tests/restart-scenarios-test.p
 # TODO: add_test(NAME consensus-validation-malicious-producers COMMAND tests/consensus-validation-malicious-producers.py -w 80 --dump-error-details WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
 add_test(NAME validate_dirty_db_test COMMAND tests/validate-dirty-db.py -v --clean-run --dump-error-detail WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
 
+# Long running tests
+add_test(NAME nodeos_sanity_lr_test COMMAND tests/nodeos_run_test.py -v --sanity-test --clean-run --dump-error-detail WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
+#add_test(NAME distributed_transactions_lr_test COMMAND tests/distributed-transactions-test.py -d 2 -p 21 -n 21 -v --clean-run --dump-error-detail WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
+set_property(TEST nodeos_sanity_lr_test PROPERTY LABELS long_runing_tests)
+#set_property(TEST distributed_transactions_lr_test PROPERTY LABELS long_runing_tests)
+
 if(ENABLE_COVERAGE_TESTING)
 
   set(Coverage_NAME ${PROJECT_NAME}_coverage)

tests/Cluster.py

@@ -13,6 +13,7 @@ import sys
 import random
 import json
 import socket
+import errno
 
 from core_symbol import CORE_SYMBOL
 from testUtils import Utils
@@ -32,7 +33,7 @@ class Cluster(object):
 
     # pylint: disable=too-many-arguments
     # walletd [True|False] Is keosd running. If not load the wallet plugin
-    def __init__(self, walletd=False, localCluster=True, host="localhost", port=8888, walletHost="localhost", walletPort=8899, enableMongo=False
+    def __init__(self, walletd=False, localCluster=True, host="localhost", port=8888, walletHost="localhost", walletPort=9899, enableMongo=False
                  , mongoHost="localhost", mongoPort=27017, mongoDb="EOStest", defproduceraPrvtKey=None, defproducerbPrvtKey=None, staging=False):
         """Cluster container.
         walletd [True|False] Is wallet keosd running. If not load the wallet plugin
@@ -191,7 +192,7 @@ class Cluster(object):
             account.activePrivateKey=keys["private"]
             account.activePublicKey=keys["public"]
 
-        for name,initKeys in producerKeys.items():
+        for name,_ in producerKeys.items():
             account=Account(name)
             initAccountKeys(account, producerKeys[name])
             self.defProducerAccounts[name] = account
@@ -322,7 +323,7 @@ class Cluster(object):
     @staticmethod
     def getClientVersion(verbose=False):
         """Returns client version (string)"""
-        p = re.compile('^Build version:\s(\w+)\n$')
+        p = re.compile(r'^Build version:\s(\w+)\n$')
         try:
             cmd="%s version client" % (Utils.EosClientPath)
             if verbose: Utils.Print("cmd: %s" % (cmd))
@@ -476,7 +477,7 @@ class Cluster(object):
             node=self.nodes[nextEosIdx]
             if Utils.Debug: Utils.Print("Wait for transaction id %s on node port %d" % (transId, node.port))
             if node.waitForTransInBlock(transId) is False:
-                Utils.Print("ERROR: Selected node never received transaction id %s" % (transId))
+                Utils.Print("ERROR: Failed to validate transaction %s got rolled into a block on server port %d." % (transId, node.port))
                 return False
 
             transferAmount -= amount
@@ -497,7 +498,7 @@ class Cluster(object):
         node=self.nodes[0]
         if Utils.Debug: Utils.Print("Wait for transaction id %s on node port %d" % (transId, node.port))
         if node.waitForTransInBlock(transId) is False:
-            Utils.Print("ERROR: Selected node never received transaction id %s" % (transId))
+            Utils.Print("ERROR: Failed to validate transaction %s got rolled into a block on server port %d." % (transId, node.port))
             return False
 
         return True
@@ -728,7 +729,9 @@ class Cluster(object):
                 accounts.append(initx)
 
             transId=Node.getTransId(trans)
-            biosNode.waitForTransInBlock(transId)
+            if not biosNode.waitForTransInBlock(transId):
+                Utils.Print("ERROR: Failed to validate transaction %s got rolled into a block on server port %d." % (transId, biosNode.port))
+                return False
 
             Utils.Print("Validating system accounts within bootstrap")
             biosNode.validateAccounts(accounts)
@@ -776,6 +779,7 @@ class Cluster(object):
                 trans=trans[1]
                 transId=Node.getTransId(trans)
                 if not biosNode.waitForTransInBlock(transId):
+                    Utils.Print("ERROR: Failed to validate transaction %s got rolled into a block on server port %d." % (transId, biosNode.port))
                     return False
 
                 # wait for block production handover (essentially a block produced by anyone but eosio).
@@ -815,7 +819,9 @@ class Cluster(object):
 
             Node.validateTransaction(trans)
             transId=Node.getTransId(trans)
-            biosNode.waitForTransInBlock(transId)
+            if not biosNode.waitForTransInBlock(transId):
+                Utils.Print("ERROR: Failed to validate transaction %s got rolled into a block on server port %d." % (transId, biosNode.port))
+                return False
 
             contract="eosio.token"
             contractDir="contracts/%s" % (contract)
@@ -840,7 +846,9 @@ class Cluster(object):
 
             Node.validateTransaction(trans[1])
             transId=Node.getTransId(trans[1])
-            biosNode.waitForTransInBlock(transId)
+            if not biosNode.waitForTransInBlock(transId):
+                Utils.Print("ERROR: Failed to validate transaction %s got rolled into a block on server port %d." % (transId, biosNode.port))
+                return False
 
             contract=eosioTokenAccount.name
             Utils.Print("push issue action to %s contract" % (contract))
@@ -856,7 +864,11 @@ class Cluster(object):
             Utils.Print("Wait for issue action transaction to become finalized.")
             transId=Node.getTransId(trans[1])
             # biosNode.waitForTransInBlock(transId)
-            biosNode.waitForTransFinalization(transId)
+            # guesstimating block finalization timeout. Two production rounds of 12 blocks per node, plus 60 seconds buffer
+            timeout = .5 * 12 * 2 * len(producerKeys) + 60
+            if not biosNode.waitForTransFinalization(transId, timeout=timeout):
+                Utils.Print("ERROR: Failed to validate transaction %s got rolled into a finalized block on server port %d." % (transId, biosNode.port))
+                return False
 
             expectedAmount="1000000000.0000 {0}".format(CORE_SYMBOL)
             Utils.Print("Verify eosio issue, Expected: %s" % (expectedAmount))
@@ -896,6 +908,7 @@ class Cluster(object):
             Utils.Print("Wait for last transfer transaction to become finalized.")
             transId=Node.getTransId(trans[1])
             if not biosNode.waitForTransInBlock(transId):
+                Utils.Print("ERROR: Failed to validate transaction %s got rolled into a block on server port %d." % (transId, biosNode.port))
                 return False
 
             Utils.Print("Cluster bootstrap done.")
@@ -1073,8 +1086,7 @@ class Cluster(object):
             node=self.nodes[0]
             if Utils.Debug: Utils.Print("Wait for transaction id %s on server port %d." % ( transId, node.port))
             if node.waitForTransInBlock(transId) is False:
-                Utils.Print("ERROR: Failed waiting for transaction id %s on server port %d." % (
-                    transId, node.port))
+                Utils.Print("ERROR: Failed to validate transaction %s got rolled into a block on server port %d." % (transId, node.port))
                 return False
 
         return True

tests/Node.py

@@ -228,7 +228,14 @@ class Node(object):
             Utils.Print("Failure in get info parsing. %s" % (info))
             raise
 
-        return True if blockNum <= node_block_num else False
+        finalized  = True if blockNum <= node_block_num else False
+        if Utils.Debug:
+            if finalized:
+                Utils.Print("Block %d is finalized." % (blockNum))
+            else:
+                Utils.Print("Block %d is not yet finalized." % (blockNum))
+
+        return finalized
 
     # pylint: disable=too-many-branches
     def getTransaction(self, transId, retry=True, silentErrors=False):
@@ -314,10 +321,13 @@ class Node(object):
         try:
             headBlockNum=int(headBlockNum)
         except(ValueError) as _:
-            Utils.Print("Info parsing failed. %s" % (headBlockNum))
+            Utils.Print("ERROR: Block info parsing failed. %s" % (headBlockNum))
+            raise
 
+        if Utils.Debug: Utils.Print("Reference block num %d, Head block num: %d" % (refBlockNum, headBlockNum))
         for blockNum in range(refBlockNum, headBlockNum+1):
             if self.isTransInBlock(str(transId), blockNum):
+                if Utils.Debug: Utils.Print("Found transaction %s in block %d" % (transId, blockNum))
                 return blockNum
 
         return None

tests/WalletMgr.py

@@ -17,7 +17,7 @@ class WalletMgr(object):
 
     # pylint: disable=too-many-arguments
     # walletd [True|False] True=Launch wallet(keosd) process; False=Manage launch process externally.
-    def __init__(self, walletd, nodeosPort=8888, nodeosHost="localhost", port=8899, host="localhost"):
+    def __init__(self, walletd, nodeosPort=8888, nodeosHost="localhost", port=9899, host="localhost"):
         self.walletd=walletd
         self.nodeosPort=nodeosPort
         self.nodeosHost=nodeosHost